@Ruikai Feng, you are absolutely right! I cannot believe I have used such a naive, faulty approach :-(. I have added a simple Microsoft.Playwright test to verify the erroneous behavior. After that, I have refactored the CustomAuthenticationStateProvider to Implement the IAuthenticationStateProvider directly and use HttpContext to obtain the custom request header:

public class CustomAuthenticationHandler : IAuthenticationHandler 
{
  public const string SchemeName = "CustomAuthenticationScheme"; 
  private const string UserNameHeaderName = "X-Claim-UserName";
  
  private HttpContext? _httpContext;
  
  public CustomAuthenticationHandler()
  {
  }
  
  public Task<AuthenticateResult> AuthenticateAsync()
  {
    if (this._httpContext is null)
    {
      return Task.FromResult(AuthenticateResult.Fail("No HttpContext"));
    }
    if (!this._httpContext.Request.Headers.TryGetValue(UserNameHeaderName, out var userName) || (userName.Count == 0))
    {
      return Task.FromResult(AuthenticateResult.Fail("No user name found in the request headers."));
    }
    return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(CreateClaimsPrincipal(userName.ToString()), SchemeName)));
  }
  // Code omitted for clarity
  public Task InitializeAsync(AuthenticationScheme scheme, HttpContext context)
  {
    this._httpContext = context;
    return Task.CompletedTask;
  }
  
  private ClaimsPrincipal CreateClaimsPrincipal(string userName = "DEFAULT")
  {
    var claims = new[] { new Claim(ClaimTypes.Name, userName) };
    var identity = new ClaimsIdentity(claims, SchemeName);
    return new ClaimsPrincipal(identity);
  }
}

Register the provider with a custom scheme in the DI container:

// Add our custom authentication scheme and handler for request headers-based authentication..
  builder.Services.AddAuthentication(options =>
  {
    options.AddScheme<CustomAuthenticationHandler>(
      name: CustomAuthenticationHandler.SchemeName, 
      displayName: CustomAuthenticationHandler.SchemeName);
  });

I hope that this is finally the correct way to do the authentication based on trusted request headers. I would really like to hear your opinion.