It turns out I needed to copy over the cookies from the original request since Blazor Server does not automatically know about them so just creating an empty CookieContainer doesn't work the same as it does in WASM.

builder.Services.AddScoped(sp =>
    new GraphQLHttpClient(config =>
    {
        var baseUri = new Uri(sp.GetRequiredService<NavigationManager>().BaseUri);
        var cookie = sp.GetRequiredService<IHttpContextAccessor>().HttpContext?.Request.Cookies[".AspNetCore.Cookies"];
        var container = new CookieContainer();
        container.Add(new Cookie(".AspNetCore.Cookies", cookie, "/", baseUri.Host));
        config.EndPoint = new Uri($"{baseUri}graphql");
        config.HttpMessageHandler = new HttpClientHandler
        {
            CookieContainer = container,
            UseCookies = true
        };
    }, new SystemTextJsonSerializer()));