If someone find's a cleaner solution, please let me know. But for now, this works:

I created a ThreadLocal to hold the header value:

object GraphQLMyHeaderThreadLocalStorage {
    private val context = ThreadLocal<String>()

    var value: String?
        get() = context.get()
        set(value) = value?.let { context.set(it) } ?: context.remove()

    fun clear() = context.remove()
}

In my resolver, I can now set this ThreadLocal with my request-specific value:

@QueryMapping
fun myResolver(
    @Argument arg1: String,
    @Argument arg2: MyInput,
): MyEntity = service.getMyEntity(arg1, arg2).also {
    GraphQLMyHeaderThreadLocalStorage.value = "whatever inferred from ${it}"
}

And I can still modify my response in a Filter if I wrap it in advance and do the modification after chain.doFilter():

class GraphQLMyHeaderFilter : Filter {
    @Throws(IOException::class, ServletException::class)
    override fun doFilter(request: ServletRequest, response: ServletResponse, chain: FilterChain) {
        if (!response.isCommitted) {
            val responseWrapper = object : HttpServletResponseWrapper(response as HttpServletResponse) {
                fun updateMyHeader(value: String?) {
                    if (value != null) {
                        setHeader("X-My-Header", value)
                    } else {
                        setHeader("X-My-Header", "default value")
                    }
                }
            }
            chain.doFilter(request, responseWrapper)

            // modify the response after the resolver was called
            if (!response.isCommitted) {
                val headerValue = try {
                    GraphQLMyHeaderThreadLocalStorage.value
                } finally {
                    GraphQLMyHeaderThreadLocalStorage.clear()
                }
                responseWrapper.updateCacheControl(headerValue)
            }
        } else {
            chain.doFilter(request, response)
        }
    }
}

@Configuration
class FilterConfig {
    @Bean
    fun graphQLMyHeaderFilter(): FilterRegistrationBean<GraphQLMyHeaderFilter> {
        val registrationBean = FilterRegistrationBean<GraphQLMyHeaderFilter>()
        registrationBean.filter = GraphQLMyHeaderFilter()
        registrationBean.addUrlPatterns("/graphql")
        return registrationBean
    }
}

Notes:

• The response.isCommitted checks were actually not necessary in my experiments, but I'm rather safe than sorry.
• I limited the Filter to only the "/graphql" endpoint in the FilterConfig. To apply it to all endpoints, you can either use the "/*" pattern instead of "/graphql" or delete the FilterConfig and annotate GraphQLMyHeaderFilter with @Component.
• Make sure to always GraphQLMyHeaderThreadLocalStorage.clear() afterwards so the state doesn't leak into following requests.
• Filter was the only option I found where I can still modify the (uncommitted) response after my resolver was called. ResponseBodyAdvice was not even called for GraphQL requests in my experiments. HandlerInterceptor was accessed, but HandlerInterceptor.preHandle() was executed before the resolver (twice even) and HandlerInterceptor.postHandle() receives the already committed response (i.e., cannot modify the response anymore).