The solution was setting ServeUnknownFileTypes to true, like so:

var builder = WebApplication.CreateBuilder(args);
var app = builder.Build();

app.UseStaticFiles(new StaticFileOptions {ServeUnknownFileTypes = true});
app.MapFallbackToFile("index.html");

app.Run();