So based on what @browsermator suggested, a much simpler solution was to change the button to a "submit" type button, add asp-controller and asp-action to the button, and have the action just return the file. So much easier. Thank you @browsemator!

View - Just the button here...

<button type="submit" asp-controller="MyPageController" asp-action="DownloadFile"> 
    Download 
</button>

Controller

[HttpPost]
[Route("MyPageController/DownloadFile")]
public ActionResult DownloadFile(string fileName)
{
    // This is an example for excel.
    string fileType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";


    MemoryStream stream = new MemoryStream();
    
    ... Load your memory stream...

    //for me and how I had to load the memory stream, 
    //i needed to reset the stream to position 0.
    stream.Position = 0;

    //return the File object.  I had no idea the post wouldn't reload the page.
    return File(stream, fileType, fileName);
}