Ok i finally figured it out. Here is a stripped down version of the code then ended working for me:

public async byte[] GetImage(string symbolpath)
{

    var getBitmapSizePath = symbolpath + "#<<ITcVnBitmapExportRpcUnlocked>>GetBitmapSize";
    var getBitmapPath = symbolpath + "#<<ITcVnBitmapExportRpcUnlocked>>GetBitmapImageRpcUnlocked";
    //see https://infosys.beckhoff.com/index.php?content=../content/1031/tf7xxx_tc3_vision/16954359435.html&id=
    var getBitmapSizeHandle = (await adsClient.CreateVariableHandleAsync(getBitmapSizePath, cancelToken)).Handle;
    var getBitmapHandle = (await adsClient.CreateVariableHandleAsync(getBitmapPath, cancelToken)).Handle;

    int status;
    ulong imageSize;
    uint width;
    uint height

    byte[] readBytes = new byte[20];
    byte[] sizeInput = new byte[8];

    var resultGetBitmapSize = await adsClient.ReadWriteAsync((uint)IndexGroupSymbolAccess.ValueByHandle, getBitmapSizeHandle, readBytes, sizeInput, cancelToken);

    //parse the result:
    using (var ms = new MemoryStream(readBytes))
    using (var reader = new BinaryReader(ms))
    {
        status = reader.ReadInt32();
        imageSize = reader.ReadUInt64();
        width = reader.ReadUInt32();
        height = reader.ReadUInt32();
    }

    //todo check resultGetBitmapSize and status on if it succeeded before continuing
    
    //now lets get the image

    //prep input 
    byte[] input = new byte[16];
    BitConverter.GetBytes(imageSize).CopyTo(input, 0);
    BitConverter.GetBytes(width).CopyTo(input, 8);
    BitConverter.GetBytes(height).CopyTo(input, 12);

    int imageBufferSize = 20 + (int)imageSize;
    byte[] buffer = new byte[imageBufferSize]; //todo use a shared array pool to limit memory use
    byte[] imageData = new byte[imageBufferSize];
    int imageStatus;

    var resultGetImage = await adsClient.ReadWriteAsync((uint)IndexGroupSymbolAccess.ValueByHandle, getBitmapHandle, buffer, input, cancelToken);

    //parse the result:
    using (var imageStream = new MemoryStream(imageDataArray))
    using (var imageReader = new BinaryReader(imageStream))
    {
        imageStatus = imageReader.ReadInt32();
        ulong byteCount = imageReader.ReadUInt64();
        imageReader.Read(imageData, 0, (int)byteCount);
    }

    //todo check resultGetImage and imageStatus to see if it was successful

    //clean up the handles
    await adsClient.DeleteVariableHandleAsync(getBitmapSizeHandle, cancelToken);
    await adsClient.DeleteVariableHandleAsync(getBitmapHandle, cancelToken);

    return imageData; //todo convert byte array to bitmap.
}

the main magic is that i needed to use ITcVnBitmapExportRpcUnlocked instead. This is documented here: https://infosys.beckhoff.com/index.php?content=../content/1031/tf7xxx_tc3_vision/16954359435.html&id=