Turns out BTLS is stricter than others about certificates' metadata.

Starting from the fact that trust-self-signed.sh "https://self-signed.badssl.com:443" worked as expected, I modeled my certificate after the one used there to include organization & locale metadata, not have X509EnhancedKeyUsageExtension and X509KeyUsageExtension, worked like a charm.

Ended up with this:

public static X509Certificate2 BuildSelfSignedServerCertificate(string host)
{
    using RSA rsa = RSA.Create(2048);
    CertificateRequest request = CreateRequest(rsa, host);
    X509Certificate2 certificate = CreateCertificate(request);
    return certificate;

    static CertificateRequest CreateRequest(RSA rsa, string host)
    {
        X500DistinguishedNameBuilder distinguishedName = new();
        distinguishedName.AddOrganizationName("Myself");
        distinguishedName.AddLocalityName("Sofia");
        distinguishedName.AddStateOrProvinceName("Sofia");
        distinguishedName.AddCountryOrRegion("BG");
        distinguishedName.AddCommonName(host);
        SubjectAlternativeNameBuilder sanExtension = new();
        sanExtension.AddDnsName(host);
        X509BasicConstraintsExtension constraintsExtension = new(
            certificateAuthority: false,
            hasPathLengthConstraint: false,
            pathLengthConstraint: 0,
            critical: false
        );
        CertificateRequest request = new(distinguishedName.Build(), rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
        request.CertificateExtensions.Add(sanExtension.Build());
        request.CertificateExtensions.Add(constraintsExtension);
        return request;
    }

    static X509Certificate2 CreateCertificate(CertificateRequest request)
    {
        X509Certificate2 certificate = request.CreateSelfSigned(
            new DateTimeOffset(DateTime.UtcNow.AddDays(-30)),
            new DateTimeOffset(DateTime.UtcNow.AddDays(365_0))
        );
        string password = $"{Guid.NewGuid():N}";
        byte[] export = certificate.Export(X509ContentType.Pfx, password);
        X509Certificate2 result = X509CertificateLoader.LoadPkcs12(export, password);
        return result;
    }
}

To compare a pair of certificates, one can use s_client -connect $HOST:$PORT to get it, save it as .crt and use the viewer built into Windows or some online viewer.