This is the ResourceResolverSPI implementation that finally worked!

import java.util.LinkedHashSet;
import java.util.Set;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.xml.security.signature.XMLSignatureException;
import org.apache.xml.security.signature.XMLSignatureInput;
import org.apache.xml.security.signature.XMLSignatureInputDebugger;
import org.apache.xml.security.signature.XMLSignatureNodeSetInput;
import org.apache.xml.security.utils.XMLUtils;
import org.apache.xml.security.utils.resolver.ResourceResolverContext;
import org.apache.xml.security.utils.resolver.ResourceResolverException;
import org.apache.xml.security.utils.resolver.ResourceResolverSpi;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class EbicsXPointerNodeSetResolver extends ResourceResolverSpi {
    
    private static final Logger log = LogManager.getLogger(EbicsXPointerNodeSetResolver.class);

    @Override
    public boolean engineCanResolveURI(ResourceResolverContext context) {
        String uri = context.uriToResolve;
        // Detecta cualquier URI que comience con #xpointer(...)
        return uri != null && uri.startsWith("#xpointer(");
    }

    @Override
    public XMLSignatureInput engineResolveURI(ResourceResolverContext context)
            throws ResourceResolverException {
        Document doc = context.attr.getOwnerDocument();
        
        try {
            XPath xpath = XPathFactory.newInstance().newXPath();
            
            // Expresión XPath para encontrar todos los elementos con @authenticate="true"
            String xpathExpr = "//*[@authenticate='true']";
            NodeList nodes = (NodeList) xpath.evaluate(xpathExpr, doc, XPathConstants.NODESET);
            
            if (nodes.getLength() == 0) {
                throw new XPathExpressionException("No se encontraron elementos con authenticate='true'");
            }
            
            Set<Node> rootSet = new LinkedHashSet<>();
            for (int i = 0; i < nodes.getLength(); i++) {
                Node node = nodes.item(i);                
                rootSet.add(node);
            }
            
            Set<Node> expandedNodeSet = new LinkedHashSet<>();
            for (Node root : rootSet) {
                XMLUtils.getSet(root, expandedNodeSet, null, false);  // Agrega root + todos descendientes, sin comentarios
            }
            
            XMLSignatureNodeSetInput input=new XMLSignatureNodeSetInput(expandedNodeSet);
            input.setExcludeComments(true);
            input.setNodeSet(true);  // Marca como nodeset para comportamiento correcto en transforms
            
            
            XMLSignatureInputDebugger debug = new XMLSignatureInputDebugger(input,Set.of());
            log.info("html debug:\n{}",debug.getHTMLRepresentation());
            return input;
        } catch (XPathExpressionException | XMLSignatureException e) {
            throw new ResourceResolverException("No nodes with authenticate=true", context.uriToResolve, context.baseUri);
        }
    }

}

Firstly, using only rootSet, the canonalizer algorithm wasn't working as expected because it was only using the nodes marked with the attribute authenticate=true but it wasn't using the child nodes, so the xml to be digested was not complete and then it failed validating the signature.

I integrated this implementation using this line: ResourceResolver.register(new EbicsXPointerNodeSetResolver(),true);

This is the code to sign the XML using XPath expression to select only those nodes with attribute authenticate=true

import java.security.InvalidAlgorithmParameterException;
import java.security.NoSuchAlgorithmException;
import java.security.Security;
import java.security.cert.X509Certificate;
import java.security.interfaces.RSAPrivateKey;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import javax.xml.crypto.MarshalException;
import javax.xml.crypto.dsig.DigestMethod;
import javax.xml.crypto.dsig.Reference;
import javax.xml.crypto.dsig.SignatureMethod;
import javax.xml.crypto.dsig.SignedInfo;
import javax.xml.crypto.dsig.Transform;
import javax.xml.crypto.dsig.XMLSignature;
import javax.xml.crypto.dsig.XMLSignatureException;
import javax.xml.crypto.dsig.XMLSignatureFactory;
import javax.xml.crypto.dsig.dom.DOMSignContext;
import javax.xml.crypto.dsig.keyinfo.KeyInfo;
import javax.xml.crypto.dsig.keyinfo.KeyInfoFactory;
import javax.xml.crypto.dsig.keyinfo.X509Data;
import javax.xml.crypto.dsig.spec.C14NMethodParameterSpec;
import javax.xml.crypto.dsig.spec.TransformParameterSpec;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.xml.security.Init;
import org.apache.xml.security.c14n.Canonicalizer;
import org.apache.xml.security.utils.resolver.ResourceResolver;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

public class XMLDSigService {
    
    private static final Logger log = LogManager.getLogger(XMLDSigService.class);
    private static final org.apache.jcp.xml.dsig.internal.dom.XMLDSigRI xmlDSigRI = new org.apache.jcp.xml.dsig.internal.dom.XMLDSigRI();
    static {
        Init.init();    
        ResourceResolver.register(new EbicsXPointerNodeSetResolver(),true); 
        System.setProperty("org.jcp.xml.dsig.provider", "org.apache.jcp.xml.dsig.internal.dom.XMLDSigRI");  
        Security.addProvider(xmlDSigRI);
    }

    public static void sign(Document doc, X509Certificate cert, RSAPrivateKey privateKey, boolean addKeyInfo) throws NoSuchAlgorithmException, InvalidAlgorithmParameterException, MarshalException, XMLSignatureException {
        
            XMLSignatureFactory sigFactory = XMLSignatureFactory.getInstance("DOM", xmlDSigRI);

            List<Transform> transforms = new ArrayList<>();
            
            Transform c14nTransform = sigFactory.newTransform(Canonicalizer.ALGO_ID_C14N_OMIT_COMMENTS, (TransformParameterSpec) null);
            transforms.add(c14nTransform);

            Reference ref = sigFactory.newReference(
                    "#xpointer(//*[@authenticate='true'])",  
                    sigFactory.newDigestMethod(DigestMethod.SHA256, null),
                    transforms,
                    null,
                    null
                    );

            SignedInfo signedInfo = sigFactory.newSignedInfo(
                    sigFactory.newCanonicalizationMethod(Canonicalizer.ALGO_ID_C14N_OMIT_COMMENTS, (C14NMethodParameterSpec) null),
                    sigFactory.newSignatureMethod(SignatureMethod.RSA_SHA256, null), Collections.singletonList(ref));


            KeyInfoFactory kif = sigFactory.getKeyInfoFactory();
            X509Data x509Data = kif.newX509Data(Collections.singletonList(cert));
            KeyInfo keyInfo = kif.newKeyInfo(Collections.singletonList(x509Data));

            XMLSignature signature = sigFactory.newXMLSignature(signedInfo,addKeyInfo? keyInfo:null);

            DOMSignContext signContext = new DOMSignContext(privateKey,
                    doc.getDocumentElement());
            signContext.setDefaultNamespacePrefix("ds");
            signature.sign(signContext);

            NodeList sigNodes = doc.getElementsByTagNameNS(XMLSignature.XMLNS, "Signature");
            if (sigNodes.getLength() > 0) {
                Element sigElem = (Element) sigNodes.item(0);

                // Renombrar nodo a AuthSignature (en el namespace EBICS, no en ds)
                doc.renameNode(sigElem, "urn:org:ebics:H005", "AuthSignature");
            }

            Element sigValueElement = (Element) doc.getElementsByTagNameNS("http://www.w3.org/2000/09/xmldsig#", "SignatureValue").item(0);
            if (sigValueElement != null) {
                log.debug("sigsignature value: {}",sigValueElement.getTextContent());
                String sigValue = sigValueElement.getTextContent().replace("\n", "").replace("\r", "");
                log.debug("signature value clean: {}",sigValue);
                sigValueElement.setTextContent(sigValue);
            }
        
    }
    
}

I tested the signed XML on this site and the result was this:

Signature Verified
Number of Reference Digests = 1
Reference 1 digest is valid.

I have checked that this works fine with one or more nodes matching the xpath expression in the source XML.