I found a way to fix this behaviour, anyone who might need it in the future here it is:

    document.addEventListener("pointerdown", (event: PointerEvent) => {
      const target = event.target as HTMLElement;
      target.setPointerCapture(event.pointerId);
    });

    document.addEventListener("pointerup", (event: PointerEvent) => {
      const target = event.target as HTMLElement;
      target.releasePointerCapture(event.pointerId);
    });