Ok, so as suggested by @pskink I've added a logic in which every time the user takes his fingers out of the screen -- or, in other words, finish a line, I store it as an image and erase the previous list of points.

It looks like this:

class DrawingCanvas extends StatelessWidget {
  const DrawingCanvas({
    super.key,
    required this.onTouchStart,
    required this.onTouchUpdate,
    required this.onTouchEnd,
    required this.onCachingDrawing,
    required this.pointsAdded,
    required this.selectedPainter,
    required this.cachedDrawing,
    required this.shouldCacheDrawing,
    required this.pageOneImage,
    this.pageTwoImage,
    required this.child,
  });

  final Widget child;
  final List<DrawingDetails> pointsAdded;
  final void Function(Offset) onTouchStart;
  final void Function(Offset) onTouchUpdate;
  final void Function() onTouchEnd;
  final void Function(ui.Image) onCachingDrawing;
  final ui.Image? cachedDrawing;
  final bool shouldCacheDrawing;
  final Paint selectedPainter;
  final ui.Image? pageOneImage;
  final ui.Image? pageTwoImage;

  @override
  Widget build(BuildContext context) {
    return GestureDetector(
      onPanStart: (details) {
        onTouchStart(details.globalPosition);
      },
      onPanUpdate: (details) {
        onTouchUpdate(details.globalPosition);
      },
      onPanEnd: (_) {
        onTouchEnd();
      },
      child: ClipPath(
        child: CustomPaint(
          isComplex: true,
          willChange: true,
          foregroundPainter: _DrawingPainter(
            drawings: pointsAdded,
            selectedPainter: selectedPainter,
            onCachingDrawing: onCachingDrawing,
            cachedDrawing: cachedDrawing,
            shouldCacheDrawing: shouldCacheDrawing,
            pageOneImage: pageOneImage,
            pageTwoImage: pageTwoImage,
          ),
          child: child,
        ),
      ),
    );
  }
}

class _DrawingPainter extends CustomPainter {
  final List<DrawingDetails> drawings;
  final Paint selectedPainter;
  final Logger logger = Logger('_DrawingPainter');
  final Function(ui.Image) onCachingDrawing;
  final bool shouldCacheDrawing;
  final ui.Image? cachedDrawing;
  final ui.Image? pageOneImage;
  final ui.Image? pageTwoImage;

  _DrawingPainter({
    required this.drawings,
    required this.selectedPainter,
    required this.onCachingDrawing,
    required this.shouldCacheDrawing,
    required this.pageOneImage,
    this.pageTwoImage,
    this.cachedDrawing,
  });

  @override
  bool shouldRepaint(_DrawingPainter oldDelegate) {
    return (drawings.isNotEmpty &&
            (drawings.length == 1 && drawings[0].points.isNotEmpty)) &&
        oldDelegate.drawings != drawings;
  }

  @override
  void paint(Canvas canvas, Size size) {
    canvas.saveLayer(Rect.largest, Paint());

    final pictureRecorder = ui.PictureRecorder();
    final pictureCanvas = Canvas(pictureRecorder);

    if (cachedDrawing != null) {
      pictureCanvas.drawImage(cachedDrawing!, Offset.zero, Paint());
    }

    for (DrawingDetails drawing in drawings) {
      if (drawing.points.isEmpty) continue;
      if (isPointMode(drawing)) {
        pictureCanvas.drawPoints(
          ui.PointMode.points,
          [drawing.points[0]!],
          drawing.paint,
        );
      } else {
        for (int i = 0; i < drawing.points.length - 1; i++) {
          if (drawing.points[i] != null && drawing.points[i + 1] != null) {
            pictureCanvas.drawLine(
              drawing.points[i]!,
              drawing.points[i + 1]!,
              drawing.paint,
            );
          }
        }
      }
    }

    final picture = pictureRecorder.endRecording();

    canvas.drawPicture(picture);

    if (shouldCacheDrawing) {
      final ui.Image cachedImage = picture.toImageSync(
        size.width.toInt(),
        size.height.toInt(),
      );
      onCachingDrawing(cachedImage);
    }

    canvas.restore();
  }

  bool isPointMode(DrawingDetails drawing) =>
      drawing.points.length == 1 && drawing.points[0] != null;
}

The key is avoiding caching it at every frame using a flag, such as shouldCacheDrawing.

So, thanks you guys and sorry for the delay to post the result.