Let's start with InteractiveViewer, instead of GestureDetector, as we need more control over transformation and interactions
I would recommend using such a structure for whiteboard layout:

Widget build(BuildContext context) {
  return LayoutBuilder(
    builder: (context, constraints) {
      return InteractiveViewer(
        transformationController: _transformationController,
        minScale: _minZoom,
        maxScale: _maxZoom,
        panEnabled: false,
        scaleEnabled: true,
        boundaryMargin: const EdgeInsets.all(double.infinity),
        onInteractionStart: onInteractionStart,
        onInteractionUpdate: onInteractionUpdate,
        onInteractionEnd: (_) => onInteractionEnd(),
        child: RepaintBoundary(
          child: SizedBox.fromSize(
            size: constraints.smallest,
            child: CustomPaint(
              key: _canvasKey,
              painter: CanvasCustomPainter(
                nodes: _nodes,
                offset: _lastFocalPoint,
                scale: _currentZoom,
                screenSize: constraints.biggest,
                transformationController: _transformationController,
              ),
            ),
          ),
        ),
      );
    },
  );
}

Now lets add a bit models for pen/drawwing logic, lets declare some abstraction, for example something like this:

abstract class WhiteboardNode {
  WhiteboardNode({required this.order});

  int order;

  NodeBoundingBox get boundingBox;

  void shift(Offset delta);
}

add a point model:

class DrawPoint extends Offset {
  DrawPoint(super.dx, super.dy, {this.visible = true});

  DrawPoint.fromOffset(Offset o)
      : visible = true,
        super(o.dx, o.dy);

  bool visible;

  @override
  DrawPoint translate(double translateX, double translateY) => DrawPoint(
        dx + translateX,
        dy + translateY,
      );
}

and finally a pen node:

class WhiteboardPenSettings {
  const WhiteboardPenSettings({
    required this.strokeWidth,
    required this.strokeCap,
    required this.currentColor,
    this.onDrawOptionChange,
  }) : assert(strokeWidth >= 0, "strokeWidth can't be negative");

  const WhiteboardPenSettings.initial()
      : strokeWidth = 2.5,
        strokeCap = StrokeCap.round,
        currentColor = AppColors.black,
        onDrawOptionChange = null;

  final double strokeWidth;
  final StrokeCap strokeCap;
  final Color currentColor;
  final ValueChanged<WhiteboardPenSettings>? onDrawOptionChange;
}

class NodeBoundingBox {
  const NodeBoundingBox({
    required this.rect,
    required this.paddingOffset,
  });

  static const NodeBoundingBox zero = NodeBoundingBox(
    rect: Rect.zero,
    paddingOffset: Offset.zero,
  );

  final Rect rect;
  final Offset paddingOffset;
}

class NodeExtremity {
  NodeExtremity({
    required this.left,
    required this.top,
    required this.right,
    required this.bottom,
  });

  NodeExtremity.initial()
      : left = 0,
        top = 0,
        right = 0,
        bottom = 0;

  double left;
  double top;
  double right;
  double bottom;
}

class PenNode extends WhiteboardNode {
  PenNode({
    required this.uuid,
    required super.order,
    required this.penSettings,
    required this.paintingStyle,
    required this.extremity,
    required this.points,
  });

  factory PenNode.fromSettings({
    required WhiteboardPenSettings settings,
    required int order,
  }) {
    return PenNode(
      uuid: const Uuid().v4(),
      penSettings: settings,
      paintingStyle: PaintingStyle.stroke,
      order: order,
      extremity: NodeExtremity.initial(),
      points: [],
    );
  }

  final String uuid;

  final List<DrawPoint> points;

  final PaintingStyle paintingStyle;
  final WhiteboardPenSettings penSettings;

  final NodeExtremity extremity;

  @override
  void shift(Offset delta) {
    for (var i = 0; i < points.length; i++) {
      points[i] = points[i].translate(delta.dx, delta.dy);
    }
  }

  @override
  NodeBoundingBox get boundingBox {
    if (points.isEmpty) return NodeBoundingBox.zero;

    var minX = double.infinity, minY = double.infinity;
    var maxX = double.negativeInfinity, maxY = double.negativeInfinity;

    for (final point in points) {
      if (point.dx < minX) minX = point.dx;
      if (point.dy < minY) minY = point.dy;
      if (point.dx > maxX) maxX = point.dx;
      if (point.dy > maxY) maxY = point.dy;
    }

    return NodeBoundingBox(
      rect: Rect.fromLTRB(minX, minY, maxX, maxY),
      paddingOffset: Offset.zero,
    );
  }
}

Soooo, yeah, we ready to go, lets focus now on custom painter logic:

class CanvasCustomPainter extends CustomPainter {
  CanvasCustomPainter({
    required this.nodes,
    required this.offset,
    required this.scale,
    required this.screenSize,
    this.transformationController,
    this.backgroundColor,
  });

  List<WhiteboardNode> nodes;

  double scale;
  Offset offset;
  Size screenSize;

  final Color? backgroundColor;

  TransformationController? transformationController;

  @override
  void paint(Canvas canvas, Size size) {
    if (backgroundColor is Color) {
      canvas.drawColor(backgroundColor!, BlendMode.src);
    }

    if (nodes.isEmpty) return;

    // we need order to pay attention to backward/forward layers
    nodes.sort((a, b) => a.order.compareTo(b.order));

    canvas.saveLayer(Rect.largest, Paint());

    for (final node in nodes) {
      if (node is! PenNode) continue;

      // if not on the screen, lets skip rendering it
      if (_checkScribbleInvisible(
        scale: scale,
        offset: offset,
        screenSize: screenSize,
        extremity: node.extremity,
      )) {
        break;
      }

      final paint = Paint()
        ..strokeCap = node.penSettings.strokeCap
        ..isAntiAlias = true
        ..color = node.penSettings.currentColor
        ..strokeWidth = node.penSettings.strokeWidth
        ..blendMode = BlendMode.srcOver;

      _drawAllPoints(points: node.points, canvas: canvas, paint: paint);
    }

    canvas.restore();
  }

  bool _checkScribbleInvisible({
    required double scale,
    required Offset offset,
    required Size screenSize,
    required NodeExtremity extremity,
  }) {
    if ((extremity.left == 0 ||
        extremity.right == 0 ||
        extremity.top == 0 ||
        extremity.bottom == 0)) {
      return false;
    }

    return (extremity.left + offset.dx < 0 && extremity.right + offset.dx < 0)
            // Check Right
            ||
            (extremity.right + offset.dx > (screenSize.width / scale) &&
                extremity.left + offset.dx > (screenSize.width / scale))
            // Check Top
            ||
            (extremity.top + offset.dy < 0 && extremity.bottom + offset.dy < 0)
            //    Check Bottom
            ||
            (extremity.bottom + offset.dy > (screenSize.height / scale) &&
                extremity.top + offset.dy > (screenSize.height / scale))
        ? true
        : false;
  }

  void _drawAllPoints({
    required Paint paint,
    required Canvas canvas,
    required List<DrawPoint> points,
  }) {
    for (var x = 0; x < points.length - 1; x++) {
      if (!points[x + 1].visible) continue;

      canvas.drawLine(points[x], points[x + 1], paint);
    }
  }

  @override
  bool shouldRepaint(CanvasCustomPainter oldDelegate) => true;
}

so now we support smart drawing, thinking about performance, and ready to add other nodes (eraser, shapes, images, text, etc.)

final step is to implement our methods to handle interactions with whiteboard:

enum WhiteboardPointerMode {
  none,
  singleTap,
  doubleTap;

  static WhiteboardPointerMode fromPointersCount(int count) {
    switch (count) {
      case 1:
        return singleTap;
      case 2:
        return doubleTap;
      default:
        return none;
    }
  }
}

int lastOrder = 0;
List<PenNode> nodes = [];
Offset lastFocalPoint = Offset.zero;
Offset? initialInteractionPoint;
WhiteboardPointerMode pointerMode = WhiteboardPointerMode.none;

// helper for transformation
Offset _toCurrentScene(
    TransformationController controller,
    Offset viewportPoint,
) {
    final inverseMatrix = Matrix4.tryInvert(controller.value);
    if (inverseMatrix is! Matrix4) return viewportPoint;
    return MatrixUtils.transformPoint(inverseMatrix, viewportPoint);
}

// helpers for drawing
List<PenNode> _startDrawing({
    required int order,
    required List<PenNode> nodes,
    required WhiteboardPenSettings penSettings,
}) {
    final node = PenNode.fromSettings(settings: penSettings, order: order);
    final tempNodes = List<PenNode>.from(nodes)..add(node);

    return tempNodes;
}

List<PenNode> _updateDrawing({
    required Offset point,
    required List<PenNode> scribbles,
}) {
    final tempNodes = List<PenNode>.from(scribbles);
    tempNodes.lastOrNull?.points.add(DrawPoint.fromOffset(point));

    return tempNodes;
}

void onInteractionStart(ScaleStartDetails details) {
     final pointerMode = WhiteboardPointerMode.fromPointersCount(
      details.pointerCount,
    );
    // we support only one finger (pointer) for drawing
    if (!pointerMode.isSingle) return;

    final lastFocalPoint = details.focalPoint;
    final point = _toCurrentScene(
      _transformationController,
      lastFocalPoint,
    );

    // start drawing here
    final penNodes = startDrawing(
        penSettings: _penSettings,
        nodes: nodes,
        order: lastOrder + 1,
    );

    pointerMode = WhiteboardPointerMode.singleTap;
    lastFocalPoint = point;
    nodes = penNodes;

    setState((){});
}

// here we will handle zoom, move and drawing at once
void onInteractionUpdate(ScaleUpdateDetails details) {
     final pointerMode = WhiteboardPointerMode.fromPointersCount(
      details.pointerCount,
    );
    final scale = _transformationController.value.getMaxScaleOnAxis();

    if (_currentZoom != scale) {
      _currentZoom = scale;
      setState((){});
    }

    // we handled zoom/move with two fingers, for drawing we need only 1
    if (!pointerMode.isSingle) return;

    final point = _toCurrentScene(
      _transformationController,
      details.localFocalPoint,
    );

    if (nodes.isEmpty) return;

    final penNodes = _updateDrawing(
        scribbles: nodes,
        point: point,
    );

    nodes = penNodes;
    setState((){});
}


// once interactions ended - reset
void onInteractionEnd() {
    pointerMode = WhiteboardPointerMode.fromPointersCount(0);
    initialInteractionPoint = null;
    setState((){});
}

And thats actually it, hope i didn't miss anything cause my original implementation is using BLOC for state and events, so feel free to comment if you have some issues or questions