Laurens code uses a custom build version of Fabric.js that has not been updated in a while (as of December 2024) and is many versions behind. I build my own version of panning and zooming for Fabric 5.3.1 - both with mouse wheel and pinch gesture - using CSS transform which is vastly more performant and then updating the canvas after the zoom ended.

I created a CodePen showing the solution I propose here: https://codepen.io/Fjonan/pen/QwLgEby

I created a second CodePen comparing a pure Fabric.js with CSS transform: https://codepen.io/Fjonan/pen/azoWXWJ (Spoiler: CSS is a lot smoother)

So this is how you can achieve panning and zooming the entire canvas with both mouse and touch.

Setup something like this:

<section class="canvas-wrapper" style="overflow:hidden; position:relative;">
  <canvas id=canvas>
  </canvas>
</section>

Fabric.js will create its own wrapper element canvas-container which I access here using canvas.wrapperEl.

This code handles dragging with mouse:

const wrapper = document.querySelector('.canvas-wrapper')
const canvas = new fabric.Canvas("canvas",{
  allowTouchScrolling: false,
  defaultCursor: 'grab',
  selection: false,
  // …
})
let lastPosX, 
    lastPosY

canvas.on("mouse:down", dragCanvasStart)
canvas.on("mouse:move", dragCanvas)

/**
 * Save reference point from which the interaction started
 */
function dragCanvasStart(event) {
  const evt = event.e || event // fabricJS event or touch event
    
  // save the position you started dragging from
  lastPosX = evt.clientX
  lastPosY = evt.clientY
}

/**
 * Start dragging the canvas using Fabric.js events
 */
function dragCanvas(event) {    
  const evt = event.e || event // fabricJS event or touch event

  // left mouse button is pressed if not a touch event
  if (1 !== evt.buttons && !(evt instanceof Touch)) {
    return
  }
    
  translateCanvas(evt)
}

/**
 * Convert movement to CSS translate which visually moves the canvas
 */
function translateCanvas(event) {    
  const transform = getTransformVals(canvas.wrapperEl)

  let offsetX = transform.translateX + (event.clientX - (lastPosX || 0))
  let offsetY = transform.translateY + (event.clientY - (lastPosY || 0))

  const viewBox = wrapper.getBoundingClientRect()

  canvas.wrapperEl.style.transform = `translate(${tVals.translateX}px, ${tVals.translateY}px) scale(${transform.scaleX})`

  lastPosX = event.clientX
  lastPosY = event.clientY
}

/**
 * Get relevant style values for the given element
 * @see https://stackoverflow.com/a/64654744/13221239
 */
function getTransformVals(element) {
  const style = window.getComputedStyle(element)
  const matrix = new DOMMatrixReadOnly(style.transform)    
  return {
    scaleX: matrix.m11,
    scaleY: matrix.m22,
    translateX: matrix.m41,
    translateY: matrix.m42,
    width: element.getBoundingClientRect().width,
    height: element.getBoundingClientRect().height,
  }
}

And this code will handle mouse zoom:

let touchZoom
canvas.on('mouse:wheel', zoomCanvasMouseWheel)

// after scaling transform the CSS to canvas zoom so it does not stay blurry
// @see https://lodash.com/docs/4.17.15#debounce
const debouncedScale2Zoom = _.debounce(canvasScaleToZoom, 1000) 

/**
 * Zoom canvas when user used mouse wheel
 */
function zoomCanvasMouseWheel(event) {
  const delta = event.e.deltaY
  let zoom = touchZoom

  zoom *= 0.999 ** delta
  const point = {x: event.e.offsetX, y: event.e.offsetY}
    
  scaleCanvas(zoom, point)
  debouncedScale2Zoom()
}

/**
 * Convert zoom to CSS scale which visually zooms the canvas
 */
function scaleCanvas(zoom, aroundPoint) {
  const tVals = getTransformVals(canvas.wrapperEl)
  const scaleFactor = tVals.scaleX / touchZoom * zoom

  canvas.wrapperEl.style.transformOrigin = `${aroundPoint.x}px ${aroundPoint.y}px`
  canvas.wrapperEl.style.transform = `translate(${tVals.translateX}px, ${tVals.translateY}px) scale(${scaleFactor})`

  touchZoom = zoom
}

/**
 * Converts CSS transform to Fabric.js zoom so the blurry image gets sharp 
 */
function canvasScaleToZoom() {    
  const transform = getTransformVals(canvas.wrapperEl)
  const canvasBox = canvas.wrapperEl.getBoundingClientRect()
  const viewBox = wrapper.getBoundingClientRect()

  // calculate the offset of the canvas inside the wrapper
  const offsetX = canvasBox.x - viewBox.x
  const offsetY = canvasBox.y - viewBox.y

  // we resize the canvas to the scaled values
  canvas.setHeight(transform.height)
  canvas.setWidth(transform.width)
  canvas.setZoom(touchZoom)

  // and reset the transform values
  canvas.wrapperEl.style.transformOrigin = `0px 0px`
  canvas.wrapperEl.style.transform = `translate(${offsetX}px, ${offsetY}px) scale(1)`

  canvas.renderAll()
}

Now for touch events we have to attach our own event listeners since Fabric.js does not (yet) support touch events as part of their regularly handled listeners.

let pinchCenter,
    initialDistance

wrapper.addEventListener('touchstart', (event) => {
  dragCanvasStart(event.targetTouches[0])
  pinchCanvasStart(event)
})
    
wrapper.addEventListener('touchmove', (event) => {      
  dragCanvas(event.targetTouches[0])
  pinchCanvas(event)
})

wrapper.addEventListener('touchend', pinchCanvasEnd)

/**
 * Save the distance between the touch points when starting the pinch
 */
function pinchCanvasStart(event) {
  if (event.touches.length !== 2) {
    return
  }
    
  initialDistance = getPinchDistance(event.touches[0], event.touches[1])
}

/**
 * Start pinch-zooming the canvas
 */
function pinchCanvas(event) {
  if (event.touches.length !== 2) {
    return
  }

  setPinchCenter(event.touches[0], event.touches[1])

  const currentDistance = getPinchDistance(event.touches[0], event.touches[1])
  let scale = (currentDistance / initialDistance).toFixed(2)
  scale = 1 + (scale - 1) / 20 // slows down scale from pinch

  scaleCanvas(scale * touchZoom, pinchCenter)
}

/**
 * Re-Draw the canvas after pinching ended
 */
function pinchCanvasEnd(event) {
  if (2 > event.touches.length) {
    debouncedScale2Zoom()
  }
}

/**
 * Putting touch point coordinates into an object
 */
function getPinchCoordinates(touch1, touch2) {
  return {
    x1: touch1.clientX,
    y1: touch1.clientY,
    x2: touch2.clientX,
    y2: touch2.clientY,
  }
}

/**
 * Returns the distance between two touch points
 */
function getPinchDistance(touch1, touch2) {
  const coord = getPinchCoordinates(touch1, touch2)
  return Math.sqrt(Math.pow(coord.x2 - coord.x1, 2) + Math.pow(coord.y2 - coord.y1, 2))
}
  
/**
 * Pinch center around wich the canvas will be scaled/zoomed
 * takes into account the translation of the container element
 */
function setPinchCenter(touch1, touch2) {    
  const coord = getPinchCoordinates(touch1, touch2)

  const currentX = (coord.x1 + coord.x2) / 2
  const currentY = (coord.y1 + coord.y2) / 2

  const transform = getTransformVals(canvas.wrapperEl)
    
  pinchCenter = {
    x: currentX - transform.translateX,
    y: currentY - transform.translateY,
  }    
}

This effectively moves the canvas inside a wrapper with overflow: hidden and updates the canvas after zoom. Add to it some boundaries to avoid the canvas from being moved out of reach and limit the zoom and you will get a performant way to pan and zoom both for mouse and touch devices. You will find additional quality of life stuff like this in my CodePen demo I left out here to not make it too complicated.