If someone wants to update the points field every time the polygon is modified, here is some code that works:

  const polyWithRecalculatedPosition = {
      points: getPoints(polygon),
      flipX: false,
      flipY: false,
      scaleX: 1,
      scaleY: 1,
      angle: 0,
  };
  
  polygon.set(polyWithRecalculatedPosition);
  polygon.setBoundingBox(true);
  canvas.requestRenderAll();
        
        
  function getPoints(poly: Polygon): XY[] {
    const matrix = poly.calcTransformMatrix();
  
    return poly.get('points')
      .map(
        (p: Point) =>
          new Point(p.x - poly.pathOffset.x, p.y - poly.pathOffset.y),
      )
      .map((p: Point) => util.transformPoint(p, matrix));
  }

Where "polygon" means polygon object that we want update. Work with moving, scaling, skewing, resizing and fliping.