2024 version based on fabric 6.4.3 and @VitaliiBratok answer:

polygon.on('modified', (o) => {
      const newCoords = polygon.get('points')
        .map(
          (p: Point) =>
            new Point(p.x - polygon.pathOffset.x, p.y - polygon.pathOffset.y),
        )
        .map((p: Point) => util.transformPoint(p, polygon.calcTransformMatrix()));

         polygon.set({'points': newCoords });
        // (...)
});