Here's an attempt to do it in SwiftUI based on @Fattie's answer.

This answer uses the native ScrollView with onScrollGeometryChange and scrollTo(y:) API's.

The movement is jittery unfortunately. I'm not sure if the jitteriness is from a weakness in my code and it could be done in a more optimized way, or if it's a weakness in SwiftUI as opposed to UiKit that we should report to Apple, and that the Apple Calendar app was made in UiKit instead of SwiftUI.

Also I simplified the code in the question to just the minimal code to focus on the anchoring part.

Open for suggestions and improvements.

import SwiftUI

struct ScrollData: Equatable {
    let height: CGFloat
    let offset: CGFloat
    let inset: CGFloat
}


final class HourItem: Identifiable {
    let id: UUID
    let hour: Int

    init(hour: Int) {
        self.id = UUID()
        self.hour = hour
    }
}

struct TestView: View {
    let minHourHeight: CGFloat = 50
    let maxHourHeight: CGFloat = 400

    @State private var isZooming: Bool = false
    @State private var previousZoomAmount: CGFloat = 0.0
    @State private var currentZoomAmount: CGFloat = 0.0

    @State private var position: ScrollPosition = ScrollPosition()
    @State private var scrollData = ScrollData(height: .zero, offset: .zero, inset: .zero)
    @State private var anchor: CGFloat = 0
    @State private var height: CGFloat = 0

    private var zoomAmount: CGFloat {
        1 + currentZoomAmount + previousZoomAmount
    }

    private var hourHeight: CGFloat {
        100 * zoomAmount
    }

    private let currentTime: Date = Date.now
    private let hourItems = (0..<25).map {
        HourItem(hour: $0)
    }

    var body: some View {
        ScrollView {
            VStack(spacing: 0) {
                ForEach(hourItems) { hourItem in
                    HourMarkView(
                        hour: hourItem.hour,
                        height: hourHeight,
                        currentTime: currentTime
                    )
                }
            }
            .simultaneousGesture(magnification)
        }
        .scrollPosition($position)
        .onScrollGeometryChange(for: ScrollData.self) { geometry in
            ScrollData(
                height: geometry.contentSize.height,
                offset: geometry.contentOffset.y,
                inset: geometry.contentInsets.top,
            )
        } action: { oldValue, newValue in
            if oldValue != newValue {
                scrollData = newValue
            }
        }
    }

    private var magnification: some Gesture {
        MagnifyGesture(minimumScaleDelta: 0)
            .onChanged(handleZoomChange)
            .onEnded(handleZoomEnd)
    }

    private func handleZoomChange(_ value: MagnifyGesture.Value) {
        if !isZooming {
            anchor = value.startAnchor.y
            height = scrollData.height
            isZooming = true
        }

        let gestureScreenOffset = value.startLocation.y - (scrollData.offset+scrollData.inset)

        let newZoomAmount = value.magnification - 1
        currentZoomAmount = clampedZoomAmount(newZoomAmount)

        position.scrollTo(y: (anchor*scrollData.height) -  gestureScreenOffset)

    }

    private func handleZoomEnd(_: MagnifyGesture.Value) {
        isZooming = false
        previousZoomAmount += currentZoomAmount
        currentZoomAmount = 0
    }

    private func clampedZoomAmount(_ newZoomAmount: CGFloat) -> CGFloat {
        if hourHeight > maxHourHeight && newZoomAmount > currentZoomAmount {
            return currentZoomAmount - 0.000001
        } else if hourHeight < minHourHeight && newZoomAmount < currentZoomAmount {
            return currentZoomAmount + 0.000001
        }

        return newZoomAmount
    }
}

struct HourMarkView: View {
    var hour: Int
    var height: CGFloat
    var currentTime: Date

    var body: some View {
        HStack(spacing: 10) {
            Text(formatTime(hour))
                .font(.caption)
                .fontWeight(.medium)
                .frame(width: 40, alignment: .trailing)
            Rectangle()
                .fill(Color.gray)
                .frame(height: 1)
        }
        .frame(height: height)
        .background(Color.white)
    }

    private func formatTime(_ hour: Int) -> String {
        return String(format: "%02d:00", hour)
    }
}