I ended up making a custom AdaptiveVGrid implementation, loosely based on this tutorial: https://www.fivestars.blog/articles/adaptive-swiftui-views/

I'm very new to SwiftUI, so there's likely some issues with the implementation. Take it with a grain of salt. It's quite verbose at least.

I'll leave this question open in case anyone has a better approach.

/// A view to display items in a grid, while adapting to the available space, item size, and number of items.
/// When items can fit in one row or column, prefer that (depending on whether we have more horizontal or vertical space).
/// Otherwise uses a LazyVGrid insize a ScrollView to display the content.
///
/// content should have the number of subviews specified as numItems.
struct AdaptiveVGrid<Content: View>: View {
  var numItems: Int
  var itemMinSize: CGSize
  var itemMaxSize: CGSize
  var itemSpacing: CGFloat
  var content: Content
  
  public init(
    numItems: Int,
    itemMinSize: CGSize,
    itemMaxSize: CGSize,
    itemSpacing: CGFloat,
    @ViewBuilder content: () -> Content
  ) {
    self.numItems = numItems
    self.itemMinSize = itemMinSize
    self.itemMaxSize = itemMaxSize
    self.itemSpacing = itemSpacing
    self.content = content()
  }
  
  var body: some View {
    GeometryReader { geometry in
      bodyImpl(availableSize: geometry.size)
        .frame(minWidth: geometry.size.width, minHeight: geometry.size.height)
    }
  }
  
  @ViewBuilder
  func bodyImpl(availableSize: CGSize) -> some View {
    let widthRatio = availableSize.width / (CGFloat(numItems) * itemMaxSize.width + CGFloat(numItems - 1) * itemSpacing)
    let heightRatio = availableSize.height / (CGFloat(numItems) * itemMaxSize.height + CGFloat(numItems - 1) * itemSpacing)
    
    if widthRatio >= heightRatio && availableSize.width >= (CGFloat(numItems) * itemMinSize.width + CGFloat(numItems - 1) * itemSpacing) {
      HStack(spacing: itemSpacing) { content }
    } else if heightRatio >= widthRatio && availableSize.height >= (CGFloat(numItems) * itemMinSize.height + CGFloat(numItems - 1) * itemSpacing) {
      VStack(spacing: itemSpacing) { content }
    } else {
      ScrollView {
        let columns = [GridItem(.adaptive(minimum: itemMinSize.width, maximum: itemMaxSize.width), spacing: itemSpacing)]
        LazyVGrid(columns: columns, alignment: .center) {
          content
        }
        .frame(minWidth: availableSize.width, minHeight: availableSize.height)
      }
    }
  }
}

struct MyView: View {
  @ObservedObject var viewModel: MyModel
  @ScaledMetric var accessabilityScale: CGFloat = 1
  
  @State private var availableSize: CGSize = .zero
  private let itemMinSizeBase = CGSize(width: 100, height: 100)
  private let itemMaxSizeBase = CGSize(width: 250, height: 250)
  private let itemSpacingBase = 20.0
  
  var body: some View {
    GeometryReader { geometry in
      AdaptiveVGrid(
        numItems: viewModel.items.count,
        itemMinSize: adjustSize(itemMinSizeBase, availableSize: geometry.size),
        itemMaxSize: adjustSize(itemMaxSizeBase, availableSize: geometry.size),
        itemSpacing: itemSpacingBase * accessabilityScale
      ) {
        ForEach(viewModel.items, id: \.id) { item in
          itemView(item: item)
            .frame(minWidth: adjustSize(itemMinSizeBase, availableSize: geometry.size).width,
                   maxWidth: CGFloat.greatestFiniteMagnitude,
                   minHeight: adjustSize(itemMinSizeBase, availableSize: geometry.size).height,
                   maxHeight: adjustSize(itemMaxSizeBase, availableSize: geometry.size).height
            )
        }
        .padding()
      }
    }
  }
  
  private func adjustSize(_ size: CGSize, availableSize: CGSize) -> CGSize {
    CGSize(width: min(size.width, availableSize.width),
           height: min(size.height * accessabilityScale, availableSize.height))
  }
}