Ok, so I came up with this custom router implementation to achieve the desired behaviour I want

import SwiftUI

protocol Closable {
  func close()
}

class Router: ObservableObject {
  @Published var values: [Int: Closable?] = [:]
  @Published var path: [Int] = [] {
    didSet {
      let difference: CollectionDifference<Int> = path.difference(from: oldValue)
      difference.forEach { change in
        switch change {
        case .remove(_, let key, _):
          values.removeValue(forKey: key)??.close()
        default:
          break
        }
      }
    }
  }
  
  func register(key: Int, value: Closable) {
    values[key] = value
  }
  
  func push(key: Int){
    values[key] = nil
    path.append(key)
  }
  
  func pop(){
    let key = path.removeLast()
    values.removeValue(forKey: key)??.close()
  }
}

struct TimerList: View {
  @State private var times = [0, 1, 2, 3, 4, 5]
  @StateObject var router = Router()
  
  var body: some View {
    NavigationStack(path: $router.path) {
      List(times, id: \.self) { time in
        Button(
          action: { router.push(key: time) },
          label: {
            Text("\(time)")
          }
        )
      }
      .navigationDestination(for: Int.self) { time in
        TimerView(time: time)
          .environmentObject(router)
      }
    }
  }
}

class TimerViewModel: ObservableObject, Closable {
  let initial: Int
  @Published var time: Int
  
  private var task: Task<Void, Never>? = nil
  
  init(time: Int) {
    self.initial = time
    self.time = time
  }
  
  func close() {
    task?.cancel()
  }
  
  @MainActor
  func start() {
    if task != nil { return }
    task = Task { [weak self] in
      guard let self = self else { return }
      repeat {
        do { try await Task.sleep(nanoseconds: 1_000_000_000) }
        catch { return }
        self.time += 1
        print("Timer \(initial) incremented to \(time)")
      } while !Task.isCancelled
    }
  }
}

struct TimerView: View {
  let time: Int
  @EnvironmentObject var router: Router
  @StateObject var viewModel: TimerViewModel
  
  init(time: Int) {
    self.time = time
    _viewModel = StateObject(wrappedValue: TimerViewModel(time: time))
  }
  
  var body: some View {
    VStack {
      Text("Timer #\(viewModel.initial) is \(viewModel.time)")
      NavigationLink(value: time + 1, label: { Text("Next") })
    }
    .onAppear {
      viewModel.start()
      router.register(key: time, value: viewModel)
    }
  }
}

Not sure if this is the best way to do it, but it does work