@BenzyNeez gave such an elegant answer, I thought I'd share a reusable view, based on Benzy's genius.

import SwiftUI

struct FancyNavTitleScrollView<TitleView: View, NavBarView: View, Content: View>: View {
    @State private var showingScrolledTitle = false
    
    let navigationTitle: String
    let titleView: () -> TitleView
    let navBarView: () -> NavBarView
    var transitionOffest: CGFloat = 30
    let content: () -> Content
    
    var body: some View {
        GeometryReader { outer in
            ScrollView {
                VStack {
                    titleView()
                        .opacity(showingScrolledTitle ? 0 : 1)
                    content()
                }
                .background {
                    scrollDetector(topInsets: outer.safeAreaInsets.top)
                }
            }
        }
        .toolbar {
            ToolbarItem(placement: .principal) {
                navBarView()
                    .opacity(showingScrolledTitle ? 1 : 0)
                    .animation(.easeInOut, value: showingScrolledTitle)
            }
        }
        .navigationTitle(navigationTitle)
        .navigationBarTitleDisplayMode(.inline)
    }
    
    private func scrollDetector(topInsets: CGFloat) -> some View {
        GeometryReader { proxy in
            let minY = proxy.frame(in: .global).minY
            let isUnderToolbar = minY - topInsets < -transitionOffest
            Color.clear
                .onChange(of: isUnderToolbar) { _, newVal in
                    showingScrolledTitle = newVal
                }
        }
    }
}


#Preview {
    NavigationStack {
        FancyNavTitleScrollView(
            navigationTitle: "Yesterday",
            titleView: {
                Text("Today")
                    .font(.custom("Chalkboard SE", size: 36))
                    .textCase(nil)
                    .bold()
            },
            navBarView: {
                Text("Today")
                    .font(.custom("Chalkboard SE", size: 16))
                    .foregroundStyle(Color.red)
            },
            content: {
                VStack {
                    ForEach(1...5, id: \.self) { val in
                        NavigationLink("List item \(val)") {
                            Text("List item \(val)")
                        }
                    }
                }
                .foregroundStyle(.indigo)
                .background(.indigo.opacity(0.1))
                .scrollContentBackground(.hidden)
                .toolbarBackground(.indigo.opacity(0.1))
                .toolbar {
                    ToolbarItem(placement: .topBarLeading) {
                        Image(systemName: "gearshape.fill")
                    }
                    ToolbarItem(placement: .topBarTrailing) {
                        Image(systemName: "calendar")
                            .padding(.trailing, 20)
                    }
                    ToolbarItem(placement: .topBarTrailing) {
                        Image(systemName: "plus.circle.fill")
                    }
                }
            }
        )
    }
}