Here's what I've come up with. It's similar to what I had originally done but I wanted to avoid the hacky extension on the View. I don't really get why we can't use a ternary operator in .labelStyle so if you have any ideas how to make it look better be my guest.

This answer for the extension: https://stackoverflow.com/a/72489274/1573326

import SwiftUI

struct SwiftUIView: View {
    @State private var hasStartDate = false
    @State private var startDate: Date? = nil
    var body: some View {
        HStack(alignment: .center) {
            Button {
                withAnimation {
                    hasStartDate.toggle()
                    if !hasStartDate {
                        startDate = nil
                    }
                }
            } label: {
                Label {
                    Text(hasStartDate ? "Remove Start Date" : "Add Start Date")
                        .foregroundColor(.primary)
                } icon: {
                    Image(systemName: hasStartDate ? "minus.circle.fill" : "plus.circle.fill")
                        .renderingMode(.original)
                        .foregroundColor(hasStartDate ? .red : .green) // Tint for remove action
                        .imageScale(.small)
                }
                .labelStyle(includingText: hasStartDate ? false  : true)
            }
            
            if hasStartDate {
                DatePicker(
                    "Start Date",
                    selection: Binding(
                        get: { startDate ?? Date() },
                        set: { startDate = $0 }
                    ),
                    displayedComponents: .date
                )
                .datePickerStyle(.compact)
                //.padding(.leading)
                // Debug control position
                .border(.green, width: 1)
                // End debug control position
            }
        }    }
}

extension View {
    @ViewBuilder
    func labelStyle(includingText: Bool) -> some View {
        if includingText {
            self.labelStyle(.titleAndIcon)
        } else {
            self.labelStyle(.iconOnly)
        }
    }
}