The above logic was good, something was wrong with my prototype. Here is some drier code that I can confirm works on ESP32C6 that also implements a suggestion by @romkey. This is running with a 10K external resistor and not esp_sleep_pd_config().

#include <Arduino.h>
#include <esp_sleep.h>

uint64_t gpio_pin_mask = (1ULL << 4);
RTC_DATA_ATTR unsigned btn_state;

void setup() {
  Serial.begin(115200);
  pinMode(4, INPUT_PULLUP);
}

void loop() {
  btn_state = digitalRead(4);
  Serial.println("btn state " + String(btn_state));
  if (btn_state == 0) {
    esp_sleep_enable_ext1_wakeup_io(gpio_pin_mask, ESP_EXT1_WAKEUP_ANY_HIGH);
  } else {
    esp_sleep_enable_ext1_wakeup_io(gpio_pin_mask, ESP_EXT1_WAKEUP_ANY_LOW);
  }
  Serial.println("Entering sleep...");
  delay(1500);
  esp_deep_sleep_start();
}