I know this is an old post but if is useful for anyone, I ran into the same problem trying to make an application launcher with an option to close when unfocused, my solution was to use a g_timeout (if I didn't do it, the program went ahead and closed) and when that timer finished, check a couple of things:

ismoving=0;

gboolean on_button_press(GtkWidget *widget, GdkEventButton *event, gpointer data)
{
   // check for mouse button 1 to set ismoving
    if (event->type == GDK_BUTTON_PRESS && event->button == 1)
    {
        ismoving = 1;
    }
    //GDK_BUTTON_RELEASE doesn't work for me in Wayland
    return FALSE;
}

gboolean close_window_if_unfocused(gpointer widget)
{
    //submenu is a popup menu, dialog is a global variable that i use for dialogs
    if (gtk_widget_get_visible(submenu) || gtk_widget_get_visible(dialog) || ismoving)
    {
        ismoving = 0; // Changing this is a workaround because i don't know other way in Wayland
        return FALSE;
    }

    // I also check for modifier keys
    GdkModifierType modifier_state = gdk_keymap_get_modifier_state(gdk_keymap_get_for_display(gdk_display_get_default()));
    GdkSeat *seat = gdk_display_get_default_seat(gdk_display_get_default());
    GdkDevice *pointer = gdk_seat_get_pointer(seat);
    guint button_state = 0;

    gdk_device_get_state(pointer, gtk_widget_get_window(GTK_WIDGET(widget)), NULL, &button_state);

    if (!gtk_window_has_toplevel_focus(GTK_WINDOW(widget)) &&
        !(modifier_state & (GDK_SHIFT_MASK | GDK_CONTROL_MASK | GDK_MOD1_MASK | GDK_SUPER_MASK)))
    {
        gtk_widget_destroy(GTK_WIDGET(widget));
    }
    return FALSE;
}

gboolean on_focus_out(GtkWidget *widget, GdkEventFocus *event, gpointer user_data)
{
    g_timeout_add(100, close_window_if_unfocused, widget);
    return FALSE;
}

...

g_signal_connect(window, "key-release-event", G_CALLBACK(on_key_release), NULL); // check input
g_signal_connect(window, "focus-out-event", G_CALLBACK(on_focus_out), window);

Please let me know if you know a better solution, this works for my situation