Yes. It was impossible to switch directly. So, I made a working switch. Here's the fix:

A post on retrocomputing gives more details. Note that LOADALL apparently couldn't do it, but was wrongly rumored to be able to:

https://retrocomputing.stackexchange.com/questions/32242/is-it-possible-to-switch-from-32-bit-protected-mode-to-real-mode-without-going-t

Pm32 -> pm16 -> real 16 (wrap function caller) -> real 16 ( the function call) -> pm32 (resume 32) -> ret to original caller.

    uint16_t result = call_real_mode_function(add16_ref, 104, 201); // argc automatically calculated
    print_args16(&args16_start);
    terminal_write_uint("\nThe result of the real mode call is: ", result);

    uint16_t result2 = call_real_mode_function(complex_operation, 104, 201, 305, 43); // argc automatically calculated
    print_args16(&args16_start);
    terminal_write_uint("\nThe result of the real mode call is: ", result2);

// Macro wrapper: automatically counts number of arguments
#define call_real_mode_function(...) \
    call_real_mode_function_with_argc(PP_NARG(__VA_ARGS__), __VA_ARGS__)

// Internal function: explicit argc
uint16_t call_real_mode_function_with_argc(uint32_t argc, ...) {

    bool optional = false;
    if (optional) {
        // This is done later anyway. But might as well for now
        GDT_ROOT gdt_root = get_gdt_root();
        args16_start.gdt_root = gdt_root;

        uint32_t esp_value;
        __asm__ volatile("mov %%esp, %0" : "=r"(esp_value));
        args16_start.esp = esp_value;
    }

    va_list args;
    va_start(args, argc);

    uint32_t func = va_arg(args, uint32_t);
    struct realmode_address rm_address = get_realmode_function_address((func_ptr_t)func);
    args16_start.func = rm_address.func_address;
    args16_start.func_cs = rm_address.func_cs;

    args16_start.argc = argc - 1;

    for (uint32_t i = 0; i < argc; i++) {
        args16_start.func_args[i] = va_arg(args, uint32_t); // read promoted uint32_t
    }

    va_end(args);
    return pm32_to_pm16();
}

GDT16_DESCRIPTOR:
    dw GDT_END - GDT_START - 1 ;limit/size
    dd GDT_START ; base

GDT_START: 
    dq 0x0 
    dq 0x0
    dq 0x00009A000000FFFF ; code 
    dq 0x000093000000FFFF ; data
GDT_END:

section .text.pm32_to_pm16

pm32_to_pm16:
    mov eax, 0xdeadfac1

    ; Save 32-bit registers and flags
    pushad
    pushfd
    push ds
    push es
    push fs
    push gs

    ; Save the stack pointer in the first 1mb (first 64kb in fact)
    ; So its accessible in 16 bit, and can be restored on the way back to 32 bit
    sgdt [args16_start + GDT_ROOT_OFFSET]
    mov [args16_start +  ESP_OFFSET], esp    ; 
    mov ax, ss
    mov [args16_start +  SS_OFFSET], ax    ; 




    mov esp, 0 ; in case i can't change esp in 16 bit mode later. Don't want the high bit to fuck us over
    mov ebp, 0 ; in case i can't change esp in 16 bit mode later. Don't want the high bit to fuck us over


    cli

    lgdt [GDT16_DESCRIPTOR]
    jmp far 0x10:pm16_to_real16

/* Reference version (purely for comparison) */
__attribute__((section(".text.realmode_functions"))) int16_t complex_operation(uint16_t a, uint16_t b, uint16_t c, uint16_t d) {

    return 2 * a + b - c + 3 * d;
}

/* Reference version (purely for comparison) */
__attribute__((section(".text.realmode_functions"))) uint16_t add16_ref(uint16_t a, uint16_t b) {

    return 2 * a + b;
}

resume32:
    ; Restore segment registers


    mov esp, [args16_start + ESP_OFFSET]
    mov ax,  [args16_start + SS_OFFSET]
    mov ss, ax 
    mov ss, ax

    pop gs
    pop fs
    pop es
    pop ds

    ; Restore general-purpose registers and flags
    popfd
    popad


    ; Retrieve result
    movzx eax, word [args16_start + RET1_OFFSET]
    ; mov eax, 15

    ret

The struct located in the first 64kb of memory, to allow multi segment data passing.

typedef struct __attribute__((packed)) Args16 {
    GDT_ROOT gdt_root;
    // uint16_t pad; // (padded due to esp wanting to)
    uint16_t ss;
    uint32_t esp;

    uint16_t ret1;
    uint16_t ret2;

    uint16_t func;
    uint16_t func_cs;

    uint16_t argc;
    uint16_t func_args[13];

} Args16;

To see a simpler version of this:
commit message: "we are so fucking back. Nicegaga"

Commit date: Nov 7, 3:51 am

(gaga typo accidentally typed)

hash: 309ca54630270c81fa6e7a66bc93

and a more modern and cleaned (The one with the code show above):
commit message: Changed readme.

commit date: Sun Nov 9 18:09:16

commit hash: a2058ca7e3f99e92ea7c76909cc3f7846674dc83

====