Found a solution based on Typescript convert type Tuple of tuple to Tuple (flatten Tuple):

// Solution using recursive type
type FlattenSortOptions<T> = T extends readonly [infer First, ...infer Rest]
  ? First extends { children: readonly unknown[] }
    ? [...FlattenSortOptions<First["children"]>, ...FlattenSortOptions<Rest>]
    : [First, ...FlattenSortOptions<Rest>]
  : [];

function flattenSortOptions<T extends SortOptions<unknown>>(
  options: T,
): FlattenSortOptions<T> {
  return options.flatMap((option) => {
    if (typeof option === "object" && option !== null && "children" in option) {
      return option.children;
    }
    return option;
  }) as FlattenSortOptions<T>;
}