I have found a way. I created a base struct that is only member variables with specializations for the 3 relevant sizes, and made my Vector struct its child.

template<typename T, int N> requires std::is_arithmetic_v<T> && (N > 1)
struct Data {
    T data[N];
};

template<typename T>
struct Data<T, 2> {
    union {
        T data[2];
        struct { T x, y; };
        struct { T r, g; };
        struct { T u, v; };
    };
};

template<typename T>
struct Data<T, 3> {
    union {
        T data[3];
        struct { T x, y, z; };
        struct { T r, g, b; };
    };
};

template<typename T>
struct Data<T, 4> {
    union {
        T data[4];
        struct { T x, y, z, w; };
        struct { T r, g, b, a; };
    };
};

export template <typename T, int N> requires std::is_arithmetic_v<T> && (N > 1)
struct Vector : Data<T, N> {
    ...
}