For anyone in the future who is curious, I ended up doing the type inference as part of parsing the response (with zod in this case).

export async function getUsers(): Promise<User[]> {
  const res = await api.get("/users");
  const parsed = userSchema.array().safeParse(res);

  if (!parsed.success) {
    throw new Error(`Users "${res}" is not valid.`);
  }

  return parsed.data;
}

Happy coding!