I have an idea, if the phone number is optional, you can do the following set up. I tells allauth than the phone number is optional.

ACCOUNT_SIGNUP_FIELDS = [
    "phone",  # <-- remove the asterisk (*)
    "email*",
    "username*",
    "password1",
    "password2"
]