It seems the culprit was somehow the header information in the curl request. So I found this solution where the access token is placed inside the URL. The critical piece here is the syntax of the env variable CURL_URL:

# ...
download-file:
    name: download file
    runs-on: ubuntu-latest

    env:
      CURL_O: "docker-compose.yml"
      CURL_PATH: "raw.githubusercontent.com/<username>/<repository name>/refs/heads/main/docker-compose.prod.yml"

    steps:
      - name: install ssh keys
        # check this thread to understand why its needed:
        # <https://stackoverflow.com/a/70447517>
        run: |
          install -m 600 -D /dev/null ~/.ssh/id_rsa
          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
          host='${{ secrets.SSH_HOST }}'
          hosts="$(dig +short "$host" | grep -v '\.$' | sed -z 's|\n|,|g')$host"
          ssh-keyscan -H "$hosts" > ~/.ssh/known_hosts

      - name: test pull single file from private repo
        env:
          CURL_URL: "https://${{secrets.PULL_ACCESS_TOKEN}}@${{env.CURL_PATH}}"
        run: |
          ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} <<'ENDSSH'
          cd test
          ls -la
          curl ${{env.CURL_URL}} -o ${{env.CURL_O}}
          ENDSSH

      - name: cleanup
        run: rm -rf ~/.ssh