Adding rm -rf /var/lib/apt/lists/*
after my apt update command solved the problem.
# Install CUDA and dependencies in a single RUN for better caching
RUN cd /tmp \
&& mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 \
&& dpkg -i cuda-repo-ubuntu2004-12-4-local_12.4.0-550.54.14-1_amd64.deb \
&& cp /var/cuda-repo-ubuntu2004-12-4-local/cuda-*-keyring.gpg /usr/share/keyrings/ \
&& apt update \
&& apt install -y --no-install-recommends cuda-toolkit-12-4 mesa-utils \
&& rm -rf /var/lib/apt/lists/* \
&& export PATH=$PATH:/usr/local/cuda/bin \
&& tar -xvf v12.4.tar.gz \
&& cd cuda-samples-12.4 \
&& make -j $(nproc)