You could use ThreadPoolExecutor and initialize workers who have a shared memory but it's affected by GIL.
you could use this simple code
from concurrent.futures import ThreadPoolExecutor
import tarfile
import os
def extract_file(fullpath, destination):
try:
with tarfile.open(fullpath, 'r:gz') as tar:
tar.extractall(destination)
except Exception as e:
print(f"Error extracting {fullpath}: {e}")
def unziptar_parallel(path):
tar_files = []
for root, dirs, files in os.walk(path):
for file in files:
if file.endswith(".tar.gz"):
fullpath = os.path.join(root, file)
tar_files.append((fullpath, root))
with ThreadPoolExecutor(max_workers=4) as executor:
tasks = []
for fullpath, destination in tar_files:
task = executor.submit(extract_file, fullpath, destination)
tasks .append(future)
# انتظار انتهاء جميع المهام
for task in tasks :
task.result()
path = 'your path'
unziptar_parallel(path)
Check these resources for more information: