import boto3 import os
def download_large_file(client, bucket, key, local_path, chunk_size=102410245): multipart = client.list_objects_v2(Bucket=bucket, Prefix=key)['Contents'] total_size = sum([part['Size'] for part in multipart]) downloaded_size = 0 num_parts = len(multipart)
with open(local_path, 'wb') as f:
for i, part in enumerate(multipart):
offset = part["Size"] * i
end = min(offset + part["Size"], total_size)
while True:
try:
response = client.get_object(Bucket=bucket, Key=key, Range=f"bytes={offset}-{end}")
chunk = response["Body"].read(chunk_size)
while chunk:
f.write(chunk)
chunk = response["Body"].read(chunk_size)
downloaded_size += end - offset
break
except Exception as e:
print(f"Download Part {i+1}/{num_parts}: {str(e)} Retrying...")
if downloaded_size != total_size:
os.remove(local_path)
raise ValueError(f"Download for {key} is incomplete. Expected {total_size} bytes but downloaded {downloaded_size} bytes.")