uniworx/k8s-gitlab-borg
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

retry download if minio closes connection

Browse Source
This commit is contained in:
Gregor Kleen 2023-05-29 14:40:35 +02:00
parent ae53231065
commit ace6dbd49f
1 changed files with 28 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
k8s_gitlab_borg/__main__.py
View File

@ -16,6 +16,8 @@ import logging
from base64 import b64decode
from minio import Minio
from http.client import IncompleteRead
from time import sleep
from datetime import datetime, timezone, timedelta
from dateutil.tz import gettz, UTC
@ -401,12 +403,32 @@ def main():
logger.debug("%s", {"import_args": import_args, "env": env})
try:
download = minio.get_object(
bucket_name="gitlab-backups",
object_name=backup.filename,
version_id=backup.version_id,
)
download_stream = download.stream(decode_content=True)
def download_iter():
offset = 0
retries = 10
while True:
download = minio.get_object(
bucket_name="gitlab-backups",
object_name=backup.filename,
version_id=backup.version_id,
offset=offset,
)
download_stream = download.stream(decode_content=True)
try:
while chunk := next(download_stream, b""):
offset += len(chunk)
retries = 10
yield chunk
else:
break
except IncompleteRead as e:
if retries <= 0:
raise e
retries -= 1
sleep(10)
download_stream = download_iter()
with subprocess.Popen(
import_args,