From ace6dbd49f117cbf82cd4877d39ff92e745bb165 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 29 May 2023 14:40:35 +0200 Subject: [PATCH] retry download if minio closes connection --- k8s_gitlab_borg/__main__.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/k8s_gitlab_borg/__main__.py b/k8s_gitlab_borg/__main__.py index aa92f31..38343fb 100644 --- a/k8s_gitlab_borg/__main__.py +++ b/k8s_gitlab_borg/__main__.py @@ -16,6 +16,8 @@ import logging from base64 import b64decode from minio import Minio +from http.client import IncompleteRead +from time import sleep from datetime import datetime, timezone, timedelta from dateutil.tz import gettz, UTC @@ -401,12 +403,32 @@ def main(): logger.debug("%s", {"import_args": import_args, "env": env}) try: - download = minio.get_object( - bucket_name="gitlab-backups", - object_name=backup.filename, - version_id=backup.version_id, - ) - download_stream = download.stream(decode_content=True) + + def download_iter(): + offset = 0 + retries = 10 + while True: + download = minio.get_object( + bucket_name="gitlab-backups", + object_name=backup.filename, + version_id=backup.version_id, + offset=offset, + ) + download_stream = download.stream(decode_content=True) + try: + while chunk := next(download_stream, b""): + offset += len(chunk) + retries = 10 + yield chunk + else: + break + except IncompleteRead as e: + if retries <= 0: + raise e + retries -= 1 + sleep(10) + + download_stream = download_iter() with subprocess.Popen( import_args,