commit a742d967b190b295de060fea86e73294009c5c5f
from: Nicolas Dandrimont <nicolas@dandrimont.eu>
date: Mon Jan 29 16:10:02 2024 UTC

loader: add some logging during packfile fetching

The packfile fetching operation can take a long time. Send one log line
every minute while it progresses.

commit - 4c9b38eda9a90a091d25041dc45c8669adf9639e
commit + a742d967b190b295de060fea86e73294009c5c5f
blob - 61d717d8fd989813ca141faf91da108914673579
blob + 25713898777ac2449410cc2a0203ab2175175064
--- swh/loader/git/loader.py
+++ swh/loader/git/loader.py
@@ -64,6 +64,7 @@ from .utils import HexBytes
 logger = logging.getLogger(__name__)
 heads_logger = logger.getChild("refs")
 remote_logger = logger.getChild("remote")
+fetch_pack_logger = logger.getChild("fetch_pack")
 
 # How often to log messages for long-running operations, in seconds
 LOGGING_INTERVAL = 30
@@ -274,16 +275,29 @@ class GitLoader(BaseGitLoader):
         logger.debug("Client %s to fetch pack at %s", client, path)
 
         size_limit = self.pack_size_bytes
+
+        last_line_logged = time.monotonic()
 
         def do_pack(data: bytes) -> None:
+            nonlocal last_line_logged
+
             cur_size = pack_buffer.tell()
             would_write = len(data)
-            if cur_size + would_write > size_limit:
+            fetched = cur_size + would_write
+            if fetched > size_limit:
                 raise IOError(
                     f"Pack file too big for repository {origin_url}, "
                     f"limit is {size_limit} bytes, current size is {cur_size}, "
                     f"would write {would_write}"
+                )
+
+            if time.monotonic() > last_line_logged + LOGGING_INTERVAL:
+                fetch_pack_logger.info(
+                    "Fetched %s packfile bytes so far (%.2f%% of configured limit)",
+                    fetched,
+                    100 * fetched / size_limit,
                 )
+                last_line_logged = time.monotonic()
 
             pack_buffer.write(data)