Commit Diff

Commit:
8848bd310b2841ff7f8c0bde78f1339b09d57c63
From:
Nicolas Dandrimont <nicolas@dandrimont.eu>
Date:
Message:
loader: add shortcuts for the connect and read timeouts This sets the connect and read timeout for both the smart loader (via urllib3/dulwich) and for the dumb loader (via requests).
Actions:
Patch | Tree
commit - c6b2b577a7ff49c11c5d1e46162c0acba48305d5
commit + 8848bd310b2841ff7f8c0bde78f1339b09d57c63
blob - d03f399da60b5916b96eaa891f278e65d989c977
blob + d4fa48814ac273c160bd930be094e137fd2b0a4b
--- swh/loader/git/loader.py
+++ swh/loader/git/loader.py
@@ -29,6 +29,7 @@ import dulwich.client
 from dulwich.object_store import ObjectStoreGraphWalker
 from dulwich.objects import Blob, Commit, ShaFile, Tag, Tree
 from dulwich.pack import PackData, PackInflater
+import urllib3.util
 
 from swh.core.statsd import Statsd
 from swh.loader.exception import NotFound
@@ -179,6 +180,8 @@ class GitLoader(BaseGitLoader):
         repo_representation: Type[RepoRepresentation] = RepoRepresentation,
         pack_size_bytes: int = 4 * 1024 * 1024 * 1024,
         temp_file_cutoff: int = 100 * 1024 * 1024,
+        connect_timeout: float = 120,
+        read_timeout: float = 60,
         urllib3_extra_kwargs: Dict[str, Any] = {},
         requests_extra_kwargs: Dict[str, Any] = {},
         **kwargs: Any,
@@ -207,7 +210,11 @@ class GitLoader(BaseGitLoader):
         self.ext_refs: Dict[bytes, Optional[Tuple[int, bytes]]] = {}
         self.repo_pack_size_bytes = 0
         self.urllib3_extra_kwargs = urllib3_extra_kwargs
+        self.urllib3_extra_kwargs["timeout"] = urllib3.util.Timeout(
+            connect=connect_timeout, read=read_timeout
+        )
         self.requests_extra_kwargs = requests_extra_kwargs
+        self.requests_extra_kwargs["timeout"] = (connect_timeout, read_timeout)
 
     def fetch_pack_from_origin(
         self,