commit f51d542ff43af954555c33e192f9a496f4fe11d6
from: Nicolas Dandrimont <nicolas@dandrimont.eu>
date: Wed Jan 24 16:16:18 2024 UTC

loader: add support for extra urllib3 kwargs

This is useful to override the default settings of the dulwich urllib3
adapter, e.g. certificate verification of connect/read timeouts.

commit - 8969d706b24bd4afe549f6e12a88e24399262181
commit + f51d542ff43af954555c33e192f9a496f4fe11d6
blob - 9bc8a7033d2ed01460e16a4d1d2e1598cc0f6d52
blob + 2122fb173037e5cc07309f99a841c36c17bd849e
--- swh/loader/git/loader.py
+++ swh/loader/git/loader.py
@@ -179,6 +179,7 @@ class GitLoader(BaseGitLoader):
         repo_representation: Type[RepoRepresentation] = RepoRepresentation,
         pack_size_bytes: int = 4 * 1024 * 1024 * 1024,
         temp_file_cutoff: int = 100 * 1024 * 1024,
+        urllib3_extra_kwargs: Dict[str, Any] = {},
         **kwargs: Any,
     ):
         """Initialize the bulk updater.
@@ -204,6 +205,7 @@ class GitLoader(BaseGitLoader):
         self.ref_object_types: Dict[bytes, Optional[TargetType]] = {}
         self.ext_refs: Dict[bytes, Optional[Tuple[int, bytes]]] = {}
         self.repo_pack_size_bytes = 0
+        self.urllib3_extra_kwargs = urllib3_extra_kwargs
 
     def fetch_pack_from_origin(
         self,
@@ -217,9 +219,21 @@ class GitLoader(BaseGitLoader):
         transport_url = origin_url
 
         logger.debug("Transport url to communicate with server: %s", transport_url)
+
+        transport_kwargs: Dict[str, Any] = {"thin_packs": False}
+
+        if transport_url.startswith(("http://", "https://")):
+            # Inject urllib3 kwargs into the pool manager
+            transport_kwargs["pool_manager"] = dulwich.client.default_urllib3_manager(
+                config=None,
+                **self.urllib3_extra_kwargs,
+            )
 
         client, path = dulwich.client.get_transport_and_path(
-            transport_url, thin_packs=False, operation="pull"
+            location=transport_url,
+            config=None,
+            operation="pull",
+            **transport_kwargs,
         )
 
         logger.debug("Client %s to fetch pack at %s", client, path)