commit - 8969d706b24bd4afe549f6e12a88e24399262181
commit + f51d542ff43af954555c33e192f9a496f4fe11d6
blob - 9bc8a7033d2ed01460e16a4d1d2e1598cc0f6d52
blob + 2122fb173037e5cc07309f99a841c36c17bd849e
--- swh/loader/git/loader.py
+++ swh/loader/git/loader.py
repo_representation: Type[RepoRepresentation] = RepoRepresentation,
pack_size_bytes: int = 4 * 1024 * 1024 * 1024,
temp_file_cutoff: int = 100 * 1024 * 1024,
+ urllib3_extra_kwargs: Dict[str, Any] = {},
**kwargs: Any,
):
"""Initialize the bulk updater.
self.ref_object_types: Dict[bytes, Optional[TargetType]] = {}
self.ext_refs: Dict[bytes, Optional[Tuple[int, bytes]]] = {}
self.repo_pack_size_bytes = 0
+ self.urllib3_extra_kwargs = urllib3_extra_kwargs
def fetch_pack_from_origin(
self,
transport_url = origin_url
logger.debug("Transport url to communicate with server: %s", transport_url)
+
+ transport_kwargs: Dict[str, Any] = {"thin_packs": False}
+
+ if transport_url.startswith(("http://", "https://")):
+ # Inject urllib3 kwargs into the pool manager
+ transport_kwargs["pool_manager"] = dulwich.client.default_urllib3_manager(
+ config=None,
+ **self.urllib3_extra_kwargs,
+ )
client, path = dulwich.client.get_transport_and_path(
- transport_url, thin_packs=False, operation="pull"
+ location=transport_url,
+ config=None,
+ operation="pull",
+ **transport_kwargs,
)
logger.debug("Client %s to fetch pack at %s", client, path)