commit 71295d087a44ad0ebfbd78ec4d6d9a8e49a9645b from: Pierre-Yves David date: Wed May 15 14:36:44 2024 UTC model: adapt to the renaming of model.TargetType to model.SnapshotTargetType commit - b15a37c16318dbf1784f9762b4f978431b3eebb9 commit + 71295d087a44ad0ebfbd78ec4d6d9a8e49a9645b blob - 28c22c15ce97bac84dc2c97cf893d9f7b09b2f7f blob + 4566e827a635100d0acf66a99a06ad411c15477b --- requirements-swh.txt +++ requirements-swh.txt @@ -1,5 +1,5 @@ swh.core >= 2.22.0 swh.loader.core >= 5.14.2 -swh.model >= 6.9.0 +swh.model >= 6.13.0 swh.scheduler >= 0.0.39 swh.storage >= 0.22.0 blob - 43aff15cdb9d055b511158359a0bb02862a0322f blob + bbd651c1db06670ea8a625d91d473b3781024322 --- swh/loader/git/converters.py +++ swh/loader/git/converters.py @@ -31,7 +31,7 @@ from swh.model.model import ( Revision, RevisionType, SkippedContent, - TargetType, + SnapshotTargetType, Timestamp, TimestampWithTimezone, ) @@ -256,10 +256,10 @@ def dulwich_commit_to_revision(obj: ShaFile) -> Revisi DULWICH_TARGET_TYPES = { - b"blob": TargetType.CONTENT, - b"tree": TargetType.DIRECTORY, - b"commit": TargetType.REVISION, - b"tag": TargetType.RELEASE, + b"blob": SnapshotTargetType.CONTENT, + b"tree": SnapshotTargetType.DIRECTORY, + b"commit": SnapshotTargetType.REVISION, + b"tag": SnapshotTargetType.RELEASE, } blob - dc07a6372a532f2b33dfb53704a8865af32a6d6d blob + ebd34fec6c1e0a26b911455f51746230823f3b93 --- swh/loader/git/directory.py +++ swh/loader/git/directory.py @@ -15,7 +15,7 @@ from swh.loader.core.loader import BaseDirectoryLoader from swh.loader.exception import NotFound from swh.loader.git.utils import raise_not_found_repository from swh.model.from_disk import ignore_empty_directories, ignore_named_directories -from swh.model.model import Snapshot, SnapshotBranch, TargetType +from swh.model.model import Snapshot, SnapshotBranch, SnapshotTargetType def git() -> str: @@ -143,12 +143,12 @@ class GitCheckoutLoader(BaseDirectoryLoader): return Snapshot( branches={ b"HEAD": SnapshotBranch( - target_type=TargetType.ALIAS, + target_type=SnapshotTargetType.ALIAS, target=branch_name, ), branch_name: SnapshotBranch( target=self.directory.hash, - target_type=TargetType.DIRECTORY, + target_type=SnapshotTargetType.DIRECTORY, ), } ) blob - 41c8882b3f1dbb29304b01950c95d2d9d451a4d1 blob + cad245e7e421db9d29ffd1f571df5588c6f8378b --- swh/loader/git/from_disk.py +++ swh/loader/git/from_disk.py @@ -17,7 +17,7 @@ import dulwich.repo from swh.loader.git.utils import raise_not_found_repository from swh.model import hashutil -from swh.model.model import Snapshot, SnapshotBranch, TargetType +from swh.model.model import Snapshot, SnapshotBranch, SnapshotTargetType from swh.storage.algos.origin import origin_get_latest_visit_status from swh.storage.interface import StorageInterface @@ -317,7 +317,9 @@ class GitLoaderFromDisk(BaseGitLoader): for ref, target in self.repo.refs.get_symrefs().items(): if utils.ignore_branch_name(ref): continue - branches[ref] = SnapshotBranch(target=target, target_type=TargetType.ALIAS) + branches[ref] = SnapshotBranch( + target=target, target_type=SnapshotTargetType.ALIAS + ) if target not in branches: # This handles the case where the pointer is "dangling". # There's a chance that a further symbolic reference will blob - 1a5d8d1c87bccb651ce57639a1c1564bcf757451 blob + b11c3fdbef25e449a5c88688057f7d19b62639d5 --- swh/loader/git/loader.py +++ swh/loader/git/loader.py @@ -50,7 +50,7 @@ from swh.model.model import ( Revision, Snapshot, SnapshotBranch, - TargetType, + SnapshotTargetType, ) from swh.model.swhids import ExtendedObjectType from swh.storage.algos.directory import directory_get @@ -108,7 +108,7 @@ class RepoRepresentation: heads_logger.debug("Heads known in the archive:") for base_snapshot in self.base_snapshots: for branch_name, branch in base_snapshot.branches.items(): - if not branch or branch.target_type == TargetType.ALIAS: + if not branch or branch.target_type == SnapshotTargetType.ALIAS: continue heads_logger.debug(" %r: %s", branch_name, branch.target.hex()) self.local_heads.add(HexBytes(hashutil.hash_to_bytehex(branch.target))) @@ -226,7 +226,7 @@ class GitLoader(BaseGitLoader): # state initialized in fetch_data self.remote_refs: Dict[bytes, HexBytes] = {} self.symbolic_refs: Dict[bytes, HexBytes] = {} - self.ref_object_types: Dict[bytes, Optional[TargetType]] = {} + self.ref_object_types: Dict[bytes, Optional[SnapshotTargetType]] = {} self.ext_refs: Dict[bytes, Optional[Tuple[int, bytes]]] = {} self.repo_pack_size_bytes = 0 self.urllib3_extra_kwargs = urllib3_extra_kwargs @@ -609,7 +609,7 @@ class GitLoader(BaseGitLoader): """Format the blobs from the git repository as swh contents""" for raw_obj in self.iter_objects(b"blob"): if raw_obj.id in self.ref_object_types: - self.ref_object_types[raw_obj.id] = TargetType.CONTENT + self.ref_object_types[raw_obj.id] = SnapshotTargetType.CONTENT yield converters.dulwich_blob_to_content( raw_obj, max_content_size=self.max_content_size @@ -619,7 +619,7 @@ class GitLoader(BaseGitLoader): """Format the trees as swh directories""" for raw_obj in self.iter_objects(b"tree"): if raw_obj.id in self.ref_object_types: - self.ref_object_types[raw_obj.id] = TargetType.DIRECTORY + self.ref_object_types[raw_obj.id] = SnapshotTargetType.DIRECTORY yield converters.dulwich_tree_to_directory(raw_obj) @@ -627,7 +627,7 @@ class GitLoader(BaseGitLoader): """Format commits as swh revisions""" for raw_obj in self.iter_objects(b"commit"): if raw_obj.id in self.ref_object_types: - self.ref_object_types[raw_obj.id] = TargetType.REVISION + self.ref_object_types[raw_obj.id] = SnapshotTargetType.REVISION yield converters.dulwich_commit_to_revision(raw_obj) @@ -635,7 +635,7 @@ class GitLoader(BaseGitLoader): """Retrieve all the release objects from the git repository""" for raw_obj in self.iter_objects(b"tag"): if raw_obj.id in self.ref_object_types: - self.ref_object_types[raw_obj.id] = TargetType.RELEASE + self.ref_object_types[raw_obj.id] = SnapshotTargetType.RELEASE yield converters.dulwich_tag_to_release(raw_obj) @@ -679,7 +679,7 @@ class GitLoader(BaseGitLoader): # Handle symbolic references as alias branches for ref_name, target in self.symbolic_refs.items(): branches[ref_name] = SnapshotBranch( - target_type=TargetType.ALIAS, + target_type=SnapshotTargetType.ALIAS, target=target, ) if target not in branches and target not in unfetched_refs: @@ -698,12 +698,12 @@ class GitLoader(BaseGitLoader): branch.target: branch for base_snapshot in reversed(self.base_snapshots) for branch in base_snapshot.branches.values() - if branch and branch.target_type != TargetType.ALIAS + if branch and branch.target_type != SnapshotTargetType.ALIAS } assert all( base_snapshot_reverse_branches[branch.target] == branch for branch in self.prev_snapshot.branches.values() - if branch and branch.target_type != TargetType.ALIAS + if branch and branch.target_type != SnapshotTargetType.ALIAS ), "base_snapshot_reverse_branches is not a superset of prev_snapshot" for ref_name, target in unfetched_refs.items(): @@ -725,10 +725,13 @@ class GitLoader(BaseGitLoader): targets_unknown = set(refs_for_target) for method, target_type in ( - (self.storage.revision_missing, TargetType.REVISION), - (self.storage.release_missing, TargetType.RELEASE), - (self.storage.directory_missing, TargetType.DIRECTORY), - (self.storage.content_missing_per_sha1_git, TargetType.CONTENT), + (self.storage.revision_missing, SnapshotTargetType.REVISION), + (self.storage.release_missing, SnapshotTargetType.RELEASE), + (self.storage.directory_missing, SnapshotTargetType.DIRECTORY), + ( + self.storage.content_missing_per_sha1_git, + SnapshotTargetType.CONTENT, + ), ): missing = set(method(list(targets_unknown))) known = targets_unknown - missing blob - 49049f02c931f750c8f4007cd075f24c8ce79caa blob + 61caae585da313f38f866d1d2f8ef8bf5b972020 --- swh/loader/git/tests/test_from_disk.py +++ swh/loader/git/tests/test_from_disk.py @@ -22,7 +22,13 @@ from swh.loader.tests import ( prepare_repository_from_archive, ) from swh.model.hashutil import bytehex_to_hash, hash_to_bytes -from swh.model.model import ObjectType, Release, Snapshot, SnapshotBranch, TargetType +from swh.model.model import ( + ObjectType, + Release, + Snapshot, + SnapshotBranch, + SnapshotTargetType, +) from swh.storage.algos.snapshot import snapshot_get_all_branches SNAPSHOT1 = Snapshot( @@ -30,27 +36,27 @@ SNAPSHOT1 = Snapshot( branches={ b"HEAD": SnapshotBranch( target=b"refs/heads/master", - target_type=TargetType.ALIAS, + target_type=SnapshotTargetType.ALIAS, ), b"refs/heads/master": SnapshotBranch( target=hash_to_bytes("2f01f5ca7e391a2f08905990277faf81e709a649"), - target_type=TargetType.REVISION, + target_type=SnapshotTargetType.REVISION, ), b"refs/heads/branch1": SnapshotBranch( target=hash_to_bytes("b0a77609903f767a2fd3d769904ef9ef68468b87"), - target_type=TargetType.REVISION, + target_type=SnapshotTargetType.REVISION, ), b"refs/heads/branch2": SnapshotBranch( target=hash_to_bytes("bd746cd1913721b269b395a56a97baf6755151c2"), - target_type=TargetType.REVISION, + target_type=SnapshotTargetType.REVISION, ), b"refs/tags/branch2-after-delete": SnapshotBranch( target=hash_to_bytes("bd746cd1913721b269b395a56a97baf6755151c2"), - target_type=TargetType.REVISION, + target_type=SnapshotTargetType.REVISION, ), b"refs/tags/branch2-before-delete": SnapshotBranch( target=hash_to_bytes("1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b"), - target_type=TargetType.REVISION, + target_type=SnapshotTargetType.REVISION, ), }, ) @@ -322,11 +328,11 @@ class FullGitLoaderTests(CommonGitLoaderTests): branches = snapshot.branches assert branches[b"HEAD"] == SnapshotBranch( target=b"refs/heads/master", - target_type=TargetType.ALIAS, + target_type=SnapshotTargetType.ALIAS, ) assert branches[b"refs/heads/master"] == SnapshotBranch( target=hash_to_bytes(new_revision), - target_type=TargetType.REVISION, + target_type=SnapshotTargetType.REVISION, ) # Merge branch1 into HEAD. @@ -380,11 +386,11 @@ class FullGitLoaderTests(CommonGitLoaderTests): merge_branches = merge_snapshot.branches assert merge_branches[b"HEAD"] == SnapshotBranch( target=b"refs/heads/master", - target_type=TargetType.ALIAS, + target_type=SnapshotTargetType.ALIAS, ) assert merge_branches[b"refs/heads/master"] == SnapshotBranch( target=hash_to_bytes(merge_commit.decode()), - target_type=TargetType.REVISION, + target_type=SnapshotTargetType.REVISION, ) def test_load_filter_branches(self): @@ -438,7 +444,7 @@ class FullGitLoaderTests(CommonGitLoaderTests): assert branches[b"HEAD"] == SnapshotBranch( target=b"refs/heads/dangling-branch", - target_type=TargetType.ALIAS, + target_type=SnapshotTargetType.ALIAS, ) assert branches[b"refs/heads/dangling-branch"] is None @@ -507,7 +513,7 @@ class FullGitLoaderTests(CommonGitLoaderTests): branches = self.loader.storage.snapshot_get_branches(self.loader.snapshot.id) branch = branches["branches"][b"refs/tags/v1.0.0"] - assert branch.target_type == TargetType.RELEASE + assert branch.target_type == SnapshotTargetType.RELEASE release = self.loader.storage.release_get([branch.target])[0] assert release.date is not None @@ -544,7 +550,7 @@ class FullGitLoaderTests(CommonGitLoaderTests): branches = self.loader.storage.snapshot_get_branches(self.loader.snapshot.id) branch = branches["branches"][b"refs/tags/v1.0.0"] - assert branch.target_type == TargetType.RELEASE + assert branch.target_type == SnapshotTargetType.RELEASE release = self.loader.storage.release_get([branch.target])[0] assert release == Release(