Be able to specify which branch of a DataFederation repo to clone

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

src/aboutcode/federated/__init__.py

@@ -566,6 +566,7 @@ def from_url(
         name: str,
         remote_root_url: str,
         local_root_dir: Path = None,
+        branch: str = "main",
     ) -> "DataFederation":
         """
         Return a DataFederation loaded from a remote configuration file.
@@ -574,6 +575,7 @@ def from_url(
             remote_root_url=remote_root_url,
             federation_name=name,
             config_filename=cls.CONFIG_FILENAME,
+            branch=branch,
         )
         headers = {"User-Agent": "AboutCode/FederatedCode"}
         response = requests.get(url=rcf_url, headers=headers)
@@ -715,6 +717,9 @@ class DataCluster:
     # this is the name of cluster
     data_kind: str
 
+    # The filename used when saving data.
+    datafile_name: str
+
     # a URI template to build the path to the datafile for this data kind.
     # this is the path relative to the root of a cluster directory. It does not
     # include directory and repository.
@@ -800,6 +805,7 @@ def from_dict(cls, data: dict) -> "DataCluster":
 
         return cls(
             data_kind=data["data_kind"],
+            datafile_name=data.get("datafile_name"),
             datafile_path_template=data.get("datafile_path_template"),
             purl_type_configs=ptcs,
             data_schema_url=data.get("data_schema_url"),
@@ -812,6 +818,7 @@ def from_dict(cls, data: dict) -> "DataCluster":
     def to_dict(self):
         return dict(
             data_kind=self.data_kind,
+            datafile_name=self.datafile_name,
             datafile_path_template=self.datafile_path_template,
             purl_type_configs=[pt.to_dict() for pt in self.purl_type_configs],
             data_schema_url=self.data_schema_url,
@@ -834,19 +841,6 @@ def get_datafile_download_url(self, purl: Union[str, PackageURL]) -> str:
         """
         raise NotImplementedError()
 
-        purl = as_purl(purl)
-        # FIXME: create as member
-        purl_type_config_by_type = {ptc.purl_type: ptc for ptc in self.purl_type_configs}
-        purl_type_config = purl_type_config_by_type(purl.type, self.default_config())
-
-        ppe = package_path_elements(purl, max_value=purl_type_config.number_of_dirs)
-        purl_hash, core_path, version, extra_path = ppe
-
-        direct_url = None
-        # construct a path based on path template
-        # construct a URL
-        return direct_url
-
     def get_local_datafile(self, purl: Union[str, PackageURL]) -> LocalDataFile:
         """
         Return a LocalDataFile of the data kind stored in this cluster given a
@@ -862,7 +856,7 @@ def get_config(self, purl_type: str) -> "PurlTypeConfig":
             return self._configs_by_purl_type["default"]
         return self._configs_by_purl_type[purl_type]
 
-    def get_datafile_relative_path(self, purl: Union[str, PackageURL]) -> str:
+    def get_datafile_relative_path(self, purl: Union[str, PackageURL], datafile_name=None) -> str:
         """
         Return the datfile path relative to the root of a cluster directory
         given a PURL.
@@ -874,11 +868,15 @@ def get_datafile_relative_path(self, purl: Union[str, PackageURL]) -> str:
                 f"DataCluster '{self.data_kind}' needs PackageURL with version to generate path."
             )
 
+        if not datafile_name:
+            datafile_name = self.datafile_name
+
         template = uritemplate.URITemplate(self.datafile_path_template)
         return template.expand(
             namespace=purl.namespace,
             name=purl.name,
             version=purl.version,
+            datafile_name=datafile_name,
         )
 
     def get_repo_and_dir_hash(self, purl: Union[str, PackageURL]) -> Tuple[str, str]:
@@ -892,14 +890,14 @@ def get_repo_and_dir_hash(self, purl: Union[str, PackageURL]) -> Tuple[str, str]
         repo_hash = purl_hash - (purl_hash % ptc.numbers_of_dirs_per_repo)
         return f"{repo_hash:04}", purl_hashid
 
-    def get_datafile_repo_and_path(self, purl: Union[str, PackageURL]) -> Tuple[str, str]:
+    def get_datafile_repo_and_path(self, purl: Union[str, PackageURL], datafile_name=None) -> Tuple[str, str]:
         """
         Return the repository name and relative path to the datafile of the data kind stored
         in this cluster given a PURL.
         """
         purl = as_purl(purl)
         repo_hash, dir_hash = self.get_repo_and_dir_hash(purl)
-        relative_datafile_path = self.get_datafile_relative_path(purl)
+        relative_datafile_path = self.get_datafile_relative_path(purl, datafile_name=datafile_name)
 
         directory_name = f"{purl.type}-{dir_hash}"
         repository_name = f"{self.data_kind}-{purl.type}-{repo_hash}"
@@ -1143,7 +1141,8 @@ def cluster_preset():
         DataCluster(
             data_kind="purls",
             description="List of fully qualified PURL strings for a package, sorted by version.",
-            datafile_path_template="{/namespace}/{name}/purls.yml",
+            datafile_name="purls.yml",
+            datafile_path_template="{/namespace}/{name}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.small_size_configs(),
             data_schema_url="",
             documentation_url="https://github.com/package-url/purl-spec/",
@@ -1155,7 +1154,8 @@ def cluster_preset():
             "Each datafile path and schema is PURL type-specific "
             "and not documented here.",
             # FIXME: a POM is in XML, some metadata files may be code
-            datafile_path_template="",
+            datafile_name="api_package_metadata.json",
+            datafile_path_template="{/namespace}/{name}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1167,7 +1167,8 @@ def cluster_preset():
             "Each datafile path and schema is PURL type-specific "
             "and not documented here.",
             # FIXME: a POM is in XML, some metadata files may be code
-            datafile_path_template="",
+            datafile_name="api_package_version_responses.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1177,7 +1178,8 @@ def cluster_preset():
             data_kind="purldb",
             description="PurlDB normalized metadata datafiles for each package "
             "versions. Does not include fingerprints and symbols.",
-            datafile_path_template="{/namespace}/{name}/{version}/purldb.json",
+            datafile_name="purldb.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1188,7 +1190,8 @@ def cluster_preset():
             data_kind="vulnerabilities",
             description="VulnerableCode vulnerabilities for each package. "
             "Also includes a separate vulnerabilities directory/",
-            datafile_path_template="{/namespace}/{name}/vulnerabilities.json",
+            datafile_name="vulnerabilities.json",
+            datafile_path_template="{/namespace}/{name}/{datafile_name}",
             purl_type_configs=[PurlTypeConfig.default_config()],
             data_schema_url="",
             documentation_url="",
@@ -1197,7 +1200,8 @@ def cluster_preset():
         DataCluster(
             data_kind="security_advisories",
             description="VulnerableCode security advisories for each package version.",
-            datafile_path_template="{/namespace}/{name}/{version}/advisories.yml",
+            datafile_name="advisories.yml",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=[PurlTypeConfig.default_config()],
             data_schema_url="",
             documentation_url="",
@@ -1206,7 +1210,8 @@ def cluster_preset():
         DataCluster(
             data_kind="scancode_toolkit_scans",
             description="scancode toolkit scans for each package version.",
-            datafile_path_template="{/namespace}/{name}/{version}/scancode-toolkit.json",
+            datafile_name="scancode-toolkit.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1215,7 +1220,8 @@ def cluster_preset():
         DataCluster(
             data_kind="scancode_fingerprints",
             description="scancode_fingerprints for each package version.",
-            datafile_path_template="{/namespace}/{name}/{version}/scancode-fingerprints.json",
+            datafile_name="scancode-fingerprints.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1224,7 +1230,8 @@ def cluster_preset():
         DataCluster(
             data_kind="cyclonedx14_sboms",
             description="CycloneDX v1.4 sboms for each package version",
-            datafile_path_template="{/namespace}/{name}/{version}/cyclonedx-14.json",
+            datafile_name="cyclonedx-14.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1233,7 +1240,8 @@ def cluster_preset():
         DataCluster(
             data_kind="cyclonedx15_sboms",
             description="CycloneDX v1.5 sboms for each package version",
-            datafile_path_template="{/namespace}/{name}/{version}/cyclonedx-15.json",
+            datafile_name="cyclonedx-15.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1242,7 +1250,8 @@ def cluster_preset():
         DataCluster(
             data_kind="cyclonedx16_sboms",
             description="CycloneDX v1.6 sboms for each package version",
-            datafile_path_template="{/namespace}/{name}/{version}/cyclonedx-16.json",
+            datafile_name="cyclonedx-16.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1251,7 +1260,8 @@ def cluster_preset():
         DataCluster(
             data_kind="spdx2_sboms",
             description="SPDX version 2.x sboms for each package version",
-            datafile_path_template="{/namespace}/{name}/{version}/spdx-2.json",
+            datafile_name="spdx-2.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1260,7 +1270,8 @@ def cluster_preset():
         DataCluster(
             data_kind="atom_slices",
             description="Atom slices for each package version",
-            datafile_path_template="{/namespace}/{name}/{version}/atom.json",
+            datafile_name="atom.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1270,7 +1281,8 @@ def cluster_preset():
             data_kind="atom_vulnerable_slices",
             description="Atom vulnerable_slices for each vulnerable package version",
             # FIXME: need to qualify these with an advisory / CVE?
-            datafile_path_template="{/namespace}/{name}/{version}/atom-vulnerable.json",
+            datafile_name="atom-vulnerable.json",
+            datafile_path_template="{/namespace}/{name}/{version}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.large_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1280,7 +1292,8 @@ def cluster_preset():
             data_kind="openssf_security_scorecards",
             description="OpenSSf security_scorecards for package",
             # FIXME: need to qualify these with an advisory / CVE?
-            datafile_path_template="{/namespace}/{name}/security_scorecard.json",
+            datafile_name="security_scorecard.json",
+            datafile_path_template="{/namespace}/{name}/{datafile_name}",
             purl_type_configs=PurlTypeConfig.medium_size_configs(),
             data_schema_url="",
             documentation_url="",
@@ -1433,6 +1446,7 @@ def build_direct_federation_config_file_url(
     remote_root_url: str,
     federation_name: str,
     config_filename: str,
+    branch:str,
 ):
     """
     Return the URL to download a remote config file for a federation
@@ -1441,7 +1455,7 @@ def build_direct_federation_config_file_url(
         root_url=remote_root_url,
         repo=federation_name,
         path=config_filename,
-        branch="main",
+        branch=branch,
     )