Merge branch 'main' into rebase_add_basic_tests

Author: r-sharp

gh_review_project/cr_deadline.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -----------------------------------------------------------------------------
 # (C) Crown copyright Met Office. All rights reserved.
 # The file LICENCE, distributed with this code, contains details of the terms

gh_review_project/finish_milestone.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -----------------------------------------------------------------------------
 # (C) Crown copyright Met Office. All rights reserved.
 # The file LICENCE, distributed with this code, contains details of the terms

gh_review_project/set_milestone.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -----------------------------------------------------------------------------
 # (C) Crown copyright Met Office. All rights reserved.
 # The file LICENCE, distributed with this code, contains details of the terms

gh_review_project/workload.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -----------------------------------------------------------------------------
 # (C) Crown copyright Met Office. All rights reserved.
 # The file LICENCE, distributed with this code, contains details of the terms
@@ -144,21 +145,31 @@ def build_table(data: ProjectData, reviewer_list: list, repos: list) -> PrettyTa
     return table
 
 
-def print_table(title: str, table: PrettyTable, sortTotal: bool) -> None:
+def print_table(
+    title: str, table: PrettyTable, sortTotal: bool, html_output: str = ""
+) -> None:
     """
     Print a pretty table and its title.
 
     title: str Title of table to be printed first
     table: PrettyTable table to be printed
     """
-    print(title)
-    # table.set_style(TableStyle.MARKDOWN) #requires newer version
+
     table.align["Reviewer"] = "l"
 
     if sortTotal:
         table.sortby = "Total"
 
-    print(table)
+    if not html_output:
+        print(title)
+        print(table)
+        return
+
+    table.format = True
+    html_table = table.get_html_string()
+    with open(html_output, "a") as f:
+        f.write(f"<h1>{title}</h1>")
+        f.write(html_table)
 
 
 def parse_args():
@@ -192,6 +203,12 @@ def parse_args():
         help="Filepath to test data for either capture the project status, "
         "or use as input data.",
     )
+    parser.add_argument(
+        "--html",
+        default="",
+        help="html file to output table contents to. If not set, an ascii formatted "
+        "table will be outputted to stdout",
+    )
 
     args = parser.parse_args()
 
@@ -236,8 +253,17 @@ def main(total: bool, test: bool, capture_project: bool, file: Path):
     tables["LFRic"] = build_table(data, reviewers, repo_list)
 
     # Print tables
+    # Check html path is valid
+    if args.html:
+        html_path = Path(args.html)
+        if html_path.is_dir():
+            raise ValueError("--html option cannot be a directory")
+        html_dir = html_path.parent
+        html_dir.mkdir(parents=True, exist_ok=True)
+        html_path.unlink(missing_ok=True)
+
     for name, table in tables.items():
-        print_table(name, table, total)
+        print_table(name, table, total, args.html)
 
 
 if __name__ == "__main__":

github_scripts/get_git_sources.py

@@ -91,22 +91,16 @@ def datetime_str() -> str:
 
 
 def clone_and_merge(
-    dependency: str,
-    opts: Union[list, dict],
-    loc: Path,
-    use_mirrors: bool,
-    mirror_loc: Path,
+    opts: Union[list, dict], loc: Path, use_mirrors: bool, mirror_loc: Path
 ) -> None:
     """
     Wrapper script for calling get_source and merge_source for a single dependency
 
-    dependency: name of the dependency
     opts: dict or list of dicts for a dependency in the dependencies file
     loc: path to location to clone to
     use_mirrors: bool, use local git mirrors if true
     mirror_loc: path to local git mirrors
     """
-
     if not isinstance(opts, list):
         opts = [opts]
 
@@ -120,7 +114,6 @@ def clone_and_merge(
                 values["source"],
                 values["ref"],
                 loc,
-                dependency,
                 use_mirrors,
                 mirror_loc,
             )
@@ -130,7 +123,6 @@ def clone_and_merge(
                 values["source"],
                 values["ref"],
                 loc,
-                dependency,
                 use_mirrors,
                 mirror_loc,
             )
@@ -140,7 +132,6 @@ def get_source(
     source: str,
     ref: str,
     dest: Path,
-    repo: str,
     use_mirrors: bool = False,
     mirror_loc: Path = Path(""),
 ) -> None:
@@ -151,26 +142,25 @@ def get_source(
     if ".git" in source:
         if use_mirrors:
             logger.info(
-                f"[{datetime_str()}] Cloning {repo} from {mirror_loc} at ref {ref}"
+                f"[{datetime_str()}] Cloning {dest.name} from {mirror_loc} at ref {ref}"
             )
-            mirror_repo = repo
-            if "jules-internal" in source:
-                mirror_repo = "jules-internal"
+            mirror_repo = re.split("[:/]", source)[-1]
             mirror_loc = Path(mirror_loc) / "MetOffice" / mirror_repo
             clone_repo_mirror(source, ref, mirror_loc, dest)
         else:
-            logger.info(f"[{datetime_str()}] Cloning {repo} from {source} at ref {ref}")
+            logger.info(
+                f"[{datetime_str()}] Cloning {dest.name} from {source} at ref {ref}"
+            )
             clone_repo(source, ref, dest)
     else:
-        logger.info(f"[{datetime_str()}] Syncing {repo} at ref {ref}")
+        logger.info(f"[{datetime_str()}] Syncing {dest.name} at ref {ref}")
         sync_repo(source, ref, dest)
 
 
 def merge_source(
     source: Union[Path, str],
     ref: str,
     dest: Path,
-    repo: str,
     use_mirrors: bool = False,
     mirror_loc: Path = Path(""),
 ) -> None:
@@ -181,17 +171,12 @@ def merge_source(
 
     logger.info(
         f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Merging "
-        f"{source} at ref {ref} into {repo}"
+        f"{source} at ref {ref} into {dest.name}"
     )
 
-    if ".git" in str(source):
-        if use_mirrors:
-            remote_path = Path(mirror_loc) / "MetOffice" / repo
-            fetch = determine_mirror_fetch(source, ref)
-        else:
-            remote_path = source
-            fetch = ref
-    else:
+    if not any(
+        [g for g in ["https:", "git@", "localmirrors:"] if str(source).startswith(g)]
+    ):
         if not ref:
             raise Exception(
                 f"Cannot merge local source '{source}' with empty ref.\n"
@@ -200,6 +185,14 @@ def merge_source(
             )
         remote_path = source
         fetch = ref
+    elif use_mirrors:
+        mirror_repo = re.split("[:/]", source)[-1]
+        remote_path = Path(mirror_loc) / "MetOffice" / mirror_repo
+        fetch = determine_mirror_fetch(source, ref)
+    else:
+        # Not using a clone or the mirrors
+        remote_path = source
+        fetch = ref
 
     run_command(f"git -C {dest} remote add local {remote_path}")
 
@@ -210,7 +203,7 @@ def merge_source(
     if result.returncode:
         unmerged_files = get_unmerged(dest)
         if unmerged_files:
-            handle_merge_conflicts(source, ref, dest, repo)
+            handle_merge_conflicts(source, ref, dest, dest.name)
         else:
             raise subprocess.CalledProcessError(
                 result.returncode, command, result.stdout, result.stderr

github_scripts/merge_sources.py

@@ -82,7 +82,7 @@ def main():
 
     for dependency, sources in dependencies.items():
         dest = args.path / dependency
-        clone_and_merge(dependency, sources, dest, args.mirrors, args.mirror_loc)
+        clone_and_merge(sources, dest, args.mirrors, args.mirror_loc)
 
 
 if __name__ == "__main__":

github_scripts/rose_stem_extract_source.py

@@ -45,7 +45,7 @@ def main() -> None:
 
     for dependency, sources in dependencies.items():
         loc = clone_loc / dependency
-        clone_and_merge(dependency, sources, loc, use_mirrors, mirror_loc)
+        clone_and_merge(sources, loc, use_mirrors, mirror_loc)
 
 
 if __name__ == "__main__":

github_scripts/tests/test_get_git_sources.py

@@ -126,7 +126,6 @@ def test_merge_sources(setup_sources):
             "https://github.com/MetOffice/SimSys_Scripts.git",
             "main",
             target_clone,
-            "SimSys_Scripts",
         )
         is None
     )