Modernize CI infra and retire support for Python 3.9

.github/workflows/formatting.yml

@@ -1,26 +1,33 @@
 name: Format and Lint Checks
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 on:
   push:
     branches:
       - main
-    paths:
-      - '*.py'
   pull_request:
     types: [ assigned, opened, synchronize, reopened ]
+
 jobs:
   check:
     name: Format and Lint Checks
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
-          cache: 'pip'
+          python-version: '3.12'
+
       - name: Check code format
-        run:  |
-          python -m pip install --upgrade pip
-          python -m pip install ruff==0.12.7
-          bash ./format.sh
+        run: |
+          set +x -euo pipefail
+          python -m pip install ruff==0.15.20
+          ./format.sh --all
           git diff --name-only
           git diff --exit-code || (echo "Code formatting failed" && exit 1)

format.sh

@@ -1,26 +1,55 @@
 #!/usr/bin/env bash
 
-# This script runs the ruff Python formatter on the whole repository.
+# This script runs the ruff Python formatter on the repository.
+# By default, on non-main branches it only formats files changed relative to
+# the main branch. On the main branch or with --all, it formats all files.
 
 # Check version
 version_string=$(ruff --version | sed -E 's/^.*(\d+\.\d+-.*).*$/\1/')
-expected_version_string='0.12.7'
+expected_version_string='0.15.20'
 if [[ "$version_string" =~ "$expected_version_string" ]]; then
     echo "ruff version '$version_string' matches '$expected_version_string'"
 else
     echo "ruff version '$version_string' doesn't match '$expected_version_string'"
     exit 1
 fi
 
-# Get all C++ files checked into the repo, excluding submodules
 root_folder=$(git rev-parse --show-toplevel)
+path_regex="^.*\(\.py\)$"
+
+format_all=false
+if [[ "$1" == "--all" ]]; then
+    format_all=true
+fi
+
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+staged_files=$( \
+    git diff --cached --name-only --diff-filter=d \
+    | grep "$path_regex" || true)
+
+if [[ "$format_all" == true ]] || [[ "$current_branch" == "main" ]]; then
+    committed_files=$( \
+        git ls-tree --full-tree -r --name-only HEAD . \
+        | grep "$path_regex" || true)
+else
+    merge_base=$(git merge-base main HEAD)
+    committed_files=$( \
+        git diff --name-only --diff-filter=d "$merge_base" \
+        | grep "$path_regex" || true)
+fi
+
 all_files=$( \
-    git ls-tree --full-tree -r --name-only HEAD . \
-    | grep "^.*\(\.py\)$" \
+    printf '%s\n' "$committed_files" "$staged_files" \
+    | grep -v '^$' | sort -u \
     | sed "s~^~$root_folder/~")
-num_files=$(echo $all_files | wc -w)
+
+if [[ -z "$all_files" ]]; then
+    echo "No Python files to format"
+    exit 0
+fi
+num_files=$(echo "$all_files" | wc -l)
 echo "Formatting ${num_files} files"
 
-# shellcheck disable=SC2086
-ruff format --config ${root_folder}/ruff.toml ${all_files}
-ruff check --config ${root_folder}/ruff.toml ${all_files} --fix
+echo "$all_files" | tr '\n' '\0' | xargs -0 ruff format --config "${root_folder}/ruff.toml"
+echo "$all_files" | tr '\n' '\0' | xargs -0 ruff check --config "${root_folder}/ruff.toml" --fix

lamaria/pipeline/estimate_to_timed_reconstruction.py

@@ -58,7 +58,7 @@ def _match_estimate_ts_to_images(
 
         matched_images.append(images[best])
         matched_timestamps.append(left_ts[best])
-    return dict(zip(matched_timestamps, matched_images))
+    return dict(zip(matched_timestamps, matched_images, strict=False))
 
 
 def convert_estimate_into_timed_reconstruction(
@@ -83,7 +83,7 @@ def convert_estimate_into_timed_reconstruction(
     image_id = 1
     frame_id_to_timestamp = dict()
     for frame_id, (pose, timestamp) in enumerate(
-        zip(estimate.poses, timestamps)
+        zip(estimate.poses, timestamps, strict=False)
     ):
         frame = pycolmap.Frame()
         frame.rig_id = 1

lamaria/pipeline/keyframe_selection.py

@@ -54,7 +54,7 @@ def _select_keyframes(self):
         new_frame_id = 1
 
         for i, (prev, curr) in enumerate(
-            zip(init_frame_ids[:-1], init_frame_ids[1:])
+            zip(init_frame_ids[:-1], init_frame_ids[1:], strict=False)
         ):
             if i == 0:
                 self.keyframe_frame_ids[new_frame_id] = prev

lamaria/pipeline/optim/callback.py

@@ -25,7 +25,9 @@ def __call__(self, summary: pyceres.IterationSummary):
         if not summary.step_is_successful:
             return pyceres.CallbackReturnType.SOLVER_CONTINUE
         diff = []
-        for pose_prev, pose in zip(self.poses_previous, self.poses):
+        for pose_prev, pose in zip(
+            self.poses_previous, self.poses, strict=False
+        ):
             pose_rel = pose_prev * pose.inverse()
             q_rel, t_rel = pose_rel.rotation.quat, pose_rel.translation
             dr = np.rad2deg(

lamaria/pipeline/triangulation.py

@@ -54,7 +54,7 @@ def pairs_from_frames(recon: pycolmap.Reconstruction):
 
     adj_pairs = set()
     for _, seq in by_index.items():
-        for a, b in zip(seq[:-1], seq[1:]):
+        for a, b in zip(seq[:-1], seq[1:], strict=False):
             adj_pairs.add((a, b))
 
     return frame_pairs, adj_pairs

lamaria/utils/aria.py

@@ -601,10 +601,10 @@ def extract_images_with_timestamps_from_vrs(
     right_img_dir = images_path / "right"
     left_images = _image_names_from_folder(left_img_dir, left_img_dir)
     right_images = _image_names_from_folder(right_img_dir, right_img_dir)
-    images = list(zip(left_images, right_images))
+    images = list(zip(left_images, right_images, strict=False))
 
     # Create a map
     assert len(left_ts) == len(images), (
         "timestamps should have the same length as images"
     )
-    return dict(zip(left_ts, images))
+    return dict(zip(left_ts, images, strict=False))

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "lamaria"
 version = "0.1.0"
-requires-python = ">=3.9,<3.15"
+requires-python = ">=3.10,<3.15"
 description = "LaMAria: Benchmarking Egocentric Visual-Inertial SLAM at City Scale"
 readme = { file = "README.md", content-type = "text/markdown" }
 

requirements.txt

@@ -8,8 +8,8 @@ evo==1.31.1
 bagpy==0.5
 cv-bridge
 scipy
-ruff==0.12.7
+ruff==0.15.20
 pyceres==2.6
-pycolmap==3.13.0.dev0 ; python_version >= "3.9" and python_version < "3.15"
+pycolmap==3.13.0.dev0 ; python_version >= "3.10" and python_version < "3.15"
 
 -e git+https://github.com/cvg/Hierarchical-Localization.git@1252817#egg=hloc

ruff.toml

@@ -1,4 +1,6 @@
 line-length = 80
+target-version = "py310"
+src = ["."]
 
 [lint]
 select = [

tools/undistort_asl_folder.py

@@ -105,7 +105,9 @@ def undistort_asl(
             image_id += 1
             colmap_images[key].append(im)
 
-    zipped_images = list(zip(*[colmap_images[key] for key, _ in ARIA_CAMERAS]))
+    zipped_images = list(
+        zip(*[colmap_images[key] for key, _ in ARIA_CAMERAS], strict=False)
+    )
 
     for j, (left_im, right_im) in enumerate(
         tqdm(

tools/vrs_to_asl_folder.py

@@ -42,10 +42,14 @@ def remove_images_when_slam_drops(
     assert len(original_right_images) == len(right_timestamps)
 
     left_camera_mapping = {
-        ts: img for ts, img in zip(left_timestamps, original_left_images)
+        ts: img
+        for ts, img in zip(left_timestamps, original_left_images, strict=False)
     }
     right_camera_mapping = {
-        ts: img for ts, img in zip(right_timestamps, original_right_images)
+        ts: img
+        for ts, img in zip(
+            right_timestamps, original_right_images, strict=False
+        )
     }
 
     matched_left_ts = [left_ts for left_ts, _ in matched_timestamps]
@@ -99,7 +103,7 @@ def rename_images_in_folder(
                     {len(image_timestamps)} in {subfolder_path}"
             )
 
-        for ts, img in zip(image_timestamps, original_images):
+        for ts, img in zip(image_timestamps, original_images, strict=False):
             old_image_path = subfolder_path / img
             new_image_path = subfolder_path / f"{ts}{image_extension}"
             os.rename(old_image_path, new_image_path)
@@ -120,7 +124,7 @@ def write_image_csv(image_timestamps, cam_folder):
     images = sorted(images, key=lambda img: int(img.split(".")[0]))
 
     assert len(images) == len(image_timestamps)
-    for ts, img in zip(image_timestamps, images):
+    for ts, img in zip(image_timestamps, images, strict=False):
         assert int(img.split(".")[0]) == ts, f"{img} != {ts}"
 
     data_csv = os.path.join(cam_folder, "data.csv")
@@ -129,7 +133,7 @@ def write_image_csv(image_timestamps, cam_folder):
 
     with open(data_csv, "w", newline="") as f:
         writer = csv.writer(f)
-        for timestamp, image in zip(image_timestamps, images):
+        for timestamp, image in zip(image_timestamps, images, strict=False):
             row = [timestamp, image]
             writer.writerow(row)