test-cnm: Support sending / polling multiple queues in one test run

test-cnm/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "test-cnm"
-version = "0.8.2"
+version = "0.9.0"
 description = ""
 authors = ["Rohan Weeden <reweeden@alaska.edu>"]
 readme = "README.md"

test-cnm/test_cnm/commands/cmd_configure.py

@@ -0,0 +1,70 @@
+import argparse
+import logging
+
+from test_cnm.config import ConfigBasic
+from test_cnm.metadata import Metadata
+from test_cnm.tester.collector import BucketTestCollector
+
+log = logging.getLogger(__name__)
+
+
+def add_parser(
+    subparsers: argparse._SubParsersAction,
+) -> argparse.ArgumentParser:
+    parser_update_metadata = subparsers.add_parser(
+        "configure",
+        help="Update metadata file to set test level configuration",
+    )
+    parser_update_metadata.add_argument(
+        "filter",
+        help="Glob pattern to filter tests by. Can include '*', '?' and '[]' expressions",
+        nargs="*",
+        default=[],
+    )
+    properties_group = parser_update_metadata.add_argument_group("properties")
+    properties_group.add_argument(
+        "--cnm-ingest-queue",
+        help="Override the cnm_ingest_queue that the CNM-S will be sent to for the tests",
+    )
+    properties_group.add_argument(
+        "--cnm-response-queue",
+        help="Override the cnm_response_queue that the CNM client will poll for the CNM-R response for the tests",
+    )
+    parser_update_metadata.set_defaults(
+        func=cmd_update_metadata,
+        config_cls=ConfigBasic,
+    )
+
+    return parser_update_metadata
+
+
+def cmd_update_metadata(
+    parser: argparse.ArgumentParser,
+    args: argparse.Namespace,
+    config: ConfigBasic,
+):
+    filters = args.filter
+
+    session = config.session()
+
+    collector = BucketTestCollector(
+        session,
+        config.test_bucket,
+    )
+    tests = collector.collect_tests(filters)
+
+    with Metadata(session, config.test_bucket) as metadata:
+        for test in tests.values():
+            test_id = test.get_id()
+            cfg = metadata.test_config[test_id]
+
+            for attr in ("cnm_ingest_queue", "cnm_response_queue"):
+                value = getattr(args, attr)
+                if value:
+                    log.debug("%s setting %s to %s", test_id, attr, value)
+                    cfg[attr] = value
+
+            if cfg:
+                log.info("%s:", test_id)
+                for attr, value in cfg.items():
+                    log.info("    %s: %s", attr, value)

test-cnm/test_cnm/commands/cmd_list.py

@@ -59,8 +59,7 @@ def cmd_list(
     for collection, grouped_tests in tests_by_collection.items():
         log.info("%s:", collection)
         for test in grouped_tests:
-            prefix = f"{collection}/"
-            test_id = test.get_id().removeprefix(prefix)
+            test_id = test.get_id()
             log.info("  - %s (%d files)", test_id, len(test.files))
             if args.files:
                 last_idx = len(test.files) - 1

test-cnm/test_cnm/commands/cmd_test.py

@@ -6,7 +6,6 @@
 from test_cnm.tester.cnm_generator import CnmSGenerator
 from test_cnm.tester.collector import BucketTestCollector
 from test_cnm.tester.executor import TestExecutor
-from test_cnm.tester.ingest_client import CnmIngestClient
 from tqdm import tqdm
 from tqdm.contrib.logging import logging_redirect_tqdm
 
@@ -46,24 +45,15 @@ def cmd_test(
     metadata = Metadata(session, config.test_bucket)
     metadata.load()
 
-    collector = BucketTestCollector(
-        session,
-        config.test_bucket,
-    )
-    ingest_client = CnmIngestClient(
+    executor = TestExecutor(
         session=session,
-        make_cnm_s=CnmSGenerator(
-            provider=args.provider or "ASF-TESTCNM",
-            trace=config.trace,
-            metadata=metadata,
+        collector=BucketTestCollector(
+            session,
+            config.test_bucket,
         ),
-        start_queue=config.cnm_ingest_queue_name(),
-        response_queue=config.cnm_response_queue_name(),
-    )
-    executor = TestExecutor(
-        collector,
-        ingest_client,
-        config.default_data_version,
+        make_cnm_s=CnmSGenerator(metadata),
+        metadata=metadata,
+        config=config,
     )
 
     log.info("Executing tests on %s", config.stack_name)

test-cnm/test_cnm/commands/cmd_tidy.py

@@ -43,6 +43,7 @@ def cmd_tidy(
 
     with Metadata(session, config.test_bucket) as metadata:
         extra_metadata = dict(metadata.metadata)
+        extra_metadata.pop(metadata.test_config_key, None)
 
         total = 0
         deleted = 0

test-cnm/test_cnm/commands/cmd_update_metadata.py

@@ -7,7 +7,8 @@
 
 import boto3
 from test_cnm.config import ConfigBasic
-from test_cnm.metadata import CHECKSUM_PATTERN, ChecksumWriter, Metadata
+from test_cnm.metadata import CHECKSUM_PATTERN, Metadata
+from test_cnm.uploader import ChecksumWriter
 
 log = logging.getLogger(__name__)
 

test-cnm/test_cnm/config.py

@@ -1,8 +1,14 @@
 import argparse
 import configparser
-from dataclasses import MISSING, dataclass, fields
+from collections.abc import Mapping
+from dataclasses import MISSING, dataclass, field, fields, replace
 from typing import Optional, Union
 
+try:
+    from typing import Self
+except ImportError:
+    Self = "ConfigBase"
+
 import boto3
 
 DEFAULT = "default"
@@ -14,12 +20,31 @@ class ConfigError(Exception):
 
 @dataclass
 class ConfigBase:
+    _options: Optional[dict[str, str]] = field(
+        default=None,
+        init=False,
+        repr=False,
+        compare=False,
+    )
+    _section: Optional[dict[str, str]] = field(
+        default=None,
+        init=False,
+        repr=False,
+        compare=False,
+    )
+    _default_section: Optional[dict[str, str]] = field(
+        default=None,
+        init=False,
+        repr=False,
+        compare=False,
+    )
+
     @classmethod
     def from_file(
         cls,
         filenames: Union[str, list[str]],
         args: Optional[argparse.Namespace] = None,
-    ) -> "ConfigBase":
+    ) -> Self:
         config = configparser.ConfigParser(default_section=None)
         config.read(filenames)
 
@@ -32,31 +57,71 @@ def from_file(
         section = config[env] if config.has_section(env) else {}
         default_section = config[DEFAULT] if config.has_section(DEFAULT) else {}
 
+        kwargs = cls._resolve_kwargs(
+            options=options,
+            section=section,
+            default_section=default_section,
+        )
+        missing = [
+            # ruff hint
+            k
+            for k, v in kwargs.items()
+            if v is MISSING
+        ]
+        if missing:
+            raise ConfigError(f"missing values for {missing} for environment '{env}'")
+
+        obj = cls(**kwargs)
+        obj._options = options
+        obj._section = {} if env == DEFAULT else dict(section)
+        obj._default_section = dict(default_section)
+
+        return obj
+
+    def dynamic_config(self, cfg: dict) -> Self:
+        if self._options is None or self._section is None or self._default_section is None:
+            obj = replace(self, **cfg)
+        else:
+            kwargs = self._resolve_kwargs(
+                options=self._options,
+                section=self._section,
+                default_section=self._default_section,
+                dynamic_config=cfg,
+            )
+            obj = self.__class__(**kwargs)
+
+        obj._options = self._options
+        obj._section = self._section
+        obj._default_section = self._default_section
+
+        return obj
+
+    @classmethod
+    def _resolve_kwargs(
+        cls,
+        options: Mapping[str, str],
+        section: Mapping[str, str],
+        default_section: Mapping[str, str],
+        dynamic_config: Optional[Mapping[str, str]] = None,
+    ) -> dict[str, str]:
         def _get_value(key: str, default=None):
             if (val := options.get(key)) is not None:
                 return val
             if (val := section.get(key)) is not None:
                 return val
+            if dynamic_config and (val := dynamic_config.get(key)) is not None:
+                return val
             if (val := default_section.get(key)) is not None:
                 return val
 
             return default
 
-        kwargs = {
+        return {
             # ruff hint
             field.name: _get_value(field.name, default=field.default)
             for field in fields(cls)
+            if not field.name.startswith("_")
         }
-        missing = [
-            # ruff hint
-            k
-            for field, (k, v) in zip(fields(cls), kwargs.items())
-            if v is MISSING
-        ]
-        if missing:
-            raise ConfigError(f"missing values for {missing} for environment '{env}'")
-
-        return cls(**kwargs)
 
 
 @dataclass
@@ -86,13 +151,14 @@ def session(self) -> boto3.Session:
         return boto3.Session(profile_name=self.profile)
 
     def cnm_ingest_queue_name(self) -> str:
-        if self.stack_name and not self.cnm_ingest_queue.startswith(self.stack_name):
-            return f"{self.stack_name}-{self.cnm_ingest_queue}"
-
-        return self.cnm_ingest_queue
+        return get_queue_name(self.stack_name, self.cnm_ingest_queue)
 
     def cnm_response_queue_name(self) -> str:
-        if self.stack_name and not self.cnm_response_queue.startswith(self.stack_name):
-            return f"{self.stack_name}-{self.cnm_response_queue}"
+        return get_queue_name(self.stack_name, self.cnm_response_queue)
+
+
+def get_queue_name(stack_name: Optional[str], queue_name: str) -> str:
+    if stack_name and not queue_name.startswith(stack_name):
+        return f"{stack_name}-{queue_name}"
 
-        return self.cnm_response_queue
+    return queue_name

test-cnm/test_cnm/main.py

@@ -16,6 +16,7 @@
 from typing import Optional
 
 from test_cnm.commands import (
+    cmd_configure,
     cmd_list,
     cmd_move,
     cmd_test,
@@ -95,6 +96,7 @@ def get_parser() -> argparse.ArgumentParser:
         dest="command",
     )
 
+    cmd_configure.add_parser(subparsers)
     cmd_list.add_parser(subparsers)
     cmd_move.add_parser(subparsers)
     cmd_test.add_parser(subparsers)