mutate: refuse a file the binary under test is not built from

scripts/mutate/mutate_core.py

@@ -2168,6 +2168,60 @@ def render_fingerprint(project, facts):
     return "\n".join(lines)
 
 
+PROBE_MARKER = "mutate_core: participation probe"
+
+
+def check_target_is_built(lane, args, log):
+    """Refuse a file the binary under test is not built from.
+
+    Every mutant in such a file comes back `survived`, because the mutation
+    never reaches the binary the tests run - so the report says "nothing
+    noticed these - whatever covers them is decoration" when the truth is that
+    nothing *could* have noticed. That is the flattering direction, and the one
+    this tool must never fail in: a 0% kill rate reads as an indictment of the
+    tests when it is really an indictment of the invocation.
+
+    Not hypothetical. oans's `src/tests.c` #includes most of `src/*.c` into a
+    single translation unit, but deliberately not `oans.c` or `run_dedupe.c`,
+    which belong to the shipped binary alone - and `make test-build` compiles
+    only `tests.c`. Sweeping `src/oans.c` planned 1,394 mutants and would have
+    scored every one of them `survived`.
+
+    The probe is an `#error` appended to the file. If the build still succeeds,
+    nothing the test binary is made of ever included it. That is deterministic
+    and needs to know nothing about the build system, which is what makes it
+    the right shape for a core shared by make, cmake and meson: a compilation
+    database can say which files are compiled, but not which are #included into
+    something that is.
+
+    It runs before the baseline build rather than after, so the tree is left
+    with a good build rather than a failed one, and it is skipped where it
+    cannot pay for itself - a run that scores nothing has nothing to be wrong
+    about.
+    """
+    with open(lane.target, encoding="utf-8") as f:
+        original = f.read()
+    log("checking the target reaches the binary under test")
+    lane.write_target("%s\n#error %s\n" % (original, PROBE_MARKER))
+    try:
+        proc = lane.run_build(args.build_timeout, jobs=os.cpu_count() or 4)
+    finally:
+        lane.write_target(original)
+    if proc is None:
+        # A timeout says nothing either way, and refusing on it would turn a
+        # slow machine into a wrong answer about the tests.
+        log("participation probe timed out; continuing without it")
+        return
+    if proc.returncode != 0:
+        return
+    raise RuntimeError(
+        "%s is not compiled into %s, so every mutant in it would come back "
+        "`survived` however good the tests are - the build succeeded with an "
+        "#error in the file. Sweep a file the test binary is actually built "
+        "from, or add this one to it."
+        % (args.file, lane.project.test_binary))
+
+
 def baseline(lane, args, log):
     """Refuse to score anything until the suite is green repeatedly.
 
@@ -2178,6 +2232,7 @@ def baseline(lane, args, log):
     are derived from - a fixed generous timeout makes every hung mutant cost
     many times what a real one does, and hangs are an expected verdict here.
     """
+    check_target_is_built(lane, args, log)
     log("baseline: building")
     proc = lane.run_build(args.build_timeout, jobs=os.cpu_count() or 4)
     if killed_for_memory(proc):

scripts/mutate/mutate_core.sha256

@@ -1 +1 @@
-a74e3a05fbc1ef5be7ec0f4e869a875d066009a57651ff910196cc3c287613e3
+c71e0f7a106d09177ef89a81bf78182f9ee4e78974d2f48aa5c80703ecb9c528

scripts/test_mutate.py

@@ -1759,6 +1759,82 @@ def test_a_run_that_timed_out_was_not_killed_for_memory(self):
         self.assertFalse(mutate.killed_for_memory(None))
 
 
+class TestTargetIsBuilt(unittest.TestCase):
+    """Refusing a file the binary under test is not built from.
+
+    This is the check for the one way this tool can be wrong that looks like a
+    finding: a file no test binary includes scores every mutant `survived`, and
+    the report says the tests are decoration when nothing was ever measured.
+    The probe appends an #error and asks whether the build notices."""
+
+    class ProbeLane:
+        def __init__(self, tmp, build_returncode):
+            self.target = os.path.join(tmp, "target.c")
+            with open(self.target, "w", encoding="utf-8") as f:
+                f.write("int answer(void) { return 42; }\n")
+            self.build_returncode = build_returncode
+            self.seen = []          # what the file held at each build
+            self.project = PROJECT
+
+        def write_target(self, text):
+            with open(self.target, "w", encoding="utf-8") as f:
+                f.write(text)
+
+        def run_build(self, timeout, jobs=None):
+            with open(self.target, encoding="utf-8") as f:
+                self.seen.append(f.read())
+            if self.build_returncode is None:
+                return None
+            return finished(self.build_returncode)
+
+    def check(self, build_returncode):
+        with tempfile.TemporaryDirectory() as tmp:
+            lane = self.ProbeLane(tmp, build_returncode)
+            args = types.SimpleNamespace(build_timeout=1, file="src/target.c")
+            try:
+                mutate.check_target_is_built(lane, args, lambda _m: None)
+                raised = None
+            except RuntimeError as exc:
+                raised = str(exc)
+            with open(lane.target, encoding="utf-8") as f:
+                left = f.read()
+            return raised, lane.seen, left
+
+    def test_a_file_nothing_includes_is_refused(self):
+        # The build shrugged off an #error, so the compiler never saw the file.
+        raised, _seen, _left = self.check(0)
+        self.assertIsNotNone(raised)
+        self.assertIn("not compiled into", raised)
+        self.assertIn("src/target.c", raised)
+        # Names the consequence, not just the condition - the whole point is
+        # that `survived` would otherwise be read as a fact about the tests.
+        self.assertIn("survived", raised)
+
+    def test_a_file_the_build_compiles_is_accepted(self):
+        raised, _seen, _left = self.check(1)
+        self.assertIsNone(raised)
+
+    def test_the_probe_really_puts_an_error_in_the_file(self):
+        # Without this the check would pass for the wrong reason on any tree
+        # whose build happens to fail, and nothing here would notice.
+        _raised, seen, _left = self.check(1)
+        self.assertEqual(1, len(seen))
+        self.assertIn("#error", seen[0])
+        self.assertIn(mutate.PROBE_MARKER, seen[0])
+        self.assertIn("int answer(void)", seen[0])   # appended, not replaced
+
+    def test_the_file_is_restored_either_way(self):
+        for rc in (0, 1, None):
+            with self.subTest(build_returncode=rc):
+                _raised, _seen, left = self.check(rc)
+                self.assertEqual("int answer(void) { return 42; }\n", left)
+
+    def test_a_timeout_does_not_refuse(self):
+        # A slow machine must not become a wrong answer about the tests.
+        raised, _seen, _left = self.check(None)
+        self.assertIsNone(raised)
+
+
 class TestEvaluate(unittest.TestCase):
     """Which verdict a mutant gets, given how its build and its suite ended.