From 9f508e78c35859d6afd000b7cb14107e80c0b03f Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:50:47 +0000 Subject: [PATCH 01/23] test(extract): add perl fixtures and RED language tests Add tests/fixtures/sample.pl + sample_module.pm and 17 Perl tests in tests/test_languages.py covering packages (incl. mid-file switch), subs, use/require imports (pragma exclusion), @ISA/use parent/use base inheritance, static + bare-sub calls (INFERRED, second-pass) and the untyped method-call negative case. Extractor (S3) and .pl/.pm dispatch (S4) not implemented yet, so all 17 fail (RED); existing suite stays green. --- tests/fixtures/sample.pl | 27 +++++ tests/fixtures/sample_module.pm | 46 +++++++++ tests/test_languages.py | 171 ++++++++++++++++++++++++++++++++ 3 files changed, 244 insertions(+) create mode 100644 tests/fixtures/sample.pl create mode 100644 tests/fixtures/sample_module.pm diff --git a/tests/fixtures/sample.pl b/tests/fixtures/sample.pl new file mode 100644 index 0000000000..102f75bf63 --- /dev/null +++ b/tests/fixtures/sample.pl @@ -0,0 +1,27 @@ +#!/usr/bin/perl +use strict; +use warnings; +use Acme::Widget; + +package Acme::Helper; + +sub emit { + my ($text) = @_; + print "$text\n"; + return length $text; +} + +sub format { + my ($text) = @_; + return uc $text; +} + +package main; + +sub run_report { + my $widget = Acme::Widget->new(); + $widget->render(); + Acme::Helper::emit("done"); +} + +run_report(); diff --git a/tests/fixtures/sample_module.pm b/tests/fixtures/sample_module.pm new file mode 100644 index 0000000000..02e0eaa201 --- /dev/null +++ b/tests/fixtures/sample_module.pm @@ -0,0 +1,46 @@ +package Acme::Widget; + +use strict; +use warnings; +use Scalar::Util qw(blessed); +use Carp; +require Acme::Helper; + +use parent -norequire, 'Acme::Role'; +use base 'Acme::Mixin'; + +our @ISA = ('Acme::Base'); + +sub new { + my ($class, %args) = @_; + my $self = { %args }; + return bless $self, $class; +} + +sub render { + my $self = shift; + my $line = format_line(); + Acme::Helper::emit($line); + $self->update(); + return $line; +} + +sub format_line { + return "rendered"; +} + +sub update { + my $self = shift; + $self->{dirty} = 1; + return $self; +} + +package Acme::Widget::Inner; + +use POSIX qw(floor); + +sub tick { + return floor(1.5); +} + +1; diff --git a/tests/test_languages.py b/tests/test_languages.py index 093a9960f5..767ddaa7e0 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4037,3 +4037,174 @@ def test_cl_ids_are_path_qualified_across_directories(tmp_path): f"same-named .lisp files in different dirs must not share ids, " f"got overlap {sorted(ids_a & ids_b)}" ) +# ── Perl ────────────────────────────────────────────────────────────────────── +# RED (S2): the `extract_perl` extractor does not exist yet (S3) and the .pl/.pm +# dispatch is not registered yet (S4). The `extract_perl` import is kept INSIDE +# each test so importing this module still succeeds and the existing suite stays +# GREEN — only the Perl tests fail (ImportError until S3, empty graph until S4). + +def test_perl_no_error(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert "error" not in r + +def test_perl_finds_packages(): + """Both the leading `package Acme::Widget` and the mid-file + `package Acme::Widget::Inner` switch must surface as nodes.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + labels = _labels(r) + assert any("Acme::Widget" in l for l in labels) + assert any("Acme::Widget::Inner" in l for l in labels) + +def test_perl_finds_subs(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + labels = _labels(r) + for name in ("new", "render", "format_line", "update", "tick"): + assert any(name in l for l in labels), f"missing sub {name!r}" + +def test_perl_finds_imports(): + """`use Module` / `require Module` become imports edges; the qw-list form + (`use Scalar::Util qw(blessed)`) and bareword `require` both count.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert "imports" in _relations(r) + import_edges = _edges_with_relation(r, "imports", "imports_from") + targets = " ".join(str(e.get("target", "")) for e in import_edges).lower() + assert "scalar" in targets or "util" in targets, "Scalar::Util import missing" + assert "carp" in targets, "Carp import missing" + assert "helper" in targets, "require Acme::Helper import missing" + +def test_perl_import_edges_have_import_context(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + assert import_edges + assert all(e.get("context") == "import" for e in import_edges) + +def test_perl_import_edges_are_extracted(): + """`use`/`require` are literal in source -> EXTRACTED confidence.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + assert import_edges + assert all(e.get("confidence") == "EXTRACTED" for e in import_edges) + +def test_perl_pragmas_not_imported(): + """`use strict` / `use warnings` are pragmas, not module dependencies, and + must NOT create imports edges (matches the pragma-exclusion in other langs).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + targets = " ".join(str(e.get("target", "")) for e in import_edges).lower() + assert "strict" not in targets, "`use strict` pragma must not be an import" + assert "warnings" not in targets, "`use warnings` pragma must not be an import" + +def test_perl_isa_inherits_edge(): + """`our @ISA = ('Acme::Base')` must emit an inherits edge to Acme::Base.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + found = any( + "Acme::Widget" in node_by_id.get(e["source"], "") + and "Acme::Base" in node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + ) + assert found, "Acme::Widget should have inherits edge to Acme::Base (our @ISA)" + +def test_perl_use_parent_inherits_edge(): + """`use parent -norequire, 'Acme::Role'` must emit an inherits edge; the + `-norequire` flag is not a parent.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + parents = { + node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + and "Acme::Widget" in node_by_id.get(e["source"], "") + } + assert any("Acme::Role" in p for p in parents), "use parent target Acme::Role missing" + assert not any("norequire" in p for p in parents), "-norequire flag must not be a parent" + +def test_perl_use_base_inherits_edge(): + """`use base 'Acme::Mixin'` must emit an inherits edge to Acme::Mixin.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + parents = { + node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + and "Acme::Widget" in node_by_id.get(e["source"], "") + } + assert any("Acme::Mixin" in p for p in parents), "use base target Acme::Mixin missing" + +def test_perl_inherits_edges_are_extracted(): + """@ISA / use parent / use base are literal in source -> EXTRACTED.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert inherits + assert all(e.get("confidence") == "EXTRACTED" for e in inherits) + +def test_perl_no_dangling_edges(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"dangling edge source: {e}" + + +# Perl call resolution runs in the top-level `extract()` second pass (raw_calls -> +# name-matched calls), so these drive the public multi-file dispatch, like the +# ObjC cross-file tests. RED until BOTH S3 (extractor) and S4 (.pl/.pm dispatch). + +def _perl_corpus(): + from graphify.extract import extract + return extract([FIXTURES / "sample.pl", FIXTURES / "sample_module.pm"], parallel=False) + +def test_perl_bare_sub_call_edge(): + """`format_line()` bare call inside `render` (same file/package) must resolve + to a calls edge render -> format_line.""" + r = _perl_corpus() + calls = _calls(r) + assert any("render" in src and "format_line" in tgt for src, tgt in calls), \ + f"expected render->format_line calls edge, got {calls}" + +def test_perl_static_qualified_call_edge(): + """`Acme::Helper::emit(...)` static, package-qualified call must resolve to a + calls edge targeting the emit sub.""" + r = _perl_corpus() + calls = _calls(r) + assert any("emit" in tgt for _src, tgt in calls), \ + f"expected a calls edge into Acme::Helper::emit, got {calls}" + +def test_perl_call_edges_are_inferred(): + """Second-pass name-matched calls carry INFERRED confidence.""" + r = _perl_corpus() + call_edges = [e for e in r["edges"] if e["relation"] == "calls"] + assert call_edges + assert all(e.get("confidence") == "INFERRED" for e in call_edges), \ + f"perl calls should be INFERRED (second-pass), got {[e.get('confidence') for e in call_edges]}" + +def test_perl_call_edges_have_call_context(): + r = _perl_corpus() + call_edges = _edges_with_relation(r, "calls") + assert call_edges + assert all(e.get("context") == "call" for e in call_edges) + +def test_perl_method_call_no_edge(): + """`$self->update()` / `$widget->render()` are untyped method dispatch and must + NOT create calls edges, even though `update` and `render` are defined subs + (naive name-matching would wrongly connect them).""" + r = _perl_corpus() + calls = _calls(r) + # Positive guard: the corpus must actually produce static/bare calls, so this + # test is not a vacuous pass on an empty graph (RED until S3+S4). + assert any("format_line" in tgt or "emit" in tgt for _src, tgt in calls), \ + f"expected the static/bare calls to resolve before asserting the negatives, got {calls}" + assert not any("update" in tgt for _src, tgt in calls), \ + f"method call $self->update() must not produce a calls edge, got {calls}" + assert not any(tgt == "render" or tgt.endswith("::render") or tgt.endswith(".render") + for _src, tgt in calls), \ + f"method call $widget->render() must not produce a calls edge, got {calls}" From 12742f3366b0c76b77d103e7dcc9ae0ccbbcdf9b Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:00:16 +0000 Subject: [PATCH 02/23] feat(extract): add perl extractor (packages, subs, imports, inherits) Bespoke tree-sitter-perl extractor under extractors/perl.py, re-exported via the extract.py facade and registered in LANGUAGE_EXTRACTORS. Emits package + sub nodes (mid-file package switches tracked), imports edges for use/require (pragmas and use parent/base excluded), and inherits edges for our \@ISA / use parent / use base. Calls are handed to the shared second pass as raw_calls (INFERRED name-matching); method calls are flagged is_member_call so that pass drops them (untyped, unresolvable). tree-sitter-perl>=1.2.0,<2.0 added as a regular dependency. --- graphify/extract.py | 1 + graphify/extractors/__init__.py | 2 + graphify/extractors/perl.py | 230 ++++++++++++++++++++++++++++++++ pyproject.toml | 1 + uv.lock | 16 +++ 5 files changed, 250 insertions(+) create mode 100644 graphify/extractors/perl.py diff --git a/graphify/extract.py b/graphify/extract.py index f68757a85f..5cf3343bcf 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -50,6 +50,7 @@ from graphify.extractors.markdown import extract_markdown, _MD_LINK_INDEX_CACHE # noqa: F401 from graphify.extractors.ocaml import extract_ocaml # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 +from graphify.extractors.perl import extract_perl # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index 68ff3340c3..6be458fb18 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -25,6 +25,7 @@ from graphify.extractors.objc import extract_objc from graphify.extractors.pascal import extract_pascal from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form +from graphify.extractors.perl import extract_perl from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest from graphify.extractors.razor import extract_razor from graphify.extractors.rust import extract_rust @@ -54,6 +55,7 @@ "markdown": extract_markdown, "objc": extract_objc, "pascal": extract_pascal, + "perl": extract_perl, "powershell": extract_powershell, "powershell_manifest": extract_powershell_manifest, "razor": extract_razor, diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py new file mode 100644 index 0000000000..6b40740d23 --- /dev/null +++ b/graphify/extractors/perl.py @@ -0,0 +1,230 @@ +"""Perl extractor: packages, subs, imports (use/require), inheritance (@ISA/parent/base). + +Deliberately untyped, consistent with the other language extractors. Calls are +handed to the shared cross-file second pass as ``raw_calls`` (INFERRED by +name-matching); method calls (``$obj->meth()``) are marked ``is_member_call`` so +that pass drops them — without receiver types they are unresolvable and naive +name-matching would wire spurious edges to same-named subs. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from graphify.extractors.base import _file_stem, _make_id + +# ``use strict`` & friends are compiler pragmas, not module dependencies — they +# must not become imports edges (matches the pragma-exclusion in other langs). +_PERL_PRAGMAS: frozenset[str] = frozenset({ + "strict", "warnings", "utf8", "constant", "vars", "lib", "feature", + "integer", "bytes", "overload", "mro", "autodie", "diagnostics", + "sort", "subs", "attributes", "fields", "encoding", "if", "less", +}) + +# ``use parent`` / ``use base`` declare inheritance, not an import. +_PERL_INHERIT_PRAGMAS: frozenset[str] = frozenset({"parent", "base"}) + +# Perl builtins that surface as function_call_expression callees; excluding them +# keeps them from accumulating spurious calls edges as god-nodes. +_PERL_BUILTINS: frozenset[str] = frozenset({ + "print", "printf", "say", "sprintf", "die", "warn", "return", "bless", + "shift", "unshift", "push", "pop", "splice", "map", "grep", "sort", + "reverse", "join", "split", "keys", "values", "each", "exists", "delete", + "defined", "ref", "scalar", "wantarray", "length", "uc", "lc", "ucfirst", + "lcfirst", "chomp", "chop", "chr", "ord", "abs", "int", "sqrt", "eval", + "local", "my", "our", "sub", "do", "require", "use", "no", +}) + + +def extract_perl(path: Path) -> dict: + """Extract packages, subs, imports and inheritance from a .pl/.pm/.t file.""" + try: + import tree_sitter_perl as tsperl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree_sitter_perl not installed"} + + try: + language = Language(tsperl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = _file_stem(path) + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + sub_bodies: list[tuple[str, Any]] = [] + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", context: str | None = None) -> None: + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0} + if context: + edge["context"] = context + edges.append(edge) + + file_nid = _make_id(str(path)) + add_node(file_nid, path.name, 1) + + def _package_name(node) -> str | None: + """`package Foo::Bar;` -> the second `package`-typed child is the name.""" + names = [c for c in node.children if c.type == "package"] + return _text(names[1]) if len(names) >= 2 else None + + def _string_parents(node) -> list[str]: + """Every string / qw-word parent named under an inheritance construct. + + Reads string_literal (`'Foo'`) and quoted_word_list (`qw(Foo Bar)`) + content anywhere below ``node``; autoquoted barewords like ``-norequire`` + are a different node type and are intentionally skipped. + """ + out: list[str] = [] + + def visit(n) -> None: + if n.type in ("string_literal", "interpolated_string_literal", "quoted_word_list"): + for c in n.children: + if c.type == "string_content": + out.extend(_text(c).split()) + return + for c in n.children: + visit(c) + + visit(node) + return out + + def add_inherits(pkg_nid: str, parent_name: str, line: int) -> None: + parent_nid = _make_id(parent_name) + add_node(parent_nid, parent_name, line) + add_edge(pkg_nid, parent_nid, "inherits", line, context="inherit") + + current_pkg_nid: str | None = None + + def handle_use(node, line: int) -> None: + # In a use_statement the `use` keyword is its own node type, so the + # module/pragma name is the first (and only) `package`-typed child — + # unlike package_statement, where `package` is also the keyword's type. + pkgs = [c for c in node.children if c.type == "package"] + if not pkgs: + return + module = _text(pkgs[0]) + if module in _PERL_INHERIT_PRAGMAS: + if current_pkg_nid: + for parent in _string_parents(node): + add_inherits(current_pkg_nid, parent, line) + return + if module in _PERL_PRAGMAS: + return + add_edge(file_nid, _make_id(module), "imports", line, context="import") + + def handle_require(req_node, line: int) -> None: + for c in req_node.children: + if c.type == "bareword": + add_edge(file_nid, _make_id(_text(c)), "imports", line, context="import") + return + + def handle_isa(assign_node, line: int) -> None: + """`our @ISA = (...)` -> inherits edges to each named parent.""" + if not current_pkg_nid: + return + is_isa = False + for child in assign_node.children: + if child.type == "variable_declaration": + for sub in child.children: + if sub.type == "array" and _text(sub) == "@ISA": + is_isa = True + if not is_isa: + return + for parent in _string_parents(assign_node): + add_inherits(current_pkg_nid, parent, line) + + def walk_statements(node) -> None: + nonlocal current_pkg_nid + for child in node.children: + line = child.start_point[0] + 1 + if child.type == "package_statement": + name = _package_name(child) + if name: + pkg_nid = _make_id(stem, name) + add_node(pkg_nid, name, line) + add_edge(file_nid, pkg_nid, "contains", line) + current_pkg_nid = pkg_nid + elif child.type == "use_statement": + handle_use(child, line) + elif child.type == "subroutine_declaration_statement": + name = None + block = None + for c in child.children: + if c.type == "bareword" and name is None: + name = _text(c) + elif c.type == "block": + block = c + if name: + container = current_pkg_nid or file_nid + sub_nid = _make_id(container, name) + add_node(sub_nid, f"{name}()", line) + add_edge(container, sub_nid, "contains", line) + if block is not None: + sub_bodies.append((sub_nid, block)) + elif child.type == "expression_statement": + for c in child.children: + if c.type == "require_expression": + handle_require(c, line) + elif c.type == "assignment_expression": + handle_isa(c, line) + + walk_statements(root) + + raw_calls: list[dict] = [] + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "function_call_expression": + for c in node.children: + if c.type == "function": + callee = _text(c).split("::")[-1] + if callee and callee not in _PERL_BUILTINS: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": False, + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + elif node.type == "method_call_expression": + for c in node.children: + if c.type == "method": + callee = _text(c).split("::")[-1] + if callee: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": True, + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + for child in node.children: + walk_calls(child, caller_nid) + + for caller_nid, body in sub_bodies: + walk_calls(body, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] == "imports")] + return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, + "input_tokens": 0, "output_tokens": 0} diff --git a/pyproject.toml b/pyproject.toml index 50f05367ae..eb38addd87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "tree-sitter-fortran>=0.6,<0.8", "tree-sitter-bash>=0.23,<0.27", "tree-sitter-json>=0.23,<0.26", + "tree-sitter-perl>=1.2.0,<2.0", ] [project.urls] diff --git a/uv.lock b/uv.lock index d772b6cf81..bb04688ffe 100644 --- a/uv.lock +++ b/uv.lock @@ -1114,6 +1114,7 @@ dependencies = [ { name = "tree-sitter-kotlin" }, { name = "tree-sitter-lua" }, { name = "tree-sitter-objc" }, + { name = "tree-sitter-perl" }, { name = "tree-sitter-php" }, { name = "tree-sitter-powershell" }, { name = "tree-sitter-python" }, @@ -1329,6 +1330,7 @@ requires-dist = [ { name = "tree-sitter-ocaml", marker = "extra == 'ocaml'" }, { name = "tree-sitter-pascal", marker = "extra == 'all'" }, { name = "tree-sitter-pascal", marker = "extra == 'pascal'" }, + { name = "tree-sitter-perl", specifier = ">=1.2.0,<2.0" }, { name = "tree-sitter-php", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-powershell", specifier = ">=0.26,<0.28" }, { name = "tree-sitter-python", specifier = ">=0.23,<0.26" }, @@ -4803,6 +4805,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/ad/e5381365ed5b247cad2e047682b081ccbb79f84bb5c1b48642ef3385f1d3/tree_sitter_pascal-0.11.0-cp38-abi3-win_arm64.whl", hash = "sha256:9d22b974a47765f1c98629338d82d1df89c35c270a73f08dda8fbd49538e32d4", size = 124914, upload-time = "2026-07-01T03:53:15.191Z" }, ] +[[package]] +name = "tree-sitter-perl" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/44/c211fda4f42c01791391928b23192b50d160b14e676f2c1fcce1850d76d9/tree_sitter_perl-1.2.1.tar.gz", hash = "sha256:7b5ce3fe9cd7d23b1619ed49d5539b106e301b68dd7139bcae6b9d4ad1e3a1d3", size = 1749866, upload-time = "2026-06-30T18:53:23.646Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/29/4cdb0da188ce7864507317e70db6c9dc045f41d6250f0865aef03294eeb2/tree_sitter_perl-1.2.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5e77a13168ae5699ee9942aaa928920b9a2d8ce051934550ba2a4b547833a8ba", size = 347325, upload-time = "2026-06-30T18:53:14.467Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b3/7489ac765c9d5e2a8ef75f43caaa5f1df3cbc93bbc5c0f49559bc4c2e280/tree_sitter_perl-1.2.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a18f09b6b76d9bb52da49f3bb4ed7fe4db0857fbf21d1aad78021abaddb47d72", size = 408980, upload-time = "2026-06-30T18:53:16.211Z" }, + { url = "https://files.pythonhosted.org/packages/a3/50/7340015653da77bc5d12e45678c9339cabf389aba7404d3f5e0514e04acc/tree_sitter_perl-1.2.1-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6cd149c5f5fb5f82d337d5764d8626eecb03b00604c8c0c7ffc4e165e26b273", size = 385181, upload-time = "2026-06-30T18:53:17.741Z" }, + { url = "https://files.pythonhosted.org/packages/8b/77/76486f28c0d837d1424d3a314736ca2b6916a56b5adf073d6e038fb03522/tree_sitter_perl-1.2.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:66361e2d61467843aeb9ef1233a461e2880ed294a368d81e4b898b33c76913fe", size = 370909, upload-time = "2026-06-30T18:53:19.223Z" }, + { url = "https://files.pythonhosted.org/packages/7f/cd/6fbecae40fb5f78bcb95432fc41d6eb0b6a246e43bb63237544c92aa271a/tree_sitter_perl-1.2.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0cfd166f0f11de834b96d99432a86b54444ccbd270d9e19eed06c2de9d6f932e", size = 373213, upload-time = "2026-06-30T18:53:20.769Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/a17bd77d350cc7344baf6f3f568f297a1d1a88d2afbf3024a8e86618e96f/tree_sitter_perl-1.2.1-cp39-abi3-win_amd64.whl", hash = "sha256:168704111ed36461b1d1da1a36b16332e2c8e2693b5daec57bd8abada1e02f58", size = 320105, upload-time = "2026-06-30T18:53:22.14Z" }, +] + [[package]] name = "tree-sitter-php" version = "0.24.1" From c8d6b121122b82bb02663868aae1441dc0963172 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:24:17 +0000 Subject: [PATCH 03/23] fix(extract): package-aware perl call resolution, inherit stubs, builtin filter, edge dedup Review findings on the Perl extractor (S3b): - F1/F3: raw_calls now carry callee_package (qualifier) + caller_package (enclosing package). The shared second pass package-aware-filters candidates: a qualified Pkg::sub() binds only to a sub in that package; a bare call prefers the caller's own package, else requires unique import evidence. No unique match drops the edge (zero-edge, not a wrong-package guess). Additive + guarded: only Perl raw_calls carry the fields, so every other language's resolution is unchanged. - F4: external inheritance parents (RTL / other module) are emitted as stubs with empty source_file instead of falsely claiming the child file. - F5: builtin filter extended to the perlfunc core (open/close/system/exec/ index/substr/time/sleep/mkdir/unlink/...). - F6: edges deduped on (source, target, relation, context). Second-pass call tests remain RED until S4 wires .pl/.pm dispatch. --- graphify/extract.py | 46 +++++++++++++++ graphify/extractors/perl.py | 80 ++++++++++++++++++++----- tests/test_languages.py | 114 ++++++++++++++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 14 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 5cf3343bcf..02c7c85a5e 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6669,6 +6669,19 @@ def _looks_like_bash(result: object) -> bool: # of these files with no import evidence is gated below (#1659). _JS_TS_CALL_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") _go_module_cache: dict[Path, str | None] = {} + + # Enclosing-package label per node id, for the package-aware call resolution + # below. Only consulted for raw_calls that carry package qualifiers (Perl): + # a sub's direct container is its package node, whose label IS the package + # name (e.g. "Acme::Widget"), so the `contains` edge target->source gives + # sub_id -> package label. Other languages never set the package fields, so + # this map is built but never read for them. + _label_by_nid = {n["id"]: n.get("label", "") for n in all_nodes} + pkg_label_by_nid: dict[str, str] = {} + for e in all_edges: + if e.get("relation") == "contains": + pkg_label_by_nid[e["target"]] = _label_by_nid.get(e["source"], "") + for rc in all_raw_calls: callee = rc.get("callee", "") if not callee: @@ -6764,6 +6777,39 @@ def _has_import_evidence(candidate_id: str) -> bool: or (candidate_file_nid is not None and candidate_file_nid in imported_modules) ) + # Package-aware pre-filter for languages that tag calls with their + # package (Perl). Additive + guarded: a raw_call only reaches this branch + # when it carries a package field, which no other language sets, so every + # other language's candidate set is untouched (the existing 314 tests + # prove the unqualified path is unchanged). Zero-edge over a wrong guess: + # an unresolvable qualifier or a foreign-package bare call is dropped, not + # bound to a same-named sub in the wrong package (#F1/#F3). + callee_package = rc.get("callee_package") + caller_package = rc.get("caller_package") + if callee_package is not None or caller_package is not None: + if callee_package is not None: + # Qualified call `Pkg::sub()`: bind only to a sub whose enclosing + # package matches the qualifier. None or several -> drop. + candidates = [ + c for c in candidates + if pkg_label_by_nid.get(c) == callee_package + ] + else: + # Bare call: prefer the caller's own package. If the sub is + # defined there, that's the target. Otherwise it can only be an + # imported sub — require unique import evidence, else drop (never + # bind a same-named sub from an unrelated package). + same_pkg = [ + c for c in candidates + if pkg_label_by_nid.get(c) == caller_package + ] + if same_pkg: + candidates = same_pkg + else: + candidates = [c for c in candidates if _has_import_evidence(c)] + if len(candidates) != 1: + continue + if len(candidates) == 1: tgt = candidates[0] has_import_evidence = go_exact_import or _has_import_evidence(tgt) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 6b40740d23..e1ce68b6b4 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -25,14 +25,32 @@ _PERL_INHERIT_PRAGMAS: frozenset[str] = frozenset({"parent", "base"}) # Perl builtins that surface as function_call_expression callees; excluding them -# keeps them from accumulating spurious calls edges as god-nodes. +# keeps them from accumulating spurious calls edges as god-nodes. Kept to the +# canonical perlfunc core so a user sub that happens to share a name with a +# builtin still resolves against real definitions in the corpus. _PERL_BUILTINS: frozenset[str] = frozenset({ - "print", "printf", "say", "sprintf", "die", "warn", "return", "bless", + # I/O & formatting + "print", "printf", "say", "sprintf", "open", "close", "read", "write", + "binmode", "eof", "seek", "tell", "sysread", "syswrite", "readline", + # process / system + "system", "exec", "fork", "wait", "waitpid", "kill", "sleep", "time", + "exit", "die", "warn", + # filesystem + "mkdir", "rmdir", "unlink", "rename", "chdir", "chmod", "chown", "stat", + "lstat", "opendir", "readdir", "closedir", "glob", + # list / hash ops "shift", "unshift", "push", "pop", "splice", "map", "grep", "sort", "reverse", "join", "split", "keys", "values", "each", "exists", "delete", - "defined", "ref", "scalar", "wantarray", "length", "uc", "lc", "ucfirst", - "lcfirst", "chomp", "chop", "chr", "ord", "abs", "int", "sqrt", "eval", - "local", "my", "our", "sub", "do", "require", "use", "no", + "wantarray", + # string ops + "index", "rindex", "substr", "length", "uc", "lc", "ucfirst", "lcfirst", + "chomp", "chop", "chr", "ord", "hex", "oct", "sprintf", "pack", "unpack", + "quotemeta", + # math + "abs", "int", "sqrt", "sin", "cos", "atan2", "exp", "log", "rand", "srand", + # scalars / refs / types + "defined", "ref", "scalar", "bless", "return", "eval", "local", "my", + "our", "sub", "do", "require", "use", "no", "wantarray", }) @@ -58,7 +76,12 @@ def extract_perl(path: Path) -> dict: nodes: list[dict] = [] edges: list[dict] = [] seen_ids: set[str] = set() - sub_bodies: list[tuple[str, Any]] = [] + # Dedup edges on their identity tuple so a repeated construct (e.g. two + # `use Foo;` in one file) yields a single edge instead of parallel dups. + seen_edges: set[tuple[str, str, str, str | None]] = set() + # sub_nid, body block, and the sub's enclosing package name (for the shared + # second pass's package-aware call resolution). + sub_bodies: list[tuple[str, Any, str | None]] = [] def _text(node) -> str: return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") @@ -69,8 +92,27 @@ def add_node(nid: str, label: str, line: int) -> None: nodes.append({"id": nid, "label": label, "file_type": "code", "source_file": str_path, "source_location": f"L{line}"}) + def add_stub_node(nid: str, label: str) -> None: + """External inheritance target (base class defined elsewhere / in the RTL). + + Emitted with an empty ``source_file`` so it does not falsely claim this + child file as the parent's source and so the corpus-level cross-file + rewire can collapse it onto the real definition; ``origin_file`` keeps + distinct same-named stubs apart in the colliding-id pass. Mirrors the + external-stub shape used by go/julia/objc. + """ + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": "", "source_location": "", + "origin_file": str_path}) + def add_edge(src: str, tgt: str, relation: str, line: int, confidence: str = "EXTRACTED", context: str | None = None) -> None: + key = (src, tgt, relation, context) + if key in seen_edges: + return + seen_edges.add(key) edge = {"source": src, "target": tgt, "relation": relation, "confidence": confidence, "source_file": str_path, "source_location": f"L{line}", "weight": 1.0} @@ -109,10 +151,11 @@ def visit(n) -> None: def add_inherits(pkg_nid: str, parent_name: str, line: int) -> None: parent_nid = _make_id(parent_name) - add_node(parent_nid, parent_name, line) + add_stub_node(parent_nid, parent_name) add_edge(pkg_nid, parent_nid, "inherits", line, context="inherit") current_pkg_nid: str | None = None + current_pkg_name: str | None = None def handle_use(node, line: int) -> None: # In a use_statement the `use` keyword is its own node type, so the @@ -153,7 +196,7 @@ def handle_isa(assign_node, line: int) -> None: add_inherits(current_pkg_nid, parent, line) def walk_statements(node) -> None: - nonlocal current_pkg_nid + nonlocal current_pkg_nid, current_pkg_name for child in node.children: line = child.start_point[0] + 1 if child.type == "package_statement": @@ -163,6 +206,7 @@ def walk_statements(node) -> None: add_node(pkg_nid, name, line) add_edge(file_nid, pkg_nid, "contains", line) current_pkg_nid = pkg_nid + current_pkg_name = name elif child.type == "use_statement": handle_use(child, line) elif child.type == "subroutine_declaration_statement": @@ -179,7 +223,7 @@ def walk_statements(node) -> None: add_node(sub_nid, f"{name}()", line) add_edge(container, sub_nid, "contains", line) if block is not None: - sub_bodies.append((sub_nid, block)) + sub_bodies.append((sub_nid, block, current_pkg_name)) elif child.type == "expression_statement": for c in child.children: if c.type == "require_expression": @@ -191,15 +235,23 @@ def walk_statements(node) -> None: raw_calls: list[dict] = [] - def walk_calls(node, caller_nid: str) -> None: + def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: if node.type == "function_call_expression": for c in node.children: if c.type == "function": - callee = _text(c).split("::")[-1] + # `Acme::Helper::emit` -> callee `emit`, callee_package + # `Acme::Helper`; a bare `emit` -> callee `emit`, no package. + # The qualifier + caller package let the shared second pass + # bind to the right same-named sub instead of any `emit()`. + parts = _text(c).split("::") + callee = parts[-1] + callee_package = "::".join(parts[:-1]) or None if callee and callee not in _PERL_BUILTINS: raw_calls.append({ "caller_nid": caller_nid, "callee": callee, + "callee_package": callee_package, + "caller_package": caller_package, "is_member_call": False, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -219,10 +271,10 @@ def walk_calls(node, caller_nid: str) -> None: }) break for child in node.children: - walk_calls(child, caller_nid) + walk_calls(child, caller_nid, caller_package) - for caller_nid, body in sub_bodies: - walk_calls(body, caller_nid) + for caller_nid, body, caller_package in sub_bodies: + walk_calls(body, caller_nid, caller_package) clean_edges = [e for e in edges if e["source"] in seen_ids and (e["target"] in seen_ids or e["relation"] == "imports")] diff --git a/tests/test_languages.py b/tests/test_languages.py index 767ddaa7e0..3ddb0c1511 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4154,6 +4154,74 @@ def test_perl_no_dangling_edges(): for e in r["edges"]: assert e["source"] in node_ids, f"dangling edge source: {e}" +def test_perl_inherit_stub_has_no_source_file(): + """Inheritance parents defined outside this file (RTL / another module) must + not claim the child file as their source; they are external stubs with an + empty source_file (mirrors go/julia/objc).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + stubs = [n for n in r["nodes"] + if n["label"] in ("Acme::Base", "Acme::Role", "Acme::Mixin")] + assert stubs, "expected external inheritance stub nodes" + for n in stubs: + assert n.get("source_file") == "", ( + f"external parent stub {n['label']!r} must have empty source_file, " + f"got {n.get('source_file')!r}" + ) + +def test_perl_builtins_not_called(tmp_path): + """Perl core builtins (open/close/system/exec/index/…) surface as call + expressions but are not user subs; they must not become raw-call edges.""" + from graphify.extract import extract_perl + src = tmp_path / "builtins.pm" + src.write_text( + "package B;\n" + "sub run {\n" + " open(my $fh, '<', 'f');\n" + " system('ls');\n" + " my $i = index('abc', 'b');\n" + " my $n = substr('abc', 1);\n" + " close($fh);\n" + "}\n" + "1;\n" + ) + r = extract_perl(src) + callees = {rc["callee"] for rc in r["raw_calls"]} + for b in ("open", "system", "index", "substr", "close"): + assert b not in callees, f"builtin {b!r} must not be a raw call, got {callees}" + +def test_perl_duplicate_use_dedup(tmp_path): + """Two identical `use Foo;` statements must dedup to a single imports edge.""" + from graphify.extract import extract_perl + src = tmp_path / "dup.pm" + src.write_text( + "package D;\n" + "use Foo;\n" + "use Foo;\n" + "1;\n" + ) + r = extract_perl(src) + foo_imports = [e for e in r["edges"] + if e["relation"] == "imports" and "foo" in str(e["target"]).lower()] + assert len(foo_imports) == 1, ( + f"duplicate `use Foo;` must dedup to one edge, got {len(foo_imports)}: {foo_imports}" + ) + +def test_perl_raw_calls_carry_package_qualifiers(): + """The extractor half of package-aware resolution: a bare call carries its + caller's package (and no callee_package); a qualified `Pkg::sub()` carries + the qualifier as callee_package. The shared second pass reads these to bind + the right same-named sub.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + by_callee = {rc["callee"]: rc for rc in r["raw_calls"] if not rc.get("is_member_call")} + bare = by_callee["format_line"] + assert bare["callee_package"] is None, bare + assert bare["caller_package"] == "Acme::Widget", bare + qualified = by_callee["emit"] + assert qualified["callee_package"] == "Acme::Helper", qualified + assert qualified["caller_package"] == "Acme::Widget", qualified + # Perl call resolution runs in the top-level `extract()` second pass (raw_calls -> # name-matched calls), so these drive the public multi-file dispatch, like the @@ -4208,3 +4276,49 @@ def test_perl_method_call_no_edge(): assert not any(tgt == "render" or tgt.endswith("::render") or tgt.endswith(".render") for _src, tgt in calls), \ f"method call $widget->render() must not produce a calls edge, got {calls}" + + +# Package-aware second-pass resolution (F1/F3). Same S4 dependency as the corpus +# tests above: the shared pass reads the raw_calls' package qualifiers, but no +# calls edge exists until the .pl/.pm dispatch is registered — RED until S4. + +def _calls_with_target_pkg(r): + """(caller_label, target_label, target_enclosing_package_label) per calls edge.""" + label = {n["id"]: n.get("label", "") for n in r["nodes"]} + contained_by = {e["target"]: e["source"] for e in r["edges"] if e["relation"] == "contains"} + return [ + (label.get(e["source"], e["source"]), + label.get(e["target"], e["target"]), + label.get(contained_by.get(e["target"]), "")) + for e in r["edges"] if e["relation"] == "calls" + ] + +def test_perl_qualified_call_binds_correct_package(tmp_path): + """Two packages define `emit`; the package-qualified call `P::A::emit()` must + bind only to P::A's emit, never the same-named sub in P::B.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "b.pm").write_text("package P::B;\nsub emit { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text("package P::C;\nsub run { P::A::emit(); }\n1;\n") + r = extract([tmp_path / "a.pm", tmp_path / "b.pm", tmp_path / "c.pm"], parallel=False) + emit_calls = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "emit" in t] + assert emit_calls, f"expected a calls edge into emit, got none: {_calls_with_target_pkg(r)}" + assert all(p == "P::A" for (_s, _t, p) in emit_calls), \ + f"qualified P::A::emit() must bind only to P::A's emit, got {emit_calls}" + +def test_perl_bare_foreign_package_call_no_edge(tmp_path): + """A bare `helper()` call resolves to a same-package sub, never to a + same-named sub in a DIFFERENT, un-imported package (zero-edge, not a guess).""" + from graphify.extract import extract + (tmp_path / "x.pm").write_text("package X;\nsub helper { return 1; }\n1;\n") + (tmp_path / "y.pm").write_text( + "package Y;\nsub run { helper(); other(); }\nsub other { return 2; }\n1;\n" + ) + r = extract([tmp_path / "x.pm", tmp_path / "y.pm"], parallel=False) + calls = _calls(r) + # Positive guard: the same-package bare call must resolve, so the negative + # assertion below is not a vacuous pass on an empty (pre-S4) graph. + assert any("run" in s and "other" in t for s, t in calls), \ + f"expected run->other same-package resolution, got {calls}" + assert not any("helper" in t for _s, t in calls), \ + f"bare helper() must not bind to X::helper without import evidence, got {calls}" From 8ffd1799220ff80b7e9b2b0523cedc488ae23bc9 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:32:05 +0000 Subject: [PATCH 04/23] feat(extract): dispatch .pl/.pm and perl shebang to perl extractor Register Perl in the four extension/dispatch tables so .pl/.pm files and extensionless #!/usr/bin/perl scripts reach extract_perl: - _DISPATCH: .pl/.pm -> extract_perl - _SHEBANG_DISPATCH: perl -> extract_perl (detect already routes perl shebang) - suffix->language map: .pl/.pm -> perl (language stats) - detect.CODE_EXTENSIONS: .pl/.pm (watch._WATCHED_EXTENSIONS inherits) .t deliberately excluded (ambiguous across ecosystems). Moves the perl example in test_extract from the unsupported-shebang test (now extract_perl) to the supported-dispatch test; tcsh takes its place as a known-but-unmapped interpreter. --- graphify/detect.py | 2 +- graphify/extract.py | 6 +++++- tests/test_extract.py | 11 ++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 3668eb6fcf..3d8c843950 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -41,7 +41,7 @@ class FileType(str, Enum): _MTIME_COARSE_S = 2.0 _MTIME_SUBSECOND_S = 0.05 -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.pl', '.pm', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index 02c7c85a5e..50dd14e88e 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2243,6 +2243,7 @@ def _lang_is_case_insensitive(source_file: object) -> bool: ".lua": "lua", ".luau": "lua", ".zig": "zig", ".ex": "elixir", ".exs": "elixir", + ".pl": "perl", ".pm": "perl", ".jl": "julia", ".dart": "dart", ".sh": "shell", ".bash": "shell", @@ -5200,6 +5201,8 @@ def add_existing_edge(edge: dict) -> None: ".psd1": extract_powershell_manifest, ".ex": extract_elixir, ".exs": extract_elixir, + ".pl": extract_perl, + ".pm": extract_perl, ".m": extract_objc, ".mm": extract_objc, ".jl": extract_julia, @@ -5296,7 +5299,7 @@ def add_existing_edge(edge: dict) -> None: # routes them to the CODE path via _shebang_interpreter; _get_extractor must # honor the same signal or these files are classified as code and then silently # dropped by extraction. Only interpreters with a real extractor are mapped — -# detect's wider set (perl, fish, tcsh, Rscript) stays unmapped and skipped. +# detect's wider set (fish, tcsh, Rscript) stays unmapped and skipped. _SHEBANG_DISPATCH: dict[str, Any] = { "python": extract_python, "python2": extract_python, @@ -5312,6 +5315,7 @@ def add_existing_edge(edge: dict) -> None: "lua": extract_lua, "php": extract_php, "julia": extract_julia, + "perl": extract_perl, } diff --git a/tests/test_extract.py b/tests/test_extract.py index 09d9b5c93f..134b12fa04 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -3438,6 +3438,11 @@ def test_extensionless_shebang_via_dispatch(tmp_path): split.write_text("#!/usr/bin/env -S bash -eu\necho hi\n") assert _get_extractor(split) is extract_bash + from graphify.extract import extract_perl + perltool = tmp_path / "legacy" + perltool.write_text("#!/usr/bin/perl\nprint 1;\n") + assert _get_extractor(perltool) is extract_perl + def test_extensionless_without_usable_shebang_stays_unsupported(tmp_path): from graphify.extract import _get_extractor @@ -3448,9 +3453,9 @@ def test_extensionless_without_usable_shebang_stays_unsupported(tmp_path): # Interpreter known to detect but with no AST extractor: stays skipped # rather than being mis-parsed by a wrong grammar. - perl = tmp_path / "legacy" - perl.write_text("#!/usr/bin/env perl\nprint 1;\n") - assert _get_extractor(perl) is None + tcsh = tmp_path / "legacy" + tcsh.write_text("#!/usr/bin/env tcsh\necho 1\n") + assert _get_extractor(tcsh) is None def test_extract_extensionless_bash_cli_end_to_end(tmp_path): From 9ada0df108abd3a88ce3455c4cce07026a9e8d44 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:40:47 +0000 Subject: [PATCH 05/23] fix(extract): bare perl call resolves to sub in use-imported package The package-aware bare-call fallback only accepted direct symbol/module import evidence, but `use P::A;` emits an imports edge to the module label id (_make_id('P::A')), never to the sub id. A bare emit() call to a sub defined in a use-imported foreign package therefore dropped even though the import unambiguously names its package. Bridge it with _has_package_import_evidence: the candidate sub's enclosing package, re-idized the same way the use target is, must be among the caller file's imports. Strictly scoped to the Perl/package branch; ambiguity (two imported packages, same sub name) still drops. Constraint: zero-edge over a guess preserved for the ambiguous case. --- graphify/extract.py | 14 +++++++++++++- tests/test_languages.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/graphify/extract.py b/graphify/extract.py index 50dd14e88e..62856204d6 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6781,6 +6781,15 @@ def _has_import_evidence(candidate_id: str) -> bool: or (candidate_file_nid is not None and candidate_file_nid in imported_modules) ) + def _has_package_import_evidence(candidate_id: str) -> bool: + # Perl-only: `use P::A;` emits an imports edge to the MODULE label id + # (`_make_id('P::A')`), never to the sub id — so a bare call to an + # imported package's sub has no direct symbol/module evidence above. + # Bridge it: the candidate sub's enclosing package, re-idized the same + # way the `use` target is, must be among the caller file's imports. + pkg = pkg_label_by_nid.get(candidate_id, "") + return bool(pkg) and _make_id(pkg) in imported_symbols + # Package-aware pre-filter for languages that tag calls with their # package (Perl). Additive + guarded: a raw_call only reaches this branch # when it carries a package field, which no other language sets, so every @@ -6810,7 +6819,10 @@ def _has_import_evidence(candidate_id: str) -> bool: if same_pkg: candidates = same_pkg else: - candidates = [c for c in candidates if _has_import_evidence(c)] + candidates = [ + c for c in candidates + if _has_import_evidence(c) or _has_package_import_evidence(c) + ] if len(candidates) != 1: continue diff --git a/tests/test_languages.py b/tests/test_languages.py index 3ddb0c1511..236e6e1449 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4322,3 +4322,40 @@ def test_perl_bare_foreign_package_call_no_edge(tmp_path): f"expected run->other same-package resolution, got {calls}" assert not any("helper" in t for _s, t in calls), \ f"bare helper() must not bind to X::helper without import evidence, got {calls}" + + +def test_perl_bare_call_binds_imported_package(tmp_path): + """A bare `emit()` call resolves to a sub in a FOREIGN package when that + package is pulled in with `use P::A;` — the `use` import is the evidence that + disambiguates the otherwise-foreign sub. `use P::A;` emits an imports edge to + the module label (`_make_id('P::A')`), not to the sub id, so plain + symbol-import evidence misses it; package-import evidence must bridge the gap.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nsub run { emit(); }\n1;\n" + ) + r = extract([tmp_path / "a.pm", tmp_path / "c.pm"], parallel=False) + emit_calls = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "emit" in t] + assert emit_calls, \ + f"bare emit() with `use P::A;` must bind to P::A::emit, got none: {_calls_with_target_pkg(r)}" + assert all(p == "P::A" for (_s, _t, p) in emit_calls), \ + f"emit() must bind only to the imported P::A::emit, got {emit_calls}" + + +def test_perl_bare_call_two_imported_packages_ambiguous_no_edge(tmp_path): + """Two imported packages both define `emit`; a bare `emit()` is ambiguous — + the `use` import evidence points at two candidates, so the edge is dropped + (zero-edge over a guess), same discipline as the qualified-drop path.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "b.pm").write_text("package P::B;\nsub emit { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nuse P::B;\nsub run { emit(); }\n1;\n" + ) + r = extract( + [tmp_path / "a.pm", tmp_path / "b.pm", tmp_path / "c.pm"], parallel=False + ) + calls = _calls(r) + assert not any("emit" in t for _s, t in calls), \ + f"bare emit() imported from two packages is ambiguous and must drop, got {calls}" From 2d1dfe4f0779aa7936acce3ba1d4a0f75e1e060c Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:09:41 +0000 Subject: [PATCH 06/23] fix(extract): repoint perl imports to in-corpus packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `use Foo::Bar;` / `require Foo::Bar;` emit an imports edge to a bare module-label id (`_make_id('Foo::Bar')`) that matches no node when Foo::Bar is a package defined elsewhere in the corpus — its package node's id is `_make_id(stem, 'Foo::Bar')`. Add `_resolve_perl_imports`, run in the top-level `extract()` AFTER the shared cross-file call pass, to re-point those edges onto the real package node keyed by package LABEL (multi-package files have unrelated stems). External modules keep their bare, dangling target, matching how other languages leave unresolved external imports. Ordering is load-bearing: `_has_package_import_evidence` reads imports targets as bare module-label ids to bind a bare call to an imported package's sub, so the re-point must run after call resolution, not before. On the Foswiki core corpus (378 files): dangling imports 671 -> 174 (497 in-corpus edges re-pointed; the 174 remaining are genuine externals). --- graphify/extract.py | 11 ++++++++- graphify/extractors/perl.py | 48 ++++++++++++++++++++++++++++++++++++ tests/test_languages.py | 49 +++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/graphify/extract.py b/graphify/extract.py index 62856204d6..5603d184f6 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -50,7 +50,7 @@ from graphify.extractors.markdown import extract_markdown, _MD_LINK_INDEX_CACHE # noqa: F401 from graphify.extractors.ocaml import extract_ocaml # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 -from graphify.extractors.perl import extract_perl # noqa: F401 +from graphify.extractors.perl import _resolve_perl_imports, extract_perl # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 @@ -6954,6 +6954,15 @@ def _has_package_import_evidence(candidate_id: str) -> bool: else: run_language_resolvers(paths, per_file, all_nodes, all_edges) + # Re-point dangling in-corpus Perl `imports` edges onto the real package node. + # Runs AFTER the call pass above so `_has_package_import_evidence` still sees the + # bare module-label ids it needs to bind bare calls to imported packages (#S5b). + try: + _resolve_perl_imports(all_nodes, all_edges) + except Exception as exc: + import logging + logging.getLogger(__name__).warning("Perl import resolution failed, skipping: %s", exc) + # Relativize source_file fields so paths are portable across machines (#555). # When the node's id was itself minted from the absolute path, remap it to a # portable id and rewrite the edge endpoints that reference it. diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index e1ce68b6b4..9353e894f9 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -280,3 +280,51 @@ def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: (e["target"] in seen_ids or e["relation"] == "imports")] return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, "input_tokens": 0, "output_tokens": 0} + + +def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: + """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. + + ``use Foo::Bar;`` / ``require Foo::Bar;`` emit an imports edge to a bare + module-label id (``_make_id('Foo::Bar')``) that matches no node: when Foo::Bar + is defined in the corpus its package node's id is ``_make_id(stem, 'Foo::Bar')`` + (and a ``package Foo::Bar;`` may live in a file whose stem is unrelated — a + multi-package file — so we key on the package LABEL, not the file stem). Bridge + module-label -> real package id so in-corpus imports connect instead of + dangling. External modules (POSIX, Carp, …) have no in-corpus package node, so + their edge keeps its bare target — matching the dangling-stub behavior other + languages leave on unresolved external imports. + + Runs AFTER the shared cross-file call pass: ``_has_package_import_evidence`` + (extract.py) reads imports targets as bare module-label ids to bind a bare call + to an imported package's sub, so re-pointing before it would break that binding. + Mutates ``all_edges`` in place; the bare target was never a node, so there is + nothing to prune. + """ + # module-label id -> package node id. First package with a given fully- + # qualified label wins (deterministic in node order); a bare `use Foo` can't + # disambiguate a re-opened same-name package, and picking one beats dangling. + pkg_by_label_id: dict[str, str] = {} + for node in all_nodes: + src = str(node.get("source_file") or "") + if not src.endswith((".pl", ".pm")): + continue + label = node.get("label", "") + nid = node.get("id", "") + # package nodes only: skip the file node (label == basename) and subs + # (label ends in `()`); neither keys an import target. + if not label or not nid or label.endswith(")") or label == Path(src).name: + continue + pkg_by_label_id.setdefault(_make_id(label), nid) + if not pkg_by_label_id: + return + for edge in all_edges: + if edge.get("relation") != "imports": + continue + # Scope to Perl imports only, so a same-named module-label id in another + # language's imports edge is never re-pointed onto a Perl package node. + if not str(edge.get("source_file") or "").endswith((".pl", ".pm")): + continue + real = pkg_by_label_id.get(edge.get("target")) + if real and real != edge["target"]: + edge["target"] = real diff --git a/tests/test_languages.py b/tests/test_languages.py index 236e6e1449..1b06f70d52 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4359,3 +4359,52 @@ def test_perl_bare_call_two_imported_packages_ambiguous_no_edge(tmp_path): calls = _calls(r) assert not any("emit" in t for _s, t in calls), \ f"bare emit() imported from two packages is ambiguous and must drop, got {calls}" + + +# In-corpus import re-pointer (S5b Befund A): a `use`/`require` whose module is a +# package defined elsewhere in the corpus must re-point its imports edge from the +# bare module-label id onto the real package node; external modules stay dangling. + +def test_perl_import_repoints_to_in_corpus_package(tmp_path): + """`use Acme::Helper;` from another file must re-point its imports edge onto + the real Acme::Helper package node (id `_make_id(stem, 'Acme::Helper')`), + not dangle on the bare module-label id `_make_id('Acme::Helper')`. An external + `use POSIX;` has no in-corpus package, so its edge keeps the bare (node-less) + target — mirroring how other languages leave external imports dangling.""" + from graphify.extract import extract + from graphify.extractors.base import _make_id + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + (tmp_path / "main.pm").write_text( + "package Main;\nuse Acme::Helper;\nuse POSIX qw(floor);\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", tmp_path / "main.pm"], parallel=False) + node_ids = {n["id"] for n in r["nodes"]} + pkg_nodes = [n for n in r["nodes"] if n.get("label") == "Acme::Helper"] + assert len(pkg_nodes) == 1, f"expected one Acme::Helper package node, got {pkg_nodes}" + pkg_id = pkg_nodes[0]["id"] + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"use Acme::Helper must re-point to package node {pkg_id}, got {[e['target'] for e in imports]}" + bare_helper = _make_id("Acme::Helper") + assert bare_helper != pkg_id + assert not any(e["target"] == bare_helper for e in imports), \ + "the in-corpus import must no longer dangle on the bare module-label id" + posix_id = _make_id("POSIX") + assert any(e["target"] == posix_id for e in imports), "external POSIX import edge missing" + assert posix_id not in node_ids, \ + "external POSIX must stay a dangling label-id stub, not become a node" + + +def test_perl_require_repoints_to_in_corpus_package(tmp_path): + """`require Acme::Helper;` (bareword require form) re-points the same way as + `use` when the module is an in-corpus package.""" + from graphify.extract import extract + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + (tmp_path / "main.pm").write_text( + "package Main;\nrequire Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", tmp_path / "main.pm"], parallel=False) + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"require Acme::Helper must re-point to package node {pkg_id}, got {[e['target'] for e in imports]}" From a9ff973d88728423c5f8f4cd051c11d51c11dfa8 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:10:30 +0000 Subject: [PATCH 07/23] fix(extract): treat indirect-object new as member call `my $cgi = new CGI();` (indirect-object constructor syntax, == `CGI->new`) parses as ambiguous_function_call_expression(function 'new', function_call_expression 'CGI()'); the inner `CGI()` was recorded as a bare function call, so a same-named sub in the corpus got a spurious `calls` edge. Detect the leading `new` on the parent node and mark the inner call `is_member_call` (edge-less), like any other untyped `$obj->meth()` dispatch. On the Foswiki core corpus this removes exactly the one wrong edge (prepareBody -> CGI, EXTRACTED) and changes nothing else. --- graphify/extractors/perl.py | 14 +++++++++++++- tests/test_languages.py | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 9353e894f9..afa8036a06 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -237,6 +237,18 @@ def walk_statements(node) -> None: def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: if node.type == "function_call_expression": + # Indirect-object constructor `new CLASS(...)` parses as + # ambiguous_function_call_expression(function 'new', function_call_expression + # 'CLASS()'); `new CLASS` == CLASS->new, an untyped member dispatch. Mark + # it a member call (edge-less) so it is not wired to a sub named CLASS. + parent = node.parent + indirect_new = ( + parent is not None + and parent.type == "ambiguous_function_call_expression" + and bool(parent.children) + and parent.children[0].type == "function" + and _text(parent.children[0]) == "new" + ) for c in node.children: if c.type == "function": # `Acme::Helper::emit` -> callee `emit`, callee_package @@ -252,7 +264,7 @@ def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: "callee": callee, "callee_package": callee_package, "caller_package": caller_package, - "is_member_call": False, + "is_member_call": indirect_new, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) diff --git a/tests/test_languages.py b/tests/test_languages.py index 1b06f70d52..23e45b9346 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4278,6 +4278,27 @@ def test_perl_method_call_no_edge(): f"method call $widget->render() must not produce a calls edge, got {calls}" +def test_perl_indirect_object_new_no_call_edge(tmp_path): + """`my $x = new Widget();` is indirect-object constructor syntax (== Widget->new), + an untyped member dispatch; it must NOT bind to a same-named sub `Widget` as if + `Widget()` were a direct function call.""" + from graphify.extract import extract + (tmp_path / "m.pm").write_text( + "package M;\n" + "sub Widget { return 1; }\n" + "sub helper { return 2; }\n" + "sub run { helper(); my $x = new Widget(); }\n" + "1;\n" + ) + r = extract([tmp_path / "m.pm"], parallel=False) + calls = _calls(r) + # Positive guard: the plain bare call resolves, so the negative isn't vacuous. + assert any("run" in s and "helper" in t for s, t in calls), \ + f"expected run->helper bare-call resolution, got {calls}" + assert not any("Widget" in t for _s, t in calls), \ + f"indirect-object `new Widget()` must not create a calls edge, got {calls}" + + # Package-aware second-pass resolution (F1/F3). Same S4 dependency as the corpus # tests above: the shared pass reads the raw_calls' package qualifiers, but no # calls edge exists until the .pl/.pm dispatch is registered — RED until S4. From 9c3d0075f46b3be1392c8c8fb44e5b89d4df34a9 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:28:28 +0000 Subject: [PATCH 08/23] fix(extract): only repoint perl imports on unique package label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A re-opened Perl package declared under the same fully-qualified label in two files (Foswiki-real: package Assert; in AssertOn.pm AND AssertOff.pm) is ambiguous: a bare `use P::A;` cannot say which file it means. The old setdefault map let the first package node in node order win, producing a guessed cross-file import edge — worse than dangling. Collect label_id -> list[node_ids] and re-point only when exactly one package node carries the label; >1 candidate stays dangling on the bare module-label id (zero-edge over a guess). Finding F1 (P1) from Review-3 of the S5b import re-pointer. --- graphify/extractors/perl.py | 20 +++++++++++--------- tests/test_languages.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index afa8036a06..f38c14b8db 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -313,10 +313,12 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: Mutates ``all_edges`` in place; the bare target was never a node, so there is nothing to prune. """ - # module-label id -> package node id. First package with a given fully- - # qualified label wins (deterministic in node order); a bare `use Foo` can't - # disambiguate a re-opened same-name package, and picking one beats dangling. - pkg_by_label_id: dict[str, str] = {} + # module-label id -> list of package node ids carrying that fully-qualified + # label. A bare `use Foo` cannot disambiguate a package re-opened under the + # same label across files (Foswiki-real: `package Assert;` in AssertOn.pm AND + # AssertOff.pm), so we re-point ONLY when exactly one package node matches; + # >1 candidate stays dangling (zero-edge over a guessed cross-file binding). + pkg_ids_by_label_id: dict[str, list[str]] = {} for node in all_nodes: src = str(node.get("source_file") or "") if not src.endswith((".pl", ".pm")): @@ -327,8 +329,8 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: # (label ends in `()`); neither keys an import target. if not label or not nid or label.endswith(")") or label == Path(src).name: continue - pkg_by_label_id.setdefault(_make_id(label), nid) - if not pkg_by_label_id: + pkg_ids_by_label_id.setdefault(_make_id(label), []).append(nid) + if not pkg_ids_by_label_id: return for edge in all_edges: if edge.get("relation") != "imports": @@ -337,6 +339,6 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: # language's imports edge is never re-pointed onto a Perl package node. if not str(edge.get("source_file") or "").endswith((".pl", ".pm")): continue - real = pkg_by_label_id.get(edge.get("target")) - if real and real != edge["target"]: - edge["target"] = real + candidates = pkg_ids_by_label_id.get(edge.get("target")) + if candidates and len(candidates) == 1 and candidates[0] != edge["target"]: + edge["target"] = candidates[0] diff --git a/tests/test_languages.py b/tests/test_languages.py index 23e45b9346..55ba7b003e 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4429,3 +4429,31 @@ def test_perl_require_repoints_to_in_corpus_package(tmp_path): imports = [e for e in r["edges"] if e["relation"] == "imports"] assert any(e["target"] == pkg_id for e in imports), \ f"require Acme::Helper must re-point to package node {pkg_id}, got {[e['target'] for e in imports]}" + + +def test_perl_import_duplicate_package_label_stays_dangling(tmp_path): + """A re-opened package declared under the SAME fully-qualified label in two + files (Foswiki-real: `package Assert;` in both AssertOn.pm and AssertOff.pm) + is ambiguous: a bare `use P::A;` cannot say which file it means. Binding it to + one file arbitrarily is a guessed edge — worse than dangling — so the imports + edge must stay on the bare module-label id (no package node) when >1 package + node carries the label. Only a unique label re-points (zero-edge over a guess).""" + from graphify.extract import extract + from graphify.extractors.base import _make_id + (tmp_path / "a1.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "a2.pm").write_text("package P::A;\nsub other { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nsub run { return 1; }\n1;\n" + ) + r = extract( + [tmp_path / "a1.pm", tmp_path / "a2.pm", tmp_path / "c.pm"], parallel=False + ) + pkg_nodes = [n for n in r["nodes"] if n.get("label") == "P::A"] + assert len(pkg_nodes) == 2, f"expected two P::A package nodes, got {pkg_nodes}" + pkg_ids = {n["id"] for n in pkg_nodes} + imports = [e for e in r["edges"] if e["relation"] == "imports"] + bare_id = _make_id("P::A") + assert any(e["target"] == bare_id for e in imports), \ + f"ambiguous use P::A; must stay dangling on bare id {bare_id}, got {[e['target'] for e in imports]}" + assert not any(e["target"] in pkg_ids for e in imports), \ + f"ambiguous use P::A; must NOT bind to either P::A file node, got {[e['target'] for e in imports]}" From fd11f56ca4e3939d63fbad1c9559d73fb2125c2f Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:29:29 +0000 Subject: [PATCH 09/23] fix(extract): scope perl import re-pointer by extractor provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-pointer scoped package nodes and imports edges by .pl/.pm suffix, so extensionless #!/usr/bin/perl scripts — dispatched to extract_perl by shebang since S4 — had their imports never re-pointed (underreporting). extract() now passes the set of Perl source paths (where _get_extractor is extract_perl) into _resolve_perl_imports, which scopes on membership instead of suffix. Keyed on str(path) to match the nodes' source_file before relativization. Falls back to suffix matching when the set is omitted (direct callers). Finding F2 (P2) from Review-3 of the S5b import re-pointer. --- graphify/extract.py | 6 +++++- graphify/extractors/perl.py | 22 +++++++++++++++++++--- tests/test_languages.py | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 5603d184f6..2f69524a07 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6957,8 +6957,12 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # Re-point dangling in-corpus Perl `imports` edges onto the real package node. # Runs AFTER the call pass above so `_has_package_import_evidence` still sees the # bare module-label ids it needs to bind bare calls to imported packages (#S5b). + # Scope by extractor provenance, not suffix, so extensionless `#!/usr/bin/perl` + # scripts (dispatched to extract_perl by shebang) are re-pointed too. Keyed on + # str(path) to match the nodes' source_file, set before the relativization below. try: - _resolve_perl_imports(all_nodes, all_edges) + _perl_sources = {str(p) for p in paths if _get_extractor(p) is extract_perl} + _resolve_perl_imports(all_nodes, all_edges, _perl_sources) except Exception as exc: import logging logging.getLogger(__name__).warning("Perl import resolution failed, skipping: %s", exc) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index f38c14b8db..30b42262bf 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -294,7 +294,11 @@ def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: "input_tokens": 0, "output_tokens": 0} -def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: +def _resolve_perl_imports( + all_nodes: list[dict], + all_edges: list[dict], + perl_source_files: set[str] | None = None, +) -> None: """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. ``use Foo::Bar;`` / ``require Foo::Bar;`` emit an imports edge to a bare @@ -307,12 +311,24 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: their edge keeps its bare target — matching the dangling-stub behavior other languages leave on unresolved external imports. + ``perl_source_files`` (absolute-path strings, from ``extract()`` where + ``_get_extractor(path) is extract_perl``) scopes both the package-node scan and + the re-pointed edges to Perl provenance. Suffix-based scoping (``.pl``/``.pm``) + silently skipped extensionless ``#!/usr/bin/perl`` scripts, which are dispatched + to extract_perl by shebang and whose imports were never re-pointed + (underreporting). When ``None`` (direct callers), falls back to suffix matching. + Runs AFTER the shared cross-file call pass: ``_has_package_import_evidence`` (extract.py) reads imports targets as bare module-label ids to bind a bare call to an imported package's sub, so re-pointing before it would break that binding. Mutates ``all_edges`` in place; the bare target was never a node, so there is nothing to prune. """ + def _is_perl_source(src: str) -> bool: + if perl_source_files is not None: + return src in perl_source_files + return src.endswith((".pl", ".pm")) + # module-label id -> list of package node ids carrying that fully-qualified # label. A bare `use Foo` cannot disambiguate a package re-opened under the # same label across files (Foswiki-real: `package Assert;` in AssertOn.pm AND @@ -321,7 +337,7 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: pkg_ids_by_label_id: dict[str, list[str]] = {} for node in all_nodes: src = str(node.get("source_file") or "") - if not src.endswith((".pl", ".pm")): + if not _is_perl_source(src): continue label = node.get("label", "") nid = node.get("id", "") @@ -337,7 +353,7 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: continue # Scope to Perl imports only, so a same-named module-label id in another # language's imports edge is never re-pointed onto a Perl package node. - if not str(edge.get("source_file") or "").endswith((".pl", ".pm")): + if not _is_perl_source(str(edge.get("source_file") or "")): continue candidates = pkg_ids_by_label_id.get(edge.get("target")) if candidates and len(candidates) == 1 and candidates[0] != edge["target"]: diff --git a/tests/test_languages.py b/tests/test_languages.py index 55ba7b003e..dd7847e585 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4457,3 +4457,21 @@ def test_perl_import_duplicate_package_label_stays_dangling(tmp_path): f"ambiguous use P::A; must stay dangling on bare id {bare_id}, got {[e['target'] for e in imports]}" assert not any(e["target"] in pkg_ids for e in imports), \ f"ambiguous use P::A; must NOT bind to either P::A file node, got {[e['target'] for e in imports]}" + + +def test_perl_import_repoints_from_shebang_perl_file(tmp_path): + """An extensionless `#!/usr/bin/perl` script is dispatched to extract_perl by + shebang, but its source_file has no .pl/.pm suffix. Scoping the re-pointer by + file suffix would silently skip such a file's imports (underreporting); scoping + by extractor provenance re-points them like any other Perl source.""" + from graphify.extract import extract + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + script = tmp_path / "runme" + script.write_text( + "#!/usr/bin/perl\npackage Main;\nuse Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", script], parallel=False) + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"use Acme::Helper in a shebang-perl file must re-point to {pkg_id}, got {[e['target'] for e in imports]}" From 1b58f6be920fcf8bfa16c7aebae2796b2e197e20 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:02:11 +0000 Subject: [PATCH 10/23] fix(extract): walk block-form perl packages without leaking package state `package Foo { ... }` created the Foo node but never walked the block, so subs inside it were lost, and the current package leaked to following top-level subs (mis-attributed to Foo). Distinguish block-form from statement-form: push package state, walk the block, restore state. --- graphify/extractors/perl.py | 16 +++++++++-- tests/test_languages.py | 53 ++++++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 30b42262bf..28d68b9643 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -205,8 +205,20 @@ def walk_statements(node) -> None: pkg_nid = _make_id(stem, name) add_node(pkg_nid, name, line) add_edge(file_nid, pkg_nid, "contains", line) - current_pkg_nid = pkg_nid - current_pkg_name = name + pkg_block = next( + (c for c in child.children if c.type == "block"), None) + if pkg_block is not None: + # Block-form `package Foo { ... }` scopes Foo to the block + # only. Walk the block under Foo, then restore the prior + # package so following top-level statements are not + # mis-attributed to Foo (and the block's subs are not lost). + prev_nid, prev_name = current_pkg_nid, current_pkg_name + current_pkg_nid, current_pkg_name = pkg_nid, name + walk_statements(pkg_block) + current_pkg_nid, current_pkg_name = prev_nid, prev_name + else: + current_pkg_nid = pkg_nid + current_pkg_name = name elif child.type == "use_statement": handle_use(child, line) elif child.type == "subroutine_declaration_statement": diff --git a/tests/test_languages.py b/tests/test_languages.py index dd7847e585..5110b09723 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4299,9 +4299,56 @@ def test_perl_indirect_object_new_no_call_edge(tmp_path): f"indirect-object `new Widget()` must not create a calls edge, got {calls}" -# Package-aware second-pass resolution (F1/F3). Same S4 dependency as the corpus -# tests above: the shared pass reads the raw_calls' package qualifiers, but no -# calls edge exists until the .pl/.pm dispatch is registered — RED until S4. +def test_perl_block_scoped_package(tmp_path): + """Block-form `package Foo { ... }` scopes Foo to the block only: subs inside + the block belong to Foo, and a sub AFTER the block belongs to the package that + was current before the block (no state leak).""" + from graphify.extract import extract_perl + src = tmp_path / "block.pm" + src.write_text( + "package Outer;\n" + "sub outer_sub { }\n" + "package Acme::Block {\n" + " sub inner { }\n" + "}\n" + "sub after_block { }\n" + "1;\n" + ) + r = extract_perl(src) + label = {n["id"]: n["label"] for n in r["nodes"]} + container_of = { + label.get(e["target"]): label.get(e["source"]) + for e in r["edges"] if e["relation"] == "contains" + } + assert "Acme::Block" in label.values(), "block-form package node must be emitted" + assert container_of.get("inner()") == "Acme::Block", ( + f"sub inside a block package must belong to it, got {container_of.get('inner()')!r}" + ) + assert container_of.get("after_block()") == "Outer", ( + f"block-form package must not leak: after_block belongs to Outer, " + f"got {container_of.get('after_block()')!r}" + ) + + +def test_perl_block_scoped_package_body_calls(tmp_path): + """A bare call inside a block-package sub resolves within that package — + proves the block body is actually walked (not silently skipped).""" + from graphify.extract import extract + (tmp_path / "b.pm").write_text( + "package B::Blk {\n" + " sub inner { helper(); }\n" + " sub helper { return 1; }\n" + "}\n" + "1;\n" + ) + r = extract([tmp_path / "b.pm"], parallel=False) + calls = _calls(r) + assert any("inner" in s and "helper" in t for s, t in calls), \ + f"expected inner->helper inside the block package, got {calls}" + + +# Package-aware second-pass resolution: the shared pass reads the raw_calls' +# package qualifiers to bind a call to the same-named sub in the right package. def _calls_with_target_pkg(r): """(caller_label, target_label, target_enclosing_package_label) per calls edge.""" From 6772976c29d8d99cd61fef015b9bcf966e48ded4 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:02:25 +0000 Subject: [PATCH 11/23] fix(extract): model qualified perl sub declarations in their named package `sub Bar::baz {...}` was stored as label `Bar::baz()` under the current package, so a call `Bar::baz()` (callee baz, package Bar) could never resolve and the body's caller package was wrong. Split the qualified name: the container is the qualified package (created if it has no `package` statement of its own), the node label is `baz()`, and the sub body carries caller package Bar. --- graphify/extractors/perl.py | 22 ++++++++++++++++++---- tests/test_languages.py | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 28d68b9643..d92dcee312 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -230,12 +230,26 @@ def walk_statements(node) -> None: elif c.type == "block": block = c if name: - container = current_pkg_nid or file_nid - sub_nid = _make_id(container, name) - add_node(sub_nid, f"{name}()", line) + if "::" in name: + # Qualified declaration `sub Pkg::sub {...}` defines the sub + # IN the named package, not the current one. Container = that + # package (created if it has no `package` statement of its + # own); the body's caller-package is the qualifier so its + # calls resolve against Pkg. + pkg_qual, _, sub_name = name.rpartition("::") + container = _make_id(stem, pkg_qual) + add_node(container, pkg_qual, line) + add_edge(file_nid, container, "contains", line) + sub_package = pkg_qual + else: + container = current_pkg_nid or file_nid + sub_name = name + sub_package = current_pkg_name + sub_nid = _make_id(container, sub_name) + add_node(sub_nid, f"{sub_name}()", line) add_edge(container, sub_nid, "contains", line) if block is not None: - sub_bodies.append((sub_nid, block, current_pkg_name)) + sub_bodies.append((sub_nid, block, sub_package)) elif child.type == "expression_statement": for c in child.children: if c.type == "require_expression": diff --git a/tests/test_languages.py b/tests/test_languages.py index 5110b09723..45dc8c9977 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4374,6 +4374,26 @@ def test_perl_qualified_call_binds_correct_package(tmp_path): assert all(p == "P::A" for (_s, _t, p) in emit_calls), \ f"qualified P::A::emit() must bind only to P::A's emit, got {emit_calls}" +def test_perl_qualified_sub_declaration_call_edge(tmp_path): + """A qualified declaration `sub Ext::Pkg::helper {...}` defines the sub IN + Ext::Pkg (not the current package). A call `Ext::Pkg::helper()` from another + package must bind to it: the node's label is `helper()` and its enclosing + package is Ext::Pkg (so callee/package-qualifier resolution matches).""" + from graphify.extract import extract + (tmp_path / "q.pm").write_text( + "package Main::Mod;\n" + "sub Ext::Pkg::helper { return 1; }\n" + "sub run { Ext::Pkg::helper(); }\n" + "1;\n" + ) + r = extract([tmp_path / "q.pm"], parallel=False) + binds = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "helper" in t] + assert binds, f"expected run->helper via qualified sub decl, got {_calls_with_target_pkg(r)}" + assert all(t == "helper()" and p == "Ext::Pkg" for (_s, t, p) in binds), ( + f"qualified sub must be labelled helper() and enclosed by Ext::Pkg, got {binds}" + ) + + def test_perl_bare_foreign_package_call_no_edge(tmp_path): """A bare `helper()` call resolves to a same-package sub, never to a same-named sub in a DIFFERENT, un-imported package (zero-edge, not a guess).""" From 08e75d146748b4d487ab878e62e329810322f81e Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:02:37 +0000 Subject: [PATCH 12/23] chore(extract): tidy perl extractor review notes for upstream - drop `.t` from the extract_perl docstring (only .pl/.pm are registered) - dedupe sprintf/wantarray in _PERL_BUILTINS - reword in-code development notes (TDD-stage and issue-shorthand markers, internal corpus references) as stable invariants - harden the method-call negative test to also reject a bare `render()` label, not only `render`/`::render`/`.render` --- graphify/extract.py | 7 +++---- graphify/extractors/perl.py | 7 +++---- tests/test_languages.py | 26 +++++++++++++------------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 2f69524a07..b1eaef2fe7 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6793,10 +6793,9 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # Package-aware pre-filter for languages that tag calls with their # package (Perl). Additive + guarded: a raw_call only reaches this branch # when it carries a package field, which no other language sets, so every - # other language's candidate set is untouched (the existing 314 tests - # prove the unqualified path is unchanged). Zero-edge over a wrong guess: - # an unresolvable qualifier or a foreign-package bare call is dropped, not - # bound to a same-named sub in the wrong package (#F1/#F3). + # other language's candidate set is untouched. Zero-edge over a wrong + # guess: an unresolvable qualifier or a foreign-package bare call is + # dropped, not bound to a same-named sub in the wrong package. callee_package = rc.get("callee_package") caller_package = rc.get("caller_package") if callee_package is not None or caller_package is not None: diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index d92dcee312..2a879c7600 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -41,10 +41,9 @@ # list / hash ops "shift", "unshift", "push", "pop", "splice", "map", "grep", "sort", "reverse", "join", "split", "keys", "values", "each", "exists", "delete", - "wantarray", # string ops "index", "rindex", "substr", "length", "uc", "lc", "ucfirst", "lcfirst", - "chomp", "chop", "chr", "ord", "hex", "oct", "sprintf", "pack", "unpack", + "chomp", "chop", "chr", "ord", "hex", "oct", "pack", "unpack", "quotemeta", # math "abs", "int", "sqrt", "sin", "cos", "atan2", "exp", "log", "rand", "srand", @@ -55,7 +54,7 @@ def extract_perl(path: Path) -> dict: - """Extract packages, subs, imports and inheritance from a .pl/.pm/.t file.""" + """Extract packages, subs, imports and inheritance from a .pl/.pm file.""" try: import tree_sitter_perl as tsperl from tree_sitter import Language, Parser @@ -357,7 +356,7 @@ def _is_perl_source(src: str) -> bool: # module-label id -> list of package node ids carrying that fully-qualified # label. A bare `use Foo` cannot disambiguate a package re-opened under the - # same label across files (Foswiki-real: `package Assert;` in AssertOn.pm AND + # same label across files (e.g. `package Assert;` in both AssertOn.pm and # AssertOff.pm), so we re-point ONLY when exactly one package node matches; # >1 candidate stays dangling (zero-edge over a guessed cross-file binding). pkg_ids_by_label_id: dict[str, list[str]] = {} diff --git a/tests/test_languages.py b/tests/test_languages.py index 45dc8c9977..5041242ed5 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4038,10 +4038,9 @@ def test_cl_ids_are_path_qualified_across_directories(tmp_path): f"got overlap {sorted(ids_a & ids_b)}" ) # ── Perl ────────────────────────────────────────────────────────────────────── -# RED (S2): the `extract_perl` extractor does not exist yet (S3) and the .pl/.pm -# dispatch is not registered yet (S4). The `extract_perl` import is kept INSIDE -# each test so importing this module still succeeds and the existing suite stays -# GREEN — only the Perl tests fail (ImportError until S3, empty graph until S4). +# `extract_perl` (and `extract`) are imported INSIDE each test rather than at +# module top so this test module still imports even if the extractor is absent — +# only the Perl tests would fail, not the whole suite's collection. def test_perl_no_error(): from graphify.extract import extract_perl @@ -4225,7 +4224,7 @@ def test_perl_raw_calls_carry_package_qualifiers(): # Perl call resolution runs in the top-level `extract()` second pass (raw_calls -> # name-matched calls), so these drive the public multi-file dispatch, like the -# ObjC cross-file tests. RED until BOTH S3 (extractor) and S4 (.pl/.pm dispatch). +# ObjC cross-file tests. def _perl_corpus(): from graphify.extract import extract @@ -4268,12 +4267,13 @@ def test_perl_method_call_no_edge(): r = _perl_corpus() calls = _calls(r) # Positive guard: the corpus must actually produce static/bare calls, so this - # test is not a vacuous pass on an empty graph (RED until S3+S4). + # test is not a vacuous pass on an empty graph. assert any("format_line" in tgt or "emit" in tgt for _src, tgt in calls), \ f"expected the static/bare calls to resolve before asserting the negatives, got {calls}" assert not any("update" in tgt for _src, tgt in calls), \ f"method call $self->update() must not produce a calls edge, got {calls}" - assert not any(tgt == "render" or tgt.endswith("::render") or tgt.endswith(".render") + assert not any(tgt in ("render", "render()") + or tgt.endswith("::render") or tgt.endswith(".render") for _src, tgt in calls), \ f"method call $widget->render() must not produce a calls edge, got {calls}" @@ -4405,7 +4405,7 @@ def test_perl_bare_foreign_package_call_no_edge(tmp_path): r = extract([tmp_path / "x.pm", tmp_path / "y.pm"], parallel=False) calls = _calls(r) # Positive guard: the same-package bare call must resolve, so the negative - # assertion below is not a vacuous pass on an empty (pre-S4) graph. + # assertion below is not a vacuous pass on an empty graph. assert any("run" in s and "other" in t for s, t in calls), \ f"expected run->other same-package resolution, got {calls}" assert not any("helper" in t for _s, t in calls), \ @@ -4449,9 +4449,9 @@ def test_perl_bare_call_two_imported_packages_ambiguous_no_edge(tmp_path): f"bare emit() imported from two packages is ambiguous and must drop, got {calls}" -# In-corpus import re-pointer (S5b Befund A): a `use`/`require` whose module is a -# package defined elsewhere in the corpus must re-point its imports edge from the -# bare module-label id onto the real package node; external modules stay dangling. +# In-corpus import re-pointer: a `use`/`require` whose module is a package defined +# elsewhere in the corpus must re-point its imports edge from the bare module-label +# id onto the real package node; external modules stay dangling. def test_perl_import_repoints_to_in_corpus_package(tmp_path): """`use Acme::Helper;` from another file must re-point its imports edge onto @@ -4500,8 +4500,8 @@ def test_perl_require_repoints_to_in_corpus_package(tmp_path): def test_perl_import_duplicate_package_label_stays_dangling(tmp_path): """A re-opened package declared under the SAME fully-qualified label in two - files (Foswiki-real: `package Assert;` in both AssertOn.pm and AssertOff.pm) - is ambiguous: a bare `use P::A;` cannot say which file it means. Binding it to + files (e.g. `package Assert;` in both AssertOn.pm and AssertOff.pm) is + ambiguous: a bare `use P::A;` cannot say which file it means. Binding it to one file arbitrarily is a guessed edge — worse than dangling — so the imports edge must stay on the bare module-label id (no package node) when >1 package node carries the label. Only a unique label re-points (zero-edge over a guess).""" From 082bf9ef0e38c26593bdcf258c6b329a2e5d3633 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:47:24 +0000 Subject: [PATCH 13/23] fix(extract): model perl main package; cache-safe re-pointer; harden walk & labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model Perl's implicit `main` package explicitly (R1): a package-less sub now lives in a lazily-created `main` node instead of hanging off the file node, so a qualified `main::helper()` binds and a bare call from a package-less file is scoped to `main` — it can no longer be mis-bound to a same-named sub in an unrelated package without import evidence. Match the import re-pointer's provenance on resolved paths (R2): a cached fragment's `source_file` returns absolutized against the resolved root, so the exact-string membership test against the raw (relative) provenance paths missed and the re-pointer silently regressed to dangling on the second (cached) run. Compare on the resolved on-disk identity so fresh and cached runs re-point alike. Walk the AST iteratively (S1): `_string_parents`, `walk_statements` and `walk_calls` were recursive, so a crafted deep nest raised RecursionError and `_safe_extract` dropped the WHOLE file. Explicit stacks plus a coarse traversal budget keep the file node and partial graph instead of losing everything. Validate inheritance targets (S2): `@ISA` / `use parent` / `use base` content is raw string data; a value that is not a well-formed package name (control chars, markdown, over-long) is discarded rather than emitted as a node label into graph.json / the Obsidian export. Stamp `lang="perl"` on raw_calls (A1, extractor half) so the shared second pass can claim exactly Perl's calls by the extractor stamp, matching the cpp/csharp/java/objc consumers. Constraint: zero-edge over a guess — unresolvable/foreign bindings are dropped. --- graphify/extractors/perl.py | 358 ++++++++++++++++++++++++------------ tests/test_languages.py | 148 +++++++++++++++ 2 files changed, 387 insertions(+), 119 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 2a879c7600..79c57dd60f 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -8,11 +8,37 @@ """ from __future__ import annotations +import logging +import re from pathlib import Path from typing import Any from graphify.extractors.base import _file_stem, _make_id +_LOG = logging.getLogger(__name__) + +# A valid Perl package/class name: a bareword component (`Foo`, `_priv`) optionally +# joined by `::`. Inheritance targets come from arbitrary string_literal / qw() +# content (`@ISA`, `use parent`, `use base`), so a crafted or malformed string +# (control chars, markdown, newlines, an over-long blob) could otherwise flow raw +# into a node label and on into graph.json / the Obsidian export. Names that do +# not match are discarded (zero-edge over a bogus stub). +_PERL_PKG_NAME_RE = re.compile(r"^[A-Za-z_]\w*(?:::\w+)*$") +_MAX_PERL_PKG_NAME_LEN = 256 + +# Coarse guard so a pathologically large or deeply nested file cannot make the +# (now iterative) tree walks run away; on exhaustion the file keeps whatever was +# already extracted (file node + partial graph) instead of nothing. +_MAX_PERL_TRAVERSAL_NODES = 2_000_000 + + +def _is_valid_perl_package_name(name: str) -> bool: + return ( + bool(name) + and len(name) <= _MAX_PERL_PKG_NAME_LEN + and _PERL_PKG_NAME_RE.match(name) is not None + ) + # ``use strict`` & friends are compiler pragmas, not module dependencies — they # must not become imports edges (matches the pragma-exclusion in other langs). _PERL_PRAGMAS: frozenset[str] = frozenset({ @@ -135,26 +161,62 @@ def _string_parents(node) -> list[str]: are a different node type and are intentionally skipped. """ out: list[str] = [] - - def visit(n) -> None: + stack = [node] + while stack: + n = stack.pop() if n.type in ("string_literal", "interpolated_string_literal", "quoted_word_list"): for c in n.children: if c.type == "string_content": out.extend(_text(c).split()) - return - for c in n.children: - visit(c) - - visit(node) + continue # an inheritance string's own children are not parents + stack.extend(reversed(n.children)) return out def add_inherits(pkg_nid: str, parent_name: str, line: int) -> None: + # Inheritance targets are raw string content; discard anything that is not + # a well-formed package name so a malformed/crafted string never becomes a + # node label or an edge (zero-edge, matching the untyped-drop discipline). + if not _is_valid_perl_package_name(parent_name): + return parent_nid = _make_id(parent_name) add_stub_node(parent_nid, parent_name) add_edge(pkg_nid, parent_nid, "inherits", line, context="inherit") current_pkg_nid: str | None = None current_pkg_name: str | None = None + main_pkg_nid: str | None = None + + def _ensure_main_pkg() -> str: + """Perl's implicit default package. Code with no ``package`` statement lives + in ``main``; modeling it explicitly (instead of hanging package-less subs off + the file node) means a qualified ``main::helper()`` call binds, and a bare + call from a package-less file is scoped to ``main`` — so it cannot be + mis-bound to a same-named sub in an unrelated package without import + evidence. Created lazily so a file with no package-less subs gets no empty + ``main`` node.""" + nonlocal main_pkg_nid + if main_pkg_nid is None: + main_pkg_nid = _make_id(stem, "main") + add_node(main_pkg_nid, "main", 1) + add_edge(file_nid, main_pkg_nid, "contains", 1) + return main_pkg_nid + + budget = [_MAX_PERL_TRAVERSAL_NODES] + budget_warned = [False] + + def _spend() -> bool: + """Charge one node against the shared traversal budget; False once spent so + the walkers stop instead of running away. Emits one bounded warning.""" + budget[0] -= 1 + if budget[0] < 0: + if not budget_warned[0]: + budget_warned[0] = True + _LOG.warning( + "perl: traversal budget exhausted for %s; graph for this file is partial", + str_path, + ) + return False + return True def handle_use(node, line: int) -> None: # In a use_statement the `use` keyword is its own node type, so the @@ -194,121 +256,152 @@ def handle_isa(assign_node, line: int) -> None: for parent in _string_parents(assign_node): add_inherits(current_pkg_nid, parent, line) - def walk_statements(node) -> None: + def walk_statements(root_node) -> None: nonlocal current_pkg_nid, current_pkg_name - for child in node.children: - line = child.start_point[0] + 1 - if child.type == "package_statement": - name = _package_name(child) - if name: - pkg_nid = _make_id(stem, name) - add_node(pkg_nid, name, line) - add_edge(file_nid, pkg_nid, "contains", line) - pkg_block = next( - (c for c in child.children if c.type == "block"), None) - if pkg_block is not None: - # Block-form `package Foo { ... }` scopes Foo to the block - # only. Walk the block under Foo, then restore the prior - # package so following top-level statements are not - # mis-attributed to Foo (and the block's subs are not lost). - prev_nid, prev_name = current_pkg_nid, current_pkg_name - current_pkg_nid, current_pkg_name = pkg_nid, name - walk_statements(pkg_block) - current_pkg_nid, current_pkg_name = prev_nid, prev_name - else: - current_pkg_nid = pkg_nid - current_pkg_name = name - elif child.type == "use_statement": - handle_use(child, line) - elif child.type == "subroutine_declaration_statement": - name = None - block = None - for c in child.children: - if c.type == "bareword" and name is None: - name = _text(c) - elif c.type == "block": - block = c - if name: - if "::" in name: - # Qualified declaration `sub Pkg::sub {...}` defines the sub - # IN the named package, not the current one. Container = that - # package (created if it has no `package` statement of its - # own); the body's caller-package is the qualifier so its - # calls resolve against Pkg. - pkg_qual, _, sub_name = name.rpartition("::") - container = _make_id(stem, pkg_qual) - add_node(container, pkg_qual, line) - add_edge(file_nid, container, "contains", line) - sub_package = pkg_qual - else: - container = current_pkg_nid or file_nid - sub_name = name - sub_package = current_pkg_name - sub_nid = _make_id(container, sub_name) - add_node(sub_nid, f"{sub_name}()", line) - add_edge(container, sub_nid, "contains", line) - if block is not None: - sub_bodies.append((sub_nid, block, sub_package)) - elif child.type == "expression_statement": - for c in child.children: - if c.type == "require_expression": - handle_require(c, line) - elif c.type == "assignment_expression": - handle_isa(c, line) + # Manual call stack in place of recursion: a pathologically deep nest of + # block-form packages would otherwise blow the Python stack, and the + # resulting RecursionError makes `_safe_extract` drop the WHOLE file. Each + # frame is an iterator over one block's statements plus the scope to restore + # once that block is exhausted; a block-form `package Foo { ... }` pushes a + # child frame under Foo's scope, so its subs are attributed to Foo and the + # prior package is restored for statements that follow the block. + stack: list[tuple[Any, str | None, str | None]] = [ + (iter(root_node.children), current_pkg_nid, current_pkg_name) + ] + while stack: + if not _spend(): + break + child_iter, restore_nid, restore_name = stack[-1] + descended = False + for child in child_iter: + line = child.start_point[0] + 1 + if child.type == "package_statement": + name = _package_name(child) + if name: + pkg_nid = _make_id(stem, name) + add_node(pkg_nid, name, line) + add_edge(file_nid, pkg_nid, "contains", line) + pkg_block = next( + (c for c in child.children if c.type == "block"), None) + if pkg_block is not None: + # Descend into the block under Foo; the frame remembers + # the pre-block scope so it is restored when the block is + # fully consumed (statements after the block are not + # mis-attributed to Foo). + prev_nid, prev_name = current_pkg_nid, current_pkg_name + current_pkg_nid, current_pkg_name = pkg_nid, name + stack.append((iter(pkg_block.children), prev_nid, prev_name)) + descended = True + break + else: + current_pkg_nid = pkg_nid + current_pkg_name = name + elif child.type == "use_statement": + handle_use(child, line) + elif child.type == "subroutine_declaration_statement": + name = None + block = None + for c in child.children: + if c.type == "bareword" and name is None: + name = _text(c) + elif c.type == "block": + block = c + if name: + if "::" in name: + # Qualified declaration `sub Pkg::sub {...}` defines the + # sub IN the named package, not the current one. Container + # = that package (created if it has no `package` statement + # of its own); the body's caller-package is the qualifier + # so its calls resolve against Pkg. + pkg_qual, _, sub_name = name.rpartition("::") + container = _make_id(stem, pkg_qual) + add_node(container, pkg_qual, line) + add_edge(file_nid, container, "contains", line) + sub_package = pkg_qual + else: + # Package-less sub → Perl's `main` (not the file node), so + # `main::sub()` binds and bare same-file calls resolve. + container = current_pkg_nid or _ensure_main_pkg() + sub_name = name + sub_package = current_pkg_name or "main" + sub_nid = _make_id(container, sub_name) + add_node(sub_nid, f"{sub_name}()", line) + add_edge(container, sub_nid, "contains", line) + if block is not None: + sub_bodies.append((sub_nid, block, sub_package)) + elif child.type == "expression_statement": + for c in child.children: + if c.type == "require_expression": + handle_require(c, line) + elif c.type == "assignment_expression": + handle_isa(c, line) + if descended: + continue + stack.pop() + current_pkg_nid, current_pkg_name = restore_nid, restore_name walk_statements(root) raw_calls: list[dict] = [] - def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: - if node.type == "function_call_expression": - # Indirect-object constructor `new CLASS(...)` parses as - # ambiguous_function_call_expression(function 'new', function_call_expression - # 'CLASS()'); `new CLASS` == CLASS->new, an untyped member dispatch. Mark - # it a member call (edge-less) so it is not wired to a sub named CLASS. - parent = node.parent - indirect_new = ( - parent is not None - and parent.type == "ambiguous_function_call_expression" - and bool(parent.children) - and parent.children[0].type == "function" - and _text(parent.children[0]) == "new" - ) - for c in node.children: - if c.type == "function": - # `Acme::Helper::emit` -> callee `emit`, callee_package - # `Acme::Helper`; a bare `emit` -> callee `emit`, no package. - # The qualifier + caller package let the shared second pass - # bind to the right same-named sub instead of any `emit()`. - parts = _text(c).split("::") - callee = parts[-1] - callee_package = "::".join(parts[:-1]) or None - if callee and callee not in _PERL_BUILTINS: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": callee, - "callee_package": callee_package, - "caller_package": caller_package, - "is_member_call": indirect_new, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - break - elif node.type == "method_call_expression": - for c in node.children: - if c.type == "method": - callee = _text(c).split("::")[-1] - if callee: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": callee, - "is_member_call": True, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - break - for child in node.children: - walk_calls(child, caller_nid, caller_package) + def walk_calls(root_node, caller_nid: str, caller_package: str | None) -> None: + # Iterative (see walk_statements): a deeply nested expression / data + # structure in a sub body would recurse once per level, and RecursionError + # here would drop the whole file via `_safe_extract`. + stack = [root_node] + while stack: + if not _spend(): + break + node = stack.pop() + if node.type == "function_call_expression": + # Indirect-object constructor `new CLASS(...)` parses as + # ambiguous_function_call_expression(function 'new', function_call_expression + # 'CLASS()'); `new CLASS` == CLASS->new, an untyped member dispatch. Mark + # it a member call (edge-less) so it is not wired to a sub named CLASS. + parent = node.parent + indirect_new = ( + parent is not None + and parent.type == "ambiguous_function_call_expression" + and bool(parent.children) + and parent.children[0].type == "function" + and _text(parent.children[0]) == "new" + ) + for c in node.children: + if c.type == "function": + # `Acme::Helper::emit` -> callee `emit`, callee_package + # `Acme::Helper`; a bare `emit` -> callee `emit`, no package. + # The qualifier + caller package let the shared second pass + # bind to the right same-named sub instead of any `emit()`. + parts = _text(c).split("::") + callee = parts[-1] + callee_package = "::".join(parts[:-1]) or None + if callee and callee not in _PERL_BUILTINS: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "callee_package": callee_package, + "caller_package": caller_package, + "is_member_call": indirect_new, + "lang": "perl", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + elif node.type == "method_call_expression": + for c in node.children: + if c.type == "method": + callee = _text(c).split("::")[-1] + if callee: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": True, + "lang": "perl", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + stack.extend(reversed(node.children)) for caller_nid, body, caller_package in sub_bodies: walk_calls(body, caller_nid, caller_package) @@ -349,10 +442,37 @@ def _resolve_perl_imports( Mutates ``all_edges`` in place; the bare target was never a node, so there is nothing to prune. """ + # Provenance matching must survive the cache round-trip. A fresh run stamps a + # node's `source_file` with the raw `str(path)` the extractor was handed, which + # is exactly what `perl_source_files` holds — so exact membership matches. But a + # cached fragment is stored relative and re-anchored on load against the RESOLVED + # cache root, so its `source_file` comes back as an absolute resolved path that + # no longer equals the raw provenance string — exact membership then misses and + # the re-pointer silently no-ops on the second (cached) run. Compare on the + # resolved form so both shapes collapse to the same on-disk identity. + resolved_perl_sources: set[str] | None = None + if perl_source_files is not None: + resolved_perl_sources = set(perl_source_files) + for s in perl_source_files: + try: + resolved_perl_sources.add(str(Path(s).resolve())) + except OSError: + pass + _resolve_memo: dict[str, bool] = {} + def _is_perl_source(src: str) -> bool: - if perl_source_files is not None: - return src in perl_source_files - return src.endswith((".pl", ".pm")) + if perl_source_files is None: + return src.endswith((".pl", ".pm")) + if src in resolved_perl_sources: + return True + hit = _resolve_memo.get(src) + if hit is None: + try: + hit = str(Path(src).resolve()) in resolved_perl_sources + except OSError: + hit = False + _resolve_memo[src] = hit + return hit # module-label id -> list of package node ids carrying that fully-qualified # label. A bare `use Foo` cannot disambiguate a package re-opened under the diff --git a/tests/test_languages.py b/tests/test_languages.py index 5041242ed5..dca7ff3c90 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4542,3 +4542,151 @@ def test_perl_import_repoints_from_shebang_perl_file(tmp_path): imports = [e for e in r["edges"] if e["relation"] == "imports"] assert any(e["target"] == pkg_id for e in imports), \ f"use Acme::Helper in a shebang-perl file must re-point to {pkg_id}, got {[e['target'] for e in imports]}" + + +# ── Perl S6c review-finding regressions ──────────────────────────────────────── + +def test_perl_raw_calls_stamped_lang_perl(): + """Every Perl raw_call carries `lang="perl"` so the shared second pass claims + exactly Perl's calls via the extractor stamp (like cpp/csharp/java/objc), + rather than sniffing for a `*_package` field (A1).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert r["raw_calls"], "expected raw_calls from the sample module" + assert all(rc.get("lang") == "perl" for rc in r["raw_calls"]), \ + f"every perl raw_call must be lang=perl, got {[rc.get('lang') for rc in r['raw_calls']]}" + + +def test_perl_main_qualified_call_binds_packageless_sub(tmp_path): + """A package-less sub lives in Perl's default `main` package; a qualified + `main::helper()` call must bind to it (R1). Before main was modeled the sub + hung off the file node with the filename as its package label, so `main::` + never resolved.""" + from graphify.extract import extract + (tmp_path / "s.pl").write_text( + "sub helper { return 1; }\nsub run { main::helper(); }\n" + ) + r = extract([tmp_path / "s.pl"], parallel=False) + calls = _calls(r) + assert any("run" in s and "helper" in t for s, t in calls), \ + f"main::helper() must bind to the package-less (main) helper, got {calls}" + + +def test_perl_packageless_bare_call_no_bind_to_foreign_package(tmp_path): + """A bare call from a package-less (main) file must NOT bind to a same-named + sub in an unrelated, un-imported package (R1). Modeling `main` scopes the call + to main; without it the call carried no package and was generically matched to + the lone same-named sub in a foreign package.""" + from graphify.extract import extract + (tmp_path / "b.pm").write_text("package B;\nsub helper { return 1; }\n1;\n") + (tmp_path / "s.pl").write_text( + "sub other { return 2; }\nsub run { helper(); other(); }\n" + ) + r = extract([tmp_path / "b.pm", tmp_path / "s.pl"], parallel=False) + calls = _calls(r) + # Positive guard: the same-file (main) bare call resolves, so the negative + # assertion below is not a vacuous pass on an empty graph. + assert any("run" in s and "other" in t for s, t in calls), \ + f"expected run->other within main, got {calls}" + assert not any("helper" in t for _s, t in calls), \ + f"bare helper() from a main file must not bind to B::helper, got {calls}" + + +def test_perl_main_package_node_emitted_for_packageless_sub(tmp_path): + """The lazily-created `main` package node contains the package-less sub (the + `contains` edge is main -> sub, not file -> sub).""" + from graphify.extract import extract_perl + src = tmp_path / "s.pl" + src.write_text("sub helper { return 1; }\n") + r = extract_perl(src) + label = {n["id"]: n.get("label", "") for n in r["nodes"]} + container_of = { + label.get(e["target"]): label.get(e["source"]) + for e in r["edges"] if e["relation"] == "contains" + } + assert "main" in label.values(), "a package-less sub must materialize a main package node" + assert container_of.get("helper()") == "main", \ + f"package-less helper must be contained by main, got {container_of.get('helper()')!r}" + + +def test_perl_import_repoint_survives_cache_roundtrip(tmp_path, monkeypatch): + """The import re-pointer must still fire on a cached (second) run (R2). A cached + fragment's source_file returns absolutized against the resolved root, so exact + string matching against the raw (relative) provenance paths missed and the + re-pointer silently regressed to dangling. Matching on the resolved path makes + both runs produce the same re-pointed edge.""" + from graphify.extract import extract + monkeypatch.chdir(tmp_path) + Path("helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + Path("main.pm").write_text( + "package Main;\nuse Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + paths = [Path("helper.pm"), Path("main.pm")] + + def _import_targets(r): + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + return pkg_id, {e["target"] for e in r["edges"] if e["relation"] == "imports"} + + r1 = extract(paths, cache_root=Path("."), parallel=False) + pkg1, targets1 = _import_targets(r1) + assert pkg1 in targets1, f"first run must re-point onto {pkg1}, got {targets1}" + assert (tmp_path / "graphify-out" / "cache").exists(), "AST cache must be written" + + r2 = extract(paths, cache_root=Path("."), parallel=False) + pkg2, targets2 = _import_targets(r2) + assert pkg2 in targets2, \ + f"cached run must ALSO re-point onto {pkg2} (not regress to dangling), got {targets2}" + assert targets1 == targets2, \ + f"import targets must be identical across fresh and cached runs: {targets1} vs {targets2}" + + +def test_perl_deeply_nested_expression_does_not_crash(tmp_path): + """A pathologically deep expression nest must not RecursionError (which would + make _safe_extract drop the whole file); the iterative walk keeps the file node + and the sub (S1).""" + from graphify.extract import extract_perl + depth = 12000 # > _RECURSION_LIMIT (10_000): the old recursive walk would blow up + body = "[" * depth + "1" + "]" * depth + src = tmp_path / "deep.pl" + src.write_text(f"sub f {{ my $x = {body}; }}\n") + r = extract_perl(src) + assert not r.get("error"), f"deep nest must not error out, got {r.get('error')!r}" + labels = {n["label"] for n in r["nodes"]} + assert "deep.pl" in labels, "file node must survive deep nesting" + assert "f()" in labels, "the sub must still be extracted despite deep nesting" + + +def test_perl_isa_rejects_non_package_string(tmp_path): + """@ISA holds arbitrary string content; a value that is not a well-formed + package name (markdown/brackets/control chars) must not become an inherits + stub node or edge (S2, zero-edge over a bogus label).""" + from graphify.extract import extract_perl + src = tmp_path / "bad.pm" + src.write_text( + 'package Child;\n' + 'our @ISA = ("weird\\nname[x](y)");\n' + '1;\n' + ) + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert not inherits, f"malformed @ISA string must not create an inherits edge, got {inherits}" + bad = [n for n in r["nodes"] + if any(ch in n.get("label", "") for ch in ("[", "(", "\n"))] + assert not bad, f"no stub node from a malformed inheritance string, got {bad}" + + +def test_perl_isa_valid_package_still_inherits(tmp_path): + """The S2 validation must not reject legitimate package names: a normal + `Foo::Bar` @ISA entry still produces an inherits edge + stub.""" + from graphify.extract import extract_perl + src = tmp_path / "ok.pm" + src.write_text( + 'package Child;\n' + 'our @ISA = ("Acme::Base");\n' + '1;\n' + ) + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert inherits, "a well-formed @ISA package must still inherit" + labels = {n.get("label") for n in r["nodes"]} + assert "Acme::Base" in labels, "the valid base class stub must be emitted" From 65c18707874e6465cae43c1284f5f8f9c1b2ac0d Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:48:01 +0000 Subject: [PATCH 14/23] refactor(extract): register perl import re-pointer via the resolver registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the Perl `imports` re-pointer off a private-underscore import called inline in `extract()`'s body onto the documented extension point (A2): it is now a registered `LanguageResolver`, so a language plugs in without editing `extract()`. It stays the last registered pass — after the shared call pass (which reads the bare module-label `use` targets via `_has_package_import_evidence`) and the member-call resolvers — the same position as before. Extend the registry with two optional hooks the re-pointer needs (both default off, so the existing 3-arg resolvers are untouched): - `activate`: a predicate over `paths` that overrides the suffix gate, so an extensionless `#!/usr/bin/perl` script (no `.pl`/`.pm` suffix, dispatched by shebang) still activates the pass by extractor PROVENANCE; - `wants_paths`: passes `paths` to `resolve` so the provenance-scoped pass can recompute which files it owns. Constraint: no behavior change to the re-pointer itself — same activation, same ordering, same fault isolation (log-and-skip). --- graphify/extract.py | 43 +++++++++++++++++++-------- graphify/resolver_registry.py | 28 ++++++++++++++++-- tests/test_language_resolvers.py | 51 ++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 15 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index b1eaef2fe7..805af85c0d 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4153,6 +4153,33 @@ def _resolve_kotlin_qualified_calls( ) +def _resolve_perl_imports_pass(per_file, all_nodes, all_edges, paths) -> None: + """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. + + Registered LAST so it runs after the shared cross-file call pass (which reads + the bare module-label ``use`` targets via ``_has_package_import_evidence``) and + after the member-call resolvers — the same position as its former inline call at + the tail of ``extract()``. Scoped by extractor PROVENANCE, not suffix: an + extensionless ``#!/usr/bin/perl`` script is dispatched to ``extract_perl`` by + shebang and must be re-pointed too, which is why it declares a custom + ``activate`` predicate (a shebang-only corpus has no ``.pl``/``.pm`` suffix) and + takes ``paths`` (``wants_paths``) to recompute that provenance set. + """ + perl_sources = {str(p) for p in paths if _get_extractor(p) is extract_perl} + _resolve_perl_imports(all_nodes, all_edges, perl_sources) + + +register_language_resolver( + LanguageResolver( + "perl_import_repoint", + frozenset({".pl", ".pm"}), + _resolve_perl_imports_pass, + activate=lambda paths: any(_get_extractor(p) is extract_perl for p in paths), + wants_paths=True, + ) +) + + # Inline markdown link: [text](target "optional title"). The negative lookbehind # excludes images (![alt](src)). The target stops at whitespace/closing paren so # an optional "title" after the URL is dropped; an optional <...> wrapper is too. @@ -6934,6 +6961,9 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # receiver-typed/qualified calls the shared pass skipped) with its own # single-definition god-node guard. Registered in graphify.resolver_registry so # a new language plugs in without editing this body (#1356 Swift, #1446 Python). + # The Perl import re-pointer (`perl_import_repoint`) is the last registered + # resolver, so it runs here — after the call pass and member-call resolvers, + # before the relativization below — the same position as its former inline call. # # #2437: on an incremental rebuild the resolvers must also see the unchanged # corpus — its nodes (types/methods, from resolution_nodes above) and its @@ -6953,19 +6983,6 @@ def _has_package_import_evidence(candidate_id: str) -> bool: else: run_language_resolvers(paths, per_file, all_nodes, all_edges) - # Re-point dangling in-corpus Perl `imports` edges onto the real package node. - # Runs AFTER the call pass above so `_has_package_import_evidence` still sees the - # bare module-label ids it needs to bind bare calls to imported packages (#S5b). - # Scope by extractor provenance, not suffix, so extensionless `#!/usr/bin/perl` - # scripts (dispatched to extract_perl by shebang) are re-pointed too. Keyed on - # str(path) to match the nodes' source_file, set before the relativization below. - try: - _perl_sources = {str(p) for p in paths if _get_extractor(p) is extract_perl} - _resolve_perl_imports(all_nodes, all_edges, _perl_sources) - except Exception as exc: - import logging - logging.getLogger(__name__).warning("Perl import resolution failed, skipping: %s", exc) - # Relativize source_file fields so paths are portable across machines (#555). # When the node's id was itself minted from the absolute path, remap it to a # portable id and rewrite the edge endpoints that reference it. diff --git a/graphify/resolver_registry.py b/graphify/resolver_registry.py index b17478a78a..4ca592ce33 100644 --- a/graphify/resolver_registry.py +++ b/graphify/resolver_registry.py @@ -33,11 +33,25 @@ class LanguageResolver: mutates ``all_nodes`` / ``all_edges`` in place, matching the existing member-call resolvers. ``suffixes`` gates activation: the pass runs only when the corpus contains at least one file with one of these extensions. + + Two optional hooks cover passes that suffix-gating cannot express (the Perl + import re-pointer needs both): + + - ``activate`` overrides the suffix gate with a predicate over ``paths``. Use it + when membership is decided by EXTRACTOR provenance rather than extension — + e.g. an extensionless ``#!/usr/bin/perl`` script has no ``.pl``/``.pm`` suffix + to gate on, yet must still activate the pass. When ``None`` the suffix gate is + used. + - ``wants_paths`` makes the driver call ``resolve`` with a fourth positional + argument, the ``paths`` sequence, so a provenance-scoped pass can recompute + which files it owns. Default ``False`` keeps the three-argument contract. """ name: str suffixes: frozenset resolve: Callable + activate: Callable | None = None + wants_paths: bool = False # Module-level registry, populated by callers via register(). Ordered: resolvers @@ -77,9 +91,19 @@ def run_language_resolvers( active = _REGISTRY if resolvers is None else resolvers suffixes_present = {p.suffix for p in paths} for resolver in active: - if not (resolver.suffixes & suffixes_present): + if resolver.activate is not None: + try: + if not resolver.activate(paths): + continue + except Exception as exc: + _LOG.warning("%s activation check failed, skipping: %s", resolver.name, exc) + continue + elif not (resolver.suffixes & suffixes_present): continue try: - resolver.resolve(per_file, all_nodes, all_edges) + if resolver.wants_paths: + resolver.resolve(per_file, all_nodes, all_edges, paths) + else: + resolver.resolve(per_file, all_nodes, all_edges) except Exception as exc: _LOG.warning("%s resolution failed, skipping: %s", resolver.name, exc) diff --git a/tests/test_language_resolvers.py b/tests/test_language_resolvers.py index 787c1d505d..bff0145766 100644 --- a/tests/test_language_resolvers.py +++ b/tests/test_language_resolvers.py @@ -72,3 +72,54 @@ def _add_edge(per_file, all_nodes, all_edges): edges: list[dict] = [] run_language_resolvers([Path("a.rb")], [], [], edges, resolvers=resolvers) assert edges == [{"source": "x", "target": "y", "relation": "calls"}] + + +def test_resolver_activate_predicate_overrides_suffix_gate() -> None: + # Provenance-gated pass: no matching suffix present, but the activate predicate + # opts it in anyway (e.g. an extensionless shebang script). + log: list[str] = [] + + def _resolve(per_file, all_nodes, all_edges): + log.append("ran") + + resolver = LanguageResolver( + "prov", frozenset({".rb"}), _resolve, + activate=lambda paths: any(p.name == "runme" for p in paths), + ) + run_language_resolvers([Path("runme")], [], [], [], resolvers=[resolver]) + assert log == ["ran"] + + +def test_resolver_activate_false_skips_even_with_matching_suffix() -> None: + log: list[str] = [] + + def _resolve(per_file, all_nodes, all_edges): + log.append("ran") + + resolver = LanguageResolver( + "prov", frozenset({".rb"}), _resolve, activate=lambda paths: False, + ) + run_language_resolvers([Path("a.rb")], [], [], [], resolvers=[resolver]) + assert log == [] # activate wins over the suffix gate + + +def test_resolver_wants_paths_receives_paths() -> None: + seen: list = [] + + def _resolve(per_file, all_nodes, all_edges, paths): + seen.append(list(paths)) + + resolver = LanguageResolver("wp", frozenset({".rb"}), _resolve, wants_paths=True) + run_language_resolvers([Path("a.rb")], [], [], [], resolvers=[resolver]) + assert seen == [[Path("a.rb")]] + + +def test_default_registry_contains_perl_import_repoint() -> None: + import graphify.extract # noqa: F401 (registers resolvers on import) + + perl = next( + (r for r in registered_resolvers() if r.name == "perl_import_repoint"), None + ) + assert perl is not None, "perl_import_repoint must be registered in the shared registry" + assert perl.wants_paths, "the re-pointer needs paths to recompute Perl provenance" + assert perl.activate is not None, "the re-pointer activates by extractor provenance, not suffix" From 82fefdc974a05492d4f9693566a7376707d62f7a Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:48:15 +0000 Subject: [PATCH 15/23] fix(extract): gate perl second pass on lang stamp; accept both import-evidence forms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate the package-aware second-pass branch on the extractor-stamped `lang == "perl"` (A1, resolver half) instead of sniffing for a `*_package` field, matching the cpp/csharp/java/objc raw-call consumers — so the branch claims exactly Perl's calls and a future language that set a package field could not fall into it. Make `_has_package_import_evidence` accept both the bare module-label id and the real package-node id as import evidence (A3). The re-pointer rewrites `use` targets from the bare id onto the package node id; matching either shape means this check no longer depends on whether the re-pointer has run, so the pass ordering stops being semantically load-bearing. The current order (re-pointer after this pass) is kept regardless. Constraint: additive + zero-edge — no other language sets `lang="perl"`, so their candidate sets are untouched. --- graphify/extract.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 805af85c0d..8a13172358 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6709,9 +6709,11 @@ def _looks_like_bash(result: object) -> bool: # this map is built but never read for them. _label_by_nid = {n["id"]: n.get("label", "") for n in all_nodes} pkg_label_by_nid: dict[str, str] = {} + pkg_nid_by_sub_nid: dict[str, str] = {} for e in all_edges: if e.get("relation") == "contains": pkg_label_by_nid[e["target"]] = _label_by_nid.get(e["source"], "") + pkg_nid_by_sub_nid[e["target"]] = e["source"] for rc in all_raw_calls: callee = rc.get("callee", "") @@ -6815,17 +6817,29 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # Bridge it: the candidate sub's enclosing package, re-idized the same # way the `use` target is, must be among the caller file's imports. pkg = pkg_label_by_nid.get(candidate_id, "") - return bool(pkg) and _make_id(pkg) in imported_symbols - - # Package-aware pre-filter for languages that tag calls with their - # package (Perl). Additive + guarded: a raw_call only reaches this branch - # when it carries a package field, which no other language sets, so every - # other language's candidate set is untouched. Zero-edge over a wrong - # guess: an unresolvable qualifier or a foreign-package bare call is - # dropped, not bound to a same-named sub in the wrong package. + if not pkg: + return False + if _make_id(pkg) in imported_symbols: + return True + # Accept the real package-node id form too. The import re-pointer rewrites + # a `use` target from the bare module-label id onto the package node id; + # matching either shape means this evidence check no longer depends on + # whether that re-pointer has run yet — the pass ordering stops being + # semantically load-bearing (A3). The current order (re-pointer after this + # pass) is kept regardless. + pkg_nid = pkg_nid_by_sub_nid.get(candidate_id) + return pkg_nid is not None and pkg_nid in imported_symbols + + # Package-aware pre-filter for Perl, which tags every call with its + # enclosing package. Gated on the extractor-stamped `lang` (matching the + # cpp/csharp/java/objc raw-call consumers) rather than field-presence, so + # the branch claims exactly Perl's raw_calls and another language that + # happened to set a `*_package` field could never fall into it. Zero-edge + # over a wrong guess: an unresolvable qualifier or a foreign-package bare + # call is dropped, not bound to a same-named sub in the wrong package. callee_package = rc.get("callee_package") caller_package = rc.get("caller_package") - if callee_package is not None or caller_package is not None: + if rc.get("lang") == "perl": if callee_package is not None: # Qualified call `Pkg::sub()`: bind only to a sub whose enclosing # package matches the qualifier. None or several -> drop. From 7a8b8da464b30ac1cfe015035940a4383da62b33 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:25:35 +0000 Subject: [PATCH 16/23] fix(extract): ASCII-validate perl package labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python `\w` is Unicode-default, so accented (`Basé`), fullwidth (`Base`) and digit-start (`Acme::1x`) names slipped the package-name validator and flowed into node labels, graph.json and the Obsidian export. Spell the classes out as explicit ASCII ranges, anchor every `::` component to a letter/underscore start, and switch to re.fullmatch so a trailing newline can no longer pass on a bare `$` anchor. 256-char cap unchanged. Constraint: fixes must not change real-corpus numbers (Foswiki core/lib: base and branch both 2776 nodes / 4611 edges / 211 inherits — Foswiki is ASCII, so no legitimate name is newly rejected). --- graphify/extractors/perl.py | 11 +++++++++-- tests/test_languages.py | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 79c57dd60f..91c2572379 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -23,7 +23,14 @@ # (control chars, markdown, newlines, an over-long blob) could otherwise flow raw # into a node label and on into graph.json / the Obsidian export. Names that do # not match are discarded (zero-edge over a bogus stub). -_PERL_PKG_NAME_RE = re.compile(r"^[A-Za-z_]\w*(?:::\w+)*$") +# +# The classes are spelled out as explicit ASCII ranges (not ``\w``): Python's +# ``\w`` is Unicode-default, so accented (``Basé``), fullwidth (``Base``) and +# other non-ASCII barewords would slip through. Every ``::`` component must begin +# with a letter/underscore, which also rejects a digit-start component +# (``Acme::1x``); ``fullmatch`` (below) anchors the whole string, so a trailing +# newline cannot pass on a ``$`` alone. +_PERL_PKG_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)*") _MAX_PERL_PKG_NAME_LEN = 256 # Coarse guard so a pathologically large or deeply nested file cannot make the @@ -36,7 +43,7 @@ def _is_valid_perl_package_name(name: str) -> bool: return ( bool(name) and len(name) <= _MAX_PERL_PKG_NAME_LEN - and _PERL_PKG_NAME_RE.match(name) is not None + and _PERL_PKG_NAME_RE.fullmatch(name) is not None ) # ``use strict`` & friends are compiler pragmas, not module dependencies — they diff --git a/tests/test_languages.py b/tests/test_languages.py index dca7ff3c90..0571544446 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4690,3 +4690,41 @@ def test_perl_isa_valid_package_still_inherits(tmp_path): assert inherits, "a well-formed @ISA package must still inherit" labels = {n.get("label") for n in r["nodes"]} assert "Acme::Base" in labels, "the valid base class stub must be emitted" + + +@pytest.mark.parametrize("bad", [ + "Acme::Basé", # accented letter — Unicode \w matched, ASCII must not + "Acme::Base", # fullwidth latin letter — Unicode \w matched, ASCII must not + "Acme::1x", # component starting with a digit after :: + "1Acme", # leading digit + "Acme::Bar\n", # trailing newline (a bare $ anchor would let this through) +]) +def test_perl_label_validator_rejects_non_ascii_component(bad): + """The package-name validator is ASCII-only and anchors every `::` component to + a letter/underscore start (F3). Python `\\w` is Unicode-default, so accented, + fullwidth and digit-start names would otherwise pass and land in graph.json / + the Obsidian export.""" + from graphify.extractors.perl import _is_valid_perl_package_name + assert not _is_valid_perl_package_name(bad), \ + f"{bad!r} must be rejected by the ASCII package-name validator" + + +@pytest.mark.parametrize("ok", ["Foo", "_priv", "Acme::Base", "A::B::C", "F0o::B4r"]) +def test_perl_label_validator_accepts_ascii_package(ok): + """Legitimate ASCII package names (including digits after the first char of a + component) still validate — the F3 tightening must not over-reject.""" + from graphify.extractors.perl import _is_valid_perl_package_name + assert _is_valid_perl_package_name(ok), f"{ok!r} must validate" + + +def test_perl_isa_digit_start_component_no_stub(tmp_path): + """End-to-end: a @ISA entry whose ::-component starts with a digit must not + produce an inherits edge or a stub node (F3 Unicode/digit-start bypass).""" + from graphify.extract import extract_perl + src = tmp_path / "bad.pm" + src.write_text('package Child;\nour @ISA = ("Acme::1Base");\n1;\n') + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert not inherits, f"digit-start component must not inherit, got {inherits}" + assert "Acme::1Base" not in {n.get("label") for n in r["nodes"]}, \ + "no stub node for a digit-start component" From b655d8a1c2512afc528e0c7103594da934c37417 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:26:23 +0000 Subject: [PATCH 17/23] fix(extract): budget every node of the perl tree walks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The traversal budget was charged once per stack frame in walk_statements, then an unbounded number of sibling children were drained for free — a broad, flat file bypassed the guard entirely. _string_parents (the inheritance-string walk) was not budgeted at all. Charge _spend() per visited sibling in walk_statements (stop cleanly on exhaustion, keeping the partial graph) and per popped node in _string_parents, so all three walkers share one bounded budget as intended. walk_calls already charged per popped node — left as is. --- graphify/extractors/perl.py | 9 ++++-- tests/test_languages.py | 55 +++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 91c2572379..e86c892850 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -170,6 +170,8 @@ def _string_parents(node) -> list[str]: out: list[str] = [] stack = [node] while stack: + if not _spend(): + break n = stack.pop() if n.type in ("string_literal", "interpolated_string_literal", "quoted_word_list"): for c in n.children: @@ -276,11 +278,14 @@ def walk_statements(root_node) -> None: (iter(root_node.children), current_pkg_nid, current_pkg_name) ] while stack: - if not _spend(): - break child_iter, restore_nid, restore_name = stack[-1] descended = False for child in child_iter: + # Charge every visited sibling, not once per frame: a broad flat + # file drains an unbounded number of children under a single frame, + # so a per-frame charge left them effectively free. + if not _spend(): + return # budget exhausted: keep the partial graph, stop walking line = child.start_point[0] + 1 if child.type == "package_statement": name = _package_name(child) diff --git a/tests/test_languages.py b/tests/test_languages.py index 0571544446..0917e1e25c 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4728,3 +4728,58 @@ def test_perl_isa_digit_start_component_no_stub(tmp_path): assert not inherits, f"digit-start component must not inherit, got {inherits}" assert "Acme::1Base" not in {n.get("label") for n in r["nodes"]}, \ "no stub node for a digit-start component" + + +def test_perl_statement_walk_charges_budget_per_sibling(tmp_path, monkeypatch): + """A broad, flat file (many sibling statements under one frame) must charge the + traversal budget per sibling, not once per stack frame (F2). Under a tiny budget + the walk stops early and emits a PARTIAL graph; the old code drained every + sibling on one charge, so all subs surfaced regardless of budget. Asserting the + partial output isolates walk_statements (the shared budget is also spent by the + later call walk, so a warning alone would not discriminate).""" + from graphify.extractors import perl + monkeypatch.setattr(perl, "_MAX_PERL_TRAVERSAL_NODES", 3) + src = tmp_path / "flat.pl" + src.write_text("package Wide;\n" + "".join( + f"sub s{i} {{ 1; }}\n" for i in range(30))) + r = perl.extract_perl(src) + sub_nodes = [n for n in r["nodes"] if n.get("label", "").endswith("()")] + assert len(sub_nodes) < 30, ( + "walk_statements must charge per sibling and stop early under a tiny budget; " + f"emitting all {len(sub_nodes)} subs means siblings were traversed for free") + + +def test_perl_statement_walk_no_warning_within_budget(tmp_path, caplog): + """Guard against a vacuous pass: a small file under the default budget extracts + fully and emits no traversal warning.""" + import logging + from graphify.extractors import perl + src = tmp_path / "small.pl" + src.write_text("package Wide;\n" + "".join( + f"sub s{i} {{ return {i}; }}\n" for i in range(20))) + with caplog.at_level(logging.WARNING, logger="graphify.extractors.perl"): + r = perl.extract_perl(src) + assert not any("traversal budget" in rec.message for rec in caplog.records), \ + "a small file must not trip the budget" + assert sum(1 for n in r["nodes"] if n.get("label", "").endswith("()")) == 20, \ + "all 20 subs extract when the budget is ample" + + +def test_perl_string_parents_charges_budget(tmp_path, monkeypatch, caplog): + """`_string_parents` shares the traversal budget (F2): an @ISA array of many + separate string literals is a per-node walk that must trip the bounded warning, + even though the file has only a couple of top-level statements. The old code + left `_string_parents` entirely unbudgeted, so it walked the whole subtree for + free. (Each literal is its own tree node — unlike a single qw() word list.)""" + import logging + from graphify.extractors import perl + # Above the ~2 top-level statement charges but far below the string-literal node + # count, so exhaustion can only come from _string_parents. + monkeypatch.setattr(perl, "_MAX_PERL_TRAVERSAL_NODES", 20) + parents = ", ".join(f"'P{i}'" for i in range(400)) + src = tmp_path / "wide_isa.pm" + src.write_text(f"package Child;\nour @ISA = ({parents});\n1;\n") + with caplog.at_level(logging.WARNING, logger="graphify.extractors.perl"): + perl.extract_perl(src) + assert any("traversal budget exhausted" in rec.message for rec in caplog.records), \ + "a wide @ISA literal array must charge the shared budget via _string_parents" From 816be5524778daac90bcb7aef66a5314e839499a Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:26:50 +0000 Subject: [PATCH 18/23] docs(resolver): generalize the LanguageResolver hook docstring The shared registry API contract named the Perl import re-pointer and a `#!/usr/bin/perl` shebang as the motivating case for the activate/wants_paths hooks. This module deliberately knows nothing about any specific language; describe the hooks generically (provenance-scoped passes gated by which extractor produced a node, not by extension). The Perl-specific rationale already lives at its registration site in extract.py. --- graphify/resolver_registry.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/graphify/resolver_registry.py b/graphify/resolver_registry.py index 4ca592ce33..cb480c5d53 100644 --- a/graphify/resolver_registry.py +++ b/graphify/resolver_registry.py @@ -34,14 +34,15 @@ class LanguageResolver: member-call resolvers. ``suffixes`` gates activation: the pass runs only when the corpus contains at least one file with one of these extensions. - Two optional hooks cover passes that suffix-gating cannot express (the Perl - import re-pointer needs both): + Two optional hooks cover passes that suffix-gating cannot express — a + provenance-scoped pass (one whose file membership is decided by which + extractor produced a node, not by extension) may need both: - ``activate`` overrides the suffix gate with a predicate over ``paths``. Use it - when membership is decided by EXTRACTOR provenance rather than extension — - e.g. an extensionless ``#!/usr/bin/perl`` script has no ``.pl``/``.pm`` suffix - to gate on, yet must still activate the pass. When ``None`` the suffix gate is - used. + when membership is decided by EXTRACTOR provenance rather than extension — a + file dispatched to an extractor by content (e.g. a shebang) may carry no + matching suffix to gate on, yet must still activate the pass. When ``None`` + the suffix gate is used. - ``wants_paths`` makes the driver call ``resolve`` with a fourth positional argument, the ``paths`` sequence, so a provenance-scoped pass can recompute which files it owns. Default ``False`` keeps the three-argument contract. From 673a6add4e4200e3220eb60951b1043ad10ef333 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:34:39 +0000 Subject: [PATCH 19/23] docs: add perl to language table and changelog --- CHANGELOG.md | 2 ++ README.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46ffcab93c..30b740cf57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -533,6 +533,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: a Java field/parameter/return-type reference to a class whose simple name is shared by two modules no longer dangles on a sourceless phantom node (#1744, thanks @aviciot). Both same-named classes already survive as distinct path-scoped nodes, but the cross-module `references` edge was left pointing at a bare no-source stub because `_resolve_java_type_references` re-pointed `implements`/`inherits`/`imports` but not `references` — so a query about the referenced class could miss it. The Java resolver now disambiguates `references` by the importing file's `import` statement (falling back to same-package), mirroring the C# resolver, and drops the orphaned phantom. +- Add: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#PR). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. + ## 0.9.11 (2026-07-08) - Fix: file enumeration no longer silently drops a directory subtree. `detect()`'s `os.walk` had no `onerror` handler, so an `os.scandir` failure (a permission error, or a directory created/deleted mid-walk by concurrent writes) was swallowed and that whole subtree vanished from the scan with no log, yielding a silently partial `graph.json`. The walk now records every skipped directory (surfaced in the result's `walk_errors`) and warns to stderr, while still enumerating the rest. Relatedly, `to_json`'s anti-shrink guard (#479) now fails safe: a non-empty but unreadable existing `graph.json` refuses the overwrite (pass `force=True` to override) instead of silently clobbering a good graph; an empty file still proceeds. diff --git a/README.md b/README.md index 0b3fba6ba3..9a22d7be2c 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (37 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .ml .mli .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`, `.ml`/`.mli` requires `uv tool install graphifyy[ocaml]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | +| Code (38 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .pl .pm .m .mm .ml .mli .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`, `.ml`/`.mli` requires `uv tool install graphifyy[ocaml]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | OCaml | `.ml .mli` (requires `uv tool install graphifyy[ocaml]`) | From 577721db9dc16dcb3bde253957b91b44e40ab7bf Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Sat, 11 Jul 2026 10:19:56 +0000 Subject: [PATCH 20/23] docs: fill in PR number in changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30b740cf57..231123c586 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -533,7 +533,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: a Java field/parameter/return-type reference to a class whose simple name is shared by two modules no longer dangles on a sourceless phantom node (#1744, thanks @aviciot). Both same-named classes already survive as distinct path-scoped nodes, but the cross-module `references` edge was left pointing at a bare no-source stub because `_resolve_java_type_references` re-pointed `implements`/`inherits`/`imports` but not `references` — so a query about the referenced class could miss it. The Java resolver now disambiguates `references` by the importing file's `import` statement (falling back to same-package), mirroring the C# resolver, and drops the orphaned phantom. -- Add: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#PR). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. +- Add: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#1788). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. ## 0.9.11 (2026-07-08) From 741aa80d76be12a25f821a8f80286d80aae90eac Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Sun, 19 Jul 2026 11:56:54 +0000 Subject: [PATCH 21/23] docs: move Perl changelog entry to Unreleased after rebase onto 0.9.20 --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 231123c586..4bf4dbbeec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.44 (2026-08-15) +- Feature: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#1788). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. - Feature: `graphify hook install` reads a committed `.graphifyrc` (`viz_node_limit=`) and bakes the visualization node limit into the generated git hooks, so a project-wide limit is shared via version control and survives hook regeneration; `hook status` reports it (#2760, thanks @hopstreax). The baked value uses a `${GRAPHIFY_VIZ_NODE_LIMIT:-}` default so an explicit per-run env var still wins, and `hook status` degrades gracefully on a malformed `.graphifyrc`. - Fix: `graphify install` (Claude always-on) now writes the CLAUDE.md registration into `$CLAUDE_CONFIG_DIR` when that env var relocates the Claude profile, instead of always mutating the default `~/.claude/CLAUDE.md` (part of #2694, thanks @AromalBiju1). - Fix: a JS/TS inline or nested function expression — including a generator function expression (`function*(k){…}`) — no longer fabricates an INFERRED `indirect_call` when one of its parameters/locals shares a name with an unrelated callable; the expression's own bindings now shadow the name (#2752, thanks @imagineers-tyler), completing the shadow family alongside catch/arrow/loop/external-import (#2757). @@ -533,8 +534,6 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: a Java field/parameter/return-type reference to a class whose simple name is shared by two modules no longer dangles on a sourceless phantom node (#1744, thanks @aviciot). Both same-named classes already survive as distinct path-scoped nodes, but the cross-module `references` edge was left pointing at a bare no-source stub because `_resolve_java_type_references` re-pointed `implements`/`inherits`/`imports` but not `references` — so a query about the referenced class could miss it. The Java resolver now disambiguates `references` by the importing file's `import` statement (falling back to same-package), mirroring the C# resolver, and drops the orphaned phantom. -- Add: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#1788). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. - ## 0.9.11 (2026-07-08) - Fix: file enumeration no longer silently drops a directory subtree. `detect()`'s `os.walk` had no `onerror` handler, so an `os.scandir` failure (a permission error, or a directory created/deleted mid-walk by concurrent writes) was swallowed and that whole subtree vanished from the scan with no log, yielding a silently partial `graph.json`. The walk now records every skipped directory (surfaced in the result's `walk_errors`) and warns to stderr, while still enumerating the rest. Relatedly, `to_json`'s anti-shrink guard (#479) now fails safe: a non-empty but unreadable existing `graph.json` refuses the overwrite (pass `force=True` to override) instead of silently clobbering a good graph; an empty file still proceeds. From bafa76fd56acb0289632be5591d0fd5136b66b12 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:25:52 +0000 Subject: [PATCH 22/23] =?UTF-8?q?fix(perl):=20hardening=20pass=20from=20sl?= =?UTF-8?q?ice=20review=20=E2=80=94=20blocks,=20phasers,=20lazy=20loads,?= =?UTF-8?q?=20name=20forms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hardening discovered while reviewing the sliced decomposition of this PR; all of it applies to the full extractor: - declarations inside bare scope blocks ({ package Inner; ... }), phaser blocks (BEGIN/CHECK/...), conditionals and loops are now traversed with the existing scope-restore discipline - root-qualified names canonicalize: ::Outer == Outer (one package node), and `sub ::foo {}` attaches to implicit main instead of minting an empty-label node - scalar inheritance strings validate intact ('Foo Bar' is one malformed name, not two parents); only qw(...) splits - bareword parents (`use parent -norequire, Foo;`) are collected from the argument subtree; -norequire's autoquoted_bareword type never matches - package-less `use parent` / @ISA attach to implicit main, materialized only after the assignment is confirmed as @ISA (ordinary assignments create no empty main node) - lazy imports surface: require/use inside sub bodies, conditional blocks, and anonymous subs are scanned under the shared traversal budget - constant literal-path requires (`require "Foo/Bar.pm";`) normalize to module dependencies; interpolated content stays skipped - `use if CONDITION, 'Module';` emits an import for its statically named module argument - pragma exclusion extended (open, locale, re, experimental, charnames, sigtrap) - import re-pointing admits unchanged packages during incremental rebuilds: candidate eligibility falls back to the .pl/.pm suffix, and resolver provenance covers resolution-context source files, so full and incremental graphs no longer diverge Real-corpus effect (Foswiki core/lib, 378 Perl files, 0 parse errors): imports 825 -> 1117, contains 2385 -> 2402, inherits 211 -> 213, calls 1088 -> 1092, nodes 2776 -> 2793. The gains are dominated by lazy requires inside BEGIN blocks and sub bodies that the previous walker never reached. --- graphify/extract.py | 25 ++-- graphify/extractors/perl.py | 196 ++++++++++++++++++++++++++---- tests/test_languages.py | 230 ++++++++++++++++++++++++++++++++++++ 3 files changed, 423 insertions(+), 28 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 8a13172358..d64fa5375a 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4156,16 +4156,25 @@ def _resolve_kotlin_qualified_calls( def _resolve_perl_imports_pass(per_file, all_nodes, all_edges, paths) -> None: """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. - Registered LAST so it runs after the shared cross-file call pass (which reads - the bare module-label ``use`` targets via ``_has_package_import_evidence``) and - after the member-call resolvers — the same position as its former inline call at - the tail of ``extract()``. Scoped by extractor PROVENANCE, not suffix: an - extensionless ``#!/usr/bin/perl`` script is dispatched to ``extract_perl`` by - shebang and must be re-pointed too, which is why it declares a custom - ``activate`` predicate (a shebang-only corpus has no ``.pl``/``.pm`` suffix) and - takes ``paths`` (``wants_paths``) to recompute that provenance set. + Registered LAST so it runs after the shared cross-file call pass — which reads + the bare module-label ``use`` targets via ``_has_package_import_evidence`` — + and after the member-call resolvers. Scoped by extractor provenance over BOTH + this run's paths AND the resolution context's source files: on an incremental + rebuild an unchanged package (extensionless shebang-Perl included) must stay a + re-point candidate, or full and incremental graphs diverge. """ perl_sources = {str(p) for p in paths if _get_extractor(p) is extract_perl} + seen_sf: set[str] = set() + for n in all_nodes: + sf = str(n.get("source_file") or "") + if not sf or sf in seen_sf: + continue + seen_sf.add(sf) + try: + if _get_extractor(Path(sf)) is extract_perl: + perl_sources.add(sf) + except Exception: + continue _resolve_perl_imports(all_nodes, all_edges, perl_sources) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index e86c892850..efa3f2817a 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -52,6 +52,7 @@ def _is_valid_perl_package_name(name: str) -> bool: "strict", "warnings", "utf8", "constant", "vars", "lib", "feature", "integer", "bytes", "overload", "mro", "autodie", "diagnostics", "sort", "subs", "attributes", "fields", "encoding", "if", "less", + "open", "locale", "re", "experimental", "charnames", "sigtrap", }) # ``use parent`` / ``use base`` declare inheritance, not an import. @@ -158,14 +159,25 @@ def add_edge(src: str, tgt: str, relation: str, line: int, def _package_name(node) -> str | None: """`package Foo::Bar;` -> the second `package`-typed child is the name.""" names = [c for c in node.children if c.type == "package"] - return _text(names[1]) if len(names) >= 2 else None + name = _text(names[1]) if len(names) >= 2 else None + # Root-qualified spelling `::Outer` is the same package as `Outer` + # (::Name == main::Name in Perl); canonicalize so both spellings key to + # one package node instead of diverging on an empty qualifier. + if name and name.startswith("::"): + name = name[2:] + # The name comes from arbitrary source text; validate before it becomes a + # label (zero-node over a malformed or crafted package statement). + return name if _is_valid_perl_package_name(name or "") else None def _string_parents(node) -> list[str]: - """Every string / qw-word parent named under an inheritance construct. - - Reads string_literal (`'Foo'`) and quoted_word_list (`qw(Foo Bar)`) - content anywhere below ``node``; autoquoted barewords like ``-norequire`` - are a different node type and are intentionally skipped. + """Every parent package named under an inheritance construct. + + A ``quoted_word_list`` (``qw(Foo Bar)``) is whitespace-separated and + yields one name per word. Ordinary string literals are SINGLE scalar + package names — their content is validated intact, never split (a + literal ``'Foo Bar'`` is one malformed name, not two parents). + Autoquoted barewords like ``-norequire`` are skipped here; bareword + parents are collected separately by ``handle_use``. """ out: list[str] = [] stack = [node] @@ -173,11 +185,16 @@ def _string_parents(node) -> list[str]: if not _spend(): break n = stack.pop() - if n.type in ("string_literal", "interpolated_string_literal", "quoted_word_list"): + if n.type == "quoted_word_list": for c in n.children: if c.type == "string_content": out.extend(_text(c).split()) - continue # an inheritance string's own children are not parents + continue # a qw-list's own children are not parents + if n.type in ("string_literal", "interpolated_string_literal"): + for c in n.children: + if c.type == "string_content": + out.append(_text(c)) + continue stack.extend(reversed(n.children)) return out @@ -235,10 +252,38 @@ def handle_use(node, line: int) -> None: if not pkgs: return module = _text(pkgs[0]) + if module == "if": + # `use if CONDITION, 'Foo::Compat';` delegates the load to the `if` + # pragma, but the module argument is statically named — emit its + # import. The condition's scalar/number tokens don't match the + # collected shapes. + for parent in _string_parents(node): + if _is_valid_perl_package_name(parent): + add_edge(file_nid, _make_id(parent), "imports", line, + context="import") + return if module in _PERL_INHERIT_PRAGMAS: - if current_pkg_nid: - for parent in _string_parents(node): - add_inherits(current_pkg_nid, parent, line) + # `use parent 'Foo'` in a package-less file applies to the implicit + # `main` package — materialize it rather than dropping the edge. + target_pkg = current_pkg_nid or _ensure_main_pkg() + parents = _string_parents(node) + # Bareword form: `use parent -norequire, Foo;` exposes Foo as a + # bareword — possibly nested in a list_expression. Collect validating + # barewords from the whole argument subtree; the -norequire flag is + # an autoquoted_bareword node (different type) and never matches. + bw_stack = [node] + while bw_stack: + if not _spend(): + break + bn = bw_stack.pop() + if bn.type == "bareword": + bw = _text(bn) + if not bw.startswith("-") and _is_valid_perl_package_name(bw): + parents.append(bw) + else: + bw_stack.extend(bn.children) + for parent in parents: + add_inherits(target_pkg, parent, line) return if module in _PERL_PRAGMAS: return @@ -249,21 +294,83 @@ def handle_require(req_node, line: int) -> None: if c.type == "bareword": add_edge(file_nid, _make_id(_text(c)), "imports", line, context="import") return + if c.type in ("string_literal", "interpolated_string_literal"): + # Constant literal path: require "Foo/Bar.pm"; — normalize to the + # module-name spelling. Interpolated content ($x) stays skipped: + # it is not a static dependency. + for sc in c.children: + if sc.type != "string_content": + continue + text_ = _text(sc) + if "$" in text_ or "@" in text_: + return + name = text_ + if name.endswith(".pm"): + name = name[:-3] + name = name.replace("/", "::") + if _is_valid_perl_package_name(name): + add_edge(file_nid, _make_id(name), "imports", line, + context="import") + # File-style literals (require "config.pl", "./Foo.pm") + # would need file-node targets with corpus-relative path + # resolution — deliberately deferred, not silently + # mis-shaped into a package import. + return def handle_isa(assign_node, line: int) -> None: - """`our @ISA = (...)` -> inherits edges to each named parent.""" - if not current_pkg_nid: - return + """`our @ISA = (...)` / bare `@ISA = (...)` -> inherits edges.""" is_isa = False for child in assign_node.children: if child.type == "variable_declaration": for sub in child.children: if sub.type == "array" and _text(sub) == "@ISA": is_isa = True + elif child.type == "array" and _text(child) == "@ISA": + # Without `our` (or split `our @ISA; @ISA = ...`), the array sits + # directly under the assignment. + is_isa = True if not is_isa: return + # Materialize implicit main only AFTER the assignment is confirmed as + # @ISA, so ordinary assignments in package-less files create no node. + target_pkg = current_pkg_nid or _ensure_main_pkg() for parent in _string_parents(assign_node): - add_inherits(current_pkg_nid, parent, line) + add_inherits(target_pkg, parent, line) + + def _scan_inner_imports(block_node) -> None: + """Collect use/require statements nested INSIDE a sub body (lazy-loading). + + ``sub load { require Foo::Bar; }`` is a static dependency even though the + walker deliberately does not descend into sub bodies for declarations. + Walks all nested nodes iteratively under the shared budget, without + changing package scope — an inner statement's imports edge is sourced by + the file either way.""" + stack = [block_node] + while stack: + if not _spend(): + return + n = stack.pop() + if n is not block_node and n.type == "use_statement": + handle_use(n, n.start_point[0] + 1) + elif n.type == "require_expression": + handle_require(n, n.start_point[0] + 1) + else: + stack.extend(reversed(n.children)) + + def _scan_anonymous_subs(node) -> None: + """Find anonymous_subroutine_expression blocks anywhere under ``node`` + and run the inner import scan on each.""" + stack = [node] + while stack: + if not _spend(): + return + n = stack.pop() + if n.type == "anonymous_subroutine_expression": + blk = next((g for g in n.children if g.type == "block"), None) + if blk is not None: + _scan_inner_imports(blk) + else: + stack.extend(n.children) def walk_statements(root_node) -> None: nonlocal current_pkg_nid, current_pkg_name @@ -308,6 +415,32 @@ def walk_statements(root_node) -> None: else: current_pkg_nid = pkg_nid current_pkg_name = name + elif child.type in ("block_statement", "block"): + # A bare `{ ... }` scope, a conditional/loop body block, or an + # else-clause may itself contain declarations and imports + # (`{ package Inner; sub f {...} }`, `if ($x) { require X; }`); + # descend without changing scope — the frame restores whatever + # the enclosing package was once the block ends. + stack.append((iter(child.children), current_pkg_nid, current_pkg_name)) + descended = True + break + elif child.type in ("conditional_statement", "while_statement", + "until_statement", "for_statement", + "foreach_statement", "c_style_for_statement"): + # Compound constructs own their blocks as children; iterate the + # construct so the block above picks up its body. + stack.append((iter(child.children), current_pkg_nid, current_pkg_name)) + descended = True + break + elif child.type == "phaser_statement": + # BEGIN/CHECK/INIT/END/UNITCHECK blocks compile their bodies + # at the surrounding scope, so declarations inside define real + # symbols; descend into the phaser's block like any scope. + blk = next((c for c in child.children if c.type == "block"), None) + if blk is not None: + stack.append((iter(blk.children), current_pkg_nid, current_pkg_name)) + descended = True + break elif child.type == "use_statement": handle_use(child, line) elif child.type == "subroutine_declaration_statement": @@ -326,10 +459,16 @@ def walk_statements(root_node) -> None: # of its own); the body's caller-package is the qualifier # so its calls resolve against Pkg. pkg_qual, _, sub_name = name.rpartition("::") - container = _make_id(stem, pkg_qual) - add_node(container, pkg_qual, line) - add_edge(file_nid, container, "contains", line) - sub_package = pkg_qual + if pkg_qual: + container = _make_id(stem, pkg_qual) + add_node(container, pkg_qual, line) + add_edge(file_nid, container, "contains", line) + sub_package = pkg_qual + else: + # Root-qualified `sub ::foo {...}` declares + # main::foo (::Name == main::Name). + container = _ensure_main_pkg() + sub_package = "main" else: # Package-less sub → Perl's `main` (not the file node), so # `main::sub()` binds and bare same-file calls resolve. @@ -341,12 +480,16 @@ def walk_statements(root_node) -> None: add_edge(container, sub_nid, "contains", line) if block is not None: sub_bodies.append((sub_nid, block, sub_package)) + _scan_inner_imports(block) elif child.type == "expression_statement": for c in child.children: if c.type == "require_expression": handle_require(c, line) elif c.type == "assignment_expression": handle_isa(c, line) + # An assignment can wrap an anonymous sub whose body + # lazy-requires (`my $loader = sub { require X; }`). + _scan_anonymous_subs(c) if descended: continue stack.pop() @@ -491,10 +634,23 @@ def _is_perl_source(src: str) -> bool: # same label across files (e.g. `package Assert;` in both AssertOn.pm and # AssertOff.pm), so we re-point ONLY when exactly one package node matches; # >1 candidate stays dangling (zero-edge over a guessed cross-file binding). + # + # Candidate eligibility: provenance set OR suffix. On an incremental rebuild + # the resolver's view includes UNCHANGED files' nodes (resolution context), + # which are not in `perl_source_files` (that set holds only the re-extracted + # paths) — a changed caller's `use` must still resolve against an unchanged + # package. The .pl/.pm suffix fallback admits those; an extensionless + # shebang-dispatched unchanged file remains covered by the caller-side + # provenance scan over resolution-context source files (extract.py). + def _is_candidate_source(src: str) -> bool: + if perl_source_files is not None and src in resolved_perl_sources: + return True + return src.endswith((".pl", ".pm")) + pkg_ids_by_label_id: dict[str, list[str]] = {} for node in all_nodes: src = str(node.get("source_file") or "") - if not _is_perl_source(src): + if not _is_candidate_source(src): continue label = node.get("label", "") nid = node.get("id", "") diff --git a/tests/test_languages.py b/tests/test_languages.py index 0917e1e25c..e0e6b59c4c 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -4783,3 +4783,233 @@ def test_perl_string_parents_charges_budget(tmp_path, monkeypatch, caplog): perl.extract_perl(src) assert any("traversal budget exhausted" in rec.message for rec in caplog.records), \ "a wide @ISA literal array must charge the shared budget via _string_parents" +def test_perl_package_inside_bare_block(tmp_path): + """`{ package Inner; sub f {...} }` — a bare scope block containing + declarations must be traversed; its subs belong to Inner and the enclosing + scope resumes after the block.""" + from graphify.extract import extract_perl + src = tmp_path / "scoped.pm" + src.write_text( + "package Outer;\n" + "{ package Inner; sub f { 42 } }\n" + "sub after { 1 }\n" + ) + r = extract_perl(src) + label = {n["id"]: n["label"] for n in r["nodes"]} + container_of = { + label.get(e["target"]): label.get(e["source"]) + for e in r["edges"] if e["relation"] == "contains" + } + assert "Inner" in label.values(), "package inside a bare block must be emitted" + assert container_of.get("f()") == "Inner", \ + f"sub inside a block-scoped package must belong to it, got {container_of.get('f()')!r}" + assert container_of.get("after()") == "Outer", "enclosing scope must resume" + + +def test_perl_phaser_block_declarations(tmp_path): + """BEGIN { package Tmp; sub g {...} } — phaser blocks compile their bodies at + the surrounding scope, so declarations inside define real symbols.""" + from graphify.extract import extract_perl + src = tmp_path / "phaser.pl" + src.write_text("BEGIN { package Tmp; sub g { 1 } }\n") + r = extract_perl(src) + label = {n["id"]: n["label"] for n in r["nodes"]} + container_of = { + label.get(e["target"]): label.get(e["source"]) + for e in r["edges"] if e["relation"] == "contains" + } + assert "Tmp" in label.values(), "package inside a phaser must be emitted" + assert container_of.get("g()") == "Tmp", \ + f"phaser-declared sub must attach to its package, got {container_of.get('g()')!r}" + + +def test_perl_root_qualified_sub_attaches_to_main(tmp_path): + """`sub ::foo {...}` declares main::foo (::Name == main::Name); it must not + mint an empty-label package node.""" + from graphify.extract import extract_perl + src = tmp_path / "rootqual.pm" + src.write_text("sub ::foo { 1 }\n") + r = extract_perl(src) + labels = [n.get("label") for n in r["nodes"]] + assert "" not in labels, "no empty-label package node for a root-qualified name" + assert "foo()" in labels, "the root-qualified sub must still be extracted" + + +def test_perl_root_qualified_package_normalizes(tmp_path): + """`package ::Outer;` is the same package as `Outer`; both spellings must key + to one package node carrying the canonical label.""" + from graphify.extract import extract_perl + src = tmp_path / "rootpkg.pm" + src.write_text("package ::Outer;\nsub h { 2 }\n") + r = extract_perl(src) + labels = [n.get("label") for n in r["nodes"]] + assert "::Outer" not in labels, "root-qualified spelling must be canonicalized" + assert "Outer" in labels, "the package must surface under its canonical label" + +def test_perl_scalar_isa_string_is_one_name(tmp_path): + """A plain string literal is ONE scalar package name: `'Foo Bar'` is a single + malformed name and must yield no inherits edge — not two fabricated parents + (only qw(...) is whitespace-split).""" + from graphify.extract import extract_perl + src = tmp_path / "scalar.pm" + src.write_text("package C;\nour @ISA = ('Foo Bar');\n1;\n") + r = extract_perl(src) + assert [e for e in r["edges"] if e["relation"] == "inherits"] == [], \ + "a malformed scalar parent must not fabricate inheritance" + + +def test_perl_qw_still_splits(tmp_path): + from graphify.extract import extract_perl + src = tmp_path / "qw.pm" + src.write_text("package C;\nour @ISA = qw(Acme::A Acme::B);\n1;\n") + r = extract_perl(src) + parents = {e["target"] for e in r["edges"] if e["relation"] == "inherits"} + assert len(parents) == 2, "qw() content is whitespace-split into parents" + + +def test_perl_require_inside_sub_body(tmp_path): + """Lazy loading — `sub load { require Foo::Bar; }` — is a static dependency + and must emit an imports edge even though the walker does not descend into + sub bodies for declarations.""" + from graphify.extract import extract_perl + src = tmp_path / "lazy.pm" + src.write_text("package L;\nsub load { require Foo::Bar; return 1; }\n1;\n") + r = extract_perl(src) + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any("foo_bar" in e["target"].lower() for e in imports), \ + "a nested require must surface as a static dependency" + + +def test_perl_packageless_use_parent_attaches_main(tmp_path): + """`use parent 'Foo'` in a package-less file applies to implicit main: the + main package node materializes and inherits, instead of the edge being dropped.""" + from graphify.extract import extract_perl + src = tmp_path / "pkgless.pm" + src.write_text("use parent 'Acme::Base';\nsub helper { 1 }\n") + r = extract_perl(src) + inh = [e for e in r["edges"] if e["relation"] == "inherits"] + assert inh, "package-less use parent must not be dropped" + labels = {n["id"]: n.get("label") for n in r["nodes"]} + main_ids = {nid for nid, lb in labels.items() if lb == "main"} + assert any(e["source"] in main_ids for e in inh), "inheritance attaches to main" + + +def test_perl_bareword_parent_form(tmp_path): + """`use parent -norequire, Foo;` exposes the parent as a bareword (nested in + a list_expression), not a string; the -norequire flag stays excluded.""" + from graphify.extract import extract_perl + src = tmp_path / "bare.pm" + src.write_text("package C;\nuse parent -norequire, Acme::Base;\n1;\n") + r = extract_perl(src) + labels = {n["id"]: n.get("label") for n in r["nodes"]} + parents = {labels[e["target"]] for e in r["edges"] if e["relation"] == "inherits"} + assert "Acme::Base" in parents, f"bareword parent must be collected, got {parents}" + assert "-norequire" not in parents, "the autoquoted flag is not a parent" + + +def test_perl_extended_pragmas_not_imported(tmp_path): + """open/locale/re/experimental/charnames are compiler pragmas, not deps.""" + from graphify.extract import extract_perl + src = tmp_path / "pragma.pm" + src.write_text( + "package P;\n" + "use open ':std';\n" + "use re 'taint';\n" + "use experimental 'signatures';\n" + "use charnames ':full';\n" + "1;\n" + ) + r = extract_perl(src) + targets = " ".join(e["target"] for e in r["edges"] if e["relation"] == "imports").lower() + for pragma in ("open", "re", "experimental", "charnames"): + assert pragma not in targets.split(), f"{pragma} is a pragma, not an import" + + +def test_perl_repoint_admits_suffix_context_candidates(tmp_path): + """On an incremental rebuild the resolver sees unchanged files' nodes as + resolution context — outside perl_source_files. A changed caller's `use` + must still re-point onto such an unchanged package (suffix fallback).""" + from pathlib import Path as _P + from graphify.extractors.perl import _resolve_perl_imports + from graphify.extractors.base import _make_id + # Mirror the real id scheme: the bare `use` target is _make_id(label); the + # unchanged package node's id is _make_id(stem, label). + ctx_pkg_nid = _make_id("helper", "Acme::Helper") + nodes = [ + {"id": "f1", "label": "main.pl", "source_file": "/abs/main.pl"}, + # unchanged context node: NOT in perl_source_files below + {"id": ctx_pkg_nid, "label": "Acme::Helper", "source_file": "/abs/helper.pm"}, + ] + edges = [{"source": "f1", "target": _make_id("Acme::Helper"), "relation": "imports", + "context": "import", "source_file": "/abs/main.pl"}] + _resolve_perl_imports(nodes, edges, {"/abs/main.pl"}) + assert edges[0]["target"] == ctx_pkg_nid, \ + "an unchanged .pm package must remain a valid re-point candidate" + + +def test_perl_bare_isa_assignment(tmp_path): + """`@ISA = qw(Base);` without `our` — the array sits directly under the + assignment — must still emit inherits.""" + from graphify.extract import extract_perl + src = tmp_path / "bareisa.pm" + src.write_text("package C;\n@ISA = qw(Acme::Base);\n1;\n") + r = extract_perl(src) + assert any(e["relation"] == "inherits" for e in r["edges"]), \ + "bare @ISA assignment must emit inheritance" + + +def test_perl_literal_path_require(tmp_path): + """`require \"Foo/Bar.pm\";` normalizes to the Foo::Bar module dependency.""" + from graphify.extract import extract_perl + src = tmp_path / "lit.pm" + src.write_text('package L;\nrequire "Foo/Bar.pm";\n1;\n') + r = extract_perl(src) + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any("foo_bar" in e["target"].lower() for e in imports), \ + "constant literal-path require must normalize to a module import" + + +def test_perl_conditional_require(tmp_path): + """`if ($x) { require Foo::Bar; }` at top level surfaces as an import.""" + from graphify.extract import extract_perl + src = tmp_path / "cond.pl" + src.write_text("if ($x) { require Foo::Bar; }\n") + r = extract_perl(src) + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any("foo_bar" in e["target"].lower() for e in imports), \ + "a conditional lazy require must surface as a static dependency" + + +def test_perl_anonymous_sub_require(tmp_path): + """`my $loader = sub { require Foo::Bar; };` surfaces as an import.""" + from graphify.extract import extract_perl + src = tmp_path / "anon.pm" + src.write_text("package A;\nmy $loader = sub { require Foo::Bar; };\n1;\n") + r = extract_perl(src) + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any("foo_bar" in e["target"].lower() for e in imports), \ + "an anonymous-sub lazy require must surface as a static dependency" + + +def test_perl_no_main_for_ordinary_packageless_assignment(tmp_path): + """A package-less file with only ordinary assignments must not mint an empty + `main` node (main is deferred until something needs it).""" + from graphify.extract import extract_perl + src = tmp_path / "plain.pl" + src.write_text("$x = 1;\nprint $x;\n") + r = extract_perl(src) + labels = [n.get("label") for n in r["nodes"]] + assert "main" not in labels, "ordinary assignments must not materialize main" + + +def test_perl_use_if_delegates_import(tmp_path): + """`use if CONDITION, 'Foo::Compat';` — the `if` pragma's statically named + module argument becomes an import; the condition tokens don't.""" + from graphify.extract import extract_perl + src = tmp_path / "useif.pm" + src.write_text("package U;\nuse if $] >= 5.010, 'Foo::Compat';\n1;\n") + r = extract_perl(src) + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any("foo_compat" in e["target"].lower() for e in imports), \ + "the if-pragma's module argument must surface as an import" + From a30a640d4e10af70d18789d267929e1cc84f83da Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:58:41 +0000 Subject: [PATCH 23/23] docs(changelog): move the Perl entry out of a released section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The entry sat under `## 0.9.44 (2026-08-15)`. It landed there during the 2026-08-16 rebase, when 0.9.44 was still the open `(unreleased)` section; the `chore: bump to 0.9.44` commit dated that heading afterwards, and the entry came along silently. Since then this branch has claimed a feature shipped in 0.9.44 that has not shipped at all. There is no open unreleased section right now — 0.9.50 was dated yesterday — so this opens `## 0.9.51 (unreleased)` in the same shape the maintainer uses (`docs(changelog): open 0.9.NN with ...`), and moves the entry there unchanged. No wording touched. --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bf4dbbeec..e8c4a7a9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.9.52 (unreleased) + +- Feature: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#1788). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. + ## 0.9.51 (2026-08-28) - Fix: the incomplete-build shrink guard now stays armed when a chunk came back hollow, unparseable, or omitting files, so a run that silently lost content can no longer overwrite the existing graph with a smaller one; a complete run and a retry-recovered chunk are unaffected, and `--allow-partial` still overrides (#3105, thanks @abhay-codes07). @@ -105,7 +109,6 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.44 (2026-08-15) -- Feature: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#1788). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. - Feature: `graphify hook install` reads a committed `.graphifyrc` (`viz_node_limit=`) and bakes the visualization node limit into the generated git hooks, so a project-wide limit is shared via version control and survives hook regeneration; `hook status` reports it (#2760, thanks @hopstreax). The baked value uses a `${GRAPHIFY_VIZ_NODE_LIMIT:-}` default so an explicit per-run env var still wins, and `hook status` degrades gracefully on a malformed `.graphifyrc`. - Fix: `graphify install` (Claude always-on) now writes the CLAUDE.md registration into `$CLAUDE_CONFIG_DIR` when that env var relocates the Claude profile, instead of always mutating the default `~/.claude/CLAUDE.md` (part of #2694, thanks @AromalBiju1). - Fix: a JS/TS inline or nested function expression — including a generator function expression (`function*(k){…}`) — no longer fabricates an INFERRED `indirect_call` when one of its parameters/locals shares a name with an unrelated callable; the expression's own bindings now shadow the name (#2752, thanks @imagineers-tyler), completing the shadow family alongside catch/arrow/loop/external-import (#2757).