Python: Add telemetry for parser usage

Adds statistics on how many files were extracted using the old parser
and using the tree-sitter parser.

Because parsing is done in parallel across many workers, I opted not to
consolidate these statistics for the entire run. Instead, we emit the
statistics for each worker and then need to aggregate themselves after
the telemetry has been ingested. (In practice the number of workers is
~16 at most, so is unlikely to be an issue.)

In terms of implementation, I opted to simply extend the existing
`DiagnosticsWriter` object (instantiatied once per worker) with methods
for counting the number of parsed files, and then thread this object
through to `modules.py` where the magic happens.

Finally, this also required instantiating such an object in cases where
we call directly into the extractor for debugging purposes (e.g. dumping
the AST or CFG). Note that in these cases we do not actually print any
diagnostics, so it's harmless to create these objects.

As for tests, we add a new separate CLI integration test that checks the
behaviour against a database that contains two files -- one that can be
parsed with the old parser and one that requires the new one. The
existing diagnostics test is modified slightly so that it ignores these
statistics (as we cannot guarantee their exact form due to worker
nondeterminism).

Author: tausbn

python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py

@@ -0,0 +1 @@
+x = 1

python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py

@@ -0,0 +1,3 @@
+match 1:
+    case 1:
+        pass

python/extractor/cli-integration-test/parser-telemetry/test.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+
+set -x
+
+CODEQL=${CODEQL:-codeql}
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$SCRIPTDIR"
+
+rm -rf db
+
+$CODEQL database create db --language python --source-root repo_dir/
+python3 test_parser_telemetry.py db
+
+rm -rf db

python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py

@@ -0,0 +1,25 @@
+import glob
+import json
+import os
+import sys
+
+
+database = sys.argv[1]
+diagnostics = []
+diagnostic_dir = os.path.join(database, "diagnostic", "extractors", "python")
+for path in glob.glob(os.path.join(diagnostic_dir, "*.jsonl")):
+    with open(path) as diagnostic_file:
+        diagnostics.extend(json.loads(line) for line in diagnostic_file)
+parser_statistics = [
+    diagnostic
+    for diagnostic in diagnostics
+    if diagnostic["source"]["id"] == "py/extractor/parser-statistics"
+]
+actual = (
+    sum(diagnostic["attributes"]["old_parser_file_count"] for diagnostic in parser_statistics),
+    sum(
+        diagnostic["attributes"]["tree_sitter_parser_file_count"]
+        for diagnostic in parser_statistics
+    ),
+)
+assert actual == (1, 1), actual

python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py

@@ -4,4 +4,11 @@
 import diagnostics_test_utils
 
 test_db = "db"
-diagnostics_test_utils.check_diagnostics(".", test_db, skip_attributes=True)
+diagnostics_test_utils.check_diagnostics(
+    ".",
+    test_db,
+    skip_attributes=True,
+    replacements={
+        r'"py/extractor/parser-statistics"': '"cli/py/extractor/parser-statistics"'
+    },
+)

python/extractor/semmle/extractors/module_printer.py

@@ -6,9 +6,9 @@ class ModulePrinter(object):
 
     name = "module printer"
 
-    def __init__(self, options, trap_folder, src_archive, renamer, logger):
+    def __init__(self, options, trap_folder, src_archive, renamer, logger, diagnostics_writer):
         self.logger = logger
-        self.py_extractor = PythonExtractor(options, trap_folder, src_archive, logger)
+        self.py_extractor = PythonExtractor(options, trap_folder, src_archive, logger, diagnostics_writer)
 
     def process(self, unit):
         imports = ()

python/extractor/semmle/extractors/py_extractor.py

@@ -16,6 +16,7 @@ def __init__(self, options, trap_folder, src_archive, logger: Logger, diagnostic
         self.module_extractor = extractor.Extractor.from_options(options, trap_folder, src_archive, logger, diagnostics_writer)
         self.finder = finder.Finder.from_options_and_env(options, logger)
         self.importer = imports.importer_from_options(options, self.finder, logger)
+        self.diagnostics_writer = diagnostics_writer
 
     def _get_module_and_imports(self, unit):
         if not isinstance(unit, util.FileExtractable):
@@ -24,7 +25,7 @@ def _get_module_and_imports(self, unit):
         module = self.finder.from_extractable(unit)
         if module is None:
             return None, ()
-        py_module = module.load(self.logger)
+        py_module = module.load(self.logger, self.diagnostics_writer)
         if py_module is None:
             return None, ()
         imports = set(mod.get_extractable() for mod in self.importer.get_imports(module, py_module))

python/extractor/semmle/logging.py

@@ -367,6 +367,14 @@ def extractor_telemetry_message():
             .telemetry()
     )
 
+def parser_statistics_telemetry_message(old_parser_file_count, tree_sitter_parser_file_count):
+    return (DiagnosticMessage(Source("py/extractor/parser-statistics", "Python parser statistics"), Severity.NOTE)
+            .markdown("Internal parser telemetry for the Python extractor.\n\nNo action needed.")
+            .attribute("old_parser_file_count", old_parser_file_count)
+            .attribute("tree_sitter_parser_file_count", tree_sitter_parser_file_count)
+            .telemetry()
+    )
+
 def get_stack_trace_lines():
     """Creates a stack trace for inclusion into the `attributes` part of a diagnostic message.
     Limits the size of the stack trace to 5000 characters, so as to not make the SARIF file overly big.

python/extractor/semmle/python/finder.py

@@ -65,8 +65,8 @@ def all_sub_modules(self):
     def get_extractable(self):
         return FileExtractable(self.path)
 
-    def load(self, logger=None):
-        return PythonSourceModule(self.name, self.path, logger=logger)
+    def load(self, logger, diagnostics_writer):
+        return PythonSourceModule(self.name, self.path, logger=logger, diagnostics_writer=diagnostics_writer)
 
     def __str__(self):
         return "Python module at %s" % self.path

python/extractor/semmle/python/modules.py

@@ -18,7 +18,7 @@ class PythonSourceModule(object):
 
     kind = None
 
-    def __init__(self, name, path, logger, bytes_source = None):
+    def __init__(self, name, path, logger, diagnostics_writer, bytes_source = None):
         assert isinstance(path, str), path
         self.name = name # May be None
         self.path = path
@@ -34,6 +34,7 @@ def __init__(self, name, path, logger, bytes_source = None):
         self._line_types = None
         self._comments = None
         self._tokens = None
+        self.diagnostics_writer = diagnostics_writer
         self.logger = logger
         with timers["decode"]:
             self.encoding, self.bytes_source  = semmle.python.parser.tokenizer.encoding_from_source(bytes_source)
@@ -113,6 +114,7 @@ def old_py_ast(self):
                 self.logger.debug("Trying old parser on %s", self.path)
                 self._py_ast = semmle.python.parser.parse(self.tokens, self.logger)
                 self.logger.debug("Old parser successful on %s", self.path)
+                self.diagnostics_writer.record_old_parser()
         else:
             self.logger.debug("Found (during old_py_ast) parse tree for %s in cache", self.path)
         return self._py_ast
@@ -147,6 +149,7 @@ def py_ast(self):
                             self.logger.debug("Trying tsg-python on %s", self.path)
                             self._py_ast = semmle.python.parser.tsg_parser.parse(self.path, self.logger)
                             self.logger.debug("tsg-python successful on %s", self.path)
+                            self.diagnostics_writer.record_tree_sitter_parser()
                         else:
                             self.logger.debug("Found (during py_ast) parse tree for %s in cache", self.path)
                     return self._py_ast