Merge branch 'yara-rules' into collect-sigma-rules-api

# Conflicts:
#	vulnerabilities/improvers/__init__.py
#	vulnerabilities/migrations/0116_detectionrule.py

Author: ziadhany

vulnerabilities/improvers/__init__.py

@@ -34,6 +34,7 @@
 from vulnerabilities.pipelines.v2_improvers import relate_severities
 from vulnerabilities.pipelines.v2_improvers import sigma_rules
 from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2
+from vulnerabilities.pipelines.v2_improvers import yara_rules
 from vulnerabilities.utils import create_registry
 
 IMPROVERS_REGISTRY = create_registry(
@@ -75,9 +76,6 @@
         compute_advisory_todo.ComputeToDo,
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
-        sigma_rules.SigmaHQImproverPipeline,
-        sigma_rules.SigmaSamuraiMDRImproverPipeline,
-        sigma_rules.SigmaMbabinskiImproverPipeline,
-        sigma_rules.P4T12ICKSigmaImproverPipeline,
+        yara_rules.YaraRulesImproverPipeline,
     ]
 )

vulnerabilities/migrations/0116_detectionrule.py

@@ -1,5 +1,3 @@
-# Generated by Django 5.2.11 on 2026-03-07 14:44
-
 import django.db.models.deletion
 from django.db import migrations, models
 

vulnerabilities/pipelines/v2_improvers/yara_rules.py

@@ -0,0 +1,179 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+from pathlib import Path
+
+from aboutcode.pipeline import LoopProgress
+from fetchcode.vcs import fetch_via_vcs
+
+from vulnerabilities.models import AdvisoryAlias
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import DetectionRule
+from vulnerabilities.models import DetectionRuleTypes
+from vulnerabilities.pipelines import VulnerableCodePipeline
+from vulnerabilities.utils import find_all_cve
+from vulnerabilities.utils import get_advisory_url
+
+
+class YaraRulesImproverPipeline(VulnerableCodePipeline):
+    pipeline_id = "yara_rules"
+
+    repo_urls = [
+        "git+https://github.com/elastic/protections-artifacts",
+        "git+https://github.com/Yara-Rules/rules",
+        "git+https://github.com/Xumeiquer/yara-forensics",
+        "git+https://github.com/reversinglabs/reversinglabs-yara-rules",
+        "git+https://github.com/advanced-threat-research/Yara-Rules",
+        "git+https://github.com/bartblaze/Yara-rules",
+        "git+https://github.com/godaddy/yara-rules",  # archived
+        "git+https://github.com/SupportIntelligence/Icewater",
+        "git+https://github.com/jeFF0Falltrades/YARA-Signatures",
+        "git+https://github.com/tjnel/yara_repo",
+        "git+https://github.com/JPCERTCC/jpcert-yara",
+        "git+https://github.com/mikesxrs/Open-Source-YARA-rules",
+        "git+https://github.com/fboldewin/YARA-rules",
+        "git+https://github.com/h3x2b/yara-rules",
+        "git+https://github.com/roadwy/DefenderYara",
+        "git+https://github.com/mthcht/ThreatHunting-Keywords-yara-rules",
+        "git+https://github.com/Neo23x0/signature-base",
+        "git+https://github.com/malpedia/signator-rules",
+        "git+https://github.com/baderj/yara",
+        "git+https://github.com/deadbits/yara-rules",  # archived
+        "git+https://github.com/pmelson/yara_rules",
+        "git+https://github.com/sbousseaden/YaraHunts",
+        "git+https://github.com/embee-research/Yara-detection-rules",
+        "git+https://github.com/RussianPanda95/Yara-Rules",
+        "git+https://github.com/ail-project/ail-yara-rules",
+        "git+https://github.com/MalGamy/YARA_Rules",
+        "git+https://github.com/elceef/yara-rulz",
+        "git+https://github.com/tenable/yara-rules",
+        "git+https://github.com/dr4k0nia/yara-rules",
+        "git+https://github.com/umair9747/yara-rules",
+    ]
+
+    license_urls = """
+    https://github.com/elastic/protections-artifacts/blob/main/LICENSE.txt
+    https://github.com/Yara-Rules/rules/blob/master/LICENSE
+    https://github.com/Xumeiquer/yara-forensics/blob/master/LICENSE
+    https://github.com/reversinglabs/reversinglabs-yara-rules/blob/develop/LICENSE
+    https://github.com/advanced-threat-research/Yara-Rules/blob/master/LICENSE
+    https://github.com/bartblaze/Yara-rules/blob/master/LICENSE
+    https://github.com/godaddy/yara-rules/blob/master/LICENSE.md
+    https://github.com/SupportIntelligence/Icewater/blob/master/LICENSE
+    https://github.com/jeFF0Falltrades/YARA-Signatures/blob/master/LICENSE.md
+    https://github.com/tjnel/yara_repo/blob/master/LICENSE
+    https://github.com/JPCERTCC/jpcert-yara/blob/main/LICENSE
+    https://github.com/mthcht/ThreatHunting-Keywords-yara-rules/blob/main/LICENSE
+    https://github.com/malpedia/signator-rules -> https://creativecommons.org/licenses/by-sa/4.0/
+    https://github.com/baderj/yara/blob/main/LICENSE
+    https://github.com/deadbits/yara-rules/blob/master/UNLICENSE
+    https://github.com/embee-research/Yara-detection-rules/tree/main?tab=readme-ov-file#detection-rule-license-drl-11
+    https://github.com/ail-project/ail-yara-rules?tab=AGPL-3.0-1-ov-file
+    https://github.com/MalGamy/YARA_Rules/blob/main/LICENSE.md
+    https://github.com/elceef/yara-rulz/tree/main?tab=MIT-1-ov-file
+    https://github.com/tenable/yara-rules/tree/master?tab=BSD-3-Clause-1-ov-file
+    https://github.com/dr4k0nia/yara-rules/blob/main/LICENSE.md
+    https://github.com/umair9747/yara-rules?tab=GPL-3.0-1-ov-file
+    
+    NO-LICENSE: https://github.com/mikesxrs/Open-Source-YARA-rules/
+    NO-LICENSE: https://github.com/fboldewin/YARA-rules
+    NO-LICENSE: https://github.com/h3x2b/yara-rules
+    NO-LICENSE: https://github.com/roadwy/DefenderYara
+    NO-LICENSE: https://github.com/pmelson/yara_rules
+    NO-LICENSE: https://github.com/sbousseaden/YaraHunts
+    NO-LICENSE: https://github.com/RussianPanda95/Yara-Rules
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.vcs_responses = []
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.clone_repos,
+            cls.collect_and_store_rules,
+            cls.clean_downloads,
+        )
+
+    def clone_repos(self):
+        for repo_url in self.repo_urls:
+            self.log(f"Cloning `{repo_url}`")
+            try:
+                response = fetch_via_vcs(repo_url)
+                if response:
+                    self.vcs_responses.append((response, repo_url))
+            except Exception as e:
+                self.log(f"Failed to clone {repo_url}: {e}")
+
+    def collect_and_store_rules(self):
+        for vcs_response, repo_url in self.vcs_responses:
+            base_directory = Path(vcs_response.dest_dir)
+            yara_files = [
+                p
+                for p in base_directory.rglob("*")
+                if p.suffix in (".yar", ".yara") and p.is_file()
+            ]
+
+            rules_count = len(yara_files)
+            self.log(f"Processing {rules_count:,d} rules from {repo_url}")
+            progress = LoopProgress(total_iterations=rules_count, logger=self.log)
+            for file_path in progress.iter(yara_files):
+                if not file_path.exists() or not file_path.is_file():
+                    self.log(
+                        f"Skipping file as it no longer exists or is not a file: {file_path}",
+                        level="warning",
+                    )
+                    continue
+
+                raw_text = file_path.read_text(encoding="utf-8", errors="ignore")
+                if not raw_text:
+                    continue
+                raw_text = raw_text.replace("\x00", "")
+
+                repo_url = repo_url.strip("git+")
+                rule_url = get_advisory_url(
+                    file=file_path,
+                    base_path=base_directory,
+                    url=f"{repo_url}/blob/master/",
+                )
+
+                cve_ids = find_all_cve(f"{file_path}\n{raw_text}")
+
+                advisories = set()
+                for cve_id in cve_ids:
+                    alias = AdvisoryAlias.objects.filter(alias=cve_id).first()
+                    if alias:
+                        for adv in alias.advisories.all():
+                            advisories.add(adv)
+                    else:
+                        advs = AdvisoryV2.objects.filter(advisory_id=cve_id)
+                        for adv in advs:
+                            advisories.add(adv)
+
+                detection_rule, _ = DetectionRule.objects.update_or_create(
+                    rule_type=DetectionRuleTypes.YARA,
+                    source_url=rule_url,
+                    defaults={
+                        "rule_text": raw_text,
+                    },
+                )
+
+                for adv in advisories:
+                    detection_rule.related_advisories.add(adv)
+
+    def clean_downloads(self):
+        for vcs_response, _ in self.vcs_responses:
+            if vcs_response:
+                self.log(f"Removing cloned repository: {vcs_response.dest_dir}")
+                vcs_response.delete()
+
+        self.vcs_responses = []
+
+    def on_failure(self):
+        self.clean_downloads()

vulnerabilities/tests/pipelines/v2_improvers/test_yara.py

@@ -0,0 +1,32 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+from vulnerabilities.models import DetectionRule
+from vulnerabilities.pipelines.v2_improvers.yara_rules import YaraRulesImproverPipeline
+
+BASE_DIR = Path(__file__).resolve().parent
+TEST_REPO_DIR = (BASE_DIR / "../../test_data/yara").resolve()
+
+
+@pytest.mark.django_db
+def test_collect_and_store_rules_from_test_repo_dir():
+    mock_vcs_response = MagicMock()
+    mock_vcs_response.dest_dir = str(TEST_REPO_DIR)
+
+    improver = YaraRulesImproverPipeline()
+    improver.vcs_responses = [(mock_vcs_response, "https://github.com/mock/repo")]
+    improver.collect_and_store_rules()
+
+    assert DetectionRule.objects.exists()
+    assert DetectionRule.objects.count() == 4

vulnerabilities/tests/test_data/yara/Linux_Backdoor_Bash.yar

@@ -0,0 +1,45 @@
+rule Windows_Trojan_Xeno_f92ffb82 {
+    meta:
+        author = "Elastic Security"
+        id = "f92ffb82-b743-4df1-9d6b-2afa3b7bb61f"
+        fingerprint = "2ae1aebd652afb7da5799f46883205b1f3a5c5b01e975b526640407d9bd0d22c"
+        creation_date = "2024-10-10"
+        last_modified = "2024-10-24"
+        threat_name = "Windows.Trojan.Xeno"
+        reference_sample = "22dbdbcdd4c8b6899006f9f07e87c19b6a2947eeff8cc89c653309379b388cf4"
+        severity = 50
+        arch_context = "x86"
+        scan_context = "file, memory"
+        license = "Elastic License v2"
+        os = "windows"
+    strings:
+        $a1 = { 28 00 00 0A 7D 0E 00 00 04 02 7B 0E 00 00 04 28 29 00 00 0A 07 7B 03 00 00 04 02 7B 0E 00 00 04 6F 2A 00 00 0A 3A F2 00 00 00 02 7B 07 00 00 04 02 7B 09 00 00 04 6F 32 00 00 06 6F 2B 00 00 0A }
+    condition:
+        all of them
+}
+
+rule Windows_Trojan_Xeno_89f9f060 {
+    meta:
+        author = "Elastic Security"
+        id = "89f9f060-afc8-427d-ad36-3672016efdf6"
+        fingerprint = "ddc5bf8c6d5140cb9ea2fbd9b6f1aaab60f506dcd6161a26961958efa4aa42e1"
+        creation_date = "2024-10-25"
+        last_modified = "2024-11-26"
+        threat_name = "Windows.Trojan.Xeno"
+        reference_sample = "b74733d68e95220ab0630a68ddf973b0c959fd421628e639c1b91e465ba9299b"
+        severity = 100
+        arch_context = "x86"
+        scan_context = "file, memory"
+        license = "Elastic License v2"
+        os = "windows"
+    strings:
+        $sc_1 = { 8B 44 24 04 89 C6 FF 56 08 68 00 04 00 00 6A 08 50 FF 16 89 C3 8B 06 89 83 2C 01 00 00 8B 46 04 89 83 30 01 00 00 8B 46 08 89 83 34 01 00 00 8B 46 0C 89 83 38 }
+        $sc_2 = { 55 48 89 E5 48 83 EC 40 49 89 CC 41 FF 54 24 10 48 89 C1 BA 08 00 00 00 41 B8 00 04 00 00 41 FF 14 24 48 89 C3 49 8B 04 24 48 89 83 90 01 00 00 49 8B 44 24 08 }
+        $str_1 = "SharpInjector" ascii fullword
+        $str_2 = "HEAVENSGATE_NON_OPERATIONAL" ascii fullword
+        $str_3 = "ChromeDecryptor" ascii fullword
+        $str_4 = "DataExtractionStructs" ascii fullword
+        $str_5 = "XenoStealer" ascii fullword
+    condition:
+        (($sc_1 or $sc_2) and ($str_1 or $str_2)) and (1 of ($str_3, $str_4, $str_5))
+}

vulnerabilities/tests/test_data/yara/Linux_Backdoor_Python.yar

@@ -0,0 +1,19 @@
+rule Linux_Backdoor_Python_00606bac {
+    meta:
+        author = "Elastic Security"
+        id = "00606bac-83eb-4a58-82d2-e4fd16d30846"
+        fingerprint = "cce1d0e7395a74c04f15ff95f6de7fd7d5f46ede83322b832df74133912c0b17"
+        creation_date = "2021-01-12"
+        last_modified = "2021-09-16"
+        threat_name = "Linux.Backdoor.Python"
+        reference_sample = "b3e3728d43535f47a1c15b915c2d29835d9769a9dc69eb1b16e40d5ba1b98460"
+        severity = 100
+        arch_context = "x86"
+        scan_context = "file, memory"
+        license = "Elastic License v2"
+        os = "linux"
+    strings:
+        $a = { F4 01 83 45 F8 01 8B 45 F8 0F B6 00 84 C0 75 F2 83 45 F8 01 8B }
+    condition:
+        all of them
+}

vulnerabilities/tests/test_data/yara/Linux_Exploit_CVE_2022_0847.yar

@@ -0,0 +1,27 @@
+rule Linux_Exploit_CVE_2022_0847_e831c285 {
+    meta:
+        author = "Elastic Security"
+        id = "e831c285-b2b9-49f3-a87c-3deb806e31e4"
+        fingerprint = "376b791f9bb5f48d0f41ead4e48b5bcc74cb68002bb7c170760428ace169457e"
+        creation_date = "2022-03-10"
+        last_modified = "2022-03-14"
+        threat_name = "Linux.Exploit.CVE-2022-0847"
+        reference_sample = "c6b2cef2f2bc04e3ae33e0d368eb39eb5ea38d1bca390df47f7096117c1aecca"
+        severity = 100
+        arch_context = "x86"
+        scan_context = "file, memory"
+        license = "Elastic License v2"
+        os = "linux"
+    strings:
+        $pp = "prepare_pipe"
+        $s1 = "splice failed"
+        $s2 = "short splice"
+        $s3 = "short write"
+        $s4 = "hijacking suid binary"
+        $s5 = "Usage: %s TARGETFILE OFFSET DATA"
+        $s6 = "Usage: %s SUID"
+        $bs1 = { B8 00 10 00 00 81 7D EC 00 10 00 00 0F 46 45 EC 89 45 FC 8B 55 FC 48 8B 45 D8 48 83 C0 04 8B 00 48 8D 35 }
+        $bs2 = { B8 00 10 00 00 81 7D F0 00 10 00 00 0F 46 45 F0 89 45 F8 8B 55 F8 48 8B 45 D8 8B 00 48 }
+    condition:
+        ($pp and 2 of ($s*)) or (all of ($bs*))
+}