Skip to content

Commit 77c356c

Browse files
ziadhanyziadhany
committed
Add support for yara rules
Signed-off-by: ziad hany <ziadhany2016@gmail.com>
1 parent 58dd7ad commit 77c356c

3 files changed

Lines changed: 215 additions & 142 deletions

File tree

‎vulnerabilities/improvers/__init__.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,42 @@
7575
unfurl_version_range_v2.UnfurlVersionRangePipeline,
7676
compute_advisory_todo.ComputeToDo,
7777
collect_ssvc_trees.CollectSSVCPipeline,
78+
7879
relate_severities.RelateSeveritiesPipeline,
79-
yara_rules.YaraRulesImproverPipeline,
80+
sigma_rules.SigmaHQImproverPipeline,
81+
sigma_rules.SigmaSamuraiMDRImproverPipeline,
82+
sigma_rules.SigmaMbabinskiImproverPipeline,
83+
sigma_rules.P4T12ICKSigmaImproverPipeline,
84+
85+
yara_rules.ProtectionsArtifactsYara,
86+
yara_rules.YaraRulesYara,
87+
yara_rules.XumeiquerForensicsYara,
88+
yara_rules.ReversinglabsYaraRules,
89+
yara_rules.AdvancedThreatResearchYara,
90+
yara_rules.BartblazeYaraRules,
91+
yara_rules.GodaddyYaraRules,
92+
yara_rules.SupportIntelligenceIcewaterYara,
93+
yara_rules.Jeff0FalltradesSignaturesYara,
94+
yara_rules.TjnelRepoYara,
95+
yara_rules.JpcertccJpcertYara,
96+
yara_rules.MikesxrsOpenSourceYaraRules,
97+
yara_rules.FboldewinYaraRules,
98+
yara_rules.H3x2bYaraRules,
99+
yara_rules.RoadwyDefenderYara,
100+
yara_rules.MthchtThreatHuntingKeywordsYara,
101+
yara_rules.Neo23x0SignatureBaseYara,
102+
yara_rules.MalpediaSignatorRulesYara,
103+
yara_rules.BaderjYara,
104+
yara_rules.DeadbitsYaraRules,
105+
yara_rules.PmelsonYaraRules,
106+
yara_rules.SbousseadenYaraHunts,
107+
yara_rules.EmbeeResearchYaraRules,
108+
yara_rules.RussianPanda95YaraRules,
109+
yara_rules.AilProjectAilYaraRules,
110+
yara_rules.MalgamyYaraRules,
111+
yara_rules.ElceefYaraRulz,
112+
yara_rules.TenableYaraRules,
113+
yara_rules.Dr4k0niaYaraRules,
114+
yara_rules.Umair9747YaraRules,
80115
]
81116
)

‎vulnerabilities/pipelines/v2_improvers/sigma_rules.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import json
1212
from pathlib import Path
1313

14-
import jsonschema
1514
import yaml
1615
from aboutcode.pipeline import LoopProgress
1716
from fetchcode.vcs import fetch_via_vcs

‎vulnerabilities/pipelines/v2_improvers/yara_rules.py‎

Lines changed: 179 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
77
# See https://aboutcode.org for more information about nexB OSS projects.
88
#
9-
from pathlib import Path
109

10+
from pathlib import Path
1111
from aboutcode.pipeline import LoopProgress
1212
from fetchcode.vcs import fetch_via_vcs
1313

@@ -19,79 +19,9 @@
1919
from vulnerabilities.utils import find_all_cve
2020
from vulnerabilities.utils import get_advisory_url
2121

22-
2322
class YaraRulesImproverPipeline(VulnerableCodePipeline):
24-
pipeline_id = "yara_rules"
25-
26-
repo_urls = [
27-
"git+https://github.com/elastic/protections-artifacts",
28-
"git+https://github.com/Yara-Rules/rules",
29-
"git+https://github.com/Xumeiquer/yara-forensics",
30-
"git+https://github.com/reversinglabs/reversinglabs-yara-rules",
31-
"git+https://github.com/advanced-threat-research/Yara-Rules",
32-
"git+https://github.com/bartblaze/Yara-rules",
33-
"git+https://github.com/godaddy/yara-rules", # archived
34-
"git+https://github.com/SupportIntelligence/Icewater",
35-
"git+https://github.com/jeFF0Falltrades/YARA-Signatures",
36-
"git+https://github.com/tjnel/yara_repo",
37-
"git+https://github.com/JPCERTCC/jpcert-yara",
38-
"git+https://github.com/mikesxrs/Open-Source-YARA-rules",
39-
"git+https://github.com/fboldewin/YARA-rules",
40-
"git+https://github.com/h3x2b/yara-rules",
41-
"git+https://github.com/roadwy/DefenderYara",
42-
"git+https://github.com/mthcht/ThreatHunting-Keywords-yara-rules",
43-
"git+https://github.com/Neo23x0/signature-base",
44-
"git+https://github.com/malpedia/signator-rules",
45-
"git+https://github.com/baderj/yara",
46-
"git+https://github.com/deadbits/yara-rules", # archived
47-
"git+https://github.com/pmelson/yara_rules",
48-
"git+https://github.com/sbousseaden/YaraHunts",
49-
"git+https://github.com/embee-research/Yara-detection-rules",
50-
"git+https://github.com/RussianPanda95/Yara-Rules",
51-
"git+https://github.com/ail-project/ail-yara-rules",
52-
"git+https://github.com/MalGamy/YARA_Rules",
53-
"git+https://github.com/elceef/yara-rulz",
54-
"git+https://github.com/tenable/yara-rules",
55-
"git+https://github.com/dr4k0nia/yara-rules",
56-
"git+https://github.com/umair9747/yara-rules",
57-
]
58-
59-
license_urls = """
60-
https://github.com/elastic/protections-artifacts/blob/main/LICENSE.txt
61-
https://github.com/Yara-Rules/rules/blob/master/LICENSE
62-
https://github.com/Xumeiquer/yara-forensics/blob/master/LICENSE
63-
https://github.com/reversinglabs/reversinglabs-yara-rules/blob/develop/LICENSE
64-
https://github.com/advanced-threat-research/Yara-Rules/blob/master/LICENSE
65-
https://github.com/bartblaze/Yara-rules/blob/master/LICENSE
66-
https://github.com/godaddy/yara-rules/blob/master/LICENSE.md
67-
https://github.com/SupportIntelligence/Icewater/blob/master/LICENSE
68-
https://github.com/jeFF0Falltrades/YARA-Signatures/blob/master/LICENSE.md
69-
https://github.com/tjnel/yara_repo/blob/master/LICENSE
70-
https://github.com/JPCERTCC/jpcert-yara/blob/main/LICENSE
71-
https://github.com/mthcht/ThreatHunting-Keywords-yara-rules/blob/main/LICENSE
72-
https://github.com/malpedia/signator-rules -> https://creativecommons.org/licenses/by-sa/4.0/
73-
https://github.com/baderj/yara/blob/main/LICENSE
74-
https://github.com/deadbits/yara-rules/blob/master/UNLICENSE
75-
https://github.com/embee-research/Yara-detection-rules/tree/main?tab=readme-ov-file#detection-rule-license-drl-11
76-
https://github.com/ail-project/ail-yara-rules?tab=AGPL-3.0-1-ov-file
77-
https://github.com/MalGamy/YARA_Rules/blob/main/LICENSE.md
78-
https://github.com/elceef/yara-rulz/tree/main?tab=MIT-1-ov-file
79-
https://github.com/tenable/yara-rules/tree/master?tab=BSD-3-Clause-1-ov-file
80-
https://github.com/dr4k0nia/yara-rules/blob/main/LICENSE.md
81-
https://github.com/umair9747/yara-rules?tab=GPL-3.0-1-ov-file
82-
83-
NO-LICENSE: https://github.com/mikesxrs/Open-Source-YARA-rules/
84-
NO-LICENSE: https://github.com/fboldewin/YARA-rules
85-
NO-LICENSE: https://github.com/h3x2b/yara-rules
86-
NO-LICENSE: https://github.com/roadwy/DefenderYara
87-
NO-LICENSE: https://github.com/pmelson/yara_rules
88-
NO-LICENSE: https://github.com/sbousseaden/YaraHunts
89-
NO-LICENSE: https://github.com/RussianPanda95/Yara-Rules
90-
"""
91-
92-
def __init__(self, *args, **kwargs):
93-
super().__init__(*args, **kwargs)
94-
self.vcs_responses = []
23+
repo_url = None
24+
rglob_patterns = ["**/*.yml"]
9525

9626
@classmethod
9727
def steps(cls):
@@ -102,78 +32,187 @@ def steps(cls):
10232
)
10333

10434
def clone_repos(self):
105-
for repo_url in self.repo_urls:
106-
self.log(f"Cloning `{repo_url}`")
107-
try:
108-
response = fetch_via_vcs(repo_url)
109-
if response:
110-
self.vcs_responses.append((response, repo_url))
111-
except Exception as e:
112-
self.log(f"Failed to clone {repo_url}: {e}")
35+
self.log(f"Cloning `{self.repo_url}`")
36+
self.vcs_response = fetch_via_vcs(f"git+{self.repo_url}")
11337

11438
def collect_and_store_rules(self):
115-
for vcs_response, repo_url in self.vcs_responses:
116-
base_directory = Path(vcs_response.dest_dir)
117-
yara_files = [
118-
p
119-
for p in base_directory.rglob("*")
120-
if p.suffix in (".yar", ".yara") and p.is_file()
121-
]
122-
123-
rules_count = len(yara_files)
124-
self.log(f"Processing {rules_count:,d} rules from {repo_url}")
125-
progress = LoopProgress(total_iterations=rules_count, logger=self.log)
126-
for file_path in progress.iter(yara_files):
127-
if not file_path.exists() or not file_path.is_file():
128-
self.log(
129-
f"Skipping file as it no longer exists or is not a file: {file_path}",
130-
level="warning",
131-
)
132-
continue
133-
134-
raw_text = file_path.read_text(encoding="utf-8", errors="ignore")
135-
if not raw_text:
136-
continue
137-
raw_text = raw_text.replace("\x00", "")
138-
139-
repo_url = repo_url.strip("git+")
140-
rule_url = get_advisory_url(
141-
file=file_path,
142-
base_path=base_directory,
143-
url=f"{repo_url}/blob/master/",
144-
)
145-
146-
cve_ids = find_all_cve(f"{file_path}\n{raw_text}")
147-
148-
advisories = set()
149-
for cve_id in cve_ids:
150-
alias = AdvisoryAlias.objects.filter(alias=cve_id).first()
151-
if alias:
152-
for adv in alias.advisories.all():
153-
advisories.add(adv)
154-
else:
155-
advs = AdvisoryV2.objects.filter(advisory_id=cve_id)
156-
for adv in advs:
157-
advisories.add(adv)
158-
159-
detection_rule, _ = DetectionRule.objects.update_or_create(
160-
rule_type=DetectionRuleTypes.YARA,
161-
source_url=rule_url,
162-
defaults={
163-
"rule_text": raw_text,
164-
},
39+
base_directory = Path(self.vcs_response.dest_dir)
40+
yara_files = set()
41+
for pattern in self.rglob_patterns:
42+
for p in base_directory.glob(pattern):
43+
if p.is_file():
44+
yara_files.add(p)
45+
46+
rules_count = len(yara_files)
47+
self.log(f"Processing {rules_count:,d} rules from {self.repo_url}")
48+
progress = LoopProgress(total_iterations=rules_count, logger=self.log)
49+
for file_path in progress.iter(yara_files):
50+
if not file_path.exists() or not file_path.is_file():
51+
self.log(
52+
f"Skipping file as it no longer exists or is not a file: {file_path}",
53+
level="warning",
16554
)
166-
167-
for adv in advisories:
168-
detection_rule.related_advisories.add(adv)
55+
continue
56+
57+
raw_text = file_path.read_text(encoding="utf-8", errors="ignore")
58+
if not raw_text:
59+
continue
60+
raw_text = raw_text.replace("\x00", "")
61+
rule_url = get_advisory_url(
62+
file=file_path,
63+
base_path=base_directory,
64+
url=f"{self.repo_url}/blob/master/",
65+
)
66+
67+
cve_ids = find_all_cve(f"{file_path}\n{raw_text}")
68+
69+
advisories = set()
70+
for cve_id in cve_ids:
71+
alias = AdvisoryAlias.objects.filter(alias=cve_id).first()
72+
if alias:
73+
for adv in alias.advisories.all():
74+
advisories.add(adv)
75+
else:
76+
advs = AdvisoryV2.objects.filter(advisory_id=cve_id)
77+
for adv in advs:
78+
advisories.add(adv)
79+
80+
detection_rule, _ = DetectionRule.objects.update_or_create(
81+
rule_type=DetectionRuleTypes.YARA,
82+
source_url=rule_url,
83+
defaults={
84+
"rule_text": raw_text,
85+
},
86+
)
87+
88+
for adv in advisories:
89+
detection_rule.related_advisories.add(adv)
16990

17091
def clean_downloads(self):
171-
for vcs_response, _ in self.vcs_responses:
172-
if vcs_response:
173-
self.log(f"Removing cloned repository: {vcs_response.dest_dir}")
174-
vcs_response.delete()
175-
176-
self.vcs_responses = []
92+
if self.vcs_response:
93+
self.log(f"Removing cloned repository: {self.vcs_response.dest_dir}")
94+
self.vcs_response.delete()
17795

17896
def on_failure(self):
17997
self.clean_downloads()
98+
99+
100+
class ProtectionsArtifactsYara(YaraRulesImproverPipeline):
101+
repo_urls = "https://github.com/elastic/protections-artifacts"
102+
license_urls = "https://github.com/elastic/protections-artifacts/blob/main/LICENSE.txt"
103+
104+
class YaraRulesYara(YaraRulesImproverPipeline):
105+
repo_urls = "https://github.com/Yara-Rules/rules"
106+
license_urls = "https://github.com/Yara-Rules/rules/blob/master/LICENSE"
107+
108+
class XumeiquerForensicsYara(YaraRulesImproverPipeline):
109+
repo_urls = "https://github.com/Xumeiquer/yara-forensics"
110+
license_urls = "https://github.com/Xumeiquer/yara-forensics/blob/master/LICENSE"
111+
112+
class ReversinglabsYaraRules(YaraRulesImproverPipeline):
113+
repo_urls = "https://github.com/reversinglabs/reversinglabs-yara-rules"
114+
license_urls = "https://github.com/reversinglabs/reversinglabs-yara-rules/blob/develop/LICENSE"
115+
116+
class AdvancedThreatResearchYara(YaraRulesImproverPipeline):
117+
repo_urls = "https://github.com/advanced-threat-research/Yara-Rules"
118+
license_urls = "https://github.com/advanced-threat-research/Yara-Rules/blob/master/LICENSE"
119+
120+
class BartblazeYaraRules(YaraRulesImproverPipeline):
121+
repo_urls = "https://github.com/bartblaze/Yara-rules"
122+
license_urls = "https://github.com/bartblaze/Yara-rules/blob/master/LICENSE"
123+
124+
class GodaddyYaraRules(YaraRulesImproverPipeline):
125+
repo_urls = "https://github.com/godaddy/yara-rules" # archived
126+
license_urls = "https://github.com/godaddy/yara-rules/blob/master/LICENSE.md"
127+
128+
class SupportIntelligenceIcewaterYara(YaraRulesImproverPipeline):
129+
repo_urls = "https://github.com/SupportIntelligence/Icewater"
130+
license_urls = "https://github.com/SupportIntelligence/Icewater/blob/master/LICENSE"
131+
132+
class Jeff0FalltradesSignaturesYara(YaraRulesImproverPipeline):
133+
repo_urls = "https://github.com/jeFF0Falltrades/YARA-Signatures"
134+
license_urls = "https://github.com/jeFF0Falltrades/YARA-Signatures/blob/master/LICENSE.md"
135+
136+
class TjnelRepoYara(YaraRulesImproverPipeline):
137+
repo_urls = "https://github.com/tjnel/yara_repo"
138+
license_urls = "https://github.com/tjnel/yara_repo/blob/master/LICENSE"
139+
140+
class JpcertccJpcertYara(YaraRulesImproverPipeline):
141+
repo_urls = "https://github.com/JPCERTCC/jpcert-yara"
142+
license_urls = "https://github.com/JPCERTCC/jpcert-yara/blob/main/LICENSE"
143+
144+
class MikesxrsOpenSourceYaraRules(YaraRulesImproverPipeline):
145+
repo_urls = "https://github.com/mikesxrs/Open-Source-YARA-rules"
146+
license_urls = None
147+
148+
class FboldewinYaraRules(YaraRulesImproverPipeline):
149+
repo_urls = "https://github.com/fboldewin/YARA-rules"
150+
license_urls = None
151+
152+
class H3x2bYaraRules(YaraRulesImproverPipeline):
153+
repo_urls = "https://github.com/h3x2b/yara-rules"
154+
license_urls = None
155+
156+
class RoadwyDefenderYara(YaraRulesImproverPipeline):
157+
repo_urls = "https://github.com/roadwy/DefenderYara"
158+
license_urls = None
159+
160+
class MthchtThreatHuntingKeywordsYara(YaraRulesImproverPipeline):
161+
repo_urls = "https://github.com/mthcht/ThreatHunting-Keywords-yara-rules"
162+
license_urls = "https://github.com/mthcht/ThreatHunting-Keywords-yara-rules/blob/main/LICENSE"
163+
164+
class Neo23x0SignatureBaseYara(YaraRulesImproverPipeline):
165+
repo_urls = "https://github.com/Neo23x0/signature-base"
166+
license_urls = None
167+
168+
class MalpediaSignatorRulesYara(YaraRulesImproverPipeline):
169+
repo_urls = "https://github.com/malpedia/signator-rules"
170+
license_urls = "https://creativecommons.org/licenses/by-sa/4.0/"
171+
172+
class BaderjYara(YaraRulesImproverPipeline):
173+
repo_urls = "https://github.com/baderj/yara"
174+
license_urls = "https://github.com/baderj/yara/blob/main/LICENSE"
175+
176+
class DeadbitsYaraRules(YaraRulesImproverPipeline):
177+
repo_urls = "https://github.com/deadbits/yara-rules"
178+
license_urls = "https://github.com/deadbits/yara-rules/blob/master/UNLICENSE"
179+
180+
class PmelsonYaraRules(YaraRulesImproverPipeline):
181+
repo_urls = "https://github.com/pmelson/yara_rules"
182+
license_urls = None
183+
184+
class SbousseadenYaraHunts(YaraRulesImproverPipeline):
185+
repo_urls = "https://github.com/sbousseaden/YaraHunts"
186+
license_urls = None
187+
188+
class EmbeeResearchYaraRules(YaraRulesImproverPipeline):
189+
repo_urls = "https://github.com/embee-research/Yara-detection-rules"
190+
license_urls = "https://github.com/embee-research/Yara-detection-rules/tree/main?tab=readme-ov-file#detection-rule-license-drl-11"
191+
192+
class RussianPanda95YaraRules(YaraRulesImproverPipeline):
193+
repo_urls = "https://github.com/RussianPanda95/Yara-Rules"
194+
license_urls = None
195+
196+
class AilProjectAilYaraRules(YaraRulesImproverPipeline):
197+
repo_urls = "https://github.com/ail-project/ail-yara-rules"
198+
license_urls = "https://github.com/ail-project/ail-yara-rules?tab=AGPL-3.0-1-ov-file"
199+
200+
class MalgamyYaraRules(YaraRulesImproverPipeline):
201+
repo_urls = "https://github.com/MalGamy/YARA_Rules"
202+
license_urls = "https://github.com/MalGamy/YARA_Rules/blob/main/LICENSE.md"
203+
204+
class ElceefYaraRulz(YaraRulesImproverPipeline):
205+
repo_urls = "https://github.com/elceef/yara-rulz"
206+
license_urls = "https://github.com/elceef/yara-rulz/tree/main?tab=MIT-1-ov-file"
207+
208+
class TenableYaraRules(YaraRulesImproverPipeline):
209+
repo_urls = "https://github.com/tenable/yara-rules"
210+
license_urls = "https://github.com/tenable/yara-rules/tree/master?tab=BSD-3-Clause-1-ov-file"
211+
212+
class Dr4k0niaYaraRules(YaraRulesImproverPipeline):
213+
repo_urls = "https://github.com/dr4k0nia/yara-rules"
214+
license_urls = "https://github.com/dr4k0nia/yara-rules/blob/main/LICENSE.md"
215+
216+
class Umair9747YaraRules(YaraRulesImproverPipeline):
217+
repo_urls = "https://github.com/umair9747/yara-rules"
218+
license_urls = "https://github.com/umair9747/yara-rules?tab=GPL-3.0-1-ov-file"

0 commit comments

Comments
 (0)