Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion bbot/core/config/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,13 +337,14 @@ class DnsConfig(BaseModel):


class CdnPortFilterConfig(BaseModel):
"""`cdn_port_filter` — drops open ports and URLs on CDN/WAF hosts outside the allowed ports."""
"""`cdn_port_filter`: drops open ports and URLs outside the allowed ports on CDN/WAF hosts and catch-all hosts."""

model_config = STRICT

enabled: Optional[bool] = None
tags: Optional[list[str]] = None
allowed_ports: Optional[list[int]] = None
max_ports: Optional[int] = None


class BodySpillConfig(BaseModel):
Expand Down
6 changes: 4 additions & 2 deletions bbot/defaults.yml
Original file line number Diff line number Diff line change
Expand Up @@ -216,18 +216,20 @@ url_querystring_collapse: True
# Reject URLs whose path repeats the same path parameter this many times (e.g. a session id appended on every redirect)
url_max_path_param_repeats: 10

# Drop open ports and URLs on CDN/WAF hosts unless the port is allowed or was explicitly targeted
# Drop open ports and URLs on CDN/WAF hosts and catch-all hosts unless the port is allowed or was explicitly targeted
# CDN edges answer on many ports (e.g. Cloudflare's 2052-2096, 8080, 8443) that rarely lead anywhere useful
cdn_port_filter:
enabled: true
# hosts with any of these tags are considered CDN/WAF hosts
tags:
- cdn
- waf
# ports that are still allowed on CDN/WAF hosts
# ports that are still allowed on CDN/WAF and catch-all hosts
allowed_ports:
- 80
- 443
# a host that a passive port source lists with more open ports than this accepts connections on any port (e.g. a load balancer or tarpit), so it's treated as catch-all (0 = off)
max_ports: 100

# Let active modules scan hosts that resolve to loopback (127.0.0.0/8, ::1, 0.0.0.0), i.e. the machine BBOT runs on
# Loopback addresses in the target are always allowed
Expand Down
30 changes: 23 additions & 7 deletions bbot/modules/censys_ip.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,32 @@ async def handle_event(self, event):
# Track what we've already emitted to avoid duplicates
seen = set()

# QUIC uses UDP as transport, so treat it as UDP
udp_transports = ("UDP", "QUIC")
services = result.get("services", [])
tcp_ports = set(
self.scan.filter_catch_all_ports(
ip,
[
service.get("port")
for service in services
if service.get("port")
and (service.get("transport_protocol") or "TCP").upper() not in udp_transports
],
)
)

# Extract data from services
for service in result.get("services", []):
for service in services:
port = service.get("port")
transport = service.get("transport_protocol", "TCP").upper()
transport = (service.get("transport_protocol") or "TCP").upper()
# a TCP port cut by filter_catch_all_ports, along with its PROTOCOL and URL
dropped = bool(port) and transport not in udp_transports and port not in tcp_ports

# Emit OPEN_TCP_PORT or OPEN_UDP_PORT for services with a port
# QUIC uses UDP as transport, so treat it as UDP
if port and (port, transport) not in seen:
if port and not dropped and (port, transport) not in seen:
seen.add((port, transport))
if transport in ("UDP", "QUIC"):
if transport in udp_transports:
event_type = "OPEN_UDP_PORT"
else:
event_type = "OPEN_TCP_PORT"
Expand All @@ -109,7 +125,7 @@ async def handle_event(self, event):
# If service_name is UNKNOWN but transport_protocol is meaningful, use that
if service_name.upper() == "UNKNOWN" and transport and transport not in ("TCP", "UDP"):
service_name = transport
if service_name and service_name.upper() not in ("HTTP", "HTTPS", "UNKNOWN"):
if service_name and not dropped and service_name.upper() not in ("HTTP", "HTTPS", "UNKNOWN"):
protocol_key = ("protocol", service_name.upper(), port)
if protocol_key not in seen:
seen.add(protocol_key)
Expand All @@ -127,7 +143,7 @@ async def handle_event(self, event):
http_data = service.get("http", {})
request = http_data.get("request", {})
uri = request.get("uri")
if uri and uri not in seen:
if uri and not dropped and uri not in seen:
seen.add(uri)
await self.emit_event(
uri,
Expand Down
21 changes: 14 additions & 7 deletions bbot/modules/shodan_enterprise.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ async def handle_event(self, event):
# NIST cvss score severity mapping
severity_map = {"NONE": 0.0, "LOW": 0.1, "MEDIUM": 4.0, "HIGH": 7.0, "CRITICAL": 9.0}

tcp_ports = set(
self.scan.filter_catch_all_ports(
ip, [data.get("port") for data in host["data"] if data.get("transport") == "tcp" and "port" in data]
)
)

for data in host["data"]:
# TECHNOLOGY Extraction
# name technologies by their CPE product, falling back to Shodan's product name when there's no CPE
Expand Down Expand Up @@ -97,13 +103,14 @@ async def handle_event(self, event):
# OPEN_TCP_PORT, OPEN_UDP_PORT Extraction
if "port" in data and "transport" in data:
if data["transport"] == "tcp":
await self.emit_event(
self.helpers.make_netloc(ip, data.get("port")),
"OPEN_TCP_PORT",
parent=event,
tags=data.get("tags") or [],
context=f"{{module}} queried Shodan API for {ip} and found OPEN_TCP_PORT: {data.get('port')}",
)
if data["port"] in tcp_ports:
await self.emit_event(
self.helpers.make_netloc(ip, data.get("port")),
"OPEN_TCP_PORT",
parent=event,
tags=data.get("tags") or [],
context=f"{{module}} queried Shodan API for {ip} and found OPEN_TCP_PORT: {data.get('port')}",
)
elif data["transport"] == "udp":
await self.emit_event(
self.helpers.make_netloc(ip, data.get("port")),
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/shodan_idb.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ async def _parse_response(self, data: dict, event, ip):
parent=event,
context=f'{{module}} queried Shodan\'s InternetDB API for "{query_host}" and found {{event.type}}: {{event.pretty_string}}',
)
for port in data.get("ports", []):
for port in self.scan.filter_catch_all_ports(event.host, data.get("ports", [])):
await self.emit_event(
self.helpers.make_netloc(event.data, port),
"OPEN_TCP_PORT",
Expand Down
2 changes: 1 addition & 1 deletion bbot/scanner/preset/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ def create_parser(self, *args, **kwargs):
output.add_argument(
"--allow-cdn-ports",
action="store_true",
help="Don't filter out open ports on CDNs/WAFs (by default only 80,443 are kept)",
help="Don't filter out open ports on CDNs/WAFs, or on hosts that passive port sources list with every port open (by default only 80,443 are kept)",
)
# deprecated: CDN/WAF port filtering is now the default
output.add_argument("--exclude-cdn", "-ec", action="store_true", help=argparse.SUPPRESS)
Expand Down
17 changes: 17 additions & 0 deletions bbot/scanner/scanner.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,7 @@ def __init__(
self.cdn_port_filter = bool(cdn_port_filter_config.get("enabled", True))
self.cdn_tags = set(cdn_port_filter_config.get("tags", []))
self.cdn_allowed_ports = set(cdn_port_filter_config.get("allowed_ports", []))
self.cdn_max_ports = int(cdn_port_filter_config.get("max_ports", 100) or 0)

# url querystring behavior
self.url_querystring_remove = self.config.get("url_querystring_remove", True)
Expand Down Expand Up @@ -1089,6 +1090,22 @@ def in_target(self, *args, **kwargs):
def blacklisted(self, *args, **kwargs):
return self.preset.blacklisted(*args, **kwargs)

def filter_catch_all_ports(self, host, ports):
"""Return a host's unique ports, cut to the allowed and targeted ones if there are more than cdn_port_filter.max_ports (it accepts connections on any port)"""
ports = list(dict.fromkeys(ports))
if not self.cdn_port_filter or not self.cdn_max_ports or len(ports) <= self.cdn_max_ports:
return ports
host = str(host)
# like the rest of cdn_port_filter, a port explicitly targeted on this host is never cut
targeted = {
seed.port for seed in self.seeds.get(host, single=False) or () if seed.port and str(seed.host) == host
}
kept = [port for port in ports if port in self.cdn_allowed_ports or port in targeted]
self.verbose(
f"{host} lists {len(ports):,} open ports, more than cdn_port_filter.max_ports ({self.cdn_max_ports}): keeping only {kept}"
)
return kept

@property
def core(self):
return self.preset.core
Expand Down
30 changes: 30 additions & 0 deletions bbot/test/test_step_1/test_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,36 @@ async def run(*targets, preset=None, config=None):
assert tagged == set()


@pytest.mark.asyncio
async def test_cdn_port_filter_catch_all(bbot_scanner):
"""A host listing more than cdn_port_filter.max_ports open ports keeps only the allowed ports, unless the filter or the cap is off"""
ports = [22, 80, 443, 8080, 8443, 80]
unique = [22, 80, 443, 8080, 8443]

def filtered(cdn_port_filter=None):
scan = bbot_scanner("1.2.3.4", config={"cdn_port_filter": cdn_port_filter or {}})
return scan.filter_catch_all_ports("1.2.3.4", ports)

# at the cap, every unique port is kept, in order
assert filtered({"max_ports": 5}) == unique
# over the cap, only the allowed ports survive
assert filtered({"max_ports": 4}) == [80, 443]
assert filtered({"max_ports": 4, "allowed_ports": [443, 8443]}) == [443, 8443]
# max_ports 0 turns off only this check, and disabling the filter (--allow-cdn-ports) turns it off too
assert filtered({"max_ports": 0}) == unique
assert filtered({"max_ports": 4, "enabled": False}) == unique

# a port explicitly targeted on the host is kept too, but not one targeted on another host
scan = bbot_scanner("1.2.3.4", "1.2.3.4:8443", "5.6.7.8:8080", config={"cdn_port_filter": {"max_ports": 4}})
assert scan.filter_catch_all_ports("1.2.3.4", ports) == [80, 443, 8443]

# the default cap is 100
scan = bbot_scanner("1.2.3.4")
hundred = list(range(1, 100)) + [443]
assert scan.filter_catch_all_ports("1.2.3.4", hundred) == hundred
assert scan.filter_catch_all_ports("1.2.3.4", hundred + [8443]) == [80, 443]


@pytest.mark.asyncio
async def test_scan_event_started_at_type(bbot_scanner):
"""Regression test: started_at must be a float on both RUNNING and FINISHED SCAN events."""
Expand Down
55 changes: 55 additions & 0 deletions bbot/test/test_step_2/module_tests/test_module_censys_ip.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,61 @@ def check(self, module_test, events):
)


class TestCensys_IP_CatchAllPorts(ModuleTestBase):
"""An IP listing more TCP ports than cdn_port_filter.max_ports keeps only the allowed ones, and drops the URLs and protocols on the others"""

module_name = "censys_ip"
targets = ["1.2.3.4"]
config_overrides = {
"modules": {"censys_ip": {"api_key": "api_id:api_secret"}},
"cdn_port_filter": {"max_ports": 3},
}

async def setup_before_prep(self, module_test):
module_test.blasthttp_mock.add_response(
url="https://search.censys.io/api/v1/account",
match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
json={"quota": {"used": 26, "allowance": 250, "resets_at": "1919-06-03 16:30:32"}},
)
services = [
{"port": port, "service_name": "HTTP", "transport_protocol": "TCP", "http": {"request": {"uri": uri}}}
for port, uri in (
(80, "http://1.2.3.4/"),
(443, "https://1.2.3.4/"),
(8080, "http://1.2.3.4:8080/"),
(8443, "https://1.2.3.4:8443/"),
)
]
services.append({"port": 2222, "service_name": "SSH", "transport_protocol": "TCP"})
# a missing transport counts as TCP
services.append({"port": 8888, "service_name": "UNKNOWN", "transport_protocol": None})
services.append({"port": 53, "service_name": "DNS", "transport_protocol": "UDP"})
module_test.blasthttp_mock.add_response(
url="https://search.censys.io/api/v2/hosts/1.2.3.4",
match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
json={"code": 200, "status": "OK", "result": {"ip": "1.2.3.4", "services": services}},
)

def check(self, module_test, events):
def from_module(event_type):
return [e for e in events if e.type == event_type and str(e.module) == self.module_name]

assert {e.port for e in from_module("OPEN_TCP_PORT")} == {80, 443}
assert {e.port for e in from_module("OPEN_UDP_PORT")} == {53}
assert {e.url for e in from_module("URL_UNVERIFIED")} == {"http://1.2.3.4/", "https://1.2.3.4/"}
assert {(e.data["protocol"], e.data.get("port")) for e in from_module("PROTOCOL")} == {("DNS", 53)}


class TestCensys_IP_CatchAllTargetedPort(TestCensys_IP_CatchAllPorts):
"""A port explicitly targeted on a catch-all host keeps its URL"""

targets = ["1.2.3.4", "1.2.3.4:8443"]

def check(self, module_test, events):
urls = {e.url for e in events if e.type == "URL_UNVERIFIED" and str(e.module) == self.module_name}
assert urls == {"http://1.2.3.4/", "https://1.2.3.4/", "https://1.2.3.4:8443/"}


class TestCensys_IP_InScopeOnly(ModuleTestBase):
"""Test that in_scope_only=True (default) does NOT query out-of-scope IPs."""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,43 @@ def check(self, module_test, events):
}


class TestShodan_Enterprise_CatchAllPorts(ModuleTestBase):
"""An IP listing more TCP ports than cdn_port_filter.max_ports keeps only the allowed ones; its UDP ports and findings are kept"""

module_name = "shodan_enterprise"
targets = ["1.2.3.4", "5.6.7.8"]
config_overrides = {
"modules": {"shodan_enterprise": {"api_key": "deadbeef"}},
"cdn_port_filter": {"max_ports": 3},
}

async def setup_before_prep(self, module_test):
for ip, tcp_ports in (("1.2.3.4", [22, 80, 443, 8080]), ("5.6.7.8", [22, 80, 443])):
data = [{"ip_str": ip, "port": port, "transport": "tcp"} for port in tcp_ports]
data.append({"ip_str": ip, "port": 53, "transport": "udp"})
data[0]["vulns"] = {"CVE-2021-12345": {"cvss": 7.5}}
module_test.blasthttp_mock.add_response(
url=f"https://api.shodan.io/shodan/host/{ip}?key=deadbeef", json={"data": data}
)

def check(self, module_test, events):
def ports(event_type):
return {
(str(e.host), e.port) for e in events if e.type == event_type and str(e.module) == self.module_name
}

assert ports("OPEN_TCP_PORT") == {
("1.2.3.4", 80),
("1.2.3.4", 443),
("5.6.7.8", 22),
("5.6.7.8", 80),
("5.6.7.8", 443),
}
assert ports("OPEN_UDP_PORT") == {("1.2.3.4", 53), ("5.6.7.8", 53)}
# the finding on port 22 is still reported where the port itself was dropped
assert {e.data["host"] for e in events if e.type == "FINDING"} == {"1.2.3.4", "5.6.7.8"}


class TestShodan_Enterprise_InScopeOnly(ModuleTestBase):
"""Test that in_scope_only=True (default) does NOT query out-of-scope IPs."""

Expand Down
26 changes: 26 additions & 0 deletions bbot/test/test_step_2/module_tests/test_module_shodan_idb.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,32 @@ def check(self, module_test, events):
assert module_test.cvedb_request_counts == {"CVE-2021-26857": 1, "CVE-2021-26855": 1}


class TestShodan_IDB_CatchAllPorts(ModuleTestBase):
"""An IP listed with more open ports than cdn_port_filter.max_ports keeps only the allowed ports; one under it keeps them all"""

module_name = "shodan_idb"
targets = ["1.2.3.4", "5.6.7.8"]
config_overrides = {"dns": {"minimal": False}}

async def setup_after_prep(self, module_test):
for ip, ports in (("1.2.3.4", list(range(1, 2001))), ("5.6.7.8", [22, 80, 443, 8080])):
module_test.blasthttp_mock.add_response(
url=f"https://internetdb.shodan.io/{ip}",
json={"cpes": [], "hostnames": [], "ip": ip, "ports": ports, "tags": [], "vulns": []},
)

def check(self, module_test, events):
ports = {(str(e.host), e.port) for e in events if e.type == "OPEN_TCP_PORT" and str(e.module) == "shodan_idb"}
assert ports == {
("1.2.3.4", 80),
("1.2.3.4", 443),
("5.6.7.8", 22),
("5.6.7.8", 80),
("5.6.7.8", 443),
("5.6.7.8", 8080),
}


class TestShodan_IDB_CrossHostCache(ModuleTestBase):
"""Same CVE on two different hosts must be fetched from CVEDB only once."""

Expand Down
4 changes: 4 additions & 0 deletions docs/scanning/tips_and_tricks.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ CDNs and WAFs like Cloudflare answer on many ports (e.g. 2052-2096, 8080, 8443)

Ports you explicitly target are never filtered. For example, `-t www.evilcorp.com:8443` keeps that port and anything discovered on it, even if the host is behind a CDN.

The same filter covers catch-all hosts, which accept a connection on any port, like some cloud load balancers and tarpits. Passive port sources can list hundreds or thousands of open ports for these, so when `shodan_idb`, `shodan_enterprise` or `censys_ip` lists more than `cdn_port_filter.max_ports` (100 by default) open TCP ports for a host, only the allowed ports and any ports you explicitly targeted are kept. The cap doesn't apply to `portscan`. Set `max_ports` to `0` to turn off just this check.

To turn the filter off and keep every port:

```bash
Expand Down Expand Up @@ -133,6 +135,8 @@ config:
- 80
- 443
- 8443
# hosts that passive port sources list with more open ports than this keep only allowed_ports (0 = off)
max_ports: 100
```

```bash
Expand Down
Loading