feat(stats): add migration v7 visit table and client-ip helpers
This commit is contained in:
1 parent
b0c53e8a2f
commit
cdd11823b9
8 files changed
+531
-15
No files matched your search
@@ -0,0 +1,221 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""来源 IP 提取与访问统计用纯函数。
|
||||||
|
|
||||||
|
服务经``域名 → CDN → 公网 EIP → 内网 → 宿主机 nginx → 容器``链路访问,容器内
|
||||||
|
socket 对端是 Docker 网关而非真实客户端,因此真实 IP 只能来自 nginx 注入的
|
||||||
|
请求头。安全前提:
|
||||||
|
|
||||||
|
* 只有当 socket 对端(``peer``)属于可信代理网段时才采信请求头;
|
||||||
|
* nginx 用 ``realip`` 模块按 CDN 网段校验后写入 ``X-Real-IP``(见文档);
|
||||||
|
* 直接解析到公网 EIP 的请求没有 CDN 头,回退到 ``peer``。
|
||||||
|
|
||||||
|
本模块不依赖网络、文件或时钟,便于离线单元测试。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
|
import re
|
||||||
|
from typing import Iterable, Mapping, Sequence
|
||||||
|
|
||||||
|
# 默认可信代理网段:回环、RFC1918、链路本地、IPv6 ULA/链路本地。
|
||||||
|
# Docker 默认桥接网段在 172.16.0.0/12 内,容器内看到的对端即网关地址。
|
||||||
|
DEFAULT_TRUSTED_PROXIES: tuple[str, ...] = (
|
||||||
|
"127.0.0.0/8",
|
||||||
|
"::1/128",
|
||||||
|
"10.0.0.0/8",
|
||||||
|
"172.16.0.0/12",
|
||||||
|
"192.168.0.0/16",
|
||||||
|
"169.254.0.0/16",
|
||||||
|
"fc00::/7",
|
||||||
|
"fe80::/10",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 依次尝试的请求头:nginx 归一化后的 X-Real-IP 优先,X-Forwarded-For 兜底。
|
||||||
|
DEFAULT_CLIENT_IP_HEADERS: tuple[str, ...] = ("X-Real-IP", "X-Forwarded-For")
|
||||||
|
|
||||||
|
# 已知前端路由:未知路径统一归为 /other,避免卡片式路径污染统计。
|
||||||
|
KNOWN_ROUTES: frozenset[str] = frozenset({
|
||||||
|
"/home", "/videos", "/ups", "/fans", "/about",
|
||||||
|
"/candidates", "/system", "/rank", "/login", "/logout",
|
||||||
|
})
|
||||||
|
|
||||||
|
_VIDEO_ROUTE_RE = re.compile(r"^/video/\d+$")
|
||||||
|
_UP_ROUTE_RE = re.compile(r"^/up/\d+$")
|
||||||
|
_MAX_ROUTE_LENGTH = 200
|
||||||
|
|
||||||
|
# 常见爬虫 / 监控 UA 关键词;命中时视为机器人(是否计数由调用方决定)。
|
||||||
|
# 这些标记在 UA 中通常作为子串出现(如 Googlebot / bingbot / Baiduspider),
|
||||||
|
# 因此不使用词边界,避免漏判。
|
||||||
|
_BOT_PATTERN = re.compile(
|
||||||
|
r"(?i)("
|
||||||
|
r"bot|crawler|spider|crawl|slurp|bingpreview|facebookexternalhit|"
|
||||||
|
r"python-requests|python-urllib|httpx|aiohttp|curl|wget|libwww|"
|
||||||
|
r"monitor|uptime|healthcheck|headlesschrome|phantomjs|"
|
||||||
|
r"semrush|ahrefs|dotbot|yandex|baiduspider|sogou|bytespider"
|
||||||
|
r")"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_ip(value: object) -> str | None:
|
||||||
|
"""规范化 IP 字符串;非法或空值返回 ``None``。
|
||||||
|
|
||||||
|
兼容 ``[::1]:443``、``1.2.3.4:5678`` 带端口写法,并把 IPv4-mapped IPv6
|
||||||
|
折叠为 IPv4(``::ffff:1.2.3.4`` → ``1.2.3.4``),保证同一客户端去重一致。
|
||||||
|
"""
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return None
|
||||||
|
text = value.strip()
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
if text.startswith("["):
|
||||||
|
end = text.find("]")
|
||||||
|
if end == -1:
|
||||||
|
return None
|
||||||
|
text = text[1:end]
|
||||||
|
ip = _parse_ip(text)
|
||||||
|
if ip is None:
|
||||||
|
# host:port(IPv6 已在上面的直接解析里成功,不会走到这里)
|
||||||
|
host, sep, port = text.rpartition(":")
|
||||||
|
if not sep or not port.isdigit() or not host:
|
||||||
|
return None
|
||||||
|
ip = _parse_ip(host)
|
||||||
|
if ip is None:
|
||||||
|
return None
|
||||||
|
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
|
||||||
|
ip = ip.ipv4_mapped
|
||||||
|
return ip.compressed
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_ip(text: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||||
|
try:
|
||||||
|
return ipaddress.ip_address(text)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_networks(values: Iterable[str]) -> tuple[ipaddress.IPv4Network | ipaddress.IPv6Network, ...]:
|
||||||
|
"""把 CIDR / 裸 IP 字符串解析为网段元组;非法项忽略。"""
|
||||||
|
networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = []
|
||||||
|
for raw in values:
|
||||||
|
if not isinstance(raw, str) or not raw.strip():
|
||||||
|
continue
|
||||||
|
text = raw.strip()
|
||||||
|
try:
|
||||||
|
networks.append(ipaddress.ip_network(text, strict=False))
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return tuple(networks)
|
||||||
|
|
||||||
|
|
||||||
|
def is_trusted_proxy(
|
||||||
|
peer_ip: str | None,
|
||||||
|
networks: Sequence[ipaddress.IPv4Network | ipaddress.IPv6Network],
|
||||||
|
) -> bool:
|
||||||
|
ip = _parse_ip(peer_ip) if peer_ip else None
|
||||||
|
if ip is None:
|
||||||
|
return False
|
||||||
|
for network in networks:
|
||||||
|
if ip.version != network.version:
|
||||||
|
continue
|
||||||
|
if ip in network:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _header_value(headers: Mapping[str, object], name: str) -> str | None:
|
||||||
|
"""大小写不敏感地读取请求头;兼容单值与列表两种取值。"""
|
||||||
|
plain = headers.get(name)
|
||||||
|
if plain is None:
|
||||||
|
lowered = name.lower()
|
||||||
|
for key, value in headers.items():
|
||||||
|
if isinstance(key, str) and key.lower() == lowered:
|
||||||
|
plain = value
|
||||||
|
break
|
||||||
|
if plain is None:
|
||||||
|
return None
|
||||||
|
if isinstance(plain, (list, tuple)):
|
||||||
|
return ",".join(str(item) for item in plain)
|
||||||
|
return str(plain)
|
||||||
|
|
||||||
|
|
||||||
|
def _rightmost_valid_forwarded(value: str) -> str | None:
|
||||||
|
"""X-Forwarded-For 取最右侧合法 IP。
|
||||||
|
|
||||||
|
nginx 的 ``$proxy_add_x_forwarded_for`` 会把真实对端追加到最右侧,客户端
|
||||||
|
自带的伪造条目在左侧,因此最右侧最可信。
|
||||||
|
"""
|
||||||
|
for part in reversed(value.split(",")):
|
||||||
|
normalized = normalize_ip(part)
|
||||||
|
if normalized is not None:
|
||||||
|
return normalized
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def extract_client_ip(
|
||||||
|
headers: Mapping[str, object],
|
||||||
|
peer_ip: str | None,
|
||||||
|
*,
|
||||||
|
trusted_proxies: Sequence[ipaddress.IPv4Network | ipaddress.IPv6Network],
|
||||||
|
header_order: Sequence[str] = DEFAULT_CLIENT_IP_HEADERS,
|
||||||
|
) -> str:
|
||||||
|
"""返回用于统计的客户端 IP。
|
||||||
|
|
||||||
|
* 对端不可信(或无法解析)时忽略所有请求头,直接回退到对端;
|
||||||
|
* 对端可信时按 ``header_order`` 依次取第一个合法值;
|
||||||
|
* 全部缺失时回退到规范化后的对端;对端也无法解析时返回 ``""``。
|
||||||
|
"""
|
||||||
|
peer = normalize_ip(peer_ip)
|
||||||
|
if peer is None:
|
||||||
|
return ""
|
||||||
|
if not is_trusted_proxy(peer, trusted_proxies):
|
||||||
|
return peer
|
||||||
|
|
||||||
|
for name in header_order:
|
||||||
|
value = _header_value(headers, name)
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
if name.lower() == "x-forwarded-for":
|
||||||
|
candidate = _rightmost_valid_forwarded(value)
|
||||||
|
else:
|
||||||
|
candidate = normalize_ip(value)
|
||||||
|
if candidate is not None:
|
||||||
|
return candidate
|
||||||
|
return peer
|
||||||
|
|
||||||
|
|
||||||
|
def is_bot(user_agent: object) -> bool:
|
||||||
|
"""UA 命中已知机器人/监控模式时返回 ``True``;空 UA 不算机器人。"""
|
||||||
|
if not isinstance(user_agent, str) or not user_agent.strip():
|
||||||
|
return False
|
||||||
|
return bool(_BOT_PATTERN.search(user_agent))
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_route(path: object) -> str:
|
||||||
|
"""把上报路径归一化为有限集合中的路由标签。
|
||||||
|
|
||||||
|
``/`` → ``/home``;``/video/123`` → ``/video/:aid``;``/up/123`` →
|
||||||
|
``/up/:mid``;已知路由原样保留;其余(含空值、超长、带查询串)→ ``/other``。
|
||||||
|
"""
|
||||||
|
if not isinstance(path, str):
|
||||||
|
return "/other"
|
||||||
|
text = path.strip()
|
||||||
|
if not text:
|
||||||
|
return "/other"
|
||||||
|
# 丢弃查询串与片段,并折叠末尾斜杠
|
||||||
|
text = text.split("?", 1)[0].split("#", 1)[0]
|
||||||
|
if not text.startswith("/"):
|
||||||
|
return "/other"
|
||||||
|
if len(text) > _MAX_ROUTE_LENGTH:
|
||||||
|
return "/other"
|
||||||
|
if text == "/" or text == "":
|
||||||
|
return "/home"
|
||||||
|
if text != "/" and text.endswith("/"):
|
||||||
|
text = text.rstrip("/") or "/home"
|
||||||
|
if _VIDEO_ROUTE_RE.match(text):
|
||||||
|
return "/video/:aid"
|
||||||
|
if _UP_ROUTE_RE.match(text):
|
||||||
|
return "/up/:mid"
|
||||||
|
if text in KNOWN_ROUTES:
|
||||||
|
return text
|
||||||
|
return "/other"
|
||||||
@@ -224,6 +224,26 @@ MIGRATIONS: tuple[Migration, ...] = (
|
|||||||
"ALTER TABLE video ADD COLUMN up_face TEXT",
|
"ALTER TABLE video ADD COLUMN up_face TEXT",
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
Migration(
|
||||||
|
version=7,
|
||||||
|
name="visit access statistics",
|
||||||
|
statements=(
|
||||||
|
# 访问统计按 (日期, IP, 路由) 聚合:PV = SUM(hits),UV = COUNT(DISTINCT ip)。
|
||||||
|
# 数据先落内存缓冲、批量 upsert,因此 hits 累加而非逐条插入。
|
||||||
|
"""
|
||||||
|
CREATE TABLE visit (
|
||||||
|
stat_date TEXT NOT NULL,
|
||||||
|
ip TEXT NOT NULL,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
hits INTEGER NOT NULL DEFAULT 0,
|
||||||
|
first_at INTEGER NOT NULL,
|
||||||
|
last_at INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (stat_date, ip, path)
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
"CREATE INDEX idx_visit_date ON visit(stat_date)",
|
||||||
|
),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -51,7 +51,7 @@ def test_init_and_status():
|
|||||||
path = write_config(tmp)
|
path = write_config(tmp)
|
||||||
code, output = run_cli(["init", "--config", str(path)])
|
code, output = run_cli(["init", "--config", str(path)])
|
||||||
assert code == 0
|
assert code == 0
|
||||||
assert '"schema_version": 6' in output
|
assert '"schema_version": 7' in output
|
||||||
|
|
||||||
db = Database(Path(tmp) / "app.db")
|
db = Database(Path(tmp) / "app.db")
|
||||||
db.initialize()
|
db.initialize()
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""来源 IP 提取与路由/机器人判定纯函数测试(离线)。
|
||||||
|
|
||||||
|
运行:uv run python tests/test_client_ip.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
sys.stdout.reconfigure(encoding="utf-8")
|
||||||
|
|
||||||
|
from reaction_board.client_ip import ( # noqa: E402
|
||||||
|
DEFAULT_TRUSTED_PROXIES,
|
||||||
|
extract_client_ip,
|
||||||
|
is_bot,
|
||||||
|
is_trusted_proxy,
|
||||||
|
normalize_ip,
|
||||||
|
normalize_route,
|
||||||
|
parse_networks,
|
||||||
|
)
|
||||||
|
|
||||||
|
TRUSTED = parse_networks(DEFAULT_TRUSTED_PROXIES)
|
||||||
|
# 容器内看到的 Docker 网关地址,属于可信代理网段。
|
||||||
|
DOCKER_GATEWAY = "172.17.0.1"
|
||||||
|
LOOPBACK = "127.0.0.1"
|
||||||
|
PUBLIC_CLIENT = "203.0.113.7"
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_ip():
|
||||||
|
assert normalize_ip("203.0.113.7") == "203.0.113.7"
|
||||||
|
assert normalize_ip(" 203.0.113.7 ") == "203.0.113.7"
|
||||||
|
assert normalize_ip("::ffff:203.0.113.7") == "203.0.113.7"
|
||||||
|
assert normalize_ip("2001:db8::1") == "2001:db8::1"
|
||||||
|
assert normalize_ip("2001:0db8:0000:0000:0000:0000:0000:0001") == "2001:db8::1"
|
||||||
|
# 带端口 / 方括号
|
||||||
|
assert normalize_ip("203.0.113.7:5678") == "203.0.113.7"
|
||||||
|
assert normalize_ip("[2001:db8::1]:443") == "2001:db8::1"
|
||||||
|
# 空值、非法、非字符串
|
||||||
|
assert normalize_ip("") is None
|
||||||
|
assert normalize_ip(" ") is None
|
||||||
|
assert normalize_ip("not-an-ip") is None
|
||||||
|
assert normalize_ip("203.0.113.7:abc") is None
|
||||||
|
assert normalize_ip("[2001:db8::1") is None
|
||||||
|
assert normalize_ip(None) is None
|
||||||
|
assert normalize_ip(123) is None
|
||||||
|
assert normalize_ip("999.1.1.1") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_trusted_proxy():
|
||||||
|
assert is_trusted_proxy(DOCKER_GATEWAY, TRUSTED)
|
||||||
|
assert is_trusted_proxy(LOOPBACK, TRUSTED)
|
||||||
|
assert is_trusted_proxy("10.10.8.5", TRUSTED)
|
||||||
|
assert is_trusted_proxy("::1", TRUSTED)
|
||||||
|
assert not is_trusted_proxy(PUBLIC_CLIENT, TRUSTED)
|
||||||
|
assert not is_trusted_proxy("2001:db8::1", TRUSTED)
|
||||||
|
assert not is_trusted_proxy(None, TRUSTED)
|
||||||
|
assert not is_trusted_proxy("garbage", TRUSTED)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_direct_public_eip():
|
||||||
|
"""场景一:域名直接解析公网 EIP,无 CDN 头,取对端。"""
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{"User-Agent": "Mozilla/5.0"},
|
||||||
|
"198.51.100.9", # 直连客户端被 nginx 记为 $remote_addr,但 app 看到的是可信对端
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
header_order=("X-Real-IP", "X-Forwarded-For"),
|
||||||
|
)
|
||||||
|
# 对端是可信 nginx,X-Real-IP 缺失 → 回退对端
|
||||||
|
assert ip == "198.51.100.9"
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_direct_uses_x_real_ip_from_nginx():
|
||||||
|
"""nginx 设置 X-Real-IP,app 采信之。"""
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{"X-Real-IP": PUBLIC_CLIENT},
|
||||||
|
DOCKER_GATEWAY,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
)
|
||||||
|
assert ip == PUBLIC_CLIENT
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_cloudflare():
|
||||||
|
"""场景二:Cloudflare 中转,nginx realip 归一化到 X-Real-IP。"""
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{
|
||||||
|
"X-Real-IP": PUBLIC_CLIENT,
|
||||||
|
"CF-Connecting-IP": PUBLIC_CLIENT,
|
||||||
|
"X-Forwarded-For": f"{PUBLIC_CLIENT}, 172.68.1.1",
|
||||||
|
},
|
||||||
|
DOCKER_GATEWAY,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
)
|
||||||
|
assert ip == PUBLIC_CLIENT
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_cloudflare_via_cf_header_only():
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{"CF-Connecting-IP": PUBLIC_CLIENT},
|
||||||
|
DOCKER_GATEWAY,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
header_order=("CF-Connecting-IP", "X-Real-IP", "X-Forwarded-For"),
|
||||||
|
)
|
||||||
|
assert ip == PUBLIC_CLIENT
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_edgeone():
|
||||||
|
"""场景三:EdgeOne 中转,X-Real-IP 由 nginx 归一化给出。"""
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{
|
||||||
|
"X-Real-IP": "198.51.100.23",
|
||||||
|
"EO-Connecting-IP": "198.51.100.23",
|
||||||
|
"X-Forwarded-For": "198.51.100.23, 43.152.0.9, 43.152.0.10",
|
||||||
|
},
|
||||||
|
DOCKER_GATEWAY,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
)
|
||||||
|
assert ip == "198.51.100.23"
|
||||||
|
|
||||||
|
|
||||||
|
def test_forwarded_for_takes_rightmost():
|
||||||
|
"""X-Forwarded-For 取最右侧(nginx 追加的真实对端),忽略左侧伪造值。"""
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{"X-Forwarded-For": "1.2.3.4, 5.6.7.8, 203.0.113.7"},
|
||||||
|
DOCKER_GATEWAY,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
header_order=("X-Forwarded-For",),
|
||||||
|
)
|
||||||
|
assert ip == "203.0.113.7"
|
||||||
|
|
||||||
|
|
||||||
|
def test_forwarded_for_skips_invalid_rightmost():
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{"X-Forwarded-For": "203.0.113.7, not-an-ip, "},
|
||||||
|
DOCKER_GATEWAY,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
header_order=("X-Forwarded-For",),
|
||||||
|
)
|
||||||
|
assert ip == "203.0.113.7"
|
||||||
|
|
||||||
|
|
||||||
|
def test_untrusted_peer_ignores_spoofed_headers():
|
||||||
|
"""不可信对端(直连容器)伪造 X-Real-IP/XFF 必须被忽略。"""
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{
|
||||||
|
"X-Real-IP": "1.1.1.1",
|
||||||
|
"X-Forwarded-For": "2.2.2.2, 3.3.3.3",
|
||||||
|
"CF-Connecting-IP": "4.4.4.4",
|
||||||
|
},
|
||||||
|
PUBLIC_CLIENT,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
header_order=("CF-Connecting-IP", "X-Real-IP", "X-Forwarded-For"),
|
||||||
|
)
|
||||||
|
assert ip == PUBLIC_CLIENT
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_fallbacks_and_edges():
|
||||||
|
# 可信对端但无任何头 → 对端
|
||||||
|
assert extract_client_ip({}, LOOPBACK, trusted_proxies=TRUSTED) == LOOPBACK
|
||||||
|
# 头部为空值 → 回退
|
||||||
|
assert extract_client_ip(
|
||||||
|
{"X-Real-IP": " "}, DOCKER_GATEWAY, trusted_proxies=TRUSTED
|
||||||
|
) == DOCKER_GATEWAY
|
||||||
|
# 头部非法 → 尝试下一个头
|
||||||
|
assert extract_client_ip(
|
||||||
|
{"X-Real-IP": "bad", "X-Forwarded-For": PUBLIC_CLIENT},
|
||||||
|
DOCKER_GATEWAY,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
) == PUBLIC_CLIENT
|
||||||
|
# 对端不可解析 → 空串
|
||||||
|
assert extract_client_ip(
|
||||||
|
{"X-Real-IP": PUBLIC_CLIENT}, "garbage", trusted_proxies=TRUSTED
|
||||||
|
) == ""
|
||||||
|
# 对端缺失 → 空串
|
||||||
|
assert extract_client_ip({"X-Real-IP": PUBLIC_CLIENT}, None, trusted_proxies=TRUSTED) == ""
|
||||||
|
# 大小写不敏感的请求头查找
|
||||||
|
assert extract_client_ip(
|
||||||
|
{"x-real-ip": PUBLIC_CLIENT}, DOCKER_GATEWAY, trusted_proxies=TRUSTED,
|
||||||
|
) == PUBLIC_CLIENT
|
||||||
|
# 多值请求头(http.server 可能给 list)
|
||||||
|
assert extract_client_ip(
|
||||||
|
{"X-Real-IP": [PUBLIC_CLIENT]}, DOCKER_GATEWAY, trusted_proxies=TRUSTED,
|
||||||
|
) == PUBLIC_CLIENT
|
||||||
|
# 空的 header_order → 回退对端
|
||||||
|
assert extract_client_ip(
|
||||||
|
{"X-Real-IP": PUBLIC_CLIENT}, DOCKER_GATEWAY,
|
||||||
|
trusted_proxies=TRUSTED, header_order=(),
|
||||||
|
) == DOCKER_GATEWAY
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_ipv6_client():
|
||||||
|
ip = extract_client_ip(
|
||||||
|
{"X-Real-IP": "2001:db8::beef"},
|
||||||
|
LOOPBACK,
|
||||||
|
trusted_proxies=TRUSTED,
|
||||||
|
)
|
||||||
|
assert ip == "2001:db8::beef"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_networks_tolerates_junk():
|
||||||
|
networks = parse_networks(["10.0.0.0/8", "bad", "", "192.168.1.1", 42])
|
||||||
|
assert len(networks) == 2
|
||||||
|
assert is_trusted_proxy("10.1.2.3", networks)
|
||||||
|
assert is_trusted_proxy("192.168.1.1", networks)
|
||||||
|
assert not is_trusted_proxy("11.0.0.1", networks)
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_bot():
|
||||||
|
assert is_bot("Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/155.0") is False
|
||||||
|
assert is_bot("") is False
|
||||||
|
assert is_bot(None) is False
|
||||||
|
assert is_bot("Googlebot/2.1 (+http://www.google.com/bot.html)") is True
|
||||||
|
assert is_bot("Mozilla/5.0 (compatible; bingbot/2.0)") is True
|
||||||
|
assert is_bot("python-requests/2.31") is True
|
||||||
|
assert is_bot("curl/8.4.0") is True
|
||||||
|
assert is_bot("Uptime-Kuma/1.23") is True
|
||||||
|
assert is_bot("Bytespider") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_route():
|
||||||
|
assert normalize_route("/") == "/home"
|
||||||
|
assert normalize_route("") == "/other"
|
||||||
|
assert normalize_route(None) == "/other"
|
||||||
|
assert normalize_route("/home") == "/home"
|
||||||
|
assert normalize_route("/videos") == "/videos"
|
||||||
|
assert normalize_route("/ups") == "/ups"
|
||||||
|
assert normalize_route("/fans") == "/fans"
|
||||||
|
assert normalize_route("/about") == "/about"
|
||||||
|
assert normalize_route("/candidates") == "/candidates"
|
||||||
|
assert normalize_route("/system") == "/system"
|
||||||
|
assert normalize_route("/rank") == "/rank"
|
||||||
|
# 带查询串 / 片段
|
||||||
|
assert normalize_route("/videos?sort=view&year=2026") == "/videos"
|
||||||
|
assert normalize_route("/video/114775988111033#top") == "/video/:aid"
|
||||||
|
# 末尾斜杠
|
||||||
|
assert normalize_route("/videos/") == "/videos"
|
||||||
|
# 动态路由折叠
|
||||||
|
assert normalize_route("/video/123") == "/video/:aid"
|
||||||
|
assert normalize_route("/up/230020975") == "/up/:mid"
|
||||||
|
# 非法 / 未知 / 超长
|
||||||
|
assert normalize_route("videos") == "/other"
|
||||||
|
assert normalize_route("/video/abc") == "/other"
|
||||||
|
assert normalize_route("/unknown") == "/other"
|
||||||
|
assert normalize_route("/" + "x" * 500) == "/other"
|
||||||
|
assert normalize_route(123) == "/other"
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
test_normalize_ip()
|
||||||
|
test_is_trusted_proxy()
|
||||||
|
test_extract_direct_public_eip()
|
||||||
|
test_extract_direct_uses_x_real_ip_from_nginx()
|
||||||
|
test_extract_cloudflare()
|
||||||
|
test_extract_cloudflare_via_cf_header_only()
|
||||||
|
test_extract_edgeone()
|
||||||
|
test_forwarded_for_takes_rightmost()
|
||||||
|
test_forwarded_for_skips_invalid_rightmost()
|
||||||
|
test_untrusted_peer_ignores_spoofed_headers()
|
||||||
|
test_extract_fallbacks_and_edges()
|
||||||
|
test_extract_ipv6_client()
|
||||||
|
test_parse_networks_tolerates_junk()
|
||||||
|
test_is_bot()
|
||||||
|
test_normalize_route()
|
||||||
|
print("OK: 来源 IP 提取与路由/机器人判定测试全部通过")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -24,7 +24,7 @@ def test_write_read_transaction():
|
|||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
db = make_db(tmp)
|
db = make_db(tmp)
|
||||||
try:
|
try:
|
||||||
assert db.schema_version == 6
|
assert db.schema_version == 7
|
||||||
db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')"))
|
db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')"))
|
||||||
assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1"
|
assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1"
|
||||||
|
|
||||||
@@ -94,8 +94,8 @@ def test_close_and_reinitialize():
|
|||||||
|
|
||||||
db2 = make_db(tmp)
|
db2 = make_db(tmp)
|
||||||
try:
|
try:
|
||||||
assert db2.schema_version == 6
|
assert db2.schema_version == 7
|
||||||
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 6
|
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 7
|
||||||
finally:
|
finally:
|
||||||
db2.close()
|
db2.close()
|
||||||
|
|
||||||
@@ -123,7 +123,7 @@ def test_migration_backup_on_existing_db():
|
|||||||
db = Database(path, backup_dir=Path(tmp) / "backup")
|
db = Database(path, backup_dir=Path(tmp) / "backup")
|
||||||
try:
|
try:
|
||||||
applied = db.initialize()
|
applied = db.initialize()
|
||||||
assert applied == [1, 2, 3, 4, 5, 6]
|
assert applied == [1, 2, 3, 4, 5, 6, 7]
|
||||||
assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1
|
assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from reaction_board.migrations import ( # noqa: E402
|
|||||||
EXPECTED_TABLES = {
|
EXPECTED_TABLES = {
|
||||||
"video", "snapshot", "folder", "folder_resource", "up_blacklist",
|
"video", "snapshot", "folder", "folder_resource", "up_blacklist",
|
||||||
"up_stat", "up_profile", "candidate", "seen_aid", "notify", "fav_action", "kv",
|
"up_stat", "up_profile", "candidate", "seen_aid", "notify", "fav_action", "kv",
|
||||||
|
"visit",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -37,8 +38,8 @@ def test_fresh_and_idempotent():
|
|||||||
conn = sqlite3.connect(Path(tmp) / "app.db")
|
conn = sqlite3.connect(Path(tmp) / "app.db")
|
||||||
try:
|
try:
|
||||||
applied = run_migrations(conn)
|
applied = run_migrations(conn)
|
||||||
assert applied == [1, 2, 3, 4, 5, 6]
|
assert applied == [1, 2, 3, 4, 5, 6, 7]
|
||||||
assert current_version(conn) == 6
|
assert current_version(conn) == 7
|
||||||
assert EXPECTED_TABLES <= table_names(conn)
|
assert EXPECTED_TABLES <= table_names(conn)
|
||||||
assert "schema_version" in table_names(conn)
|
assert "schema_version" in table_names(conn)
|
||||||
columns = {
|
columns = {
|
||||||
@@ -63,10 +64,16 @@ def test_fresh_and_idempotent():
|
|||||||
"mid", "follower_first", "following_first", "first_captured_at",
|
"mid", "follower_first", "following_first", "first_captured_at",
|
||||||
"follower_latest", "last_refreshed",
|
"follower_latest", "last_refreshed",
|
||||||
} <= profile_columns
|
} <= profile_columns
|
||||||
|
visit_columns = {
|
||||||
|
row[1] for row in conn.execute("PRAGMA table_info(visit)")
|
||||||
|
}
|
||||||
|
assert {
|
||||||
|
"stat_date", "ip", "path", "hits", "first_at", "last_at",
|
||||||
|
} <= visit_columns
|
||||||
# 再跑一次不应重复应用
|
# 再跑一次不应重复应用
|
||||||
assert run_migrations(conn) == []
|
assert run_migrations(conn) == []
|
||||||
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
|
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
|
||||||
assert row[0] == 6
|
assert row[0] == 7
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
@@ -81,15 +88,15 @@ def test_backup_before_upgrade():
|
|||||||
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
|
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
v7 = Migration(7, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
v8 = Migration(8, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
||||||
applied = run_migrations(
|
applied = run_migrations(
|
||||||
conn,
|
conn,
|
||||||
backup_dir=backup_dir,
|
backup_dir=backup_dir,
|
||||||
now=1_800_000_000,
|
now=1_800_000_000,
|
||||||
migrations=MIGRATIONS + (v7,),
|
migrations=MIGRATIONS + (v8,),
|
||||||
)
|
)
|
||||||
assert applied == [7]
|
assert applied == [8]
|
||||||
backups = list(backup_dir.glob("app-pre-v6-*.db"))
|
backups = list(backup_dir.glob("app-pre-v7-*.db"))
|
||||||
assert len(backups) == 1
|
assert len(backups) == 1
|
||||||
check = sqlite3.connect(backups[0])
|
check = sqlite3.connect(backups[0])
|
||||||
try:
|
try:
|
||||||
@@ -174,7 +181,7 @@ def test_v5_normalizes_unfaved():
|
|||||||
)
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
applied = run_migrations(conn)
|
applied = run_migrations(conn)
|
||||||
assert applied == [5, 6]
|
assert applied == [5, 6, 7]
|
||||||
rows = dict(conn.execute("SELECT aid, status FROM video").fetchall())
|
rows = dict(conn.execute("SELECT aid, status FROM video").fetchall())
|
||||||
assert rows == {1: "active", 2: "dropped"}
|
assert rows == {1: "active", 2: "dropped"}
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
@@ -459,7 +459,7 @@ def test_about_payload():
|
|||||||
assert about["snapshot_days"] == 3
|
assert about["snapshot_days"] == 3
|
||||||
assert about["snapshot_first_date"] == DATES[0]
|
assert about["snapshot_first_date"] == DATES[0]
|
||||||
assert about["fans_start_date"] == DATES[0]
|
assert about["fans_start_date"] == DATES[0]
|
||||||
assert about["schema_version"] == 6
|
assert about["schema_version"] == 7
|
||||||
assert about["analytics"] == {
|
assert about["analytics"] == {
|
||||||
"min_duration_seconds": 600,
|
"min_duration_seconds": 600,
|
||||||
"maturity_days": 2,
|
"maturity_days": 2,
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ def test_readonly_routes_and_static():
|
|||||||
assert status == 200 and payload["ok"] is True
|
assert status == 200 and payload["ok"] is True
|
||||||
status, _, payload = running.json("/api/about")
|
status, _, payload = running.json("/api/about")
|
||||||
assert status == 200
|
assert status == 200
|
||||||
assert payload["data"]["schema_version"] == 6
|
assert payload["data"]["schema_version"] == 7
|
||||||
assert payload["data"]["analytics"]["min_duration_seconds"] == 600
|
assert payload["data"]["analytics"]["min_duration_seconds"] == 600
|
||||||
|
|
||||||
status, _, overview = running.json("/api/overview")
|
status, _, overview = running.json("/api/overview")
|
||||||
|
|||||||
Reference in new issue
Block a user