feat(web): add read-only dashboard server and query layer

This commit is contained in:
kairee committed 2026-10-31 23:13:54 +08:00
1 parent e7cae035e0
commit 19572171b8
10 files changed
+2603 -10

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
# -*- coding: utf-8 -*-
"""离线测试与共享夹具。"""
+29
View File
@@ -184,6 +184,34 @@ def test_run_once_offline():
assert '"name": "snapshot"' in output
assert selected and all(credentials.mid == 1 for credentials in selected)
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
for name in ("folder_sync", "snapshot", "discovery", "candidate_cleanup"):
assert repo.kv_get(f"task:last:{name}"), name
finally:
db.close()
def test_default_command_starts_web_and_scheduler():
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp)
calls = []
original = cli.cmd_run
def fake_run(args, logger, *, serve_web=False):
calls.append((args.command, serve_web, args.once, args.allow_empty_folder))
return 0
cli.cmd_run = fake_run
try:
code = main(["--config", str(path)])
finally:
cli.cmd_run = original
assert code == 0
assert calls == [("run", True, False, False)]
def test_snapshot_backup_failure_returns_error_and_retries():
import reaction_board.services.snapshot as snapshot_module
@@ -271,6 +299,7 @@ def main_test():
test_auth_check_without_cookie()
test_auth_check_all_and_auth_dispatch()
test_run_once_offline()
test_default_command_starts_web_and_scheduler()
test_snapshot_backup_failure_returns_error_and_retries()
test_candidates_and_discover_commands()
print("OK: CLI init/sync/snapshot/auth-check/run/status/错误路径测试全部通过")
+242
View File
@@ -0,0 +1,242 @@
# -*- coding: utf-8 -*-
"""只读查询层测试(离线): uv run python tests/test_web_queries.py"""
import os
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.stdout.reconfigure(encoding="utf-8")
from reaction_board.config import ConfigStore # noqa: E402
from reaction_board.database import Database # noqa: E402
from reaction_board.web.queries import DashboardQueries # noqa: E402
from tests.web_fixtures import DATES, build_db, write_config # noqa: E402
FIXED_NOW = 1_793_459_000 # 2026-10-31 前后
def setup(tmp: str):
build_db(tmp)
config = write_config(tmp)
db = Database(Path(tmp) / "app.db")
db.initialize()
store = ConfigStore(config)
queries = DashboardQueries(db, store, clock=lambda: FIXED_NOW)
return db, queries
def test_overview():
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
overview = q.overview()
assert overview["data_date"] == DATES[-1]
assert overview["snapshot_days"] == 3
assert overview["is_first_day"] is False
assert overview["counts"]["active"] == 2
assert overview["counts"]["unfaved"] == 1
assert overview["counts"]["removed"] == 1
assert overview["counts"]["active_ups"] == 2
assert overview["counts"]["candidates_pending"] == 1
assert overview["totals"]["view"] == 1200 + 5200 + 120
assert overview["interactions"] > 0
assert overview["view_delta"] == (100 + 100 + 0) # 101/102 的日增量
assert overview["snapshot_health"]["total"] == 3
assert overview["folders"][0]["media_count"] == 3
assert overview["folders"][0]["ratio"] < 1
assert overview["fans"]["follower"] == 1020
assert overview["fans"]["delta_1d"] == 10
assert overview["preview"] is False
assert "snapshot" in overview["tasks"]
finally:
db.close()
def test_video_listing_filters_and_sorting():
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
heat = q.list_videos(sort="heat")
aids = [item["aid"] for item in heat["items"]]
assert 104 not in aids, "removed 视频不应进入榜单"
assert aids[0] == 102, aids
assert heat["items"][0]["heat"] > heat["items"][-1]["heat"]
assert heat["items"][0]["heat_percentile"] == 100.0
views = q.list_videos(sort="view")
assert [item["aid"] for item in views["items"]] == [102, 101, 103]
interactions = q.list_videos(sort="interaction")
assert interactions["items"][0]["aid"] == 101
added = q.list_videos(sort="added", within_days=1)
assert [item["aid"] for item in added["items"]] == [103]
published = q.list_videos(sort="published", within_days=2)
assert {item["aid"] for item in published["items"]} == {101, 103}
searched = q.list_videos(sort="view", q="UP B")
assert [item["aid"] for item in searched["items"]] == [103]
searched = q.list_videos(sort="view", q="视频 102")
assert [item["aid"] for item in searched["items"]] == [102]
folder = q.list_videos(sort="view", folder_id=3700023351)
assert folder["total"] == 3
folder = q.list_videos(sort="view", folder_id=999)
assert folder["total"] == 0
page = q.list_videos(sort="view", page=2, page_size=1)
assert page["total"] == 3
assert page["pages"] == 3
assert [item["aid"] for item in page["items"]] == [101]
finally:
db.close()
def test_growth_and_cold_start():
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
growth = q.list_videos(sort="growth", growth_days=2)
assert growth["available"] is True
assert {item["aid"] for item in growth["items"]} == {101, 102}
first = next(item for item in growth["items"] if item["aid"] == 101)
assert first["growth"]["available"] is True
assert first["growth"]["delta"] == 200
assert first["growth"]["gap_days"] == 2
rate = q.list_videos(sort="growth_rate", growth_days=2)
assert {item["aid"] for item in rate["items"]} == {101, 102}
# 只有一个快照日时增长不可用,不能返回假 0
for day in DATES[:-1]:
db.write(lambda conn, day=day: conn.execute(
"DELETE FROM snapshot WHERE stat_date = ?", (day,)
))
cold = q.list_videos(sort="growth", growth_days=2)
assert cold["available"] is False
assert cold["items"] == []
assert "两天" in cold["reason"]
finally:
db.close()
def test_video_detail_and_trend():
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
detail = q.video_detail(101)
assert detail["aid"] == 101
assert detail["metrics"]["view"] == 1200
assert detail["growth"]["delta"] == 100
assert detail["snapshot_count"] == 3
assert detail["folder_title"] == "2026"
assert detail["url"].endswith("BV101")
assert q.video_detail(999999) is None
trend = q.video_trend(101, days=3)
assert trend["dates"] == list(DATES)
assert trend["series"]["view"] == [1000, 1100, 1200]
assert trend["available"] is True
assert len(trend["points"]) == 3
# 删除中间一天,折线在缺失日期应为 None,而不是插值
db.write(lambda conn: conn.execute(
"DELETE FROM snapshot WHERE aid=101 AND stat_date=?",
("2026-10-30",),
))
trend = q.video_trend(101, days=3)
assert trend["series"]["view"] == [1000, None, 1200]
# 失败快照也视为缺失
db.write(lambda conn: conn.execute(
"UPDATE snapshot SET ok=0, view=NULL WHERE aid=101 AND stat_date=?",
(DATES[-1],),
))
trend = q.video_trend(101, days=3)
assert trend["series"]["view"][-1] is None
finally:
db.close()
def test_up_queries():
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
ups = q.list_ups()
assert ups["total"] == 2
assert ups["items"][0]["up_mid"] == 1
assert ups["items"][0]["video_count"] == 2
assert ups["items"][0]["total_views"] == 6400
assert ups["items"][0]["avg_views"] == 3200.0
one = q.list_ups(min_videos=2)
assert one["total"] == 1
detail = q.up_detail(1)
assert detail["video_count"] == 2
assert detail["total_views"] == 6400
assert detail["blacklisted"] is False
assert detail["series"]["dates"] == list(DATES)
assert detail["series"]["views"] == [6000, 6200, 6400]
assert detail["space_url"].endswith("/1")
blacklisted = q.up_detail(2)
assert blacklisted["blacklisted"] is True
assert blacklisted["video_count"] == 1
assert q.up_detail(999999) is None
finally:
db.close()
def test_fans_candidates_system_and_export():
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
fans = q.fans(days=3)
assert fans["mid"] == 37754047
assert fans["name"] == "咻咻满"
assert fans["dates"] == list(DATES)
assert fans["series"] == [1000, 1010, 1020]
assert fans["delta_1d"] == 10
assert fans["delta_7d"] is None
assert fans["delta_cum"] == 20
assert fans["options"][0]["mid"] == 37754047
candidates = q.candidates()
assert candidates["total"] == 2
assert candidates["pending"] == 1
assert candidates["items"][0]["aid"] == 201
assert 1 <= candidates["items"][0]["days_left"] <= 7
assert candidates["items"][1]["decision"] == "rejected"
system = q.system()
assert system["data_date"] == DATES[-1]
assert system["snapshot_days"] == 3
assert system["blacklist"][0]["video_count"] == 1
assert system["tasks"]["snapshot"]["ok"] is True
assert system["errors"] == []
assert system["preview"] is False
assert system["next_runs"]["snapshot"] > 0
exported = q.export_videos(sort="view")
assert exported["total"] == 3
assert all("metrics" in item for item in exported["items"])
finally:
db.close()
def main_test():
test_overview()
test_video_listing_filters_and_sorting()
test_growth_and_cold_start()
test_video_detail_and_trend()
test_up_queries()
test_fans_candidates_system_and_export()
print("OK: Web 查询层总览/榜单/详情/UP/粉丝/候选/系统测试全部通过")
if __name__ == "__main__":
main_test()
+215
View File
@@ -0,0 +1,215 @@
# -*- coding: utf-8 -*-
"""只读 Web 服务测试(离线): uv run python tests/test_web_server.py"""
import http.client
import json
import os
import sys
import tempfile
import threading
import urllib.parse
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.stdout.reconfigure(encoding="utf-8")
from reaction_board.config import ConfigStore # noqa: E402
from reaction_board.database import Database # noqa: E402
from reaction_board.web.server import create_server # noqa: E402
from tests.web_fixtures import build_db, write_config # noqa: E402
class RunningServer:
def __init__(self, tmp: str, *, token: str = "", preview: bool = False):
self.config = write_config(tmp, token=token)
build_db(tmp)
self.db = Database(Path(tmp) / "app.db")
self.db.initialize()
if preview:
self.db.write(lambda conn: conn.execute(
"INSERT OR REPLACE INTO kv(k, v) VALUES ('preview:seeded', '1')"
))
self.store = ConfigStore(self.config)
self.server = create_server(
self.store, self.db, port=0, preview=preview,
clock=lambda: 1_793_459_000,
)
self.port = self.server.server_address[1]
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
def request(self, path: str, *, method: str = "GET", headers=None, body=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=5)
conn.request(method, path, body=body, headers=headers or {})
response = conn.getresponse()
payload = response.read()
result = response.status, dict(response.getheaders()), payload
conn.close()
return result
def json(self, path: str, **kwargs):
status, headers, payload = self.request(path, **kwargs)
return status, headers, json.loads(payload.decode("utf-8"))
def close(self):
self.server.shutdown()
self.server.server_close()
self.db.close()
def test_readonly_routes_and_static():
with tempfile.TemporaryDirectory() as tmp:
running = RunningServer(tmp)
try:
status, _, health = running.json("/api/health")
assert status == 200 and health["ok"] is True
assert health["data"]["preview"] is False
status, _, overview = running.json("/api/overview")
assert status == 200 and overview["ok"] is True
assert overview["data"]["data_date"] == "2026-10-31"
status, headers, payload = running.request("/")
assert status == 200
assert "text/html" in headers["Content-Type"]
assert b"<html" in payload.lower()
status, _, payload = running.request("/static/../config.json")
assert status == 404
status, _, payload = running.request("/api/videos")
assert status == 200
items = json.loads(payload)["data"]["items"]
assert items and items[0]["aid"] == 102
status, headers, payload = running.request("/api/videos", method="HEAD")
assert status == 200 and payload == b""
assert "Content-Length" in headers
# 只读:所有业务写方法都被拒绝,且数据库不变
before = running.db.read(lambda conn: conn.execute(
"SELECT COUNT(*) FROM candidate"
).fetchone()[0])
for method in ("POST", "PUT", "PATCH", "DELETE"):
status, _, _ = running.request("/api/videos/101", method=method)
assert status == 405, method
after = running.db.read(lambda conn: conn.execute(
"SELECT COUNT(*) FROM candidate"
).fetchone()[0])
assert after == before
status, _, error = running.json("/api/videos?sort=bad")
assert status == 400 and error["ok"] is False
status, _, error = running.json("/api/videos?page=0")
assert status == 400
status, _, error = running.json("/api/videos/abc")
assert status == 400
status, _, error = running.json("/api/nope")
assert status == 404
status, _, payload = running.request("/api/export.csv?sort=view")
assert status == 200
assert payload.startswith(b"\xef\xbb\xbf")
assert "咻咻满".encode("utf-8") in payload
status, headers, _ = running.request("/api/overview")
assert headers["X-Content-Type-Options"] == "nosniff"
assert "default-src 'self'" in headers["Content-Security-Policy"]
assert headers["X-Frame-Options"] == "DENY"
finally:
running.close()
def test_token_auth_and_login():
with tempfile.TemporaryDirectory() as tmp:
running = RunningServer(tmp, token="secret-token")
try:
status, _, body = running.json("/api/overview")
assert status == 401 and body["error"]["code"] == "unauthorized"
status, headers, _ = running.request("/")
assert status == 303 and headers["Location"] == "/login"
status, _, payload = running.request("/login")
assert status == 200 and b"web.auth_token" in payload
status, _, _ = running.request(
"/login",
method="POST",
headers={"Content-Type": "application/x-www-form-urlencoded"},
body="token=wrong",
)
assert status == 401
status, headers, _ = running.request(
"/login",
method="POST",
headers={"Content-Type": "application/x-www-form-urlencoded"},
body="token=secret-token",
)
assert status == 303
cookie = headers["Set-Cookie"].split(";", 1)[0]
assert cookie.startswith("rb_session=")
status, _, body = running.json("/api/overview", headers={"Cookie": cookie})
assert status == 200 and body["ok"] is True
status, _, body = running.json(
"/api/overview", headers={"Authorization": "Bearer secret-token"}
)
assert status == 200
tampered = cookie[:-1] + ("0" if cookie[-1] != "0" else "1")
status, _, _ = running.json("/api/overview", headers={"Cookie": tampered})
assert status == 401
# 健康检查始终匿名
status, _, _ = running.json("/api/health")
assert status == 200
finally:
running.close()
def test_preview_and_loopback_guards():
with tempfile.TemporaryDirectory() as tmp:
config = write_config(tmp)
build_db(tmp)
db = Database(Path(tmp) / "app.db")
db.initialize()
store = ConfigStore(config)
try:
try:
create_server(store, db, host="0.0.0.0", port=0)
except ValueError as exc:
assert "web.auth_token" in str(exc)
else:
raise AssertionError("非回环地址且无 token 应拒绝启动")
try:
create_server(store, db, port=0, preview=True)
except ValueError as exc:
assert "preview:seeded" in str(exc)
else:
raise AssertionError("非预览库不应允许 --preview")
db.write(lambda conn: conn.execute(
"INSERT OR REPLACE INTO kv(k, v) VALUES ('preview:seeded', '1')"
))
server = create_server(store, db, port=0, preview=True)
try:
assert server.context.preview is True
finally:
server.server_close()
finally:
db.close()
def main_test():
test_readonly_routes_and_static()
test_token_auth_and_login()
test_preview_and_loopback_guards()
print("OK: Web 只读路由/鉴权/预览/安全边界测试全部通过")
if __name__ == "__main__":
main_test()
+177
View File
@@ -0,0 +1,177 @@
# -*- coding: utf-8 -*-
"""Web 查询/服务测试共用的临时数据库与配置。"""
from __future__ import annotations
import json
from datetime import date
from pathlib import Path
from reaction_board.bilibili.models import FavResource
from reaction_board.config import FolderConfig
from reaction_board.database import Database
from reaction_board.repository import Repository
from reaction_board.timeutil import day_bounds
FOLDER = FolderConfig(year=2026, media_id=3700023351, title="2026")
DATES = ("2026-10-29", "2026-10-30", "2026-10-31")
def write_config(
tmp: str | Path,
*,
token: str = "",
host: str = "127.0.0.1",
port: int = 8080,
) -> Path:
root = Path(tmp)
path = root / "config.json"
path.write_text(json.dumps({
"bilibili": {
"read": {"mid": 1, "csrf": "read-csrf", "cookie": "SESSDATA=fake-read"},
"write": {"mid": 2, "csrf": "write-csrf", "cookie": "SESSDATA=fake-write"},
},
"telegram": {"enabled": False, "bot_token": "", "chat_id": 0},
"web": {
"host": host, "port": port, "auth_token": token,
"public_base_url": "",
},
"storage": {
"database_path": str(root / "app.db"),
"backup_dir": str(root / "backup"),
"backup_keep_days": 7,
},
"folders": [
{"year": 2026, "media_id": 3700023351, "title": "2026"},
],
"track_fans": [37754047],
"up_blacklist_seed": [],
"schedule": {
"discover_minutes": 60,
"folder_sync_minutes": 60,
"full_folder_refresh_hours": 6,
"snapshot_at": "00:01",
"candidate_ttl_days": 7,
},
}, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def resource(
aid: int,
*,
up_mid: int,
up_name: str,
pubdate: int,
attr: int = 0,
title: str | None = None,
) -> FavResource:
return FavResource(
aid=aid,
bvid=f"BV{aid}",
title=title or f"咻咻满 reaction 视频 {aid}",
up_mid=up_mid,
up_name=up_name,
pubdate=pubdate,
duration=7200,
cover=f"//i0.hdslb.com/{aid}.jpg",
attr=attr,
fav_time=None,
)
def build_db(tmp: str | Path) -> Path:
root = Path(tmp)
path = root / "app.db"
db = Database(path)
db.initialize()
try:
repo = Repository(db)
repo.ensure_folder(FOLDER)
repo.seed_blacklist([2])
def ts(day: str) -> int:
return int(day_bounds(day)[0])
videos = {
101: (1, "UP A", int(day_bounds("2026-10-30")[0]), 0, "2026-10-29"),
102: (1, "UP A", int(day_bounds("2026-10-20")[0]), 0, "2026-10-29"),
103: (2, "UP B", int(day_bounds("2026-10-31")[0]), 0, "2026-10-31"),
104: (3, "UP C", int(day_bounds("2026-09-01")[0]), 9, "2026-10-29"),
}
for aid, (up_mid, up_name, pubdate, attr, first_day) in videos.items():
repo.upsert_from_fav(
resource(aid, up_mid=up_mid, up_name=up_name, pubdate=pubdate, attr=attr),
FOLDER.media_id,
was_added=True,
now=ts(first_day),
)
repo.set_video_status(103, "unfaved")
metrics = {
101: {"view": 1000, "like": 100, "coin": 20, "favorite": 30, "share": 2, "reply": 10, "danmaku": 12},
102: {"view": 5000, "like": 200, "coin": 50, "favorite": 100, "share": 10, "reply": 30, "danmaku": 40},
104: {"view": 50, "like": 1, "coin": 0, "favorite": 0, "share": 0, "reply": 0, "danmaku": 0},
}
for index, day in enumerate(DATES):
for aid in (101, 102):
base = metrics[aid]
values = {}
for key, value in base.items():
values[key] = value + (index * 100 if key == "view" else index * 2)
repo.upsert_snapshot(
day, aid, metrics=values, ok=True, err=None,
captured_at=ts(day) + 60,
)
repo.upsert_up_stat(
day, 37754047, follower=1000 + index * 10, following=5,
ok=True, captured_at=ts(day) + 60,
)
# 103 只有最后一天快照,用于验证样本不足;104 另加一条失败快照
repo.upsert_snapshot(
DATES[-1], 103,
metrics={"view": 120, "like": 5, "coin": 1, "favorite": 2, "share": 0, "reply": 1, "danmaku": 1},
ok=True, err=None, captured_at=ts(DATES[-1]) + 60,
)
repo.upsert_snapshot(
"2026-10-30", 104,
metrics=None, ok=False, err="simulated failure",
captured_at=ts("2026-10-30") + 60,
)
repo.upsert_candidate(
aid=201, bvid="BV201", title="咻咻满 reaction 候选",
up_mid=9, up_name="候选UP", pubdate=1_790_000_000,
cover=None, score=6, channel="search:test#p1",
now=ts("2026-10-30"), play=1000, like=50,
)
repo.mark_notified(
201, channel="telegram", batch_id="b1", ok=True,
now=ts("2026-10-30") + 10,
)
repo.upsert_candidate(
aid=202, bvid="BV202", title="咻咻满 切片候选",
up_mid=10, up_name="切片UP", pubdate=1_790_000_000,
cover=None, score=1, channel="search:test#p1",
now=ts("2026-10-29"),
)
repo.mark_candidate_decision(202, "rejected", now=ts("2026-10-30"))
repo.finish_folder_sync(FOLDER, media_count=3, full_refresh=True)
repo.kv_set("snapshot:last_completed", DATES[-1])
repo.kv_set("auth:ok", "1")
repo.kv_set("auth:message", "OK")
repo.kv_set(
"task:last:snapshot",
json.dumps({
"name": "snapshot", "ok": True,
"started_at": ts(DATES[-1]), "finished_at": ts(DATES[-1]) + 60,
"error": None, "summary": {"completed": True},
}),
)
finally:
db.close()
return path
def day_date(value: str) -> date:
return date.fromisoformat(value)