feat(p1): add daily snapshots and fan tracking
Implement P1: seven-metric view snapshots, relation/stat fan snapshots, failure placeholders, resume by daily completion markers, Cookie login warnings, daily SQLite backups, Shanghai daily scheduling, and CLI snapshot/auth-check commands. Real read-only run captured 795/795 videos successfully over 1124s with a verified daily backup.
This commit is contained in:
1 parent
ee289bfa56
commit
5c2ab7b187
20 files changed
+1680
-54
No files matched your search
@@ -161,3 +161,108 @@ def info_for(
|
||||
f"bilibili://music/playlist/playpage/3728825651?page_type=3&oid={aid}&otype=2"
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
# GET x/web-interface/view?aid=117530286889638(真实七项 stat)
|
||||
VIEW_PAYLOAD: dict[str, Any] = {
|
||||
"code": 0,
|
||||
"message": "0",
|
||||
"ttl": 1,
|
||||
"data": {
|
||||
"aid": 117530286889638,
|
||||
"bvid": "BV1xvxi6LEZD",
|
||||
"cid": 43184885058,
|
||||
"title": "【咻咻满充电专场①】百变音色机能怪,还有什么风格是你驾驭不了的?!",
|
||||
"pubdate": 1793370433,
|
||||
"ctime": 1793370433,
|
||||
"duration": 9504,
|
||||
"owner": {"mid": 434442072, "name": "_-抱抱-_", "face": ""},
|
||||
"stat": {
|
||||
"aid": 117530286889638,
|
||||
"view": 819,
|
||||
"danmaku": 7,
|
||||
"reply": 16,
|
||||
"favorite": 43,
|
||||
"coin": 9,
|
||||
"share": 1,
|
||||
"now_rank": 0,
|
||||
"his_rank": 0,
|
||||
"like": 44,
|
||||
"dislike": 0,
|
||||
"evaluation": "",
|
||||
"vt": 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# 当前实测:收藏夹里已不可访问的 aid 返回 62012,而不是旧文档里的 -404
|
||||
VIEW_UNAVAILABLE: dict[str, Any] = {
|
||||
"code": 62012,
|
||||
"message": "62012",
|
||||
"ttl": 1,
|
||||
"data": None,
|
||||
}
|
||||
|
||||
# GET x/relation/stat?vmid=37754047
|
||||
RELATION_PAYLOAD: dict[str, Any] = {
|
||||
"code": 0,
|
||||
"message": "0",
|
||||
"ttl": 1,
|
||||
"data": {
|
||||
"mid": 37754047,
|
||||
"following": 95,
|
||||
"whisper": 0,
|
||||
"black": 0,
|
||||
"follower": 2767268,
|
||||
"fans_medal_toast": None,
|
||||
"fans_effect": None,
|
||||
},
|
||||
}
|
||||
|
||||
# GET x/web-interface/nav(已登录)
|
||||
NAV_PAYLOAD: dict[str, Any] = {
|
||||
"code": 0,
|
||||
"message": "0",
|
||||
"ttl": 1,
|
||||
"data": {
|
||||
"isLogin": True,
|
||||
"mid": 28802351,
|
||||
"uname": "冷萃编译器",
|
||||
"face": "https://i0.hdslb.com/bfs/face/example.jpg",
|
||||
"level_info": {"current_level": 5},
|
||||
"vipStatus": 0,
|
||||
"wbi_img": {
|
||||
"img_url": "https://i0.hdslb.com/bfs/wbi/example.png",
|
||||
"sub_url": "https://i0.hdslb.com/bfs/wbi/example-sub.png",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def view_for(
|
||||
aid: int,
|
||||
*,
|
||||
view: int = 819,
|
||||
like: int = 44,
|
||||
coin: int = 9,
|
||||
favorite: int = 43,
|
||||
share: int = 1,
|
||||
reply: int = 16,
|
||||
danmaku: int = 7,
|
||||
bvid: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
item = copy.deepcopy(VIEW_PAYLOAD)
|
||||
data = item["data"]
|
||||
data["aid"] = aid
|
||||
data["bvid"] = bvid or f"BV{aid}"
|
||||
data["stat"].update({
|
||||
"aid": aid,
|
||||
"view": view,
|
||||
"like": like,
|
||||
"coin": coin,
|
||||
"favorite": favorite,
|
||||
"share": share,
|
||||
"reply": reply,
|
||||
"danmaku": danmaku,
|
||||
})
|
||||
return item
|
||||
@@ -25,8 +25,13 @@ from fixtures import ( # noqa: E402
|
||||
FOLDERS_PAYLOAD,
|
||||
IDS_PAYLOAD,
|
||||
INVALID_INFO,
|
||||
NAV_PAYLOAD,
|
||||
RELATION_PAYLOAD,
|
||||
VALID_INFO,
|
||||
VIEW_PAYLOAD,
|
||||
VIEW_UNAVAILABLE,
|
||||
info_for,
|
||||
view_for,
|
||||
)
|
||||
|
||||
|
||||
@@ -274,6 +279,88 @@ def test_timeout_and_method_forwarded():
|
||||
assert call["timeout"] == 30
|
||||
|
||||
|
||||
def test_view_seven_metrics():
|
||||
transport = QueueTransport([http_json(VIEW_PAYLOAD)])
|
||||
client, _ = make_client(transport)
|
||||
stat = client.view(117530286889638)
|
||||
assert stat.aid == 117530286889638
|
||||
assert stat.bvid == "BV1xvxi6LEZD"
|
||||
assert stat.metrics() == {
|
||||
"view": 819, "like": 44, "favorite": 43, "coin": 9,
|
||||
"share": 1, "reply": 16, "danmaku": 7,
|
||||
}
|
||||
assert "aid=117530286889638" in transport.calls[0]["url"]
|
||||
|
||||
|
||||
def test_view_unavailable_and_malformed():
|
||||
transport = QueueTransport([
|
||||
http_json(VIEW_UNAVAILABLE), http_json(VIEW_UNAVAILABLE),
|
||||
])
|
||||
client, _ = make_client(transport)
|
||||
try:
|
||||
client.view(1)
|
||||
except BilibiliAPIError as exc:
|
||||
assert exc.code == 62012
|
||||
else:
|
||||
raise AssertionError("不可访问视频应抛业务错误")
|
||||
|
||||
payload = view_for(1)
|
||||
payload["data"]["stat"].pop("coin")
|
||||
bad = http_json(payload)
|
||||
transport = QueueTransport([bad, bad])
|
||||
client, _ = make_client(transport)
|
||||
try:
|
||||
client.view(1)
|
||||
except BilibiliProtocolError as exc:
|
||||
assert "coin" in str(exc)
|
||||
else:
|
||||
raise AssertionError("缺少 stat 字段应报协议错误")
|
||||
|
||||
|
||||
def test_relation_stat():
|
||||
transport = QueueTransport([http_json(RELATION_PAYLOAD)])
|
||||
client, _ = make_client(transport)
|
||||
fan = client.relation_stat(37754047)
|
||||
assert fan.mid == 37754047
|
||||
assert fan.follower == 2767268
|
||||
assert fan.following == 95
|
||||
assert "vmid=37754047" in transport.calls[0]["url"]
|
||||
|
||||
|
||||
def test_check_login():
|
||||
transport = QueueTransport([http_json(NAV_PAYLOAD)])
|
||||
client, _ = make_client(transport)
|
||||
state = client.check_login()
|
||||
assert state.is_login is True
|
||||
assert state.mid == 28802351
|
||||
assert state.uname == "冷萃编译器"
|
||||
assert "SESSDATA=secret" in transport.calls[0]["headers"]["cookie"]
|
||||
|
||||
transport = QueueTransport([http_json({
|
||||
"code": 0, "message": "0", "ttl": 1,
|
||||
"data": {"isLogin": False, "mid": 0, "uname": ""},
|
||||
})])
|
||||
client, _ = make_client(transport)
|
||||
state = client.check_login()
|
||||
assert state.is_login is False and state.mid == 0
|
||||
|
||||
transport = QueueTransport([http_json({"code": -101, "message": "账号未登录", "data": None})])
|
||||
client, _ = make_client(transport)
|
||||
state = client.check_login()
|
||||
assert state.is_login is False
|
||||
assert state.code == -101
|
||||
assert "账号未登录" in state.message
|
||||
|
||||
|
||||
def test_check_login_without_cookie_makes_no_request():
|
||||
transport = QueueTransport([])
|
||||
client, _ = make_client(transport, cookie="")
|
||||
state = client.check_login()
|
||||
assert state.is_login is False
|
||||
assert state.message == "未配置 cookie"
|
||||
assert transport.calls == []
|
||||
|
||||
|
||||
def main():
|
||||
test_fav_ids_parse_and_anonymous_header()
|
||||
test_anonymous_failure_falls_back_to_full_cookie()
|
||||
@@ -288,6 +375,11 @@ def main():
|
||||
test_fav_infos_chunk_size_validation()
|
||||
test_pacer_delays_after_first_request()
|
||||
test_timeout_and_method_forwarded()
|
||||
test_view_seven_metrics()
|
||||
test_view_unavailable_and_malformed()
|
||||
test_relation_stat()
|
||||
test_check_login()
|
||||
test_check_login_without_cookie_makes_no_request()
|
||||
print("OK: B 站只读客户端契约与重试测试全部通过")
|
||||
|
||||
|
||||
|
||||
+44
-5
@@ -12,9 +12,11 @@ from pathlib import Path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
from reaction_board.bilibili.models import LoginState # noqa: E402
|
||||
from reaction_board.cli import main # noqa: E402
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.repository import Repository # noqa: E402
|
||||
import reaction_board.cli as cli # noqa: E402
|
||||
|
||||
|
||||
def write_config(tmp, *, cookie="SESSDATA=x; buvid3=y", csrf="csrf"):
|
||||
@@ -86,24 +88,61 @@ def test_sync_requires_auth_without_network():
|
||||
assert not (Path(tmp) / "app.db").exists()
|
||||
|
||||
|
||||
def test_run_once_without_folders():
|
||||
def test_snapshot_offline_empty_db():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = write_config(tmp, cookie="", csrf="")
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
data["folders"] = []
|
||||
data["track_fans"] = []
|
||||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
code, output = run_cli(["snapshot", "--config", str(path)])
|
||||
assert code == 0
|
||||
assert '"completed": true' in output
|
||||
assert '"total": 0' in output
|
||||
assert len(list((Path(tmp) / "backup").glob("app-*.db"))) == 1
|
||||
|
||||
|
||||
def test_auth_check_without_cookie():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = write_config(tmp, cookie="", csrf="")
|
||||
code, output = run_cli(["auth-check", "--config", str(path)])
|
||||
assert code == 1
|
||||
payload = json.loads(output)
|
||||
assert payload["is_login"] is False
|
||||
assert "未配置 cookie" in payload["message"]
|
||||
|
||||
|
||||
def test_run_once_offline():
|
||||
class FakeClient:
|
||||
def check_login(self):
|
||||
return LoginState(True, 1, "tester", 0, "OK")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = write_config(tmp)
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
data["folders"] = []
|
||||
data["track_fans"] = []
|
||||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
code, output = run_cli(["run", "--once", "--config", str(path)])
|
||||
|
||||
original = cli._client
|
||||
cli._client = lambda *args, **kwargs: FakeClient()
|
||||
try:
|
||||
code, output = run_cli(["run", "--once", "--config", str(path)])
|
||||
finally:
|
||||
cli._client = original
|
||||
assert code == 0
|
||||
assert '"name": "folder_sync"' in output
|
||||
assert '"ok": true' in output
|
||||
assert '"name": "snapshot"' in output
|
||||
|
||||
|
||||
def main_test():
|
||||
test_init_and_status()
|
||||
test_missing_config()
|
||||
test_sync_requires_auth_without_network()
|
||||
test_run_once_without_folders()
|
||||
print("OK: CLI init/sync/run/status/错误路径测试全部通过")
|
||||
test_snapshot_offline_empty_db()
|
||||
test_auth_check_without_cookie()
|
||||
test_run_once_offline()
|
||||
print("OK: CLI init/sync/snapshot/auth-check/run/status/错误路径测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""每日备份与保留期测试: uv run python tests/test_maintenance.py"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.services.maintenance import ( # noqa: E402
|
||||
backup_daily,
|
||||
daily_backup_path,
|
||||
prune_daily_backups,
|
||||
)
|
||||
|
||||
|
||||
def test_daily_backup_and_restore():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = Database(Path(tmp) / "app.db")
|
||||
db.initialize()
|
||||
try:
|
||||
db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('x', '42')"))
|
||||
target = backup_daily(db, Path(tmp) / "backup", "2026-10-31")
|
||||
assert target == daily_backup_path(Path(tmp) / "backup", "2026-10-31")
|
||||
assert target.name == "app-20261031.db"
|
||||
assert target.exists()
|
||||
|
||||
check = sqlite3.connect(target)
|
||||
try:
|
||||
assert check.execute("SELECT v FROM kv WHERE k='x'").fetchone()[0] == "42"
|
||||
finally:
|
||||
check.close()
|
||||
|
||||
# 覆盖同日备份也安全
|
||||
db.write(lambda c: c.execute("UPDATE kv SET v='43' WHERE k='x'"))
|
||||
backup_daily(db, Path(tmp) / "backup", "2026-10-31")
|
||||
check = sqlite3.connect(target)
|
||||
try:
|
||||
assert check.execute("SELECT v FROM kv WHERE k='x'").fetchone()[0] == "43"
|
||||
finally:
|
||||
check.close()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_prune_daily_backups_only():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
backup_dir = Path(tmp)
|
||||
old_daily = backup_dir / "app-20260101.db"
|
||||
fresh_daily = backup_dir / "app-20261031.db"
|
||||
old_migration = backup_dir / "app-pre-v1-20260101.db"
|
||||
other = backup_dir / "keep.txt"
|
||||
for path in (old_daily, fresh_daily, old_migration, other):
|
||||
path.write_bytes(b"x")
|
||||
now = 1_800_000_000
|
||||
old_time = now - 20 * 86400
|
||||
for path in (old_daily, old_migration):
|
||||
os.utime(path, (old_time, old_time))
|
||||
os.utime(fresh_daily, (now, now))
|
||||
removed = prune_daily_backups(backup_dir, keep_days=14, now=now)
|
||||
assert removed == [old_daily]
|
||||
assert fresh_daily.exists()
|
||||
assert old_migration.exists() # 迁移备份不属于日备份清理范围
|
||||
assert other.exists()
|
||||
|
||||
|
||||
def main():
|
||||
test_daily_backup_and_restore()
|
||||
test_prune_daily_backups_only()
|
||||
print("OK: 每日备份与保留期测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -110,8 +110,27 @@ def test_max_iterations_and_validation():
|
||||
raise AssertionError(f"应当失败: {tasks}, {kwargs}")
|
||||
|
||||
|
||||
def test_next_run_fn_controls_reschedule():
|
||||
clock = [1_000.0]
|
||||
calls = []
|
||||
task = ScheduledTask(
|
||||
"daily", 86400, lambda: calls.append(clock[0]),
|
||||
next_run_fn=lambda started: started + 123,
|
||||
)
|
||||
scheduler = Scheduler([task], clock=lambda: clock[0])
|
||||
scheduler.run_due()
|
||||
assert calls == [1_000.0]
|
||||
assert task.next_run == 1_123.0
|
||||
clock[0] = 1_122.0
|
||||
assert scheduler.run_due() == []
|
||||
clock[0] = 1_123.0
|
||||
scheduler.run_due()
|
||||
assert calls == [1_000.0, 1_123.0]
|
||||
|
||||
|
||||
def main():
|
||||
test_run_when_due_and_reschedule()
|
||||
test_next_run_fn_controls_reschedule()
|
||||
test_task_error_does_not_block_other_tasks()
|
||||
test_run_forever_with_stop_event()
|
||||
test_max_iterations_and_validation()
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""每日快照与粉丝采集测试(transport 边界 Mock,不联网):
|
||||
uv run python tests/test_snapshot.py
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from fixtures import ( # noqa: E402
|
||||
NAV_PAYLOAD,
|
||||
RELATION_PAYLOAD,
|
||||
VIEW_UNAVAILABLE,
|
||||
view_for,
|
||||
)
|
||||
from reaction_board.bilibili.client import ( # noqa: E402
|
||||
BilibiliClient,
|
||||
HTTPResponse,
|
||||
Pacer,
|
||||
)
|
||||
from reaction_board.bilibili.models import FavResource # noqa: E402
|
||||
from reaction_board.config import ConfigStore, FolderConfig # noqa: E402
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.logging_json import JsonLogger # noqa: E402
|
||||
from reaction_board.repository import Repository # noqa: E402
|
||||
from reaction_board.services.snapshot import SnapshotService # noqa: E402
|
||||
|
||||
DATE = "2026-10-31"
|
||||
FOLDER = FolderConfig(year=2026, media_id=3700023351, title="2026")
|
||||
|
||||
|
||||
def http_json(payload):
|
||||
return HTTPResponse(
|
||||
200, {"content-type": "application/json"},
|
||||
json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
)
|
||||
|
||||
|
||||
class SnapshotTransport:
|
||||
def __init__(self, views, *, nav=None, fan=None):
|
||||
self.views = views
|
||||
self.nav = NAV_PAYLOAD if nav is None else nav
|
||||
self.fan = RELATION_PAYLOAD if fan is None else fan
|
||||
self.calls = []
|
||||
|
||||
def request(self, method, url, headers, data=None, timeout=30):
|
||||
self.calls.append(url)
|
||||
if "/x/web-interface/nav" in url:
|
||||
return http_json(self.nav)
|
||||
if "/x/web-interface/view" in url:
|
||||
aid = int(urllib.parse.parse_qs(
|
||||
urllib.parse.urlparse(url).query
|
||||
)["aid"][0])
|
||||
payload = self.views[aid]
|
||||
if isinstance(payload, HTTPResponse):
|
||||
return payload
|
||||
return http_json(payload)
|
||||
if "/x/relation/stat" in url:
|
||||
return http_json(self.fan)
|
||||
raise AssertionError(f"未预期的 URL: {url}")
|
||||
|
||||
def count(self, part):
|
||||
return sum(1 for url in self.calls if part in url)
|
||||
|
||||
|
||||
def make_service(tmp, transport, *, track_fans=(37754047,), logger=None,
|
||||
retries=3, cookie="SESSDATA=secret; buvid3=device"):
|
||||
path = Path(tmp) / "config.json"
|
||||
path.write_text(json.dumps({
|
||||
"mid": 28802351,
|
||||
"csrf": "csrf",
|
||||
"cookie": cookie,
|
||||
"storage": {
|
||||
"database_path": str(Path(tmp) / "app.db"),
|
||||
"backup_dir": str(Path(tmp) / "backup"),
|
||||
"backup_keep_days": 14,
|
||||
},
|
||||
"folders": [],
|
||||
"track_fans": list(track_fans),
|
||||
}, ensure_ascii=False), encoding="utf-8")
|
||||
store = ConfigStore(path, env={}, require_auth=False)
|
||||
db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup")
|
||||
db.initialize()
|
||||
repo = Repository(db, clock=lambda: 1_800_000_000)
|
||||
repo.ensure_folder(FOLDER)
|
||||
client = BilibiliClient(
|
||||
cookie=cookie,
|
||||
csrf="csrf",
|
||||
mid=28802351,
|
||||
transport=transport,
|
||||
pacer=Pacer(0, 0),
|
||||
retries=1,
|
||||
)
|
||||
service = SnapshotService(
|
||||
store, repo, client,
|
||||
clock=lambda: 1_800_000_000,
|
||||
sleeper=lambda seconds: None,
|
||||
logger=logger,
|
||||
retries=retries,
|
||||
backoff=0,
|
||||
)
|
||||
return db, repo, service
|
||||
|
||||
|
||||
def add_video(repo, aid, *, status="active"):
|
||||
repo.upsert_from_fav(
|
||||
FavResource(
|
||||
aid=aid, bvid=f"BV{aid}", title=f"视频{aid}", up_mid=100,
|
||||
up_name="UP", pubdate=1_700_000_000, duration=600,
|
||||
cover=None, attr=0, fav_time=None,
|
||||
),
|
||||
FOLDER.media_id,
|
||||
was_added=True,
|
||||
)
|
||||
if status != "active":
|
||||
repo.set_video_status(aid, status)
|
||||
|
||||
|
||||
def test_snapshot_success_resume_and_force():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({
|
||||
1: view_for(1, view=100, like=10, coin=2, favorite=3,
|
||||
share=1, reply=4, danmaku=5),
|
||||
2: view_for(2, view=200, like=20, coin=4, favorite=6,
|
||||
share=2, reply=8, danmaku=10),
|
||||
})
|
||||
db, repo, service = make_service(tmp, transport)
|
||||
try:
|
||||
add_video(repo, 1)
|
||||
add_video(repo, 2)
|
||||
result = service.run(date=DATE)
|
||||
assert result.completed is True
|
||||
assert result.total == 2
|
||||
assert result.success == 2
|
||||
assert result.failed == 0
|
||||
assert result.fans_ok == 1
|
||||
assert result.fans_failed == 0
|
||||
assert result.auth_ok is True
|
||||
assert Path(result.backup_path or "").exists()
|
||||
|
||||
row = repo.get_snapshot(DATE, 1)
|
||||
assert (row["view"], row["like"], row["favorite"], row["coin"],
|
||||
row["share"], row["reply"], row["danmaku"]) == (100, 10, 3, 2, 1, 4, 5)
|
||||
assert row["ok"] == 1 and row["src"] == "view"
|
||||
fan = repo.get_up_stat(DATE, 37754047)
|
||||
assert fan["follower"] == 2767268 and fan["ok"] == 1
|
||||
assert repo.kv_get(service.completed_key(DATE)) == "1"
|
||||
assert repo.kv_get("snapshot:last_completed") == DATE
|
||||
assert repo.snapshot_counts(DATE) == {"total": 2, "ok": 2, "failed": 0}
|
||||
|
||||
# 已完成 -> 不再请求
|
||||
view_before = transport.count("/web-interface/view")
|
||||
again = service.run_if_needed(date=DATE)
|
||||
assert again.already_completed is True
|
||||
assert transport.count("/web-interface/view") == view_before
|
||||
|
||||
# force 重新采集
|
||||
forced = service.run(date=DATE, force=True)
|
||||
assert forced.success == 2
|
||||
assert transport.count("/web-interface/view") == view_before + 2
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_resume_skips_ok_and_retries_failed():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({
|
||||
1: view_for(1, view=111),
|
||||
2: view_for(2, view=222),
|
||||
})
|
||||
db, repo, service = make_service(tmp, transport)
|
||||
try:
|
||||
add_video(repo, 1)
|
||||
add_video(repo, 2)
|
||||
repo.upsert_snapshot(
|
||||
DATE, 1, metrics={"view": 1, "like": 1, "favorite": 1,
|
||||
"coin": 1, "share": 1, "reply": 1, "danmaku": 1},
|
||||
ok=True, err=None, captured_at=1,
|
||||
)
|
||||
repo.upsert_snapshot(
|
||||
DATE, 2, metrics=None, ok=False, err="previous error",
|
||||
captured_at=1,
|
||||
)
|
||||
result = service.run(date=DATE)
|
||||
assert result.skipped == 1
|
||||
assert result.success == 1
|
||||
assert result.completed is True
|
||||
assert transport.count("/web-interface/view") == 1
|
||||
assert repo.get_snapshot(DATE, 1)["view"] == 1 # 未被覆盖
|
||||
assert repo.get_snapshot(DATE, 2)["view"] == 222
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_unavailable_marks_removed():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({
|
||||
1: VIEW_UNAVAILABLE,
|
||||
2: HTTPResponse(404, {}, b"not found"),
|
||||
})
|
||||
db, repo, service = make_service(tmp, transport, track_fans=())
|
||||
try:
|
||||
add_video(repo, 1)
|
||||
add_video(repo, 2)
|
||||
result = service.run(date=DATE)
|
||||
assert result.removed == 2
|
||||
assert result.failed == 0
|
||||
assert result.completed is True
|
||||
assert repo.get_video(1)["status"] == "removed"
|
||||
assert repo.get_video(2)["status"] == "removed"
|
||||
assert repo.get_snapshot(DATE, 1) is None
|
||||
assert repo.get_snapshot(DATE, 2) is None
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_other_error_writes_failed_placeholder():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({
|
||||
1: {"code": -500, "message": "服务异常", "data": None},
|
||||
})
|
||||
db, repo, service = make_service(tmp, transport, track_fans=(), retries=3)
|
||||
try:
|
||||
add_video(repo, 1)
|
||||
result = service.run(date=DATE)
|
||||
assert result.failed == 1
|
||||
assert result.success == 0
|
||||
assert result.completed is False
|
||||
row = repo.get_snapshot(DATE, 1)
|
||||
assert row["ok"] == 0
|
||||
assert row["view"] is None and row["like"] is None
|
||||
assert "code=-500" in row["err"]
|
||||
assert repo.kv_get(service.completed_key(DATE)) == "0"
|
||||
# 每次业务失败会走匿名+完整 Cookie 两条请求,服务层重试 3 次
|
||||
assert transport.count("/web-interface/view") == 6
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_fan_failure_placeholder():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({}, fan={"code": -500, "message": "x", "data": None})
|
||||
db, repo, service = make_service(tmp, transport, retries=2)
|
||||
try:
|
||||
result = service.run(date=DATE)
|
||||
assert result.total == 0
|
||||
assert result.fans_failed == 1
|
||||
assert result.completed is False
|
||||
row = repo.get_up_stat(DATE, 37754047)
|
||||
assert row["ok"] == 0
|
||||
assert row["follower"] is None and row["following"] is None
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_auth_invalid_is_warned_but_snapshot_continues():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
stream = io.StringIO()
|
||||
logger = JsonLogger(stream)
|
||||
transport = SnapshotTransport(
|
||||
{}, nav={"code": 0, "message": "0", "ttl": 1,
|
||||
"data": {"isLogin": False, "mid": 0, "uname": ""}},
|
||||
fan=RELATION_PAYLOAD,
|
||||
)
|
||||
db, repo, service = make_service(tmp, transport, logger=logger)
|
||||
try:
|
||||
result = service.run(date=DATE)
|
||||
assert result.auth_ok is False
|
||||
assert result.completed is True
|
||||
assert repo.kv_get("auth:ok") == "0"
|
||||
events = [json.loads(line) for line in stream.getvalue().splitlines()]
|
||||
assert any(e["event"] == "auth.invalid" for e in events)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_no_cookie_skips_nav_request():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({}, fan=RELATION_PAYLOAD)
|
||||
db, repo, service = make_service(tmp, transport, cookie="")
|
||||
try:
|
||||
result = service.run(date=DATE)
|
||||
assert result.auth_ok is False
|
||||
assert result.auth_message == "未配置 cookie"
|
||||
assert transport.count("/web-interface/nav") == 0
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_force_failure_clears_last_completed():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({1: view_for(1, view=100)})
|
||||
db, repo, service = make_service(tmp, transport, track_fans=(), retries=1)
|
||||
try:
|
||||
add_video(repo, 1)
|
||||
first = service.run(date=DATE)
|
||||
assert first.completed is True
|
||||
assert repo.kv_get("snapshot:last_completed") == DATE
|
||||
|
||||
transport.views[1] = {"code": -500, "message": "x", "data": None}
|
||||
second = service.run(date=DATE, force=True)
|
||||
assert second.failed == 1
|
||||
assert second.completed is False
|
||||
assert repo.kv_get("snapshot:last_completed") == ""
|
||||
assert repo.get_snapshot(DATE, 1)["ok"] == 0
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
test_snapshot_success_resume_and_force()
|
||||
test_resume_skips_ok_and_retries_failed()
|
||||
test_unavailable_marks_removed()
|
||||
test_other_error_writes_failed_placeholder()
|
||||
test_fan_failure_placeholder()
|
||||
test_auth_invalid_is_warned_but_snapshot_continues()
|
||||
test_no_cookie_skips_nav_request()
|
||||
test_force_failure_clears_last_completed()
|
||||
print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,84 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""上海时区与每日时间计算测试: uv run python tests/test_timeutil.py"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
from reaction_board.timeutil import ( # noqa: E402
|
||||
SHANGHAI_TZ,
|
||||
day_bounds,
|
||||
is_date,
|
||||
next_daily_run,
|
||||
parse_hhmm,
|
||||
shanghai_date,
|
||||
shanghai_datetime,
|
||||
)
|
||||
|
||||
|
||||
def ts(text):
|
||||
return datetime.strptime(text, "%Y-%m-%d %H:%M:%S").replace(tzinfo=SHANGHAI_TZ).timestamp()
|
||||
|
||||
|
||||
def test_shanghai_date_boundary():
|
||||
# UTC 2026-10-31 15:59:59 = 上海 2026-10-31 23:59:59
|
||||
assert shanghai_date(ts("2026-10-31 23:59:59")) == "2026-10-31"
|
||||
# UTC 2026-10-31 16:00:00 = 上海 2026-11-01 00:00:00
|
||||
assert shanghai_date(ts("2026-11-01 00:00:00")) == "2026-11-01"
|
||||
assert shanghai_datetime().tzinfo is not None
|
||||
|
||||
|
||||
def test_parse_hhmm():
|
||||
assert parse_hhmm("00:01") == (0, 1)
|
||||
assert parse_hhmm("23:59") == (23, 59)
|
||||
for bad in ("24:00", "1:01", "00:60", "", "abc"):
|
||||
try:
|
||||
parse_hhmm(bad)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"{bad!r} 应失败")
|
||||
|
||||
|
||||
def test_next_daily_run():
|
||||
# 当天 00:01 之前 -> 当天 00:01
|
||||
now = ts("2026-11-01 00:00:10")
|
||||
nxt = next_daily_run(now, "00:01")
|
||||
assert nxt == ts("2026-11-01 00:01:00")
|
||||
# 正好 00:01:00 -> 下一天(严格大于)
|
||||
assert next_daily_run(ts("2026-11-01 00:01:00"), "00:01") == ts("2026-11-02 00:01:00")
|
||||
# 当天 00:01 之后 -> 次日
|
||||
assert next_daily_run(ts("2026-11-01 23:59:59"), "00:01") == ts("2026-11-02 00:01:00")
|
||||
# 跨月
|
||||
assert next_daily_run(ts("2026-10-31 12:00:00"), "00:01") == ts("2026-11-01 00:01:00")
|
||||
|
||||
|
||||
def test_day_bounds_and_date_validation():
|
||||
start, end = day_bounds("2026-11-01")
|
||||
assert start == ts("2026-11-01 00:00:00")
|
||||
assert end == ts("2026-11-02 00:00:00")
|
||||
assert end - start == 86400
|
||||
assert is_date("2026-11-01")
|
||||
for bad in ("2026-13-01", "2026-02-30", "2026/11/01", "20261101", ""):
|
||||
assert not is_date(bad), bad
|
||||
try:
|
||||
day_bounds("bad")
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("非法日期应失败")
|
||||
|
||||
|
||||
def main():
|
||||
test_shanghai_date_boundary()
|
||||
test_parse_hhmm()
|
||||
test_next_daily_run()
|
||||
test_day_bounds_and_date_validation()
|
||||
print("OK: 上海时区与每日时间计算测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user