fix: close data integrity and hot-reload review findings

Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
This commit is contained in:
kairee committed 2026-10-31 20:58:36 +08:00
1 parent 178ab64b18
commit 6a21b5d905
21 files changed
+895 -82

No files matched your search

+38
View File
@@ -361,6 +361,42 @@ def test_check_login_without_cookie_makes_no_request():
assert transport.calls == []
def test_missing_data_is_protocol_error():
for payload in (
{"code": 0, "message": "OK"},
{"code": 0, "message": "OK", "data": None},
):
transport = QueueTransport([http_json(payload)])
client, _ = make_client(transport)
try:
client.fav_ids(1)
except BilibiliProtocolError as exc:
assert "data" in str(exc)
else:
raise AssertionError(f"ids 缺少 data 应报协议错误: {payload}")
transport = QueueTransport([http_json(payload)])
client, _ = make_client(transport)
try:
client.fav_infos(1, [123])
except BilibiliProtocolError as exc:
assert "data" in str(exc)
else:
raise AssertionError(f"infos 缺少 data 应报协议错误: {payload}")
def test_empty_data_list_is_valid():
transport = QueueTransport([http_json({"code": 0, "message": "OK", "data": []})])
client, _ = make_client(transport)
assert client.fav_ids(4413147139) == []
transport = QueueTransport([http_json({"code": 0, "message": "OK", "data": []})])
client, _ = make_client(transport)
result = client.fav_infos(1, [123, 456])
assert result.items == {}
assert result.missing == (123, 456)
def main():
test_fav_ids_parse_and_anonymous_header()
test_anonymous_failure_falls_back_to_full_cookie()
@@ -380,6 +416,8 @@ def main():
test_relation_stat()
test_check_login()
test_check_login_without_cookie_makes_no_request()
test_missing_data_is_protocol_error()
test_empty_data_list_is_valid()
print("OK: B 站只读客户端契约与重试测试全部通过")
+37 -1
View File
@@ -51,7 +51,7 @@ def test_init_and_status():
path = write_config(tmp)
code, output = run_cli(["init", "--config", str(path)])
assert code == 0
assert '"schema_version": 1' in output
assert '"schema_version": 2' in output
db = Database(Path(tmp) / "app.db")
db.initialize()
@@ -174,6 +174,41 @@ def test_run_once_offline():
assert selected and all(credentials.mid == 1 for credentials in selected)
def test_snapshot_backup_failure_returns_error_and_retries():
import reaction_board.services.snapshot as snapshot_module
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
data = json.loads(path.read_text(encoding="utf-8"))
data["folders"] = []
data["track_fans"] = []
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
original = snapshot_module.backup_daily
def fail_backup(*args, **kwargs):
raise OSError("disk full")
snapshot_module.backup_daily = fail_backup
try:
code, output = run_cli(["snapshot", "--config", str(path)])
finally:
snapshot_module.backup_daily = original
assert code == 1
marker = output.find("{\n")
assert marker >= 0, output
payload = json.loads(output[marker:])
assert payload["completed"] is True
assert "disk full" in payload["backup_error"]
code, output = run_cli(["snapshot", "--config", str(path)])
assert code == 0
marker = output.find("{\n")
payload = json.loads(output[marker:])
assert payload["backup_error"] is None
assert Path(payload["backup_path"]).exists()
def main_test():
test_init_and_status()
test_missing_config()
@@ -182,6 +217,7 @@ def main_test():
test_auth_check_without_cookie()
test_auth_check_all_and_auth_dispatch()
test_run_once_offline()
test_snapshot_backup_failure_returns_error_and_retries()
print("OK: CLI init/sync/snapshot/auth-check/run/status/错误路径测试全部通过")
+4 -4
View File
@@ -24,7 +24,7 @@ def test_write_read_transaction():
with tempfile.TemporaryDirectory() as tmp:
db = make_db(tmp)
try:
assert db.schema_version == 1
assert db.schema_version == 2
db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')"))
assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1"
@@ -94,8 +94,8 @@ def test_close_and_reinitialize():
db2 = make_db(tmp)
try:
assert db2.schema_version == 1
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 1
assert db2.schema_version == 2
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 2
finally:
db2.close()
@@ -123,7 +123,7 @@ def test_migration_backup_on_existing_db():
db = Database(path, backup_dir=Path(tmp) / "backup")
try:
applied = db.initialize()
assert applied == [1]
assert applied == [1, 2]
assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1
finally:
db.close()
+100
View File
@@ -0,0 +1,100 @@
# -*- coding: utf-8 -*-
"""favsort run 参数校验与资源类型测试(不联网): uv run python tests/test_favsort_run.py"""
import os
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.stdout.reconfigure(encoding="utf-8")
import favsort # noqa: E402
class Args:
media_id = 1
verify_every = 1
def config():
return {
"mid": 1,
"csrf": "csrf",
"cookie": "cookie",
"overrides": {},
"sleep": {
"min": 0, "max": 0,
"long_every_min": 1000, "long_every_max": 1000,
"long_min": 0, "long_max": 0,
"retry_wait": 0,
},
}
def test_zero_verify_every_rejected_before_writes():
calls = []
originals = (favsort.load_plan, favsort.fetch_ids, favsort.post_form)
favsort.load_plan = lambda media_id: {"target_order": [], "titles": {}}
favsort.fetch_ids = lambda cfg, media_id: calls.append("fetch") or []
favsort.post_form = lambda *args, **kwargs: calls.append("post") or {}
args = Args()
args.verify_every = 0
try:
code = favsort.cmd_run(config(), args)
finally:
favsort.load_plan, favsort.fetch_ids, favsort.post_form = originals
assert code == 2
assert calls == []
def test_sort_preserves_resource_types():
with tempfile.TemporaryDirectory() as tmp:
plan = {
"media_id": 1,
"target_order": [1, 2],
"titles": {"1": {"title": "音频"}, "2": {"title": "视频"}},
"types": {1: 2, 2: 12},
}
bodies = []
orders = [[2, 1]]
originals = (
favsort.load_plan,
favsort.fetch_ids,
favsort.post_form,
favsort.plan_path,
favsort.journal_path,
)
favsort.load_plan = lambda media_id: plan
favsort.plan_path = lambda media_id: str(Path(tmp) / "plan.json")
favsort.journal_path = lambda media_id: str(Path(tmp) / "journal.jsonl")
def fetch_ids(cfg, media_id):
order = orders.pop(0) if orders else [1, 2]
return [{"id": aid} for aid in order]
favsort.fetch_ids = fetch_ids
favsort.post_form = lambda cfg, url, body: bodies.append(body) or {"code": 0}
args = Args()
try:
code = favsort.cmd_run(config(), args)
finally:
(
favsort.load_plan,
favsort.fetch_ids,
favsort.post_form,
favsort.plan_path,
favsort.journal_path,
) = originals
assert code == 0
assert [body["sort"] for body in bodies] == ["1:2:2:12"]
def main():
test_zero_verify_every_rejected_before_writes()
test_sort_preserves_resource_types()
print("OK: favsort run 参数校验与资源类型测试全部通过")
if __name__ == "__main__":
main()
+51 -1
View File
@@ -38,10 +38,11 @@ def http_json(payload):
class FakeFavTransport:
def __init__(self, folders, infos, *, fail_media=None):
def __init__(self, folders, infos, *, fail_media=None, omit_ids_data=None):
self.folders = {k: list(v) for k, v in folders.items()}
self.infos = dict(infos)
self.fail_media = fail_media
self.omit_ids_data = omit_ids_data
self.ids_calls = []
self.info_calls = []
@@ -52,6 +53,8 @@ class FakeFavTransport:
self.ids_calls.append(media_id)
if self.fail_media == media_id:
raise TransportError("ids 网络错误")
if self.omit_ids_data == media_id:
return http_json({"code": 0, "message": "OK", "ttl": 1})
data_rows = [
{"id": aid, "type": 2, "bv_id": f"BV{aid}", "bvid": f"BV{aid}"}
for aid in self.folders.get(media_id, [])
@@ -284,6 +287,51 @@ def test_blacklist_seed_only_once():
db.close()
def test_empty_remote_requires_explicit_opt_in():
with tempfile.TemporaryDirectory() as tmp:
transport = FakeFavTransport(
{F1: [1101, 1102], F2: []},
{1101: valid(1101), 1102: valid(1102)},
)
clock = [1_800_000_000]
db, repo, service = make_service(tmp, transport, clock)
try:
service.run()
transport.folders[F1] = []
clock[0] += 3600
summary = service.run()
assert summary.ok is False
assert "疑似接口异常" in summary.folders[0].error
assert repo.get_video(1101)["status"] == "active"
assert repo.folder_active_aids(F1) == {1101, 1102}
service.allow_empty_folders = True
summary = service.run()
assert summary.ok is True
assert repo.get_video(1101)["status"] == "unfaved"
assert repo.folder_active_aids(F1) == set()
finally:
db.close()
def test_missing_ids_data_aborts_folder():
with tempfile.TemporaryDirectory() as tmp:
transport = FakeFavTransport(
{F1: [1101], F2: []},
{1101: valid(1101)},
omit_ids_data=F1,
)
clock = [1_800_000_000]
db, repo, service = make_service(tmp, transport, clock)
try:
summary = service.run()
assert summary.ok is False
assert "data" in summary.folders[0].error
assert repo.get_video(1101) is None
finally:
db.close()
def main():
test_initial_import_and_incremental_skip()
test_diff_add_and_remove()
@@ -291,6 +339,8 @@ def main():
test_capacity_warning_does_not_move_folder()
test_folder_error_does_not_block_other_folders()
test_blacklist_seed_only_once()
test_empty_remote_requires_explicit_opt_in()
test_missing_ids_data_aborts_folder()
print("OK: 收藏夹初始导入、差集同步、失效识别与容量预警测试全部通过")
+11 -7
View File
@@ -37,14 +37,18 @@ def test_fresh_and_idempotent():
conn = sqlite3.connect(Path(tmp) / "app.db")
try:
applied = run_migrations(conn)
assert applied == [1]
assert current_version(conn) == 1
assert applied == [1, 2]
assert current_version(conn) == 2
assert EXPECTED_TABLES <= table_names(conn)
assert "schema_version" in table_names(conn)
columns = {
row[1] for row in conn.execute("PRAGMA table_info(folder_resource)")
}
assert "fav_time" in columns
# 再跑一次不应重复应用
assert run_migrations(conn) == []
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
assert row[0] == 1
assert row[0] == 2
finally:
conn.close()
@@ -59,15 +63,15 @@ def test_backup_before_upgrade():
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
conn.commit()
v2 = Migration(2, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
v3 = Migration(3, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
applied = run_migrations(
conn,
backup_dir=backup_dir,
now=1_800_000_000,
migrations=MIGRATIONS + (v2,),
migrations=MIGRATIONS + (v3,),
)
assert applied == [2]
backups = list(backup_dir.glob("app-pre-v1-*.db"))
assert applied == [3]
backups = list(backup_dir.glob("app-pre-v2-*.db"))
assert len(backups) == 1
check = sqlite3.connect(backups[0])
try:
+106
View File
@@ -0,0 +1,106 @@
# -*- coding: utf-8 -*-
"""凭据变化时客户端重建测试(不联网): uv run python tests/test_provider.py"""
import json
import os
import sys
import tempfile
import time
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.stdout.reconfigure(encoding="utf-8")
from reaction_board.bilibili.client import BilibiliClient, Pacer # noqa: E402
from reaction_board.bilibili.provider import ClientProvider # noqa: E402
from reaction_board.config import ConfigStore # noqa: E402
def write_config(path: Path, *, cookie: str, mid: int) -> None:
path.write_text(json.dumps({
"bilibili": {
"read": {"mid": mid, "csrf": "read-csrf", "cookie": cookie},
"write": {"mid": 2, "csrf": "write-csrf", "cookie": "write-cookie"},
},
"folders": [],
}, ensure_ascii=False), encoding="utf-8")
def bump(path: Path) -> None:
target = max(path.stat().st_mtime_ns, time.time_ns()) + 1_000_000
os.utime(path, ns=(target, target))
def test_rebuild_only_when_credentials_change():
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "config.json"
write_config(path, cookie="cookie-a", mid=1)
store = ConfigStore(path, env={})
created = []
def factory(credentials):
created.append((credentials.mid, credentials.cookie))
return BilibiliClient(
cookie=credentials.cookie,
csrf=credentials.csrf,
mid=credentials.mid,
pacer=Pacer(1.0, 1.6),
)
provider = ClientProvider(store, "read", factory)
first = provider.get()
assert provider.get() is first
assert created == [(1, "cookie-a")]
write_config(path, cookie="cookie-b", mid=1)
bump(path)
second = provider.get()
assert second is not first
assert created == [(1, "cookie-a"), (1, "cookie-b")]
assert second.pacer.min_delay == 1.0
assert second.pacer.max_delay == 1.6
# 仅 mtime 变化但凭据相同:不重建
bump(path)
assert provider.get() is second
assert len(created) == 2
def test_mid_change_rebuilds():
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "config.json"
write_config(path, cookie="cookie-a", mid=1)
store = ConfigStore(path, env={})
provider = ClientProvider(
store, "read",
lambda credentials: BilibiliClient(mid=credentials.mid),
)
assert provider.get().mid == 1
write_config(path, cookie="cookie-a", mid=9)
bump(path)
assert provider.get().mid == 9
def test_config_revision_increments():
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "config.json"
write_config(path, cookie="cookie-a", mid=1)
store = ConfigStore(path, env={})
assert store.revision == 0
write_config(path, cookie="cookie-b", mid=1)
bump(path)
store.get()
assert store.revision == 1
store.get()
assert store.revision == 1
def main():
test_rebuild_only_when_credentials_change()
test_mid_change_rebuilds()
test_config_revision_increments()
print("OK: 凭据变化重建客户端与配置 revision 测试全部通过")
if __name__ == "__main__":
main()
+71 -2
View File
@@ -18,7 +18,7 @@ F1 = FolderConfig(year=2025, media_id=4451053851, title="2025")
F2 = FolderConfig(year=2026, media_id=3700023351, title="2026")
def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None):
def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None):
return FavResource(
aid=aid,
bvid=bvid or f"BV{aid}",
@@ -29,7 +29,7 @@ def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None):
duration=600,
cover="http://example/cover.jpg",
attr=attr,
fav_time=None,
fav_time=fav_time,
)
@@ -227,6 +227,72 @@ def test_folder_sync_metadata_and_counts():
db.close()
def test_primary_folder_is_order_independent():
with tempfile.TemporaryDirectory() as tmp:
db, repo = setup(tmp)
try:
repo.upsert_from_fav(resource(1, fav_time=100), F1.media_id, was_added=True)
assert repo.get_video(1)["folder_id"] == F1.media_id
repo.upsert_from_fav(resource(1, fav_time=200), F2.media_id, was_added=True)
assert repo.get_video(1)["folder_id"] == F2.media_id
# 全量刷新按相反顺序遍历,主归属仍应指向 fav_time 更新的夹
repo.upsert_from_fav(resource(1, fav_time=100), F1.media_id, was_added=False)
assert repo.get_video(1)["folder_id"] == F2.media_id
repo.upsert_from_fav(resource(1, fav_time=200), F2.media_id, was_added=False)
assert repo.get_video(1)["folder_id"] == F2.media_id
# 从主夹移除、另一个夹仍持有时,主归属切换
status = repo.mark_resource_removed(F2.media_id, 1)
assert status == "active"
assert repo.get_video(1)["folder_id"] == F1.media_id
finally:
db.close()
def test_dropped_invalid_preserves_original_title():
with tempfile.TemporaryDirectory() as tmp:
db, repo = setup(tmp)
try:
repo.upsert_from_fav(
resource(1, title="原始标题"), F1.media_id, was_added=True
)
repo.set_video_status(1, "dropped")
repo.upsert_from_fav(
resource(1, title="已失效视频", attr=9),
F1.media_id,
was_added=False,
)
video = repo.get_video(1)
assert video["status"] == "dropped"
assert video["title"] == "原始标题"
finally:
db.close()
def test_pending_snapshot_aids():
with tempfile.TemporaryDirectory() as tmp:
db, repo = setup(tmp)
try:
repo.upsert_from_fav(resource(1), F1.media_id, was_added=True)
repo.upsert_from_fav(resource(2), F1.media_id, was_added=True)
repo.upsert_from_fav(resource(3), F1.media_id, was_added=True)
repo.set_video_status(3, "unfaved")
repo.upsert_snapshot(
"2026-10-31", 1,
metrics={"view": 1, "like": 1, "favorite": 1, "coin": 1,
"share": 1, "reply": 1, "danmaku": 1},
ok=True, err=None, captured_at=1,
)
repo.upsert_snapshot(
"2026-10-31", 2,
metrics=None, ok=False, err="x", captured_at=1,
)
assert repo.pending_snapshot_aids("2026-10-31") == [2, 3]
finally:
db.close()
def main():
test_upsert_and_folder_resource()
test_invalid_attr_and_missing()
@@ -236,6 +302,9 @@ def main():
test_multi_folder_moves_primary_and_warns()
test_blacklist_seed_idempotent()
test_folder_sync_metadata_and_counts()
test_primary_folder_is_order_independent()
test_dropped_invalid_preserves_original_title()
test_pending_snapshot_aids()
print("OK: 收藏夹/视频状态机与黑名单测试全部通过")
+115
View File
@@ -60,6 +60,8 @@ class SnapshotTransport:
urllib.parse.urlparse(url).query
)["aid"][0])
payload = self.views[aid]
if isinstance(payload, BaseException):
raise payload
if isinstance(payload, HTTPResponse):
return payload
return http_json(payload)
@@ -316,6 +318,115 @@ def test_force_failure_clears_last_completed():
db.close()
def test_new_video_after_completion_backfills_same_day():
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({1: view_for(1, view=100)})
db, repo, service = make_service(tmp, transport, track_fans=())
try:
add_video(repo, 1)
first = service.run(date=DATE)
assert first.completed is True
assert transport.count("/web-interface/view") == 1
# 当天 11:00 新入库:完成标记仍是 1,但必须补采新视频
add_video(repo, 2)
transport.views[2] = view_for(2, view=222)
second = service.run_if_needed(date=DATE)
assert second.already_completed is False
assert second.success == 1
assert second.skipped == 1
assert transport.count("/web-interface/view") == 2
assert repo.get_snapshot(DATE, 2)["view"] == 222
assert repo.get_snapshot(DATE, 1)["view"] == 100
finally:
db.close()
def test_fan_resume_preserves_successful_value():
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport(
{1: {"code": -500, "message": "x", "data": None}},
)
db, repo, service = make_service(tmp, transport, retries=1)
try:
add_video(repo, 1)
first = service.run(date=DATE)
assert first.failed == 1
assert first.fans_ok == 1
assert repo.get_up_stat(DATE, 37754047)["follower"] == 2767268
transport.views[1] = view_for(1, view=111)
transport.fan = {"code": -500, "message": "x", "data": None}
second = service.run(date=DATE)
assert second.fans_skipped == 1
assert second.fans_failed == 0
fan = repo.get_up_stat(DATE, 37754047)
assert fan["follower"] == 2767268
assert fan["ok"] == 1
finally:
db.close()
def test_force_interrupt_leaves_incomplete_marker():
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({
1: view_for(1),
2: RuntimeError("simulated interrupt"),
})
db, repo, service = make_service(tmp, transport, track_fans=())
try:
add_video(repo, 1)
add_video(repo, 2)
repo.kv_set(service.completed_key(DATE), "1")
repo.kv_set("snapshot:last_completed", DATE)
try:
service.run(date=DATE, force=True)
except RuntimeError:
pass
else:
raise AssertionError("应模拟中断")
assert repo.kv_get(service.completed_key(DATE)) == "0"
assert repo.kv_get("snapshot:last_completed") == ""
transport.views[2] = view_for(2)
resumed = service.run_if_needed(date=DATE)
assert resumed.completed is True
assert resumed.success == 1
assert resumed.skipped == 1
finally:
db.close()
def test_backup_failure_retries_without_api():
import reaction_board.services.snapshot as snapshot_module
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({1: view_for(1)})
db, repo, service = make_service(tmp, transport, track_fans=())
original = snapshot_module.backup_daily
def fail_backup(*args, **kwargs):
raise OSError("disk full")
snapshot_module.backup_daily = fail_backup
try:
first = service.run(date=DATE)
finally:
snapshot_module.backup_daily = original
assert first.completed is True
assert first.backup_error
assert repo.kv_get(service.backup_key(DATE)) == "0"
view_calls = transport.count("/web-interface/view")
retried = service.run_if_needed(date=DATE)
assert retried.already_completed is True
assert retried.backup_error is None
assert Path(retried.backup_path or "").exists()
assert repo.kv_get(service.backup_key(DATE)) == "1"
assert transport.count("/web-interface/view") == view_calls
db.close()
def main():
test_snapshot_success_resume_and_force()
test_resume_skips_ok_and_retries_failed()
@@ -325,6 +436,10 @@ def main():
test_auth_invalid_is_warned_but_snapshot_continues()
test_no_cookie_skips_nav_request()
test_force_failure_clears_last_completed()
test_new_video_after_completion_backfills_same_day()
test_fan_resume_preserves_successful_value()
test_force_interrupt_leaves_incomplete_marker()
test_backup_failure_retries_without_api()
print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过")