fix: close data integrity and hot-reload review findings

Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
This commit is contained in:
kairee committed 2026-10-31 20:58:36 +08:00
1 parent 178ab64b18
commit 6a21b5d905
21 files changed
+895 -82

No files matched your search

+115
View File
@@ -60,6 +60,8 @@ class SnapshotTransport:
urllib.parse.urlparse(url).query
)["aid"][0])
payload = self.views[aid]
if isinstance(payload, BaseException):
raise payload
if isinstance(payload, HTTPResponse):
return payload
return http_json(payload)
@@ -316,6 +318,115 @@ def test_force_failure_clears_last_completed():
db.close()
def test_new_video_after_completion_backfills_same_day():
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({1: view_for(1, view=100)})
db, repo, service = make_service(tmp, transport, track_fans=())
try:
add_video(repo, 1)
first = service.run(date=DATE)
assert first.completed is True
assert transport.count("/web-interface/view") == 1
# 当天 11:00 新入库:完成标记仍是 1,但必须补采新视频
add_video(repo, 2)
transport.views[2] = view_for(2, view=222)
second = service.run_if_needed(date=DATE)
assert second.already_completed is False
assert second.success == 1
assert second.skipped == 1
assert transport.count("/web-interface/view") == 2
assert repo.get_snapshot(DATE, 2)["view"] == 222
assert repo.get_snapshot(DATE, 1)["view"] == 100
finally:
db.close()
def test_fan_resume_preserves_successful_value():
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport(
{1: {"code": -500, "message": "x", "data": None}},
)
db, repo, service = make_service(tmp, transport, retries=1)
try:
add_video(repo, 1)
first = service.run(date=DATE)
assert first.failed == 1
assert first.fans_ok == 1
assert repo.get_up_stat(DATE, 37754047)["follower"] == 2767268
transport.views[1] = view_for(1, view=111)
transport.fan = {"code": -500, "message": "x", "data": None}
second = service.run(date=DATE)
assert second.fans_skipped == 1
assert second.fans_failed == 0
fan = repo.get_up_stat(DATE, 37754047)
assert fan["follower"] == 2767268
assert fan["ok"] == 1
finally:
db.close()
def test_force_interrupt_leaves_incomplete_marker():
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({
1: view_for(1),
2: RuntimeError("simulated interrupt"),
})
db, repo, service = make_service(tmp, transport, track_fans=())
try:
add_video(repo, 1)
add_video(repo, 2)
repo.kv_set(service.completed_key(DATE), "1")
repo.kv_set("snapshot:last_completed", DATE)
try:
service.run(date=DATE, force=True)
except RuntimeError:
pass
else:
raise AssertionError("应模拟中断")
assert repo.kv_get(service.completed_key(DATE)) == "0"
assert repo.kv_get("snapshot:last_completed") == ""
transport.views[2] = view_for(2)
resumed = service.run_if_needed(date=DATE)
assert resumed.completed is True
assert resumed.success == 1
assert resumed.skipped == 1
finally:
db.close()
def test_backup_failure_retries_without_api():
import reaction_board.services.snapshot as snapshot_module
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({1: view_for(1)})
db, repo, service = make_service(tmp, transport, track_fans=())
original = snapshot_module.backup_daily
def fail_backup(*args, **kwargs):
raise OSError("disk full")
snapshot_module.backup_daily = fail_backup
try:
first = service.run(date=DATE)
finally:
snapshot_module.backup_daily = original
assert first.completed is True
assert first.backup_error
assert repo.kv_get(service.backup_key(DATE)) == "0"
view_calls = transport.count("/web-interface/view")
retried = service.run_if_needed(date=DATE)
assert retried.already_completed is True
assert retried.backup_error is None
assert Path(retried.backup_path or "").exists()
assert repo.kv_get(service.backup_key(DATE)) == "1"
assert transport.count("/web-interface/view") == view_calls
db.close()
def main():
test_snapshot_success_resume_and_force()
test_resume_skips_ok_and_retries_failed()
@@ -325,6 +436,10 @@ def main():
test_auth_invalid_is_warned_but_snapshot_continues()
test_no_cookie_skips_nav_request()
test_force_failure_clears_last_completed()
test_new_video_after_completion_backfills_same_day()
test_fan_resume_preserves_successful_value()
test_force_interrupt_leaves_incomplete_marker()
test_backup_failure_retries_without_api()
print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过")