fix: close data integrity and hot-reload review findings
Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
This commit is contained in:
1 parent
178ab64b18
commit
6a21b5d905
21 files changed
+895
-82
No files matched your search
@@ -60,6 +60,8 @@ class SnapshotTransport:
|
||||
urllib.parse.urlparse(url).query
|
||||
)["aid"][0])
|
||||
payload = self.views[aid]
|
||||
if isinstance(payload, BaseException):
|
||||
raise payload
|
||||
if isinstance(payload, HTTPResponse):
|
||||
return payload
|
||||
return http_json(payload)
|
||||
@@ -316,6 +318,115 @@ def test_force_failure_clears_last_completed():
|
||||
db.close()
|
||||
|
||||
|
||||
def test_new_video_after_completion_backfills_same_day():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({1: view_for(1, view=100)})
|
||||
db, repo, service = make_service(tmp, transport, track_fans=())
|
||||
try:
|
||||
add_video(repo, 1)
|
||||
first = service.run(date=DATE)
|
||||
assert first.completed is True
|
||||
assert transport.count("/web-interface/view") == 1
|
||||
|
||||
# 当天 11:00 新入库:完成标记仍是 1,但必须补采新视频
|
||||
add_video(repo, 2)
|
||||
transport.views[2] = view_for(2, view=222)
|
||||
second = service.run_if_needed(date=DATE)
|
||||
assert second.already_completed is False
|
||||
assert second.success == 1
|
||||
assert second.skipped == 1
|
||||
assert transport.count("/web-interface/view") == 2
|
||||
assert repo.get_snapshot(DATE, 2)["view"] == 222
|
||||
assert repo.get_snapshot(DATE, 1)["view"] == 100
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_fan_resume_preserves_successful_value():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport(
|
||||
{1: {"code": -500, "message": "x", "data": None}},
|
||||
)
|
||||
db, repo, service = make_service(tmp, transport, retries=1)
|
||||
try:
|
||||
add_video(repo, 1)
|
||||
first = service.run(date=DATE)
|
||||
assert first.failed == 1
|
||||
assert first.fans_ok == 1
|
||||
assert repo.get_up_stat(DATE, 37754047)["follower"] == 2767268
|
||||
|
||||
transport.views[1] = view_for(1, view=111)
|
||||
transport.fan = {"code": -500, "message": "x", "data": None}
|
||||
second = service.run(date=DATE)
|
||||
assert second.fans_skipped == 1
|
||||
assert second.fans_failed == 0
|
||||
fan = repo.get_up_stat(DATE, 37754047)
|
||||
assert fan["follower"] == 2767268
|
||||
assert fan["ok"] == 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_force_interrupt_leaves_incomplete_marker():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({
|
||||
1: view_for(1),
|
||||
2: RuntimeError("simulated interrupt"),
|
||||
})
|
||||
db, repo, service = make_service(tmp, transport, track_fans=())
|
||||
try:
|
||||
add_video(repo, 1)
|
||||
add_video(repo, 2)
|
||||
repo.kv_set(service.completed_key(DATE), "1")
|
||||
repo.kv_set("snapshot:last_completed", DATE)
|
||||
try:
|
||||
service.run(date=DATE, force=True)
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("应模拟中断")
|
||||
assert repo.kv_get(service.completed_key(DATE)) == "0"
|
||||
assert repo.kv_get("snapshot:last_completed") == ""
|
||||
|
||||
transport.views[2] = view_for(2)
|
||||
resumed = service.run_if_needed(date=DATE)
|
||||
assert resumed.completed is True
|
||||
assert resumed.success == 1
|
||||
assert resumed.skipped == 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_backup_failure_retries_without_api():
|
||||
import reaction_board.services.snapshot as snapshot_module
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = SnapshotTransport({1: view_for(1)})
|
||||
db, repo, service = make_service(tmp, transport, track_fans=())
|
||||
original = snapshot_module.backup_daily
|
||||
|
||||
def fail_backup(*args, **kwargs):
|
||||
raise OSError("disk full")
|
||||
|
||||
snapshot_module.backup_daily = fail_backup
|
||||
try:
|
||||
first = service.run(date=DATE)
|
||||
finally:
|
||||
snapshot_module.backup_daily = original
|
||||
assert first.completed is True
|
||||
assert first.backup_error
|
||||
assert repo.kv_get(service.backup_key(DATE)) == "0"
|
||||
view_calls = transport.count("/web-interface/view")
|
||||
|
||||
retried = service.run_if_needed(date=DATE)
|
||||
assert retried.already_completed is True
|
||||
assert retried.backup_error is None
|
||||
assert Path(retried.backup_path or "").exists()
|
||||
assert repo.kv_get(service.backup_key(DATE)) == "1"
|
||||
assert transport.count("/web-interface/view") == view_calls
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
test_snapshot_success_resume_and_force()
|
||||
test_resume_skips_ok_and_retries_failed()
|
||||
@@ -325,6 +436,10 @@ def main():
|
||||
test_auth_invalid_is_warned_but_snapshot_continues()
|
||||
test_no_cookie_skips_nav_request()
|
||||
test_force_failure_clears_last_completed()
|
||||
test_new_video_after_completion_backfills_same_day()
|
||||
test_fan_resume_preserves_successful_value()
|
||||
test_force_interrupt_leaves_incomplete_marker()
|
||||
test_backup_failure_retries_without_api()
|
||||
print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过")
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user