kairee
6a21b5d905
Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
349 lines
13 KiB
Python
349 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""收藏夹同步集成测试（用真实接口形状的 transport Mock，不联网）:
|
||
uv run python tests/test_folder_sync.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sys
|
||
import tempfile
|
||
import urllib.parse
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
sys.stdout.reconfigure(encoding="utf-8")
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
from fixtures import info_for # noqa: E402
|
||
from reaction_board.bilibili.client import ( # noqa: E402
|
||
BilibiliClient,
|
||
HTTPResponse,
|
||
Pacer,
|
||
TransportError,
|
||
)
|
||
from reaction_board.config import ConfigStore # noqa: E402
|
||
from reaction_board.database import Database # noqa: E402
|
||
from reaction_board.repository import Repository # noqa: E402
|
||
from reaction_board.services.folder_sync import FolderSyncService # noqa: E402
|
||
|
||
F1 = 4451053851
|
||
F2 = 3700023351
|
||
|
||
|
||
def http_json(payload):
|
||
return HTTPResponse(
|
||
200, {"content-type": "application/json"},
|
||
json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||
)
|
||
|
||
|
||
class FakeFavTransport:
|
||
def __init__(self, folders, infos, *, fail_media=None, omit_ids_data=None):
|
||
self.folders = {k: list(v) for k, v in folders.items()}
|
||
self.infos = dict(infos)
|
||
self.fail_media = fail_media
|
||
self.omit_ids_data = omit_ids_data
|
||
self.ids_calls = []
|
||
self.info_calls = []
|
||
|
||
def request(self, method, url, headers, data=None, timeout=30):
|
||
query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||
if "/x/v3/fav/resource/ids" in url:
|
||
media_id = int(query["media_id"][0])
|
||
self.ids_calls.append(media_id)
|
||
if self.fail_media == media_id:
|
||
raise TransportError("ids 网络错误")
|
||
if self.omit_ids_data == media_id:
|
||
return http_json({"code": 0, "message": "OK", "ttl": 1})
|
||
data_rows = [
|
||
{"id": aid, "type": 2, "bv_id": f"BV{aid}", "bvid": f"BV{aid}"}
|
||
for aid in self.folders.get(media_id, [])
|
||
]
|
||
return http_json({"code": 0, "message": "0", "ttl": 1, "data": data_rows})
|
||
if "/x/v3/fav/resource/infos" in url:
|
||
media_id = int(query["folder_id"][0])
|
||
resources = query["resources"][0]
|
||
aids = [int(part.split(":")[0]) for part in resources.split(",")]
|
||
self.info_calls.append((media_id, aids))
|
||
rows = [self.infos[aid] for aid in aids if aid in self.infos]
|
||
return http_json({"code": 0, "message": "0", "ttl": 1, "data": rows})
|
||
if "/x/v3/fav/folder/created/list-all" in url:
|
||
return http_json({
|
||
"code": 0, "message": "0", "ttl": 1,
|
||
"data": {"count": 0, "list": []},
|
||
})
|
||
raise AssertionError(f"未预期的 URL: {url}")
|
||
|
||
|
||
def write_config(tmp, folders):
|
||
data = {
|
||
"bilibili": {
|
||
"read": {"mid": 28802351, "csrf": "csrf-secret", "cookie": "SESSDATA=secret; bili_jct=csrf-secret; buvid3=device"},
|
||
"write": {"mid": 2, "csrf": "write-csrf", "cookie": "write-cookie"},
|
||
},
|
||
"storage": {
|
||
"database_path": str(Path(tmp) / "app.db"),
|
||
"backup_dir": str(Path(tmp) / "backup"),
|
||
},
|
||
"folders": folders,
|
||
"up_blacklist_seed": [999],
|
||
"schedule": {
|
||
"discover_minutes": 60,
|
||
"folder_sync_minutes": 60,
|
||
"full_folder_refresh_hours": 6,
|
||
"snapshot_at": "00:01",
|
||
"candidate_ttl_days": 7,
|
||
},
|
||
}
|
||
path = Path(tmp) / "config.json"
|
||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def make_service(tmp, transport, clock, *, capacity_warn=950):
|
||
folders = [
|
||
{"year": 2025, "media_id": F1, "title": "2025"},
|
||
{"year": 2026, "media_id": F2, "title": "2026"},
|
||
]
|
||
path = write_config(tmp, folders)
|
||
store = ConfigStore(path, env={}, require_auth=True)
|
||
db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup")
|
||
db.initialize()
|
||
repo = Repository(db, clock=lambda: clock[0])
|
||
client = BilibiliClient(
|
||
cookie="SESSDATA=secret; bvid3=device",
|
||
csrf="csrf",
|
||
mid=28802351,
|
||
transport=transport,
|
||
pacer=Pacer(0, 0),
|
||
retries=1,
|
||
)
|
||
service = FolderSyncService(
|
||
store, repo, client, clock=lambda: clock[0], capacity_warn=capacity_warn
|
||
)
|
||
return db, repo, service
|
||
|
||
|
||
def valid(aid, *, up_mid=100, title=None):
|
||
return info_for(aid, title=title or f"视频{aid}", up_mid=up_mid, up_name="UP")
|
||
|
||
|
||
def test_initial_import_and_incremental_skip():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101, 1102, 1103, 1104], F2: []},
|
||
{
|
||
1101: valid(1101),
|
||
1102: valid(1102, up_mid=999), # 黑名单 UP 也必须正常入库
|
||
1103: valid(1103, title="已失效", up_mid=100),
|
||
},
|
||
)
|
||
# 1103 标记为 attr=9；1104 静默缺失
|
||
transport.infos[1103]["attr"] = 9
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
summary = service.run()
|
||
assert summary.ok
|
||
assert summary.added == 4
|
||
assert summary.missing == 1
|
||
assert summary.refreshed == 3
|
||
assert repo.seed_blacklist([999], now=0) == 0 # 已由 service 写入
|
||
|
||
assert repo.get_video(1101)["status"] == "active"
|
||
assert repo.get_video(1102)["status"] == "active" # 黑名单不影响收藏夹导入
|
||
assert repo.get_video(1103)["status"] == "removed"
|
||
assert repo.get_video(1104)["status"] == "removed"
|
||
assert repo.get_video(1104)["bvid"] == "BV1104"
|
||
assert repo.folder_active_aids(F1) == {1101, 1102, 1103, 1104}
|
||
assert repo.is_blacklisted(999)
|
||
|
||
folder = repo.get_folder(F1)
|
||
assert folder.media_count == 4
|
||
assert folder.last_full_at == 1_800_000_000
|
||
|
||
# 6 小时内再次运行：没有新增，不请求 infos
|
||
clock[0] += 60
|
||
info_before = len(transport.info_calls)
|
||
summary2 = service.run()
|
||
assert summary2.added == 0
|
||
assert summary2.refreshed == 0
|
||
assert len(transport.info_calls) == info_before
|
||
assert summary2.folders[0].full_refresh is False
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_diff_add_and_remove():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101, 1102], F2: []},
|
||
{1101: valid(1101), 1102: valid(1102)},
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
service.run()
|
||
assert repo.get_video(1101)["status"] == "active"
|
||
assert repo.get_video(1102)["status"] == "active"
|
||
|
||
transport.folders[F1] = [1102, 1103]
|
||
transport.infos[1103] = valid(1103)
|
||
clock[0] += 3600
|
||
summary = service.run()
|
||
assert summary.added == 1
|
||
assert summary.removed == 1
|
||
assert summary.refreshed == 1
|
||
assert repo.get_video(1101)["status"] == "unfaved"
|
||
assert repo.get_video(1102)["status"] == "active"
|
||
assert repo.get_video(1103)["status"] == "active"
|
||
assert repo.folder_active_aids(F1) == {1102, 1103}
|
||
assert repo.get_folder(F1).media_count == 2
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_full_refresh_marks_invalid_and_missing():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101, 1102], F2: []},
|
||
{1101: valid(1101), 1102: valid(1102)},
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
service.run()
|
||
assert repo.get_video(1101)["status"] == "active"
|
||
assert repo.get_video(1102)["status"] == "active"
|
||
|
||
# 6 小时后：1101 失效，1102 静默缺失
|
||
clock[0] += 6 * 3600 + 1
|
||
transport.infos[1101]["attr"] = 9
|
||
del transport.infos[1102]
|
||
summary = service.run()
|
||
assert summary.folders[0].full_refresh is True
|
||
assert summary.missing == 1
|
||
assert repo.get_video(1101)["status"] == "removed"
|
||
assert repo.get_video(1102)["status"] == "removed"
|
||
assert repo.get_folder(F1).last_full_at == clock[0]
|
||
# 全量刷新请求了夹内全部 aid
|
||
assert set(transport.info_calls[-1][1]) == {1101, 1102}
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_capacity_warning_does_not_move_folder():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1, 2, 3], F2: []},
|
||
{i: valid(i) for i in (1, 2, 3)},
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock, capacity_warn=3)
|
||
try:
|
||
summary = service.run()
|
||
assert summary.folders[0].warning
|
||
assert "预警线 3" in summary.folders[0].warning
|
||
assert "不会自动换夹" in summary.folders[0].warning
|
||
# 视频仍在原夹，不被移动
|
||
assert all(repo.get_video(i)["folder_id"] == F1 for i in (1, 2, 3))
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_folder_error_does_not_block_other_folders():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101], F2: [2201]},
|
||
{1101: valid(1101), 2201: valid(2201)},
|
||
fail_media=F1,
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
summary = service.run()
|
||
assert summary.ok is False
|
||
assert len(summary.errors) == 1 and str(F1) in summary.errors[0]
|
||
assert summary.folders[0].error
|
||
assert summary.folders[1].error is None
|
||
assert repo.get_video(2201)["status"] == "active"
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_blacklist_seed_only_once():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport({F1: [1101], F2: []}, {1101: valid(1101)})
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
first = service.run()
|
||
assert first.blacklist_seeded == 1
|
||
clock[0] += 60
|
||
second = service.run()
|
||
assert second.blacklist_seeded == 0
|
||
assert len(repo.blacklist()) == 1
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_empty_remote_requires_explicit_opt_in():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101, 1102], F2: []},
|
||
{1101: valid(1101), 1102: valid(1102)},
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
service.run()
|
||
transport.folders[F1] = []
|
||
clock[0] += 3600
|
||
summary = service.run()
|
||
assert summary.ok is False
|
||
assert "疑似接口异常" in summary.folders[0].error
|
||
assert repo.get_video(1101)["status"] == "active"
|
||
assert repo.folder_active_aids(F1) == {1101, 1102}
|
||
|
||
service.allow_empty_folders = True
|
||
summary = service.run()
|
||
assert summary.ok is True
|
||
assert repo.get_video(1101)["status"] == "unfaved"
|
||
assert repo.folder_active_aids(F1) == set()
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_missing_ids_data_aborts_folder():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101], F2: []},
|
||
{1101: valid(1101)},
|
||
omit_ids_data=F1,
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
summary = service.run()
|
||
assert summary.ok is False
|
||
assert "data" in summary.folders[0].error
|
||
assert repo.get_video(1101) is None
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def main():
|
||
test_initial_import_and_incremental_skip()
|
||
test_diff_add_and_remove()
|
||
test_full_refresh_marks_invalid_and_missing()
|
||
test_capacity_warning_does_not_move_folder()
|
||
test_folder_error_does_not_block_other_folders()
|
||
test_blacklist_seed_only_once()
|
||
test_empty_remote_requires_explicit_opt_in()
|
||
test_missing_ids_data_aborts_folder()
|
||
print("OK: 收藏夹初始导入、差集同步、失效识别与容量预警测试全部通过")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|