374 lines
14 KiB
Python
374 lines
14 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""收藏夹同步集成测试（用真实接口形状的 transport Mock，不联网）:
|
||
uv run python tests/test_folder_sync.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sys
|
||
import tempfile
|
||
import urllib.parse
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
sys.stdout.reconfigure(encoding="utf-8")
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
from fixtures import info_for # noqa: E402
|
||
from reaction_board.bilibili.client import ( # noqa: E402
|
||
BilibiliClient,
|
||
HTTPResponse,
|
||
Pacer,
|
||
TransportError,
|
||
)
|
||
from reaction_board.config import ConfigStore # noqa: E402
|
||
from reaction_board.database import Database # noqa: E402
|
||
from reaction_board.repository import Repository # noqa: E402
|
||
from reaction_board.services.folder_sync import FolderSyncService # noqa: E402
|
||
|
||
F1 = 4451053851
|
||
F2 = 3700023351
|
||
|
||
|
||
def http_json(payload):
|
||
return HTTPResponse(
|
||
200, {"content-type": "application/json"},
|
||
json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||
)
|
||
|
||
|
||
class FakeFavTransport:
|
||
def __init__(self, folders, infos, *, fail_media=None, omit_ids_data=None):
|
||
self.folders = {k: list(v) for k, v in folders.items()}
|
||
self.infos = dict(infos)
|
||
self.fail_media = fail_media
|
||
self.omit_ids_data = omit_ids_data
|
||
self.ids_calls = []
|
||
self.info_calls = []
|
||
self.relation_calls = []
|
||
|
||
def request(self, method, url, headers, data=None, timeout=30):
|
||
query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||
if "/x/relation/stat" in url:
|
||
mid = int(query["vmid"][0])
|
||
self.relation_calls.append(mid)
|
||
return http_json({
|
||
"code": 0, "message": "0", "ttl": 1,
|
||
"data": {"mid": mid, "following": 5, "follower": 1000 + mid},
|
||
})
|
||
if "/x/v3/fav/resource/ids" in url:
|
||
media_id = int(query["media_id"][0])
|
||
self.ids_calls.append(media_id)
|
||
if self.fail_media == media_id:
|
||
raise TransportError("ids 网络错误")
|
||
if self.omit_ids_data == media_id:
|
||
return http_json({"code": 0, "message": "OK", "ttl": 1})
|
||
data_rows = [
|
||
{"id": aid, "type": 2, "bv_id": f"BV{aid}", "bvid": f"BV{aid}"}
|
||
for aid in self.folders.get(media_id, [])
|
||
]
|
||
return http_json({"code": 0, "message": "0", "ttl": 1, "data": data_rows})
|
||
if "/x/v3/fav/resource/infos" in url:
|
||
media_id = int(query["folder_id"][0])
|
||
resources = query["resources"][0]
|
||
aids = [int(part.split(":")[0]) for part in resources.split(",")]
|
||
self.info_calls.append((media_id, aids))
|
||
rows = [self.infos[aid] for aid in aids if aid in self.infos]
|
||
return http_json({"code": 0, "message": "0", "ttl": 1, "data": rows})
|
||
if "/x/v3/fav/folder/created/list-all" in url:
|
||
return http_json({
|
||
"code": 0, "message": "0", "ttl": 1,
|
||
"data": {"count": 0, "list": []},
|
||
})
|
||
raise AssertionError(f"未预期的 URL: {url}")
|
||
|
||
|
||
def write_config(tmp, folders):
|
||
data = {
|
||
"bilibili": {
|
||
"read": {"mid": 28802351, "csrf": "csrf-secret", "cookie": "SESSDATA=secret; bili_jct=csrf-secret; buvid3=device"},
|
||
"write": {"mid": 2, "csrf": "write-csrf", "cookie": "write-cookie"},
|
||
},
|
||
"storage": {
|
||
"database_path": str(Path(tmp) / "app.db"),
|
||
"backup_dir": str(Path(tmp) / "backup"),
|
||
},
|
||
"folders": folders,
|
||
"up_blacklist_seed": [999],
|
||
"schedule": {
|
||
"discover_minutes": 60,
|
||
"folder_sync_minutes": 60,
|
||
"full_folder_refresh_hours": 6,
|
||
"snapshot_at": "00:01",
|
||
"candidate_ttl_days": 7,
|
||
},
|
||
}
|
||
path = Path(tmp) / "config.json"
|
||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def make_service(tmp, transport, clock, *, capacity_warn=950):
|
||
folders = [
|
||
{"year": 2025, "media_id": F1, "title": "2025"},
|
||
{"year": 2026, "media_id": F2, "title": "2026"},
|
||
]
|
||
path = write_config(tmp, folders)
|
||
store = ConfigStore(path, env={}, require_auth=True)
|
||
db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup")
|
||
db.initialize()
|
||
repo = Repository(db, clock=lambda: clock[0])
|
||
client = BilibiliClient(
|
||
cookie="SESSDATA=secret; bvid3=device",
|
||
csrf="csrf",
|
||
mid=28802351,
|
||
transport=transport,
|
||
pacer=Pacer(0, 0),
|
||
retries=1,
|
||
)
|
||
service = FolderSyncService(
|
||
store, repo, client, clock=lambda: clock[0], capacity_warn=capacity_warn
|
||
)
|
||
return db, repo, service
|
||
|
||
|
||
def valid(aid, *, up_mid=100, title=None):
|
||
return info_for(aid, title=title or f"视频{aid}", up_mid=up_mid, up_name="UP")
|
||
|
||
|
||
def test_initial_import_and_incremental_skip():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101, 1102, 1103, 1104], F2: []},
|
||
{
|
||
1101: valid(1101),
|
||
1102: valid(1102, up_mid=999), # 黑名单 UP 也必须正常入库
|
||
1103: valid(1103, title="已失效", up_mid=100),
|
||
},
|
||
)
|
||
# 1103 标记为 attr=9；1104 静默缺失
|
||
transport.infos[1103]["attr"] = 9
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
repo.upsert_candidate(
|
||
aid=1101, bvid="BV1101", title="咻咻满 reaction",
|
||
up_mid=100, up_name="UP", pubdate=1_799_000_000,
|
||
cover=None, score=3, channel="search:test#p1",
|
||
now=1_799_999_000,
|
||
)
|
||
summary = service.run()
|
||
assert summary.ok
|
||
assert summary.added == 4
|
||
assert summary.missing == 1
|
||
assert summary.refreshed == 3
|
||
# 人工加入收藏夹后，同 aid 候选自动移除
|
||
assert repo.get_candidate(1101) is None
|
||
assert repo.seed_blacklist([999], now=0) == 0 # 已由 service 写入
|
||
|
||
assert repo.get_video(1101)["status"] == "active"
|
||
assert repo.get_video(1102)["status"] == "active" # 黑名单不影响收藏夹导入
|
||
assert repo.get_video(1103)["status"] == "removed"
|
||
assert repo.get_video(1104)["status"] == "removed"
|
||
assert repo.get_video(1104)["bvid"] == "BV1104"
|
||
assert repo.folder_active_aids(F1) == {1101, 1102, 1103, 1104}
|
||
assert repo.is_blacklisted(999)
|
||
# 新入库视频记录入库时粉丝数；失效视频不再采集
|
||
assert summary.followers_updated == 2
|
||
assert summary.followers_failed == 0
|
||
assert repo.get_video(1101)["up_follower"] == 1100
|
||
assert repo.get_video(1101)["up_follower_backfilled"] == 0
|
||
assert repo.get_video(1102)["up_follower"] == 1999
|
||
assert sorted(transport.relation_calls) == [100, 999]
|
||
|
||
folder = repo.get_folder(F1)
|
||
assert folder.media_count == 4
|
||
assert folder.last_full_at == 1_800_000_000
|
||
|
||
# 6 小时内再次运行：没有新增，不请求 infos
|
||
clock[0] += 60
|
||
info_before = len(transport.info_calls)
|
||
relation_before = len(transport.relation_calls)
|
||
summary2 = service.run()
|
||
assert summary2.added == 0
|
||
assert summary2.refreshed == 0
|
||
assert len(transport.info_calls) == info_before
|
||
assert len(transport.relation_calls) == relation_before
|
||
assert summary2.folders[0].full_refresh is False
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_diff_add_and_remove():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101, 1102], F2: []},
|
||
{1101: valid(1101), 1102: valid(1102)},
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
service.run()
|
||
assert repo.get_video(1101)["status"] == "active"
|
||
assert repo.get_video(1102)["status"] == "active"
|
||
|
||
transport.folders[F1] = [1102, 1103]
|
||
transport.infos[1103] = valid(1103)
|
||
clock[0] += 3600
|
||
summary = service.run()
|
||
assert summary.added == 1
|
||
assert summary.removed == 1
|
||
assert summary.refreshed == 1
|
||
assert repo.get_video(1101)["status"] == "unfaved"
|
||
assert repo.get_video(1102)["status"] == "active"
|
||
assert repo.get_video(1103)["status"] == "active"
|
||
assert repo.folder_active_aids(F1) == {1102, 1103}
|
||
assert repo.get_folder(F1).media_count == 2
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_full_refresh_marks_invalid_and_missing():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101, 1102], F2: []},
|
||
{1101: valid(1101), 1102: valid(1102)},
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
service.run()
|
||
assert repo.get_video(1101)["status"] == "active"
|
||
assert repo.get_video(1102)["status"] == "active"
|
||
|
||
# 6 小时后：1101 失效，1102 静默缺失
|
||
clock[0] += 6 * 3600 + 1
|
||
transport.infos[1101]["attr"] = 9
|
||
del transport.infos[1102]
|
||
summary = service.run()
|
||
assert summary.folders[0].full_refresh is True
|
||
assert summary.missing == 1
|
||
assert repo.get_video(1101)["status"] == "removed"
|
||
assert repo.get_video(1102)["status"] == "removed"
|
||
assert repo.get_folder(F1).last_full_at == clock[0]
|
||
# 全量刷新请求了夹内全部 aid
|
||
assert set(transport.info_calls[-1][1]) == {1101, 1102}
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_capacity_warning_does_not_move_folder():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1, 2, 3], F2: []},
|
||
{i: valid(i) for i in (1, 2, 3)},
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock, capacity_warn=3)
|
||
try:
|
||
summary = service.run()
|
||
assert summary.folders[0].warning
|
||
assert "预警线 3" in summary.folders[0].warning
|
||
assert "不会自动换夹" in summary.folders[0].warning
|
||
# 视频仍在原夹，不被移动
|
||
assert all(repo.get_video(i)["folder_id"] == F1 for i in (1, 2, 3))
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_folder_error_does_not_block_other_folders():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101], F2: [2201]},
|
||
{1101: valid(1101), 2201: valid(2201)},
|
||
fail_media=F1,
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
summary = service.run()
|
||
assert summary.ok is False
|
||
assert len(summary.errors) == 1 and str(F1) in summary.errors[0]
|
||
assert summary.folders[0].error
|
||
assert summary.folders[1].error is None
|
||
assert repo.get_video(2201)["status"] == "active"
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_blacklist_seed_only_once():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport({F1: [1101], F2: []}, {1101: valid(1101)})
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
first = service.run()
|
||
assert first.blacklist_seeded == 1
|
||
clock[0] += 60
|
||
second = service.run()
|
||
assert second.blacklist_seeded == 0
|
||
assert len(repo.blacklist()) == 1
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_empty_remote_requires_explicit_opt_in():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101, 1102], F2: []},
|
||
{1101: valid(1101), 1102: valid(1102)},
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
service.run()
|
||
transport.folders[F1] = []
|
||
clock[0] += 3600
|
||
summary = service.run()
|
||
assert summary.ok is False
|
||
assert "疑似接口异常" in summary.folders[0].error
|
||
assert repo.get_video(1101)["status"] == "active"
|
||
assert repo.folder_active_aids(F1) == {1101, 1102}
|
||
|
||
service.allow_empty_folders = True
|
||
summary = service.run()
|
||
assert summary.ok is True
|
||
assert repo.get_video(1101)["status"] == "unfaved"
|
||
assert repo.folder_active_aids(F1) == set()
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def test_missing_ids_data_aborts_folder():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
transport = FakeFavTransport(
|
||
{F1: [1101], F2: []},
|
||
{1101: valid(1101)},
|
||
omit_ids_data=F1,
|
||
)
|
||
clock = [1_800_000_000]
|
||
db, repo, service = make_service(tmp, transport, clock)
|
||
try:
|
||
summary = service.run()
|
||
assert summary.ok is False
|
||
assert "data" in summary.folders[0].error
|
||
assert repo.get_video(1101) is None
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def main():
|
||
test_initial_import_and_incremental_skip()
|
||
test_diff_add_and_remove()
|
||
test_full_refresh_marks_invalid_and_missing()
|
||
test_capacity_warning_does_not_move_folder()
|
||
test_folder_error_does_not_block_other_folders()
|
||
test_blacklist_seed_only_once()
|
||
test_empty_remote_requires_explicit_opt_in()
|
||
test_missing_ids_data_aborts_folder()
|
||
print("OK: 收藏夹初始导入、差集同步、失效识别与容量预警测试全部通过")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|