fix: close data integrity and hot-reload review findings

Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
This commit is contained in:
kairee committed 2026-10-31 20:58:36 +08:00
1 parent 178ab64b18
commit 6a21b5d905
21 files changed
+895 -82

No files matched your search

+58 -12
View File
@@ -21,6 +21,7 @@ from typing import Sequence
from .bilibili.auth_flow import AuthService
from .bilibili.client import BilibiliClient, Pacer
from .bilibili.provider import ClientProvider
from .config import ConfigError, ConfigStore
from .database import Database
from .logging_json import JsonLogger
@@ -53,6 +54,11 @@ def _build_parser() -> argparse.ArgumentParser:
sync.add_argument(
"--full", action="store_true", help="强制全量刷新元数据(忽略 6 小时周期)"
)
sync.add_argument(
"--allow-empty-folder",
action="store_true",
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
)
snapshot = sub.add_parser(
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
)
@@ -72,6 +78,11 @@ def _build_parser() -> argparse.ArgumentParser:
run.add_argument(
"--once", action="store_true", help="只执行一次到期任务后退出"
)
run.add_argument(
"--allow-empty-folder",
action="store_true",
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
)
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
return parser
@@ -130,7 +141,13 @@ def cmd_sync(args, logger: JsonLogger) -> int:
try:
repo = Repository(db)
client = _client(cfg.bilibili.read, logger)
service = FolderSyncService(store, repo, client, logger=logger)
service = FolderSyncService(
store,
repo,
client,
logger=logger,
allow_empty_folders=args.allow_empty_folder,
)
summary = service.run(full_refresh=args.full)
payload = summary.as_dict()
payload["database"] = str(cfg.storage.database_path)
@@ -157,7 +174,7 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
payload["counts"] = repo.counts()
_print(payload)
return 0 if summary.completed else 1
return 0 if summary.completed and not summary.backup_error else 1
finally:
db.close()
@@ -205,16 +222,26 @@ def cmd_run(args, logger: JsonLogger) -> int:
last: dict[str, object] = {}
try:
repo = Repository(db)
folder_client = _client(cfg.bilibili.read, logger)
folder_service = FolderSyncService(store, repo, folder_client, logger=logger)
snapshot_client = _client(cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=1)
snapshot_service = SnapshotService(store, repo, snapshot_client, logger=logger)
def run_sync():
summary = folder_service.run()
last["folder"] = summary
logger.info("folder_sync.summary", **summary.as_dict())
return summary
folder_provider = ClientProvider(
store, "read", lambda credentials: _client(credentials, logger)
)
snapshot_provider = ClientProvider(
store,
"read",
lambda credentials: _client(
credentials, logger, pacer=Pacer(1.0, 1.6), retries=1
),
)
folder_service = FolderSyncService(
store,
repo,
client_provider=folder_provider,
logger=logger,
allow_empty_folders=args.allow_empty_folder,
)
snapshot_service = SnapshotService(
store, repo, client_provider=snapshot_provider, logger=logger
)
def run_snapshot():
summary = snapshot_service.run_if_needed()
@@ -222,6 +249,22 @@ def cmd_run(args, logger: JsonLogger) -> int:
logger.info("snapshot.summary", **summary.as_dict())
return summary
def run_sync():
summary = folder_service.run()
last["folder"] = summary
logger.info("folder_sync.summary", **summary.as_dict())
# 新入库/恢复的视频立即补当天快照,而不是等到次日。
if summary.added > 0:
run_snapshot()
return summary
def run_config_watch():
before = store.revision
store.get()
if store.revision != before:
logger.info("config.reloaded", revision=store.revision)
return {"revision": store.revision}
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
snapshot_at = f"{hh:02d}:{mm:02d}"
scheduler = Scheduler(
@@ -238,6 +281,7 @@ def cmd_run(args, logger: JsonLogger) -> int:
next_run=0.0,
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
),
ScheduledTask("config_watch", 60.0, run_config_watch),
],
logger=logger,
)
@@ -275,6 +319,8 @@ def cmd_status(args, logger: JsonLogger) -> int:
"last_completed": repo.kv_get("snapshot:last_completed"),
"auth_ok": repo.kv_get("auth:ok"),
"auth_message": repo.kv_get("auth:message"),
"backup_ok": repo.kv_get(f"snapshot:backup:{today}"),
"backup_error": repo.kv_get(f"snapshot:backup_error:{today}"),
})
_print({
"ok": True,