fix: close data integrity and hot-reload review findings
Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
This commit is contained in:
1 parent
178ab64b18
commit
6a21b5d905
21 files changed
+895
-82
No files matched your search
+58
-12
@@ -21,6 +21,7 @@ from typing import Sequence
|
||||
|
||||
from .bilibili.auth_flow import AuthService
|
||||
from .bilibili.client import BilibiliClient, Pacer
|
||||
from .bilibili.provider import ClientProvider
|
||||
from .config import ConfigError, ConfigStore
|
||||
from .database import Database
|
||||
from .logging_json import JsonLogger
|
||||
@@ -53,6 +54,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
sync.add_argument(
|
||||
"--full", action="store_true", help="强制全量刷新元数据(忽略 6 小时周期)"
|
||||
)
|
||||
sync.add_argument(
|
||||
"--allow-empty-folder",
|
||||
action="store_true",
|
||||
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
|
||||
)
|
||||
snapshot = sub.add_parser(
|
||||
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
|
||||
)
|
||||
@@ -72,6 +78,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
run.add_argument(
|
||||
"--once", action="store_true", help="只执行一次到期任务后退出"
|
||||
)
|
||||
run.add_argument(
|
||||
"--allow-empty-folder",
|
||||
action="store_true",
|
||||
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
|
||||
)
|
||||
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
|
||||
return parser
|
||||
|
||||
@@ -130,7 +141,13 @@ def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = _client(cfg.bilibili.read, logger)
|
||||
service = FolderSyncService(store, repo, client, logger=logger)
|
||||
service = FolderSyncService(
|
||||
store,
|
||||
repo,
|
||||
client,
|
||||
logger=logger,
|
||||
allow_empty_folders=args.allow_empty_folder,
|
||||
)
|
||||
summary = service.run(full_refresh=args.full)
|
||||
payload = summary.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
@@ -157,7 +174,7 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||||
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
|
||||
payload["counts"] = repo.counts()
|
||||
_print(payload)
|
||||
return 0 if summary.completed else 1
|
||||
return 0 if summary.completed and not summary.backup_error else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -205,16 +222,26 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
last: dict[str, object] = {}
|
||||
try:
|
||||
repo = Repository(db)
|
||||
folder_client = _client(cfg.bilibili.read, logger)
|
||||
folder_service = FolderSyncService(store, repo, folder_client, logger=logger)
|
||||
snapshot_client = _client(cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=1)
|
||||
snapshot_service = SnapshotService(store, repo, snapshot_client, logger=logger)
|
||||
|
||||
def run_sync():
|
||||
summary = folder_service.run()
|
||||
last["folder"] = summary
|
||||
logger.info("folder_sync.summary", **summary.as_dict())
|
||||
return summary
|
||||
folder_provider = ClientProvider(
|
||||
store, "read", lambda credentials: _client(credentials, logger)
|
||||
)
|
||||
snapshot_provider = ClientProvider(
|
||||
store,
|
||||
"read",
|
||||
lambda credentials: _client(
|
||||
credentials, logger, pacer=Pacer(1.0, 1.6), retries=1
|
||||
),
|
||||
)
|
||||
folder_service = FolderSyncService(
|
||||
store,
|
||||
repo,
|
||||
client_provider=folder_provider,
|
||||
logger=logger,
|
||||
allow_empty_folders=args.allow_empty_folder,
|
||||
)
|
||||
snapshot_service = SnapshotService(
|
||||
store, repo, client_provider=snapshot_provider, logger=logger
|
||||
)
|
||||
|
||||
def run_snapshot():
|
||||
summary = snapshot_service.run_if_needed()
|
||||
@@ -222,6 +249,22 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
logger.info("snapshot.summary", **summary.as_dict())
|
||||
return summary
|
||||
|
||||
def run_sync():
|
||||
summary = folder_service.run()
|
||||
last["folder"] = summary
|
||||
logger.info("folder_sync.summary", **summary.as_dict())
|
||||
# 新入库/恢复的视频立即补当天快照,而不是等到次日。
|
||||
if summary.added > 0:
|
||||
run_snapshot()
|
||||
return summary
|
||||
|
||||
def run_config_watch():
|
||||
before = store.revision
|
||||
store.get()
|
||||
if store.revision != before:
|
||||
logger.info("config.reloaded", revision=store.revision)
|
||||
return {"revision": store.revision}
|
||||
|
||||
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
|
||||
snapshot_at = f"{hh:02d}:{mm:02d}"
|
||||
scheduler = Scheduler(
|
||||
@@ -238,6 +281,7 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
next_run=0.0,
|
||||
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
|
||||
),
|
||||
ScheduledTask("config_watch", 60.0, run_config_watch),
|
||||
],
|
||||
logger=logger,
|
||||
)
|
||||
@@ -275,6 +319,8 @@ def cmd_status(args, logger: JsonLogger) -> int:
|
||||
"last_completed": repo.kv_get("snapshot:last_completed"),
|
||||
"auth_ok": repo.kv_get("auth:ok"),
|
||||
"auth_message": repo.kv_get("auth:message"),
|
||||
"backup_ok": repo.kv_get(f"snapshot:backup:{today}"),
|
||||
"backup_error": repo.kv_get(f"snapshot:backup_error:{today}"),
|
||||
})
|
||||
_print({
|
||||
"ok": True,
|
||||
|
||||
Reference in new issue
Block a user