feat(p0): add favorite folder sync and sqlite data foundation
Implement P0 of the reaction dashboard: config loading with hot reload, SQLite migrations with backups, single-writer queue, read-only Bilibili fav client, folder import/diff sync, invalid-resource handling, blacklist seed, scheduler, and JSON redacted logging. Includes offline unit and transport-level contract tests; all existing and new tests pass.
This commit is contained in:
kairee committed
commit
e2cdf997af
39 files changed
+8575
No files matched your search
@@ -0,0 +1,217 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""P0 命令行入口。
|
||||
|
||||
python app.py init # 建库 / 迁移 / 写入黑名单种子（离线）
|
||||
python app.py sync # 收藏夹只读同步（首次导入或增量）
|
||||
python app.py run # 按 folder_sync_minutes 常驻周期同步（--once 只跑一次）
|
||||
python app.py status # 查看库内状态（离线）
|
||||
|
||||
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
from .bilibili.client import BilibiliClient
|
||||
from .config import ConfigError, ConfigStore
|
||||
from .database import Database
|
||||
from .logging_json import JsonLogger
|
||||
from .repository import Repository
|
||||
from .scheduler import ScheduledTask, Scheduler
|
||||
from .services.folder_sync import FolderSyncService
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_CONFIG = ROOT / "config.json"
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
common = argparse.ArgumentParser(add_help=False)
|
||||
common.add_argument(
|
||||
"--config",
|
||||
default=str(DEFAULT_CONFIG),
|
||||
help="配置文件路径（默认项目根目录 config.json）",
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="app.py",
|
||||
description="咻咻满 Reaction 数据看板 —— P0 数据底座",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
|
||||
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
|
||||
sync.add_argument(
|
||||
"--full", action="store_true", help="强制全量刷新元数据（忽略 6 小时周期）"
|
||||
)
|
||||
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行收藏夹同步")
|
||||
run.add_argument(
|
||||
"--once", action="store_true", help="只执行一次到期任务后退出"
|
||||
)
|
||||
sub.add_parser("status", parents=[common], help="查看数据库状态（离线）")
|
||||
return parser
|
||||
|
||||
|
||||
def _open_db(cfg, logger: JsonLogger) -> Database:
|
||||
db = Database(
|
||||
cfg.storage.database_path,
|
||||
backup_dir=cfg.storage.backup_dir,
|
||||
backup_keep_days=cfg.storage.backup_keep_days,
|
||||
logger=logger,
|
||||
)
|
||||
db.initialize()
|
||||
return db
|
||||
|
||||
|
||||
def _print(payload) -> None:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
|
||||
|
||||
|
||||
def cmd_init(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
for folder in cfg.folders:
|
||||
repo.ensure_folder(folder)
|
||||
seeded = repo.seed_blacklist(cfg.up_blacklist_seed)
|
||||
_print({
|
||||
"ok": True,
|
||||
"schema_version": db.schema_version,
|
||||
"database": str(cfg.storage.database_path),
|
||||
"blacklist_seeded": seeded,
|
||||
**repo.counts(),
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=True)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = BilibiliClient(
|
||||
cookie=cfg.bilibili.cookie,
|
||||
csrf=cfg.bilibili.csrf,
|
||||
mid=cfg.bilibili.mid,
|
||||
logger=logger,
|
||||
)
|
||||
service = FolderSyncService(store, repo, client, logger=logger)
|
||||
summary = service.run(full_refresh=args.full)
|
||||
payload = summary.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
payload["counts"] = repo.counts()
|
||||
_print(payload)
|
||||
return 0 if summary.ok else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_run(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=True)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
last: dict[str, object] = {}
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = BilibiliClient(
|
||||
cookie=cfg.bilibili.cookie,
|
||||
csrf=cfg.bilibili.csrf,
|
||||
mid=cfg.bilibili.mid,
|
||||
logger=logger,
|
||||
)
|
||||
service = FolderSyncService(store, repo, client, logger=logger)
|
||||
|
||||
def run_sync():
|
||||
summary = service.run()
|
||||
last["summary"] = summary
|
||||
logger.info("folder_sync.summary", **summary.as_dict())
|
||||
return summary
|
||||
|
||||
scheduler = Scheduler(
|
||||
[ScheduledTask(
|
||||
"folder_sync",
|
||||
float(cfg.schedule.folder_sync_minutes * 60),
|
||||
run_sync,
|
||||
)],
|
||||
logger=logger,
|
||||
)
|
||||
if args.once:
|
||||
results = scheduler.run_due()
|
||||
_print([result.as_dict() for result in results])
|
||||
else:
|
||||
logger.info(
|
||||
"scheduler.start",
|
||||
interval_seconds=cfg.schedule.folder_sync_minutes * 60,
|
||||
)
|
||||
scheduler.run_forever()
|
||||
summary = last.get("summary")
|
||||
return 0 if summary is None or getattr(summary, "ok", True) else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_status(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
_print({
|
||||
"ok": True,
|
||||
"schema_version": db.schema_version,
|
||||
"database": str(cfg.storage.database_path),
|
||||
"configured_folders": [
|
||||
{
|
||||
"media_id": row.media_id,
|
||||
"year": row.year,
|
||||
"title": row.title,
|
||||
"media_count": row.media_count,
|
||||
"last_ids_at": row.last_ids_at,
|
||||
"last_full_at": row.last_full_at,
|
||||
"enabled": row.enabled,
|
||||
}
|
||||
for row in repo.list_folders()
|
||||
],
|
||||
**repo.counts(),
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _build_parser().parse_args(list(argv) if argv is not None else None)
|
||||
logger = JsonLogger()
|
||||
try:
|
||||
if args.command == "init":
|
||||
return cmd_init(args, logger)
|
||||
if args.command == "sync":
|
||||
return cmd_sync(args, logger)
|
||||
if args.command == "run":
|
||||
return cmd_run(args, logger)
|
||||
if args.command == "status":
|
||||
return cmd_status(args, logger)
|
||||
except ConfigError as exc:
|
||||
logger.error("config.error", error=str(exc))
|
||||
return 2
|
||||
except KeyboardInterrupt:
|
||||
logger.warning("app.interrupted")
|
||||
return 130
|
||||
except Exception as exc: # noqa: BLE001 CLI 边界统一记录
|
||||
logger.error("app.error", error=str(exc))
|
||||
return 2
|
||||
logger.error("app.unknown_command", command=getattr(args, "command", None))
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user