# -*- coding: utf-8 -*- """看板命令行入口。 python app.py init # 建库 / 迁移 / 写入黑名单种子（离线） python app.py ingest # 手动入库 B 站视频链接 / BV 号 / av 号（只写本地库） python app.py snapshot # 采集当日七项快照与粉丝数 python app.py discover # 执行一次搜索发现并写入候选池 python app.py candidates # 查看候选池（离线） python app.py auth-check # 检查 Cookie 登录态 python app.py run # 按配置周期常驻运行（--once 只跑一次） python app.py status # 查看库内状态（离线） 收藏夹重排序使用独立的 ``favsort.py``（唯一使用 ``bilibili.write`` 的功能）。 P2 起 ``python app.py`` 将在此基础上启动本地看板服务。 """ from __future__ import annotations import argparse import json import sys import threading import time from pathlib import Path from typing import Sequence from .bilibili.auth_flow import AuthService from .bilibili.client import BilibiliClient, Pacer from .bilibili.provider import ClientProvider from .bilibili.search import SearchClient from .config import ConfigError, ConfigStore from .database import Database from .logging_json import JsonLogger from .notify.telegram import TelegramClient from .repository import Repository from .scheduler import ScheduledTask, Scheduler from .services.discovery import DiscoveryService from .services.library import LibraryService from .services.snapshot import SnapshotService from .services.telegram_poller import TelegramPoller from .services.up_profile import UpProfileService from .timeutil import next_daily_run, parse_hhmm, shanghai_date ROOT = Path(__file__).resolve().parent.parent DEFAULT_CONFIG = ROOT / "config.json" def _build_parser() -> argparse.ArgumentParser: common = argparse.ArgumentParser(add_help=False) common.add_argument( "--config", default=argparse.SUPPRESS, help="配置文件路径（默认项目根目录 config.json）", ) parser = argparse.ArgumentParser( prog="app.py", description="咻咻满 Reaction 数据看板 —— 数据底座、发现与每日快照", ) parser.add_argument( "--config", default=str(DEFAULT_CONFIG), help="配置文件路径（默认项目根目录 config.json）", ) sub = parser.add_subparsers(dest="command", required=False) sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子") snapshot = sub.add_parser( "snapshot", parents=[common], help="采集当日七项快照与粉丝数" ) snapshot.add_argument( "--force", action="store_true", help="忽略当日完成标记，重新采集" ) discover = sub.add_parser( "discover", parents=[common], help="执行一次搜索发现并写入候选池" ) discover.add_argument( "--no-notify", action="store_true", help="只写候选池，不发送 Telegram 通知", ) candidates = sub.add_parser( "candidates", parents=[common], help="查看候选池（离线）" ) candidates.add_argument( "--decision", choices=("accepted", "rejected"), default=None, help="只查看指定决策状态的候选", ) ingest = sub.add_parser( "ingest", parents=[common], help="手动入库：B 站视频链接、BV 号或 av 号（只写本地库）", ) ingest.add_argument( "refs", nargs="+", metavar="REF", help="视频链接、BV 号（BV…）、av 号（av12345）或含它们的字符串", ) auth = sub.add_parser("auth", parents=[common], help="扫码授权并更新指定账号 Cookie") auth.add_argument("--account", choices=("read", "write"), default="read", help="要授权的账号角色（默认 read）") auth.add_argument("--no-browser", action="store_true", help="不自动打开二维码图片（仍在终端打印 ASCII 二维码）") auth.add_argument("--timeout", type=int, default=180, metavar="SEC", help="等待扫码的超时时间（默认 180 秒）") auth_check = sub.add_parser("auth-check", parents=[common], help="检查账号登录态") auth_check.add_argument("--account", choices=("read", "write", "all"), default="read", help="要检查的账号角色（默认 read）") run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行发现与快照") run.add_argument( "--once", action="store_true", help="只执行一次到期任务后退出" ) serve = sub.add_parser("serve", parents=[common], help="启动只读本地看板（不运行调度）") serve.add_argument( "--preview", action="store_true", help="以预览模式启动（要求数据库含 preview:seeded=1 标记）", ) backfill = sub.add_parser( "backfill-followers", parents=[common], help="为历史视频回填 UP 粉丝数快照（只补空值）", ) backfill.add_argument("--limit", type=int, default=None, help="最多处理多少个 UP") backfill.add_argument("--dry-run", action="store_true", help="只统计待回填数量，不发请求") sub.add_parser("status", parents=[common], help="查看数据库状态（离线）") return parser def _open_db(cfg, logger: JsonLogger) -> Database: db = Database( cfg.storage.database_path, backup_dir=cfg.storage.backup_dir, backup_keep_days=cfg.storage.backup_keep_days, logger=logger, ) db.initialize() return db def _print(payload) -> None: print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True) def _task_running_key(name: str) -> str: return f"task:running:{name}" def _mark_task_running(repo: Repository, name: str, *, started_at: int | None = None) -> None: """任务开始时写入 running 标记，供 /system 显示「运行中」。""" now = int(time.time()) record = { "name": name, "started_at": int(started_at if started_at is not None else now), } try: repo.kv_set(_task_running_key(name), json.dumps(record, ensure_ascii=False)) except Exception: # noqa: BLE001 pass def _clear_task_running(repo: Repository, name: str) -> None: try: repo.kv_delete(_task_running_key(name)) except Exception: # noqa: BLE001 pass def _is_task_running(repo: Repository, name: str) -> bool: try: return bool(repo.kv_get(_task_running_key(name))) except Exception: # noqa: BLE001 return False def _record_task( repo: Repository, name: str, *, ok: bool, started_at: int | None = None, finished_at: int | None = None, payload=None, error: str | None = None, ) -> None: """把最近一次任务结果写入 kv，并清除 running 标记。""" now = int(time.time()) record = { "name": name, "ok": bool(ok), "started_at": int(started_at if started_at is not None else now), "finished_at": int(finished_at if finished_at is not None else now), "error": error, "summary": payload, } try: repo.kv_set(f"task:last:{name}", json.dumps(record, ensure_ascii=False)) except Exception: # noqa: BLE001 任务状态记录失败不影响主流程 pass _clear_task_running(repo, name) def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient: return BilibiliClient( cookie=credentials.cookie, csrf=credentials.csrf, mid=credentials.mid, pacer=pacer, retries=retries, logger=logger, ) def cmd_init(args, logger: JsonLogger) -> int: store = ConfigStore(args.config, require_auth=False) cfg = store.get() db = _open_db(cfg, logger) try: repo = Repository(db) for folder in cfg.folders: repo.ensure_folder(folder) seeded = repo.seed_blacklist(cfg.up_blacklist_seed) _print({ "ok": True, "schema_version": db.schema_version, "database": str(cfg.storage.database_path), "blacklist_seeded": seeded, **repo.counts(), }) finally: db.close() return 0 def cmd_ingest(args, logger: JsonLogger) -> int: store = ConfigStore(args.config, require_auth=False) cfg = store.get() db = _open_db(cfg, logger) try: repo = Repository(db) client = _client(cfg.bilibili.read, logger) up_profile = UpProfileService(repo, lambda: client, logger=logger) service = LibraryService( store, repo, lambda: client, up_profile_service=up_profile, logger=logger, ) text = " ".join(args.refs) result = service.ingest_text(text) payload = result.as_dict() payload["database"] = str(cfg.storage.database_path) _record_task( repo, "ingest", ok=bool(result.results) and not any( item.state == "error" for item in result.results ), payload=payload, ) _print(payload) return 0 if result.results and not any( item.state == "error" for item in result.results ) else 1 finally: db.close() def cmd_backfill_followers(args, logger: JsonLogger) -> int: store = ConfigStore(args.config, require_auth=False) cfg = store.get() db = _open_db(cfg, logger) try: repo = Repository(db) client = _client( cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=2 ) service = UpProfileService(repo, lambda: client, logger=logger) started = int(time.time()) result = service.backfill(limit=args.limit, dry_run=args.dry_run) payload = result.as_dict() payload["dry_run"] = bool(args.dry_run) payload["coverage"] = repo.up_follower_coverage() payload["database"] = str(cfg.storage.database_path) _record_task( repo, "backfill_followers", ok=result.failed == 0, started_at=started, finished_at=int(time.time()), payload=payload, error="；".join(result.errors) if result.errors else None, ) _print(payload) return 0 if result.failed == 0 else 1 finally: db.close() def cmd_snapshot(args, logger: JsonLogger) -> int: store = ConfigStore(args.config, require_auth=False) cfg = store.get() db = _open_db(cfg, logger) try: repo = Repository(db) client = _client( cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=1 ) service = SnapshotService(store, repo, client, logger=logger) started = int(time.time()) _mark_task_running(repo, "snapshot", started_at=started) try: video_summary = service.run_if_needed(force=args.force) except Exception as exc: # noqa: BLE001 _record_task( repo, "snapshot", ok=False, started_at=started, finished_at=int(time.time()), error=str(exc), ) raise video_payload = video_summary.as_dict() video_payload["database"] = str(cfg.storage.database_path) video_payload["snapshot_counts"] = repo.snapshot_counts(video_summary.stat_date) video_payload["counts"] = repo.counts() video_ok = bool(video_summary.completed and not video_summary.backup_error) _record_task( repo, "snapshot", ok=video_ok, started_at=int(video_summary.started_at or started), finished_at=int(video_summary.finished_at or time.time()), payload=video_payload, error=video_summary.backup_error, ) fans_started = int(time.time()) _mark_task_running(repo, "fans", started_at=fans_started) try: fans_summary = service.run_fans_if_needed( date=video_summary.stat_date, force=args.force, ) except Exception as exc: # noqa: BLE001 _record_task( repo, "fans", ok=False, started_at=fans_started, finished_at=int(time.time()), error=str(exc), ) raise fans_payload = fans_summary.as_dict() fans_ok = fans_summary.fans_failed == 0 _record_task( repo, "fans", ok=fans_ok, started_at=int(fans_summary.started_at or fans_started), finished_at=int(fans_summary.finished_at or time.time()), payload=fans_payload, error="；".join( err for err in fans_summary.errors if str(err).startswith("fans ") ) or None, ) payload = { **video_payload, "fans_ok": fans_summary.fans_ok, "fans_failed": fans_summary.fans_failed, "fans_skipped": fans_summary.fans_skipped, } _print(payload) return 0 if video_ok and fans_ok else 1 finally: db.close() def _telegram(cfg, logger: JsonLogger): if not cfg.telegram.enabled: return None if not cfg.telegram.bot_token or not cfg.telegram.chat_id: raise ConfigError( "telegram.enabled=true 时必须配置 bot_token 和 chat_id" ) return TelegramClient( cfg.telegram.bot_token, cfg.telegram.chat_id, logger=logger, ) def cmd_discover(args, logger: JsonLogger) -> int: store = ConfigStore(args.config, require_auth=True) cfg = store.get() db = _open_db(cfg, logger) try: repo = Repository(db) provider = ClientProvider( store, "read", lambda credentials: _client(credentials, logger) ) service = DiscoveryService( store, repo, SearchClient(provider.get), notifier=None if args.no_notify else _telegram(cfg, logger), logger=logger, ) started = int(time.time()) _mark_task_running(repo, "discovery", started_at=started) try: summary = service.run() except Exception as exc: # noqa: BLE001 _record_task( repo, "discovery", ok=False, started_at=started, finished_at=int(time.time()), error=str(exc), ) raise payload = summary.as_dict() payload["candidate_total"] = len(repo.list_candidates()) payload["database"] = str(cfg.storage.database_path) _record_task( repo, "discovery", ok=not summary.errors, started_at=int(summary.started_at or started), finished_at=int(summary.finished_at or time.time()), payload=payload, error="；".join(summary.errors) if summary.errors else None, ) _print(payload) return 0 if not summary.errors else 1 finally: db.close() def cmd_candidates(args, logger: JsonLogger) -> int: store = ConfigStore(args.config, require_auth=False) cfg = store.get() db = _open_db(cfg, logger) try: repo = Repository(db) rows = repo.list_candidates(args.decision) _print({ "ok": True, "count": len(rows), "candidates": rows, "database": str(cfg.storage.database_path), }) return 0 finally: db.close() def cmd_auth(args, logger: JsonLogger) -> int: if args.timeout < 1: raise ConfigError("auth --timeout 必须至少为 1 秒") return AuthService(Path(args.config), account=args.account, logger=logger).run( timeout=args.timeout, no_browser=args.no_browser ) def _auth_check(cfg, account: str, logger: JsonLogger) -> dict[str, object]: credentials = getattr(cfg.bilibili, account) if not credentials.cookie: return {"account": account, "ok": False, "is_login": False, "message": "未配置 cookie"} state = _client(credentials, logger, retries=1).check_login() return { "account": account, "ok": state.is_login, "is_login": state.is_login, "mid": state.mid, "uname": state.uname, "code": state.code, "message": state.message, } def cmd_auth_check(args, logger: JsonLogger) -> int: store = ConfigStore(args.config, require_auth=False) cfg = store.get() if args.account != "all": result = _auth_check(cfg, args.account, logger) _print(result) return 0 if result["ok"] else 1 results = [_auth_check(cfg, account, logger) for account in ("read", "write")] _print({"ok": all(result["ok"] for result in results), "accounts": results}) return 0 if all(result["ok"] for result in results) else 1 def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: store = ConfigStore(args.config, require_auth=True) cfg = store.get() db = _open_db(cfg, logger) last: dict[str, object] = {} web_server = None try: repo = Repository(db) if cfg.up_blacklist_seed: repo.seed_blacklist(cfg.up_blacklist_seed) read_provider = ClientProvider( store, "read", lambda credentials: _client(credentials, logger) ) snapshot_provider = ClientProvider( store, "read", lambda credentials: _client( credentials, logger, pacer=Pacer(1.0, 1.6), retries=1 ), ) snapshot_service = SnapshotService( store, repo, client_provider=snapshot_provider, logger=logger ) up_profile_service = UpProfileService(repo, read_provider.get, logger=logger) library_service = LibraryService( store, repo, read_provider, up_profile_service=up_profile_service, logger=logger, ) notifier = _telegram(cfg, logger) discovery_service = DiscoveryService( store, repo, SearchClient(read_provider.get), notifier=notifier, logger=logger, ) def run_discover(): started = int(time.time()) _mark_task_running(repo, "discovery", started_at=started) try: summary = discovery_service.run() except Exception as exc: # noqa: BLE001 记录后交给调度器隔离 _record_task( repo, "discovery", ok=False, started_at=started, finished_at=int(time.time()), error=str(exc), ) raise payload = summary.as_dict() _record_task( repo, "discovery", ok=not summary.errors, started_at=summary.started_at, finished_at=summary.finished_at or int(time.time()), payload=payload, error="；".join(summary.errors) if summary.errors else None, ) last["discover"] = summary logger.info("discovery.summary", **payload) return summary def run_cleanup(): started = int(time.time()) removed = repo.delete_expired_candidates( ttl_days=cfg.schedule.candidate_ttl_days ) expired = repo.expire_fav_actions() if removed or expired: repo.bump_cache_version("videos") logger.info( "cleanup.done", candidates_removed=removed, fav_actions_expired=expired, ) _record_task( repo, "candidate_cleanup", ok=True, started_at=started, finished_at=int(time.time()), payload={ "candidates_removed": removed, "fav_actions_expired": expired, }, ) return { "candidates_removed": removed, "fav_actions_expired": expired, } def run_snapshot(): started = int(time.time()) if _is_task_running(repo, "snapshot_recent"): logger.info( "snapshot.skip", reason="snapshot_recent_running", ) _record_task( repo, "snapshot", ok=True, started_at=started, finished_at=int(time.time()), payload={"skipped": True, "reason": "snapshot_recent_running"}, ) return None _mark_task_running(repo, "snapshot", started_at=started) try: summary = snapshot_service.run_if_needed() except Exception as exc: # noqa: BLE001 _record_task( repo, "snapshot", ok=False, started_at=started, finished_at=int(time.time()), error=str(exc), ) raise payload = summary.as_dict() ok = bool(summary.completed and not summary.backup_error) _record_task( repo, "snapshot", ok=ok, started_at=summary.started_at, finished_at=summary.finished_at or int(time.time()), payload=payload, error=summary.backup_error, ) last["snapshot"] = summary logger.info("snapshot.summary", **payload) return summary def run_snapshot_recent(): started = int(time.time()) if _is_task_running(repo, "snapshot"): logger.info( "snapshot.recent_skip", reason="snapshot_running", ) _record_task( repo, "snapshot_recent", ok=True, started_at=started, finished_at=int(time.time()), payload={"skipped": True, "reason": "snapshot_running"}, ) return None _mark_task_running(repo, "snapshot_recent", started_at=started) try: summary = snapshot_service.run_recent_videos() except Exception as exc: # noqa: BLE001 _record_task( repo, "snapshot_recent", ok=False, started_at=started, finished_at=int(time.time()), error=str(exc), ) raise payload = summary.as_dict() ok = summary.failed == 0 error = "；".join(summary.errors) if summary.errors else None _record_task( repo, "snapshot_recent", ok=ok, started_at=summary.started_at, finished_at=summary.finished_at or int(time.time()), payload=payload, error=error, ) last["snapshot_recent"] = summary logger.info("snapshot.recent_summary", **payload) return summary def run_fans(): started = int(time.time()) _mark_task_running(repo, "fans", started_at=started) try: summary = snapshot_service.run_fans_if_needed() except Exception as exc: # noqa: BLE001 _record_task( repo, "fans", ok=False, started_at=started, finished_at=int(time.time()), error=str(exc), ) raise payload = summary.as_dict() ok = summary.fans_failed == 0 error = "；".join( err for err in summary.errors if str(err).startswith("fans ") ) or None _record_task( repo, "fans", ok=ok, started_at=summary.started_at, finished_at=summary.finished_at or int(time.time()), payload=payload, error=error, ) last["fans"] = summary logger.info("fans.summary", **payload) return summary def run_config_watch(): before = store.revision store.get() if store.revision != before: logger.info("config.reloaded", revision=store.revision) return {"revision": store.revision} hh, mm = parse_hhmm(cfg.schedule.snapshot_at) snapshot_at = f"{hh:02d}:{mm:02d}" recent_minutes = int(store.get().schedule.recent_snapshot_minutes) scheduler = Scheduler( [ ScheduledTask( "snapshot", 86400.0, run_snapshot, next_run=0.0, next_run_fn=lambda started: next_daily_run(started, snapshot_at), ), ScheduledTask( "fans", 86400.0, run_fans, next_run=0.0, next_run_fn=lambda started: next_daily_run(started, snapshot_at), ), ScheduledTask( "snapshot_recent", float(recent_minutes * 60), run_snapshot_recent, ), ScheduledTask( "discovery", float(cfg.schedule.discover_minutes * 60), run_discover, ), ScheduledTask("candidate_cleanup", 86400.0, run_cleanup), ScheduledTask("config_watch", 60.0, run_config_watch), ], logger=logger, ) if serve_web and not args.once: from .web.server import create_server, serve_forever web_server = create_server(store, db, logger=logger) threading.Thread( target=lambda: serve_forever(web_server, logger=logger), name="web-server", daemon=True, ).start() if notifier is not None and not args.once: poller = TelegramPoller(notifier, library_service, logger=logger) threading.Thread( target=poller.run_forever, name="telegram-poller", daemon=True, ).start() logger.info("telegram.polling_started") if args.once: results = scheduler.run_due() _print([result.as_dict() for result in results]) else: logger.info( "scheduler.start", snapshot_at=snapshot_at, ) scheduler.run_forever() snapshot_summary = last.get("snapshot") fans_summary = last.get("fans") discover_summary = last.get("discover") snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True) fans_ok = fans_summary is None or getattr(fans_summary, "fans_failed", 0) == 0 discover_ok = discover_summary is None or not getattr(discover_summary, "errors", []) return 0 if snapshot_ok and fans_ok and discover_ok else 1 finally: if web_server is not None: try: web_server.shutdown() finally: web_server.server_close() db.close() def _assert_preview_database(path: Path) -> None: import sqlite3 if not path.exists(): raise ConfigError(f"预览数据库不存在: {path}") uri = f"file:{path.resolve().as_posix()}?mode=ro" try: conn = sqlite3.connect(uri, uri=True) except sqlite3.Error as exc: raise ConfigError(f"无法读取预览数据库: {exc}") from exc try: row = conn.execute( "SELECT v FROM kv WHERE k = 'preview:seeded'" ).fetchone() except sqlite3.Error as exc: raise ConfigError(f"目标数据库不是预览库: {exc}") from exc finally: conn.close() if row is None or str(row[0]) != "1": raise ConfigError( "目标数据库缺少 preview:seeded=1 标记；请先运行 " "tools/seed_preview_db.py，或去掉 --preview" ) def cmd_serve(args, logger: JsonLogger) -> int: from .web.server import create_server, serve_forever store = ConfigStore(args.config, require_auth=False) cfg = store.get() if args.preview: _assert_preview_database(cfg.storage.database_path) db = _open_db(cfg, logger) try: server = create_server(store, db, preview=args.preview, logger=logger) serve_forever(server, logger=logger) return 0 finally: db.close() def cmd_status(args, logger: JsonLogger) -> int: store = ConfigStore(args.config, require_auth=False) cfg = store.get() db = _open_db(cfg, logger) try: repo = Repository(db) today = shanghai_date() snapshot = repo.snapshot_counts(today) snapshot.update({ "date": today, "last_attempt": repo.kv_get("snapshot:last_attempt"), "last_completed": repo.kv_get("snapshot:last_completed"), "auth_ok": repo.kv_get("auth:ok"), "auth_message": repo.kv_get("auth:message"), "backup_ok": repo.kv_get(f"snapshot:backup:{today}"), "backup_error": repo.kv_get(f"snapshot:backup_error:{today}"), }) _print({ "ok": True, "schema_version": db.schema_version, "database": str(cfg.storage.database_path), "configured_folders": [ { "media_id": row.media_id, "year": row.year, "title": row.title, "media_count": row.media_count, "last_ids_at": row.last_ids_at, "last_full_at": row.last_full_at, "enabled": row.enabled, } for row in repo.list_folders() ], "snapshot": snapshot, **repo.counts(), }) finally: db.close() return 0 def main(argv: Sequence[str] | None = None) -> int: args = _build_parser().parse_args(list(argv) if argv is not None else None) logger = JsonLogger() try: if args.command is None: # 无子命令时启动完整服务：只读 Web + 调度 + Telegram。 args.command = "run" args.once = False return cmd_run(args, logger, serve_web=True) if args.command == "serve": return cmd_serve(args, logger) if args.command == "init": return cmd_init(args, logger) if args.command == "snapshot": return cmd_snapshot(args, logger) if args.command == "ingest": return cmd_ingest(args, logger) if args.command == "backfill-followers": return cmd_backfill_followers(args, logger) if args.command == "discover": return cmd_discover(args, logger) if args.command == "candidates": return cmd_candidates(args, logger) if args.command == "auth": return cmd_auth(args, logger) if args.command == "auth-check": return cmd_auth_check(args, logger) if args.command == "run": return cmd_run(args, logger) if args.command == "status": return cmd_status(args, logger) except ConfigError as exc: logger.error("config.error", error=str(exc)) return 2 except KeyboardInterrupt: logger.warning("app.interrupted") return 130 except Exception as exc: # noqa: BLE001 CLI 边界统一记录 logger.error("app.error", error=str(exc)) return 2 logger.error("app.unknown_command", command=getattr(args, "command", None)) return 2 if __name__ == "__main__": sys.exit(main())