feat(web): add read-only dashboard server and query layer

This commit is contained in:
kairee committed 2026-10-31 23:13:54 +08:00
1 parent e7cae035e0
commit 19572171b8
10 files changed
+2603 -10

No files matched your search

+184 -10
View File
@@ -19,6 +19,7 @@ import argparse
import json
import sys
import threading
import time
from pathlib import Path
from typing import Sequence
@@ -48,7 +49,7 @@ def _build_parser() -> argparse.ArgumentParser:
common = argparse.ArgumentParser(add_help=False)
common.add_argument(
"--config",
default=str(DEFAULT_CONFIG),
default=argparse.SUPPRESS,
help="配置文件路径(默认项目根目录 config.json)",
)
@@ -56,7 +57,12 @@ def _build_parser() -> argparse.ArgumentParser:
prog="app.py",
description="咻咻满 Reaction 数据看板 —— 数据底座、同步与每日快照",
)
sub = parser.add_subparsers(dest="command", required=True)
parser.add_argument(
"--config",
default=str(DEFAULT_CONFIG),
help="配置文件路径(默认项目根目录 config.json)",
)
sub = parser.add_subparsers(dest="command", required=False)
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
@@ -109,6 +115,11 @@ def _build_parser() -> argparse.ArgumentParser:
action="store_true",
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
)
serve = sub.add_parser("serve", parents=[common], help="启动只读本地看板(不运行调度)")
serve.add_argument(
"--preview", action="store_true",
help="以预览模式启动(要求数据库含 preview:seeded=1 标记)",
)
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
return parser
@@ -128,6 +139,32 @@ def _print(payload) -> None:
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
def _record_task(
repo: Repository,
name: str,
*,
ok: bool,
started_at: int | None = None,
finished_at: int | None = None,
payload=None,
error: str | None = None,
) -> None:
"""把最近一次任务结果写入 kv,供只读系统页展示。"""
now = int(time.time())
record = {
"name": name,
"ok": bool(ok),
"started_at": int(started_at if started_at is not None else now),
"finished_at": int(finished_at if finished_at is not None else now),
"error": error,
"summary": payload,
}
try:
repo.kv_set(f"task:last:{name}", json.dumps(record, ensure_ascii=False))
except Exception: # noqa: BLE001 任务状态记录失败不影响主流程
pass
def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
return BilibiliClient(
cookie=credentials.cookie,
@@ -174,10 +211,15 @@ def cmd_sync(args, logger: JsonLogger) -> int:
logger=logger,
allow_empty_folders=args.allow_empty_folder,
)
started = int(time.time())
summary = service.run(full_refresh=args.full)
payload = summary.as_dict()
payload["database"] = str(cfg.storage.database_path)
payload["counts"] = repo.counts()
_record_task(
repo, "folder_sync", ok=summary.ok, started_at=started,
finished_at=int(time.time()), payload=payload,
)
_print(payload)
return 0 if summary.ok else 1
finally:
@@ -199,8 +241,15 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
payload["database"] = str(cfg.storage.database_path)
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
payload["counts"] = repo.counts()
ok = bool(summary.completed and not summary.backup_error)
_record_task(
repo, "snapshot", ok=ok,
started_at=int(summary.started_at or time.time()),
finished_at=int(time.time()), payload=payload,
error=summary.backup_error,
)
_print(payload)
return 0 if summary.completed and not summary.backup_error else 1
return 0 if ok else 1
finally:
db.close()
@@ -239,6 +288,13 @@ def cmd_discover(args, logger: JsonLogger) -> int:
payload = summary.as_dict()
payload["candidate_total"] = len(repo.list_candidates())
payload["database"] = str(cfg.storage.database_path)
_record_task(
repo, "discovery", ok=not summary.errors,
started_at=int(summary.started_at),
finished_at=int(summary.finished_at or time.time()),
payload=payload,
error=";".join(summary.errors) if summary.errors else None,
)
_print(payload)
return 0 if not summary.errors else 1
finally:
@@ -299,11 +355,12 @@ def cmd_auth_check(args, logger: JsonLogger) -> int:
return 0 if all(result["ok"] for result in results) else 1
def cmd_run(args, logger: JsonLogger) -> int:
def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
store = ConfigStore(args.config, require_auth=True)
cfg = store.get()
db = _open_db(cfg, logger)
last: dict[str, object] = {}
web_server = None
try:
repo = Repository(db)
folder_provider = ClientProvider(
@@ -349,12 +406,29 @@ def cmd_run(args, logger: JsonLogger) -> int:
)
def run_discover():
summary = discovery_service.run()
started = int(time.time())
try:
summary = discovery_service.run()
except Exception as exc: # noqa: BLE001 记录后交给调度器隔离
_record_task(
repo, "discovery", ok=False, started_at=started,
finished_at=int(time.time()), error=str(exc),
)
raise
payload = summary.as_dict()
_record_task(
repo, "discovery", ok=not summary.errors,
started_at=summary.started_at,
finished_at=summary.finished_at or int(time.time()),
payload=payload,
error=";".join(summary.errors) if summary.errors else None,
)
last["discover"] = summary
logger.info("discovery.summary", **summary.as_dict())
logger.info("discovery.summary", **payload)
return summary
def run_cleanup():
started = int(time.time())
removed = repo.delete_expired_candidates(
ttl_days=cfg.schedule.candidate_ttl_days
)
@@ -363,21 +437,58 @@ def cmd_run(args, logger: JsonLogger) -> int:
"cleanup.done",
candidates_removed=removed, fav_actions_expired=expired,
)
_record_task(
repo, "candidate_cleanup", ok=True, started_at=started,
finished_at=int(time.time()),
payload={
"candidates_removed": removed,
"fav_actions_expired": expired,
},
)
return {
"candidates_removed": removed,
"fav_actions_expired": expired,
}
def run_snapshot():
summary = snapshot_service.run_if_needed()
started = int(time.time())
try:
summary = snapshot_service.run_if_needed()
except Exception as exc: # noqa: BLE001
_record_task(
repo, "snapshot", ok=False, started_at=started,
finished_at=int(time.time()), error=str(exc),
)
raise
payload = summary.as_dict()
ok = bool(summary.completed and not summary.backup_error)
_record_task(
repo, "snapshot", ok=ok, started_at=summary.started_at,
finished_at=summary.finished_at or int(time.time()),
payload=payload, error=summary.backup_error,
)
last["snapshot"] = summary
logger.info("snapshot.summary", **summary.as_dict())
logger.info("snapshot.summary", **payload)
return summary
def run_sync():
summary = folder_service.run()
started = int(time.time())
try:
summary = folder_service.run()
except Exception as exc: # noqa: BLE001
_record_task(
repo, "folder_sync", ok=False, started_at=started,
finished_at=int(time.time()), error=str(exc),
)
raise
payload = summary.as_dict()
_record_task(
repo, "folder_sync", ok=summary.ok, started_at=started,
finished_at=int(time.time()), payload=payload,
error=None if summary.ok else ";".join(summary.errors),
)
last["folder"] = summary
logger.info("folder_sync.summary", **summary.as_dict())
logger.info("folder_sync.summary", **payload)
# 新入库/恢复的视频立即补当天快照,而不是等到次日。
if summary.added > 0:
run_snapshot()
@@ -416,6 +527,15 @@ def cmd_run(args, logger: JsonLogger) -> int:
],
logger=logger,
)
if serve_web and not args.once:
from .web.server import create_server, serve_forever
web_server = create_server(store, db, logger=logger)
threading.Thread(
target=lambda: serve_forever(web_server, logger=logger),
name="web-server",
daemon=True,
).start()
if notifier is not None and not args.once:
poller = TelegramPoller(notifier, favorite_service, logger=logger)
threading.Thread(
@@ -442,6 +562,52 @@ def cmd_run(args, logger: JsonLogger) -> int:
snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True)
discover_ok = discover_summary is None or not getattr(discover_summary, "errors", [])
return 0 if folder_ok and snapshot_ok and discover_ok else 1
finally:
if web_server is not None:
try:
web_server.shutdown()
finally:
web_server.server_close()
db.close()
def _assert_preview_database(path: Path) -> None:
import sqlite3
if not path.exists():
raise ConfigError(f"预览数据库不存在: {path}")
uri = f"file:{path.resolve().as_posix()}?mode=ro"
try:
conn = sqlite3.connect(uri, uri=True)
except sqlite3.Error as exc:
raise ConfigError(f"无法读取预览数据库: {exc}") from exc
try:
row = conn.execute(
"SELECT v FROM kv WHERE k = 'preview:seeded'"
).fetchone()
except sqlite3.Error as exc:
raise ConfigError(f"目标数据库不是预览库: {exc}") from exc
finally:
conn.close()
if row is None or str(row[0]) != "1":
raise ConfigError(
"目标数据库缺少 preview:seeded=1 标记;请先运行 "
"tools/seed_preview_db.py,或去掉 --preview"
)
def cmd_serve(args, logger: JsonLogger) -> int:
from .web.server import create_server, serve_forever
store = ConfigStore(args.config, require_auth=False)
cfg = store.get()
if args.preview:
_assert_preview_database(cfg.storage.database_path)
db = _open_db(cfg, logger)
try:
server = create_server(store, db, preview=args.preview, logger=logger)
serve_forever(server, logger=logger)
return 0
finally:
db.close()
@@ -491,6 +657,14 @@ def main(argv: Sequence[str] | None = None) -> int:
args = _build_parser().parse_args(list(argv) if argv is not None else None)
logger = JsonLogger()
try:
if args.command is None:
# 无子命令时启动完整服务:只读 Web + 调度 + Telegram。
args.command = "run"
args.once = False
args.allow_empty_folder = False
return cmd_run(args, logger, serve_web=True)
if args.command == "serve":
return cmd_serve(args, logger)
if args.command == "init":
return cmd_init(args, logger)
if args.command == "sync":