feat(web): add read-only dashboard server and query layer
This commit is contained in:
1 parent
e7cae035e0
commit
19572171b8
10 files changed
+2603
-10
No files matched your search
+184
-10
@@ -19,6 +19,7 @@ import argparse
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
@@ -48,7 +49,7 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
common = argparse.ArgumentParser(add_help=False)
|
||||
common.add_argument(
|
||||
"--config",
|
||||
default=str(DEFAULT_CONFIG),
|
||||
default=argparse.SUPPRESS,
|
||||
help="配置文件路径(默认项目根目录 config.json)",
|
||||
)
|
||||
|
||||
@@ -56,7 +57,12 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
prog="app.py",
|
||||
description="咻咻满 Reaction 数据看板 —— 数据底座、同步与每日快照",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
default=str(DEFAULT_CONFIG),
|
||||
help="配置文件路径(默认项目根目录 config.json)",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", required=False)
|
||||
|
||||
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
|
||||
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
|
||||
@@ -109,6 +115,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
action="store_true",
|
||||
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
|
||||
)
|
||||
serve = sub.add_parser("serve", parents=[common], help="启动只读本地看板(不运行调度)")
|
||||
serve.add_argument(
|
||||
"--preview", action="store_true",
|
||||
help="以预览模式启动(要求数据库含 preview:seeded=1 标记)",
|
||||
)
|
||||
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
|
||||
return parser
|
||||
|
||||
@@ -128,6 +139,32 @@ def _print(payload) -> None:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
|
||||
|
||||
|
||||
def _record_task(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
ok: bool,
|
||||
started_at: int | None = None,
|
||||
finished_at: int | None = None,
|
||||
payload=None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""把最近一次任务结果写入 kv,供只读系统页展示。"""
|
||||
now = int(time.time())
|
||||
record = {
|
||||
"name": name,
|
||||
"ok": bool(ok),
|
||||
"started_at": int(started_at if started_at is not None else now),
|
||||
"finished_at": int(finished_at if finished_at is not None else now),
|
||||
"error": error,
|
||||
"summary": payload,
|
||||
}
|
||||
try:
|
||||
repo.kv_set(f"task:last:{name}", json.dumps(record, ensure_ascii=False))
|
||||
except Exception: # noqa: BLE001 任务状态记录失败不影响主流程
|
||||
pass
|
||||
|
||||
|
||||
def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
|
||||
return BilibiliClient(
|
||||
cookie=credentials.cookie,
|
||||
@@ -174,10 +211,15 @@ def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
logger=logger,
|
||||
allow_empty_folders=args.allow_empty_folder,
|
||||
)
|
||||
started = int(time.time())
|
||||
summary = service.run(full_refresh=args.full)
|
||||
payload = summary.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
payload["counts"] = repo.counts()
|
||||
_record_task(
|
||||
repo, "folder_sync", ok=summary.ok, started_at=started,
|
||||
finished_at=int(time.time()), payload=payload,
|
||||
)
|
||||
_print(payload)
|
||||
return 0 if summary.ok else 1
|
||||
finally:
|
||||
@@ -199,8 +241,15 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
|
||||
payload["counts"] = repo.counts()
|
||||
ok = bool(summary.completed and not summary.backup_error)
|
||||
_record_task(
|
||||
repo, "snapshot", ok=ok,
|
||||
started_at=int(summary.started_at or time.time()),
|
||||
finished_at=int(time.time()), payload=payload,
|
||||
error=summary.backup_error,
|
||||
)
|
||||
_print(payload)
|
||||
return 0 if summary.completed and not summary.backup_error else 1
|
||||
return 0 if ok else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -239,6 +288,13 @@ def cmd_discover(args, logger: JsonLogger) -> int:
|
||||
payload = summary.as_dict()
|
||||
payload["candidate_total"] = len(repo.list_candidates())
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
_record_task(
|
||||
repo, "discovery", ok=not summary.errors,
|
||||
started_at=int(summary.started_at),
|
||||
finished_at=int(summary.finished_at or time.time()),
|
||||
payload=payload,
|
||||
error=";".join(summary.errors) if summary.errors else None,
|
||||
)
|
||||
_print(payload)
|
||||
return 0 if not summary.errors else 1
|
||||
finally:
|
||||
@@ -299,11 +355,12 @@ def cmd_auth_check(args, logger: JsonLogger) -> int:
|
||||
return 0 if all(result["ok"] for result in results) else 1
|
||||
|
||||
|
||||
def cmd_run(args, logger: JsonLogger) -> int:
|
||||
def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
store = ConfigStore(args.config, require_auth=True)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
last: dict[str, object] = {}
|
||||
web_server = None
|
||||
try:
|
||||
repo = Repository(db)
|
||||
folder_provider = ClientProvider(
|
||||
@@ -349,12 +406,29 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
)
|
||||
|
||||
def run_discover():
|
||||
summary = discovery_service.run()
|
||||
started = int(time.time())
|
||||
try:
|
||||
summary = discovery_service.run()
|
||||
except Exception as exc: # noqa: BLE001 记录后交给调度器隔离
|
||||
_record_task(
|
||||
repo, "discovery", ok=False, started_at=started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
)
|
||||
raise
|
||||
payload = summary.as_dict()
|
||||
_record_task(
|
||||
repo, "discovery", ok=not summary.errors,
|
||||
started_at=summary.started_at,
|
||||
finished_at=summary.finished_at or int(time.time()),
|
||||
payload=payload,
|
||||
error=";".join(summary.errors) if summary.errors else None,
|
||||
)
|
||||
last["discover"] = summary
|
||||
logger.info("discovery.summary", **summary.as_dict())
|
||||
logger.info("discovery.summary", **payload)
|
||||
return summary
|
||||
|
||||
def run_cleanup():
|
||||
started = int(time.time())
|
||||
removed = repo.delete_expired_candidates(
|
||||
ttl_days=cfg.schedule.candidate_ttl_days
|
||||
)
|
||||
@@ -363,21 +437,58 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
"cleanup.done",
|
||||
candidates_removed=removed, fav_actions_expired=expired,
|
||||
)
|
||||
_record_task(
|
||||
repo, "candidate_cleanup", ok=True, started_at=started,
|
||||
finished_at=int(time.time()),
|
||||
payload={
|
||||
"candidates_removed": removed,
|
||||
"fav_actions_expired": expired,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"candidates_removed": removed,
|
||||
"fav_actions_expired": expired,
|
||||
}
|
||||
|
||||
def run_snapshot():
|
||||
summary = snapshot_service.run_if_needed()
|
||||
started = int(time.time())
|
||||
try:
|
||||
summary = snapshot_service.run_if_needed()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_record_task(
|
||||
repo, "snapshot", ok=False, started_at=started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
)
|
||||
raise
|
||||
payload = summary.as_dict()
|
||||
ok = bool(summary.completed and not summary.backup_error)
|
||||
_record_task(
|
||||
repo, "snapshot", ok=ok, started_at=summary.started_at,
|
||||
finished_at=summary.finished_at or int(time.time()),
|
||||
payload=payload, error=summary.backup_error,
|
||||
)
|
||||
last["snapshot"] = summary
|
||||
logger.info("snapshot.summary", **summary.as_dict())
|
||||
logger.info("snapshot.summary", **payload)
|
||||
return summary
|
||||
|
||||
def run_sync():
|
||||
summary = folder_service.run()
|
||||
started = int(time.time())
|
||||
try:
|
||||
summary = folder_service.run()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_record_task(
|
||||
repo, "folder_sync", ok=False, started_at=started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
)
|
||||
raise
|
||||
payload = summary.as_dict()
|
||||
_record_task(
|
||||
repo, "folder_sync", ok=summary.ok, started_at=started,
|
||||
finished_at=int(time.time()), payload=payload,
|
||||
error=None if summary.ok else ";".join(summary.errors),
|
||||
)
|
||||
last["folder"] = summary
|
||||
logger.info("folder_sync.summary", **summary.as_dict())
|
||||
logger.info("folder_sync.summary", **payload)
|
||||
# 新入库/恢复的视频立即补当天快照,而不是等到次日。
|
||||
if summary.added > 0:
|
||||
run_snapshot()
|
||||
@@ -416,6 +527,15 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
],
|
||||
logger=logger,
|
||||
)
|
||||
if serve_web and not args.once:
|
||||
from .web.server import create_server, serve_forever
|
||||
|
||||
web_server = create_server(store, db, logger=logger)
|
||||
threading.Thread(
|
||||
target=lambda: serve_forever(web_server, logger=logger),
|
||||
name="web-server",
|
||||
daemon=True,
|
||||
).start()
|
||||
if notifier is not None and not args.once:
|
||||
poller = TelegramPoller(notifier, favorite_service, logger=logger)
|
||||
threading.Thread(
|
||||
@@ -442,6 +562,52 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True)
|
||||
discover_ok = discover_summary is None or not getattr(discover_summary, "errors", [])
|
||||
return 0 if folder_ok and snapshot_ok and discover_ok else 1
|
||||
finally:
|
||||
if web_server is not None:
|
||||
try:
|
||||
web_server.shutdown()
|
||||
finally:
|
||||
web_server.server_close()
|
||||
db.close()
|
||||
|
||||
|
||||
def _assert_preview_database(path: Path) -> None:
|
||||
import sqlite3
|
||||
|
||||
if not path.exists():
|
||||
raise ConfigError(f"预览数据库不存在: {path}")
|
||||
uri = f"file:{path.resolve().as_posix()}?mode=ro"
|
||||
try:
|
||||
conn = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as exc:
|
||||
raise ConfigError(f"无法读取预览数据库: {exc}") from exc
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT v FROM kv WHERE k = 'preview:seeded'"
|
||||
).fetchone()
|
||||
except sqlite3.Error as exc:
|
||||
raise ConfigError(f"目标数据库不是预览库: {exc}") from exc
|
||||
finally:
|
||||
conn.close()
|
||||
if row is None or str(row[0]) != "1":
|
||||
raise ConfigError(
|
||||
"目标数据库缺少 preview:seeded=1 标记;请先运行 "
|
||||
"tools/seed_preview_db.py,或去掉 --preview"
|
||||
)
|
||||
|
||||
|
||||
def cmd_serve(args, logger: JsonLogger) -> int:
|
||||
from .web.server import create_server, serve_forever
|
||||
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
if args.preview:
|
||||
_assert_preview_database(cfg.storage.database_path)
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
server = create_server(store, db, preview=args.preview, logger=logger)
|
||||
serve_forever(server, logger=logger)
|
||||
return 0
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -491,6 +657,14 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _build_parser().parse_args(list(argv) if argv is not None else None)
|
||||
logger = JsonLogger()
|
||||
try:
|
||||
if args.command is None:
|
||||
# 无子命令时启动完整服务:只读 Web + 调度 + Telegram。
|
||||
args.command = "run"
|
||||
args.once = False
|
||||
args.allow_empty_folder = False
|
||||
return cmd_run(args, logger, serve_web=True)
|
||||
if args.command == "serve":
|
||||
return cmd_serve(args, logger)
|
||||
if args.command == "init":
|
||||
return cmd_init(args, logger)
|
||||
if args.command == "sync":
|
||||
|
||||
Reference in new issue
Block a user