feat(p0): add favorite folder sync and sqlite data foundation
Implement P0 of the reaction dashboard: config loading with hot reload, SQLite migrations with backups, single-writer queue, read-only Bilibili fav client, folder import/diff sync, invalid-resource handling, blacklist seed, scheduler, and JSON redacted logging. Includes offline unit and transport-level contract tests; all existing and new tests pass.
This commit is contained in:
kairee committed
commit
e2cdf997af
39 files changed
+8575
No files matched your search
@@ -0,0 +1,495 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""运行期配置加载、校验与热加载。
|
||||
|
||||
支持两种写法：
|
||||
|
||||
* 新格式：``{"bilibili": {"mid": ..., "cookie": ...}, ...}``；
|
||||
* 旧 ``favsort.py`` 格式：顶层 ``mid`` / ``cookie`` / ``csrf``。
|
||||
|
||||
旧的扁平字段仍然被读取，因此 ``bili-fav auth`` 写回的 config.json 无需迁移即可
|
||||
被看板使用。敏感值只保存在配置对象中，日志输出必须先经过 ``logging_json`` 脱敏。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
DEFAULT_CLASSIFY: dict[str, Any] = {
|
||||
"require_title": "咻咻满",
|
||||
"positive": [
|
||||
"reaction", "re", "反应", "看", "听", "初听", "首次",
|
||||
"点评", "锐评", "路人", "震惊", "双人",
|
||||
],
|
||||
"charge": ["充电", "加急", "专场"],
|
||||
"negative": ["歌切", "切片", "纯享", "搬运", "完整版"],
|
||||
"duration_boost": {"gt_30min": 1, "gt_60min": 2},
|
||||
}
|
||||
|
||||
DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
||||
"view": 0.30,
|
||||
"like": 0.20,
|
||||
"coin": 0.20,
|
||||
"favorite": 0.15,
|
||||
"share": 0.10,
|
||||
"reply": 0.03,
|
||||
"danmaku": 0.02,
|
||||
}
|
||||
|
||||
DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"bilibili": {"mid": 0, "csrf": "", "cookie": ""},
|
||||
"telegram": {"enabled": False, "bot_token": "", "chat_id": 0},
|
||||
"web": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 8080,
|
||||
"auth_token": "",
|
||||
"public_base_url": "",
|
||||
},
|
||||
"storage": {
|
||||
"database_path": "data/app.db",
|
||||
"backup_dir": "data/backup",
|
||||
"backup_keep_days": 14,
|
||||
},
|
||||
"folders": [],
|
||||
"track_fans": [],
|
||||
"up_blacklist_seed": [],
|
||||
"schedule": {
|
||||
"discover_minutes": 60,
|
||||
"folder_sync_minutes": 60,
|
||||
"full_folder_refresh_hours": 6,
|
||||
"snapshot_at": "00:01",
|
||||
"candidate_ttl_days": 7,
|
||||
},
|
||||
"notify": {"min_score": 2, "max_items_per_message": 10},
|
||||
"classify": DEFAULT_CLASSIFY,
|
||||
"heat_weights": DEFAULT_HEAT_WEIGHTS,
|
||||
}
|
||||
|
||||
# 环境变量只作为 CI / 临时调试的覆盖入口，不作为主路径。
|
||||
ENV_OVERRIDES: dict[str, tuple[tuple[str, ...], type]] = {
|
||||
"BILI_COOKIE": (("bilibili", "cookie"), str),
|
||||
"BILI_CSRF": (("bilibili", "csrf"), str),
|
||||
"BILI_MID": (("bilibili", "mid"), int),
|
||||
"TG_BOT_TOKEN": (("telegram", "bot_token"), str),
|
||||
"TG_CHAT_ID": (("telegram", "chat_id"), int),
|
||||
"DASHBOARD_AUTH_TOKEN": (("web", "auth_token"), str),
|
||||
"DATABASE_PATH": (("storage", "database_path"), str),
|
||||
}
|
||||
|
||||
_SNAPSHOT_AT_RE = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$")
|
||||
_HEAT_METRICS = tuple(DEFAULT_HEAT_WEIGHTS)
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""配置文件缺失、格式错误或字段非法。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BilibiliConfig:
|
||||
mid: int
|
||||
csrf: str
|
||||
cookie: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TelegramConfig:
|
||||
enabled: bool
|
||||
bot_token: str
|
||||
chat_id: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WebConfig:
|
||||
host: str
|
||||
port: int
|
||||
auth_token: str
|
||||
public_base_url: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StorageConfig:
|
||||
database_path: Path
|
||||
backup_dir: Path
|
||||
backup_keep_days: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FolderConfig:
|
||||
year: int
|
||||
media_id: int
|
||||
title: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScheduleConfig:
|
||||
discover_minutes: int
|
||||
folder_sync_minutes: int
|
||||
full_folder_refresh_hours: int
|
||||
snapshot_at: str
|
||||
candidate_ttl_days: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NotifyConfig:
|
||||
min_score: int
|
||||
max_items_per_message: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppConfig:
|
||||
path: Path
|
||||
bilibili: BilibiliConfig
|
||||
telegram: TelegramConfig
|
||||
web: WebConfig
|
||||
storage: StorageConfig
|
||||
folders: tuple[FolderConfig, ...]
|
||||
track_fans: tuple[int, ...]
|
||||
up_blacklist_seed: tuple[int, ...]
|
||||
schedule: ScheduleConfig
|
||||
notify: NotifyConfig
|
||||
classify: Mapping[str, Any]
|
||||
heat_weights: Mapping[str, float]
|
||||
raw: Mapping[str, Any] = field(repr=False, compare=False)
|
||||
|
||||
def folder_for_year(self, year: int) -> FolderConfig | None:
|
||||
for folder in self.folders:
|
||||
if folder.year == year:
|
||||
return folder
|
||||
return None
|
||||
|
||||
def latest_folder(self) -> FolderConfig | None:
|
||||
if not self.folders:
|
||||
return None
|
||||
return max(self.folders, key=lambda f: f.year)
|
||||
|
||||
|
||||
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
|
||||
out = copy.deepcopy(dict(base))
|
||||
for key, value in override.items():
|
||||
if isinstance(value, Mapping) and isinstance(out.get(key), Mapping):
|
||||
out[key] = _deep_merge(out[key], value)
|
||||
else:
|
||||
out[key] = copy.deepcopy(value)
|
||||
return out
|
||||
|
||||
|
||||
def _get_path(data: Mapping[str, Any], path: tuple[str, ...]) -> Any:
|
||||
cur: Any = data
|
||||
for key in path:
|
||||
if not isinstance(cur, Mapping) or key not in cur:
|
||||
return None
|
||||
cur = cur[key]
|
||||
return cur
|
||||
|
||||
|
||||
def _set_path(data: dict[str, Any], path: tuple[str, ...], value: Any) -> None:
|
||||
cur = data
|
||||
for key in path[:-1]:
|
||||
cur = cur.setdefault(key, {})
|
||||
cur[path[-1]] = value
|
||||
|
||||
|
||||
def _as_int(value: Any, name: str, *, minimum: int | None = None,
|
||||
maximum: int | None = None) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ConfigError(f"{name} 必须是整数")
|
||||
if minimum is not None and value < minimum:
|
||||
raise ConfigError(f"{name} 不能小于 {minimum}")
|
||||
if maximum is not None and value > maximum:
|
||||
raise ConfigError(f"{name} 不能大于 {maximum}")
|
||||
return value
|
||||
|
||||
|
||||
def _as_str(value: Any, name: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ConfigError(f"{name} 必须是字符串")
|
||||
return value
|
||||
|
||||
|
||||
def _as_int_list(value: Any, name: str) -> tuple[int, ...]:
|
||||
if not isinstance(value, list):
|
||||
raise ConfigError(f"{name} 必须是数组")
|
||||
out: list[int] = []
|
||||
for item in value:
|
||||
mid = _as_int(item, f"{name}[]", minimum=1)
|
||||
if mid not in out:
|
||||
out.append(mid)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _as_str_list(value: Any, name: str) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
raise ConfigError(f"{name} 必须是数组")
|
||||
out = []
|
||||
for item in value:
|
||||
if not isinstance(item, str) or not item:
|
||||
raise ConfigError(f"{name}[] 必须是非空字符串")
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
def _legacy_to_current(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""把 favsort 的扁平字段映射到新格式，不修改调用方对象。"""
|
||||
data = copy.deepcopy(dict(raw))
|
||||
if "bilibili" not in data:
|
||||
data["bilibili"] = {
|
||||
"mid": raw.get("mid", 0),
|
||||
"csrf": raw.get("csrf", ""),
|
||||
"cookie": raw.get("cookie", ""),
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _validate_folders(value: Any) -> tuple[FolderConfig, ...]:
|
||||
if not isinstance(value, list):
|
||||
raise ConfigError("folders 必须是数组")
|
||||
folders: list[FolderConfig] = []
|
||||
seen_ids: set[int] = set()
|
||||
seen_years: set[int] = set()
|
||||
for index, item in enumerate(value):
|
||||
name = f"folders[{index}]"
|
||||
if not isinstance(item, Mapping):
|
||||
raise ConfigError(f"{name} 必须是对象")
|
||||
year = _as_int(item.get("year"), f"{name}.year", minimum=1900, maximum=9999)
|
||||
media_id = _as_int(item.get("media_id"), f"{name}.media_id", minimum=1)
|
||||
title = _as_str(item.get("title", "") or "", f"{name}.title")
|
||||
if media_id in seen_ids:
|
||||
raise ConfigError(f"{name}.media_id 重复: {media_id}")
|
||||
if year in seen_years:
|
||||
raise ConfigError(f"{name}.year 重复: {year}")
|
||||
seen_ids.add(media_id)
|
||||
seen_years.add(year)
|
||||
folders.append(FolderConfig(year=year, media_id=media_id, title=title))
|
||||
return tuple(folders)
|
||||
|
||||
|
||||
def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
b = data["bilibili"]
|
||||
if not isinstance(b, Mapping):
|
||||
raise ConfigError("bilibili 必须是对象")
|
||||
bilibili = BilibiliConfig(
|
||||
mid=_as_int(b.get("mid", 0), "bilibili.mid", minimum=0),
|
||||
csrf=_as_str(b.get("csrf", "") or "", "bilibili.csrf"),
|
||||
cookie=_as_str(b.get("cookie", "") or "", "bilibili.cookie"),
|
||||
)
|
||||
|
||||
tg = data["telegram"]
|
||||
if not isinstance(tg, Mapping):
|
||||
raise ConfigError("telegram 必须是对象")
|
||||
telegram = TelegramConfig(
|
||||
enabled=bool(tg.get("enabled", False)),
|
||||
bot_token=_as_str(tg.get("bot_token", "") or "", "telegram.bot_token"),
|
||||
chat_id=_as_int(tg.get("chat_id", 0), "telegram.chat_id"),
|
||||
)
|
||||
|
||||
web_raw = data["web"]
|
||||
if not isinstance(web_raw, Mapping):
|
||||
raise ConfigError("web 必须是对象")
|
||||
web = WebConfig(
|
||||
host=_as_str(web_raw.get("host", "127.0.0.1"), "web.host"),
|
||||
port=_as_int(web_raw.get("port", 8080), "web.port", minimum=1, maximum=65535),
|
||||
auth_token=_as_str(web_raw.get("auth_token", "") or "", "web.auth_token"),
|
||||
public_base_url=_as_str(
|
||||
web_raw.get("public_base_url", "") or "", "web.public_base_url"
|
||||
),
|
||||
)
|
||||
|
||||
storage_raw = data["storage"]
|
||||
if not isinstance(storage_raw, Mapping):
|
||||
raise ConfigError("storage 必须是对象")
|
||||
base_dir = path.parent
|
||||
database_path = Path(_as_str(
|
||||
storage_raw.get("database_path", "data/app.db") or "data/app.db",
|
||||
"storage.database_path",
|
||||
))
|
||||
backup_dir = Path(_as_str(
|
||||
storage_raw.get("backup_dir", "data/backup") or "data/backup",
|
||||
"storage.backup_dir",
|
||||
))
|
||||
if not database_path.is_absolute():
|
||||
database_path = base_dir / database_path
|
||||
if not backup_dir.is_absolute():
|
||||
backup_dir = base_dir / backup_dir
|
||||
storage = StorageConfig(
|
||||
database_path=database_path,
|
||||
backup_dir=backup_dir,
|
||||
backup_keep_days=_as_int(
|
||||
storage_raw.get("backup_keep_days", 14),
|
||||
"storage.backup_keep_days",
|
||||
minimum=1,
|
||||
),
|
||||
)
|
||||
|
||||
sched_raw = data["schedule"]
|
||||
if not isinstance(sched_raw, Mapping):
|
||||
raise ConfigError("schedule 必须是对象")
|
||||
snapshot_at = _as_str(sched_raw.get("snapshot_at", "00:01"), "schedule.snapshot_at")
|
||||
if not _SNAPSHOT_AT_RE.match(snapshot_at):
|
||||
raise ConfigError("schedule.snapshot_at 必须是 HH:MM（24 小时制）")
|
||||
schedule = ScheduleConfig(
|
||||
discover_minutes=_as_int(
|
||||
sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1
|
||||
),
|
||||
folder_sync_minutes=_as_int(
|
||||
sched_raw.get("folder_sync_minutes", 60),
|
||||
"schedule.folder_sync_minutes",
|
||||
minimum=1,
|
||||
),
|
||||
full_folder_refresh_hours=_as_int(
|
||||
sched_raw.get("full_folder_refresh_hours", 6),
|
||||
"schedule.full_folder_refresh_hours",
|
||||
minimum=1,
|
||||
),
|
||||
snapshot_at=snapshot_at,
|
||||
candidate_ttl_days=_as_int(
|
||||
sched_raw.get("candidate_ttl_days", 7),
|
||||
"schedule.candidate_ttl_days",
|
||||
minimum=1,
|
||||
),
|
||||
)
|
||||
|
||||
notify_raw = data["notify"]
|
||||
if not isinstance(notify_raw, Mapping):
|
||||
raise ConfigError("notify 必须是对象")
|
||||
notify = NotifyConfig(
|
||||
min_score=_as_int(notify_raw.get("min_score", 2), "notify.min_score"),
|
||||
max_items_per_message=_as_int(
|
||||
notify_raw.get("max_items_per_message", 10),
|
||||
"notify.max_items_per_message",
|
||||
minimum=1,
|
||||
),
|
||||
)
|
||||
|
||||
classify_raw = data["classify"]
|
||||
if not isinstance(classify_raw, Mapping):
|
||||
raise ConfigError("classify 必须是对象")
|
||||
classify = copy.deepcopy(dict(classify_raw))
|
||||
classify["require_title"] = _as_str(
|
||||
classify.get("require_title", "咻咻满"), "classify.require_title"
|
||||
)
|
||||
if not classify["require_title"]:
|
||||
raise ConfigError("classify.require_title 不能为空")
|
||||
for key in ("positive", "charge", "negative"):
|
||||
classify[key] = _as_str_list(classify.get(key, []), f"classify.{key}")
|
||||
boost = classify.get("duration_boost", {})
|
||||
if not isinstance(boost, Mapping):
|
||||
raise ConfigError("classify.duration_boost 必须是对象")
|
||||
classify["duration_boost"] = {
|
||||
"gt_30min": _as_int(boost.get("gt_30min", 1), "classify.duration_boost.gt_30min"),
|
||||
"gt_60min": _as_int(boost.get("gt_60min", 2), "classify.duration_boost.gt_60min"),
|
||||
}
|
||||
|
||||
heat_raw = data["heat_weights"]
|
||||
if not isinstance(heat_raw, Mapping):
|
||||
raise ConfigError("heat_weights 必须是对象")
|
||||
heat: dict[str, float] = {}
|
||||
for metric in _HEAT_METRICS:
|
||||
value = heat_raw.get(metric, DEFAULT_HEAT_WEIGHTS[metric])
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
|
||||
raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
|
||||
heat[metric] = float(value)
|
||||
|
||||
return AppConfig(
|
||||
path=path,
|
||||
bilibili=bilibili,
|
||||
telegram=telegram,
|
||||
web=web,
|
||||
storage=storage,
|
||||
folders=_validate_folders(data["folders"]),
|
||||
track_fans=_as_int_list(data["track_fans"], "track_fans"),
|
||||
up_blacklist_seed=_as_int_list(data["up_blacklist_seed"], "up_blacklist_seed"),
|
||||
schedule=schedule,
|
||||
notify=notify,
|
||||
classify=classify,
|
||||
heat_weights=heat,
|
||||
raw=data,
|
||||
)
|
||||
|
||||
|
||||
def load_config(
|
||||
path: str | os.PathLike[str],
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
require_auth: bool = False,
|
||||
) -> AppConfig:
|
||||
"""读取并校验配置。``env`` 默认为 ``os.environ``。"""
|
||||
config_path = Path(path)
|
||||
if not config_path.exists():
|
||||
raise ConfigError(f"缺少配置文件: {config_path}")
|
||||
try:
|
||||
with open(config_path, encoding="utf-8") as fh:
|
||||
raw = json.load(fh)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ConfigError(f"配置文件不是合法 JSON: {config_path}: {exc}") from exc
|
||||
except OSError as exc:
|
||||
raise ConfigError(f"无法读取配置文件: {config_path}: {exc}") from exc
|
||||
if not isinstance(raw, Mapping):
|
||||
raise ConfigError("配置文件顶层必须是 JSON 对象")
|
||||
|
||||
data = _deep_merge(DEFAULT_CONFIG, _legacy_to_current(raw))
|
||||
|
||||
env_map = os.environ if env is None else env
|
||||
for env_name, (target, caster) in ENV_OVERRIDES.items():
|
||||
if env_name not in env_map:
|
||||
continue
|
||||
value: Any = env_map[env_name]
|
||||
if caster is int:
|
||||
try:
|
||||
value = int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ConfigError(f"环境变量 {env_name} 必须是整数") from exc
|
||||
_set_path(data, target, value)
|
||||
|
||||
cfg = _build_config(data, config_path)
|
||||
if require_auth and (not cfg.bilibili.cookie or not cfg.bilibili.csrf):
|
||||
raise ConfigError(
|
||||
"config.json 中的 cookie / csrf 不能为空（可先运行 bili-fav auth）"
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
class ConfigStore:
|
||||
"""按文件 mtime 热加载配置；坏配置不会覆盖上一份可用配置。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str | os.PathLike[str],
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
require_auth: bool = False,
|
||||
) -> None:
|
||||
self.path = Path(path)
|
||||
self._env = env
|
||||
self._require_auth = require_auth
|
||||
self._lock = threading.RLock()
|
||||
self._config = load_config(path, env=env, require_auth=require_auth)
|
||||
self._mtime_ns = self._read_mtime_ns()
|
||||
self.last_error: str | None = None
|
||||
|
||||
def _read_mtime_ns(self) -> int | None:
|
||||
try:
|
||||
return self.path.stat().st_mtime_ns
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
def get(self) -> AppConfig:
|
||||
with self._lock:
|
||||
mtime_ns = self._read_mtime_ns()
|
||||
if mtime_ns is None or mtime_ns == self._mtime_ns:
|
||||
return self._config
|
||||
self._mtime_ns = mtime_ns
|
||||
try:
|
||||
self._config = load_config(
|
||||
self.path, env=self._env, require_auth=self._require_auth
|
||||
)
|
||||
except ConfigError as exc:
|
||||
self.last_error = str(exc)
|
||||
return self._config
|
||||
self.last_error = None
|
||||
return self._config
|
||||
Reference in new issue
Block a user