kairee
6a21b5d905
Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
505 lines
16 KiB
Python
505 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""运行期配置加载、校验与热加载。
|
||
|
||
B 站凭据必须采用 ``bilibili.read`` / ``bilibili.write`` 双账号格式。敏感值只
|
||
保存在配置对象中，日志输出必须先经过 ``logging_json`` 脱敏。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import json
|
||
import os
|
||
import re
|
||
import threading
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any, Mapping
|
||
|
||
DEFAULT_CLASSIFY: dict[str, Any] = {
|
||
"require_title": "咻咻满",
|
||
"positive": [
|
||
"reaction", "re", "反应", "看", "听", "初听", "首次",
|
||
"点评", "锐评", "路人", "震惊", "双人",
|
||
],
|
||
"charge": ["充电", "加急", "专场"],
|
||
"negative": ["歌切", "切片", "纯享", "搬运", "完整版"],
|
||
"duration_boost": {"gt_30min": 1, "gt_60min": 2},
|
||
}
|
||
|
||
DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
||
"view": 0.30,
|
||
"like": 0.20,
|
||
"coin": 0.20,
|
||
"favorite": 0.15,
|
||
"share": 0.10,
|
||
"reply": 0.03,
|
||
"danmaku": 0.02,
|
||
}
|
||
|
||
DEFAULT_CONFIG: dict[str, Any] = {
|
||
"bilibili": {
|
||
"read": {"mid": 0, "csrf": "", "cookie": ""},
|
||
"write": {"mid": 0, "csrf": "", "cookie": ""},
|
||
},
|
||
"telegram": {"enabled": False, "bot_token": "", "chat_id": 0},
|
||
"web": {
|
||
"host": "127.0.0.1",
|
||
"port": 8080,
|
||
"auth_token": "",
|
||
"public_base_url": "",
|
||
},
|
||
"storage": {
|
||
"database_path": "data/app.db",
|
||
"backup_dir": "data/backup",
|
||
"backup_keep_days": 14,
|
||
},
|
||
"folders": [],
|
||
"track_fans": [],
|
||
"up_blacklist_seed": [],
|
||
"schedule": {
|
||
"discover_minutes": 60,
|
||
"folder_sync_minutes": 60,
|
||
"full_folder_refresh_hours": 6,
|
||
"snapshot_at": "00:01",
|
||
"candidate_ttl_days": 7,
|
||
},
|
||
"notify": {"min_score": 2, "max_items_per_message": 10},
|
||
"classify": DEFAULT_CLASSIFY,
|
||
"heat_weights": DEFAULT_HEAT_WEIGHTS,
|
||
}
|
||
|
||
# 环境变量只作为 CI / 临时调试的覆盖入口，不作为主路径。
|
||
ENV_OVERRIDES: dict[str, tuple[tuple[str, ...], type]] = {
|
||
"BILI_READ_COOKIE": (("bilibili", "read", "cookie"), str),
|
||
"BILI_READ_CSRF": (("bilibili", "read", "csrf"), str),
|
||
"BILI_READ_MID": (("bilibili", "read", "mid"), int),
|
||
"BILI_WRITE_COOKIE": (("bilibili", "write", "cookie"), str),
|
||
"BILI_WRITE_CSRF": (("bilibili", "write", "csrf"), str),
|
||
"BILI_WRITE_MID": (("bilibili", "write", "mid"), int),
|
||
"TG_BOT_TOKEN": (("telegram", "bot_token"), str),
|
||
"TG_CHAT_ID": (("telegram", "chat_id"), int),
|
||
"DASHBOARD_AUTH_TOKEN": (("web", "auth_token"), str),
|
||
"DATABASE_PATH": (("storage", "database_path"), str),
|
||
}
|
||
|
||
_SNAPSHOT_AT_RE = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$")
|
||
_HEAT_METRICS = tuple(DEFAULT_HEAT_WEIGHTS)
|
||
|
||
|
||
class ConfigError(ValueError):
|
||
"""配置文件缺失、格式错误或字段非法。"""
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class BilibiliCredentials:
|
||
mid: int
|
||
csrf: str
|
||
cookie: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class BilibiliConfig:
|
||
read: BilibiliCredentials
|
||
write: BilibiliCredentials
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class TelegramConfig:
|
||
enabled: bool
|
||
bot_token: str
|
||
chat_id: int
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class WebConfig:
|
||
host: str
|
||
port: int
|
||
auth_token: str
|
||
public_base_url: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class StorageConfig:
|
||
database_path: Path
|
||
backup_dir: Path
|
||
backup_keep_days: int
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class FolderConfig:
|
||
year: int
|
||
media_id: int
|
||
title: str = ""
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ScheduleConfig:
|
||
discover_minutes: int
|
||
folder_sync_minutes: int
|
||
full_folder_refresh_hours: int
|
||
snapshot_at: str
|
||
candidate_ttl_days: int
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class NotifyConfig:
|
||
min_score: int
|
||
max_items_per_message: int
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class AppConfig:
|
||
path: Path
|
||
bilibili: BilibiliConfig
|
||
telegram: TelegramConfig
|
||
web: WebConfig
|
||
storage: StorageConfig
|
||
folders: tuple[FolderConfig, ...]
|
||
track_fans: tuple[int, ...]
|
||
up_blacklist_seed: tuple[int, ...]
|
||
schedule: ScheduleConfig
|
||
notify: NotifyConfig
|
||
classify: Mapping[str, Any]
|
||
heat_weights: Mapping[str, float]
|
||
raw: Mapping[str, Any] = field(repr=False, compare=False)
|
||
|
||
def folder_for_year(self, year: int) -> FolderConfig | None:
|
||
for folder in self.folders:
|
||
if folder.year == year:
|
||
return folder
|
||
return None
|
||
|
||
def latest_folder(self) -> FolderConfig | None:
|
||
if not self.folders:
|
||
return None
|
||
return max(self.folders, key=lambda f: f.year)
|
||
|
||
|
||
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
|
||
out = copy.deepcopy(dict(base))
|
||
for key, value in override.items():
|
||
if isinstance(value, Mapping) and isinstance(out.get(key), Mapping):
|
||
out[key] = _deep_merge(out[key], value)
|
||
else:
|
||
out[key] = copy.deepcopy(value)
|
||
return out
|
||
|
||
|
||
def _get_path(data: Mapping[str, Any], path: tuple[str, ...]) -> Any:
|
||
cur: Any = data
|
||
for key in path:
|
||
if not isinstance(cur, Mapping) or key not in cur:
|
||
return None
|
||
cur = cur[key]
|
||
return cur
|
||
|
||
|
||
def _set_path(data: dict[str, Any], path: tuple[str, ...], value: Any) -> None:
|
||
cur = data
|
||
for key in path[:-1]:
|
||
cur = cur.setdefault(key, {})
|
||
cur[path[-1]] = value
|
||
|
||
|
||
def _as_int(value: Any, name: str, *, minimum: int | None = None,
|
||
maximum: int | None = None) -> int:
|
||
if isinstance(value, bool) or not isinstance(value, int):
|
||
raise ConfigError(f"{name} 必须是整数")
|
||
if minimum is not None and value < minimum:
|
||
raise ConfigError(f"{name} 不能小于 {minimum}")
|
||
if maximum is not None and value > maximum:
|
||
raise ConfigError(f"{name} 不能大于 {maximum}")
|
||
return value
|
||
|
||
|
||
def _as_str(value: Any, name: str) -> str:
|
||
if not isinstance(value, str):
|
||
raise ConfigError(f"{name} 必须是字符串")
|
||
return value
|
||
|
||
|
||
def _as_int_list(value: Any, name: str) -> tuple[int, ...]:
|
||
if not isinstance(value, list):
|
||
raise ConfigError(f"{name} 必须是数组")
|
||
out: list[int] = []
|
||
for item in value:
|
||
mid = _as_int(item, f"{name}[]", minimum=1)
|
||
if mid not in out:
|
||
out.append(mid)
|
||
return tuple(out)
|
||
|
||
|
||
def _as_str_list(value: Any, name: str) -> list[str]:
|
||
if not isinstance(value, list):
|
||
raise ConfigError(f"{name} 必须是数组")
|
||
out = []
|
||
for item in value:
|
||
if not isinstance(item, str) or not item:
|
||
raise ConfigError(f"{name}[] 必须是非空字符串")
|
||
out.append(item)
|
||
return out
|
||
|
||
|
||
def _validate_folders(value: Any) -> tuple[FolderConfig, ...]:
|
||
if not isinstance(value, list):
|
||
raise ConfigError("folders 必须是数组")
|
||
folders: list[FolderConfig] = []
|
||
seen_ids: set[int] = set()
|
||
seen_years: set[int] = set()
|
||
for index, item in enumerate(value):
|
||
name = f"folders[{index}]"
|
||
if not isinstance(item, Mapping):
|
||
raise ConfigError(f"{name} 必须是对象")
|
||
year = _as_int(item.get("year"), f"{name}.year", minimum=1900, maximum=9999)
|
||
media_id = _as_int(item.get("media_id"), f"{name}.media_id", minimum=1)
|
||
title = _as_str(item.get("title", "") or "", f"{name}.title")
|
||
if media_id in seen_ids:
|
||
raise ConfigError(f"{name}.media_id 重复: {media_id}")
|
||
if year in seen_years:
|
||
raise ConfigError(f"{name}.year 重复: {year}")
|
||
seen_ids.add(media_id)
|
||
seen_years.add(year)
|
||
folders.append(FolderConfig(year=year, media_id=media_id, title=title))
|
||
return tuple(folders)
|
||
|
||
|
||
def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||
b = data["bilibili"]
|
||
if not isinstance(b, Mapping):
|
||
raise ConfigError("bilibili 必须是对象")
|
||
def credentials(account: str) -> BilibiliCredentials:
|
||
value = b.get(account)
|
||
if not isinstance(value, Mapping):
|
||
raise ConfigError(f"bilibili.{account} 必须是对象")
|
||
return BilibiliCredentials(
|
||
mid=_as_int(value.get("mid", 0), f"bilibili.{account}.mid", minimum=0),
|
||
csrf=_as_str(value.get("csrf", "") or "", f"bilibili.{account}.csrf"),
|
||
cookie=_as_str(value.get("cookie", "") or "", f"bilibili.{account}.cookie"),
|
||
)
|
||
|
||
bilibili = BilibiliConfig(read=credentials("read"), write=credentials("write"))
|
||
|
||
tg = data["telegram"]
|
||
if not isinstance(tg, Mapping):
|
||
raise ConfigError("telegram 必须是对象")
|
||
telegram = TelegramConfig(
|
||
enabled=bool(tg.get("enabled", False)),
|
||
bot_token=_as_str(tg.get("bot_token", "") or "", "telegram.bot_token"),
|
||
chat_id=_as_int(tg.get("chat_id", 0), "telegram.chat_id"),
|
||
)
|
||
|
||
web_raw = data["web"]
|
||
if not isinstance(web_raw, Mapping):
|
||
raise ConfigError("web 必须是对象")
|
||
web = WebConfig(
|
||
host=_as_str(web_raw.get("host", "127.0.0.1"), "web.host"),
|
||
port=_as_int(web_raw.get("port", 8080), "web.port", minimum=1, maximum=65535),
|
||
auth_token=_as_str(web_raw.get("auth_token", "") or "", "web.auth_token"),
|
||
public_base_url=_as_str(
|
||
web_raw.get("public_base_url", "") or "", "web.public_base_url"
|
||
),
|
||
)
|
||
|
||
storage_raw = data["storage"]
|
||
if not isinstance(storage_raw, Mapping):
|
||
raise ConfigError("storage 必须是对象")
|
||
base_dir = path.parent
|
||
database_path = Path(_as_str(
|
||
storage_raw.get("database_path", "data/app.db") or "data/app.db",
|
||
"storage.database_path",
|
||
))
|
||
backup_dir = Path(_as_str(
|
||
storage_raw.get("backup_dir", "data/backup") or "data/backup",
|
||
"storage.backup_dir",
|
||
))
|
||
if not database_path.is_absolute():
|
||
database_path = base_dir / database_path
|
||
if not backup_dir.is_absolute():
|
||
backup_dir = base_dir / backup_dir
|
||
storage = StorageConfig(
|
||
database_path=database_path,
|
||
backup_dir=backup_dir,
|
||
backup_keep_days=_as_int(
|
||
storage_raw.get("backup_keep_days", 14),
|
||
"storage.backup_keep_days",
|
||
minimum=1,
|
||
),
|
||
)
|
||
|
||
sched_raw = data["schedule"]
|
||
if not isinstance(sched_raw, Mapping):
|
||
raise ConfigError("schedule 必须是对象")
|
||
snapshot_at = _as_str(sched_raw.get("snapshot_at", "00:01"), "schedule.snapshot_at")
|
||
if not _SNAPSHOT_AT_RE.match(snapshot_at):
|
||
raise ConfigError("schedule.snapshot_at 必须是 HH:MM（24 小时制）")
|
||
schedule = ScheduleConfig(
|
||
discover_minutes=_as_int(
|
||
sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1
|
||
),
|
||
folder_sync_minutes=_as_int(
|
||
sched_raw.get("folder_sync_minutes", 60),
|
||
"schedule.folder_sync_minutes",
|
||
minimum=1,
|
||
),
|
||
full_folder_refresh_hours=_as_int(
|
||
sched_raw.get("full_folder_refresh_hours", 6),
|
||
"schedule.full_folder_refresh_hours",
|
||
minimum=1,
|
||
),
|
||
snapshot_at=snapshot_at,
|
||
candidate_ttl_days=_as_int(
|
||
sched_raw.get("candidate_ttl_days", 7),
|
||
"schedule.candidate_ttl_days",
|
||
minimum=1,
|
||
),
|
||
)
|
||
|
||
notify_raw = data["notify"]
|
||
if not isinstance(notify_raw, Mapping):
|
||
raise ConfigError("notify 必须是对象")
|
||
notify = NotifyConfig(
|
||
min_score=_as_int(notify_raw.get("min_score", 2), "notify.min_score"),
|
||
max_items_per_message=_as_int(
|
||
notify_raw.get("max_items_per_message", 10),
|
||
"notify.max_items_per_message",
|
||
minimum=1,
|
||
),
|
||
)
|
||
|
||
classify_raw = data["classify"]
|
||
if not isinstance(classify_raw, Mapping):
|
||
raise ConfigError("classify 必须是对象")
|
||
classify = copy.deepcopy(dict(classify_raw))
|
||
classify["require_title"] = _as_str(
|
||
classify.get("require_title", "咻咻满"), "classify.require_title"
|
||
)
|
||
if not classify["require_title"]:
|
||
raise ConfigError("classify.require_title 不能为空")
|
||
for key in ("positive", "charge", "negative"):
|
||
classify[key] = _as_str_list(classify.get(key, []), f"classify.{key}")
|
||
boost = classify.get("duration_boost", {})
|
||
if not isinstance(boost, Mapping):
|
||
raise ConfigError("classify.duration_boost 必须是对象")
|
||
classify["duration_boost"] = {
|
||
"gt_30min": _as_int(boost.get("gt_30min", 1), "classify.duration_boost.gt_30min"),
|
||
"gt_60min": _as_int(boost.get("gt_60min", 2), "classify.duration_boost.gt_60min"),
|
||
}
|
||
|
||
heat_raw = data["heat_weights"]
|
||
if not isinstance(heat_raw, Mapping):
|
||
raise ConfigError("heat_weights 必须是对象")
|
||
heat: dict[str, float] = {}
|
||
for metric in _HEAT_METRICS:
|
||
value = heat_raw.get(metric, DEFAULT_HEAT_WEIGHTS[metric])
|
||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
|
||
raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
|
||
heat[metric] = float(value)
|
||
|
||
return AppConfig(
|
||
path=path,
|
||
bilibili=bilibili,
|
||
telegram=telegram,
|
||
web=web,
|
||
storage=storage,
|
||
folders=_validate_folders(data["folders"]),
|
||
track_fans=_as_int_list(data["track_fans"], "track_fans"),
|
||
up_blacklist_seed=_as_int_list(data["up_blacklist_seed"], "up_blacklist_seed"),
|
||
schedule=schedule,
|
||
notify=notify,
|
||
classify=classify,
|
||
heat_weights=heat,
|
||
raw=data,
|
||
)
|
||
|
||
|
||
def load_config(
|
||
path: str | os.PathLike[str],
|
||
*,
|
||
env: Mapping[str, str] | None = None,
|
||
require_auth: bool = False,
|
||
) -> AppConfig:
|
||
"""读取并校验配置。``env`` 默认为 ``os.environ``。"""
|
||
config_path = Path(path)
|
||
if not config_path.exists():
|
||
raise ConfigError(f"缺少配置文件: {config_path}")
|
||
try:
|
||
with open(config_path, encoding="utf-8") as fh:
|
||
raw = json.load(fh)
|
||
except json.JSONDecodeError as exc:
|
||
raise ConfigError(f"配置文件不是合法 JSON: {config_path}: {exc}") from exc
|
||
except OSError as exc:
|
||
raise ConfigError(f"无法读取配置文件: {config_path}: {exc}") from exc
|
||
if not isinstance(raw, Mapping):
|
||
raise ConfigError("配置文件顶层必须是 JSON 对象")
|
||
raw_bilibili = raw.get("bilibili")
|
||
if not isinstance(raw_bilibili, Mapping):
|
||
raise ConfigError("缺少 bilibili.read / bilibili.write 双账号配置")
|
||
for account in ("read", "write"):
|
||
if account not in raw_bilibili:
|
||
raise ConfigError(f"缺少 bilibili.{account} 账号配置")
|
||
|
||
data = _deep_merge(DEFAULT_CONFIG, raw)
|
||
|
||
env_map = os.environ if env is None else env
|
||
for env_name, (target, caster) in ENV_OVERRIDES.items():
|
||
if env_name not in env_map:
|
||
continue
|
||
value: Any = env_map[env_name]
|
||
if caster is int:
|
||
try:
|
||
value = int(value)
|
||
except (TypeError, ValueError) as exc:
|
||
raise ConfigError(f"环境变量 {env_name} 必须是整数") from exc
|
||
_set_path(data, target, value)
|
||
|
||
cfg = _build_config(data, config_path)
|
||
if require_auth and (not cfg.bilibili.read.cookie or not cfg.bilibili.read.csrf):
|
||
raise ConfigError(
|
||
"config.json 中的 bilibili.read.cookie / csrf 不能为空（可先运行 python app.py auth）"
|
||
)
|
||
return cfg
|
||
|
||
|
||
class ConfigStore:
|
||
"""按文件 mtime 热加载配置；坏配置不会覆盖上一份可用配置。"""
|
||
|
||
def __init__(
|
||
self,
|
||
path: str | os.PathLike[str],
|
||
*,
|
||
env: Mapping[str, str] | None = None,
|
||
require_auth: bool = False,
|
||
) -> None:
|
||
self.path = Path(path)
|
||
self._env = env
|
||
self._require_auth = require_auth
|
||
self._lock = threading.RLock()
|
||
self._config = load_config(path, env=env, require_auth=require_auth)
|
||
self._mtime_ns = self._read_mtime_ns()
|
||
self.last_error: str | None = None
|
||
self.revision = 0
|
||
|
||
def _read_mtime_ns(self) -> int | None:
|
||
try:
|
||
return self.path.stat().st_mtime_ns
|
||
except OSError:
|
||
return None
|
||
|
||
def get(self) -> AppConfig:
|
||
with self._lock:
|
||
mtime_ns = self._read_mtime_ns()
|
||
if mtime_ns is None or mtime_ns == self._mtime_ns:
|
||
return self._config
|
||
self._mtime_ns = mtime_ns
|
||
try:
|
||
self._config = load_config(
|
||
self.path, env=self._env, require_auth=self._require_auth
|
||
)
|
||
except ConfigError as exc:
|
||
self.last_error = str(exc)
|
||
return self._config
|
||
self.last_error = None
|
||
self.revision += 1
|
||
return self._config
|