feat: switch config to JSONC + split secrets.json
Load business config as JSONC (comments + trailing commas) via an in-repo stripper then stdlib json — no new deps. Default path prefers config.jsonc, falls back to legacy config.json. Auth write-back targets only sibling secrets.json so JSONC comments are never wiped; secrets overlay merges on load. Examples, compose, Docker, docs, favsort and tools all share the same loader. Phase A only — no DB settings migration.
This commit was merged in pull request #4.
This commit is contained in:
1 parent
b4f0f45c4e
commit
8e7ead1669
21 files changed
+652
-182
No files matched your search
@@ -1,8 +1,9 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""扫码授权流程;仅由 ``python app.py auth`` 调用。
|
||||
|
||||
认证结果只写入指定的 ``bilibili.read`` 或 ``bilibili.write`` 段,两个账号的
|
||||
身份及设备指纹 Cookie 绝不互相覆盖。
|
||||
认证结果只写入凭据文件(默认与主配置同目录的 ``secrets.json``)里指定的
|
||||
``bilibili.read`` 或 ``bilibili.write`` 段,两个账号的身份及设备指纹 Cookie
|
||||
绝不互相覆盖。主配置(可为 JSONC)不会被回写,注释得以保留。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,6 +22,8 @@ import zlib
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
from ..config import resolve_secrets_path
|
||||
from .. import jsonc
|
||||
from .auth import (
|
||||
IDENTITY_KEYS,
|
||||
cookie_value,
|
||||
@@ -154,22 +157,71 @@ class AuthService:
|
||||
def __init__(self, config_path: str | Path, *, account: str, logger=None) -> None:
|
||||
if account not in {"read", "write"}:
|
||||
raise ValueError(f"未知账号角色: {account}")
|
||||
self.path = Path(config_path)
|
||||
self.config_path = Path(config_path)
|
||||
# 兼容旧属性名:历史代码/测试可能读 path;现指向凭据文件
|
||||
self.account = account
|
||||
self.logger = logger
|
||||
self.secrets_path = self._resolve_secrets_path()
|
||||
self.path = self.secrets_path
|
||||
|
||||
def _resolve_secrets_path(self) -> Path:
|
||||
raw: dict[str, Any] | None = None
|
||||
if self.config_path.exists():
|
||||
try:
|
||||
loaded = jsonc.load(self.config_path)
|
||||
except (OSError, ValueError):
|
||||
loaded = None
|
||||
if isinstance(loaded, dict):
|
||||
raw = loaded
|
||||
return resolve_secrets_path(self.config_path, raw)
|
||||
|
||||
@property
|
||||
def backup_path(self) -> Path:
|
||||
return self.path.with_name(self.path.name + ".bak")
|
||||
return self.secrets_path.with_name(self.secrets_path.name + ".bak")
|
||||
|
||||
def _empty_account(self) -> dict[str, Any]:
|
||||
return {"mid": 0, "csrf": "", "cookie": ""}
|
||||
|
||||
def _load_raw(self) -> dict[str, Any]:
|
||||
with self.path.open(encoding="utf-8") as fh:
|
||||
raw = json.load(fh)
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError("配置文件顶层必须是对象")
|
||||
"""加载凭据文件;若尚不存在则从主配置 bilibili 段迁移种子。"""
|
||||
if self.secrets_path.exists():
|
||||
raw = jsonc.load(self.secrets_path)
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError("凭据文件顶层必须是对象")
|
||||
else:
|
||||
raw = {}
|
||||
if self.config_path.exists():
|
||||
try:
|
||||
main = jsonc.load(self.config_path)
|
||||
except (OSError, ValueError):
|
||||
main = None
|
||||
if isinstance(main, dict) and isinstance(main.get("bilibili"), dict):
|
||||
# 仅迁移凭据段,不把整份主配置拷进 secrets
|
||||
raw["bilibili"] = main["bilibili"]
|
||||
|
||||
section = raw.get("bilibili")
|
||||
if not isinstance(section, dict) or not isinstance(section.get(self.account), dict):
|
||||
raise ValueError(f"缺少 bilibili.{self.account} 账号配置")
|
||||
if not isinstance(section, dict):
|
||||
section = {}
|
||||
raw["bilibili"] = section
|
||||
if not isinstance(section.get(self.account), dict):
|
||||
section[self.account] = self._empty_account()
|
||||
# 保证另一账号键存在,避免后续手工编辑困惑
|
||||
other = "write" if self.account == "read" else "read"
|
||||
if not isinstance(section.get(other), dict):
|
||||
# 尝试从主配置补齐;否则空壳
|
||||
filled = False
|
||||
if self.config_path.exists():
|
||||
try:
|
||||
main = jsonc.load(self.config_path)
|
||||
except (OSError, ValueError):
|
||||
main = None
|
||||
if isinstance(main, dict) and isinstance(main.get("bilibili"), dict):
|
||||
other_cred = main["bilibili"].get(other)
|
||||
if isinstance(other_cred, dict):
|
||||
section[other] = dict(other_cred)
|
||||
filled = True
|
||||
if not filled:
|
||||
section[other] = self._empty_account()
|
||||
return raw
|
||||
|
||||
@staticmethod
|
||||
@@ -209,7 +261,9 @@ class AuthService:
|
||||
if new_mid.isdigit():
|
||||
credentials["mid"] = int(new_mid)
|
||||
|
||||
shutil.copy2(self.path, self.backup_path)
|
||||
had_secrets = self.secrets_path.exists()
|
||||
if had_secrets:
|
||||
shutil.copy2(self.secrets_path, self.backup_path)
|
||||
self._write_raw(raw)
|
||||
try:
|
||||
nav = self._get_json(credentials, f"{BASE}/x/web-interface/nav", True)
|
||||
@@ -218,8 +272,11 @@ class AuthService:
|
||||
except Exception as exc: # noqa: BLE001
|
||||
data, ok = {"uname": str(exc)}, False
|
||||
if not ok:
|
||||
shutil.copy2(self.backup_path, self.path)
|
||||
print(f"[auth] {self.account} 登录校验失败,已回滚 config.json")
|
||||
if had_secrets and self.backup_path.exists():
|
||||
shutil.copy2(self.backup_path, self.secrets_path)
|
||||
elif self.secrets_path.exists():
|
||||
self.secrets_path.unlink()
|
||||
print(f"[auth] {self.account} 登录校验失败,已回滚 {self.secrets_path.name}")
|
||||
return 4
|
||||
|
||||
if isinstance(data.get("mid"), int):
|
||||
|
||||
@@ -28,7 +28,7 @@ from .bilibili.auth_flow import AuthService
|
||||
from .bilibili.client import BilibiliClient, Pacer
|
||||
from .bilibili.provider import ClientProvider
|
||||
from .bilibili.search import SearchClient
|
||||
from .config import ConfigError, ConfigStore
|
||||
from .config import ConfigError, ConfigStore, default_config_path
|
||||
from .database import Database
|
||||
from .logging_json import JsonLogger
|
||||
from .notify.telegram import TelegramClient
|
||||
@@ -42,7 +42,7 @@ from .services.up_profile import UpProfileService
|
||||
from .timeutil import next_daily_run, parse_hhmm, shanghai_date
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_CONFIG = ROOT / "config.json"
|
||||
DEFAULT_CONFIG = default_config_path(ROOT)
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
@@ -50,7 +50,7 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
common.add_argument(
|
||||
"--config",
|
||||
default=argparse.SUPPRESS,
|
||||
help="配置文件路径(默认项目根目录 config.json)",
|
||||
help="配置文件路径(默认优先 config.jsonc,否则 config.json)",
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -60,7 +60,7 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
default=str(DEFAULT_CONFIG),
|
||||
help="配置文件路径(默认项目根目录 config.json)",
|
||||
help="配置文件路径(默认优先 config.jsonc,否则 config.json)",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", required=False)
|
||||
|
||||
|
||||
+95
-22
@@ -8,7 +8,6 @@ B 站凭据必须采用 ``bilibili.read`` / ``bilibili.write`` 双账号格式
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
@@ -16,12 +15,15 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from . import jsonc
|
||||
from .client_ip import (
|
||||
DEFAULT_CLIENT_IP_HEADERS,
|
||||
DEFAULT_TRUSTED_PROXIES,
|
||||
parse_networks,
|
||||
)
|
||||
|
||||
DEFAULT_SECRETS_NAME = "secrets.json"
|
||||
|
||||
DEFAULT_CLASSIFY: dict[str, Any] = {
|
||||
"require_title": "咻咻满",
|
||||
"self_mid": 37754047,
|
||||
@@ -213,6 +215,7 @@ class AnalyticsConfig:
|
||||
@dataclass(frozen=True)
|
||||
class AppConfig:
|
||||
path: Path
|
||||
secrets_path: Path
|
||||
bilibili: BilibiliConfig
|
||||
telegram: TelegramConfig
|
||||
web: WebConfig
|
||||
@@ -352,7 +355,7 @@ def _validate_folders(value: Any) -> tuple[FolderConfig, ...]:
|
||||
return tuple(folders)
|
||||
|
||||
|
||||
def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
def _build_config(data: Mapping[str, Any], path: Path, *, secrets_path: Path) -> AppConfig:
|
||||
b = data["bilibili"]
|
||||
if not isinstance(b, Mapping):
|
||||
raise ConfigError("bilibili 必须是对象")
|
||||
@@ -593,6 +596,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
|
||||
return AppConfig(
|
||||
path=path,
|
||||
secrets_path=secrets_path,
|
||||
bilibili=bilibili,
|
||||
telegram=telegram,
|
||||
web=web,
|
||||
@@ -610,33 +614,87 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
)
|
||||
|
||||
|
||||
def resolve_secrets_path(
|
||||
config_path: Path,
|
||||
raw: Mapping[str, Any] | None = None,
|
||||
) -> Path:
|
||||
"""解析凭据文件路径。
|
||||
|
||||
优先读配置里的 ``secrets_path``(相对配置文件目录或绝对路径);
|
||||
否则默认同目录 ``secrets.json``。
|
||||
"""
|
||||
base = Path(config_path).resolve().parent
|
||||
if raw and "secrets_path" in raw and raw["secrets_path"] not in (None, ""):
|
||||
value = raw["secrets_path"]
|
||||
if not isinstance(value, str):
|
||||
raise ConfigError("secrets_path 必须是字符串")
|
||||
path = Path(value)
|
||||
return path if path.is_absolute() else (base / path)
|
||||
return base / DEFAULT_SECRETS_NAME
|
||||
|
||||
|
||||
def default_config_path(root: str | os.PathLike[str] | None = None) -> Path:
|
||||
"""默认配置:同目录下优先 ``config.jsonc``,否则 ``config.json``。
|
||||
|
||||
两者都不存在时返回 ``config.jsonc``(作为建议创建路径)。
|
||||
"""
|
||||
base = Path(root) if root is not None else Path(__file__).resolve().parent.parent
|
||||
jsonc_path = base / "config.jsonc"
|
||||
json_path = base / "config.json"
|
||||
if jsonc_path.exists():
|
||||
return jsonc_path
|
||||
if json_path.exists():
|
||||
return json_path
|
||||
return jsonc_path
|
||||
|
||||
|
||||
def _read_mapping_file(path: Path, *, kind: str) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
raise ConfigError(f"缺少{kind}: {path}")
|
||||
try:
|
||||
raw = jsonc.load(path)
|
||||
except ValueError as exc:
|
||||
raise ConfigError(f"{kind}不是合法 JSON/JSONC: {path}: {exc}") from exc
|
||||
except OSError as exc:
|
||||
raise ConfigError(f"无法读取{kind}: {path}: {exc}") from exc
|
||||
if not isinstance(raw, Mapping):
|
||||
raise ConfigError(f"{kind}顶层必须是 JSON 对象: {path}")
|
||||
return dict(raw)
|
||||
|
||||
|
||||
def load_config(
|
||||
path: str | os.PathLike[str],
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
require_auth: bool = False,
|
||||
) -> AppConfig:
|
||||
"""读取并校验配置。``env`` 默认为 ``os.environ``。"""
|
||||
"""读取并校验配置。``env`` 默认为 ``os.environ``。
|
||||
|
||||
主文件支持 ``.jsonc`` / ``.json``(注释与尾逗号)。敏感凭据优先从同目录
|
||||
``secrets.json``(或 ``secrets_path`` 指定文件)合并;``auth`` 只回写该凭据
|
||||
文件,避免冲掉 JSONC 注释。旧版把 ``bilibili`` 写在主配置里的用法仍可用。
|
||||
"""
|
||||
config_path = Path(path)
|
||||
if not config_path.exists():
|
||||
raise ConfigError(f"缺少配置文件: {config_path}")
|
||||
try:
|
||||
with open(config_path, encoding="utf-8") as fh:
|
||||
raw = json.load(fh)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ConfigError(f"配置文件不是合法 JSON: {config_path}: {exc}") from exc
|
||||
except OSError as exc:
|
||||
raise ConfigError(f"无法读取配置文件: {config_path}: {exc}") from exc
|
||||
if not isinstance(raw, Mapping):
|
||||
raise ConfigError("配置文件顶层必须是 JSON 对象")
|
||||
raw = _read_mapping_file(config_path, kind="配置文件")
|
||||
secrets_path = resolve_secrets_path(config_path, raw)
|
||||
if secrets_path.exists():
|
||||
secrets_raw = _read_mapping_file(secrets_path, kind="凭据文件")
|
||||
# 凭据覆盖主配置中的同名段(深度合并)
|
||||
raw = _deep_merge(raw, secrets_raw)
|
||||
|
||||
raw_bilibili = raw.get("bilibili")
|
||||
if not isinstance(raw_bilibili, Mapping):
|
||||
raise ConfigError("缺少 bilibili.read / bilibili.write 双账号配置")
|
||||
raise ConfigError(
|
||||
"缺少 bilibili.read / bilibili.write 双账号配置"
|
||||
f"(可写在主配置或 {secrets_path.name})"
|
||||
)
|
||||
for account in ("read", "write"):
|
||||
if account not in raw_bilibili:
|
||||
raise ConfigError(f"缺少 bilibili.{account} 账号配置")
|
||||
|
||||
data = _deep_merge(DEFAULT_CONFIG, raw)
|
||||
# secrets_path 不是业务字段
|
||||
data.pop("secrets_path", None)
|
||||
|
||||
env_map = os.environ if env is None else env
|
||||
for env_name, (target, caster) in ENV_OVERRIDES.items():
|
||||
@@ -650,16 +708,17 @@ def load_config(
|
||||
raise ConfigError(f"环境变量 {env_name} 必须是整数") from exc
|
||||
_set_path(data, target, value)
|
||||
|
||||
cfg = _build_config(data, config_path)
|
||||
cfg = _build_config(data, config_path, secrets_path=secrets_path)
|
||||
if require_auth and (not cfg.bilibili.read.cookie or not cfg.bilibili.read.csrf):
|
||||
raise ConfigError(
|
||||
"config.json 中的 bilibili.read.cookie / csrf 不能为空(可先运行 python app.py auth)"
|
||||
"bilibili.read.cookie / csrf 不能为空"
|
||||
f"(写入 {secrets_path.name},或运行 python app.py auth)"
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
class ConfigStore:
|
||||
"""按文件 mtime 热加载配置;坏配置不会覆盖上一份可用配置。"""
|
||||
"""按主配置与凭据文件 mtime 热加载;坏配置不会覆盖上一份可用配置。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -677,11 +736,25 @@ class ConfigStore:
|
||||
self.last_error: str | None = None
|
||||
self.revision = 0
|
||||
|
||||
def _watched_paths(self) -> tuple[Path, ...]:
|
||||
paths = [self.path]
|
||||
secrets = None
|
||||
if getattr(self, "_config", None) is not None:
|
||||
secrets = self._config.secrets_path
|
||||
if secrets is None:
|
||||
secrets = resolve_secrets_path(self.path)
|
||||
if secrets not in paths:
|
||||
paths.append(secrets)
|
||||
return tuple(paths)
|
||||
|
||||
def _read_mtime_ns(self) -> int | None:
|
||||
try:
|
||||
return self.path.stat().st_mtime_ns
|
||||
except OSError:
|
||||
return None
|
||||
times: list[int] = []
|
||||
for candidate in self._watched_paths():
|
||||
try:
|
||||
times.append(candidate.stat().st_mtime_ns)
|
||||
except OSError:
|
||||
continue
|
||||
return max(times) if times else None
|
||||
|
||||
def get(self) -> AppConfig:
|
||||
with self._lock:
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""JSONC(JSON with Comments)加载。
|
||||
|
||||
选择**自研轻量剥离**而非引入第三方依赖:本仓库运行依赖目前只有 ``httpx``,
|
||||
配置加载属于启动/热更新热路径,JSONC 只需注释与尾逗号。实现按字符扫描,
|
||||
在字符串字面量内保留 ``//``、``/*``、引号转义,避免误伤。
|
||||
|
||||
支持:
|
||||
- ``//`` 行注释、``/* ... */`` 块注释
|
||||
- 对象/数组中的尾逗号
|
||||
- UTF-8 BOM
|
||||
不支持 JSON5 扩展(无引号键、单引号、十六进制、``Infinity`` 等)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
def strip_jsonc(text: str) -> str:
|
||||
"""去掉 JSONC 注释与尾逗号,返回可交给 ``json.loads`` 的文本。"""
|
||||
if text.startswith("\ufeff"):
|
||||
text = text[1:]
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
n = len(text)
|
||||
in_string = False
|
||||
escape = False
|
||||
while i < n:
|
||||
ch = text[i]
|
||||
if in_string:
|
||||
out.append(ch)
|
||||
if escape:
|
||||
escape = False
|
||||
elif ch == "\\":
|
||||
escape = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
out.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == "/" and i + 1 < n:
|
||||
nxt = text[i + 1]
|
||||
if nxt == "/":
|
||||
i += 2
|
||||
while i < n and text[i] not in "\r\n":
|
||||
i += 1
|
||||
continue
|
||||
if nxt == "*":
|
||||
i += 2
|
||||
while i + 1 < n and not (text[i] == "*" and text[i + 1] == "/"):
|
||||
i += 1
|
||||
i = min(i + 2, n)
|
||||
continue
|
||||
out.append(ch)
|
||||
i += 1
|
||||
|
||||
stripped = "".join(out)
|
||||
# 去掉尾逗号:`,}` / `,]`(可夹空白)
|
||||
cleaned: list[str] = []
|
||||
i = 0
|
||||
m = len(stripped)
|
||||
in_string = False
|
||||
escape = False
|
||||
while i < m:
|
||||
ch = stripped[i]
|
||||
if in_string:
|
||||
cleaned.append(ch)
|
||||
if escape:
|
||||
escape = False
|
||||
elif ch == "\\":
|
||||
escape = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
cleaned.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == ",":
|
||||
j = i + 1
|
||||
while j < m and stripped[j] in " \t\r\n":
|
||||
j += 1
|
||||
if j < m and stripped[j] in "}]":
|
||||
i += 1
|
||||
continue
|
||||
cleaned.append(ch)
|
||||
i += 1
|
||||
return "".join(cleaned)
|
||||
|
||||
|
||||
def loads(text: str) -> Any:
|
||||
"""解析 JSONC 文本。"""
|
||||
return json.loads(strip_jsonc(text))
|
||||
|
||||
|
||||
def load(path: str | bytes, *, encoding: str = "utf-8") -> Any:
|
||||
"""从路径读取并解析 JSONC / JSON。"""
|
||||
with open(path, encoding=encoding) as fh:
|
||||
return loads(fh.read())
|
||||
Reference in new issue
Block a user