diff --git a/.dockerignore b/.dockerignore index 6181cd8..08ae3cc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,13 @@ # 本地凭据与运行期数据 config.json +config.jsonc config.json.bak +config.jsonc.bak config.json.tmp +config.jsonc.tmp +secrets.json +secrets.json.bak +secrets.json.tmp data/ # 开发与测试 diff --git a/.gitignore b/.gitignore index fcc0392..b746784 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ -# 本地凭据,绝对不要提交 +# 本地凭据与业务配置,绝对不要提交 config.json +config.jsonc config.json.bak +config.jsonc.bak config.json.tmp +config.jsonc.tmp +secrets.json +secrets.json.bak +secrets.json.tmp # 计划 / 日志 / 快照 data/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3115f73..e2d4d35 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -113,14 +113,14 @@ uv run python app.py status # P2 起启动本地看板服务 uv run python app.py serve -uv run python app.py serve --config .\config.json +uv run python app.py serve --config .\config.jsonc # 无子命令启动完整服务:只读 Web + 调度 + Telegram uv run python app.py # 生成 7 天预览库并启动预览 uv run python tools/seed_preview_db.py --days 7 -uv run python app.py serve --preview --config data/preview/config.json +uv run python app.py serve --preview --config data/preview/config.jsonc # 从 B 站粉丝历史 CSV 回填(先 dry-run;首次完整导入可用 --replace) uv run python tools/import_fans_csv.py --csv E:\Downloads\history.csv --dry-run @@ -152,8 +152,8 @@ uv sync --all-groups 不得提交: - `.venv/` -- `config.json` -- `config.json.bak` +- `config.jsonc` / `config.json` +- `secrets.json` / `secrets.json.bak` - `data/` 中的运行期数据库、日志、计划和备份。 --- @@ -205,8 +205,11 @@ PRAGMA synchronous = NORMAL; ├── pyproject.toml # uv / Python 项目元数据 ├── uv.lock # uv 生成并提交的锁文件 ├── .python-version # 固定 Python 3.14 -├── config.json # 运行期私密配置,不提交 -├── config.example.json # 脱敏配置样例 +├── config.jsonc # 运行期业务配置(JSONC,不提交) +├── secrets.json # 运行期凭据(auth 回写目标,不提交) +├── config.example.jsonc # 脱敏业务配置样例(含注释) +├── secrets.example.json # 脱敏凭据样例 +├── config.docker.example.jsonc # 服务器业务配置样例 ├── DEVELOPMENT.md │ ├── reaction_board/ @@ -261,24 +264,22 @@ PRAGMA synchronous = NORMAL; ### 5.1 运行期配置 -所有凭据和可变业务参数放在 `config.json` 中。文件必须被 Git 忽略,日志中不得输出其敏感值。 +业务参数放在 `config.jsonc`(JSONC:`//` / `/* */` 注释与尾逗号;实现为自研轻量剥离后再 `json.loads`,无额外依赖)。 +敏感凭据放同目录 `secrets.json`(纯 JSON,供 `auth` 原子回写)。二者都必须被 Git 忽略,日志中不得输出敏感值。 +默认路径优先 `config.jsonc`,否则回落 `config.json`;旧单体 `config.json`(凭据内嵌)仍可加载,若存在 `secrets.json` 则凭据以它为准。 + +`config.jsonc` 示例: ```jsonc { - "bilibili": { - "mid": 28802351, - "csrf": "", - "cookie": "SESSDATA=...; bili_jct=...; buvid3=..." - }, + // bilibili / bot_token / auth_token → secrets.json "telegram": { "enabled": true, - "bot_token": "123456:ABC", - "chat_id": 123456789 }, "web": { "host": "127.0.0.1", "port": 8080, - "auth_token": "replace-with-a-long-random-token", + // auth_token → secrets.json "public_base_url": "" }, "storage": { @@ -303,11 +304,14 @@ PRAGMA synchronous = NORMAL; ``` - `folders` 仅作为历史来源与 `favsort.py` 重排序目标,系统不再自动同步或建夹; -- 配置必须为 `bilibili.read` / `bilibili.write` 双账号结构;不兼容顶层扁平 `mid` / `csrf` / `cookie`; +- `bilibili.read` / `bilibili.write` 写在 `secrets.json`(或旧单体主配置内);不兼容顶层扁平 `mid` / `csrf` / `cookie`; +- `python app.py auth` 只原子回写 `secrets.json`(并保留 `.bak`),不改主 JSONC; - P0/P3 已实现的命令行:`python app.py init`、`python app.py ingest`、`python app.py status`、`python app.py run`。 -### 5.2 安全规则- Windows 本地开发时应限制 `config.json` 的文件访问权限;部署到 Linux 时权限必须为 `600`; -- 配置文件每 60 秒检查一次 mtime,凭据、分类规则和权重变更后无需重启; +### 5.2 安全规则 + +- Windows 本地开发时应限制 `config.jsonc` / `secrets.json` 的文件访问权限;部署到 Linux 时权限必须为 `600`; +- 主配置与凭据文件每 60 秒检查一次 mtime,凭据、分类规则和权重变更后无需重启; - Cookie / CSRF 变化时由 `ClientProvider` 重建 B 站请求客户端,并保留该角色的限速与重试策略; - `schedule.discover_minutes`、`schedule.snapshot_at` 与 `schedule.seed_at` 在启动时固化,修改后需重启 `run`; - Telegram 操作必须同时验证 `from.id` 和 `chat.id` 是否为配置的私聊用户; @@ -788,7 +792,7 @@ uv run python tests/test_moves.py - `reaction_board/client_ip.py`:来源 IP 提取、路由归一化与机器人判定纯函数(三种来源 + 伪造防护); - `reaction_board/web/visits.py`:访问上报先落内存缓冲,后台每分钟批量 upsert,支持单键上限与保留期清理,关闭时最后一次 flush; - 前端 `navigator.sendBeacon("/api/visit")` 埋点;Web 只读边界放宽为显式 POST 白名单(`/api/visit` + `/login`); -- `config.json` 新增 `web.track`(`enabled`/`header_order`/`trusted_proxies`/`count_bots`/`flush_seconds`/`keep_days`/`max_hits_per_ip_path_per_day`); +- `config` 新增 `web.track`(`enabled`/`header_order`/`trusted_proxies`/`count_bots`/`flush_seconds`/`keep_days`/`max_hits_per_ip_path_per_day`); - 系统状态页展示近 30 天 PV/UV 与 Top 页面(`GET /api/stats`,需登录,合并未落库缓冲); - `deploy/nginx/` 提供直连 EIP / Cloudflare / EdgeOne 的 realip 与站点配置、网段刷新脚本和说明; - 自动化测试新增 `tests/test_client_ip.py`、`tests/test_visits.py`,并扩展 `test_config.py`、`test_web_queries.py`、`test_web_server.py`、`test_migrations.py` 等。 @@ -796,7 +800,7 @@ uv run python tests/test_moves.py ### P4:可选部署 - 反向代理、TLS、远程访问鉴权; -- 仓库已提供 `Dockerfile` / `compose.example.yml` / `config.docker.example.json`;容器内执行 `python app.py --config /config/config.json`,挂载 `/config` 与 `/data`; +- 仓库已提供 `Dockerfile` / `compose.example.yml` / `config.docker.example.jsonc` / `secrets.example.json`;容器内执行 `python app.py --config /config/config.jsonc`,挂载 `/config` 与 `/data`; - Docker 不作为本机开发、联调或日常运行的前置条件。 --- diff --git a/Dockerfile b/Dockerfile index 3f1f2e1..2145a29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,4 +30,4 @@ EXPOSE 8080 HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/api/health', timeout=3)" -CMD ["python", "app.py", "--config", "/config/config.json"] \ No newline at end of file +CMD ["python", "app.py", "--config", "/config/config.jsonc"] \ No newline at end of file diff --git a/README.md b/README.md index 83a58ba..af26790 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ uv run python app.py # 生成 7 天预览库并启动预览 uv run python tools/seed_preview_db.py --days 7 -uv run python app.py serve --preview --config data/preview/config.json +uv run python app.py serve --preview --config data/preview/config.jsonc ``` - 首页默认“最新动态”:标题下显示「收录视频 N」,顶部四个数据卡片为「今日入库 / 播放增量 / 7 日粉丝变化 / 数据日期」(前三项可点击跳转);下方为最近发布、增长动态、热度速览三个区块,统一使用竖排卡片(封面 + 两行截断标题 + 作者/数据行 + 高亮数据行),并统一为响应式网格(移动端每行 2 个、PC 每行 3 个),每处各取前 6 条;采集状态等运行期信息只在登录后的「系统」页可见;有候选且已登录时,首页显示「N 条候选待确认」提示条(访客不显示); @@ -231,7 +231,7 @@ uv run python tools/import_fans_csv.py --csv E:\Downloads\history.csv --replace - 如果数据库里已有晚于 CSV 末尾的记录,`--replace` 会拒绝执行;确认要删除时才加 `--allow-lossy-replace`;只想补齐 CSV 日期并保留后续采集时用 `--force`; - 页面的“日增”只在存在前一天数据时计算,跨多天不会标成日增; - 写入前建议先备份 `data/app.db`,例如 `data/backup/app-pre-fans-YYYYMMDD.db`; -- 容器内运行时用 `--db /data/app.db` 或 `--config /config/config.json`,避免默认去找 `/app/config.json`。 +- 容器内运行时用 `--db /data/app.db` 或 `--config /config/config.jsonc`,避免默认去找 `/app/config.jsonc`。 ## 访问统计与真实来源 IP @@ -251,17 +251,19 @@ uv run python tools/import_fans_csv.py --csv E:\Downloads\history.csv --replace - 三种来源(直连 EIP / Cloudflare / EdgeOne)的完整 nginx 配置、网段刷新脚本与 配置项说明见 [`deploy/nginx/README.md`](deploy/nginx/README.md)。 -`config.json` 中的 `web.track` 控制统计行为:`enabled` / `header_order` / +`config.jsonc` 中的 `web.track` 控制统计行为:`enabled` / `header_order` / `trusted_proxies` / `count_bots` / `flush_seconds` / `keep_days` / `max_hits_per_ip_path_per_day`;默认过滤常见爬虫 UA、每分钟落库、保留 180 天。 ## 快速部署(Docker) -仓库根目录已包含 `Dockerfile`、`compose.example.yml`、`.dockerignore` 和 `config.docker.example.json`。Compose 文件是示例,需要显式指定: +仓库根目录已包含 `Dockerfile`、`compose.example.yml`、`.dockerignore`、`config.docker.example.jsonc` 与 `secrets.example.json`。Compose 文件是示例,需要显式指定: ```powershell -# 1. 准备服务器配置;至少填写 B 站 Cookie / Telegram,并保留 /data 绝对路径 -Copy-Item config.docker.example.json config.json +# 1. 准备服务器配置:业务 JSONC + 无注释凭据文件;保留 /data 绝对路径 +Copy-Item config.docker.example.jsonc config.jsonc +Copy-Item secrets.example.json secrets.json +# 编辑 secrets.json 填入 B 站 Cookie / Telegram / web.auth_token # 2. 构建并后台启动 docker compose -f compose.example.yml up -d --build @@ -278,11 +280,11 @@ Copy-Item compose.example.yml compose.yml docker compose up -d --build ``` -- 容器执行 `python app.py --config /config/config.json`,同时运行只读 Web、调度和 Telegram; -- `config.json` 挂载到 `/config/config.json`,`data/` 挂载到 `/data/`; -- 服务器配置需使用 `web.host = 0.0.0.0` 并设置 `web.auth_token`,否则服务拒绝启动; +- 容器执行 `python app.py --config /config/config.jsonc`,同时运行只读 Web、调度和 Telegram; +- `config.jsonc` 与 `secrets.json` 挂到 `/config/`,`data/` 挂到 `/data/`; +- 服务器配置需使用 `web.host = 0.0.0.0`,并在 `secrets.json` 设置 `web.auth_token`,否则服务拒绝启动; - `storage.database_path` 使用 `/data/app.db`,`storage.backup_dir` 使用 `/data/backup`; -- 首次部署可先在本地执行 `uv run python app.py auth --account read` 扫码,再把生成好的 `config.json` 放到服务器; +- 首次部署可先在本地执行 `uv run python app.py auth --account read` 扫码(只回写 `secrets.json`),再把 `config.jsonc` + `secrets.json` 放到服务器; - 也可在容器内扫码:`docker compose -f compose.example.yml run --rm board python app.py auth --account read --no-browser`,终端会打印 ASCII 二维码;PNG 仍写在临时文件路径,可用 `docker cp` 取出; - 镜像包含 `tools/`,可直接在容器内执行 `python tools/import_fans_csv.py ...` 回填粉丝历史; - 数据卷包含数据库、备份与预览库,建议定期备份 `/data`。 @@ -293,20 +295,26 @@ docker compose up -d --build docker build -t reaction-board . docker run -d --name reaction-board --restart unless-stopped ` -p 8080:8080 ` - -v ${PWD}/config.json:/config/config.json ` + -v ${PWD}/config.jsonc:/config/config.jsonc ` + -v ${PWD}/secrets.json:/config/secrets.json ` -v ${PWD}/data:/data ` reaction-board ``` ## 配置 -复制样例配置,并填写 B 站账号信息: +复制样例:业务配置用 JSONC(支持 `//` / `/* */` 注释与尾逗号),凭据单独放无注释的 `secrets.json`: ```powershell -Copy-Item config.example.json config.json +Copy-Item config.example.jsonc config.jsonc +Copy-Item secrets.example.json secrets.json ``` -`config.json` 已被 `.gitignore` 忽略,**不得提交**。配置必须使用双账号格式: +`config.jsonc` / `secrets.json`(以及旧名 `config.json`)已被 `.gitignore` 忽略,**不得提交**。 +默认启动优先读 `config.jsonc`,若不存在则回落 `config.json`(旧单体文件仍可用)。 +`python app.py auth` **只回写** `secrets.json`,不会冲掉 JSONC 注释。 + +凭据文件双账号格式: ```json { @@ -338,7 +346,7 @@ Copy-Item config.example.json config.json 所有命令建议通过 `uv run` 执行: ```powershell -# 查看全部命令;无需 config.json +# 查看全部命令;无需配置文件 uv run bili-fav help # 为写账号扫码登录与检查状态(认证入口统一在 app.py) @@ -386,7 +394,7 @@ uv run python app.py auth-check --account all - 更新 `SESSDATA`、`bili_jct`、`DedeUserID`、`sid` 等身份字段; - 保留 `buvid3`、`buvid4`、`_uuid`、`buvid_fp` 等设备指纹; - 将选中账号的 `csrf` 同步为新的 `bili_jct`,并从 `nav` 回写该账号实际 mid; -- 原子写入 `config.json`,并保留 `config.json.bak`; +- 原子写入 `secrets.json`,并保留 `secrets.json.bak`(主配置 JSONC 不被改写); - 通过 `x/web-interface/nav` 验证新登录态,失败时自动回滚;不会修改另一个账号。 常用选项: @@ -459,7 +467,7 @@ uv run python tests/test_cli.py # CLI 离线路径 ## 运行期文件 ```text -config.json.bak 最近一次 auth 更新前的配置备份 +secrets.json.bak 最近一次 auth 更新前的凭据备份 data/plan-.json 排序计划、顺序快照和元数据缓存 data/journal-.jsonl 排序执行日志 data/app.db SQLite 数据库(视频、快照、粉丝、黑名单、历史收藏夹) @@ -467,7 +475,7 @@ data/backup/app-YYYYMMDD.db 每日快照后的 SQLite 备份 data/backup/app-pre-v*.db 数据库迁移前的自动备份 data/backup/app-pre-fans-*.db 粉丝历史回填前的手动备份 data/preview/app.db 合成预览数据库(不提交) -data/preview/config.json 无凭据的预览配置(不提交) +data/preview/config.jsonc 无凭据的预览配置(不提交) ``` ## 文档索引 diff --git a/compose.example.yml b/compose.example.yml index c726672..a56300f 100644 --- a/compose.example.yml +++ b/compose.example.yml @@ -10,6 +10,7 @@ services: ports: - "8080:8080" volumes: - - ./config.json:/config/config.json + - ./config.jsonc:/config/config.jsonc + - ./secrets.json:/config/secrets.json - ./data:/data - command: ["python", "app.py", "--config", "/config/config.json"] \ No newline at end of file + command: ["python", "app.py", "--config", "/config/config.jsonc"] diff --git a/config.docker.example.json b/config.docker.example.jsonc similarity index 69% rename from config.docker.example.json rename to config.docker.example.jsonc index b0111a8..0af70f4 100644 --- a/config.docker.example.json +++ b/config.docker.example.jsonc @@ -1,25 +1,13 @@ +// Docker / 服务器业务配置。复制为 config.jsonc,与 secrets.json 一并挂到 /config。 +// 敏感项见 secrets.example.json → secrets.json;auth 只回写 secrets.json。 { - "bilibili": { - "read": { - "mid": 0, - "csrf": "从 read 账号 cookie 中的 bili_jct 取值", - "cookie": "只读账号的完整 Cookie(需含 SESSDATA、bili_jct、DedeUserID 等)" - }, - "write": { - "mid": 0, - "csrf": "从 write 账号 cookie 中的 bili_jct 取值", - "cookie": "写入账号的完整 Cookie(仅用于 favsort.py 收藏夹重排序)" - } - }, "telegram": { "enabled": false, - "bot_token": "replace-with-telegram-bot-token", - "chat_id": 0 }, + "web": { "host": "0.0.0.0", "port": 8080, - "auth_token": "replace-with-a-long-random-token", "public_base_url": "", "track": { "enabled": true, @@ -32,22 +20,24 @@ "192.168.0.0/16", "169.254.0.0/16", "fc00::/7", - "fe80::/10" + "fe80::/10", ], "count_bots": false, "flush_seconds": 60, "keep_days": 180, - "max_hits_per_ip_path_per_day": 0 - } + "max_hits_per_ip_path_per_day": 0, + }, }, + "storage": { "database_path": "/data/app.db", "backup_dir": "/data/backup", - "backup_keep_days": 14 + "backup_keep_days": 14, }, + "folders": [ { "year": 2025, "media_id": 4451053851, "title": "咻咻满-reaction-2025" }, - { "year": 2026, "media_id": 3700023351, "title": "咻咻满 reaction 合集" } + { "year": 2026, "media_id": 3700023351, "title": "咻咻满 reaction 合集" }, ], "track_fans": [37754047], "up_blacklist_seed": [ @@ -59,26 +49,30 @@ 145178393, 66389997, 293976264, - 3026109 + 3026109, ], + "schedule": { "discover_minutes": 60, "snapshot_at": "23:15", "seed_at": "00:00", "candidate_ttl_days": 7, "recent_snapshot_minutes": 60, - "recent_video_days": 2 + "recent_video_days": 2, }, + "discover": { "keywords": ["咻咻满"], "pages": 3, "window_days": 7, - "candidate_min_score": 4 + "candidate_min_score": 4, }, + "notify": { "min_score": 4, - "max_items_per_message": 10 + "max_items_per_message": 10, }, + "classify": { "require_title": "咻咻满", "self_mid": 37754047, @@ -91,11 +85,12 @@ "搬运", "完整版", "录播", - "录像" + "录像", ], "date_negative": true, - "duration_boost": { "gt_30min": 1, "gt_60min": 2 } + "duration_boost": { "gt_30min": 1, "gt_60min": 2 }, }, + "heat_weights": { "view": 0.3, "like": 0.2, @@ -103,14 +98,15 @@ "favorite": 0.15, "share": 0.1, "reply": 0.03, - "danmaku": 0.02 + "danmaku": 0.02, }, + "analytics": { "min_duration_seconds": 600, "maturity_days": 2, "min_view_for_ranking": 2000, "video_shrink_minutes": 60, "shrink_minutes": null, - "home_growth_days": "auto" - } -} \ No newline at end of file + "home_growth_days": "auto", + }, +} diff --git a/config.example.json b/config.example.jsonc similarity index 64% rename from config.example.json rename to config.example.jsonc index 47df51e..f7b06d7 100644 --- a/config.example.json +++ b/config.example.jsonc @@ -1,16 +1,13 @@ +// 业务配置(可含注释与尾逗号)。复制为 config.jsonc 后按需修改。 +// 敏感凭据请放同目录 secrets.json(见 secrets.example.json); +// `python app.py auth` 只回写 secrets.json,不会冲掉本文件注释。 +// +// 兼容:仍可读旧版单一 config.json(凭据可内嵌);若存在 secrets.json 则凭据以它为准。 { - "bilibili": { - "read": { - "mid": 0, - "csrf": "从 read 账号 cookie 中的 bili_jct 取值", - "cookie": "只读账号的完整 Cookie(需含 SESSDATA、bili_jct、DedeUserID 等)" - }, - "write": { - "mid": 0, - "csrf": "从 write 账号 cookie 中的 bili_jct 取值", - "cookie": "写入账号的完整 Cookie(仅用于 favsort.py 收藏夹重排序)" - } - }, + // 可选:自定义凭据路径(相对本文件目录或绝对路径)。默认 secrets.json + // "secrets_path": "secrets.json", + + // favsort 代发归属等(非密钥) "overrides": {}, "sleep": { "min": 2.5, @@ -19,17 +16,18 @@ "long_every_max": 40, "long_min": 30, "long_max": 90, - "retry_wait": 45 + "retry_wait": 45, }, + "telegram": { "enabled": false, - "bot_token": "replace-with-telegram-bot-token", - "chat_id": 0 + // bot_token / chat_id 建议写在 secrets.json }, + "web": { "host": "127.0.0.1", "port": 8080, - "auth_token": "replace-with-a-long-random-token", + // auth_token 建议写在 secrets.json "public_base_url": "", "track": { "enabled": true, @@ -42,22 +40,24 @@ "192.168.0.0/16", "169.254.0.0/16", "fc00::/7", - "fe80::/10" + "fe80::/10", ], "count_bots": false, "flush_seconds": 60, "keep_days": 180, - "max_hits_per_ip_path_per_day": 0 - } + "max_hits_per_ip_path_per_day": 0, + }, }, + "storage": { "database_path": "data/app.db", "backup_dir": "data/backup", - "backup_keep_days": 14 + "backup_keep_days": 14, }, + "folders": [ { "year": 2025, "media_id": 4451053851, "title": "咻咻满-reaction-2025" }, - { "year": 2026, "media_id": 3700023351, "title": "咻咻满 reaction 合集" } + { "year": 2026, "media_id": 3700023351, "title": "咻咻满 reaction 合集" }, ], "track_fans": [37754047], "up_blacklist_seed": [ @@ -69,26 +69,31 @@ 145178393, 66389997, 293976264, - 3026109 + 3026109, ], + + // 日更默认 23:15 落盘;午夜 seed_at 用昨收盘克隆当日壳(不推进 completed) "schedule": { "discover_minutes": 60, "snapshot_at": "23:15", "seed_at": "00:00", "candidate_ttl_days": 7, "recent_snapshot_minutes": 60, - "recent_video_days": 2 + "recent_video_days": 2, }, + "discover": { "keywords": ["咻咻满"], "pages": 3, "window_days": 7, - "candidate_min_score": 4 + "candidate_min_score": 4, }, + "notify": { "min_score": 4, - "max_items_per_message": 10 + "max_items_per_message": 10, }, + "classify": { "require_title": "咻咻满", "self_mid": 37754047, @@ -97,8 +102,9 @@ "weak_positive": ["双人", "路人", "第一次", "首次"], "negative": ["歌切", "切片", "搬运", "完整版", "录播", "录像"], "date_negative": true, - "duration_boost": { "gt_30min": 1, "gt_60min": 2 } + "duration_boost": { "gt_30min": 1, "gt_60min": 2 }, }, + "heat_weights": { "view": 0.3, "like": 0.2, @@ -106,14 +112,16 @@ "favorite": 0.15, "share": 0.1, "reply": 0.03, - "danmaku": 0.02 + "danmaku": 0.02, }, + "analytics": { "min_duration_seconds": 600, "maturity_days": 2, "min_view_for_ranking": 2000, "video_shrink_minutes": 60, "shrink_minutes": null, - "home_growth_days": "auto" - } + // "auto" = 7 日有数据用 7,否则 1 + "home_growth_days": "auto", + }, } diff --git a/favsort.py b/favsort.py index c7afb79..32216f5 100644 --- a/favsort.py +++ b/favsort.py @@ -6,7 +6,7 @@ bili-fav / favsort —— B 站收藏夹批量重排序工具 1) 同一个 upper 的视频必须连续(聚成一组),组内按发布时间倒序(并列按 avid 升序) 2) 组间按"该 upper 在本收藏夹中最早视频的发布时间"倒序(并列按该组最小 avid 升序) 3) 失效/异常资源置于末尾,保持原相对顺序 - 4) config.json 里的 overrides 可把指定视频强制归入另一个 upper(代发等场景) + 4) 配置里的 overrides 可把指定视频强制归入另一个 upper(代发等场景) 指令 python favsort.py help [子命令] @@ -36,11 +36,16 @@ sys.stdout.reconfigure(encoding="utf-8") BASE = "https://api.bilibili.com" ROOT = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join(ROOT, "data") -CONFIG_PATH = os.path.join(ROOT, "config.json") - from reaction_board.bilibili.auth import ( # noqa: E402 make_headers, ) +from reaction_board.config import ( # noqa: E402 + ConfigError, + default_config_path, + load_config as load_app_config, +) + +CONFIG_PATH = str(default_config_path(ROOT)) CHUNK = 20 # infos 接口单次 resources 数量(实测 20 稳定) API = { @@ -70,18 +75,19 @@ def load_config(require_auth=True): if not os.path.exists(CONFIG_PATH): raise SystemExit( f"缺少 {CONFIG_PATH}\n" - f"请复制 config.example.json 为 config.json 并填入 cookie / csrf / mid。" + f"请复制 config.example.jsonc 为 config.jsonc,并复制 secrets.example.json 为 secrets.json 后填入凭据\n" + f"(或运行 python app.py auth --account write)。" ) - with open(CONFIG_PATH, encoding="utf-8") as f: - raw = json.load(f) - bilibili = raw.get("bilibili") - if not isinstance(bilibili, dict) or not isinstance(bilibili.get("write"), dict): - raise SystemExit("config.json 必须配置 bilibili.write(扫码请运行 python app.py auth --account write)") - write = bilibili["write"] + try: + app_cfg = load_app_config(CONFIG_PATH, require_auth=False) + except ConfigError as exc: + raise SystemExit(str(exc)) from exc + write = app_cfg.bilibili.write + raw = app_cfg.raw cfg = { - "mid": write.get("mid", 0), - "csrf": write.get("csrf", ""), - "cookie": write.get("cookie", ""), + "mid": write.mid, + "csrf": write.csrf, + "cookie": write.cookie, "overrides": raw.get("overrides", {}), "sleep": raw.get("sleep", {}), } @@ -90,7 +96,10 @@ def load_config(require_auth=True): for k, v in DEFAULT_CONFIG["sleep"].items(): cfg["sleep"].setdefault(k, v) if require_auth and (not cfg["cookie"] or not cfg["csrf"]): - raise SystemExit("config.json 中的 bilibili.write.cookie / csrf 不能为空(可先运行 python app.py auth --account write)") + raise SystemExit( + "bilibili.write.cookie / csrf 不能为空" + f"(写入 {app_cfg.secrets_path.name},或运行 python app.py auth --account write)" + ) return cfg diff --git a/project.md b/project.md index c8aa7fb..4261107 100644 --- a/project.md +++ b/project.md @@ -85,11 +85,14 @@ Telegram 私聊发链接/BV 号 ───┘ - **日更对象** = 库内 `active` 视频; - **黑名单只挡"发现"**:已入库的黑名单 UP 视频一律当普通视频处理,不判断、不排除、也不反向更新黑名单。 -### 1.5 配置与凭据(统一放配置文件) +### 1.5 配置与凭据(JSONC + secrets) -所有可变项与敏感信息都在一个配置文件里(建议 `config.json`,`chmod 600`,`.gitignore` 排除): +业务项放 `config.jsonc`(可注释,`chmod 600`,`.gitignore` 排除);敏感凭据放同目录 `secrets.json`(纯 JSON,供 auth 回写)。 +默认优先读 `config.jsonc`,否则 `config.json`。旧单体文件仍可用;存在 `secrets.json` 时凭据以它为准。 -```jsonc +`secrets.json`: + +```json { "bilibili": { "read": { @@ -107,6 +110,15 @@ Telegram 私聊发链接/BV 号 ───┘ "bot_token": "123456:ABC...", "chat_id": "-1001234567890" }, + "web": { "auth_token": "..." } +} +``` + +`config.jsonc`(节选): + +```jsonc +{ + "telegram": { "enabled": true }, "folders": [ { "year": 2025, "media_id": 4451053851, "title": "咻咻满-reaction-2025" }, { "year": 2026, "media_id": 3700023351, "title": "咻咻满 reaction 合集" } @@ -139,8 +151,8 @@ Telegram 私聊发链接/BV 号 ───┘ - 支持热加载(每 60 秒检查 mtime),改收藏夹/黑名单不用重启;`schedule` 中的周期与每日时间需重启生效; - **黑名单本身放数据库**(`up_blacklist` 表),因为要从通知里动态拉黑;配置文件只做初始种子; - 日志里**永不打印 token / cookie**,启动时只打印"已配置/未配置"; -- 容器里把配置目录挂成**可写卷**(`-v ./config:/config`,权限 600),因为 `auth` 扫码续期后要回写 cookie; -- `python app.py auth --account read|write` 对指定账号扫码续期,只替换该账号身份字段、保留该账号设备指纹; +- 容器里把配置目录挂成**可写卷**(挂 `config.jsonc` + `secrets.json`,权限 600),因为 `auth` 扫码续期后要回写 `secrets.json`; +- `python app.py auth --account read|write` 对指定账号扫码续期,只写 `secrets.json` 中该账号身份字段、保留设备指纹,**不改**主 JSONC; - `read` 用于快照、搜索发现、入库与所有查询;`write` 仅用于 `favsort.py` 收藏夹重排序(`sort`);不支持顶层扁平凭据; - 可选的 `ENV` 覆盖(`TG_BOT_TOKEN` 等)留给 CI/临时调试,不作为主路径。 @@ -586,7 +598,7 @@ CREATE TABLE kv (k TEXT PRIMARY KEY, v TEXT); **开发阶段只做 Telegram**: - Bot 用 `getUpdates` long polling 收 `callback_query`,本机没有公网 IP 也能联调; -- `bot_token` / `chat_id` 从 `config.json` 读(见 §1.5),不写死在代码里; +- `bot_token` / `chat_id` 从配置/secrets 读(见 §1.5),不写死在代码里; - **按私聊设计**:`chat_id` 是用户 id(正数),Bot 只服务你一个人;回调 `callback_query` 直接用 `answerCallbackQuery` 回执; - 上线后服务有域名 + TLS,再补"签名链接"路径给企业微信/邮件/Webhook; - 两种路径共用同一个 `fav_action` 状态机,只是入口不同(`callback_data` vs `POST /a//confirm`)。 @@ -767,9 +779,10 @@ UP 内容效率 = EB 收缩后的 Σview / Σ内容小时 ```text Dockerfile 单容器镜像(只安装 httpx 0.28.1) -compose.example.yml 挂载 ./config.json → /config、./data → /data(复制为 compose.yml 后使用) -.dockerignore 排除凭据、数据、测试与本地环境 -config.docker.example.json 服务器配置样例(0.0.0.0 + /data 绝对路径) +compose.example.yml 挂载 config.jsonc + secrets.json → /config、./data → /data +.dockerignore 排除凭据、数据、测试与本地环境 +config.docker.example.jsonc 服务器业务配置样例(0.0.0.0 + /data 绝对路径) +secrets.example.json 凭据样例(auth 回写目标) ``` - 单容器跑「调度 + Web」,数据卷 `/data/app.db`; diff --git a/reaction_board/bilibili/auth_flow.py b/reaction_board/bilibili/auth_flow.py index 83f88ac..24004c7 100644 --- a/reaction_board/bilibili/auth_flow.py +++ b/reaction_board/bilibili/auth_flow.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- """扫码授权流程;仅由 ``python app.py auth`` 调用。 -认证结果只写入指定的 ``bilibili.read`` 或 ``bilibili.write`` 段,两个账号的 -身份及设备指纹 Cookie 绝不互相覆盖。 +认证结果只写入凭据文件(默认与主配置同目录的 ``secrets.json``)里指定的 +``bilibili.read`` 或 ``bilibili.write`` 段,两个账号的身份及设备指纹 Cookie +绝不互相覆盖。主配置(可为 JSONC)不会被回写,注释得以保留。 """ from __future__ import annotations @@ -21,6 +22,8 @@ import zlib from pathlib import Path from typing import Any, Callable, Mapping +from ..config import resolve_secrets_path +from .. import jsonc from .auth import ( IDENTITY_KEYS, cookie_value, @@ -154,22 +157,71 @@ class AuthService: def __init__(self, config_path: str | Path, *, account: str, logger=None) -> None: if account not in {"read", "write"}: raise ValueError(f"未知账号角色: {account}") - self.path = Path(config_path) + self.config_path = Path(config_path) + # 兼容旧属性名:历史代码/测试可能读 path;现指向凭据文件 self.account = account self.logger = logger + self.secrets_path = self._resolve_secrets_path() + self.path = self.secrets_path + + def _resolve_secrets_path(self) -> Path: + raw: dict[str, Any] | None = None + if self.config_path.exists(): + try: + loaded = jsonc.load(self.config_path) + except (OSError, ValueError): + loaded = None + if isinstance(loaded, dict): + raw = loaded + return resolve_secrets_path(self.config_path, raw) @property def backup_path(self) -> Path: - return self.path.with_name(self.path.name + ".bak") + return self.secrets_path.with_name(self.secrets_path.name + ".bak") + + def _empty_account(self) -> dict[str, Any]: + return {"mid": 0, "csrf": "", "cookie": ""} def _load_raw(self) -> dict[str, Any]: - with self.path.open(encoding="utf-8") as fh: - raw = json.load(fh) - if not isinstance(raw, dict): - raise ValueError("配置文件顶层必须是对象") + """加载凭据文件;若尚不存在则从主配置 bilibili 段迁移种子。""" + if self.secrets_path.exists(): + raw = jsonc.load(self.secrets_path) + if not isinstance(raw, dict): + raise ValueError("凭据文件顶层必须是对象") + else: + raw = {} + if self.config_path.exists(): + try: + main = jsonc.load(self.config_path) + except (OSError, ValueError): + main = None + if isinstance(main, dict) and isinstance(main.get("bilibili"), dict): + # 仅迁移凭据段,不把整份主配置拷进 secrets + raw["bilibili"] = main["bilibili"] + section = raw.get("bilibili") - if not isinstance(section, dict) or not isinstance(section.get(self.account), dict): - raise ValueError(f"缺少 bilibili.{self.account} 账号配置") + if not isinstance(section, dict): + section = {} + raw["bilibili"] = section + if not isinstance(section.get(self.account), dict): + section[self.account] = self._empty_account() + # 保证另一账号键存在,避免后续手工编辑困惑 + other = "write" if self.account == "read" else "read" + if not isinstance(section.get(other), dict): + # 尝试从主配置补齐;否则空壳 + filled = False + if self.config_path.exists(): + try: + main = jsonc.load(self.config_path) + except (OSError, ValueError): + main = None + if isinstance(main, dict) and isinstance(main.get("bilibili"), dict): + other_cred = main["bilibili"].get(other) + if isinstance(other_cred, dict): + section[other] = dict(other_cred) + filled = True + if not filled: + section[other] = self._empty_account() return raw @staticmethod @@ -209,7 +261,9 @@ class AuthService: if new_mid.isdigit(): credentials["mid"] = int(new_mid) - shutil.copy2(self.path, self.backup_path) + had_secrets = self.secrets_path.exists() + if had_secrets: + shutil.copy2(self.secrets_path, self.backup_path) self._write_raw(raw) try: nav = self._get_json(credentials, f"{BASE}/x/web-interface/nav", True) @@ -218,8 +272,11 @@ class AuthService: except Exception as exc: # noqa: BLE001 data, ok = {"uname": str(exc)}, False if not ok: - shutil.copy2(self.backup_path, self.path) - print(f"[auth] {self.account} 登录校验失败,已回滚 config.json") + if had_secrets and self.backup_path.exists(): + shutil.copy2(self.backup_path, self.secrets_path) + elif self.secrets_path.exists(): + self.secrets_path.unlink() + print(f"[auth] {self.account} 登录校验失败,已回滚 {self.secrets_path.name}") return 4 if isinstance(data.get("mid"), int): diff --git a/reaction_board/cli.py b/reaction_board/cli.py index 83c11d7..90d996c 100644 --- a/reaction_board/cli.py +++ b/reaction_board/cli.py @@ -28,7 +28,7 @@ from .bilibili.auth_flow import AuthService from .bilibili.client import BilibiliClient, Pacer from .bilibili.provider import ClientProvider from .bilibili.search import SearchClient -from .config import ConfigError, ConfigStore +from .config import ConfigError, ConfigStore, default_config_path from .database import Database from .logging_json import JsonLogger from .notify.telegram import TelegramClient @@ -42,7 +42,7 @@ from .services.up_profile import UpProfileService from .timeutil import next_daily_run, parse_hhmm, shanghai_date ROOT = Path(__file__).resolve().parent.parent -DEFAULT_CONFIG = ROOT / "config.json" +DEFAULT_CONFIG = default_config_path(ROOT) def _build_parser() -> argparse.ArgumentParser: @@ -50,7 +50,7 @@ def _build_parser() -> argparse.ArgumentParser: common.add_argument( "--config", default=argparse.SUPPRESS, - help="配置文件路径(默认项目根目录 config.json)", + help="配置文件路径(默认优先 config.jsonc,否则 config.json)", ) parser = argparse.ArgumentParser( @@ -60,7 +60,7 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument( "--config", default=str(DEFAULT_CONFIG), - help="配置文件路径(默认项目根目录 config.json)", + help="配置文件路径(默认优先 config.jsonc,否则 config.json)", ) sub = parser.add_subparsers(dest="command", required=False) diff --git a/reaction_board/config.py b/reaction_board/config.py index 103e614..d241f15 100644 --- a/reaction_board/config.py +++ b/reaction_board/config.py @@ -8,7 +8,6 @@ B 站凭据必须采用 ``bilibili.read`` / ``bilibili.write`` 双账号格式 from __future__ import annotations import copy -import json import os import re import threading @@ -16,12 +15,15 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any, Mapping +from . import jsonc from .client_ip import ( DEFAULT_CLIENT_IP_HEADERS, DEFAULT_TRUSTED_PROXIES, parse_networks, ) +DEFAULT_SECRETS_NAME = "secrets.json" + DEFAULT_CLASSIFY: dict[str, Any] = { "require_title": "咻咻满", "self_mid": 37754047, @@ -213,6 +215,7 @@ class AnalyticsConfig: @dataclass(frozen=True) class AppConfig: path: Path + secrets_path: Path bilibili: BilibiliConfig telegram: TelegramConfig web: WebConfig @@ -352,7 +355,7 @@ def _validate_folders(value: Any) -> tuple[FolderConfig, ...]: return tuple(folders) -def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: +def _build_config(data: Mapping[str, Any], path: Path, *, secrets_path: Path) -> AppConfig: b = data["bilibili"] if not isinstance(b, Mapping): raise ConfigError("bilibili 必须是对象") @@ -593,6 +596,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: return AppConfig( path=path, + secrets_path=secrets_path, bilibili=bilibili, telegram=telegram, web=web, @@ -610,33 +614,87 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: ) +def resolve_secrets_path( + config_path: Path, + raw: Mapping[str, Any] | None = None, +) -> Path: + """解析凭据文件路径。 + + 优先读配置里的 ``secrets_path``(相对配置文件目录或绝对路径); + 否则默认同目录 ``secrets.json``。 + """ + base = Path(config_path).resolve().parent + if raw and "secrets_path" in raw and raw["secrets_path"] not in (None, ""): + value = raw["secrets_path"] + if not isinstance(value, str): + raise ConfigError("secrets_path 必须是字符串") + path = Path(value) + return path if path.is_absolute() else (base / path) + return base / DEFAULT_SECRETS_NAME + + +def default_config_path(root: str | os.PathLike[str] | None = None) -> Path: + """默认配置:同目录下优先 ``config.jsonc``,否则 ``config.json``。 + + 两者都不存在时返回 ``config.jsonc``(作为建议创建路径)。 + """ + base = Path(root) if root is not None else Path(__file__).resolve().parent.parent + jsonc_path = base / "config.jsonc" + json_path = base / "config.json" + if jsonc_path.exists(): + return jsonc_path + if json_path.exists(): + return json_path + return jsonc_path + + +def _read_mapping_file(path: Path, *, kind: str) -> dict[str, Any]: + if not path.exists(): + raise ConfigError(f"缺少{kind}: {path}") + try: + raw = jsonc.load(path) + except ValueError as exc: + raise ConfigError(f"{kind}不是合法 JSON/JSONC: {path}: {exc}") from exc + except OSError as exc: + raise ConfigError(f"无法读取{kind}: {path}: {exc}") from exc + if not isinstance(raw, Mapping): + raise ConfigError(f"{kind}顶层必须是 JSON 对象: {path}") + return dict(raw) + + def load_config( path: str | os.PathLike[str], *, env: Mapping[str, str] | None = None, require_auth: bool = False, ) -> AppConfig: - """读取并校验配置。``env`` 默认为 ``os.environ``。""" + """读取并校验配置。``env`` 默认为 ``os.environ``。 + + 主文件支持 ``.jsonc`` / ``.json``(注释与尾逗号)。敏感凭据优先从同目录 + ``secrets.json``(或 ``secrets_path`` 指定文件)合并;``auth`` 只回写该凭据 + 文件,避免冲掉 JSONC 注释。旧版把 ``bilibili`` 写在主配置里的用法仍可用。 + """ config_path = Path(path) - if not config_path.exists(): - raise ConfigError(f"缺少配置文件: {config_path}") - try: - with open(config_path, encoding="utf-8") as fh: - raw = json.load(fh) - except json.JSONDecodeError as exc: - raise ConfigError(f"配置文件不是合法 JSON: {config_path}: {exc}") from exc - except OSError as exc: - raise ConfigError(f"无法读取配置文件: {config_path}: {exc}") from exc - if not isinstance(raw, Mapping): - raise ConfigError("配置文件顶层必须是 JSON 对象") + raw = _read_mapping_file(config_path, kind="配置文件") + secrets_path = resolve_secrets_path(config_path, raw) + if secrets_path.exists(): + secrets_raw = _read_mapping_file(secrets_path, kind="凭据文件") + # 凭据覆盖主配置中的同名段(深度合并) + raw = _deep_merge(raw, secrets_raw) + raw_bilibili = raw.get("bilibili") if not isinstance(raw_bilibili, Mapping): - raise ConfigError("缺少 bilibili.read / bilibili.write 双账号配置") + raise ConfigError( + "缺少 bilibili.read / bilibili.write 双账号配置" + f"(可写在主配置或 {secrets_path.name})" + ) for account in ("read", "write"): if account not in raw_bilibili: raise ConfigError(f"缺少 bilibili.{account} 账号配置") data = _deep_merge(DEFAULT_CONFIG, raw) + # secrets_path 不是业务字段 + data.pop("secrets_path", None) env_map = os.environ if env is None else env for env_name, (target, caster) in ENV_OVERRIDES.items(): @@ -650,16 +708,17 @@ def load_config( raise ConfigError(f"环境变量 {env_name} 必须是整数") from exc _set_path(data, target, value) - cfg = _build_config(data, config_path) + cfg = _build_config(data, config_path, secrets_path=secrets_path) if require_auth and (not cfg.bilibili.read.cookie or not cfg.bilibili.read.csrf): raise ConfigError( - "config.json 中的 bilibili.read.cookie / csrf 不能为空(可先运行 python app.py auth)" + "bilibili.read.cookie / csrf 不能为空" + f"(写入 {secrets_path.name},或运行 python app.py auth)" ) return cfg class ConfigStore: - """按文件 mtime 热加载配置;坏配置不会覆盖上一份可用配置。""" + """按主配置与凭据文件 mtime 热加载;坏配置不会覆盖上一份可用配置。""" def __init__( self, @@ -677,11 +736,25 @@ class ConfigStore: self.last_error: str | None = None self.revision = 0 + def _watched_paths(self) -> tuple[Path, ...]: + paths = [self.path] + secrets = None + if getattr(self, "_config", None) is not None: + secrets = self._config.secrets_path + if secrets is None: + secrets = resolve_secrets_path(self.path) + if secrets not in paths: + paths.append(secrets) + return tuple(paths) + def _read_mtime_ns(self) -> int | None: - try: - return self.path.stat().st_mtime_ns - except OSError: - return None + times: list[int] = [] + for candidate in self._watched_paths(): + try: + times.append(candidate.stat().st_mtime_ns) + except OSError: + continue + return max(times) if times else None def get(self) -> AppConfig: with self._lock: diff --git a/reaction_board/jsonc.py b/reaction_board/jsonc.py new file mode 100644 index 0000000..2daab1f --- /dev/null +++ b/reaction_board/jsonc.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +"""JSONC(JSON with Comments)加载。 + +选择**自研轻量剥离**而非引入第三方依赖:本仓库运行依赖目前只有 ``httpx``, +配置加载属于启动/热更新热路径,JSONC 只需注释与尾逗号。实现按字符扫描, +在字符串字面量内保留 ``//``、``/*``、引号转义,避免误伤。 + +支持: +- ``//`` 行注释、``/* ... */`` 块注释 +- 对象/数组中的尾逗号 +- UTF-8 BOM +不支持 JSON5 扩展(无引号键、单引号、十六进制、``Infinity`` 等)。 +""" + +from __future__ import annotations + +import json +from typing import Any + + +def strip_jsonc(text: str) -> str: + """去掉 JSONC 注释与尾逗号,返回可交给 ``json.loads`` 的文本。""" + if text.startswith("\ufeff"): + text = text[1:] + out: list[str] = [] + i = 0 + n = len(text) + in_string = False + escape = False + while i < n: + ch = text[i] + if in_string: + out.append(ch) + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_string = False + i += 1 + continue + if ch == '"': + in_string = True + out.append(ch) + i += 1 + continue + if ch == "/" and i + 1 < n: + nxt = text[i + 1] + if nxt == "/": + i += 2 + while i < n and text[i] not in "\r\n": + i += 1 + continue + if nxt == "*": + i += 2 + while i + 1 < n and not (text[i] == "*" and text[i + 1] == "/"): + i += 1 + i = min(i + 2, n) + continue + out.append(ch) + i += 1 + + stripped = "".join(out) + # 去掉尾逗号:`,}` / `,]`(可夹空白) + cleaned: list[str] = [] + i = 0 + m = len(stripped) + in_string = False + escape = False + while i < m: + ch = stripped[i] + if in_string: + cleaned.append(ch) + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_string = False + i += 1 + continue + if ch == '"': + in_string = True + cleaned.append(ch) + i += 1 + continue + if ch == ",": + j = i + 1 + while j < m and stripped[j] in " \t\r\n": + j += 1 + if j < m and stripped[j] in "}]": + i += 1 + continue + cleaned.append(ch) + i += 1 + return "".join(cleaned) + + +def loads(text: str) -> Any: + """解析 JSONC 文本。""" + return json.loads(strip_jsonc(text)) + + +def load(path: str | bytes, *, encoding: str = "utf-8") -> Any: + """从路径读取并解析 JSONC / JSON。""" + with open(path, encoding=encoding) as fh: + return loads(fh.read()) diff --git a/secrets.example.json b/secrets.example.json new file mode 100644 index 0000000..64359f1 --- /dev/null +++ b/secrets.example.json @@ -0,0 +1,21 @@ +{ + "bilibili": { + "read": { + "mid": 0, + "csrf": "", + "cookie": "" + }, + "write": { + "mid": 0, + "csrf": "", + "cookie": "" + } + }, + "telegram": { + "bot_token": "replace-with-telegram-bot-token", + "chat_id": 0 + }, + "web": { + "auth_token": "replace-with-a-long-random-token" + } +} diff --git a/tests/test_auth.py b/tests/test_auth.py index 38cf807..c832399 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -58,28 +58,56 @@ def test_cookie_helpers(): def test_finish_updates_only_selected_account_and_rolls_back(): with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "config.json" + # 主配置为 JSONC(含注释);auth 只写 secrets.json + config_path = Path(tmp) / "config.jsonc" + secrets_path = Path(tmp) / "secrets.json" original = raw_config() - path.write_text(json.dumps(original, ensure_ascii=False), encoding="utf-8") - service = AuthService(path, account="write") + config_body = ( + "// keep me\n" + + json.dumps({"folders": [], "telegram": {"enabled": False}}, ensure_ascii=False) + ) + config_path.write_text(config_body, encoding="utf-8") + secrets_path.write_text(json.dumps(original, ensure_ascii=False), encoding="utf-8") + service = AuthService(config_path, account="write") + assert service.secrets_path == secrets_path service._get_json = lambda credentials, url, auth: { # type: ignore[method-assign] "code": 0, "data": {"isLogin": True, "mid": 99, "uname": "writer"} } assert service.finish(service._load_raw(), NEW) == 0 - saved = json.loads(path.read_text(encoding="utf-8")) + saved = json.loads(secrets_path.read_text(encoding="utf-8")) assert saved["bilibili"]["read"] == original["bilibili"]["read"] assert saved["bilibili"]["write"]["mid"] == 99 assert saved["bilibili"]["write"]["csrf"] == "newcsrf" assert cookie_value(saved["bilibili"]["write"]["cookie"], "buvid3") == "WRITE-B3" - assert path.with_name("config.json.bak").exists() + assert secrets_path.with_name("secrets.json.bak").exists() + assert config_path.read_text(encoding="utf-8") == config_body + assert "// keep me" in config_path.read_text(encoding="utf-8") - before = path.read_text(encoding="utf-8") + before = secrets_path.read_text(encoding="utf-8") service._get_json = lambda credentials, url, auth: { # type: ignore[method-assign] "code": -101, "data": {"isLogin": False} } assert service.finish(service._load_raw(), NEW) == 4 + assert secrets_path.read_text(encoding="utf-8") == before + assert config_path.read_text(encoding="utf-8") == config_body + + +def test_finish_seeds_secrets_from_legacy_config_without_touching_it(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + original = raw_config() + path.write_text(json.dumps(original, ensure_ascii=False), encoding="utf-8") + service = AuthService(path, account="write") + service._get_json = lambda credentials, url, auth: { # type: ignore[method-assign] + "code": 0, "data": {"isLogin": True, "mid": 99, "uname": "writer"} + } + before = path.read_text(encoding="utf-8") + assert service.finish(service._load_raw(), NEW) == 0 assert path.read_text(encoding="utf-8") == before + secrets = json.loads((Path(tmp) / "secrets.json").read_text(encoding="utf-8")) + assert secrets["bilibili"]["write"]["mid"] == 99 + assert secrets["bilibili"]["read"] == original["bilibili"]["read"] def test_qr_png(): @@ -207,12 +235,13 @@ def test_make_headers_drops_placeholder_and_non_latin1_cookies(): def main(): test_cookie_helpers() test_finish_updates_only_selected_account_and_rolls_back() + test_finish_seeds_secrets_from_legacy_config_without_touching_it() test_qr_png() test_render_qr_ascii_roundtrip_and_quiet_zone() test_run_prints_ascii_qr_without_network() test_make_headers_matches_raw_request() test_make_headers_drops_placeholder_and_non_latin1_cookies() - print("OK: 双账号 auth Cookie 合并、写回隔离、终端二维码与回滚测试全部通过") + print("OK: 双账号 auth Cookie 合并、secrets 写回隔离、终端二维码与回滚测试全部通过") if __name__ == "__main__": diff --git a/tests/test_config.py b/tests/test_config.py index a03419c..2824011 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,7 +11,13 @@ from pathlib import Path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.stdout.reconfigure(encoding="utf-8") -from reaction_board.config import ConfigError, ConfigStore, load_config # noqa: E402 +from reaction_board.config import ( # noqa: E402 + ConfigError, + ConfigStore, + default_config_path, + load_config, + resolve_secrets_path, +) def write(path: Path, data) -> None: @@ -294,6 +300,78 @@ def test_schedule_recent_validation(): + +def test_jsonc_and_secrets_merge_and_hot_reload(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + config_path = root / "config.jsonc" + secrets_path = root / "secrets.json" + config_path.write_text( + """ + // 业务配置 + { + "folders": [ + {"year": 2026, "media_id": 1, "title": "A"}, + ], + "track_fans": [], + "up_blacklist_seed": [], + "telegram": { "enabled": true }, + "web": { "host": "127.0.0.1", "port": 8080 }, + } + """, + encoding="utf-8", + ) + secrets_path.write_text(json.dumps({ + "bilibili": { + "read": {"mid": 11, "csrf": "rc", "cookie": "rk"}, + "write": {"mid": 22, "csrf": "wc", "cookie": "wk"}, + }, + "telegram": {"bot_token": "tok", "chat_id": 9}, + "web": {"auth_token": "secret-token"}, + }), encoding="utf-8") + + cfg = load_config(config_path, env={}) + assert cfg.secrets_path == secrets_path + assert cfg.bilibili.read.cookie == "rk" + assert cfg.telegram.enabled is True + assert cfg.telegram.bot_token == "tok" + assert cfg.telegram.chat_id == 9 + assert cfg.web.auth_token == "secret-token" + assert resolve_secrets_path(config_path) == secrets_path + + # 主配置注释在热更新凭据后仍保留 + original = config_path.read_text(encoding="utf-8") + store = ConfigStore(config_path, env={}) + secrets = json.loads(secrets_path.read_text(encoding="utf-8")) + secrets["bilibili"]["read"]["cookie"] = "rk2" + secrets_path.write_text(json.dumps(secrets), encoding="utf-8") + bump_mtime(secrets_path) + assert store.get().bilibili.read.cookie == "rk2" + assert config_path.read_text(encoding="utf-8") == original + assert "// 业务配置" in original + + +def test_legacy_monolithic_json_still_loads(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + write(path, base_config()) + cfg = load_config(path, env={}) + assert cfg.bilibili.read.mid == 11 + assert cfg.secrets_path == Path(tmp) / "secrets.json" + + +def test_default_config_path_prefers_jsonc(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + json_path = root / "config.json" + jsonc_path = root / "config.jsonc" + json_path.write_text("{}", encoding="utf-8") + assert default_config_path(root) == json_path + jsonc_path.write_text("{}", encoding="utf-8") + assert default_config_path(root) == jsonc_path + + + def main(): test_dual_accounts_and_env_override() test_flat_and_missing_account_rejected() @@ -302,7 +380,10 @@ def main(): test_analytics_defaults_and_validation() test_schedule_recent_validation() test_web_track_defaults_and_validation() - print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖、热加载与 analytics/web.track 测试全部通过") + test_jsonc_and_secrets_merge_and_hot_reload() + test_legacy_monolithic_json_still_loads() + test_default_config_path_prefers_jsonc() + print("OK: 双账号配置加载、JSONC/secrets、拒绝 flat 格式、ENV 覆盖、热加载与 analytics/web.track 测试全部通过") if __name__ == "__main__": diff --git a/tests/test_jsonc.py b/tests/test_jsonc.py new file mode 100644 index 0000000..53f714b --- /dev/null +++ b/tests/test_jsonc.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +"""JSONC 剥离与解析测试: uv run python tests/test_jsonc.py""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.jsonc import loads, strip_jsonc # noqa: E402 + + +def test_comments_trailing_comma_and_strings(): + text = ''' + // head comment + { + "a": 1, // trail + "url": "https://ex.com//path", /* block */ + "list": [1, 2, ], + "nested": { "x": true, }, + } + ''' + data = loads(text) + assert data == { + "a": 1, + "url": "https://ex.com//path", + "list": [1, 2], + "nested": {"x": True}, + } + # 字符串内双斜杠不得被当成注释吃掉 + assert "https://ex.com//path" in strip_jsonc(text) + + +def test_block_comment_and_bom(): + text = '\ufeff{"k": /* c */ "v",}' + assert loads(text) == {"k": "v"} + + +def main(): + test_comments_trailing_comma_and_strings() + test_block_comment_and_bom() + print("OK: JSONC 注释、尾逗号与字符串保护测试通过") + + +if __name__ == "__main__": + main() diff --git a/tools/import_fans_csv.py b/tools/import_fans_csv.py index 0552986..3fc4da8 100644 --- a/tools/import_fans_csv.py +++ b/tools/import_fans_csv.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) +from reaction_board.config import default_config_path # noqa: E402 from reaction_board.database import Database # noqa: E402 from reaction_board.repository import Repository # noqa: E402 from reaction_board.timeutil import SHANGHAI_TZ # noqa: E402 @@ -226,19 +227,19 @@ def _database_path(config_path: str, override: str | None) -> Path: return Path(override) from reaction_board.config import ConfigStore - candidates = [Path(config_path), Path("/config/config.json")] + candidates = [Path(config_path), Path("/config/config.jsonc"), Path("/config/config.json")] for candidate in candidates: if candidate.exists(): return ConfigStore(str(candidate), require_auth=False).get().storage.database_path raise ValueError( - "找不到配置文件;请用 --db /data/app.db 或 --config /config/config.json" + "找不到配置文件;请用 --db /data/app.db 或 --config /config/config.jsonc" ) def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description="从 B 站粉丝历史 CSV 回填 up_stat") parser.add_argument("--csv", required=True, help="粉丝历史 CSV 路径") - parser.add_argument("--config", default=str(ROOT / "config.json"), help="配置文件") + parser.add_argument("--config", default=str(default_config_path(ROOT)), help="配置文件") parser.add_argument("--db", default=None, help="直接指定数据库(覆盖配置)") parser.add_argument("--mid", type=int, default=DEFAULT_MID, help="写入的 B 站 mid") parser.add_argument("--force", action="store_true", help="覆盖同日期已有 ok=1 记录") diff --git a/tools/probe_anon.py b/tools/probe_anon.py index 5f5df23..910a37a 100644 --- a/tools/probe_anon.py +++ b/tools/probe_anon.py @@ -49,8 +49,9 @@ def get(url, cookie): def main(): - cfg = json.load(open(os.path.join(ROOT, "config.json"), encoding="utf-8")) - full = cfg["cookie"] + from reaction_board.config import default_config_path, load_config + app = load_config(default_config_path(ROOT), require_auth=False) + full = app.bilibili.read.cookie or app.bilibili.write.cookie anon = strip_identity(full) variants = [("完整 cookie", full), ("去掉身份字段", anon), ("完全无 cookie", None)] diff --git a/tools/seed_preview_db.py b/tools/seed_preview_db.py index 1a93994..0ed8973 100644 --- a/tools/seed_preview_db.py +++ b/tools/seed_preview_db.py @@ -8,7 +8,7 @@ 用法:: uv run python tools/seed_preview_db.py --days 7 --seed 20261031 - uv run python app.py serve --preview --config data/preview/config.json + uv run python app.py serve --preview --config data/preview/config.jsonc """ from __future__ import annotations @@ -31,7 +31,9 @@ from reaction_board.timeutil import SHANGHAI_TZ, shanghai_datetime # noqa: E402 DEFAULT_SOURCE = ROOT / "data" / "app.db" DEFAULT_TARGET = ROOT / "data" / "preview" / "app.db" -DEFAULT_CONFIG_SOURCE = ROOT / "config.json" +from reaction_board.config import default_config_path # noqa: E402 + +DEFAULT_CONFIG_SOURCE = default_config_path(ROOT) BASE_DAILY_RATES: dict[str, float] = { "view": 0.0040, @@ -187,8 +189,9 @@ def _read_source_config(path: Path | None) -> dict[str, Any] | None: if path is None or not path.exists(): return None try: - data = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + from reaction_board import jsonc + data = jsonc.load(path) + except (OSError, ValueError): return None return data if isinstance(data, dict) else None @@ -617,7 +620,7 @@ def seed_preview( config_target = ( Path(config_out).resolve() if config_out is not None - else target_path.parent / "config.json" + else target_path.parent / "config.jsonc" ) config_target.parent.mkdir(parents=True, exist_ok=True)