From e2cdf997af17f44d186d309aba7625cfb6047614 Mon Sep 17 00:00:00 2001 From: kaireewu Date: Sat, 31 Oct 2026 18:08:33 +0800 Subject: [PATCH] feat(p0): add favorite folder sync and sqlite data foundation Implement P0 of the reaction dashboard: config loading with hot reload, SQLite migrations with backups, single-writer queue, read-only Bilibili fav client, folder import/diff sync, invalid-resource handling, blacklist seed, scheduler, and JSON redacted logging. Includes offline unit and transport-level contract tests; all existing and new tests pass. --- .gitignore | 17 + .python-version | 1 + AGENTS.md | 80 +++ DEVELOPMENT.md | 595 ++++++++++++++++ README.md | 221 ++++++ app.py | 13 + config.example.json | 87 +++ favsort.py | 944 +++++++++++++++++++++++++ project.md | 801 +++++++++++++++++++++ pyproject.toml | 23 + reaction_board/__init__.py | 2 + reaction_board/bilibili/__init__.py | 2 + reaction_board/bilibili/auth.py | 143 ++++ reaction_board/bilibili/client.py | 362 ++++++++++ reaction_board/bilibili/models.py | 87 +++ reaction_board/cli.py | 217 ++++++ reaction_board/config.py | 495 +++++++++++++ reaction_board/database.py | 175 +++++ reaction_board/logging_json.py | 101 +++ reaction_board/migrations.py | 305 ++++++++ reaction_board/repository.py | 441 ++++++++++++ reaction_board/scheduler.py | 115 +++ reaction_board/services/__init__.py | 2 + reaction_board/services/folder_sync.py | 278 ++++++++ tests/fixtures.py | 163 +++++ tests/test_auth.py | 171 +++++ tests/test_bilibili_client.py | 295 ++++++++ tests/test_cli.py | 110 +++ tests/test_config.py | 210 ++++++ tests/test_database.py | 142 ++++ tests/test_folder_sync.py | 297 ++++++++ tests/test_logging.py | 94 +++ tests/test_migrations.py | 150 ++++ tests/test_moves.py | 57 ++ tests/test_repository.py | 243 +++++++ tests/test_scheduler.py | 122 ++++ tools/probe_anon.py | 94 +++ uv.lock | 13 + vendor/qrcodegen.py | 907 ++++++++++++++++++++++++ 39 files changed, 8575 insertions(+) create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 AGENTS.md create mode 100644 DEVELOPMENT.md create mode 100644 README.md create mode 100644 app.py create mode 100644 config.example.json create mode 100644 favsort.py create mode 100644 project.md create mode 100644 pyproject.toml create mode 100644 reaction_board/__init__.py create mode 100644 reaction_board/bilibili/__init__.py create mode 100644 reaction_board/bilibili/auth.py create mode 100644 reaction_board/bilibili/client.py create mode 100644 reaction_board/bilibili/models.py create mode 100644 reaction_board/cli.py create mode 100644 reaction_board/config.py create mode 100644 reaction_board/database.py create mode 100644 reaction_board/logging_json.py create mode 100644 reaction_board/migrations.py create mode 100644 reaction_board/repository.py create mode 100644 reaction_board/scheduler.py create mode 100644 reaction_board/services/__init__.py create mode 100644 reaction_board/services/folder_sync.py create mode 100644 tests/fixtures.py create mode 100644 tests/test_auth.py create mode 100644 tests/test_bilibili_client.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_config.py create mode 100644 tests/test_database.py create mode 100644 tests/test_folder_sync.py create mode 100644 tests/test_logging.py create mode 100644 tests/test_migrations.py create mode 100644 tests/test_moves.py create mode 100644 tests/test_repository.py create mode 100644 tests/test_scheduler.py create mode 100644 tools/probe_anon.py create mode 100644 uv.lock create mode 100644 vendor/qrcodegen.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..09732d6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# 本地凭据，绝对不要提交 +config.json +config.json.bak +config.json.tmp + +# 计划 / 日志 / 快照 +data/ +*.journal.jsonl + +# Python +__pycache__/ +*.pyc +.venv/ + +# uv / test tooling cache +.uv/ +.pytest_cache/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8215e4d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,80 @@ +# 开发约束 + +本文件适用于本仓库内的所有代码、测试、配置和文档变更。后续开发必须遵守以下约束；如与更深层目录中的 `AGENTS.md` 冲突，以更深层文件为准。 + +## 1. 测试是实现的一部分 + +- 新增或修改任何方法、函数、类方法或业务逻辑时，必须同步新增或更新自动化测试。 +- **不依赖外部接口、文件系统、时钟、随机数或进程环境的纯函数**必须有可靠的单元测试，覆盖： + - 正常输入与预期输出； + - 边界值； + - 空值、异常值或非法输入（如函数设计需要处理）； + - 已修复缺陷对应的回归用例。 +- 不得以“仅人工运行验证”“代码简单”或“调用方已有测试”为由省略纯函数测试。 +- 测试必须确定、可重复、离线运行；不得访问真实 B 站、Telegram 或其他外部服务。 + +## 2. 外部 API 调用必须使用契约一致的 Mock 验证 + +- 所有依赖外部 HTTP/API 接口的方法或函数必须有自动化测试。 +- 测试中的 Mock 响应必须与实际接口的返回格式一致，包括： + - 顶层字段和嵌套结构； + - 字段类型（例如整数、字符串、数组、`null`）； + - 成功响应的 `code` / `data` 等包装层； + - 项目需要处理的失败响应、缺字段、空数组和异常响应。 +- Mock 数据应优先来自已记录的真实接口样本，并完成脱敏；不得为方便测试而编造与接口契约不一致的简化结构。 +- 每个 API 客户端至少覆盖： + - 成功响应的解析与字段映射； + - API 业务错误码； + - HTTP/网络错误； + - 本项目依赖的边界情形（例如 B 站 `infos` 静默缺失资源、`view` 返回 `-404`、鉴权失效）。 +- Mock 应替换网络传输边界（HTTP client / transport），而不是替换正在测试的业务解析与错误处理逻辑。 + +## 3. 测试执行要求 + +- 所有测试命令必须通过 `uv run` 在项目锁定的 Python 3.14 环境中运行。 +- 提交前至少运行受改动影响的测试；涉及共享模块、配置、数据库迁移、API 客户端或调度逻辑时，必须运行完整测试集。 +- 测试失败时不得提交“已验证”的实现；应先修复代码或更新正确的测试预期。 +- 测试命令、结果和未执行测试的原因必须在最终开发说明中如实记录。 + +当前已有测试示例： + +```powershell +uv run python tests/test_auth.py +uv run python tests/test_moves.py +``` + +新增测试应按功能放入 `tests/`，命名清晰，并保持可由 `uv run python <测试文件>` 直接执行；如后续引入测试框架，必须同时更新 `pyproject.toml`、`uv.lock` 与相关文档。 + +## 4. 阶段性本地 Git 提交 + +- 完成一个可独立验证的关键步骤后，必须先运行对应测试，再创建一次**本地 Git 提交**；不得推送远程仓库，除非用户明确要求。 +- 关键步骤包括但不限于： + - 一个完整的 API 客户端或认证能力； + - 一个数据库迁移或数据模型阶段； + - 一个调度任务； + - 一个端到端业务动作（如候选收藏、忽略、拉黑）； + - 一个完整页面/API 范围； + - 重要缺陷修复或安全修复。 +- 提交前检查工作区，确保不包含凭据、Cookie、数据库、日志、缓存、构建产物或其他运行期文件。 +- 提交信息使用简洁的 Conventional Commits 风格，例如： + +```text +feat(sync): add favorite-folder import +fix(auth): preserve device fingerprint cookies +test(api): cover missing favorite resource responses +docs: document local uv workflow +``` + +- 若当前目录尚未初始化为 Git 仓库，或 Git 用户身份未配置，应向用户说明这一阻塞项；**不得未经明确授权自行执行 `git init`、配置 Git 身份或创建提交。** +- 不得执行会丢失用户工作区内容的 Git 命令，例如 `git reset --hard`、`git checkout -- .` 或强制清理未跟踪文件，除非用户明确要求。 + +## 5. 完成定义 + +一个开发任务只有在以下条件满足后才可视为完成： + +1. 实现与文档/需求口径一致； +2. 新增或变更的纯函数已有可靠离线单元测试； +3. API 相关逻辑已有契约一致的 Mock 测试； +4. 受影响测试已通过； +5. 关键步骤已经创建本地 Git 提交，或已明确记录无法提交的原因； +6. 最终说明包含改动文件、验证命令与结果，以及未验证项（如有）。 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..23d6427 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,595 @@ +# 咻咻满 Reaction 视频数据看板开发文档 + +> **状态：** P0（数据底座与收藏夹同步）已实施并通过只读联调；P1 待开始 +> +> **技术基线：** Python 3.14、Python 标准库、SQLite、原生 HTML/CSS/JavaScript、`uv` +> +> **本地开发：** 不依赖 Docker；使用 `uv` 创建和管理隔离环境。 + +--- + +## 1. 技术选型 + +| 范围 | 选型 | 说明 | +|---|---|---| +| 开发语言 | Python 3.14 | 与现有 `favsort.py` 一致，复用 Cookie、扫码授权与收藏夹排序能力。 | +| 包与环境管理 | `uv` | 管理 Python 版本、`.venv`、依赖同步及统一命令。 | +| 后端服务 | Python 标准库 `ThreadingHTTPServer` | 个人单用户看板，接口与并发规模有限，不引入 Web 框架。 | +| HTTP 客户端 | `urllib.request` | 现有 B 站请求代码已使用；无需额外依赖。 | +| 数据库 | SQLite（WAL） | 单用户、单机、低并发写入；备份和迁移成本低。 | +| 调度 | `threading`、`queue.Queue`、标准库时间工具 | 单进程内运行发现、同步、快照和清理任务。 | +| 前端 | 原生 HTML/CSS/JavaScript | 无 Node.js、无构建链；前端仅渲染 `/api/*` JSON。 | +| 图表 | 仓库内置 ECharts | 离线静态资源，Apache-2.0 许可。 | +| 通知 | Telegram Bot long polling | 具备按钮回调能力，开发和本机运行不需要公网 URL。 | +| 容器化 | 可选的后续部署方式 | 本地开发、调试和日常运行均不要求 Docker。 | + +### 1.1 选型原则 + +1. **复用现有能力。** 当前仓库的 `favsort.py` 已实现 B 站设备指纹 Cookie、扫码登录、收藏夹读取和排序逻辑；新功能应拆分复用，而不是另起技术栈。 +2. **保持单体。** 当前规模约数百到数千视频，每日采集量约千级请求；单进程、SQLite 和内置调度足够。 +3. **尽量标准库。** 当前没有第三方 Python 依赖，第一期不因便利性引入 Web 框架、ORM、任务队列或 Redis。 +4. **本地优先。** 可在 Windows 上通过 `uv` 直接运行，不要求 WSL、Docker、数据库服务或宿主机 cron。 +5. **未来可部署。** 服务保持无状态代码与可挂载数据目录分离，未来可选用 Docker 与 Caddy/Nginx 部署，但不是开发前置条件。 + +--- + +## 2. 使用 uv 的本地开发环境 + +### 2.1 前置条件 + +- 安装 [uv](https://docs.astral.sh/uv/)； +- 网络联通以便 `uv` 首次下载 Python 3.14； +- Windows PowerShell、Windows Terminal 或任意可运行 Python 的终端； +- 不需要安装 Docker、Node.js、npm、SQLite 命令行工具或 PostgreSQL。 + +项目通过根目录的 `.python-version` 固定使用 Python 3.14。`uv` 会在需要时自动安装相应解释器，并在仓库根目录创建 `.venv`。 + +### 2.2 初始化与同步 + +在项目根目录执行： + +```powershell +uv sync --all-groups +``` + +该命令会： + +1. 读取 `pyproject.toml`； +2. 确保 Python 3.14 可用； +3. 创建或更新 `.venv`； +4. 安装项目本身及后续声明的依赖； +5. 根据 `uv.lock` 锁定可复现的环境。 + +目前项目功能仅依赖 Python 标准库，因此首次同步不会安装额外运行时包。这是预期行为，不表示 `uv` 没有生效。 + +### 2.3 常用命令 + +```powershell +# 查看 Python 与 uv 管理的虚拟环境是否可用 +uv run python --version + +# 查看现有收藏夹工具帮助 +uv run bili-fav help + +# 扫码登录并原子更新 config.json +uv run bili-fav auth + +# 检查现有 B 站登录状态 +uv run bili-fav auth --check + +# 执行当前无网络依赖的测试 +uv run python tests/test_auth.py +uv run python tests/test_moves.py +``` + +未来看板服务实现后，统一使用： + +```powershell +# 建库、迁移与黑名单种子（离线，P0） +uv run python app.py init + +# 收藏夹只读同步/首次导入（P0） +uv run python app.py sync + +# 按配置周期常驻同步（P0）；--once 只跑一次 +uv run python app.py run +uv run python app.py run --once + +# 查看数据库状态（离线，P0） +uv run python app.py status + +# P2 起启动本地看板服务 +uv run python app.py +uv run python app.py --config .\config.json +``` + +不建议直接激活 `.venv`；优先使用 `uv run ...`，这样命令总是运行在项目锁定的解释器和环境中。 + +### 2.4 依赖变更规范 + +若后续确实需要新增依赖，应使用 `uv` 修改项目元数据，而不是直接使用 `pip install`： + +```powershell +# 新增运行依赖（仅在标准库无法合理满足需求时） +uv add + +# 新增仅用于开发的依赖 +uv add --dev + +# 同步锁文件和环境 +uv sync --all-groups +``` + +提交依赖变更时，必须同时提交： + +- `pyproject.toml` +- `uv.lock` + +不得提交： + +- `.venv/` +- `config.json` +- `config.json.bak` +- `data/` 中的运行期数据库、日志、计划和备份。 + +--- + +## 3. 总体架构 + +```text +浏览器 / 手机 Telegram 私聊 + │ │ + ├──────────── HTTP / JSON ─────────────┤ + │ │ +┌─────▼──────────────────────────────────────▼─────┐ +│ 单一 Python 进程 │ +│ │ +│ Web 线程：ThreadingHTTPServer :8080 │ +│ 调度线程：发现 / 收藏夹同步 / 快照 / 清理 │ +│ Telegram：getUpdates long polling │ +│ 写入线程：SQLite 单写队列 │ +│ │ +│ BilibiliClient ── B 站 Web API │ +│ SQLite (WAL) ── ./data/app.db 或配置的数据目录 │ +└────────────────────────────────────────────────────┘ +``` + +### 3.1 并发与事务原则 + +- Web 请求、调度任务和 Telegram 长轮询可以并发； +- 所有 SQLite 写操作经单写队列串行执行，避免 `database is locked`； +- 读操作使用短生命周期连接； +- 网络请求期间不得持有数据库写事务； +- 收藏操作通过 `fav_action` 状态锁保证幂等； +- SQLite 启用 WAL、外键和合理的 busy timeout。 + +```sql +PRAGMA journal_mode = WAL; +PRAGMA foreign_keys = ON; +PRAGMA busy_timeout = 5000; +PRAGMA synchronous = NORMAL; +``` + +--- + +## 4. 推荐代码结构 + +```text +. +├── app.py # 看板服务入口（P0 起新增） +├── favsort.py # 现有收藏夹排序与授权工具 +├── pyproject.toml # uv / Python 项目元数据 +├── uv.lock # uv 生成并提交的锁文件 +├── .python-version # 固定 Python 3.14 +├── config.json # 运行期私密配置，不提交 +├── config.example.json # 脱敏配置样例 +├── DEVELOPMENT.md +│ +├── reaction_board/ +│ ├── config.py # 配置、ENV 覆盖、mtime 热加载 +│ ├── database.py # SQLite 连接、迁移、单写队列 +│ ├── migrations.py # schema_version 与迁移 +│ ├── scheduler.py # 内置任务调度 +│ ├── logging_json.py # 脱敏 JSON 日志 +│ ├── bilibili/ +│ │ ├── client.py # HTTP、WBI、重试与限速 +│ │ ├── auth.py # 从 favsort 抽取的扫码授权能力 +│ │ └── favorite.py # 加入收藏夹与排序定位 +│ ├── services/ +│ │ ├── discovery.py +│ │ ├── folder_sync.py +│ │ ├── snapshot.py +│ │ ├── fans.py +│ │ ├── candidates.py +│ │ ├── rankings.py +│ │ └── maintenance.py +│ ├── notify/telegram.py +│ └── web/ +│ ├── server.py +│ ├── api.py +│ └── static/ +│ ├── index.html +│ ├── app.js +│ ├── styles.css +│ └── vendor/echarts.min.js +│ +├── tests/ +└── data/ # 本地运行期数据，默认不提交 +``` + +`favsort.py` 继续保持可独立运行。P0 实施时优先抽取其通用的 Cookie、请求头、扫码授权和收藏夹 API 逻辑；不要直接复制出第二套不一致的身份与请求实现。 + +--- + +## 5. 配置与凭据 + +### 5.1 运行期配置 + +所有凭据和可变业务参数放在 `config.json` 中。文件必须被 Git 忽略，日志中不得输出其敏感值。 + +```jsonc +{ + "bilibili": { + "mid": 28802351, + "csrf": "", + "cookie": "SESSDATA=...; bili_jct=...; buvid3=..." + }, + "telegram": { + "enabled": true, + "bot_token": "123456:ABC", + "chat_id": 123456789 + }, + "web": { + "host": "127.0.0.1", + "port": 8080, + "auth_token": "replace-with-a-long-random-token", + "public_base_url": "" + }, + "storage": { + "database_path": "data/app.db", + "backup_dir": "data/backup", + "backup_keep_days": 14 + }, + "folders": [ + { "year": 2025, "media_id": 4451053851, "title": "咻咻满-reaction-2025" }, + { "year": 2026, "media_id": 3700023351, "title": "咻咻满 reaction 合集" } + ], + "track_fans": [37754047], + "up_blacklist_seed": [261052419, 1908537643, 1701121088, 542262], + "schedule": { + "discover_minutes": 60, + "folder_sync_minutes": 60, + "full_folder_refresh_hours": 6, + "snapshot_at": "00:01", + "candidate_ttl_days": 7 + }, + "notify": { "min_score": 2, "max_items_per_message": 10 } +} +``` + +- `folders` 中的年度夹由人工创建，系统只读； +- 兼容现有 `favsort.py` 的扁平 `mid` / `csrf` / `cookie` 写法；存在 `bilibili` 嵌套对象时以嵌套值为准； +- P0 已实现的命令行：`python app.py init`、`python app.py sync [--full]`、`python app.py status`。 + +### 5.2 安全规则- Windows 本地开发时应限制 `config.json` 的文件访问权限；部署到 Linux 时权限必须为 `600`； +- 配置文件每 60 秒检查一次 mtime，收藏夹、分类规则和权重变更后无需重启； +- Cookie 或 CSRF 改变时重建 B 站请求客户端； +- Telegram 操作必须同时验证 `from.id` 和 `chat.id` 是否为配置的私聊用户； +- 仅允许环境变量临时覆盖敏感配置，不能将真实凭据硬编码进源代码； +- `GET` 路由绝不进行收藏、忽略、拉黑等写操作。 + +--- + +## 6. 数据模型 + +### 6.1 核心状态 + +| 表 | 状态 | 含义 | 是否参与每日快照 | +|---|---|---|---| +| `video` | `active` | 在库追踪的视频 | 是 | +| `video` | `unfaved` | 已从收藏夹移除，但保留历史追踪 | 是 | +| `video` | `removed` | 视频失效、删除或不可访问 | 否 | +| `video` | `dropped` | 在看板中手动移出库 | 否 | +| `candidate` | `NULL` | 待人工决策 | 否 | +| `candidate` | `rejected` | 已忽略 | 否 | +| `candidate` | `blacklisted` | 所属 UP 已拉黑 | 否 | + +### 6.2 建表原则 + +最终 DDL 由迁移模块维护，关键约束如下： + +- `snapshot` 主键为 `(stat_date, aid)`，保证同日幂等； +- `up_stat` 主键为 `(stat_date, mid)`； +- `folder_resource(media_id, aid)` 记录每个收藏夹实际拥有的资源，不能仅依赖 `video.folder_id`； +- `video.folder_id` 表示视频的主归属夹，用于新视频同 UP 归属和排序； +- `up_blacklist` 只在发现任务中生效，收藏夹同步不得用它过滤既有视频； +- `seen_aid` 永久保留搜索去重标记，候选过期删除后也不重复通知； +- `fav_action` 保存 Telegram 或签名链接的动作状态，防止重复点击造成重复写入。 + +### 6.3 迁移规范 + +- 使用 `schema_version(version, applied_at)`； +- 每个版本迁移必须包在单一 SQLite 事务； +- 迁移只能向前追加，不能用“删除数据库重建”替代升级； +- 开发和正式运行产生的数据均保留； +- 迁移前自动创建 SQLite 备份，失败时回滚事务并保留原库。 + +--- + +## 7. B 站接口与任务设计 + +### 7.1 接口职责 + +| 能力 | 接口 | 用途 | +|---|---|---| +| 发现 | `x/web-interface/wbi/search/type` | 每小时搜索候选 | +| 搜索降级 | `x/web-interface/search/type` | WBI 搜索失败时使用 | +| 收藏夹 ID | `x/v3/fav/resource/ids` | 计算新增和移除差集 | +| 收藏夹详情 | `x/v3/fav/resource/infos` | 元数据同步、失效识别 | +| 视频统计 | `x/web-interface/view?aid=` | 日更七项指标的唯一事实来源 | +| 粉丝数 | `x/relation/stat?vmid=` | 每日粉丝快照 | +| 收藏夹列表 | `x/v3/fav/folder/created/list-all` | 容量监控 | +| 加入收藏 | `x/v3/fav/resource/deal` | 确认候选后收藏 | +| 收藏夹排序 | `x/v3/fav/resource/sort` | 定位至同 UP 分组 | + +### 7.2 发现任务 + +每小时扫描两个关键词、每个关键词前两页： + +```text +关键词：咻咻满 reaction、咻咻满 充电 +排序：pubdate +页面：1、2 +``` + +处理规则： + +1. 已在 `video` 或 `seen_aid` 的 aid 跳过； +2. 发布时间超过 14 天的结果跳过； +3. 标题不含“咻咻满”跳过； +4. 命中 `up_blacklist` 的 UP 跳过，不写候选池； +5. 通过分类规则计算分数； +6. 分数低于零丢弃； +7. 其余写入候选池；达到通知阈值的候选发送 Telegram 消息； +8. 请求之间随机等待 1.5～2.5 秒。 + +分类规则 v1： + +```text ++2：标题含 reaction / re / 反应 / 看 / 听 / 初听 / 首次 / 点评 / 锐评 / 路人 / 震惊 / 双人 ++2：标题含 充电 / 加急 / 专场 ++2：时长大于 60 分钟 ++1：时长大于 30 分钟且不超过 60 分钟 +-5：标题含 歌切 / 切片 / 纯享 / 搬运 / 完整版 +-3：UP 为咻咻满本人 + +score >= 2：发送通知 +0 <= score < 2：仅候选池展示 +score < 0：丢弃 +``` + +### 7.3 收藏夹同步 + +每小时对所有已配置年度夹执行： + +1. 请求 `resource/ids` 获取远端 aid 集合； +2. 与 `folder_resource` 的有效本地集合做差集； +3. 对新增 aid 每 200 条请求 `resource/infos` 并入库； +4. 对移除 aid 标记对应 `folder_resource.active=0`；若视频不在任何配置夹中，视频状态改为 `unfaved`； +5. 每 6 小时全量刷新 infos，刷新元数据、`attr` 和收藏夹容量； +6. `attr=9` 或请求集合中静默缺失的资源改为 `removed`； +7. 容量达到 950 条时发送预警；系统绝不自动换夹。 + +`infos` 只能作为元数据和失效同步来源。它缺少点赞、投币、转发，且评论字段不可靠，不能替代 `view` 日更。 + +### 7.4 每日快照与粉丝采集 + +每天 `00:01`（`Asia/Shanghai`）执行： + +- 对 `status IN ('active', 'unfaved')` 的视频逐条请求 `view?aid=`； +- 成功写入播放、点赞、收藏、投币、转发、评论、弹幕七项数据； +- `-404` 时标记 `removed`； +- 其他错误最多重试 3 次，仍失败则写入 `ok=0` 与错误信息，不写伪造的零值； +- 每条视频请求之间随机等待 1.0～1.6 秒； +- 对 `track_fans` 中的每个 mid 请求一次 `relation/stat` 并写入 `up_stat`； +- 任务结束向 Telegram 发送成功数、失败数、移除数和总耗时摘要。 + +服务启动后若当天快照尚未成功完成，应补跑当日任务；已成功的 `(stat_date, aid)` 不重复请求。 + +--- + +## 8. Telegram 与收藏操作 + +### 8.1 通知动作 + +候选通知提供： + +- **收藏**：加入目标收藏夹并入库； +- **忽略**：候选标记 `rejected`，不再通知； +- **拉黑**：写入 `up_blacklist`，同时拒绝该 UP 的其他待决策候选； +- **打开视频**：外链到 B 站视频页。 + +拉黑只影响未来搜索发现；不清理、不停止追踪已有视频，也不影响从收藏夹同步进来的视频。 + +### 8.2 幂等与失败处理 + +Telegram `callback_data` 只包含短操作 ID，例如 `fav:`；真实 aid、目标夹、过期时间和状态保存在 `fav_action` 表。 + +收藏流程： + +1. 将动作从 `pending` 原子切换为 `processing`； +2. 先读取目标夹 ids，若视频已存在则按成功处理； +3. 按“同 UP 最早视频所在夹优先，否则当年夹”的规则选择目标夹； +4. 调用 `deal` 加入收藏； +5. 调用 `sort` 将视频放进同 UP 连续分组； +6. 回读 ids 校验最终位置； +7. 成功：动作标记 `done`，视频标记 `active`，删除候选； +8. 失败：动作标记 `failed`，保留候选与原始错误，允许用户重试。 + +GET 页面或链接预览只能展示信息，绝不能触发任何写操作。 + +--- + +## 9. Web API 与页面 + +### 9.1 API 原则 + +- `/api/health` 可匿名，仅返回健康状态； +- 其他 `/api/*` 默认要求 `Authorization: Bearer `； +- 写操作额外要求 CSRF 防护与 Origin/Referer 校验； +- API 返回 JSON；前端不复制热度、状态机或收藏夹归属判断； +- CSV 导出由后端根据当前筛选条件生成。 + +核心接口： + +| 方法 | 路径 | 用途 | +|---|---|---| +| `GET` | `/api/overview` | 总览指标、容量、采集状态 | +| `GET` | `/api/videos` | 视频分页、筛选和排序 | +| `GET` | `/api/videos/{aid}` | 视频详情 | +| `GET` | `/api/videos/{aid}/trend` | 七项趋势数据 | +| `GET` | `/api/rankings` | 热度、增长、互动率与单项榜 | +| `GET` | `/api/ups` | UP 聚合榜单 | +| `GET` | `/api/fans` | 粉丝曲线与增量 | +| `GET` | `/api/candidates` | 候选池 | +| `GET` | `/api/system/status` | 最近任务、下次任务、错误日志 | +| `POST` | `/api/candidates/{aid}/favorite` | 收藏候选 | +| `POST` | `/api/candidates/{aid}/ignore` | 忽略候选 | +| `POST` | `/api/candidates/{aid}/blacklist` | 拉黑候选所属 UP | +| `DELETE` | `/api/blacklist/{mid}` | 移除黑名单 | +| `POST` | `/api/videos/{aid}/drop` | 手动停止追踪 | + +### 9.2 页面 + +| 页面 | 内容 | +|---|---| +| 总览 | 库内数、UP 数、今日新增、增量、容量条、采集成功率、故障提示。 | +| 榜单 | 综合热度、七项单项、增长、增速、互动率、UP 聚合、筛选和 CSV。 | +| 视频详情 | 元数据、七项趋势、快照表、失败标记、同 UP 对比。 | +| UP 页 | UP 的全部视频、聚合趋势和平均互动率。 | +| 粉丝页 | 粉丝总数、日增、7 日净增、累计增长；不叠加视频数据。 | +| 候选池 | 待确认候选、规则命中、剩余保留时间和三个操作。 | +| 系统状态 | 最近扫描、同步、快照、下一次运行时间和错误。 | + +--- + +## 10. 热度与趋势口径 + +### 10.1 综合热度 + +对每项最新有效快照值计算： + +```text +x_i = log1p(metric_i) +z_i = (x_i - median(x)) / (IQR(x) / 1.349) +score = Σ weight_i * z_i +``` + +默认权重： + +```text +view 0.30, like 0.20, coin 0.20, favorite 0.15, +share 0.10, reply 0.03, danmaku 0.02 +``` + +权重从配置读取，改动后查询时重算，不存冗余结果。 + +### 10.2 增长与互动率 + +```text +d7 = (view_today - view_previous) / actual_day_gap +r7 = (view_today - view_previous) / max(view_previous, 1) +interaction_rate = (like + coin + favorite) / max(view, 1) +``` + +- 查询前一有效快照而不是填补缺失日期； +- 缺失日期在图表中断开； +- 增速榜要求播放基数至少 200； +- 互动率榜要求播放至少 100； +- 样本不足两天的视频明确标注，不伪造增长数据。 + +--- + +## 11. 测试与质量要求 + +在引入第三方测试框架前，测试继续使用可直接运行的 Python 脚本： + +```powershell +uv run python tests/test_auth.py +uv run python tests/test_moves.py +``` + +新增模块至少覆盖： + +- 标题归一化、时长解析、分类打分； +- 黑名单只影响发现； +- `infos` 静默缺失资源的识别； +- `active`、`unfaved`、`removed`、`dropped` 状态迁移； +- 数据库迁移与幂等写入； +- 每日快照缺失处理； +- 热度、增长和互动率计算； +- 收藏夹目标选择与排序锚点； +- `fav_action` 的重复点击、失败重试和过期处理； +- Telegram 用户与 chat ID 授权校验。 + +只有当标准库测试维护成本明显过高时，才考虑通过 `uv add --dev pytest` 引入 pytest。 + +--- + +## 12. 实施阶段 + +### P0：数据底座与收藏夹同步 ✅ + +- SQLite 迁移、配置加载、B 站只读客户端； +- 收藏夹首次导入与每小时差集同步； +- 全量 infos 元数据刷新、失效识别、黑名单种子； +- 在 Windows + `uv` 环境完成只读联调。 + +完成情况（2026-10-31）： + +- 已实现 `reaction_board/`：配置热加载、单写队列、`schema_version` 迁移与升级前备份； +- 已实现内置调度器，按 `schedule.folder_sync_minutes` 周期运行收藏夹同步； +- 已实现 `fav/resource/ids`、`fav/resource/infos`、`fav/folder/created/list-all` 只读客户端； +- 首次导入两个年度夹共 831 条资源：795 `active`、36 `removed`（20 条 `attr=9` + 16 条 `infos` 静默缺失），与探测结果一致； +- 增量同步在无变化时只发 2 次 `ids` 请求，不请求 `infos`； +- 黑名单 9 个 mid 已写入 `up_blacklist`，且不参与收藏夹同步过滤； +- 自动化测试见 `tests/test_config.py`、`test_migrations.py`、`test_database.py`、`test_repository.py`、`test_bilibili_client.py`、`test_folder_sync.py`、`test_scheduler.py`、`test_logging.py`、`test_cli.py`。 + +### P1：快照与粉丝追踪 + +- 每日七项视频快照； +- 粉丝快照； +- 失败占位、断点续跑、Cookie 失效告警、SQLite 备份。 + +### P2：本地看板 + +- 总览、榜单、详情、UP、粉丝、候选和系统状态页面； +- 原生前端和 ECharts； +- 本机默认绑定 `127.0.0.1:8080`。 + +### P3：Telegram 与一键加入 + +- long polling、候选通知、收藏/忽略/拉黑； +- `deal`、`sort`、回读验证和失败重试； +- 对真实写入操作使用独立测试视频完成授权验证后再启用。 + +### P4：可选部署 + +- 反向代理、TLS、远程访问鉴权； +- 如有需要再提供 Dockerfile； +- Docker 不作为本机开发、联调或日常运行的前置条件。 + +--- + +## 13. 明确非目标 + +第一期不实现： + +- 多用户、注册、角色权限； +- 自动创建收藏夹或容量满后自动换夹； +- AI 自动判定 reaction； +- UP 白名单； +- 视频历史统计回填； +- Redis、消息队列、微服务、云数据库； +- Node.js 前端工程； +- Docker 作为本地开发依赖。 diff --git a/README.md b/README.md new file mode 100644 index 0000000..c938953 --- /dev/null +++ b/README.md @@ -0,0 +1,221 @@ +# bili-fav + +**咻咻满 Reaction 视频数据看板与收藏夹管理工具。** + +项目以 B 站年度收藏夹为可信视频来源，补充关键词发现和 Telegram 人工确认，持续采集 reaction 视频的七项数据与咻咻满粉丝数，并提供本地数据看板。现有的收藏夹排序工具仍保留并作为一键加入后的排序能力基础。 + +> 当前状态：**P0（数据底座与收藏夹同步）已实施，P1 待开始。** +> +> 当前已可用：扫码授权、收藏夹读取、排序计划生成、执行与校验；SQLite 建库迁移、收藏夹首次导入与增量同步、失效识别、黑名单种子。 +> +> 详细业务口径、接口验证和风险记录见 [project.md](project.md)；开发架构与实施计划见 [DEVELOPMENT.md](DEVELOPMENT.md)。 + +## 功能概览 + +| 功能 | 状态 | 说明 | +|---|---|---| +| 收藏夹排序 | 可用 | 同一 UP 连续，组内按发布时间倒序，组间按该 UP 最早视频发布时间倒序。 | +| 扫码授权与 Cookie 续期 | 可用 | 本地生成二维码；更新身份 Cookie 时保留设备指纹。 | +| 收藏夹同步与首次导入 | 可用（P0） | 监控已配置年度夹，自动发现人工新收藏的视频与失效资源。 | +| 视频日快照 | 规划（P1） | 逐条采集播放、点赞、投币、收藏、转发、评论、弹幕。 | +| 粉丝追踪 | 规划（P1） | 每日记录咻咻满等配置 UP 的粉丝数。 | +| 本地数据看板 | 规划（P2） | 总览、榜单、视频/UP 详情、粉丝曲线、候选池和系统状态。 | +| Telegram 发现与一键加入 | 规划（P3） | 每小时发现候选，支持收藏、忽略、拉黑和收藏夹内定位。 | + +## 核心工作流 + +```text +关键词搜索（补漏） ──► 候选池 / Telegram 人工确认 ──► 收藏夹 + 视频库 + │ +人工加入年度收藏夹 ──► 收藏夹同步（真相源）───────────┤ + ▼ + 每日指标与粉丝快照 + ▼ + 本地看板 +``` + +- **收录范围**：只收录他人观看、试听或点评咻咻满内容的 reaction 视频；最终是否收录由人工确认。 +- **收藏夹是真相源**：手工加入已配置年度夹的视频自动入库；关键词搜索仅负责发现候选。 +- **黑名单仅作用于发现**：已收藏或已入库的黑名单 UP 视频不会被清理，也会继续追踪。 +- **年度夹不自动创建或切换**：同一 UP 的新视频优先进入该 UP 最早视频所在的收藏夹；容量接近上限时预警，失败时告警，不擅自改夹。 +- **数据不回填**：B 站不提供任意日期的历史指标；新视频从首次入库日起开始积累快照。 + +## 本地开发环境 + +项目默认使用 **Python 3.14** 和 [uv](https://docs.astral.sh/uv/)。本地开发、测试和现有排序工具的运行均不依赖 Docker、Node.js、数据库服务或宿主机 cron。 + +```powershell +# 在项目根目录初始化或同步环境 +uv sync --all-groups + +# 确认锁定的解释器版本 +uv run python --version + +# 查看现有 CLI 帮助 +uv run bili-fav help +``` + +完整的 `uv` 使用方式、依赖规范、架构与开发约定见 [DEVELOPMENT.md](DEVELOPMENT.md)。 + +## P0：数据底座与收藏夹同步 + +P0 已实现 SQLite 迁移、配置热加载、B 站只读客户端、收藏夹首次导入与增量同步、`infos` 全量元数据刷新、失效识别、容量预警和黑名单种子。所有请求均为只读，不会修改收藏夹。 + +```powershell +# 建库、执行迁移、写入配置文件夹与黑名单种子（离线） +uv run python app.py init + +# 首次导入或增量同步配置中的年度收藏夹（只读联网） +uv run python app.py sync + +# 按 folder_sync_minutes 常驻周期同步；--once 只跑一次 +uv run python app.py run +uv run python app.py run --once + +# 强制全量刷新元数据（每 6 小时自动执行一次） +uv run python app.py sync --full + +# 查看库内状态（离线） +uv run python app.py status +``` + +- 数据写入 `data/app.db`（SQLite WAL）；升级已有数据库前会自动备份到 `data/backup/`； +- 内置调度器按 `schedule.folder_sync_minutes` 周期运行，`run` 常驻、`sync` 单次； +- `infos` 静默缺失或 `attr=9` 的资源标记为 `removed`，保留历史； +- 从收藏夹移除的资源标记为 `unfaved`，仍保留数据； +- 黑名单只作为发现阶段的种子写入，**不会过滤收藏夹同步**； +- 收藏夹达到 950 条时预警，系统绝不自动换夹。 + +## 配置 + +复制样例配置，并填写 B 站账号信息： + +```powershell +Copy-Item config.example.json config.json +``` + +`config.json` 已被 `.gitignore` 忽略，**不得提交**。其中： + +- `mid` / `cookie` / `csrf`：B 站账号 UID、完整 Cookie 与 `bili_jct`（兼容 `favsort.py` 的扁平写法；也支持 `bilibili` 嵌套对象）； +- `overrides`：可选，为代发或联合投稿视频指定归属 UP； +- `folders`：年度收藏夹配置（`year` + `media_id` + `title`），系统不自动建夹； +- `up_blacklist_seed`：首次启动写入数据库的 UP 黑名单种子； +- `track_fans`：需要每日追踪粉丝数的 UP mid； +- `storage`：数据库与备份目录； +- `schedule` / `notify` / `classify` / `heat_weights`：任务节奏、通知阈值、分类规则和热度权重。 + +完整配置口径以 [project.md](project.md) 和 [DEVELOPMENT.md](DEVELOPMENT.md) 为准。 + +## 当前可用：收藏夹排序 CLI + +所有命令建议通过 `uv run` 执行： + +```powershell +# 查看全部命令；无需 config.json +uv run bili-fav help + +# 扫码登录，更新 config.json 中失效的身份 Cookie +uv run bili-fav auth +uv run bili-fav auth --check + +# 列出本人创建的收藏夹 +uv run bili-fav folders + +# 只读：拉取收藏夹，生成排序计划 +uv run bili-fav plan -v + +# 查看或离线重算已有计划 +uv run bili-fav show +uv run bili-fav replan + +# 写操作：按计划逐步排序；完成后校验 +uv run bili-fav run +uv run bili-fav verify +``` + +`plan` 会生成 `data/plan-.json`，其中含原始顺序、目标顺序、移动脚本及视频元数据缓存。`run` 直接使用该计划，不会重复请求 `infos`；如排序算法变更，可用 `replan` 离线重算。 + +### 排序规则 + +1. **同一 UP 必须连续**：按 `upper.mid` 分组，组内按发布时间倒序；发布时间相同则按 `avid` 升序。 +2. **组间按最早投稿倒序**：取该 UP 在当前收藏夹中最早视频的发布时间作为组键，按组键倒序；并列时按组内最小 `avid` 升序。 +3. **失效或异常资源置底**：保持它们原有的相对顺序。 +4. **可配置归属修正**：`config.json` 的 `overrides` 可为指定视频指定实际归属 UP。 + +> 组间规则是有意设计：某个 UP 只要存在较老视频，即使近期继续投稿，整个分组也会靠后。 + +## 扫码授权 + +当接口返回 `-101` 或 Cookie 即将过期时，运行： + +```powershell +uv run bili-fav auth +uv run bili-fav auth --check +``` + +授权流程在本地生成二维码图片并等待手机客户端扫码确认。成功后： + +- 更新 `SESSDATA`、`bili_jct`、`DedeUserID`、`sid` 等身份字段； +- 保留 `buvid3`、`buvid4`、`_uuid`、`buvid_fp` 等设备指纹； +- 将 `csrf` 同步为新的 `bili_jct`； +- 原子写入 `config.json`，并保留 `config.json.bak`； +- 通过 `x/web-interface/nav` 验证新登录态，失败时自动回滚。 + +常用选项： + +```powershell +uv run bili-fav auth --no-browser # 只生成二维码图片并输出路径 +uv run bili-fav auth --timeout 180 # 修改扫码等待时间（秒） +``` + +## B 站访问与安全约束 + +| 接口 | 当前用途 | 身份 Cookie | +|---|---|---| +| `fav/resource/ids` | 收藏夹资源 ID | 不需要账号身份；可匿名读取。 | +| `fav/resource/infos` | 收藏夹元数据 | 不需要账号身份，但需要设备指纹，完全无 Cookie 时可能返回 HTTP 412。 | +| `fav/folder/created/list-all` | 列出自建收藏夹 | 需要完整登录 Cookie。 | +| `fav/resource/sort` | 收藏夹排序 | 写操作；需要完整 Cookie 和 `csrf`。 | + +现有工具对 `ids` / `infos` 默认剥离账号身份 Cookie，仅保留设备指纹；如果只读请求失败，才使用完整 Cookie 重试。所有写操作使用完整 Cookie 与 CSRF。 + +`x/v3/fav/resource/sort` 的实测语义为： + +```text +sort=A:2:B:2 → 将 B 插到 A 之后 +``` + +运行排序时会重新读取实际顺序、逐步回读校验，并在收藏夹内容于执行期间变动时中止，避免用过期计划强制排序。写操作存在风险，请确认计划后再执行；中断后可直接再次运行 `run` 继续收敛。 + +## 测试 + +测试全部离线运行，外部接口只在 transport 边界用真实响应形状的 Mock 替换： + +```powershell +uv run python tests/test_auth.py # Cookie 合并、设备指纹、Set-Cookie 解析 +uv run python tests/test_moves.py # 排序移动脚本：随机排列与头部用例 +uv run python tests/test_config.py # 配置加载、校验、ENV 覆盖与热加载 +uv run python tests/test_migrations.py # 迁移、备份、回滚与保留期 +uv run python tests/test_database.py # 单写队列、事务与并发 +uv run python tests/test_repository.py # 收藏夹/视频状态机与黑名单 +uv run python tests/test_bilibili_client.py # ids/infos 契约、分块、缺失与重试 +uv run python tests/test_folder_sync.py # 首次导入、差集、失效识别、容量预警 +uv run python tests/test_scheduler.py # 内置调度器到期/失败隔离 +uv run python tests/test_logging.py # JSON 日志脱敏 +uv run python tests/test_cli.py # CLI 离线路径 +``` + +## 运行期文件 + +```text +config.json.bak 最近一次 auth 更新前的配置备份 +data/plan-.json 排序计划、顺序快照和元数据缓存 +data/journal-.jsonl 排序执行日志 +data/app.db P0 起的 SQLite 数据库（收藏夹、视频、黑名单） +data/backup/ 数据库迁移前的自动备份 +``` + +## 文档索引 + +- [project.md](project.md)：完整可行性分析、业务口径、接口验证、数据模型草案和风险清单。 +- [DEVELOPMENT.md](DEVELOPMENT.md)：开发架构、数据库迁移、API、调度、Telegram 与分阶段实施计划。 diff --git a/app.py b/app.py new file mode 100644 index 0000000..e42112b --- /dev/null +++ b/app.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +"""看板服务入口。 + +P0 提供 ``init`` / ``sync`` / ``run`` / ``status`` 子命令；后续阶段会在同一入口启动 +本地 Web 服务。 +""" + +import sys + +from reaction_board.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config.example.json b/config.example.json new file mode 100644 index 0000000..545b949 --- /dev/null +++ b/config.example.json @@ -0,0 +1,87 @@ +{ + "mid": 0, + "csrf": "从 cookie 中的 bili_jct 取值", + "cookie": "粘贴浏览器请求头里的完整 Cookie（需含 SESSDATA、bili_jct、DedeUserID 等）", + "overrides": {}, + "sleep": { + "min": 2.5, + "max": 7.5, + "long_every_min": 20, + "long_every_max": 40, + "long_min": 30, + "long_max": 90, + "retry_wait": 45 + }, + "telegram": { + "enabled": false, + "bot_token": "replace-with-telegram-bot-token", + "chat_id": 0 + }, + "web": { + "host": "127.0.0.1", + "port": 8080, + "auth_token": "replace-with-a-long-random-token", + "public_base_url": "" + }, + "storage": { + "database_path": "data/app.db", + "backup_dir": "data/backup", + "backup_keep_days": 14 + }, + "folders": [ + { "year": 2025, "media_id": 4451053851, "title": "咻咻满-reaction-2025" }, + { "year": 2026, "media_id": 3700023351, "title": "咻咻满 reaction 合集" } + ], + "track_fans": [37754047], + "up_blacklist_seed": [ + 261052419, + 1908537643, + 1701121088, + 542262, + 376316958, + 145178393, + 66389997, + 293976264, + 3026109 + ], + "schedule": { + "discover_minutes": 60, + "folder_sync_minutes": 60, + "full_folder_refresh_hours": 6, + "snapshot_at": "00:01", + "candidate_ttl_days": 7 + }, + "notify": { + "min_score": 2, + "max_items_per_message": 10 + }, + "classify": { + "require_title": "咻咻满", + "positive": [ + "reaction", + "re", + "反应", + "看", + "听", + "初听", + "首次", + "点评", + "锐评", + "路人", + "震惊", + "双人" + ], + "charge": ["充电", "加急", "专场"], + "negative": ["歌切", "切片", "纯享", "搬运", "完整版"], + "duration_boost": { "gt_30min": 1, "gt_60min": 2 } + }, + "heat_weights": { + "view": 0.3, + "like": 0.2, + "coin": 0.2, + "favorite": 0.15, + "share": 0.1, + "reply": 0.03, + "danmaku": 0.02 + } +} diff --git a/favsort.py b/favsort.py new file mode 100644 index 0000000..d351af9 --- /dev/null +++ b/favsort.py @@ -0,0 +1,944 @@ +# -*- coding: utf-8 -*- +""" +bili-fav / favsort —— B 站收藏夹批量重排序工具 + +排序规则 + 1) 同一个 upper 的视频必须连续（聚成一组），组内按发布时间倒序（并列按 avid 升序） + 2) 组间按"该 upper 在本收藏夹中最早视频的发布时间"倒序（并列按该组最小 avid 升序） + 3) 失效/异常资源置于末尾，保持原相对顺序 + 4) config.json 里的 overrides 可把指定视频强制归入另一个 upper（代发等场景） + +指令 + python favsort.py help [子命令] + python favsort.py auth [--check] [--no-browser] [--timeout SEC] + python favsort.py folders + python favsort.py plan [--folder-mid N] [-v] + python favsort.py show + python favsort.py replan + python favsort.py run [--verify-every N] + python favsort.py verify + +鉴权 + auth 会申请扫码登录，在本地生成二维码图片（vendor/qrcodegen.py，纯 Python、MIT， + 无需 pip 安装），用默认看图工具打开给你扫；拿到新的身份 cookie 后只替换 + SESSDATA / bili_jct / DedeUserID 等身份字段，buvid3 / buvid4 / _uuid / buvid_fp + 等设备指纹一律保留。只有设备 cookie 缺失时才用公开的 finger/spi 接口补， + 拿不到的（如 buvid_fp）保持原样。 + ids / infos / folders 为只读请求，默认剥离账号身份 cookie；sort 为写操作， + 始终使用完整 cookie + csrf。 +""" + +import argparse +import json +import os +import random +import shutil +import struct +import sys +import tempfile +import time +import urllib.parse +import urllib.request +import webbrowser +import zlib + +sys.stdout.reconfigure(encoding="utf-8") + +BASE = "https://api.bilibili.com" +PASSPORT = "https://passport.bilibili.com" +ROOT = os.path.dirname(os.path.abspath(__file__)) +DATA_DIR = os.path.join(ROOT, "data") +CONFIG_PATH = os.path.join(ROOT, "config.json") +CONFIG_BACKUP = CONFIG_PATH + ".bak" + +sys.path.insert(0, os.path.join(ROOT, "vendor")) +try: + import qrcodegen # noqa: E402 纯 Python，MIT，见 vendor/qrcodegen.py +except ImportError: # pragma: no cover + qrcodegen = None + +from reaction_board.bilibili.auth import ( # noqa: E402 + IDENTITY_KEYS, + cookie_value, + cookies_from_set_cookie, + gen_uuid, + make_headers, + merge_cookie, + parse_cookie, + sessdata_expiry, + strip_identity, +) + +CHUNK = 20 # infos 接口单次 resources 数量（实测 20 稳定） +API = { + "ids": "{base}/x/v3/fav/resource/ids?media_id={mid}&platform=web", + "infos": "{base}/x/v3/fav/resource/infos?resources={res}&platform=web&folder_mid={fmid}&folder_id={mid}", + "sort": "{base}/x/v3/fav/resource/sort", + "folders": "{base}/x/v3/fav/folder/created/list-all?up_mid={mid}&jsonp=jsonp", + "nav": "{base}/x/web-interface/nav", + "spi": "{base}/x/frontend/finger/spi", + "qr_generate": "{passport}/x/passport-login/web/qrcode/generate?source=main-fe-header", + "qr_poll": "{passport}/x/passport-login/web/qrcode/poll?qrcode_key={key}", +} + +# ---------------------------------------------------------------- config + +DEFAULT_CONFIG = { + "mid": 0, + "csrf": "", + "cookie": "", + "overrides": {}, + "sleep": { + "min": 2.5, + "max": 7.5, + "long_every_min": 20, + "long_every_max": 40, + "long_min": 30, + "long_max": 90, + "retry_wait": 45, + }, +} + + +def load_config(require_auth=True): + if not os.path.exists(CONFIG_PATH): + raise SystemExit( + f"缺少 {CONFIG_PATH}\n" + f"请复制 config.example.json 为 config.json 并填入 cookie / csrf / mid。" + ) + with open(CONFIG_PATH, encoding="utf-8") as f: + cfg = json.load(f) + for k, v in DEFAULT_CONFIG.items(): + cfg.setdefault(k, v) + for k, v in DEFAULT_CONFIG["sleep"].items(): + cfg["sleep"].setdefault(k, v) + if require_auth and (not cfg["cookie"] or not cfg["csrf"]): + raise SystemExit("config.json 中的 cookie / csrf 不能为空（可先运行 python favsort.py auth）") + return cfg + + +# ---------------------------------------------------------------- http + +def _get_json(cfg, url, auth): + req = urllib.request.Request(url, headers=make_headers(cfg, auth=auth)) + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode("utf-8", "replace")) + + +def get_json(cfg, url, auth=False): + """只读请求：默认用去掉账号身份的 cookie。 + + 万一匿名请求被拒（HTTP 错误，或 code != 0），自动带完整 cookie 重试一次， + 保证功能不会因为剥离身份而回退。 + """ + if auth: + return _get_json(cfg, url, auth=True) + try: + r = _get_json(cfg, url, auth=False) + if r.get("code") == 0: + return r + except Exception: # noqa: BLE001 网络错误 / 412 风控都走回退 + pass + return _get_json(cfg, url, auth=True) + + +def post_form(cfg, url, data): + headers = make_headers(cfg, auth=True) + headers["content-type"] = "application/x-www-form-urlencoded" + req = urllib.request.Request( + url, data=urllib.parse.urlencode(data).encode(), headers=headers, method="POST" + ) + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode("utf-8", "replace")) + + +# ---------------------------------------------------------------- cookies + +def ensure_device_cookies(cfg): + """补齐缺失的设备 cookie；已有的一律不动（设备指纹换新反而更容易触发风控）。""" + pairs = parse_cookie(cfg["cookie"]) + lower = {k.lower() for k, _ in pairs} + added = [] + + if "buvid3" not in lower or "buvid4" not in lower: + try: + r = get_json(cfg, API["spi"].format(base=BASE), auth=True) + data = r.get("data") or {} + except Exception: # noqa: BLE001 补不到就算了，保留原有数据 + data = {} + if data.get("b_3") and "buvid3" not in lower: + pairs.append(("buvid3", data["b_3"])) + lower.add("buvid3") + added.append("buvid3") + if data.get("b_4") and "buvid4" not in lower: + pairs.append(("buvid4", data["b_4"])) + lower.add("buvid4") + added.append("buvid4") + + if "_uuid" not in lower: + pairs.append(("_uuid", gen_uuid())) + lower.add("_uuid") + added.append("_uuid") + + if "b_nut" not in lower and "buvid3" in added: + pairs.append(("b_nut", str(int(time.time())))) + added.append("b_nut") + + cfg["cookie"] = "; ".join(f"{k}={v}" for k, v in pairs) + return added + + +def save_config(cfg): + """原子写入 config.json，并留一份 .bak。""" + if os.path.exists(CONFIG_PATH): + shutil.copy2(CONFIG_PATH, CONFIG_BACKUP) + tmp = CONFIG_PATH + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(cfg, f, ensure_ascii=False, indent=2) + f.write("\n") + os.replace(tmp, CONFIG_PATH) + + +# ---------------------------------------------------------------- qr + +def write_qr_png(text, path, scale=8, border=4): + """把文本编码成 QR 并写成 8bit 灰度 PNG（不依赖任何图像库）。""" + if qrcodegen is None: + raise RuntimeError("缺少 vendor/qrcodegen.py") + qr = qrcodegen.QrCode.encode_text(text, qrcodegen.QrCode.Ecc.MEDIUM) + size = qr.get_size() + dim = (size + border * 2) * scale + + rows = [] + for y in range(dim): + my = y // scale - border + row = bytearray([0]) # PNG 行过滤类型 0 + for x in range(dim): + mx = x // scale - border + dark = 0 <= mx < size and 0 <= my < size and qr.get_module(mx, my) + row.append(0 if dark else 255) + rows.append(bytes(row)) + raw = b"".join(rows) + + def chunk(tag, data): + return (struct.pack(">I", len(data)) + tag + data + + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)) + + png = b"\x89PNG\r\n\x1a\n" + png += chunk(b"IHDR", struct.pack(">IIBBBBB", dim, dim, 8, 0, 0, 0, 0)) + png += chunk(b"IDAT", zlib.compress(raw, 9)) + png += chunk(b"IEND", b"") + with open(path, "wb") as f: + f.write(png) + return path + + +def open_path(path): + """用系统默认程序打开文件（Windows 用 startfile，其它平台用 webbrowser）。""" + if os.name == "nt": + os.startfile(path) # type: ignore[attr-defined] + return True + return webbrowser.open("file://" + path) + + +def fetch_ids(cfg, media_id): + r = get_json(cfg, API["ids"].format(base=BASE, mid=media_id)) + if r.get("code") != 0: + raise SystemExit(f"ids 失败: code={r.get('code')} msg={r.get('message')}") + return r.get("data") or [] + + +def fetch_infos(cfg, media_id, folder_mid, id_list): + items, calls = [], 0 + for i in range(0, len(id_list), CHUNK): + chunk = id_list[i:i + CHUNK] + res = ",".join(f"{x['id']}:{x['type']}" for x in chunk) + url = API["infos"].format( + base=BASE, res=urllib.parse.quote(res), fmid=folder_mid, mid=media_id + ) + r = get_json(cfg, url) + calls += 1 + if r.get("code") != 0: + raise SystemExit(f"infos 失败 @{i}: code={r.get('code')} msg={r.get('message')}") + m = {g["id"]: g for g in (r.get("data") or [])} + for c in chunk: + g = m.get(c["id"]) or {} + up = g.get("upper") or {} + items.append({ + "avid": c["id"], + "type": c["type"], + "title": g.get("title") or "(缺失)", + "upper_mid": up.get("mid"), + "upper_name": up.get("name") or "(未知UP)", + "pubtime": g.get("pubtime") or g.get("ctime") or 0, + "attr": g.get("attr", 0), + "found": bool(g), + }) + return items, calls + + +# ---------------------------------------------------------------- core + +def build_target(items, overrides): + applied = [] + for v in items: + w = dict(v) + o = overrides.get(str(v["avid"])) or overrides.get(v["avid"]) + if o: + w["upper_mid"] = o["mid"] + w["upper_name"] = o["name"] + w["overridden"] = True + applied.append(w) + + invalid = [x for x in applied if (not x["found"]) or x["attr"] != 0 or not x["upper_mid"]] + valid = [x for x in applied if x not in invalid] + + groups = {} + for v in valid: + g = groups.setdefault(v["upper_mid"], {"mid": v["upper_mid"], "name": v["upper_name"], "vids": []}) + g["vids"].append(v) + garr = list(groups.values()) + for g in garr: + g["vids"].sort(key=lambda v: (-v["pubtime"], v["avid"])) + g["key"] = min(v["pubtime"] for v in g["vids"]) + g["min_avid"] = min(v["avid"] for v in g["vids"]) + garr.sort(key=lambda g: (-g["key"], g["min_avid"])) + + target = [v for g in garr for v in g["vids"]] + invalid + return applied, garr, target + + +def lcs_keep(cur, tgt): + n = len(cur) + dp = [[0] * (n + 1) for _ in range(n + 1)] + for i in range(n - 1, -1, -1): + for j in range(n - 1, -1, -1): + dp[i][j] = dp[i + 1][j + 1] + 1 if cur[i] == tgt[j] else max(dp[i + 1][j], dp[i][j + 1]) + i = j = 0 + kept = set() + while i < n and j < n: + if cur[i] == tgt[j]: + kept.add(cur[i]); i += 1; j += 1 + elif dp[i + 1][j] >= dp[i][j + 1]: + i += 1 + else: + j += 1 + return kept + + +def _one_pass(work, tgt): + """单轮：保留 LCS 里的元素不动，其余元素逐个插到"目标前驱"之后。 + + 原地修改 work，返回本轮动作。返回后 work 会非常接近 tgt（通常只差 0~5 个位置）， + 由 build_moves 多跑几轮收敛。 + """ + kept = lcs_keep(work, tgt) + pos = {a: i for i, a in enumerate(tgt)} + moves = [] + + # 首位：API 只能"插到某人之后"，要占住第 1 位得走两步 + if work[0] != tgt[0]: + a = tgt[0] + first = work[0] + f = work.index(a) + if f != 1: + work.pop(f) + work.insert(1, a) + moves.append({"moved": a, "pred": first, "from": f + 1, "to": 2}) + k = pos[first] + work.remove(first) + pred = work[k - 1] + work.insert(k, first) + moves.append({"moved": first, "pred": pred, "from": 1, "to": k + 1}) + kept.discard(a) + kept.discard(first) + + for t in range(1, len(tgt)): + a = tgt[t] + if work[t] == a or a in kept: + continue + f = work.index(a) + work.pop(f) + i = work.index(tgt[t - 1]) + work.insert(i + 1, a) + moves.append({"moved": a, "pred": tgt[t - 1], "from": f + 1, "to": i + 2}) + kept.discard(a) + return moves + + +def build_moves(cur, tgt): + """生成单步移动脚本。 + + 服务端语义（实测）：sort=A:2:B:2 => 把 B 插到 A 之后（B 可以往前也可以往后）。 + 最少步数 ≈ n - LCS(cur, tgt)：保留最长公共子序列不动，只搬其余元素。 + 由于"保留元素"需要靠其它元素被搬走才会滑到正确位置，单轮往往差几个位置， + 多跑一两轮即可收敛（实测 658 条约 100 步，正好等于理论下界）。 + """ + work = list(cur) + moves = [] + for _ in range(20): + if work == tgt: + break + part = _one_pass(work, tgt) + if not part: + break + moves.extend(part) + assert work == tgt, "内部错误：生成的动作未能到达目标顺序" + return moves + + +def apply_move(work, m): + work = list(work) + work.remove(m["moved"]) + i = work.index(m["pred"]) + work.insert(i + 1, m["moved"]) + return work + + +# ---------------------------------------------------------------- io + +def ensure_data_dir(): + os.makedirs(DATA_DIR, exist_ok=True) + + +def plan_path(media_id): + return os.path.join(DATA_DIR, f"plan-{media_id}.json") + + +def journal_path(media_id): + return os.path.join(DATA_DIR, f"journal-{media_id}.jsonl") + + +def load_plan(media_id): + p = plan_path(media_id) + if not os.path.exists(p): + raise SystemExit(f"找不到计划文件 {p}，请先运行: python favsort.py plan {media_id}") + with open(p, encoding="utf-8") as f: + return json.load(f) + + +# ---------------------------------------------------------------- commands + +def poll_qr(cfg, qrcode_key): + """轮询扫码状态，同时拿到本轮的 Set-Cookie。""" + url = API["qr_poll"].format(passport=PASSPORT, key=urllib.parse.quote(qrcode_key)) + req = urllib.request.Request(url, headers=make_headers(cfg, auth=False)) + with urllib.request.urlopen(req, timeout=30) as r: + payload = json.loads(r.read().decode("utf-8", "replace")) + return payload, (r.headers.get_all("Set-Cookie") or []) + + +def fetch_nav(cfg): + return get_json(cfg, API["nav"].format(base=BASE), auth=True) + + +def auth_check(cfg): + """只读：检查当前 cookie 是否有效、SESSDATA 何时到期、设备 cookie 是否齐全。""" + device = [k for k in ("buvid3", "buvid4", "_uuid", "buvid_fp") + if cookie_value(cfg["cookie"], k)] + print(f"[auth] 设备 cookie: {', '.join(device) if device else '（缺失）'}") + if not device: + print("[auth] 提示：缺少 buvid3/buvid4 等设备指纹，infos 接口可能返回 412") + + exp = sessdata_expiry(cfg["cookie"]) + if exp: + days = (exp - time.time()) / 86400 + print(f"[auth] SESSDATA 到期: {time.strftime('%Y-%m-%d', time.localtime(exp))}" + f"（剩余 {days:.0f} 天）") + + if not cfg["cookie"]: + print("[auth] 未登录：config.json 里没有 cookie") + return 1 + + try: + r = fetch_nav(cfg) + except Exception as e: # noqa: BLE001 + print(f"[auth] 检查失败: {e}") + return 1 + data = r.get("data") or {} + if r.get("code") != 0 or not data.get("isLogin"): + print(f"[auth] 登录已失效 ❌ code={r.get('code')} {r.get('message')}") + print("[auth] 运行 `python favsort.py auth` 扫码重新登录") + return 1 + + print(f"[auth] 已登录 ✅ mid={data.get('mid')} uname={data.get('uname')}") + if exp and exp < time.time(): + print("[auth] 警告：SESSDATA 时间戳已过期但接口仍返回登录态，建议尽快更新") + return 0 + + +def finish_auth(cfg, pairs): + """合并新身份 cookie -> 补设备 cookie -> 写配置 -> 校验，失败回滚。""" + old_cookie = cfg["cookie"] + merged = merge_cookie(old_cookie, pairs) + if not cookie_value(merged, "SESSDATA") or not cookie_value(merged, "bili_jct"): + print("[auth] 合并后缺少 SESSDATA / bili_jct，中止") + return 2 + + cfg["cookie"] = merged + cfg["csrf"] = cookie_value(merged, "bili_jct") + updated = sorted(k for k in pairs if k.lower() in IDENTITY_KEYS) + print(f"[auth] 登录成功，更新身份字段: {', '.join(updated)}") + + added = ensure_device_cookies(cfg) + if added: + print(f"[auth] 补齐缺失的设备字段: {', '.join(added)}") + kept = [k for k in ("buvid3", "buvid4", "_uuid", "buvid_fp") + if cookie_value(cfg["cookie"], k)] + print(f"[auth] 保留设备指纹: {', '.join(kept) if kept else '（无）'}") + if not cookie_value(cfg["cookie"], "buvid3") or not cookie_value(cfg["cookie"], "buvid4"): + print("[auth] 警告：buvid3/buvid4 仍缺失，infos 接口可能返回 412") + + new_mid = cookie_value(cfg["cookie"], "DedeUserID") + if new_mid and new_mid.isdigit() and int(cfg.get("mid") or 0) != int(new_mid): + print(f"[auth] 注意：登录账号 mid={new_mid}，与 config 里的 mid={cfg.get('mid')} 不同，" + f"已同步为 {new_mid}") + cfg["mid"] = int(new_mid) + + exp = sessdata_expiry(cfg["cookie"]) + if exp: + print(f"[auth] 新 SESSDATA 到期: " + f"{time.strftime('%Y-%m-%d', time.localtime(exp))}") + + save_config(cfg) + print(f"[auth] 已写入 {CONFIG_PATH}（备份 {os.path.basename(CONFIG_BACKUP)}）") + + try: + r = fetch_nav(cfg) + ok = r.get("code") == 0 and (r.get("data") or {}).get("isLogin") + uname = (r.get("data") or {}).get("uname") + except Exception as e: # noqa: BLE001 + ok, uname = False, str(e) + + if not ok: + if os.path.exists(CONFIG_BACKUP): + shutil.copy2(CONFIG_BACKUP, CONFIG_PATH) + print("[auth] 登录校验失败，已回滚 config.json") + else: + print("[auth] 登录校验失败（没有备份可回滚）") + return 4 + + print(f"[auth] 校验通过 ✅ uname={uname}") + return 0 + + +def cmd_auth(cfg, args): + if args.check: + return auth_check(cfg) + + print("[auth] 申请扫码登录 ...") + gen = get_json(cfg, API["qr_generate"].format(passport=PASSPORT), auth=False) + data = gen.get("data") or {} + qrcode_key, url = data.get("qrcode_key"), data.get("url") + if gen.get("code") != 0 or not qrcode_key or not url: + print(f"[auth] 申请二维码失败: code={gen.get('code')} {gen.get('message')}") + return 2 + + qr_path = os.path.join(tempfile.gettempdir(), f"bili-fav-login-{qrcode_key[:8]}.png") + try: + write_qr_png(url, qr_path) + print(f"[auth] 二维码已生成: {qr_path}") + except Exception as e: # noqa: BLE001 + print(f"[auth] 本地生成二维码失败: {e}") + print(f"[auth] 请手动处理这个链接: {url}") + else: + if not args.no_browser: + try: + if open_path(qr_path): + print("[auth] 已用默认看图工具打开二维码") + except Exception as e: # noqa: BLE001 + print(f"[auth] 打开图片失败: {e}，请手动打开上面的路径") + + print(f"[auth] 请在 {args.timeout} 秒内用 B 站客户端扫码并确认（二维码约 180 秒有效）") + sys.stdout.flush() + + deadline = time.time() + args.timeout + last = None + while time.time() < deadline: + try: + payload, set_cookies = poll_qr(cfg, qrcode_key) + except Exception as e: # noqa: BLE001 + print(f"[auth] 轮询异常: {e}") + time.sleep(3) + continue + + data = payload.get("data") or {} + state = data.get("code") + if state is None: + print(f"[auth] 轮询失败: code={payload.get('code')} {payload.get('message')}") + return 2 + + if state == 0: + pairs = cookies_from_set_cookie(set_cookies) + # 老版本会把 cookie 放在跨域 URL 的 query 里，作为兜底保留 + if data.get("url"): + for part in urllib.parse.urlparse(data["url"]).query.split("&"): + k, _, v = part.partition("=") + if k and v: + pairs.setdefault(k, v) + got = "; ".join(f"{k}={v}" for k, v in pairs.items()) + if not cookie_value(got, "SESSDATA") or not cookie_value(got, "bili_jct"): + print("[auth] 扫码成功，但响应里没有 SESSDATA / bili_jct，无法继续") + return 2 + return finish_auth(cfg, pairs) + + if state == 86038: + print("[auth] 二维码已过期，请重新运行 auth") + return 3 + + if state != last: + msg = data.get("message") or { + 86101: "未扫码", + 86090: "已扫码，等待确认", + }.get(state, f"状态 {state}") + print(f"[auth] {msg} ...") + last = state + time.sleep(2) + + print("[auth] 等待超时，二维码已失效，请重新运行 auth") + return 3 + + +def cmd_folders(cfg, args): + # 注意：list-all 是"列出我自己的收藏夹"，匿名调用只会返回公开收藏夹（实测 11 -> 3）， + # 因此这里必须带完整 cookie（仍然只是 GET，不会改动任何数据）。 + r = get_json(cfg, API["folders"].format(base=BASE, mid=cfg["mid"]), auth=True) + if r.get("code") != 0: + raise SystemExit(f"获取收藏夹列表失败: {r.get('code')} {r.get('message')}") + lst = (r.get("data") or {}).get("list") or [] + print(f"共 {len(lst)} 个自建收藏夹（mid={cfg['mid']}）") + for f in sorted(lst, key=lambda x: -x.get("media_count", 0)): + print(f" {f['id']:<12} {f.get('media_count', 0):>5} 条 {f['title']}") + + +def cmd_plan(cfg, args): + media_id = args.media_id + folder_mid = args.folder_mid or cfg["mid"] + ensure_data_dir() + + print(f"[plan] 拉取资源 id (media_id={media_id}) ...") + id_list = fetch_ids(cfg, media_id) + print(f"[plan] {len(id_list)} 条资源，分批拉取详情 ...") + items, calls = fetch_infos(cfg, media_id, folder_mid, id_list) + + applied, garr, target = build_target(items, cfg["overrides"]) + cur = [v["avid"] for v in applied] + tgt = [v["avid"] for v in target] + kept = lcs_keep(cur, tgt) + moves = build_moves(cur, tgt) + + plan = { + "media_id": media_id, + "folder_mid": folder_mid, + "created_at": time.strftime("%Y-%m-%d %H:%M:%S"), + "requests": {"ids": 1, "infos": calls}, + "n": len(cur), + "groups": len(garr), + "lcs": len(kept), + "current_order": cur, + "target_order": tgt, + "moves": moves, + "titles": { + v["avid"]: { + "title": v["title"], + "upper": v["upper_name"], + "pubtime": v["pubtime"], + "overridden": v.get("overridden", False), + } + for v in applied + }, + "group_summary": [ + { + "name": g["name"], + "mid": g["mid"], + "key": g["key"], + "count": len(g["vids"]), + "newest": max(v["pubtime"] for v in g["vids"]), + } + for g in garr + ], + "unchanged": cur == tgt, + } + with open(plan_path(media_id), "w", encoding="utf-8") as f: + json.dump(plan, f, ensure_ascii=False, indent=1) + + print(f"[plan] n={plan['n']} 分组={plan['groups']} 不动={plan['lcs']} " + f"需移动={len(moves)} 已是最优={plan['unchanged']}") + print(f"[plan] 已保存 -> {plan_path(media_id)}") + if moves and args.verbose: + local_titles = {v["avid"]: v["title"] for v in applied} + for i, m in enumerate(moves, 1): + print(f" {i:>4}. {local_titles[m['moved']][:44]:<45} {m['from']:>4} -> {m['to']:<4}") + if moves and not args.verbose: + print("[plan] 加 --verbose 可查看完整移动脚本") + + +def cmd_show(cfg, args): + plan = load_plan(args.media_id) + print(f"media={plan['media_id']} n={plan['n']} 分组={plan['groups']} 移动={len(plan['moves'])}") + print(f"plan 生成于 {plan.get('created_at')}，执行于 {plan.get('applied_at', '未执行')}") + for i, m in enumerate(plan["moves"], 1): + t = plan["titles"][str(m["moved"])] + print(f" {i:>4}. {t['title'][:44]:<45} {m['from']:>4} -> {m['to']:<4}") + + +def cmd_run(cfg, args): + media_id = args.media_id + plan = load_plan(media_id) + target = plan["target_order"] + titles = plan["titles"] + sc = cfg["sleep"] + verify_every = args.verify_every + + print("[run] 读取当前顺序 ...") + actual = [x["id"] for x in fetch_ids(cfg, media_id)] + if actual == target: + print("[run] 顺序已与目标一致，无需操作") + return 0 + + if set(actual) != set(target): + added = set(actual) - set(target) + removed = set(target) - set(actual) + print(f"[run] 收藏夹内容已变化：计划 {len(target)} 条 / 现在 {len(actual)} 条，" + f"新增 {len(added)} 条、移除 {len(removed)} 条") + print("[run] 目标序已过期，请先重新生成计划: python favsort.py plan %d" % media_id) + return 6 + + moves = build_moves(actual, target) + if any(m["pred"] is None for m in moves): + print("[run] 存在需移动到第 1 位（无前驱锚点）的步骤，当前版本不支持") + return 5 + + print(f"[run] 从当前状态重算: {len(moves)} 步 / 目标 {len(target)} 条") + for i, m in enumerate(moves, 1): + t = titles[str(m["moved"])] + print(f" {i:>4}. {t['title'][:40]:<41} {m['from']:>4} -> {m['to']:<4}") + + open(journal_path(media_id), "w", encoding="utf-8").close() + work = list(actual) + total = len(moves) + long_after = random.randint(sc["long_every_min"], sc["long_every_max"]) + + for i, m in enumerate(moves, 1): + body = {"media_id": media_id, "sort": f"{m['pred']}:2:{m['moved']}:2", "csrf": cfg["csrf"]} + print(f"[run] {i}/{total} {titles[str(m['moved'])]['title'][:30]} " + f"-> after {m['pred']}", end=" ", flush=True) + resp = post_form(cfg, API["sort"].format(base=BASE), body) + code = resp.get("code") + print(f"code={code}") + if code != 0: + wait = sc["retry_wait"] + print(f"[run] 返回非 0，冷却 {wait}s 后重试一次") + time.sleep(wait) + resp = post_form(cfg, API["sort"].format(base=BASE), body) + code = resp.get("code") + print(f"[run] retry code={code}") + if code != 0: + print("[run] 重试仍失败，中止（可用 run 续跑）") + return 2 + + work = apply_move(work, m) + with open(journal_path(media_id), "a", encoding="utf-8") as f: + f.write(json.dumps({ + "i": i, "moved": m["moved"], "pred": m["pred"], "to": m["to"], + "code": code, "ts": time.strftime("%Y-%m-%d %H:%M:%S"), + }, ensure_ascii=False) + "\n") + + time.sleep(random.uniform(sc["min"], sc["max"])) + + if i <= 3 or i % verify_every == 0 or i == total: + now = [x["id"] for x in fetch_ids(cfg, media_id)] + if now != work: + print(f"[verify] 第 {i} 步后与模拟不符，停止") + for k in range(min(len(now), len(work))): + if now[k] != work[k]: + print(f" 首个差异 idx={k} 实际={now[k]} 预期={work[k]}") + break + return 3 + print(f"[verify] 第 {i} 步 OK") + time.sleep(random.uniform(sc["min"], sc["max"])) + + if i % long_after == 0 and i != total: + pause = random.uniform(sc["long_min"], sc["long_max"]) + print(f"[run] 长冷却 {pause:.0f}s") + time.sleep(pause) + long_after = random.randint(sc["long_every_min"], sc["long_every_max"]) + + final = [x["id"] for x in fetch_ids(cfg, media_id)] + ok = final == target + print(f"[done] 最终顺序 {'与目标完全一致 ✅' if ok else '与目标不一致 ❌'}") + plan["moves"] = moves + plan["applied_at"] = time.strftime("%Y-%m-%d %H:%M:%S") + with open(plan_path(media_id), "w", encoding="utf-8") as f: + json.dump(plan, f, ensure_ascii=False, indent=1) + if not ok: + for k in range(min(len(final), len(target))): + if final[k] != target[k]: + print(f" 首个差异 idx={k} 实际={final[k]} 目标={target[k]}") + break + return 4 + return 0 + + +def cmd_replan(cfg, args): + """不联网：用 plan 里保存的 current_order / target_order 重新生成移动脚本。""" + media_id = args.media_id + plan = load_plan(media_id) + cur, tgt = plan["current_order"], plan["target_order"] + if cur == tgt: + print("[replan] 原序即目标序，无需移动") + return 0 + kept = lcs_keep(cur, tgt) + moves = build_moves(cur, tgt) + plan["moves"] = moves + plan["lcs"] = len(kept) + plan["replanned_at"] = time.strftime("%Y-%m-%d %H:%M:%S") + with open(plan_path(media_id), "w", encoding="utf-8") as f: + json.dump(plan, f, ensure_ascii=False, indent=1) + print(f"[replan] n={plan['n']} 不动={len(kept)} 理论下界={plan['n'] - len(kept)} " + f"实际步数={len(moves)}") + titles = plan["titles"] + for i, m in enumerate(moves, 1): + t = titles[str(m["moved"])] + print(f" {i:>4}. {t['title'][:40]:<41} {m['from']:>4} -> {m['to']:<4}") + return 0 + + +def cmd_verify(cfg, args): + plan = load_plan(args.media_id) + target = plan["target_order"] + cur = [x["id"] for x in fetch_ids(cfg, args.media_id)] + if cur == target: + print(f"[verify] 与目标顺序完全一致 ✅（n={len(cur)}）") + return 0 + diffs = [i for i in range(min(len(cur), len(target))) if cur[i] != target[i]] + print(f"[verify] 不一致 ❌ 差异位置 {len(diffs)} 处，前 5 处：") + for i in diffs[:5]: + t = plan["titles"] + print(f" idx={i} 当前={t.get(str(cur[i]), {}).get('title', '?')[:36]}") + print(f" 目标={t.get(str(target[i]), {}).get('title', '?')[:36]}") + return 1 + + +# ---------------------------------------------------------------- main + +COMMAND_HELP = { + "auth": "扫码登录并更新 config.json 里失效的身份 cookie（保留设备指纹）", + "folders": "列出自己（config.mid）的自建收藏夹及其 media_id", + "plan": "拉取收藏夹数据并生成排序计划（只读，不改动收藏夹）", + "show": "查看已生成的计划与完整移动脚本（离线）", + "replan": "离线重算移动脚本（不联网，不改目标顺序）", + "run": "按计划逐步执行移动（写操作，每步一个 sort 请求）", + "verify": "对比当前实际顺序与计划目标", + "help": "列出所有子命令与可用选项", +} + + +def build_parser(): + p = argparse.ArgumentParser( + prog="favsort", + description="B 站收藏夹批量重排序：同 UP 聚在一起，组间按该 UP 最早投稿时间倒序。", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", title="子命令", metavar="") + + sub.add_parser("folders", help=COMMAND_HELP["folders"]) + + pa = sub.add_parser("auth", help=COMMAND_HELP["auth"]) + pa.add_argument("--check", action="store_true", + help="只检查当前登录状态与到期时间，不修改配置") + pa.add_argument("--no-browser", action="store_true", + help="不自动打开二维码图片，只打印图片路径") + pa.add_argument("--timeout", type=int, default=180, metavar="SEC", + help="等待扫码的超时时间（默认 180 秒，与二维码有效期一致）") + + pp = sub.add_parser("plan", help=COMMAND_HELP["plan"]) + pp.add_argument("media_id", type=int, help="收藏夹 media_id（数字，见 folders 输出）") + pp.add_argument("--folder-mid", type=int, default=None, metavar="N", + help="收藏夹归属用户 mid（默认取 config.json 的 mid）") + pp.add_argument("--verbose", "-v", action="store_true", help="打印完整移动脚本") + + ps = sub.add_parser("show", help=COMMAND_HELP["show"]) + ps.add_argument("media_id", type=int, help="收藏夹 media_id") + + prp = sub.add_parser("replan", help=COMMAND_HELP["replan"]) + prp.add_argument("media_id", type=int, help="收藏夹 media_id") + + pr = sub.add_parser("run", help=COMMAND_HELP["run"]) + pr.add_argument("media_id", type=int, help="收藏夹 media_id") + pr.add_argument("--verify-every", type=int, default=1, metavar="N", + help="每 N 步回读校验一次（默认 1；前 3 步始终逐步校验）") + + pv = sub.add_parser("verify", help=COMMAND_HELP["verify"]) + pv.add_argument("media_id", type=int, help="收藏夹 media_id") + + ph = sub.add_parser("help", help=COMMAND_HELP["help"]) + ph.add_argument("topic", nargs="?", default=None, metavar="", + help="查看某个子命令的详细帮助，例如: help run") + return p, sub + + +def print_help(parser, sub, topic=None): + """打印全部子命令与选项；指定 topic 时只打印该子命令的详细帮助。""" + if topic: + sp = sub.choices.get(topic) + if sp is None: + print(f"未知子命令: {topic}\n") + print(parser.format_help()) + return 1 + print(sp.format_help()) + return 0 + + print(parser.format_help()) + print("子命令与可用选项：") + for name, sp in sub.choices.items(): + head = sp.format_usage().strip().removeprefix("usage: ") + print(f"\n {head}") + desc = COMMAND_HELP.get(name, "") + if desc: + print(f" {desc}") + for a in sp._actions: # noqa: SLF001 argparse 没有公开接口 + if a.dest == "help": + continue + if a.option_strings: + flags = ", ".join(a.option_strings) + extra = "" if a.nargs == 0 else f" {a.metavar or a.dest.upper()}" + print(f" {flags + extra:<26} {a.help or ''}") + else: + print(f" {a.dest.upper():<26} {a.help or ''}") + print("\n示例：") + print(" python favsort.py auth") + print(" python favsort.py auth --check") + print(" python favsort.py folders") + print(" python favsort.py plan 3700023351 -v") + print(" python favsort.py run 3700023351 --verify-every 25") + print(" python favsort.py verify 3700023351") + print("\n鉴权：auth 扫码后只更新身份 cookie，保留设备指纹；" + "只读请求自动去掉账号身份；run 的 sort 仍使用完整 cookie + csrf。") + return 0 + + +def main(): + p, sub = build_parser() + args = p.parse_args() + + if args.cmd in (None, "help"): + return print_help(p, sub, getattr(args, "topic", None)) + + cfg = load_config(require_auth=args.cmd != "auth") + ensure_data_dir() + + handler = { + "auth": cmd_auth, + "folders": cmd_folders, + "plan": cmd_plan, + "show": cmd_show, + "run": cmd_run, + "verify": cmd_verify, + "replan": cmd_replan, + }[args.cmd] + return handler(cfg, args) or 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/project.md b/project.md new file mode 100644 index 0000000..8dbc060 --- /dev/null +++ b/project.md @@ -0,0 +1,801 @@ +# 咻咻满 Reaction 视频数据看板 —— 可行性分析与技术路线 + +> 状态：**设计定稿；P0（数据底座与收藏夹同步）已实施并完成只读联调** +> 最后更新：2026-10-31（第 5 轮：P0 落地） +> 全部探测均为只读（GET），未调用 `fav/resource/deal`、`fav/resource/sort` + +--- + +## 0. 结论摘要 + +**判定：可行，工程难度中等偏低。** 四个功能点都有稳定可用的官方 Web 接口，且匿名即可取数。 + +| 功能点 | 主力接口 | 实测 | +|---|---|---| +| 视频发现 | `wbi/search/type` | ✅ `code=0`，匿名可用，单关键词上限 800~1000 条 | +| 收藏夹同步（库的真相源） | `fav/resource/ids` + `fav/resource/infos` | ✅ infos **一次 400 条 / 0.2s**，664 条全量 2 次请求 | +| 每日统计 | `view?aid=` | ✅ 7 项指标齐全，0.07~0.14s/条，无风控 | +| 一键加入 | `fav/resource/deal` + `fav/resource/sort` | 排序语义前作已验证；加入待授权实测 | +| 看板 | 本地 SQLite | ✅ 无外部依赖 | + +**本轮确认的设计前提**（来自你的两轮答复）： + +1. 搜索频率 1 次/小时，**只取前 2 页，允许遗漏**——搜索是"代替人工发现 + 即时提醒"，人工加收藏兜底； +2. 搜索结果的**分类靠人工点击确认**（通知即确认），不追求自动判定； +3. **不建 UP 白名单**（reaction UP 多为一次性投稿，收益低），改为**可配置的 UP 黑名单**（专发直播回放/切片的号直接排除，**仅作用于发现，不影响库内已有视频**）； +4. 收藏夹**按年分夹**，但**新建夹由人工完成**，系统只读配置；新视频优先入"同 UP 最早视频所在夹"； +5. **通知渠道先做 Telegram**（支持按钮；开发阶段用 long polling，不需要公网 URL）； +6. 收藏夹是"新增视频来源"的兜底，**日更对象是库内已入库视频**，与年度夹解耦； +7. 用户取消收藏的视频**继续日更**，但允许打标记"移出库"后停止发现与追踪； +8. 新增追踪指标：**咻咻满粉丝数**（只画粉丝曲线，不叠加视频数据）； +9. 所有凭据（Telegram token/chat_id、收藏夹 id、cookie）**统一放配置文件**； +10. 通知提供**收藏 / 忽略 / 拉黑**三个动作，拉黑只作用于"未来发现"。 + +**三个仍需注意的硬约束**： + +- `fav/resource/infos` 只能替代 7 项里的 3 项（play/favorite/danmaku），`reply` 字段恒为 0，且没有 `like/coin/share` → **日更仍必须逐条 `view`**； +- 搜索 `order=pubdate` 排序不稳定（page 20 会跳到 3 月），**不能"翻到旧视频就停"**； +- 收藏夹约 **1000 条/夹**，2026 夹已 664 条，年末约 740 条 → 容量安全但需预警。 + +--- + +## 1. 已确认的需求口径 + +### 1.1 收录范围 + +- **只收「他人 reaction 咻咻满」**：别的 UP 主观看/试听/点评咻咻满的直播、唱歌、充电内容。 +- **不收**：咻咻满本人投稿、纯直播歌切、粉丝切片（除非人工明确加入收藏夹）、搬运、纯翻唱。 +- 判定责任在**人**：搜索只负责"可能相关"的候选，用户点确认才入库。 + +### 1.2 "充电"关键词的真实含义 + +- 不是"充电才能观看的付费视频"； +- 而是很多 reaction UP 把"充电用户点播"的专场投稿标题写成 `【加急充电-咻咻满专场】`、`【咻咻满充电专场①】`，**标题里没有 reaction 字样**； +- 所以 `咻咻满 充电` 是**为了命中这批漏网的 reaction 视频**，实测 537 条结果、近 48h 新增 5 条，确有必要。 +- 实测这些视频 `view` 正常返回 stat，`is_upower_exclusive=False`，不存在取不到数据的问题。 + +### 1.3 收藏夹按年（人工建夹 + 同 UP 归属） + +- `4451053851` = `咻咻满-reaction-2025`（167 条） +- `3700023351` = `咻咻满 reaction 合集`（664 条，实为 2026 年） +- **年度夹只作为配置项**，系统不自动创建；年末人工建夹后把 `media_id` 加进配置。 + +**入哪个夹的判定（按优先级）**： + +1. 该 UP 在库中已有视频 → **入"同 UP 最早视频所在夹"**。 + 例：某 UP 在 2026-12-25 发过视频、在 2026 夹，那么他 2027-01-02 的新视频**仍入 2026 夹**。 +2. 该 UP 在库中无视频 → 入**当前自然年**对应的夹（找不到则用配置里的最新年度夹，并提示"建议新建 2027 夹"）。 + +> 这样同一个 UP 的视频永远在同一个夹里，配合 §7.3 的"同 UP 连续"规则，收藏夹不会因为跨年把一个 UP 拆成两段。 +> **容量处理（不做自动改夹）**：目标夹满了就让它失败——`deal` 返回错误时记 `fav_action.state='failed'` 并通过通知渠道告警（含"请新建年度夹并更新配置"）。可在接近上限（如 `media_count >= 950`）时提前发一次预警，但**系统绝不擅自把视频塞进另一个夹**。 +> **注意**：这只是"收藏夹归属"，`snapshot` 与看板**不按年分表**，全部视频统一查询、统一出榜。 + +### 1.4 两个数据来源的关系 + +``` +人工加收藏（兜底、可信） ──┐ + ├──► video 表（status=active）──► 每日统计 ──► 看板 +搜索候选 + 点击确认（补充） ──┘ +``` + +- **收藏夹是真相源**：用户手工加进去的，默认就是"符合条件"，系统自动入库； +- **搜索是补漏**：只发通知，点确认才入库，同时顺手加进目标收藏夹； +- **日更对象** = 库内 `active` 视频（跨年度夹累计），与收藏夹当前内容解耦； +- **黑名单只挡"发现"**：收藏夹里已有的黑名单 UP 视频一律当普通视频处理，不判断、不排除、也不反向更新黑名单。 + +### 1.5 配置与凭据（统一放配置文件） + +所有可变项与敏感信息都在一个配置文件里（建议 `config.json`，`chmod 600`，`.gitignore` 排除）： + +```jsonc +{ + "bilibili": { + "cookie": "SESSDATA=...; bili_jct=...; buvid3=...; ...", + "csrf": "", + "mid": 28802351 + }, + "telegram": { + "bot_token": "123456:ABC...", + "chat_id": "-1001234567890" + }, + "folders": [ + { "year": 2025, "media_id": 4451053851, "title": "咻咻满-reaction-2025" }, + { "year": 2026, "media_id": 3700023351, "title": "咻咻满 reaction 合集" } + ], + "track_fans": [37754047], + "up_blacklist_seed": [ + 261052419, 1908537643, 1701121088, 542262, 376316958, + 145178393, 66389997, 293976264, 3026109 + ], + "schedule": { "discover_minutes": 60, "folder_sync_minutes": 60, + "snapshot_at": "00:01", "candidate_ttl_days": 7 }, + "notify": { "min_score": 2, "max_items_per_message": 10 }, + "classify": { + "require_title": "咻咻满", + "positive": ["reaction", "re", "反应", "看", "听", "初听", "首次", "点评", "锐评", "路人", "震惊", "双人"], + "charge": ["充电", "加急", "专场"], + "negative": ["歌切", "切片", "纯享", "搬运", "完整版"], + "duration_boost": { "gt_30min": 1, "gt_60min": 2 } + }, + "heat_weights": { "view": 0.30, "like": 0.20, "coin": 0.20, + "favorite": 0.15, "share": 0.10, "reply": 0.03, "danmaku": 0.02 } +} +``` + +- 支持热加载（`SIGHUP` 或每 60 秒检查 mtime），改收藏夹/黑名单不用重启； +- **黑名单本身放数据库**（`up_blacklist` 表），因为要从通知里动态拉黑；配置文件只做初始种子； +- 日志里**永不打印 token / cookie**，启动时只打印"已配置/未配置"； +- 容器里把配置目录挂成**可写卷**（`-v ./config:/config`，权限 600），因为 `auth` 扫码续期后要回写 cookie； +- `python app.py auth` 复用 `bili-fav` 的扫码流程，只替换身份字段、保留设备指纹； +- 可选的 `ENV` 覆盖（`TG_BOT_TOKEN` 等）留给 CI/临时调试，不作为主路径。 + +### 1.6 初始黑名单（已核对昵称，实测 2026-10-31） + +| mid | 昵称 | 粉丝 | 类型（人工判断） | +|---|---|---|---| +| 261052419 | TL二十度灰 | 1,099 | 直播歌切 | +| 1908537643 | 咻咻满在发光 | 7,289 | 粉丝切片号 | +| 1701121088 | TTsssolo | 2,393 | 切片/搬运 | +| 542262 | 咻咻满的表情包 | 16,436 | 切片号 | +| 376316958 | 好心的王二哈 | 2,779 | 切片/直播相关 | +| 145178393 | 雨落柃云 | 1,545 | 切片/搬运 | +| 66389997 | 至尊煎饼 | 216 | 切片/搬运 | +| 293976264 | 宁牙XML | 158 | 切片/搬运 | +| 3026109 | 切片不切丝QAQ | 7,471 | 切片号 | + +- 这 9 个作为 `up_blacklist` 的**初始种子**，首次启动时写入数据库（`INSERT OR IGNORE`）； +- 之后通过通知里的「拉黑」动态追加； +- 已入库/收藏夹里的这些 UP 的视频**不受影响**（§1.4、§3.4）； +- 误拉黑可在看板移除，或删库里的行。 + +--- + +## 2. 接口能力矩阵（2026-10-31 实测） + +| 能力 | 接口 | 需登录 | 实测结果 | 用途 | +|---|---|---|---|---| +| 关键词搜索 | `GET x/web-interface/wbi/search/type` | 否 | ✅ 800~1000 条上限 | 发现 | +| 旧版搜索（降级） | `GET x/web-interface/search/type` | 否 | ✅ `code=0`，结果略少 | 兜底 | +| UP 主搜索 | 同上 `search_type=bili_user` | 否 | ✅ 找到 咻咻满 mid=37754047 | 仅元数据 | +| 空间投稿 | `GET x/space/wbi/arc/search` | 否（需 WBI） | ✅ `code=0` | 本方案不使用 | +| 视频完整统计 | `GET x/web-interface/view?aid=/bvid=` | **否** | ✅ 7 项齐全，0.1s | **日更主力** | +| 收藏夹 ID 列表 | `GET x/v3/fav/resource/ids` | 是 | ✅ 664 条 1 次拿全 | 收藏夹监控 | +| 收藏夹批量详情 | `GET x/v3/fav/resource/infos` | 是 | ✅ **一次 400 条**，0.2s；664 条返回 648（缺 16） | 元数据同步/导入 | +| 加入收藏夹 | `POST x/v3/fav/resource/deal` | 是 + csrf | 文档明确，待实测 | 一键加入 | +| 收藏夹内排序 | `POST x/v3/fav/resource/sort` | 是 + csrf | ✅ 已验证 `A:2:B:2` | 定位 | +| 收藏夹列表 | `GET x/v3/fav/folder/created/list-all` | 是 | ✅ 返回 media_count | 容量监控 | +| 粉丝数 | `GET x/relation/stat?vmid=` | **否** | ✅ `code=0`，匿名可用 | 粉丝追踪 | +| ~~视频统计（旧）~~ | `x/web-interface/archive/stat` | — | ❌ **HTTP 404，已下线** | 不可用 | +| ~~批量 view~~ | `view?aid=1,2` | — | ❌ `-400` | 不可用 | + +### 2.1 `infos` 到底能替代什么（本轮重点实测） + +逐条与 `view` 对照（同一批 6 条视频）： + +| 指标 | infos | view | 是否一致 | +|---|---|---|---| +| 播放 `play` / `view` | ✅ | ✅ | **完全一致**（746/746、1316/1316…） | +| 收藏 `collect` / `favorite` | ✅ | ✅ | **完全一致**（43/43、66/66…） | +| 弹幕 `danmaku` | ✅ | ✅ | **完全一致**（7/7、26/26…） | +| 评论 `reply` | ❌ 恒为 **0** | ✅ 16/24/7/38/37/109 | **字段已失效，不能用** | +| 点赞 `like` | ❌ 无此字段 | ✅ | 缺失 | +| 投币 `coin` | ❌ 无此字段 | ✅ | 缺失 | +| 转发 `share` | ❌ 无此字段 | ✅ | 缺失 | + +**结论：`infos` 不能替代 `archive/stat` 做"7 项日更"。** + +但它是**极好的批处理通道**，有不可替代的价值： + +- 单次 **400 条 / 0.2 秒**（664 条 URL 约 12KB，一次请求 0.2s 返回 648 条）； +- 返回 `upper.mid/name`、`bvid`、`title`、`pubtime`、`ctime`、`duration`、`attr`（失效标记）； +- 664 条收藏的**全量元数据同步只需 2 次请求**（按 200 条分块更稳）。 + +**因此日更拆成两段**： + +1. **元数据 + 3 项指标**（play/favorite/danmaku）：`infos` 批量，2 次请求拿全库 → 秒级； +2. **其余 4 项**（like/coin/share/reply）：`view` 逐条，831 条约 17 分钟。 + +> 注意：第一段不能只跑一次就当作日更，因为两张表口径必须一致；实际做法是**以 `view` 为唯一事实来源**，`infos` 只用于收藏夹同步/导入/校验。若某天 `view` 大面积失败，可用 `infos` 的三项先兜底并标记"部分指标缺失"。 + +### 2.2 `infos` 的两个坑（实测） + +1. **静默丢条**：请求 664 条只返回 648 条，缺失的 16 条是"账号已注销/视频已删"。做差集时**不能用返回条数判断成功**，要 `请求 ids - 返回 ids` 得到缺失集合。 +2. **失效标记**：`attr=9` 表示"已失效视频"，664 条里有 20 条（3.0%）；加上静默缺失的 16 条，共 **36 条 / 664 = 5.4% 已不可用**。日更必须跳过并把状态置为 `removed`，否则每天 36 次无效请求 + 报错。 + +--- + +## 3. 发现环节设计（每小时，前 2 页） + +### 3.1 为什么"前 2 页就够" + +实测（`order=pubdate`）： + +| 关键词 | 第 1 页时间跨度 | 第 2 页覆盖到 | 近 48h 新增 | +|---|---|---|---| +| `咻咻满 reaction` | 10-31 → 10-23 | 10-08 | 6 | +| `咻咻满 充电` | 10-31 → 10-17 | 10-02 | 5 | + +第 1~2 页覆盖 20~30 天，而扫描间隔只有 1 小时，**理论冗余 500 倍**。即使排序抖动导致某条新视频落到第 3 页，下一轮也会被捞到（抖动主要是"旧视频位置漂移"，不是"新视频沉底"）。 + +**代价可接受**：漏掉的由人工加收藏兜底，且收藏夹监控会自动发现。 + +### 3.2 扫描流程 + +``` +每 60 分钟（整点 ± 抖动）： + for kw in ["咻咻满 reaction", "咻咻满 充电"]: # 2 个关键词 + for page in (1, 2): # 每词 2 页 + r = wbi_search(kw, page, order=pubdate) # 共 4 次请求 + for item in r.result: + if item.aid in seen_aid or item.aid in video: continue # 永久去重 + if now - item.pubdate > 14 天: continue # 时间窗，防历史回填 + insert(seen_aid, item.aid) + record(candidate, score=classify(item)) # 进候选池 + if score >= NOTIFY_THRESHOLD: enqueue_notify(item) + sleep(1.5~2.5s jitter) + + # 收藏夹监控（独立任务，见 §4），发现人工新增会立即入库 +``` + +请求量：**每小时 4 次搜索**，一天 96 次。远低于风控阈值（实测 1.5s 间隔连续 15 次无 `-352`）。 + +### 3.3 轻量降噪规则（只决定"要不要发通知"，不决定"要不要入库"） + +``` +# 硬门槛 +title 含 "咻咻满"（全半角/大小写归一化后），否则直接丢弃 +up_mid 命中黑名单 → 直接排除（永不通知，连候选池都不进） + +score = 0 ++2 标题含 reaction|re|反应|看|听|初听|首次|点评|锐评|路人|震惊|双人 ++2 标题含 充电|加急|专场 ++2 时长 > 60 分钟 ++1 时长 > 30 分钟（与上一条不叠加，取大） +-5 标题含 歌切|切片|纯享|搬运|完整版 +-3 UP 为 咻咻满本人（mid=37754047） + +通知阈值 = 2 +0 <= score < 2 且含"咻咻满" → 进看板"候选池"（7 天后删除），不通知 +score < 0 → 丢弃 +``` + +**几条设计依据**： + +- 对现有 664 条收藏回测，**92.6% 的标题含 reaction 类词**，规则召回足够；误判代价只是"多发/少发一条通知"，最终由人点击决定； +- **"直播回放"不再算负向词**：确实有人先直播看、再发直播回放，这类仍是 reaction；改由"UP 黑名单"处理专发回放/切片的号； +- **时长是强正向信号**：实测这批 reaction 视频时长普遍 100~336 分钟（`105:05`、`158:24`、`336:31`），30/60 分钟阈值能把大多数切片（十几分钟）区分开； +- 时长直接取**搜索结果自带的 `duration` 字段**（格式是 `分钟:秒`，超过 60 分钟写作 `105:5`），**不需要额外请求**； +- UP 黑名单在配置文件里维护，可热加载，后续从"误通知"记录里一键加入。 + +### 3.4 通知内容 + +每条候选一条消息，包含： + +- 封面缩略图 + 标题 + UP 名 + 发布时间 + **时长** + 当前播放/点赞 +- 三个动作按钮： + - **「收藏」**：加入目标收藏夹并入库（见 §7） + - **「忽略」**：该 aid 标 `rejected`，不再提醒（同一轮同 UP 的其他候选不受影响） + - **「拉黑」**：把该视频的 **UP 拉入 `up_blacklist`**，同时把该 UP 的其他待决策候选一并标记为 `rejected`；**已入库/收藏夹里的该 UP 视频不受影响** +- 「打开视频」链接（`https://www.bilibili.com/video/`） + +**黑名单语义（明确边界）**： + +1. 只过滤"未来发现"——命中黑名单的 UP 新视频不进候选池、不发通知； +2. **不做回溯清理**：库内已有的、收藏夹里已有的该 UP 视频照常日更、照常上榜； +3. **不从收藏夹反推黑名单**：人工加进收藏夹的行为不会修改黑名单； +4. 可在看板里移除黑名单条目（误拉黑可恢复）。 + +> 同一轮多条候选**合并成一条消息**（避免刷屏），单条消息最多 10 个按钮。 + +**零额外请求**：搜索结果的 item 里已经带 `play / like / favorites / danmaku / review / duration / cover / pubdate / uname / mid`，发通知不需要额外调用 `view`。注意这些是索引值，可能比实时值滞后几分钟到几小时，通知里标注"（索引值）"即可。 + +--- + +## 4. 收藏夹监控（库的真相源） + +### 4.1 为什么必须有 + +- 人工加收藏是最可靠的"确认"动作，系统必须能感知； +- 同时解决"标题/UP 改名、视频失效、用户移出收藏夹"的状态同步。 + +### 4.2 流程（每小时，与搜索同频） + +``` +for folder in 配置的年度夹（2025、2026、…）: + ids = get(ids?media_id=folder) # 1 次，返回全部 aid + diff_add = ids - db.known_ids(folder) + diff_del = db.known_ids(folder) - ids + if diff_add: + for chunk in chunks(diff_add, 200): # 通常 1 次 + infos = get(infos?resources=...) + for it in infos: + upsert(video, source='fav:', status='active') + enqueue_snapshot_today(it.id) # 当天补一次快照 + notify("收藏夹发现 N 条新视频，已自动入库") + if diff_del: + for aid in diff_del: + mark(video.status='removed' or 'unfaved') + # 元数据刷新：每 6 小时用 infos 全量刷一次标题/UP/封面/失效状态 +``` + +- 请求量：2 个夹 × （1 次 ids + 0~1 次 infos）/小时，**基本可忽略**； +- `infos` 返回 `attr=9` 或缺失 → 标记 `removed`，保留历史快照； +- 用户移出收藏夹（`diff_del` 且视频仍可访问）→ 标记 `unfaved`，**数据继续跟踪**（库不因取消收藏而丢历史），只是看板可筛"仅在库/在收藏夹"。 +- 被用户手动"移出库"（`dropped`）的视频若又出现在收藏夹（`diff_add`）→ **自动恢复为 `active`**，并补当天快照；这是唯一会"复活"的状态。 +- 监控的是**所有配置的年度夹**（2025、2026…），所以人工把新视频加进旧夹也能被发现； +- `video.folder_id` 以"实际所在夹"为准，若同一 aid 同时出现在多个夹（极少见），取最新加入的那个并告警。 + +### 4.3 首次导入 + +- `infos` 按 200 条分块，664 条 = 4 次请求，秒级完成； +- 两个夹共 831 条，一次性导入 `video` 表； +- **历史快照无法回填**：B 站不提供任意日期历史数据，只能从部署当天开始积累； +- **开发联调阶段产生的数据一律保留**（不清库、不重建），正式上线直接续用同一个 `app.db`； +- 新发现/新入库的视频从"入库当天"开始积累快照，第一天没有增量是正常的，看板对"仅 1 个数据点"的视频标注"样本不足"。 + +--- + +## 5. 每日数据追踪 + +### 5.1 任务 + +``` +每天 00:01:00（Asia/Shanghai）： + videos = db.where(status='active') # 跨所有年度夹 + for v in videos: + if exists(snapshot[today, v.aid]) and ok: continue # 断点续跑 + r = view(aid=v.aid) # 匿名 + 设备指纹 + if code == 0: upsert(snapshot, today, r.stat) + elif code == -404: v.status='removed' + else: retry(3, backoff); snapshot(ok=0, err=...) + sleep(1.0~1.6s jitter) + 发送完成摘要（总数/成功/失败/耗时） +``` + +### 5.2 容量与耗时 + +| 库规模 | 请求数 | 预估耗时（1.2s/条 + 网络） | +|---|---|---| +| 831（当前） | 831 | 约 17 分钟 | +| 1600（2027 年末） | 1600 | 约 33 分钟 | +| 3000（2029 年） | 3000 | 约 60 分钟 | + +- 00:01 启动，最晚 01:00 前完成，不影响白天使用； +- 若单条 0.1s + sleep 1.2s ≈ 1.3s，实际按 1.2s 估算偏保守。 + +### 5.3 幂等与口径 + +- 唯一键 `(stat_date, aid)`，`INSERT ... ON CONFLICT DO UPDATE`； +- `stat_date` 用本地日期，不用 UTC（避免 00:01 落到前一天）； +- 失败重试 3 次后写 `ok=0` 占位，**不写假数据**；看板显示"缺失"而不是 0； +- 日增量查询时用 `LAG()` 跳过缺失日，不插值。 + +### 5.4 粉丝数追踪（新增） + +``` +每天 00:01 与视频快照同一个任务里，追加 1 次请求： + r = GET x/relation/stat?vmid=37754047 # 匿名可用，实测 0.1s + upsert(up_stat, today, mid=37754047, follower=r.follower, following=r.following) +``` + +- 接口实测：`{"mid":37754047,"following":95,"follower":2767264}`，**匿名`code=0`**，无需登录； +- 唯一键 `(stat_date, mid)`，同样幂等； +- 只加 1 次请求/天，成本可忽略； +- 支持配置多个 mid（`track_fans: [37754047]`），但看板默认只突出咻咻满； +- 粉丝日增 = 今日 - 昨日，缺失日不插值，曲线断开。 + +--- + +## 6. 数据模型（定稿草案） + +```sql +-- 视频主表 +CREATE TABLE video ( + aid INTEGER PRIMARY KEY, + bvid TEXT UNIQUE NOT NULL, + title TEXT NOT NULL, + up_mid INTEGER, + up_name TEXT, + pubdate INTEGER NOT NULL, + duration INTEGER, + cover TEXT, + folder_id INTEGER, -- 来源年度夹（可空=纯搜索确认） + fav_time INTEGER, -- 加入收藏夹时间 + status TEXT NOT NULL, -- active|unfaved|removed|dropped|rejected + source TEXT, -- 'fav:3700023351' | 'search:咻咻满 reaction' + first_seen INTEGER NOT NULL, + last_sync INTEGER, + attr INTEGER +); +CREATE INDEX idx_video_status ON video(status, pubdate DESC); +CREATE INDEX idx_video_up ON video(up_mid, pubdate DESC); + +-- 每日快照（唯一键 = 日期 + 视频） +CREATE TABLE snapshot ( + stat_date TEXT NOT NULL, -- 'YYYY-MM-DD' + aid INTEGER NOT NULL, + view INTEGER, like INTEGER, favorite INTEGER, coin INTEGER, + share INTEGER, reply INTEGER, danmaku INTEGER, + captured_at INTEGER NOT NULL, + ok INTEGER NOT NULL DEFAULT 1, + err TEXT, + src TEXT DEFAULT 'view', -- view | infos（降级时标记） + PRIMARY KEY (stat_date, aid) +); +CREATE INDEX idx_snap_aid ON snapshot(aid, stat_date); + +-- 收藏夹监控状态 +CREATE TABLE folder ( + media_id INTEGER PRIMARY KEY, + year INTEGER NOT NULL, + title TEXT, + media_count INTEGER, + last_ids_at INTEGER, + last_full_at INTEGER, + enabled INTEGER DEFAULT 1 +); + +-- UP 黑名单（专发直播回放/切片等，直接排除，不进候选） +CREATE TABLE up_blacklist ( + mid INTEGER PRIMARY KEY, + name TEXT, + reason TEXT, + created_at INTEGER NOT NULL +); + +-- UP 粉丝/关注数快照（每天一条） +CREATE TABLE up_stat ( + stat_date TEXT NOT NULL, + mid INTEGER NOT NULL, + follower INTEGER, + following INTEGER, + captured_at INTEGER NOT NULL, + ok INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY (stat_date, mid) +); + +-- 候选与扫描流水（漏抓分析 + 去重 + 候选池） +CREATE TABLE candidate ( + aid INTEGER PRIMARY KEY, + bvid TEXT, + title TEXT, + up_mid INTEGER, + up_name TEXT, + pubdate INTEGER, + cover TEXT, + score REAL, + channel TEXT, -- 'search:咻咻满 reaction#p1' + first_seen INTEGER NOT NULL, + notified_at INTEGER, + decision TEXT, -- accepted | rejected | NULL(待定) + decided_at INTEGER +); + +-- 永久去重标记：candidate 删除后仍能防重复通知 +CREATE TABLE seen_aid ( + aid INTEGER PRIMARY KEY, + first_seen INTEGER NOT NULL, + source TEXT +); + +-- 通知与一键加入 +CREATE TABLE notify ( + aid INTEGER NOT NULL, + channel TEXT NOT NULL, + batch_id TEXT, + sent_at INTEGER NOT NULL, + ok INTEGER NOT NULL, + err TEXT, + PRIMARY KEY (aid, channel) +); + +CREATE TABLE fav_action ( + token TEXT PRIMARY KEY, -- HMAC 签名串 + aid INTEGER NOT NULL, + folder_id INTEGER NOT NULL, + state TEXT NOT NULL, -- pending | done | failed | expired + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + done_at INTEGER, + position INTEGER, + err TEXT +); + +CREATE TABLE kv (k TEXT PRIMARY KEY, v TEXT); +``` + +**要点** + +- `status` 是核心状态机： + - `pending` 搜索候选，等确认（只在 `candidate` 表，不进 `video`） + - `active` 在库，参与日更 + - `unfaved` 用户取消收藏，**仍继续日更**，看板可筛 + - `removed` 视频失效（`-404` / `attr=9` / `infos` 缺失），停止请求但保留历史 + - `dropped` 用户手动"移出库"，**停止发现与追踪**；若之后又出现在收藏夹，同步任务会自动改回 `active` + - `rejected` 用户点过"忽略"，不再通知 +- `candidate` 与 `video` 分表：候选可能永远不入库，避免污染主表；候选 `first_seen` 超过 **7 天**且 `decision IS NULL` → **直接删除 candidate 行**（按你的偏好），去重由 `seen_aid` 保证，不会重复通知； +- `seen_aid` 只存 `(aid, first_seen, source)`，每条约 30 字节，按 5 条/天增长，10 年也不到 100KB； +- `up_blacklist` 只影响发现：`folder_sync` **不查**这张表，收藏夹里黑名单 UP 的视频按普通视频入库； +- `folder_id` 的取值遵循 §1.3：优先同 UP 最早视频所在夹，其次当前自然年夹； +- 不预存"日增量"，全部查询时算（历史数据修正后不会自相矛盾）。 + +--- + +## 7. 通知与「一键加入」 + +### 7.1 渠道抽象 + +统一接口 `notify.send(candidates, actions) -> ok`，实现： + +| 优先级 | 渠道 | 按钮能力 | 备注 | +|---|---|---|---| +| **P0** | **Telegram Bot** | ✅ Inline Keyboard | `callback_data` 直接回调，**long polling 即可，不需要公网 URL** | +| P1 | 企业微信机器人 | ⚠️ 仅 markdown 链接 | 不支持按钮，只能点链接 | +| P1 | 邮件 | ⚠️ HTML 链接 | 兼容性最好 | +| P1 | 通用 Webhook | ✅ 自定义 JSON | 给用户自己接（Bark/ntfy/钉钉） | + +**开发阶段只做 Telegram**： + +- Bot 用 `getUpdates` long polling 收 `callback_query`，本机没有公网 IP 也能联调； +- `bot_token` / `chat_id` 从 `config.json` 读（见 §1.5），不写死在代码里； +- **按私聊设计**：`chat_id` 是用户 id（正数），Bot 只服务你一个人；回调 `callback_query` 直接用 `answerCallbackQuery` 回执； +- 上线后服务有域名 + TLS，再补"签名链接"路径给企业微信/邮件/Webhook； +- 两种路径共用同一个 `fav_action` 状态机，只是入口不同（`callback_data` vs `POST /a//confirm`）。 + +### 7.2 链接与防误触 + +``` +通知链接： https:///a/ +token = base64url(aid.folder.exp.nonce) + "." + HMAC-SHA256(secret, payload) +``` + +- `GET /a/`：**只读**，校验签名/过期/未使用 → 渲染确认页（封面、标题、UP、当前数据、加入后位置预览）→ 页面按钮 `POST /a//confirm`； +- `POST`：再次校验 → 调 `deal` → 调 `sort` 定位 → 写 `fav_action` → upsert `video(status='active')` → **删除 candidate 行** → 返回"已加入，位置 #N"； +- 任一步失败（夹满 / csrf 失效 / 风控 / 网络）：`fav_action.state='failed'`、保留 candidate、把错误原文通过通知渠道发回，用户可再点一次； +- 忽略：`candidate.decision='rejected'`，7 天后随候选一起删除； +- 拉黑：写 `up_blacklist` + 该 UP 所有待决策候选 `rejected`，不触碰已有 `video`； +- token **一次性**（成功后作废）、**24 小时过期**、`nonce` 防重放； +- **GET 绝不产生写副作用**，从根上避免 Telegram/微信/钉钉的链接预览预取误写； +- 若某渠道不支持按钮又想要"一步到位"：确认页可用 JS 在加载后自动 POST（预览机器人不执行 JS），但仍保留手动按钮兜底。 + +### 7.3 加入后的位置计算 + +沿用 `bili-fav` 已验证的规则（同一 UP 连续、组内发布时间倒序、组间按该 UP 最早视频时间倒序）： + +``` +target_folder = + 同 UP 在库视频中最早一条的 folder_id # 优先，见 §1.3 + 若没有 → 当前自然年的夹 + 若该年未配置 → 最新年度夹 + 提示"建议新建新夹" + +if 已在夹内: return "已在收藏夹，位置 #N" +anchor = 该 UP 组内比它新的最后一条 + - 若同 UP 组存在：插到"该 UP 最新的那条"之前（即组头） + - 若不存在：新组，按组间规则插到"最早视频时间比它小的第一个组"之前 +调用 deal(add_media_ids=target_folder, rid=aid, type=2, csrf=...) +调用 sort(sort=":2::2") # 把 aid 插到 prev 之后 +校验 ids 序列里 aid 的位置 +``` + +> 举例：某 UP 的旧视频在 2026 夹，他 2027-01-02 的新视频**仍进 2026 夹**，并插在该 UP 组头——这样收藏夹里同一 UP 永远是连续一段，不会跨年断裂。 +> +> **失败处理**：`deal` 报错（夹满/权限/风控）→ `fav_action.state='failed'` + 原样记录错误信息 → 通过通知渠道告知用户 → 视频保持 `candidate` 状态，仍可重试。**不自动换夹、不自动丢弃。** + +**幂等**：加入前先查 `ids` 是否已在夹内；并发/重复点击用 `fav_action.state` 锁。 + +### 7.4 年度夹切换 + +- 配置 `folders: [{year: 2025, media_id: 4451053851}, {year: 2026, media_id: 3700023351}]`； +- **系统不建夹**，年末人工建好后把 `media_id` 加进配置（可热加载，无需重启）； +- 选夹规则见 §1.3（同 UP 最早视频所在夹优先）； +- 12 月中旬开始在看板/通知里提醒"距年末 N 天，请准备下一年度收藏夹"，避免跨年时新 UP 的视频无处可放。 + +--- + +## 8. 热度算法 + +### 8.1 综合热度（存量） + +``` +x_i = log1p(metric_i) +z_i = (x_i - median(x)) / (IQR(x)/1.349) # 稳健标准化 +score = Σ w_i * z_i +w = {view:0.30, like:0.20, coin:0.20, favorite:0.15, + share:0.10, reply:0.03, danmaku:0.02} +``` + +- 权重可配置，看板可切预设（均衡/互动向/传播向）； +- 用中位数 + IQR 而非 max-min，避免单个爆款把其他视频压平； +- 硬币权重高：B 站硬币有每日限额，是强意愿信号； +- 另设"互动率榜"：`(like+coin+fav)/view`，识别小而精。 + +### 8.2 增长榜 + +``` +d7 = (view_today - view_{today-7}) / 实际间隔天数 +r7 = (view_today - view_{today-7}) / max(view_{today-7}, 1) +``` + +| 榜 | 口径 | 基数门槛 | +|---|---|---| +| 累计热度 | §8.1 | — | +| 近 7 天播放增量 | `d7(view)` | — | +| 近 7 天增速 | `r7(view)` | `view >= 200` | +| 互动率 | `(like+coin+fav)/view` | `view >= 100` | +| 单项榜 | 7 项直接排序 | — | +| UP 榜 | 按 `up_mid` 聚合 Σ/AVG | — | + +**冷启动**：不足 2 天数据的视频用"发布后 24h 增量"，并标注"样本不足"；缺失日不插值。 + +### 8.3 趋势 + +- 单视频：7 项折线，缺失日期断开； +- 按 UP：该 UP 全部视频的多线/堆叠； +- 区间：7/30/90/全部。 + +### 8.4 粉丝维度（独立，不参与热度计算） + +``` +fan_delta_1d(d) = follower(d) - follower(d-1) +fan_delta_7d(d) = follower(d) - follower(d-7) +fan_delta_cum = follower(d) - follower(首日) +``` + +- 单独一条曲线（默认近 90 天，可切 7/30/全部）； +- **只画粉丝数**，不叠加 reaction 视频数/播放量（两者与咻咻满本人涨粉没有稳定因果关系）； +- 缺失日不插值，曲线断开并标注； +- 首日没有前值，第一条增量从第 2 天开始。 + +--- + +## 9. 看板 + +### 9.1 页面 + +| 页面 | 内容 | +|---|---| +| 总览 | 库内视频数、活跃 UP 数、今日新增、今日总播放/增量、**收藏夹容量条**（664/1000）、昨日采集成功率 | +| 榜单 | Tab：综合热度/播放/点赞/硬币/收藏/转发/评论/弹幕/增长最快/互动率；筛选 UP、搜标题、导出 CSV | +| 视频详情 | 元数据 + 7 项趋势 + 同 UP 对比 + 快照表（含失败标记）+ 「加入收藏夹」按钮 | +| UP 页 | 该 UP 全部视频、聚合趋势、平均互动率 | +| **粉丝页** | 咻咻满粉丝数曲线 + 每日/7 日净增 + 累计增长（只画粉丝，不叠加视频数据） | +| **候选池** | 未达通知阈值 / 未决策的候选（7 天后自动删除），支持收藏/忽略/拉黑 | +| **系统状态** | 最近扫描、采集成功率、收藏夹同步时间、下次任务时间、错误日志 | + +### 9.2 技术 + +- 单 `index.html` + 原生 JS，无构建链； +- 图表 **vendored ECharts**（Apache-2.0，离线）； +- `/api/*` JSON，前端只渲染； +- 移动端优先（通知点进来多为手机）。 + +### 9.3 鉴权 + +- 公网部署，默认开启：`Authorization: Bearer ` 或一次性登录换 HttpOnly cookie； +- 写操作（加入/忽略）同样校验，CSRF 用签名 token； +- 不做多用户（YAGNI）。 + +--- + +## 10. 部署 + +```dockerfile +FROM python:3.14-slim +WORKDIR /app +COPY . /app +ENV TZ=Asia/Shanghai PYTHONUNBUFFERED=1 +VOLUME ["/data"] +EXPOSE 8080 +HEALTHCHECK CMD python -c "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/api/health')" +CMD ["python", "app.py"] +``` + +- 单容器跑「调度 + Web」，数据卷 `/data/app.db`； +- **不拆多容器、不依赖宿主 cron**：运营量级是"每小时几次请求 + 每天千次请求"，单体进程完全够；调度用内置线程，避免宿主机差异； +- 镜像预计 120~150MB，多架构 amd64/arm64； +- 公网：Nginx/Caddy 反代 → TLS（Caddy 自动证书最省事）； +- 日志 stdout（JSON 行）； +- 备份：每日 `sqlite3 /data/app.db ".backup /data/backup/app-YYYYMMDD.db"`，保留 14 天。 + +**进程结构** + +``` +scheduler 线程：discovery(每 60min) / folder_sync(每 60min) / snapshot(每日 00:01) +web 线程：ThreadingHTTPServer :8080 +storage：SQLite WAL，单写线程 +``` + +--- + +## 11. 风险清单（更新） + +| 风险 | 影响 | 概率 | 缓解 | +|---|---|---|---| +| 搜索排序抖动 + 只取 2 页 | 漏抓个别新视频 | 中 | 每小时扫一次 + 收藏夹监控兜底 + 候选池；业务上可接受 | +| 搜索结果噪声 | 通知太吵 | 中 | 轻量规则降噪 + 人点确认；不达阈值进候选池 | +| 收藏夹 1000 条上限 | 夹满后无法加入 | 中 | 接近 950 时预警；夹满则 `deal` 失败 → **报错并通知**（不自动改夹），由人工新建年度夹并更新配置 | +| 同 UP 归属导致旧夹持续增长 | 2026 夹被 2027 视频撑爆 | **高** | 与上一条同一处置：预警 + 失败告警 + 人工建新夹；不加自动改夹逻辑 | +| 黑名单误伤 | 漏掉正常 reaction | 低 | 黑名单可随时移除；被排除的 UP 不写候选表，但可从日志审计 | +| `infos` 静默丢条 | 误判同步成功 | **高** | 用 `请求 ids - 返回 ids` 判断缺失；缺失即 `removed` | +| `infos.reply` 恒 0 / 缺 like/coin/share | 统计不全 | **确定** | 日更以 `view` 为唯一事实来源，`infos` 只做同步/降级 | +| GET 链接被预览预取 | 误加入收藏夹 | 中 | GET 只渲染页面，写操作走 POST | +| `deal` / `sort` 写失败 | 收藏夹状态不一致 | 中 | 先查 ids 幂等、失败重试、`fav_action` 记录、页面展示失败原因 | +| 视频失效（当前 5.4%） | 日更报错 | **高** | `-404` / `attr=9` / 缺失 → `removed`，不再请求 | +| 接口下线（`archive/stat` 已 404） | 功能中断 | 中 | 接口层抽象 + 降级（旧版 search / infos 三项）+ 监控告警 | +| 风控 `-352/-412` | 采集中断 | 低（实测未见） | 限速 1.2s + 抖动 + 指数退避 + 告警 | +| 跨年切换遗漏 | 新视频进错夹 | 低 | 按 pubdate 选夹 + 1 月 1 日提醒 | +| SQLite 并发 | 写冲突 | 低 | WAL + 单写线程 | +| cookie 过期 | 收藏夹相关接口失败 | 中 | 复用 `bili-fav auth` 扫码续期；`view`/搜索匿名可用作降级 | + +--- + +## 12. 待确认 + +**无。** 三轮问答已覆盖：收录口径、发现策略、收藏夹归属与容量、通知渠道与动作、黑名单语义、候选池生命周期、粉丝追踪、部署形态。 + +进入定稿/实施前只需要你提供**运行期凭据**（不建议写进本文档）： + +- Telegram `bot_token` + 私聊 `chat_id` +- B 站 cookie（也可以先用 `auth` 扫码生成） +- 年度收藏夹 `media_id`（已有 4451053851 / 3700023351） + +--- + +## 13. 验证记录 + +环境：Windows / Python 3.14 / 现有 `bili-fav/config.json`（mid=28802351，SESSDATA 有效至 2027-04-29）。 + +| # | 验证项 | 结果 | +|---|---|---| +| 1 | WBI 关键词搜索 × 4 | 全部 `code=0` | +| 2 | WBI 搜索匿名 | `code=0` 可用 | +| 3 | 旧版 `search/type` | `code=0`，可降级 | +| 4 | UP 主搜索 | 咻咻满 mid=37754047（276.7 万粉） | +| 5 | 空间投稿 | Reaction厂 mid=230020975，251 投稿 | +| 6 | `view` 取 7 项 stat | 完整 | +| 7 | `view` 匿名 / 仅设备指纹 | 均 `code=0` | +| 8 | `archive/stat` | **HTTP 404 已下线** | +| 9 | 批量 `view?aid=1,2` | `-400` 不支持 | +| 10 | 连续 12 次 `view`（0.6s 间隔） | 无风控，0.07~0.14s/条 | +| 11 | 搜索分页深度 | 上限 ~1000（50 页），排序非严格时间序 | +| 12 | 收藏夹容量 | 3700023351=664、4451053851=167 | +| 13 | 充电专场视频 stat | 正常，非付费墙 | +| 14 | 664 条收藏 UP 分布 | 392 个 UP；269 个只投 1 条；Top100 覆盖 52.6% | +| 15 | 664 条标题规则回测 | 92.6% 含 reaction 类词；4.7% 含切片类词 | +| 16 | **`infos` 批量上限** | **400 条/次 OK（0.2s）**；664 条一次请求 URL 12KB，返回 648 条 | +| 17 | **`infos` vs `view` 对照** | play/favorite/danmaku **完全一致**；`reply` **恒 0**；无 like/coin/share | +| 18 | **失效判定** | `attr=9` 20 条 + 静默缺失 16 条 = **36/664（5.4%）** | +| 19 | `infos` 200 条分块 | `code=0`，稳定 | +| 20 | **粉丝数接口** `x/relation/stat` | `code=0`，匿名可用，`follower=2767264` | +| 21 | **搜索 item 的 duration 格式** | `分钟:秒`（`105:5`、`336:31`），超 60 分钟不转小时 | +| 22 | **搜索 item 自带统计字段** | `play/like/favorites/danmaku/review/duration/cover/pubdate/mid/uname` 齐全，通知可用索引值 | +| 23 | **初始黑名单 9 个 mid 核对** | `x/web-interface/card` 全部 `code=0`，昵称/粉丝数已记录（§1.6） | + +**未验证（需授权或外部配合）**： + +- `fav/resource/deal` 实际加入 + `sort` 定位（会写收藏夹）； +- 手机扫码 `auth` 完整链路； +- 各通知渠道的按钮/链接实际表现； +- 长期（>72h）每小时扫描的风控表现； +- 收藏夹真实上限是否 1000。 + +--- + +## 14. 下一步 + +所有决策点已确认（§12 无遗留）。下一步是**方案定稿**，产出： + +1. 最终 DDL + 迁移脚本设计（含 `up_stat`、`up_blacklist`）； +2. 四个任务的调度/限速/重试/幂等伪代码（发现 / 收藏夹同步 / 快照 / 粉丝）； +3. 分类规则 v1（含时长权重、UP 黑名单）+ 用 831 条收藏回测方案； +4. 热度算法参数表与榜单口径； +5. 前端线框 + API 契约（含粉丝页、候选池页）； +6. 一键加入时序图（Telegram callback 与 签名链接 两条入口）； +7. 分阶段实施计划： + - P0 收藏夹同步 + 导入 + 黑名单种子（先把库建起来） + - P1 每日快照 + 粉丝追踪（开始积累数据） + - P2 看板（榜单/详情/粉丝页） + - P3 Telegram 通知 + 一键加入 + - P4 容器化 + 反代 + 备份 + +> 当前**不写实现代码**，等明确进入下一阶段。 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bb4c530 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "bili-fav" +version = "0.1.0" +description = "Bilibili favorite-folder tools and Reaction video dashboard" +readme = "README.md" +requires-python = ">=3.14,<3.15" +license = { text = "MIT" } +authors = [{ name = "bili-fav contributors" }] +dependencies = [] + +[project.scripts] +bili-fav = "favsort:main" +reaction-board = "reaction_board.cli:main" + +[dependency-groups] +dev = [] + +[tool.hatch.build.targets.wheel] +only-include = ["favsort.py", "app.py", "vendor", "reaction_board"] diff --git a/reaction_board/__init__.py b/reaction_board/__init__.py new file mode 100644 index 0000000..9b60e27 --- /dev/null +++ b/reaction_board/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +"""Reaction video dashboard backend package.""" diff --git a/reaction_board/bilibili/__init__.py b/reaction_board/bilibili/__init__.py new file mode 100644 index 0000000..8ca57a6 --- /dev/null +++ b/reaction_board/bilibili/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +"""B 站接口访问层。""" diff --git a/reaction_board/bilibili/auth.py b/reaction_board/bilibili/auth.py new file mode 100644 index 0000000..069abf1 --- /dev/null +++ b/reaction_board/bilibili/auth.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +"""B 站 Cookie 与请求头工具。 + +从 ``favsort.py`` 抽取，供现有排序工具和看板共用，避免出现两套不一致的 +身份 Cookie / 设备指纹处理逻辑。 +""" + +from __future__ import annotations + +import random +import urllib.parse +import uuid + +IDENTITY_KEYS = { + "sessdata", + "bili_jct", + "dedeuserid", + "dedeuserid__ckmd5", + "bili_ticket", + "bili_ticket_expires", + "sid", + "b_lsid", +} + +DROP_STALE = {"sid", "b_lsid"} # 会话级字段，刷新时应丢弃旧值 + + +def strip_identity(cookie: str) -> str: + """剥离账号身份类 cookie，保留 buvid3 / buvid4 / _uuid 等设备指纹。 + + 实测结论： + - ids：完全无 cookie 也能读（连私密收藏夹都返回全量）； + - infos：不需要账号身份，但缺设备指纹 cookie 会被 HTTP 412 拦； + - sort：必须带完整 cookie + csrf，不能走这里。 + """ + kept = [] + for part in (cookie or "").split(";"): + part = part.strip() + if not part: + continue + key = part.split("=", 1)[0].strip().lower() + if key in IDENTITY_KEYS: + continue + kept.append(part) + return "; ".join(kept) + + +def make_headers(cfg: dict, auth: bool = False) -> dict[str, str]: + """构造 B 站 Web 请求头；``auth=False`` 时去掉账号身份 cookie。""" + return { + "accept": "*/*", + "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7", + "cookie": cfg["cookie"] if auth else strip_identity(cfg["cookie"]), + "dnt": "1", + "origin": "https://www.bilibili.com", + "referer": "https://www.bilibili.com/", + "sec-ch-ua": '"Google Chrome";v="155", "Chromium";v="155", "Not(A:Brand";v="24"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-site", + "user-agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36" + ), + } + + +def parse_cookie(cookie: str) -> list[tuple[str, str]]: + """把 cookie 字符串解析成 [(key, value)]，保留原顺序。""" + pairs = [] + for part in (cookie or "").split(";"): + part = part.strip() + if not part or "=" not in part: + continue + k, v = part.split("=", 1) + pairs.append((k.strip(), v.strip())) + return pairs + + +def cookie_value(cookie: str, name: str) -> str: + name = name.lower() + for k, v in parse_cookie(cookie): + if k.lower() == name: + return v + return "" + + +def cookies_from_set_cookie(headers) -> dict[str, str]: + """从 Set-Cookie 头里取 name=value（忽略 Path/Domain/Expires 等属性）。""" + pairs = {} + for h in headers or []: + first = h.split(";", 1)[0].strip() + if "=" not in first: + continue + k, v = first.split("=", 1) + pairs[k.strip()] = v.strip() + return pairs + + +def merge_cookie(old: str, new_pairs: dict[str, str]) -> str: + """只替换身份类 cookie，其它字段（设备指纹等）原样保留。 + + 新值里没有的身份字段不会被删除（例如服务端没下发新的 sid 时保留旧值）， + 只有 DROP_STALE 里的会话字段会在刷新时清掉。 + """ + new_l = { + k.lower(): (k, v) + for k, v in new_pairs.items() + if k.lower() in IDENTITY_KEYS + } + out, seen = [], set() + for k, v in parse_cookie(old): + kl = k.lower() + if kl in new_l: + nk, nv = new_l[kl] + out.append((nk, nv)) + seen.add(kl) + elif kl in DROP_STALE: + continue + else: + out.append((k, v)) + seen.add(kl) + for kl, (k, v) in new_l.items(): + if kl not in seen: + out.append((k, v)) + return "; ".join(f"{k}={v}" for k, v in out) + + +def gen_uuid() -> str: + """生成 B 站风格的 _uuid（前端本来就是本地生成的，服务端不下发）。""" + suffix = "".join(random.choices("0123456789ABCDEF", k=7)) + return f"{str(uuid.uuid4()).upper()}{suffix}infoc" + + +def sessdata_expiry(cookie: str) -> int | None: + """SESSDATA 形如 <值>%2C<过期时间戳>%2C<校验>，取中间的时间戳。""" + value = urllib.parse.unquote(cookie_value(cookie, "SESSDATA")) + parts = value.split(",") + if len(parts) >= 2 and parts[1].isdigit(): + return int(parts[1]) + return None diff --git a/reaction_board/bilibili/client.py b/reaction_board/bilibili/client.py new file mode 100644 index 0000000..5a2b9bd --- /dev/null +++ b/reaction_board/bilibili/client.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +"""只读 B 站 Web API 客户端（P0 范围）。 + +P0 只需要收藏夹读取能力：``ids`` / ``infos`` / ``folder/created/list-all``。 +所有网络访问都经过 ``Transport`` 边界，测试可直接替换该边界，而不需要访问 +真实 B 站。写操作（``deal`` / ``sort``）不在本模块实现。 +""" + +from __future__ import annotations + +import json +import random +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any, Mapping, Protocol, Sequence + +from .auth import make_headers +from .models import FavId, FavResource, FolderInfo, InfosResult + +BASE = "https://api.bilibili.com" +DEFAULT_CHUNK = 200 +MAX_CHUNK = 400 + + +class BilibiliError(RuntimeError): + """B 站访问层基础异常。""" + + +class TransportError(BilibiliError): + """网络层错误，可重试。""" + + retryable = True + + +class BilibiliHTTPError(BilibiliError): + def __init__(self, status: int, message: str = "") -> None: + self.status = status + self.message = message + self.retryable = status >= 500 or status == 429 + super().__init__(f"HTTP {status}: {message}".rstrip()) + + def __str__(self) -> str: + return f"HTTP {self.status}: {self.message}".rstrip(": ") + + +class BilibiliAPIError(BilibiliError): + def __init__(self, code: int, message: str = "") -> None: + self.code = code + self.message = message + super().__init__(f"B 站接口业务错误 code={code}: {message}".rstrip(": ")) + + def __str__(self) -> str: + return f"B 站接口业务错误 code={self.code}: {self.message}".rstrip(": ") + + +class BilibiliProtocolError(BilibiliError): + """响应结构与已知契约不一致。""" + + +@dataclass(frozen=True) +class HTTPResponse: + status: int + headers: Mapping[str, str] + body: bytes + + def json(self) -> Any: + try: + return json.loads(self.body.decode("utf-8", "replace")) + except json.JSONDecodeError as exc: + raise BilibiliProtocolError(f"响应不是合法 JSON: {exc}") from exc + + +class Transport(Protocol): + def request( + self, + method: str, + url: str, + headers: Mapping[str, str], + data: bytes | None = None, + timeout: float = 30, + ) -> HTTPResponse: ... + + +class UrllibTransport: + """基于 ``urllib.request`` 的默认传输实现。""" + + def request( + self, + method: str, + url: str, + headers: Mapping[str, str], + data: bytes | None = None, + timeout: float = 30, + ) -> HTTPResponse: + req = urllib.request.Request( + url, data=data, headers=dict(headers), method=method + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return HTTPResponse( + status=int(resp.status), + headers=dict(resp.headers.items()), + body=resp.read(), + ) + except urllib.error.HTTPError as exc: + try: + body = exc.read() + except Exception: # noqa: BLE001 + body = b"" + return HTTPResponse( + status=int(exc.code), + headers=dict(exc.headers.items()) if exc.headers else {}, + body=body, + ) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise TransportError(str(exc)) from exc + + +class Pacer: + """请求间随机等待；第一个请求不等待。""" + + def __init__( + self, + min_delay: float = 1.5, + max_delay: float = 2.5, + *, + sleep=time.sleep, + rng: random.Random | None = None, + ) -> None: + if min_delay < 0 or max_delay < min_delay: + raise ValueError("非法的请求间隔") + self.min_delay = min_delay + self.max_delay = max_delay + self._sleep = sleep + self._rng = rng or random.Random() + self._first = True + + def wait(self) -> float: + if self._first: + self._first = False + return 0.0 + delay = self._rng.uniform(self.min_delay, self.max_delay) + if delay > 0: + self._sleep(delay) + return delay + + +def _chunks(seq: Sequence[int], size: int): + for index in range(0, len(seq), size): + yield seq[index:index + size] + + +class BilibiliClient: + """只读收藏夹 API 客户端。""" + + def __init__( + self, + *, + cookie: str = "", + csrf: str = "", + mid: int = 0, + transport: Transport | None = None, + pacer: Pacer | None = None, + sleeper=time.sleep, + retries: int = 3, + backoff: float = 1.0, + timeout: float = 30, + logger=None, + ) -> None: + if retries < 1: + raise ValueError("retries 必须 >= 1") + self.cookie = cookie or "" + self.csrf = csrf or "" + self.mid = mid + self.transport = transport or UrllibTransport() + self.pacer = pacer or Pacer(sleep=sleeper) + self._sleeper = sleeper + self.retries = retries + self.backoff = backoff + self.timeout = timeout + self.logger = logger + + # ------------------------------------------------------------ core + + def _headers(self, auth: bool) -> dict[str, str]: + return make_headers({"cookie": self.cookie}, auth=auth) + + def _request_json(self, url: str, *, auth: bool) -> dict[str, Any]: + self.pacer.wait() + resp = self.transport.request( + "GET", url, self._headers(auth), data=None, timeout=self.timeout + ) + if resp.status >= 400: + raise BilibiliHTTPError(resp.status, resp.body.decode("utf-8", "replace")[:200]) + payload = resp.json() + if not isinstance(payload, dict): + raise BilibiliProtocolError("接口响应顶层不是 JSON 对象") + return payload + + def _request_with_retry(self, url: str, *, auth: bool) -> dict[str, Any]: + last: BilibiliError | None = None + for attempt in range(self.retries): + try: + return self._request_json(url, auth=auth) + except (TransportError, BilibiliHTTPError) as exc: + last = exc + if not exc.retryable or attempt >= self.retries - 1: + raise + delay = self.backoff * (2 ** attempt) + if self.logger is not None: + self.logger.warning( + "bili.retry", + attempt=attempt + 1, + delay=delay, + status=getattr(exc, "status", None), + ) + if delay > 0: + self._sleeper(delay) + raise last or TransportError("未知网络错误") + + @staticmethod + def _api_error(payload: Mapping[str, Any]) -> BilibiliAPIError: + code = payload.get("code") + if isinstance(code, bool) or not isinstance(code, int): + raise BilibiliProtocolError("接口响应缺少整数 code") + return BilibiliAPIError(code, str(payload.get("message") or "")) + + def get_json(self, url: str, *, auth: bool = False) -> dict[str, Any]: + """GET JSON；匿名失败时自动带完整 Cookie 重试一次（与 favsort 一致）。""" + if auth: + payload = self._request_with_retry(url, auth=True) + if payload.get("code") != 0: + raise self._api_error(payload) + return payload + + anon_error: BilibiliError | None = None + try: + payload = self._request_with_retry(url, auth=False) + if payload.get("code") == 0: + return payload + anon_error = self._api_error(payload) + except BilibiliError as exc: + anon_error = exc + + if self.logger is not None and anon_error is not None: + self.logger.debug("bili.anon_fallback", error=str(anon_error)) + payload = self._request_with_retry(url, auth=True) + if payload.get("code") != 0: + raise self._api_error(payload) + return payload + + # ------------------------------------------------------------ fav + + def fav_ids(self, media_id: int) -> list[FavId]: + url = ( + f"{BASE}/x/v3/fav/resource/ids?media_id={int(media_id)}&platform=web" + ) + payload = self.get_json(url) + data = payload.get("data") + if data is None: + data = [] + if not isinstance(data, list): + raise BilibiliProtocolError("fav/resource/ids data 不是数组") + out: list[FavId] = [] + for index, item in enumerate(data): + if not isinstance(item, Mapping): + raise BilibiliProtocolError(f"fav/resource/ids data[{index}] 不是对象") + aid = item.get("id") + if isinstance(aid, bool) or not isinstance(aid, int): + raise BilibiliProtocolError(f"fav/resource/ids data[{index}].id 不是整数") + raw_type = item.get("type", 2) + if isinstance(raw_type, bool) or not isinstance(raw_type, int): + raise BilibiliProtocolError(f"fav/resource/ids data[{index}].type 不是整数") + bvid = item.get("bvid") or item.get("bv_id") + if not isinstance(bvid, str) or not bvid: + bvid = None + out.append(FavId(aid=aid, type=raw_type, bvid=bvid)) + return out + + def fav_infos( + self, + media_id: int, + aids: Sequence[int], + *, + folder_mid: int | None = None, + chunk_size: int = DEFAULT_CHUNK, + ) -> InfosResult: + if not 1 <= chunk_size <= MAX_CHUNK: + raise ValueError(f"chunk_size 必须在 1..{MAX_CHUNK} 之间") + requested: list[int] = [] + seen: set[int] = set() + for raw in aids: + aid = int(raw) + if aid not in seen: + seen.add(aid) + requested.append(aid) + if not requested: + return InfosResult(requested=(), items={}, missing=()) + + fmid = self.mid if folder_mid is None else int(folder_mid) + items: dict[int, FavResource] = {} + for chunk in _chunks(requested, chunk_size): + resources = ",".join(f"{aid}:2" for aid in chunk) + url = ( + f"{BASE}/x/v3/fav/resource/infos" + f"?resources={urllib.parse.quote(resources)}" + f"&platform=web&folder_mid={fmid}&folder_id={int(media_id)}" + ) + payload = self.get_json(url) + data = payload.get("data") + if data is None: + data = [] + if not isinstance(data, list): + raise BilibiliProtocolError("fav/resource/infos data 不是数组") + chunk_set = set(chunk) + for item in data: + if not isinstance(item, Mapping): + raise BilibiliProtocolError("fav/resource/infos item 不是对象") + try: + parsed = FavResource.from_api(item) + except (TypeError, ValueError) as exc: + raise BilibiliProtocolError(f"fav/resource/infos 解析失败: {exc}") from exc + if parsed.aid not in chunk_set: + raise BilibiliProtocolError( + f"fav/resource/infos 返回了未请求的 aid={parsed.aid}" + ) + items[parsed.aid] = parsed + + missing = tuple(aid for aid in requested if aid not in items) + return InfosResult(requested=tuple(requested), items=items, missing=missing) + + def fav_folders(self, up_mid: int | None = None) -> list[FolderInfo]: + mid = self.mid if up_mid is None else int(up_mid) + url = f"{BASE}/x/v3/fav/folder/created/list-all?up_mid={mid}&jsonp=jsonp" + payload = self.get_json(url, auth=True) + data = payload.get("data") or {} + if not isinstance(data, Mapping): + raise BilibiliProtocolError("fav/folder/created/list-all data 不是对象") + rows = data.get("list") + if rows is None: + return [] + if not isinstance(rows, list): + raise BilibiliProtocolError("fav/folder/created/list-all data.list 不是数组") + out: list[FolderInfo] = [] + for index, row in enumerate(rows): + if not isinstance(row, Mapping): + raise BilibiliProtocolError(f"folder list[{index}] 不是对象") + media_id = row.get("id") + if isinstance(media_id, bool) or not isinstance(media_id, int): + raise BilibiliProtocolError(f"folder list[{index}].id 不是整数") + count = row.get("media_count", 0) + if isinstance(count, bool) or not isinstance(count, int): + count = 0 + out.append(FolderInfo( + media_id=media_id, + title=str(row.get("title") or ""), + media_count=count, + )) + return out diff --git a/reaction_board/bilibili/models.py b/reaction_board/bilibili/models.py new file mode 100644 index 0000000..d55bc83 --- /dev/null +++ b/reaction_board/bilibili/models.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""B 站收藏夹接口的返回模型。""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + + +@dataclass(frozen=True) +class FavId: + """``fav/resource/ids`` 的一条资源。""" + + aid: int + type: int + bvid: str | None = None + + +@dataclass(frozen=True) +class FavResource: + """``fav/resource/infos`` 的一条视频元数据。""" + + aid: int + bvid: str | None + title: str + up_mid: int | None + up_name: str | None + pubdate: int + duration: int | None + cover: str | None + attr: int + fav_time: int | None + + @property + def invalid(self) -> bool: + # 实测 attr=9 表示"已失效视频"。 + return self.attr == 9 + + @classmethod + def from_api(cls, item: Mapping[str, Any]) -> "FavResource": + aid = item.get("id") + if isinstance(aid, bool) or not isinstance(aid, int): + raise ValueError("infos item.id 不是整数") + upper = item.get("upper") or {} + if not isinstance(upper, Mapping): + upper = {} + up_mid = upper.get("mid") + if isinstance(up_mid, bool) or not isinstance(up_mid, int): + up_mid = None + raw_duration = item.get("duration") + duration = raw_duration if isinstance(raw_duration, int) and not isinstance( + raw_duration, bool + ) else None + raw_attr = item.get("attr", 0) + attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0 + fav_time = item.get("fav_time") + if isinstance(fav_time, bool) or not isinstance(fav_time, int) or fav_time <= 0: + fav_time = None + bvid = item.get("bvid") or item.get("bv_id") + if not isinstance(bvid, str) or not bvid: + bvid = None + return cls( + aid=aid, + bvid=bvid, + title=str(item.get("title") or ""), + up_mid=up_mid, + up_name=str(upper.get("name") or "") or None, + pubdate=int(item.get("pubtime") or item.get("ctime") or 0), + duration=duration, + cover=str(item.get("cover") or "") or None, + attr=attr, + fav_time=fav_time, + ) + + +@dataclass(frozen=True) +class FolderInfo: + media_id: int + title: str + media_count: int + + +@dataclass(frozen=True) +class InfosResult: + requested: tuple[int, ...] + items: dict[int, FavResource] + missing: tuple[int, ...] diff --git a/reaction_board/cli.py b/reaction_board/cli.py new file mode 100644 index 0000000..3af4a1c --- /dev/null +++ b/reaction_board/cli.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +"""P0 命令行入口。 + + python app.py init # 建库 / 迁移 / 写入黑名单种子（离线） + python app.py sync # 收藏夹只读同步（首次导入或增量） + python app.py run # 按 folder_sync_minutes 常驻周期同步（--once 只跑一次） + python app.py status # 查看库内状态（离线） + +P2 起 ``python app.py`` 将在此基础上启动本地看板服务。 +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Sequence + +from .bilibili.client import BilibiliClient +from .config import ConfigError, ConfigStore +from .database import Database +from .logging_json import JsonLogger +from .repository import Repository +from .scheduler import ScheduledTask, Scheduler +from .services.folder_sync import FolderSyncService + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_CONFIG = ROOT / "config.json" + + +def _build_parser() -> argparse.ArgumentParser: + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--config", + default=str(DEFAULT_CONFIG), + help="配置文件路径（默认项目根目录 config.json）", + ) + + parser = argparse.ArgumentParser( + prog="app.py", + description="咻咻满 Reaction 数据看板 —— P0 数据底座", + ) + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子") + sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹") + sync.add_argument( + "--full", action="store_true", help="强制全量刷新元数据（忽略 6 小时周期）" + ) + run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行收藏夹同步") + run.add_argument( + "--once", action="store_true", help="只执行一次到期任务后退出" + ) + sub.add_parser("status", parents=[common], help="查看数据库状态（离线）") + return parser + + +def _open_db(cfg, logger: JsonLogger) -> Database: + db = Database( + cfg.storage.database_path, + backup_dir=cfg.storage.backup_dir, + backup_keep_days=cfg.storage.backup_keep_days, + logger=logger, + ) + db.initialize() + return db + + +def _print(payload) -> None: + print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True) + + +def cmd_init(args, logger: JsonLogger) -> int: + store = ConfigStore(args.config, require_auth=False) + cfg = store.get() + db = _open_db(cfg, logger) + try: + repo = Repository(db) + for folder in cfg.folders: + repo.ensure_folder(folder) + seeded = repo.seed_blacklist(cfg.up_blacklist_seed) + _print({ + "ok": True, + "schema_version": db.schema_version, + "database": str(cfg.storage.database_path), + "blacklist_seeded": seeded, + **repo.counts(), + }) + finally: + db.close() + return 0 + + +def cmd_sync(args, logger: JsonLogger) -> int: + store = ConfigStore(args.config, require_auth=True) + cfg = store.get() + db = _open_db(cfg, logger) + try: + repo = Repository(db) + client = BilibiliClient( + cookie=cfg.bilibili.cookie, + csrf=cfg.bilibili.csrf, + mid=cfg.bilibili.mid, + logger=logger, + ) + service = FolderSyncService(store, repo, client, logger=logger) + summary = service.run(full_refresh=args.full) + payload = summary.as_dict() + payload["database"] = str(cfg.storage.database_path) + payload["counts"] = repo.counts() + _print(payload) + return 0 if summary.ok else 1 + finally: + db.close() + + +def cmd_run(args, logger: JsonLogger) -> int: + store = ConfigStore(args.config, require_auth=True) + cfg = store.get() + db = _open_db(cfg, logger) + last: dict[str, object] = {} + try: + repo = Repository(db) + client = BilibiliClient( + cookie=cfg.bilibili.cookie, + csrf=cfg.bilibili.csrf, + mid=cfg.bilibili.mid, + logger=logger, + ) + service = FolderSyncService(store, repo, client, logger=logger) + + def run_sync(): + summary = service.run() + last["summary"] = summary + logger.info("folder_sync.summary", **summary.as_dict()) + return summary + + scheduler = Scheduler( + [ScheduledTask( + "folder_sync", + float(cfg.schedule.folder_sync_minutes * 60), + run_sync, + )], + logger=logger, + ) + if args.once: + results = scheduler.run_due() + _print([result.as_dict() for result in results]) + else: + logger.info( + "scheduler.start", + interval_seconds=cfg.schedule.folder_sync_minutes * 60, + ) + scheduler.run_forever() + summary = last.get("summary") + return 0 if summary is None or getattr(summary, "ok", True) else 1 + finally: + db.close() + + +def cmd_status(args, logger: JsonLogger) -> int: + store = ConfigStore(args.config, require_auth=False) + cfg = store.get() + db = _open_db(cfg, logger) + try: + repo = Repository(db) + _print({ + "ok": True, + "schema_version": db.schema_version, + "database": str(cfg.storage.database_path), + "configured_folders": [ + { + "media_id": row.media_id, + "year": row.year, + "title": row.title, + "media_count": row.media_count, + "last_ids_at": row.last_ids_at, + "last_full_at": row.last_full_at, + "enabled": row.enabled, + } + for row in repo.list_folders() + ], + **repo.counts(), + }) + finally: + db.close() + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + args = _build_parser().parse_args(list(argv) if argv is not None else None) + logger = JsonLogger() + try: + if args.command == "init": + return cmd_init(args, logger) + if args.command == "sync": + return cmd_sync(args, logger) + if args.command == "run": + return cmd_run(args, logger) + if args.command == "status": + return cmd_status(args, logger) + except ConfigError as exc: + logger.error("config.error", error=str(exc)) + return 2 + except KeyboardInterrupt: + logger.warning("app.interrupted") + return 130 + except Exception as exc: # noqa: BLE001 CLI 边界统一记录 + logger.error("app.error", error=str(exc)) + return 2 + logger.error("app.unknown_command", command=getattr(args, "command", None)) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/reaction_board/config.py b/reaction_board/config.py new file mode 100644 index 0000000..3414a4c --- /dev/null +++ b/reaction_board/config.py @@ -0,0 +1,495 @@ +# -*- coding: utf-8 -*- +"""运行期配置加载、校验与热加载。 + +支持两种写法： + +* 新格式：``{"bilibili": {"mid": ..., "cookie": ...}, ...}``； +* 旧 ``favsort.py`` 格式：顶层 ``mid`` / ``cookie`` / ``csrf``。 + +旧的扁平字段仍然被读取，因此 ``bili-fav auth`` 写回的 config.json 无需迁移即可 +被看板使用。敏感值只保存在配置对象中，日志输出必须先经过 ``logging_json`` 脱敏。 +""" + +from __future__ import annotations + +import copy +import json +import os +import re +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping + +DEFAULT_CLASSIFY: dict[str, Any] = { + "require_title": "咻咻满", + "positive": [ + "reaction", "re", "反应", "看", "听", "初听", "首次", + "点评", "锐评", "路人", "震惊", "双人", + ], + "charge": ["充电", "加急", "专场"], + "negative": ["歌切", "切片", "纯享", "搬运", "完整版"], + "duration_boost": {"gt_30min": 1, "gt_60min": 2}, +} + +DEFAULT_HEAT_WEIGHTS: dict[str, float] = { + "view": 0.30, + "like": 0.20, + "coin": 0.20, + "favorite": 0.15, + "share": 0.10, + "reply": 0.03, + "danmaku": 0.02, +} + +DEFAULT_CONFIG: dict[str, Any] = { + "bilibili": {"mid": 0, "csrf": "", "cookie": ""}, + "telegram": {"enabled": False, "bot_token": "", "chat_id": 0}, + "web": { + "host": "127.0.0.1", + "port": 8080, + "auth_token": "", + "public_base_url": "", + }, + "storage": { + "database_path": "data/app.db", + "backup_dir": "data/backup", + "backup_keep_days": 14, + }, + "folders": [], + "track_fans": [], + "up_blacklist_seed": [], + "schedule": { + "discover_minutes": 60, + "folder_sync_minutes": 60, + "full_folder_refresh_hours": 6, + "snapshot_at": "00:01", + "candidate_ttl_days": 7, + }, + "notify": {"min_score": 2, "max_items_per_message": 10}, + "classify": DEFAULT_CLASSIFY, + "heat_weights": DEFAULT_HEAT_WEIGHTS, +} + +# 环境变量只作为 CI / 临时调试的覆盖入口，不作为主路径。 +ENV_OVERRIDES: dict[str, tuple[tuple[str, ...], type]] = { + "BILI_COOKIE": (("bilibili", "cookie"), str), + "BILI_CSRF": (("bilibili", "csrf"), str), + "BILI_MID": (("bilibili", "mid"), int), + "TG_BOT_TOKEN": (("telegram", "bot_token"), str), + "TG_CHAT_ID": (("telegram", "chat_id"), int), + "DASHBOARD_AUTH_TOKEN": (("web", "auth_token"), str), + "DATABASE_PATH": (("storage", "database_path"), str), +} + +_SNAPSHOT_AT_RE = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$") +_HEAT_METRICS = tuple(DEFAULT_HEAT_WEIGHTS) + + +class ConfigError(ValueError): + """配置文件缺失、格式错误或字段非法。""" + + +@dataclass(frozen=True) +class BilibiliConfig: + mid: int + csrf: str + cookie: str + + +@dataclass(frozen=True) +class TelegramConfig: + enabled: bool + bot_token: str + chat_id: int + + +@dataclass(frozen=True) +class WebConfig: + host: str + port: int + auth_token: str + public_base_url: str + + +@dataclass(frozen=True) +class StorageConfig: + database_path: Path + backup_dir: Path + backup_keep_days: int + + +@dataclass(frozen=True) +class FolderConfig: + year: int + media_id: int + title: str = "" + + +@dataclass(frozen=True) +class ScheduleConfig: + discover_minutes: int + folder_sync_minutes: int + full_folder_refresh_hours: int + snapshot_at: str + candidate_ttl_days: int + + +@dataclass(frozen=True) +class NotifyConfig: + min_score: int + max_items_per_message: int + + +@dataclass(frozen=True) +class AppConfig: + path: Path + bilibili: BilibiliConfig + telegram: TelegramConfig + web: WebConfig + storage: StorageConfig + folders: tuple[FolderConfig, ...] + track_fans: tuple[int, ...] + up_blacklist_seed: tuple[int, ...] + schedule: ScheduleConfig + notify: NotifyConfig + classify: Mapping[str, Any] + heat_weights: Mapping[str, float] + raw: Mapping[str, Any] = field(repr=False, compare=False) + + def folder_for_year(self, year: int) -> FolderConfig | None: + for folder in self.folders: + if folder.year == year: + return folder + return None + + def latest_folder(self) -> FolderConfig | None: + if not self.folders: + return None + return max(self.folders, key=lambda f: f.year) + + +def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]: + out = copy.deepcopy(dict(base)) + for key, value in override.items(): + if isinstance(value, Mapping) and isinstance(out.get(key), Mapping): + out[key] = _deep_merge(out[key], value) + else: + out[key] = copy.deepcopy(value) + return out + + +def _get_path(data: Mapping[str, Any], path: tuple[str, ...]) -> Any: + cur: Any = data + for key in path: + if not isinstance(cur, Mapping) or key not in cur: + return None + cur = cur[key] + return cur + + +def _set_path(data: dict[str, Any], path: tuple[str, ...], value: Any) -> None: + cur = data + for key in path[:-1]: + cur = cur.setdefault(key, {}) + cur[path[-1]] = value + + +def _as_int(value: Any, name: str, *, minimum: int | None = None, + maximum: int | None = None) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ConfigError(f"{name} 必须是整数") + if minimum is not None and value < minimum: + raise ConfigError(f"{name} 不能小于 {minimum}") + if maximum is not None and value > maximum: + raise ConfigError(f"{name} 不能大于 {maximum}") + return value + + +def _as_str(value: Any, name: str) -> str: + if not isinstance(value, str): + raise ConfigError(f"{name} 必须是字符串") + return value + + +def _as_int_list(value: Any, name: str) -> tuple[int, ...]: + if not isinstance(value, list): + raise ConfigError(f"{name} 必须是数组") + out: list[int] = [] + for item in value: + mid = _as_int(item, f"{name}[]", minimum=1) + if mid not in out: + out.append(mid) + return tuple(out) + + +def _as_str_list(value: Any, name: str) -> list[str]: + if not isinstance(value, list): + raise ConfigError(f"{name} 必须是数组") + out = [] + for item in value: + if not isinstance(item, str) or not item: + raise ConfigError(f"{name}[] 必须是非空字符串") + out.append(item) + return out + + +def _legacy_to_current(raw: Mapping[str, Any]) -> dict[str, Any]: + """把 favsort 的扁平字段映射到新格式，不修改调用方对象。""" + data = copy.deepcopy(dict(raw)) + if "bilibili" not in data: + data["bilibili"] = { + "mid": raw.get("mid", 0), + "csrf": raw.get("csrf", ""), + "cookie": raw.get("cookie", ""), + } + return data + + +def _validate_folders(value: Any) -> tuple[FolderConfig, ...]: + if not isinstance(value, list): + raise ConfigError("folders 必须是数组") + folders: list[FolderConfig] = [] + seen_ids: set[int] = set() + seen_years: set[int] = set() + for index, item in enumerate(value): + name = f"folders[{index}]" + if not isinstance(item, Mapping): + raise ConfigError(f"{name} 必须是对象") + year = _as_int(item.get("year"), f"{name}.year", minimum=1900, maximum=9999) + media_id = _as_int(item.get("media_id"), f"{name}.media_id", minimum=1) + title = _as_str(item.get("title", "") or "", f"{name}.title") + if media_id in seen_ids: + raise ConfigError(f"{name}.media_id 重复: {media_id}") + if year in seen_years: + raise ConfigError(f"{name}.year 重复: {year}") + seen_ids.add(media_id) + seen_years.add(year) + folders.append(FolderConfig(year=year, media_id=media_id, title=title)) + return tuple(folders) + + +def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: + b = data["bilibili"] + if not isinstance(b, Mapping): + raise ConfigError("bilibili 必须是对象") + bilibili = BilibiliConfig( + mid=_as_int(b.get("mid", 0), "bilibili.mid", minimum=0), + csrf=_as_str(b.get("csrf", "") or "", "bilibili.csrf"), + cookie=_as_str(b.get("cookie", "") or "", "bilibili.cookie"), + ) + + tg = data["telegram"] + if not isinstance(tg, Mapping): + raise ConfigError("telegram 必须是对象") + telegram = TelegramConfig( + enabled=bool(tg.get("enabled", False)), + bot_token=_as_str(tg.get("bot_token", "") or "", "telegram.bot_token"), + chat_id=_as_int(tg.get("chat_id", 0), "telegram.chat_id"), + ) + + web_raw = data["web"] + if not isinstance(web_raw, Mapping): + raise ConfigError("web 必须是对象") + web = WebConfig( + host=_as_str(web_raw.get("host", "127.0.0.1"), "web.host"), + port=_as_int(web_raw.get("port", 8080), "web.port", minimum=1, maximum=65535), + auth_token=_as_str(web_raw.get("auth_token", "") or "", "web.auth_token"), + public_base_url=_as_str( + web_raw.get("public_base_url", "") or "", "web.public_base_url" + ), + ) + + storage_raw = data["storage"] + if not isinstance(storage_raw, Mapping): + raise ConfigError("storage 必须是对象") + base_dir = path.parent + database_path = Path(_as_str( + storage_raw.get("database_path", "data/app.db") or "data/app.db", + "storage.database_path", + )) + backup_dir = Path(_as_str( + storage_raw.get("backup_dir", "data/backup") or "data/backup", + "storage.backup_dir", + )) + if not database_path.is_absolute(): + database_path = base_dir / database_path + if not backup_dir.is_absolute(): + backup_dir = base_dir / backup_dir + storage = StorageConfig( + database_path=database_path, + backup_dir=backup_dir, + backup_keep_days=_as_int( + storage_raw.get("backup_keep_days", 14), + "storage.backup_keep_days", + minimum=1, + ), + ) + + sched_raw = data["schedule"] + if not isinstance(sched_raw, Mapping): + raise ConfigError("schedule 必须是对象") + snapshot_at = _as_str(sched_raw.get("snapshot_at", "00:01"), "schedule.snapshot_at") + if not _SNAPSHOT_AT_RE.match(snapshot_at): + raise ConfigError("schedule.snapshot_at 必须是 HH:MM（24 小时制）") + schedule = ScheduleConfig( + discover_minutes=_as_int( + sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1 + ), + folder_sync_minutes=_as_int( + sched_raw.get("folder_sync_minutes", 60), + "schedule.folder_sync_minutes", + minimum=1, + ), + full_folder_refresh_hours=_as_int( + sched_raw.get("full_folder_refresh_hours", 6), + "schedule.full_folder_refresh_hours", + minimum=1, + ), + snapshot_at=snapshot_at, + candidate_ttl_days=_as_int( + sched_raw.get("candidate_ttl_days", 7), + "schedule.candidate_ttl_days", + minimum=1, + ), + ) + + notify_raw = data["notify"] + if not isinstance(notify_raw, Mapping): + raise ConfigError("notify 必须是对象") + notify = NotifyConfig( + min_score=_as_int(notify_raw.get("min_score", 2), "notify.min_score"), + max_items_per_message=_as_int( + notify_raw.get("max_items_per_message", 10), + "notify.max_items_per_message", + minimum=1, + ), + ) + + classify_raw = data["classify"] + if not isinstance(classify_raw, Mapping): + raise ConfigError("classify 必须是对象") + classify = copy.deepcopy(dict(classify_raw)) + classify["require_title"] = _as_str( + classify.get("require_title", "咻咻满"), "classify.require_title" + ) + if not classify["require_title"]: + raise ConfigError("classify.require_title 不能为空") + for key in ("positive", "charge", "negative"): + classify[key] = _as_str_list(classify.get(key, []), f"classify.{key}") + boost = classify.get("duration_boost", {}) + if not isinstance(boost, Mapping): + raise ConfigError("classify.duration_boost 必须是对象") + classify["duration_boost"] = { + "gt_30min": _as_int(boost.get("gt_30min", 1), "classify.duration_boost.gt_30min"), + "gt_60min": _as_int(boost.get("gt_60min", 2), "classify.duration_boost.gt_60min"), + } + + heat_raw = data["heat_weights"] + if not isinstance(heat_raw, Mapping): + raise ConfigError("heat_weights 必须是对象") + heat: dict[str, float] = {} + for metric in _HEAT_METRICS: + value = heat_raw.get(metric, DEFAULT_HEAT_WEIGHTS[metric]) + if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0: + raise ConfigError(f"heat_weights.{metric} 必须是非负数字") + heat[metric] = float(value) + + return AppConfig( + path=path, + bilibili=bilibili, + telegram=telegram, + web=web, + storage=storage, + folders=_validate_folders(data["folders"]), + track_fans=_as_int_list(data["track_fans"], "track_fans"), + up_blacklist_seed=_as_int_list(data["up_blacklist_seed"], "up_blacklist_seed"), + schedule=schedule, + notify=notify, + classify=classify, + heat_weights=heat, + raw=data, + ) + + +def load_config( + path: str | os.PathLike[str], + *, + env: Mapping[str, str] | None = None, + require_auth: bool = False, +) -> AppConfig: + """读取并校验配置。``env`` 默认为 ``os.environ``。""" + config_path = Path(path) + if not config_path.exists(): + raise ConfigError(f"缺少配置文件: {config_path}") + try: + with open(config_path, encoding="utf-8") as fh: + raw = json.load(fh) + except json.JSONDecodeError as exc: + raise ConfigError(f"配置文件不是合法 JSON: {config_path}: {exc}") from exc + except OSError as exc: + raise ConfigError(f"无法读取配置文件: {config_path}: {exc}") from exc + if not isinstance(raw, Mapping): + raise ConfigError("配置文件顶层必须是 JSON 对象") + + data = _deep_merge(DEFAULT_CONFIG, _legacy_to_current(raw)) + + env_map = os.environ if env is None else env + for env_name, (target, caster) in ENV_OVERRIDES.items(): + if env_name not in env_map: + continue + value: Any = env_map[env_name] + if caster is int: + try: + value = int(value) + except (TypeError, ValueError) as exc: + raise ConfigError(f"环境变量 {env_name} 必须是整数") from exc + _set_path(data, target, value) + + cfg = _build_config(data, config_path) + if require_auth and (not cfg.bilibili.cookie or not cfg.bilibili.csrf): + raise ConfigError( + "config.json 中的 cookie / csrf 不能为空（可先运行 bili-fav auth）" + ) + return cfg + + +class ConfigStore: + """按文件 mtime 热加载配置；坏配置不会覆盖上一份可用配置。""" + + def __init__( + self, + path: str | os.PathLike[str], + *, + env: Mapping[str, str] | None = None, + require_auth: bool = False, + ) -> None: + self.path = Path(path) + self._env = env + self._require_auth = require_auth + self._lock = threading.RLock() + self._config = load_config(path, env=env, require_auth=require_auth) + self._mtime_ns = self._read_mtime_ns() + self.last_error: str | None = None + + def _read_mtime_ns(self) -> int | None: + try: + return self.path.stat().st_mtime_ns + except OSError: + return None + + def get(self) -> AppConfig: + with self._lock: + mtime_ns = self._read_mtime_ns() + if mtime_ns is None or mtime_ns == self._mtime_ns: + return self._config + self._mtime_ns = mtime_ns + try: + self._config = load_config( + self.path, env=self._env, require_auth=self._require_auth + ) + except ConfigError as exc: + self.last_error = str(exc) + return self._config + self.last_error = None + return self._config diff --git a/reaction_board/database.py b/reaction_board/database.py new file mode 100644 index 0000000..43a06e7 --- /dev/null +++ b/reaction_board/database.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +"""SQLite 连接、迁移入口与单写队列。 + +所有写操作经单个后台线程串行执行，避免并发任务出现 ``database is locked``； +读操作使用短生命周期连接。网络请求绝不在写事务内执行。 +""" + +from __future__ import annotations + +import queue +import sqlite3 +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, TypeVar + +from .migrations import current_version, run_migrations + +T = TypeVar("T") + + +class DatabaseClosed(RuntimeError): + pass + + +@dataclass +class _Job: + fn: Callable[[sqlite3.Connection], Any] + done: threading.Event = field(default_factory=threading.Event) + result: Any = None + error: BaseException | None = None + + +def _configure(conn: sqlite3.Connection) -> sqlite3.Connection: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("PRAGMA busy_timeout = 5000") + conn.execute("PRAGMA synchronous = NORMAL") + conn.execute("PRAGMA journal_mode = WAL") + return conn + + +class Database: + """线程安全的 SQLite 访问入口。""" + + def __init__( + self, + path: str | Path, + *, + backup_dir: str | Path | None = None, + backup_keep_days: int = 14, + logger=None, + ) -> None: + self.path = Path(path) + self.backup_dir = Path(backup_dir) if backup_dir is not None else None + self.backup_keep_days = backup_keep_days + self.logger = logger + self.schema_version: int | None = None + self._queue: queue.Queue[_Job | None] = queue.Queue() + self._thread: threading.Thread | None = None + self._ready = threading.Event() + self._writer_error: BaseException | None = None + self._closed = False + + # ------------------------------------------------------------ setup + + def _connect(self) -> sqlite3.Connection: + # isolation_level=None：单条 DML 立即提交；显式事务由 transaction() 管理。 + return _configure( + sqlite3.connect(self.path, timeout=5.0, isolation_level=None) + ) + + def initialize(self) -> list[int]: + """创建目录、执行迁移并启动单写线程。""" + self.path.parent.mkdir(parents=True, exist_ok=True) + conn = self._connect() + try: + applied = run_migrations( + conn, + backup_dir=self.backup_dir, + backup_keep_days=self.backup_keep_days, + logger=self.logger, + ) + self.schema_version = current_version(conn) + finally: + conn.close() + self._start_writer() + if self.logger is not None: + self.logger.info( + "db.ready", path=str(self.path), schema_version=self.schema_version + ) + return applied + + def _start_writer(self) -> None: + if self._thread is not None: + return + self._thread = threading.Thread(target=self._writer_loop, name="sqlite-writer", daemon=True) + self._thread.start() + self._ready.wait(timeout=10) + if self._writer_error is not None: + raise RuntimeError(f"SQLite 写线程启动失败: {self._writer_error}") from self._writer_error + + def _writer_loop(self) -> None: + conn: sqlite3.Connection | None = None + try: + conn = self._connect() + except BaseException as exc: # noqa: BLE001 + self._writer_error = exc + self._ready.set() + return + self._ready.set() + try: + while True: + job = self._queue.get() + if job is None: + break + try: + job.result = job.fn(conn) + except BaseException as exc: # noqa: BLE001 原样返回给调用方 + job.error = exc + finally: + job.done.set() + finally: + conn.close() + + # ------------------------------------------------------------ access + + def write(self, fn: Callable[[sqlite3.Connection], T]) -> T: + """把函数交给单写线程执行，并同步等待结果。""" + if self._closed: + raise DatabaseClosed("数据库已关闭") + if self._thread is None: + raise RuntimeError("请先调用 initialize()") + job = _Job(fn=fn) + self._queue.put(job) + job.done.wait() + if job.error is not None: + raise job.error + return job.result + + def transaction(self, fn: Callable[[sqlite3.Connection], T]) -> T: + """在单写线程内执行一个显式事务。""" + def run(conn: sqlite3.Connection) -> T: + conn.execute("BEGIN IMMEDIATE") + try: + result = fn(conn) + conn.execute("COMMIT") + return result + except BaseException: + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise + + return self.write(run) + + def read(self, fn: Callable[[sqlite3.Connection], T]) -> T: + """使用独立的短生命周期连接执行只读查询。""" + if self._closed: + raise DatabaseClosed("数据库已关闭") + conn = self._connect() + try: + return fn(conn) + finally: + conn.close() + + def close(self) -> None: + if self._closed: + return + self._closed = True + if self._thread is not None: + self._queue.put(None) + self._thread.join(timeout=10) + self._thread = None diff --git a/reaction_board/logging_json.py b/reaction_board/logging_json.py new file mode 100644 index 0000000..28c7666 --- /dev/null +++ b/reaction_board/logging_json.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +"""脱敏 JSON 行日志。 + +日志只写 stdout，敏感字段（cookie / csrf / token / secret 等）在序列化前统一 +替换为 ``***``；即使调用方误把异常文本或 Cookie 塞进普通字段，也会在字符串 +层面再脱敏一次。 +""" + +from __future__ import annotations + +import json +import re +import sys +import threading +from datetime import datetime, timezone +from typing import Any, IO, Mapping + +REDACTED = "***" + +_SENSITIVE_KEY_PARTS = ( + "cookie", + "csrf", + "token", + "secret", + "password", + "authorization", + "sessdata", + "bili_jct", +) + +_STRING_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + ( + re.compile( + r"(?i)\b(SESSDATA|bili_jct|buvid3|buvid4|DedeUserID__ckMd5|DedeUserID)" + r"\s*=\s*([^;\s]+)" + ), + r"\1=" + REDACTED, + ), + (re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer " + REDACTED), + (re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{25,}\b"), REDACTED), +) + + +def _is_sensitive_key(key: str) -> bool: + normalized = key.lower().replace("-", "_") + return any(part in normalized for part in _SENSITIVE_KEY_PARTS) + + +def _redact_string(value: str) -> str: + for pattern, replacement in _STRING_PATTERNS: + value = pattern.sub(replacement, value) + return value + + +def redact(value: Any, *, key: str | None = None) -> Any: + """递归脱敏。敏感键的整个值替换为 ``***``。""" + if key is not None and _is_sensitive_key(key): + return REDACTED + if isinstance(value, str): + return _redact_string(value) + if isinstance(value, Mapping): + return {k: redact(v, key=str(k)) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [redact(item) for item in value] + return value + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +class JsonLogger: + """线程安全的 JSON 行 logger。""" + + def __init__(self, stream: IO[str] | None = None) -> None: + self.stream = stream if stream is not None else sys.stdout + self._lock = threading.Lock() + + def emit(self, level: str, event: str, **fields: Any) -> dict[str, Any]: + payload: dict[str, Any] = { + "ts": _now_iso(), + "level": level, + "event": event, + } + payload.update(redact(fields)) + line = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + with self._lock: + print(line, file=self.stream, flush=True) + return payload + + def debug(self, event: str, **fields: Any) -> dict[str, Any]: + return self.emit("debug", event, **fields) + + def info(self, event: str, **fields: Any) -> dict[str, Any]: + return self.emit("info", event, **fields) + + def warning(self, event: str, **fields: Any) -> dict[str, Any]: + return self.emit("warning", event, **fields) + + def error(self, event: str, **fields: Any) -> dict[str, Any]: + return self.emit("error", event, **fields) diff --git a/reaction_board/migrations.py b/reaction_board/migrations.py new file mode 100644 index 0000000..cfddd9e --- /dev/null +++ b/reaction_board/migrations.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- +"""SQLite schema 迁移。 + +规则（DEVELOPMENT.md §6.3）： + +* 使用 ``schema_version(version, applied_at)`` 记录已应用版本； +* 每个版本包在单一事务里，失败整体回滚； +* 只向前追加，不允许删库重建； +* 升级已有数据库前自动创建 SQLite 备份。 +""" + +from __future__ import annotations + +import sqlite3 +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + + +@dataclass(frozen=True) +class Migration: + version: int + name: str + statements: tuple[str, ...] + + +MIGRATIONS: tuple[Migration, ...] = ( + Migration( + version=1, + name="initial schema", + statements=( + """ + CREATE TABLE video ( + aid INTEGER PRIMARY KEY, + bvid TEXT UNIQUE, + title TEXT NOT NULL, + up_mid INTEGER, + up_name TEXT, + pubdate INTEGER NOT NULL, + duration INTEGER, + cover TEXT, + folder_id INTEGER, + fav_time INTEGER, + status TEXT NOT NULL, + source TEXT, + first_seen INTEGER NOT NULL, + last_sync INTEGER, + attr INTEGER + ) + """, + "CREATE INDEX idx_video_status ON video(status, pubdate DESC)", + "CREATE INDEX idx_video_up ON video(up_mid, pubdate DESC)", + """ + CREATE TABLE snapshot ( + stat_date TEXT NOT NULL, + aid INTEGER NOT NULL, + view INTEGER, + like INTEGER, + favorite INTEGER, + coin INTEGER, + share INTEGER, + reply INTEGER, + danmaku INTEGER, + captured_at INTEGER NOT NULL, + ok INTEGER NOT NULL DEFAULT 1, + err TEXT, + src TEXT DEFAULT 'view', + PRIMARY KEY (stat_date, aid) + ) + """, + "CREATE INDEX idx_snap_aid ON snapshot(aid, stat_date)", + """ + CREATE TABLE folder ( + media_id INTEGER PRIMARY KEY, + year INTEGER NOT NULL, + title TEXT, + media_count INTEGER, + last_ids_at INTEGER, + last_full_at INTEGER, + enabled INTEGER NOT NULL DEFAULT 1 + ) + """, + """ + CREATE TABLE folder_resource ( + media_id INTEGER NOT NULL, + aid INTEGER NOT NULL, + first_seen INTEGER NOT NULL, + active INTEGER NOT NULL DEFAULT 1, + last_seen INTEGER, + PRIMARY KEY (media_id, aid) + ) + """, + "CREATE INDEX idx_folder_resource_aid ON folder_resource(aid, active)", + """ + CREATE TABLE up_blacklist ( + mid INTEGER PRIMARY KEY, + name TEXT, + reason TEXT, + created_at INTEGER NOT NULL + ) + """, + """ + CREATE TABLE up_stat ( + stat_date TEXT NOT NULL, + mid INTEGER NOT NULL, + follower INTEGER, + following INTEGER, + captured_at INTEGER NOT NULL, + ok INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY (stat_date, mid) + ) + """, + """ + CREATE TABLE candidate ( + aid INTEGER PRIMARY KEY, + bvid TEXT, + title TEXT, + up_mid INTEGER, + up_name TEXT, + pubdate INTEGER, + cover TEXT, + score REAL, + channel TEXT, + first_seen INTEGER NOT NULL, + notified_at INTEGER, + decision TEXT, + decided_at INTEGER + ) + """, + """ + CREATE TABLE seen_aid ( + aid INTEGER PRIMARY KEY, + first_seen INTEGER NOT NULL, + source TEXT + ) + """, + """ + CREATE TABLE notify ( + aid INTEGER NOT NULL, + channel TEXT NOT NULL, + batch_id TEXT, + sent_at INTEGER NOT NULL, + ok INTEGER NOT NULL, + err TEXT, + PRIMARY KEY (aid, channel) + ) + """, + """ + CREATE TABLE fav_action ( + token TEXT PRIMARY KEY, + aid INTEGER NOT NULL, + folder_id INTEGER NOT NULL, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + done_at INTEGER, + position INTEGER, + err TEXT + ) + """, + """ + CREATE TABLE kv ( + k TEXT PRIMARY KEY, + v TEXT + ) + """, + ), + ), +) + + +class MigrationError(RuntimeError): + """迁移失败，原事务已回滚。""" + + +def _schema_version_table(conn: sqlite3.Connection) -> None: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_version (" + "version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)" + ) + + +def current_version(conn: sqlite3.Connection) -> int: + _schema_version_table(conn) + row = conn.execute("SELECT MAX(version) FROM schema_version").fetchone() + return int(row[0] or 0) if row else 0 + + +def _user_tables(conn: sqlite3.Connection) -> list[str]: + rows = conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'schema_version'" + ).fetchall() + return [str(row[0]) for row in rows] + + +def backup_database( + conn: sqlite3.Connection, + backup_dir: str | Path, + *, + version: int, + now: float | None = None, +) -> Path | None: + """用 SQLite backup API 生成一致性备份；内存库返回 None。""" + try: + row = conn.execute("PRAGMA database_list").fetchone() + except sqlite3.Error: + return None + if not row or not row[2] or str(row[2]) == ":memory:": + return None + + target_dir = Path(backup_dir) + target_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(now)) + target = target_dir / f"app-pre-v{version}-{stamp}.db" + suffix = 1 + while target.exists(): + target = target_dir / f"app-pre-v{version}-{stamp}-{suffix}.db" + suffix += 1 + + dst = sqlite3.connect(target) + try: + conn.backup(dst) + finally: + dst.close() + return target + + +def prune_backups( + backup_dir: str | Path, *, keep_days: int, now: float | None = None +) -> list[Path]: + """删除超过保留期的迁移备份。""" + target_dir = Path(backup_dir) + if not target_dir.is_dir(): + return [] + cutoff = (time.time() if now is None else now) - keep_days * 86400 + removed: list[Path] = [] + for path in target_dir.glob("app-pre-v*.db"): + try: + if path.stat().st_mtime < cutoff: + path.unlink() + removed.append(path) + except OSError: + continue + return removed + + +def _validate_migrations(migrations: Sequence[Migration]) -> None: + versions = [m.version for m in migrations] + if versions != sorted(versions): + raise ValueError("迁移版本必须按升序排列") + if len(set(versions)) != len(versions): + raise ValueError("迁移版本不能重复") + if versions and versions[0] < 1: + raise ValueError("迁移版本必须从 1 开始") + + +def run_migrations( + conn: sqlite3.Connection, + *, + backup_dir: str | Path | None = None, + backup_keep_days: int = 14, + now: float | None = None, + migrations: Sequence[Migration] = MIGRATIONS, + logger=None, +) -> list[int]: + """应用所有待执行迁移，返回本次应用的版本列表。""" + _validate_migrations(migrations) + _schema_version_table(conn) + version = current_version(conn) + pending = [m for m in migrations if m.version > version] + if not pending: + return [] + + if backup_dir is not None and _user_tables(conn): + backup = backup_database(conn, backup_dir, version=version, now=now) + prune_backups(backup_dir, keep_days=backup_keep_days) + if logger is not None and backup is not None: + logger.info("db.backup", path=str(backup), version=version) + + applied: list[int] = [] + for migration in pending: + try: + conn.execute("BEGIN IMMEDIATE") + for statement in migration.statements: + conn.execute(statement) + conn.execute( + "INSERT INTO schema_version(version, applied_at) VALUES (?, ?)", + (migration.version, int(time.time() if now is None else now)), + ) + conn.execute("COMMIT") + except Exception as exc: # noqa: BLE001 任何错误都必须回滚 + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise MigrationError( + f"迁移 v{migration.version} ({migration.name}) 失败，已回滚: {exc}" + ) from exc + applied.append(migration.version) + if logger is not None: + logger.info("db.migrated", version=migration.version, name=migration.name) + + return applied diff --git a/reaction_board/repository.py b/reaction_board/repository.py new file mode 100644 index 0000000..8a5d04c --- /dev/null +++ b/reaction_board/repository.py @@ -0,0 +1,441 @@ +# -*- coding: utf-8 -*- +"""P0 数据访问层：收藏夹、视频与黑名单。 + +业务状态迁移集中在这里，服务层只负责请求与编排。所有写操作经 ``Database`` +的单写队列串行执行。 +""" + +from __future__ import annotations + +import sqlite3 +import time +from dataclasses import dataclass +from typing import Any, Iterable + +from .config import FolderConfig +from .database import Database + + +@dataclass(frozen=True) +class UpsertOutcome: + aid: int + status: str + previous_folder_id: int | None + moved_folder: bool + + +@dataclass(frozen=True) +class FolderRow: + media_id: int + year: int + title: str + media_count: int | None + last_ids_at: int | None + last_full_at: int | None + enabled: bool + + +class Repository: + def __init__(self, db: Database, *, clock=time.time) -> None: + self.db = db + self._clock = clock + + def _now(self) -> int: + return int(self._clock()) + + # ------------------------------------------------------------ folder + + def ensure_folder(self, folder: FolderConfig, *, now: int | None = None) -> None: + now = self._now() if now is None else int(now) + def run(conn: sqlite3.Connection) -> None: + conn.execute( + """ + INSERT INTO folder(media_id, year, title, media_count, + last_ids_at, last_full_at, enabled) + VALUES (?, ?, ?, NULL, NULL, NULL, 1) + ON CONFLICT(media_id) DO UPDATE SET + year = excluded.year, + title = COALESCE(NULLIF(excluded.title, ''), folder.title), + enabled = 1 + """, + (folder.media_id, folder.year, folder.title), + ) + self.db.write(run) + + def disable_folders_except(self, media_ids: Iterable[int]) -> None: + ids = [int(mid) for mid in media_ids] + def run(conn: sqlite3.Connection) -> None: + if ids: + placeholders = ",".join("?" for _ in ids) + conn.execute( + f"UPDATE folder SET enabled = 0 WHERE media_id NOT IN ({placeholders})", + ids, + ) + else: + conn.execute("UPDATE folder SET enabled = 0") + self.db.write(run) + + def finish_folder_sync( + self, + folder: FolderConfig, + *, + media_count: int, + full_refresh: bool, + now: int | None = None, + ) -> None: + now = self._now() if now is None else int(now) + def run(conn: sqlite3.Connection) -> None: + conn.execute( + """ + INSERT INTO folder(media_id, year, title, media_count, + last_ids_at, last_full_at, enabled) + VALUES (?, ?, ?, ?, ?, ?, 1) + ON CONFLICT(media_id) DO UPDATE SET + year = excluded.year, + title = COALESCE(NULLIF(excluded.title, ''), folder.title), + media_count = excluded.media_count, + last_ids_at = excluded.last_ids_at, + last_full_at = CASE + WHEN ? THEN excluded.last_full_at + ELSE folder.last_full_at + END, + enabled = 1 + """, + ( + folder.media_id, folder.year, folder.title, int(media_count), + now, now if full_refresh else None, 1 if full_refresh else 0, + ), + ) + self.db.write(run) + + def get_folder(self, media_id: int) -> FolderRow | None: + def run(conn: sqlite3.Connection) -> FolderRow | None: + row = conn.execute( + "SELECT media_id, year, title, media_count, last_ids_at, " + "last_full_at, enabled FROM folder WHERE media_id = ?", + (int(media_id),), + ).fetchone() + if row is None: + return None + return FolderRow( + media_id=int(row["media_id"]), + year=int(row["year"]), + title=str(row["title"] or ""), + media_count=row["media_count"], + last_ids_at=row["last_ids_at"], + last_full_at=row["last_full_at"], + enabled=bool(row["enabled"]), + ) + return self.db.read(run) + + def list_folders(self) -> list[FolderRow]: + def run(conn: sqlite3.Connection) -> list[FolderRow]: + rows = conn.execute( + "SELECT media_id, year, title, media_count, last_ids_at, " + "last_full_at, enabled FROM folder ORDER BY year, media_id" + ).fetchall() + return [ + FolderRow( + media_id=int(row["media_id"]), + year=int(row["year"]), + title=str(row["title"] or ""), + media_count=row["media_count"], + last_ids_at=row["last_ids_at"], + last_full_at=row["last_full_at"], + enabled=bool(row["enabled"]), + ) + for row in rows + ] + return self.db.read(run) + + def folder_active_aids(self, media_id: int) -> set[int]: + def run(conn: sqlite3.Connection) -> set[int]: + rows = conn.execute( + "SELECT aid FROM folder_resource WHERE media_id = ? AND active = 1", + (int(media_id),), + ).fetchall() + return {int(row["aid"]) for row in rows} + return self.db.read(run) + + def touch_folder_resources(self, media_id: int, *, now: int | None = None) -> None: + now = self._now() if now is None else int(now) + self.db.write(lambda conn: conn.execute( + "UPDATE folder_resource SET last_seen = ? WHERE media_id = ? AND active = 1", + (now, int(media_id)), + )) + + # ------------------------------------------------------------ video + + def upsert_from_fav( + self, + resource, + folder_id: int, + *, + was_added: bool, + now: int | None = None, + ) -> UpsertOutcome: + now = self._now() if now is None else int(now) + folder_id = int(folder_id) + invalid = bool(getattr(resource, "invalid", False)) + + def run(conn: sqlite3.Connection) -> UpsertOutcome: + previous = conn.execute( + "SELECT status, folder_id FROM video WHERE aid = ?", (resource.aid,) + ).fetchone() + previous_status = str(previous["status"]) if previous else None + previous_folder = ( + int(previous["folder_id"]) + if previous and previous["folder_id"] is not None else None + ) + + if previous_status is None: + status = "removed" if invalid else "active" + elif previous_status == "dropped": + status = "active" if (was_added and not invalid) else "dropped" + elif invalid: + status = "removed" + else: + status = "active" + + conn.execute( + """ + INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate, + duration, cover, folder_id, fav_time, status, + source, first_seen, last_sync, attr) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(aid) DO UPDATE SET + bvid = COALESCE(excluded.bvid, video.bvid), + title = CASE + WHEN excluded.status = 'removed' AND video.title != '' + THEN video.title + WHEN excluded.title != '' THEN excluded.title + ELSE video.title + END, + up_mid = COALESCE(excluded.up_mid, video.up_mid), + up_name = COALESCE(excluded.up_name, video.up_name), + pubdate = CASE + WHEN excluded.pubdate > 0 THEN excluded.pubdate + ELSE video.pubdate + END, + duration = COALESCE(excluded.duration, video.duration), + cover = COALESCE(excluded.cover, video.cover), + folder_id = excluded.folder_id, + fav_time = COALESCE(excluded.fav_time, video.fav_time), + status = excluded.status, + source = COALESCE(video.source, excluded.source), + last_sync = excluded.last_sync, + attr = excluded.attr + """, + ( + resource.aid, + resource.bvid, + resource.title or "(无标题)", + resource.up_mid, + resource.up_name, + int(resource.pubdate or 0), + resource.duration, + resource.cover, + folder_id, + resource.fav_time, + status, + f"fav:{folder_id}", + now, + now, + resource.attr, + ), + ) + conn.execute( + """ + INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen) + VALUES (?, ?, ?, 1, ?) + ON CONFLICT(media_id, aid) DO UPDATE SET + active = 1, + last_seen = excluded.last_seen + """, + (folder_id, resource.aid, now, now), + ) + return UpsertOutcome( + aid=resource.aid, + status=status, + previous_folder_id=previous_folder, + moved_folder=previous_folder is not None and previous_folder != folder_id, + ) + return self.db.write(run) + + def mark_missing_resource( + self, + aid: int, + *, + folder_id: int, + bvid: str | None = None, + now: int | None = None, + ) -> str: + """处理 infos 静默缺失：写入/更新为 removed，保留历史。""" + now = self._now() if now is None else int(now) + aid = int(aid) + folder_id = int(folder_id) + + def run(conn: sqlite3.Connection) -> str: + row = conn.execute("SELECT status FROM video WHERE aid = ?", (aid,)).fetchone() + previous_status = str(row["status"]) if row else None + status = "dropped" if previous_status == "dropped" else "removed" + conn.execute( + """ + INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate, + duration, cover, folder_id, fav_time, status, + source, first_seen, last_sync, attr) + VALUES (?, ?, '(资源已失效)', NULL, NULL, 0, NULL, NULL, ?, NULL, + ?, ?, ?, ?, 9) + ON CONFLICT(aid) DO UPDATE SET + bvid = COALESCE(video.bvid, excluded.bvid), + status = ?, + last_sync = excluded.last_sync, + attr = 9 + """, + (aid, bvid, folder_id, status, f"fav:{folder_id}", now, now, status), + ) + conn.execute( + """ + INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen) + VALUES (?, ?, ?, 1, ?) + ON CONFLICT(media_id, aid) DO UPDATE SET + active = 1, + last_seen = excluded.last_seen + """, + (folder_id, aid, now, now), + ) + return status + return self.db.write(run) + + def mark_resource_removed( + self, media_id: int, aid: int, *, now: int | None = None + ) -> str | None: + """资源从收藏夹移除：active -> unfaved，其他（dropped/removed）保持不变。""" + now = self._now() if now is None else int(now) + media_id, aid = int(media_id), int(aid) + + def run(conn: sqlite3.Connection) -> str | None: + conn.execute( + "UPDATE folder_resource SET active = 0, last_seen = ? " + "WHERE media_id = ? AND aid = ?", + (now, media_id, aid), + ) + other = conn.execute( + """ + SELECT 1 + FROM folder_resource fr + JOIN folder f ON f.media_id = fr.media_id + WHERE fr.aid = ? AND fr.active = 1 AND f.enabled = 1 + LIMIT 1 + """, + (aid,), + ).fetchone() + if other is None: + conn.execute( + "UPDATE video SET status = 'unfaved', last_sync = ? " + "WHERE aid = ? AND status IN ('active', 'unfaved')", + (now, aid), + ) + row = conn.execute("SELECT status FROM video WHERE aid = ?", (aid,)).fetchone() + return str(row["status"]) if row else None + return self.db.write(run) + + def set_video_status(self, aid: int, status: str, *, now: int | None = None) -> None: + now = self._now() if now is None else int(now) + self.db.write(lambda conn: conn.execute( + "UPDATE video SET status = ?, last_sync = ? WHERE aid = ?", + (status, now, int(aid)), + )) + + def get_video(self, aid: int) -> dict[str, Any] | None: + def run(conn: sqlite3.Connection) -> dict[str, Any] | None: + row = conn.execute("SELECT * FROM video WHERE aid = ?", (int(aid),)).fetchone() + return dict(row) if row else None + return self.db.read(run) + + def list_videos(self) -> list[dict[str, Any]]: + def run(conn: sqlite3.Connection) -> list[dict[str, Any]]: + rows = conn.execute("SELECT * FROM video ORDER BY aid").fetchall() + return [dict(row) for row in rows] + return self.db.read(run) + + def folder_resources(self, media_id: int) -> list[dict[str, Any]]: + def run(conn: sqlite3.Connection) -> list[dict[str, Any]]: + rows = conn.execute( + "SELECT * FROM folder_resource WHERE media_id = ? ORDER BY aid", + (int(media_id),), + ).fetchall() + return [dict(row) for row in rows] + return self.db.read(run) + + # ------------------------------------------------------------ blacklist + + def seed_blacklist(self, mids: Iterable[int], *, now: int | None = None) -> int: + now = self._now() if now is None else int(now) + def run(conn: sqlite3.Connection) -> int: + inserted = 0 + for mid in mids: + cur = conn.execute( + "INSERT OR IGNORE INTO up_blacklist(mid, name, reason, created_at) " + "VALUES (?, NULL, 'seed', ?)", + (int(mid), now), + ) + inserted += max(cur.rowcount, 0) + return inserted + return self.db.write(run) + + def add_blacklist( + self, + mid: int, + *, + name: str | None = None, + reason: str | None = None, + now: int | None = None, + ) -> None: + now = self._now() if now is None else int(now) + self.db.write(lambda conn: conn.execute( + """ + INSERT INTO up_blacklist(mid, name, reason, created_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(mid) DO UPDATE SET + name = COALESCE(excluded.name, up_blacklist.name), + reason = COALESCE(excluded.reason, up_blacklist.reason) + """, + (int(mid), name, reason, now), + )) + + def is_blacklisted(self, mid: int) -> bool: + def run(conn: sqlite3.Connection) -> bool: + row = conn.execute( + "SELECT 1 FROM up_blacklist WHERE mid = ?", (int(mid),) + ).fetchone() + return row is not None + return self.db.read(run) + + def blacklist(self) -> list[dict[str, Any]]: + def run(conn: sqlite3.Connection) -> list[dict[str, Any]]: + rows = conn.execute( + "SELECT mid, name, reason, created_at FROM up_blacklist ORDER BY created_at, mid" + ).fetchall() + return [dict(row) for row in rows] + return self.db.read(run) + + # ------------------------------------------------------------ summary + + def counts(self) -> dict[str, Any]: + def run(conn: sqlite3.Connection) -> dict[str, Any]: + status_rows = conn.execute( + "SELECT status, COUNT(*) AS n FROM video GROUP BY status" + ).fetchall() + by_status = {str(row["status"]): int(row["n"]) for row in status_rows} + folder_count = conn.execute("SELECT COUNT(*) FROM folder").fetchone()[0] + blacklist_count = conn.execute("SELECT COUNT(*) FROM up_blacklist").fetchone()[0] + candidate_count = conn.execute("SELECT COUNT(*) FROM candidate").fetchone()[0] + return { + "video_total": sum(by_status.values()), + "video_by_status": by_status, + "folders": int(folder_count), + "blacklist": int(blacklist_count), + "candidates": int(candidate_count), + } + return self.db.read(run) diff --git a/reaction_board/scheduler.py b/reaction_board/scheduler.py new file mode 100644 index 0000000..5acfb43 --- /dev/null +++ b/reaction_board/scheduler.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +"""单进程内置调度器。 + +P0 只注册收藏夹同步任务；P1/P3 会在同一调度器上追加快照、粉丝和发现任务。 +调度只负责“什么时候运行”，具体任务保持幂等，并在自己的服务层处理重试。 +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any, Callable, Sequence + + +@dataclass +class ScheduledTask: + name: str + interval: float + fn: Callable[[], Any] + next_run: float = 0.0 + + +@dataclass +class TaskResult: + name: str + started_at: float + finished_at: float + ok: bool + error: str | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "started_at": int(self.started_at), + "finished_at": int(self.finished_at), + "ok": self.ok, + "error": self.error, + } + + +class Scheduler: + def __init__( + self, + tasks: Sequence[ScheduledTask], + *, + clock=time.time, + sleeper=time.sleep, + logger=None, + ) -> None: + if not tasks: + raise ValueError("至少需要一个调度任务") + for task in tasks: + if task.interval <= 0: + raise ValueError(f"任务 {task.name} 的 interval 必须大于 0") + self.tasks = list(tasks) + self._clock = clock + self._sleeper = sleeper + self.logger = logger + + def _log(self, level: str, event: str, **fields) -> None: + if self.logger is not None: + getattr(self.logger, level)(event, **fields) + + def run_due(self, *, now: float | None = None) -> list[TaskResult]: + current = self._clock() if now is None else now + results: list[TaskResult] = [] + for task in self.tasks: + if task.next_run > current: + continue + started = current + error: str | None = None + try: + task.fn() + except Exception as exc: # noqa: BLE001 单任务失败不终止调度 + error = str(exc) + self._log("error", "scheduler.task_error", task=task.name, error=error) + else: + self._log("info", "scheduler.task_ok", task=task.name) + finished = self._clock() + task.next_run = started + task.interval + result = TaskResult( + name=task.name, + started_at=started, + finished_at=finished, + ok=error is None, + error=error, + ) + results.append(result) + return results + + def run_forever( + self, + *, + stop_event=None, + poll_seconds: float = 30.0, + max_iterations: int | None = None, + ) -> list[TaskResult]: + """阻塞运行；``stop_event`` 或 ``max_iterations`` 用于可控退出/测试。""" + if poll_seconds <= 0: + raise ValueError("poll_seconds 必须大于 0") + all_results: list[TaskResult] = [] + iterations = 0 + while True: + if stop_event is not None and stop_event.is_set(): + break + all_results.extend(self.run_due()) + iterations += 1 + if max_iterations is not None and iterations >= max_iterations: + break + if stop_event is not None: + if stop_event.wait(poll_seconds): + break + else: + self._sleeper(poll_seconds) + return all_results diff --git a/reaction_board/services/__init__.py b/reaction_board/services/__init__.py new file mode 100644 index 0000000..07f2f76 --- /dev/null +++ b/reaction_board/services/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +"""业务服务层。""" diff --git a/reaction_board/services/folder_sync.py b/reaction_board/services/folder_sync.py new file mode 100644 index 0000000..4a0442b --- /dev/null +++ b/reaction_board/services/folder_sync.py @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +"""收藏夹同步（库的真相源）。 + +每次运行： + +1. 读取每个配置年度夹的全部 aid； +2. 与本地 ``folder_resource`` 做差集； +3. 新增资源按 200 条分块拉取 ``infos`` 并入库； +4. 从收藏夹移除的资源标记 ``unfaved``（不删除历史）； +5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源； +6. 容量接近上限时预警，但绝不自动换夹。 + +黑名单只在发现流程生效，本服务不读取 ``up_blacklist``。 +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +from ..bilibili.client import BilibiliClient +from ..config import AppConfig, ConfigStore, FolderConfig +from ..repository import Repository + +CAPACITY_WARN = 950 + + +@dataclass +class FolderSyncResult: + media_id: int + year: int + title: str + remote_count: int = 0 + added: int = 0 + removed: int = 0 + refreshed: int = 0 + missing: int = 0 + full_refresh: bool = False + warning: str | None = None + error: str | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "media_id": self.media_id, + "year": self.year, + "title": self.title, + "remote_count": self.remote_count, + "added": self.added, + "removed": self.removed, + "refreshed": self.refreshed, + "missing": self.missing, + "full_refresh": self.full_refresh, + "warning": self.warning, + "error": self.error, + } + + +@dataclass +class SyncSummary: + started_at: int + finished_at: int = 0 + folders: list[FolderSyncResult] = field(default_factory=list) + blacklist_seeded: int = 0 + + @property + def added(self) -> int: + return sum(f.added for f in self.folders) + + @property + def removed(self) -> int: + return sum(f.removed for f in self.folders) + + @property + def refreshed(self) -> int: + return sum(f.refreshed for f in self.folders) + + @property + def missing(self) -> int: + return sum(f.missing for f in self.folders) + + @property + def errors(self) -> list[str]: + return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error] + + @property + def warnings(self) -> list[str]: + return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning] + + @property + def ok(self) -> bool: + return not self.errors + + def as_dict(self) -> dict[str, Any]: + return { + "started_at": self.started_at, + "finished_at": self.finished_at, + "elapsed": max(0, self.finished_at - self.started_at), + "added": self.added, + "removed": self.removed, + "refreshed": self.refreshed, + "missing": self.missing, + "blacklist_seeded": self.blacklist_seeded, + "folders": [f.as_dict() for f in self.folders], + "errors": self.errors, + "warnings": self.warnings, + "ok": self.ok, + } + + +class FolderSyncService: + def __init__( + self, + config_store: ConfigStore, + repository: Repository, + client: BilibiliClient, + *, + clock=time.time, + logger=None, + capacity_warn: int = CAPACITY_WARN, + ) -> None: + self.config_store = config_store + self.repo = repository + self.client = client + self._clock = clock + self.logger = logger + self.capacity_warn = capacity_warn + + def _now(self) -> int: + return int(self._clock()) + + def _log(self, level: str, event: str, **fields) -> None: + if self.logger is not None: + getattr(self.logger, level)(event, **fields) + + def run(self, *, full_refresh: bool = False) -> SyncSummary: + cfg = self.config_store.get() + now = self._now() + summary = SyncSummary(started_at=now) + self._log( + "info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh + ) + + for folder in cfg.folders: + self.repo.ensure_folder(folder, now=now) + self.repo.disable_folders_except(f.media_id for f in cfg.folders) + if cfg.up_blacklist_seed: + summary.blacklist_seeded = self.repo.seed_blacklist( + cfg.up_blacklist_seed, now=now + ) + + for folder in cfg.folders: + try: + result = self._sync_folder(cfg, folder, now=now, force_full=full_refresh) + except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹 + result = FolderSyncResult( + media_id=folder.media_id, + year=folder.year, + title=folder.title, + error=str(exc), + ) + self._log( + "error", + "folder_sync.folder_error", + media_id=folder.media_id, + error=str(exc), + ) + else: + self._log("info", "folder_sync.folder", **result.as_dict()) + if result.warning: + self._log( + "warning", + "folder_sync.capacity", + media_id=folder.media_id, + warning=result.warning, + ) + summary.folders.append(result) + + summary.finished_at = self._now() + self._log( + "info" if summary.ok else "error", + "folder_sync.done", + added=summary.added, + removed=summary.removed, + refreshed=summary.refreshed, + missing=summary.missing, + errors=len(summary.errors), + elapsed=max(0, summary.finished_at - summary.started_at), + ) + return summary + + def _sync_folder( + self, + cfg: AppConfig, + folder: FolderConfig, + *, + now: int, + force_full: bool, + ) -> FolderSyncResult: + row = self.repo.get_folder(folder.media_id) + full_due = ( + force_full + or row is None + or row.last_full_at is None + or now - int(row.last_full_at) + >= cfg.schedule.full_folder_refresh_hours * 3600 + ) + + resources = self.client.fav_ids(folder.media_id) + remote_map = {item.aid: item for item in resources if item.type == 2} + remote = set(remote_map) + local = self.repo.folder_active_aids(folder.media_id) + added = remote - local + removed = local - remote + + result = FolderSyncResult( + media_id=folder.media_id, + year=folder.year, + title=folder.title or (row.title if row else ""), + remote_count=len(remote), + added=len(added), + removed=len(removed), + full_refresh=full_due, + ) + + targets = set(added) + if full_due: + targets |= remote + + if targets: + infos = self.client.fav_infos( + folder.media_id, + sorted(targets), + folder_mid=cfg.bilibili.mid, + ) + for aid, resource in infos.items.items(): + if aid not in remote: + continue + outcome = self.repo.upsert_from_fav( + resource, folder.media_id, was_added=aid in added, now=now + ) + if outcome.moved_folder: + self._log( + "warning", + "folder_sync.multi_folder", + aid=aid, + previous_folder_id=outcome.previous_folder_id, + folder_id=folder.media_id, + ) + for aid in infos.missing: + source = remote_map.get(aid) + self.repo.mark_missing_resource( + aid, + folder_id=folder.media_id, + bvid=source.bvid if source else None, + now=now, + ) + result.refreshed = len(infos.items) + result.missing = len(infos.missing) + + for aid in removed: + self.repo.mark_resource_removed(folder.media_id, aid, now=now) + + self.repo.touch_folder_resources(folder.media_id, now=now) + self.repo.finish_folder_sync( + folder, + media_count=len(remote), + full_refresh=full_due, + now=now, + ) + + if len(remote) >= self.capacity_warn: + result.warning = ( + f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条，" + f"达到预警线 {self.capacity_warn}；请人工准备下一年度收藏夹并更新配置，" + f"系统不会自动换夹" + ) + return result diff --git a/tests/fixtures.py b/tests/fixtures.py new file mode 100644 index 0000000..f835d61 --- /dev/null +++ b/tests/fixtures.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +"""脱敏后的真实接口样本（2026-10-31 只读抓取）。 + +样本只保留 P0 需要的字段结构，但包含真实接口的完整嵌套形状： +``upper`` / ``cnt_info`` / ``attr`` / ``bv_id`` / ``bvid`` 等。 +不包含 Cookie、token 或任何凭据。 +""" + +from __future__ import annotations + +import copy +from typing import Any + +# GET x/v3/fav/resource/ids?media_id=3700023351（两条有效 + 两条后来静默缺失） +IDS_PAYLOAD: dict[str, Any] = { + "code": 0, + "message": "0", + "ttl": 1, + "data": [ + {"id": 117530286889638, "type": 2, "bv_id": "BV1xvxi6LEZD", "bvid": "BV1xvxi6LEZD"}, + {"id": 117014421051497, "type": 2, "bv_id": "BV17tGN6kETF", "bvid": "BV17tGN6kETF"}, + {"id": 116911710935631, "type": 2, "bv_id": "BV1JoNy66Exp", "bvid": "BV1JoNy66Exp"}, + {"id": 116769540805008, "type": 2, "bv_id": "BV183jw6hEpt", "bvid": "BV183jw6hEpt"}, + ], +} + +# GET x/v3/fav/resource/infos 的真实有效条目 +VALID_INFO: dict[str, Any] = { + "id": 117530286889638, + "type": 2, + "title": "【咻咻满充电专场①】百变音色机能怪，还有什么风格是你驾驭不了的？！", + "cover": "http://i1.hdslb.com/bfs/archive/e5d3d4f25777b08c1dfae8bc9dc3d56b21d548f1.jpg", + "intro": "视频唱见咻咻满充电专场，感谢宝宝的充电@推堡儿", + "page": 2, + "duration": 9504, + "upper": { + "mid": 434442072, + "name": "_-抱抱-_", + "face": "https://i0.hdslb.com/bfs/face/3176d3d551875a4dd7dac202963ec76d393ddd18.jpg", + "jump_link": "", + }, + "attr": 0, + "cnt_info": { + "collect": 43, + "play": 799, + "danmaku": 7, + "vt": 0, + "play_switch": 0, + "reply": 0, + "view_text_1": "799", + }, + "link": "bilibili://video/117530286889638", + "ctime": 1793370433, + "pubtime": 1793370433, + "fav_time": 0, + "bv_id": "BV1xvxi6LEZD", + "bvid": "BV1xvxi6LEZD", + "season": None, + "ogv": None, + "ugc": {"first_cid": 43184885058}, + "media_list_link": "bilibili://music/playlist/playpage/3728825651?page_type=3&oid=117530286889638&otype=2", +} + +# attr=9 表示已失效视频（真实样本） +INVALID_INFO: dict[str, Any] = { + "id": 116769540805008, + "type": 2, + "title": "已失效视频", + "cover": "http://i0.hdslb.com/bfs/archive/be27fd62c99036dce67efface486fb0a88ffed06.jpg", + "intro": "-", + "page": 1, + "duration": 4504, + "upper": { + "mid": 3707011906669054, + "name": "裤裤喝可乐", + "face": "https://i1.hdslb.com/bfs/face/c1d2eb1c9e5c783b8b0aa6e6154892724dfe2812.jpg", + "jump_link": "", + }, + "attr": 9, + "cnt_info": { + "collect": 27, + "play": 977, + "danmaku": 0, + "vt": 0, + "play_switch": 0, + "reply": 0, + "view_text_1": "977", + }, + "link": "bilibili://video/116769540805008", + "ctime": 1781762147, + "pubtime": 1781762147, + "fav_time": 0, + "bv_id": "BV183jw6hEpt", + "bvid": "BV183jw6hEpt", + "season": None, + "ogv": None, + "ugc": {"first_cid": 39207702089}, + "media_list_link": "bilibili://music/playlist/playpage/3728825651?page_type=3&oid=116769540805008&otype=2", +} + +# GET x/v3/fav/folder/created/list-all 的真实条目 +FOLDERS_PAYLOAD: dict[str, Any] = { + "code": 0, + "message": "0", + "ttl": 1, + "data": { + "count": 2, + "list": [ + { + "id": 4451053851, + "fid": 44510538, + "mid": 28802351, + "attr": 2, + "title": "咻咻满-reaction-2025", + "fav_state": 0, + "media_count": 167, + "is_kid_playlist": False, + "kid_playlist_desc": "", + }, + { + "id": 3700023351, + "fid": 37000233, + "mid": 28802351, + "attr": 54, + "title": "咻咻满 -reaction-2026", + "fav_state": 0, + "media_count": 664, + "is_kid_playlist": False, + "kid_playlist_desc": "", + }, + ], + }, +} + + +def info_for( + aid: int, + *, + title: str, + up_mid: int, + up_name: str, + pubdate: int = 1793370433, + duration: int = 9504, + attr: int = 0, + bvid: str | None = None, +) -> dict[str, Any]: + """基于真实样本复制一条 infos 条目，只改测试需要的字段。""" + item = copy.deepcopy(VALID_INFO) + item["id"] = aid + item["title"] = title + item["upper"]["mid"] = up_mid + item["upper"]["name"] = up_name + item["pubtime"] = pubdate + item["ctime"] = pubdate + item["duration"] = duration + item["attr"] = attr + item["bvid"] = bvid or f"BV{aid}" + item["bv_id"] = item["bvid"] + item["link"] = f"bilibili://video/{aid}" + item["media_list_link"] = ( + f"bilibili://music/playlist/playpage/3728825651?page_type=3&oid={aid}&otype=2" + ) + return item diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..35c44af --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +"""auth 相关纯函数自测（不联网）: python tests/test_auth.py""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +import favsort as f # noqa: E402 + +OLD = ( + "buvid3=OLD-B3; buvid4=OLD-B4; _uuid=OLD-UUID; buvid_fp=OLD-FP; " + "b_nut=1700000000; SESSDATA=old%2C1700000000%2Cabc; bili_jct=oldcsrf; " + "DedeUserID=28802351; DedeUserID__ckMd5=oldmd5; sid=oldsid; b_lsid=OLDL; " + "bili_ticket=ticket; home_feed_column=5" +) + + +def test_parse_and_merge(): + pairs = f.parse_cookie(OLD) + assert pairs[0] == ("buvid3", "OLD-B3") + assert f.cookie_value(OLD, "SESSDATA") == "old%2C1700000000%2Cabc" + + new = { + "SESSDATA": "new%2C1800000000%2Cxyz", + "bili_jct": "newcsrf", + "DedeUserID": "28802351", + "DedeUserID__ckMd5": "newmd5", + "sid": "newsid", + } + merged = f.merge_cookie(OLD, new) + # 身份字段被替换 + assert f.cookie_value(merged, "SESSDATA") == "new%2C1800000000%2Cxyz" + assert f.cookie_value(merged, "bili_jct") == "newcsrf" + assert f.cookie_value(merged, "sid") == "newsid" + # 设备指纹原样保留 + assert f.cookie_value(merged, "buvid3") == "OLD-B3" + assert f.cookie_value(merged, "buvid4") == "OLD-B4" + assert f.cookie_value(merged, "_uuid") == "OLD-UUID" + assert f.cookie_value(merged, "buvid_fp") == "OLD-FP" + assert f.cookie_value(merged, "b_nut") == "1700000000" + assert f.cookie_value(merged, "home_feed_column") == "5" + # 旧会话字段在没有新值时应被丢弃 + assert f.cookie_value(merged, "b_lsid") == "" + + +def test_set_cookie_and_url_fallback(): + headers = [ + "SESSDATA=abc%2C1800000000%2Cdef; Path=/; Domain=.bilibili.com; HttpOnly", + "bili_jct=csrf2; Domain=.bilibili.com", + "buvid3=SHOULD-NOT-OVERWRITE; Domain=.bilibili.com", + ] + got = f.cookies_from_set_cookie(headers) + assert got["SESSDATA"] == "abc%2C1800000000%2Cdef" + assert got["bili_jct"] == "csrf2" + + merged = f.merge_cookie(OLD, got) + assert f.cookie_value(merged, "buvid3") == "OLD-B3" # 设备字段不被 Set-Cookie 覆盖 + + +def test_expiry(): + assert f.sessdata_expiry(OLD) == 1700000000 + assert f.sessdata_expiry("SESSDATA=bad") is None + assert f.sessdata_expiry("") is None + + +def test_ensure_device_cookies(): + called = {"n": 0} + + def fake_get_json(cfg, url, auth=False): + called["n"] += 1 + return {"code": 0, "data": {"b_3": "NEW-B3", "b_4": "NEW-B4"}} + + orig = f.get_json + f.get_json = fake_get_json + try: + cfg = {"cookie": "SESSDATA=x; bili_jct=y"} + added = f.ensure_device_cookies(cfg) + assert "buvid3" in added and "buvid4" in added and "_uuid" in added + assert f.cookie_value(cfg["cookie"], "buvid3") == "NEW-B3" + assert f.cookie_value(cfg["cookie"], "buvid4") == "NEW-B4" + assert f.cookie_value(cfg["cookie"], "_uuid").endswith("infoc") + + # 已有设备字段时不应调用 spi，也不应改动 + called["n"] = 0 + cfg2 = {"cookie": OLD} + added2 = f.ensure_device_cookies(cfg2) + assert added2 == [] and called["n"] == 0 + assert cfg2["cookie"] == OLD + finally: + f.get_json = orig + + +def test_finish_auth_write_and_rollback(): + import json + import tempfile + + new = {"SESSDATA": "new%2C1800000000%2Cxyz", "bili_jct": "newcsrf", + "DedeUserID": "28802351", "DedeUserID__ckMd5": "newmd5"} + cfg = {"mid": 28802351, "csrf": "oldcsrf", "cookie": OLD, + "overrides": {}, "sleep": {}} + old_raw = json.dumps(cfg, ensure_ascii=False, indent=2) + + orig = (f.CONFIG_PATH, f.CONFIG_BACKUP, f.fetch_nav, f.ensure_device_cookies) + + def fake_nav_ok(c): + return {"code": 0, "data": {"isLogin": True, "uname": "tester", "mid": 28802351}} + + def fake_nav_bad(c): + return {"code": -101, "message": "账号未登录", "data": {"isLogin": False}} + + def fake_device(c): + return [] + + try: + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "config.json") + f.CONFIG_PATH, f.CONFIG_BACKUP = path, path + ".bak" + f.ensure_device_cookies = fake_device + + with open(path, "w", encoding="utf-8") as fh: + fh.write(old_raw) + + # 成功：写入新身份字段，设备字段不变，并生成备份 + f.fetch_nav = fake_nav_ok + assert f.finish_auth(cfg, dict(new)) == 0 + with open(path, encoding="utf-8") as fh: + saved = json.load(fh) + assert "newcsrf" == saved["csrf"] + assert f.cookie_value(saved["cookie"], "buvid3") == "OLD-B3" + assert f.cookie_value(saved["cookie"], "SESSDATA") == "new%2C1800000000%2Cxyz" + assert os.path.exists(path + ".bak") + + # 失败：用上一份备份回滚 + before = open(path, encoding="utf-8").read() + cfg2 = dict(saved) + f.fetch_nav = fake_nav_bad + assert f.finish_auth(cfg2, {"SESSDATA": "x%2C1%2Cy", "bili_jct": "z"}) == 4 + after = open(path, encoding="utf-8").read() + assert after == before + finally: + f.CONFIG_PATH, f.CONFIG_BACKUP, f.fetch_nav, f.ensure_device_cookies = orig + + +def test_qr_png(): + import tempfile + + path = os.path.join(tempfile.gettempdir(), "favsort-test-qr.png") + try: + f.write_qr_png("https://example.com/login?key=abc&from=test", path) + with open(path, "rb") as fh: + data = fh.read() + assert data.startswith(b"\x89PNG\r\n\x1a\n") + assert len(data) > 500 + finally: + if os.path.exists(path): + os.remove(path) + + +def main(): + test_parse_and_merge() + test_set_cookie_and_url_fallback() + test_expiry() + test_ensure_device_cookies() + test_finish_auth_write_and_rollback() + test_qr_png() + print("OK: auth 纯函数测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_bilibili_client.py b/tests/test_bilibili_client.py new file mode 100644 index 0000000..b0ecaf5 --- /dev/null +++ b/tests/test_bilibili_client.py @@ -0,0 +1,295 @@ +# -*- coding: utf-8 -*- +"""B 站只读客户端测试（transport 边界 Mock，不联网）: +uv run python tests/test_bilibili_client.py +""" + +import json +import os +import sys +import urllib.parse +from dataclasses import dataclass, field + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.bilibili.client import ( # noqa: E402 + BilibiliAPIError, + BilibiliClient, + BilibiliHTTPError, + BilibiliProtocolError, + HTTPResponse, + Pacer, + TransportError, +) +from fixtures import ( # noqa: E402 + FOLDERS_PAYLOAD, + IDS_PAYLOAD, + INVALID_INFO, + VALID_INFO, + info_for, +) + + +def http_json(payload, status=200): + return HTTPResponse( + status=status, + headers={"content-type": "application/json"}, + body=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + ) + + +@dataclass +class QueueTransport: + responses: list + calls: list = field(default_factory=list) + + def request(self, method, url, headers, data=None, timeout=30): + self.calls.append({ + "method": method, "url": url, "headers": dict(headers), + "data": data, "timeout": timeout, + }) + item = self.responses.pop(0) + if isinstance(item, BaseException): + raise item + return item + + +@dataclass +class RouteTransport: + handler: object + calls: list = field(default_factory=list) + + def request(self, method, url, headers, data=None, timeout=30): + self.calls.append({ + "method": method, "url": url, "headers": dict(headers), + "data": data, "timeout": timeout, + }) + return self.handler(url, headers) + + +def make_client(transport, *, cookie="SESSDATA=secret; buvid3=device", retries=3): + sleeps = [] + client = BilibiliClient( + cookie=cookie, + csrf="csrf", + mid=28802351, + transport=transport, + pacer=Pacer(0, 0, sleep=sleeps.append), + sleeper=sleeps.append, + retries=retries, + backoff=1.0, + ) + return client, sleeps + + +def test_fav_ids_parse_and_anonymous_header(): + transport = QueueTransport([http_json(IDS_PAYLOAD)]) + client, _ = make_client(transport) + ids = client.fav_ids(3700023351) + assert len(ids) == 4 + assert ids[0].aid == 117530286889638 + assert ids[0].type == 2 + assert ids[0].bvid == "BV1xvxi6LEZD" + call = transport.calls[0] + assert "media_id=3700023351" in call["url"] + assert "SESSDATA" not in call["headers"]["cookie"] + assert "buvid3=device" in call["headers"]["cookie"] + + +def test_anonymous_failure_falls_back_to_full_cookie(): + transport = QueueTransport([ + http_json({"code": -101, "message": "账号未登录", "data": None}), + http_json(IDS_PAYLOAD), + ]) + client, _ = make_client(transport) + ids = client.fav_ids(3700023351) + assert len(ids) == 4 + assert len(transport.calls) == 2 + assert "SESSDATA" not in transport.calls[0]["headers"]["cookie"] + assert "SESSDATA=secret" in transport.calls[1]["headers"]["cookie"] + + +def test_business_error_is_not_retried(): + error = {"code": -404, "message": "啥都木有", "data": None} + transport = QueueTransport([http_json(error), http_json(error)]) + client, sleeps = make_client(transport) + try: + client.fav_ids(3700023351) + except BilibiliAPIError as exc: + assert exc.code == -404 + assert "啥都木有" in str(exc) + else: + raise AssertionError("业务错误应抛出") + assert len(transport.calls) == 2 # 匿名 + 完整 Cookie 各一次，不重试 + assert sleeps == [] + + +def test_http_500_retries_with_backoff(): + transport = QueueTransport([ + http_json({"code": 0}, status=500), + http_json({"code": 0}, status=500), + http_json({"code": 0}, status=500), + ]) + client, sleeps = make_client(transport, retries=3) + try: + client.get_json("https://api.bilibili.com/x/test", auth=True) + except BilibiliHTTPError as exc: + assert exc.status == 500 + else: + raise AssertionError("HTTP 500 应抛出") + assert len(transport.calls) == 3 + assert sleeps == [1.0, 2.0] + + +def test_network_error_retry_then_success(): + transport = QueueTransport([TransportError("reset"), http_json(IDS_PAYLOAD)]) + client, sleeps = make_client(transport) + ids = client.fav_ids(3700023351) + assert len(ids) == 4 + assert len(transport.calls) == 2 + assert sleeps == [1.0] + + +def test_invalid_json_is_protocol_error(): + bad = HTTPResponse(200, {}, b"not json") + transport = QueueTransport([bad, bad]) + client, _ = make_client(transport) + try: + client.fav_ids(3700023351) + except BilibiliProtocolError as exc: + assert "JSON" in str(exc) + else: + raise AssertionError("非法 JSON 应抛出协议错误") + + +def test_infos_chunking_and_silent_missing(): + aids = list(range(1000, 1450)) + missing = 1003 + + def handler(url, headers): + query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query) + resources = query["resources"][0] + requested = [int(part.split(":")[0]) for part in resources.split(",")] + data = [ + info_for(aid, title=f"视频{aid}", up_mid=100, up_name="UP") + for aid in requested if aid != missing + ] + return http_json({"code": 0, "message": "0", "ttl": 1, "data": data}) + + transport = RouteTransport(handler) + client, _ = make_client(transport) + result = client.fav_infos(3700023351, aids, folder_mid=28802351, chunk_size=200) + assert len(transport.calls) == 3 + sizes = [] + for call in transport.calls: + query = urllib.parse.parse_qs(urllib.parse.urlparse(call["url"]).query) + sizes.append(len(query["resources"][0].split(","))) + assert sizes == [200, 200, 50] + assert result.requested == tuple(aids) + assert result.missing == (missing,) + assert len(result.items) == 449 + assert result.items[1000].title == "视频1000" + assert result.items[1000].invalid is False + + +def test_infos_invalid_attr_and_empty(): + transport = QueueTransport([ + http_json({"code": 0, "message": "0", "ttl": 1, "data": [INVALID_INFO]}) + ]) + client, _ = make_client(transport) + result = client.fav_infos(3700023351, [INVALID_INFO["id"]]) + assert result.items[INVALID_INFO["id"]].invalid is True + assert result.missing == () + + # 空输入不发起请求 + empty = client.fav_infos(3700023351, []) + assert empty.requested == () and empty.items == {} + assert len(transport.calls) == 1 + + +def test_infos_unexpected_aid_is_protocol_error(): + transport = QueueTransport([http_json({ + "code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO], + })]) + client, _ = make_client(transport) + try: + client.fav_infos(3700023351, [1]) + except BilibiliProtocolError as exc: + assert "未请求" in str(exc) + else: + raise AssertionError("返回未请求 aid 应报错") + + +def test_fav_folders_uses_full_cookie(): + transport = QueueTransport([http_json(FOLDERS_PAYLOAD)]) + client, _ = make_client(transport) + folders = client.fav_folders() + assert [(f.media_id, f.media_count) for f in folders] == [ + (4451053851, 167), (3700023351, 664) + ] + call = transport.calls[0] + assert "up_mid=28802351" in call["url"] + assert "SESSDATA=secret" in call["headers"]["cookie"] + + +def test_fav_infos_chunk_size_validation(): + client, _ = make_client(QueueTransport([])) + for size in (0, 401): + try: + client.fav_infos(1, [1], chunk_size=size) + except ValueError: + pass + else: + raise AssertionError(f"chunk_size={size} 应失败") + + +def test_pacer_delays_after_first_request(): + sleeps = [] + + class Rng: + def uniform(self, a, b): + assert (a, b) == (1.5, 2.5) + return 2.0 + + pacer = Pacer(1.5, 2.5, sleep=sleeps.append, rng=Rng()) + assert pacer.wait() == 0.0 + assert pacer.wait() == 2.0 + assert pacer.wait() == 2.0 + assert sleeps == [2.0, 2.0] + try: + Pacer(2.0, 1.0) + except ValueError: + pass + else: + raise AssertionError("非法间隔应失败") + + +def test_timeout_and_method_forwarded(): + transport = QueueTransport([http_json(IDS_PAYLOAD)]) + client, _ = make_client(transport) + client.fav_ids(1) + call = transport.calls[0] + assert call["method"] == "GET" + assert call["data"] is None + assert call["timeout"] == 30 + + +def main(): + test_fav_ids_parse_and_anonymous_header() + test_anonymous_failure_falls_back_to_full_cookie() + test_business_error_is_not_retried() + test_http_500_retries_with_backoff() + test_network_error_retry_then_success() + test_invalid_json_is_protocol_error() + test_infos_chunking_and_silent_missing() + test_infos_invalid_attr_and_empty() + test_infos_unexpected_aid_is_protocol_error() + test_fav_folders_uses_full_cookie() + test_fav_infos_chunk_size_validation() + test_pacer_delays_after_first_request() + test_timeout_and_method_forwarded() + print("OK: B 站只读客户端契约与重试测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..9d04cbf --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""CLI 入口测试（离线）: uv run python tests/test_cli.py""" + +import contextlib +import io +import json +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.cli import main # noqa: E402 +from reaction_board.database import Database # noqa: E402 +from reaction_board.repository import Repository # noqa: E402 + + +def write_config(tmp, *, cookie="SESSDATA=x; buvid3=y", csrf="csrf"): + path = Path(tmp) / "config.json" + path.write_text(json.dumps({ + "mid": 1, + "csrf": csrf, + "cookie": cookie, + "storage": { + "database_path": str(Path(tmp) / "app.db"), + "backup_dir": str(Path(tmp) / "backup"), + }, + "folders": [ + {"year": 2025, "media_id": 11, "title": "2025"}, + {"year": 2026, "media_id": 22, "title": "2026"}, + ], + "up_blacklist_seed": [101, 202], + }, ensure_ascii=False), encoding="utf-8") + return path + + +def run_cli(argv): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = main(argv) + return code, out.getvalue() + + +def test_init_and_status(): + with tempfile.TemporaryDirectory() as tmp: + path = write_config(tmp) + code, output = run_cli(["init", "--config", str(path)]) + assert code == 0 + assert '"schema_version": 1' in output + + db = Database(Path(tmp) / "app.db") + db.initialize() + try: + repo = Repository(db) + assert len(repo.list_folders()) == 2 + assert [r["mid"] for r in repo.blacklist()] == [101, 202] + assert repo.counts()["video_total"] == 0 + finally: + db.close() + + code, output = run_cli(["status", "--config", str(path)]) + assert code == 0 + assert '"configured_folders"' in output + assert '"media_count": null' in output + assert '"video_total": 0' in output + + +def test_missing_config(): + with tempfile.TemporaryDirectory() as tmp: + code, output = run_cli(["init", "--config", str(Path(tmp) / "nope.json")]) + assert code == 2 + payload = json.loads(output.strip().splitlines()[-1]) + assert payload["level"] == "error" + assert "缺少配置文件" in payload["error"] + + +def test_sync_requires_auth_without_network(): + with tempfile.TemporaryDirectory() as tmp: + path = write_config(tmp, cookie="", csrf="") + code, output = run_cli(["sync", "--config", str(path)]) + assert code == 2 + payload = json.loads(output.strip().splitlines()[-1]) + assert "cookie" in payload["error"] + assert not (Path(tmp) / "app.db").exists() + + +def test_run_once_without_folders(): + with tempfile.TemporaryDirectory() as tmp: + path = write_config(tmp) + data = json.loads(path.read_text(encoding="utf-8")) + data["folders"] = [] + path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + code, output = run_cli(["run", "--once", "--config", str(path)]) + assert code == 0 + assert '"name": "folder_sync"' in output + assert '"ok": true' in output + + +def main_test(): + test_init_and_status() + test_missing_config() + test_sync_requires_auth_without_network() + test_run_once_without_folders() + print("OK: CLI init/sync/run/status/错误路径测试全部通过") + + +if __name__ == "__main__": + main_test() diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..a8f1ca5 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +"""配置加载 / 校验 / 热加载测试（不联网）: uv run python tests/test_config.py""" + +import json +import os +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.config import ConfigError, ConfigStore, load_config # noqa: E402 + +COOKIE = "SESSDATA=secret%2C1800000000%2Cabc; bili_jct=csrf-secret; buvid3=dev" + + +def write(path: Path, data) -> None: + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def bump_mtime(path: Path) -> None: + """保证 mtime 严格大于上一次，避免 Windows 时间精度导致热加载测试偶发失败。""" + target = max(path.stat().st_mtime_ns, time.time_ns()) + 1_000_000 + os.utime(path, ns=(target, target)) + + +def base_legacy() -> dict: + return { + "mid": 28802351, + "csrf": "csrf-secret", + "cookie": COOKIE, + "overrides": {"1": {"mid": 2, "name": "x"}}, + "sleep": {"min": 1}, + "folders": [ + {"year": 2025, "media_id": 4451053851, "title": "2025"}, + {"year": 2026, "media_id": 3700023351, "title": "2026"}, + ], + "track_fans": [37754047], + "up_blacklist_seed": [1, 2, 2], + } + + +def test_legacy_flat_mapping(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + write(path, base_legacy()) + cfg = load_config(path, env={}) + assert cfg.bilibili.mid == 28802351 + assert cfg.bilibili.csrf == "csrf-secret" + assert cfg.bilibili.cookie == COOKIE + assert cfg.storage.database_path == Path(tmp) / "data" / "app.db" + assert cfg.storage.backup_dir == Path(tmp) / "data" / "backup" + assert cfg.schedule.snapshot_at == "00:01" + assert [f.media_id for f in cfg.folders] == [4451053851, 3700023351] + assert cfg.track_fans == (37754047,) + assert cfg.up_blacklist_seed == (1, 2) # 去重 + assert cfg.raw["overrides"] # 旧字段仍保留给 favsort + assert set(cfg.heat_weights) == { + "view", "like", "coin", "favorite", "share", "reply", "danmaku" + } + + +def test_nested_and_env_override(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + write(path, { + "bilibili": {"mid": 1, "csrf": "old", "cookie": "old-cookie"}, + "storage": {"database_path": "var/app.db", "backup_dir": "var/bak"}, + "folders": [], + }) + env = { + "BILI_MID": "999", + "BILI_CSRF": "env-csrf", + "BILI_COOKIE": "env-cookie", + "TG_BOT_TOKEN": "123456:abcdefghijklmnopqrstuvwxyzABCDE", + "TG_CHAT_ID": "42", + "DASHBOARD_AUTH_TOKEN": "web-token", + "DATABASE_PATH": "custom.db", + } + cfg = load_config(path, env=env) + assert cfg.bilibili.mid == 999 + assert cfg.bilibili.csrf == "env-csrf" + assert cfg.bilibili.cookie == "env-cookie" + assert cfg.telegram.chat_id == 42 + assert cfg.telegram.bot_token.startswith("123456:") + assert cfg.web.auth_token == "web-token" + assert cfg.storage.database_path == Path(tmp) / "custom.db" + # 备份目录仍按配置文件目录解析 + assert cfg.storage.backup_dir == Path(tmp) / "var" / "bak" + + +def test_absolute_paths_kept(): + with tempfile.TemporaryDirectory() as tmp: + db = str(Path(tmp) / "abs" / "app.db") + path = Path(tmp) / "config.json" + write(path, { + "bilibili": {"mid": 1, "csrf": "c", "cookie": "k"}, + "storage": {"database_path": db, "backup_dir": str(Path(tmp) / "b")}, + "folders": [], + }) + cfg = load_config(path, env={}) + assert cfg.storage.database_path == Path(db) + + +def test_invalid_configs(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + cases = [ + ({"folders": [{"year": 2025, "media_id": 1}, + {"year": 2026, "media_id": 1}]}, "media_id 重复"), + ({"folders": [{"year": 2025, "media_id": 1}, + {"year": 2025, "media_id": 2}]}, "year 重复"), + ({"folders": [{"year": 2025, "media_id": 0}]}, "不能小于 1"), + ({"schedule": {"snapshot_at": "24:00"}}, "snapshot_at"), + ({"schedule": {"snapshot_at": "1:01"}}, "snapshot_at"), + ({"notify": {"max_items_per_message": 0}}, "max_items_per_message"), + ({"heat_weights": {"view": -1}}, "heat_weights.view"), + ({"classify": {"require_title": ""}}, "require_title"), + ] + base = { + "bilibili": {"mid": 1, "csrf": "c", "cookie": "k"}, + "folders": [], + } + for patch, needle in cases: + data = json.loads(json.dumps(base)) + data.update(patch) + write(path, data) + try: + load_config(path, env={}) + except ConfigError as exc: + assert needle in str(exc), (needle, str(exc)) + else: + raise AssertionError(f"应当失败: {patch}") + + +def test_require_auth(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + write(path, {"bilibili": {"mid": 1, "csrf": "", "cookie": ""}, "folders": []}) + load_config(path, env={}, require_auth=False) + try: + load_config(path, env={}, require_auth=True) + except ConfigError as exc: + assert "cookie" in str(exc) + else: + raise AssertionError("require_auth 应拒绝空凭据") + + +def test_env_int_invalid(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + write(path, base_legacy()) + try: + load_config(path, env={"BILI_MID": "abc"}) + except ConfigError as exc: + assert "BILI_MID" in str(exc) + else: + raise AssertionError("非法整数环境变量应失败") + + +def test_hot_reload_keeps_last_good_config(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + write(path, base_legacy()) + store = ConfigStore(path, env={}) + assert store.get().folders[0].title == "2025" + + data = base_legacy() + data["folders"][0]["title"] = "2025-新" + write(path, data) + bump_mtime(path) + assert store.get().folders[0].title == "2025-新" + assert store.last_error is None + + path.write_text("{not json", encoding="utf-8") + bump_mtime(path) + assert store.get().folders[0].title == "2025-新" + assert store.last_error is not None + + write(path, data) + bump_mtime(path) + assert store.get().folders[0].title == "2025-新" + assert store.last_error is None + + +def test_missing_file(): + try: + load_config(Path(tempfile.gettempdir()) / "no-such-bili-fav-config.json", env={}) + except ConfigError as exc: + assert "缺少配置文件" in str(exc) + else: + raise AssertionError("缺失配置应失败") + + +def main(): + test_legacy_flat_mapping() + test_nested_and_env_override() + test_absolute_paths_kept() + test_invalid_configs() + test_require_auth() + test_env_int_invalid() + test_hot_reload_keeps_last_good_config() + test_missing_file() + print("OK: 配置加载、校验与热加载测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..ee37229 --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +"""数据库单写队列测试（不联网）: uv run python tests/test_database.py""" + +import os +import sqlite3 +import sys +import tempfile +import threading +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.database import Database, DatabaseClosed # noqa: E402 + + +def make_db(tmp): + db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup") + db.initialize() + return db + + +def test_write_read_transaction(): + with tempfile.TemporaryDirectory() as tmp: + db = make_db(tmp) + try: + assert db.schema_version == 1 + db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')")) + assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1" + + def boom(conn): + conn.execute("UPDATE kv SET v='2' WHERE k='a'") + raise RuntimeError("boom") + try: + db.transaction(boom) + except RuntimeError as exc: + assert str(exc) == "boom" + else: + raise AssertionError("事务应抛错") + assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1" + finally: + db.close() + + +def test_concurrent_writes_are_serialized(): + with tempfile.TemporaryDirectory() as tmp: + db = make_db(tmp) + try: + db.write(lambda c: c.execute( + "CREATE TABLE counter (id INTEGER PRIMARY KEY, n INTEGER NOT NULL)" + )) + db.write(lambda c: c.execute("INSERT INTO counter(id, n) VALUES (1, 0)")) + errors = [] + + def worker(): + try: + for _ in range(10): + db.transaction( + lambda conn: conn.execute( + "UPDATE counter SET n = n + 1 WHERE id = 1" + ) + ) + except Exception as exc: # noqa: BLE001 + errors.append(exc) + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert not errors, errors + assert db.read(lambda c: c.execute("SELECT n FROM counter").fetchone()[0]) == 80 + finally: + db.close() + + +def test_close_and_reinitialize(): + with tempfile.TemporaryDirectory() as tmp: + db = make_db(tmp) + db.close() + db.close() # 幂等 + try: + db.write(lambda c: None) + except DatabaseClosed: + pass + else: + raise AssertionError("关闭后写入应失败") + try: + db.read(lambda c: None) + except DatabaseClosed: + pass + else: + raise AssertionError("关闭后读取应失败") + + db2 = make_db(tmp) + try: + assert db2.schema_version == 1 + assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 1 + finally: + db2.close() + + +def test_wal_enabled(): + with tempfile.TemporaryDirectory() as tmp: + db = make_db(tmp) + try: + mode = db.read(lambda c: c.execute("PRAGMA journal_mode").fetchone()[0]) + assert str(mode).lower() == "wal" + assert db.read(lambda c: c.execute("PRAGMA foreign_keys").fetchone()[0]) == 1 + finally: + db.close() + + +def test_migration_backup_on_existing_db(): + """已有数据升级时由 Database 自动备份。""" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "app.db" + # 先建一个旧库（只有 schema_version v0），Database.initialize 会升级并备份 + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE legacy (x INTEGER)") + conn.commit() + conn.close() + db = Database(path, backup_dir=Path(tmp) / "backup") + try: + applied = db.initialize() + assert applied == [1] + assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1 + finally: + db.close() + + +def main(): + test_write_read_transaction() + test_concurrent_writes_are_serialized() + test_close_and_reinitialize() + test_wal_enabled() + test_migration_backup_on_existing_db() + print("OK: 数据库单写队列与迁移入口测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_folder_sync.py b/tests/test_folder_sync.py new file mode 100644 index 0000000..21bd602 --- /dev/null +++ b/tests/test_folder_sync.py @@ -0,0 +1,297 @@ +# -*- coding: utf-8 -*- +"""收藏夹同步集成测试（用真实接口形状的 transport Mock，不联网）: +uv run python tests/test_folder_sync.py +""" + +import json +import os +import sys +import tempfile +import urllib.parse +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from fixtures import info_for # noqa: E402 +from reaction_board.bilibili.client import ( # noqa: E402 + BilibiliClient, + HTTPResponse, + Pacer, + TransportError, +) +from reaction_board.config import ConfigStore # noqa: E402 +from reaction_board.database import Database # noqa: E402 +from reaction_board.repository import Repository # noqa: E402 +from reaction_board.services.folder_sync import FolderSyncService # noqa: E402 + +F1 = 4451053851 +F2 = 3700023351 + + +def http_json(payload): + return HTTPResponse( + 200, {"content-type": "application/json"}, + json.dumps(payload, ensure_ascii=False).encode("utf-8"), + ) + + +class FakeFavTransport: + def __init__(self, folders, infos, *, fail_media=None): + self.folders = {k: list(v) for k, v in folders.items()} + self.infos = dict(infos) + self.fail_media = fail_media + self.ids_calls = [] + self.info_calls = [] + + def request(self, method, url, headers, data=None, timeout=30): + query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query) + if "/x/v3/fav/resource/ids" in url: + media_id = int(query["media_id"][0]) + self.ids_calls.append(media_id) + if self.fail_media == media_id: + raise TransportError("ids 网络错误") + data_rows = [ + {"id": aid, "type": 2, "bv_id": f"BV{aid}", "bvid": f"BV{aid}"} + for aid in self.folders.get(media_id, []) + ] + return http_json({"code": 0, "message": "0", "ttl": 1, "data": data_rows}) + if "/x/v3/fav/resource/infos" in url: + media_id = int(query["folder_id"][0]) + resources = query["resources"][0] + aids = [int(part.split(":")[0]) for part in resources.split(",")] + self.info_calls.append((media_id, aids)) + rows = [self.infos[aid] for aid in aids if aid in self.infos] + return http_json({"code": 0, "message": "0", "ttl": 1, "data": rows}) + if "/x/v3/fav/folder/created/list-all" in url: + return http_json({ + "code": 0, "message": "0", "ttl": 1, + "data": {"count": 0, "list": []}, + }) + raise AssertionError(f"未预期的 URL: {url}") + + +def write_config(tmp, folders): + data = { + "mid": 28802351, + "csrf": "csrf-secret", + "cookie": "SESSDATA=secret; bili_jct=csrf-secret; buvid3=device", + "storage": { + "database_path": str(Path(tmp) / "app.db"), + "backup_dir": str(Path(tmp) / "backup"), + }, + "folders": folders, + "up_blacklist_seed": [999], + "schedule": { + "discover_minutes": 60, + "folder_sync_minutes": 60, + "full_folder_refresh_hours": 6, + "snapshot_at": "00:01", + "candidate_ttl_days": 7, + }, + } + path = Path(tmp) / "config.json" + path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + return path + + +def make_service(tmp, transport, clock, *, capacity_warn=950): + folders = [ + {"year": 2025, "media_id": F1, "title": "2025"}, + {"year": 2026, "media_id": F2, "title": "2026"}, + ] + path = write_config(tmp, folders) + store = ConfigStore(path, env={}, require_auth=True) + db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup") + db.initialize() + repo = Repository(db, clock=lambda: clock[0]) + client = BilibiliClient( + cookie="SESSDATA=secret; bvid3=device", + csrf="csrf", + mid=28802351, + transport=transport, + pacer=Pacer(0, 0), + retries=1, + ) + service = FolderSyncService( + store, repo, client, clock=lambda: clock[0], capacity_warn=capacity_warn + ) + return db, repo, service + + +def valid(aid, *, up_mid=100, title=None): + return info_for(aid, title=title or f"视频{aid}", up_mid=up_mid, up_name="UP") + + +def test_initial_import_and_incremental_skip(): + with tempfile.TemporaryDirectory() as tmp: + transport = FakeFavTransport( + {F1: [1101, 1102, 1103, 1104], F2: []}, + { + 1101: valid(1101), + 1102: valid(1102, up_mid=999), # 黑名单 UP 也必须正常入库 + 1103: valid(1103, title="已失效", up_mid=100), + }, + ) + # 1103 标记为 attr=9；1104 静默缺失 + transport.infos[1103]["attr"] = 9 + clock = [1_800_000_000] + db, repo, service = make_service(tmp, transport, clock) + try: + summary = service.run() + assert summary.ok + assert summary.added == 4 + assert summary.missing == 1 + assert summary.refreshed == 3 + assert repo.seed_blacklist([999], now=0) == 0 # 已由 service 写入 + + assert repo.get_video(1101)["status"] == "active" + assert repo.get_video(1102)["status"] == "active" # 黑名单不影响收藏夹导入 + assert repo.get_video(1103)["status"] == "removed" + assert repo.get_video(1104)["status"] == "removed" + assert repo.get_video(1104)["bvid"] == "BV1104" + assert repo.folder_active_aids(F1) == {1101, 1102, 1103, 1104} + assert repo.is_blacklisted(999) + + folder = repo.get_folder(F1) + assert folder.media_count == 4 + assert folder.last_full_at == 1_800_000_000 + + # 6 小时内再次运行：没有新增，不请求 infos + clock[0] += 60 + info_before = len(transport.info_calls) + summary2 = service.run() + assert summary2.added == 0 + assert summary2.refreshed == 0 + assert len(transport.info_calls) == info_before + assert summary2.folders[0].full_refresh is False + finally: + db.close() + + +def test_diff_add_and_remove(): + with tempfile.TemporaryDirectory() as tmp: + transport = FakeFavTransport( + {F1: [1101, 1102], F2: []}, + {1101: valid(1101), 1102: valid(1102)}, + ) + clock = [1_800_000_000] + db, repo, service = make_service(tmp, transport, clock) + try: + service.run() + assert repo.get_video(1101)["status"] == "active" + assert repo.get_video(1102)["status"] == "active" + + transport.folders[F1] = [1102, 1103] + transport.infos[1103] = valid(1103) + clock[0] += 3600 + summary = service.run() + assert summary.added == 1 + assert summary.removed == 1 + assert summary.refreshed == 1 + assert repo.get_video(1101)["status"] == "unfaved" + assert repo.get_video(1102)["status"] == "active" + assert repo.get_video(1103)["status"] == "active" + assert repo.folder_active_aids(F1) == {1102, 1103} + assert repo.get_folder(F1).media_count == 2 + finally: + db.close() + + +def test_full_refresh_marks_invalid_and_missing(): + with tempfile.TemporaryDirectory() as tmp: + transport = FakeFavTransport( + {F1: [1101, 1102], F2: []}, + {1101: valid(1101), 1102: valid(1102)}, + ) + clock = [1_800_000_000] + db, repo, service = make_service(tmp, transport, clock) + try: + service.run() + assert repo.get_video(1101)["status"] == "active" + assert repo.get_video(1102)["status"] == "active" + + # 6 小时后：1101 失效，1102 静默缺失 + clock[0] += 6 * 3600 + 1 + transport.infos[1101]["attr"] = 9 + del transport.infos[1102] + summary = service.run() + assert summary.folders[0].full_refresh is True + assert summary.missing == 1 + assert repo.get_video(1101)["status"] == "removed" + assert repo.get_video(1102)["status"] == "removed" + assert repo.get_folder(F1).last_full_at == clock[0] + # 全量刷新请求了夹内全部 aid + assert set(transport.info_calls[-1][1]) == {1101, 1102} + finally: + db.close() + + +def test_capacity_warning_does_not_move_folder(): + with tempfile.TemporaryDirectory() as tmp: + transport = FakeFavTransport( + {F1: [1, 2, 3], F2: []}, + {i: valid(i) for i in (1, 2, 3)}, + ) + clock = [1_800_000_000] + db, repo, service = make_service(tmp, transport, clock, capacity_warn=3) + try: + summary = service.run() + assert summary.folders[0].warning + assert "预警线 3" in summary.folders[0].warning + assert "不会自动换夹" in summary.folders[0].warning + # 视频仍在原夹，不被移动 + assert all(repo.get_video(i)["folder_id"] == F1 for i in (1, 2, 3)) + finally: + db.close() + + +def test_folder_error_does_not_block_other_folders(): + with tempfile.TemporaryDirectory() as tmp: + transport = FakeFavTransport( + {F1: [1101], F2: [2201]}, + {1101: valid(1101), 2201: valid(2201)}, + fail_media=F1, + ) + clock = [1_800_000_000] + db, repo, service = make_service(tmp, transport, clock) + try: + summary = service.run() + assert summary.ok is False + assert len(summary.errors) == 1 and str(F1) in summary.errors[0] + assert summary.folders[0].error + assert summary.folders[1].error is None + assert repo.get_video(2201)["status"] == "active" + finally: + db.close() + + +def test_blacklist_seed_only_once(): + with tempfile.TemporaryDirectory() as tmp: + transport = FakeFavTransport({F1: [1101], F2: []}, {1101: valid(1101)}) + clock = [1_800_000_000] + db, repo, service = make_service(tmp, transport, clock) + try: + first = service.run() + assert first.blacklist_seeded == 1 + clock[0] += 60 + second = service.run() + assert second.blacklist_seeded == 0 + assert len(repo.blacklist()) == 1 + finally: + db.close() + + +def main(): + test_initial_import_and_incremental_skip() + test_diff_add_and_remove() + test_full_refresh_marks_invalid_and_missing() + test_capacity_warning_does_not_move_folder() + test_folder_error_does_not_block_other_folders() + test_blacklist_seed_only_once() + print("OK: 收藏夹初始导入、差集同步、失效识别与容量预警测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 0000000..9ff1cdd --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +"""JSON 日志脱敏测试（不联网）: uv run python tests/test_logging.py""" + +import io +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.logging_json import JsonLogger, redact # noqa: E402 + +SECRET_COOKIE = "SESSDATA=abc%2C1800000000%2Cdef; bili_jct=csrf123; buvid3=device123" +SECRET_TOKEN = "123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghi" + + +def test_redact_keys_and_nested(): + data = { + "cookie": SECRET_COOKIE, + "nested": { + "csrf": "csrf123", + "bot_token": SECRET_TOKEN, + "authorization": "Bearer abc.def.ghi", + "safe": "visible", + }, + "items": [{"sessdata": "x"}, {"note": "ok"}], + } + out = redact(data) + assert out["cookie"] == "***" + assert out["nested"]["csrf"] == "***" + assert out["nested"]["bot_token"] == "***" + assert out["nested"]["authorization"] == "***" + assert out["nested"]["safe"] == "visible" + assert out["items"][0]["sessdata"] == "***" + assert out["items"][1]["note"] == "ok" + + +def test_redact_secrets_inside_ordinary_strings(): + text = ( + f"request failed cookie={SECRET_COOKIE} token={SECRET_TOKEN} " + "Authorization: Bearer abc.def" + ) + out = redact({"error": text}) + assert "abc%2C1800000000" not in out["error"] + assert "csrf123" not in out["error"] + assert SECRET_TOKEN not in out["error"] + assert "***" in out["error"] + + +def test_logger_emits_json_lines_without_secrets(): + stream = io.StringIO() + logger = JsonLogger(stream) + logger.info( + "test.event", + cookie=SECRET_COOKIE, + message=f"bad cookie {SECRET_COOKIE}", + nested={"bili_jct": "csrf123", "count": 2}, + list_value=[SECRET_TOKEN], + ) + lines = [line for line in stream.getvalue().splitlines() if line] + assert len(lines) == 1 + payload = json.loads(lines[0]) + assert payload["level"] == "info" + assert payload["event"] == "test.event" + assert payload["cookie"] == "***" + assert payload["nested"]["bili_jct"] == "***" + assert payload["list_value"] == ["***"] + assert "SESSDATA=***" in lines[0] + assert SECRET_TOKEN not in lines[0] + assert "1800000000" not in lines[0] + + +def test_logger_levels(): + stream = io.StringIO() + logger = JsonLogger(stream) + logger.debug("d", x=1) + logger.warning("w", x=2) + logger.error("e", x=3) + rows = [json.loads(line) for line in stream.getvalue().splitlines()] + assert [r["level"] for r in rows] == ["debug", "warning", "error"] + assert [r["event"] for r in rows] == ["d", "w", "e"] + + +def main(): + test_redact_keys_and_nested() + test_redact_secrets_inside_ordinary_strings() + test_logger_emits_json_lines_without_secrets() + test_logger_levels() + print("OK: JSON 日志脱敏测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..50f36ff --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +"""SQLite 迁移测试（不联网）: uv run python tests/test_migrations.py""" + +import os +import sqlite3 +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.migrations import ( # noqa: E402 + MIGRATIONS, + Migration, + MigrationError, + current_version, + prune_backups, + run_migrations, +) + +EXPECTED_TABLES = { + "video", "snapshot", "folder", "folder_resource", "up_blacklist", + "up_stat", "candidate", "seen_aid", "notify", "fav_action", "kv", +} + + +def table_names(conn): + rows = conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ).fetchall() + return {row[0] for row in rows} + + +def test_fresh_and_idempotent(): + with tempfile.TemporaryDirectory() as tmp: + conn = sqlite3.connect(Path(tmp) / "app.db") + try: + applied = run_migrations(conn) + assert applied == [1] + assert current_version(conn) == 1 + assert EXPECTED_TABLES <= table_names(conn) + assert "schema_version" in table_names(conn) + # 再跑一次不应重复应用 + assert run_migrations(conn) == [] + row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone() + assert row[0] == 1 + finally: + conn.close() + + +def test_backup_before_upgrade(): + with tempfile.TemporaryDirectory() as tmp: + db_path = Path(tmp) / "app.db" + backup_dir = Path(tmp) / "backup" + conn = sqlite3.connect(db_path) + try: + run_migrations(conn, backup_dir=backup_dir) + conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')") + conn.commit() + + v2 = Migration(2, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",)) + applied = run_migrations( + conn, + backup_dir=backup_dir, + now=1_800_000_000, + migrations=MIGRATIONS + (v2,), + ) + assert applied == [2] + backups = list(backup_dir.glob("app-pre-v1-*.db")) + assert len(backups) == 1 + check = sqlite3.connect(backups[0]) + try: + assert check.execute("SELECT v FROM kv WHERE k='hello'").fetchone()[0] == "world" + assert "probe" not in table_names(check) + finally: + check.close() + finally: + conn.close() + + +def test_failed_migration_rolls_back(): + with tempfile.TemporaryDirectory() as tmp: + conn = sqlite3.connect(Path(tmp) / "app.db") + bad = Migration( + 1, + "broken", + ( + "CREATE TABLE should_not_exist (id INTEGER PRIMARY KEY)", + "CREATE TABLE broken (", + ), + ) + try: + try: + run_migrations(conn, migrations=(bad,)) + except MigrationError as exc: + assert "已回滚" in str(exc) + else: + raise AssertionError("坏迁移应抛错") + assert "should_not_exist" not in table_names(conn) + assert current_version(conn) == 0 + finally: + conn.close() + + +def test_prune_backups(): + with tempfile.TemporaryDirectory() as tmp: + backup_dir = Path(tmp) + old = backup_dir / "app-pre-v1-old.db" + fresh = backup_dir / "app-pre-v1-fresh.db" + other = backup_dir / "keep.txt" + for path in (old, fresh, other): + path.write_bytes(b"x") + now = 1_800_000_000 + old_time = now - 20 * 86400 + os.utime(old, (old_time, old_time)) + os.utime(fresh, (now, now)) + removed = prune_backups(backup_dir, keep_days=14, now=now) + assert removed == [old] + assert fresh.exists() and other.exists() + + +def test_validate_migration_order(): + with tempfile.TemporaryDirectory() as tmp: + conn = sqlite3.connect(Path(tmp) / "app.db") + try: + try: + run_migrations(conn, migrations=( + Migration(2, "x", ()), + Migration(1, "y", ()), + )) + except ValueError as exc: + assert "升序" in str(exc) + else: + raise AssertionError("乱序迁移应失败") + finally: + conn.close() + + +def main(): + test_fresh_and_idempotent() + test_backup_before_upgrade() + test_failed_migration_rolls_back() + test_prune_backups() + test_validate_migration_order() + print("OK: SQLite 迁移、备份与回滚测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_moves.py b/tests/test_moves.py new file mode 100644 index 0000000..26c5901 --- /dev/null +++ b/tests/test_moves.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +"""build_moves / apply_move 的正确性自测（无需 pytest）: python tests/test_moves.py""" +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from favsort import apply_move, build_moves, lcs_keep # noqa: E402 + + +def run_case(cur, tgt, label): + moves = build_moves(cur, tgt) + work = list(cur) + for m in moves: + assert m["pred"] is not None, f"{label}: pred 为空" + work = apply_move(work, m) + assert work == tgt, f"{label}: 结果不符\n cur={cur}\n tgt={tgt}\n got={work}" + lower = len(lcs_keep(cur, tgt)) + # 不追求严格最优（多轮 LCS 会略多于下界），但不能出现病态膨胀 + assert len(moves) <= len(cur) + 8, ( + f"{label}: 步数异常 {len(moves)} (n={len(cur)}, 下界={len(cur) - lower})" + ) + return len(moves) + + +def main(): + random.seed(20261031) + total = 0 + for n in range(1, 13): + ids = [1000 + i for i in range(n)] + for _ in range(200): + cur = ids[:] + tgt = ids[:] + random.shuffle(cur) + random.shuffle(tgt) + run_case(cur, tgt, f"n={n}") + total += 1 + + # 头部必须换人的用例 + cases = [ + ([3, 2, 1], [1, 2, 3]), + ([5, 4, 3, 2, 1], [1, 2, 3, 4, 5]), + ([2, 3, 1], [1, 2, 3]), + ([4, 1, 2, 3], [1, 2, 3, 4]), + ] + for cur, tgt in cases: + run_case(cur, tgt, f"head {cur}->{tgt}") + + # 已排序不应产生移动 + assert build_moves([1, 2, 3], [1, 2, 3]) == [] + print(f"OK: {total} 个随机用例 + {len(cases)} 个头部用例全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_repository.py b/tests/test_repository.py new file mode 100644 index 0000000..d7a182d --- /dev/null +++ b/tests/test_repository.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- +"""收藏夹/视频状态机测试（不联网）: uv run python tests/test_repository.py""" + +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.bilibili.models import FavResource # noqa: E402 +from reaction_board.config import FolderConfig # noqa: E402 +from reaction_board.database import Database # noqa: E402 +from reaction_board.repository import Repository # noqa: E402 + +F1 = FolderConfig(year=2025, media_id=4451053851, title="2025") +F2 = FolderConfig(year=2026, media_id=3700023351, title="2026") + + +def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None): + return FavResource( + aid=aid, + bvid=bvid or f"BV{aid}", + title=title, + up_mid=up_mid, + up_name="UP", + pubdate=1_700_000_000, + duration=600, + cover="http://example/cover.jpg", + attr=attr, + fav_time=None, + ) + + +def setup(tmp): + db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup") + db.initialize() + repo = Repository(db, clock=lambda: 1_800_000_000) + repo.ensure_folder(F1) + repo.ensure_folder(F2) + return db, repo + + +def test_upsert_and_folder_resource(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + outcome = repo.upsert_from_fav(resource(1), F1.media_id, was_added=True) + assert outcome.status == "active" + assert outcome.previous_folder_id is None + video = repo.get_video(1) + assert video["bvid"] == "BV1" + assert video["folder_id"] == F1.media_id + assert video["source"] == f"fav:{F1.media_id}" + assert video["first_seen"] == 1_800_000_000 + assert repo.folder_active_aids(F1.media_id) == {1} + + # 幂等：重复导入不新增行、不改 first_seen + repo.upsert_from_fav(resource(1, title="新标题"), F1.media_id, was_added=False) + assert len(repo.list_videos()) == 1 + video = repo.get_video(1) + assert video["title"] == "新标题" + assert video["first_seen"] == 1_800_000_000 + finally: + db.close() + + +def test_invalid_attr_and_missing(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + out = repo.upsert_from_fav(resource(2, attr=9), F1.media_id, was_added=True) + assert out.status == "removed" + assert repo.get_video(2)["attr"] == 9 + + # 原本有效的视频后来失效：保留原历史标题 + repo.upsert_from_fav( + resource(4, title="原始标题"), F1.media_id, was_added=True + ) + repo.upsert_from_fav( + resource(4, title="已失效视频", attr=9), F1.media_id, was_added=False + ) + video4 = repo.get_video(4) + assert video4["status"] == "removed" + assert video4["title"] == "原始标题" + + status = repo.mark_missing_resource( + 3, folder_id=F1.media_id, bvid="BV3" + ) + assert status == "removed" + video = repo.get_video(3) + assert video["status"] == "removed" + assert video["bvid"] == "BV3" + assert video["title"] == "(资源已失效)" + assert video["attr"] == 9 + assert 3 in repo.folder_active_aids(F1.media_id) + finally: + db.close() + + +def test_remove_from_folder_transitions(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + for aid in (1, 2): + repo.upsert_from_fav(resource(aid), F1.media_id, was_added=True) + # 2 同时出现在两个夹：从 F1 移除后仍 active + repo.upsert_from_fav(resource(2), F2.media_id, was_added=True) + status = repo.mark_resource_removed(F1.media_id, 2) + assert status == "active" + assert repo.get_video(2)["status"] == "active" + + # 1 只在 F1：移除 -> unfaved，但数据保留 + status = repo.mark_resource_removed(F1.media_id, 1) + assert status == "unfaved" + assert repo.get_video(1)["status"] == "unfaved" + assert repo.get_video(1)["bvid"] == "BV1" + assert 1 not in repo.folder_active_aids(F1.media_id) + + # 再次出现在收藏夹 -> active + out = repo.upsert_from_fav(resource(1), F1.media_id, was_added=True) + assert out.status == "active" + finally: + db.close() + + +def test_dropped_only_revives_on_reappearing_in_folder(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + repo.upsert_from_fav(resource(1), F1.media_id, was_added=True) + repo.set_video_status(1, "dropped") + # 全量刷新（未重新加入）保持 dropped + out = repo.upsert_from_fav(resource(1), F1.media_id, was_added=False) + assert out.status == "dropped" + # 从收藏夹移除也不改变 dropped + repo.mark_resource_removed(F1.media_id, 1) + assert repo.get_video(1)["status"] == "dropped" + # 重新出现在收藏夹 -> 自动恢复 active + out = repo.upsert_from_fav(resource(1), F1.media_id, was_added=True) + assert out.status == "active" + finally: + db.close() + + +def test_removed_stays_removed_when_folder_removed(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + repo.upsert_from_fav(resource(1, attr=9), F1.media_id, was_added=True) + assert repo.get_video(1)["status"] == "removed" + status = repo.mark_resource_removed(F1.media_id, 1) + assert status == "removed" + finally: + db.close() + + +def test_multi_folder_moves_primary_and_warns(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + repo.upsert_from_fav(resource(1), F1.media_id, was_added=True) + out = repo.upsert_from_fav(resource(1), F2.media_id, was_added=True) + assert out.moved_folder is True + assert out.previous_folder_id == F1.media_id + assert repo.get_video(1)["folder_id"] == F2.media_id + # 两个夹的资源关系都保留 + assert repo.folder_active_aids(F1.media_id) == {1} + assert repo.folder_active_aids(F2.media_id) == {1} + finally: + db.close() + + +def test_blacklist_seed_idempotent(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + assert repo.seed_blacklist([1, 2, 3], now=100) == 3 + assert repo.seed_blacklist([2, 3, 4], now=200) == 1 + rows = repo.blacklist() + assert [r["mid"] for r in rows] == [1, 2, 3, 4] + assert all(r["reason"] == "seed" for r in rows) + assert repo.is_blacklisted(2) and not repo.is_blacklisted(99) + repo.add_blacklist(9, name="切片号", reason="manual") + assert repo.is_blacklisted(9) + repo.add_blacklist(9, reason="updated") + row = next(r for r in repo.blacklist() if r["mid"] == 9) + assert row["name"] == "切片号" + assert row["reason"] == "updated" + finally: + db.close() + + +def test_folder_sync_metadata_and_counts(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + repo.upsert_from_fav(resource(1), F1.media_id, was_added=True) + repo.finish_folder_sync( + F1, media_count=1, full_refresh=True, now=1_800_000_000 + ) + row = repo.get_folder(F1.media_id) + assert row.media_count == 1 + assert row.last_ids_at == 1_800_000_000 + assert row.last_full_at == 1_800_000_000 + + repo.finish_folder_sync( + F1, media_count=2, full_refresh=False, now=1_800_000_100 + ) + row = repo.get_folder(F1.media_id) + assert row.media_count == 2 + assert row.last_ids_at == 1_800_000_100 + assert row.last_full_at == 1_800_000_000 + + # 配置中移除的年度夹停用，重新加入时恢复 + repo.disable_folders_except([F1.media_id]) + assert repo.get_folder(F2.media_id).enabled is False + repo.ensure_folder(F2) + assert repo.get_folder(F2.media_id).enabled is True + + counts = repo.counts() + assert counts["video_total"] == 1 + assert counts["video_by_status"] == {"active": 1} + assert counts["folders"] == 2 + finally: + db.close() + + +def main(): + test_upsert_and_folder_resource() + test_invalid_attr_and_missing() + test_remove_from_folder_transitions() + test_dropped_only_revives_on_reappearing_in_folder() + test_removed_stays_removed_when_folder_removed() + test_multi_folder_moves_primary_and_warns() + test_blacklist_seed_idempotent() + test_folder_sync_metadata_and_counts() + print("OK: 收藏夹/视频状态机与黑名单测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 0000000..f6669d3 --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +"""内置调度器测试（离线）: uv run python tests/test_scheduler.py""" + +import os +import sys +import threading + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding="utf-8") + +from reaction_board.scheduler import ScheduledTask, Scheduler # noqa: E402 + + +def test_run_when_due_and_reschedule(): + clock = [1_000.0] + calls = [] + + task = ScheduledTask("sync", 60, lambda: calls.append(clock[0])) + scheduler = Scheduler([task], clock=lambda: clock[0]) + assert scheduler.run_due() and len(calls) == 1 + assert task.next_run == 1_060.0 + + clock[0] = 1_059.0 + assert scheduler.run_due() == [] + assert len(calls) == 1 + + clock[0] = 1_060.0 + results = scheduler.run_due() + assert len(results) == 1 + assert calls == [1_000.0, 1_060.0] + assert task.next_run == 1_120.0 + + +def test_task_error_does_not_block_other_tasks(): + clock = [100.0] + calls = [] + + def boom(): + raise RuntimeError("api down") + + scheduler = Scheduler( + [ + ScheduledTask("bad", 10, boom), + ScheduledTask("good", 10, lambda: calls.append("ok")), + ], + clock=lambda: clock[0], + ) + results = scheduler.run_due() + assert [r.ok for r in results] == [False, True] + assert results[0].error == "api down" + assert calls == ["ok"] + # 失败任务仍然被重新排期 + assert all(t.next_run == 110.0 for t in scheduler.tasks) + + +def test_run_forever_with_stop_event(): + clock = [0.0] + calls = [] + + def tick(): + calls.append(clock[0]) + clock[0] += 10 + + stop = threading.Event() + scheduler = Scheduler( + [ScheduledTask("tick", 10, tick)], + clock=lambda: clock[0], + sleeper=lambda seconds: None, + ) + + def stop_soon(): + while len(calls) < 3: + pass + stop.set() + + waiter = threading.Thread(target=stop_soon) + waiter.start() + scheduler.run_forever(stop_event=stop, poll_seconds=0.001) + waiter.join(timeout=2) + assert len(calls) >= 3 + + +def test_max_iterations_and_validation(): + calls = [] + tick_clock = [0.0] + + def next_time(): + tick_clock[0] += 1.0 + return tick_clock[0] + + scheduler = Scheduler( + [ScheduledTask("t", 1, lambda: calls.append(1))], + clock=next_time, + sleeper=lambda seconds: None, + ) + scheduler.run_forever(max_iterations=2) + assert len(calls) == 2 + + for tasks, kwargs in [ + ([], {}), + ([ScheduledTask("bad", 0, lambda: None)], {}), + ([ScheduledTask("t", 1, lambda: None)], {"poll_seconds": 0}), + ]: + try: + s = Scheduler(tasks, sleeper=lambda seconds: None) + s.run_forever(max_iterations=1, **kwargs) + except ValueError: + pass + else: + raise AssertionError(f"应当失败: {tasks}, {kwargs}") + + +def main(): + test_run_when_due_and_reschedule() + test_task_error_does_not_block_other_tasks() + test_run_forever_with_stop_event() + test_max_iterations_and_validation() + print("OK: 内置调度器测试全部通过") + + +if __name__ == "__main__": + main() diff --git a/tools/probe_anon.py b/tools/probe_anon.py new file mode 100644 index 0000000..5f5df23 --- /dev/null +++ b/tools/probe_anon.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +"""只读接口的身份校验探测：对比 完整 cookie / 无 cookie / 去掉身份字段 三种情况。 + +用法: python tools/probe_anon.py +""" +import json +import os +import sys +import time +import urllib.parse +import urllib.request + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +sys.stdout.reconfigure(encoding="utf-8") + +from favsort import strip_identity # noqa: E402 与主程序保持同一套剥离规则 + +BASE = "https://api.bilibili.com" +UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36") + +BASE_HEADERS = { + "accept": "*/*", + "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7", + "dnt": "1", + "origin": "https://www.bilibili.com", + "referer": "https://www.bilibili.com/", + "sec-ch-ua": '"Google Chrome";v="155", "Chromium";v="155", "Not(A:Brand";v="24"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-site", + "user-agent": UA, +} + + +def get(url, cookie): + headers = dict(BASE_HEADERS) + if cookie is not None: + headers["cookie"] = cookie + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode("utf-8", "replace")) + except Exception as e: # noqa: BLE001 + return {"code": "EXC", "message": str(e)[:60]} + + +def main(): + cfg = json.load(open(os.path.join(ROOT, "config.json"), encoding="utf-8")) + full = cfg["cookie"] + anon = strip_identity(full) + + variants = [("完整 cookie", full), ("去掉身份字段", anon), ("完全无 cookie", None)] + folders = [(3700023351, "咻咻满 reaction 合集 (658)"), + (4451053851, "咻咻满-reaction-2025 (167)"), + (85420451, "默认收藏夹 (29)")] + + print("=== ids 接口 ===") + for name, ck in variants: + for mid, label in folders: + url = f"{BASE}/x/v3/fav/resource/ids?media_id={mid}&platform=web" + r = get(url, ck) + n = len(r.get("data") or []) if isinstance(r.get("data"), list) else "-" + print(f" [{name:<14}] media={mid:<11} code={r.get('code')!s:<5} " + f"条数={n!s:<5} msg={r.get('message')}") + time.sleep(1.2) + print() + + print("=== infos 接口（各取前 3 条）===") + for mid, label in folders: + ids = get(f"{BASE}/x/v3/fav/resource/ids?media_id={mid}&platform=web", full) + arr = ids.get("data") or [] + res = ",".join(f"{x['id']}:{x['type']}" for x in arr[:3]) + if not res: + print(f" media={mid}: 没有可测资源") + continue + for name, ck in variants: + url = (f"{BASE}/x/v3/fav/resource/infos?resources={urllib.parse.quote(res)}" + f"&platform=web&folder_mid={cfg['mid']}&folder_id={mid}") + r = get(url, ck) + d = r.get("data") + n = len(d) if isinstance(d, list) else "-" + first = (d[0].get("title", "")[:18] if isinstance(d, list) and d else "") + print(f" [{name:<14}] media={mid:<11} code={r.get('code')!s:<5} " + f"返回={n!s:<3} msg={r.get('message')} {first}") + time.sleep(1.2) + print() + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..67d7bc6 --- /dev/null +++ b/uv.lock @@ -0,0 +1,13 @@ +version = 1 +revision = 3 +requires-python = "==3.14.*" + +[[package]] +name = "bili-fav" +version = "0.1.0" +source = { editable = "." } + +[package.metadata] + +[package.metadata.requires-dev] +dev = [] diff --git a/vendor/qrcodegen.py b/vendor/qrcodegen.py new file mode 100644 index 0000000..0eb24ea --- /dev/null +++ b/vendor/qrcodegen.py @@ -0,0 +1,907 @@ +# +# QR Code generator library (Python) +# +# Copyright (c) Project Nayuki. (MIT License) +# https://www.nayuki.io/page/qr-code-generator-library +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# - The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# - The Software is provided "as is", without warranty of any kind, express or +# implied, including but not limited to the warranties of merchantability, +# fitness for a particular purpose and noninfringement. In no event shall the +# authors or copyright holders be liable for any claim, damages or other +# liability, whether in an action of contract, tort or otherwise, arising from, +# out of or in connection with the Software or the use or other dealings in the +# Software. +# + +from __future__ import annotations +import collections, itertools, re +from collections.abc import Sequence +from typing import Optional, Union + + +# ---- QR Code symbol class ---- + +class QrCode: + """A QR Code symbol, which is a type of two-dimension barcode. + Invented by Denso Wave and described in the ISO/IEC 18004 standard. + Instances of this class represent an immutable square grid of dark and light cells. + The class provides static factory functions to create a QR Code from text or binary data. + The class covers the QR Code Model 2 specification, supporting all versions (sizes) + from 1 to 40, all 4 error correction levels, and 4 character encoding modes. + + Ways to create a QR Code object: + - High level: Take the payload data and call QrCode.encode_text() or QrCode.encode_binary(). + - Mid level: Custom-make the list of segments and call QrCode.encode_segments(). + - Low level: Custom-make the array of data codeword bytes (including + segment headers and final padding, excluding error correction codewords), + supply the appropriate version number, and call the QrCode() constructor. + (Note that all ways require supplying the desired error correction level.)""" + + # ---- Static factory functions (high level) ---- + + @staticmethod + def encode_text(text: str, ecl: QrCode.Ecc) -> QrCode: + """Returns a QR Code representing the given Unicode text string at the given error correction level. + As a conservative upper bound, this function is guaranteed to succeed for strings that have 738 or fewer + Unicode code points (not UTF-16 code units) if the low error correction level is used. The smallest possible + QR Code version is automatically chosen for the output. The ECC level of the result may be higher than the + ecl argument if it can be done without increasing the version.""" + segs: list[QrSegment] = QrSegment.make_segments(text) + return QrCode.encode_segments(segs, ecl) + + + @staticmethod + def encode_binary(data: Union[bytes,Sequence[int]], ecl: QrCode.Ecc) -> QrCode: + """Returns a QR Code representing the given binary data at the given error correction level. + This function always encodes using the binary segment mode, not any text mode. The maximum number of + bytes allowed is 2953. The smallest possible QR Code version is automatically chosen for the output. + The ECC level of the result may be higher than the ecl argument if it can be done without increasing the version.""" + return QrCode.encode_segments([QrSegment.make_bytes(data)], ecl) + + + # ---- Static factory functions (mid level) ---- + + @staticmethod + def encode_segments(segs: Sequence[QrSegment], ecl: QrCode.Ecc, minversion: int = 1, maxversion: int = 40, mask: int = -1, boostecl: bool = True) -> QrCode: + """Returns a QR Code representing the given segments with the given encoding parameters. + The smallest possible QR Code version within the given range is automatically + chosen for the output. Iff boostecl is true, then the ECC level of the result + may be higher than the ecl argument if it can be done without increasing the + version. The mask number is either between 0 to 7 (inclusive) to force that + mask, or -1 to automatically choose an appropriate mask (which may be slow). + This function allows the user to create a custom sequence of segments that switches + between modes (such as alphanumeric and byte) to encode text in less space. + This is a mid-level API; the high-level API is encode_text() and encode_binary().""" + + if not (QrCode.MIN_VERSION <= minversion <= maxversion <= QrCode.MAX_VERSION) or not (-1 <= mask <= 7): + raise ValueError("Invalid value") + + # Find the minimal version number to use + for version in range(minversion, maxversion + 1): + datacapacitybits: int = QrCode._get_num_data_codewords(version, ecl) * 8 # Number of data bits available + datausedbits: Optional[int] = QrSegment.get_total_bits(segs, version) + if (datausedbits is not None) and (datausedbits <= datacapacitybits): + break # This version number is found to be suitable + if version >= maxversion: # All versions in the range could not fit the given data + msg: str = "Segment too long" + if datausedbits is not None: + msg = f"Data length = {datausedbits} bits, Max capacity = {datacapacitybits} bits" + raise DataTooLongError(msg) + assert datausedbits is not None + + # Increase the error correction level while the data still fits in the current version number + for newecl in (QrCode.Ecc.MEDIUM, QrCode.Ecc.QUARTILE, QrCode.Ecc.HIGH): # From low to high + if boostecl and (datausedbits <= QrCode._get_num_data_codewords(version, newecl) * 8): + ecl = newecl + + # Concatenate all segments to create the data bit string + bb = _BitBuffer() + for seg in segs: + bb.append_bits(seg.get_mode().get_mode_bits(), 4) + bb.append_bits(seg.get_num_chars(), seg.get_mode().num_char_count_bits(version)) + bb.extend(seg._bitdata) + assert len(bb) == datausedbits + + # Add terminator and pad up to a byte if applicable + datacapacitybits = QrCode._get_num_data_codewords(version, ecl) * 8 + assert len(bb) <= datacapacitybits + bb.append_bits(0, min(4, datacapacitybits - len(bb))) + bb.append_bits(0, -len(bb) % 8) # Note: Python's modulo on negative numbers behaves better than C family languages + assert len(bb) % 8 == 0 + + # Pad with alternating bytes until data capacity is reached + for padbyte in itertools.cycle((0xEC, 0x11)): + if len(bb) >= datacapacitybits: + break + bb.append_bits(padbyte, 8) + + # Pack bits into bytes in big endian + datacodewords = bytearray([0] * (len(bb) // 8)) + for (i, bit) in enumerate(bb): + datacodewords[i >> 3] |= bit << (7 - (i & 7)) + + # Create the QR Code object + return QrCode(version, ecl, datacodewords, mask) + + + # ---- Private fields ---- + + # The version number of this QR Code, which is between 1 and 40 (inclusive). + # This determines the size of this barcode. + _version: int + + # The width and height of this QR Code, measured in modules, between + # 21 and 177 (inclusive). This is equal to version * 4 + 17. + _size: int + + # The error correction level used in this QR Code. + _errcorlvl: QrCode.Ecc + + # The index of the mask pattern used in this QR Code, which is between 0 and 7 (inclusive). + # Even if a QR Code is created with automatic masking requested (mask = -1), + # the resulting object still has a mask value between 0 and 7. + _mask: int + + # The modules of this QR Code (False = light, True = dark). + # Immutable after constructor finishes. Accessed through get_module(). + _modules: list[list[bool]] + + # Indicates function modules that are not subjected to masking. Discarded when constructor finishes. + _isfunction: list[list[bool]] + + + # ---- Constructor (low level) ---- + + def __init__(self, version: int, errcorlvl: QrCode.Ecc, datacodewords: Union[bytes,Sequence[int]], msk: int) -> None: + """Creates a new QR Code with the given version number, + error correction level, data codeword bytes, and mask number. + This is a low-level API that most users should not use directly. + A mid-level API is the encode_segments() function.""" + + # Check scalar arguments and set fields + if not (QrCode.MIN_VERSION <= version <= QrCode.MAX_VERSION): + raise ValueError("Version value out of range") + if not (-1 <= msk <= 7): + raise ValueError("Mask value out of range") + + self._version = version + self._size = version * 4 + 17 + self._errcorlvl = errcorlvl + + # Initialize both grids to be size*size arrays of Boolean false + self._modules = [[False] * self._size for _ in range(self._size)] # Initially all light + self._isfunction = [[False] * self._size for _ in range(self._size)] + + # Compute ECC, draw modules + self._draw_function_patterns() + allcodewords: bytes = self._add_ecc_and_interleave(bytearray(datacodewords)) + self._draw_codewords(allcodewords) + + # Do masking + if msk == -1: # Automatically choose best mask + minpenalty: int = 1 << 32 + for i in range(8): + self._apply_mask(i) + self._draw_format_bits(i) + penalty = self._get_penalty_score() + if penalty < minpenalty: + msk = i + minpenalty = penalty + self._apply_mask(i) # Undoes the mask due to XOR + assert 0 <= msk <= 7 + self._mask = msk + self._apply_mask(msk) # Apply the final choice of mask + self._draw_format_bits(msk) # Overwrite old format bits + + del self._isfunction + + + # ---- Accessor methods ---- + + def get_version(self) -> int: + """Returns this QR Code's version number, in the range [1, 40].""" + return self._version + + def get_size(self) -> int: + """Returns this QR Code's size, in the range [21, 177].""" + return self._size + + def get_error_correction_level(self) -> QrCode.Ecc: + """Returns this QR Code's error correction level.""" + return self._errcorlvl + + def get_mask(self) -> int: + """Returns this QR Code's mask, in the range [0, 7].""" + return self._mask + + def get_module(self, x: int, y: int) -> bool: + """Returns the color of the module (pixel) at the given coordinates, which is False + for light or True for dark. The top left corner has the coordinates (x=0, y=0). + If the given coordinates are out of bounds, then False (light) is returned.""" + return (0 <= x < self._size) and (0 <= y < self._size) and self._modules[y][x] + + + # ---- Private helper methods for constructor: Drawing function modules ---- + + def _draw_function_patterns(self) -> None: + """Reads this object's version field, and draws and marks all function modules.""" + # Draw horizontal and vertical timing patterns + for i in range(self._size): + self._set_function_module(6, i, i % 2 == 0) + self._set_function_module(i, 6, i % 2 == 0) + + # Draw 3 finder patterns (all corners except bottom right; overwrites some timing modules) + self._draw_finder_pattern(3, 3) + self._draw_finder_pattern(self._size - 4, 3) + self._draw_finder_pattern(3, self._size - 4) + + # Draw numerous alignment patterns + alignpatpos: list[int] = self._get_alignment_pattern_positions() + numalign: int = len(alignpatpos) + skips: Sequence[tuple[int,int]] = ((0, 0), (0, numalign - 1), (numalign - 1, 0)) + for i in range(numalign): + for j in range(numalign): + if (i, j) not in skips: # Don't draw on the three finder corners + self._draw_alignment_pattern(alignpatpos[i], alignpatpos[j]) + + # Draw configuration data + self._draw_format_bits(0) # Dummy mask value; overwritten later in the constructor + self._draw_version() + + + def _draw_format_bits(self, mask: int) -> None: + """Draws two copies of the format bits (with its own error correction code) + based on the given mask and this object's error correction level field.""" + # Calculate error correction code and pack bits + data: int = self._errcorlvl.formatbits << 3 | mask # errCorrLvl is uint2, mask is uint3 + rem: int = data + for _ in range(10): + rem = (rem << 1) ^ ((rem >> 9) * 0x537) + bits: int = (data << 10 | rem) ^ 0x5412 # uint15 + assert bits >> 15 == 0 + + # Draw first copy + for i in range(0, 6): + self._set_function_module(8, i, _get_bit(bits, i)) + self._set_function_module(8, 7, _get_bit(bits, 6)) + self._set_function_module(8, 8, _get_bit(bits, 7)) + self._set_function_module(7, 8, _get_bit(bits, 8)) + for i in range(9, 15): + self._set_function_module(14 - i, 8, _get_bit(bits, i)) + + # Draw second copy + for i in range(0, 8): + self._set_function_module(self._size - 1 - i, 8, _get_bit(bits, i)) + for i in range(8, 15): + self._set_function_module(8, self._size - 15 + i, _get_bit(bits, i)) + self._set_function_module(8, self._size - 8, True) # Always dark + + + def _draw_version(self) -> None: + """Draws two copies of the version bits (with its own error correction code), + based on this object's version field, iff 7 <= version <= 40.""" + if self._version < 7: + return + + # Calculate error correction code and pack bits + rem: int = self._version # version is uint6, in the range [7, 40] + for _ in range(12): + rem = (rem << 1) ^ ((rem >> 11) * 0x1F25) + bits: int = self._version << 12 | rem # uint18 + assert bits >> 18 == 0 + + # Draw two copies + for i in range(18): + bit: bool = _get_bit(bits, i) + a: int = self._size - 11 + i % 3 + b: int = i // 3 + self._set_function_module(a, b, bit) + self._set_function_module(b, a, bit) + + + def _draw_finder_pattern(self, x: int, y: int) -> None: + """Draws a 9*9 finder pattern including the border separator, + with the center module at (x, y). Modules can be out of bounds.""" + for dy in range(-4, 5): + for dx in range(-4, 5): + xx, yy = x + dx, y + dy + if (0 <= xx < self._size) and (0 <= yy < self._size): + # Chebyshev/infinity norm + self._set_function_module(xx, yy, max(abs(dx), abs(dy)) not in (2, 4)) + + + def _draw_alignment_pattern(self, x: int, y: int) -> None: + """Draws a 5*5 alignment pattern, with the center module + at (x, y). All modules must be in bounds.""" + for dy in range(-2, 3): + for dx in range(-2, 3): + self._set_function_module(x + dx, y + dy, max(abs(dx), abs(dy)) != 1) + + + def _set_function_module(self, x: int, y: int, isdark: bool) -> None: + """Sets the color of a module and marks it as a function module. + Only used by the constructor. Coordinates must be in bounds.""" + assert type(isdark) is bool + self._modules[y][x] = isdark + self._isfunction[y][x] = True + + + # ---- Private helper methods for constructor: Codewords and masking ---- + + def _add_ecc_and_interleave(self, data: bytearray) -> bytes: + """Returns a new byte string representing the given data with the appropriate error correction + codewords appended to it, based on this object's version and error correction level.""" + version: int = self._version + assert len(data) == QrCode._get_num_data_codewords(version, self._errcorlvl) + + # Calculate parameter numbers + numblocks: int = QrCode._NUM_ERROR_CORRECTION_BLOCKS[self._errcorlvl.ordinal][version] + blockecclen: int = QrCode._ECC_CODEWORDS_PER_BLOCK [self._errcorlvl.ordinal][version] + rawcodewords: int = QrCode._get_num_raw_data_modules(version) // 8 + numshortblocks: int = numblocks - rawcodewords % numblocks + shortblocklen: int = rawcodewords // numblocks + + # Split data into blocks and append ECC to each block + blocks: list[bytes] = [] + rsdiv: bytes = QrCode._reed_solomon_compute_divisor(blockecclen) + k: int = 0 + for i in range(numblocks): + dat: bytearray = data[k : k + shortblocklen - blockecclen + (0 if i < numshortblocks else 1)] + k += len(dat) + ecc: bytes = QrCode._reed_solomon_compute_remainder(dat, rsdiv) + if i < numshortblocks: + dat.append(0) + blocks.append(dat + ecc) + assert k == len(data) + + # Interleave (not concatenate) the bytes from every block into a single sequence + result = bytearray() + for i in range(len(blocks[0])): + for (j, blk) in enumerate(blocks): + # Skip the padding byte in short blocks + if (i != shortblocklen - blockecclen) or (j >= numshortblocks): + result.append(blk[i]) + assert len(result) == rawcodewords + return result + + + def _draw_codewords(self, data: bytes) -> None: + """Draws the given sequence of 8-bit codewords (data and error correction) onto the entire + data area of this QR Code. Function modules need to be marked off before this is called.""" + assert len(data) == QrCode._get_num_raw_data_modules(self._version) // 8 + + i: int = 0 # Bit index into the data + # Do the funny zigzag scan + for right in range(self._size - 1, 0, -2): # Index of right column in each column pair + if right <= 6: + right -= 1 + for vert in range(self._size): # Vertical counter + for j in range(2): + x: int = right - j # Actual x coordinate + upward: bool = (right + 1) & 2 == 0 + y: int = (self._size - 1 - vert) if upward else vert # Actual y coordinate + if (not self._isfunction[y][x]) and (i < len(data) * 8): + self._modules[y][x] = _get_bit(data[i >> 3], 7 - (i & 7)) + i += 1 + # If this QR Code has any remainder bits (0 to 7), they were assigned as + # 0/false/light by the constructor and are left unchanged by this method + assert i == len(data) * 8 + + + def _apply_mask(self, mask: int) -> None: + """XORs the codeword modules in this QR Code with the given mask pattern. + The function modules must be marked and the codeword bits must be drawn + before masking. Due to the arithmetic of XOR, calling _apply_mask() with + the same mask value a second time will undo the mask. A final well-formed + QR Code needs exactly one (not zero, two, etc.) mask applied.""" + if not (0 <= mask <= 7): + raise ValueError("Mask value out of range") + masker: collections.abc.Callable[[int,int],int] = QrCode._MASK_PATTERNS[mask] + for y in range(self._size): + for x in range(self._size): + self._modules[y][x] ^= (masker(x, y) == 0) and (not self._isfunction[y][x]) + + + def _get_penalty_score(self) -> int: + """Calculates and returns the penalty score based on state of this QR Code's current modules. + This is used by the automatic mask choice algorithm to find the mask pattern that yields the lowest score.""" + result: int = 0 + size: int = self._size + modules: list[list[bool]] = self._modules + + # Adjacent modules in row having same color, and finder-like patterns + for y in range(size): + runcolor: bool = False + runx: int = 0 + runhistory = collections.deque([0] * 7, 7) + for x in range(size): + if modules[y][x] == runcolor: + runx += 1 + if runx == 5: + result += QrCode._PENALTY_N1 + elif runx > 5: + result += 1 + else: + self._finder_penalty_add_history(runx, runhistory) + if not runcolor: + result += self._finder_penalty_count_patterns(runhistory) * QrCode._PENALTY_N3 + runcolor = modules[y][x] + runx = 1 + result += self._finder_penalty_terminate_and_count(runcolor, runx, runhistory) * QrCode._PENALTY_N3 + # Adjacent modules in column having same color, and finder-like patterns + for x in range(size): + runcolor = False + runy: int = 0 + runhistory = collections.deque([0] * 7, 7) + for y in range(size): + if modules[y][x] == runcolor: + runy += 1 + if runy == 5: + result += QrCode._PENALTY_N1 + elif runy > 5: + result += 1 + else: + self._finder_penalty_add_history(runy, runhistory) + if not runcolor: + result += self._finder_penalty_count_patterns(runhistory) * QrCode._PENALTY_N3 + runcolor = modules[y][x] + runy = 1 + result += self._finder_penalty_terminate_and_count(runcolor, runy, runhistory) * QrCode._PENALTY_N3 + + # 2*2 blocks of modules having same color + for y in range(size - 1): + for x in range(size - 1): + if modules[y][x] == modules[y][x + 1] == modules[y + 1][x] == modules[y + 1][x + 1]: + result += QrCode._PENALTY_N2 + + # Balance of dark and light modules + dark: int = sum((1 if cell else 0) for row in modules for cell in row) + total: int = size**2 # Note that size is odd, so dark/total != 1/2 + # Compute the smallest integer k >= 0 such that (45-5k)% <= dark/total <= (55+5k)% + k: int = (abs(dark * 20 - total * 10) + total - 1) // total - 1 + assert 0 <= k <= 9 + result += k * QrCode._PENALTY_N4 + assert 0 <= result <= 2568888 # Non-tight upper bound based on default values of PENALTY_N1, ..., N4 + return result + + + # ---- Private helper functions ---- + + def _get_alignment_pattern_positions(self) -> list[int]: + """Returns an ascending list of positions of alignment patterns for this version number. + Each position is in the range [0,177), and are used on both the x and y axes. + This could be implemented as lookup table of 40 variable-length lists of integers.""" + if self._version == 1: + return [] + else: + numalign: int = self._version // 7 + 2 + step: int = (self._version * 8 + numalign * 3 + 5) // (numalign * 4 - 4) * 2 + result: list[int] = [(self._size - 7 - i * step) for i in range(numalign - 1)] + [6] + return list(reversed(result)) + + + @staticmethod + def _get_num_raw_data_modules(ver: int) -> int: + """Returns the number of data bits that can be stored in a QR Code of the given version number, after + all function modules are excluded. This includes remainder bits, so it might not be a multiple of 8. + The result is in the range [208, 29648]. This could be implemented as a 40-entry lookup table.""" + if not (QrCode.MIN_VERSION <= ver <= QrCode.MAX_VERSION): + raise ValueError("Version number out of range") + result: int = (16 * ver + 128) * ver + 64 + if ver >= 2: + numalign: int = ver // 7 + 2 + result -= (25 * numalign - 10) * numalign - 55 + if ver >= 7: + result -= 36 + assert 208 <= result <= 29648 + return result + + + @staticmethod + def _get_num_data_codewords(ver: int, ecl: QrCode.Ecc) -> int: + """Returns the number of 8-bit data (i.e. not error correction) codewords contained in any + QR Code of the given version number and error correction level, with remainder bits discarded. + This stateless pure function could be implemented as a (40*4)-cell lookup table.""" + return QrCode._get_num_raw_data_modules(ver) // 8 \ + - QrCode._ECC_CODEWORDS_PER_BLOCK [ecl.ordinal][ver] \ + * QrCode._NUM_ERROR_CORRECTION_BLOCKS[ecl.ordinal][ver] + + + @staticmethod + def _reed_solomon_compute_divisor(degree: int) -> bytes: + """Returns a Reed-Solomon ECC generator polynomial for the given degree. This could be + implemented as a lookup table over all possible parameter values, instead of as an algorithm.""" + if not (1 <= degree <= 255): + raise ValueError("Degree out of range") + # Polynomial coefficients are stored from highest to lowest power, excluding the leading term which is always 1. + # For example the polynomial x^3 + 255x^2 + 8x + 93 is stored as the uint8 array [255, 8, 93]. + result = bytearray([0] * (degree - 1) + [1]) # Start off with the monomial x^0 + + # Compute the product polynomial (x - r^0) * (x - r^1) * (x - r^2) * ... * (x - r^{degree-1}), + # and drop the highest monomial term which is always 1x^degree. + # Note that r = 0x02, which is a generator element of this field GF(2^8/0x11D). + root: int = 1 + for _ in range(degree): # Unused variable i + # Multiply the current product by (x - r^i) + for j in range(degree): + result[j] = QrCode._reed_solomon_multiply(result[j], root) + if j + 1 < degree: + result[j] ^= result[j + 1] + root = QrCode._reed_solomon_multiply(root, 0x02) + return result + + + @staticmethod + def _reed_solomon_compute_remainder(data: bytes, divisor: bytes) -> bytes: + """Returns the Reed-Solomon error correction codeword for the given data and divisor polynomials.""" + result = bytearray([0] * len(divisor)) + for b in data: # Polynomial division + factor: int = b ^ result.pop(0) + result.append(0) + for (i, coef) in enumerate(divisor): + result[i] ^= QrCode._reed_solomon_multiply(coef, factor) + return result + + + @staticmethod + def _reed_solomon_multiply(x: int, y: int) -> int: + """Returns the product of the two given field elements modulo GF(2^8/0x11D). The arguments and result + are unsigned 8-bit integers. This could be implemented as a lookup table of 256*256 entries of uint8.""" + if (x >> 8 != 0) or (y >> 8 != 0): + raise ValueError("Byte out of range") + # Russian peasant multiplication + z: int = 0 + for i in reversed(range(8)): + z = (z << 1) ^ ((z >> 7) * 0x11D) + z ^= ((y >> i) & 1) * x + assert z >> 8 == 0 + return z + + + def _finder_penalty_count_patterns(self, runhistory: collections.deque[int]) -> int: + """Can only be called immediately after a light run is added, and + returns either 0, 1, or 2. A helper function for _get_penalty_score().""" + n: int = runhistory[1] + assert n <= self._size * 3 + core: bool = n > 0 and (runhistory[2] == runhistory[4] == runhistory[5] == n) and runhistory[3] == n * 3 + return (1 if (core and runhistory[0] >= n * 4 and runhistory[6] >= n) else 0) \ + + (1 if (core and runhistory[6] >= n * 4 and runhistory[0] >= n) else 0) + + + def _finder_penalty_terminate_and_count(self, currentruncolor: bool, currentrunlength: int, runhistory: collections.deque[int]) -> int: + """Must be called at the end of a line (row or column) of modules. A helper function for _get_penalty_score().""" + if currentruncolor: # Terminate dark run + self._finder_penalty_add_history(currentrunlength, runhistory) + currentrunlength = 0 + currentrunlength += self._size # Add light border to final run + self._finder_penalty_add_history(currentrunlength, runhistory) + return self._finder_penalty_count_patterns(runhistory) + + + def _finder_penalty_add_history(self, currentrunlength: int, runhistory: collections.deque[int]) -> None: + if runhistory[0] == 0: + currentrunlength += self._size # Add light border to initial run + runhistory.appendleft(currentrunlength) + + + # ---- Constants and tables ---- + + MIN_VERSION: int = 1 # The minimum version number supported in the QR Code Model 2 standard + MAX_VERSION: int = 40 # The maximum version number supported in the QR Code Model 2 standard + + # For use in _get_penalty_score(), when evaluating which mask is best. + _PENALTY_N1: int = 3 + _PENALTY_N2: int = 3 + _PENALTY_N3: int = 40 + _PENALTY_N4: int = 10 + + _ECC_CODEWORDS_PER_BLOCK: Sequence[Sequence[int]] = ( + # Version: (note that index 0 is for padding, and is set to an illegal value) + # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 Error correction level + (-1, 7, 10, 15, 20, 26, 18, 20, 24, 30, 18, 20, 24, 26, 30, 22, 24, 28, 30, 28, 28, 28, 28, 30, 30, 26, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30), # Low + (-1, 10, 16, 26, 18, 24, 16, 18, 22, 22, 26, 30, 22, 22, 24, 24, 28, 28, 26, 26, 26, 26, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28), # Medium + (-1, 13, 22, 18, 26, 18, 24, 18, 22, 20, 24, 28, 26, 24, 20, 30, 24, 28, 28, 26, 30, 28, 30, 30, 30, 30, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30), # Quartile + (-1, 17, 28, 22, 16, 22, 28, 26, 26, 24, 28, 24, 28, 22, 24, 24, 30, 28, 28, 26, 28, 30, 24, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30)) # High + + _NUM_ERROR_CORRECTION_BLOCKS: Sequence[Sequence[int]] = ( + # Version: (note that index 0 is for padding, and is set to an illegal value) + # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 Error correction level + (-1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4, 4, 4, 4, 4, 6, 6, 6, 6, 7, 8, 8, 9, 9, 10, 12, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 24, 25), # Low + (-1, 1, 1, 1, 2, 2, 4, 4, 4, 5, 5, 5, 8, 9, 9, 10, 10, 11, 13, 14, 16, 17, 17, 18, 20, 21, 23, 25, 26, 28, 29, 31, 33, 35, 37, 38, 40, 43, 45, 47, 49), # Medium + (-1, 1, 1, 2, 2, 4, 4, 6, 6, 8, 8, 8, 10, 12, 16, 12, 17, 16, 18, 21, 20, 23, 23, 25, 27, 29, 34, 34, 35, 38, 40, 43, 45, 48, 51, 53, 56, 59, 62, 65, 68), # Quartile + (-1, 1, 1, 2, 4, 4, 4, 5, 6, 8, 8, 11, 11, 16, 16, 18, 16, 19, 21, 25, 25, 25, 34, 30, 32, 35, 37, 40, 42, 45, 48, 51, 54, 57, 60, 63, 66, 70, 74, 77, 81)) # High + + _MASK_PATTERNS: Sequence[collections.abc.Callable[[int,int],int]] = ( + (lambda x, y: (x + y) % 2 ), + (lambda x, y: y % 2 ), + (lambda x, y: x % 3 ), + (lambda x, y: (x + y) % 3 ), + (lambda x, y: (x // 3 + y // 2) % 2 ), + (lambda x, y: x * y % 2 + x * y % 3 ), + (lambda x, y: (x * y % 2 + x * y % 3) % 2 ), + (lambda x, y: ((x + y) % 2 + x * y % 3) % 2), + ) + + + # ---- Public helper enumeration ---- + + class Ecc: + ordinal: int # (Public) In the range 0 to 3 (unsigned 2-bit integer) + formatbits: int # (Package-private) In the range 0 to 3 (unsigned 2-bit integer) + + """The error correction level in a QR Code symbol. Immutable.""" + # Private constructor + def __init__(self, i: int, fb: int) -> None: + self.ordinal = i + self.formatbits = fb + + # Placeholders + LOW : QrCode.Ecc + MEDIUM : QrCode.Ecc + QUARTILE: QrCode.Ecc + HIGH : QrCode.Ecc + + # Public constants. Create them outside the class. + Ecc.LOW = Ecc(0, 1) # The QR Code can tolerate about 7% erroneous codewords + Ecc.MEDIUM = Ecc(1, 0) # The QR Code can tolerate about 15% erroneous codewords + Ecc.QUARTILE = Ecc(2, 3) # The QR Code can tolerate about 25% erroneous codewords + Ecc.HIGH = Ecc(3, 2) # The QR Code can tolerate about 30% erroneous codewords + + + +# ---- Data segment class ---- + +class QrSegment: + """A segment of character/binary/control data in a QR Code symbol. + Instances of this class are immutable. + The mid-level way to create a segment is to take the payload data + and call a static factory function such as QrSegment.make_numeric(). + The low-level way to create a segment is to custom-make the bit buffer + and call the QrSegment() constructor with appropriate values. + This segment class imposes no length restrictions, but QR Codes have restrictions. + Even in the most favorable conditions, a QR Code can only hold 7089 characters of data. + Any segment longer than this is meaningless for the purpose of generating QR Codes.""" + + # ---- Static factory functions (mid level) ---- + + @staticmethod + def make_bytes(data: Union[bytes,Sequence[int]]) -> QrSegment: + """Returns a segment representing the given binary data encoded in byte mode. + All input byte lists are acceptable. Any text string can be converted to + UTF-8 bytes (s.encode("UTF-8")) and encoded as a byte mode segment.""" + bb = _BitBuffer() + for b in data: + bb.append_bits(b, 8) + return QrSegment(QrSegment.Mode.BYTE, len(data), bb) + + + @staticmethod + def make_numeric(digits: str) -> QrSegment: + """Returns a segment representing the given string of decimal digits encoded in numeric mode.""" + if not QrSegment.is_numeric(digits): + raise ValueError("String contains non-numeric characters") + bb = _BitBuffer() + i: int = 0 + while i < len(digits): # Consume up to 3 digits per iteration + n: int = min(len(digits) - i, 3) + bb.append_bits(int(digits[i : i + n]), n * 3 + 1) + i += n + return QrSegment(QrSegment.Mode.NUMERIC, len(digits), bb) + + + @staticmethod + def make_alphanumeric(text: str) -> QrSegment: + """Returns a segment representing the given text string encoded in alphanumeric mode. + The characters allowed are: 0 to 9, A to Z (uppercase only), space, + dollar, percent, asterisk, plus, hyphen, period, slash, colon.""" + if not QrSegment.is_alphanumeric(text): + raise ValueError("String contains unencodable characters in alphanumeric mode") + bb = _BitBuffer() + for i in range(0, len(text) - 1, 2): # Process groups of 2 + temp: int = QrSegment._ALPHANUMERIC_ENCODING_TABLE[text[i]] * 45 + temp += QrSegment._ALPHANUMERIC_ENCODING_TABLE[text[i + 1]] + bb.append_bits(temp, 11) + if len(text) % 2 > 0: # 1 character remaining + bb.append_bits(QrSegment._ALPHANUMERIC_ENCODING_TABLE[text[-1]], 6) + return QrSegment(QrSegment.Mode.ALPHANUMERIC, len(text), bb) + + + @staticmethod + def make_segments(text: str) -> list[QrSegment]: + """Returns a new mutable list of zero or more segments to represent the given Unicode text string. + The result may use various segment modes and switch modes to optimize the length of the bit stream.""" + + # Select the most efficient segment encoding automatically + if text == "": + return [] + elif QrSegment.is_numeric(text): + return [QrSegment.make_numeric(text)] + elif QrSegment.is_alphanumeric(text): + return [QrSegment.make_alphanumeric(text)] + else: + return [QrSegment.make_bytes(text.encode("UTF-8"))] + + + @staticmethod + def make_eci(assignval: int) -> QrSegment: + """Returns a segment representing an Extended Channel Interpretation + (ECI) designator with the given assignment value.""" + bb = _BitBuffer() + if assignval < 0: + raise ValueError("ECI assignment value out of range") + elif assignval < (1 << 7): + bb.append_bits(assignval, 8) + elif assignval < (1 << 14): + bb.append_bits(0b10, 2) + bb.append_bits(assignval, 14) + elif assignval < 1000000: + bb.append_bits(0b110, 3) + bb.append_bits(assignval, 21) + else: + raise ValueError("ECI assignment value out of range") + return QrSegment(QrSegment.Mode.ECI, 0, bb) + + + # Tests whether the given string can be encoded as a segment in numeric mode. + # A string is encodable iff each character is in the range 0 to 9. + @staticmethod + def is_numeric(text: str) -> bool: + return QrSegment._NUMERIC_REGEX.fullmatch(text) is not None + + + # Tests whether the given string can be encoded as a segment in alphanumeric mode. + # A string is encodable iff each character is in the following set: 0 to 9, A to Z + # (uppercase only), space, dollar, percent, asterisk, plus, hyphen, period, slash, colon. + @staticmethod + def is_alphanumeric(text: str) -> bool: + return QrSegment._ALPHANUMERIC_REGEX.fullmatch(text) is not None + + + # ---- Private fields ---- + + # The mode indicator of this segment. Accessed through get_mode(). + _mode: QrSegment.Mode + + # The length of this segment's unencoded data. Measured in characters for + # numeric/alphanumeric/kanji mode, bytes for byte mode, and 0 for ECI mode. + # Always zero or positive. Not the same as the data's bit length. + # Accessed through get_num_chars(). + _numchars: int + + # The data bits of this segment. Accessed through get_data(). + _bitdata: list[int] + + + # ---- Constructor (low level) ---- + + def __init__(self, mode: QrSegment.Mode, numch: int, bitdata: Sequence[int]) -> None: + """Creates a new QR Code segment with the given attributes and data. + The character count (numch) must agree with the mode and the bit buffer length, + but the constraint isn't checked. The given bit buffer is cloned and stored.""" + if numch < 0: + raise ValueError() + self._mode = mode + self._numchars = numch + self._bitdata = list(bitdata) # Make defensive copy + + + # ---- Accessor methods ---- + + def get_mode(self) -> QrSegment.Mode: + """Returns the mode field of this segment.""" + return self._mode + + def get_num_chars(self) -> int: + """Returns the character count field of this segment.""" + return self._numchars + + def get_data(self) -> list[int]: + """Returns a new copy of the data bits of this segment.""" + return list(self._bitdata) # Make defensive copy + + + # Package-private function + @staticmethod + def get_total_bits(segs: Sequence[QrSegment], version: int) -> Optional[int]: + """Calculates the number of bits needed to encode the given segments at + the given version. Returns a non-negative number if successful. Otherwise + returns None if a segment has too many characters to fit its length field.""" + result = 0 + for seg in segs: + ccbits: int = seg.get_mode().num_char_count_bits(version) + if seg.get_num_chars() >= (1 << ccbits): + return None # The segment's length doesn't fit the field's bit width + result += 4 + ccbits + len(seg._bitdata) + return result + + + # ---- Constants ---- + + # Describes precisely all strings that are encodable in numeric mode. + _NUMERIC_REGEX: re.Pattern[str] = re.compile(r"[0-9]*") + + # Describes precisely all strings that are encodable in alphanumeric mode. + _ALPHANUMERIC_REGEX: re.Pattern[str] = re.compile(r"[A-Z0-9 $%*+./:-]*") + + # Dictionary of "0"->0, "A"->10, "$"->37, etc. + _ALPHANUMERIC_ENCODING_TABLE: dict[str,int] = {ch: i for (i, ch) in enumerate("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ $%*+-./:")} + + + # ---- Public helper enumeration ---- + + class Mode: + """Describes how a segment's data bits are interpreted. Immutable.""" + + _modebits: int # The mode indicator bits, which is a uint4 value (range 0 to 15) + _charcounts: tuple[int,int,int] # Number of character count bits for three different version ranges + + # Private constructor + def __init__(self, modebits: int, charcounts: tuple[int,int,int]): + self._modebits = modebits + self._charcounts = charcounts + + # Package-private method + def get_mode_bits(self) -> int: + """Returns an unsigned 4-bit integer value (range 0 to 15) representing the mode indicator bits for this mode object.""" + return self._modebits + + # Package-private method + def num_char_count_bits(self, ver: int) -> int: + """Returns the bit width of the character count field for a segment in this mode + in a QR Code at the given version number. The result is in the range [0, 16].""" + return self._charcounts[(ver + 7) // 17] + + # Placeholders + NUMERIC : QrSegment.Mode + ALPHANUMERIC: QrSegment.Mode + BYTE : QrSegment.Mode + KANJI : QrSegment.Mode + ECI : QrSegment.Mode + + # Public constants. Create them outside the class. + Mode.NUMERIC = Mode(0x1, (10, 12, 14)) + Mode.ALPHANUMERIC = Mode(0x2, ( 9, 11, 13)) + Mode.BYTE = Mode(0x4, ( 8, 16, 16)) + Mode.KANJI = Mode(0x8, ( 8, 10, 12)) + Mode.ECI = Mode(0x7, ( 0, 0, 0)) + + + +# ---- Private helper class ---- + +class _BitBuffer(list[int]): + """An appendable sequence of bits (0s and 1s). Mainly used by QrSegment.""" + + def append_bits(self, val: int, n: int) -> None: + """Appends the given number of low-order bits of the given + value to this buffer. Requires n >= 0 and 0 <= val < 2^n.""" + if (n < 0) or (val >> n != 0): + raise ValueError("Value out of range") + self.extend(((val >> i) & 1) for i in reversed(range(n))) + + +def _get_bit(x: int, i: int) -> bool: + """Returns true iff the i'th bit of x is set to 1.""" + return (x >> i) & 1 != 0 + + + +class DataTooLongError(ValueError): + """Raised when the supplied data does not fit any QR Code version. Ways to handle this exception include: + - Decrease the error correction level if it was greater than Ecc.LOW. + - If the encode_segments() function was called with a maxversion argument, then increase + it if it was less than QrCode.MAX_VERSION. (This advice does not apply to the other + factory functions because they search all versions up to QrCode.MAX_VERSION.) + - Split the text data into better or optimal segments in order to reduce the number of bits required. + - Change the text or binary data to be shorter. + - Change the text to fit the character set of a particular segment mode (e.g. alphanumeric). + - Propagate the error upward to the caller/user.""" + pass