# -*- coding: utf-8 -*- """从小满虫之家公开数据回填粉丝历史。 数据源是站点 ``assets/js/data.js`` 中的 ``window.XXM`` 快照:``followers`` 提供锚点,``corr`` 提供每日净增。脚本只写入 ``up_stat``,不会修改视频快照、 收藏夹或其它表。 用法:: uv run python tools/import_fans_history.py --days 90 --dry-run uv run python tools/import_fans_history.py --days 90 如果站点暂时不可达,可先保存 data.js 再离线导入:: uv run python tools/import_fans_history.py --from-file data.js --days 90 """ from __future__ import annotations import argparse import json import os import sys import urllib.request from datetime import date, datetime, time, timedelta from pathlib import Path from typing import Any, Mapping, Sequence ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) from reaction_board.database import Database # noqa: E402 from reaction_board.repository import Repository # noqa: E402 from reaction_board.timeutil import SHANGHAI_TZ # noqa: E402 DEFAULT_URL = "https://www.xxm0319.com/assets/js/data.js" DEFAULT_ACCOUNT = "咻咻满" DEFAULT_MID = 37754047 USER_AGENT = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36" ) def parse_data_js(text: str) -> dict[str, Any]: """从 ``window.XXM = {...};`` 中提取 JSON 对象。""" if not isinstance(text, str) or not text.strip(): raise ValueError("data.js 内容为空") start = text.find("window.XXM") if start < 0: raise ValueError("data.js 中找不到 window.XXM") equals = text.find("=", start) if equals < 0: raise ValueError("data.js 中 window.XXM 缺少赋值") end = text.rfind(";") if end <= equals: raise ValueError("data.js 中 window.XXM 缺少结尾分号") raw = text[equals + 1:end].strip() try: payload = json.loads(raw) except json.JSONDecodeError as exc: raise ValueError(f"window.XXM 不是合法 JSON: {exc}") from exc if not isinstance(payload, dict): raise ValueError("window.XXM 顶层必须是对象") return payload def _parse_date(value: str) -> date: text = str(value).strip().replace("/", "-") try: return date.fromisoformat(text) except ValueError as exc: raise ValueError(f"日期格式不正确: {value!r}") from exc def _day_timestamp(day: date) -> int: return int(datetime.combine(day, time(0, 1), tzinfo=SHANGHAI_TZ).timestamp()) def build_follower_series( payload: Mapping[str, Any], *, account: str = DEFAULT_ACCOUNT, days: int = 90, ) -> dict[str, Any]: """根据锚点和每日净增构建连续粉丝序列。 返回的 ``points`` 按日期升序,包含 ``date`` 和 ``follower``。若数据出现 日期缺口,只回填锚点之前连续可计算的部分,不插值。 """ if days < 1: raise ValueError("days 必须至少为 1") followers = payload.get("followers") if not isinstance(followers, Mapping): raise ValueError("data.js 缺少 followers") account_data = followers.get(account) if not isinstance(account_data, Mapping): raise ValueError(f"data.js 缺少账号: {account}") week = account_data.get("week") if not isinstance(week, list) or not week: raise ValueError(f"{account} 缺少 week 锚点") last = week[-1] if not isinstance(last, list) or len(last) < 2: raise ValueError("week 锚点结构不正确") anchor_day = _parse_date(last[0]) anchor_value = int(last[1]) total = account_data.get("total") warnings: list[str] = [] if isinstance(total, int) and total != anchor_value: warnings.append(f"week 锚点 {anchor_value} 与 total {total} 不一致,使用 week 锚点") corr = payload.get("corr") if not isinstance(corr, list) or not corr: raise ValueError("data.js 缺少 corr 每日净增") deltas: dict[date, int] = {} for item in corr: if not isinstance(item, list) or len(item) < 3: raise ValueError("corr 条目必须包含日期、占位值和净增") day = _parse_date(item[0]) try: delta = int(item[2]) except (TypeError, ValueError) as exc: raise ValueError(f"corr 净增不是整数: {item!r}") from exc deltas[day] = delta values: dict[date, int] = {anchor_day: anchor_value} ordered = sorted(day for day in deltas if day <= anchor_day) if anchor_day not in deltas: ordered.append(anchor_day) ordered = sorted(set(ordered)) stopped = False for current, next_day in zip(reversed(ordered[:-1]), reversed(ordered[1:])): if (next_day - current).days != 1: warnings.append( f"粉丝数据在 {current.isoformat()} 与 {next_day.isoformat()} 之间不连续" ) stopped = True break delta = deltas.get(next_day) if delta is None: warnings.append(f"缺少 {next_day.isoformat()} 的净增") stopped = True break value = values[next_day] - delta if value < 0: raise ValueError( f"计算结果为负: {current.isoformat()} follower={value}" ) values[current] = value # 用 week 中的多个点校验每日净增口径,防止站点字段语义变化。 for item in week: if not isinstance(item, list) or len(item) < 2: continue day = _parse_date(item[0]) if day in values and int(item[1]) != values[day]: raise ValueError( f"week 与 corr 计算结果不一致: {day.isoformat()} " f"{item[1]} != {values[day]}" ) points = [ {"date": day.isoformat(), "follower": values[day]} for day in sorted(values)[-days:] ] if not points: raise ValueError("没有可用的粉丝历史数据") return { "account": account, "source_updated": str((payload.get("meta") or {}).get("updated") or anchor_day), "anchor_date": anchor_day.isoformat(), "anchor_follower": anchor_value, "first_date": points[0]["date"], "last_date": points[-1]["date"], "points": points, "warnings": warnings, "stopped_at_gap": stopped, } def import_follower_series( db: Database, series: Mapping[str, Any], *, mid: int = DEFAULT_MID, force: bool = False, dry_run: bool = False, ) -> dict[str, Any]: """把序列写入 ``up_stat``;默认跳过已有成功记录。""" repo = Repository(db) inserted = 0 updated = 0 skipped = 0 for point in series.get("points", []): day = str(point["date"]) follower = int(point["follower"]) existing = repo.get_up_stat(day, mid) if existing is not None and int(existing.get("ok") or 0) == 1 and not force: skipped += 1 continue if existing is None: inserted += 1 else: updated += 1 if dry_run: continue following = None if existing is None else existing.get("following") repo.upsert_up_stat( day, mid, follower=follower, following=following, ok=True, captured_at=_day_timestamp(date.fromisoformat(day)), ) return { "ok": True, "dry_run": bool(dry_run), "mid": int(mid), "inserted": inserted, "updated": updated, "skipped": skipped, "total": len(series.get("points", [])), "first_date": series.get("first_date"), "last_date": series.get("last_date"), "source_updated": series.get("source_updated"), "warnings": list(series.get("warnings", [])), } def fetch_data_js(url: str, *, timeout: float = 30.0) -> str: request = urllib.request.Request( url, headers={ "User-Agent": USER_AGENT, "Accept": "application/javascript,text/javascript,*/*", "Referer": "https://www.xxm0319.com/data.html", }, ) with urllib.request.urlopen(request, timeout=timeout) as response: return response.read().decode("utf-8") def _database_path(config_path: str, override: str | None) -> Path: if override: return Path(override) from reaction_board.config import ConfigStore return ConfigStore(config_path, require_auth=False).get().storage.database_path def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description="从小满虫之家回填粉丝历史到 up_stat") parser.add_argument("--config", default=str(ROOT / "config.json"), help="配置文件") parser.add_argument("--db", default=None, help="直接指定数据库(覆盖配置)") parser.add_argument("--url", default=DEFAULT_URL, help="data.js 地址") parser.add_argument("--from-file", default=None, help="离线 data.js 文件") parser.add_argument("--account", default=DEFAULT_ACCOUNT, help="数据源账号名") parser.add_argument("--mid", type=int, default=DEFAULT_MID, help="写入的 B 站 mid") parser.add_argument("--days", type=int, default=90, help="保留最近天数") parser.add_argument("--force", action="store_true", help="覆盖已有 ok=1 记录") parser.add_argument("--dry-run", action="store_true", help="只计算,不写数据库") args = parser.parse_args(list(argv) if argv is not None else None) try: if args.from_file: text = Path(args.from_file).read_text(encoding="utf-8") else: text = fetch_data_js(args.url) payload = parse_data_js(text) series = build_follower_series( payload, account=args.account, days=args.days ) db_path = _database_path(args.config, args.db) db = Database(db_path) db.initialize() try: summary = import_follower_series( db, series, mid=args.mid, force=args.force, dry_run=args.dry_run ) finally: db.close() summary.update({ "database": str(db_path), "source": args.from_file or args.url, }) print(json.dumps(summary, ensure_ascii=False, indent=2)) return 0 except (ValueError, OSError, RuntimeError) as exc: print(json.dumps({"ok": False, "error": str(exc)}, ensure_ascii=False)) return 2 if __name__ == "__main__": raise SystemExit(main())