feat(tools): import 90-day fan history from public data source
This commit is contained in:
kairee committed
1 parent
f9a5478800
commit
ff101803cd
3 files changed
+475
No files matched your search
@@ -0,0 +1,295 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""从小满虫之家公开数据回填粉丝历史。
|
||||
|
||||
数据源是站点 ``assets/js/data.js`` 中的 ``window.XXM`` 快照：``followers``
|
||||
提供锚点，``corr`` 提供每日净增。脚本只写入 ``up_stat``，不会修改视频快照、
|
||||
收藏夹或其它表。
|
||||
|
||||
用法::
|
||||
|
||||
uv run python tools/import_fans_history.py --days 90 --dry-run
|
||||
uv run python tools/import_fans_history.py --days 90
|
||||
|
||||
如果站点暂时不可达，可先保存 data.js 再离线导入::
|
||||
|
||||
uv run python tools/import_fans_history.py --from-file data.js --days 90
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import date, datetime, time, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.repository import Repository # noqa: E402
|
||||
from reaction_board.timeutil import SHANGHAI_TZ # noqa: E402
|
||||
|
||||
DEFAULT_URL = "https://www.xxm0319.com/assets/js/data.js"
|
||||
DEFAULT_ACCOUNT = "咻咻满"
|
||||
DEFAULT_MID = 37754047
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
def parse_data_js(text: str) -> dict[str, Any]:
|
||||
"""从 ``window.XXM = {...};`` 中提取 JSON 对象。"""
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ValueError("data.js 内容为空")
|
||||
start = text.find("window.XXM")
|
||||
if start < 0:
|
||||
raise ValueError("data.js 中找不到 window.XXM")
|
||||
equals = text.find("=", start)
|
||||
if equals < 0:
|
||||
raise ValueError("data.js 中 window.XXM 缺少赋值")
|
||||
end = text.rfind(";")
|
||||
if end <= equals:
|
||||
raise ValueError("data.js 中 window.XXM 缺少结尾分号")
|
||||
raw = text[equals + 1:end].strip()
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"window.XXM 不是合法 JSON: {exc}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("window.XXM 顶层必须是对象")
|
||||
return payload
|
||||
|
||||
|
||||
def _parse_date(value: str) -> date:
|
||||
text = str(value).strip().replace("/", "-")
|
||||
try:
|
||||
return date.fromisoformat(text)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"日期格式不正确: {value!r}") from exc
|
||||
|
||||
|
||||
def _day_timestamp(day: date) -> int:
|
||||
return int(datetime.combine(day, time(0, 1), tzinfo=SHANGHAI_TZ).timestamp())
|
||||
|
||||
|
||||
def build_follower_series(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
account: str = DEFAULT_ACCOUNT,
|
||||
days: int = 90,
|
||||
) -> dict[str, Any]:
|
||||
"""根据锚点和每日净增构建连续粉丝序列。
|
||||
|
||||
返回的 ``points`` 按日期升序，包含 ``date`` 和 ``follower``。若数据出现
|
||||
日期缺口，只回填锚点之前连续可计算的部分，不插值。
|
||||
"""
|
||||
if days < 1:
|
||||
raise ValueError("days 必须至少为 1")
|
||||
followers = payload.get("followers")
|
||||
if not isinstance(followers, Mapping):
|
||||
raise ValueError("data.js 缺少 followers")
|
||||
account_data = followers.get(account)
|
||||
if not isinstance(account_data, Mapping):
|
||||
raise ValueError(f"data.js 缺少账号: {account}")
|
||||
week = account_data.get("week")
|
||||
if not isinstance(week, list) or not week:
|
||||
raise ValueError(f"{account} 缺少 week 锚点")
|
||||
last = week[-1]
|
||||
if not isinstance(last, list) or len(last) < 2:
|
||||
raise ValueError("week 锚点结构不正确")
|
||||
anchor_day = _parse_date(last[0])
|
||||
anchor_value = int(last[1])
|
||||
total = account_data.get("total")
|
||||
warnings: list[str] = []
|
||||
if isinstance(total, int) and total != anchor_value:
|
||||
warnings.append(f"week 锚点 {anchor_value} 与 total {total} 不一致，使用 week 锚点")
|
||||
|
||||
corr = payload.get("corr")
|
||||
if not isinstance(corr, list) or not corr:
|
||||
raise ValueError("data.js 缺少 corr 每日净增")
|
||||
deltas: dict[date, int] = {}
|
||||
for item in corr:
|
||||
if not isinstance(item, list) or len(item) < 3:
|
||||
raise ValueError("corr 条目必须包含日期、占位值和净增")
|
||||
day = _parse_date(item[0])
|
||||
try:
|
||||
delta = int(item[2])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"corr 净增不是整数: {item!r}") from exc
|
||||
deltas[day] = delta
|
||||
|
||||
values: dict[date, int] = {anchor_day: anchor_value}
|
||||
ordered = sorted(day for day in deltas if day <= anchor_day)
|
||||
if anchor_day not in deltas:
|
||||
ordered.append(anchor_day)
|
||||
ordered = sorted(set(ordered))
|
||||
stopped = False
|
||||
for current, next_day in zip(reversed(ordered[:-1]), reversed(ordered[1:])):
|
||||
if (next_day - current).days != 1:
|
||||
warnings.append(
|
||||
f"粉丝数据在 {current.isoformat()} 与 {next_day.isoformat()} 之间不连续"
|
||||
)
|
||||
stopped = True
|
||||
break
|
||||
delta = deltas.get(next_day)
|
||||
if delta is None:
|
||||
warnings.append(f"缺少 {next_day.isoformat()} 的净增")
|
||||
stopped = True
|
||||
break
|
||||
value = values[next_day] - delta
|
||||
if value < 0:
|
||||
raise ValueError(
|
||||
f"计算结果为负: {current.isoformat()} follower={value}"
|
||||
)
|
||||
values[current] = value
|
||||
|
||||
# 用 week 中的多个点校验每日净增口径，防止站点字段语义变化。
|
||||
for item in week:
|
||||
if not isinstance(item, list) or len(item) < 2:
|
||||
continue
|
||||
day = _parse_date(item[0])
|
||||
if day in values and int(item[1]) != values[day]:
|
||||
raise ValueError(
|
||||
f"week 与 corr 计算结果不一致: {day.isoformat()} "
|
||||
f"{item[1]} != {values[day]}"
|
||||
)
|
||||
|
||||
points = [
|
||||
{"date": day.isoformat(), "follower": values[day]}
|
||||
for day in sorted(values)[-days:]
|
||||
]
|
||||
if not points:
|
||||
raise ValueError("没有可用的粉丝历史数据")
|
||||
return {
|
||||
"account": account,
|
||||
"source_updated": str((payload.get("meta") or {}).get("updated") or anchor_day),
|
||||
"anchor_date": anchor_day.isoformat(),
|
||||
"anchor_follower": anchor_value,
|
||||
"first_date": points[0]["date"],
|
||||
"last_date": points[-1]["date"],
|
||||
"points": points,
|
||||
"warnings": warnings,
|
||||
"stopped_at_gap": stopped,
|
||||
}
|
||||
|
||||
|
||||
def import_follower_series(
|
||||
db: Database,
|
||||
series: Mapping[str, Any],
|
||||
*,
|
||||
mid: int = DEFAULT_MID,
|
||||
force: bool = False,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""把序列写入 ``up_stat``；默认跳过已有成功记录。"""
|
||||
repo = Repository(db)
|
||||
inserted = 0
|
||||
updated = 0
|
||||
skipped = 0
|
||||
for point in series.get("points", []):
|
||||
day = str(point["date"])
|
||||
follower = int(point["follower"])
|
||||
existing = repo.get_up_stat(day, mid)
|
||||
if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
|
||||
skipped += 1
|
||||
continue
|
||||
if existing is None:
|
||||
inserted += 1
|
||||
else:
|
||||
updated += 1
|
||||
if dry_run:
|
||||
continue
|
||||
following = None if existing is None else existing.get("following")
|
||||
repo.upsert_up_stat(
|
||||
day,
|
||||
mid,
|
||||
follower=follower,
|
||||
following=following,
|
||||
ok=True,
|
||||
captured_at=_day_timestamp(date.fromisoformat(day)),
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"dry_run": bool(dry_run),
|
||||
"mid": int(mid),
|
||||
"inserted": inserted,
|
||||
"updated": updated,
|
||||
"skipped": skipped,
|
||||
"total": len(series.get("points", [])),
|
||||
"first_date": series.get("first_date"),
|
||||
"last_date": series.get("last_date"),
|
||||
"source_updated": series.get("source_updated"),
|
||||
"warnings": list(series.get("warnings", [])),
|
||||
}
|
||||
|
||||
|
||||
def fetch_data_js(url: str, *, timeout: float = 30.0) -> str:
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/javascript,text/javascript,*/*",
|
||||
"Referer": "https://www.xxm0319.com/data.html",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.read().decode("utf-8")
|
||||
|
||||
|
||||
def _database_path(config_path: str, override: str | None) -> Path:
|
||||
if override:
|
||||
return Path(override)
|
||||
from reaction_board.config import ConfigStore
|
||||
|
||||
return ConfigStore(config_path, require_auth=False).get().storage.database_path
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="从小满虫之家回填粉丝历史到 up_stat")
|
||||
parser.add_argument("--config", default=str(ROOT / "config.json"), help="配置文件")
|
||||
parser.add_argument("--db", default=None, help="直接指定数据库（覆盖配置）")
|
||||
parser.add_argument("--url", default=DEFAULT_URL, help="data.js 地址")
|
||||
parser.add_argument("--from-file", default=None, help="离线 data.js 文件")
|
||||
parser.add_argument("--account", default=DEFAULT_ACCOUNT, help="数据源账号名")
|
||||
parser.add_argument("--mid", type=int, default=DEFAULT_MID, help="写入的 B 站 mid")
|
||||
parser.add_argument("--days", type=int, default=90, help="保留最近天数")
|
||||
parser.add_argument("--force", action="store_true", help="覆盖已有 ok=1 记录")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只计算，不写数据库")
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
try:
|
||||
if args.from_file:
|
||||
text = Path(args.from_file).read_text(encoding="utf-8")
|
||||
else:
|
||||
text = fetch_data_js(args.url)
|
||||
payload = parse_data_js(text)
|
||||
series = build_follower_series(
|
||||
payload, account=args.account, days=args.days
|
||||
)
|
||||
db_path = _database_path(args.config, args.db)
|
||||
db = Database(db_path)
|
||||
db.initialize()
|
||||
try:
|
||||
summary = import_follower_series(
|
||||
db, series, mid=args.mid, force=args.force, dry_run=args.dry_run
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
summary.update({
|
||||
"database": str(db_path),
|
||||
"source": args.from_file or args.url,
|
||||
})
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
except (ValueError, OSError, RuntimeError) as exc:
|
||||
print(json.dumps({"ok": False, "error": str(exc)}, ensure_ascii=False))
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user