feat(p1): add daily snapshots and fan tracking

Implement P1: seven-metric view snapshots, relation/stat fan snapshots, failure placeholders, resume by daily completion markers, Cookie login warnings, daily SQLite backups, Shanghai daily scheduling, and CLI snapshot/auth-check commands. Real read-only run captured 795/795 videos successfully over 1124s with a verified daily backup.
This commit is contained in:
kairee committed 2026-10-31 19:19:24 +08:00
1 parent ee289bfa56
commit 5c2ab7b187
20 files changed
+1680 -54

No files matched your search

+131 -1
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import sqlite3
import time
from dataclasses import dataclass
from typing import Any, Iterable
from typing import Any, Iterable, Mapping
from .config import FolderConfig
from .database import Database
@@ -368,6 +368,136 @@ class Repository:
return [dict(row) for row in rows]
return self.db.read(run)
# ------------------------------------------------------------ snapshot
def video_aids_for_snapshot(self) -> list[int]:
def run(conn: sqlite3.Connection) -> list[int]:
rows = conn.execute(
"SELECT aid FROM video WHERE status IN ('active', 'unfaved') "
"ORDER BY aid"
).fetchall()
return [int(row["aid"]) for row in rows]
return self.db.read(run)
def get_snapshot(self, stat_date: str, aid: int) -> dict[str, Any] | None:
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
row = conn.execute(
"SELECT * FROM snapshot WHERE stat_date = ? AND aid = ?",
(stat_date, int(aid)),
).fetchone()
return dict(row) if row else None
return self.db.read(run)
def upsert_snapshot(
self,
stat_date: str,
aid: int,
*,
metrics: Mapping[str, int] | None,
ok: bool,
err: str | None,
captured_at: int,
src: str = "view",
) -> None:
values = metrics or {}
def run(conn: sqlite3.Connection) -> None:
conn.execute(
"""
INSERT INTO snapshot(stat_date, aid, "view", "like", favorite, coin,
share, reply, danmaku, captured_at, ok, err, src)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(stat_date, aid) DO UPDATE SET
"view" = excluded."view",
"like" = excluded."like",
favorite = excluded.favorite,
coin = excluded.coin,
share = excluded.share,
reply = excluded.reply,
danmaku = excluded.danmaku,
captured_at = excluded.captured_at,
ok = excluded.ok,
err = excluded.err,
src = excluded.src
""",
(
stat_date,
int(aid),
values.get("view"),
values.get("like"),
values.get("favorite"),
values.get("coin"),
values.get("share"),
values.get("reply"),
values.get("danmaku"),
int(captured_at),
1 if ok else 0,
err,
src,
),
)
self.db.write(run)
def snapshot_counts(self, stat_date: str) -> dict[str, int]:
def run(conn: sqlite3.Connection) -> dict[str, int]:
rows = conn.execute(
"SELECT ok, COUNT(*) AS n FROM snapshot WHERE stat_date = ? GROUP BY ok",
(stat_date,),
).fetchall()
by_ok = {int(row["ok"]): int(row["n"]) for row in rows}
return {
"total": sum(by_ok.values()),
"ok": by_ok.get(1, 0),
"failed": by_ok.get(0, 0),
}
return self.db.read(run)
def upsert_up_stat(
self,
stat_date: str,
mid: int,
*,
follower: int | None,
following: int | None,
ok: bool,
captured_at: int,
) -> None:
self.db.write(lambda conn: conn.execute(
"""
INSERT INTO up_stat(stat_date, mid, follower, following, captured_at, ok)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(stat_date, mid) DO UPDATE SET
follower = excluded.follower,
following = excluded.following,
captured_at = excluded.captured_at,
ok = excluded.ok
""",
(stat_date, int(mid), follower, following, int(captured_at), 1 if ok else 0),
))
def get_up_stat(self, stat_date: str, mid: int) -> dict[str, Any] | None:
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
row = conn.execute(
"SELECT * FROM up_stat WHERE stat_date = ? AND mid = ?",
(stat_date, int(mid)),
).fetchone()
return dict(row) if row else None
return self.db.read(run)
# ------------------------------------------------------------ kv
def kv_get(self, key: str) -> str | None:
def run(conn: sqlite3.Connection) -> str | None:
row = conn.execute("SELECT v FROM kv WHERE k = ?", (key,)).fetchone()
return None if row is None else str(row["v"])
return self.db.read(run)
def kv_set(self, key: str, value: str) -> None:
self.db.write(lambda conn: conn.execute(
"INSERT INTO kv(k, v) VALUES (?, ?) "
"ON CONFLICT(k) DO UPDATE SET v = excluded.v",
(key, str(value)),
))
# ------------------------------------------------------------ blacklist
def seed_blacklist(self, mids: Iterable[int], *, now: int | None = None) -> int: