feat(p1): add daily snapshots and fan tracking
Implement P1: seven-metric view snapshots, relation/stat fan snapshots, failure placeholders, resume by daily completion markers, Cookie login warnings, daily SQLite backups, Shanghai daily scheduling, and CLI snapshot/auth-check commands. Real read-only run captured 795/795 videos successfully over 1124s with a verified daily backup.
This commit is contained in:
1 parent
ee289bfa56
commit
5c2ab7b187
20 files changed
+1680
-54
No files matched your search
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
import sqlite3
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
from .config import FolderConfig
|
||||
from .database import Database
|
||||
@@ -368,6 +368,136 @@ class Repository:
|
||||
return [dict(row) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
# ------------------------------------------------------------ snapshot
|
||||
|
||||
def video_aids_for_snapshot(self) -> list[int]:
|
||||
def run(conn: sqlite3.Connection) -> list[int]:
|
||||
rows = conn.execute(
|
||||
"SELECT aid FROM video WHERE status IN ('active', 'unfaved') "
|
||||
"ORDER BY aid"
|
||||
).fetchall()
|
||||
return [int(row["aid"]) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
def get_snapshot(self, stat_date: str, aid: int) -> dict[str, Any] | None:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM snapshot WHERE stat_date = ? AND aid = ?",
|
||||
(stat_date, int(aid)),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
return self.db.read(run)
|
||||
|
||||
def upsert_snapshot(
|
||||
self,
|
||||
stat_date: str,
|
||||
aid: int,
|
||||
*,
|
||||
metrics: Mapping[str, int] | None,
|
||||
ok: bool,
|
||||
err: str | None,
|
||||
captured_at: int,
|
||||
src: str = "view",
|
||||
) -> None:
|
||||
values = metrics or {}
|
||||
def run(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO snapshot(stat_date, aid, "view", "like", favorite, coin,
|
||||
share, reply, danmaku, captured_at, ok, err, src)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(stat_date, aid) DO UPDATE SET
|
||||
"view" = excluded."view",
|
||||
"like" = excluded."like",
|
||||
favorite = excluded.favorite,
|
||||
coin = excluded.coin,
|
||||
share = excluded.share,
|
||||
reply = excluded.reply,
|
||||
danmaku = excluded.danmaku,
|
||||
captured_at = excluded.captured_at,
|
||||
ok = excluded.ok,
|
||||
err = excluded.err,
|
||||
src = excluded.src
|
||||
""",
|
||||
(
|
||||
stat_date,
|
||||
int(aid),
|
||||
values.get("view"),
|
||||
values.get("like"),
|
||||
values.get("favorite"),
|
||||
values.get("coin"),
|
||||
values.get("share"),
|
||||
values.get("reply"),
|
||||
values.get("danmaku"),
|
||||
int(captured_at),
|
||||
1 if ok else 0,
|
||||
err,
|
||||
src,
|
||||
),
|
||||
)
|
||||
self.db.write(run)
|
||||
|
||||
def snapshot_counts(self, stat_date: str) -> dict[str, int]:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, int]:
|
||||
rows = conn.execute(
|
||||
"SELECT ok, COUNT(*) AS n FROM snapshot WHERE stat_date = ? GROUP BY ok",
|
||||
(stat_date,),
|
||||
).fetchall()
|
||||
by_ok = {int(row["ok"]): int(row["n"]) for row in rows}
|
||||
return {
|
||||
"total": sum(by_ok.values()),
|
||||
"ok": by_ok.get(1, 0),
|
||||
"failed": by_ok.get(0, 0),
|
||||
}
|
||||
return self.db.read(run)
|
||||
|
||||
def upsert_up_stat(
|
||||
self,
|
||||
stat_date: str,
|
||||
mid: int,
|
||||
*,
|
||||
follower: int | None,
|
||||
following: int | None,
|
||||
ok: bool,
|
||||
captured_at: int,
|
||||
) -> None:
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"""
|
||||
INSERT INTO up_stat(stat_date, mid, follower, following, captured_at, ok)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(stat_date, mid) DO UPDATE SET
|
||||
follower = excluded.follower,
|
||||
following = excluded.following,
|
||||
captured_at = excluded.captured_at,
|
||||
ok = excluded.ok
|
||||
""",
|
||||
(stat_date, int(mid), follower, following, int(captured_at), 1 if ok else 0),
|
||||
))
|
||||
|
||||
def get_up_stat(self, stat_date: str, mid: int) -> dict[str, Any] | None:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM up_stat WHERE stat_date = ? AND mid = ?",
|
||||
(stat_date, int(mid)),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
return self.db.read(run)
|
||||
|
||||
# ------------------------------------------------------------ kv
|
||||
|
||||
def kv_get(self, key: str) -> str | None:
|
||||
def run(conn: sqlite3.Connection) -> str | None:
|
||||
row = conn.execute("SELECT v FROM kv WHERE k = ?", (key,)).fetchone()
|
||||
return None if row is None else str(row["v"])
|
||||
return self.db.read(run)
|
||||
|
||||
def kv_set(self, key: str, value: str) -> None:
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"INSERT INTO kv(k, v) VALUES (?, ?) "
|
||||
"ON CONFLICT(k) DO UPDATE SET v = excluded.v",
|
||||
(key, str(value)),
|
||||
))
|
||||
|
||||
# ------------------------------------------------------------ blacklist
|
||||
|
||||
def seed_blacklist(self, mids: Iterable[int], *, now: int | None = None) -> int:
|
||||
|
||||
Reference in new issue
Block a user