Files
re-xxm/reaction_board/repository.py
T
kairee 6a21b5d905 fix: close data integrity and hot-reload review findings
Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
2026-10-31 20:58:36 +08:00

629 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""P0 数据访问层:收藏夹、视频与黑名单。
业务状态迁移集中在这里,服务层只负责请求与编排。所有写操作经 ``Database``
的单写队列串行执行。
"""
from __future__ import annotations
import sqlite3
import time
from dataclasses import dataclass
from typing import Any, Iterable, Mapping
from .config import FolderConfig
from .database import Database
@dataclass(frozen=True)
class UpsertOutcome:
aid: int
status: str
previous_folder_id: int | None
moved_folder: bool
@dataclass(frozen=True)
class FolderRow:
media_id: int
year: int
title: str
media_count: int | None
last_ids_at: int | None
last_full_at: int | None
enabled: bool
class Repository:
def __init__(self, db: Database, *, clock=time.time) -> None:
self.db = db
self._clock = clock
def _now(self) -> int:
return int(self._clock())
# ------------------------------------------------------------ folder
def ensure_folder(self, folder: FolderConfig, *, now: int | None = None) -> None:
now = self._now() if now is None else int(now)
def run(conn: sqlite3.Connection) -> None:
conn.execute(
"""
INSERT INTO folder(media_id, year, title, media_count,
last_ids_at, last_full_at, enabled)
VALUES (?, ?, ?, NULL, NULL, NULL, 1)
ON CONFLICT(media_id) DO UPDATE SET
year = excluded.year,
title = COALESCE(NULLIF(excluded.title, ''), folder.title),
enabled = 1
""",
(folder.media_id, folder.year, folder.title),
)
self.db.write(run)
def disable_folders_except(self, media_ids: Iterable[int]) -> None:
ids = [int(mid) for mid in media_ids]
def run(conn: sqlite3.Connection) -> None:
if ids:
placeholders = ",".join("?" for _ in ids)
conn.execute(
f"UPDATE folder SET enabled = 0 WHERE media_id NOT IN ({placeholders})",
ids,
)
else:
conn.execute("UPDATE folder SET enabled = 0")
self.db.write(run)
def finish_folder_sync(
self,
folder: FolderConfig,
*,
media_count: int,
full_refresh: bool,
now: int | None = None,
) -> None:
now = self._now() if now is None else int(now)
def run(conn: sqlite3.Connection) -> None:
conn.execute(
"""
INSERT INTO folder(media_id, year, title, media_count,
last_ids_at, last_full_at, enabled)
VALUES (?, ?, ?, ?, ?, ?, 1)
ON CONFLICT(media_id) DO UPDATE SET
year = excluded.year,
title = COALESCE(NULLIF(excluded.title, ''), folder.title),
media_count = excluded.media_count,
last_ids_at = excluded.last_ids_at,
last_full_at = CASE
WHEN ? THEN excluded.last_full_at
ELSE folder.last_full_at
END,
enabled = 1
""",
(
folder.media_id, folder.year, folder.title, int(media_count),
now, now if full_refresh else None, 1 if full_refresh else 0,
),
)
self.db.write(run)
def get_folder(self, media_id: int) -> FolderRow | None:
def run(conn: sqlite3.Connection) -> FolderRow | None:
row = conn.execute(
"SELECT media_id, year, title, media_count, last_ids_at, "
"last_full_at, enabled FROM folder WHERE media_id = ?",
(int(media_id),),
).fetchone()
if row is None:
return None
return FolderRow(
media_id=int(row["media_id"]),
year=int(row["year"]),
title=str(row["title"] or ""),
media_count=row["media_count"],
last_ids_at=row["last_ids_at"],
last_full_at=row["last_full_at"],
enabled=bool(row["enabled"]),
)
return self.db.read(run)
def list_folders(self) -> list[FolderRow]:
def run(conn: sqlite3.Connection) -> list[FolderRow]:
rows = conn.execute(
"SELECT media_id, year, title, media_count, last_ids_at, "
"last_full_at, enabled FROM folder ORDER BY year, media_id"
).fetchall()
return [
FolderRow(
media_id=int(row["media_id"]),
year=int(row["year"]),
title=str(row["title"] or ""),
media_count=row["media_count"],
last_ids_at=row["last_ids_at"],
last_full_at=row["last_full_at"],
enabled=bool(row["enabled"]),
)
for row in rows
]
return self.db.read(run)
def folder_active_aids(self, media_id: int) -> set[int]:
def run(conn: sqlite3.Connection) -> set[int]:
rows = conn.execute(
"SELECT aid FROM folder_resource WHERE media_id = ? AND active = 1",
(int(media_id),),
).fetchall()
return {int(row["aid"]) for row in rows}
return self.db.read(run)
def touch_folder_resources(self, media_id: int, *, now: int | None = None) -> None:
now = self._now() if now is None else int(now)
self.db.write(lambda conn: conn.execute(
"UPDATE folder_resource SET last_seen = ? WHERE media_id = ? AND active = 1",
(now, int(media_id)),
))
# ------------------------------------------------------------ video
@staticmethod
def _recompute_primary_folder(conn: sqlite3.Connection, aid: int) -> int | None:
"""按有效收藏关系重算主归属,避免依赖收藏夹遍历顺序。
优先使用逐夹 ``fav_time``(后加入者优先),其次 ``first_seen``,
最后用年度与 media_id 做确定性决胜。
"""
row = conn.execute(
"""
SELECT fr.media_id
FROM folder_resource fr
JOIN folder f ON f.media_id = fr.media_id
WHERE fr.aid = ? AND fr.active = 1 AND f.enabled = 1
ORDER BY COALESCE(fr.fav_time, 0) DESC,
fr.first_seen DESC,
f.year DESC,
fr.media_id DESC
LIMIT 1
""",
(int(aid),),
).fetchone()
if row is None:
return None
folder_id = int(row["media_id"])
conn.execute(
"UPDATE video SET folder_id = ? WHERE aid = ?", (folder_id, int(aid))
)
return folder_id
def upsert_from_fav(
self,
resource,
folder_id: int,
*,
was_added: bool,
now: int | None = None,
) -> UpsertOutcome:
now = self._now() if now is None else int(now)
folder_id = int(folder_id)
invalid = bool(getattr(resource, "invalid", False))
def run(conn: sqlite3.Connection) -> UpsertOutcome:
previous = conn.execute(
"SELECT status, folder_id FROM video WHERE aid = ?", (resource.aid,)
).fetchone()
previous_status = str(previous["status"]) if previous else None
previous_folder = (
int(previous["folder_id"])
if previous and previous["folder_id"] is not None else None
)
if previous_status is None:
status = "removed" if invalid else "active"
elif previous_status == "dropped":
status = "active" if (was_added and not invalid) else "dropped"
elif invalid:
status = "removed"
else:
status = "active"
conn.execute(
"""
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
duration, cover, folder_id, fav_time, status,
source, first_seen, last_sync, attr)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(aid) DO UPDATE SET
bvid = COALESCE(excluded.bvid, video.bvid),
title = CASE
WHEN (excluded.attr = 9 OR excluded.status = 'removed')
AND video.title != ''
THEN video.title
WHEN excluded.title != '' THEN excluded.title
ELSE video.title
END,
up_mid = COALESCE(excluded.up_mid, video.up_mid),
up_name = COALESCE(excluded.up_name, video.up_name),
pubdate = CASE
WHEN excluded.pubdate > 0 THEN excluded.pubdate
ELSE video.pubdate
END,
duration = COALESCE(excluded.duration, video.duration),
cover = COALESCE(excluded.cover, video.cover),
folder_id = CASE
WHEN video.folder_id IS NULL THEN excluded.folder_id
ELSE video.folder_id
END,
fav_time = COALESCE(excluded.fav_time, video.fav_time),
status = excluded.status,
source = COALESCE(video.source, excluded.source),
last_sync = excluded.last_sync,
attr = excluded.attr
""",
(
resource.aid,
resource.bvid,
resource.title or "(无标题)",
resource.up_mid,
resource.up_name,
int(resource.pubdate or 0),
resource.duration,
resource.cover,
folder_id,
resource.fav_time,
status,
f"fav:{folder_id}",
now,
now,
resource.attr,
),
)
conn.execute(
"""
INSERT INTO folder_resource(media_id, aid, first_seen, active,
last_seen, fav_time)
VALUES (?, ?, ?, 1, ?, ?)
ON CONFLICT(media_id, aid) DO UPDATE SET
active = 1,
last_seen = excluded.last_seen,
fav_time = COALESCE(excluded.fav_time, folder_resource.fav_time)
""",
(folder_id, resource.aid, now, now, resource.fav_time),
)
primary = self._recompute_primary_folder(conn, resource.aid)
return UpsertOutcome(
aid=resource.aid,
status=status,
previous_folder_id=previous_folder,
moved_folder=previous_folder is not None and previous_folder != primary,
)
return self.db.write(run)
def mark_missing_resource(
self,
aid: int,
*,
folder_id: int,
bvid: str | None = None,
now: int | None = None,
) -> str:
"""处理 infos 静默缺失:写入/更新为 removed,保留历史。"""
now = self._now() if now is None else int(now)
aid = int(aid)
folder_id = int(folder_id)
def run(conn: sqlite3.Connection) -> str:
row = conn.execute("SELECT status FROM video WHERE aid = ?", (aid,)).fetchone()
previous_status = str(row["status"]) if row else None
status = "dropped" if previous_status == "dropped" else "removed"
conn.execute(
"""
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
duration, cover, folder_id, fav_time, status,
source, first_seen, last_sync, attr)
VALUES (?, ?, '(资源已失效)', NULL, NULL, 0, NULL, NULL, ?, NULL,
?, ?, ?, ?, 9)
ON CONFLICT(aid) DO UPDATE SET
bvid = COALESCE(video.bvid, excluded.bvid),
status = ?,
last_sync = excluded.last_sync,
attr = 9
""",
(aid, bvid, folder_id, status, f"fav:{folder_id}", now, now, status),
)
conn.execute(
"""
INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen)
VALUES (?, ?, ?, 1, ?)
ON CONFLICT(media_id, aid) DO UPDATE SET
active = 1,
last_seen = excluded.last_seen
""",
(folder_id, aid, now, now),
)
self._recompute_primary_folder(conn, aid)
return status
return self.db.write(run)
def mark_resource_removed(
self, media_id: int, aid: int, *, now: int | None = None
) -> str | None:
"""资源从收藏夹移除:active -> unfaved,其他(dropped/removed)保持不变。"""
now = self._now() if now is None else int(now)
media_id, aid = int(media_id), int(aid)
def run(conn: sqlite3.Connection) -> str | None:
conn.execute(
"UPDATE folder_resource SET active = 0, last_seen = ? "
"WHERE media_id = ? AND aid = ?",
(now, media_id, aid),
)
self._recompute_primary_folder(conn, aid)
other = conn.execute(
"""
SELECT 1
FROM folder_resource fr
JOIN folder f ON f.media_id = fr.media_id
WHERE fr.aid = ? AND fr.active = 1 AND f.enabled = 1
LIMIT 1
""",
(aid,),
).fetchone()
if other is None:
conn.execute(
"UPDATE video SET status = 'unfaved', last_sync = ? "
"WHERE aid = ? AND status IN ('active', 'unfaved')",
(now, aid),
)
row = conn.execute("SELECT status FROM video WHERE aid = ?", (aid,)).fetchone()
return str(row["status"]) if row else None
return self.db.write(run)
def set_video_status(self, aid: int, status: str, *, now: int | None = None) -> None:
now = self._now() if now is None else int(now)
self.db.write(lambda conn: conn.execute(
"UPDATE video SET status = ?, last_sync = ? WHERE aid = ?",
(status, now, int(aid)),
))
def get_video(self, aid: int) -> dict[str, Any] | None:
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
row = conn.execute("SELECT * FROM video WHERE aid = ?", (int(aid),)).fetchone()
return dict(row) if row else None
return self.db.read(run)
def list_videos(self) -> list[dict[str, Any]]:
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
rows = conn.execute("SELECT * FROM video ORDER BY aid").fetchall()
return [dict(row) for row in rows]
return self.db.read(run)
def folder_resources(self, media_id: int) -> list[dict[str, Any]]:
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
rows = conn.execute(
"SELECT * FROM folder_resource WHERE media_id = ? ORDER BY aid",
(int(media_id),),
).fetchall()
return [dict(row) for row in rows]
return self.db.read(run)
# ------------------------------------------------------------ snapshot
def video_aids_for_snapshot(self) -> list[int]:
def run(conn: sqlite3.Connection) -> list[int]:
rows = conn.execute(
"SELECT aid FROM video WHERE status IN ('active', 'unfaved') "
"ORDER BY aid"
).fetchall()
return [int(row["aid"]) for row in rows]
return self.db.read(run)
def pending_snapshot_aids(self, stat_date: str) -> list[int]:
"""当天仍缺成功快照的追踪视频;用于新入库/恢复后的同日补采。"""
def run(conn: sqlite3.Connection) -> list[int]:
rows = conn.execute(
"""
SELECT v.aid
FROM video v
WHERE v.status IN ('active', 'unfaved')
AND NOT EXISTS (
SELECT 1 FROM snapshot s
WHERE s.stat_date = ? AND s.aid = v.aid AND s.ok = 1
)
ORDER BY v.aid
""",
(stat_date,),
).fetchall()
return [int(row["aid"]) for row in rows]
return self.db.read(run)
def get_snapshot(self, stat_date: str, aid: int) -> dict[str, Any] | None:
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
row = conn.execute(
"SELECT * FROM snapshot WHERE stat_date = ? AND aid = ?",
(stat_date, int(aid)),
).fetchone()
return dict(row) if row else None
return self.db.read(run)
def upsert_snapshot(
self,
stat_date: str,
aid: int,
*,
metrics: Mapping[str, int] | None,
ok: bool,
err: str | None,
captured_at: int,
src: str = "view",
) -> None:
values = metrics or {}
def run(conn: sqlite3.Connection) -> None:
conn.execute(
"""
INSERT INTO snapshot(stat_date, aid, "view", "like", favorite, coin,
share, reply, danmaku, captured_at, ok, err, src)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(stat_date, aid) DO UPDATE SET
"view" = excluded."view",
"like" = excluded."like",
favorite = excluded.favorite,
coin = excluded.coin,
share = excluded.share,
reply = excluded.reply,
danmaku = excluded.danmaku,
captured_at = excluded.captured_at,
ok = excluded.ok,
err = excluded.err,
src = excluded.src
""",
(
stat_date,
int(aid),
values.get("view"),
values.get("like"),
values.get("favorite"),
values.get("coin"),
values.get("share"),
values.get("reply"),
values.get("danmaku"),
int(captured_at),
1 if ok else 0,
err,
src,
),
)
self.db.write(run)
def snapshot_counts(self, stat_date: str) -> dict[str, int]:
def run(conn: sqlite3.Connection) -> dict[str, int]:
rows = conn.execute(
"SELECT ok, COUNT(*) AS n FROM snapshot WHERE stat_date = ? GROUP BY ok",
(stat_date,),
).fetchall()
by_ok = {int(row["ok"]): int(row["n"]) for row in rows}
return {
"total": sum(by_ok.values()),
"ok": by_ok.get(1, 0),
"failed": by_ok.get(0, 0),
}
return self.db.read(run)
def upsert_up_stat(
self,
stat_date: str,
mid: int,
*,
follower: int | None,
following: int | None,
ok: bool,
captured_at: int,
) -> None:
self.db.write(lambda conn: conn.execute(
"""
INSERT INTO up_stat(stat_date, mid, follower, following, captured_at, ok)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(stat_date, mid) DO UPDATE SET
follower = excluded.follower,
following = excluded.following,
captured_at = excluded.captured_at,
ok = excluded.ok
""",
(stat_date, int(mid), follower, following, int(captured_at), 1 if ok else 0),
))
def get_up_stat(self, stat_date: str, mid: int) -> dict[str, Any] | None:
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
row = conn.execute(
"SELECT * FROM up_stat WHERE stat_date = ? AND mid = ?",
(stat_date, int(mid)),
).fetchone()
return dict(row) if row else None
return self.db.read(run)
# ------------------------------------------------------------ kv
def kv_get(self, key: str) -> str | None:
def run(conn: sqlite3.Connection) -> str | None:
row = conn.execute("SELECT v FROM kv WHERE k = ?", (key,)).fetchone()
return None if row is None else str(row["v"])
return self.db.read(run)
def kv_set(self, key: str, value: str) -> None:
self.db.write(lambda conn: conn.execute(
"INSERT INTO kv(k, v) VALUES (?, ?) "
"ON CONFLICT(k) DO UPDATE SET v = excluded.v",
(key, str(value)),
))
# ------------------------------------------------------------ blacklist
def seed_blacklist(self, mids: Iterable[int], *, now: int | None = None) -> int:
now = self._now() if now is None else int(now)
def run(conn: sqlite3.Connection) -> int:
inserted = 0
for mid in mids:
cur = conn.execute(
"INSERT OR IGNORE INTO up_blacklist(mid, name, reason, created_at) "
"VALUES (?, NULL, 'seed', ?)",
(int(mid), now),
)
inserted += max(cur.rowcount, 0)
return inserted
return self.db.write(run)
def add_blacklist(
self,
mid: int,
*,
name: str | None = None,
reason: str | None = None,
now: int | None = None,
) -> None:
now = self._now() if now is None else int(now)
self.db.write(lambda conn: conn.execute(
"""
INSERT INTO up_blacklist(mid, name, reason, created_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(mid) DO UPDATE SET
name = COALESCE(excluded.name, up_blacklist.name),
reason = COALESCE(excluded.reason, up_blacklist.reason)
""",
(int(mid), name, reason, now),
))
def is_blacklisted(self, mid: int) -> bool:
def run(conn: sqlite3.Connection) -> bool:
row = conn.execute(
"SELECT 1 FROM up_blacklist WHERE mid = ?", (int(mid),)
).fetchone()
return row is not None
return self.db.read(run)
def blacklist(self) -> list[dict[str, Any]]:
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
rows = conn.execute(
"SELECT mid, name, reason, created_at FROM up_blacklist ORDER BY created_at, mid"
).fetchall()
return [dict(row) for row in rows]
return self.db.read(run)
# ------------------------------------------------------------ summary
def counts(self) -> dict[str, Any]:
def run(conn: sqlite3.Connection) -> dict[str, Any]:
status_rows = conn.execute(
"SELECT status, COUNT(*) AS n FROM video GROUP BY status"
).fetchall()
by_status = {str(row["status"]): int(row["n"]) for row in status_rows}
folder_count = conn.execute("SELECT COUNT(*) FROM folder").fetchone()[0]
blacklist_count = conn.execute("SELECT COUNT(*) FROM up_blacklist").fetchone()[0]
candidate_count = conn.execute("SELECT COUNT(*) FROM candidate").fetchone()[0]
return {
"video_total": sum(by_status.values()),
"video_by_status": by_status,
"folders": int(folder_count),
"blacklist": int(blacklist_count),
"candidates": int(candidate_count),
}
return self.db.read(run)