kairee
c5b7185675
Let 1-day view growth use a zero baseline for videos published today, and the first successful sample for older videos ingested today, without mixing those into 7/30-day rankings.
395 lines
13 KiB
Python
395 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""SQLite schema 迁移。
|
||
|
||
规则（DEVELOPMENT.md §6.3）：
|
||
|
||
* 使用 ``schema_version(version, applied_at)`` 记录已应用版本；
|
||
* 每个版本包在单一事务里，失败整体回滚；
|
||
* 只向前追加，不允许删库重建；
|
||
* 升级已有数据库前自动创建 SQLite 备份。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import sqlite3
|
||
import time
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Sequence
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Migration:
|
||
version: int
|
||
name: str
|
||
statements: tuple[str, ...]
|
||
|
||
|
||
MIGRATIONS: tuple[Migration, ...] = (
|
||
Migration(
|
||
version=1,
|
||
name="initial schema",
|
||
statements=(
|
||
"""
|
||
CREATE TABLE video (
|
||
aid INTEGER PRIMARY KEY,
|
||
bvid TEXT UNIQUE,
|
||
title TEXT NOT NULL,
|
||
up_mid INTEGER,
|
||
up_name TEXT,
|
||
pubdate INTEGER NOT NULL,
|
||
duration INTEGER,
|
||
cover TEXT,
|
||
folder_id INTEGER,
|
||
fav_time INTEGER,
|
||
status TEXT NOT NULL,
|
||
source TEXT,
|
||
first_seen INTEGER NOT NULL,
|
||
last_sync INTEGER,
|
||
attr INTEGER
|
||
)
|
||
""",
|
||
"CREATE INDEX idx_video_status ON video(status, pubdate DESC)",
|
||
"CREATE INDEX idx_video_up ON video(up_mid, pubdate DESC)",
|
||
"""
|
||
CREATE TABLE snapshot (
|
||
stat_date TEXT NOT NULL,
|
||
aid INTEGER NOT NULL,
|
||
view INTEGER,
|
||
like INTEGER,
|
||
favorite INTEGER,
|
||
coin INTEGER,
|
||
share INTEGER,
|
||
reply INTEGER,
|
||
danmaku INTEGER,
|
||
captured_at INTEGER NOT NULL,
|
||
ok INTEGER NOT NULL DEFAULT 1,
|
||
err TEXT,
|
||
src TEXT DEFAULT 'view',
|
||
PRIMARY KEY (stat_date, aid)
|
||
)
|
||
""",
|
||
"CREATE INDEX idx_snap_aid ON snapshot(aid, stat_date)",
|
||
"""
|
||
CREATE TABLE folder (
|
||
media_id INTEGER PRIMARY KEY,
|
||
year INTEGER NOT NULL,
|
||
title TEXT,
|
||
media_count INTEGER,
|
||
last_ids_at INTEGER,
|
||
last_full_at INTEGER,
|
||
enabled INTEGER NOT NULL DEFAULT 1
|
||
)
|
||
""",
|
||
"""
|
||
CREATE TABLE folder_resource (
|
||
media_id INTEGER NOT NULL,
|
||
aid INTEGER NOT NULL,
|
||
first_seen INTEGER NOT NULL,
|
||
active INTEGER NOT NULL DEFAULT 1,
|
||
last_seen INTEGER,
|
||
PRIMARY KEY (media_id, aid)
|
||
)
|
||
""",
|
||
"CREATE INDEX idx_folder_resource_aid ON folder_resource(aid, active)",
|
||
"""
|
||
CREATE TABLE up_blacklist (
|
||
mid INTEGER PRIMARY KEY,
|
||
name TEXT,
|
||
reason TEXT,
|
||
created_at INTEGER NOT NULL
|
||
)
|
||
""",
|
||
"""
|
||
CREATE TABLE up_stat (
|
||
stat_date TEXT NOT NULL,
|
||
mid INTEGER NOT NULL,
|
||
follower INTEGER,
|
||
following INTEGER,
|
||
captured_at INTEGER NOT NULL,
|
||
ok INTEGER NOT NULL DEFAULT 1,
|
||
PRIMARY KEY (stat_date, mid)
|
||
)
|
||
""",
|
||
"""
|
||
CREATE TABLE candidate (
|
||
aid INTEGER PRIMARY KEY,
|
||
bvid TEXT,
|
||
title TEXT,
|
||
up_mid INTEGER,
|
||
up_name TEXT,
|
||
pubdate INTEGER,
|
||
cover TEXT,
|
||
score REAL,
|
||
channel TEXT,
|
||
first_seen INTEGER NOT NULL,
|
||
notified_at INTEGER,
|
||
decision TEXT,
|
||
decided_at INTEGER
|
||
)
|
||
""",
|
||
"""
|
||
CREATE TABLE seen_aid (
|
||
aid INTEGER PRIMARY KEY,
|
||
first_seen INTEGER NOT NULL,
|
||
source TEXT
|
||
)
|
||
""",
|
||
"""
|
||
CREATE TABLE notify (
|
||
aid INTEGER NOT NULL,
|
||
channel TEXT NOT NULL,
|
||
batch_id TEXT,
|
||
sent_at INTEGER NOT NULL,
|
||
ok INTEGER NOT NULL,
|
||
err TEXT,
|
||
PRIMARY KEY (aid, channel)
|
||
)
|
||
""",
|
||
"""
|
||
CREATE TABLE fav_action (
|
||
token TEXT PRIMARY KEY,
|
||
aid INTEGER NOT NULL,
|
||
folder_id INTEGER NOT NULL,
|
||
state TEXT NOT NULL,
|
||
created_at INTEGER NOT NULL,
|
||
expires_at INTEGER NOT NULL,
|
||
done_at INTEGER,
|
||
position INTEGER,
|
||
err TEXT
|
||
)
|
||
""",
|
||
"""
|
||
CREATE TABLE kv (
|
||
k TEXT PRIMARY KEY,
|
||
v TEXT
|
||
)
|
||
""",
|
||
),
|
||
),
|
||
Migration(
|
||
version=2,
|
||
name="folder_resource fav_time",
|
||
statements=(
|
||
"ALTER TABLE folder_resource ADD COLUMN fav_time INTEGER",
|
||
),
|
||
),
|
||
Migration(
|
||
version=3,
|
||
name="candidate stats",
|
||
statements=(
|
||
"ALTER TABLE candidate ADD COLUMN duration INTEGER",
|
||
"ALTER TABLE candidate ADD COLUMN play INTEGER",
|
||
"ALTER TABLE candidate ADD COLUMN like INTEGER",
|
||
"ALTER TABLE candidate ADD COLUMN favorite INTEGER",
|
||
"ALTER TABLE candidate ADD COLUMN danmaku INTEGER",
|
||
"ALTER TABLE candidate ADD COLUMN review INTEGER",
|
||
),
|
||
),
|
||
Migration(
|
||
version=4,
|
||
name="video pages and up follower snapshot",
|
||
statements=(
|
||
"ALTER TABLE video ADD COLUMN pages INTEGER",
|
||
"ALTER TABLE video ADD COLUMN up_follower INTEGER",
|
||
"ALTER TABLE video ADD COLUMN up_follower_at INTEGER",
|
||
"ALTER TABLE video ADD COLUMN up_follower_backfilled INTEGER NOT NULL DEFAULT 0",
|
||
"""
|
||
CREATE TABLE up_profile (
|
||
mid INTEGER PRIMARY KEY,
|
||
name TEXT,
|
||
follower_first INTEGER,
|
||
following_first INTEGER,
|
||
first_captured_at INTEGER,
|
||
follower_latest INTEGER,
|
||
last_refreshed INTEGER
|
||
)
|
||
""",
|
||
),
|
||
),
|
||
Migration(
|
||
version=5,
|
||
name="stop favorite-folder tracking",
|
||
statements=(
|
||
# 不再追踪收藏夹后，“取消收藏”状态失去语义；历史 unfaved 行归一到 active。
|
||
# 表结构与历史数据保留，仅停止写入。
|
||
"UPDATE video SET status = 'active' WHERE status = 'unfaved'",
|
||
),
|
||
),
|
||
Migration(
|
||
version=6,
|
||
name="video up face",
|
||
statements=(
|
||
# UP 头像 URL（来自 view 的 owner.face），用于 UP 列表/详情展示。
|
||
"ALTER TABLE video ADD COLUMN up_face TEXT",
|
||
),
|
||
),
|
||
Migration(
|
||
version=7,
|
||
name="visit access statistics",
|
||
statements=(
|
||
# 访问统计按 (日期, IP, 路由) 聚合：PV = SUM(hits)，UV = COUNT(DISTINCT ip)。
|
||
# 数据先落内存缓冲、批量 upsert，因此 hits 累加而非逐条插入。
|
||
"""
|
||
CREATE TABLE visit (
|
||
stat_date TEXT NOT NULL,
|
||
ip TEXT NOT NULL,
|
||
path TEXT NOT NULL,
|
||
hits INTEGER NOT NULL DEFAULT 0,
|
||
first_at INTEGER NOT NULL,
|
||
last_at INTEGER NOT NULL,
|
||
PRIMARY KEY (stat_date, ip, path)
|
||
)
|
||
""",
|
||
"CREATE INDEX idx_visit_date ON visit(stat_date)",
|
||
),
|
||
),
|
||
Migration(
|
||
version=8,
|
||
name="snapshot open view",
|
||
statements=(
|
||
# 同日第一次成功采样的播放量，供近 1 日「入库后增量」做基准。
|
||
# 已有成功行无法找回被覆盖的入库值，只能把当前 view 当作开盘。
|
||
'ALTER TABLE snapshot ADD COLUMN view_open INTEGER',
|
||
"ALTER TABLE snapshot ADD COLUMN open_captured_at INTEGER",
|
||
'UPDATE snapshot SET view_open = "view", open_captured_at = captured_at '
|
||
"WHERE ok = 1 AND view_open IS NULL",
|
||
),
|
||
),
|
||
)
|
||
|
||
|
||
class MigrationError(RuntimeError):
|
||
"""迁移失败，原事务已回滚。"""
|
||
|
||
|
||
def _schema_version_table(conn: sqlite3.Connection) -> None:
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS schema_version ("
|
||
"version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)"
|
||
)
|
||
|
||
|
||
def current_version(conn: sqlite3.Connection) -> int:
|
||
_schema_version_table(conn)
|
||
row = conn.execute("SELECT MAX(version) FROM schema_version").fetchone()
|
||
return int(row[0] or 0) if row else 0
|
||
|
||
|
||
def _user_tables(conn: sqlite3.Connection) -> list[str]:
|
||
rows = conn.execute(
|
||
"SELECT name FROM sqlite_master "
|
||
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'schema_version'"
|
||
).fetchall()
|
||
return [str(row[0]) for row in rows]
|
||
|
||
|
||
def backup_database(
|
||
conn: sqlite3.Connection,
|
||
backup_dir: str | Path,
|
||
*,
|
||
version: int,
|
||
now: float | None = None,
|
||
) -> Path | None:
|
||
"""用 SQLite backup API 生成一致性备份；内存库返回 None。"""
|
||
try:
|
||
row = conn.execute("PRAGMA database_list").fetchone()
|
||
except sqlite3.Error:
|
||
return None
|
||
if not row or not row[2] or str(row[2]) == ":memory:":
|
||
return None
|
||
|
||
target_dir = Path(backup_dir)
|
||
target_dir.mkdir(parents=True, exist_ok=True)
|
||
stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(now))
|
||
target = target_dir / f"app-pre-v{version}-{stamp}.db"
|
||
suffix = 1
|
||
while target.exists():
|
||
target = target_dir / f"app-pre-v{version}-{stamp}-{suffix}.db"
|
||
suffix += 1
|
||
|
||
dst = sqlite3.connect(target)
|
||
try:
|
||
conn.backup(dst)
|
||
finally:
|
||
dst.close()
|
||
return target
|
||
|
||
|
||
def prune_backups(
|
||
backup_dir: str | Path, *, keep_days: int, now: float | None = None
|
||
) -> list[Path]:
|
||
"""删除超过保留期的迁移备份。"""
|
||
target_dir = Path(backup_dir)
|
||
if not target_dir.is_dir():
|
||
return []
|
||
cutoff = (time.time() if now is None else now) - keep_days * 86400
|
||
removed: list[Path] = []
|
||
for path in target_dir.glob("app-pre-v*.db"):
|
||
try:
|
||
if path.stat().st_mtime < cutoff:
|
||
path.unlink()
|
||
removed.append(path)
|
||
except OSError:
|
||
continue
|
||
return removed
|
||
|
||
|
||
def _validate_migrations(migrations: Sequence[Migration]) -> None:
|
||
versions = [m.version for m in migrations]
|
||
if versions != sorted(versions):
|
||
raise ValueError("迁移版本必须按升序排列")
|
||
if len(set(versions)) != len(versions):
|
||
raise ValueError("迁移版本不能重复")
|
||
if versions and versions[0] < 1:
|
||
raise ValueError("迁移版本必须从 1 开始")
|
||
|
||
|
||
def run_migrations(
|
||
conn: sqlite3.Connection,
|
||
*,
|
||
backup_dir: str | Path | None = None,
|
||
backup_keep_days: int = 14,
|
||
now: float | None = None,
|
||
migrations: Sequence[Migration] = MIGRATIONS,
|
||
logger=None,
|
||
) -> list[int]:
|
||
"""应用所有待执行迁移，返回本次应用的版本列表。"""
|
||
_validate_migrations(migrations)
|
||
_schema_version_table(conn)
|
||
version = current_version(conn)
|
||
pending = [m for m in migrations if m.version > version]
|
||
if not pending:
|
||
return []
|
||
|
||
if backup_dir is not None and _user_tables(conn):
|
||
backup = backup_database(conn, backup_dir, version=version, now=now)
|
||
prune_backups(backup_dir, keep_days=backup_keep_days)
|
||
if logger is not None and backup is not None:
|
||
logger.info("db.backup", path=str(backup), version=version)
|
||
|
||
applied: list[int] = []
|
||
for migration in pending:
|
||
try:
|
||
conn.execute("BEGIN IMMEDIATE")
|
||
for statement in migration.statements:
|
||
conn.execute(statement)
|
||
conn.execute(
|
||
"INSERT INTO schema_version(version, applied_at) VALUES (?, ?)",
|
||
(migration.version, int(time.time() if now is None else now)),
|
||
)
|
||
conn.execute("COMMIT")
|
||
except Exception as exc: # noqa: BLE001 任何错误都必须回滚
|
||
try:
|
||
conn.execute("ROLLBACK")
|
||
except sqlite3.Error:
|
||
pass
|
||
raise MigrationError(
|
||
f"迁移 v{migration.version} ({migration.name}) 失败，已回滚: {exc}"
|
||
) from exc
|
||
applied.append(migration.version)
|
||
if logger is not None:
|
||
logger.info("db.migrated", version=migration.version, name=migration.name)
|
||
|
||
return applied
|