feat(db): add video pages and up follower snapshot schema

This commit is contained in:
kairee committed 2026-11-01 11:14:36 +08:00
1 parent 707ed7d7d7
commit 4942d1d161
10 files changed
+521 -13

No files matched your search

+125
View File
@@ -35,6 +35,9 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
MIN_VIEW_FOR_INTERACTION = 100 MIN_VIEW_FOR_INTERACTION = 100
MIN_VIEW_FOR_GROWTH_RATE = 200 MIN_VIEW_FOR_GROWTH_RATE = 200
MIN_EFFICIENCY_DURATION = 600
DEFAULT_MATURITY_DAYS = 14
DEFAULT_SHRINK_MINUTES = 120.0
_IQR_TO_SIGMA = 1.349 _IQR_TO_SIGMA = 1.349
@@ -83,6 +86,128 @@ def _metric_value(row: Mapping[str, object], metric: str) -> float:
return 0.0 return 0.0
def content_hours(duration_seconds: object) -> float | None:
"""把视频总时长(秒)换算为内容小时;缺失或非正数返回 ``None``。
多 P 视频的 ``duration`` 已是各 P 之和,因此这里不需要再乘 P 数。
"""
if duration_seconds is None or isinstance(duration_seconds, bool):
return None
try:
seconds = float(duration_seconds)
except (TypeError, ValueError):
return None
if seconds <= 0:
return None
return seconds / 3600.0
def views_per_hour(view: object, duration_seconds: object) -> float | None:
"""播放/内容小时;时长缺失或非正数时返回 ``None``,不做除零。"""
hours = content_hours(duration_seconds)
if hours is None:
return None
return _metric_value({"view": view}, "view") / hours
def interactions_per_hour(
metrics: Mapping[str, object] | None,
duration_seconds: object,
*,
interaction_metrics: Sequence[str] = ("like", "coin", "favorite"),
) -> float | None:
"""互动/内容小时;统计缺失或时长无效时返回 ``None``。"""
hours = content_hours(duration_seconds)
if hours is None or metrics is None:
return None
interactions = sum(_metric_value(metrics, metric) for metric in interaction_metrics)
return interactions / hours
def age_days(pubdate: object, reference: int | float) -> int | None:
"""发布到参考时点(通常为最新数据日)的整天数;无效发布时间返回 ``None``。"""
if pubdate is None or isinstance(pubdate, bool):
return None
try:
published = int(pubdate)
now = int(reference)
except (TypeError, ValueError):
return None
if published <= 0:
return None
return max(0, (now - published) // 86400)
def is_mature(
pubdate: object,
reference: int | float,
maturity_days: int = DEFAULT_MATURITY_DAYS,
) -> bool:
"""是否已达到成熟窗口;发布时间缺失视为未成熟。"""
days = age_days(pubdate, reference)
return days is not None and days >= max(0, int(maturity_days))
def median_of(values: Iterable[object]) -> float | None:
"""忽略无效值的中位数;没有有效值返回 ``None``。"""
numbers: list[float] = []
for value in values:
if value is None or isinstance(value, bool):
continue
try:
numbers.append(float(value))
except (TypeError, ValueError):
continue
if not numbers:
return None
return float(_stat_median(numbers))
def effective_shrink_minutes(
per_up_minutes: Iterable[object],
fallback: float = DEFAULT_SHRINK_MINUTES,
) -> float:
"""自适应收缩强度:各 UP 总内容分钟的中位数,缺失时回退固定值。"""
positives: list[float] = []
for minutes in per_up_minutes:
if minutes is None or isinstance(minutes, bool):
continue
try:
value = float(minutes)
except (TypeError, ValueError):
continue
if value > 0:
positives.append(value)
value = median_of(positives)
if value is None or value <= 0:
return float(fallback)
return value
def eb_rate(
total: object,
exposure: object,
prior_rate: float,
prior_strength: float,
) -> float | None:
"""Gamma–Poisson 经验贝叶斯收缩后的速率。
``exposure`` 与 ``prior_strength`` 必须使用同一单位(本项目中为分钟),
返回单位与 ``prior_rate`` 一致(每分钟)。
"""
if prior_strength < 0:
raise ValueError("prior_strength 不能为负")
try:
total_value = max(0.0, float(total))
exposure_value = max(0.0, float(exposure))
prior = max(0.0, float(prior_rate))
except (TypeError, ValueError):
return None
if prior_strength <= 0:
return None if exposure_value <= 0 else total_value / exposure_value
return (total_value + prior_strength * prior) / (exposure_value + prior_strength)
def normalized_weights(weights: Mapping[str, float] | None = None) -> dict[str, float]: def normalized_weights(weights: Mapping[str, float] | None = None) -> dict[str, float]:
source = weights or DEFAULT_HEAT_WEIGHTS source = weights or DEFAULT_HEAT_WEIGHTS
result: dict[str, float] = {} result: dict[str, float] = {}
+8
View File
@@ -30,6 +30,7 @@ class FavResource:
cover: str | None cover: str | None
attr: int attr: int
fav_time: int | None fav_time: int | None
pages: int | None = None
@property @property
def invalid(self) -> bool: def invalid(self) -> bool:
@@ -51,6 +52,12 @@ class FavResource:
duration = raw_duration if isinstance(raw_duration, int) and not isinstance( duration = raw_duration if isinstance(raw_duration, int) and not isinstance(
raw_duration, bool raw_duration, bool
) else None ) else None
raw_pages = item.get("page")
pages = raw_pages if (
isinstance(raw_pages, int)
and not isinstance(raw_pages, bool)
and raw_pages > 0
) else None
raw_attr = item.get("attr", 0) raw_attr = item.get("attr", 0)
attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0 attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0
fav_time = item.get("fav_time") fav_time = item.get("fav_time")
@@ -70,6 +77,7 @@ class FavResource:
cover=str(item.get("cover") or "") or None, cover=str(item.get("cover") or "") or None,
attr=attr, attr=attr,
fav_time=fav_time, fav_time=fav_time,
pages=pages,
) )
+45
View File
@@ -38,6 +38,13 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
"danmaku": 0.02, "danmaku": 0.02,
} }
DEFAULT_ANALYTICS: dict[str, Any] = {
"min_duration_seconds": 600,
"maturity_days": 14,
"min_view_for_efficiency": 0,
"shrink_minutes": None,
}
DEFAULT_CONFIG: dict[str, Any] = { DEFAULT_CONFIG: dict[str, Any] = {
"bilibili": { "bilibili": {
"read": {"mid": 0, "csrf": "", "cookie": ""}, "read": {"mid": 0, "csrf": "", "cookie": ""},
@@ -73,6 +80,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
"notify": {"min_score": 2, "max_items_per_message": 10}, "notify": {"min_score": 2, "max_items_per_message": 10},
"classify": DEFAULT_CLASSIFY, "classify": DEFAULT_CLASSIFY,
"heat_weights": DEFAULT_HEAT_WEIGHTS, "heat_weights": DEFAULT_HEAT_WEIGHTS,
"analytics": DEFAULT_ANALYTICS,
} }
# 环境变量只作为 CI / 临时调试的覆盖入口,不作为主路径。 # 环境变量只作为 CI / 临时调试的覆盖入口,不作为主路径。
@@ -161,6 +169,14 @@ class DiscoverConfig:
window_days: int window_days: int
@dataclass(frozen=True)
class AnalyticsConfig:
min_duration_seconds: int
maturity_days: int
min_view_for_efficiency: int
shrink_minutes: int | None
@dataclass(frozen=True) @dataclass(frozen=True)
class AppConfig: class AppConfig:
path: Path path: Path
@@ -176,6 +192,7 @@ class AppConfig:
discover: DiscoverConfig discover: DiscoverConfig
classify: Mapping[str, Any] classify: Mapping[str, Any]
heat_weights: Mapping[str, float] heat_weights: Mapping[str, float]
analytics: AnalyticsConfig
raw: Mapping[str, Any] = field(repr=False, compare=False) raw: Mapping[str, Any] = field(repr=False, compare=False)
def folder_for_year(self, year: int) -> FolderConfig | None: def folder_for_year(self, year: int) -> FolderConfig | None:
@@ -431,6 +448,33 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
raise ConfigError(f"heat_weights.{metric} 必须是非负数字") raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
heat[metric] = float(value) heat[metric] = float(value)
analytics_raw = data["analytics"]
if not isinstance(analytics_raw, Mapping):
raise ConfigError("analytics 必须是对象")
shrink_raw = analytics_raw.get("shrink_minutes")
analytics = AnalyticsConfig(
min_duration_seconds=_as_int(
analytics_raw.get("min_duration_seconds", 600),
"analytics.min_duration_seconds",
minimum=0,
),
maturity_days=_as_int(
analytics_raw.get("maturity_days", 14),
"analytics.maturity_days",
minimum=0,
),
min_view_for_efficiency=_as_int(
analytics_raw.get("min_view_for_efficiency", 0),
"analytics.min_view_for_efficiency",
minimum=0,
),
shrink_minutes=(
None if shrink_raw is None else _as_int(
shrink_raw, "analytics.shrink_minutes", minimum=1
)
),
)
return AppConfig( return AppConfig(
path=path, path=path,
bilibili=bilibili, bilibili=bilibili,
@@ -445,6 +489,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
discover=discover, discover=discover,
classify=classify, classify=classify,
heat_weights=heat, heat_weights=heat,
analytics=analytics,
raw=data, raw=data,
) )
+21
View File
@@ -186,6 +186,27 @@ MIGRATIONS: tuple[Migration, ...] = (
"ALTER TABLE candidate ADD COLUMN review INTEGER", "ALTER TABLE candidate ADD COLUMN review INTEGER",
), ),
), ),
Migration(
version=4,
name="video pages and up follower snapshot",
statements=(
"ALTER TABLE video ADD COLUMN pages INTEGER",
"ALTER TABLE video ADD COLUMN up_follower INTEGER",
"ALTER TABLE video ADD COLUMN up_follower_at INTEGER",
"ALTER TABLE video ADD COLUMN up_follower_backfilled INTEGER NOT NULL DEFAULT 0",
"""
CREATE TABLE up_profile (
mid INTEGER PRIMARY KEY,
name TEXT,
follower_first INTEGER,
following_first INTEGER,
first_captured_at INTEGER,
follower_latest INTEGER,
last_refreshed INTEGER
)
""",
),
),
) )
+103 -2
View File
@@ -229,9 +229,9 @@ class Repository:
conn.execute( conn.execute(
""" """
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate, INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
duration, cover, folder_id, fav_time, status, duration, pages, cover, folder_id, fav_time, status,
source, first_seen, last_sync, attr) source, first_seen, last_sync, attr)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(aid) DO UPDATE SET ON CONFLICT(aid) DO UPDATE SET
bvid = COALESCE(excluded.bvid, video.bvid), bvid = COALESCE(excluded.bvid, video.bvid),
title = CASE title = CASE
@@ -248,6 +248,7 @@ class Repository:
ELSE video.pubdate ELSE video.pubdate
END, END,
duration = COALESCE(excluded.duration, video.duration), duration = COALESCE(excluded.duration, video.duration),
pages = COALESCE(excluded.pages, video.pages),
cover = COALESCE(excluded.cover, video.cover), cover = COALESCE(excluded.cover, video.cover),
folder_id = CASE folder_id = CASE
WHEN video.folder_id IS NULL THEN excluded.folder_id WHEN video.folder_id IS NULL THEN excluded.folder_id
@@ -267,6 +268,7 @@ class Repository:
resource.up_name, resource.up_name,
int(resource.pubdate or 0), int(resource.pubdate or 0),
resource.duration, resource.duration,
resource.pages,
resource.cover, resource.cover,
folder_id, folder_id,
resource.fav_time, resource.fav_time,
@@ -397,6 +399,105 @@ class Repository:
return [dict(row) for row in rows] return [dict(row) for row in rows]
return self.db.read(run) return self.db.read(run)
# ------------------------------------------------------- up profile
def apply_up_follower(
self,
*,
mid: int,
name: str | None,
follower: int,
following: int | None = None,
now: int | None = None,
backfilled: bool = False,
) -> int:
"""写入 UP 粉丝快照,并补齐该 UP 尚未记录粉丝数的视频。
``follower_first`` 只在首次插入时写入;``follower_latest`` 每次刷新都会
更新。视频上的 ``up_follower`` 是入库时快照,已有值不会被覆盖。返回本次
补齐的视频数。
"""
now = self._now() if now is None else int(now)
mid = int(mid)
follower = max(0, int(follower))
following_value = None if following is None else max(0, int(following))
backfill_flag = 1 if backfilled else 0
def run(conn: sqlite3.Connection) -> int:
conn.execute(
"""
INSERT INTO up_profile(mid, name, follower_first, following_first,
first_captured_at, follower_latest, last_refreshed)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(mid) DO UPDATE SET
name = COALESCE(NULLIF(excluded.name, ''), up_profile.name),
follower_latest = excluded.follower_latest,
last_refreshed = excluded.last_refreshed
""",
(mid, name, follower, following_value, now, follower, now),
)
cursor = conn.execute(
"""
UPDATE video
SET up_follower = ?, up_follower_at = ?, up_follower_backfilled = ?
WHERE up_mid = ? AND up_follower IS NULL
""",
(follower, now, backfill_flag, mid),
)
return max(cursor.rowcount, 0)
return self.db.write(run)
def get_up_profile(self, mid: int) -> dict[str, Any] | None:
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
row = conn.execute(
"SELECT * FROM up_profile WHERE mid = ?", (int(mid),)
).fetchone()
return dict(row) if row else None
return self.db.read(run)
def up_mids_missing_follower(self) -> list[dict[str, Any]]:
"""追踪中但仍有视频缺少入库粉丝数的 UP。"""
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
rows = conn.execute(
"""
SELECT up_mid, COALESCE(MAX(up_name), '') AS up_name,
COUNT(*) AS missing
FROM video
WHERE status IN ('active', 'unfaved')
AND up_mid IS NOT NULL
AND up_follower IS NULL
GROUP BY up_mid
ORDER BY up_mid
"""
).fetchall()
return [dict(row) for row in rows]
return self.db.read(run)
def up_follower_coverage(self) -> dict[str, int]:
"""返回追踪视频的粉丝快照覆盖情况。"""
def run(conn: sqlite3.Connection) -> dict[str, int]:
row = conn.execute(
"""
SELECT COUNT(*) AS total,
SUM(CASE WHEN up_follower IS NOT NULL THEN 1 ELSE 0 END) AS covered,
COUNT(DISTINCT CASE WHEN up_follower IS NOT NULL
THEN up_mid END) AS ups_covered
FROM video
WHERE status IN ('active', 'unfaved')
AND up_mid IS NOT NULL
"""
).fetchone()
total = int(row["total"] or 0)
covered = int(row["covered"] or 0)
return {
"total": total,
"covered": covered,
"missing": max(0, total - covered),
"ups_covered": int(row["ups_covered"] or 0),
}
return self.db.read(run)
def folder_resources(self, media_id: int) -> list[dict[str, Any]]: def folder_resources(self, media_id: int) -> list[dict[str, Any]]:
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]: def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
rows = conn.execute( rows = conn.execute(
+78 -1
View File
@@ -10,15 +10,23 @@ sys.stdout.reconfigure(encoding="utf-8")
from reaction_board.analytics import ( # noqa: E402 from reaction_board.analytics import ( # noqa: E402
DEFAULT_HEAT_WEIGHTS, DEFAULT_HEAT_WEIGHTS,
METRICS, METRICS,
age_days,
compute_growth, compute_growth,
content_hours,
eb_rate,
effective_shrink_minutes,
heat_scores, heat_scores,
interactions_per_hour,
interaction_rate, interaction_rate,
is_loopback, is_loopback,
is_mature,
median_of,
normalize_cover, normalize_cover,
normalized_weights, normalized_weights,
percentile, percentile,
robust_center_scale, robust_center_scale,
score_percentiles, score_percentiles,
views_per_hour,
) )
@@ -157,6 +165,72 @@ def test_normalize_cover_and_hosts():
assert not is_loopback("192.168.1.2") assert not is_loopback("192.168.1.2")
def test_content_efficiency_helpers():
assert content_hours(3600) == 1.0
assert content_hours(7200) == 2.0
assert content_hours(0) is None
assert content_hours(-1) is None
assert content_hours(None) is None
assert content_hours(True) is None
assert content_hours("bad") is None
assert views_per_hour(1000, 3600) == 1000.0
assert views_per_hour(1000, 7200) == 500.0
assert views_per_hour(None, 3600) == 0.0
assert views_per_hour(1000, None) is None
assert views_per_hour(1000, 0) is None
metrics = {"like": 30, "coin": 10, "favorite": 20, "view": 999}
assert interactions_per_hour(metrics, 3600) == 60.0
assert interactions_per_hour(metrics, 7200) == 30.0
assert interactions_per_hour(None, 3600) is None
assert interactions_per_hour(metrics, None) is None
# 单项缺失按 0 处理,不抛异常
assert interactions_per_hour({"like": None, "coin": "bad"}, 3600) == 0.0
def test_age_and_maturity():
reference = 1_800_000_000
assert age_days(reference - 86400 * 10, reference) == 10
assert age_days(reference, reference) == 0
# 未来时间不产生负年龄
assert age_days(reference + 86400, reference) == 0
assert age_days(None, reference) is None
assert age_days(0, reference) is None
assert age_days("bad", reference) is None
assert is_mature(reference - 86400 * 14, reference, 14) is True
assert is_mature(reference - 86400 * 13, reference, 14) is False
assert is_mature(None, reference, 14) is False
assert is_mature(reference - 86400 * 14, reference, 0) is True
def test_shrinkage_helpers():
assert median_of([]) is None
assert median_of([None, "bad", True]) is None
assert median_of([1, 3, 2]) == 2.0
# 自适应 κ 取各 UP 总分钟的中位数
assert effective_shrink_minutes([10, 30, 20]) == 20.0
assert effective_shrink_minutes([0, -5, None]) == 120.0
assert effective_shrink_minutes([]) == 120.0
assert effective_shrink_minutes([], fallback=60) == 60.0
# κ 越大越靠近先验;exposure 为 0 时直接返回先验
assert abs(eb_rate(0, 0, 0.5, 100) - 0.5) < 1e-12
assert eb_rate(100, 100, 1.0, 0) == 1.0
assert eb_rate(100, 0, 1.0, 0) is None
# (100 + 100*0.5) / (100 + 100) = 0.75
assert abs(eb_rate(100, 100, 0.5, 100) - 0.75) < 1e-12
assert eb_rate(None, 100, 0.5, 100) is None
try:
eb_rate(1, 1, 0.5, -1)
except ValueError:
pass
else:
raise AssertionError("负 prior_strength 必须报错")
def main_test(): def main_test():
test_percentile_and_robust_scale() test_percentile_and_robust_scale()
test_normalized_weights() test_normalized_weights()
@@ -166,7 +240,10 @@ def main_test():
test_interaction_rate_thresholds() test_interaction_rate_thresholds()
test_compute_growth() test_compute_growth()
test_normalize_cover_and_hosts() test_normalize_cover_and_hosts()
print("OK: analytics 热度/增长/互动率/封面测试全部通过") test_content_efficiency_helpers()
test_age_and_maturity()
test_shrinkage_helpers()
print("OK: analytics 热度/增长/互动率/时长效率/收缩测试全部通过")
if __name__ == "__main__": if __name__ == "__main__":
+19
View File
@@ -216,6 +216,25 @@ def test_infos_invalid_attr_and_empty():
assert len(transport.calls) == 1 assert len(transport.calls) == 1
def test_infos_parses_page_count():
payload = {"code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO, INVALID_INFO]}
client, _ = make_client(QueueTransport([http_json(payload)]))
result = client.fav_infos(
3700023351, [VALID_INFO["id"], INVALID_INFO["id"]]
)
assert result.items[VALID_INFO["id"]].pages == 2
assert result.items[INVALID_INFO["id"]].pages == 1
# page 字段缺失或非法时保持 None,不把 0 当分 P 数
broken = json.loads(json.dumps(VALID_INFO))
broken["id"] = 999
broken["page"] = 0
client, _ = make_client(QueueTransport([http_json({
"code": 0, "message": "0", "ttl": 1, "data": [broken],
})]))
assert client.fav_infos(3700023351, [999]).items[999].pages is None
def test_infos_unexpected_aid_is_protocol_error(): def test_infos_unexpected_aid_is_protocol_error():
transport = QueueTransport([http_json({ transport = QueueTransport([http_json({
"code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO], "code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO],
+44 -1
View File
@@ -126,12 +126,55 @@ def test_invalid_values():
raise AssertionError("非法环境变量必须拒绝") raise AssertionError("非法环境变量必须拒绝")
def test_analytics_defaults_and_validation():
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "config.json"
write(path, base_config())
cfg = load_config(path, env={})
assert cfg.analytics.min_duration_seconds == 600
assert cfg.analytics.maturity_days == 14
assert cfg.analytics.min_view_for_efficiency == 0
assert cfg.analytics.shrink_minutes is None
data = base_config()
data["analytics"] = {
"min_duration_seconds": 300,
"maturity_days": 7,
"min_view_for_efficiency": 100,
"shrink_minutes": 90,
}
write(path, data)
cfg = load_config(path, env={})
assert cfg.analytics.min_duration_seconds == 300
assert cfg.analytics.maturity_days == 7
assert cfg.analytics.min_view_for_efficiency == 100
assert cfg.analytics.shrink_minutes == 90
for bad, hint in (
({"min_duration_seconds": -1}, "min_duration_seconds"),
({"maturity_days": -1}, "maturity_days"),
({"min_view_for_efficiency": -1}, "min_view_for_efficiency"),
({"shrink_minutes": 0}, "shrink_minutes"),
({"shrink_minutes": "auto"}, "shrink_minutes"),
):
data = base_config()
data["analytics"] = bad
write(path, data)
try:
load_config(path, env={})
except ConfigError as exc:
assert hint in str(exc), (bad, str(exc))
else:
raise AssertionError(f"非法 analytics 必须拒绝: {bad}")
def main(): def main():
test_dual_accounts_and_env_override() test_dual_accounts_and_env_override()
test_flat_and_missing_account_rejected() test_flat_and_missing_account_rejected()
test_validation_require_read_auth_and_hot_reload() test_validation_require_read_auth_and_hot_reload()
test_invalid_values() test_invalid_values()
print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖与热加载测试全部通过") test_analytics_defaults_and_validation()
print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖、热加载与 analytics 测试全部通过")
if __name__ == "__main__": if __name__ == "__main__":
+22 -8
View File
@@ -21,7 +21,7 @@ from reaction_board.migrations import ( # noqa: E402
EXPECTED_TABLES = { EXPECTED_TABLES = {
"video", "snapshot", "folder", "folder_resource", "up_blacklist", "video", "snapshot", "folder", "folder_resource", "up_blacklist",
"up_stat", "candidate", "seen_aid", "notify", "fav_action", "kv", "up_stat", "up_profile", "candidate", "seen_aid", "notify", "fav_action", "kv",
} }
@@ -37,8 +37,8 @@ def test_fresh_and_idempotent():
conn = sqlite3.connect(Path(tmp) / "app.db") conn = sqlite3.connect(Path(tmp) / "app.db")
try: try:
applied = run_migrations(conn) applied = run_migrations(conn)
assert applied == [1, 2, 3] assert applied == [1, 2, 3, 4]
assert current_version(conn) == 3 assert current_version(conn) == 4
assert EXPECTED_TABLES <= table_names(conn) assert EXPECTED_TABLES <= table_names(conn)
assert "schema_version" in table_names(conn) assert "schema_version" in table_names(conn)
columns = { columns = {
@@ -49,10 +49,24 @@ def test_fresh_and_idempotent():
row[1] for row in conn.execute("PRAGMA table_info(candidate)") row[1] for row in conn.execute("PRAGMA table_info(candidate)")
} }
assert {"duration", "play", "like", "favorite", "danmaku", "review"} <= candidate_columns assert {"duration", "play", "like", "favorite", "danmaku", "review"} <= candidate_columns
video_columns = {
row[1] for row in conn.execute("PRAGMA table_info(video)")
}
assert {
"pages", "duration", "up_follower", "up_follower_at",
"up_follower_backfilled",
} <= video_columns
profile_columns = {
row[1] for row in conn.execute("PRAGMA table_info(up_profile)")
}
assert {
"mid", "follower_first", "following_first", "first_captured_at",
"follower_latest", "last_refreshed",
} <= profile_columns
# 再跑一次不应重复应用 # 再跑一次不应重复应用
assert run_migrations(conn) == [] assert run_migrations(conn) == []
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone() row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
assert row[0] == 3 assert row[0] == 4
finally: finally:
conn.close() conn.close()
@@ -67,15 +81,15 @@ def test_backup_before_upgrade():
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')") conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
conn.commit() conn.commit()
v4 = Migration(4, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",)) v5 = Migration(5, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
applied = run_migrations( applied = run_migrations(
conn, conn,
backup_dir=backup_dir, backup_dir=backup_dir,
now=1_800_000_000, now=1_800_000_000,
migrations=MIGRATIONS + (v4,), migrations=MIGRATIONS + (v5,),
) )
assert applied == [4] assert applied == [5]
backups = list(backup_dir.glob("app-pre-v3-*.db")) backups = list(backup_dir.glob("app-pre-v4-*.db"))
assert len(backups) == 1 assert len(backups) == 1
check = sqlite3.connect(backups[0]) check = sqlite3.connect(backups[0])
try: try:
+56 -1
View File
@@ -18,7 +18,8 @@ F1 = FolderConfig(year=2025, media_id=4451053851, title="2025")
F2 = FolderConfig(year=2026, media_id=3700023351, title="2026") F2 = FolderConfig(year=2026, media_id=3700023351, title="2026")
def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None): def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None,
pages=2):
return FavResource( return FavResource(
aid=aid, aid=aid,
bvid=bvid or f"BV{aid}", bvid=bvid or f"BV{aid}",
@@ -30,6 +31,7 @@ def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=Non
cover="http://example/cover.jpg", cover="http://example/cover.jpg",
attr=attr, attr=attr,
fav_time=fav_time, fav_time=fav_time,
pages=pages,
) )
@@ -54,6 +56,7 @@ def test_upsert_and_folder_resource():
assert video["folder_id"] == F1.media_id assert video["folder_id"] == F1.media_id
assert video["source"] == f"fav:{F1.media_id}" assert video["source"] == f"fav:{F1.media_id}"
assert video["first_seen"] == 1_800_000_000 assert video["first_seen"] == 1_800_000_000
assert video["pages"] == 2
assert repo.folder_active_aids(F1.media_id) == {1} assert repo.folder_active_aids(F1.media_id) == {1}
# 幂等:重复导入不新增行、不改 first_seen # 幂等:重复导入不新增行、不改 first_seen
@@ -293,6 +296,57 @@ def test_pending_snapshot_aids():
db.close() db.close()
def test_up_follower_snapshot_and_backfill():
with tempfile.TemporaryDirectory() as tmp:
db, repo = setup(tmp)
try:
repo.upsert_from_fav(resource(1, up_mid=100), F1.media_id, was_added=True)
repo.upsert_from_fav(resource(2, up_mid=100), F1.media_id, was_added=True)
repo.upsert_from_fav(resource(3, up_mid=200), F1.media_id, was_added=True)
missing = {row["up_mid"] for row in repo.up_mids_missing_follower()}
assert missing == {100, 200}
now = 1_800_000_000
assert repo.apply_up_follower(
mid=100, name="UP", follower=1234, following=5, now=now
) == 2
first = repo.get_video(1)
assert first["up_follower"] == 1234
assert first["up_follower_at"] == now
assert first["up_follower_backfilled"] == 0
profile = repo.get_up_profile(100)
assert profile["follower_first"] == 1234
assert profile["follower_latest"] == 1234
assert {
row["up_mid"] for row in repo.up_mids_missing_follower()
} == {200}
# 同一 UP 后来新增的视频记录当下的粉丝快照,不覆盖旧视频
repo.upsert_from_fav(resource(4, up_mid=100), F1.media_id, was_added=True)
repo.apply_up_follower(
mid=100, name="UP", follower=2000, following=6, now=now + 60
)
assert repo.get_video(4)["up_follower"] == 2000
assert repo.get_video(1)["up_follower"] == 1234
profile = repo.get_up_profile(100)
assert profile["follower_first"] == 1234
assert profile["follower_latest"] == 2000
# 回填只补空值,并标记 backfilled
assert repo.apply_up_follower(
mid=200, name="UP2", follower=88, now=now + 120, backfilled=True
) == 1
assert repo.get_video(3)["up_follower"] == 88
assert repo.get_video(3)["up_follower_backfilled"] == 1
coverage = repo.up_follower_coverage()
assert coverage == {
"total": 4, "covered": 4, "missing": 0, "ups_covered": 2,
}
finally:
db.close()
def main(): def main():
test_upsert_and_folder_resource() test_upsert_and_folder_resource()
test_invalid_attr_and_missing() test_invalid_attr_and_missing()
@@ -305,6 +359,7 @@ def main():
test_primary_folder_is_order_independent() test_primary_folder_is_order_independent()
test_dropped_invalid_preserves_original_title() test_dropped_invalid_preserves_original_title()
test_pending_snapshot_aids() test_pending_snapshot_aids()
test_up_follower_snapshot_and_backfill()
print("OK: 收藏夹/视频状态机与黑名单测试全部通过") print("OK: 收藏夹/视频状态机与黑名单测试全部通过")