feat(db): add video pages and up follower snapshot schema
This commit is contained in:
1 parent
707ed7d7d7
commit
4942d1d161
10 files changed
+521
-13
No files matched your search
@@ -35,6 +35,9 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
||||
|
||||
MIN_VIEW_FOR_INTERACTION = 100
|
||||
MIN_VIEW_FOR_GROWTH_RATE = 200
|
||||
MIN_EFFICIENCY_DURATION = 600
|
||||
DEFAULT_MATURITY_DAYS = 14
|
||||
DEFAULT_SHRINK_MINUTES = 120.0
|
||||
_IQR_TO_SIGMA = 1.349
|
||||
|
||||
|
||||
@@ -83,6 +86,128 @@ def _metric_value(row: Mapping[str, object], metric: str) -> float:
|
||||
return 0.0
|
||||
|
||||
|
||||
def content_hours(duration_seconds: object) -> float | None:
|
||||
"""把视频总时长(秒)换算为内容小时;缺失或非正数返回 ``None``。
|
||||
|
||||
多 P 视频的 ``duration`` 已是各 P 之和,因此这里不需要再乘 P 数。
|
||||
"""
|
||||
if duration_seconds is None or isinstance(duration_seconds, bool):
|
||||
return None
|
||||
try:
|
||||
seconds = float(duration_seconds)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if seconds <= 0:
|
||||
return None
|
||||
return seconds / 3600.0
|
||||
|
||||
|
||||
def views_per_hour(view: object, duration_seconds: object) -> float | None:
|
||||
"""播放/内容小时;时长缺失或非正数时返回 ``None``,不做除零。"""
|
||||
hours = content_hours(duration_seconds)
|
||||
if hours is None:
|
||||
return None
|
||||
return _metric_value({"view": view}, "view") / hours
|
||||
|
||||
|
||||
def interactions_per_hour(
|
||||
metrics: Mapping[str, object] | None,
|
||||
duration_seconds: object,
|
||||
*,
|
||||
interaction_metrics: Sequence[str] = ("like", "coin", "favorite"),
|
||||
) -> float | None:
|
||||
"""互动/内容小时;统计缺失或时长无效时返回 ``None``。"""
|
||||
hours = content_hours(duration_seconds)
|
||||
if hours is None or metrics is None:
|
||||
return None
|
||||
interactions = sum(_metric_value(metrics, metric) for metric in interaction_metrics)
|
||||
return interactions / hours
|
||||
|
||||
|
||||
def age_days(pubdate: object, reference: int | float) -> int | None:
|
||||
"""发布到参考时点(通常为最新数据日)的整天数;无效发布时间返回 ``None``。"""
|
||||
if pubdate is None or isinstance(pubdate, bool):
|
||||
return None
|
||||
try:
|
||||
published = int(pubdate)
|
||||
now = int(reference)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if published <= 0:
|
||||
return None
|
||||
return max(0, (now - published) // 86400)
|
||||
|
||||
|
||||
def is_mature(
|
||||
pubdate: object,
|
||||
reference: int | float,
|
||||
maturity_days: int = DEFAULT_MATURITY_DAYS,
|
||||
) -> bool:
|
||||
"""是否已达到成熟窗口;发布时间缺失视为未成熟。"""
|
||||
days = age_days(pubdate, reference)
|
||||
return days is not None and days >= max(0, int(maturity_days))
|
||||
|
||||
|
||||
def median_of(values: Iterable[object]) -> float | None:
|
||||
"""忽略无效值的中位数;没有有效值返回 ``None``。"""
|
||||
numbers: list[float] = []
|
||||
for value in values:
|
||||
if value is None or isinstance(value, bool):
|
||||
continue
|
||||
try:
|
||||
numbers.append(float(value))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if not numbers:
|
||||
return None
|
||||
return float(_stat_median(numbers))
|
||||
|
||||
|
||||
def effective_shrink_minutes(
|
||||
per_up_minutes: Iterable[object],
|
||||
fallback: float = DEFAULT_SHRINK_MINUTES,
|
||||
) -> float:
|
||||
"""自适应收缩强度:各 UP 总内容分钟的中位数,缺失时回退固定值。"""
|
||||
positives: list[float] = []
|
||||
for minutes in per_up_minutes:
|
||||
if minutes is None or isinstance(minutes, bool):
|
||||
continue
|
||||
try:
|
||||
value = float(minutes)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if value > 0:
|
||||
positives.append(value)
|
||||
value = median_of(positives)
|
||||
if value is None or value <= 0:
|
||||
return float(fallback)
|
||||
return value
|
||||
|
||||
|
||||
def eb_rate(
|
||||
total: object,
|
||||
exposure: object,
|
||||
prior_rate: float,
|
||||
prior_strength: float,
|
||||
) -> float | None:
|
||||
"""Gamma–Poisson 经验贝叶斯收缩后的速率。
|
||||
|
||||
``exposure`` 与 ``prior_strength`` 必须使用同一单位(本项目中为分钟),
|
||||
返回单位与 ``prior_rate`` 一致(每分钟)。
|
||||
"""
|
||||
if prior_strength < 0:
|
||||
raise ValueError("prior_strength 不能为负")
|
||||
try:
|
||||
total_value = max(0.0, float(total))
|
||||
exposure_value = max(0.0, float(exposure))
|
||||
prior = max(0.0, float(prior_rate))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if prior_strength <= 0:
|
||||
return None if exposure_value <= 0 else total_value / exposure_value
|
||||
return (total_value + prior_strength * prior) / (exposure_value + prior_strength)
|
||||
|
||||
|
||||
def normalized_weights(weights: Mapping[str, float] | None = None) -> dict[str, float]:
|
||||
source = weights or DEFAULT_HEAT_WEIGHTS
|
||||
result: dict[str, float] = {}
|
||||
|
||||
@@ -30,6 +30,7 @@ class FavResource:
|
||||
cover: str | None
|
||||
attr: int
|
||||
fav_time: int | None
|
||||
pages: int | None = None
|
||||
|
||||
@property
|
||||
def invalid(self) -> bool:
|
||||
@@ -51,6 +52,12 @@ class FavResource:
|
||||
duration = raw_duration if isinstance(raw_duration, int) and not isinstance(
|
||||
raw_duration, bool
|
||||
) else None
|
||||
raw_pages = item.get("page")
|
||||
pages = raw_pages if (
|
||||
isinstance(raw_pages, int)
|
||||
and not isinstance(raw_pages, bool)
|
||||
and raw_pages > 0
|
||||
) else None
|
||||
raw_attr = item.get("attr", 0)
|
||||
attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0
|
||||
fav_time = item.get("fav_time")
|
||||
@@ -70,6 +77,7 @@ class FavResource:
|
||||
cover=str(item.get("cover") or "") or None,
|
||||
attr=attr,
|
||||
fav_time=fav_time,
|
||||
pages=pages,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -38,6 +38,13 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
||||
"danmaku": 0.02,
|
||||
}
|
||||
|
||||
DEFAULT_ANALYTICS: dict[str, Any] = {
|
||||
"min_duration_seconds": 600,
|
||||
"maturity_days": 14,
|
||||
"min_view_for_efficiency": 0,
|
||||
"shrink_minutes": None,
|
||||
}
|
||||
|
||||
DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"bilibili": {
|
||||
"read": {"mid": 0, "csrf": "", "cookie": ""},
|
||||
@@ -73,6 +80,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"notify": {"min_score": 2, "max_items_per_message": 10},
|
||||
"classify": DEFAULT_CLASSIFY,
|
||||
"heat_weights": DEFAULT_HEAT_WEIGHTS,
|
||||
"analytics": DEFAULT_ANALYTICS,
|
||||
}
|
||||
|
||||
# 环境变量只作为 CI / 临时调试的覆盖入口,不作为主路径。
|
||||
@@ -161,6 +169,14 @@ class DiscoverConfig:
|
||||
window_days: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AnalyticsConfig:
|
||||
min_duration_seconds: int
|
||||
maturity_days: int
|
||||
min_view_for_efficiency: int
|
||||
shrink_minutes: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppConfig:
|
||||
path: Path
|
||||
@@ -176,6 +192,7 @@ class AppConfig:
|
||||
discover: DiscoverConfig
|
||||
classify: Mapping[str, Any]
|
||||
heat_weights: Mapping[str, float]
|
||||
analytics: AnalyticsConfig
|
||||
raw: Mapping[str, Any] = field(repr=False, compare=False)
|
||||
|
||||
def folder_for_year(self, year: int) -> FolderConfig | None:
|
||||
@@ -431,6 +448,33 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
|
||||
heat[metric] = float(value)
|
||||
|
||||
analytics_raw = data["analytics"]
|
||||
if not isinstance(analytics_raw, Mapping):
|
||||
raise ConfigError("analytics 必须是对象")
|
||||
shrink_raw = analytics_raw.get("shrink_minutes")
|
||||
analytics = AnalyticsConfig(
|
||||
min_duration_seconds=_as_int(
|
||||
analytics_raw.get("min_duration_seconds", 600),
|
||||
"analytics.min_duration_seconds",
|
||||
minimum=0,
|
||||
),
|
||||
maturity_days=_as_int(
|
||||
analytics_raw.get("maturity_days", 14),
|
||||
"analytics.maturity_days",
|
||||
minimum=0,
|
||||
),
|
||||
min_view_for_efficiency=_as_int(
|
||||
analytics_raw.get("min_view_for_efficiency", 0),
|
||||
"analytics.min_view_for_efficiency",
|
||||
minimum=0,
|
||||
),
|
||||
shrink_minutes=(
|
||||
None if shrink_raw is None else _as_int(
|
||||
shrink_raw, "analytics.shrink_minutes", minimum=1
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
return AppConfig(
|
||||
path=path,
|
||||
bilibili=bilibili,
|
||||
@@ -445,6 +489,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
discover=discover,
|
||||
classify=classify,
|
||||
heat_weights=heat,
|
||||
analytics=analytics,
|
||||
raw=data,
|
||||
)
|
||||
|
||||
|
||||
@@ -186,6 +186,27 @@ MIGRATIONS: tuple[Migration, ...] = (
|
||||
"ALTER TABLE candidate ADD COLUMN review INTEGER",
|
||||
),
|
||||
),
|
||||
Migration(
|
||||
version=4,
|
||||
name="video pages and up follower snapshot",
|
||||
statements=(
|
||||
"ALTER TABLE video ADD COLUMN pages INTEGER",
|
||||
"ALTER TABLE video ADD COLUMN up_follower INTEGER",
|
||||
"ALTER TABLE video ADD COLUMN up_follower_at INTEGER",
|
||||
"ALTER TABLE video ADD COLUMN up_follower_backfilled INTEGER NOT NULL DEFAULT 0",
|
||||
"""
|
||||
CREATE TABLE up_profile (
|
||||
mid INTEGER PRIMARY KEY,
|
||||
name TEXT,
|
||||
follower_first INTEGER,
|
||||
following_first INTEGER,
|
||||
first_captured_at INTEGER,
|
||||
follower_latest INTEGER,
|
||||
last_refreshed INTEGER
|
||||
)
|
||||
""",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -229,9 +229,9 @@ class Repository:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
|
||||
duration, cover, folder_id, fav_time, status,
|
||||
duration, pages, cover, folder_id, fav_time, status,
|
||||
source, first_seen, last_sync, attr)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(aid) DO UPDATE SET
|
||||
bvid = COALESCE(excluded.bvid, video.bvid),
|
||||
title = CASE
|
||||
@@ -248,6 +248,7 @@ class Repository:
|
||||
ELSE video.pubdate
|
||||
END,
|
||||
duration = COALESCE(excluded.duration, video.duration),
|
||||
pages = COALESCE(excluded.pages, video.pages),
|
||||
cover = COALESCE(excluded.cover, video.cover),
|
||||
folder_id = CASE
|
||||
WHEN video.folder_id IS NULL THEN excluded.folder_id
|
||||
@@ -267,6 +268,7 @@ class Repository:
|
||||
resource.up_name,
|
||||
int(resource.pubdate or 0),
|
||||
resource.duration,
|
||||
resource.pages,
|
||||
resource.cover,
|
||||
folder_id,
|
||||
resource.fav_time,
|
||||
@@ -397,6 +399,105 @@ class Repository:
|
||||
return [dict(row) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
# ------------------------------------------------------- up profile
|
||||
|
||||
def apply_up_follower(
|
||||
self,
|
||||
*,
|
||||
mid: int,
|
||||
name: str | None,
|
||||
follower: int,
|
||||
following: int | None = None,
|
||||
now: int | None = None,
|
||||
backfilled: bool = False,
|
||||
) -> int:
|
||||
"""写入 UP 粉丝快照,并补齐该 UP 尚未记录粉丝数的视频。
|
||||
|
||||
``follower_first`` 只在首次插入时写入;``follower_latest`` 每次刷新都会
|
||||
更新。视频上的 ``up_follower`` 是入库时快照,已有值不会被覆盖。返回本次
|
||||
补齐的视频数。
|
||||
"""
|
||||
now = self._now() if now is None else int(now)
|
||||
mid = int(mid)
|
||||
follower = max(0, int(follower))
|
||||
following_value = None if following is None else max(0, int(following))
|
||||
backfill_flag = 1 if backfilled else 0
|
||||
|
||||
def run(conn: sqlite3.Connection) -> int:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO up_profile(mid, name, follower_first, following_first,
|
||||
first_captured_at, follower_latest, last_refreshed)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(mid) DO UPDATE SET
|
||||
name = COALESCE(NULLIF(excluded.name, ''), up_profile.name),
|
||||
follower_latest = excluded.follower_latest,
|
||||
last_refreshed = excluded.last_refreshed
|
||||
""",
|
||||
(mid, name, follower, following_value, now, follower, now),
|
||||
)
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
UPDATE video
|
||||
SET up_follower = ?, up_follower_at = ?, up_follower_backfilled = ?
|
||||
WHERE up_mid = ? AND up_follower IS NULL
|
||||
""",
|
||||
(follower, now, backfill_flag, mid),
|
||||
)
|
||||
return max(cursor.rowcount, 0)
|
||||
|
||||
return self.db.write(run)
|
||||
|
||||
def get_up_profile(self, mid: int) -> dict[str, Any] | None:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM up_profile WHERE mid = ?", (int(mid),)
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
return self.db.read(run)
|
||||
|
||||
def up_mids_missing_follower(self) -> list[dict[str, Any]]:
|
||||
"""追踪中但仍有视频缺少入库粉丝数的 UP。"""
|
||||
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT up_mid, COALESCE(MAX(up_name), '') AS up_name,
|
||||
COUNT(*) AS missing
|
||||
FROM video
|
||||
WHERE status IN ('active', 'unfaved')
|
||||
AND up_mid IS NOT NULL
|
||||
AND up_follower IS NULL
|
||||
GROUP BY up_mid
|
||||
ORDER BY up_mid
|
||||
"""
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
def up_follower_coverage(self) -> dict[str, int]:
|
||||
"""返回追踪视频的粉丝快照覆盖情况。"""
|
||||
def run(conn: sqlite3.Connection) -> dict[str, int]:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS total,
|
||||
SUM(CASE WHEN up_follower IS NOT NULL THEN 1 ELSE 0 END) AS covered,
|
||||
COUNT(DISTINCT CASE WHEN up_follower IS NOT NULL
|
||||
THEN up_mid END) AS ups_covered
|
||||
FROM video
|
||||
WHERE status IN ('active', 'unfaved')
|
||||
AND up_mid IS NOT NULL
|
||||
"""
|
||||
).fetchone()
|
||||
total = int(row["total"] or 0)
|
||||
covered = int(row["covered"] or 0)
|
||||
return {
|
||||
"total": total,
|
||||
"covered": covered,
|
||||
"missing": max(0, total - covered),
|
||||
"ups_covered": int(row["ups_covered"] or 0),
|
||||
}
|
||||
return self.db.read(run)
|
||||
|
||||
def folder_resources(self, media_id: int) -> list[dict[str, Any]]:
|
||||
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||
rows = conn.execute(
|
||||
|
||||
+78
-1
@@ -10,15 +10,23 @@ sys.stdout.reconfigure(encoding="utf-8")
|
||||
from reaction_board.analytics import ( # noqa: E402
|
||||
DEFAULT_HEAT_WEIGHTS,
|
||||
METRICS,
|
||||
age_days,
|
||||
compute_growth,
|
||||
content_hours,
|
||||
eb_rate,
|
||||
effective_shrink_minutes,
|
||||
heat_scores,
|
||||
interactions_per_hour,
|
||||
interaction_rate,
|
||||
is_loopback,
|
||||
is_mature,
|
||||
median_of,
|
||||
normalize_cover,
|
||||
normalized_weights,
|
||||
percentile,
|
||||
robust_center_scale,
|
||||
score_percentiles,
|
||||
views_per_hour,
|
||||
)
|
||||
|
||||
|
||||
@@ -157,6 +165,72 @@ def test_normalize_cover_and_hosts():
|
||||
assert not is_loopback("192.168.1.2")
|
||||
|
||||
|
||||
def test_content_efficiency_helpers():
|
||||
assert content_hours(3600) == 1.0
|
||||
assert content_hours(7200) == 2.0
|
||||
assert content_hours(0) is None
|
||||
assert content_hours(-1) is None
|
||||
assert content_hours(None) is None
|
||||
assert content_hours(True) is None
|
||||
assert content_hours("bad") is None
|
||||
|
||||
assert views_per_hour(1000, 3600) == 1000.0
|
||||
assert views_per_hour(1000, 7200) == 500.0
|
||||
assert views_per_hour(None, 3600) == 0.0
|
||||
assert views_per_hour(1000, None) is None
|
||||
assert views_per_hour(1000, 0) is None
|
||||
|
||||
metrics = {"like": 30, "coin": 10, "favorite": 20, "view": 999}
|
||||
assert interactions_per_hour(metrics, 3600) == 60.0
|
||||
assert interactions_per_hour(metrics, 7200) == 30.0
|
||||
assert interactions_per_hour(None, 3600) is None
|
||||
assert interactions_per_hour(metrics, None) is None
|
||||
# 单项缺失按 0 处理,不抛异常
|
||||
assert interactions_per_hour({"like": None, "coin": "bad"}, 3600) == 0.0
|
||||
|
||||
|
||||
def test_age_and_maturity():
|
||||
reference = 1_800_000_000
|
||||
assert age_days(reference - 86400 * 10, reference) == 10
|
||||
assert age_days(reference, reference) == 0
|
||||
# 未来时间不产生负年龄
|
||||
assert age_days(reference + 86400, reference) == 0
|
||||
assert age_days(None, reference) is None
|
||||
assert age_days(0, reference) is None
|
||||
assert age_days("bad", reference) is None
|
||||
|
||||
assert is_mature(reference - 86400 * 14, reference, 14) is True
|
||||
assert is_mature(reference - 86400 * 13, reference, 14) is False
|
||||
assert is_mature(None, reference, 14) is False
|
||||
assert is_mature(reference - 86400 * 14, reference, 0) is True
|
||||
|
||||
|
||||
def test_shrinkage_helpers():
|
||||
assert median_of([]) is None
|
||||
assert median_of([None, "bad", True]) is None
|
||||
assert median_of([1, 3, 2]) == 2.0
|
||||
|
||||
# 自适应 κ 取各 UP 总分钟的中位数
|
||||
assert effective_shrink_minutes([10, 30, 20]) == 20.0
|
||||
assert effective_shrink_minutes([0, -5, None]) == 120.0
|
||||
assert effective_shrink_minutes([]) == 120.0
|
||||
assert effective_shrink_minutes([], fallback=60) == 60.0
|
||||
|
||||
# κ 越大越靠近先验;exposure 为 0 时直接返回先验
|
||||
assert abs(eb_rate(0, 0, 0.5, 100) - 0.5) < 1e-12
|
||||
assert eb_rate(100, 100, 1.0, 0) == 1.0
|
||||
assert eb_rate(100, 0, 1.0, 0) is None
|
||||
# (100 + 100*0.5) / (100 + 100) = 0.75
|
||||
assert abs(eb_rate(100, 100, 0.5, 100) - 0.75) < 1e-12
|
||||
assert eb_rate(None, 100, 0.5, 100) is None
|
||||
try:
|
||||
eb_rate(1, 1, 0.5, -1)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("负 prior_strength 必须报错")
|
||||
|
||||
|
||||
def main_test():
|
||||
test_percentile_and_robust_scale()
|
||||
test_normalized_weights()
|
||||
@@ -166,7 +240,10 @@ def main_test():
|
||||
test_interaction_rate_thresholds()
|
||||
test_compute_growth()
|
||||
test_normalize_cover_and_hosts()
|
||||
print("OK: analytics 热度/增长/互动率/封面测试全部通过")
|
||||
test_content_efficiency_helpers()
|
||||
test_age_and_maturity()
|
||||
test_shrinkage_helpers()
|
||||
print("OK: analytics 热度/增长/互动率/时长效率/收缩测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -216,6 +216,25 @@ def test_infos_invalid_attr_and_empty():
|
||||
assert len(transport.calls) == 1
|
||||
|
||||
|
||||
def test_infos_parses_page_count():
|
||||
payload = {"code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO, INVALID_INFO]}
|
||||
client, _ = make_client(QueueTransport([http_json(payload)]))
|
||||
result = client.fav_infos(
|
||||
3700023351, [VALID_INFO["id"], INVALID_INFO["id"]]
|
||||
)
|
||||
assert result.items[VALID_INFO["id"]].pages == 2
|
||||
assert result.items[INVALID_INFO["id"]].pages == 1
|
||||
|
||||
# page 字段缺失或非法时保持 None,不把 0 当分 P 数
|
||||
broken = json.loads(json.dumps(VALID_INFO))
|
||||
broken["id"] = 999
|
||||
broken["page"] = 0
|
||||
client, _ = make_client(QueueTransport([http_json({
|
||||
"code": 0, "message": "0", "ttl": 1, "data": [broken],
|
||||
})]))
|
||||
assert client.fav_infos(3700023351, [999]).items[999].pages is None
|
||||
|
||||
|
||||
def test_infos_unexpected_aid_is_protocol_error():
|
||||
transport = QueueTransport([http_json({
|
||||
"code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO],
|
||||
|
||||
+44
-1
@@ -126,12 +126,55 @@ def test_invalid_values():
|
||||
raise AssertionError("非法环境变量必须拒绝")
|
||||
|
||||
|
||||
def test_analytics_defaults_and_validation():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "config.json"
|
||||
write(path, base_config())
|
||||
cfg = load_config(path, env={})
|
||||
assert cfg.analytics.min_duration_seconds == 600
|
||||
assert cfg.analytics.maturity_days == 14
|
||||
assert cfg.analytics.min_view_for_efficiency == 0
|
||||
assert cfg.analytics.shrink_minutes is None
|
||||
|
||||
data = base_config()
|
||||
data["analytics"] = {
|
||||
"min_duration_seconds": 300,
|
||||
"maturity_days": 7,
|
||||
"min_view_for_efficiency": 100,
|
||||
"shrink_minutes": 90,
|
||||
}
|
||||
write(path, data)
|
||||
cfg = load_config(path, env={})
|
||||
assert cfg.analytics.min_duration_seconds == 300
|
||||
assert cfg.analytics.maturity_days == 7
|
||||
assert cfg.analytics.min_view_for_efficiency == 100
|
||||
assert cfg.analytics.shrink_minutes == 90
|
||||
|
||||
for bad, hint in (
|
||||
({"min_duration_seconds": -1}, "min_duration_seconds"),
|
||||
({"maturity_days": -1}, "maturity_days"),
|
||||
({"min_view_for_efficiency": -1}, "min_view_for_efficiency"),
|
||||
({"shrink_minutes": 0}, "shrink_minutes"),
|
||||
({"shrink_minutes": "auto"}, "shrink_minutes"),
|
||||
):
|
||||
data = base_config()
|
||||
data["analytics"] = bad
|
||||
write(path, data)
|
||||
try:
|
||||
load_config(path, env={})
|
||||
except ConfigError as exc:
|
||||
assert hint in str(exc), (bad, str(exc))
|
||||
else:
|
||||
raise AssertionError(f"非法 analytics 必须拒绝: {bad}")
|
||||
|
||||
|
||||
def main():
|
||||
test_dual_accounts_and_env_override()
|
||||
test_flat_and_missing_account_rejected()
|
||||
test_validation_require_read_auth_and_hot_reload()
|
||||
test_invalid_values()
|
||||
print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖与热加载测试全部通过")
|
||||
test_analytics_defaults_and_validation()
|
||||
print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖、热加载与 analytics 测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -21,7 +21,7 @@ from reaction_board.migrations import ( # noqa: E402
|
||||
|
||||
EXPECTED_TABLES = {
|
||||
"video", "snapshot", "folder", "folder_resource", "up_blacklist",
|
||||
"up_stat", "candidate", "seen_aid", "notify", "fav_action", "kv",
|
||||
"up_stat", "up_profile", "candidate", "seen_aid", "notify", "fav_action", "kv",
|
||||
}
|
||||
|
||||
|
||||
@@ -37,8 +37,8 @@ def test_fresh_and_idempotent():
|
||||
conn = sqlite3.connect(Path(tmp) / "app.db")
|
||||
try:
|
||||
applied = run_migrations(conn)
|
||||
assert applied == [1, 2, 3]
|
||||
assert current_version(conn) == 3
|
||||
assert applied == [1, 2, 3, 4]
|
||||
assert current_version(conn) == 4
|
||||
assert EXPECTED_TABLES <= table_names(conn)
|
||||
assert "schema_version" in table_names(conn)
|
||||
columns = {
|
||||
@@ -49,10 +49,24 @@ def test_fresh_and_idempotent():
|
||||
row[1] for row in conn.execute("PRAGMA table_info(candidate)")
|
||||
}
|
||||
assert {"duration", "play", "like", "favorite", "danmaku", "review"} <= candidate_columns
|
||||
video_columns = {
|
||||
row[1] for row in conn.execute("PRAGMA table_info(video)")
|
||||
}
|
||||
assert {
|
||||
"pages", "duration", "up_follower", "up_follower_at",
|
||||
"up_follower_backfilled",
|
||||
} <= video_columns
|
||||
profile_columns = {
|
||||
row[1] for row in conn.execute("PRAGMA table_info(up_profile)")
|
||||
}
|
||||
assert {
|
||||
"mid", "follower_first", "following_first", "first_captured_at",
|
||||
"follower_latest", "last_refreshed",
|
||||
} <= profile_columns
|
||||
# 再跑一次不应重复应用
|
||||
assert run_migrations(conn) == []
|
||||
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
|
||||
assert row[0] == 3
|
||||
assert row[0] == 4
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -67,15 +81,15 @@ def test_backup_before_upgrade():
|
||||
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
|
||||
conn.commit()
|
||||
|
||||
v4 = Migration(4, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
||||
v5 = Migration(5, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
||||
applied = run_migrations(
|
||||
conn,
|
||||
backup_dir=backup_dir,
|
||||
now=1_800_000_000,
|
||||
migrations=MIGRATIONS + (v4,),
|
||||
migrations=MIGRATIONS + (v5,),
|
||||
)
|
||||
assert applied == [4]
|
||||
backups = list(backup_dir.glob("app-pre-v3-*.db"))
|
||||
assert applied == [5]
|
||||
backups = list(backup_dir.glob("app-pre-v4-*.db"))
|
||||
assert len(backups) == 1
|
||||
check = sqlite3.connect(backups[0])
|
||||
try:
|
||||
|
||||
@@ -18,7 +18,8 @@ F1 = FolderConfig(year=2025, media_id=4451053851, title="2025")
|
||||
F2 = FolderConfig(year=2026, media_id=3700023351, title="2026")
|
||||
|
||||
|
||||
def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None):
|
||||
def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None,
|
||||
pages=2):
|
||||
return FavResource(
|
||||
aid=aid,
|
||||
bvid=bvid or f"BV{aid}",
|
||||
@@ -30,6 +31,7 @@ def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=Non
|
||||
cover="http://example/cover.jpg",
|
||||
attr=attr,
|
||||
fav_time=fav_time,
|
||||
pages=pages,
|
||||
)
|
||||
|
||||
|
||||
@@ -54,6 +56,7 @@ def test_upsert_and_folder_resource():
|
||||
assert video["folder_id"] == F1.media_id
|
||||
assert video["source"] == f"fav:{F1.media_id}"
|
||||
assert video["first_seen"] == 1_800_000_000
|
||||
assert video["pages"] == 2
|
||||
assert repo.folder_active_aids(F1.media_id) == {1}
|
||||
|
||||
# 幂等:重复导入不新增行、不改 first_seen
|
||||
@@ -293,6 +296,57 @@ def test_pending_snapshot_aids():
|
||||
db.close()
|
||||
|
||||
|
||||
def test_up_follower_snapshot_and_backfill():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo = setup(tmp)
|
||||
try:
|
||||
repo.upsert_from_fav(resource(1, up_mid=100), F1.media_id, was_added=True)
|
||||
repo.upsert_from_fav(resource(2, up_mid=100), F1.media_id, was_added=True)
|
||||
repo.upsert_from_fav(resource(3, up_mid=200), F1.media_id, was_added=True)
|
||||
missing = {row["up_mid"] for row in repo.up_mids_missing_follower()}
|
||||
assert missing == {100, 200}
|
||||
|
||||
now = 1_800_000_000
|
||||
assert repo.apply_up_follower(
|
||||
mid=100, name="UP", follower=1234, following=5, now=now
|
||||
) == 2
|
||||
first = repo.get_video(1)
|
||||
assert first["up_follower"] == 1234
|
||||
assert first["up_follower_at"] == now
|
||||
assert first["up_follower_backfilled"] == 0
|
||||
profile = repo.get_up_profile(100)
|
||||
assert profile["follower_first"] == 1234
|
||||
assert profile["follower_latest"] == 1234
|
||||
assert {
|
||||
row["up_mid"] for row in repo.up_mids_missing_follower()
|
||||
} == {200}
|
||||
|
||||
# 同一 UP 后来新增的视频记录当下的粉丝快照,不覆盖旧视频
|
||||
repo.upsert_from_fav(resource(4, up_mid=100), F1.media_id, was_added=True)
|
||||
repo.apply_up_follower(
|
||||
mid=100, name="UP", follower=2000, following=6, now=now + 60
|
||||
)
|
||||
assert repo.get_video(4)["up_follower"] == 2000
|
||||
assert repo.get_video(1)["up_follower"] == 1234
|
||||
profile = repo.get_up_profile(100)
|
||||
assert profile["follower_first"] == 1234
|
||||
assert profile["follower_latest"] == 2000
|
||||
|
||||
# 回填只补空值,并标记 backfilled
|
||||
assert repo.apply_up_follower(
|
||||
mid=200, name="UP2", follower=88, now=now + 120, backfilled=True
|
||||
) == 1
|
||||
assert repo.get_video(3)["up_follower"] == 88
|
||||
assert repo.get_video(3)["up_follower_backfilled"] == 1
|
||||
|
||||
coverage = repo.up_follower_coverage()
|
||||
assert coverage == {
|
||||
"total": 4, "covered": 4, "missing": 0, "ups_covered": 2,
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
test_upsert_and_folder_resource()
|
||||
test_invalid_attr_and_missing()
|
||||
@@ -305,6 +359,7 @@ def main():
|
||||
test_primary_folder_is_order_independent()
|
||||
test_dropped_invalid_preserves_original_title()
|
||||
test_pending_snapshot_aids()
|
||||
test_up_follower_snapshot_and_backfill()
|
||||
print("OK: 收藏夹/视频状态机与黑名单测试全部通过")
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user