feat(db): add video pages and up follower snapshot schema
This commit is contained in:
1 parent
707ed7d7d7
commit
4942d1d161
10 files changed
+521
-13
No files matched your search
@@ -35,6 +35,9 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
|||||||
|
|
||||||
MIN_VIEW_FOR_INTERACTION = 100
|
MIN_VIEW_FOR_INTERACTION = 100
|
||||||
MIN_VIEW_FOR_GROWTH_RATE = 200
|
MIN_VIEW_FOR_GROWTH_RATE = 200
|
||||||
|
MIN_EFFICIENCY_DURATION = 600
|
||||||
|
DEFAULT_MATURITY_DAYS = 14
|
||||||
|
DEFAULT_SHRINK_MINUTES = 120.0
|
||||||
_IQR_TO_SIGMA = 1.349
|
_IQR_TO_SIGMA = 1.349
|
||||||
|
|
||||||
|
|
||||||
@@ -83,6 +86,128 @@ def _metric_value(row: Mapping[str, object], metric: str) -> float:
|
|||||||
return 0.0
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def content_hours(duration_seconds: object) -> float | None:
|
||||||
|
"""把视频总时长(秒)换算为内容小时;缺失或非正数返回 ``None``。
|
||||||
|
|
||||||
|
多 P 视频的 ``duration`` 已是各 P 之和,因此这里不需要再乘 P 数。
|
||||||
|
"""
|
||||||
|
if duration_seconds is None or isinstance(duration_seconds, bool):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
seconds = float(duration_seconds)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if seconds <= 0:
|
||||||
|
return None
|
||||||
|
return seconds / 3600.0
|
||||||
|
|
||||||
|
|
||||||
|
def views_per_hour(view: object, duration_seconds: object) -> float | None:
|
||||||
|
"""播放/内容小时;时长缺失或非正数时返回 ``None``,不做除零。"""
|
||||||
|
hours = content_hours(duration_seconds)
|
||||||
|
if hours is None:
|
||||||
|
return None
|
||||||
|
return _metric_value({"view": view}, "view") / hours
|
||||||
|
|
||||||
|
|
||||||
|
def interactions_per_hour(
|
||||||
|
metrics: Mapping[str, object] | None,
|
||||||
|
duration_seconds: object,
|
||||||
|
*,
|
||||||
|
interaction_metrics: Sequence[str] = ("like", "coin", "favorite"),
|
||||||
|
) -> float | None:
|
||||||
|
"""互动/内容小时;统计缺失或时长无效时返回 ``None``。"""
|
||||||
|
hours = content_hours(duration_seconds)
|
||||||
|
if hours is None or metrics is None:
|
||||||
|
return None
|
||||||
|
interactions = sum(_metric_value(metrics, metric) for metric in interaction_metrics)
|
||||||
|
return interactions / hours
|
||||||
|
|
||||||
|
|
||||||
|
def age_days(pubdate: object, reference: int | float) -> int | None:
|
||||||
|
"""发布到参考时点(通常为最新数据日)的整天数;无效发布时间返回 ``None``。"""
|
||||||
|
if pubdate is None or isinstance(pubdate, bool):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
published = int(pubdate)
|
||||||
|
now = int(reference)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if published <= 0:
|
||||||
|
return None
|
||||||
|
return max(0, (now - published) // 86400)
|
||||||
|
|
||||||
|
|
||||||
|
def is_mature(
|
||||||
|
pubdate: object,
|
||||||
|
reference: int | float,
|
||||||
|
maturity_days: int = DEFAULT_MATURITY_DAYS,
|
||||||
|
) -> bool:
|
||||||
|
"""是否已达到成熟窗口;发布时间缺失视为未成熟。"""
|
||||||
|
days = age_days(pubdate, reference)
|
||||||
|
return days is not None and days >= max(0, int(maturity_days))
|
||||||
|
|
||||||
|
|
||||||
|
def median_of(values: Iterable[object]) -> float | None:
|
||||||
|
"""忽略无效值的中位数;没有有效值返回 ``None``。"""
|
||||||
|
numbers: list[float] = []
|
||||||
|
for value in values:
|
||||||
|
if value is None or isinstance(value, bool):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
numbers.append(float(value))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if not numbers:
|
||||||
|
return None
|
||||||
|
return float(_stat_median(numbers))
|
||||||
|
|
||||||
|
|
||||||
|
def effective_shrink_minutes(
|
||||||
|
per_up_minutes: Iterable[object],
|
||||||
|
fallback: float = DEFAULT_SHRINK_MINUTES,
|
||||||
|
) -> float:
|
||||||
|
"""自适应收缩强度:各 UP 总内容分钟的中位数,缺失时回退固定值。"""
|
||||||
|
positives: list[float] = []
|
||||||
|
for minutes in per_up_minutes:
|
||||||
|
if minutes is None or isinstance(minutes, bool):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
value = float(minutes)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if value > 0:
|
||||||
|
positives.append(value)
|
||||||
|
value = median_of(positives)
|
||||||
|
if value is None or value <= 0:
|
||||||
|
return float(fallback)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def eb_rate(
|
||||||
|
total: object,
|
||||||
|
exposure: object,
|
||||||
|
prior_rate: float,
|
||||||
|
prior_strength: float,
|
||||||
|
) -> float | None:
|
||||||
|
"""Gamma–Poisson 经验贝叶斯收缩后的速率。
|
||||||
|
|
||||||
|
``exposure`` 与 ``prior_strength`` 必须使用同一单位(本项目中为分钟),
|
||||||
|
返回单位与 ``prior_rate`` 一致(每分钟)。
|
||||||
|
"""
|
||||||
|
if prior_strength < 0:
|
||||||
|
raise ValueError("prior_strength 不能为负")
|
||||||
|
try:
|
||||||
|
total_value = max(0.0, float(total))
|
||||||
|
exposure_value = max(0.0, float(exposure))
|
||||||
|
prior = max(0.0, float(prior_rate))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if prior_strength <= 0:
|
||||||
|
return None if exposure_value <= 0 else total_value / exposure_value
|
||||||
|
return (total_value + prior_strength * prior) / (exposure_value + prior_strength)
|
||||||
|
|
||||||
|
|
||||||
def normalized_weights(weights: Mapping[str, float] | None = None) -> dict[str, float]:
|
def normalized_weights(weights: Mapping[str, float] | None = None) -> dict[str, float]:
|
||||||
source = weights or DEFAULT_HEAT_WEIGHTS
|
source = weights or DEFAULT_HEAT_WEIGHTS
|
||||||
result: dict[str, float] = {}
|
result: dict[str, float] = {}
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ class FavResource:
|
|||||||
cover: str | None
|
cover: str | None
|
||||||
attr: int
|
attr: int
|
||||||
fav_time: int | None
|
fav_time: int | None
|
||||||
|
pages: int | None = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def invalid(self) -> bool:
|
def invalid(self) -> bool:
|
||||||
@@ -51,6 +52,12 @@ class FavResource:
|
|||||||
duration = raw_duration if isinstance(raw_duration, int) and not isinstance(
|
duration = raw_duration if isinstance(raw_duration, int) and not isinstance(
|
||||||
raw_duration, bool
|
raw_duration, bool
|
||||||
) else None
|
) else None
|
||||||
|
raw_pages = item.get("page")
|
||||||
|
pages = raw_pages if (
|
||||||
|
isinstance(raw_pages, int)
|
||||||
|
and not isinstance(raw_pages, bool)
|
||||||
|
and raw_pages > 0
|
||||||
|
) else None
|
||||||
raw_attr = item.get("attr", 0)
|
raw_attr = item.get("attr", 0)
|
||||||
attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0
|
attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0
|
||||||
fav_time = item.get("fav_time")
|
fav_time = item.get("fav_time")
|
||||||
@@ -70,6 +77,7 @@ class FavResource:
|
|||||||
cover=str(item.get("cover") or "") or None,
|
cover=str(item.get("cover") or "") or None,
|
||||||
attr=attr,
|
attr=attr,
|
||||||
fav_time=fav_time,
|
fav_time=fav_time,
|
||||||
|
pages=pages,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -38,6 +38,13 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
|||||||
"danmaku": 0.02,
|
"danmaku": 0.02,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DEFAULT_ANALYTICS: dict[str, Any] = {
|
||||||
|
"min_duration_seconds": 600,
|
||||||
|
"maturity_days": 14,
|
||||||
|
"min_view_for_efficiency": 0,
|
||||||
|
"shrink_minutes": None,
|
||||||
|
}
|
||||||
|
|
||||||
DEFAULT_CONFIG: dict[str, Any] = {
|
DEFAULT_CONFIG: dict[str, Any] = {
|
||||||
"bilibili": {
|
"bilibili": {
|
||||||
"read": {"mid": 0, "csrf": "", "cookie": ""},
|
"read": {"mid": 0, "csrf": "", "cookie": ""},
|
||||||
@@ -73,6 +80,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
|||||||
"notify": {"min_score": 2, "max_items_per_message": 10},
|
"notify": {"min_score": 2, "max_items_per_message": 10},
|
||||||
"classify": DEFAULT_CLASSIFY,
|
"classify": DEFAULT_CLASSIFY,
|
||||||
"heat_weights": DEFAULT_HEAT_WEIGHTS,
|
"heat_weights": DEFAULT_HEAT_WEIGHTS,
|
||||||
|
"analytics": DEFAULT_ANALYTICS,
|
||||||
}
|
}
|
||||||
|
|
||||||
# 环境变量只作为 CI / 临时调试的覆盖入口,不作为主路径。
|
# 环境变量只作为 CI / 临时调试的覆盖入口,不作为主路径。
|
||||||
@@ -161,6 +169,14 @@ class DiscoverConfig:
|
|||||||
window_days: int
|
window_days: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AnalyticsConfig:
|
||||||
|
min_duration_seconds: int
|
||||||
|
maturity_days: int
|
||||||
|
min_view_for_efficiency: int
|
||||||
|
shrink_minutes: int | None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class AppConfig:
|
class AppConfig:
|
||||||
path: Path
|
path: Path
|
||||||
@@ -176,6 +192,7 @@ class AppConfig:
|
|||||||
discover: DiscoverConfig
|
discover: DiscoverConfig
|
||||||
classify: Mapping[str, Any]
|
classify: Mapping[str, Any]
|
||||||
heat_weights: Mapping[str, float]
|
heat_weights: Mapping[str, float]
|
||||||
|
analytics: AnalyticsConfig
|
||||||
raw: Mapping[str, Any] = field(repr=False, compare=False)
|
raw: Mapping[str, Any] = field(repr=False, compare=False)
|
||||||
|
|
||||||
def folder_for_year(self, year: int) -> FolderConfig | None:
|
def folder_for_year(self, year: int) -> FolderConfig | None:
|
||||||
@@ -431,6 +448,33 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
|||||||
raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
|
raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
|
||||||
heat[metric] = float(value)
|
heat[metric] = float(value)
|
||||||
|
|
||||||
|
analytics_raw = data["analytics"]
|
||||||
|
if not isinstance(analytics_raw, Mapping):
|
||||||
|
raise ConfigError("analytics 必须是对象")
|
||||||
|
shrink_raw = analytics_raw.get("shrink_minutes")
|
||||||
|
analytics = AnalyticsConfig(
|
||||||
|
min_duration_seconds=_as_int(
|
||||||
|
analytics_raw.get("min_duration_seconds", 600),
|
||||||
|
"analytics.min_duration_seconds",
|
||||||
|
minimum=0,
|
||||||
|
),
|
||||||
|
maturity_days=_as_int(
|
||||||
|
analytics_raw.get("maturity_days", 14),
|
||||||
|
"analytics.maturity_days",
|
||||||
|
minimum=0,
|
||||||
|
),
|
||||||
|
min_view_for_efficiency=_as_int(
|
||||||
|
analytics_raw.get("min_view_for_efficiency", 0),
|
||||||
|
"analytics.min_view_for_efficiency",
|
||||||
|
minimum=0,
|
||||||
|
),
|
||||||
|
shrink_minutes=(
|
||||||
|
None if shrink_raw is None else _as_int(
|
||||||
|
shrink_raw, "analytics.shrink_minutes", minimum=1
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
return AppConfig(
|
return AppConfig(
|
||||||
path=path,
|
path=path,
|
||||||
bilibili=bilibili,
|
bilibili=bilibili,
|
||||||
@@ -445,6 +489,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
|||||||
discover=discover,
|
discover=discover,
|
||||||
classify=classify,
|
classify=classify,
|
||||||
heat_weights=heat,
|
heat_weights=heat,
|
||||||
|
analytics=analytics,
|
||||||
raw=data,
|
raw=data,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -186,6 +186,27 @@ MIGRATIONS: tuple[Migration, ...] = (
|
|||||||
"ALTER TABLE candidate ADD COLUMN review INTEGER",
|
"ALTER TABLE candidate ADD COLUMN review INTEGER",
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
Migration(
|
||||||
|
version=4,
|
||||||
|
name="video pages and up follower snapshot",
|
||||||
|
statements=(
|
||||||
|
"ALTER TABLE video ADD COLUMN pages INTEGER",
|
||||||
|
"ALTER TABLE video ADD COLUMN up_follower INTEGER",
|
||||||
|
"ALTER TABLE video ADD COLUMN up_follower_at INTEGER",
|
||||||
|
"ALTER TABLE video ADD COLUMN up_follower_backfilled INTEGER NOT NULL DEFAULT 0",
|
||||||
|
"""
|
||||||
|
CREATE TABLE up_profile (
|
||||||
|
mid INTEGER PRIMARY KEY,
|
||||||
|
name TEXT,
|
||||||
|
follower_first INTEGER,
|
||||||
|
following_first INTEGER,
|
||||||
|
first_captured_at INTEGER,
|
||||||
|
follower_latest INTEGER,
|
||||||
|
last_refreshed INTEGER
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -229,9 +229,9 @@ class Repository:
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
|
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
|
||||||
duration, cover, folder_id, fav_time, status,
|
duration, pages, cover, folder_id, fav_time, status,
|
||||||
source, first_seen, last_sync, attr)
|
source, first_seen, last_sync, attr)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
ON CONFLICT(aid) DO UPDATE SET
|
ON CONFLICT(aid) DO UPDATE SET
|
||||||
bvid = COALESCE(excluded.bvid, video.bvid),
|
bvid = COALESCE(excluded.bvid, video.bvid),
|
||||||
title = CASE
|
title = CASE
|
||||||
@@ -248,6 +248,7 @@ class Repository:
|
|||||||
ELSE video.pubdate
|
ELSE video.pubdate
|
||||||
END,
|
END,
|
||||||
duration = COALESCE(excluded.duration, video.duration),
|
duration = COALESCE(excluded.duration, video.duration),
|
||||||
|
pages = COALESCE(excluded.pages, video.pages),
|
||||||
cover = COALESCE(excluded.cover, video.cover),
|
cover = COALESCE(excluded.cover, video.cover),
|
||||||
folder_id = CASE
|
folder_id = CASE
|
||||||
WHEN video.folder_id IS NULL THEN excluded.folder_id
|
WHEN video.folder_id IS NULL THEN excluded.folder_id
|
||||||
@@ -267,6 +268,7 @@ class Repository:
|
|||||||
resource.up_name,
|
resource.up_name,
|
||||||
int(resource.pubdate or 0),
|
int(resource.pubdate or 0),
|
||||||
resource.duration,
|
resource.duration,
|
||||||
|
resource.pages,
|
||||||
resource.cover,
|
resource.cover,
|
||||||
folder_id,
|
folder_id,
|
||||||
resource.fav_time,
|
resource.fav_time,
|
||||||
@@ -397,6 +399,105 @@ class Repository:
|
|||||||
return [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
return self.db.read(run)
|
return self.db.read(run)
|
||||||
|
|
||||||
|
# ------------------------------------------------------- up profile
|
||||||
|
|
||||||
|
def apply_up_follower(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
mid: int,
|
||||||
|
name: str | None,
|
||||||
|
follower: int,
|
||||||
|
following: int | None = None,
|
||||||
|
now: int | None = None,
|
||||||
|
backfilled: bool = False,
|
||||||
|
) -> int:
|
||||||
|
"""写入 UP 粉丝快照,并补齐该 UP 尚未记录粉丝数的视频。
|
||||||
|
|
||||||
|
``follower_first`` 只在首次插入时写入;``follower_latest`` 每次刷新都会
|
||||||
|
更新。视频上的 ``up_follower`` 是入库时快照,已有值不会被覆盖。返回本次
|
||||||
|
补齐的视频数。
|
||||||
|
"""
|
||||||
|
now = self._now() if now is None else int(now)
|
||||||
|
mid = int(mid)
|
||||||
|
follower = max(0, int(follower))
|
||||||
|
following_value = None if following is None else max(0, int(following))
|
||||||
|
backfill_flag = 1 if backfilled else 0
|
||||||
|
|
||||||
|
def run(conn: sqlite3.Connection) -> int:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO up_profile(mid, name, follower_first, following_first,
|
||||||
|
first_captured_at, follower_latest, last_refreshed)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(mid) DO UPDATE SET
|
||||||
|
name = COALESCE(NULLIF(excluded.name, ''), up_profile.name),
|
||||||
|
follower_latest = excluded.follower_latest,
|
||||||
|
last_refreshed = excluded.last_refreshed
|
||||||
|
""",
|
||||||
|
(mid, name, follower, following_value, now, follower, now),
|
||||||
|
)
|
||||||
|
cursor = conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE video
|
||||||
|
SET up_follower = ?, up_follower_at = ?, up_follower_backfilled = ?
|
||||||
|
WHERE up_mid = ? AND up_follower IS NULL
|
||||||
|
""",
|
||||||
|
(follower, now, backfill_flag, mid),
|
||||||
|
)
|
||||||
|
return max(cursor.rowcount, 0)
|
||||||
|
|
||||||
|
return self.db.write(run)
|
||||||
|
|
||||||
|
def get_up_profile(self, mid: int) -> dict[str, Any] | None:
|
||||||
|
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT * FROM up_profile WHERE mid = ?", (int(mid),)
|
||||||
|
).fetchone()
|
||||||
|
return dict(row) if row else None
|
||||||
|
return self.db.read(run)
|
||||||
|
|
||||||
|
def up_mids_missing_follower(self) -> list[dict[str, Any]]:
|
||||||
|
"""追踪中但仍有视频缺少入库粉丝数的 UP。"""
|
||||||
|
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT up_mid, COALESCE(MAX(up_name), '') AS up_name,
|
||||||
|
COUNT(*) AS missing
|
||||||
|
FROM video
|
||||||
|
WHERE status IN ('active', 'unfaved')
|
||||||
|
AND up_mid IS NOT NULL
|
||||||
|
AND up_follower IS NULL
|
||||||
|
GROUP BY up_mid
|
||||||
|
ORDER BY up_mid
|
||||||
|
"""
|
||||||
|
).fetchall()
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
return self.db.read(run)
|
||||||
|
|
||||||
|
def up_follower_coverage(self) -> dict[str, int]:
|
||||||
|
"""返回追踪视频的粉丝快照覆盖情况。"""
|
||||||
|
def run(conn: sqlite3.Connection) -> dict[str, int]:
|
||||||
|
row = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) AS total,
|
||||||
|
SUM(CASE WHEN up_follower IS NOT NULL THEN 1 ELSE 0 END) AS covered,
|
||||||
|
COUNT(DISTINCT CASE WHEN up_follower IS NOT NULL
|
||||||
|
THEN up_mid END) AS ups_covered
|
||||||
|
FROM video
|
||||||
|
WHERE status IN ('active', 'unfaved')
|
||||||
|
AND up_mid IS NOT NULL
|
||||||
|
"""
|
||||||
|
).fetchone()
|
||||||
|
total = int(row["total"] or 0)
|
||||||
|
covered = int(row["covered"] or 0)
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"covered": covered,
|
||||||
|
"missing": max(0, total - covered),
|
||||||
|
"ups_covered": int(row["ups_covered"] or 0),
|
||||||
|
}
|
||||||
|
return self.db.read(run)
|
||||||
|
|
||||||
def folder_resources(self, media_id: int) -> list[dict[str, Any]]:
|
def folder_resources(self, media_id: int) -> list[dict[str, Any]]:
|
||||||
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
|
|||||||
+78
-1
@@ -10,15 +10,23 @@ sys.stdout.reconfigure(encoding="utf-8")
|
|||||||
from reaction_board.analytics import ( # noqa: E402
|
from reaction_board.analytics import ( # noqa: E402
|
||||||
DEFAULT_HEAT_WEIGHTS,
|
DEFAULT_HEAT_WEIGHTS,
|
||||||
METRICS,
|
METRICS,
|
||||||
|
age_days,
|
||||||
compute_growth,
|
compute_growth,
|
||||||
|
content_hours,
|
||||||
|
eb_rate,
|
||||||
|
effective_shrink_minutes,
|
||||||
heat_scores,
|
heat_scores,
|
||||||
|
interactions_per_hour,
|
||||||
interaction_rate,
|
interaction_rate,
|
||||||
is_loopback,
|
is_loopback,
|
||||||
|
is_mature,
|
||||||
|
median_of,
|
||||||
normalize_cover,
|
normalize_cover,
|
||||||
normalized_weights,
|
normalized_weights,
|
||||||
percentile,
|
percentile,
|
||||||
robust_center_scale,
|
robust_center_scale,
|
||||||
score_percentiles,
|
score_percentiles,
|
||||||
|
views_per_hour,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -157,6 +165,72 @@ def test_normalize_cover_and_hosts():
|
|||||||
assert not is_loopback("192.168.1.2")
|
assert not is_loopback("192.168.1.2")
|
||||||
|
|
||||||
|
|
||||||
|
def test_content_efficiency_helpers():
|
||||||
|
assert content_hours(3600) == 1.0
|
||||||
|
assert content_hours(7200) == 2.0
|
||||||
|
assert content_hours(0) is None
|
||||||
|
assert content_hours(-1) is None
|
||||||
|
assert content_hours(None) is None
|
||||||
|
assert content_hours(True) is None
|
||||||
|
assert content_hours("bad") is None
|
||||||
|
|
||||||
|
assert views_per_hour(1000, 3600) == 1000.0
|
||||||
|
assert views_per_hour(1000, 7200) == 500.0
|
||||||
|
assert views_per_hour(None, 3600) == 0.0
|
||||||
|
assert views_per_hour(1000, None) is None
|
||||||
|
assert views_per_hour(1000, 0) is None
|
||||||
|
|
||||||
|
metrics = {"like": 30, "coin": 10, "favorite": 20, "view": 999}
|
||||||
|
assert interactions_per_hour(metrics, 3600) == 60.0
|
||||||
|
assert interactions_per_hour(metrics, 7200) == 30.0
|
||||||
|
assert interactions_per_hour(None, 3600) is None
|
||||||
|
assert interactions_per_hour(metrics, None) is None
|
||||||
|
# 单项缺失按 0 处理,不抛异常
|
||||||
|
assert interactions_per_hour({"like": None, "coin": "bad"}, 3600) == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_age_and_maturity():
|
||||||
|
reference = 1_800_000_000
|
||||||
|
assert age_days(reference - 86400 * 10, reference) == 10
|
||||||
|
assert age_days(reference, reference) == 0
|
||||||
|
# 未来时间不产生负年龄
|
||||||
|
assert age_days(reference + 86400, reference) == 0
|
||||||
|
assert age_days(None, reference) is None
|
||||||
|
assert age_days(0, reference) is None
|
||||||
|
assert age_days("bad", reference) is None
|
||||||
|
|
||||||
|
assert is_mature(reference - 86400 * 14, reference, 14) is True
|
||||||
|
assert is_mature(reference - 86400 * 13, reference, 14) is False
|
||||||
|
assert is_mature(None, reference, 14) is False
|
||||||
|
assert is_mature(reference - 86400 * 14, reference, 0) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_shrinkage_helpers():
|
||||||
|
assert median_of([]) is None
|
||||||
|
assert median_of([None, "bad", True]) is None
|
||||||
|
assert median_of([1, 3, 2]) == 2.0
|
||||||
|
|
||||||
|
# 自适应 κ 取各 UP 总分钟的中位数
|
||||||
|
assert effective_shrink_minutes([10, 30, 20]) == 20.0
|
||||||
|
assert effective_shrink_minutes([0, -5, None]) == 120.0
|
||||||
|
assert effective_shrink_minutes([]) == 120.0
|
||||||
|
assert effective_shrink_minutes([], fallback=60) == 60.0
|
||||||
|
|
||||||
|
# κ 越大越靠近先验;exposure 为 0 时直接返回先验
|
||||||
|
assert abs(eb_rate(0, 0, 0.5, 100) - 0.5) < 1e-12
|
||||||
|
assert eb_rate(100, 100, 1.0, 0) == 1.0
|
||||||
|
assert eb_rate(100, 0, 1.0, 0) is None
|
||||||
|
# (100 + 100*0.5) / (100 + 100) = 0.75
|
||||||
|
assert abs(eb_rate(100, 100, 0.5, 100) - 0.75) < 1e-12
|
||||||
|
assert eb_rate(None, 100, 0.5, 100) is None
|
||||||
|
try:
|
||||||
|
eb_rate(1, 1, 0.5, -1)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise AssertionError("负 prior_strength 必须报错")
|
||||||
|
|
||||||
|
|
||||||
def main_test():
|
def main_test():
|
||||||
test_percentile_and_robust_scale()
|
test_percentile_and_robust_scale()
|
||||||
test_normalized_weights()
|
test_normalized_weights()
|
||||||
@@ -166,7 +240,10 @@ def main_test():
|
|||||||
test_interaction_rate_thresholds()
|
test_interaction_rate_thresholds()
|
||||||
test_compute_growth()
|
test_compute_growth()
|
||||||
test_normalize_cover_and_hosts()
|
test_normalize_cover_and_hosts()
|
||||||
print("OK: analytics 热度/增长/互动率/封面测试全部通过")
|
test_content_efficiency_helpers()
|
||||||
|
test_age_and_maturity()
|
||||||
|
test_shrinkage_helpers()
|
||||||
|
print("OK: analytics 热度/增长/互动率/时长效率/收缩测试全部通过")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -216,6 +216,25 @@ def test_infos_invalid_attr_and_empty():
|
|||||||
assert len(transport.calls) == 1
|
assert len(transport.calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_infos_parses_page_count():
|
||||||
|
payload = {"code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO, INVALID_INFO]}
|
||||||
|
client, _ = make_client(QueueTransport([http_json(payload)]))
|
||||||
|
result = client.fav_infos(
|
||||||
|
3700023351, [VALID_INFO["id"], INVALID_INFO["id"]]
|
||||||
|
)
|
||||||
|
assert result.items[VALID_INFO["id"]].pages == 2
|
||||||
|
assert result.items[INVALID_INFO["id"]].pages == 1
|
||||||
|
|
||||||
|
# page 字段缺失或非法时保持 None,不把 0 当分 P 数
|
||||||
|
broken = json.loads(json.dumps(VALID_INFO))
|
||||||
|
broken["id"] = 999
|
||||||
|
broken["page"] = 0
|
||||||
|
client, _ = make_client(QueueTransport([http_json({
|
||||||
|
"code": 0, "message": "0", "ttl": 1, "data": [broken],
|
||||||
|
})]))
|
||||||
|
assert client.fav_infos(3700023351, [999]).items[999].pages is None
|
||||||
|
|
||||||
|
|
||||||
def test_infos_unexpected_aid_is_protocol_error():
|
def test_infos_unexpected_aid_is_protocol_error():
|
||||||
transport = QueueTransport([http_json({
|
transport = QueueTransport([http_json({
|
||||||
"code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO],
|
"code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO],
|
||||||
|
|||||||
+44
-1
@@ -126,12 +126,55 @@ def test_invalid_values():
|
|||||||
raise AssertionError("非法环境变量必须拒绝")
|
raise AssertionError("非法环境变量必须拒绝")
|
||||||
|
|
||||||
|
|
||||||
|
def test_analytics_defaults_and_validation():
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "config.json"
|
||||||
|
write(path, base_config())
|
||||||
|
cfg = load_config(path, env={})
|
||||||
|
assert cfg.analytics.min_duration_seconds == 600
|
||||||
|
assert cfg.analytics.maturity_days == 14
|
||||||
|
assert cfg.analytics.min_view_for_efficiency == 0
|
||||||
|
assert cfg.analytics.shrink_minutes is None
|
||||||
|
|
||||||
|
data = base_config()
|
||||||
|
data["analytics"] = {
|
||||||
|
"min_duration_seconds": 300,
|
||||||
|
"maturity_days": 7,
|
||||||
|
"min_view_for_efficiency": 100,
|
||||||
|
"shrink_minutes": 90,
|
||||||
|
}
|
||||||
|
write(path, data)
|
||||||
|
cfg = load_config(path, env={})
|
||||||
|
assert cfg.analytics.min_duration_seconds == 300
|
||||||
|
assert cfg.analytics.maturity_days == 7
|
||||||
|
assert cfg.analytics.min_view_for_efficiency == 100
|
||||||
|
assert cfg.analytics.shrink_minutes == 90
|
||||||
|
|
||||||
|
for bad, hint in (
|
||||||
|
({"min_duration_seconds": -1}, "min_duration_seconds"),
|
||||||
|
({"maturity_days": -1}, "maturity_days"),
|
||||||
|
({"min_view_for_efficiency": -1}, "min_view_for_efficiency"),
|
||||||
|
({"shrink_minutes": 0}, "shrink_minutes"),
|
||||||
|
({"shrink_minutes": "auto"}, "shrink_minutes"),
|
||||||
|
):
|
||||||
|
data = base_config()
|
||||||
|
data["analytics"] = bad
|
||||||
|
write(path, data)
|
||||||
|
try:
|
||||||
|
load_config(path, env={})
|
||||||
|
except ConfigError as exc:
|
||||||
|
assert hint in str(exc), (bad, str(exc))
|
||||||
|
else:
|
||||||
|
raise AssertionError(f"非法 analytics 必须拒绝: {bad}")
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
test_dual_accounts_and_env_override()
|
test_dual_accounts_and_env_override()
|
||||||
test_flat_and_missing_account_rejected()
|
test_flat_and_missing_account_rejected()
|
||||||
test_validation_require_read_auth_and_hot_reload()
|
test_validation_require_read_auth_and_hot_reload()
|
||||||
test_invalid_values()
|
test_invalid_values()
|
||||||
print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖与热加载测试全部通过")
|
test_analytics_defaults_and_validation()
|
||||||
|
print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖、热加载与 analytics 测试全部通过")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from reaction_board.migrations import ( # noqa: E402
|
|||||||
|
|
||||||
EXPECTED_TABLES = {
|
EXPECTED_TABLES = {
|
||||||
"video", "snapshot", "folder", "folder_resource", "up_blacklist",
|
"video", "snapshot", "folder", "folder_resource", "up_blacklist",
|
||||||
"up_stat", "candidate", "seen_aid", "notify", "fav_action", "kv",
|
"up_stat", "up_profile", "candidate", "seen_aid", "notify", "fav_action", "kv",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -37,8 +37,8 @@ def test_fresh_and_idempotent():
|
|||||||
conn = sqlite3.connect(Path(tmp) / "app.db")
|
conn = sqlite3.connect(Path(tmp) / "app.db")
|
||||||
try:
|
try:
|
||||||
applied = run_migrations(conn)
|
applied = run_migrations(conn)
|
||||||
assert applied == [1, 2, 3]
|
assert applied == [1, 2, 3, 4]
|
||||||
assert current_version(conn) == 3
|
assert current_version(conn) == 4
|
||||||
assert EXPECTED_TABLES <= table_names(conn)
|
assert EXPECTED_TABLES <= table_names(conn)
|
||||||
assert "schema_version" in table_names(conn)
|
assert "schema_version" in table_names(conn)
|
||||||
columns = {
|
columns = {
|
||||||
@@ -49,10 +49,24 @@ def test_fresh_and_idempotent():
|
|||||||
row[1] for row in conn.execute("PRAGMA table_info(candidate)")
|
row[1] for row in conn.execute("PRAGMA table_info(candidate)")
|
||||||
}
|
}
|
||||||
assert {"duration", "play", "like", "favorite", "danmaku", "review"} <= candidate_columns
|
assert {"duration", "play", "like", "favorite", "danmaku", "review"} <= candidate_columns
|
||||||
|
video_columns = {
|
||||||
|
row[1] for row in conn.execute("PRAGMA table_info(video)")
|
||||||
|
}
|
||||||
|
assert {
|
||||||
|
"pages", "duration", "up_follower", "up_follower_at",
|
||||||
|
"up_follower_backfilled",
|
||||||
|
} <= video_columns
|
||||||
|
profile_columns = {
|
||||||
|
row[1] for row in conn.execute("PRAGMA table_info(up_profile)")
|
||||||
|
}
|
||||||
|
assert {
|
||||||
|
"mid", "follower_first", "following_first", "first_captured_at",
|
||||||
|
"follower_latest", "last_refreshed",
|
||||||
|
} <= profile_columns
|
||||||
# 再跑一次不应重复应用
|
# 再跑一次不应重复应用
|
||||||
assert run_migrations(conn) == []
|
assert run_migrations(conn) == []
|
||||||
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
|
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
|
||||||
assert row[0] == 3
|
assert row[0] == 4
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
@@ -67,15 +81,15 @@ def test_backup_before_upgrade():
|
|||||||
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
|
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
v4 = Migration(4, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
v5 = Migration(5, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
||||||
applied = run_migrations(
|
applied = run_migrations(
|
||||||
conn,
|
conn,
|
||||||
backup_dir=backup_dir,
|
backup_dir=backup_dir,
|
||||||
now=1_800_000_000,
|
now=1_800_000_000,
|
||||||
migrations=MIGRATIONS + (v4,),
|
migrations=MIGRATIONS + (v5,),
|
||||||
)
|
)
|
||||||
assert applied == [4]
|
assert applied == [5]
|
||||||
backups = list(backup_dir.glob("app-pre-v3-*.db"))
|
backups = list(backup_dir.glob("app-pre-v4-*.db"))
|
||||||
assert len(backups) == 1
|
assert len(backups) == 1
|
||||||
check = sqlite3.connect(backups[0])
|
check = sqlite3.connect(backups[0])
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ F1 = FolderConfig(year=2025, media_id=4451053851, title="2025")
|
|||||||
F2 = FolderConfig(year=2026, media_id=3700023351, title="2026")
|
F2 = FolderConfig(year=2026, media_id=3700023351, title="2026")
|
||||||
|
|
||||||
|
|
||||||
def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None):
|
def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None,
|
||||||
|
pages=2):
|
||||||
return FavResource(
|
return FavResource(
|
||||||
aid=aid,
|
aid=aid,
|
||||||
bvid=bvid or f"BV{aid}",
|
bvid=bvid or f"BV{aid}",
|
||||||
@@ -30,6 +31,7 @@ def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=Non
|
|||||||
cover="http://example/cover.jpg",
|
cover="http://example/cover.jpg",
|
||||||
attr=attr,
|
attr=attr,
|
||||||
fav_time=fav_time,
|
fav_time=fav_time,
|
||||||
|
pages=pages,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -54,6 +56,7 @@ def test_upsert_and_folder_resource():
|
|||||||
assert video["folder_id"] == F1.media_id
|
assert video["folder_id"] == F1.media_id
|
||||||
assert video["source"] == f"fav:{F1.media_id}"
|
assert video["source"] == f"fav:{F1.media_id}"
|
||||||
assert video["first_seen"] == 1_800_000_000
|
assert video["first_seen"] == 1_800_000_000
|
||||||
|
assert video["pages"] == 2
|
||||||
assert repo.folder_active_aids(F1.media_id) == {1}
|
assert repo.folder_active_aids(F1.media_id) == {1}
|
||||||
|
|
||||||
# 幂等:重复导入不新增行、不改 first_seen
|
# 幂等:重复导入不新增行、不改 first_seen
|
||||||
@@ -293,6 +296,57 @@ def test_pending_snapshot_aids():
|
|||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_up_follower_snapshot_and_backfill():
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db, repo = setup(tmp)
|
||||||
|
try:
|
||||||
|
repo.upsert_from_fav(resource(1, up_mid=100), F1.media_id, was_added=True)
|
||||||
|
repo.upsert_from_fav(resource(2, up_mid=100), F1.media_id, was_added=True)
|
||||||
|
repo.upsert_from_fav(resource(3, up_mid=200), F1.media_id, was_added=True)
|
||||||
|
missing = {row["up_mid"] for row in repo.up_mids_missing_follower()}
|
||||||
|
assert missing == {100, 200}
|
||||||
|
|
||||||
|
now = 1_800_000_000
|
||||||
|
assert repo.apply_up_follower(
|
||||||
|
mid=100, name="UP", follower=1234, following=5, now=now
|
||||||
|
) == 2
|
||||||
|
first = repo.get_video(1)
|
||||||
|
assert first["up_follower"] == 1234
|
||||||
|
assert first["up_follower_at"] == now
|
||||||
|
assert first["up_follower_backfilled"] == 0
|
||||||
|
profile = repo.get_up_profile(100)
|
||||||
|
assert profile["follower_first"] == 1234
|
||||||
|
assert profile["follower_latest"] == 1234
|
||||||
|
assert {
|
||||||
|
row["up_mid"] for row in repo.up_mids_missing_follower()
|
||||||
|
} == {200}
|
||||||
|
|
||||||
|
# 同一 UP 后来新增的视频记录当下的粉丝快照,不覆盖旧视频
|
||||||
|
repo.upsert_from_fav(resource(4, up_mid=100), F1.media_id, was_added=True)
|
||||||
|
repo.apply_up_follower(
|
||||||
|
mid=100, name="UP", follower=2000, following=6, now=now + 60
|
||||||
|
)
|
||||||
|
assert repo.get_video(4)["up_follower"] == 2000
|
||||||
|
assert repo.get_video(1)["up_follower"] == 1234
|
||||||
|
profile = repo.get_up_profile(100)
|
||||||
|
assert profile["follower_first"] == 1234
|
||||||
|
assert profile["follower_latest"] == 2000
|
||||||
|
|
||||||
|
# 回填只补空值,并标记 backfilled
|
||||||
|
assert repo.apply_up_follower(
|
||||||
|
mid=200, name="UP2", follower=88, now=now + 120, backfilled=True
|
||||||
|
) == 1
|
||||||
|
assert repo.get_video(3)["up_follower"] == 88
|
||||||
|
assert repo.get_video(3)["up_follower_backfilled"] == 1
|
||||||
|
|
||||||
|
coverage = repo.up_follower_coverage()
|
||||||
|
assert coverage == {
|
||||||
|
"total": 4, "covered": 4, "missing": 0, "ups_covered": 2,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
test_upsert_and_folder_resource()
|
test_upsert_and_folder_resource()
|
||||||
test_invalid_attr_and_missing()
|
test_invalid_attr_and_missing()
|
||||||
@@ -305,6 +359,7 @@ def main():
|
|||||||
test_primary_folder_is_order_independent()
|
test_primary_folder_is_order_independent()
|
||||||
test_dropped_invalid_preserves_original_title()
|
test_dropped_invalid_preserves_original_title()
|
||||||
test_pending_snapshot_aids()
|
test_pending_snapshot_aids()
|
||||||
|
test_up_follower_snapshot_and_backfill()
|
||||||
print("OK: 收藏夹/视频状态机与黑名单测试全部通过")
|
print("OK: 收藏夹/视频状态机与黑名单测试全部通过")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user