feat: home growth auto(7→1) + hourly recent-video snapshot #2
No files matched your search
@@ -64,7 +64,9 @@
|
|||||||
"schedule": {
|
"schedule": {
|
||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "00:01",
|
"snapshot_at": "00:01",
|
||||||
"candidate_ttl_days": 7
|
"candidate_ttl_days": 7,
|
||||||
|
"recent_snapshot_minutes": 60,
|
||||||
|
"recent_video_days": 2
|
||||||
},
|
},
|
||||||
"discover": {
|
"discover": {
|
||||||
"keywords": ["咻咻满"],
|
"keywords": ["咻咻满"],
|
||||||
@@ -107,6 +109,7 @@
|
|||||||
"maturity_days": 2,
|
"maturity_days": 2,
|
||||||
"min_view_for_ranking": 2000,
|
"min_view_for_ranking": 2000,
|
||||||
"video_shrink_minutes": 60,
|
"video_shrink_minutes": 60,
|
||||||
"shrink_minutes": null
|
"shrink_minutes": null,
|
||||||
|
"home_growth_days": "auto"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+5
-2
@@ -74,7 +74,9 @@
|
|||||||
"schedule": {
|
"schedule": {
|
||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "00:01",
|
"snapshot_at": "00:01",
|
||||||
"candidate_ttl_days": 7
|
"candidate_ttl_days": 7,
|
||||||
|
"recent_snapshot_minutes": 60,
|
||||||
|
"recent_video_days": 2
|
||||||
},
|
},
|
||||||
"discover": {
|
"discover": {
|
||||||
"keywords": ["咻咻满"],
|
"keywords": ["咻咻满"],
|
||||||
@@ -110,6 +112,7 @@
|
|||||||
"maturity_days": 2,
|
"maturity_days": 2,
|
||||||
"min_view_for_ranking": 2000,
|
"min_view_for_ranking": 2000,
|
||||||
"video_shrink_minutes": 60,
|
"video_shrink_minutes": 60,
|
||||||
"shrink_minutes": null
|
"shrink_minutes": null,
|
||||||
|
"home_growth_days": "auto"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -163,6 +163,13 @@ def _clear_task_running(repo: Repository, name: str) -> None:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _is_task_running(repo: Repository, name: str) -> bool:
|
||||||
|
try:
|
||||||
|
return bool(repo.kv_get(_task_running_key(name)))
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _record_task(
|
def _record_task(
|
||||||
repo: Repository,
|
repo: Repository,
|
||||||
name: str,
|
name: str,
|
||||||
@@ -547,6 +554,16 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
|||||||
|
|
||||||
def run_snapshot():
|
def run_snapshot():
|
||||||
started = int(time.time())
|
started = int(time.time())
|
||||||
|
if _is_task_running(repo, "snapshot_recent"):
|
||||||
|
logger.info(
|
||||||
|
"snapshot.skip", reason="snapshot_recent_running",
|
||||||
|
)
|
||||||
|
_record_task(
|
||||||
|
repo, "snapshot", ok=True, started_at=started,
|
||||||
|
finished_at=int(time.time()),
|
||||||
|
payload={"skipped": True, "reason": "snapshot_recent_running"},
|
||||||
|
)
|
||||||
|
return None
|
||||||
_mark_task_running(repo, "snapshot", started_at=started)
|
_mark_task_running(repo, "snapshot", started_at=started)
|
||||||
try:
|
try:
|
||||||
summary = snapshot_service.run_if_needed()
|
summary = snapshot_service.run_if_needed()
|
||||||
@@ -567,6 +584,39 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
|||||||
logger.info("snapshot.summary", **payload)
|
logger.info("snapshot.summary", **payload)
|
||||||
return summary
|
return summary
|
||||||
|
|
||||||
|
def run_snapshot_recent():
|
||||||
|
started = int(time.time())
|
||||||
|
if _is_task_running(repo, "snapshot"):
|
||||||
|
logger.info(
|
||||||
|
"snapshot.recent_skip", reason="snapshot_running",
|
||||||
|
)
|
||||||
|
_record_task(
|
||||||
|
repo, "snapshot_recent", ok=True, started_at=started,
|
||||||
|
finished_at=int(time.time()),
|
||||||
|
payload={"skipped": True, "reason": "snapshot_running"},
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
_mark_task_running(repo, "snapshot_recent", started_at=started)
|
||||||
|
try:
|
||||||
|
summary = snapshot_service.run_recent_videos()
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
_record_task(
|
||||||
|
repo, "snapshot_recent", ok=False, started_at=started,
|
||||||
|
finished_at=int(time.time()), error=str(exc),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
payload = summary.as_dict()
|
||||||
|
ok = summary.failed == 0
|
||||||
|
error = ";".join(summary.errors) if summary.errors else None
|
||||||
|
_record_task(
|
||||||
|
repo, "snapshot_recent", ok=ok, started_at=summary.started_at,
|
||||||
|
finished_at=summary.finished_at or int(time.time()),
|
||||||
|
payload=payload, error=error,
|
||||||
|
)
|
||||||
|
last["snapshot_recent"] = summary
|
||||||
|
logger.info("snapshot.recent_summary", **payload)
|
||||||
|
return summary
|
||||||
|
|
||||||
def run_fans():
|
def run_fans():
|
||||||
started = int(time.time())
|
started = int(time.time())
|
||||||
_mark_task_running(repo, "fans", started_at=started)
|
_mark_task_running(repo, "fans", started_at=started)
|
||||||
@@ -601,6 +651,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
|||||||
|
|
||||||
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
|
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
|
||||||
snapshot_at = f"{hh:02d}:{mm:02d}"
|
snapshot_at = f"{hh:02d}:{mm:02d}"
|
||||||
|
recent_minutes = int(store.get().schedule.recent_snapshot_minutes)
|
||||||
scheduler = Scheduler(
|
scheduler = Scheduler(
|
||||||
[
|
[
|
||||||
ScheduledTask(
|
ScheduledTask(
|
||||||
@@ -617,6 +668,11 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
|||||||
next_run=0.0,
|
next_run=0.0,
|
||||||
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
|
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
|
||||||
),
|
),
|
||||||
|
ScheduledTask(
|
||||||
|
"snapshot_recent",
|
||||||
|
float(recent_minutes * 60),
|
||||||
|
run_snapshot_recent,
|
||||||
|
),
|
||||||
ScheduledTask(
|
ScheduledTask(
|
||||||
"discovery",
|
"discovery",
|
||||||
float(cfg.schedule.discover_minutes * 60),
|
float(cfg.schedule.discover_minutes * 60),
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ DEFAULT_ANALYTICS: dict[str, Any] = {
|
|||||||
"min_view_for_ranking": 2000,
|
"min_view_for_ranking": 2000,
|
||||||
"video_shrink_minutes": 60,
|
"video_shrink_minutes": 60,
|
||||||
"shrink_minutes": None,
|
"shrink_minutes": None,
|
||||||
|
"home_growth_days": "auto",
|
||||||
}
|
}
|
||||||
|
|
||||||
DEFAULT_CONFIG: dict[str, Any] = {
|
DEFAULT_CONFIG: dict[str, Any] = {
|
||||||
@@ -84,6 +85,8 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
|||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "00:01",
|
"snapshot_at": "00:01",
|
||||||
"candidate_ttl_days": 7,
|
"candidate_ttl_days": 7,
|
||||||
|
"recent_snapshot_minutes": 60,
|
||||||
|
"recent_video_days": 2,
|
||||||
},
|
},
|
||||||
"discover": {
|
"discover": {
|
||||||
"keywords": ["咻咻满"],
|
"keywords": ["咻咻满"],
|
||||||
@@ -177,6 +180,8 @@ class ScheduleConfig:
|
|||||||
discover_minutes: int
|
discover_minutes: int
|
||||||
snapshot_at: str
|
snapshot_at: str
|
||||||
candidate_ttl_days: int
|
candidate_ttl_days: int
|
||||||
|
recent_snapshot_minutes: int
|
||||||
|
recent_video_days: int
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -200,6 +205,7 @@ class AnalyticsConfig:
|
|||||||
min_view_for_ranking: int
|
min_view_for_ranking: int
|
||||||
video_shrink_minutes: int
|
video_shrink_minutes: int
|
||||||
shrink_minutes: int | None
|
shrink_minutes: int | None
|
||||||
|
home_growth_days: str | int
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -456,6 +462,16 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
|||||||
"schedule.candidate_ttl_days",
|
"schedule.candidate_ttl_days",
|
||||||
minimum=1,
|
minimum=1,
|
||||||
),
|
),
|
||||||
|
recent_snapshot_minutes=_as_int(
|
||||||
|
sched_raw.get("recent_snapshot_minutes", 60),
|
||||||
|
"schedule.recent_snapshot_minutes",
|
||||||
|
minimum=1,
|
||||||
|
),
|
||||||
|
recent_video_days=_as_int(
|
||||||
|
sched_raw.get("recent_video_days", 2),
|
||||||
|
"schedule.recent_video_days",
|
||||||
|
minimum=1,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
notify_raw = data["notify"]
|
notify_raw = data["notify"]
|
||||||
@@ -533,6 +549,13 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
|||||||
if not isinstance(analytics_raw, Mapping):
|
if not isinstance(analytics_raw, Mapping):
|
||||||
raise ConfigError("analytics 必须是对象")
|
raise ConfigError("analytics 必须是对象")
|
||||||
shrink_raw = analytics_raw.get("shrink_minutes")
|
shrink_raw = analytics_raw.get("shrink_minutes")
|
||||||
|
home_growth_raw = analytics_raw.get("home_growth_days", "auto")
|
||||||
|
if home_growth_raw == "auto":
|
||||||
|
home_growth_days: str | int = "auto"
|
||||||
|
elif home_growth_raw in (1, 7):
|
||||||
|
home_growth_days = int(home_growth_raw)
|
||||||
|
else:
|
||||||
|
raise ConfigError('analytics.home_growth_days 必须是 "auto"、1 或 7')
|
||||||
analytics = AnalyticsConfig(
|
analytics = AnalyticsConfig(
|
||||||
min_duration_seconds=_as_int(
|
min_duration_seconds=_as_int(
|
||||||
analytics_raw.get("min_duration_seconds", 600),
|
analytics_raw.get("min_duration_seconds", 600),
|
||||||
@@ -559,6 +582,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
|||||||
shrink_raw, "analytics.shrink_minutes", minimum=1
|
shrink_raw, "analytics.shrink_minutes", minimum=1
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
|
home_growth_days=home_growth_days,
|
||||||
)
|
)
|
||||||
|
|
||||||
return AppConfig(
|
return AppConfig(
|
||||||
|
|||||||
@@ -637,6 +637,19 @@ class Repository:
|
|||||||
return [int(row["aid"]) for row in rows]
|
return [int(row["aid"]) for row in rows]
|
||||||
return self.db.read(run)
|
return self.db.read(run)
|
||||||
|
|
||||||
|
def video_aids_recent_for_snapshot(self, *, since_pubdate: int) -> list[int]:
|
||||||
|
"""近 N 天新视频:按 pubdate 墙钟窗口(since_pubdate = now - days*86400)。"""
|
||||||
|
def run(conn: sqlite3.Connection) -> list[int]:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT aid FROM video "
|
||||||
|
"WHERE status IN ('active', 'unfaved') "
|
||||||
|
"AND pubdate IS NOT NULL AND pubdate >= ? "
|
||||||
|
"ORDER BY aid",
|
||||||
|
(int(since_pubdate),),
|
||||||
|
).fetchall()
|
||||||
|
return [int(row["aid"]) for row in rows]
|
||||||
|
return self.db.read(run)
|
||||||
|
|
||||||
def pending_snapshot_aids(self, stat_date: str) -> list[int]:
|
def pending_snapshot_aids(self, stat_date: str) -> list[int]:
|
||||||
"""当天仍缺成功快照的追踪视频;用于新入库/恢复后的同日补采。"""
|
"""当天仍缺成功快照的追踪视频;用于新入库/恢复后的同日补采。"""
|
||||||
def run(conn: sqlite3.Connection) -> list[int]:
|
def run(conn: sqlite3.Connection) -> list[int]:
|
||||||
|
|||||||
@@ -4,7 +4,8 @@
|
|||||||
规则(DEVELOPMENT.md §7.4):
|
规则(DEVELOPMENT.md §7.4):
|
||||||
|
|
||||||
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
|
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
|
||||||
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑);
|
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
|
||||||
|
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
|
||||||
* 视频不可访问时标记 ``removed``;
|
* 视频不可访问时标记 ``removed``;
|
||||||
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
|
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
|
||||||
* 对 ``track_fans`` 逐个请求 ``relation/stat``(独立任务,不影响视频完成日);
|
* 对 ``track_fans`` 逐个请求 ``relation/stat``(独立任务,不影响视频完成日);
|
||||||
@@ -198,6 +199,8 @@ class SnapshotService:
|
|||||||
for index, aid in enumerate(videos, 1):
|
for index, aid in enumerate(videos, 1):
|
||||||
existing = self.repo.get_snapshot(stat_date, aid)
|
existing = self.repo.get_snapshot(stat_date, aid)
|
||||||
if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
|
if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
|
||||||
|
# 断点续跑:同日已成功则跳过。小时任务刷新过的行也是 ok=1,
|
||||||
|
# 自然覆盖「1 小时内 captured_at 已更新则不再次更新」的要求。
|
||||||
summary.skipped += 1
|
summary.skipped += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -269,6 +272,98 @@ class SnapshotService:
|
|||||||
)
|
)
|
||||||
return summary
|
return summary
|
||||||
|
|
||||||
|
def run_recent_videos(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
date: str | None = None,
|
||||||
|
recent_days: int | None = None,
|
||||||
|
) -> SnapshotSummary:
|
||||||
|
"""刷新近 N 天新视频的当日快照;不碰完成日标记与日备份。
|
||||||
|
|
||||||
|
「近 N 天」按 pubdate 墙钟窗口:``now - recent_video_days * 86400``,
|
||||||
|
不是上海日历日差。允许覆盖同日已有 ``ok=1`` 行以便小时级刷新。
|
||||||
|
"""
|
||||||
|
cfg = self.config_store.get()
|
||||||
|
client = self._client()
|
||||||
|
now = self._now()
|
||||||
|
stat_date = date or shanghai_date(now)
|
||||||
|
if not is_date(stat_date):
|
||||||
|
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||||
|
days = (
|
||||||
|
int(cfg.schedule.recent_video_days)
|
||||||
|
if recent_days is None else int(recent_days)
|
||||||
|
)
|
||||||
|
if days < 1:
|
||||||
|
raise ValueError("recent_days 必须 >= 1")
|
||||||
|
# 墙钟窗口,而非按上海日历截断到日界。
|
||||||
|
since_pubdate = now - days * 86400
|
||||||
|
|
||||||
|
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||||||
|
self._log(
|
||||||
|
"info", "snapshot.recent_start",
|
||||||
|
stat_date=stat_date, recent_days=days, since_pubdate=since_pubdate,
|
||||||
|
)
|
||||||
|
videos = self.repo.video_aids_recent_for_snapshot(since_pubdate=since_pubdate)
|
||||||
|
summary.total = len(videos)
|
||||||
|
for index, aid in enumerate(videos, 1):
|
||||||
|
stat, error, unavailable = self._fetch_view(client, aid)
|
||||||
|
if unavailable:
|
||||||
|
summary.removed += 1
|
||||||
|
self.repo.set_video_status(aid, "removed", now=now)
|
||||||
|
self._log(
|
||||||
|
"warning", "snapshot.recent_removed",
|
||||||
|
stat_date=stat_date, aid=aid, error=error,
|
||||||
|
)
|
||||||
|
elif stat is not None:
|
||||||
|
self.repo.upsert_snapshot(
|
||||||
|
stat_date,
|
||||||
|
aid,
|
||||||
|
metrics=stat.metrics(),
|
||||||
|
ok=True,
|
||||||
|
err=None,
|
||||||
|
captured_at=self._now(),
|
||||||
|
)
|
||||||
|
if getattr(stat, "up_face", None):
|
||||||
|
self.repo.apply_video_up_face(aid, stat.up_face)
|
||||||
|
summary.success += 1
|
||||||
|
else:
|
||||||
|
summary.failed += 1
|
||||||
|
self.repo.upsert_snapshot(
|
||||||
|
stat_date,
|
||||||
|
aid,
|
||||||
|
metrics=None,
|
||||||
|
ok=False,
|
||||||
|
err=error,
|
||||||
|
captured_at=self._now(),
|
||||||
|
)
|
||||||
|
summary.errors.append(f"aid {aid}: {error}")
|
||||||
|
self._log(
|
||||||
|
"error", "snapshot.recent_failed",
|
||||||
|
stat_date=stat_date, aid=aid, error=error,
|
||||||
|
)
|
||||||
|
|
||||||
|
if self.progress_every > 0 and (
|
||||||
|
index % self.progress_every == 0 or index == summary.total
|
||||||
|
):
|
||||||
|
self._log(
|
||||||
|
"info", "snapshot.recent_progress",
|
||||||
|
stat_date=stat_date, done=index, total=summary.total,
|
||||||
|
success=summary.success, failed=summary.failed,
|
||||||
|
removed=summary.removed,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 刻意不写 snapshot:completed / last_completed / 日备份:完成日只归全量任务。
|
||||||
|
summary.completed = summary.failed == 0
|
||||||
|
if summary.success or summary.removed or summary.failed:
|
||||||
|
self.repo.bump_cache_version("videos")
|
||||||
|
summary.finished_at = self._now()
|
||||||
|
self._log(
|
||||||
|
"info" if summary.completed else "warning",
|
||||||
|
"snapshot.recent_done",
|
||||||
|
**summary.as_dict(),
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
def run_fans_if_needed(
|
def run_fans_if_needed(
|
||||||
self, *, date: str | None = None, force: bool = False
|
self, *, date: str | None = None, force: bool = False
|
||||||
) -> SnapshotSummary:
|
) -> SnapshotSummary:
|
||||||
|
|||||||
@@ -699,7 +699,7 @@ class DashboardQueries:
|
|||||||
sort: str = "heat",
|
sort: str = "heat",
|
||||||
within_days: int | None = None,
|
within_days: int | None = None,
|
||||||
year: int | None = None,
|
year: int | None = None,
|
||||||
growth_days: int = 7,
|
growth_days: int | str = 7,
|
||||||
folder_id: int | None = None,
|
folder_id: int | None = None,
|
||||||
q: str | None = None,
|
q: str | None = None,
|
||||||
page: int = 1,
|
page: int = 1,
|
||||||
@@ -734,7 +734,7 @@ class DashboardQueries:
|
|||||||
sort: str = "heat",
|
sort: str = "heat",
|
||||||
within_days: int | None = None,
|
within_days: int | None = None,
|
||||||
year: int | None = None,
|
year: int | None = None,
|
||||||
growth_days: int = 7,
|
growth_days: int | str = 7,
|
||||||
folder_id: int | None = None,
|
folder_id: int | None = None,
|
||||||
q: str | None = None,
|
q: str | None = None,
|
||||||
page: int = 1,
|
page: int = 1,
|
||||||
@@ -742,6 +742,25 @@ class DashboardQueries:
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
if sort not in VIDEO_SORTS:
|
if sort not in VIDEO_SORTS:
|
||||||
raise ValueError(f"不支持的排序: {sort}")
|
raise ValueError(f"不支持的排序: {sort}")
|
||||||
|
if growth_days == "auto":
|
||||||
|
preferred = self.config_store.get().analytics.home_growth_days
|
||||||
|
if preferred in (1, 7):
|
||||||
|
growth_days = int(preferred)
|
||||||
|
else:
|
||||||
|
trial = self._list_videos_uncached(
|
||||||
|
sort=sort,
|
||||||
|
within_days=within_days,
|
||||||
|
year=year,
|
||||||
|
growth_days=7,
|
||||||
|
folder_id=folder_id,
|
||||||
|
q=q,
|
||||||
|
page=page,
|
||||||
|
page_size=page_size,
|
||||||
|
)
|
||||||
|
if trial.get("available") and int(trial.get("total") or 0) > 0:
|
||||||
|
return trial
|
||||||
|
growth_days = 1
|
||||||
|
growth_days = int(growth_days)
|
||||||
if growth_days < 1 or growth_days > 365:
|
if growth_days < 1 or growth_days > 365:
|
||||||
raise ValueError("growth_days 必须在 1~365 之间")
|
raise ValueError("growth_days 必须在 1~365 之间")
|
||||||
page = max(1, int(page))
|
page = max(1, int(page))
|
||||||
@@ -900,7 +919,9 @@ class DashboardQueries:
|
|||||||
reason = None
|
reason = None
|
||||||
if sort in {"growth", "growth_rate"} and not growth_available:
|
if sort in {"growth", "growth_rate"} and not growth_available:
|
||||||
available = False
|
available = False
|
||||||
reason = "需要至少两天的快照才能计算增长"
|
reason = (
|
||||||
|
f"近 {growth_days} 日增长需要至少 {growth_days + 1} 天快照"
|
||||||
|
)
|
||||||
duration_filter = None
|
duration_filter = None
|
||||||
if sort in duration_sorts:
|
if sort in duration_sorts:
|
||||||
duration_filter = {
|
duration_filter = {
|
||||||
@@ -1571,6 +1592,7 @@ class DashboardQueries:
|
|||||||
"min_view_for_ranking": cfg.analytics.min_view_for_ranking,
|
"min_view_for_ranking": cfg.analytics.min_view_for_ranking,
|
||||||
"video_shrink_minutes": cfg.analytics.video_shrink_minutes,
|
"video_shrink_minutes": cfg.analytics.video_shrink_minutes,
|
||||||
"shrink_minutes": cfg.analytics.shrink_minutes,
|
"shrink_minutes": cfg.analytics.shrink_minutes,
|
||||||
|
"home_growth_days": cfg.analytics.home_growth_days,
|
||||||
},
|
},
|
||||||
"heat_weights": {
|
"heat_weights": {
|
||||||
metric: float(value)
|
metric: float(value)
|
||||||
@@ -1747,23 +1769,30 @@ class DashboardQueries:
|
|||||||
running = self._task_running(conn)
|
running = self._task_running(conn)
|
||||||
now = self._now()
|
now = self._now()
|
||||||
discover_interval = int(cfg.schedule.discover_minutes) * 60
|
discover_interval = int(cfg.schedule.discover_minutes) * 60
|
||||||
|
recent_interval = int(cfg.schedule.recent_snapshot_minutes) * 60
|
||||||
next_runs = {
|
next_runs = {
|
||||||
"fans": int(next_daily_run(now, cfg.schedule.snapshot_at)),
|
"fans": int(next_daily_run(now, cfg.schedule.snapshot_at)),
|
||||||
"snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)),
|
"snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)),
|
||||||
"discover": int(now + discover_interval),
|
"discover": int(now + discover_interval),
|
||||||
|
"snapshot_recent": int(now + recent_interval),
|
||||||
}
|
}
|
||||||
for name, key in (
|
for name, key in (
|
||||||
("fans", "fans"),
|
("fans", "fans"),
|
||||||
("snapshot", "snapshot"),
|
("snapshot", "snapshot"),
|
||||||
("discover", "discovery"),
|
("discover", "discovery"),
|
||||||
|
("snapshot_recent", "snapshot_recent"),
|
||||||
):
|
):
|
||||||
record = tasks.get(key)
|
record = tasks.get(key)
|
||||||
if not (isinstance(record, dict) and record.get("finished_at")):
|
if not (isinstance(record, dict) and record.get("finished_at")):
|
||||||
continue
|
continue
|
||||||
finished_at = int(record["finished_at"])
|
finished_at = int(record["finished_at"])
|
||||||
if name == "discover":
|
if name in {"discover", "snapshot_recent"}:
|
||||||
# 只按上次完成推下一档;进行中由 running 展示,不按墙钟抢跑。
|
# 只按上次完成推下一档;进行中由 running 展示,不按墙钟抢跑。
|
||||||
next_runs[name] = finished_at + discover_interval
|
interval = (
|
||||||
|
recent_interval if name == "snapshot_recent"
|
||||||
|
else discover_interval
|
||||||
|
)
|
||||||
|
next_runs[name] = finished_at + interval
|
||||||
else:
|
else:
|
||||||
next_runs[name] = int(
|
next_runs[name] = int(
|
||||||
next_daily_run(finished_at, cfg.schedule.snapshot_at)
|
next_daily_run(finished_at, cfg.schedule.snapshot_at)
|
||||||
@@ -1805,6 +1834,8 @@ class DashboardQueries:
|
|||||||
"discover_minutes": cfg.schedule.discover_minutes,
|
"discover_minutes": cfg.schedule.discover_minutes,
|
||||||
"snapshot_at": cfg.schedule.snapshot_at,
|
"snapshot_at": cfg.schedule.snapshot_at,
|
||||||
"candidate_ttl_days": cfg.schedule.candidate_ttl_days,
|
"candidate_ttl_days": cfg.schedule.candidate_ttl_days,
|
||||||
|
"recent_snapshot_minutes": cfg.schedule.recent_snapshot_minutes,
|
||||||
|
"recent_video_days": cfg.schedule.recent_video_days,
|
||||||
},
|
},
|
||||||
"telegram_enabled": bool(cfg.telegram.enabled),
|
"telegram_enabled": bool(cfg.telegram.enabled),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -373,7 +373,7 @@ border-color:#3a2c22;color:#f3e9dc}button{background:#e9b97c;color:#2a1a10}}
|
|||||||
sort=_param(query, "sort", "heat"),
|
sort=_param(query, "sort", "heat"),
|
||||||
within_days=_optional_int(query, "within_days"),
|
within_days=_optional_int(query, "within_days"),
|
||||||
year=_year_param(query),
|
year=_year_param(query),
|
||||||
growth_days=_int_param(query, "growth_days", 7, 1, 365),
|
growth_days=_growth_days_param(query),
|
||||||
folder_id=_optional_int(query, "folder"),
|
folder_id=_optional_int(query, "folder"),
|
||||||
q=_optional_str(query, "q"),
|
q=_optional_str(query, "q"),
|
||||||
page=_int_param(query, "page", 1, 1, 1_000_000),
|
page=_int_param(query, "page", 1, 1, 1_000_000),
|
||||||
@@ -422,7 +422,7 @@ border-color:#3a2c22;color:#f3e9dc}button{background:#e9b97c;color:#2a1a10}}
|
|||||||
sort=_param(query, "sort", "heat"),
|
sort=_param(query, "sort", "heat"),
|
||||||
within_days=_optional_int(query, "within_days"),
|
within_days=_optional_int(query, "within_days"),
|
||||||
year=_year_param(query),
|
year=_year_param(query),
|
||||||
growth_days=_int_param(query, "growth_days", 7, 1, 365),
|
growth_days=_growth_days_param(query),
|
||||||
folder_id=_optional_int(query, "folder"),
|
folder_id=_optional_int(query, "folder"),
|
||||||
q=_optional_str(query, "q"),
|
q=_optional_str(query, "q"),
|
||||||
page=_int_param(query, "page", 1, 1, 1_000_000),
|
page=_int_param(query, "page", 1, 1, 1_000_000),
|
||||||
@@ -649,6 +649,28 @@ def _year_param(query: Mapping[str, list[str]]) -> int | None:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _growth_days_param(query: Mapping[str, list[str]]) -> int | str:
|
||||||
|
"""榜单传 1~365;首页可传 auto(由查询层按 home_growth_days 解析)。"""
|
||||||
|
raw = _optional_str(query, "growth_days")
|
||||||
|
if raw is None:
|
||||||
|
return 7
|
||||||
|
if raw == "auto":
|
||||||
|
return "auto"
|
||||||
|
try:
|
||||||
|
value = int(raw)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ApiError(
|
||||||
|
HTTPStatus.BAD_REQUEST, "bad_request",
|
||||||
|
"growth_days 必须是整数或 auto",
|
||||||
|
) from exc
|
||||||
|
if value < 1 or value > 365:
|
||||||
|
raise ApiError(
|
||||||
|
HTTPStatus.BAD_REQUEST, "bad_request",
|
||||||
|
"growth_days 必须在 1~365 之间",
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
def _int_param(
|
def _int_param(
|
||||||
query: Mapping[str, list[str]],
|
query: Mapping[str, list[str]],
|
||||||
name: str,
|
name: str,
|
||||||
|
|||||||
@@ -510,6 +510,7 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo
|
|||||||
.status-grid strong { display: block; margin-top: 2px; font-size: 12.5px; font-weight: 600; }
|
.status-grid strong { display: block; margin-top: 2px; font-size: 12.5px; font-weight: 600; }
|
||||||
/* 预计下次运行等 3 项:PC 与移动端都保持一行 */
|
/* 预计下次运行等 3 项:PC 与移动端都保持一行 */
|
||||||
.status-grid.status-row-3 { grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
.status-grid.status-row-3 { grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
||||||
|
.status-grid.status-row-4 { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||||
|
|
||||||
.segmented {
|
.segmented {
|
||||||
display: inline-flex;
|
display: inline-flex;
|
||||||
@@ -993,4 +994,4 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo
|
|||||||
.about-page .panel h2 { font-size: 15px; margin-bottom: 8px; }
|
.about-page .panel h2 { font-size: 15px; margin-bottom: 8px; }
|
||||||
.about-list { margin: 0; padding-left: 18px; display: grid; gap: 6px; font-size: 13px; line-height: 1.55; }
|
.about-list { margin: 0; padding-left: 18px; display: grid; gap: 6px; font-size: 13px; line-height: 1.55; }
|
||||||
.about-list code { padding: 1px 4px; border-radius: 4px; background: var(--accent-soft); font-size: 12px; }
|
.about-list code { padding: 1px 4px; border-radius: 4px; background: var(--accent-soft); font-size: 12px; }
|
||||||
.about-list.about-weights { list-style: none; padding-left: 0; grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
.about-list.about-weights { list-style: none; padding-left: 0; grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||||
@@ -47,7 +47,8 @@ export async function renderHome(root) {
|
|||||||
api("/api/overview"),
|
api("/api/overview"),
|
||||||
api("/api/videos", { sort: "published", within_days: 14, limit: 6 }),
|
api("/api/videos", { sort: "published", within_days: 14, limit: 6 }),
|
||||||
api("/api/videos", { sort: "heat", limit: 6 }),
|
api("/api/videos", { sort: "heat", limit: 6 }),
|
||||||
api("/api/videos", { sort: "growth", growth_days: 7, limit: 6 }),
|
// auto:服务端按 analytics.home_growth_days 解析;默认先试 7 天再降到 1 天
|
||||||
|
api("/api/videos", { sort: "growth", growth_days: "auto", limit: 6 }),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const fans = overview.fans;
|
const fans = overview.fans;
|
||||||
@@ -59,6 +60,13 @@ export async function renderHome(root) {
|
|||||||
const pending = overview.counts?.candidates_pending || 0;
|
const pending = overview.counts?.candidates_pending || 0;
|
||||||
// 候选提示仅对已登录(或未启用鉴权)用户可见,访客不展示运行期信息。
|
// 候选提示仅对已登录(或未启用鉴权)用户可见,访客不展示运行期信息。
|
||||||
const showCandidates = pending > 0 && canAccessProtected();
|
const showCandidates = pending > 0 && canAccessProtected();
|
||||||
|
const growthDays = Number(growth.growth_days) || 1;
|
||||||
|
const growthLabel = `近 ${growthDays} 日`;
|
||||||
|
const growthEmptyTitle = growth.reason
|
||||||
|
|| `近 ${growthDays} 日增长暂不可用`;
|
||||||
|
const growthEmptyHint = growthDays <= 1
|
||||||
|
? "当前快照不足,完成至少 2 天快照后即可显示播放增量。"
|
||||||
|
: `需要至少 ${growthDays + 1} 天快照才能计算近 ${growthDays} 日增长。`;
|
||||||
|
|
||||||
root.innerHTML = `<section class="page home-page">
|
root.innerHTML = `<section class="page home-page">
|
||||||
<header class="page-head home-head">
|
<header class="page-head home-head">
|
||||||
@@ -101,12 +109,12 @@ export async function renderHome(root) {
|
|||||||
|
|
||||||
<section class="panel">
|
<section class="panel">
|
||||||
<div class="section-head">
|
<div class="section-head">
|
||||||
<h2>增长动态</h2>
|
<h2>增长动态 · ${escapeHtml(growthLabel)}</h2>
|
||||||
<a class="text-link" href="/videos?sort=growth&growth_days=7">增长榜</a>
|
<a class="text-link" href="/videos?sort=growth&growth_days=${growthDays}">增长榜</a>
|
||||||
</div>
|
</div>
|
||||||
${growth.available && growth.items.length
|
${growth.available && growth.items.length
|
||||||
? `<div class="media-grid">${growth.items.map((item, index) => growthCard(item, index)).join("")}</div>`
|
? `<div class="media-grid">${growth.items.map((item, index) => growthCard(item, index)).join("")}</div>`
|
||||||
: emptyState("需要至少 2 天快照", growth.reason || "当前只有 1 个数据点,明天开始显示播放增量。")}
|
: emptyState(growthEmptyTitle, growthEmptyHint)}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section class="panel">
|
<section class="panel">
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ export async function renderSystem(root) {
|
|||||||
const health = system.snapshot_health || { total: 0, ok: 0, failed: 0 };
|
const health = system.snapshot_health || { total: 0, ok: 0, failed: 0 };
|
||||||
const counts = system.counts || {};
|
const counts = system.counts || {};
|
||||||
const next = system.next_runs || {};
|
const next = system.next_runs || {};
|
||||||
|
const running = system.running || {};
|
||||||
|
|
||||||
const visitExtra = visitStats
|
const visitExtra = visitStats
|
||||||
? `<span class="section-note">近 ${visitStats.days} 天 · ${visitStats.pending ? "含未落库缓冲" : "已落库"}</span>`
|
? `<span class="section-note">近 ${visitStats.days} 天 · ${visitStats.pending ? "含未落库缓冲" : "已落库"}</span>`
|
||||||
@@ -92,12 +93,13 @@ export async function renderSystem(root) {
|
|||||||
|
|
||||||
<section class="panel">
|
<section class="panel">
|
||||||
${sectionHeader("预计下次运行")}
|
${sectionHeader("预计下次运行")}
|
||||||
<div class="status-grid status-row-3">
|
<div class="status-grid status-row-4">
|
||||||
<div><span>咻咻满粉丝数更新</span><strong>${running.fans ? "运行中" : escapeHtml(formatDateTime(next.fans))}</strong></div>
|
<div><span>咻咻满粉丝数更新</span><strong>${running.fans ? "运行中" : escapeHtml(formatDateTime(next.fans))}</strong></div>
|
||||||
<div><span>视频数据快照 + UP 信息</span><strong>${running.snapshot ? "运行中" : escapeHtml(formatDateTime(next.snapshot))}</strong></div>
|
<div><span>视频数据快照 + UP 信息</span><strong>${running.snapshot ? "运行中" : escapeHtml(formatDateTime(next.snapshot))}</strong></div>
|
||||||
|
<div><span>新视频小时同步</span><strong>${running.snapshot_recent ? "运行中" : escapeHtml(formatDateTime(next.snapshot_recent))}</strong></div>
|
||||||
<div><span>新视频搜索</span><strong>${running.discover || running.discovery ? "运行中" : escapeHtml(formatDateTime(next.discover))}</strong></div>
|
<div><span>新视频搜索</span><strong>${running.discover || running.discovery ? "运行中" : escapeHtml(formatDateTime(next.discover))}</strong></div>
|
||||||
</div>
|
</div>
|
||||||
<p class="section-note">进行中显示「运行中」;空闲时按该任务上次完成时间推算下次运行,不按墙钟抢跑。</p>
|
<p class="section-note">进行中显示「运行中」;空闲时按该任务上次完成时间推算下次运行,不按墙钟抢跑。新视频小时同步与全站快照互斥。</p>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section class="panel">
|
<section class="panel">
|
||||||
|
|||||||
@@ -139,6 +139,9 @@ def test_analytics_defaults_and_validation():
|
|||||||
assert cfg.analytics.min_view_for_ranking == 2000
|
assert cfg.analytics.min_view_for_ranking == 2000
|
||||||
assert cfg.analytics.video_shrink_minutes == 60
|
assert cfg.analytics.video_shrink_minutes == 60
|
||||||
assert cfg.analytics.shrink_minutes is None
|
assert cfg.analytics.shrink_minutes is None
|
||||||
|
assert cfg.analytics.home_growth_days == "auto"
|
||||||
|
assert cfg.schedule.recent_snapshot_minutes == 60
|
||||||
|
assert cfg.schedule.recent_video_days == 2
|
||||||
|
|
||||||
data = base_config()
|
data = base_config()
|
||||||
data["analytics"] = {
|
data["analytics"] = {
|
||||||
@@ -147,6 +150,14 @@ def test_analytics_defaults_and_validation():
|
|||||||
"min_view_for_ranking": 100,
|
"min_view_for_ranking": 100,
|
||||||
"video_shrink_minutes": 30,
|
"video_shrink_minutes": 30,
|
||||||
"shrink_minutes": 90,
|
"shrink_minutes": 90,
|
||||||
|
"home_growth_days": 1,
|
||||||
|
}
|
||||||
|
data["schedule"] = {
|
||||||
|
"discover_minutes": 30,
|
||||||
|
"snapshot_at": "00:01",
|
||||||
|
"candidate_ttl_days": 7,
|
||||||
|
"recent_snapshot_minutes": 15,
|
||||||
|
"recent_video_days": 3,
|
||||||
}
|
}
|
||||||
write(path, data)
|
write(path, data)
|
||||||
cfg = load_config(path, env={})
|
cfg = load_config(path, env={})
|
||||||
@@ -155,6 +166,14 @@ def test_analytics_defaults_and_validation():
|
|||||||
assert cfg.analytics.min_view_for_ranking == 100
|
assert cfg.analytics.min_view_for_ranking == 100
|
||||||
assert cfg.analytics.video_shrink_minutes == 30
|
assert cfg.analytics.video_shrink_minutes == 30
|
||||||
assert cfg.analytics.shrink_minutes == 90
|
assert cfg.analytics.shrink_minutes == 90
|
||||||
|
assert cfg.analytics.home_growth_days == 1
|
||||||
|
assert cfg.schedule.recent_snapshot_minutes == 15
|
||||||
|
assert cfg.schedule.recent_video_days == 3
|
||||||
|
|
||||||
|
data = base_config()
|
||||||
|
data["analytics"] = {"home_growth_days": 7}
|
||||||
|
write(path, data)
|
||||||
|
assert load_config(path, env={}).analytics.home_growth_days == 7
|
||||||
|
|
||||||
for bad, hint in (
|
for bad, hint in (
|
||||||
({"min_duration_seconds": -1}, "min_duration_seconds"),
|
({"min_duration_seconds": -1}, "min_duration_seconds"),
|
||||||
@@ -163,6 +182,9 @@ def test_analytics_defaults_and_validation():
|
|||||||
({"video_shrink_minutes": -1}, "video_shrink_minutes"),
|
({"video_shrink_minutes": -1}, "video_shrink_minutes"),
|
||||||
({"shrink_minutes": 0}, "shrink_minutes"),
|
({"shrink_minutes": 0}, "shrink_minutes"),
|
||||||
({"shrink_minutes": "auto"}, "shrink_minutes"),
|
({"shrink_minutes": "auto"}, "shrink_minutes"),
|
||||||
|
({"home_growth_days": 3}, "home_growth_days"),
|
||||||
|
({"home_growth_days": "7"}, "home_growth_days"),
|
||||||
|
({"home_growth_days": 0}, "home_growth_days"),
|
||||||
):
|
):
|
||||||
data = base_config()
|
data = base_config()
|
||||||
data["analytics"] = bad
|
data["analytics"] = bad
|
||||||
@@ -243,12 +265,38 @@ def test_web_track_defaults_and_validation():
|
|||||||
raise AssertionError("web.track 必须是对象")
|
raise AssertionError("web.track 必须是对象")
|
||||||
|
|
||||||
|
|
||||||
|
def test_schedule_recent_validation():
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "config.json"
|
||||||
|
for bad, hint in (
|
||||||
|
({"recent_snapshot_minutes": 0}, "recent_snapshot_minutes"),
|
||||||
|
({"recent_video_days": 0}, "recent_video_days"),
|
||||||
|
({"recent_snapshot_minutes": -5}, "recent_snapshot_minutes"),
|
||||||
|
):
|
||||||
|
data = base_config()
|
||||||
|
data["schedule"] = {
|
||||||
|
"discover_minutes": 60,
|
||||||
|
"snapshot_at": "00:01",
|
||||||
|
"candidate_ttl_days": 7,
|
||||||
|
**bad,
|
||||||
|
}
|
||||||
|
write(path, data)
|
||||||
|
try:
|
||||||
|
load_config(path, env={})
|
||||||
|
except ConfigError as exc:
|
||||||
|
assert hint in str(exc), (bad, str(exc))
|
||||||
|
else:
|
||||||
|
raise AssertionError(f"非法 schedule 必须拒绝: {bad}")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
test_dual_accounts_and_env_override()
|
test_dual_accounts_and_env_override()
|
||||||
test_flat_and_missing_account_rejected()
|
test_flat_and_missing_account_rejected()
|
||||||
test_validation_require_read_auth_and_hot_reload()
|
test_validation_require_read_auth_and_hot_reload()
|
||||||
test_invalid_values()
|
test_invalid_values()
|
||||||
test_analytics_defaults_and_validation()
|
test_analytics_defaults_and_validation()
|
||||||
|
test_schedule_recent_validation()
|
||||||
test_web_track_defaults_and_validation()
|
test_web_track_defaults_and_validation()
|
||||||
print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖、热加载与 analytics/web.track 测试全部通过")
|
print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖、热加载与 analytics/web.track 测试全部通过")
|
||||||
|
|
||||||
|
|||||||
@@ -436,6 +436,100 @@ def test_backup_failure_retries_without_api():
|
|||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_recent_videos_refreshes_without_completion_markers():
|
||||||
|
"""小时任务只刷新近窗新片,不写完成日 / 备份。"""
|
||||||
|
clock = 1_800_000_000
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
transport = SnapshotTransport({
|
||||||
|
1: view_for(1, view=111),
|
||||||
|
2: view_for(2, view=222),
|
||||||
|
3: view_for(3, view=333),
|
||||||
|
})
|
||||||
|
db, repo, service = make_service(tmp, transport, track_fans=())
|
||||||
|
try:
|
||||||
|
# 1/2 在墙钟近 2 天内;3 为旧片
|
||||||
|
add_video(repo, 1)
|
||||||
|
add_video(repo, 2)
|
||||||
|
add_video(repo, 3)
|
||||||
|
repo.db.write(lambda conn: conn.executemany(
|
||||||
|
"UPDATE video SET pubdate = ? WHERE aid = ?",
|
||||||
|
[
|
||||||
|
(clock - 3600, 1),
|
||||||
|
(clock - 86400, 2),
|
||||||
|
(clock - 10 * 86400, 3),
|
||||||
|
],
|
||||||
|
))
|
||||||
|
# 预置同日 ok=1,验证小时任务可覆盖刷新
|
||||||
|
repo.upsert_snapshot(
|
||||||
|
DATE, 1,
|
||||||
|
metrics={"view": 1, "like": 0, "coin": 0, "favorite": 0,
|
||||||
|
"share": 0, "reply": 0, "danmaku": 0},
|
||||||
|
ok=True, err=None, captured_at=clock - 100,
|
||||||
|
)
|
||||||
|
assert repo.kv_get(service.completed_key(DATE)) is None
|
||||||
|
|
||||||
|
summary = service.run_recent_videos(date=DATE, recent_days=2)
|
||||||
|
assert summary.total == 2
|
||||||
|
assert summary.success == 2
|
||||||
|
assert summary.failed == 0
|
||||||
|
assert set(repo.video_aids_recent_for_snapshot(
|
||||||
|
since_pubdate=clock - 2 * 86400
|
||||||
|
)) == {1, 2}
|
||||||
|
|
||||||
|
row = repo.db.read(lambda conn: dict(conn.execute(
|
||||||
|
"SELECT view, ok, captured_at FROM snapshot WHERE stat_date=? AND aid=1",
|
||||||
|
(DATE,),
|
||||||
|
).fetchone()))
|
||||||
|
assert row["view"] == 111
|
||||||
|
assert row["ok"] == 1
|
||||||
|
assert row["captured_at"] == clock
|
||||||
|
|
||||||
|
# 完成日 / 备份标记保持原样
|
||||||
|
assert repo.kv_get(service.completed_key(DATE)) is None
|
||||||
|
assert repo.kv_get("snapshot:last_completed") is None
|
||||||
|
assert repo.kv_get(service.backup_key(DATE)) is None
|
||||||
|
assert transport.count("/web-interface/view") == 2
|
||||||
|
|
||||||
|
# 全量任务随后才写完成日;且已 ok=1 的跳过
|
||||||
|
full = service.run_videos(date=DATE)
|
||||||
|
assert full.skipped >= 2 # 1、2 已被小时任务写过
|
||||||
|
assert full.success >= 1 # 至少旧片 3
|
||||||
|
assert repo.kv_get(service.completed_key(DATE)) == "1"
|
||||||
|
assert repo.kv_get("snapshot:last_completed") == DATE
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_full_snapshot_skips_ok_rows_including_hourly_refresh():
|
||||||
|
"""00:01 全量:同日 ok=1(含小时任务刚刷)直接跳过。"""
|
||||||
|
clock = 1_800_000_000
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
transport = SnapshotTransport({1: view_for(1), 2: view_for(2)})
|
||||||
|
db, repo, service = make_service(tmp, transport, track_fans=())
|
||||||
|
try:
|
||||||
|
add_video(repo, 1)
|
||||||
|
add_video(repo, 2)
|
||||||
|
repo.upsert_snapshot(
|
||||||
|
DATE, 1,
|
||||||
|
metrics={"view": 9, "like": 0, "coin": 0, "favorite": 0,
|
||||||
|
"share": 0, "reply": 0, "danmaku": 0},
|
||||||
|
ok=True, err=None, captured_at=clock - 30, # 「1 小时内」
|
||||||
|
)
|
||||||
|
result = service.run_videos(date=DATE)
|
||||||
|
assert result.skipped == 1
|
||||||
|
assert result.success == 1
|
||||||
|
# aid=1 保持小时任务写入的 view=9,未被全量覆盖
|
||||||
|
row = repo.db.read(lambda conn: dict(conn.execute(
|
||||||
|
"SELECT view FROM snapshot WHERE stat_date=? AND aid=1",
|
||||||
|
(DATE,),
|
||||||
|
).fetchone()))
|
||||||
|
assert row["view"] == 9
|
||||||
|
assert transport.count("/web-interface/view") == 1
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
test_snapshot_success_resume_and_force()
|
test_snapshot_success_resume_and_force()
|
||||||
test_resume_skips_ok_and_retries_failed()
|
test_resume_skips_ok_and_retries_failed()
|
||||||
@@ -449,6 +543,8 @@ def main():
|
|||||||
test_fan_resume_preserves_successful_value()
|
test_fan_resume_preserves_successful_value()
|
||||||
test_force_interrupt_leaves_incomplete_marker()
|
test_force_interrupt_leaves_incomplete_marker()
|
||||||
test_backup_failure_retries_without_api()
|
test_backup_failure_retries_without_api()
|
||||||
|
test_run_recent_videos_refreshes_without_completion_markers()
|
||||||
|
test_full_snapshot_skips_ok_rows_including_hourly_refresh()
|
||||||
print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过")
|
print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -124,7 +124,8 @@ def test_growth_and_cold_start():
|
|||||||
cold = q.list_videos(sort="growth", growth_days=2)
|
cold = q.list_videos(sort="growth", growth_days=2)
|
||||||
assert cold["available"] is False
|
assert cold["available"] is False
|
||||||
assert cold["items"] == []
|
assert cold["items"] == []
|
||||||
assert "两天" in cold["reason"]
|
assert "至少 3 天快照" in cold["reason"]
|
||||||
|
assert cold["growth_days"] == 2
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
@@ -361,10 +362,15 @@ def test_fans_candidates_system_and_export():
|
|||||||
assert system["errors"] == []
|
assert system["errors"] == []
|
||||||
assert system["preview"] is False
|
assert system["preview"] is False
|
||||||
assert system["running"] == {}
|
assert system["running"] == {}
|
||||||
assert set(system["next_runs"]) >= {"fans", "snapshot", "discover"}
|
assert set(system["next_runs"]) >= {
|
||||||
|
"fans", "snapshot", "discover", "snapshot_recent",
|
||||||
|
}
|
||||||
assert system["next_runs"]["snapshot"] > 0
|
assert system["next_runs"]["snapshot"] > 0
|
||||||
assert system["next_runs"]["fans"] > 0
|
assert system["next_runs"]["fans"] > 0
|
||||||
assert system["next_runs"]["discover"] > 0
|
assert system["next_runs"]["discover"] > 0
|
||||||
|
assert system["next_runs"]["snapshot_recent"] > 0
|
||||||
|
assert system["schedule"]["recent_snapshot_minutes"] == 60
|
||||||
|
assert system["schedule"]["recent_video_days"] == 2
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
@@ -483,6 +489,7 @@ def test_about_payload():
|
|||||||
"min_view_for_ranking": 2000,
|
"min_view_for_ranking": 2000,
|
||||||
"video_shrink_minutes": 60,
|
"video_shrink_minutes": 60,
|
||||||
"shrink_minutes": None,
|
"shrink_minutes": None,
|
||||||
|
"home_growth_days": "auto",
|
||||||
}
|
}
|
||||||
assert about["heat_weights"]["view"] == 0.30
|
assert about["heat_weights"]["view"] == 0.30
|
||||||
assert about["followers"]["videos_total"] == 3
|
assert about["followers"]["videos_total"] == 3
|
||||||
@@ -553,6 +560,65 @@ def test_stats_query():
|
|||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_growth_auto_falls_back_to_one_day():
|
||||||
|
"""3 天快照撑不起 7 日增长;auto 应降到 1 日。"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db, q = setup(tmp)
|
||||||
|
try:
|
||||||
|
auto = q.list_videos(sort="growth", growth_days="auto", page_size=6)
|
||||||
|
assert auto["available"] is True
|
||||||
|
assert auto["growth_days"] == 1
|
||||||
|
assert {item["aid"] for item in auto["items"]} == {101, 102}
|
||||||
|
first = next(item for item in auto["items"] if item["aid"] == 101)
|
||||||
|
assert first["growth"]["delta"] == 100 # 相邻日
|
||||||
|
|
||||||
|
forced7 = q.list_videos(sort="growth", growth_days=7)
|
||||||
|
assert forced7["available"] is False
|
||||||
|
assert "至少 8 天快照" in forced7["reason"]
|
||||||
|
|
||||||
|
# 配置钉死 7 天:auto 不再降级
|
||||||
|
path = Path(tmp) / "config.json"
|
||||||
|
data = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
data.setdefault("analytics", {})["home_growth_days"] = 7
|
||||||
|
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||||
|
target = max(path.stat().st_mtime_ns, time.time_ns()) + 1_000_000
|
||||||
|
os.utime(path, ns=(target, target))
|
||||||
|
DASHBOARD_CACHE.clear()
|
||||||
|
pinned = q.list_videos(sort="growth", growth_days="auto")
|
||||||
|
assert pinned["growth_days"] == 7
|
||||||
|
assert pinned["available"] is False
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_snapshot_recent_next_run_follows_completion():
|
||||||
|
import json
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db, q = setup(tmp)
|
||||||
|
try:
|
||||||
|
repo = Repository(db)
|
||||||
|
finished = FIXED_NOW - 10
|
||||||
|
repo.kv_set(
|
||||||
|
"task:last:snapshot_recent",
|
||||||
|
json.dumps({
|
||||||
|
"name": "snapshot_recent", "ok": True,
|
||||||
|
"started_at": finished - 5, "finished_at": finished,
|
||||||
|
"error": None, "summary": {},
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
DASHBOARD_CACHE.clear()
|
||||||
|
system = q.system()
|
||||||
|
assert system["next_runs"]["snapshot_recent"] == finished + 3600
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
def main_test():
|
def main_test():
|
||||||
test_overview()
|
test_overview()
|
||||||
test_video_listing_filters_and_sorting()
|
test_video_listing_filters_and_sorting()
|
||||||
@@ -568,6 +634,8 @@ def main_test():
|
|||||||
test_fans_candidates_system_and_export()
|
test_fans_candidates_system_and_export()
|
||||||
test_system_status_uses_completed_date_and_running_flags()
|
test_system_status_uses_completed_date_and_running_flags()
|
||||||
test_system_next_runs_follow_last_completion_not_wall_clock()
|
test_system_next_runs_follow_last_completion_not_wall_clock()
|
||||||
|
test_growth_auto_falls_back_to_one_day()
|
||||||
|
test_snapshot_recent_next_run_follows_completion()
|
||||||
print("OK: Web 查询层总览/榜单/详情/UP/时长效率/粉丝/候选/系统/访问统计测试全部通过")
|
print("OK: Web 查询层总览/榜单/详情/UP/时长效率/粉丝/候选/系统/访问统计测试全部通过")
|
||||||
|
|
||||||
|
|
||||||
@@ -605,7 +673,9 @@ def test_system_status_uses_completed_date_and_running_flags():
|
|||||||
system = q.system()
|
system = q.system()
|
||||||
assert "snapshot" in system["running"]
|
assert "snapshot" in system["running"]
|
||||||
assert system["running"]["snapshot"]["running"] is True
|
assert system["running"]["snapshot"]["running"] is True
|
||||||
assert set(system["next_runs"]) >= {"fans", "snapshot", "discover"}
|
assert set(system["next_runs"]) >= {
|
||||||
|
"fans", "snapshot", "discover", "snapshot_recent",
|
||||||
|
}
|
||||||
|
|
||||||
# 失败记录保留,完成日不变
|
# 失败记录保留,完成日不变
|
||||||
repo.kv_set(
|
repo.kv_set(
|
||||||
|
|||||||
@@ -50,6 +50,8 @@ def write_config(
|
|||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "00:01",
|
"snapshot_at": "00:01",
|
||||||
"candidate_ttl_days": 7,
|
"candidate_ttl_days": 7,
|
||||||
|
"recent_snapshot_minutes": 60,
|
||||||
|
"recent_video_days": 2,
|
||||||
},
|
},
|
||||||
}, ensure_ascii=False, indent=2), encoding="utf-8")
|
}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
return path
|
return path
|
||||||
|
|||||||
Reference in new issue
Block a user