1103 lines
45 KiB
Python
1103 lines
45 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""只读看板查询层。
|
|
|
|
所有 SQL 都在这里集中，Web 层只负责参数解析与 JSON 序列化。查询不会写入
|
|
数据库，也不会访问 B 站。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import sqlite3
|
|
import time
|
|
from datetime import date, timedelta
|
|
from typing import Any, Iterable, Mapping, Sequence
|
|
|
|
from ..analytics import (
|
|
DEFAULT_HEAT_WEIGHTS,
|
|
METRICS,
|
|
MIN_VIEW_FOR_GROWTH_RATE,
|
|
compute_growth,
|
|
heat_scores,
|
|
interaction_rate,
|
|
normalize_cover,
|
|
score_percentiles,
|
|
)
|
|
from ..config import ConfigStore
|
|
from ..database import Database
|
|
from ..timeutil import day_bounds, next_daily_run, shanghai_date, shanghai_datetime
|
|
|
|
TRACKED_STATUSES = ("active", "unfaved")
|
|
VIDEO_SORTS = {
|
|
"heat", "view", "like", "coin", "favorite", "share", "reply", "danmaku",
|
|
"interaction", "growth", "growth_rate", "added", "published",
|
|
}
|
|
UP_SORTS = {"views", "count", "avg_views", "interaction", "avg_interaction", "latest"}
|
|
CAPACITY = 1000
|
|
CAPACITY_WARNING = 950
|
|
TRACK_FANS_NAMES = {37754047: "咻咻满"}
|
|
|
|
|
|
def _chunks(values: Sequence[int], size: int = 400) -> Iterable[Sequence[int]]:
|
|
for index in range(0, len(values), size):
|
|
yield values[index:index + size]
|
|
|
|
|
|
def _date_minus(day: str, days: int) -> str:
|
|
return (date.fromisoformat(day) - timedelta(days=int(days))).isoformat()
|
|
|
|
|
|
def _date_range(start: str, end: str) -> list[str]:
|
|
first = date.fromisoformat(start)
|
|
last = date.fromisoformat(end)
|
|
if last < first:
|
|
return []
|
|
return [
|
|
(first + timedelta(days=offset)).isoformat()
|
|
for offset in range((last - first).days + 1)
|
|
]
|
|
|
|
|
|
def _ts_date(timestamp: int | float | None) -> str | None:
|
|
if timestamp is None:
|
|
return None
|
|
return shanghai_date(float(timestamp))
|
|
|
|
|
|
def _like_pattern(value: str) -> str:
|
|
escaped = value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
return f"%{escaped}%"
|
|
|
|
|
|
def _metric_map(row: Mapping[str, Any] | None) -> dict[str, int] | None:
|
|
if row is None:
|
|
return None
|
|
result: dict[str, int] = {}
|
|
for metric in METRICS:
|
|
value = row.get(metric)
|
|
result[metric] = 0 if value is None else int(value)
|
|
return result
|
|
|
|
|
|
def _safe_float(value: Any) -> float | None:
|
|
if value is None or isinstance(value, bool):
|
|
return None
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
class DashboardQueries:
|
|
def __init__(
|
|
self,
|
|
db: Database,
|
|
config_store: ConfigStore,
|
|
*,
|
|
clock=time.time,
|
|
) -> None:
|
|
self.db = db
|
|
self.config_store = config_store
|
|
self._clock = clock
|
|
|
|
# ------------------------------------------------------------ helpers
|
|
|
|
def _now(self) -> float:
|
|
return float(self._clock())
|
|
|
|
def _today(self) -> str:
|
|
return shanghai_date(self._now())
|
|
|
|
@staticmethod
|
|
def _latest_date(conn: sqlite3.Connection) -> str | None:
|
|
row = conn.execute(
|
|
"SELECT MAX(stat_date) AS d FROM snapshot WHERE ok = 1"
|
|
).fetchone()
|
|
return None if row is None or row["d"] is None else str(row["d"])
|
|
|
|
@staticmethod
|
|
def _snapshot_days(conn: sqlite3.Connection) -> int:
|
|
row = conn.execute(
|
|
"SELECT COUNT(DISTINCT stat_date) AS n FROM snapshot WHERE ok = 1"
|
|
).fetchone()
|
|
return int(row["n"] or 0)
|
|
|
|
@staticmethod
|
|
def _snapshot_map(
|
|
conn: sqlite3.Connection,
|
|
aids: Sequence[int],
|
|
*,
|
|
before: str | None = None,
|
|
before_inclusive: bool = False,
|
|
) -> dict[int, dict[str, Any]]:
|
|
if not aids:
|
|
return {}
|
|
result: dict[int, dict[str, Any]] = {}
|
|
for chunk in _chunks(list(aids)):
|
|
placeholders = ",".join("?" for _ in chunk)
|
|
if before is None:
|
|
sql = f"""
|
|
SELECT s.* FROM snapshot s
|
|
JOIN (
|
|
SELECT aid, MAX(stat_date) AS stat_date
|
|
FROM snapshot WHERE ok = 1 AND aid IN ({placeholders})
|
|
GROUP BY aid
|
|
) m ON m.aid = s.aid AND m.stat_date = s.stat_date
|
|
"""
|
|
params: list[Any] = list(chunk)
|
|
else:
|
|
operator = "<=" if before_inclusive else "<"
|
|
sql = f"""
|
|
SELECT s.* FROM snapshot s
|
|
JOIN (
|
|
SELECT aid, MAX(stat_date) AS stat_date
|
|
FROM snapshot
|
|
WHERE ok = 1 AND aid IN ({placeholders}) AND stat_date {operator} ?
|
|
GROUP BY aid
|
|
) m ON m.aid = s.aid AND m.stat_date = s.stat_date
|
|
"""
|
|
params = [*chunk, before]
|
|
for row in conn.execute(sql, params).fetchall():
|
|
data = dict(row)
|
|
result[int(data["aid"])] = data
|
|
return result
|
|
|
|
@staticmethod
|
|
def _videos(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
statuses: Sequence[str] = TRACKED_STATUSES,
|
|
folder_id: int | None = None,
|
|
q: str | None = None,
|
|
within_field: str | None = None,
|
|
within_start: str | None = None,
|
|
up_mid: int | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
clauses = [f"v.status IN ({','.join('?' for _ in statuses)})"]
|
|
params: list[Any] = list(statuses)
|
|
if folder_id is not None:
|
|
clauses.append("v.folder_id = ?")
|
|
params.append(int(folder_id))
|
|
if up_mid is not None:
|
|
clauses.append("v.up_mid = ?")
|
|
params.append(int(up_mid))
|
|
if q:
|
|
pattern = _like_pattern(q)
|
|
clauses.append(
|
|
"(v.title LIKE ? ESCAPE '\\' OR v.up_name LIKE ? ESCAPE '\\')"
|
|
)
|
|
params.extend([pattern, pattern])
|
|
if within_field and within_start:
|
|
if within_field not in {"pubdate", "first_seen"}:
|
|
raise ValueError("within_field 必须是 pubdate 或 first_seen")
|
|
start_ts, _ = day_bounds(within_start)
|
|
clauses.append(f"v.{within_field} >= ?")
|
|
params.append(start_ts)
|
|
sql = (
|
|
"SELECT v.*, f.title AS folder_title, f.year AS folder_year "
|
|
"FROM video v LEFT JOIN folder f ON f.media_id = v.folder_id "
|
|
f"WHERE {' AND '.join(clauses)} ORDER BY v.aid"
|
|
)
|
|
return [dict(row) for row in conn.execute(sql, params).fetchall()]
|
|
|
|
def _global_heat(
|
|
self, conn: sqlite3.Connection
|
|
) -> tuple[list[dict[str, Any]], dict[int, dict[str, Any]], dict[int, float], dict[int, float]]:
|
|
videos = self._videos(conn)
|
|
snapshots = self._snapshot_map(conn, [int(v["aid"]) for v in videos])
|
|
rows = []
|
|
for video in videos:
|
|
snapshot = snapshots.get(int(video["aid"]))
|
|
if snapshot is None:
|
|
continue
|
|
item = {"aid": int(video["aid"])}
|
|
item.update(_metric_map(snapshot) or {})
|
|
rows.append(item)
|
|
weights = self.config_store.get().heat_weights
|
|
scores = heat_scores(rows, weights or DEFAULT_HEAT_WEIGHTS)
|
|
percentiles = score_percentiles(scores)
|
|
return videos, snapshots, scores, percentiles
|
|
|
|
@staticmethod
|
|
def _growth_for(
|
|
snapshot: Mapping[str, Any] | None,
|
|
previous: Mapping[str, Any] | None,
|
|
metric: str = "view",
|
|
):
|
|
if snapshot is None:
|
|
return compute_growth(None, None, None)
|
|
current = snapshot.get(metric)
|
|
before = None if previous is None else previous.get(metric)
|
|
gap = None
|
|
if previous is not None and snapshot.get("stat_date") and previous.get("stat_date"):
|
|
try:
|
|
gap = (
|
|
date.fromisoformat(str(snapshot["stat_date"]))
|
|
- date.fromisoformat(str(previous["stat_date"]))
|
|
).days
|
|
except ValueError:
|
|
gap = None
|
|
return compute_growth(current, before, gap)
|
|
|
|
def _item(
|
|
self,
|
|
video: Mapping[str, Any],
|
|
snapshot: Mapping[str, Any] | None,
|
|
*,
|
|
heat: float | None = None,
|
|
percentile: float | None = None,
|
|
previous: Mapping[str, Any] | None = None,
|
|
include_growth: bool = False,
|
|
) -> dict[str, Any]:
|
|
metrics = _metric_map(snapshot)
|
|
item: dict[str, Any] = {
|
|
"aid": int(video["aid"]),
|
|
"bvid": video.get("bvid"),
|
|
"title": video.get("title") or "(无标题)",
|
|
"up_mid": video.get("up_mid"),
|
|
"up_name": video.get("up_name"),
|
|
"pubdate": int(video.get("pubdate") or 0),
|
|
"duration": video.get("duration"),
|
|
"cover": normalize_cover(video.get("cover")),
|
|
"folder_id": video.get("folder_id"),
|
|
"folder_title": video.get("folder_title"),
|
|
"folder_year": video.get("folder_year"),
|
|
"status": video.get("status"),
|
|
"first_seen": int(video.get("first_seen") or 0),
|
|
"attr": video.get("attr"),
|
|
"metrics": metrics,
|
|
"snapshot_date": None if snapshot is None else snapshot.get("stat_date"),
|
|
"heat": None if heat is None else round(float(heat), 3),
|
|
"heat_percentile": percentile,
|
|
"interaction_rate": None if metrics is None else interaction_rate(metrics),
|
|
}
|
|
if include_growth:
|
|
item["growth"] = self._growth_for(snapshot, previous).as_dict()
|
|
return item
|
|
|
|
# ------------------------------------------------------------ overview
|
|
|
|
def overview(self) -> dict[str, Any]:
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
|
latest = self._latest_date(conn)
|
|
snapshot_days = self._snapshot_days(conn)
|
|
counts_rows = conn.execute(
|
|
"SELECT status, COUNT(*) AS n FROM video GROUP BY status"
|
|
).fetchall()
|
|
by_status = {str(row["status"]): int(row["n"]) for row in counts_rows}
|
|
videos = self._videos(conn)
|
|
aids = [int(video["aid"]) for video in videos]
|
|
snapshots = self._snapshot_map(conn, aids)
|
|
previous = self._snapshot_map(conn, aids, before=latest or self._today())
|
|
totals = {metric: 0 for metric in METRICS}
|
|
for video in videos:
|
|
metrics = _metric_map(snapshots.get(int(video["aid"])))
|
|
if metrics:
|
|
for metric in METRICS:
|
|
totals[metric] += metrics[metric]
|
|
|
|
view_delta: int | None = None
|
|
available_deltas = 0
|
|
view_delta_sum = 0
|
|
for video in videos:
|
|
aid = int(video["aid"])
|
|
snapshot = snapshots.get(aid)
|
|
prior = previous.get(aid)
|
|
if snapshot is None or prior is None:
|
|
continue
|
|
growth = self._growth_for(snapshot, prior)
|
|
if growth.available and growth.delta is not None:
|
|
view_delta_sum += growth.delta
|
|
available_deltas += 1
|
|
if available_deltas:
|
|
view_delta = view_delta_sum
|
|
|
|
first_seen_row = conn.execute(
|
|
"SELECT MIN(first_seen) AS ts FROM video"
|
|
).fetchone()
|
|
first_seen_ts = first_seen_row["ts"] if first_seen_row else None
|
|
first_seen_date = _ts_date(first_seen_ts)
|
|
initial_import_count = 0
|
|
if first_seen_ts is not None:
|
|
initial_import_count = int(conn.execute(
|
|
"SELECT COUNT(*) FROM video WHERE first_seen = ? "
|
|
"AND status IN ('active','unfaved')",
|
|
(int(first_seen_ts),),
|
|
).fetchone()[0])
|
|
|
|
start_ts, end_ts = day_bounds(self._today())
|
|
added_today = int(conn.execute(
|
|
"SELECT COUNT(*) FROM video WHERE first_seen >= ? AND first_seen < ?",
|
|
(start_ts, end_ts),
|
|
).fetchone()[0])
|
|
is_first_day = snapshot_days <= 1
|
|
if is_first_day:
|
|
added_today = 0
|
|
|
|
snapshot_health = {"total": 0, "ok": 0, "failed": 0}
|
|
if latest:
|
|
rows = conn.execute(
|
|
"SELECT ok, COUNT(*) AS n FROM snapshot WHERE stat_date = ? GROUP BY ok",
|
|
(latest,),
|
|
).fetchall()
|
|
by_ok = {int(row["ok"]): int(row["n"]) for row in rows}
|
|
snapshot_health = {
|
|
"total": sum(by_ok.values()),
|
|
"ok": by_ok.get(1, 0),
|
|
"failed": by_ok.get(0, 0),
|
|
}
|
|
|
|
candidate_pending = int(conn.execute(
|
|
"SELECT COUNT(*) FROM candidate WHERE decision IS NULL"
|
|
).fetchone()[0])
|
|
candidate_total = int(conn.execute(
|
|
"SELECT COUNT(*) FROM candidate"
|
|
).fetchone()[0])
|
|
|
|
cfg = self.config_store.get()
|
|
fans = self._fans_overview(conn, cfg.track_fans, latest, snapshot_days)
|
|
folders = self._folders_overview(conn)
|
|
tasks = self._task_records(conn)
|
|
auth = {
|
|
"ok": self._kv(conn, "auth:ok"),
|
|
"message": self._kv(conn, "auth:message"),
|
|
"last_check": self._kv(conn, "auth:last_check"),
|
|
"mid": self._kv(conn, "auth:mid"),
|
|
}
|
|
preview = self._kv(conn, "preview:seeded") == "1"
|
|
snapshot_completed_today = self._kv(
|
|
conn, f"snapshot:completed:{self._today()}"
|
|
) == "1"
|
|
return {
|
|
"generated_at": int(self._now()),
|
|
"today": self._today(),
|
|
"data_date": latest,
|
|
"snapshot_days": snapshot_days,
|
|
"first_seen_date": first_seen_date,
|
|
"is_first_day": is_first_day,
|
|
"initial_import_count": initial_import_count,
|
|
"added_today": added_today,
|
|
"counts": {
|
|
**by_status,
|
|
"active_ups": int(conn.execute(
|
|
"SELECT COUNT(DISTINCT up_mid) FROM video "
|
|
"WHERE status IN ('active','unfaved') AND up_mid IS NOT NULL"
|
|
).fetchone()[0]),
|
|
"candidates_pending": candidate_pending,
|
|
"candidates_total": candidate_total,
|
|
},
|
|
"totals": totals,
|
|
"interactions": totals["like"] + totals["coin"] + totals["favorite"],
|
|
"view_delta": view_delta,
|
|
"view_delta_available": view_delta is not None,
|
|
"snapshot_health": snapshot_health,
|
|
"snapshot_completed_today": snapshot_completed_today,
|
|
"fans": fans,
|
|
"folders": folders,
|
|
"tasks": tasks,
|
|
"auth": auth,
|
|
"preview": preview,
|
|
"capacity": CAPACITY,
|
|
"capacity_warning": CAPACITY_WARNING,
|
|
}
|
|
return self.db.read(run)
|
|
|
|
@staticmethod
|
|
def _kv(conn: sqlite3.Connection, key: str) -> str | None:
|
|
row = conn.execute("SELECT v FROM kv WHERE k = ?", (key,)).fetchone()
|
|
return None if row is None else str(row["v"])
|
|
|
|
def _task_records(self, conn: sqlite3.Connection) -> dict[str, Any]:
|
|
rows = conn.execute(
|
|
"SELECT k, v FROM kv WHERE k LIKE 'task:last:%' ORDER BY k"
|
|
).fetchall()
|
|
result: dict[str, Any] = {}
|
|
for row in rows:
|
|
name = str(row["k"]).removeprefix("task:last:")
|
|
try:
|
|
value = json.loads(str(row["v"]))
|
|
except (TypeError, json.JSONDecodeError):
|
|
value = {"raw": row["v"]}
|
|
result[name] = value
|
|
return result
|
|
|
|
def _folders_overview(self, conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT f.*,
|
|
(SELECT COUNT(*) FROM folder_resource fr
|
|
WHERE fr.media_id = f.media_id AND fr.active = 1) AS local_count
|
|
FROM folder f ORDER BY f.year DESC, f.media_id DESC
|
|
"""
|
|
).fetchall()
|
|
result = []
|
|
for row in rows:
|
|
media_count = row["media_count"]
|
|
count = int(media_count) if media_count is not None else int(row["local_count"] or 0)
|
|
result.append({
|
|
"media_id": int(row["media_id"]),
|
|
"year": int(row["year"]),
|
|
"title": row["title"] or "",
|
|
"media_count": count,
|
|
"local_count": int(row["local_count"] or 0),
|
|
"last_ids_at": row["last_ids_at"],
|
|
"last_full_at": row["last_full_at"],
|
|
"enabled": bool(row["enabled"]),
|
|
"capacity": CAPACITY,
|
|
"ratio": min(1.0, count / CAPACITY),
|
|
"warning": count >= CAPACITY_WARNING,
|
|
})
|
|
return result
|
|
|
|
def _fans_overview(
|
|
self,
|
|
conn: sqlite3.Connection,
|
|
mid_list: Sequence[int],
|
|
latest: str | None,
|
|
snapshot_days: int,
|
|
) -> dict[str, Any] | None:
|
|
mids = list(mid_list) or [
|
|
int(row["mid"])
|
|
for row in conn.execute("SELECT DISTINCT mid FROM up_stat ORDER BY mid").fetchall()
|
|
]
|
|
if not mids:
|
|
return None
|
|
mid = int(mids[0])
|
|
rows = conn.execute(
|
|
"SELECT * FROM up_stat WHERE mid = ? ORDER BY stat_date", (mid,)
|
|
).fetchall()
|
|
if not rows:
|
|
return None
|
|
ok_rows = [dict(row) for row in rows if int(row["ok"]) == 1]
|
|
if not ok_rows:
|
|
return None
|
|
current = ok_rows[-1]
|
|
previous = ok_rows[-2] if len(ok_rows) >= 2 else None
|
|
baseline = None
|
|
if latest:
|
|
cutoff = _date_minus(latest, 7)
|
|
candidates = [row for row in ok_rows if str(row["stat_date"]) <= cutoff]
|
|
baseline = candidates[-1] if candidates else None
|
|
first = ok_rows[0]
|
|
return {
|
|
"mid": mid,
|
|
"name": TRACK_FANS_NAMES.get(mid, f"UP {mid}"),
|
|
"stat_date": current["stat_date"],
|
|
"follower": current["follower"],
|
|
"following": current["following"],
|
|
"delta_1d": None if previous is None else int(current["follower"] or 0) - int(previous["follower"] or 0),
|
|
"delta_7d": None if baseline is None else int(current["follower"] or 0) - int(baseline["follower"] or 0),
|
|
"delta_cum": int(current["follower"] or 0) - int(first["follower"] or 0),
|
|
"first_date": first["stat_date"],
|
|
"snapshot_days": snapshot_days,
|
|
"points": [
|
|
{
|
|
"date": row["stat_date"],
|
|
"follower": row["follower"] if int(row["ok"]) == 1 else None,
|
|
"ok": bool(row["ok"]),
|
|
}
|
|
for row in rows[-30:]
|
|
],
|
|
}
|
|
|
|
# ------------------------------------------------------------ videos
|
|
|
|
def list_videos(
|
|
self,
|
|
*,
|
|
sort: str = "heat",
|
|
within_days: int | None = None,
|
|
growth_days: int = 7,
|
|
folder_id: int | None = None,
|
|
q: str | None = None,
|
|
page: int = 1,
|
|
page_size: int = 20,
|
|
export: bool = False,
|
|
) -> dict[str, Any]:
|
|
if sort not in VIDEO_SORTS:
|
|
raise ValueError(f"不支持的排序: {sort}")
|
|
if growth_days < 1 or growth_days > 365:
|
|
raise ValueError("growth_days 必须在 1~365 之间")
|
|
page = max(1, int(page))
|
|
page_size = max(1, min(int(page_size), 200 if not export else 10_000))
|
|
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
|
latest = self._latest_date(conn) or self._today()
|
|
within_field = None
|
|
if within_days is not None:
|
|
if within_days < 1:
|
|
raise ValueError("within_days 必须大于 0")
|
|
within_field = "first_seen" if sort == "added" else "pubdate"
|
|
videos = self._videos(
|
|
conn,
|
|
folder_id=folder_id,
|
|
q=q,
|
|
within_field=within_field,
|
|
within_start=(
|
|
_date_minus(latest, int(within_days) - 1)
|
|
if within_days is not None else None
|
|
),
|
|
)
|
|
aids = [int(video["aid"]) for video in videos]
|
|
snapshots = self._snapshot_map(conn, aids)
|
|
_, _, heat, percentiles = self._global_heat(conn)
|
|
|
|
previous: dict[int, dict[str, Any]] = {}
|
|
if sort in {"growth", "growth_rate", "interaction"}:
|
|
baseline_date = _date_minus(latest, growth_days)
|
|
previous = self._snapshot_map(conn, aids, before=baseline_date, before_inclusive=True)
|
|
elif sort in {"heat", "view", "like", "coin", "favorite", "share", "reply", "danmaku"}:
|
|
previous = {}
|
|
|
|
items = [
|
|
self._item(
|
|
video,
|
|
snapshots.get(int(video["aid"])),
|
|
heat=heat.get(int(video["aid"])),
|
|
percentile=percentiles.get(int(video["aid"])),
|
|
previous=previous.get(int(video["aid"])),
|
|
include_growth=sort in {"growth", "growth_rate"},
|
|
)
|
|
for video in videos
|
|
]
|
|
|
|
if sort == "interaction":
|
|
items = [item for item in items if item["interaction_rate"] is not None]
|
|
if sort in {"growth", "growth_rate"}:
|
|
items = [
|
|
item for item in items
|
|
if item.get("growth", {}).get("available")
|
|
]
|
|
if sort == "growth_rate":
|
|
items = [
|
|
item for item in items
|
|
if (item["metrics"] or {}).get("view", 0) >= MIN_VIEW_FOR_GROWTH_RATE
|
|
]
|
|
|
|
def sort_key(item: Mapping[str, Any]):
|
|
if sort == "added":
|
|
return (int(item["first_seen"]), int(item["pubdate"]))
|
|
if sort == "published":
|
|
return (int(item["pubdate"]), int(item["first_seen"]))
|
|
if sort == "heat":
|
|
return item["heat"] if item["heat"] is not None else float("-inf")
|
|
if sort == "interaction":
|
|
return item["interaction_rate"] if item["interaction_rate"] is not None else float("-inf")
|
|
if sort in {"growth", "growth_rate"}:
|
|
field = "delta" if sort == "growth" else "rate"
|
|
value = item.get("growth", {}).get(field)
|
|
return value if value is not None else float("-inf")
|
|
value = (item["metrics"] or {}).get(sort)
|
|
return value if value is not None else -1
|
|
|
|
items.sort(key=lambda item: item["aid"])
|
|
items.sort(key=sort_key, reverse=True)
|
|
total = len(items)
|
|
growth_available = any(
|
|
item.get("growth", {}).get("available") for item in items
|
|
)
|
|
if not export:
|
|
start = (page - 1) * page_size
|
|
items = items[start:start + page_size]
|
|
available = True
|
|
reason = None
|
|
if sort in {"growth", "growth_rate"} and not growth_available:
|
|
available = False
|
|
reason = "需要至少两天的快照才能计算增长"
|
|
return {
|
|
"sort": sort,
|
|
"within_days": within_days,
|
|
"growth_days": growth_days,
|
|
"page": page,
|
|
"page_size": page_size,
|
|
"total": total,
|
|
"pages": max(1, math.ceil(total / page_size)) if total else 0,
|
|
"items": items,
|
|
"available": available,
|
|
"reason": reason,
|
|
}
|
|
return self.db.read(run)
|
|
|
|
def video_detail(self, aid: int) -> dict[str, Any] | None:
|
|
aid = int(aid)
|
|
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
|
video_rows = self._videos(conn)
|
|
video = next((item for item in video_rows if int(item["aid"]) == aid), None)
|
|
if video is None:
|
|
row = conn.execute(
|
|
"SELECT v.*, f.title AS folder_title, f.year AS folder_year "
|
|
"FROM video v LEFT JOIN folder f ON f.media_id = v.folder_id "
|
|
"WHERE v.aid = ?",
|
|
(aid,),
|
|
).fetchone()
|
|
if row is None:
|
|
return None
|
|
video = dict(row)
|
|
snapshots = self._snapshot_map(conn, [aid])
|
|
snapshot = snapshots.get(aid)
|
|
previous = self._snapshot_map(conn, [aid], before=snapshot["stat_date"] if snapshot else None).get(aid)
|
|
_, _, heat, percentiles = self._global_heat(conn)
|
|
growth = self._growth_for(snapshot, previous)
|
|
item = self._item(
|
|
video,
|
|
snapshot,
|
|
heat=heat.get(aid),
|
|
percentile=percentiles.get(aid),
|
|
)
|
|
item["growth"] = growth.as_dict()
|
|
item["snapshot_count"] = int(conn.execute(
|
|
"SELECT COUNT(*) FROM snapshot WHERE aid = ?", (aid,)
|
|
).fetchone()[0])
|
|
item["blacklisted"] = bool(conn.execute(
|
|
"SELECT 1 FROM up_blacklist WHERE mid = ?", (video.get("up_mid"),)
|
|
).fetchone()) if video.get("up_mid") is not None else False
|
|
item["url"] = (
|
|
f"https://www.bilibili.com/video/{video.get('bvid')}"
|
|
if video.get("bvid") else None
|
|
)
|
|
return item
|
|
return self.db.read(run)
|
|
|
|
def video_trend(self, aid: int, *, days: int | None = 30) -> dict[str, Any]:
|
|
aid = int(aid)
|
|
if days is not None and days < 1:
|
|
raise ValueError("days 必须大于 0")
|
|
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
|
latest = self._latest_date(conn)
|
|
rows = conn.execute(
|
|
"SELECT * FROM snapshot WHERE aid = ? ORDER BY stat_date", (aid,)
|
|
).fetchall()
|
|
points = [dict(row) for row in rows]
|
|
if latest is None or not points:
|
|
return {
|
|
"aid": aid,
|
|
"days": days,
|
|
"dates": [],
|
|
"series": {metric: [] for metric in METRICS},
|
|
"points": [],
|
|
"available": False,
|
|
}
|
|
requested_start = (
|
|
_date_minus(latest, days - 1) if days is not None
|
|
else str(points[0]["stat_date"])
|
|
)
|
|
first_point = str(points[0]["stat_date"])
|
|
start = max(requested_start, first_point)
|
|
dates = _date_range(start, latest)
|
|
by_date = {str(point["stat_date"]): point for point in points}
|
|
series: dict[str, list[int | None]] = {}
|
|
for metric in METRICS:
|
|
values: list[int | None] = []
|
|
for day in dates:
|
|
point = by_date.get(day)
|
|
if point is None or int(point["ok"]) != 1 or point.get(metric) is None:
|
|
values.append(None)
|
|
else:
|
|
values.append(int(point[metric]))
|
|
series[metric] = values
|
|
filtered_points = [
|
|
point for point in points if str(point["stat_date"]) in set(dates)
|
|
]
|
|
return {
|
|
"aid": aid,
|
|
"days": days,
|
|
"dates": dates,
|
|
"series": series,
|
|
"points": filtered_points,
|
|
"available": any(point["ok"] == 1 for point in filtered_points),
|
|
}
|
|
return self.db.read(run)
|
|
|
|
# ------------------------------------------------------------ ups
|
|
|
|
def list_ups(
|
|
self,
|
|
*,
|
|
sort: str = "views",
|
|
within_days: int | None = None,
|
|
q: str | None = None,
|
|
min_videos: int = 1,
|
|
page: int = 1,
|
|
page_size: int = 20,
|
|
export: bool = False,
|
|
) -> dict[str, Any]:
|
|
if sort not in UP_SORTS:
|
|
raise ValueError(f"不支持的 UP 排序: {sort}")
|
|
page = max(1, int(page))
|
|
page_size = max(1, min(int(page_size), 200 if not export else 10_000))
|
|
min_videos = max(1, int(min_videos))
|
|
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
|
latest = self._latest_date(conn) or self._today()
|
|
videos = self._videos(
|
|
conn,
|
|
q=q,
|
|
within_field="pubdate" if within_days is not None else None,
|
|
within_start=(
|
|
_date_minus(latest, int(within_days) - 1)
|
|
if within_days is not None else None
|
|
),
|
|
)
|
|
snapshots = self._snapshot_map(conn, [int(video["aid"]) for video in videos])
|
|
groups: dict[int, dict[str, Any]] = {}
|
|
for video in videos:
|
|
mid = video.get("up_mid")
|
|
if mid is None:
|
|
continue
|
|
mid = int(mid)
|
|
group = groups.setdefault(mid, {
|
|
"up_mid": mid,
|
|
"up_name": video.get("up_name") or f"UP {mid}",
|
|
"videos": [],
|
|
})
|
|
metrics = _metric_map(snapshots.get(int(video["aid"])))
|
|
group["videos"].append((video, metrics))
|
|
items = []
|
|
for group in groups.values():
|
|
group_videos = group["videos"]
|
|
metrics_list = [metrics for _, metrics in group_videos if metrics]
|
|
total_views = sum(metrics["view"] for metrics in metrics_list)
|
|
total_interactions = sum(
|
|
metrics["like"] + metrics["coin"] + metrics["favorite"]
|
|
for metrics in metrics_list
|
|
)
|
|
rates = [
|
|
interaction_rate(metrics)
|
|
for metrics in metrics_list
|
|
if interaction_rate(metrics) is not None
|
|
]
|
|
best = max(
|
|
group_videos,
|
|
key=lambda pair: (pair[1] or {}).get("view", 0),
|
|
default=None,
|
|
)
|
|
items.append({
|
|
"up_mid": group["up_mid"],
|
|
"up_name": group["up_name"],
|
|
"video_count": len(group_videos),
|
|
"total_views": total_views,
|
|
"avg_views": round(total_views / len(metrics_list), 1) if metrics_list else None,
|
|
"total_interactions": total_interactions,
|
|
"avg_interaction": round(sum(rates) / len(rates), 6) if rates else None,
|
|
"latest_pubdate": max(int(video["pubdate"]) for video, _ in group_videos),
|
|
"first_pubdate": min(int(video["pubdate"]) for video, _ in group_videos),
|
|
"best_video": None if best is None else {
|
|
"aid": int(best[0]["aid"]),
|
|
"title": best[0].get("title"),
|
|
"cover": normalize_cover(best[0].get("cover")),
|
|
"view": (best[1] or {}).get("view"),
|
|
},
|
|
})
|
|
items = [item for item in items if item["video_count"] >= min_videos]
|
|
sort_map = {
|
|
"views": lambda item: item["total_views"],
|
|
"count": lambda item: item["video_count"],
|
|
"avg_views": lambda item: item["avg_views"] or 0,
|
|
"interaction": lambda item: item["total_interactions"],
|
|
"avg_interaction": lambda item: item["avg_interaction"] or 0,
|
|
"latest": lambda item: item["latest_pubdate"],
|
|
}
|
|
key = sort_map[sort]
|
|
items.sort(key=lambda item: (key(item), item["up_mid"]), reverse=True)
|
|
total = len(items)
|
|
if not export:
|
|
start = (page - 1) * page_size
|
|
items = items[start:start + page_size]
|
|
return {
|
|
"sort": sort,
|
|
"within_days": within_days,
|
|
"min_videos": min_videos,
|
|
"page": page,
|
|
"page_size": page_size,
|
|
"total": total,
|
|
"pages": max(1, math.ceil(total / page_size)) if total else 0,
|
|
"items": items,
|
|
}
|
|
return self.db.read(run)
|
|
|
|
def up_detail(self, mid: int) -> dict[str, Any] | None:
|
|
mid = int(mid)
|
|
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
|
latest = self._latest_date(conn) or self._today()
|
|
videos = self._videos(conn, up_mid=mid)
|
|
if not videos:
|
|
row = conn.execute(
|
|
"SELECT 1 FROM up_blacklist WHERE mid = ?", (mid,)
|
|
).fetchone()
|
|
if row is None:
|
|
return None
|
|
return {
|
|
"up_mid": mid,
|
|
"up_name": f"UP {mid}",
|
|
"videos": [],
|
|
"video_count": 0,
|
|
"blacklisted": True,
|
|
"series": {"dates": [], "views": []},
|
|
}
|
|
snapshots = self._snapshot_map(conn, [int(video["aid"]) for video in videos])
|
|
items = [
|
|
self._item(video, snapshots.get(int(video["aid"])))
|
|
for video in videos
|
|
]
|
|
metrics_list = [item["metrics"] for item in items if item["metrics"]]
|
|
rates = [item["interaction_rate"] for item in items if item["interaction_rate"] is not None]
|
|
total_views = sum(metrics["view"] for metrics in metrics_list)
|
|
total_interactions = sum(
|
|
metrics["like"] + metrics["coin"] + metrics["favorite"]
|
|
for metrics in metrics_list
|
|
)
|
|
aids = [int(video["aid"]) for video in videos]
|
|
rows = conn.execute(
|
|
f"SELECT stat_date, SUM(view) AS view FROM snapshot "
|
|
f"WHERE ok = 1 AND aid IN ({','.join('?' for _ in aids)}) "
|
|
"GROUP BY stat_date ORDER BY stat_date",
|
|
aids,
|
|
).fetchall()
|
|
view_by_date = {str(row["stat_date"]): int(row["view"] or 0) for row in rows}
|
|
dates = _date_range(min(view_by_date), latest) if view_by_date else []
|
|
items.sort(key=lambda item: ((item["metrics"] or {}).get("view", 0), item["pubdate"]), reverse=True)
|
|
folder_titles = sorted({
|
|
str(item["folder_title"]) for item in items if item.get("folder_title")
|
|
})
|
|
return {
|
|
"up_mid": mid,
|
|
"up_name": items[0].get("up_name") or f"UP {mid}",
|
|
"video_count": len(items),
|
|
"total_views": total_views,
|
|
"avg_views": round(total_views / len(metrics_list), 1) if metrics_list else None,
|
|
"total_interactions": total_interactions,
|
|
"avg_interaction": round(sum(rates) / len(rates), 6) if rates else None,
|
|
"first_pubdate": min(item["pubdate"] for item in items),
|
|
"latest_pubdate": max(item["pubdate"] for item in items),
|
|
"folders": folder_titles,
|
|
"blacklisted": bool(conn.execute(
|
|
"SELECT 1 FROM up_blacklist WHERE mid = ?", (mid,)
|
|
).fetchone()),
|
|
"space_url": f"https://space.bilibili.com/{mid}",
|
|
"videos": items,
|
|
"series": {
|
|
"dates": dates,
|
|
"views": [view_by_date.get(day) for day in dates],
|
|
},
|
|
}
|
|
return self.db.read(run)
|
|
|
|
# ------------------------------------------------------------ fans
|
|
|
|
def fans(self, mid: int | None = None, *, days: int = 90) -> dict[str, Any]:
|
|
if days < 1:
|
|
raise ValueError("days 必须大于 0")
|
|
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
|
cfg = self.config_store.get()
|
|
mids = list(cfg.track_fans)
|
|
if not mids:
|
|
mids = [
|
|
int(row["mid"]) for row in conn.execute(
|
|
"SELECT DISTINCT mid FROM up_stat ORDER BY mid"
|
|
).fetchall()
|
|
]
|
|
selected = int(mid) if mid is not None else (mids[0] if mids else None)
|
|
options = [
|
|
{"mid": value, "name": TRACK_FANS_NAMES.get(value, f"UP {value}")}
|
|
for value in mids
|
|
]
|
|
if selected is None:
|
|
return {
|
|
"mid": None, "name": None, "available": False,
|
|
"options": options, "dates": [], "series": [],
|
|
"points": [], "days": days,
|
|
}
|
|
rows = [
|
|
dict(row) for row in conn.execute(
|
|
"SELECT * FROM up_stat WHERE mid = ? ORDER BY stat_date",
|
|
(selected,),
|
|
).fetchall()
|
|
]
|
|
if not rows:
|
|
return {
|
|
"mid": selected,
|
|
"name": TRACK_FANS_NAMES.get(selected, f"UP {selected}"),
|
|
"available": False,
|
|
"options": options,
|
|
"dates": [],
|
|
"series": [],
|
|
"points": [],
|
|
"days": days,
|
|
}
|
|
latest = max(str(row["stat_date"]) for row in rows if int(row["ok"]) == 1) if any(int(row["ok"]) == 1 for row in rows) else str(rows[-1]["stat_date"])
|
|
first_date = str(rows[0]["stat_date"])
|
|
start = max(_date_minus(latest, days - 1), first_date)
|
|
dates = _date_range(start, latest)
|
|
by_date = {str(row["stat_date"]): row for row in rows}
|
|
series = []
|
|
for day in dates:
|
|
row = by_date.get(day)
|
|
if row is None or int(row["ok"]) != 1:
|
|
series.append(None)
|
|
else:
|
|
series.append(row["follower"])
|
|
ok_rows = [row for row in rows if int(row["ok"]) == 1]
|
|
current = ok_rows[-1] if ok_rows else rows[-1]
|
|
previous = ok_rows[-2] if len(ok_rows) >= 2 else None
|
|
cutoff = _date_minus(latest, 7)
|
|
baselines = [row for row in ok_rows if str(row["stat_date"]) <= cutoff]
|
|
baseline = baselines[-1] if baselines else None
|
|
first = ok_rows[0] if ok_rows else rows[0]
|
|
return {
|
|
"mid": selected,
|
|
"name": TRACK_FANS_NAMES.get(selected, f"UP {selected}"),
|
|
"available": bool(ok_rows),
|
|
"options": options,
|
|
"days": days,
|
|
"dates": dates,
|
|
"series": series,
|
|
"points": rows,
|
|
"stat_date": current["stat_date"],
|
|
"follower": current.get("follower"),
|
|
"following": current.get("following"),
|
|
"delta_1d": None if previous is None else int(current.get("follower") or 0) - int(previous.get("follower") or 0),
|
|
"delta_7d": None if baseline is None else int(current.get("follower") or 0) - int(baseline.get("follower") or 0),
|
|
"delta_cum": int(current.get("follower") or 0) - int(first.get("follower") or 0),
|
|
"first_date": first["stat_date"],
|
|
}
|
|
return self.db.read(run)
|
|
|
|
# ------------------------------------------------------------ candidates
|
|
|
|
def candidates(self) -> dict[str, Any]:
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
|
cfg = self.config_store.get()
|
|
now = int(self._now())
|
|
ttl = int(cfg.schedule.candidate_ttl_days)
|
|
rows = conn.execute(
|
|
"SELECT * FROM candidate ORDER BY "
|
|
"CASE WHEN decision IS NULL THEN 0 ELSE 1 END, "
|
|
"COALESCE(score, 0) DESC, first_seen DESC, aid"
|
|
).fetchall()
|
|
items = []
|
|
for row in rows:
|
|
data = dict(row)
|
|
first_seen = int(data.get("first_seen") or 0)
|
|
expires_at = first_seen + ttl * 86400
|
|
items.append({
|
|
**data,
|
|
"cover": normalize_cover(data.get("cover")),
|
|
"expires_at": expires_at,
|
|
"days_left": max(0, math.ceil((expires_at - now) / 86400)),
|
|
"url": (
|
|
f"https://www.bilibili.com/video/{data.get('bvid')}"
|
|
if data.get("bvid") else None
|
|
),
|
|
})
|
|
pending = sum(1 for item in items if item.get("decision") is None)
|
|
return {
|
|
"items": items,
|
|
"total": len(items),
|
|
"pending": pending,
|
|
"ttl_days": ttl,
|
|
"telegram_enabled": bool(cfg.telegram.enabled),
|
|
"now": now,
|
|
}
|
|
return self.db.read(run)
|
|
|
|
# ------------------------------------------------------------ system
|
|
|
|
def system(self) -> dict[str, Any]:
|
|
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
|
cfg = self.config_store.get()
|
|
latest = self._latest_date(conn)
|
|
snapshot_days = self._snapshot_days(conn)
|
|
health = {"total": 0, "ok": 0, "failed": 0}
|
|
if latest:
|
|
rows = conn.execute(
|
|
"SELECT ok, COUNT(*) AS n FROM snapshot WHERE stat_date = ? GROUP BY ok",
|
|
(latest,),
|
|
).fetchall()
|
|
by_ok = {int(row["ok"]): int(row["n"]) for row in rows}
|
|
health = {
|
|
"total": sum(by_ok.values()),
|
|
"ok": by_ok.get(1, 0),
|
|
"failed": by_ok.get(0, 0),
|
|
}
|
|
folders = self._folders_overview(conn)
|
|
blacklist = [
|
|
dict(row) for row in conn.execute(
|
|
"""
|
|
SELECT b.*,
|
|
(SELECT COUNT(*) FROM video v WHERE v.up_mid = b.mid
|
|
AND v.status IN ('active','unfaved')) AS video_count
|
|
FROM up_blacklist b ORDER BY b.created_at, b.mid
|
|
"""
|
|
).fetchall()
|
|
]
|
|
tasks = self._task_records(conn)
|
|
now = self._now()
|
|
next_runs = {
|
|
"discover": int(now + cfg.schedule.discover_minutes * 60),
|
|
"folder_sync": int(now + cfg.schedule.folder_sync_minutes * 60),
|
|
"snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)),
|
|
}
|
|
for name, key, interval_minutes in (
|
|
("discover", "discovery", cfg.schedule.discover_minutes),
|
|
("folder_sync", "folder_sync", cfg.schedule.folder_sync_minutes),
|
|
):
|
|
record = tasks.get(key)
|
|
if isinstance(record, dict) and record.get("finished_at"):
|
|
interval = int(interval_minutes) * 60
|
|
candidate = int(record["finished_at"]) + interval
|
|
while candidate <= now:
|
|
candidate += interval
|
|
next_runs[name] = candidate
|
|
errors = [
|
|
{"task": name, **record}
|
|
for name, record in tasks.items()
|
|
if isinstance(record, dict) and record.get("ok") is False
|
|
]
|
|
counts = {str(row["status"]): int(row["n"]) for row in conn.execute(
|
|
"SELECT status, COUNT(*) AS n FROM video GROUP BY status"
|
|
).fetchall()}
|
|
return {
|
|
"generated_at": int(now),
|
|
"today": self._today(),
|
|
"data_date": latest,
|
|
"snapshot_days": snapshot_days,
|
|
"snapshot_health": health,
|
|
"counts": counts,
|
|
"folders": folders,
|
|
"blacklist": blacklist,
|
|
"tasks": tasks,
|
|
"errors": errors,
|
|
"next_runs": next_runs,
|
|
"auth": {
|
|
"ok": self._kv(conn, "auth:ok"),
|
|
"message": self._kv(conn, "auth:message"),
|
|
"last_check": self._kv(conn, "auth:last_check"),
|
|
"mid": self._kv(conn, "auth:mid"),
|
|
},
|
|
"backup": {
|
|
"last_completed": self._kv(conn, "snapshot:last_completed"),
|
|
"last_attempt": self._kv(conn, "snapshot:last_attempt"),
|
|
"ok": self._kv(conn, f"snapshot:backup:{latest}" if latest else "snapshot:backup"),
|
|
"error": self._kv(conn, f"snapshot:backup_error:{latest}" if latest else "snapshot:backup_error"),
|
|
},
|
|
"preview": self._kv(conn, "preview:seeded") == "1",
|
|
"schedule": {
|
|
"discover_minutes": cfg.schedule.discover_minutes,
|
|
"folder_sync_minutes": cfg.schedule.folder_sync_minutes,
|
|
"snapshot_at": cfg.schedule.snapshot_at,
|
|
"candidate_ttl_days": cfg.schedule.candidate_ttl_days,
|
|
},
|
|
"telegram_enabled": bool(cfg.telegram.enabled),
|
|
}
|
|
return self.db.read(run)
|
|
|
|
def export_videos(self, **filters) -> dict[str, Any]:
|
|
filters.pop("page", None)
|
|
filters.pop("page_size", None)
|
|
return self.list_videos(**filters, export=True)
|