# -*- coding: utf-8 -*- """只读看板查询层。 所有 SQL 都在这里集中，Web 层只负责参数解析与 JSON 序列化。查询不会写入 数据库，也不会访问 B 站。 """ from __future__ import annotations import json import math import sqlite3 import time from datetime import date, timedelta from typing import Any, Iterable, Mapping, Sequence from ..analytics import ( DEFAULT_HEAT_WEIGHTS, METRICS, MIN_VIEW_FOR_GROWTH_RATE, compute_growth, heat_scores, interaction_rate, normalize_cover, score_percentiles, ) from ..config import ConfigStore from ..database import Database from ..timeutil import day_bounds, next_daily_run, shanghai_date, shanghai_datetime TRACKED_STATUSES = ("active", "unfaved") VIDEO_SORTS = { "heat", "view", "like", "coin", "favorite", "share", "reply", "danmaku", "interaction", "growth", "growth_rate", "added", "published", } UP_SORTS = {"views", "count", "avg_views", "interaction", "avg_interaction", "latest"} CAPACITY = 1000 CAPACITY_WARNING = 950 TRACK_FANS_NAMES = {37754047: "咻咻满"} def _chunks(values: Sequence[int], size: int = 400) -> Iterable[Sequence[int]]: for index in range(0, len(values), size): yield values[index:index + size] def _date_minus(day: str, days: int) -> str: return (date.fromisoformat(day) - timedelta(days=int(days))).isoformat() def _date_range(start: str, end: str) -> list[str]: first = date.fromisoformat(start) last = date.fromisoformat(end) if last < first: return [] return [ (first + timedelta(days=offset)).isoformat() for offset in range((last - first).days + 1) ] def _ts_date(timestamp: int | float | None) -> str | None: if timestamp is None: return None return shanghai_date(float(timestamp)) def _like_pattern(value: str) -> str: escaped = value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") return f"%{escaped}%" def _metric_map(row: Mapping[str, Any] | None) -> dict[str, int] | None: if row is None: return None result: dict[str, int] = {} for metric in METRICS: value = row.get(metric) result[metric] = 0 if value is None else int(value) return result def _safe_float(value: Any) -> float | None: if value is None or isinstance(value, bool): return None try: return float(value) except (TypeError, ValueError): return None class DashboardQueries: def __init__( self, db: Database, config_store: ConfigStore, *, clock=time.time, ) -> None: self.db = db self.config_store = config_store self._clock = clock # ------------------------------------------------------------ helpers def _now(self) -> float: return float(self._clock()) def _today(self) -> str: return shanghai_date(self._now()) @staticmethod def _latest_date(conn: sqlite3.Connection) -> str | None: row = conn.execute( "SELECT MAX(stat_date) AS d FROM snapshot WHERE ok = 1" ).fetchone() return None if row is None or row["d"] is None else str(row["d"]) @staticmethod def _snapshot_days(conn: sqlite3.Connection) -> int: row = conn.execute( "SELECT COUNT(DISTINCT stat_date) AS n FROM snapshot WHERE ok = 1" ).fetchone() return int(row["n"] or 0) @staticmethod def _snapshot_map( conn: sqlite3.Connection, aids: Sequence[int], *, before: str | None = None, before_inclusive: bool = False, ) -> dict[int, dict[str, Any]]: if not aids: return {} result: dict[int, dict[str, Any]] = {} for chunk in _chunks(list(aids)): placeholders = ",".join("?" for _ in chunk) if before is None: sql = f""" SELECT s.* FROM snapshot s JOIN ( SELECT aid, MAX(stat_date) AS stat_date FROM snapshot WHERE ok = 1 AND aid IN ({placeholders}) GROUP BY aid ) m ON m.aid = s.aid AND m.stat_date = s.stat_date """ params: list[Any] = list(chunk) else: operator = "<=" if before_inclusive else "<" sql = f""" SELECT s.* FROM snapshot s JOIN ( SELECT aid, MAX(stat_date) AS stat_date FROM snapshot WHERE ok = 1 AND aid IN ({placeholders}) AND stat_date {operator} ? GROUP BY aid ) m ON m.aid = s.aid AND m.stat_date = s.stat_date """ params = [*chunk, before] for row in conn.execute(sql, params).fetchall(): data = dict(row) result[int(data["aid"])] = data return result @staticmethod def _videos( conn: sqlite3.Connection, *, statuses: Sequence[str] = TRACKED_STATUSES, folder_id: int | None = None, q: str | None = None, within_field: str | None = None, within_start: str | None = None, up_mid: int | None = None, ) -> list[dict[str, Any]]: clauses = [f"v.status IN ({','.join('?' for _ in statuses)})"] params: list[Any] = list(statuses) if folder_id is not None: clauses.append("v.folder_id = ?") params.append(int(folder_id)) if up_mid is not None: clauses.append("v.up_mid = ?") params.append(int(up_mid)) if q: pattern = _like_pattern(q) clauses.append( "(v.title LIKE ? ESCAPE '\\' OR v.up_name LIKE ? ESCAPE '\\')" ) params.extend([pattern, pattern]) if within_field and within_start: if within_field not in {"pubdate", "first_seen"}: raise ValueError("within_field 必须是 pubdate 或 first_seen") start_ts, _ = day_bounds(within_start) clauses.append(f"v.{within_field} >= ?") params.append(start_ts) sql = ( "SELECT v.*, f.title AS folder_title, f.year AS folder_year " "FROM video v LEFT JOIN folder f ON f.media_id = v.folder_id " f"WHERE {' AND '.join(clauses)} ORDER BY v.aid" ) return [dict(row) for row in conn.execute(sql, params).fetchall()] def _global_heat( self, conn: sqlite3.Connection ) -> tuple[list[dict[str, Any]], dict[int, dict[str, Any]], dict[int, float], dict[int, float]]: videos = self._videos(conn) snapshots = self._snapshot_map(conn, [int(v["aid"]) for v in videos]) rows = [] for video in videos: snapshot = snapshots.get(int(video["aid"])) if snapshot is None: continue item = {"aid": int(video["aid"])} item.update(_metric_map(snapshot) or {}) rows.append(item) weights = self.config_store.get().heat_weights scores = heat_scores(rows, weights or DEFAULT_HEAT_WEIGHTS) percentiles = score_percentiles(scores) return videos, snapshots, scores, percentiles @staticmethod def _growth_for( snapshot: Mapping[str, Any] | None, previous: Mapping[str, Any] | None, metric: str = "view", ): if snapshot is None: return compute_growth(None, None, None) current = snapshot.get(metric) before = None if previous is None else previous.get(metric) gap = None if previous is not None and snapshot.get("stat_date") and previous.get("stat_date"): try: gap = ( date.fromisoformat(str(snapshot["stat_date"])) - date.fromisoformat(str(previous["stat_date"])) ).days except ValueError: gap = None return compute_growth(current, before, gap) def _item( self, video: Mapping[str, Any], snapshot: Mapping[str, Any] | None, *, heat: float | None = None, percentile: float | None = None, previous: Mapping[str, Any] | None = None, include_growth: bool = False, ) -> dict[str, Any]: metrics = _metric_map(snapshot) item: dict[str, Any] = { "aid": int(video["aid"]), "bvid": video.get("bvid"), "title": video.get("title") or "(无标题)", "up_mid": video.get("up_mid"), "up_name": video.get("up_name"), "pubdate": int(video.get("pubdate") or 0), "duration": video.get("duration"), "cover": normalize_cover(video.get("cover")), "folder_id": video.get("folder_id"), "folder_title": video.get("folder_title"), "folder_year": video.get("folder_year"), "status": video.get("status"), "first_seen": int(video.get("first_seen") or 0), "attr": video.get("attr"), "metrics": metrics, "snapshot_date": None if snapshot is None else snapshot.get("stat_date"), "heat": None if heat is None else round(float(heat), 3), "heat_percentile": percentile, "interaction_rate": None if metrics is None else interaction_rate(metrics), } if include_growth: item["growth"] = self._growth_for(snapshot, previous).as_dict() return item # ------------------------------------------------------------ overview def overview(self) -> dict[str, Any]: def run(conn: sqlite3.Connection) -> dict[str, Any]: latest = self._latest_date(conn) snapshot_days = self._snapshot_days(conn) counts_rows = conn.execute( "SELECT status, COUNT(*) AS n FROM video GROUP BY status" ).fetchall() by_status = {str(row["status"]): int(row["n"]) for row in counts_rows} videos = self._videos(conn) aids = [int(video["aid"]) for video in videos] snapshots = self._snapshot_map(conn, aids) previous = self._snapshot_map(conn, aids, before=latest or self._today()) totals = {metric: 0 for metric in METRICS} for video in videos: metrics = _metric_map(snapshots.get(int(video["aid"]))) if metrics: for metric in METRICS: totals[metric] += metrics[metric] view_delta: int | None = None available_deltas = 0 view_delta_sum = 0 for video in videos: aid = int(video["aid"]) snapshot = snapshots.get(aid) prior = previous.get(aid) if snapshot is None or prior is None: continue growth = self._growth_for(snapshot, prior) if growth.available and growth.delta is not None: view_delta_sum += growth.delta available_deltas += 1 if available_deltas: view_delta = view_delta_sum first_seen_row = conn.execute( "SELECT MIN(first_seen) AS ts FROM video" ).fetchone() first_seen_ts = first_seen_row["ts"] if first_seen_row else None first_seen_date = _ts_date(first_seen_ts) initial_import_count = 0 if first_seen_ts is not None: initial_import_count = int(conn.execute( "SELECT COUNT(*) FROM video WHERE first_seen = ?", (int(first_seen_ts),), ).fetchone()[0]) start_ts, end_ts = day_bounds(self._today()) added_today = int(conn.execute( "SELECT COUNT(*) FROM video WHERE first_seen >= ? AND first_seen < ?", (start_ts, end_ts), ).fetchone()[0]) is_first_day = snapshot_days <= 1 if is_first_day: added_today = 0 snapshot_health = {"total": 0, "ok": 0, "failed": 0} if latest: rows = conn.execute( "SELECT ok, COUNT(*) AS n FROM snapshot WHERE stat_date = ? GROUP BY ok", (latest,), ).fetchall() by_ok = {int(row["ok"]): int(row["n"]) for row in rows} snapshot_health = { "total": sum(by_ok.values()), "ok": by_ok.get(1, 0), "failed": by_ok.get(0, 0), } candidate_pending = int(conn.execute( "SELECT COUNT(*) FROM candidate WHERE decision IS NULL" ).fetchone()[0]) candidate_total = int(conn.execute( "SELECT COUNT(*) FROM candidate" ).fetchone()[0]) cfg = self.config_store.get() fans = self._fans_overview(conn, cfg.track_fans, latest, snapshot_days) folders = self._folders_overview(conn) tasks = self._task_records(conn) auth = { "ok": self._kv(conn, "auth:ok"), "message": self._kv(conn, "auth:message"), "last_check": self._kv(conn, "auth:last_check"), "mid": self._kv(conn, "auth:mid"), } preview = self._kv(conn, "preview:seeded") == "1" snapshot_completed_today = self._kv( conn, f"snapshot:completed:{self._today()}" ) == "1" return { "generated_at": int(self._now()), "today": self._today(), "data_date": latest, "snapshot_days": snapshot_days, "first_seen_date": first_seen_date, "is_first_day": is_first_day, "initial_import_count": initial_import_count, "added_today": added_today, "counts": { **by_status, "active_ups": int(conn.execute( "SELECT COUNT(DISTINCT up_mid) FROM video " "WHERE status IN ('active','unfaved') AND up_mid IS NOT NULL" ).fetchone()[0]), "candidates_pending": candidate_pending, "candidates_total": candidate_total, }, "totals": totals, "interactions": totals["like"] + totals["coin"] + totals["favorite"], "view_delta": view_delta, "view_delta_available": view_delta is not None, "snapshot_health": snapshot_health, "snapshot_completed_today": snapshot_completed_today, "fans": fans, "folders": folders, "tasks": tasks, "auth": auth, "preview": preview, "capacity": CAPACITY, "capacity_warning": CAPACITY_WARNING, } return self.db.read(run) @staticmethod def _kv(conn: sqlite3.Connection, key: str) -> str | None: row = conn.execute("SELECT v FROM kv WHERE k = ?", (key,)).fetchone() return None if row is None else str(row["v"]) def _task_records(self, conn: sqlite3.Connection) -> dict[str, Any]: rows = conn.execute( "SELECT k, v FROM kv WHERE k LIKE 'task:last:%' ORDER BY k" ).fetchall() result: dict[str, Any] = {} for row in rows: name = str(row["k"]).removeprefix("task:last:") try: value = json.loads(str(row["v"])) except (TypeError, json.JSONDecodeError): value = {"raw": row["v"]} result[name] = value return result def _folders_overview(self, conn: sqlite3.Connection) -> list[dict[str, Any]]: rows = conn.execute( """ SELECT f.*, (SELECT COUNT(*) FROM folder_resource fr WHERE fr.media_id = f.media_id AND fr.active = 1) AS local_count FROM folder f ORDER BY f.year DESC, f.media_id DESC """ ).fetchall() result = [] for row in rows: media_count = row["media_count"] count = int(media_count) if media_count is not None else int(row["local_count"] or 0) result.append({ "media_id": int(row["media_id"]), "year": int(row["year"]), "title": row["title"] or "", "media_count": count, "local_count": int(row["local_count"] or 0), "last_ids_at": row["last_ids_at"], "last_full_at": row["last_full_at"], "enabled": bool(row["enabled"]), "capacity": CAPACITY, "ratio": min(1.0, count / CAPACITY), "warning": count >= CAPACITY_WARNING, }) return result def _fans_overview( self, conn: sqlite3.Connection, mid_list: Sequence[int], latest: str | None, snapshot_days: int, ) -> dict[str, Any] | None: mids = list(mid_list) or [ int(row["mid"]) for row in conn.execute("SELECT DISTINCT mid FROM up_stat ORDER BY mid").fetchall() ] if not mids: return None mid = int(mids[0]) rows = conn.execute( "SELECT * FROM up_stat WHERE mid = ? ORDER BY stat_date", (mid,) ).fetchall() if not rows: return None ok_rows = [dict(row) for row in rows if int(row["ok"]) == 1] if not ok_rows: return None current = ok_rows[-1] previous = ok_rows[-2] if len(ok_rows) >= 2 else None baseline = None if latest: cutoff = _date_minus(latest, 7) candidates = [row for row in ok_rows if str(row["stat_date"]) <= cutoff] baseline = candidates[-1] if candidates else None first = ok_rows[0] return { "mid": mid, "name": TRACK_FANS_NAMES.get(mid, f"UP {mid}"), "stat_date": current["stat_date"], "follower": current["follower"], "following": current["following"], "delta_1d": None if previous is None else int(current["follower"] or 0) - int(previous["follower"] or 0), "delta_7d": None if baseline is None else int(current["follower"] or 0) - int(baseline["follower"] or 0), "delta_cum": int(current["follower"] or 0) - int(first["follower"] or 0), "first_date": first["stat_date"], "snapshot_days": snapshot_days, "points": [ { "date": row["stat_date"], "follower": row["follower"] if int(row["ok"]) == 1 else None, "ok": bool(row["ok"]), } for row in rows[-30:] ], } # ------------------------------------------------------------ videos def list_videos( self, *, sort: str = "heat", within_days: int | None = None, growth_days: int = 7, folder_id: int | None = None, q: str | None = None, page: int = 1, page_size: int = 20, export: bool = False, ) -> dict[str, Any]: if sort not in VIDEO_SORTS: raise ValueError(f"不支持的排序: {sort}") if growth_days < 1 or growth_days > 365: raise ValueError("growth_days 必须在 1~365 之间") page = max(1, int(page)) page_size = max(1, min(int(page_size), 200 if not export else 10_000)) def run(conn: sqlite3.Connection) -> dict[str, Any]: latest = self._latest_date(conn) or self._today() within_field = None if within_days is not None: if within_days < 1: raise ValueError("within_days 必须大于 0") within_field = "first_seen" if sort == "added" else "pubdate" videos = self._videos( conn, folder_id=folder_id, q=q, within_field=within_field, within_start=( _date_minus(latest, int(within_days) - 1) if within_days is not None else None ), ) aids = [int(video["aid"]) for video in videos] snapshots = self._snapshot_map(conn, aids) _, _, heat, percentiles = self._global_heat(conn) previous: dict[int, dict[str, Any]] = {} if sort in {"growth", "growth_rate", "interaction"}: baseline_date = _date_minus(latest, growth_days) previous = self._snapshot_map(conn, aids, before=baseline_date, before_inclusive=True) elif sort in {"heat", "view", "like", "coin", "favorite", "share", "reply", "danmaku"}: previous = {} items = [ self._item( video, snapshots.get(int(video["aid"])), heat=heat.get(int(video["aid"])), percentile=percentiles.get(int(video["aid"])), previous=previous.get(int(video["aid"])), include_growth=sort in {"growth", "growth_rate"}, ) for video in videos ] if sort == "interaction": items = [item for item in items if item["interaction_rate"] is not None] if sort in {"growth", "growth_rate"}: items = [ item for item in items if item.get("growth", {}).get("available") ] if sort == "growth_rate": items = [ item for item in items if (item["metrics"] or {}).get("view", 0) >= MIN_VIEW_FOR_GROWTH_RATE ] def sort_key(item: Mapping[str, Any]): if sort == "added": return (int(item["first_seen"]), int(item["pubdate"])) if sort == "published": return (int(item["pubdate"]), int(item["first_seen"])) if sort == "heat": return item["heat"] if item["heat"] is not None else float("-inf") if sort == "interaction": return item["interaction_rate"] if item["interaction_rate"] is not None else float("-inf") if sort in {"growth", "growth_rate"}: field = "delta" if sort == "growth" else "rate" value = item.get("growth", {}).get(field) return value if value is not None else float("-inf") value = (item["metrics"] or {}).get(sort) return value if value is not None else -1 items.sort(key=lambda item: item["aid"]) items.sort(key=sort_key, reverse=True) total = len(items) growth_available = any( item.get("growth", {}).get("available") for item in items ) if not export: start = (page - 1) * page_size items = items[start:start + page_size] available = True reason = None if sort in {"growth", "growth_rate"} and not growth_available: available = False reason = "需要至少两天的快照才能计算增长" return { "sort": sort, "within_days": within_days, "growth_days": growth_days, "page": page, "page_size": page_size, "total": total, "pages": max(1, math.ceil(total / page_size)) if total else 0, "items": items, "available": available, "reason": reason, } return self.db.read(run) def video_detail(self, aid: int) -> dict[str, Any] | None: aid = int(aid) def run(conn: sqlite3.Connection) -> dict[str, Any] | None: video_rows = self._videos(conn) video = next((item for item in video_rows if int(item["aid"]) == aid), None) if video is None: row = conn.execute( "SELECT v.*, f.title AS folder_title, f.year AS folder_year " "FROM video v LEFT JOIN folder f ON f.media_id = v.folder_id " "WHERE v.aid = ?", (aid,), ).fetchone() if row is None: return None video = dict(row) snapshots = self._snapshot_map(conn, [aid]) snapshot = snapshots.get(aid) previous = self._snapshot_map(conn, [aid], before=snapshot["stat_date"] if snapshot else None).get(aid) _, _, heat, percentiles = self._global_heat(conn) growth = self._growth_for(snapshot, previous) item = self._item( video, snapshot, heat=heat.get(aid), percentile=percentiles.get(aid), ) item["growth"] = growth.as_dict() item["snapshot_count"] = int(conn.execute( "SELECT COUNT(*) FROM snapshot WHERE aid = ?", (aid,) ).fetchone()[0]) item["blacklisted"] = bool(conn.execute( "SELECT 1 FROM up_blacklist WHERE mid = ?", (video.get("up_mid"),) ).fetchone()) if video.get("up_mid") is not None else False item["url"] = ( f"https://www.bilibili.com/video/{video.get('bvid')}" if video.get("bvid") else None ) return item return self.db.read(run) def video_trend(self, aid: int, *, days: int | None = 30) -> dict[str, Any]: aid = int(aid) if days is not None and days < 1: raise ValueError("days 必须大于 0") def run(conn: sqlite3.Connection) -> dict[str, Any]: latest = self._latest_date(conn) rows = conn.execute( "SELECT * FROM snapshot WHERE aid = ? ORDER BY stat_date", (aid,) ).fetchall() points = [dict(row) for row in rows] if latest is None or not points: return { "aid": aid, "days": days, "dates": [], "series": {metric: [] for metric in METRICS}, "points": [], "available": False, } requested_start = ( _date_minus(latest, days - 1) if days is not None else str(points[0]["stat_date"]) ) first_point = str(points[0]["stat_date"]) start = max(requested_start, first_point) dates = _date_range(start, latest) by_date = {str(point["stat_date"]): point for point in points} series: dict[str, list[int | None]] = {} for metric in METRICS: values: list[int | None] = [] for day in dates: point = by_date.get(day) if point is None or int(point["ok"]) != 1 or point.get(metric) is None: values.append(None) else: values.append(int(point[metric])) series[metric] = values filtered_points = [ point for point in points if str(point["stat_date"]) in set(dates) ] return { "aid": aid, "days": days, "dates": dates, "series": series, "points": filtered_points, "available": any(point["ok"] == 1 for point in filtered_points), } return self.db.read(run) # ------------------------------------------------------------ ups def list_ups( self, *, sort: str = "views", within_days: int | None = None, q: str | None = None, min_videos: int = 1, page: int = 1, page_size: int = 20, export: bool = False, ) -> dict[str, Any]: if sort not in UP_SORTS: raise ValueError(f"不支持的 UP 排序: {sort}") page = max(1, int(page)) page_size = max(1, min(int(page_size), 200 if not export else 10_000)) min_videos = max(1, int(min_videos)) def run(conn: sqlite3.Connection) -> dict[str, Any]: latest = self._latest_date(conn) or self._today() videos = self._videos( conn, q=q, within_field="pubdate" if within_days is not None else None, within_start=( _date_minus(latest, int(within_days) - 1) if within_days is not None else None ), ) snapshots = self._snapshot_map(conn, [int(video["aid"]) for video in videos]) groups: dict[int, dict[str, Any]] = {} for video in videos: mid = video.get("up_mid") if mid is None: continue mid = int(mid) group = groups.setdefault(mid, { "up_mid": mid, "up_name": video.get("up_name") or f"UP {mid}", "videos": [], }) metrics = _metric_map(snapshots.get(int(video["aid"]))) group["videos"].append((video, metrics)) items = [] for group in groups.values(): group_videos = group["videos"] metrics_list = [metrics for _, metrics in group_videos if metrics] total_views = sum(metrics["view"] for metrics in metrics_list) total_interactions = sum( metrics["like"] + metrics["coin"] + metrics["favorite"] for metrics in metrics_list ) rates = [ interaction_rate(metrics) for metrics in metrics_list if interaction_rate(metrics) is not None ] best = max( group_videos, key=lambda pair: (pair[1] or {}).get("view", 0), default=None, ) items.append({ "up_mid": group["up_mid"], "up_name": group["up_name"], "video_count": len(group_videos), "total_views": total_views, "avg_views": round(total_views / len(metrics_list), 1) if metrics_list else None, "total_interactions": total_interactions, "avg_interaction": round(sum(rates) / len(rates), 6) if rates else None, "latest_pubdate": max(int(video["pubdate"]) for video, _ in group_videos), "first_pubdate": min(int(video["pubdate"]) for video, _ in group_videos), "best_video": None if best is None else { "aid": int(best[0]["aid"]), "title": best[0].get("title"), "cover": normalize_cover(best[0].get("cover")), "view": (best[1] or {}).get("view"), }, }) items = [item for item in items if item["video_count"] >= min_videos] sort_map = { "views": lambda item: item["total_views"], "count": lambda item: item["video_count"], "avg_views": lambda item: item["avg_views"] or 0, "interaction": lambda item: item["total_interactions"], "avg_interaction": lambda item: item["avg_interaction"] or 0, "latest": lambda item: item["latest_pubdate"], } key = sort_map[sort] items.sort(key=lambda item: (key(item), item["up_mid"]), reverse=True) total = len(items) if not export: start = (page - 1) * page_size items = items[start:start + page_size] return { "sort": sort, "within_days": within_days, "min_videos": min_videos, "page": page, "page_size": page_size, "total": total, "pages": max(1, math.ceil(total / page_size)) if total else 0, "items": items, } return self.db.read(run) def up_detail(self, mid: int) -> dict[str, Any] | None: mid = int(mid) def run(conn: sqlite3.Connection) -> dict[str, Any] | None: latest = self._latest_date(conn) or self._today() videos = self._videos(conn, up_mid=mid) if not videos: row = conn.execute( "SELECT 1 FROM up_blacklist WHERE mid = ?", (mid,) ).fetchone() if row is None: return None return { "up_mid": mid, "up_name": f"UP {mid}", "videos": [], "video_count": 0, "blacklisted": True, "series": {"dates": [], "views": []}, } snapshots = self._snapshot_map(conn, [int(video["aid"]) for video in videos]) items = [ self._item(video, snapshots.get(int(video["aid"]))) for video in videos ] metrics_list = [item["metrics"] for item in items if item["metrics"]] rates = [item["interaction_rate"] for item in items if item["interaction_rate"] is not None] total_views = sum(metrics["view"] for metrics in metrics_list) total_interactions = sum( metrics["like"] + metrics["coin"] + metrics["favorite"] for metrics in metrics_list ) aids = [int(video["aid"]) for video in videos] rows = conn.execute( f"SELECT stat_date, SUM(view) AS view FROM snapshot " f"WHERE ok = 1 AND aid IN ({','.join('?' for _ in aids)}) " "GROUP BY stat_date ORDER BY stat_date", aids, ).fetchall() view_by_date = {str(row["stat_date"]): int(row["view"] or 0) for row in rows} dates = _date_range(min(view_by_date), latest) if view_by_date else [] items.sort(key=lambda item: ((item["metrics"] or {}).get("view", 0), item["pubdate"]), reverse=True) folder_titles = sorted({ str(item["folder_title"]) for item in items if item.get("folder_title") }) return { "up_mid": mid, "up_name": items[0].get("up_name") or f"UP {mid}", "video_count": len(items), "total_views": total_views, "avg_views": round(total_views / len(metrics_list), 1) if metrics_list else None, "total_interactions": total_interactions, "avg_interaction": round(sum(rates) / len(rates), 6) if rates else None, "first_pubdate": min(item["pubdate"] for item in items), "latest_pubdate": max(item["pubdate"] for item in items), "folders": folder_titles, "blacklisted": bool(conn.execute( "SELECT 1 FROM up_blacklist WHERE mid = ?", (mid,) ).fetchone()), "space_url": f"https://space.bilibili.com/{mid}", "videos": items, "series": { "dates": dates, "views": [view_by_date.get(day) for day in dates], }, } return self.db.read(run) # ------------------------------------------------------------ fans def fans(self, mid: int | None = None, *, days: int = 90) -> dict[str, Any]: if days < 1: raise ValueError("days 必须大于 0") def run(conn: sqlite3.Connection) -> dict[str, Any]: cfg = self.config_store.get() mids = list(cfg.track_fans) if not mids: mids = [ int(row["mid"]) for row in conn.execute( "SELECT DISTINCT mid FROM up_stat ORDER BY mid" ).fetchall() ] selected = int(mid) if mid is not None else (mids[0] if mids else None) options = [ {"mid": value, "name": TRACK_FANS_NAMES.get(value, f"UP {value}")} for value in mids ] if selected is None: return { "mid": None, "name": None, "available": False, "options": options, "dates": [], "series": [], "points": [], "days": days, } rows = [ dict(row) for row in conn.execute( "SELECT * FROM up_stat WHERE mid = ? ORDER BY stat_date", (selected,), ).fetchall() ] if not rows: return { "mid": selected, "name": TRACK_FANS_NAMES.get(selected, f"UP {selected}"), "available": False, "options": options, "dates": [], "series": [], "points": [], "days": days, } latest = max(str(row["stat_date"]) for row in rows if int(row["ok"]) == 1) if any(int(row["ok"]) == 1 for row in rows) else str(rows[-1]["stat_date"]) first_date = str(rows[0]["stat_date"]) start = max(_date_minus(latest, days - 1), first_date) dates = _date_range(start, latest) by_date = {str(row["stat_date"]): row for row in rows} series = [] for day in dates: row = by_date.get(day) if row is None or int(row["ok"]) != 1: series.append(None) else: series.append(row["follower"]) ok_rows = [row for row in rows if int(row["ok"]) == 1] current = ok_rows[-1] if ok_rows else rows[-1] previous = ok_rows[-2] if len(ok_rows) >= 2 else None cutoff = _date_minus(latest, 7) baselines = [row for row in ok_rows if str(row["stat_date"]) <= cutoff] baseline = baselines[-1] if baselines else None first = ok_rows[0] if ok_rows else rows[0] return { "mid": selected, "name": TRACK_FANS_NAMES.get(selected, f"UP {selected}"), "available": bool(ok_rows), "options": options, "days": days, "dates": dates, "series": series, "points": rows, "stat_date": current["stat_date"], "follower": current.get("follower"), "following": current.get("following"), "delta_1d": None if previous is None else int(current.get("follower") or 0) - int(previous.get("follower") or 0), "delta_7d": None if baseline is None else int(current.get("follower") or 0) - int(baseline.get("follower") or 0), "delta_cum": int(current.get("follower") or 0) - int(first.get("follower") or 0), "first_date": first["stat_date"], } return self.db.read(run) # ------------------------------------------------------------ candidates def candidates(self) -> dict[str, Any]: def run(conn: sqlite3.Connection) -> dict[str, Any]: cfg = self.config_store.get() now = int(self._now()) ttl = int(cfg.schedule.candidate_ttl_days) rows = conn.execute( "SELECT * FROM candidate ORDER BY " "CASE WHEN decision IS NULL THEN 0 ELSE 1 END, " "COALESCE(score, 0) DESC, first_seen DESC, aid" ).fetchall() items = [] for row in rows: data = dict(row) first_seen = int(data.get("first_seen") or 0) expires_at = first_seen + ttl * 86400 items.append({ **data, "cover": normalize_cover(data.get("cover")), "expires_at": expires_at, "days_left": max(0, math.ceil((expires_at - now) / 86400)), "url": ( f"https://www.bilibili.com/video/{data.get('bvid')}" if data.get("bvid") else None ), }) pending = sum(1 for item in items if item.get("decision") is None) return { "items": items, "total": len(items), "pending": pending, "ttl_days": ttl, "telegram_enabled": bool(cfg.telegram.enabled), "now": now, } return self.db.read(run) # ------------------------------------------------------------ system def system(self) -> dict[str, Any]: def run(conn: sqlite3.Connection) -> dict[str, Any]: cfg = self.config_store.get() latest = self._latest_date(conn) snapshot_days = self._snapshot_days(conn) health = {"total": 0, "ok": 0, "failed": 0} if latest: rows = conn.execute( "SELECT ok, COUNT(*) AS n FROM snapshot WHERE stat_date = ? GROUP BY ok", (latest,), ).fetchall() by_ok = {int(row["ok"]): int(row["n"]) for row in rows} health = { "total": sum(by_ok.values()), "ok": by_ok.get(1, 0), "failed": by_ok.get(0, 0), } folders = self._folders_overview(conn) blacklist = [ dict(row) for row in conn.execute( """ SELECT b.*, (SELECT COUNT(*) FROM video v WHERE v.up_mid = b.mid AND v.status IN ('active','unfaved')) AS video_count FROM up_blacklist b ORDER BY b.created_at, b.mid """ ).fetchall() ] tasks = self._task_records(conn) now = self._now() next_runs = { "discover": int(now + cfg.schedule.discover_minutes * 60), "folder_sync": int(now + cfg.schedule.folder_sync_minutes * 60), "snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)), } for name, key, interval_minutes in ( ("discover", "discovery", cfg.schedule.discover_minutes), ("folder_sync", "folder_sync", cfg.schedule.folder_sync_minutes), ): record = tasks.get(key) if isinstance(record, dict) and record.get("finished_at"): interval = int(interval_minutes) * 60 candidate = int(record["finished_at"]) + interval while candidate <= now: candidate += interval next_runs[name] = candidate errors = [ {"task": name, **record} for name, record in tasks.items() if isinstance(record, dict) and record.get("ok") is False ] counts = {str(row["status"]): int(row["n"]) for row in conn.execute( "SELECT status, COUNT(*) AS n FROM video GROUP BY status" ).fetchall()} return { "generated_at": int(now), "today": self._today(), "data_date": latest, "snapshot_days": snapshot_days, "snapshot_health": health, "counts": counts, "folders": folders, "blacklist": blacklist, "tasks": tasks, "errors": errors, "next_runs": next_runs, "auth": { "ok": self._kv(conn, "auth:ok"), "message": self._kv(conn, "auth:message"), "last_check": self._kv(conn, "auth:last_check"), "mid": self._kv(conn, "auth:mid"), }, "backup": { "last_completed": self._kv(conn, "snapshot:last_completed"), "last_attempt": self._kv(conn, "snapshot:last_attempt"), "ok": self._kv(conn, f"snapshot:backup:{latest}" if latest else "snapshot:backup"), "error": self._kv(conn, f"snapshot:backup_error:{latest}" if latest else "snapshot:backup_error"), }, "preview": self._kv(conn, "preview:seeded") == "1", "schedule": { "discover_minutes": cfg.schedule.discover_minutes, "folder_sync_minutes": cfg.schedule.folder_sync_minutes, "snapshot_at": cfg.schedule.snapshot_at, "candidate_ttl_days": cfg.schedule.candidate_ttl_days, }, "telegram_enabled": bool(cfg.telegram.enabled), } return self.db.read(run) def export_videos(self, **filters) -> dict[str, Any]: filters.pop("page", None) filters.pop("page_size", None) return self.list_videos(**filters, export=True)