feat(stats): buffer visits in memory and expose POST /api/visit + /api/stats
This commit is contained in:
kairee committed
1 parent
cdd11823b9
commit
3c16caac1a
8 files changed
+905
-5
No files matched your search
@@ -1560,6 +1560,91 @@ class DashboardQueries:
|
||||
}
|
||||
return self.db.read(run)
|
||||
|
||||
# ------------------------------------------------------------ stats
|
||||
|
||||
def stats(
|
||||
self,
|
||||
*,
|
||||
days: int = 30,
|
||||
pending: Sequence[Mapping[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""访问统计：区间内每日 PV/UV 与 Top 页面。
|
||||
|
||||
``pending`` 为内存缓冲中尚未落库的上报行；与已落库数据按
|
||||
(日期, IP, 路由) 合并去重后统一聚合，因此页面基本是实时可见的。
|
||||
"""
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
||||
cfg = self.config_store.get()
|
||||
today = self._today()
|
||||
start = _date_minus(today, int(days) - 1)
|
||||
rows: list[Mapping[str, Any]] = [
|
||||
dict(row) for row in conn.execute(
|
||||
"SELECT stat_date, ip, path, hits, first_at, last_at FROM visit "
|
||||
"WHERE stat_date >= ? AND stat_date <= ?",
|
||||
(start, today),
|
||||
).fetchall()
|
||||
]
|
||||
if pending:
|
||||
rows.extend(dict(row) for row in pending)
|
||||
|
||||
buckets: dict[tuple[str, str, str], list[int]] = {}
|
||||
for row in rows:
|
||||
day = str(row["stat_date"])
|
||||
ip = str(row["ip"])
|
||||
path = str(row["path"])
|
||||
hits = int(row.get("hits") or 0)
|
||||
first = int(row.get("first_at") or 0)
|
||||
last = int(row.get("last_at") or 0)
|
||||
bucket = buckets.get((day, ip, path))
|
||||
if bucket is None:
|
||||
buckets[(day, ip, path)] = [hits, first, last]
|
||||
else:
|
||||
bucket[0] += hits
|
||||
bucket[1] = min(bucket[1], first) if first else bucket[1]
|
||||
bucket[2] = max(bucket[2], last)
|
||||
|
||||
dates = _date_range(start, today)
|
||||
daily_pv = {day: 0 for day in dates}
|
||||
daily_uv: dict[str, set[str]] = {day: set() for day in dates}
|
||||
page_pv: dict[str, int] = {}
|
||||
page_uv: dict[str, set[str]] = {}
|
||||
total_pv = 0
|
||||
total_ips: set[str] = set()
|
||||
for (day, ip, path), bucket in buckets.items():
|
||||
if day not in daily_pv:
|
||||
continue
|
||||
daily_pv[day] += bucket[0]
|
||||
daily_uv[day].add(ip)
|
||||
total_pv += bucket[0]
|
||||
total_ips.add(ip)
|
||||
page_pv[path] = page_pv.get(path, 0) + bucket[0]
|
||||
page_uv.setdefault(path, set()).add(ip)
|
||||
|
||||
top_pages = [
|
||||
{"path": path, "pv": hits, "uv": len(page_uv[path])}
|
||||
for path, hits in sorted(
|
||||
page_pv.items(), key=lambda kv: (-kv[1], kv[0])
|
||||
)[:10]
|
||||
]
|
||||
return {
|
||||
"enabled": bool(cfg.web.track.enabled),
|
||||
"days": int(days),
|
||||
"range": {"start": start, "end": today},
|
||||
"today": today,
|
||||
"today_pv": daily_pv.get(today, 0),
|
||||
"today_uv": len(daily_uv.get(today, set())),
|
||||
"total_pv": total_pv,
|
||||
"total_uv": len(total_ips),
|
||||
"series": {
|
||||
"dates": dates,
|
||||
"pv": [daily_pv[day] for day in dates],
|
||||
"uv": [len(daily_uv[day]) for day in dates],
|
||||
},
|
||||
"top_pages": top_pages,
|
||||
"pending": bool(pending),
|
||||
}
|
||||
return self.db.read(run)
|
||||
|
||||
def candidates(self) -> dict[str, Any]:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
||||
cfg = self.config_store.get()
|
||||
|
||||
@@ -21,15 +21,20 @@ from typing import Any, Callable, Mapping
|
||||
from urllib.parse import parse_qs, quote, unquote, urlsplit
|
||||
|
||||
from ..analytics import is_loopback
|
||||
from ..client_ip import extract_client_ip, is_bot, normalize_route, parse_networks
|
||||
from ..config import ConfigStore
|
||||
from ..database import Database
|
||||
from .queries import DashboardQueries
|
||||
from .visits import VisitCollector
|
||||
|
||||
STATIC_ROOT = Path(__file__).resolve().parent / "static"
|
||||
COOKIE_NAME = "rb_session"
|
||||
SESSION_TTL = 7 * 86400
|
||||
PROTECTED_API = {"/api/candidates", "/api/system"}
|
||||
PROTECTED_API = {"/api/candidates", "/api/system", "/api/stats"}
|
||||
PROTECTED_PAGES = {"/candidates", "/system"}
|
||||
# 显式放行的 POST 端点：访问统计埋点（业务写操作仍统一 405）。
|
||||
POST_API = {"/api/visit"}
|
||||
VISIT_BODY_LIMIT = 2048
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
@@ -49,6 +54,7 @@ class WebContext:
|
||||
clock: Callable[[], float] = time.time
|
||||
db_path: str | None = None
|
||||
schema_version: int | None = None
|
||||
visits: VisitCollector | None = None
|
||||
|
||||
@property
|
||||
def auth_token(self) -> str:
|
||||
@@ -63,6 +69,13 @@ class DashboardHTTPServer(ThreadingHTTPServer):
|
||||
self.context = context
|
||||
super().__init__(address, DashboardHandler)
|
||||
|
||||
def server_close(self) -> None:
|
||||
# 关闭前停止访问统计刷新线程，保证最后一次缓冲落库。
|
||||
collector = getattr(self.context, "visits", None)
|
||||
if collector is not None:
|
||||
collector.stop()
|
||||
super().server_close()
|
||||
|
||||
|
||||
class DashboardHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
@@ -142,6 +155,16 @@ class DashboardHandler(BaseHTTPRequestHandler):
|
||||
self._send_error(HTTPStatus.UNAUTHORIZED, "unauthorized", "需要登录")
|
||||
return
|
||||
if method == "POST":
|
||||
if path in POST_API:
|
||||
self._api_post(path, query)
|
||||
return
|
||||
self._send_error(
|
||||
HTTPStatus.METHOD_NOT_ALLOWED,
|
||||
"method_not_allowed",
|
||||
"不支持该操作",
|
||||
)
|
||||
return
|
||||
if path in POST_API:
|
||||
self._send_error(
|
||||
HTTPStatus.METHOD_NOT_ALLOWED,
|
||||
"method_not_allowed",
|
||||
@@ -285,6 +308,61 @@ border-color:#3a2c22;color:#f3e9dc}button{background:#e9b97c;color:#2a1a10}}
|
||||
|
||||
# ------------------------------------------------------------ API
|
||||
|
||||
def _api_post(self, path: str, query: Mapping[str, list[str]]) -> None:
|
||||
if path == "/api/visit":
|
||||
self._visit_post(query)
|
||||
return
|
||||
raise ApiError(HTTPStatus.METHOD_NOT_ALLOWED, "method_not_allowed", "不支持该操作")
|
||||
|
||||
def _visit_post(self, query: Mapping[str, list[str]]) -> None:
|
||||
"""记录一次页面浏览；前端 sendBeacon 上报，写内存后定期批量落库。"""
|
||||
track = self.context.config_store.get().web.track
|
||||
collector = self.context.visits
|
||||
if not track.enabled or collector is None:
|
||||
self._send_json(HTTPStatus.OK, {"recorded": False, "reason": "disabled"})
|
||||
return
|
||||
|
||||
path_value = _param(query, "path", "")
|
||||
if not path_value:
|
||||
body = self._read_small_body()
|
||||
if body:
|
||||
path_value = _json_path(body)
|
||||
|
||||
user_agent = self.headers.get("User-Agent")
|
||||
if not track.count_bots and is_bot(user_agent):
|
||||
self._send_json(HTTPStatus.OK, {"recorded": False, "reason": "bot"})
|
||||
return
|
||||
|
||||
peer = self.client_address[0] if self.client_address else None
|
||||
ip = extract_client_ip(
|
||||
self.headers,
|
||||
peer,
|
||||
trusted_proxies=parse_networks(track.trusted_proxies),
|
||||
header_order=track.header_order,
|
||||
)
|
||||
if not ip:
|
||||
self._send_json(HTTPStatus.OK, {"recorded": False, "reason": "no_ip"})
|
||||
return
|
||||
|
||||
route = normalize_route(path_value)
|
||||
recorded = collector.record(ip, route)
|
||||
self._send_json(HTTPStatus.OK, {
|
||||
"recorded": bool(recorded),
|
||||
"route": route,
|
||||
"reason": None if recorded else "capped",
|
||||
})
|
||||
|
||||
def _read_small_body(self) -> bytes:
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except (TypeError, ValueError):
|
||||
length = 0
|
||||
if length <= 0:
|
||||
return b""
|
||||
if length > VISIT_BODY_LIMIT:
|
||||
raise ApiError(HTTPStatus.BAD_REQUEST, "bad_request", "请求体过大")
|
||||
return self.rfile.read(length)
|
||||
|
||||
def _api(self, path: str, query: Mapping[str, list[str]], method: str) -> None:
|
||||
queries = self.context.queries
|
||||
if path == "/api/overview":
|
||||
@@ -392,6 +470,16 @@ border-color:#3a2c22;color:#f3e9dc}button{background:#e9b97c;color:#2a1a10}}
|
||||
if path == "/api/about":
|
||||
self._send_json(HTTPStatus.OK, queries.about())
|
||||
return
|
||||
if path == "/api/stats":
|
||||
pending = (
|
||||
self.context.visits.pending_rows()
|
||||
if self.context.visits is not None else []
|
||||
)
|
||||
self._send_json(HTTPStatus.OK, queries.stats(
|
||||
days=_int_param(query, "days", 30, 1, 3650),
|
||||
pending=pending,
|
||||
))
|
||||
return
|
||||
if path == "/api/candidates":
|
||||
self._send_json(HTTPStatus.OK, queries.candidates())
|
||||
return
|
||||
@@ -510,6 +598,22 @@ def _safe_next(value: str | None) -> str:
|
||||
return text
|
||||
|
||||
|
||||
def _json_path(body: bytes) -> str:
|
||||
"""从 JSON（或表单编码）请求体里取 ``path`` 字段；失败返回空串。"""
|
||||
text = body.decode("utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return ""
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except (ValueError, TypeError):
|
||||
data = None
|
||||
if isinstance(data, Mapping):
|
||||
value = data.get("path")
|
||||
return value if isinstance(value, str) else ""
|
||||
values = parse_qs(text).get("path")
|
||||
return values[0] if values else ""
|
||||
|
||||
|
||||
def _param(query: Mapping[str, list[str]], name: str, default: str) -> str:
|
||||
values = query.get(name)
|
||||
return default if not values else str(values[0])
|
||||
@@ -604,6 +708,15 @@ def create_server(
|
||||
)
|
||||
if preview and not _preview_marker(db):
|
||||
raise ValueError("目标数据库不是预览库（缺少 preview:seeded=1 标记）")
|
||||
track = cfg.web.track
|
||||
visits = VisitCollector(
|
||||
db,
|
||||
clock=clock,
|
||||
flush_seconds=track.flush_seconds,
|
||||
keep_days=track.keep_days,
|
||||
max_hits_per_key=track.max_hits_per_ip_path_per_day,
|
||||
logger=logger,
|
||||
)
|
||||
context = WebContext(
|
||||
queries=DashboardQueries(db, config_store, clock=clock),
|
||||
config_store=config_store,
|
||||
@@ -612,8 +725,11 @@ def create_server(
|
||||
clock=clock,
|
||||
db_path=str(db.path),
|
||||
schema_version=db.schema_version,
|
||||
visits=visits,
|
||||
)
|
||||
return DashboardHTTPServer((bind_host, bind_port), context)
|
||||
server = DashboardHTTPServer((bind_host, bind_port), context)
|
||||
visits.start()
|
||||
return server
|
||||
|
||||
|
||||
def serve_forever(server: DashboardHTTPServer, *, logger: Any = None) -> None:
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""访问统计的内存缓冲与批量入库。
|
||||
|
||||
前端通过 ``POST /api/visit`` 上报页面浏览。为降低 SQLite 压力，上报先写入进程内
|
||||
聚合缓冲，再由后台线程按固定周期（默认 60 秒）批量 upsert 到 ``visit`` 表。按
|
||||
(日期, IP, 路由) 聚合，同一分钟的重复浏览只累加计数。
|
||||
|
||||
进程被强杀时最多丢失一个刷新周期的数据，属可接受范围。批量写入经 Database 的
|
||||
单写线程串行执行，网络/请求路径不直接触碰数据库。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
from ..database import Database
|
||||
from ..timeutil import shanghai_date
|
||||
|
||||
# 缓冲里每个键保存 [hits, first_at, last_at]。
|
||||
_UPSERT_SQL = """
|
||||
INSERT INTO visit(stat_date, ip, path, hits, first_at, last_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(stat_date, ip, path) DO UPDATE SET
|
||||
hits = visit.hits + excluded.hits,
|
||||
first_at = MIN(visit.first_at, excluded.first_at),
|
||||
last_at = MAX(visit.last_at, excluded.last_at)
|
||||
"""
|
||||
|
||||
|
||||
class VisitCollector:
|
||||
def __init__(
|
||||
self,
|
||||
db: Database,
|
||||
*,
|
||||
clock: Callable[[], float] = time.time,
|
||||
flush_seconds: int = 60,
|
||||
keep_days: int = 180,
|
||||
max_hits_per_key: int = 0,
|
||||
logger: Any = None,
|
||||
) -> None:
|
||||
self._db = db
|
||||
self._clock = clock
|
||||
self._flush_seconds = max(0, int(flush_seconds))
|
||||
self._keep_days = max(1, int(keep_days))
|
||||
self._max_hits = max(0, int(max_hits_per_key))
|
||||
self._logger = logger
|
||||
self._lock = threading.RLock()
|
||||
self._buckets: dict[tuple[str, str, str], list[int]] = {}
|
||||
self._last_prune_date: str | None = None
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
# ------------------------------------------------------------ buffering
|
||||
|
||||
def record(self, ip: str, path: str, *, now: float | None = None) -> bool:
|
||||
"""把一次浏览累加到内存缓冲；超过单键上限时返回 ``False``。"""
|
||||
if not ip or not path:
|
||||
return False
|
||||
ts = int(self._clock() if now is None else now)
|
||||
key = (shanghai_date(ts), ip, path)
|
||||
with self._lock:
|
||||
bucket = self._buckets.get(key)
|
||||
if bucket is None:
|
||||
bucket = [0, ts, ts]
|
||||
self._buckets[key] = bucket
|
||||
if self._max_hits and bucket[0] >= self._max_hits:
|
||||
return False
|
||||
bucket[0] += 1
|
||||
bucket[1] = min(bucket[1], ts)
|
||||
bucket[2] = max(bucket[2], ts)
|
||||
return True
|
||||
|
||||
def pending_rows(self) -> list[dict[str, Any]]:
|
||||
"""返回未落库缓冲的快照，供 /api/stats 合并展示。"""
|
||||
with self._lock:
|
||||
return [
|
||||
{
|
||||
"stat_date": day,
|
||||
"ip": ip,
|
||||
"path": path,
|
||||
"hits": bucket[0],
|
||||
"first_at": bucket[1],
|
||||
"last_at": bucket[2],
|
||||
}
|
||||
for (day, ip, path), bucket in self._buckets.items()
|
||||
]
|
||||
|
||||
@property
|
||||
def pending_keys(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._buckets)
|
||||
|
||||
# ------------------------------------------------------------ flush
|
||||
|
||||
def flush(self) -> int:
|
||||
"""把当前缓冲批量写入数据库，返回写入的聚合行数。"""
|
||||
with self._lock:
|
||||
if not self._buckets:
|
||||
return 0
|
||||
take = self._buckets
|
||||
self._buckets = {}
|
||||
rows = [
|
||||
(day, ip, path, bucket[0], bucket[1], bucket[2])
|
||||
for (day, ip, path), bucket in take.items()
|
||||
]
|
||||
|
||||
def write(conn) -> None:
|
||||
conn.executemany(_UPSERT_SQL, rows)
|
||||
|
||||
try:
|
||||
self._db.transaction(write)
|
||||
except Exception as exc: # noqa: BLE001 写失败不应影响服务
|
||||
# 按用户容忍度可丢弃，但内存极小，这里并回缓冲等待下次重试。
|
||||
with self._lock:
|
||||
self._merge_back(take)
|
||||
self._log("error", "visit.flush_error", error=str(exc))
|
||||
return 0
|
||||
self._maybe_prune()
|
||||
self._log(
|
||||
"info", "visit.flush",
|
||||
rows=len(rows), hits=sum(row[3] for row in rows),
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
def _merge_back(self, take: dict[tuple[str, str, str], list[int]]) -> None:
|
||||
for key, bucket in take.items():
|
||||
current = self._buckets.get(key)
|
||||
if current is None:
|
||||
self._buckets[key] = list(bucket)
|
||||
else:
|
||||
current[0] += bucket[0]
|
||||
current[1] = min(current[1], bucket[1])
|
||||
current[2] = max(current[2], bucket[2])
|
||||
|
||||
def _maybe_prune(self) -> None:
|
||||
if self._keep_days <= 0:
|
||||
return
|
||||
today = shanghai_date(self._clock())
|
||||
if today == self._last_prune_date:
|
||||
return
|
||||
cutoff = shanghai_date(
|
||||
self._clock() - self._keep_days * 86400
|
||||
)
|
||||
|
||||
def prune(conn) -> int:
|
||||
cur = conn.execute(
|
||||
"DELETE FROM visit WHERE stat_date < ?", (cutoff,)
|
||||
)
|
||||
return int(cur.rowcount or 0)
|
||||
|
||||
try:
|
||||
removed = self._db.write(prune)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._log("error", "visit.prune_error", error=str(exc))
|
||||
return
|
||||
self._last_prune_date = today
|
||||
if removed:
|
||||
self._log("info", "visit.prune", removed=removed, cutoff=cutoff)
|
||||
|
||||
# ------------------------------------------------------------ lifecycle
|
||||
|
||||
def start(self) -> None:
|
||||
if self._flush_seconds <= 0 or self._thread is not None:
|
||||
return
|
||||
self._stop.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._loop, name="visit-flusher", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _loop(self) -> None:
|
||||
while not self._stop.wait(self._flush_seconds):
|
||||
try:
|
||||
self.flush()
|
||||
except Exception as exc: # noqa: BLE001 后台线程自保
|
||||
self._log("error", "visit.flush_loop_error", error=str(exc))
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止后台线程并做最后一次 flush。"""
|
||||
self._stop.set()
|
||||
thread = self._thread
|
||||
if thread is not None:
|
||||
thread.join(timeout=5)
|
||||
self._thread = None
|
||||
try:
|
||||
self.flush()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._log("error", "visit.stop_flush_error", error=str(exc))
|
||||
|
||||
def _log(self, level: str, event: str, **fields: Any) -> None:
|
||||
if self._logger is None:
|
||||
return
|
||||
method = getattr(self._logger, level, None)
|
||||
if method is not None:
|
||||
method(event, **fields)
|
||||
Reference in new issue
Block a user