fix(up): hard timeout and progress for follower capture

This commit is contained in:
kairee committed 2026-11-01 13:24:56 +08:00
1 parent 98cf42a113
commit e903c49921
5 files changed
+88 -3

No files matched your search

+39 -2
View File
@@ -11,6 +11,7 @@
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Callable, Iterable, Mapping
@@ -42,11 +43,15 @@ class UpProfileService:
*,
clock=time.time,
logger=None,
request_timeout: float = 45.0,
heartbeat: int = 25,
) -> None:
self.repo = repository
self.client_provider = client_provider
self._clock = clock
self.logger = logger
self.request_timeout = max(1.0, float(request_timeout))
self.heartbeat = max(1, int(heartbeat))
def _now(self) -> int:
return int(self._clock())
@@ -55,6 +60,32 @@ class UpProfileService:
if self.logger is not None:
getattr(self.logger, level)(event, **fields)
def _fetch_relation(self, mid: int):
"""带硬超时的单条请求。
正常网络错误已由 ``BilibiliClient`` 的 timeout/retry 覆盖;但 DNS 解析或
传输层偶发挂起可能不触发 timeout,这里用工作线程兜底,避免整批任务静默
卡死。
"""
box: dict[str, Any] = {}
def work() -> None:
try:
box["value"] = self.client_provider().relation_stat(mid)
except BaseException as exc: # noqa: BLE001 结果由主线程统一处理
box["error"] = exc
worker = threading.Thread(target=work, name=f"relation-{mid}", daemon=True)
worker.start()
worker.join(self.request_timeout)
if worker.is_alive():
raise TimeoutError(
f"relation/stat timeout after {self.request_timeout:.0f}s"
)
if "error" in box:
raise box["error"]
return box["value"]
def capture(
self,
mids: Iterable[int],
@@ -86,9 +117,10 @@ class UpProfileService:
return result
now = self._now()
names = names or {}
for mid in ordered:
total = len(ordered)
for index, mid in enumerate(ordered, 1):
try:
fan = self.client_provider().relation_stat(mid)
fan = self._fetch_relation(mid)
updated = self.repo.apply_up_follower(
mid=mid,
name=names.get(mid) or getattr(fan, "uname", None),
@@ -110,6 +142,11 @@ class UpProfileService:
"info", "up_profile.captured",
mid=mid, follower=int(fan.follower), updated=updated,
)
if index % self.heartbeat == 0 or index == total:
self._log(
"info", "up_profile.progress",
processed=index, total=total, failed=result.failed,
)
return result
def backfill(