fix(up): hard timeout and progress for follower capture
This commit is contained in:
1 parent
98cf42a113
commit
e903c49921
5 files changed
+88
-3
No files matched your search
@@ -11,6 +11,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
@@ -42,11 +43,15 @@ class UpProfileService:
|
||||
*,
|
||||
clock=time.time,
|
||||
logger=None,
|
||||
request_timeout: float = 45.0,
|
||||
heartbeat: int = 25,
|
||||
) -> None:
|
||||
self.repo = repository
|
||||
self.client_provider = client_provider
|
||||
self._clock = clock
|
||||
self.logger = logger
|
||||
self.request_timeout = max(1.0, float(request_timeout))
|
||||
self.heartbeat = max(1, int(heartbeat))
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
@@ -55,6 +60,32 @@ class UpProfileService:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def _fetch_relation(self, mid: int):
|
||||
"""带硬超时的单条请求。
|
||||
|
||||
正常网络错误已由 ``BilibiliClient`` 的 timeout/retry 覆盖;但 DNS 解析或
|
||||
传输层偶发挂起可能不触发 timeout,这里用工作线程兜底,避免整批任务静默
|
||||
卡死。
|
||||
"""
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def work() -> None:
|
||||
try:
|
||||
box["value"] = self.client_provider().relation_stat(mid)
|
||||
except BaseException as exc: # noqa: BLE001 结果由主线程统一处理
|
||||
box["error"] = exc
|
||||
|
||||
worker = threading.Thread(target=work, name=f"relation-{mid}", daemon=True)
|
||||
worker.start()
|
||||
worker.join(self.request_timeout)
|
||||
if worker.is_alive():
|
||||
raise TimeoutError(
|
||||
f"relation/stat timeout after {self.request_timeout:.0f}s"
|
||||
)
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
return box["value"]
|
||||
|
||||
def capture(
|
||||
self,
|
||||
mids: Iterable[int],
|
||||
@@ -86,9 +117,10 @@ class UpProfileService:
|
||||
return result
|
||||
now = self._now()
|
||||
names = names or {}
|
||||
for mid in ordered:
|
||||
total = len(ordered)
|
||||
for index, mid in enumerate(ordered, 1):
|
||||
try:
|
||||
fan = self.client_provider().relation_stat(mid)
|
||||
fan = self._fetch_relation(mid)
|
||||
updated = self.repo.apply_up_follower(
|
||||
mid=mid,
|
||||
name=names.get(mid) or getattr(fan, "uname", None),
|
||||
@@ -110,6 +142,11 @@ class UpProfileService:
|
||||
"info", "up_profile.captured",
|
||||
mid=mid, follower=int(fan.follower), updated=updated,
|
||||
)
|
||||
if index % self.heartbeat == 0 or index == total:
|
||||
self._log(
|
||||
"info", "up_profile.progress",
|
||||
processed=index, total=total, failed=result.failed,
|
||||
)
|
||||
return result
|
||||
|
||||
def backfill(
|
||||
|
||||
Reference in new issue
Block a user