kcloud/re-xxm
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(http): use httpx with pooled connections and browser headers

Browse Source 
Add httpx and route Bilibili and Telegram requests through pooled clients with keepalive and explicit timeouts. Align Bilibili headers with data/raw-request.txt: Chrome 155 UA, sec-ch-ua, space.bilibili.com origin, folder-specific referer, and priority header. Close replaced clients when credentials change. Add transport contract tests and live read-only validation for nav, folder sync, WBI search, and Telegram getUpdates.
This commit is contained in:
kairee committed 2026-10-31 22:28:32 +08:00
1 parent 71a0e8c1d6
commit 5b22ed1c5d
12 files changed
+430 -25
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

reaction_board/bilibili/auth.py
+5 -4
View File
@@ -45,15 +45,16 @@ def strip_identity(cookie: str) -> str:
return "; ".join(kept)
def make_headers(cfg: dict, auth: bool = False) -> dict[str, str]:
"""构造 B 站 Web 请求头；``auth=False`` 时去掉账号身份 cookie。"""
def make_headers(cfg: dict, auth: bool = False, *, referer: str | None = None) -> dict[str, str]:
"""构造与 ``data/raw-request.txt`` 对齐的 B 站 Web 请求头。"""
return {
"accept": "*/*",
"accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
"cookie": cfg["cookie"] if auth else strip_identity(cfg["cookie"]),
"dnt": "1",
"origin": "https://www.bilibili.com",
"referer": "https://www.bilibili.com/",
"origin": "https://space.bilibili.com",
"priority": "u=1, i",
"referer": referer or "https://space.bilibili.com/",
"sec-ch-ua": '"Google Chrome";v="155", "Chromium";v="155", "Not(A:Brand";v="24"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Windows"',
reaction_board/bilibili/client.py
+87 -14
View File
@@ -17,6 +17,8 @@ import urllib.request
from dataclasses import dataclass
from typing import Any, Mapping, Protocol, Sequence
import httpx
from .auth import make_headers
from .models import (
FanStat,
@@ -129,6 +131,54 @@ class UrllibTransport:
raise TransportError(str(exc)) from exc
class HttpxTransport:
"""基于 httpx 的连接池传输，默认用于 B 站请求。"""
def __init__(
self,
*,
timeout: float = 30,
client: httpx.Client | None = None,
) -> None:
self.timeout = timeout
self._client = client or httpx.Client(
timeout=httpx.Timeout(timeout, connect=10),
limits=httpx.Limits(
max_keepalive_connections=10,
keepalive_expiry=30,
),
follow_redirects=False,
trust_env=True,
)
def request(
self,
method: str,
url: str,
headers: Mapping[str, str],
data: bytes | None = None,
timeout: float = 30,
) -> HTTPResponse:
try:
response = self._client.request(
method,
url,
headers=dict(headers),
content=data,
timeout=timeout,
)
except httpx.TransportError as exc:
raise TransportError(str(exc)) from exc
return HTTPResponse(
status=int(response.status_code),
headers=dict(response.headers),
body=response.content,
)
def close(self) -> None:
self._client.close()
class Pacer:
"""请求间随机等待；第一个请求不等待。"""
@@ -185,7 +235,7 @@ class BilibiliClient:
self.cookie = cookie or ""
self.csrf = csrf or ""
self.mid = mid
self.transport = transport or UrllibTransport()
self.transport = transport or HttpxTransport()
self.pacer = pacer or Pacer(sleep=sleeper)
self._sleeper = sleeper
self.retries = retries
@@ -193,15 +243,22 @@ class BilibiliClient:
self.timeout = timeout
self.logger = logger
def close(self) -> None:
close = getattr(self.transport, "close", None)
if callable(close):
close()
# ------------------------------------------------------------ core
def _headers(self, auth: bool) -> dict[str, str]:
return make_headers({"cookie": self.cookie}, auth=auth)
def _headers(self, auth: bool, referer: str | None = None) -> dict[str, str]:
return make_headers({"cookie": self.cookie}, auth=auth, referer=referer)
def _request_json(self, url: str, *, auth: bool) -> dict[str, Any]:
def _request_json(
self, url: str, *, auth: bool, referer: str | None = None
) -> dict[str, Any]:
self.pacer.wait()
resp = self.transport.request(
"GET", url, self._headers(auth), data=None, timeout=self.timeout
"GET", url, self._headers(auth, referer), data=None, timeout=self.timeout
)
if resp.status >= 400:
raise BilibiliHTTPError(resp.status, resp.body.decode("utf-8", "replace")[:200])
@@ -210,11 +267,13 @@ class BilibiliClient:
raise BilibiliProtocolError("接口响应顶层不是 JSON 对象")
return payload
def _request_with_retry(self, url: str, *, auth: bool) -> dict[str, Any]:
def _request_with_retry(
self, url: str, *, auth: bool, referer: str | None = None
) -> dict[str, Any]:
last: BilibiliError | None = None
for attempt in range(self.retries):
try:
return self._request_json(url, auth=auth)
return self._request_json(url, auth=auth, referer=referer)
except (TransportError, BilibiliHTTPError) as exc:
last = exc
if not exc.retryable or attempt >= self.retries - 1:
@@ -238,17 +297,19 @@ class BilibiliClient:
raise BilibiliProtocolError("接口响应缺少整数 code")
return BilibiliAPIError(code, str(payload.get("message") or ""))
def get_json(self, url: str, *, auth: bool = False) -> dict[str, Any]:
def get_json(
self, url: str, *, auth: bool = False, referer: str | None = None
) -> dict[str, Any]:
"""GET JSON；匿名失败时自动带完整 Cookie 重试一次（与 favsort 一致）。"""
if auth:
payload = self._request_with_retry(url, auth=True)
payload = self._request_with_retry(url, auth=True, referer=referer)
if payload.get("code") != 0:
raise self._api_error(payload)
return payload
anon_error: BilibiliError | None = None
try:
payload = self._request_with_retry(url, auth=False)
payload = self._request_with_retry(url, auth=False, referer=referer)
if payload.get("code") == 0:
return payload
anon_error = self._api_error(payload)
@@ -257,7 +318,7 @@ class BilibiliClient:
if self.logger is not None and anon_error is not None:
self.logger.debug("bili.anon_fallback", error=str(anon_error))
payload = self._request_with_retry(url, auth=True)
payload = self._request_with_retry(url, auth=True, referer=referer)
if payload.get("code") != 0:
raise self._api_error(payload)
return payload
@@ -282,7 +343,11 @@ class BilibiliClient:
url = (
f"{BASE}/x/v3/fav/resource/ids?media_id={int(media_id)}&platform=web"
)
payload = self.get_json(url)
referer = (
f"https://space.bilibili.com/{self.mid}/favlist"
f"?fid={int(media_id)}&ftype=create"
)
payload = self.get_json(url, referer=referer)
data = self._data_list(payload, "fav/resource/ids")
out: list[FavId] = []
for index, item in enumerate(data):
@@ -321,6 +386,10 @@ class BilibiliClient:
return InfosResult(requested=(), items={}, missing=())
fmid = self.mid if folder_mid is None else int(folder_mid)
referer = (
f"https://space.bilibili.com/{fmid}/favlist"
f"?fid={int(media_id)}&ftype=create"
)
items: dict[int, FavResource] = {}
for chunk in _chunks(requested, chunk_size):
resources = ",".join(f"{aid}:2" for aid in chunk)
@@ -329,7 +398,7 @@ class BilibiliClient:
f"?resources={urllib.parse.quote(resources)}"
f"&platform=web&folder_mid={fmid}&folder_id={int(media_id)}"
)
payload = self.get_json(url)
payload = self.get_json(url, referer=referer)
data = self._data_list(payload, "fav/resource/infos")
chunk_set = set(chunk)
for item in data:
@@ -351,7 +420,11 @@ class BilibiliClient:
def fav_folders(self, up_mid: int | None = None) -> list[FolderInfo]:
mid = self.mid if up_mid is None else int(up_mid)
url = f"{BASE}/x/v3/fav/folder/created/list-all?up_mid={mid}&jsonp=jsonp"
payload = self.get_json(url, auth=True)
payload = self.get_json(
url,
auth=True,
referer=f"https://space.bilibili.com/{mid}/favlist",
)
data = payload.get("data") or {}
if not isinstance(data, Mapping):
raise BilibiliProtocolError("fav/folder/created/list-all data 不是对象")
reaction_board/bilibili/provider.py
+4
View File
@@ -33,6 +33,10 @@ class ClientProvider:
credentials = getattr(self.store.get().bilibili, self.role)
key = (credentials.mid, credentials.cookie, credentials.csrf)
if self._client is None or key != self._key:
if self._client is not None:
close = getattr(self._client, "close", None)
if callable(close):
close()
self._client = self.factory(credentials)
self._key = key
return self._client
reaction_board/bilibili/write_client.py
+12 -2
View File
@@ -18,9 +18,11 @@ from .client import (
class BilibiliWriteClient(BilibiliClient):
def post_form(self, url: str, data: Mapping[str, Any]) -> dict[str, Any]:
def post_form(
self, url: str, data: Mapping[str, Any], *, referer: str | None = None
) -> dict[str, Any]:
self.pacer.wait()
headers = self._headers(auth=True)
headers = self._headers(auth=True, referer=referer)
headers["content-type"] = "application/x-www-form-urlencoded"
body = urllib.parse.urlencode(data).encode("utf-8")
response = self.transport.request(
@@ -47,6 +49,10 @@ class BilibiliWriteClient(BilibiliClient):
"del_media_ids": "",
"csrf": self.csrf,
},
referer=(
f"https://space.bilibili.com/{self.mid}/favlist"
f"?fid={int(folder_id)}&ftype=create"
),
)
def sort(self, media_id: int, sort_spec: str) -> dict[str, Any]:
@@ -57,4 +63,8 @@ class BilibiliWriteClient(BilibiliClient):
"sort": sort_spec,
"csrf": self.csrf,
},
referer=(
f"https://space.bilibili.com/{self.mid}/favlist"
f"?fid={int(media_id)}&ftype=create"
),
)