feat(tools): replace xxm fan import with bilibili csv
This commit is contained in:
kairee committed
1 parent
58df38814a
commit
c4c9c0b038
4 files changed
+392
-469
No files matched your search
@@ -0,0 +1,132 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""粉丝历史 CSV 回填测试（离线）: uv run python tests/test_fans_csv.py"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.repository import Repository # noqa: E402
|
||||
from tools.import_fans_csv import ( # noqa: E402
|
||||
import_fans_series,
|
||||
parse_fans_csv,
|
||||
)
|
||||
|
||||
CSV_TEXT = """\ufeff时间,粉丝,7日变化,1日变化
|
||||
2026-01-03 10:00,1003,0,1
|
||||
2026-01-01 09:00,1001,0,0
|
||||
2026-01-02 08:30,1002,0,1
|
||||
2026-01-02 23:30,1005,0,4
|
||||
2026-01-04,1007,0,1
|
||||
"""
|
||||
|
||||
|
||||
def test_parse_csv():
|
||||
series = parse_fans_csv(CSV_TEXT)
|
||||
assert series["first_date"] == "2026-01-01"
|
||||
assert series["last_date"] == "2026-01-04"
|
||||
assert [point["date"] for point in series["points"]] == [
|
||||
"2026-01-01", "2026-01-02", "2026-01-03", "2026-01-04",
|
||||
]
|
||||
# 同一天两行，保留时间较晚的一条
|
||||
assert [point["follower"] for point in series["points"]] == [1001, 1005, 1003, 1007]
|
||||
assert series["unique_days"] == 4
|
||||
assert series["total_rows"] == 5
|
||||
assert series["gaps"] == 0
|
||||
assert any("多行" in warning for warning in series["warnings"])
|
||||
|
||||
for bad in ("", "时间,粉丝\n2026-01-01\n", "date,follower\nx,1\n", "时间,粉丝\n"):
|
||||
try:
|
||||
parse_fans_csv(bad)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"非法 CSV 应报错: {bad!r}")
|
||||
|
||||
|
||||
def test_parse_rejects_bad_values():
|
||||
for bad in (
|
||||
"时间,粉丝\n2026-01-01,nope\n",
|
||||
"时间,粉丝\nnot-a-date,100\n",
|
||||
"时间,粉丝\n2026-01-01,-1\n",
|
||||
):
|
||||
try:
|
||||
parse_fans_csv(bad)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"非法行应报错: {bad!r}")
|
||||
|
||||
|
||||
def test_import_default_force_and_replace():
|
||||
series = parse_fans_csv(CSV_TEXT)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = Database(Path(tmp) / "app.db")
|
||||
db.initialize()
|
||||
try:
|
||||
repo = Repository(db)
|
||||
repo.upsert_up_stat("2026-01-01", 37754047, follower=1, following=9, ok=True, captured_at=1)
|
||||
repo.upsert_up_stat("2026-01-02", 37754047, follower=None, following=None, ok=False, captured_at=1)
|
||||
repo.upsert_up_stat("2026-01-03", 99999, follower=1, following=9, ok=True, captured_at=1)
|
||||
|
||||
dry = import_fans_series(db, series, dry_run=True)
|
||||
assert dry["inserted"] == 2
|
||||
assert dry["updated"] == 1
|
||||
assert dry["skipped"] == 1
|
||||
assert repo.get_up_stat("2026-01-02", 37754047)["ok"] == 0
|
||||
|
||||
summary = import_fans_series(db, series)
|
||||
assert summary["inserted"] == 2
|
||||
assert summary["updated"] == 1
|
||||
assert summary["skipped"] == 1
|
||||
assert repo.get_up_stat("2026-01-01", 37754047)["follower"] == 1
|
||||
assert repo.get_up_stat("2026-01-02", 37754047)["follower"] == 1005
|
||||
assert repo.get_up_stat("2026-01-02", 37754047)["ok"] == 1
|
||||
assert repo.get_up_stat("2026-01-04", 37754047)["follower"] == 1007
|
||||
# 另一个 mid 不受影响
|
||||
assert repo.get_up_stat("2026-01-03", 99999)["follower"] == 1
|
||||
|
||||
forced = import_fans_series(db, series, force=True)
|
||||
assert forced["updated"] == 4
|
||||
assert repo.get_up_stat("2026-01-01", 37754047)["follower"] == 1001
|
||||
|
||||
replaced = import_fans_series(db, series, replace=True)
|
||||
assert replaced["deleted"] == 4
|
||||
assert replaced["inserted"] == 4
|
||||
assert replaced["updated"] == 0
|
||||
assert repo.get_up_stat("2026-01-01", 37754047)["follower"] == 1001
|
||||
assert repo.get_up_stat("2026-01-03", 99999)["follower"] == 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_replace_dry_run_counts_only():
|
||||
series = parse_fans_csv(CSV_TEXT)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = Database(Path(tmp) / "app.db")
|
||||
db.initialize()
|
||||
try:
|
||||
repo = Repository(db)
|
||||
repo.upsert_up_stat("2026-01-01", 37754047, follower=1, following=None, ok=True, captured_at=1)
|
||||
result = import_fans_series(db, series, replace=True, dry_run=True)
|
||||
assert result["deleted"] == 1
|
||||
assert result["inserted"] == 4
|
||||
assert repo.get_up_stat("2026-01-01", 37754047)["follower"] == 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def main_test():
|
||||
test_parse_csv()
|
||||
test_parse_rejects_bad_values()
|
||||
test_import_default_force_and_replace()
|
||||
test_replace_dry_run_counts_only()
|
||||
print("OK: 粉丝 CSV 解析/导入/替换/幂等测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_test()
|
||||
@@ -1,174 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""粉丝历史回填测试（离线）: uv run python tests/test_fans_history.py"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.repository import Repository # noqa: E402
|
||||
from tools.import_fans_history import ( # noqa: E402
|
||||
build_follower_series,
|
||||
import_follower_series,
|
||||
parse_data_js,
|
||||
)
|
||||
|
||||
|
||||
def make_payload(*, days=10, gap_day=None, mismatch=False):
|
||||
start = date(2026, 1, 1)
|
||||
dates = [start + timedelta(days=index) for index in range(days)]
|
||||
anchor = dates[-1]
|
||||
# 从 900 起每天 +10，最后一天为 900 + (days-1)*10
|
||||
values = {day: 900 + index * 10 for index, day in enumerate(dates)}
|
||||
corr = []
|
||||
for index, day in enumerate(dates):
|
||||
delta = 0 if index == 0 else values[day] - values[dates[index - 1]]
|
||||
corr.append([day.isoformat(), 0, delta])
|
||||
if gap_day is not None:
|
||||
corr = [row for row in corr if row[0] != gap_day]
|
||||
week = [
|
||||
[day.strftime("%Y/%m/%d"), values[day], 0]
|
||||
for day in dates[-3:]
|
||||
]
|
||||
if mismatch:
|
||||
week[-1][1] += 7
|
||||
payload = {
|
||||
"meta": {"updated": anchor.isoformat()},
|
||||
"followers": {
|
||||
"咻咻满": {
|
||||
"total": values[anchor],
|
||||
"week": week,
|
||||
}
|
||||
},
|
||||
"corr": corr,
|
||||
}
|
||||
return payload, values
|
||||
|
||||
|
||||
def test_parse_data_js():
|
||||
payload, _ = make_payload()
|
||||
text = "/* demo */\nwindow.XXM = " + json.dumps(payload, ensure_ascii=False) + ";\n"
|
||||
parsed = parse_data_js(text)
|
||||
assert parsed["meta"]["updated"] == "2026-01-10"
|
||||
|
||||
for bad in ("", "<html></html>", "window.XXM = {}"):
|
||||
try:
|
||||
parse_data_js(bad)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"非法输入应报错: {bad!r}")
|
||||
|
||||
|
||||
def test_build_follower_series():
|
||||
payload, values = make_payload(days=10)
|
||||
series = build_follower_series(payload, days=10)
|
||||
assert series["anchor_follower"] == values[date(2026, 1, 10)]
|
||||
assert series["first_date"] == "2026-01-01"
|
||||
assert series["last_date"] == "2026-01-10"
|
||||
assert series["points"][0] == {"date": "2026-01-01", "follower": 900}
|
||||
assert series["points"][-1] == {"date": "2026-01-10", "follower": 990}
|
||||
assert series["warnings"] == []
|
||||
assert series["stopped_at_gap"] is False
|
||||
|
||||
limited = build_follower_series(payload, days=3)
|
||||
assert [point["date"] for point in limited["points"]] == [
|
||||
"2026-01-08", "2026-01-09", "2026-01-10",
|
||||
]
|
||||
|
||||
try:
|
||||
build_follower_series(payload, days=0)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("days=0 应报错")
|
||||
|
||||
|
||||
def test_gap_and_mismatch():
|
||||
payload, _ = make_payload(days=10, gap_day="2026-01-05")
|
||||
series = build_follower_series(payload, days=10)
|
||||
assert series["stopped_at_gap"] is True
|
||||
assert series["first_date"] == "2026-01-06"
|
||||
assert series["last_date"] == "2026-01-10"
|
||||
assert any("不连续" in warning for warning in series["warnings"])
|
||||
|
||||
payload, _ = make_payload(days=10, mismatch=True)
|
||||
try:
|
||||
build_follower_series(payload, days=10)
|
||||
except ValueError as exc:
|
||||
assert "不一致" in str(exc)
|
||||
else:
|
||||
raise AssertionError("week 与 corr 不一致时应报错")
|
||||
|
||||
|
||||
def test_import_and_force():
|
||||
payload, values = make_payload(days=10)
|
||||
series = build_follower_series(payload, days=10)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = Database(Path(tmp) / "app.db")
|
||||
db.initialize()
|
||||
try:
|
||||
repo = Repository(db)
|
||||
repo.upsert_up_stat(
|
||||
"2026-01-08", 37754047, follower=1, following=5,
|
||||
ok=True, captured_at=1,
|
||||
)
|
||||
repo.upsert_up_stat(
|
||||
"2026-01-09", 37754047, follower=None, following=None,
|
||||
ok=False, captured_at=1,
|
||||
)
|
||||
repo.upsert_up_stat(
|
||||
"2026-01-10", 37754047, follower=2, following=None,
|
||||
ok=True, captured_at=1,
|
||||
)
|
||||
|
||||
dry = import_follower_series(
|
||||
db, series, mid=37754047, dry_run=True
|
||||
)
|
||||
assert dry["inserted"] == 7
|
||||
assert dry["updated"] == 1
|
||||
assert dry["skipped"] == 2
|
||||
assert repo.get_up_stat("2026-01-09", 37754047)["ok"] == 0
|
||||
|
||||
summary = import_follower_series(db, series, mid=37754047)
|
||||
assert summary["inserted"] == 7
|
||||
assert summary["updated"] == 1
|
||||
assert summary["skipped"] == 2
|
||||
assert repo.get_up_stat("2026-01-01", 37754047)["follower"] == 900
|
||||
row = repo.get_up_stat("2026-01-09", 37754047)
|
||||
assert row["follower"] == values[date(2026, 1, 9)]
|
||||
assert row["ok"] == 1
|
||||
assert row["following"] is None
|
||||
preserved = repo.get_up_stat("2026-01-08", 37754047)
|
||||
assert preserved["follower"] == 1
|
||||
assert preserved["following"] == 5
|
||||
|
||||
again = import_follower_series(db, series, mid=37754047)
|
||||
assert again["inserted"] == 0
|
||||
assert again["skipped"] == 10
|
||||
|
||||
forced = import_follower_series(
|
||||
db, series, mid=37754047, force=True
|
||||
)
|
||||
assert forced["updated"] == 10
|
||||
assert repo.get_up_stat("2026-01-08", 37754047)["follower"] == values[date(2026, 1, 8)]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def main_test():
|
||||
test_parse_data_js()
|
||||
test_build_follower_series()
|
||||
test_gap_and_mismatch()
|
||||
test_import_and_force()
|
||||
print("OK: 粉丝历史解析/回填/缺口/幂等测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_test()
|
||||
Reference in new issue
Block a user