fix(tools): guard csv replace against newer fan records
This commit is contained in:
1 parent
dd6b09807d
commit
998d0fd0c0
4 files changed
+43
-1
No files matched your search
+1
-1
@@ -439,7 +439,7 @@ P3 实现说明:
|
||||
- 其他错误最多重试 3 次,仍失败则写入 `ok=0` 与错误信息,不写伪造的零值;
|
||||
- 每条视频请求之间随机等待 1.0~1.6 秒;
|
||||
- 对 `track_fans` 中的每个 mid 请求一次 `relation/stat` 并写入 `up_stat`;
|
||||
- 可用 `tools/import_fans_csv.py` 从 B 站粉丝历史 CSV 回填 `up_stat`;只写 `up_stat`,默认跳过已有成功记录,`--replace` 可用完整历史覆盖重建;
|
||||
- 可用 `tools/import_fans_csv.py` 从 B 站粉丝历史 CSV 回填 `up_stat`;只写 `up_stat`,默认跳过已有成功记录,`--force` 覆盖同日期,`--replace` 用完整历史覆盖重建(存在晚于 CSV 末尾的记录时默认拒绝,需显式 `--allow-lossy-replace`);
|
||||
- 任务结束向 Telegram 发送成功数、失败数、移除数和总耗时摘要。
|
||||
|
||||
服务启动后若当天快照尚未成功完成,应补跑当日任务;已成功的 `(stat_date, aid)` 不重复请求。
|
||||
|
||||
@@ -214,6 +214,7 @@ uv run python tools/import_fans_csv.py --csv E:\Downloads\history.csv --replace
|
||||
- 缺失日期不插值,粉丝曲线会断开;
|
||||
- 默认跳过已有 `ok=1` 记录;`ok=0` 的历史记录会被修复;
|
||||
- `--replace` 会先删除该 mid 的全部 `up_stat`,再按 CSV 重建,适合用完整历史覆盖旧数据;
|
||||
- 如果数据库里已有晚于 CSV 末尾的记录,`--replace` 会拒绝执行;确认要删除时才加 `--allow-lossy-replace`;只想补齐 CSV 日期并保留后续采集时用 `--force`;
|
||||
- 页面的“日增”只在存在前一天数据时计算,跨多天不会标成日增;
|
||||
- 写入前建议先备份 `data/app.db`,例如 `data/backup/app-pre-fans-YYYYMMDD.db`。
|
||||
|
||||
|
||||
@@ -104,6 +104,29 @@ def test_import_default_force_and_replace():
|
||||
db.close()
|
||||
|
||||
|
||||
def test_replace_guard_for_newer_rows():
|
||||
series = parse_fans_csv(CSV_TEXT)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = Database(Path(tmp) / "app.db")
|
||||
db.initialize()
|
||||
try:
|
||||
repo = Repository(db)
|
||||
repo.upsert_up_stat("2026-02-01", 37754047, follower=2000, following=None, ok=True, captured_at=1)
|
||||
try:
|
||||
import_fans_series(db, series, replace=True)
|
||||
except ValueError as exc:
|
||||
assert "allow-lossy-replace" in str(exc)
|
||||
else:
|
||||
raise AssertionError("存在更新记录时应拒绝 --replace")
|
||||
result = import_fans_series(
|
||||
db, series, replace=True, allow_lossy_replace=True, dry_run=True
|
||||
)
|
||||
assert result["deleted"] == 1
|
||||
assert repo.get_up_stat("2026-02-01", 37754047)["follower"] == 2000
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_replace_dry_run_counts_only():
|
||||
series = parse_fans_csv(CSV_TEXT)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
@@ -124,6 +147,7 @@ def main_test():
|
||||
test_parse_csv()
|
||||
test_parse_rejects_bad_values()
|
||||
test_import_default_force_and_replace()
|
||||
test_replace_guard_for_newer_rows()
|
||||
test_replace_dry_run_counts_only()
|
||||
print("OK: 粉丝 CSV 解析/导入/替换/幂等测试全部通过")
|
||||
|
||||
|
||||
@@ -149,12 +149,24 @@ def import_fans_series(
|
||||
mid: int = DEFAULT_MID,
|
||||
force: bool = False,
|
||||
replace: bool = False,
|
||||
allow_lossy_replace: bool = False,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""把解析后的序列写入 ``up_stat``。"""
|
||||
points = list(series.get("points", []))
|
||||
repo = Repository(db)
|
||||
deleted = 0
|
||||
last_csv = str(series.get("last_date") or "")
|
||||
if replace and last_csv:
|
||||
newer = db.read(lambda conn: int(conn.execute(
|
||||
"SELECT COUNT(*) FROM up_stat WHERE mid = ? AND stat_date > ?",
|
||||
(int(mid), last_csv),
|
||||
).fetchone()[0]))
|
||||
if newer and not allow_lossy_replace:
|
||||
raise ValueError(
|
||||
f"数据库中存在 {newer} 条晚于 CSV 末尾 {last_csv} 的粉丝记录;"
|
||||
"确认要用 CSV 覆盖并删除这些记录时请加 --allow-lossy-replace"
|
||||
)
|
||||
if replace:
|
||||
if dry_run:
|
||||
deleted = db.read(lambda conn: int(conn.execute(
|
||||
@@ -226,6 +238,10 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
"--replace", action="store_true",
|
||||
help="先清空该 mid 的全部粉丝历史,再用 CSV 重建",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-lossy-replace", action="store_true",
|
||||
help="配合 --replace:允许删除数据库中晚于 CSV 末尾的记录",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="只计算,不写数据库")
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
@@ -242,6 +258,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
mid=args.mid,
|
||||
force=args.force,
|
||||
replace=args.replace,
|
||||
allow_lossy_replace=args.allow_lossy_replace,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
finally:
|
||||
|
||||
Reference in new issue
Block a user