feat(up): capture follower snapshots at ingest and backfill history
This commit is contained in:
1 parent
4942d1d161
commit
519fc39fca
8 files changed
+401
-14
No files matched your search
@@ -39,6 +39,7 @@ from .services.favorites import FavoriteActionService
|
||||
from .services.folder_sync import FolderSyncService
|
||||
from .services.snapshot import SnapshotService
|
||||
from .services.telegram_poller import TelegramPoller
|
||||
from .services.up_profile import UpProfileService
|
||||
from .timeutil import next_daily_run, parse_hhmm, shanghai_date
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -120,6 +121,12 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
"--preview", action="store_true",
|
||||
help="以预览模式启动(要求数据库含 preview:seeded=1 标记)",
|
||||
)
|
||||
backfill = sub.add_parser(
|
||||
"backfill-followers", parents=[common],
|
||||
help="为历史视频回填 UP 粉丝数快照(只补空值)",
|
||||
)
|
||||
backfill.add_argument("--limit", type=int, default=None, help="最多处理多少个 UP")
|
||||
backfill.add_argument("--dry-run", action="store_true", help="只统计待回填数量,不发请求")
|
||||
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
|
||||
return parser
|
||||
|
||||
@@ -226,6 +233,33 @@ def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_backfill_followers(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = _client(
|
||||
cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=2
|
||||
)
|
||||
service = UpProfileService(repo, lambda: client, logger=logger)
|
||||
started = int(time.time())
|
||||
result = service.backfill(limit=args.limit, dry_run=args.dry_run)
|
||||
payload = result.as_dict()
|
||||
payload["dry_run"] = bool(args.dry_run)
|
||||
payload["coverage"] = repo.up_follower_coverage()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
_record_task(
|
||||
repo, "backfill_followers", ok=result.failed == 0,
|
||||
started_at=started, finished_at=int(time.time()), payload=payload,
|
||||
error=";".join(result.errors) if result.errors else None,
|
||||
)
|
||||
_print(payload)
|
||||
return 0 if result.failed == 0 else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
@@ -673,6 +707,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
return cmd_sync(args, logger)
|
||||
if args.command == "snapshot":
|
||||
return cmd_snapshot(args, logger)
|
||||
if args.command == "backfill-followers":
|
||||
return cmd_backfill_followers(args, logger)
|
||||
if args.command == "discover":
|
||||
return cmd_discover(args, logger)
|
||||
if args.command == "candidates":
|
||||
|
||||
Reference in new issue
Block a user