feat(p3): add discovery, telegram notifications, and one-click favorites
Implement WBI video search with legacy fallback, classification scoring, candidate pool, permanent seen-aid dedupe, blacklist filtering, and batched Telegram notifications. Add write client for fav deal/sort, target-folder selection, insertion position calculation, read-back verification, fav_action locking, retry on failure, ignore and blacklist actions. Wire discovery, candidate cleanup, Telegram long polling, and new discover/candidates CLI commands into the scheduler. Add migration v3 for candidate duration and index stats.
This commit is contained in:
kairee committed
1 parent
6a21b5d905
commit
d971e718e9
31 files changed
+2886
-55
No files matched your search
@@ -555,6 +555,296 @@ class Repository:
|
||||
(key, str(value)),
|
||||
))
|
||||
|
||||
# ------------------------------------------------------------ candidate
|
||||
|
||||
def known_aid(self, aid: int) -> bool:
|
||||
def run(conn: sqlite3.Connection) -> bool:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM video WHERE aid = ? "
|
||||
"UNION ALL SELECT 1 FROM seen_aid WHERE aid = ? LIMIT 1",
|
||||
(int(aid), int(aid)),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
return self.db.read(run)
|
||||
|
||||
def record_seen(
|
||||
self, aid: int, *, source: str, now: int | None = None
|
||||
) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"INSERT INTO seen_aid(aid, first_seen, source) VALUES (?, ?, ?) "
|
||||
"ON CONFLICT(aid) DO NOTHING",
|
||||
(int(aid), now, source),
|
||||
))
|
||||
|
||||
def upsert_candidate(
|
||||
self,
|
||||
*,
|
||||
aid: int,
|
||||
bvid: str | None,
|
||||
title: str,
|
||||
up_mid: int | None,
|
||||
up_name: str | None,
|
||||
pubdate: int,
|
||||
cover: str | None,
|
||||
score: int,
|
||||
channel: str,
|
||||
duration: int = 0,
|
||||
play: int = 0,
|
||||
like: int = 0,
|
||||
favorite: int = 0,
|
||||
danmaku: int = 0,
|
||||
review: int = 0,
|
||||
now: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = self._now() if now is None else int(now)
|
||||
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO candidate(aid, bvid, title, up_mid, up_name, pubdate,
|
||||
cover, score, channel, first_seen,
|
||||
notified_at, decision, decided_at,
|
||||
duration, play, like, favorite, danmaku, review)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL,
|
||||
?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(aid) DO UPDATE SET
|
||||
bvid = COALESCE(excluded.bvid, candidate.bvid),
|
||||
title = COALESCE(NULLIF(excluded.title, ''), candidate.title),
|
||||
up_mid = COALESCE(excluded.up_mid, candidate.up_mid),
|
||||
up_name = COALESCE(excluded.up_name, candidate.up_name),
|
||||
pubdate = CASE WHEN excluded.pubdate > 0
|
||||
THEN excluded.pubdate ELSE candidate.pubdate END,
|
||||
cover = COALESCE(excluded.cover, candidate.cover),
|
||||
score = excluded.score,
|
||||
channel = excluded.channel,
|
||||
duration = excluded.duration,
|
||||
play = excluded.play,
|
||||
"like" = excluded."like",
|
||||
favorite = excluded.favorite,
|
||||
danmaku = excluded.danmaku,
|
||||
review = excluded.review
|
||||
""",
|
||||
(
|
||||
int(aid), bvid, title, up_mid, up_name, int(pubdate),
|
||||
cover, float(score), channel, now,
|
||||
int(duration or 0), int(play or 0), int(like or 0),
|
||||
int(favorite or 0), int(danmaku or 0), int(review or 0),
|
||||
),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT * FROM candidate WHERE aid = ?", (int(aid),)
|
||||
).fetchone()
|
||||
return dict(row) if row else {}
|
||||
return self.db.write(run)
|
||||
|
||||
def get_candidate(self, aid: int) -> dict[str, Any] | None:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM candidate WHERE aid = ?", (int(aid),)
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
return self.db.read(run)
|
||||
|
||||
def list_candidates(self, decision: str | None = None) -> list[dict[str, Any]]:
|
||||
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||
if decision is None:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM candidate ORDER BY COALESCE(score, 0) DESC, "
|
||||
"first_seen DESC, aid"
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM candidate WHERE decision = ? "
|
||||
"ORDER BY COALESCE(score, 0) DESC, first_seen DESC, aid",
|
||||
(decision,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
def mark_candidate_decision(
|
||||
self, aid: int, decision: str, *, now: int | None = None
|
||||
) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"UPDATE candidate SET decision = ?, decided_at = ? WHERE aid = ?",
|
||||
(decision, now, int(aid)),
|
||||
))
|
||||
|
||||
def reject_pending_for_up(self, up_mid: int, *, now: int | None = None) -> int:
|
||||
now = self._now() if now is None else int(now)
|
||||
def run(conn: sqlite3.Connection) -> int:
|
||||
cur = conn.execute(
|
||||
"UPDATE candidate SET decision = 'rejected', decided_at = ? "
|
||||
"WHERE up_mid = ? AND decision IS NULL",
|
||||
(now, int(up_mid)),
|
||||
)
|
||||
return max(cur.rowcount, 0)
|
||||
return self.db.write(run)
|
||||
|
||||
def delete_candidate(self, aid: int) -> None:
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"DELETE FROM candidate WHERE aid = ?", (int(aid),)
|
||||
))
|
||||
|
||||
def delete_expired_candidates(
|
||||
self, *, ttl_days: int, now: int | None = None
|
||||
) -> int:
|
||||
now = self._now() if now is None else int(now)
|
||||
cutoff = now - int(ttl_days) * 86400
|
||||
def run(conn: sqlite3.Connection) -> int:
|
||||
cur = conn.execute(
|
||||
"DELETE FROM candidate WHERE decision IS NULL AND first_seen < ?",
|
||||
(cutoff,),
|
||||
)
|
||||
return max(cur.rowcount, 0)
|
||||
return self.db.write(run)
|
||||
|
||||
def mark_notified(
|
||||
self,
|
||||
aid: int,
|
||||
*,
|
||||
channel: str,
|
||||
batch_id: str | None,
|
||||
ok: bool,
|
||||
err: str | None = None,
|
||||
now: int | None = None,
|
||||
) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
|
||||
def run(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO notify(aid, channel, batch_id, sent_at, ok, err)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(aid, channel) DO UPDATE SET
|
||||
batch_id = excluded.batch_id,
|
||||
sent_at = excluded.sent_at,
|
||||
ok = excluded.ok,
|
||||
err = excluded.err
|
||||
""",
|
||||
(int(aid), channel, batch_id, now, 1 if ok else 0, err),
|
||||
)
|
||||
if ok:
|
||||
conn.execute(
|
||||
"UPDATE candidate SET notified_at = ? WHERE aid = ?",
|
||||
(now, int(aid)),
|
||||
)
|
||||
self.db.write(run)
|
||||
|
||||
# ------------------------------------------------------------ fav action
|
||||
|
||||
def video_metadata(self, aids: Iterable[int]) -> dict[int, dict[str, Any]]:
|
||||
ids = [int(aid) for aid in aids]
|
||||
if not ids:
|
||||
return {}
|
||||
|
||||
def run(conn: sqlite3.Connection) -> dict[int, dict[str, Any]]:
|
||||
placeholders = ",".join("?" for _ in ids)
|
||||
rows = conn.execute(
|
||||
f"SELECT aid, up_mid, pubdate FROM video WHERE aid IN ({placeholders})",
|
||||
ids,
|
||||
).fetchall()
|
||||
return {
|
||||
int(row["aid"]): {
|
||||
"up_mid": row["up_mid"],
|
||||
"pubdate": int(row["pubdate"] or 0),
|
||||
}
|
||||
for row in rows
|
||||
}
|
||||
return self.db.read(run)
|
||||
|
||||
def folder_for_up(self, up_mid: int) -> int | None:
|
||||
def run(conn: sqlite3.Connection) -> int | None:
|
||||
row = conn.execute(
|
||||
"SELECT folder_id FROM video "
|
||||
"WHERE up_mid = ? AND folder_id IS NOT NULL "
|
||||
"AND status IN ('active', 'unfaved') "
|
||||
"ORDER BY pubdate ASC, aid ASC LIMIT 1",
|
||||
(int(up_mid),),
|
||||
).fetchone()
|
||||
return int(row["folder_id"]) if row else None
|
||||
return self.db.read(run)
|
||||
|
||||
def begin_fav_action(
|
||||
self,
|
||||
aid: int,
|
||||
folder_id: int,
|
||||
*,
|
||||
now: int | None = None,
|
||||
ttl: int = 86400,
|
||||
) -> tuple[str, str]:
|
||||
now = self._now() if now is None else int(now)
|
||||
token = f"fav:{int(aid)}"
|
||||
|
||||
def run(conn: sqlite3.Connection) -> tuple[str, str]:
|
||||
row = conn.execute(
|
||||
"SELECT state, expires_at FROM fav_action WHERE token = ?",
|
||||
(token,),
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
state = str(row["state"])
|
||||
if state == "done":
|
||||
return token, "done"
|
||||
if state == "processing" and int(row["expires_at"]) > now:
|
||||
return token, "busy"
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO fav_action(token, aid, folder_id, state, created_at,
|
||||
expires_at, done_at, position, err)
|
||||
VALUES (?, ?, ?, 'processing', ?, ?, NULL, NULL, NULL)
|
||||
ON CONFLICT(token) DO UPDATE SET
|
||||
folder_id = excluded.folder_id,
|
||||
state = 'processing',
|
||||
expires_at = excluded.expires_at,
|
||||
done_at = NULL,
|
||||
position = NULL,
|
||||
err = NULL
|
||||
""",
|
||||
(token, int(aid), int(folder_id), now, now + int(ttl)),
|
||||
)
|
||||
return token, "processing"
|
||||
return self.db.write(run)
|
||||
|
||||
def finish_fav_action(
|
||||
self, token: str, *, position: int, now: int | None = None
|
||||
) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"UPDATE fav_action SET state = 'done', done_at = ?, position = ?, err = NULL "
|
||||
"WHERE token = ?",
|
||||
(now, int(position), token),
|
||||
))
|
||||
|
||||
def fail_fav_action(
|
||||
self, token: str, *, error: str, now: int | None = None
|
||||
) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"UPDATE fav_action SET state = 'failed', done_at = ?, err = ? "
|
||||
"WHERE token = ?",
|
||||
(now, error, token),
|
||||
))
|
||||
|
||||
def get_fav_action(self, token: str) -> dict[str, Any] | None:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM fav_action WHERE token = ?", (token,)
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
return self.db.read(run)
|
||||
|
||||
def expire_fav_actions(self, *, now: int | None = None) -> int:
|
||||
now = self._now() if now is None else int(now)
|
||||
def run(conn: sqlite3.Connection) -> int:
|
||||
cur = conn.execute(
|
||||
"UPDATE fav_action SET state = 'failed', done_at = ?, "
|
||||
"err = '处理超时' WHERE state = 'processing' AND expires_at < ?",
|
||||
(now, now),
|
||||
)
|
||||
return max(cur.rowcount, 0)
|
||||
return self.db.write(run)
|
||||
|
||||
# ------------------------------------------------------------ blacklist
|
||||
|
||||
def seed_blacklist(self, mids: Iterable[int], *, now: int | None = None) -> int:
|
||||
|
||||
Reference in new issue
Block a user