Use Plex's lastViewedAt for the date correction, not just Tautulli
The correction shipped in the previous commit fired zero times on the live library. It compared Plex's addedAt against MediaShelf's first completed Tautulli play, but Tautulli history only starts 2025-03-08 and the wrong dates are overwhelmingly older than that. The evidence that proved the bug in the first place was Plex's own lastViewedAt - which the provider parsed and then dropped on the floor. Now stored as provider_last_viewed_at and folded into the bound: added_at = MIN(provider_added_at, first_watched_at, provider_last_viewed_at). Seasons also get their own provider_added_at (MIN over episodes), without which their added_at_source could never be computed. Fixture gains the case that actually failed: added last week, last viewed 1500 days ago, no Tautulli history at all - so only lastViewedAt carries it. Also tightened test_correction_leaves_unwatched_items_alone, which defined "unwatched" as watch_count=0 and so wrongly expected an item with Plex view evidence to be left alone. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GVbG48GAXfCZatcmX123Ra
This commit is contained in:
parent
e819548ff2
commit
edbb6f088f
6 changed files with 134 additions and 36 deletions
|
|
@ -380,18 +380,19 @@ class Ingest:
|
|||
item.title, item.sort_title, item.year, None, None,
|
||||
item.added_at, item.added_at, item.updated_at, 0, item.size_bytes,
|
||||
item.duration_ms, len(item.parts), primary, item.resolution,
|
||||
item.video_codec, item.view_count, "present", scan_id, scan_id, now,
|
||||
item.video_codec, item.view_count, item.last_viewed_at,
|
||||
"present", scan_id, scan_id, now,
|
||||
)
|
||||
if existing:
|
||||
self.db.execute(
|
||||
"UPDATE media_item SET library_id=?, guid=?, title=?, sort_title=?, year=?, "
|
||||
"provider_added_at=?, updated_at=?, size_bytes=?, duration_ms=?, part_count=?, "
|
||||
"primary_path=?, resolution=?, video_codec=?, provider_view_count=?, "
|
||||
"status='present', last_seen_scan_id=? WHERE id=?",
|
||||
"provider_last_viewed_at=?, status='present', last_seen_scan_id=? WHERE id=?",
|
||||
(library_id, item.guid, item.title, item.sort_title, item.year,
|
||||
item.added_at, item.updated_at, item.size_bytes, item.duration_ms,
|
||||
len(item.parts), primary, item.resolution, item.video_codec,
|
||||
item.view_count, scan_id, existing["id"]),
|
||||
item.view_count, item.last_viewed_at, scan_id, existing["id"]),
|
||||
)
|
||||
item_id = existing["id"]
|
||||
result.items_updated += 1
|
||||
|
|
@ -401,9 +402,9 @@ class Ingest:
|
|||
"guid, show_guid, title, sort_title, year, parent_id, season_number, "
|
||||
"added_at, provider_added_at, updated_at, episode_count, size_bytes, "
|
||||
"duration_ms, part_count, primary_path, resolution, video_codec, "
|
||||
"provider_view_count, status, first_seen_scan_id, last_seen_scan_id, "
|
||||
"first_seen_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals,
|
||||
"provider_view_count, provider_last_viewed_at, status, "
|
||||
"first_seen_scan_id, last_seen_scan_id, first_seen_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals,
|
||||
)
|
||||
item_id = cur.lastrowid
|
||||
result.items_added += 1
|
||||
|
|
@ -484,20 +485,23 @@ class Ingest:
|
|||
ep_id = row["id"]
|
||||
self.db.execute(
|
||||
"UPDATE episode SET season_item_id=?, episode_number=?, title=?, "
|
||||
"provider_added_at=?, duration_ms=?, size_bytes=?, part_count=?, "
|
||||
"provider_added_at=?, provider_last_viewed_at=?, duration_ms=?, "
|
||||
"size_bytes=?, part_count=?, "
|
||||
"status='present', last_seen_scan_id=? WHERE id=?",
|
||||
(season_id, ep.episode_number, ep.title, ep.added_at, ep.duration_ms,
|
||||
(season_id, ep.episode_number, ep.title, ep.added_at,
|
||||
ep.last_viewed_at, ep.duration_ms,
|
||||
ep.size_bytes, primary_count, scan_id, ep_id),
|
||||
)
|
||||
else:
|
||||
cur = self.db.execute(
|
||||
"INSERT INTO episode (season_item_id, provider_item_id, episode_number, "
|
||||
"title, added_at, provider_added_at, duration_ms, size_bytes, part_count, "
|
||||
"title, added_at, provider_added_at, provider_last_viewed_at, "
|
||||
"duration_ms, size_bytes, part_count, "
|
||||
"status, last_seen_scan_id, first_seen_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?, 'present', ?, ?)",
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,?, 'present', ?, ?)",
|
||||
(season_id, ep.provider_item_id, ep.episode_number, ep.title,
|
||||
ep.added_at, ep.added_at, ep.duration_ms, ep.size_bytes,
|
||||
primary_count, scan_id, int(time.time())),
|
||||
ep.added_at, ep.added_at, ep.last_viewed_at, ep.duration_ms,
|
||||
ep.size_bytes, primary_count, scan_id, int(time.time())),
|
||||
)
|
||||
ep_id = cur.lastrowid
|
||||
self.db.execute("DELETE FROM media_part WHERE episode_id=?", (ep_id,))
|
||||
|
|
@ -597,27 +601,39 @@ class Ingest:
|
|||
"""Repair added_at where Plex's value is provably wrong.
|
||||
|
||||
Plex's addedAt tracks the FILE, not the library entry: replace or
|
||||
re-encode a file and Date Added resets while the item and its watch
|
||||
history survive. Measured on the live server, 19% of items report a
|
||||
last-watch EARLIER than their added date.
|
||||
re-encode a file and Date Added resets while the item, its ratingKey and
|
||||
its watch history all survive. Measured on the live server, 19% of items
|
||||
report a view EARLIER than their added date.
|
||||
|
||||
A play is proof the item already existed, so the first completed view is
|
||||
a lower bound on the true add date. That is not the exact date Plex
|
||||
never kept, but it is strictly better than a value we can show is
|
||||
impossible — and it matters, because pre_history is derived from
|
||||
added_at and a wrongly-recent date promotes an item into the CONFIDENT
|
||||
A view is proof the item already existed, so the earliest view is a
|
||||
lower bound on the true add date. Two independent witnesses:
|
||||
|
||||
* Plex's own lastViewedAt, which reaches back as far as the server does
|
||||
* MediaShelf's first completed play from Tautulli, which is more
|
||||
precise but only covers the history window (here, 2025-03 onward)
|
||||
|
||||
The first version of this used only Tautulli and fired zero times,
|
||||
because the wrong dates are mostly older than the history window. Plex's
|
||||
lastViewedAt is the field that actually carries the evidence.
|
||||
|
||||
This is a lower bound, not the date Plex never kept. But it beats a value
|
||||
we can prove impossible, and it matters: pre_history is derived from
|
||||
added_at, so a wrongly-recent date promotes an item into the CONFIDENT
|
||||
reclaim pool when it belongs in the uncertain one.
|
||||
|
||||
Items nobody ever watched keep Plex's value; nothing contradicts it.
|
||||
first_seen_at is authoritative for anything MediaShelf sees appear.
|
||||
Items nobody has ever watched keep Plex's value; nothing contradicts it.
|
||||
"""
|
||||
c = self.db.conn
|
||||
|
||||
# episodes first, so season rollups inherit corrected dates
|
||||
c.execute("UPDATE episode SET added_at = provider_added_at "
|
||||
"WHERE provider_added_at IS NOT NULL")
|
||||
c.execute("""
|
||||
UPDATE episode SET added_at = first_watched_at
|
||||
WHERE first_watched_at IS NOT NULL AND first_watched_at > 0
|
||||
AND (added_at IS NULL OR added_at > first_watched_at)
|
||||
UPDATE episode SET added_at = MIN(
|
||||
COALESCE(added_at, 253402300799),
|
||||
COALESCE(NULLIF(first_watched_at, 0), 253402300799),
|
||||
COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799))
|
||||
WHERE (first_watched_at > 0 OR provider_last_viewed_at > 0)
|
||||
""")
|
||||
|
||||
c.execute("UPDATE media_item SET added_at = provider_added_at, "
|
||||
|
|
@ -625,17 +641,23 @@ class Ingest:
|
|||
"WHERE kind = 'movie' AND provider_added_at IS NOT NULL "
|
||||
"AND provider_id = ?", (provider_id,))
|
||||
cur = c.execute("""
|
||||
UPDATE media_item SET added_at = first_watched_at,
|
||||
added_at_source = 'first_watch'
|
||||
UPDATE media_item SET added_at = MIN(
|
||||
COALESCE(added_at, 253402300799),
|
||||
COALESCE(NULLIF(first_watched_at, 0), 253402300799),
|
||||
COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799)),
|
||||
added_at_source = 'first_watch'
|
||||
WHERE kind = 'movie' AND provider_id = ?
|
||||
AND first_watched_at IS NOT NULL AND first_watched_at > 0
|
||||
AND (added_at IS NULL OR added_at > first_watched_at)
|
||||
AND (first_watched_at > 0 OR provider_last_viewed_at > 0)
|
||||
AND MIN(COALESCE(NULLIF(first_watched_at, 0), 253402300799),
|
||||
COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799))
|
||||
< COALESCE(added_at, 253402300799)
|
||||
""", (provider_id,))
|
||||
corrected = cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
|
||||
if corrected:
|
||||
self._warn(
|
||||
"%d movie(s) had a Plex addedAt later than their first play; "
|
||||
"corrected to the first play date" % corrected)
|
||||
"%d movie(s) had a Plex addedAt later than a recorded view; "
|
||||
"corrected to the earliest view (Plex's Date Added follows the "
|
||||
"file, so replacing one resets it)" % corrected)
|
||||
|
||||
def _rollup_seasons(self, provider_id: int) -> None:
|
||||
c = self.db.conn
|
||||
|
|
@ -664,6 +686,13 @@ class Ingest:
|
|||
WHERE kind = 'season' AND provider_id = ?
|
||||
""", (provider_id,))
|
||||
|
||||
c.execute("""
|
||||
UPDATE media_item SET provider_added_at = (
|
||||
SELECT MIN(e.provider_added_at) FROM episode e
|
||||
WHERE e.season_item_id = media_item.id AND e.status='present')
|
||||
WHERE kind = 'season' AND provider_id = ?
|
||||
""", (provider_id,))
|
||||
|
||||
c.execute("""
|
||||
UPDATE media_item SET added_at_source = CASE
|
||||
WHEN provider_added_at IS NOT NULL AND added_at < provider_added_at
|
||||
|
|
|
|||
13
mediashelf/migrations/003_provider_last_viewed.sql
Normal file
13
mediashelf/migrations/003_provider_last_viewed.sql
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
-- The added_at correction added in 002 could only use Tautulli's history, which
|
||||
-- starts 2025-03-08. But the evidence that proved Plex's addedAt wrong is Plex's
|
||||
-- own lastViewedAt, which reaches back years further — 2001: A Space Odyssey
|
||||
-- reports added 2026-07-31 and last watched 2017-08-26.
|
||||
--
|
||||
-- That field was parsed by the provider and then dropped on the floor, so the
|
||||
-- correction fired zero times on the live library. Store it.
|
||||
--
|
||||
-- Seasons also need a provider_added_at of their own (the MIN over their
|
||||
-- episodes), or their added_at_source can never be computed.
|
||||
|
||||
ALTER TABLE media_item ADD COLUMN provider_last_viewed_at INTEGER;
|
||||
ALTER TABLE episode ADD COLUMN provider_last_viewed_at INTEGER;
|
||||
|
|
@ -66,6 +66,7 @@ BASE_COLUMNS = """
|
|||
i.distinct_watcher_count, i.pre_history, i.kept, i.kept_via, i.kept_mark_id,
|
||||
i.provider_view_count, i.status, i.parent_id, i.season_number,
|
||||
i.provider_added_at, i.added_at_source, i.first_seen_at,
|
||||
i.provider_last_viewed_at,
|
||||
parent.title AS show_title
|
||||
"""
|
||||
|
||||
|
|
@ -224,6 +225,8 @@ class Query:
|
|||
"provider_added_at": d.get("provider_added_at"),
|
||||
"added_at_source": d.get("added_at_source"),
|
||||
"first_seen_at": d.get("first_seen_at"),
|
||||
"first_watched_at": d.get("first_watched_at"),
|
||||
"provider_last_viewed_at": d.get("provider_last_viewed_at"),
|
||||
"last_watched_at": d.get("last_watched_at"),
|
||||
"last_touched_at": d.get("last_touched_at"),
|
||||
"watch_count": d.get("watch_count") or 0,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue