diff --git a/docs/design.md b/docs/design.md index 3ccb91d..fa0fc4d 100644 --- a/docs/design.md +++ b/docs/design.md @@ -388,13 +388,23 @@ whose file was replaced looks post-coverage and is promoted into the **confident** reclaim pool — the one pool that is supposed to be trustworthy. The new-arrival grace is affected the same way in reverse. -**What MediaShelf does about it.** A completed play is proof the item already -existed, so the first play is a lower bound on the true add date: +**What MediaShelf does about it.** A view is proof the item already existed, so +the earliest view is a lower bound on the true add date. Two independent +witnesses, and both are needed: ``` -added_at = MIN(provider_added_at, first_watched_at) +added_at = MIN(provider_added_at, first_watched_at, provider_last_viewed_at) ``` +`first_watched_at` is MediaShelf's first completed Tautulli play — precise, but +only inside the history window. `provider_last_viewed_at` is Plex's own +`lastViewedAt`, which reaches back as far as the server does. + +The first version of this correction used Tautulli alone and **fired zero times +on the live library**, because the wrong dates are overwhelmingly older than the +2025-03 history window. Plex's `lastViewedAt` is the field that actually carries +the evidence, and it was being parsed and discarded. + `provider_added_at` keeps Plex's raw value, `added_at_source` records which applied, and the UI explains the substitution on any corrected row rather than silently showing a different date than Plex does. Items nobody ever watched keep diff --git a/mediashelf/ingest.py b/mediashelf/ingest.py index a08d557..0f2992a 100644 --- a/mediashelf/ingest.py +++ b/mediashelf/ingest.py @@ -380,18 +380,19 @@ class Ingest: item.title, item.sort_title, item.year, None, None, item.added_at, item.added_at, item.updated_at, 0, item.size_bytes, item.duration_ms, len(item.parts), primary, item.resolution, - item.video_codec, item.view_count, "present", scan_id, scan_id, now, + item.video_codec, item.view_count, item.last_viewed_at, + "present", scan_id, scan_id, now, ) if existing: self.db.execute( "UPDATE media_item SET library_id=?, guid=?, title=?, sort_title=?, year=?, " "provider_added_at=?, updated_at=?, size_bytes=?, duration_ms=?, part_count=?, " "primary_path=?, resolution=?, video_codec=?, provider_view_count=?, " - "status='present', last_seen_scan_id=? WHERE id=?", + "provider_last_viewed_at=?, status='present', last_seen_scan_id=? WHERE id=?", (library_id, item.guid, item.title, item.sort_title, item.year, item.added_at, item.updated_at, item.size_bytes, item.duration_ms, len(item.parts), primary, item.resolution, item.video_codec, - item.view_count, scan_id, existing["id"]), + item.view_count, item.last_viewed_at, scan_id, existing["id"]), ) item_id = existing["id"] result.items_updated += 1 @@ -401,9 +402,9 @@ class Ingest: "guid, show_guid, title, sort_title, year, parent_id, season_number, " "added_at, provider_added_at, updated_at, episode_count, size_bytes, " "duration_ms, part_count, primary_path, resolution, video_codec, " - "provider_view_count, status, first_seen_scan_id, last_seen_scan_id, " - "first_seen_at) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals, + "provider_view_count, provider_last_viewed_at, status, " + "first_seen_scan_id, last_seen_scan_id, first_seen_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals, ) item_id = cur.lastrowid result.items_added += 1 @@ -484,20 +485,23 @@ class Ingest: ep_id = row["id"] self.db.execute( "UPDATE episode SET season_item_id=?, episode_number=?, title=?, " - "provider_added_at=?, duration_ms=?, size_bytes=?, part_count=?, " + "provider_added_at=?, provider_last_viewed_at=?, duration_ms=?, " + "size_bytes=?, part_count=?, " "status='present', last_seen_scan_id=? WHERE id=?", - (season_id, ep.episode_number, ep.title, ep.added_at, ep.duration_ms, + (season_id, ep.episode_number, ep.title, ep.added_at, + ep.last_viewed_at, ep.duration_ms, ep.size_bytes, primary_count, scan_id, ep_id), ) else: cur = self.db.execute( "INSERT INTO episode (season_item_id, provider_item_id, episode_number, " - "title, added_at, provider_added_at, duration_ms, size_bytes, part_count, " + "title, added_at, provider_added_at, provider_last_viewed_at, " + "duration_ms, size_bytes, part_count, " "status, last_seen_scan_id, first_seen_at) " - "VALUES (?,?,?,?,?,?,?,?,?, 'present', ?, ?)", + "VALUES (?,?,?,?,?,?,?,?,?,?, 'present', ?, ?)", (season_id, ep.provider_item_id, ep.episode_number, ep.title, - ep.added_at, ep.added_at, ep.duration_ms, ep.size_bytes, - primary_count, scan_id, int(time.time())), + ep.added_at, ep.added_at, ep.last_viewed_at, ep.duration_ms, + ep.size_bytes, primary_count, scan_id, int(time.time())), ) ep_id = cur.lastrowid self.db.execute("DELETE FROM media_part WHERE episode_id=?", (ep_id,)) @@ -597,27 +601,39 @@ class Ingest: """Repair added_at where Plex's value is provably wrong. Plex's addedAt tracks the FILE, not the library entry: replace or - re-encode a file and Date Added resets while the item and its watch - history survive. Measured on the live server, 19% of items report a - last-watch EARLIER than their added date. + re-encode a file and Date Added resets while the item, its ratingKey and + its watch history all survive. Measured on the live server, 19% of items + report a view EARLIER than their added date. - A play is proof the item already existed, so the first completed view is - a lower bound on the true add date. That is not the exact date Plex - never kept, but it is strictly better than a value we can show is - impossible — and it matters, because pre_history is derived from - added_at and a wrongly-recent date promotes an item into the CONFIDENT + A view is proof the item already existed, so the earliest view is a + lower bound on the true add date. Two independent witnesses: + + * Plex's own lastViewedAt, which reaches back as far as the server does + * MediaShelf's first completed play from Tautulli, which is more + precise but only covers the history window (here, 2025-03 onward) + + The first version of this used only Tautulli and fired zero times, + because the wrong dates are mostly older than the history window. Plex's + lastViewedAt is the field that actually carries the evidence. + + This is a lower bound, not the date Plex never kept. But it beats a value + we can prove impossible, and it matters: pre_history is derived from + added_at, so a wrongly-recent date promotes an item into the CONFIDENT reclaim pool when it belongs in the uncertain one. - Items nobody ever watched keep Plex's value; nothing contradicts it. - first_seen_at is authoritative for anything MediaShelf sees appear. + Items nobody has ever watched keep Plex's value; nothing contradicts it. """ c = self.db.conn + + # episodes first, so season rollups inherit corrected dates c.execute("UPDATE episode SET added_at = provider_added_at " "WHERE provider_added_at IS NOT NULL") c.execute(""" - UPDATE episode SET added_at = first_watched_at - WHERE first_watched_at IS NOT NULL AND first_watched_at > 0 - AND (added_at IS NULL OR added_at > first_watched_at) + UPDATE episode SET added_at = MIN( + COALESCE(added_at, 253402300799), + COALESCE(NULLIF(first_watched_at, 0), 253402300799), + COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799)) + WHERE (first_watched_at > 0 OR provider_last_viewed_at > 0) """) c.execute("UPDATE media_item SET added_at = provider_added_at, " @@ -625,17 +641,23 @@ class Ingest: "WHERE kind = 'movie' AND provider_added_at IS NOT NULL " "AND provider_id = ?", (provider_id,)) cur = c.execute(""" - UPDATE media_item SET added_at = first_watched_at, - added_at_source = 'first_watch' + UPDATE media_item SET added_at = MIN( + COALESCE(added_at, 253402300799), + COALESCE(NULLIF(first_watched_at, 0), 253402300799), + COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799)), + added_at_source = 'first_watch' WHERE kind = 'movie' AND provider_id = ? - AND first_watched_at IS NOT NULL AND first_watched_at > 0 - AND (added_at IS NULL OR added_at > first_watched_at) + AND (first_watched_at > 0 OR provider_last_viewed_at > 0) + AND MIN(COALESCE(NULLIF(first_watched_at, 0), 253402300799), + COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799)) + < COALESCE(added_at, 253402300799) """, (provider_id,)) corrected = cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0 if corrected: self._warn( - "%d movie(s) had a Plex addedAt later than their first play; " - "corrected to the first play date" % corrected) + "%d movie(s) had a Plex addedAt later than a recorded view; " + "corrected to the earliest view (Plex's Date Added follows the " + "file, so replacing one resets it)" % corrected) def _rollup_seasons(self, provider_id: int) -> None: c = self.db.conn @@ -664,6 +686,13 @@ class Ingest: WHERE kind = 'season' AND provider_id = ? """, (provider_id,)) + c.execute(""" + UPDATE media_item SET provider_added_at = ( + SELECT MIN(e.provider_added_at) FROM episode e + WHERE e.season_item_id = media_item.id AND e.status='present') + WHERE kind = 'season' AND provider_id = ? + """, (provider_id,)) + c.execute(""" UPDATE media_item SET added_at_source = CASE WHEN provider_added_at IS NOT NULL AND added_at < provider_added_at diff --git a/mediashelf/migrations/003_provider_last_viewed.sql b/mediashelf/migrations/003_provider_last_viewed.sql new file mode 100644 index 0000000..1f4f9c0 --- /dev/null +++ b/mediashelf/migrations/003_provider_last_viewed.sql @@ -0,0 +1,13 @@ +-- The added_at correction added in 002 could only use Tautulli's history, which +-- starts 2025-03-08. But the evidence that proved Plex's addedAt wrong is Plex's +-- own lastViewedAt, which reaches back years further — 2001: A Space Odyssey +-- reports added 2026-07-31 and last watched 2017-08-26. +-- +-- That field was parsed by the provider and then dropped on the floor, so the +-- correction fired zero times on the live library. Store it. +-- +-- Seasons also need a provider_added_at of their own (the MIN over their +-- episodes), or their added_at_source can never be computed. + +ALTER TABLE media_item ADD COLUMN provider_last_viewed_at INTEGER; +ALTER TABLE episode ADD COLUMN provider_last_viewed_at INTEGER; diff --git a/mediashelf/queries.py b/mediashelf/queries.py index ee6443a..f800e28 100644 --- a/mediashelf/queries.py +++ b/mediashelf/queries.py @@ -66,6 +66,7 @@ BASE_COLUMNS = """ i.distinct_watcher_count, i.pre_history, i.kept, i.kept_via, i.kept_mark_id, i.provider_view_count, i.status, i.parent_id, i.season_number, i.provider_added_at, i.added_at_source, i.first_seen_at, + i.provider_last_viewed_at, parent.title AS show_title """ @@ -224,6 +225,8 @@ class Query: "provider_added_at": d.get("provider_added_at"), "added_at_source": d.get("added_at_source"), "first_seen_at": d.get("first_seen_at"), + "first_watched_at": d.get("first_watched_at"), + "provider_last_viewed_at": d.get("provider_last_viewed_at"), "last_watched_at": d.get("last_watched_at"), "last_touched_at": d.get("last_touched_at"), "watch_count": d.get("watch_count") or 0, diff --git a/tests/test_ingest.py b/tests/test_ingest.py index a6ac2a3..250d9cc 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -154,9 +154,12 @@ def test_added_at_is_corrected_when_plex_claims_it_postdates_a_play(scanned): def test_correction_leaves_unwatched_items_alone(scanned): - """Nothing contradicts Plex for an item nobody ever played.""" + """Nothing contradicts Plex for an item with no evidence of ever being + viewed — by Tautulli OR by Plex's own lastViewedAt.""" rows = scanned.query( "SELECT * FROM media_item WHERE kind='movie' AND watch_count = 0 " + "AND COALESCE(first_watched_at,0) = 0 " + "AND COALESCE(provider_last_viewed_at,0) = 0 " "AND provider_added_at IS NOT NULL LIMIT 20") assert rows for r in rows: @@ -188,3 +191,24 @@ def test_added_at_correction_is_idempotent(scanned, rescan): "SELECT id, added_at, provider_added_at, added_at_source FROM media_item " "ORDER BY id") assert [dict(r) for r in before] == [dict(r) for r in after] + + +def test_correction_works_from_plex_last_viewed_alone(scanned): + """The live failure: the wrong dates are mostly older than Tautulli's history + window, so only Plex's lastViewedAt carries the evidence. The first version + of this correction used Tautulli alone and fired zero times on 65 TB.""" + row = scanned.one("SELECT * FROM media_item WHERE title = 'Plex Evidence Only'") + assert row is not None + assert not row["first_watched_at"], "fixture must have no Tautulli history" + assert row["provider_last_viewed_at"] < row["provider_added_at"] + assert row["added_at"] == row["provider_last_viewed_at"], \ + "correction ignored Plex's lastViewedAt" + assert row["added_at_source"] == "first_watch" + + +def test_seasons_get_a_provider_added_at(scanned): + """Without one, a season's added_at_source can never be computed.""" + n = scanned.scalar( + "SELECT COUNT(*) FROM media_item WHERE kind='season' AND episode_count > 0 " + "AND provider_added_at IS NULL") + assert n == 0 diff --git a/tools/mockserver.py b/tools/mockserver.py index ae19897..d37fc67 100644 --- a/tools/mockserver.py +++ b/tools/mockserver.py @@ -83,6 +83,21 @@ def _build(seed=7): "size": 9 * 10**9}]}], }) + # The case that actually failed on the live library: Plex says it was added + # last week, Plex ALSO says it was last watched years ago, and Tautulli has + # no record at all because its history only starts recently. Only + # lastViewedAt carries the evidence here. + movies.append({ + "sectionKey": "1", "ratingKey": "1998", "guid": "plex://movie/plexonly", + "type": "movie", "title": "Plex Evidence Only", "year": 1999, + "addedAt": NOW - 7 * DAY, "updatedAt": NOW - 7 * DAY, + "lastViewedAt": NOW - 1500 * DAY, # long before Tautulli existed + "duration": 100 * 60000, "viewCount": 2, + "Media": [{"videoResolution": "1080", "Part": [ + {"id": 6998, "file": "/mnt/titan4/Movies/Plex Evidence Only/x.mkv", + "size": 7 * 10**9}]}], + }) + # ── shows / seasons / episodes ───────────────────────────────────── ep_rk = 50000 for s in range(6): @@ -129,7 +144,11 @@ def _build(seed=7): ep_rk += 1 # ── Tautulli history ─────────────────────────────────────────────── - watchable = [m["ratingKey"] for m in movies] + [e["ratingKey"] for e in episodes] + # 1998/1999 are the date-correction fixtures: their history is set + # explicitly below, so keep the random plays away from them. + SPECIAL = {"1998", "1999"} + watchable = [m["ratingKey"] for m in movies if m["ratingKey"] not in SPECIAL] \ + + [e["ratingKey"] for e in episodes] row_id = 1 for _ in range(1200): rk = rnd.choice(watchable)