Use Plex's lastViewedAt for the date correction, not just Tautulli

The correction shipped in the previous commit fired zero times on the live
library. It compared Plex's addedAt against MediaShelf's first completed
Tautulli play, but Tautulli history only starts 2025-03-08 and the wrong dates
are overwhelmingly older than that. The evidence that proved the bug in the
first place was Plex's own lastViewedAt - which the provider parsed and then
dropped on the floor.

Now stored as provider_last_viewed_at and folded into the bound:
added_at = MIN(provider_added_at, first_watched_at, provider_last_viewed_at).
Seasons also get their own provider_added_at (MIN over episodes), without which
their added_at_source could never be computed.

Fixture gains the case that actually failed: added last week, last viewed 1500
days ago, no Tautulli history at all - so only lastViewedAt carries it. Also
tightened test_correction_leaves_unwatched_items_alone, which defined
"unwatched" as watch_count=0 and so wrongly expected an item with Plex view
evidence to be left alone.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GVbG48GAXfCZatcmX123Ra

docs/design.md

@@ -388,13 +388,23 @@ whose file was replaced looks post-coverage and is promoted into the
 **confident** reclaim pool — the one pool that is supposed to be trustworthy. The
 new-arrival grace is affected the same way in reverse.
 
-**What MediaShelf does about it.** A completed play is proof the item already
+**What MediaShelf does about it.** A view is proof the item already existed, so
-existed, so the first play is a lower bound on the true add date:
+the earliest view is a lower bound on the true add date. Two independent
+witnesses, and both are needed:
 
 ```
-added_at = MIN(provider_added_at, first_watched_at)
+added_at = MIN(provider_added_at, first_watched_at, provider_last_viewed_at)
 ```
 
+`first_watched_at` is MediaShelf's first completed Tautulli play — precise, but
+only inside the history window. `provider_last_viewed_at` is Plex's own
+`lastViewedAt`, which reaches back as far as the server does.
+
+The first version of this correction used Tautulli alone and **fired zero times
+on the live library**, because the wrong dates are overwhelmingly older than the
+2025-03 history window. Plex's `lastViewedAt` is the field that actually carries
+the evidence, and it was being parsed and discarded.
+
 `provider_added_at` keeps Plex's raw value, `added_at_source` records which
 applied, and the UI explains the substitution on any corrected row rather than
 silently showing a different date than Plex does. Items nobody ever watched keep

mediashelf/ingest.py

@@ -380,18 +380,19 @@ class Ingest:
             item.title, item.sort_title, item.year, None, None,
             item.added_at, item.added_at, item.updated_at, 0, item.size_bytes,
             item.duration_ms, len(item.parts), primary, item.resolution,
-            item.video_codec, item.view_count, "present", scan_id, scan_id, now,
+            item.video_codec, item.view_count, item.last_viewed_at,
+            "present", scan_id, scan_id, now,
         )
         if existing:
             self.db.execute(
                 "UPDATE media_item SET library_id=?, guid=?, title=?, sort_title=?, year=?, "
                 "provider_added_at=?, updated_at=?, size_bytes=?, duration_ms=?, part_count=?, "
                 "primary_path=?, resolution=?, video_codec=?, provider_view_count=?, "
-                "status='present', last_seen_scan_id=? WHERE id=?",
+                "provider_last_viewed_at=?, status='present', last_seen_scan_id=? WHERE id=?",
                 (library_id, item.guid, item.title, item.sort_title, item.year,
                  item.added_at, item.updated_at, item.size_bytes, item.duration_ms,
                  len(item.parts), primary, item.resolution, item.video_codec,
-                 item.view_count, scan_id, existing["id"]),
+                 item.view_count, item.last_viewed_at, scan_id, existing["id"]),
             )
             item_id = existing["id"]
             result.items_updated += 1
@@ -401,9 +402,9 @@ class Ingest:
                 "guid, show_guid, title, sort_title, year, parent_id, season_number, "
                 "added_at, provider_added_at, updated_at, episode_count, size_bytes, "
                 "duration_ms, part_count, primary_path, resolution, video_codec, "
-                "provider_view_count, status, first_seen_scan_id, last_seen_scan_id, "
+                "provider_view_count, provider_last_viewed_at, status, "
-                "first_seen_at) "
+                "first_seen_scan_id, last_seen_scan_id, first_seen_at) "
-                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals,
+                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals,
             )
             item_id = cur.lastrowid
             result.items_added += 1
@@ -484,20 +485,23 @@ class Ingest:
                 ep_id = row["id"]
                 self.db.execute(
                     "UPDATE episode SET season_item_id=?, episode_number=?, title=?, "
-                    "provider_added_at=?, duration_ms=?, size_bytes=?, part_count=?, "
+                    "provider_added_at=?, provider_last_viewed_at=?, duration_ms=?, "
+                    "size_bytes=?, part_count=?, "
                     "status='present', last_seen_scan_id=? WHERE id=?",
-                    (season_id, ep.episode_number, ep.title, ep.added_at, ep.duration_ms,
+                    (season_id, ep.episode_number, ep.title, ep.added_at,
+                     ep.last_viewed_at, ep.duration_ms,
                      ep.size_bytes, primary_count, scan_id, ep_id),
                 )
             else:
                 cur = self.db.execute(
                     "INSERT INTO episode (season_item_id, provider_item_id, episode_number, "
-                    "title, added_at, provider_added_at, duration_ms, size_bytes, part_count, "
+                    "title, added_at, provider_added_at, provider_last_viewed_at, "
+                    "duration_ms, size_bytes, part_count, "
                     "status, last_seen_scan_id, first_seen_at) "
-                    "VALUES (?,?,?,?,?,?,?,?,?, 'present', ?, ?)",
+                    "VALUES (?,?,?,?,?,?,?,?,?,?, 'present', ?, ?)",
                     (season_id, ep.provider_item_id, ep.episode_number, ep.title,
-                     ep.added_at, ep.added_at, ep.duration_ms, ep.size_bytes,
+                     ep.added_at, ep.added_at, ep.last_viewed_at, ep.duration_ms,
-                     primary_count, scan_id, int(time.time())),
+                     ep.size_bytes, primary_count, scan_id, int(time.time())),
                 )
                 ep_id = cur.lastrowid
             self.db.execute("DELETE FROM media_part WHERE episode_id=?", (ep_id,))
@@ -597,27 +601,39 @@ class Ingest:
         """Repair added_at where Plex's value is provably wrong.
 
         Plex's addedAt tracks the FILE, not the library entry: replace or
-        re-encode a file and Date Added resets while the item and its watch
+        re-encode a file and Date Added resets while the item, its ratingKey and
-        history survive. Measured on the live server, 19% of items report a
+        its watch history all survive. Measured on the live server, 19% of items
-        last-watch EARLIER than their added date.
+        report a view EARLIER than their added date.
 
-        A play is proof the item already existed, so the first completed view is
+        A view is proof the item already existed, so the earliest view is a
-        a lower bound on the true add date. That is not the exact date Plex
+        lower bound on the true add date. Two independent witnesses:
-        never kept, but it is strictly better than a value we can show is
+
-        impossible — and it matters, because pre_history is derived from
+          * Plex's own lastViewedAt, which reaches back as far as the server does
-        added_at and a wrongly-recent date promotes an item into the CONFIDENT
+          * MediaShelf's first completed play from Tautulli, which is more
+            precise but only covers the history window (here, 2025-03 onward)
+
+        The first version of this used only Tautulli and fired zero times,
+        because the wrong dates are mostly older than the history window. Plex's
+        lastViewedAt is the field that actually carries the evidence.
+
+        This is a lower bound, not the date Plex never kept. But it beats a value
+        we can prove impossible, and it matters: pre_history is derived from
+        added_at, so a wrongly-recent date promotes an item into the CONFIDENT
         reclaim pool when it belongs in the uncertain one.
 
-        Items nobody ever watched keep Plex's value; nothing contradicts it.
+        Items nobody has ever watched keep Plex's value; nothing contradicts it.
-        first_seen_at is authoritative for anything MediaShelf sees appear.
         """
         c = self.db.conn
+
+        # episodes first, so season rollups inherit corrected dates
         c.execute("UPDATE episode SET added_at = provider_added_at "
                   "WHERE provider_added_at IS NOT NULL")
         c.execute("""
-            UPDATE episode SET added_at = first_watched_at
+            UPDATE episode SET added_at = MIN(
-            WHERE first_watched_at IS NOT NULL AND first_watched_at > 0
+                    COALESCE(added_at, 253402300799),
-              AND (added_at IS NULL OR added_at > first_watched_at)
+                    COALESCE(NULLIF(first_watched_at, 0), 253402300799),
+                    COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799))
+            WHERE (first_watched_at > 0 OR provider_last_viewed_at > 0)
         """)
 
         c.execute("UPDATE media_item SET added_at = provider_added_at, "
@@ -625,17 +641,23 @@ class Ingest:
                   "WHERE kind = 'movie' AND provider_added_at IS NOT NULL "
                   "AND provider_id = ?", (provider_id,))
         cur = c.execute("""
-            UPDATE media_item SET added_at = first_watched_at,
+            UPDATE media_item SET added_at = MIN(
-                                  added_at_source = 'first_watch'
+                    COALESCE(added_at, 253402300799),
+                    COALESCE(NULLIF(first_watched_at, 0), 253402300799),
+                    COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799)),
+                added_at_source = 'first_watch'
             WHERE kind = 'movie' AND provider_id = ?
-              AND first_watched_at IS NOT NULL AND first_watched_at > 0
+              AND (first_watched_at > 0 OR provider_last_viewed_at > 0)
-              AND (added_at IS NULL OR added_at > first_watched_at)
+              AND MIN(COALESCE(NULLIF(first_watched_at, 0), 253402300799),
+                      COALESCE(NULLIF(provider_last_viewed_at, 0), 253402300799))
+                  < COALESCE(added_at, 253402300799)
         """, (provider_id,))
         corrected = cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
         if corrected:
             self._warn(
-                "%d movie(s) had a Plex addedAt later than their first play; "
+                "%d movie(s) had a Plex addedAt later than a recorded view; "
-                "corrected to the first play date" % corrected)
+                "corrected to the earliest view (Plex's Date Added follows the "
+                "file, so replacing one resets it)" % corrected)
 
     def _rollup_seasons(self, provider_id: int) -> None:
         c = self.db.conn
@@ -664,6 +686,13 @@ class Ingest:
             WHERE kind = 'season' AND provider_id = ?
         """, (provider_id,))
 
+        c.execute("""
+            UPDATE media_item SET provider_added_at = (
+                SELECT MIN(e.provider_added_at) FROM episode e
+                WHERE e.season_item_id = media_item.id AND e.status='present')
+            WHERE kind = 'season' AND provider_id = ?
+        """, (provider_id,))
+
         c.execute("""
             UPDATE media_item SET added_at_source = CASE
                 WHEN provider_added_at IS NOT NULL AND added_at < provider_added_at

mediashelf/migrations/003_provider_last_viewed.sql

@@ -0,0 +1,13 @@
+-- The added_at correction added in 002 could only use Tautulli's history, which
+-- starts 2025-03-08. But the evidence that proved Plex's addedAt wrong is Plex's
+-- own lastViewedAt, which reaches back years further — 2001: A Space Odyssey
+-- reports added 2026-07-31 and last watched 2017-08-26.
+--
+-- That field was parsed by the provider and then dropped on the floor, so the
+-- correction fired zero times on the live library. Store it.
+--
+-- Seasons also need a provider_added_at of their own (the MIN over their
+-- episodes), or their added_at_source can never be computed.
+
+ALTER TABLE media_item ADD COLUMN provider_last_viewed_at INTEGER;
+ALTER TABLE episode ADD COLUMN provider_last_viewed_at INTEGER;

mediashelf/queries.py

@@ -66,6 +66,7 @@ BASE_COLUMNS = """
     i.distinct_watcher_count, i.pre_history, i.kept, i.kept_via, i.kept_mark_id,
     i.provider_view_count, i.status, i.parent_id, i.season_number,
     i.provider_added_at, i.added_at_source, i.first_seen_at,
+    i.provider_last_viewed_at,
     parent.title AS show_title
 """
 
@@ -224,6 +225,8 @@ class Query:
             "provider_added_at": d.get("provider_added_at"),
             "added_at_source": d.get("added_at_source"),
             "first_seen_at": d.get("first_seen_at"),
+            "first_watched_at": d.get("first_watched_at"),
+            "provider_last_viewed_at": d.get("provider_last_viewed_at"),
             "last_watched_at": d.get("last_watched_at"),
             "last_touched_at": d.get("last_touched_at"),
             "watch_count": d.get("watch_count") or 0,

tests/test_ingest.py

@@ -154,9 +154,12 @@ def test_added_at_is_corrected_when_plex_claims_it_postdates_a_play(scanned):
 
 
 def test_correction_leaves_unwatched_items_alone(scanned):
-    """Nothing contradicts Plex for an item nobody ever played."""
+    """Nothing contradicts Plex for an item with no evidence of ever being
+    viewed — by Tautulli OR by Plex's own lastViewedAt."""
     rows = scanned.query(
         "SELECT * FROM media_item WHERE kind='movie' AND watch_count = 0 "
+        "AND COALESCE(first_watched_at,0) = 0 "
+        "AND COALESCE(provider_last_viewed_at,0) = 0 "
         "AND provider_added_at IS NOT NULL LIMIT 20")
     assert rows
     for r in rows:
@@ -188,3 +191,24 @@ def test_added_at_correction_is_idempotent(scanned, rescan):
         "SELECT id, added_at, provider_added_at, added_at_source FROM media_item "
         "ORDER BY id")
     assert [dict(r) for r in before] == [dict(r) for r in after]
+
+
+def test_correction_works_from_plex_last_viewed_alone(scanned):
+    """The live failure: the wrong dates are mostly older than Tautulli's history
+    window, so only Plex's lastViewedAt carries the evidence. The first version
+    of this correction used Tautulli alone and fired zero times on 65 TB."""
+    row = scanned.one("SELECT * FROM media_item WHERE title = 'Plex Evidence Only'")
+    assert row is not None
+    assert not row["first_watched_at"], "fixture must have no Tautulli history"
+    assert row["provider_last_viewed_at"] < row["provider_added_at"]
+    assert row["added_at"] == row["provider_last_viewed_at"], \
+        "correction ignored Plex's lastViewedAt"
+    assert row["added_at_source"] == "first_watch"
+
+
+def test_seasons_get_a_provider_added_at(scanned):
+    """Without one, a season's added_at_source can never be computed."""
+    n = scanned.scalar(
+        "SELECT COUNT(*) FROM media_item WHERE kind='season' AND episode_count > 0 "
+        "AND provider_added_at IS NULL")
+    assert n == 0

tools/mockserver.py

@@ -83,6 +83,21 @@ def _build(seed=7):
              "size": 9 * 10**9}]}],
     })
 
+    # The case that actually failed on the live library: Plex says it was added
+    # last week, Plex ALSO says it was last watched years ago, and Tautulli has
+    # no record at all because its history only starts recently. Only
+    # lastViewedAt carries the evidence here.
+    movies.append({
+        "sectionKey": "1", "ratingKey": "1998", "guid": "plex://movie/plexonly",
+        "type": "movie", "title": "Plex Evidence Only", "year": 1999,
+        "addedAt": NOW - 7 * DAY, "updatedAt": NOW - 7 * DAY,
+        "lastViewedAt": NOW - 1500 * DAY,        # long before Tautulli existed
+        "duration": 100 * 60000, "viewCount": 2,
+        "Media": [{"videoResolution": "1080", "Part": [
+            {"id": 6998, "file": "/mnt/titan4/Movies/Plex Evidence Only/x.mkv",
+             "size": 7 * 10**9}]}],
+    })
+
     # ── shows / seasons / episodes ─────────────────────────────────────
     ep_rk = 50000
     for s in range(6):
@@ -129,7 +144,11 @@ def _build(seed=7):
         ep_rk += 1
 
     # ── Tautulli history ───────────────────────────────────────────────
-    watchable = [m["ratingKey"] for m in movies] + [e["ratingKey"] for e in episodes]
+    # 1998/1999 are the date-correction fixtures: their history is set
+    # explicitly below, so keep the random plays away from them.
+    SPECIAL = {"1998", "1999"}
+    watchable = [m["ratingKey"] for m in movies if m["ratingKey"] not in SPECIAL] \
+                + [e["ratingKey"] for e in episodes]
     row_id = 1
     for _ in range(1200):
         rk = rnd.choice(watchable)