2026-09-10 17:35:48 +00:00 · 2026-09-10 17:35:48 +00:00 · e819548ff2
commit e819548ff2
parent d12cbc62ee
9 changed files with 234 additions and 15 deletions
--- a/mediashelf/ingest.py
+++ b/mediashelf/ingest.py
@ -160,6 +160,7 @@ class Ingest:

        self._progress(scan_id, "rolling up")
        self._apply_watch_rollups(provider_id)
+        self._correct_added_at(provider_id)
        self._rollup_seasons(provider_id)
        self._rollup_shows(provider_id)
        self._apply_pre_history(provider_id, coverage)
@ -373,17 +374,18 @@ class Ingest:
            (provider_id, item.provider_item_id),
        )
        primary = item.parts[0].file_path if item.parts else None
+        now = int(time.time())
        vals = (
            provider_id, library_id, item.provider_item_id, "movie", item.guid, None,
            item.title, item.sort_title, item.year, None, None,
-            item.added_at, item.updated_at, 0, item.size_bytes, item.duration_ms,
-            len(item.parts), primary, item.resolution, item.video_codec,
-            item.view_count, "present", scan_id, scan_id,
+            item.added_at, item.added_at, item.updated_at, 0, item.size_bytes,
+            item.duration_ms, len(item.parts), primary, item.resolution,
+            item.video_codec, item.view_count, "present", scan_id, scan_id, now,
        )
        if existing:
            self.db.execute(
                "UPDATE media_item SET library_id=?, guid=?, title=?, sort_title=?, year=?, "
-                "added_at=?, updated_at=?, size_bytes=?, duration_ms=?, part_count=?, "
+                "provider_added_at=?, updated_at=?, size_bytes=?, duration_ms=?, part_count=?, "
                "primary_path=?, resolution=?, video_codec=?, provider_view_count=?, "
                "status='present', last_seen_scan_id=? WHERE id=?",
                (library_id, item.guid, item.title, item.sort_title, item.year,
@ -397,10 +399,11 @@ class Ingest:
            cur = self.db.execute(
                "INSERT INTO media_item (provider_id, library_id, provider_item_id, kind, "
                "guid, show_guid, title, sort_title, year, parent_id, season_number, "
-                "added_at, updated_at, episode_count, size_bytes, duration_ms, part_count, "
-                "primary_path, resolution, video_codec, provider_view_count, status, "
-                "first_seen_scan_id, last_seen_scan_id) "
-                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals,
+                "added_at, provider_added_at, updated_at, episode_count, size_bytes, "
+                "duration_ms, part_count, primary_path, resolution, video_codec, "
+                "provider_view_count, status, first_seen_scan_id, last_seen_scan_id, "
+                "first_seen_at) "
+                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals,
            )
            item_id = cur.lastrowid
            result.items_added += 1
@ -481,7 +484,7 @@ class Ingest:
                ep_id = row["id"]
                self.db.execute(
                    "UPDATE episode SET season_item_id=?, episode_number=?, title=?, "
-                    "added_at=?, duration_ms=?, size_bytes=?, part_count=?, "
+                    "provider_added_at=?, duration_ms=?, size_bytes=?, part_count=?, "
                    "status='present', last_seen_scan_id=? WHERE id=?",
                    (season_id, ep.episode_number, ep.title, ep.added_at, ep.duration_ms,
                     ep.size_bytes, primary_count, scan_id, ep_id),
@ -489,10 +492,12 @@ class Ingest:
            else:
                cur = self.db.execute(
                    "INSERT INTO episode (season_item_id, provider_item_id, episode_number, "
-                    "title, added_at, duration_ms, size_bytes, part_count, status, last_seen_scan_id) "
-                    "VALUES (?,?,?,?,?,?,?,?, 'present', ?)",
+                    "title, added_at, provider_added_at, duration_ms, size_bytes, part_count, "
+                    "status, last_seen_scan_id, first_seen_at) "
+                    "VALUES (?,?,?,?,?,?,?,?,?, 'present', ?, ?)",
                    (season_id, ep.provider_item_id, ep.episode_number, ep.title,
-                     ep.added_at, ep.duration_ms, ep.size_bytes, primary_count, scan_id),
+                     ep.added_at, ep.added_at, ep.duration_ms, ep.size_bytes,
+                     primary_count, scan_id, int(time.time())),
                )
                ep_id = cur.lastrowid
            self.db.execute("DELETE FROM media_part WHERE episode_id=?", (ep_id,))
@ -546,7 +551,7 @@ class Ingest:
        """)
        c.execute("""
            UPDATE episode SET watch_count=0, partial_count=0, abandoned_count=0,
-                last_watched_at=NULL, last_touched_at=NULL
+                last_watched_at=NULL, last_touched_at=NULL, first_watched_at=NULL
        """)

        agg = """
@ -584,9 +589,54 @@ class Ingest:
                partial_count = COALESCE((SELECT partial FROM _wagg WHERE pid = episode.provider_item_id), 0),
                abandoned_count = COALESCE((SELECT abandoned FROM _wagg WHERE pid = episode.provider_item_id), 0),
                last_watched_at = (SELECT last_watched FROM _wagg WHERE pid = episode.provider_item_id),
+                first_watched_at = (SELECT first_watched FROM _wagg WHERE pid = episode.provider_item_id),
                last_touched_at = (SELECT last_touched FROM _wagg WHERE pid = episode.provider_item_id)
        """)

+    def _correct_added_at(self, provider_id: int) -> None:
+        """Repair added_at where Plex's value is provably wrong.
+
+        Plex's addedAt tracks the FILE, not the library entry: replace or
+        re-encode a file and Date Added resets while the item and its watch
+        history survive. Measured on the live server, 19% of items report a
+        last-watch EARLIER than their added date.
+
+        A play is proof the item already existed, so the first completed view is
+        a lower bound on the true add date. That is not the exact date Plex
+        never kept, but it is strictly better than a value we can show is
+        impossible — and it matters, because pre_history is derived from
+        added_at and a wrongly-recent date promotes an item into the CONFIDENT
+        reclaim pool when it belongs in the uncertain one.
+
+        Items nobody ever watched keep Plex's value; nothing contradicts it.
+        first_seen_at is authoritative for anything MediaShelf sees appear.
+        """
+        c = self.db.conn
+        c.execute("UPDATE episode SET added_at = provider_added_at "
+                  "WHERE provider_added_at IS NOT NULL")
+        c.execute("""
+            UPDATE episode SET added_at = first_watched_at
+            WHERE first_watched_at IS NOT NULL AND first_watched_at > 0
+              AND (added_at IS NULL OR added_at > first_watched_at)
+        """)
+
+        c.execute("UPDATE media_item SET added_at = provider_added_at, "
+                  "added_at_source = 'provider' "
+                  "WHERE kind = 'movie' AND provider_added_at IS NOT NULL "
+                  "AND provider_id = ?", (provider_id,))
+        cur = c.execute("""
+            UPDATE media_item SET added_at = first_watched_at,
+                                  added_at_source = 'first_watch'
+            WHERE kind = 'movie' AND provider_id = ?
+              AND first_watched_at IS NOT NULL AND first_watched_at > 0
+              AND (added_at IS NULL OR added_at > first_watched_at)
+        """, (provider_id,))
+        corrected = cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
+        if corrected:
+            self._warn(
+                "%d movie(s) had a Plex addedAt later than their first play; "
+                "corrected to the first play date" % corrected)
+
    def _rollup_seasons(self, provider_id: int) -> None:
        c = self.db.conn
        c.execute("""
@ -614,6 +664,13 @@ class Ingest:
            WHERE kind = 'season' AND provider_id = ?
        """, (provider_id,))

+        c.execute("""
+            UPDATE media_item SET added_at_source = CASE
+                WHEN provider_added_at IS NOT NULL AND added_at < provider_added_at
+                THEN 'first_watch' ELSE 'provider' END
+            WHERE kind = 'season' AND provider_id = ?
+        """, (provider_id,))
+
        # distinct watchers across the season's episodes
        c.execute("""
            UPDATE media_item SET distinct_watcher_count = COALESCE((
--- a/mediashelf/migrations/002_added_at_provenance.sql
+++ b/mediashelf/migrations/002_added_at_provenance.sql
@ -0,0 +1,26 @@
+-- Plex's addedAt tracks the FILE, not the library entry. Replacing or
+-- re-encoding a file resets Date Added while the item and its watch history
+-- survive, so 19% of the live library (22% of TV Show Archive) reports a
+-- lastViewedAt EARLIER than its addedAt — watched before it was "added".
+--
+-- That matters beyond cosmetics: pre_history is derived from added_at, so an
+-- old item whose file was replaced looks post-coverage and lands in the
+-- CONFIDENT reclaim pool when it belongs in the uncertain one.
+--
+-- Keep Plex's raw value for reference, and let added_at hold the best estimate
+-- (see ingest._correct_added_at). Also record when MediaShelf itself first saw
+-- a row, which is authoritative from now on regardless of what Plex does.
+
+ALTER TABLE media_item ADD COLUMN provider_added_at INTEGER;
+ALTER TABLE media_item ADD COLUMN added_at_source TEXT;
+ALTER TABLE media_item ADD COLUMN first_seen_at INTEGER;
+
+ALTER TABLE episode ADD COLUMN provider_added_at INTEGER;
+ALTER TABLE episode ADD COLUMN first_watched_at INTEGER;
+ALTER TABLE episode ADD COLUMN first_seen_at INTEGER;
+
+-- Backfill: existing rows carry Plex's value in added_at.
+UPDATE media_item SET provider_added_at = added_at WHERE provider_added_at IS NULL;
+UPDATE episode SET provider_added_at = added_at WHERE provider_added_at IS NULL;
+
+CREATE INDEX ix_item_first_seen ON media_item(first_seen_at);
--- a/mediashelf/queries.py
+++ b/mediashelf/queries.py
@ -65,6 +65,7 @@ BASE_COLUMNS = """
    i.last_watched_at, i.last_touched_at, i.first_watched_at,
    i.distinct_watcher_count, i.pre_history, i.kept, i.kept_via, i.kept_mark_id,
    i.provider_view_count, i.status, i.parent_id, i.season_number,
+    i.provider_added_at, i.added_at_source, i.first_seen_at,
    parent.title AS show_title
 """

@ -200,6 +201,8 @@ class Query:
            flags.append("duplicate")
        if (d.get("provider_view_count") or 0) > 0 and not d.get("watch_count"):
            flags.append("history_gap")
+        if d.get("added_at_source") == "first_watch":
+            flags.append("added_corrected")
        # A season has one part per episode, so part_count > 1 is normal there;
        # only flag genuinely extra files (a movie held twice, or a split episode).
        parts = d.get("part_count") or 0
@ -218,6 +221,9 @@ class Query:
            "library": {"id": d["library_id"], "title": d["library_title"]},
            "size_bytes": d.get("size_bytes") or 0,
            "added_at": d.get("added_at"),
+            "provider_added_at": d.get("provider_added_at"),
+            "added_at_source": d.get("added_at_source"),
+            "first_seen_at": d.get("first_seen_at"),
            "last_watched_at": d.get("last_watched_at"),
            "last_touched_at": d.get("last_touched_at"),
            "watch_count": d.get("watch_count") or 0,
--- a/mediashelf/rules.py
+++ b/mediashelf/rules.py
@ -25,6 +25,9 @@ FIELDS: dict[str, tuple[str, str]] = {
    "year":                   ("i.year", "int"),
    "size_bytes":             ("i.size_bytes", "int"),
    "added_at":               ("i.added_at", "ts"),
+    "provider_added_at":      ("i.provider_added_at", "ts"),
+    "added_at_source":        ("i.added_at_source", "str"),
+    "first_seen_at":          ("i.first_seen_at", "ts"),
    "updated_at":             ("i.updated_at", "ts"),
    "episode_count":          ("i.episode_count", "int"),
    "part_count":             ("i.part_count", "int"),
--- a/mediashelf/static/app.css
+++ b/mediashelf/static/app.css
@ -156,6 +156,7 @@ td.title-cell { max-width: 380px; overflow: hidden; text-overflow: ellipsis; }
 .flag.duplicate { color: var(--accent); border-color: var(--accent); }
 .flag.kept { color: var(--kept); border-color: var(--kept); }
 .flag.multi_part, .flag.history_gap { color: var(--faint); border-color: var(--faint); }
+.flag.added_corrected { color: var(--accent); border-color: var(--accent); opacity: .8; }

 .pager { display: flex; gap: .6rem; align-items: center; justify-content: center; padding: .7rem; color: var(--dim); font-size: .85rem; }

--- a/mediashelf/static/app.js
+++ b/mediashelf/static/app.js
@ -551,7 +551,15 @@ async function openItem(id) {
  const add = (k, v) => { dl.append(el('dt', { text: k }), el('dd', { text: v })); };
  add('Library', it.library.title);
  add('Size', `${bytes(it.size_bytes)} (${exact(it.size_bytes)})`);
-  add('Added', date(it.added_at));
+  if (it.added_at_source === 'first_watch') {
+    add('Added', `${date(it.added_at)} (corrected)`);
+    add('', `Plex reports ${date(it.provider_added_at)}, but it was already being ` +
+            `watched before then — Plex's Date Added follows the file, so replacing ` +
+            `or re-encoding one resets it. The first play is used as a lower bound.`);
+  } else {
+    add('Added', date(it.added_at));
+  }
+  if (it.first_seen_at) add('First seen by MediaShelf', date(it.first_seen_at));
  add('Last played', it.last_watched_at ? `${date(it.last_watched_at)} (${agoPhrase(it.last_watched_at)})` : 'never');
  add('Plays', `${it.watch_count} finished · ${it.partial_count} partial · ${it.abandoned_count} abandoned`);
  add('Distinct viewers', it.distinct_watcher_count);