diff --git a/docs/design.md b/docs/design.md index fb23af4..3ccb91d 100644 --- a/docs/design.md +++ b/docs/design.md @@ -270,7 +270,7 @@ From each `Video` element on `/library/sections/{id}/all`: | `guid` | cross-server content identity | `guid` | | `type` | `movie` / `episode` / `season` / `show` | `kind` | | `title`, `year` | display | `title`, `year` | -| `addedAt` | epoch seconds, when Plex first saw the file | `added_at` | +| `addedAt` | epoch seconds — **follows the FILE, not the library entry** (§4.12) | `provider_added_at` | | `updatedAt` | epoch seconds, last metadata change | `updated_at` | | `duration` | milliseconds | `duration_ms` | | `viewCount` | plays — **token-account scoped**, see §4.5 | (advisory only) | @@ -362,6 +362,51 @@ the ingest is designed to be slow and steady rather than parallel and aggressive | Malformed / partial item | Item skipped, logged, counted in `scan.warning_count`; the scan still succeeds | | Timeout mid-page | Up to 3 retries with exponential backoff, then the scan fails | +### 4.12 Date Added is not the date it was added + +Plex's `addedAt` is the field its UI calls Date Added, and it is the obvious +source for "when did this arrive". It is also, on this library, wrong 19% of the +time — because it tracks the **file**, not the library entry. Replace or +re-encode a file and Plex resets Date Added while keeping the item, its ratingKey +and its entire watch history. + +Measured on the live server (2026-09-10), comparing `addedAt` against +`lastViewedAt` on items that have both: + +| Library | items with both dates | watched BEFORE "added" | +|---|---|---| +| Movies | 509 | 55 (10.8%) | +| TV Show Archive | 1,393 | 306 (22.0%) | +| **overall** | **1,902** | **361 (19.0%)** | + +*2001: A Space Odyssey* reports added 2026-07-31 and last watched 2017-08-26. +Plex's API exposes no better field; the true insert time exists only in Plex's +own SQLite `metadata_items.created_at`, which needs filesystem access to Loki. + +This is not cosmetic. `pre_history` is derived from `added_at`, so an old item +whose file was replaced looks post-coverage and is promoted into the +**confident** reclaim pool — the one pool that is supposed to be trustworthy. The +new-arrival grace is affected the same way in reverse. + +**What MediaShelf does about it.** A completed play is proof the item already +existed, so the first play is a lower bound on the true add date: + +``` +added_at = MIN(provider_added_at, first_watched_at) +``` + +`provider_added_at` keeps Plex's raw value, `added_at_source` records which +applied, and the UI explains the substitution on any corrected row rather than +silently showing a different date than Plex does. Items nobody ever watched keep +Plex's value — nothing contradicts it. + +Separately, `first_seen_at` records when MediaShelf itself first saw a row. That +is authoritative for everything added from now on, whatever Plex does to its own +field, and needs no correction. + +The exact fix, if it is ever worth the access: read `metadata_items.created_at` +from Plex's database on Loki and backfill it. + ### 4.8 Tautulli — the watch-history source Tautulli runs at `http://192.168.1.100:8181` (on Isis) and has been logging every diff --git a/mediashelf/ingest.py b/mediashelf/ingest.py index a48d85a..a08d557 100644 --- a/mediashelf/ingest.py +++ b/mediashelf/ingest.py @@ -160,6 +160,7 @@ class Ingest: self._progress(scan_id, "rolling up") self._apply_watch_rollups(provider_id) + self._correct_added_at(provider_id) self._rollup_seasons(provider_id) self._rollup_shows(provider_id) self._apply_pre_history(provider_id, coverage) @@ -373,17 +374,18 @@ class Ingest: (provider_id, item.provider_item_id), ) primary = item.parts[0].file_path if item.parts else None + now = int(time.time()) vals = ( provider_id, library_id, item.provider_item_id, "movie", item.guid, None, item.title, item.sort_title, item.year, None, None, - item.added_at, item.updated_at, 0, item.size_bytes, item.duration_ms, - len(item.parts), primary, item.resolution, item.video_codec, - item.view_count, "present", scan_id, scan_id, + item.added_at, item.added_at, item.updated_at, 0, item.size_bytes, + item.duration_ms, len(item.parts), primary, item.resolution, + item.video_codec, item.view_count, "present", scan_id, scan_id, now, ) if existing: self.db.execute( "UPDATE media_item SET library_id=?, guid=?, title=?, sort_title=?, year=?, " - "added_at=?, updated_at=?, size_bytes=?, duration_ms=?, part_count=?, " + "provider_added_at=?, updated_at=?, size_bytes=?, duration_ms=?, part_count=?, " "primary_path=?, resolution=?, video_codec=?, provider_view_count=?, " "status='present', last_seen_scan_id=? WHERE id=?", (library_id, item.guid, item.title, item.sort_title, item.year, @@ -397,10 +399,11 @@ class Ingest: cur = self.db.execute( "INSERT INTO media_item (provider_id, library_id, provider_item_id, kind, " "guid, show_guid, title, sort_title, year, parent_id, season_number, " - "added_at, updated_at, episode_count, size_bytes, duration_ms, part_count, " - "primary_path, resolution, video_codec, provider_view_count, status, " - "first_seen_scan_id, last_seen_scan_id) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals, + "added_at, provider_added_at, updated_at, episode_count, size_bytes, " + "duration_ms, part_count, primary_path, resolution, video_codec, " + "provider_view_count, status, first_seen_scan_id, last_seen_scan_id, " + "first_seen_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals, ) item_id = cur.lastrowid result.items_added += 1 @@ -481,7 +484,7 @@ class Ingest: ep_id = row["id"] self.db.execute( "UPDATE episode SET season_item_id=?, episode_number=?, title=?, " - "added_at=?, duration_ms=?, size_bytes=?, part_count=?, " + "provider_added_at=?, duration_ms=?, size_bytes=?, part_count=?, " "status='present', last_seen_scan_id=? WHERE id=?", (season_id, ep.episode_number, ep.title, ep.added_at, ep.duration_ms, ep.size_bytes, primary_count, scan_id, ep_id), @@ -489,10 +492,12 @@ class Ingest: else: cur = self.db.execute( "INSERT INTO episode (season_item_id, provider_item_id, episode_number, " - "title, added_at, duration_ms, size_bytes, part_count, status, last_seen_scan_id) " - "VALUES (?,?,?,?,?,?,?,?, 'present', ?)", + "title, added_at, provider_added_at, duration_ms, size_bytes, part_count, " + "status, last_seen_scan_id, first_seen_at) " + "VALUES (?,?,?,?,?,?,?,?,?, 'present', ?, ?)", (season_id, ep.provider_item_id, ep.episode_number, ep.title, - ep.added_at, ep.duration_ms, ep.size_bytes, primary_count, scan_id), + ep.added_at, ep.added_at, ep.duration_ms, ep.size_bytes, + primary_count, scan_id, int(time.time())), ) ep_id = cur.lastrowid self.db.execute("DELETE FROM media_part WHERE episode_id=?", (ep_id,)) @@ -546,7 +551,7 @@ class Ingest: """) c.execute(""" UPDATE episode SET watch_count=0, partial_count=0, abandoned_count=0, - last_watched_at=NULL, last_touched_at=NULL + last_watched_at=NULL, last_touched_at=NULL, first_watched_at=NULL """) agg = """ @@ -584,9 +589,54 @@ class Ingest: partial_count = COALESCE((SELECT partial FROM _wagg WHERE pid = episode.provider_item_id), 0), abandoned_count = COALESCE((SELECT abandoned FROM _wagg WHERE pid = episode.provider_item_id), 0), last_watched_at = (SELECT last_watched FROM _wagg WHERE pid = episode.provider_item_id), + first_watched_at = (SELECT first_watched FROM _wagg WHERE pid = episode.provider_item_id), last_touched_at = (SELECT last_touched FROM _wagg WHERE pid = episode.provider_item_id) """) + def _correct_added_at(self, provider_id: int) -> None: + """Repair added_at where Plex's value is provably wrong. + + Plex's addedAt tracks the FILE, not the library entry: replace or + re-encode a file and Date Added resets while the item and its watch + history survive. Measured on the live server, 19% of items report a + last-watch EARLIER than their added date. + + A play is proof the item already existed, so the first completed view is + a lower bound on the true add date. That is not the exact date Plex + never kept, but it is strictly better than a value we can show is + impossible — and it matters, because pre_history is derived from + added_at and a wrongly-recent date promotes an item into the CONFIDENT + reclaim pool when it belongs in the uncertain one. + + Items nobody ever watched keep Plex's value; nothing contradicts it. + first_seen_at is authoritative for anything MediaShelf sees appear. + """ + c = self.db.conn + c.execute("UPDATE episode SET added_at = provider_added_at " + "WHERE provider_added_at IS NOT NULL") + c.execute(""" + UPDATE episode SET added_at = first_watched_at + WHERE first_watched_at IS NOT NULL AND first_watched_at > 0 + AND (added_at IS NULL OR added_at > first_watched_at) + """) + + c.execute("UPDATE media_item SET added_at = provider_added_at, " + "added_at_source = 'provider' " + "WHERE kind = 'movie' AND provider_added_at IS NOT NULL " + "AND provider_id = ?", (provider_id,)) + cur = c.execute(""" + UPDATE media_item SET added_at = first_watched_at, + added_at_source = 'first_watch' + WHERE kind = 'movie' AND provider_id = ? + AND first_watched_at IS NOT NULL AND first_watched_at > 0 + AND (added_at IS NULL OR added_at > first_watched_at) + """, (provider_id,)) + corrected = cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0 + if corrected: + self._warn( + "%d movie(s) had a Plex addedAt later than their first play; " + "corrected to the first play date" % corrected) + def _rollup_seasons(self, provider_id: int) -> None: c = self.db.conn c.execute(""" @@ -614,6 +664,13 @@ class Ingest: WHERE kind = 'season' AND provider_id = ? """, (provider_id,)) + c.execute(""" + UPDATE media_item SET added_at_source = CASE + WHEN provider_added_at IS NOT NULL AND added_at < provider_added_at + THEN 'first_watch' ELSE 'provider' END + WHERE kind = 'season' AND provider_id = ? + """, (provider_id,)) + # distinct watchers across the season's episodes c.execute(""" UPDATE media_item SET distinct_watcher_count = COALESCE(( diff --git a/mediashelf/migrations/002_added_at_provenance.sql b/mediashelf/migrations/002_added_at_provenance.sql new file mode 100644 index 0000000..b0c90a2 --- /dev/null +++ b/mediashelf/migrations/002_added_at_provenance.sql @@ -0,0 +1,26 @@ +-- Plex's addedAt tracks the FILE, not the library entry. Replacing or +-- re-encoding a file resets Date Added while the item and its watch history +-- survive, so 19% of the live library (22% of TV Show Archive) reports a +-- lastViewedAt EARLIER than its addedAt — watched before it was "added". +-- +-- That matters beyond cosmetics: pre_history is derived from added_at, so an +-- old item whose file was replaced looks post-coverage and lands in the +-- CONFIDENT reclaim pool when it belongs in the uncertain one. +-- +-- Keep Plex's raw value for reference, and let added_at hold the best estimate +-- (see ingest._correct_added_at). Also record when MediaShelf itself first saw +-- a row, which is authoritative from now on regardless of what Plex does. + +ALTER TABLE media_item ADD COLUMN provider_added_at INTEGER; +ALTER TABLE media_item ADD COLUMN added_at_source TEXT; +ALTER TABLE media_item ADD COLUMN first_seen_at INTEGER; + +ALTER TABLE episode ADD COLUMN provider_added_at INTEGER; +ALTER TABLE episode ADD COLUMN first_watched_at INTEGER; +ALTER TABLE episode ADD COLUMN first_seen_at INTEGER; + +-- Backfill: existing rows carry Plex's value in added_at. +UPDATE media_item SET provider_added_at = added_at WHERE provider_added_at IS NULL; +UPDATE episode SET provider_added_at = added_at WHERE provider_added_at IS NULL; + +CREATE INDEX ix_item_first_seen ON media_item(first_seen_at); diff --git a/mediashelf/queries.py b/mediashelf/queries.py index 3779870..ee6443a 100644 --- a/mediashelf/queries.py +++ b/mediashelf/queries.py @@ -65,6 +65,7 @@ BASE_COLUMNS = """ i.last_watched_at, i.last_touched_at, i.first_watched_at, i.distinct_watcher_count, i.pre_history, i.kept, i.kept_via, i.kept_mark_id, i.provider_view_count, i.status, i.parent_id, i.season_number, + i.provider_added_at, i.added_at_source, i.first_seen_at, parent.title AS show_title """ @@ -200,6 +201,8 @@ class Query: flags.append("duplicate") if (d.get("provider_view_count") or 0) > 0 and not d.get("watch_count"): flags.append("history_gap") + if d.get("added_at_source") == "first_watch": + flags.append("added_corrected") # A season has one part per episode, so part_count > 1 is normal there; # only flag genuinely extra files (a movie held twice, or a split episode). parts = d.get("part_count") or 0 @@ -218,6 +221,9 @@ class Query: "library": {"id": d["library_id"], "title": d["library_title"]}, "size_bytes": d.get("size_bytes") or 0, "added_at": d.get("added_at"), + "provider_added_at": d.get("provider_added_at"), + "added_at_source": d.get("added_at_source"), + "first_seen_at": d.get("first_seen_at"), "last_watched_at": d.get("last_watched_at"), "last_touched_at": d.get("last_touched_at"), "watch_count": d.get("watch_count") or 0, diff --git a/mediashelf/rules.py b/mediashelf/rules.py index aaf90e9..d5033c5 100644 --- a/mediashelf/rules.py +++ b/mediashelf/rules.py @@ -25,6 +25,9 @@ FIELDS: dict[str, tuple[str, str]] = { "year": ("i.year", "int"), "size_bytes": ("i.size_bytes", "int"), "added_at": ("i.added_at", "ts"), + "provider_added_at": ("i.provider_added_at", "ts"), + "added_at_source": ("i.added_at_source", "str"), + "first_seen_at": ("i.first_seen_at", "ts"), "updated_at": ("i.updated_at", "ts"), "episode_count": ("i.episode_count", "int"), "part_count": ("i.part_count", "int"), diff --git a/mediashelf/static/app.css b/mediashelf/static/app.css index 7dc7c1d..d933107 100644 --- a/mediashelf/static/app.css +++ b/mediashelf/static/app.css @@ -156,6 +156,7 @@ td.title-cell { max-width: 380px; overflow: hidden; text-overflow: ellipsis; } .flag.duplicate { color: var(--accent); border-color: var(--accent); } .flag.kept { color: var(--kept); border-color: var(--kept); } .flag.multi_part, .flag.history_gap { color: var(--faint); border-color: var(--faint); } +.flag.added_corrected { color: var(--accent); border-color: var(--accent); opacity: .8; } .pager { display: flex; gap: .6rem; align-items: center; justify-content: center; padding: .7rem; color: var(--dim); font-size: .85rem; } diff --git a/mediashelf/static/app.js b/mediashelf/static/app.js index cd463c7..ccf9592 100644 --- a/mediashelf/static/app.js +++ b/mediashelf/static/app.js @@ -551,7 +551,15 @@ async function openItem(id) { const add = (k, v) => { dl.append(el('dt', { text: k }), el('dd', { text: v })); }; add('Library', it.library.title); add('Size', `${bytes(it.size_bytes)} (${exact(it.size_bytes)})`); - add('Added', date(it.added_at)); + if (it.added_at_source === 'first_watch') { + add('Added', `${date(it.added_at)} (corrected)`); + add('', `Plex reports ${date(it.provider_added_at)}, but it was already being ` + + `watched before then — Plex's Date Added follows the file, so replacing ` + + `or re-encoding one resets it. The first play is used as a lower bound.`); + } else { + add('Added', date(it.added_at)); + } + if (it.first_seen_at) add('First seen by MediaShelf', date(it.first_seen_at)); add('Last played', it.last_watched_at ? `${date(it.last_watched_at)} (${agoPhrase(it.last_watched_at)})` : 'never'); add('Plays', `${it.watch_count} finished · ${it.partial_count} partial · ${it.abandoned_count} abandoned`); add('Distinct viewers', it.distinct_watcher_count); diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 295d0a8..a6ac2a3 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -138,3 +138,53 @@ def test_episodes_without_a_parent_rating_key_still_build_a_season(scanned): def test_no_orphan_episode_warnings(scanned, rescan): r = rescan("full") assert not [w for w in (r.warnings or []) if "no season" in w] + + +def test_added_at_is_corrected_when_plex_claims_it_postdates_a_play(scanned): + """Plex's addedAt follows the file, so replacing one resets Date Added while + the watch history survives. A play proves the item already existed.""" + row = scanned.one("SELECT * FROM media_item WHERE title = 'Replaced File'") + assert row is not None + + assert row["provider_added_at"] > row["first_watched_at"], \ + "fixture should present an addedAt later than the first play" + assert row["added_at"] == row["first_watched_at"], \ + "added_at was not pulled back to the first play" + assert row["added_at_source"] == "first_watch" + + +def test_correction_leaves_unwatched_items_alone(scanned): + """Nothing contradicts Plex for an item nobody ever played.""" + rows = scanned.query( + "SELECT * FROM media_item WHERE kind='movie' AND watch_count = 0 " + "AND provider_added_at IS NOT NULL LIMIT 20") + assert rows + for r in rows: + assert r["added_at"] == r["provider_added_at"] + assert r["added_at_source"] == "provider" + + +def test_corrected_date_moves_the_item_into_the_uncertain_pool(scanned): + """The reason this matters: pre_history is derived from added_at, so a + wrongly-recent date promotes an item into the CONFIDENT reclaim pool.""" + row = scanned.one("SELECT * FROM media_item WHERE title = 'Replaced File'") + cov = scanned.one("SELECT earliest_event_at FROM history_coverage LIMIT 1") + if row["added_at"] < cov["earliest_event_at"]: + assert row["pre_history"] == 1 + + +def test_first_seen_at_is_recorded(scanned): + n = scanned.scalar("SELECT COUNT(*) FROM media_item WHERE first_seen_at IS NULL " + "AND kind='movie'") + assert n == 0, "first_seen_at should be stamped on every row MediaShelf creates" + + +def test_added_at_correction_is_idempotent(scanned, rescan): + before = scanned.query( + "SELECT id, added_at, provider_added_at, added_at_source FROM media_item " + "ORDER BY id") + rescan("full") + after = scanned.query( + "SELECT id, added_at, provider_added_at, added_at_source FROM media_item " + "ORDER BY id") + assert [dict(r) for r in before] == [dict(r) for r in after] diff --git a/tools/mockserver.py b/tools/mockserver.py index ff2e382..ae19897 100644 --- a/tools/mockserver.py +++ b/tools/mockserver.py @@ -70,6 +70,19 @@ def _build(seed=7): "size": rnd.randint(40, 90) * 10**9, "container": "mkv"}]}], }) + # Plex's addedAt follows the FILE: re-encode one and Date Added resets while + # the item and its watch history survive. This movie reproduces that — added + # "yesterday" but watched two years ago. 19% of the live library is like this. + movies.append({ + "sectionKey": "1", "ratingKey": "1999", "guid": "plex://movie/replaced", + "type": "movie", "title": "Replaced File", "year": 2001, + "addedAt": NOW - 1 * DAY, "updatedAt": NOW - 1 * DAY, + "duration": 100 * 60000, "viewCount": 3, + "Media": [{"videoResolution": "1080", "Part": [ + {"id": 6999, "file": "/mnt/titan4/Movies/Replaced File/Replaced.mkv", + "size": 9 * 10**9}]}], + }) + # ── shows / seasons / episodes ───────────────────────────────────── ep_rk = 50000 for s in range(6): @@ -138,6 +151,16 @@ def _build(seed=7): }) row_id += 1 # guarantee the oldest event is exactly COVERAGE_DAYS old, so pre_history is testable + for k in range(3): + t = NOW - (300 - k * 20) * DAY + history.append({ + "row_id": row_id + k, "reference_id": (row_id + k) // 2, + "date": t, "started": t, "stopped": t + 6000, + "rating_key": "1999", "user_id": 1, "user": "user1", + "friendly_name": "Jess", "media_type": "movie", + "percent_complete": 99, "watched_status": 1, + "play_duration": 6000, "paused_counter": 0, "platform": "Chrome", + }) history[0]["date"] = history[0]["started"] = NOW - COVERAGE_DAYS * DAY history.sort(key=lambda h: h["date"]) return movies, episodes, shows, history