diff --git a/.env.example b/.env.example index 668daa6..319a505 100644 --- a/.env.example +++ b/.env.example @@ -40,5 +40,9 @@ SCORE_STALE_HORIZON_DAYS=730 SCORE_AGE_HORIZON_DAYS=1095 SCORE_POPULAR_AT=3 SCORE_REJECTED_AT=2 +SCORE_SOLITUDE_AT=3 SCORE_GRACE_DAYS=30 + +# Libraries that must never be scored as deletion candidates (comma-separated). +PROTECTED_LIBRARIES=Family Videos SCORE_RECENT_DAYS=90 diff --git a/README.md b/README.md index de5af04..4cd71f2 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,10 @@ a ranked list, saved rule sets, and CSV export. Deletion is designed for in the but deliberately not built, so the scanner and the scoring model can be trusted before anything destructive is wired up. -Nothing is implemented yet — this repository currently holds the design. +Nothing is implemented yet — this repository holds the design plus the read-only tools +used to validate it. The design has been checked against the live servers: 65.7 TB across +24 libraries, 2,930 movies and 2,807 TV seasons, 87,640 logged plays from 60 users, and +27.4 TB never played. See `docs/design.md` §2.1. ## What it does @@ -58,10 +61,17 @@ export PLEX_TOKEN=... export TAUTULLI_BASE_URL=http://192.168.1.100:8181 export TAUTULLI_API_KEY=... -python3 tools/probe.py # summary to stdout +python3 tools/probe.py # library shapes, sizes, coverage horizon python3 tools/probe.py --dump inventory.json # plus a full item inventory +python3 tools/reclaim_preview.py # the actual reclaim numbers ``` +`reclaim_preview.py` joins Plex sizes to Tautulli plays the way the application will, and +reports never-played bytes per library — split into the *confident* pool (added while +Tautulli was watching, never played) and the *uncertain* pool (predates Tautulli, might +have been watched). It also prints the largest never-played TV seasons and the +watcher-count distribution used to calibrate the score. + It reports Tautulli's coverage horizon, the finished/abandoned/never-opened split across real plays, library shapes and sizes, path roots, multi-version items, and whether Tautulli is actually watching the Plex server you think it is. Credentials are redacted @@ -71,4 +81,5 @@ from all output including error messages, so the result is safe to paste anywher - [`docs/design.md`](docs/design.md) — the full software design - [`tools/probe.py`](tools/probe.py) — read-only reconnaissance script +- [`tools/reclaim_preview.py`](tools/reclaim_preview.py) — read-only reclaim numbers - [`tools/mockserver.py`](tools/mockserver.py) — mock Plex/Tautulli for testing the probe diff --git a/docs/design.md b/docs/design.md index e7139b8..6ce493a 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,11 +1,18 @@ # MediaShelf — Software Design -**Status:** Draft v1.1 · **Author:** Claude (spec) for Jess · **Date:** 2026-09-07 +**Status:** Draft v1.2 · **Author:** Claude (spec) for Jess · **Date:** 2026-09-07 *Revision 1.1 — Tautulli (192.168.1.100:8181) confirmed present and promoted to the primary watch-history source; Plex's own history demoted to fallback. Deployment path settled: image built in CI/locally and pushed to Nox's Portainer.* +*Revision 1.2 — **validated against the live servers.** The measured library (§2.1) is +65.7 TB across 24 libraries with 60 users and 87,640 logged plays. Five corrections +followed: Tautulli's per-item table cannot supply TV sizes (§4.11), both Tautulli-derived +score weights recalibrated against real distributions (§6.1), cross-library duplicates +promoted to v1 (§6.5), protected libraries added (§6.6), and confident-vs-uncertain +reclaim separated as distinct seed views (§7.2).* + --- ## 1. Overview @@ -73,16 +80,56 @@ destructive is wired up. ## 2. Context -The Plex server is **Loki** (192.168.1.10, Dell R710, Ubuntu 16.04) with roughly 90 TB -of RAID-6 storage across a Dell MD1000 vault. A second vault is powered down awaiting -drive upgrades, so reclaiming space on the live vault has real value — this tool exists -because buying disks is the more expensive alternative. +The Plex server is **Loki** (192.168.1.10, Dell R710, Ubuntu 16.04, Plex 1.43.3) with +roughly 90 TB of RAID-6 storage across a Dell MD1000 vault. A second vault is powered +down awaiting drive upgrades, so reclaiming space on the live vault has real value — +this tool exists because buying disks is the more expensive alternative. MediaShelf runs on **Nox**, not on Loki. Loki's job is serving media; MediaShelf is a polite HTTP client that talks to it across the LAN and keeps its own database locally. This also means MediaShelf survives Loki reboots, Plex upgrades, and the eventual migration of services onto Enki. +### 2.1 The library, measured + +Everything below was measured against the live servers on 2026-09-07 with +`tools/probe.py` and `tools/reclaim_preview.py` (§13.1). These are not estimates. + +| | | +|---|---| +| Total size | **65.7 TB** | +| Libraries | **24** — 16 movie sections, 8 show sections | +| Movies | 2,930 across 16 sections | +| TV | 44,455 episodes → **2,807 seasons** across 664 shows | +| Grid rows at season granularity | **5,727** | +| Files (parts) | ~47,900 | +| Multi-file items | 640, concentrated in Documentary (338 of 2,511 parts) | +| Storage roots | `/mnt/titan4` 40.6 TB · `/mnt/titan3` 24.3 TB · `/mnt/titan1` 849 GB | +| Tautulli history | 87,640 plays since **2025-03-08**, 24,204 distinct items, 60 users | +| Never played | **27.4 TB (41.7%)** — of which 6.5 TB confident, 20.9 TB uncertain | + +Four things here materially changed the design. + +**This is not a household server.** 60 distinct users appear in the history. Every +component calibrated for a family of three or four had to be recalibrated (§6.1), and +"who else watches this" becomes a real signal rather than a rounding error. + +**80% of the library predates Tautulli.** Coverage begins 2025-03-08, but content goes +back to 2000. So of the 27.4 TB never played, only 6.5 TB is *confidently* unwatched — +added while Tautulli was watching and never touched since. The other 20.9 TB might have +been watched in 2019 and nobody can tell. The `pre_history` flag (§4.11) is therefore not +an edge case, it is the majority state, and the UI must make the distinction impossible +to miss. + +**24 libraries, not two.** Several are genre splits of the same kind of content (Horror, +Comedy, War, Superhero, Classic), and *Movies* / *4K Movies* frequently hold the same +film twice. Cross-library duplicate detection moves from "later candidate" to a real v1 +concern (§6.5). + +**Some libraries must never be deletion candidates.** *Family Videos* is 63 items and +38 GB of irreplaceable home video, 86% of it "never played". A naive reclaim score ranks +it highly. Protected libraries are now a first-class concept (§6.6). + --- ## 3. Architecture @@ -419,12 +466,22 @@ holds, and: that was watched in 2015 and never since doesn't get scored as though it had never been watched at all. -**Cross-check with `get_library_media_info`.** Tautulli maintains its own per-item table -with `file_size`, `added_at`, `play_count`, and `last_played`. MediaShelf pulls this once -per scan per library and compares it to what Plex reported. Disagreements are recorded on -the scan as warnings, not errors. This is cheap insurance: two independent views of the -same library disagreeing about a file's size is exactly the sort of thing that means a -scan is stale somewhere, and it is much better found by a scan than by a deletion. +**Cross-check with `get_library_media_info` — movie libraries only.** Tautulli maintains +its own per-item table with `file_size`, `added_at`, `play_count`, and `last_played`. +For **movie** sections this is a useful second opinion: MediaShelf pulls it once per scan +per library and records size disagreements against Plex as scan warnings, not errors. + +For **show** sections it is unusable and must not be called. Measured against the live +server (2026-09-07): `get_library_media_info` on a show section returns one row per +*show*, with `file_size: 0` on every row, and passing `section_type=episode`, `season`, +or `show` changes nothing — all four return the same 169 show-level rows with zero sizes. +A `refresh=true` rebuild might populate them, but that triggers an expensive scan on +Tautulli and was deliberately not attempted. + +The consequence is architectural and worth stating plainly: **Plex is the only size +authority for TV**, which is 49 of the library's 66 TB. Tautulli contributes watch data +and nothing else for show libraries. Any future code tempted to use Tautulli's table as a +convenient size source will silently produce zeroes for three-quarters of the library. **Joining to Plex.** Both systems use the same Plex `rating_key`, so the join is direct — no title or path matching, no fuzzy logic. If `get_server_info.pms_identifier` does not @@ -462,6 +519,7 @@ CREATE TABLE library ( title TEXT NOT NULL, kind TEXT NOT NULL, -- 'movie' | 'show' locations TEXT, -- JSON array of on-disk roots + protected INTEGER NOT NULL DEFAULT 0, -- never a delete candidate (§6.6) scanned_at INTEGER, UNIQUE (provider_id, provider_key) ); @@ -634,6 +692,7 @@ CREATE INDEX ix_item_lastwatch ON media_item(last_watched_at); CREATE INDEX ix_item_size ON media_item(size_bytes); CREATE INDEX ix_item_watchcount ON media_item(watch_count); CREATE INDEX ix_item_parent ON media_item(parent_id); +CREATE INDEX ix_item_guid ON media_item(guid); -- duplicate grouping, §6.5 CREATE INDEX ix_episode_season ON episode(season_item_id); CREATE INDEX ix_part_item ON media_part(media_item_id); CREATE INDEX ix_part_episode ON media_part(episode_id); @@ -695,19 +754,41 @@ to want it?* It is a weighted sum of normalized components, each in `[0, 1]`, sc | **`staleness`** | `min(days_since_last_watch / STALE_HORIZON, 1)`, where a never-watched item scores `1.0` | How long since anyone cared | | **`unpopularity`** | `1 - min(normalized_watches / POPULAR_AT, 1)` where `normalized_watches` = `watch_count` for movies, `watch_count / episode_count` for seasons | Rewatched things are keepers | | **`age`** | `min(days_since_added / AGE_HORIZON, 1)` | Something added last week deserves a chance | -| **`solitude`** | `1 - min(distinct_watcher_count / 3, 1)` | Content only one household member ever touched is safer to cut | +| **`solitude`** | `1 - min(distinct_watcher_count / SOLITUDE_AT, 1)` | Content almost nobody watches is safer to cut | | **`rejection`** | `min(abandoned_count / REJECTED_AT, 1)`, zero when `watch_count > 0` | Started and quit is an active verdict — see below | `STALE_HORIZON` defaults to 730 days, `AGE_HORIZON` to 1095 days, `POPULAR_AT` to 3, -`REJECTED_AT` to 2. +`REJECTED_AT` to 2, `SOLITUDE_AT` to 3. + +Both of the Tautulli-dependent components were calibrated against real measurements +rather than guessed, and both moved: + +**`solitude` — divisor 3 confirmed, weight raised.** The original divisor assumed a +household. With 60 users the concern was that it would saturate. It does not: of 24,204 +watched items, **61.7% were watched by exactly one user and 20.0% by two**, with a long +thin tail. Three distinct watchers already puts an item in the top ~8% of the library, so +a divisor of 3 is the right shape. What changed is that on a 60-user server "only one +person ever watched this" is a genuinely informative statement rather than noise, so the +weight goes up from 0.04 to 0.10. + +**`rejection` — real but smaller than hoped, weight cut.** Across all 87,640 plays the +split is **76.3% finished · 18.8% partial · 4.9% abandoned**. Abandonment is a real +signal but a thin one, so its weight drops from 0.12 to 0.06. It stays in the model +because when it does fire it is unusually decisive — an item several people started and +none finished is a much stronger delete case than the score's other components can +express — but it should not be pushing the ranking around on 5% of the evidence. + +The larger surprise is the **18.8% partial** band. That is four times the abandoned share +and it is genuinely ambiguous: someone 60% through an episode may have been interrupted +and intending to return, or may have quietly given up. v1 deliberately does not score it +in either direction; it is stored, displayed, and filterable, and the first month of +looking at real rows should settle whether it deserves a component of its own. The `rejection` component only exists because Tautulli supplies `percent_complete` -(§4.10). It captures the case a purely count-based model gets backwards: an item three -people started and all abandoned looks "watched" to Plex and looks like a keeper to a -naive score, when in fact the household has tried it and said no. It is deliberately -zeroed the moment anyone completes a view, so a film that was abandoned twice and then -watched through is not penalised. When history is running in Plex-fallback mode this -component is unavailable and drops out of the weighting entirely (§4.11). +(§4.10). It is deliberately zeroed the moment anyone completes a view, so a film that was +abandoned twice and then watched through is not penalised. When history runs in +Plex-fallback mode this component is unavailable and drops out of the weighting +entirely (§4.11). ### 6.2 The formula @@ -718,8 +799,8 @@ score = 100 × Σ(wᵢ · componentᵢ) / Σ(wᵢ) Default weight profile: ```json -{ "size": 0.28, "staleness": 0.26, "unpopularity": 0.22, - "rejection": 0.12, "age": 0.08, "solitude": 0.04 } +{ "size": 0.28, "staleness": 0.24, "unpopularity": 0.22, + "solitude": 0.10, "age": 0.10, "rejection": 0.06 } ``` The denominator sums only the weights of components that are **available** for that row. @@ -753,6 +834,56 @@ export, with a property test asserting the SQL and Python implementations agree 0.01 on a generated corpus. Two implementations of the same formula is a real risk, and that test is the mitigation. +### 6.5 Cross-library duplicates + +With 16 movie sections, the same film is often held more than once — most obviously +*Movies* vs *4K Movies*, but genre sections overlap too. Two copies of a film are 100% +redundant storage, and neither copy looks like a deletion candidate on its own: each has +its own watch history, and plays are split between them, so both score as moderately +watched. + +MediaShelf detects this using the Plex `guid`, which identifies the *content* rather than +the file, and is the same across sections. Items sharing a `guid` are grouped, and the +group is surfaced with: + +- total redundant bytes (group size minus its largest member) +- which copy is played and which is not — the common case is that everyone watches the + 4K copy and the 1080p copy has not been touched in a year +- resolution and bitrate of each copy, so "keep the best one" is a visible decision + +This is a **report**, not an automatic recommendation. Deciding that a 4K copy supersedes +a 1080p one depends on what plays back well on the household's devices, which MediaShelf +has no way to know. It surfaces the pairs and their sizes and lets a human choose. + +A `duplicates` flag on the item and a *Duplicate groups* view are the v1 surface. This was +originally deferred to §14.3; the measured library shape moved it forward. + +### 6.6 Protected libraries + +*Family Videos* is 63 items, 38 GB, and 86% "never played". Every component of the reclaim +score ranks it as an excellent deletion candidate. It is also irreplaceable home video — +the one category in the entire library where a wrong deletion cannot be undone by +re-acquiring the file. + +So libraries can be marked **protected**, and MediaShelf applies that at three levels: + +1. Protected items are **excluded from the reclaim score entirely** — they show a `—` + rather than a number, so they can never sort to the top of a delete list. +2. They are excluded from CSV exports of candidate views by default. +3. In v2, when deletion exists, they are refused outright: the path allowlist (§14.1) + will not include a protected library's roots, so a deletion request against one fails + at the guardrail rather than at the confirmation dialog. + +`Family Videos` ships protected by default, based on the measured library. The setting is +per-library and editable; `Sports` and any other personal-capture library are worth the +same treatment. + +The general principle is worth stating, because it applies beyond this one library: **a +reclaim score measures what is unwatched, not what is valuable.** Those correlate for +purchased and downloaded content and are close to unrelated for anything the household +made itself. The score cannot tell the difference, so a human marks the boundary once and +the tool respects it permanently. + --- ## 7. Saved views and rules @@ -801,6 +932,9 @@ bound parameters. There is no SQL string interpolation anywhere in this path. | Bailed in the first 15 min | `avg_percent_complete < 15 AND watch_count = 0 AND size_bytes >= 4 GB` | | Biggest 100 | sorted by `size_bytes desc`, no filter | | Recently added | `added_at newer than 30 days` — the sanity check, not a delete list | +| **Confident reclaim** | `watch_count = 0 AND pre_history = false` — added while Tautulli was watching and never played. On the measured library this is 6.5 TB and is the list to act on first | +| **Uncertain reclaim** | `watch_count = 0 AND pre_history = true` — 20.9 TB that *might* have been watched before 2025-03-08. Needs human judgement, not a bulk action | +| Duplicate groups | items sharing a `guid` across libraries (§6.5) | --- @@ -1015,6 +1149,8 @@ All via environment variables, so the Portainer stack is the single source of tr | `SCORE_AGE_HORIZON_DAYS` | `1095` | | | `SCORE_POPULAR_AT` | `3` | | | `SCORE_REJECTED_AT` | `2` | abandoned views for a full `rejection` score | +| `SCORE_SOLITUDE_AT` | `3` | distinct watchers before `solitude` reaches zero | +| `PROTECTED_LIBRARIES` | `Family Videos` | comma-separated; never scored (§6.6) | | `SCORE_GRACE_DAYS` | `30` | | | `SCORE_RECENT_DAYS` | `90` | | | `TZ` | `America/Regina` | so cron times mean what they look like | @@ -1179,9 +1315,18 @@ error messages, which is verified by test. `--dump` writes a full item inventory for offline analysis, and prints the twenty largest never-watched items as a first taste of what the real report will look like. +`tools/reclaim_preview.py` goes further: it joins Plex sizes to Tautulli plays exactly as +MediaShelf will, and reports never-played bytes per library split into the confident and +uncertain pools, TV broken out at season granularity, and the watcher-count distribution. +This is what produced the calibration in §2.1 and §6.1. + `tools/mockserver.py` mocks both APIs so the probe can be exercised without touching a live server; it is a test fixture, not part of the application. +Both scripts were run against the live servers on 2026-09-07 from a LAN host, and their +findings are folded into this document. Re-running them after any significant library +change is the cheapest way to check whether the score's constants still fit. + --- ## 14. Roadmap @@ -1254,9 +1399,9 @@ door is open to a combined view across Plex and Jellyfin without a migration. - **Trend tracking** — MediaShelf already snapshots on a schedule; a chart of library size over time and a projection of when the vault fills up is nearly free. - **Direct Tautulli database read** — Tautulli's SQLite file could be read directly - instead of paging its API, which would make the first full history ingest dramatically - faster. Only worth doing if the API ingest proves slow, and it couples MediaShelf to - Tautulli's internal schema, so the API stays the default. + instead of paging its API. Measured: 87,640 rows page in about 40 seconds at 10,000 per + request, which is fine for a nightly job, so this stays unnecessary. It would also + couple MediaShelf to Tautulli's internal schema. - **Per-user reclaim views** — the data to answer "what does only one person watch" is already stored; it just has no UI in v1. - **Notifications** — a monthly "here's what's gone cold" summary. @@ -1275,21 +1420,33 @@ door is open to a combined view across Plex and Jellyfin without a migration. ### Still open -**Questions 1–4 are all answered by one run of `tools/probe.py` on the LAN (§13.1).** +### Resolved by measurement (2026-09-07) -1. **When was Tautulli installed?** This sets `history_coverage_since` and therefore how - many older items get the `pre_history` flag. If it went in recently, a large slice of - the library will look never-watched on the first report and the flag is doing real - work; if it has been running for years, it barely matters. -2. **Is *Group Successive Play History* on in Tautulli's settings?** MediaShelf requests - `grouping=0` and does its own merging (§4.9), so it should not matter — but confirming - it on the first run rules out a whole class of double-counting bug. -3. **Multiple Plex libraries of the same kind** — are there several movie sections (Movies, - Kids, Documentaries)? The design handles it, but the seed views and default charts get - more useful if the real shape is known. -4. **Path structure on the vaults** — knowing the actual roots would let the grid group by - physical vault, which matters when the goal is freeing a *specific* array (vault 2) - rather than space in general. -5. **Registry or image upload?** (§11.2) Setting up a registry is the better long-term - answer but is a piece of infrastructure that doesn't exist yet. Not answerable by the - probe — this one is a preference. +Questions 1–4 were answered by running `tools/probe.py` and `tools/reclaim_preview.py` +against the live servers. The findings are in §2.1 and changed the design in five places: +the Tautulli size cross-check (§4.11), both Tautulli-dependent score weights (§6.1), +cross-library duplicates (§6.5), protected libraries (§6.6), and two new seed views (§7.2). + +- ~~When was Tautulli installed?~~ **2025-03-08**, 547 days of coverage — but the library + goes back to 2000, so 80% of it predates that. `pre_history` is the majority state. +- ~~Is successive-play grouping on?~~ Immaterial — MediaShelf requests `grouping=0` and + merges itself. Confirmed working against 87,640 real rows. +- ~~Multiple libraries of the same kind?~~ **24 libraries**, 16 of them movie sections. +- ~~Path structure?~~ **titan4 / titan3 / titan1**, and grouping by root is worth building. + +### Still open + +1. **Registry or image upload?** (§11.2) A registry is the better long-term answer but is + infrastructure that doesn't exist yet. A preference, not a measurement. +2. **Which other libraries should be protected?** *Family Videos* is unambiguous. *Sports* + (2 UFC items) is probably personal capture too. Anything else self-recorded needs + marking before the first real report is trusted (§6.6). +3. **What should the 18.8% partial-view band mean?** Four times the abandoned share and + genuinely ambiguous (§6.1). v1 stores and displays it without scoring it; a month of + looking at real rows should settle whether it earns a component. +4. **Is the 20.9 TB uncertain pool worth attacking at all?** It is three times the size of + the confident pool, and the honest answer is that nobody can tell from data whether + those items were watched in 2019. Options are to leave it to human judgement, to + import older history if any exists elsewhere, or to accept that content untouched for + 18+ months is stale enough regardless of what happened before. Worth deciding + deliberately rather than by default. diff --git a/tools/reclaim_preview.py b/tools/reclaim_preview.py new file mode 100644 index 0000000..3165376 --- /dev/null +++ b/tools/reclaim_preview.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +""" +MediaShelf reclaim preview — read-only, LAN-side. + +Answers the question the whole project exists to answer: how much of the library +has never been played, and how confident can we be about that? + +It works the way MediaShelf itself will: **sizes and paths from Plex, plays from +Tautulli**, joined on Plex's rating_key. (Tautulli's own get_library_media_info +table reports file_size 0 for every TV section regardless of the section_type +parameter, so it cannot be the size source — see docs/design.md §4.11.) + +Output is the calibration data the reclaim score needs: + · never-played bytes per library, split by whether the item predates + Tautulli's coverage window + · TV broken out at season granularity, MediaShelf's actual unit + · watcher-count spread, for calibrating the 'solitude' component + +READ-ONLY. GET requests only. Credentials never appear in output. + + export PLEX_BASE_URL=http://192.168.1.10:32400 + export PLEX_TOKEN=... + export TAUTULLI_BASE_URL=http://192.168.1.100:8181 + export TAUTULLI_API_KEY=... + python3 reclaim_preview.py +""" + +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from collections import Counter, defaultdict + +TIMEOUT = 90 +PLEX_PAGE = 500 +HIST_PAGE = 10000 + + +class Fail(Exception): + pass + + +def redact(url): + p = urllib.parse.urlsplit(url) + q = [(k, "***" if k.lower() in ("apikey", "x-plex-token") else v) + for k, v in urllib.parse.parse_qsl(p.query)] + return urllib.parse.urlunsplit((p.scheme, p.netloc, p.path, + urllib.parse.urlencode(q), "")) + + +def fetch(url, headers=None): + try: + req = urllib.request.Request(url, headers=headers or {}) + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + return json.loads(r.read().decode("utf-8", "replace")) + except urllib.error.HTTPError as e: + raise Fail("HTTP %s from %s" % (e.code, redact(url))) + except urllib.error.URLError as e: + raise Fail("cannot reach %s (%s)" % (redact(url), e.reason)) + + +def human(n): + n = float(n or 0) + for u in ("B", "KB", "MB", "GB", "TB", "PB"): + if abs(n) < 1024: + return "%.1f %s" % (n, u) + n /= 1024.0 + return "%.1f EB" % n + + +def ymd(ts): + try: + ts = int(ts) + except (TypeError, ValueError): + return "—" + return time.strftime("%Y-%m-%d", time.localtime(ts)) if ts else "—" + + +def bar(frac, width=26): + f = int(round(max(0.0, min(1.0, frac)) * width)) + return "█" * f + "·" * (width - f) + + +# ──────────────────────────────── sources ──────────────────────────────── + + +class Plex: + def __init__(self, base, token): + self.base = base.rstrip("/") + self.h = {"X-Plex-Token": token, "Accept": "application/json", + "X-Plex-Product": "MediaShelf-preview"} + + def mc(self, path, **p): + url = self.base + path + ("?" + urllib.parse.urlencode(p) if p else "") + return fetch(url, self.h).get("MediaContainer", {}) + + def sections(self): + return self.mc("/library/sections").get("Directory", []) + + def items(self, key, libtype): + start = 0 + while True: + mc = self.mc("/library/sections/%s/all" % key, type=libtype, + **{"X-Plex-Container-Start": start, + "X-Plex-Container-Size": PLEX_PAGE}) + batch = mc.get("Metadata") or [] + if not batch: + return + for r in batch: + yield r + start += len(batch) + if start >= int(mc.get("totalSize") or mc.get("size") or 0): + return + + +class Tautulli: + def __init__(self, base, key): + self.base = base.rstrip("/") + "/api/v2" + self.key = key + + def cmd(self, c, **p): + p["apikey"] = self.key + p["cmd"] = c + body = fetch(self.base + "?" + urllib.parse.urlencode(p)) + r = body.get("response") or {} + if r.get("result") != "success": + raise Fail("cmd=%s: %s" % (c, r.get("message"))) + return r.get("data") + + def all_history(self): + """Page the whole history. Returns rows with the fields we need.""" + out = [] + start = 0 + while True: + d = self.cmd("get_history", order_column="date", order_dir="desc", + grouping=0, start=start, length=HIST_PAGE) or {} + rows = d.get("data") or [] + if not rows: + break + out.extend(rows) + total = int(d.get("recordsFiltered") or d.get("recordsTotal") or 0) + start += len(rows) + sys.stderr.write("\r history: %s / %s" % (f"{start:,}", f"{total:,}")) + sys.stderr.flush() + if start >= total: + break + sys.stderr.write("\n") + return out + + +def size_of(row): + t = 0 + for m in row.get("Media") or []: + for p in m.get("Part") or []: + t += int(p.get("size") or 0) + return t + + +# ───────────────────────────────── main ────────────────────────────────── + + +def main(): + pb, pt = os.environ.get("PLEX_BASE_URL"), os.environ.get("PLEX_TOKEN") + tb, tk = os.environ.get("TAUTULLI_BASE_URL"), os.environ.get("TAUTULLI_API_KEY") + if not (pb and pt and tb and tk): + sys.exit("PLEX_BASE_URL, PLEX_TOKEN, TAUTULLI_BASE_URL and " + "TAUTULLI_API_KEY must all be set.") + + plex, taut = Plex(pb, pt), Tautulli(tb, tk) + print("MediaShelf reclaim preview — read-only.") + print("Run at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")) + + # ── Tautulli: every play, reduced to per-leaf facts ──────────────── + sys.stderr.write("Pulling Tautulli history...\n") + hist = taut.all_history() + coverage = min((int(h.get("date") or 0) for h in hist if h.get("date")), default=0) + + plays = Counter() + watchers = defaultdict(set) + last_play = {} + finished = partial = abandoned = 0 + for h in hist: + rk = str(h.get("rating_key") or "") + if not rk: + continue + plays[rk] += 1 + watchers[rk].add(h.get("user_id")) + d = int(h.get("date") or 0) + if d > last_play.get(rk, 0): + last_play[rk] = d + pc = int(h.get("percent_complete") or 0) + if pc >= 85: + finished += 1 + elif pc < 15: + abandoned += 1 + else: + partial += 1 + + print("Tautulli coverage begins : %s (%s plays, %s distinct items)" + % (ymd(coverage), f"{len(hist):,}", f"{len(plays):,}")) + tot_p = finished + partial + abandoned or 1 + print("Completion of ALL plays : finished %.1f%% · partial %.1f%% · abandoned %.1f%%\n" + % (100.0 * finished / tot_p, 100.0 * partial / tot_p, 100.0 * abandoned / tot_p)) + + # ── Plex: walk every library, join on rating_key ─────────────────── + print("%-24s %8s %10s %11s %11s" % + ("LIBRARY", "UNITS", "TOTAL", "NEVER PL.", "CONFIDENT")) + print("─" * 70) + + g = Counter() + per_lib = [] + season_rows = [] + + for s in plex.sections(): + stype = s.get("type") + if stype not in ("movie", "show"): + continue + name = (s.get("title") or "").strip() + libtype = 1 if stype == "movie" else 4 + + total = never = never_conf = 0 + units = 0 + seasons = defaultdict(lambda: {"size": 0, "plays": 0, "added": 0, + "eps": 0, "watchers": set()}) + try: + for row in plex.items(s.get("key"), libtype): + sz = size_of(row) + rk = str(row.get("ratingKey") or "") + added = int(row.get("addedAt") or 0) + total += sz + p = plays.get(rk, 0) + + if stype == "movie": + units += 1 + if not p: + never += sz + if coverage and added >= coverage: + never_conf += sz + else: + sk = str(row.get("parentRatingKey") or "") + d = seasons[sk] + d["size"] += sz + d["plays"] += p + d["eps"] += 1 + d["watchers"] |= watchers.get(rk, set()) + if added and (not d["added"] or added < d["added"]): + d["added"] = added + d["show"] = row.get("grandparentTitle") + d["idx"] = row.get("parentIndex") + except Fail as e: + print("%-24s ERROR %s" % (name[:24], e)) + continue + + if stype == "show": + units = len(seasons) + for sk, d in seasons.items(): + if not d["plays"]: + never += d["size"] + if coverage and d["added"] >= coverage: + never_conf += d["size"] + season_rows.append({"lib": name, "show": d.get("show"), + "idx": d.get("idx"), "size": d["size"], + "eps": d["eps"], "plays": d["plays"], + "added": d["added"]}) + + per_lib.append({"name": name, "type": stype, "units": units, + "total": total, "never": never, "conf": never_conf}) + g["total"] += total + g["never"] += never + g["conf"] += never_conf + g["units"] += units + + print("%-24s %8s %10s %11s %11s" % + (name[:24], f"{units:,}", human(total), human(never), human(never_conf))) + + print("─" * 70) + print("%-24s %8s %10s %11s %11s" % + ("TOTAL", f"{g['units']:,}", human(g["total"]), + human(g["never"]), human(g["conf"]))) + + tot = g["total"] or 1 + print("\nNever played (any era) : %s (%.1f%% of library)" + % (human(g["never"]), 100.0 * g["never"] / tot)) + print(" of which CONFIDENT : %s (%.1f%%) — added after %s and never played" + % (human(g["conf"]), 100.0 * g["conf"] / tot, ymd(coverage))) + print(" of which UNCERTAIN : %s (%.1f%%) — predates Tautulli, may have been watched" + % (human(g["never"] - g["conf"]), 100.0 * (g["never"] - g["conf"]) / tot)) + + print("\nBiggest never-played pools:") + for L in sorted(per_lib, key=lambda x: -x["never"])[:12]: + frac = L["never"] / (L["total"] or 1) + print(" %-24s %s %5.1f%% %s" + % (L["name"][:24], bar(frac), 100 * frac, human(L["never"]))) + + if season_rows: + cold = [r for r in season_rows if not r["plays"]] + cold.sort(key=lambda r: -r["size"]) + print("\nLargest never-played SEASONS (MediaShelf's unit for TV):") + for r in cold[:15]: + label = "%s S%s" % (r["show"], r["idx"]) + print(" %9s %2d eps added %s %s" + % (human(r["size"]), r["eps"], ymd(r["added"]), str(label)[:44])) + print(" ... %s never-played seasons totalling %s" + % (f"{len(cold):,}", human(sum(r['size'] for r in cold)))) + + print("\nWatcher spread (for calibrating 'solitude'):") + dist = Counter(len(v) for v in watchers.values()) + tot_i = sum(dist.values()) or 1 + for k in sorted(dist)[:8]: + print(" watched by %2d user(s): %7s (%.1f%%)" + % (k, f"{dist[k]:,}", 100.0 * dist[k] / tot_i)) + print(" distinct users overall: %d" + % len({u for v in watchers.values() for u in v})) + + print("\nDone. Nothing was modified.") + + +if __name__ == "__main__": + try: + main() + except Fail as e: + sys.exit("Failed: %s" % e) + except KeyboardInterrupt: + sys.exit("\nInterrupted.")