MediaShelf/tools/reclaim_preview.py

#!/usr/bin/env python3
"""
MediaShelf reclaim preview — read-only, LAN-side.

Answers the question the whole project exists to answer: how much of the library
has never been played, and how confident can we be about that?

It works the way MediaShelf itself will: **sizes and paths from Plex, plays from
Tautulli**, joined on Plex's rating_key. (Tautulli's own get_library_media_info
table reports file_size 0 for every TV section regardless of the section_type
parameter, so it cannot be the size source — see docs/design.md §4.11.)

Output is the calibration data the reclaim score needs:
  · never-played bytes per library, split by whether the item predates
    Tautulli's coverage window
  · TV broken out at season granularity, MediaShelf's actual unit
  · watcher-count spread, for calibrating the 'solitude' component

READ-ONLY. GET requests only. Credentials never appear in output.

    export PLEX_BASE_URL=http://192.168.1.10:32400
    export PLEX_TOKEN=...
    export TAUTULLI_BASE_URL=http://192.168.1.100:8181
    export TAUTULLI_API_KEY=...
    python3 reclaim_preview.py
"""

import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from collections import Counter, defaultdict

TIMEOUT = 90
PLEX_PAGE = 500
HIST_PAGE = 10000


class Fail(Exception):
    pass


def redact(url):
    p = urllib.parse.urlsplit(url)
    q = [(k, "***" if k.lower() in ("apikey", "x-plex-token") else v)
         for k, v in urllib.parse.parse_qsl(p.query)]
    return urllib.parse.urlunsplit((p.scheme, p.netloc, p.path,
                                    urllib.parse.urlencode(q), ""))


def fetch(url, headers=None):
    try:
        req = urllib.request.Request(url, headers=headers or {})
        with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
            return json.loads(r.read().decode("utf-8", "replace"))
    except urllib.error.HTTPError as e:
        raise Fail("HTTP %s from %s" % (e.code, redact(url)))
    except urllib.error.URLError as e:
        raise Fail("cannot reach %s (%s)" % (redact(url), e.reason))


def human(n):
    n = float(n or 0)
    for u in ("B", "KB", "MB", "GB", "TB", "PB"):
        if abs(n) < 1024:
            return "%.1f %s" % (n, u)
        n /= 1024.0
    return "%.1f EB" % n


def ymd(ts):
    try:
        ts = int(ts)
    except (TypeError, ValueError):
        return "—"
    return time.strftime("%Y-%m-%d", time.localtime(ts)) if ts else "—"


def bar(frac, width=26):
    f = int(round(max(0.0, min(1.0, frac)) * width))
    return "█" * f + "·" * (width - f)


# ──────────────────────────────── sources ────────────────────────────────


class Plex:
    def __init__(self, base, token):
        self.base = base.rstrip("/")
        self.h = {"X-Plex-Token": token, "Accept": "application/json",
                  "X-Plex-Product": "MediaShelf-preview"}

    def mc(self, path, **p):
        url = self.base + path + ("?" + urllib.parse.urlencode(p) if p else "")
        return fetch(url, self.h).get("MediaContainer", {})

    def sections(self):
        return self.mc("/library/sections").get("Directory", [])

    def items(self, key, libtype):
        start = 0
        while True:
            mc = self.mc("/library/sections/%s/all" % key, type=libtype,
                         **{"X-Plex-Container-Start": start,
                            "X-Plex-Container-Size": PLEX_PAGE})
            batch = mc.get("Metadata") or []
            if not batch:
                return
            for r in batch:
                yield r
            start += len(batch)
            if start >= int(mc.get("totalSize") or mc.get("size") or 0):
                return


class Tautulli:
    def __init__(self, base, key):
        self.base = base.rstrip("/") + "/api/v2"
        self.key = key

    def cmd(self, c, **p):
        p["apikey"] = self.key
        p["cmd"] = c
        body = fetch(self.base + "?" + urllib.parse.urlencode(p))
        r = body.get("response") or {}
        if r.get("result") != "success":
            raise Fail("cmd=%s: %s" % (c, r.get("message")))
        return r.get("data")

    def all_history(self):
        """Page the whole history. Returns rows with the fields we need."""
        out = []
        start = 0
        while True:
            d = self.cmd("get_history", order_column="date", order_dir="desc",
                         grouping=0, start=start, length=HIST_PAGE) or {}
            rows = d.get("data") or []
            if not rows:
                break
            out.extend(rows)
            total = int(d.get("recordsFiltered") or d.get("recordsTotal") or 0)
            start += len(rows)
            sys.stderr.write("\r  history: %s / %s" % (f"{start:,}", f"{total:,}"))
            sys.stderr.flush()
            if start >= total:
                break
        sys.stderr.write("\n")
        return out


def size_of(row):
    t = 0
    for m in row.get("Media") or []:
        for p in m.get("Part") or []:
            t += int(p.get("size") or 0)
    return t


# ───────────────────────────────── main ──────────────────────────────────


def main():
    pb, pt = os.environ.get("PLEX_BASE_URL"), os.environ.get("PLEX_TOKEN")
    tb, tk = os.environ.get("TAUTULLI_BASE_URL"), os.environ.get("TAUTULLI_API_KEY")
    if not (pb and pt and tb and tk):
        sys.exit("PLEX_BASE_URL, PLEX_TOKEN, TAUTULLI_BASE_URL and "
                 "TAUTULLI_API_KEY must all be set.")

    plex, taut = Plex(pb, pt), Tautulli(tb, tk)
    print("MediaShelf reclaim preview — read-only.")
    print("Run at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S"))

    # ── Tautulli: every play, reduced to per-leaf facts ────────────────
    sys.stderr.write("Pulling Tautulli history...\n")
    hist = taut.all_history()
    coverage = min((int(h.get("date") or 0) for h in hist if h.get("date")), default=0)

    plays = Counter()
    watchers = defaultdict(set)
    last_play = {}
    finished = partial = abandoned = 0
    for h in hist:
        rk = str(h.get("rating_key") or "")
        if not rk:
            continue
        plays[rk] += 1
        watchers[rk].add(h.get("user_id"))
        d = int(h.get("date") or 0)
        if d > last_play.get(rk, 0):
            last_play[rk] = d
        pc = int(h.get("percent_complete") or 0)
        if pc >= 85:
            finished += 1
        elif pc < 15:
            abandoned += 1
        else:
            partial += 1

    print("Tautulli coverage begins : %s  (%s plays, %s distinct items)"
          % (ymd(coverage), f"{len(hist):,}", f"{len(plays):,}"))
    tot_p = finished + partial + abandoned or 1
    print("Completion of ALL plays  : finished %.1f%% · partial %.1f%% · abandoned %.1f%%\n"
          % (100.0 * finished / tot_p, 100.0 * partial / tot_p, 100.0 * abandoned / tot_p))

    # ── Plex: walk every library, join on rating_key ───────────────────
    print("%-24s %8s %10s %11s %11s" %
          ("LIBRARY", "UNITS", "TOTAL", "NEVER PL.", "CONFIDENT"))
    print("─" * 70)

    g = Counter()
    per_lib = []
    season_rows = []

    for s in plex.sections():
        stype = s.get("type")
        if stype not in ("movie", "show"):
            continue
        name = (s.get("title") or "").strip()
        libtype = 1 if stype == "movie" else 4

        total = never = never_conf = 0
        units = 0
        seasons = defaultdict(lambda: {"size": 0, "plays": 0, "added": 0,
                                       "eps": 0, "watchers": set()})
        try:
            for row in plex.items(s.get("key"), libtype):
                sz = size_of(row)
                rk = str(row.get("ratingKey") or "")
                added = int(row.get("addedAt") or 0)
                total += sz
                p = plays.get(rk, 0)

                if stype == "movie":
                    units += 1
                    if not p:
                        never += sz
                        if coverage and added >= coverage:
                            never_conf += sz
                else:
                    sk = str(row.get("parentRatingKey") or "")
                    d = seasons[sk]
                    d["size"] += sz
                    d["plays"] += p
                    d["eps"] += 1
                    d["watchers"] |= watchers.get(rk, set())
                    if added and (not d["added"] or added < d["added"]):
                        d["added"] = added
                    d["show"] = row.get("grandparentTitle")
                    d["idx"] = row.get("parentIndex")
        except Fail as e:
            print("%-24s ERROR %s" % (name[:24], e))
            continue

        if stype == "show":
            units = len(seasons)
            for sk, d in seasons.items():
                if not d["plays"]:
                    never += d["size"]
                    if coverage and d["added"] >= coverage:
                        never_conf += d["size"]
                season_rows.append({"lib": name, "show": d.get("show"),
                                    "idx": d.get("idx"), "size": d["size"],
                                    "eps": d["eps"], "plays": d["plays"],
                                    "added": d["added"]})

        per_lib.append({"name": name, "type": stype, "units": units,
                        "total": total, "never": never, "conf": never_conf})
        g["total"] += total
        g["never"] += never
        g["conf"] += never_conf
        g["units"] += units

        print("%-24s %8s %10s %11s %11s" %
              (name[:24], f"{units:,}", human(total), human(never), human(never_conf)))

    print("─" * 70)
    print("%-24s %8s %10s %11s %11s" %
          ("TOTAL", f"{g['units']:,}", human(g["total"]),
           human(g["never"]), human(g["conf"])))

    tot = g["total"] or 1
    print("\nNever played (any era)      : %s  (%.1f%% of library)"
          % (human(g["never"]), 100.0 * g["never"] / tot))
    print("  of which CONFIDENT        : %s  (%.1f%%) — added after %s and never played"
          % (human(g["conf"]), 100.0 * g["conf"] / tot, ymd(coverage)))
    print("  of which UNCERTAIN        : %s  (%.1f%%) — predates Tautulli, may have been watched"
          % (human(g["never"] - g["conf"]), 100.0 * (g["never"] - g["conf"]) / tot))

    print("\nBiggest never-played pools:")
    for L in sorted(per_lib, key=lambda x: -x["never"])[:12]:
        frac = L["never"] / (L["total"] or 1)
        print("   %-24s %s %5.1f%%  %s"
              % (L["name"][:24], bar(frac), 100 * frac, human(L["never"])))

    if season_rows:
        cold = [r for r in season_rows if not r["plays"]]
        cold.sort(key=lambda r: -r["size"])
        print("\nLargest never-played SEASONS (MediaShelf's unit for TV):")
        for r in cold[:15]:
            label = "%s S%s" % (r["show"], r["idx"])
            print("   %9s  %2d eps  added %s  %s"
                  % (human(r["size"]), r["eps"], ymd(r["added"]), str(label)[:44]))
        print("   ... %s never-played seasons totalling %s"
              % (f"{len(cold):,}", human(sum(r['size'] for r in cold))))

    print("\nWatcher spread (for calibrating 'solitude'):")
    dist = Counter(len(v) for v in watchers.values())
    tot_i = sum(dist.values()) or 1
    for k in sorted(dist)[:8]:
        print("   watched by %2d user(s): %7s  (%.1f%%)"
              % (k, f"{dist[k]:,}", 100.0 * dist[k] / tot_i))
    print("   distinct users overall: %d"
          % len({u for v in watchers.values() for u in v}))

    print("\nDone. Nothing was modified.")


if __name__ == "__main__":
    try:
        main()
    except Fail as e:
        sys.exit("Failed: %s" % e)
    except KeyboardInterrupt:
        sys.exit("\nInterrupted.")