Implement MediaShelf v1

The application the design describes: Flask + SQLite, Plex for library data,
Tautulli for watch history, report-only.

Structure follows the design's seams. providers/ splits MediaProvider from
HistoryProvider, because on this network library data and watch data live on
different machines and Jellyfin later will have no Tautulli equivalent.
scoring.py implements the reclaim score twice - as a SQL expression for the
live grid (weights change on every slider drag, so storing it would mean
rewriting thousands of rows per drag) and in Python for CSV export and tests,
with a property test over 500 generated rows asserting the two agree.
rules.py compiles saved views to parameterized SQL through a field/operator
whitelist; nothing user-supplied is ever interpolated.

Three properties are enforced by test rather than asserted in prose:

- Ingest is idempotent. Three consecutive full scans leave every count and
  every byte total unchanged. A scanner that double-counts produces a report
  that looks plausible and is wrong.
- Keep marks survive Plex reassigning every rating key in the library. They
  are keyed on content GUID, scoped per library so the Movies and 4K Movies
  copies of the same film mark independently.
- Every config variable the app reads is declared in docker-compose.yml, so
  a variable set in Portainer can never silently do nothing.

Also found and fixed while verifying against a fake Plex+Tautulli pair:
executescript() commits the pending transaction, so migrations needed their
BEGIN/COMMIT inside the script; replaceChildren() renders null as the literal
text "null"; a hash-only URL change does not reload the document, so deep
links needed a hashchange listener; and SQLite ROUND rounds half away from
zero where Python rounds half to even.

73 tests, no live server required.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GVbG48GAXfCZatcmX123Ra

mediashelf/api.py

@@ -0,0 +1,497 @@
+"""JSON API (§8). No auth in v1 — LAN-only, single admin view (§12)."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import logging
+import time
+
+from flask import Blueprint, Response, current_app, jsonify, request
+
+from . import keeps, queries, scanner
+from .rules import RuleError
+from .keeps import KeepError
+
+log = logging.getLogger(__name__)
+
+bp = Blueprint("api", __name__, url_prefix="/api/v1")
+
+
+def db():
+    return current_app.extensions["mediashelf"]["db"]
+
+
+def cfg():
+    return current_app.extensions["mediashelf"]["config"]
+
+
+@bp.errorhandler(RuleError)
+def _rule_error(e):
+    return jsonify({"error": "invalid_rule", "message": str(e)}), 400
+
+
+@bp.errorhandler(KeepError)
+def _keep_error(e):
+    return jsonify({"error": "invalid_keep", "message": str(e)}), 400
+
+
+def _json_arg(name):
+    raw = request.args.get(name)
+    if not raw:
+        return None
+    try:
+        return json.loads(raw)
+    except (TypeError, ValueError):
+        raise RuleError("%s must be valid JSON" % name)
+
+
+def _bool_arg(name, default=False):
+    v = request.args.get(name)
+    if v is None:
+        return default
+    return v.lower() in ("1", "true", "yes")
+
+
+def _query_kwargs():
+    rule_group = _json_arg("rules")
+    weights = _json_arg("weights")
+
+    view_id = request.args.get("view_id")
+    sort = request.args.get("sort")
+    if view_id:
+        row = db().one("SELECT * FROM saved_view WHERE id = ?", (view_id,))
+        if row is None:
+            raise RuleError("no such view")
+        rule_group = json.loads(row["rules"]) if row["rules"] else None
+        sort = sort or row["sort"]
+        if row["weights"] and not weights:
+            weights = json.loads(row["weights"])
+
+    return dict(
+        library_ids=request.args.getlist("library_id") or None,
+        kinds=request.args.getlist("kind") or None,
+        q=request.args.get("q"),
+        rule_group=rule_group,
+        include_missing=_bool_arg("include_missing"),
+        include_kept=_bool_arg("include_kept"),
+        include_shows=_bool_arg("include_shows"),
+        weights=weights,
+    ), sort
+
+
+# ── items ────────────────────────────────────────────────────────────────
+
+@bp.get("/items")
+def items():
+    kw, sort = _query_kwargs()
+    q = queries.Query(db(), cfg())
+    return jsonify(q.page(
+        sort=sort,
+        page=int(request.args.get("page", 1)),
+        page_size=int(request.args.get("page_size", 100)),
+        **kw,
+    ))
+
+
+@bp.get("/items/<int:item_id>")
+def item_detail(item_id):
+    d = db()
+    q = queries.Query(d, cfg())
+    expr, where, params = q.build(include_kept=True, include_missing=True, include_shows=True)
+    params["item_id"] = item_id
+    sql = ("SELECT " + queries.BASE_COLUMNS + ", " + expr + " AS reclaim_score, "
+           + "NULL AS grace, COALESCE(d.dupe_count,1) AS duplicate_count "
+           + queries.FROM_CLAUSE + " WHERE i.id = :item_id")
+    row = d.one(sql, params)
+    if row is None:
+        return jsonify({"error": "not_found"}), 404
+
+    out = q.serialize(row)
+    out["parts"] = [dict(r) for r in d.query(
+        "SELECT * FROM media_part WHERE media_item_id = ? ORDER BY file_path", (item_id,))]
+
+    if out["kind"] == "season":
+        out["episodes"] = [dict(r) for r in d.query(
+            "SELECT * FROM episode WHERE season_item_id = ? ORDER BY episode_number",
+            (item_id,))]
+        pids = [e["provider_item_id"] for e in out["episodes"]]
+    else:
+        pids = [d.scalar("SELECT provider_item_id FROM media_item WHERE id=?", (item_id,))]
+
+    if pids:
+        marks = ",".join("?" * len(pids))
+        out["watch_history"] = [dict(r) for r in d.query(
+            "SELECT w.viewed_at, w.percent_complete, w.disposition, w.account_id, "
+            "COALESCE(a.friendly_name, a.name, w.account_id) AS who, w.platform "
+            "FROM watch_event w LEFT JOIN account a ON a.account_id = w.account_id "
+            f"WHERE w.provider_item_id IN ({marks}) ORDER BY w.viewed_at DESC LIMIT 500",
+            tuple(pids))]
+    else:
+        out["watch_history"] = []
+
+    if out.get("guid"):
+        out["duplicates"] = [dict(r) for r in d.query(
+            "SELECT i.id, i.title, i.size_bytes, i.resolution, i.watch_count, "
+            "lib.title AS library_title FROM media_item i "
+            "JOIN library lib ON lib.id=i.library_id "
+            "WHERE i.guid = ? AND i.id != ? AND i.kind='movie' AND i.status='present'",
+            (out["guid"], item_id))]
+    return jsonify(out)
+
+
+@bp.get("/libraries")
+def libraries():
+    return jsonify({"libraries": queries.size_by_library(db())})
+
+
+@bp.get("/accounts")
+def accounts():
+    return jsonify({"accounts": [dict(r) for r in db().query(
+        "SELECT a.*, (SELECT COUNT(*) FROM watch_event w WHERE w.account_id=a.account_id) "
+        "AS plays FROM account a ORDER BY plays DESC")]})
+
+
+# ── stats ────────────────────────────────────────────────────────────────
+
+@bp.get("/stats/overview")
+def stats_overview():
+    return jsonify(queries.overview(db(), cfg()))
+
+
+@bp.get("/stats/size-by-library")
+def stats_size_by_library():
+    return jsonify({"libraries": queries.size_by_library(db())})
+
+
+@bp.get("/stats/added-over-time")
+def stats_added_over_time():
+    return jsonify({"buckets": queries.added_over_time(
+        db(), request.args.get("bucket", "month"))})
+
+
+@bp.get("/stats/completion")
+def stats_completion():
+    return jsonify(queries.completion_split(db()))
+
+
+@bp.get("/stats/size-vs-lastwatched")
+def stats_scatter():
+    return jsonify({"points": queries.size_vs_lastwatched(db())})
+
+
+@bp.get("/stats/watch-distribution")
+def stats_watch_distribution():
+    return jsonify({"buckets": [dict(r) for r in db().query("""
+        SELECT CASE WHEN watch_count = 0 THEN '0'
+                    WHEN watch_count = 1 THEN '1'
+                    WHEN watch_count <= 3 THEN '2-3'
+                    WHEN watch_count <= 10 THEN '4-10'
+                    ELSE '10+' END AS bucket,
+               COUNT(*) AS items, COALESCE(SUM(size_bytes),0) AS size_bytes
+        FROM media_item WHERE kind IN ('movie','season') AND status='present'
+        GROUP BY bucket""")]})
+
+
+@bp.get("/duplicates")
+def duplicates():
+    groups = queries.duplicate_groups(db())
+    return jsonify({
+        "groups": groups,
+        "total_redundant_bytes": sum(g["redundant_bytes"] for g in groups),
+        "group_count": len(groups),
+    })
+
+
+@bp.get("/sources")
+def sources():
+    d = db()
+    c = cfg()
+    cov = [dict(r) for r in d.query("SELECT * FROM history_coverage")]
+    return jsonify({
+        "plex": {"configured": c.plex_configured, "base_url": c.plex_base_url},
+        "tautulli": {"configured": c.tautulli_configured, "base_url": c.tautulli_base_url},
+        "history_source": (cov[0]["source"] if cov else None),
+        "has_completion_data": queries.history_has_completion(d),
+        "coverage": cov,
+        "providers": [dict(r) for r in d.query("SELECT * FROM provider")],
+    })
+
+
+# ── keep marks (§8.2) ────────────────────────────────────────────────────
+
+@bp.get("/keeps")
+def list_keeps():
+    d = db()
+    counts = keeps.mark_matches(d)
+    rows = []
+    for r in d.query(
+        "SELECT k.*, lib.title AS library_title FROM keep_mark k "
+        "JOIN library lib ON lib.id = k.library_id ORDER BY k.created_at DESC"
+    ):
+        item = dict(r)
+        item["resolved_items"] = counts.get(r["id"], 0)
+        item["resolved_bytes"] = d.scalar(
+            "SELECT COALESCE(SUM(size_bytes),0) FROM media_item WHERE kept_mark_id = ?",
+            (r["id"],)) or 0
+        item["orphaned"] = item["resolved_items"] == 0
+        rows.append(item)
+    libs = [dict(r) for r in d.query(
+        "SELECT id, title, keep_all FROM library WHERE keep_all = 1 ORDER BY title")]
+    return jsonify({
+        "marks": rows,
+        "library_rules": libs,
+        "orphan_count": sum(1 for r in rows if r["orphaned"]),
+        "kept_bytes": d.scalar("SELECT COALESCE(SUM(size_bytes),0) FROM media_item "
+                               "WHERE kept=1 AND kind IN ('movie','season')") or 0,
+        "kept_items": d.scalar("SELECT COUNT(*) FROM media_item "
+                               "WHERE kept=1 AND kind IN ('movie','season')") or 0,
+    })
+
+
+@bp.get("/keeps/orphans")
+def keep_orphans():
+    d = db()
+    counts = keeps.mark_matches(d)
+    ids = [k for k, v in counts.items() if v == 0]
+    if not ids:
+        return jsonify({"marks": []})
+    marks = ",".join("?" * len(ids))
+    return jsonify({"marks": [dict(r) for r in d.query(
+        f"SELECT k.*, lib.title AS library_title FROM keep_mark k "
+        f"JOIN library lib ON lib.id=k.library_id WHERE k.id IN ({marks})", tuple(ids))]})
+
+
+@bp.post("/keeps")
+def create_keep():
+    payload = request.get_json(silent=True) or {}
+    d = db()
+    mode = payload.get("mode", "keep")
+    note = payload.get("note")
+    if payload.get("item_id"):
+        mark_id = keeps.create_from_item(d, int(payload["item_id"]), mode, note)
+    else:
+        required = ("scope", "library_id", "guid")
+        if not all(payload.get(k) for k in required):
+            raise KeepError("need item_id, or scope + library_id + guid")
+        mark_id = keeps.upsert(
+            d, payload["scope"], mode, int(payload["library_id"]), payload["guid"],
+            payload.get("season_number"), label=payload.get("label") or payload["guid"],
+            note=note)
+    keeps.resolve_all(d)
+    row = d.one("SELECT * FROM keep_mark WHERE id = ?", (mark_id,))
+    counts = keeps.mark_matches(d)
+    out = dict(row)
+    out["resolved_items"] = counts.get(mark_id, 0)
+    out["resolved_bytes"] = d.scalar(
+        "SELECT COALESCE(SUM(size_bytes),0) FROM media_item WHERE kept_mark_id=?",
+        (mark_id,)) or 0
+    return jsonify(out), 201
+
+
+@bp.post("/keeps/bulk")
+def bulk_keep():
+    payload = request.get_json(silent=True) or {}
+    ids = payload.get("item_ids") or []
+    if not isinstance(ids, list) or not ids:
+        raise KeepError("item_ids must be a non-empty list")
+    if len(ids) > 5000:
+        raise KeepError("too many items in one request")
+    mode = payload.get("mode", "keep")
+    note = payload.get("note")
+    d = db()
+    created, failed = [], []
+    for item_id in ids:
+        try:
+            created.append(keeps.create_from_item(d, int(item_id), mode, note))
+        except KeepError as e:
+            failed.append({"item_id": item_id, "reason": str(e)})
+    keeps.resolve_all(d)
+    return jsonify({
+        "created": len(created), "failed": failed,
+        "kept_bytes": d.scalar("SELECT COALESCE(SUM(size_bytes),0) FROM media_item "
+                               "WHERE kept=1 AND kind IN ('movie','season')") or 0,
+    })
+
+
+@bp.patch("/keeps/<int:mark_id>")
+def patch_keep(mark_id):
+    payload = request.get_json(silent=True) or {}
+    db().execute("UPDATE keep_mark SET note=?, updated_at=? WHERE id=?",
+                 (payload.get("note"), int(time.time()), mark_id))
+    return jsonify({"ok": True})
+
+
+@bp.delete("/keeps/<int:mark_id>")
+def delete_keep(mark_id):
+    d = db()
+    ok = keeps.delete(d, mark_id)
+    keeps.resolve_all(d)
+    return jsonify({"deleted": ok}), (200 if ok else 404)
+
+
+@bp.put("/libraries/<int:library_id>/keep_all")
+def library_keep_all(library_id):
+    payload = request.get_json(silent=True) or {}
+    d = db()
+    keeps.set_library_keep_all(d, library_id, bool(payload.get("keep_all")))
+    keeps.resolve_all(d)
+    return jsonify({"ok": True, "library_id": library_id,
+                    "keep_all": bool(payload.get("keep_all"))})
+
+
+@bp.get("/keeps/export")
+def export_keeps():
+    return Response(json.dumps(keeps.export(db()), indent=2),
+                    mimetype="application/json",
+                    headers={"Content-Disposition": "attachment; filename=keeps.json"})
+
+
+@bp.post("/keeps/import")
+def import_keeps():
+    payload = request.get_json(silent=True) or {}
+    return jsonify(keeps.import_(db(), payload))
+
+
+# ── saved views (§8.3) ───────────────────────────────────────────────────
+
+@bp.get("/views")
+def list_views():
+    return jsonify({"views": [dict(r) for r in db().query(
+        "SELECT * FROM saved_view ORDER BY builtin DESC, name")]})
+
+
+@bp.post("/views")
+def create_view():
+    p = request.get_json(silent=True) or {}
+    if not p.get("name"):
+        return jsonify({"error": "name is required"}), 400
+    now = int(time.time())
+    cur = db().execute(
+        "INSERT INTO saved_view (name, description, rules, sort, columns, weights, "
+        "created_at, updated_at) VALUES (?,?,?,?,?,?,?,?)",
+        (p["name"], p.get("description"), json.dumps(p.get("rules") or {}),
+         p.get("sort"), json.dumps(p.get("columns") or []),
+         json.dumps(p.get("weights") or {}), now, now))
+    return jsonify(dict(db().one("SELECT * FROM saved_view WHERE id=?", (cur.lastrowid,)))), 201
+
+
+@bp.get("/views/<int:view_id>")
+def get_view(view_id):
+    row = db().one("SELECT * FROM saved_view WHERE id=?", (view_id,))
+    return (jsonify(dict(row)), 200) if row else (jsonify({"error": "not_found"}), 404)
+
+
+@bp.put("/views/<int:view_id>")
+def update_view(view_id):
+    p = request.get_json(silent=True) or {}
+    db().execute(
+        "UPDATE saved_view SET name=COALESCE(?,name), description=?, rules=?, sort=?, "
+        "columns=?, weights=?, updated_at=? WHERE id=?",
+        (p.get("name"), p.get("description"), json.dumps(p.get("rules") or {}),
+         p.get("sort"), json.dumps(p.get("columns") or []),
+         json.dumps(p.get("weights") or {}), int(time.time()), view_id))
+    return jsonify({"ok": True})
+
+
+@bp.delete("/views/<int:view_id>")
+def delete_view(view_id):
+    cur = db().execute("DELETE FROM saved_view WHERE id=? AND builtin=0", (view_id,))
+    return jsonify({"deleted": cur.rowcount > 0})
+
+
+# ── scans (§8.4) ─────────────────────────────────────────────────────────
+
+@bp.get("/scans")
+def list_scans():
+    return jsonify({"scans": [dict(r) for r in db().query(
+        "SELECT * FROM scan ORDER BY id DESC LIMIT 50")]})
+
+
+@bp.get("/scans/current")
+def current_scan():
+    row = db().one("SELECT * FROM scan WHERE status='running' ORDER BY id DESC LIMIT 1")
+    return jsonify(dict(row) if row else None)
+
+
+@bp.post("/scans")
+def start_scan():
+    p = request.get_json(silent=True) or {}
+    mode = p.get("mode", "incremental")
+    if mode not in ("full", "incremental"):
+        return jsonify({"error": "mode must be 'full' or 'incremental'"}), 400
+    started = scanner.start_background_scan(current_app, mode, "manual")
+    if not started:
+        return jsonify({"error": "a scan is already running"}), 409
+    return jsonify({"status": "started", "mode": mode}), 202
+
+
+# ── export (§8.5) ────────────────────────────────────────────────────────
+
+CSV_COLUMNS = [
+    "id", "kind", "library", "title", "show_title", "season_number", "year",
+    "size_bytes", "size_human", "added_at_iso", "last_watched_at_iso",
+    "watch_count", "partial_count", "abandoned_count", "distinct_watcher_count",
+    "episode_count", "part_count", "resolution", "duplicate_count",
+    "pre_history", "kept", "kept_via", "reclaim_score", "primary_path",
+]
+
+
+def _human(n):
+    n = float(n or 0)
+    for u in ("B", "KB", "MB", "GB", "TB", "PB"):
+        if abs(n) < 1024:
+            return "%.1f %s" % (n, u)
+        n /= 1024.0
+    return "%.1f EB" % n
+
+
+def _iso(ts):
+    if not ts:
+        return ""
+    return time.strftime("%Y-%m-%d", time.localtime(int(ts)))
+
+
+@bp.get("/export.csv")
+def export_csv():
+    kw, sort = _query_kwargs()
+    d = db()
+    q = queries.Query(d, cfg())
+
+    def generate():
+        buf = io.StringIO()
+        w = csv.writer(buf)
+        w.writerow(CSV_COLUMNS)
+        yield buf.getvalue()
+        buf.seek(0), buf.truncate(0)
+        for row in q.iter_all(sort=sort, **kw):
+            r = dict(row)
+            w.writerow([
+                r["id"], r["kind"], r["library_title"], r["title"],
+                r.get("show_title") or "", r.get("season_number") or "",
+                r.get("year") or "", r.get("size_bytes") or 0,
+                _human(r.get("size_bytes")), _iso(r.get("added_at")),
+                _iso(r.get("last_watched_at")), r.get("watch_count") or 0,
+                r.get("partial_count") or 0, r.get("abandoned_count") or 0,
+                r.get("distinct_watcher_count") or 0,
+                r.get("episode_count") if r["kind"] == "season" else "",
+                r.get("part_count") or 0, r.get("resolution") or "",
+                r.get("duplicate_count") or 1,
+                "yes" if r.get("pre_history") else "no",
+                "yes" if r.get("kept") else "no", r.get("kept_via") or "",
+                r.get("reclaim_score"), r.get("primary_path") or "",
+            ])
+            yield buf.getvalue()
+            buf.seek(0), buf.truncate(0)
+
+    stamp = time.strftime("%Y%m%d-%H%M")
+    return Response(generate(), mimetype="text/csv", headers={
+        "Content-Disposition": "attachment; filename=mediashelf-%s.csv" % stamp})
+
+
+# ── settings ─────────────────────────────────────────────────────────────
+
+@bp.get("/settings")
+def settings():
+    return jsonify(cfg().redacted())