Implement MediaShelf v1

The application the design describes: Flask + SQLite, Plex for library data,
Tautulli for watch history, report-only.

Structure follows the design's seams. providers/ splits MediaProvider from
HistoryProvider, because on this network library data and watch data live on
different machines and Jellyfin later will have no Tautulli equivalent.
scoring.py implements the reclaim score twice - as a SQL expression for the
live grid (weights change on every slider drag, so storing it would mean
rewriting thousands of rows per drag) and in Python for CSV export and tests,
with a property test over 500 generated rows asserting the two agree.
rules.py compiles saved views to parameterized SQL through a field/operator
whitelist; nothing user-supplied is ever interpolated.

Three properties are enforced by test rather than asserted in prose:

- Ingest is idempotent. Three consecutive full scans leave every count and
  every byte total unchanged. A scanner that double-counts produces a report
  that looks plausible and is wrong.
- Keep marks survive Plex reassigning every rating key in the library. They
  are keyed on content GUID, scoped per library so the Movies and 4K Movies
  copies of the same film mark independently.
- Every config variable the app reads is declared in docker-compose.yml, so
  a variable set in Portainer can never silently do nothing.

Also found and fixed while verifying against a fake Plex+Tautulli pair:
executescript() commits the pending transaction, so migrations needed their
BEGIN/COMMIT inside the script; replaceChildren() renders null as the literal
text "null"; a hash-only URL change does not reload the document, so deep
links needed a hashchange listener; and SQLite ROUND rounds half away from
zero where Python rounds half to even.

73 tests, no live server required.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GVbG48GAXfCZatcmX123Ra
This commit is contained in:
Jess Hallsworth 2026-09-07 14:59:03 +00:00
parent 58c2883492
commit 6a557bcdd9
No known key found for this signature in database
37 changed files with 6486 additions and 85 deletions

0
mediashelf/__init__.py Normal file
View file

497
mediashelf/api.py Normal file
View file

@ -0,0 +1,497 @@
"""JSON API (§8). No auth in v1 — LAN-only, single admin view (§12)."""
from __future__ import annotations
import csv
import io
import json
import logging
import time
from flask import Blueprint, Response, current_app, jsonify, request
from . import keeps, queries, scanner
from .rules import RuleError
from .keeps import KeepError
log = logging.getLogger(__name__)
bp = Blueprint("api", __name__, url_prefix="/api/v1")
def db():
return current_app.extensions["mediashelf"]["db"]
def cfg():
return current_app.extensions["mediashelf"]["config"]
@bp.errorhandler(RuleError)
def _rule_error(e):
return jsonify({"error": "invalid_rule", "message": str(e)}), 400
@bp.errorhandler(KeepError)
def _keep_error(e):
return jsonify({"error": "invalid_keep", "message": str(e)}), 400
def _json_arg(name):
raw = request.args.get(name)
if not raw:
return None
try:
return json.loads(raw)
except (TypeError, ValueError):
raise RuleError("%s must be valid JSON" % name)
def _bool_arg(name, default=False):
v = request.args.get(name)
if v is None:
return default
return v.lower() in ("1", "true", "yes")
def _query_kwargs():
rule_group = _json_arg("rules")
weights = _json_arg("weights")
view_id = request.args.get("view_id")
sort = request.args.get("sort")
if view_id:
row = db().one("SELECT * FROM saved_view WHERE id = ?", (view_id,))
if row is None:
raise RuleError("no such view")
rule_group = json.loads(row["rules"]) if row["rules"] else None
sort = sort or row["sort"]
if row["weights"] and not weights:
weights = json.loads(row["weights"])
return dict(
library_ids=request.args.getlist("library_id") or None,
kinds=request.args.getlist("kind") or None,
q=request.args.get("q"),
rule_group=rule_group,
include_missing=_bool_arg("include_missing"),
include_kept=_bool_arg("include_kept"),
include_shows=_bool_arg("include_shows"),
weights=weights,
), sort
# ── items ────────────────────────────────────────────────────────────────
@bp.get("/items")
def items():
kw, sort = _query_kwargs()
q = queries.Query(db(), cfg())
return jsonify(q.page(
sort=sort,
page=int(request.args.get("page", 1)),
page_size=int(request.args.get("page_size", 100)),
**kw,
))
@bp.get("/items/<int:item_id>")
def item_detail(item_id):
d = db()
q = queries.Query(d, cfg())
expr, where, params = q.build(include_kept=True, include_missing=True, include_shows=True)
params["item_id"] = item_id
sql = ("SELECT " + queries.BASE_COLUMNS + ", " + expr + " AS reclaim_score, "
+ "NULL AS grace, COALESCE(d.dupe_count,1) AS duplicate_count "
+ queries.FROM_CLAUSE + " WHERE i.id = :item_id")
row = d.one(sql, params)
if row is None:
return jsonify({"error": "not_found"}), 404
out = q.serialize(row)
out["parts"] = [dict(r) for r in d.query(
"SELECT * FROM media_part WHERE media_item_id = ? ORDER BY file_path", (item_id,))]
if out["kind"] == "season":
out["episodes"] = [dict(r) for r in d.query(
"SELECT * FROM episode WHERE season_item_id = ? ORDER BY episode_number",
(item_id,))]
pids = [e["provider_item_id"] for e in out["episodes"]]
else:
pids = [d.scalar("SELECT provider_item_id FROM media_item WHERE id=?", (item_id,))]
if pids:
marks = ",".join("?" * len(pids))
out["watch_history"] = [dict(r) for r in d.query(
"SELECT w.viewed_at, w.percent_complete, w.disposition, w.account_id, "
"COALESCE(a.friendly_name, a.name, w.account_id) AS who, w.platform "
"FROM watch_event w LEFT JOIN account a ON a.account_id = w.account_id "
f"WHERE w.provider_item_id IN ({marks}) ORDER BY w.viewed_at DESC LIMIT 500",
tuple(pids))]
else:
out["watch_history"] = []
if out.get("guid"):
out["duplicates"] = [dict(r) for r in d.query(
"SELECT i.id, i.title, i.size_bytes, i.resolution, i.watch_count, "
"lib.title AS library_title FROM media_item i "
"JOIN library lib ON lib.id=i.library_id "
"WHERE i.guid = ? AND i.id != ? AND i.kind='movie' AND i.status='present'",
(out["guid"], item_id))]
return jsonify(out)
@bp.get("/libraries")
def libraries():
return jsonify({"libraries": queries.size_by_library(db())})
@bp.get("/accounts")
def accounts():
return jsonify({"accounts": [dict(r) for r in db().query(
"SELECT a.*, (SELECT COUNT(*) FROM watch_event w WHERE w.account_id=a.account_id) "
"AS plays FROM account a ORDER BY plays DESC")]})
# ── stats ────────────────────────────────────────────────────────────────
@bp.get("/stats/overview")
def stats_overview():
return jsonify(queries.overview(db(), cfg()))
@bp.get("/stats/size-by-library")
def stats_size_by_library():
return jsonify({"libraries": queries.size_by_library(db())})
@bp.get("/stats/added-over-time")
def stats_added_over_time():
return jsonify({"buckets": queries.added_over_time(
db(), request.args.get("bucket", "month"))})
@bp.get("/stats/completion")
def stats_completion():
return jsonify(queries.completion_split(db()))
@bp.get("/stats/size-vs-lastwatched")
def stats_scatter():
return jsonify({"points": queries.size_vs_lastwatched(db())})
@bp.get("/stats/watch-distribution")
def stats_watch_distribution():
return jsonify({"buckets": [dict(r) for r in db().query("""
SELECT CASE WHEN watch_count = 0 THEN '0'
WHEN watch_count = 1 THEN '1'
WHEN watch_count <= 3 THEN '2-3'
WHEN watch_count <= 10 THEN '4-10'
ELSE '10+' END AS bucket,
COUNT(*) AS items, COALESCE(SUM(size_bytes),0) AS size_bytes
FROM media_item WHERE kind IN ('movie','season') AND status='present'
GROUP BY bucket""")]})
@bp.get("/duplicates")
def duplicates():
groups = queries.duplicate_groups(db())
return jsonify({
"groups": groups,
"total_redundant_bytes": sum(g["redundant_bytes"] for g in groups),
"group_count": len(groups),
})
@bp.get("/sources")
def sources():
d = db()
c = cfg()
cov = [dict(r) for r in d.query("SELECT * FROM history_coverage")]
return jsonify({
"plex": {"configured": c.plex_configured, "base_url": c.plex_base_url},
"tautulli": {"configured": c.tautulli_configured, "base_url": c.tautulli_base_url},
"history_source": (cov[0]["source"] if cov else None),
"has_completion_data": queries.history_has_completion(d),
"coverage": cov,
"providers": [dict(r) for r in d.query("SELECT * FROM provider")],
})
# ── keep marks (§8.2) ────────────────────────────────────────────────────
@bp.get("/keeps")
def list_keeps():
d = db()
counts = keeps.mark_matches(d)
rows = []
for r in d.query(
"SELECT k.*, lib.title AS library_title FROM keep_mark k "
"JOIN library lib ON lib.id = k.library_id ORDER BY k.created_at DESC"
):
item = dict(r)
item["resolved_items"] = counts.get(r["id"], 0)
item["resolved_bytes"] = d.scalar(
"SELECT COALESCE(SUM(size_bytes),0) FROM media_item WHERE kept_mark_id = ?",
(r["id"],)) or 0
item["orphaned"] = item["resolved_items"] == 0
rows.append(item)
libs = [dict(r) for r in d.query(
"SELECT id, title, keep_all FROM library WHERE keep_all = 1 ORDER BY title")]
return jsonify({
"marks": rows,
"library_rules": libs,
"orphan_count": sum(1 for r in rows if r["orphaned"]),
"kept_bytes": d.scalar("SELECT COALESCE(SUM(size_bytes),0) FROM media_item "
"WHERE kept=1 AND kind IN ('movie','season')") or 0,
"kept_items": d.scalar("SELECT COUNT(*) FROM media_item "
"WHERE kept=1 AND kind IN ('movie','season')") or 0,
})
@bp.get("/keeps/orphans")
def keep_orphans():
d = db()
counts = keeps.mark_matches(d)
ids = [k for k, v in counts.items() if v == 0]
if not ids:
return jsonify({"marks": []})
marks = ",".join("?" * len(ids))
return jsonify({"marks": [dict(r) for r in d.query(
f"SELECT k.*, lib.title AS library_title FROM keep_mark k "
f"JOIN library lib ON lib.id=k.library_id WHERE k.id IN ({marks})", tuple(ids))]})
@bp.post("/keeps")
def create_keep():
payload = request.get_json(silent=True) or {}
d = db()
mode = payload.get("mode", "keep")
note = payload.get("note")
if payload.get("item_id"):
mark_id = keeps.create_from_item(d, int(payload["item_id"]), mode, note)
else:
required = ("scope", "library_id", "guid")
if not all(payload.get(k) for k in required):
raise KeepError("need item_id, or scope + library_id + guid")
mark_id = keeps.upsert(
d, payload["scope"], mode, int(payload["library_id"]), payload["guid"],
payload.get("season_number"), label=payload.get("label") or payload["guid"],
note=note)
keeps.resolve_all(d)
row = d.one("SELECT * FROM keep_mark WHERE id = ?", (mark_id,))
counts = keeps.mark_matches(d)
out = dict(row)
out["resolved_items"] = counts.get(mark_id, 0)
out["resolved_bytes"] = d.scalar(
"SELECT COALESCE(SUM(size_bytes),0) FROM media_item WHERE kept_mark_id=?",
(mark_id,)) or 0
return jsonify(out), 201
@bp.post("/keeps/bulk")
def bulk_keep():
payload = request.get_json(silent=True) or {}
ids = payload.get("item_ids") or []
if not isinstance(ids, list) or not ids:
raise KeepError("item_ids must be a non-empty list")
if len(ids) > 5000:
raise KeepError("too many items in one request")
mode = payload.get("mode", "keep")
note = payload.get("note")
d = db()
created, failed = [], []
for item_id in ids:
try:
created.append(keeps.create_from_item(d, int(item_id), mode, note))
except KeepError as e:
failed.append({"item_id": item_id, "reason": str(e)})
keeps.resolve_all(d)
return jsonify({
"created": len(created), "failed": failed,
"kept_bytes": d.scalar("SELECT COALESCE(SUM(size_bytes),0) FROM media_item "
"WHERE kept=1 AND kind IN ('movie','season')") or 0,
})
@bp.patch("/keeps/<int:mark_id>")
def patch_keep(mark_id):
payload = request.get_json(silent=True) or {}
db().execute("UPDATE keep_mark SET note=?, updated_at=? WHERE id=?",
(payload.get("note"), int(time.time()), mark_id))
return jsonify({"ok": True})
@bp.delete("/keeps/<int:mark_id>")
def delete_keep(mark_id):
d = db()
ok = keeps.delete(d, mark_id)
keeps.resolve_all(d)
return jsonify({"deleted": ok}), (200 if ok else 404)
@bp.put("/libraries/<int:library_id>/keep_all")
def library_keep_all(library_id):
payload = request.get_json(silent=True) or {}
d = db()
keeps.set_library_keep_all(d, library_id, bool(payload.get("keep_all")))
keeps.resolve_all(d)
return jsonify({"ok": True, "library_id": library_id,
"keep_all": bool(payload.get("keep_all"))})
@bp.get("/keeps/export")
def export_keeps():
return Response(json.dumps(keeps.export(db()), indent=2),
mimetype="application/json",
headers={"Content-Disposition": "attachment; filename=keeps.json"})
@bp.post("/keeps/import")
def import_keeps():
payload = request.get_json(silent=True) or {}
return jsonify(keeps.import_(db(), payload))
# ── saved views (§8.3) ───────────────────────────────────────────────────
@bp.get("/views")
def list_views():
return jsonify({"views": [dict(r) for r in db().query(
"SELECT * FROM saved_view ORDER BY builtin DESC, name")]})
@bp.post("/views")
def create_view():
p = request.get_json(silent=True) or {}
if not p.get("name"):
return jsonify({"error": "name is required"}), 400
now = int(time.time())
cur = db().execute(
"INSERT INTO saved_view (name, description, rules, sort, columns, weights, "
"created_at, updated_at) VALUES (?,?,?,?,?,?,?,?)",
(p["name"], p.get("description"), json.dumps(p.get("rules") or {}),
p.get("sort"), json.dumps(p.get("columns") or []),
json.dumps(p.get("weights") or {}), now, now))
return jsonify(dict(db().one("SELECT * FROM saved_view WHERE id=?", (cur.lastrowid,)))), 201
@bp.get("/views/<int:view_id>")
def get_view(view_id):
row = db().one("SELECT * FROM saved_view WHERE id=?", (view_id,))
return (jsonify(dict(row)), 200) if row else (jsonify({"error": "not_found"}), 404)
@bp.put("/views/<int:view_id>")
def update_view(view_id):
p = request.get_json(silent=True) or {}
db().execute(
"UPDATE saved_view SET name=COALESCE(?,name), description=?, rules=?, sort=?, "
"columns=?, weights=?, updated_at=? WHERE id=?",
(p.get("name"), p.get("description"), json.dumps(p.get("rules") or {}),
p.get("sort"), json.dumps(p.get("columns") or []),
json.dumps(p.get("weights") or {}), int(time.time()), view_id))
return jsonify({"ok": True})
@bp.delete("/views/<int:view_id>")
def delete_view(view_id):
cur = db().execute("DELETE FROM saved_view WHERE id=? AND builtin=0", (view_id,))
return jsonify({"deleted": cur.rowcount > 0})
# ── scans (§8.4) ─────────────────────────────────────────────────────────
@bp.get("/scans")
def list_scans():
return jsonify({"scans": [dict(r) for r in db().query(
"SELECT * FROM scan ORDER BY id DESC LIMIT 50")]})
@bp.get("/scans/current")
def current_scan():
row = db().one("SELECT * FROM scan WHERE status='running' ORDER BY id DESC LIMIT 1")
return jsonify(dict(row) if row else None)
@bp.post("/scans")
def start_scan():
p = request.get_json(silent=True) or {}
mode = p.get("mode", "incremental")
if mode not in ("full", "incremental"):
return jsonify({"error": "mode must be 'full' or 'incremental'"}), 400
started = scanner.start_background_scan(current_app, mode, "manual")
if not started:
return jsonify({"error": "a scan is already running"}), 409
return jsonify({"status": "started", "mode": mode}), 202
# ── export (§8.5) ────────────────────────────────────────────────────────
CSV_COLUMNS = [
"id", "kind", "library", "title", "show_title", "season_number", "year",
"size_bytes", "size_human", "added_at_iso", "last_watched_at_iso",
"watch_count", "partial_count", "abandoned_count", "distinct_watcher_count",
"episode_count", "part_count", "resolution", "duplicate_count",
"pre_history", "kept", "kept_via", "reclaim_score", "primary_path",
]
def _human(n):
n = float(n or 0)
for u in ("B", "KB", "MB", "GB", "TB", "PB"):
if abs(n) < 1024:
return "%.1f %s" % (n, u)
n /= 1024.0
return "%.1f EB" % n
def _iso(ts):
if not ts:
return ""
return time.strftime("%Y-%m-%d", time.localtime(int(ts)))
@bp.get("/export.csv")
def export_csv():
kw, sort = _query_kwargs()
d = db()
q = queries.Query(d, cfg())
def generate():
buf = io.StringIO()
w = csv.writer(buf)
w.writerow(CSV_COLUMNS)
yield buf.getvalue()
buf.seek(0), buf.truncate(0)
for row in q.iter_all(sort=sort, **kw):
r = dict(row)
w.writerow([
r["id"], r["kind"], r["library_title"], r["title"],
r.get("show_title") or "", r.get("season_number") or "",
r.get("year") or "", r.get("size_bytes") or 0,
_human(r.get("size_bytes")), _iso(r.get("added_at")),
_iso(r.get("last_watched_at")), r.get("watch_count") or 0,
r.get("partial_count") or 0, r.get("abandoned_count") or 0,
r.get("distinct_watcher_count") or 0,
r.get("episode_count") if r["kind"] == "season" else "",
r.get("part_count") or 0, r.get("resolution") or "",
r.get("duplicate_count") or 1,
"yes" if r.get("pre_history") else "no",
"yes" if r.get("kept") else "no", r.get("kept_via") or "",
r.get("reclaim_score"), r.get("primary_path") or "",
])
yield buf.getvalue()
buf.seek(0), buf.truncate(0)
stamp = time.strftime("%Y%m%d-%H%M")
return Response(generate(), mimetype="text/csv", headers={
"Content-Disposition": "attachment; filename=mediashelf-%s.csv" % stamp})
# ── settings ─────────────────────────────────────────────────────────────
@bp.get("/settings")
def settings():
return jsonify(cfg().redacted())

153
mediashelf/app.py Normal file
View file

@ -0,0 +1,153 @@
"""Flask app factory."""
from __future__ import annotations
import json
import logging
import time
from flask import Flask, jsonify, render_template
from . import queries, scanner
from .config import Config
from .db import Database
from .scoring import register_sqlite_functions
log = logging.getLogger(__name__)
# Shipped read-only so the first run has somewhere useful to start (§7.2).
SEED_VIEWS = [
("Confident reclaim",
"Added while watch history was being recorded, and never played since. "
"The list to act on first.",
{"op": "and", "rules": [
{"field": "watch_count", "op": "eq", "value": 0},
{"field": "pre_history", "op": "eq", "value": False}]},
"reclaim_score:desc"),
("Uncertain reclaim",
"Never played, but added before history began — it may have been watched "
"and nobody can tell. Needs judgement, not a bulk action.",
{"op": "and", "rules": [
{"field": "watch_count", "op": "eq", "value": 0},
{"field": "pre_history", "op": "eq", "value": True}]},
"size_bytes:desc"),
("Never watched, large",
"Nothing has ever played it and it is over 8 GB.",
{"op": "and", "rules": [
{"field": "watch_count", "op": "eq", "value": 0},
{"field": "size_bytes", "op": "gte", "value": 8 * 1024**3}]},
"size_bytes:desc"),
("Cold storage",
"Not watched in over three years.",
{"op": "and", "rules": [
{"field": "last_watched_at", "op": "older_than_days", "value": 1095}]},
"size_bytes:desc"),
("One-and-done movies",
"Watched exactly once, over two years ago.",
{"op": "and", "rules": [
{"field": "kind", "op": "eq", "value": "movie"},
{"field": "watch_count", "op": "eq", "value": 1},
{"field": "last_watched_at", "op": "older_than_days", "value": 730}]},
"size_bytes:desc"),
("Tried and rejected",
"Two or more people started it and nobody finished it. Needs Tautulli.",
{"op": "and", "rules": [
{"field": "abandoned_count", "op": "gte", "value": 2},
{"field": "watch_count", "op": "eq", "value": 0}]},
"size_bytes:desc"),
("Abandoned seasons",
"Seasons barely watched through, added over a year ago.",
{"op": "and", "rules": [
{"field": "kind", "op": "eq", "value": "season"},
{"field": "watch_ratio", "op": "lt", "value": 0.3},
{"field": "added_at", "op": "older_than_days", "value": 365}]},
"size_bytes:desc"),
("Biggest 100",
"Simply the largest things on disk.",
{}, "size_bytes:desc"),
("Recently added",
"The sanity check, not a delete list.",
{"op": "and", "rules": [
{"field": "added_at", "op": "newer_than_days", "value": 30}]},
"added_at:desc"),
("Kept",
"What you have decided to keep, and what it costs.",
{"op": "and", "rules": [{"field": "kept", "op": "eq", "value": True}]},
"size_bytes:desc"),
]
def seed_views(db) -> None:
now = int(time.time())
for name, desc, rules, sort in SEED_VIEWS:
if db.one("SELECT id FROM saved_view WHERE name = ?", (name,)):
continue
db.execute(
"INSERT INTO saved_view (name, description, rules, sort, columns, weights, "
"builtin, created_at, updated_at) VALUES (?,?,?,?,'[]','{}',1,?,?)",
(name, desc, json.dumps(rules), sort, now, now))
def create_app(config: Config | None = None, *, start_scheduler: bool = True) -> Flask:
cfg = config or Config.from_env()
logging.basicConfig(
level=getattr(logging, cfg.log_level, logging.INFO),
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s")
app = Flask(__name__)
app.config["SECRET_KEY"] = cfg.secret_key
app.config["SESSION_COOKIE_SAMESITE"] = "Strict"
app.config["JSON_SORT_KEYS"] = False
db = Database(cfg.database_path)
db.migrate()
register_sqlite_functions(db.conn)
seed_views(db)
app.extensions["mediashelf"] = {
"config": cfg, "db": db, "degraded_reason": None,
"started_at": int(time.time()),
}
@app.before_request
def _ensure_functions():
# Each thread gets its own connection, so LOG() must be registered on it.
register_sqlite_functions(db.conn)
from .api import bp as api_bp
app.register_blueprint(api_bp)
from .web import bp as web_bp
app.register_blueprint(web_bp)
@app.get("/healthz")
def healthz():
ext = app.extensions["mediashelf"]
out = {"status": "ok", "db": "ok", "plex": "unknown", "tautulli": "disabled"}
try:
db.scalar("SELECT 1")
except Exception as e: # noqa: BLE001
out["status"], out["db"] = "error", str(e)
row = db.one("SELECT * FROM scan WHERE status='succeeded' ORDER BY id DESC LIMIT 1")
out["last_scan_at"] = row["finished_at"] if row else None
out["stale"] = bool(row is None or (time.time() - (row["finished_at"] or 0)) > 48 * 3600)
cov = db.one("SELECT source FROM history_coverage ORDER BY event_count DESC LIMIT 1")
out["history_source"] = cov["source"] if cov else None
out["plex"] = "configured" if cfg.plex_configured else "unconfigured"
out["tautulli"] = "configured" if cfg.tautulli_configured else "disabled"
out["degraded"] = ext.get("degraded_reason")
return jsonify(out), (200 if out["status"] == "ok" else 503)
@app.errorhandler(404)
def _404(e):
return jsonify({"error": "not_found"}), 404
@app.errorhandler(500)
def _500(e):
log.exception("unhandled error")
return jsonify({"error": "internal_error"}), 500
if start_scheduler:
scanner.start_scheduler(app)
return app

112
mediashelf/cli.py Normal file
View file

@ -0,0 +1,112 @@
"""Command line: run a scan, serve in development, or export keeps.
python -m mediashelf.cli scan --full
python -m mediashelf.cli serve --port 8080
python -m mediashelf.cli export-keeps > keeps.json
"""
from __future__ import annotations
import argparse
import json
import logging
import sys
from . import ingest, keeps, providers
from .app import create_app
from .config import Config
from .db import Database
def _load_dotenv(path=".env"):
"""Minimal .env support for development. Production uses the stack env."""
import os
try:
with open(path) as fh:
for line in fh:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
os.environ.setdefault(k.strip(), v.strip())
except OSError:
pass
def cmd_scan(args) -> int:
cfg = Config.from_env()
db = Database(cfg.database_path)
db.migrate()
media = providers.build_media_provider(cfg)
history, degraded = providers.build_history_provider(cfg, media)
if degraded:
print("WARNING: history degraded — %s" % degraded, file=sys.stderr)
print("history source: %s" % (history.name if history else "none"), file=sys.stderr)
result = ingest.Ingest(db, cfg, media, history).run(
"full" if args.full else "incremental", "manual")
print(json.dumps({
"status": result.status, "scan_id": result.scan_id,
"items_seen": result.items_seen, "items_added": result.items_added,
"items_updated": result.items_updated, "items_missing": result.items_missing,
"events_added": result.events_added, "warnings": len(result.warnings or []),
"error": result.error,
}, indent=2))
return 0 if result.status == "succeeded" else 1
def cmd_serve(args) -> int:
app = create_app(start_scheduler=not args.no_scheduler)
app.run(host=args.host, port=args.port, debug=args.debug, threaded=True)
return 0
def cmd_export_keeps(args) -> int:
cfg = Config.from_env()
db = Database(cfg.database_path)
db.migrate()
json.dump(keeps.export(db), sys.stdout, indent=2)
return 0
def cmd_import_keeps(args) -> int:
cfg = Config.from_env()
db = Database(cfg.database_path)
db.migrate()
with open(args.path) as fh:
payload = json.load(fh)
print(json.dumps(keeps.import_(db, payload), indent=2))
return 0
def main(argv=None) -> int:
_load_dotenv()
logging.basicConfig(level=logging.INFO,
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s")
p = argparse.ArgumentParser(prog="mediashelf")
sub = p.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("scan", help="run a scan now")
s.add_argument("--full", action="store_true", help="full sweep rather than incremental")
s.set_defaults(func=cmd_scan)
s = sub.add_parser("serve", help="development server")
s.add_argument("--host", default="127.0.0.1")
s.add_argument("--port", type=int, default=8080)
s.add_argument("--debug", action="store_true")
s.add_argument("--no-scheduler", action="store_true")
s.set_defaults(func=cmd_serve)
s = sub.add_parser("export-keeps", help="write keep marks to stdout as JSON")
s.set_defaults(func=cmd_export_keeps)
s = sub.add_parser("import-keeps", help="restore keep marks from a JSON export")
s.add_argument("path")
s.set_defaults(func=cmd_import_keeps)
args = p.parse_args(argv)
return args.func(args)
if __name__ == "__main__":
sys.exit(main())

173
mediashelf/config.py Normal file
View file

@ -0,0 +1,173 @@
"""Configuration, read once from the environment.
Every value here is documented in docs/design.md §10. Defaults match that table.
The Portainer stack is the single source of truth in production; .env is used in
development via `python -m mediashelf.cli`.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
def _b(name: str, default: bool) -> bool:
v = os.environ.get(name)
if v is None or v == "":
return default
return v.strip().lower() in ("1", "true", "yes", "on")
def _i(name: str, default: int) -> int:
v = os.environ.get(name)
if v is None or v == "":
return default
try:
return int(v)
except ValueError:
return default
def _f(name: str, default: float) -> float:
v = os.environ.get(name)
if v is None or v == "":
return default
try:
return float(v)
except ValueError:
return default
def _s(name: str, default: str = "") -> str:
v = os.environ.get(name)
return default if v is None else v.strip()
def _csv(name: str) -> list[str]:
raw = _s(name)
return [p.strip() for p in raw.split(",") if p.strip()]
@dataclass(frozen=True)
class ScoreConfig:
"""Constants for the reclaim score (§6)."""
stale_horizon_days: int = 730
age_horizon_days: int = 1095
popular_at: int = 3
rejected_at: int = 2
solitude_at: int = 3
grace_days: int = 30
recent_days: int = 90
# Default weight profile, calibrated against the measured library (§6.1/§6.2).
weights: dict[str, float] = field(
default_factory=lambda: {
"size": 0.28,
"staleness": 0.24,
"unpopularity": 0.22,
"solitude": 0.10,
"age": 0.10,
"rejection": 0.06,
}
)
@classmethod
def from_env(cls) -> "ScoreConfig":
return cls(
stale_horizon_days=_i("SCORE_STALE_HORIZON_DAYS", 730),
age_horizon_days=_i("SCORE_AGE_HORIZON_DAYS", 1095),
popular_at=_i("SCORE_POPULAR_AT", 3),
rejected_at=_i("SCORE_REJECTED_AT", 2),
solitude_at=_i("SCORE_SOLITUDE_AT", 3),
grace_days=_i("SCORE_GRACE_DAYS", 30),
recent_days=_i("SCORE_RECENT_DAYS", 90),
)
@dataclass(frozen=True)
class Config:
secret_key: str = "dev-insecure"
log_level: str = "INFO"
tz: str = "America/Regina"
plex_base_url: str = ""
plex_token: str = ""
plex_verify_ssl: bool = True
plex_timeout_s: int = 30
plex_page_size: int = 500
plex_request_delay_ms: int = 0
tautulli_base_url: str = ""
tautulli_api_key: str = ""
tautulli_timeout_s: int = 30
tautulli_page_size: int = 1000
history_source: str = "auto" # auto | tautulli | plex
session_merge_window_h: int = 6
completion_threshold: int = 85
abandon_ceiling: int = 15
database_path: str = "/data/mediashelf.db"
scan_schedule_cron: str = "0 4 * * *"
scan_full_sweep_cron: str = "0 3 * * 0"
scan_on_startup: bool = False
scan_lock_timeout_s: int = 7200
scheduler_enabled: bool = True
keep_all_libraries: list[str] = field(default_factory=list)
score: ScoreConfig = field(default_factory=ScoreConfig)
@classmethod
def from_env(cls) -> "Config":
return cls(
secret_key=_s("MEDIASHELF_SECRET_KEY", "dev-insecure"),
log_level=_s("LOG_LEVEL", "INFO").upper(),
tz=_s("TZ", "America/Regina"),
plex_base_url=_s("PLEX_BASE_URL").rstrip("/"),
plex_token=_s("PLEX_TOKEN"),
plex_verify_ssl=_b("PLEX_VERIFY_SSL", True),
plex_timeout_s=_i("PLEX_TIMEOUT_S", 30),
plex_page_size=_i("PLEX_PAGE_SIZE", 500),
plex_request_delay_ms=_i("PLEX_REQUEST_DELAY_MS", 0),
tautulli_base_url=_s("TAUTULLI_BASE_URL").rstrip("/"),
tautulli_api_key=_s("TAUTULLI_API_KEY"),
tautulli_timeout_s=_i("TAUTULLI_TIMEOUT_S", 30),
tautulli_page_size=_i("TAUTULLI_PAGE_SIZE", 1000),
history_source=_s("HISTORY_SOURCE", "auto").lower(),
session_merge_window_h=_i("SESSION_MERGE_WINDOW_H", 6),
completion_threshold=_i("COMPLETION_THRESHOLD", 85),
abandon_ceiling=_i("ABANDON_CEILING", 15),
database_path=_s("DATABASE_PATH", "/data/mediashelf.db"),
scan_schedule_cron=_s("SCAN_SCHEDULE_CRON", "0 4 * * *"),
scan_full_sweep_cron=_s("SCAN_FULL_SWEEP_CRON", "0 3 * * 0"),
scan_on_startup=_b("SCAN_ON_STARTUP", False),
scan_lock_timeout_s=_i("SCAN_LOCK_TIMEOUT_S", 7200),
scheduler_enabled=_b("SCHEDULER_ENABLED", True),
keep_all_libraries=_csv("KEEP_ALL_LIBRARIES"),
score=ScoreConfig.from_env(),
)
# ── derived ──────────────────────────────────────────────────────────
@property
def tautulli_configured(self) -> bool:
return bool(self.tautulli_base_url and self.tautulli_api_key)
@property
def plex_configured(self) -> bool:
return bool(self.plex_base_url and self.plex_token)
def redacted(self) -> dict:
"""Safe for API responses and logs. Never leaks a credential (§12)."""
out = {}
for k, v in self.__dict__.items():
if k in ("plex_token", "tautulli_api_key", "secret_key"):
out[k] = "***" if v else ""
elif k == "score":
out[k] = v.__dict__
else:
out[k] = v
return out

158
mediashelf/db.py Normal file
View file

@ -0,0 +1,158 @@
"""SQLite access and migrations.
One writer (the scan job), many readers. WAL mode makes that work without
readers blocking. Migrations are plain numbered .sql files applied in order at
startup and tracked in schema_version no ORM migration framework (§5).
"""
from __future__ import annotations
import logging
import os
import sqlite3
import threading
from pathlib import Path
log = logging.getLogger(__name__)
MIGRATIONS_DIR = Path(__file__).parent / "migrations"
_local = threading.local()
def connect(path: str, *, read_only: bool = False) -> sqlite3.Connection:
"""Open a tuned connection. Callers own the connection lifecycle."""
if path != ":memory:":
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
conn = sqlite3.connect(path, timeout=30.0, isolation_level=None)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA foreign_keys=ON")
conn.execute("PRAGMA synchronous=NORMAL")
conn.execute("PRAGMA busy_timeout=30000")
conn.execute("PRAGMA temp_store=MEMORY")
if read_only:
conn.execute("PRAGMA query_only=ON")
return conn
def migrate(conn: sqlite3.Connection) -> int:
"""Apply any unapplied migrations. Returns the resulting version."""
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_version ("
" version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)"
)
row = conn.execute("SELECT COALESCE(MAX(version), 0) AS v FROM schema_version").fetchone()
current = row["v"]
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
applied = current
for f in files:
try:
version = int(f.name.split("_", 1)[0])
except ValueError:
log.warning("skipping unnumbered migration %s", f.name)
continue
if version <= current:
continue
log.info("applying migration %s", f.name)
sql = f.read_text()
# Migrations contain triggers with internal semicolons, so the whole file
# runs as a script rather than being split on ';'. executescript() commits
# any pending transaction before it runs, so the BEGIN/COMMIT must live
# inside the script itself for the migration to be atomic.
script = (
"BEGIN;\n"
+ sql
+ f"\nINSERT INTO schema_version (version, applied_at)"
f" VALUES ({version}, strftime('%s','now'));\nCOMMIT;"
)
try:
conn.executescript(script)
except Exception:
if conn.in_transaction:
conn.execute("ROLLBACK")
raise
applied = version
return applied
class Database:
"""Thin wrapper giving each thread its own connection.
Flask serves requests on multiple threads and the scheduler runs on its own,
and sqlite3 connections are not shareable across threads.
"""
def __init__(self, path: str):
self.path = path
@property
def conn(self) -> sqlite3.Connection:
c = getattr(_local, "conn", None)
if c is None or getattr(_local, "path", None) != self.path:
c = connect(self.path)
_local.conn = c
_local.path = self.path
return c
def close(self) -> None:
c = getattr(_local, "conn", None)
if c is not None:
c.close()
_local.conn = None
# ── convenience ──────────────────────────────────────────────────────
def query(self, sql: str, params: tuple | dict = ()) -> list[sqlite3.Row]:
return self.conn.execute(sql, params).fetchall()
def one(self, sql: str, params: tuple | dict = ()) -> sqlite3.Row | None:
return self.conn.execute(sql, params).fetchone()
def scalar(self, sql: str, params: tuple | dict = ()):
row = self.conn.execute(sql, params).fetchone()
return None if row is None else row[0]
def execute(self, sql: str, params: tuple | dict = ()) -> sqlite3.Cursor:
return self.conn.execute(sql, params)
def executemany(self, sql: str, seq) -> sqlite3.Cursor:
return self.conn.executemany(sql, seq)
def migrate(self) -> int:
return migrate(self.conn)
# ── settings ─────────────────────────────────────────────────────────
def get_setting(self, key: str, default=None):
row = self.one("SELECT value FROM setting WHERE key = ?", (key,))
return default if row is None else row["value"]
def set_setting(self, key: str, value: str) -> None:
self.execute(
"INSERT INTO setting (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, str(value)),
)
class transaction:
"""Context manager for an explicit write transaction.
isolation_level is None (autocommit), so transactions are explicit here.
"""
def __init__(self, db: Database):
self.db = db
def __enter__(self) -> sqlite3.Connection:
self.db.conn.execute("BEGIN IMMEDIATE")
return self.db.conn
def __exit__(self, exc_type, exc, tb) -> bool:
if exc_type is None:
self.db.conn.execute("COMMIT")
else:
self.db.conn.execute("ROLLBACK")
return False

696
mediashelf/ingest.py Normal file
View file

@ -0,0 +1,696 @@
"""Scan orchestration (§4.6).
Pulls libraries, items and history; normalizes; upserts inside one transaction
per library; rolls episodes up to seasons; resolves keep marks; marks vanished
items missing.
The property that matters most here is idempotency. A scanner that double-counts
sizes or duplicates history events produces a report that looks entirely
plausible and is wrong, which is worse than one that crashes.
"""
from __future__ import annotations
import json
import logging
import os
import time
from dataclasses import dataclass
from . import keeps
from .db import Database
from .providers.base import (
Account,
AuthError,
Coverage,
HistoryProvider,
Item,
Library,
MediaProvider,
ProviderError,
WatchEvent,
)
log = logging.getLogger(__name__)
class ScanBusy(RuntimeError):
pass
@dataclass
class ScanResult:
scan_id: int
status: str
items_seen: int = 0
items_added: int = 0
items_updated: int = 0
items_missing: int = 0
events_added: int = 0
warnings: list[str] = None
error: str | None = None
class Ingest:
def __init__(self, db: Database, cfg, media: MediaProvider,
history: HistoryProvider | None):
self.db = db
self.cfg = cfg
self.media = media
self.history = history
self.warnings: list[str] = []
# ── locking ──────────────────────────────────────────────────────────
def _acquire_lock(self, scan_id: int) -> None:
now = int(time.time())
row = self.db.one("SELECT * FROM scan_lock WHERE id = 1")
if row and row["scan_id"] is not None:
age = now - (row["acquired_at"] or 0)
if age < self.cfg.scan_lock_timeout_s:
raise ScanBusy("a scan is already running (started %ds ago)" % age)
# Stale lock: the previous scan died. Mark it failed and take over.
log.warning("breaking stale scan lock held by scan %s", row["scan_id"])
self.db.execute(
"UPDATE scan SET status='failed', finished_at=?, "
"error='abandoned - lock timed out' WHERE id=? AND status='running'",
(now, row["scan_id"]),
)
self.db.execute(
"INSERT INTO scan_lock (id, scan_id, holder, acquired_at) VALUES (1,?,?,?) "
"ON CONFLICT(id) DO UPDATE SET scan_id=excluded.scan_id, "
"holder=excluded.holder, acquired_at=excluded.acquired_at",
(scan_id, "%s:%s" % (os.uname().nodename, os.getpid()), now),
)
def _release_lock(self) -> None:
self.db.execute("UPDATE scan_lock SET scan_id=NULL, holder=NULL WHERE id=1")
def _progress(self, scan_id: int, text: str) -> None:
self.db.execute("UPDATE scan SET progress=? WHERE id=?", (text, scan_id))
def _warn(self, msg: str) -> None:
log.warning("scan warning: %s", msg)
if len(self.warnings) < 200:
self.warnings.append(msg)
# ── entry point ──────────────────────────────────────────────────────
def run(self, mode: str = "full", trigger: str = "manual") -> ScanResult:
now = int(time.time())
cur = self.db.execute(
"INSERT INTO scan (mode, trigger, status, started_at, history_source) "
"VALUES (?,?,'running',?,?)",
(mode, trigger, now, self.history.name if self.history else None),
)
scan_id = cur.lastrowid
try:
self._acquire_lock(scan_id)
except ScanBusy:
self.db.execute(
"UPDATE scan SET status='failed', finished_at=?, error=? WHERE id=?",
(now, "another scan is already running", scan_id),
)
raise
result = ScanResult(scan_id=scan_id, status="running", warnings=[])
try:
self._run_inner(scan_id, mode, result)
result.status = "succeeded"
except AuthError as e:
result.status, result.error = "failed", str(e)
log.error("scan %s failed on auth: %s", scan_id, e)
except Exception as e: # noqa: BLE001
result.status, result.error = "failed", str(e)
log.exception("scan %s failed", scan_id)
finally:
result.warnings = self.warnings
self.db.execute(
"UPDATE scan SET status=?, finished_at=?, items_seen=?, items_added=?, "
"items_updated=?, items_missing=?, events_added=?, warning_count=?, "
"warnings=?, error=?, progress=NULL WHERE id=?",
(result.status, int(time.time()), result.items_seen, result.items_added,
result.items_updated, result.items_missing, result.events_added,
len(self.warnings), json.dumps(self.warnings[:200]), result.error, scan_id),
)
self._release_lock()
return result
def _run_inner(self, scan_id: int, mode: str, result: ScanResult) -> None:
self._progress(scan_id, "connecting")
info = self.media.server_info()
provider_id = self._upsert_provider(info)
self.db.execute("UPDATE scan SET provider_id=? WHERE id=?", (provider_id, scan_id))
self._check_history_pairing(provider_id, info)
self._progress(scan_id, "reading libraries")
libraries = self.media.libraries()
lib_ids = self._upsert_libraries(provider_id, libraries)
self._seed_keep_all_libraries()
# History first: item rollups need it in place.
self._progress(scan_id, "reading watch history")
coverage = self._ingest_history(provider_id, mode, result)
for lib in libraries:
self._progress(scan_id, "scanning %s" % lib.title)
self._ingest_library(provider_id, lib, lib_ids[lib.provider_key],
scan_id, result)
self._progress(scan_id, "rolling up")
self._apply_watch_rollups(provider_id)
self._rollup_seasons(provider_id)
self._rollup_shows(provider_id)
self._apply_pre_history(provider_id, coverage)
if mode == "full":
self._mark_missing(provider_id, scan_id, result)
self._progress(scan_id, "resolving keeps")
keeps.resolve_all(self.db)
orphans = keeps.stamp_matches(self.db, scan_id)
if orphans:
self._warn("%d keep mark(s) matched nothing this scan" % orphans)
self.db.execute("UPDATE provider SET last_scan_id=? WHERE id=?", (scan_id, provider_id))
# ── provider / libraries ─────────────────────────────────────────────
def _upsert_provider(self, info) -> int:
now = int(time.time())
self.db.execute(
"INSERT INTO provider (kind, name, base_url, server_id, version, created_at) "
"VALUES (?,?,?,?,?,?) ON CONFLICT(kind, base_url) DO UPDATE SET "
"name=excluded.name, server_id=excluded.server_id, version=excluded.version",
(info.kind, info.name, info.base_url, info.server_id, info.version, now),
)
return self.db.scalar(
"SELECT id FROM provider WHERE kind=? AND base_url=?",
(info.kind, info.base_url),
)
def _check_history_pairing(self, provider_id: int, media_info) -> None:
"""Refuse to join history from a different Plex server (§4.11)."""
if self.history is None or self.history.name != "tautulli":
return
try:
hinfo = self.history.server_info()
except ProviderError as e:
self._warn("could not read Tautulli server info: %s" % e)
return
if hinfo.server_id and media_info.server_id and hinfo.server_id != media_info.server_id:
raise ProviderError(
"Tautulli is watching a different Plex server "
"(%s != %s) - refusing to join unrelated history data"
% (hinfo.server_id[:8], media_info.server_id[:8])
)
def _upsert_libraries(self, provider_id: int, libraries: list[Library]) -> dict[str, int]:
out = {}
now = int(time.time())
for lib in libraries:
self.db.execute(
"INSERT INTO library (provider_id, provider_key, title, kind, locations, scanned_at) "
"VALUES (?,?,?,?,?,?) ON CONFLICT(provider_id, provider_key) DO UPDATE SET "
"title=excluded.title, kind=excluded.kind, locations=excluded.locations, "
"scanned_at=excluded.scanned_at",
(provider_id, lib.provider_key, lib.title, lib.kind,
json.dumps(lib.locations), now),
)
out[lib.provider_key] = self.db.scalar(
"SELECT id FROM library WHERE provider_id=? AND provider_key=?",
(provider_id, lib.provider_key),
)
return out
def _seed_keep_all_libraries(self) -> None:
"""Apply KEEP_ALL_LIBRARIES once, on first run only.
Empty by default: nothing is ever kept unless a person says so (§6.6).
Re-applying on every start would silently re-enable a rule the user
turned off in the UI, so a marker setting guards it.
"""
if not self.cfg.keep_all_libraries:
return
if self.db.get_setting("keep_all_seeded"):
return
for title in self.cfg.keep_all_libraries:
row = self.db.one("SELECT id FROM library WHERE title = ?", (title,))
if row:
self.db.execute("UPDATE library SET keep_all = 1 WHERE id = ?", (row["id"],))
log.info("seeded keep_all for library %r", title)
else:
self._warn("KEEP_ALL_LIBRARIES names %r, which is not a library" % title)
self.db.set_setting("keep_all_seeded", "1")
# ── history ──────────────────────────────────────────────────────────
def _disposition(self, pc: int | None) -> str:
if pc is None:
return "completed" # Plex fallback: only a play is recorded (§4.11)
if pc >= self.cfg.completion_threshold:
return "completed"
if pc < self.cfg.abandon_ceiling:
return "abandoned"
return "partial"
def _ingest_history(self, provider_id: int, mode: str, result: ScanResult) -> Coverage | None:
if self.history is None:
return None
try:
for acct in self.history.accounts():
self.db.execute(
"INSERT INTO account (provider_id, account_id, name, friendly_name) "
"VALUES (?,?,?,?) ON CONFLICT(provider_id, account_id) DO UPDATE SET "
"name=excluded.name, friendly_name=excluded.friendly_name",
(provider_id, acct.account_id, acct.name, acct.friendly_name),
)
except ProviderError as e:
self._warn("could not read accounts: %s" % e)
since = None
if mode != "full":
since = self.db.scalar(
"SELECT MAX(viewed_at) FROM watch_event WHERE provider_id=? AND source=?",
(provider_id, self.history.name),
)
window = self.cfg.session_merge_window_h * 3600
batch: list[tuple] = []
added = 0
def flush():
nonlocal added, batch
if not batch:
return
cur = self.db.executemany(
"INSERT INTO watch_event (provider_id, source, source_row_id, reference_id, "
"provider_item_id, account_id, viewed_at, stopped_at, play_duration_s, "
"paused_counter_s, percent_complete, watched_status, disposition, session_id, "
"media_type, platform) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "
"ON CONFLICT(provider_id, source, source_row_id) DO NOTHING",
batch,
)
added += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
batch = []
for ev in self.history.watch_events(since):
# Session key: same item + same user within the merge window (§4.9).
bucket = ev.viewed_at // window if window > 0 else ev.viewed_at
session_id = "%s:%s:%s" % (ev.provider_item_id, ev.account_id or "-", bucket)
batch.append((
provider_id, ev.source, ev.source_row_id, ev.reference_id,
ev.provider_item_id, ev.account_id, ev.viewed_at, ev.stopped_at,
ev.play_duration_s, ev.paused_counter_s, ev.percent_complete,
ev.watched_status, self._disposition(ev.percent_complete), session_id,
ev.media_type, ev.platform,
))
if len(batch) >= 2000:
flush()
flush()
result.events_added = added
cov = self.db.one(
"SELECT MIN(viewed_at) AS lo, MAX(viewed_at) AS hi, COUNT(*) AS n "
"FROM watch_event WHERE provider_id=? AND source=?",
(provider_id, self.history.name),
)
coverage = Coverage(cov["lo"], cov["hi"], cov["n"] or 0)
self.db.execute(
"INSERT INTO history_coverage (provider_id, source, earliest_event_at, "
"latest_event_at, event_count, updated_at) VALUES (?,?,?,?,?,?) "
"ON CONFLICT(provider_id, source) DO UPDATE SET "
"earliest_event_at=excluded.earliest_event_at, "
"latest_event_at=excluded.latest_event_at, "
"event_count=excluded.event_count, updated_at=excluded.updated_at",
(provider_id, self.history.name, coverage.earliest_event_at,
coverage.latest_event_at, coverage.event_count, int(time.time())),
)
return coverage
# ── items ────────────────────────────────────────────────────────────
def _ingest_library(self, provider_id: int, lib: Library, library_id: int,
scan_id: int, result: ScanResult) -> None:
show_guids: dict[str, str] = {}
if lib.kind == "show":
try:
show_guids = self.media.show_guids(lib)
except Exception as e: # noqa: BLE001
self._warn("could not read show GUIDs for %s: %s" % (lib.title, e))
seasons: dict[str, dict] = {}
shows: dict[str, dict] = {}
n = 0
conn = self.db.conn
conn.execute("BEGIN IMMEDIATE")
try:
for item in self.media.items(lib):
n += 1
if item.kind == "movie":
self._upsert_movie(provider_id, library_id, item, scan_id, result)
else:
self._collect_episode(provider_id, library_id, item, show_guids,
seasons, shows, scan_id, result)
# season/show container rows
for key, s in seasons.items():
self._upsert_season(provider_id, library_id, s, scan_id, result)
for key, s in shows.items():
self._upsert_show(provider_id, library_id, s, scan_id, result)
conn.execute("COMMIT")
except Exception:
conn.execute("ROLLBACK")
raise
result.items_seen += n
def _upsert_movie(self, provider_id, library_id, item: Item, scan_id, result) -> None:
existing = self.db.one(
"SELECT id FROM media_item WHERE provider_id=? AND provider_item_id=?",
(provider_id, item.provider_item_id),
)
primary = item.parts[0].file_path if item.parts else None
vals = (
provider_id, library_id, item.provider_item_id, "movie", item.guid, None,
item.title, item.sort_title, item.year, None, None,
item.added_at, item.updated_at, 0, item.size_bytes, item.duration_ms,
len(item.parts), primary, item.resolution, item.video_codec,
item.view_count, "present", scan_id, scan_id,
)
if existing:
self.db.execute(
"UPDATE media_item SET library_id=?, guid=?, title=?, sort_title=?, year=?, "
"added_at=?, updated_at=?, size_bytes=?, duration_ms=?, part_count=?, "
"primary_path=?, resolution=?, video_codec=?, provider_view_count=?, "
"status='present', last_seen_scan_id=? WHERE id=?",
(library_id, item.guid, item.title, item.sort_title, item.year,
item.added_at, item.updated_at, item.size_bytes, item.duration_ms,
len(item.parts), primary, item.resolution, item.video_codec,
item.view_count, scan_id, existing["id"]),
)
item_id = existing["id"]
result.items_updated += 1
else:
cur = self.db.execute(
"INSERT INTO media_item (provider_id, library_id, provider_item_id, kind, "
"guid, show_guid, title, sort_title, year, parent_id, season_number, "
"added_at, updated_at, episode_count, size_bytes, duration_ms, part_count, "
"primary_path, resolution, video_codec, provider_view_count, status, "
"first_seen_scan_id, last_seen_scan_id) "
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", vals,
)
item_id = cur.lastrowid
result.items_added += 1
# Parts are replaced wholesale — cheap, and the only way to stay correct
# when a version is removed or a split file is re-encoded into one.
self.db.execute("DELETE FROM media_part WHERE media_item_id=?", (item_id,))
self._insert_parts(item, media_item_id=item_id)
def _insert_parts(self, item: Item, *, media_item_id=None, episode_id=None) -> None:
if not item.parts:
return
self.db.executemany(
"INSERT INTO media_part (media_item_id, episode_id, provider_part_id, file_path, "
"size_bytes, container, resolution, video_codec, audio_codec, bitrate) "
"VALUES (?,?,?,?,?,?,?,?,?,?)",
[(media_item_id, episode_id, p.provider_part_id, p.file_path, p.size_bytes,
p.container, p.resolution, p.video_codec, p.audio_codec, p.bitrate)
for p in item.parts],
)
def _collect_episode(self, provider_id, library_id, item: Item, show_guids,
seasons, shows, scan_id, result) -> None:
if not item.season_id:
self._warn("episode %s has no season; skipped" % item.provider_item_id)
return
show_guid = show_guids.get(item.show_id or "") or None
s = seasons.setdefault(item.season_id, {
"provider_item_id": item.season_id,
"show_id": item.show_id,
"show_guid": show_guid,
"show_title": item.show_title,
"season_number": item.season_number,
"episodes": [],
})
s["episodes"].append(item)
sh = shows.setdefault(item.show_id or "?", {
"provider_item_id": item.show_id,
"guid": show_guid,
"title": item.show_title or "(unknown show)",
"seasons": set(),
})
sh["seasons"].add(item.season_id)
def _upsert_season(self, provider_id, library_id, s: dict, scan_id, result) -> None:
eps: list[Item] = s["episodes"]
title = "Season %s" % (s["season_number"] if s["season_number"] is not None else "?")
existing = self.db.one(
"SELECT id FROM media_item WHERE provider_id=? AND provider_item_id=?",
(provider_id, s["provider_item_id"]),
)
if existing:
season_id = existing["id"]
self.db.execute(
"UPDATE media_item SET library_id=?, show_guid=?, title=?, season_number=?, "
"status='present', last_seen_scan_id=? WHERE id=?",
(library_id, s["show_guid"], title, s["season_number"], scan_id, season_id),
)
result.items_updated += 1
else:
cur = self.db.execute(
"INSERT INTO media_item (provider_id, library_id, provider_item_id, kind, "
"show_guid, title, season_number, status, first_seen_scan_id, last_seen_scan_id) "
"VALUES (?,?,?,'season',?,?,?,'present',?,?)",
(provider_id, library_id, s["provider_item_id"], s["show_guid"],
title, s["season_number"], scan_id, scan_id),
)
season_id = cur.lastrowid
result.items_added += 1
for ep in eps:
row = self.db.one("SELECT id FROM episode WHERE provider_item_id=?",
(ep.provider_item_id,))
primary_count = len(ep.parts)
if row:
ep_id = row["id"]
self.db.execute(
"UPDATE episode SET season_item_id=?, episode_number=?, title=?, "
"added_at=?, duration_ms=?, size_bytes=?, part_count=?, "
"status='present', last_seen_scan_id=? WHERE id=?",
(season_id, ep.episode_number, ep.title, ep.added_at, ep.duration_ms,
ep.size_bytes, primary_count, scan_id, ep_id),
)
else:
cur = self.db.execute(
"INSERT INTO episode (season_item_id, provider_item_id, episode_number, "
"title, added_at, duration_ms, size_bytes, part_count, status, last_seen_scan_id) "
"VALUES (?,?,?,?,?,?,?,?, 'present', ?)",
(season_id, ep.provider_item_id, ep.episode_number, ep.title,
ep.added_at, ep.duration_ms, ep.size_bytes, primary_count, scan_id),
)
ep_id = cur.lastrowid
self.db.execute("DELETE FROM media_part WHERE episode_id=?", (ep_id,))
self._insert_parts(ep, episode_id=ep_id)
def _upsert_show(self, provider_id, library_id, sh: dict, scan_id, result) -> None:
if not sh["provider_item_id"]:
return
existing = self.db.one(
"SELECT id FROM media_item WHERE provider_id=? AND provider_item_id=?",
(provider_id, sh["provider_item_id"]),
)
if existing:
show_id = existing["id"]
self.db.execute(
"UPDATE media_item SET library_id=?, guid=?, title=?, status='present', "
"last_seen_scan_id=? WHERE id=?",
(library_id, sh["guid"], sh["title"], scan_id, show_id),
)
else:
cur = self.db.execute(
"INSERT INTO media_item (provider_id, library_id, provider_item_id, kind, "
"guid, title, status, first_seen_scan_id, last_seen_scan_id) "
"VALUES (?,?,?,'show',?,?,'present',?,?)",
(provider_id, library_id, sh["provider_item_id"], sh["guid"],
sh["title"], scan_id, scan_id),
)
show_id = cur.lastrowid
# link seasons to their show
self.db.execute(
"UPDATE media_item SET parent_id=? WHERE provider_id=? AND kind='season' "
"AND provider_item_id IN (%s)" % ",".join("?" * len(sh["seasons"])),
tuple([show_id, provider_id] + list(sh["seasons"])),
)
# ── rollups ──────────────────────────────────────────────────────────
def _apply_watch_rollups(self, provider_id: int) -> None:
"""Aggregate watch_event onto movies and episodes.
Distinct sessions, not raw events: a paused-and-resumed play is one
viewing (§4.9). Everything is recomputed from scratch each scan, which is
what makes re-running a scan idempotent.
"""
c = self.db.conn
c.execute("""
UPDATE media_item SET watch_count=0, partial_count=0, abandoned_count=0,
last_watched_at=NULL, last_touched_at=NULL, first_watched_at=NULL,
distinct_watcher_count=0, avg_percent_complete=NULL
WHERE kind='movie'
""")
c.execute("""
UPDATE episode SET watch_count=0, partial_count=0, abandoned_count=0,
last_watched_at=NULL, last_touched_at=NULL
""")
agg = """
SELECT provider_item_id AS pid,
COUNT(DISTINCT CASE WHEN disposition='completed' THEN session_id END) AS completed,
COUNT(DISTINCT CASE WHEN disposition='partial' THEN session_id END) AS partial,
COUNT(DISTINCT CASE WHEN disposition='abandoned' THEN session_id END) AS abandoned,
MAX(CASE WHEN disposition='completed' THEN viewed_at END) AS last_watched,
MIN(CASE WHEN disposition='completed' THEN viewed_at END) AS first_watched,
MAX(viewed_at) AS last_touched,
COUNT(DISTINCT account_id) AS watchers,
AVG(percent_complete) AS avg_pc
FROM watch_event WHERE provider_id = ?
GROUP BY provider_item_id
"""
c.execute("DROP TABLE IF EXISTS _wagg")
c.execute("CREATE TEMP TABLE _wagg AS " + agg, (provider_id,))
c.execute("CREATE INDEX _wagg_pid ON _wagg(pid)")
c.execute("""
UPDATE media_item SET
watch_count = COALESCE((SELECT completed FROM _wagg WHERE pid = media_item.provider_item_id), 0),
partial_count = COALESCE((SELECT partial FROM _wagg WHERE pid = media_item.provider_item_id), 0),
abandoned_count = COALESCE((SELECT abandoned FROM _wagg WHERE pid = media_item.provider_item_id), 0),
last_watched_at = (SELECT last_watched FROM _wagg WHERE pid = media_item.provider_item_id),
first_watched_at = (SELECT first_watched FROM _wagg WHERE pid = media_item.provider_item_id),
last_touched_at = (SELECT last_touched FROM _wagg WHERE pid = media_item.provider_item_id),
distinct_watcher_count = COALESCE((SELECT watchers FROM _wagg WHERE pid = media_item.provider_item_id), 0),
avg_percent_complete = (SELECT avg_pc FROM _wagg WHERE pid = media_item.provider_item_id)
WHERE kind = 'movie'
""")
c.execute("""
UPDATE episode SET
watch_count = COALESCE((SELECT completed FROM _wagg WHERE pid = episode.provider_item_id), 0),
partial_count = COALESCE((SELECT partial FROM _wagg WHERE pid = episode.provider_item_id), 0),
abandoned_count = COALESCE((SELECT abandoned FROM _wagg WHERE pid = episode.provider_item_id), 0),
last_watched_at = (SELECT last_watched FROM _wagg WHERE pid = episode.provider_item_id),
last_touched_at = (SELECT last_touched FROM _wagg WHERE pid = episode.provider_item_id)
""")
def _rollup_seasons(self, provider_id: int) -> None:
c = self.db.conn
c.execute("""
UPDATE media_item SET
episode_count = COALESCE((SELECT COUNT(*) FROM episode e
WHERE e.season_item_id = media_item.id AND e.status='present'), 0),
size_bytes = COALESCE((SELECT SUM(e.size_bytes) FROM episode e
WHERE e.season_item_id = media_item.id AND e.status='present'), 0),
duration_ms = COALESCE((SELECT SUM(e.duration_ms) FROM episode e
WHERE e.season_item_id = media_item.id AND e.status='present'), 0),
part_count = COALESCE((SELECT SUM(e.part_count) FROM episode e
WHERE e.season_item_id = media_item.id AND e.status='present'), 0),
added_at = (SELECT MIN(e.added_at) FROM episode e
WHERE e.season_item_id = media_item.id AND e.status='present'),
watch_count = COALESCE((SELECT SUM(e.watch_count) FROM episode e
WHERE e.season_item_id = media_item.id), 0),
partial_count = COALESCE((SELECT SUM(e.partial_count) FROM episode e
WHERE e.season_item_id = media_item.id), 0),
abandoned_count = COALESCE((SELECT SUM(e.abandoned_count) FROM episode e
WHERE e.season_item_id = media_item.id), 0),
last_watched_at = (SELECT MAX(e.last_watched_at) FROM episode e
WHERE e.season_item_id = media_item.id),
last_touched_at = (SELECT MAX(e.last_touched_at) FROM episode e
WHERE e.season_item_id = media_item.id)
WHERE kind = 'season' AND provider_id = ?
""", (provider_id,))
# distinct watchers across the season's episodes
c.execute("""
UPDATE media_item SET distinct_watcher_count = COALESCE((
SELECT COUNT(DISTINCT w.account_id) FROM watch_event w
JOIN episode e ON e.provider_item_id = w.provider_item_id
WHERE e.season_item_id = media_item.id), 0)
WHERE kind = 'season' AND provider_id = ?
""", (provider_id,))
# representative path: the common directory of its episodes
c.execute("""
UPDATE media_item SET primary_path = (
SELECT p.file_path FROM media_part p
JOIN episode e ON e.id = p.episode_id
WHERE e.season_item_id = media_item.id
ORDER BY e.episode_number LIMIT 1)
WHERE kind = 'season' AND provider_id = ?
""", (provider_id,))
c.execute("""
UPDATE media_item SET resolution = (
SELECT p.resolution FROM media_part p
JOIN episode e ON e.id = p.episode_id
WHERE e.season_item_id = media_item.id AND p.resolution IS NOT NULL
LIMIT 1)
WHERE kind = 'season' AND provider_id = ?
""", (provider_id,))
def _rollup_shows(self, provider_id: int) -> None:
self.db.execute("""
UPDATE media_item SET
episode_count = COALESCE((SELECT SUM(s.episode_count) FROM media_item s
WHERE s.parent_id = media_item.id), 0),
size_bytes = COALESCE((SELECT SUM(s.size_bytes) FROM media_item s
WHERE s.parent_id = media_item.id), 0),
part_count = COALESCE((SELECT SUM(s.part_count) FROM media_item s
WHERE s.parent_id = media_item.id), 0),
duration_ms = COALESCE((SELECT SUM(s.duration_ms) FROM media_item s
WHERE s.parent_id = media_item.id), 0),
added_at = (SELECT MIN(s.added_at) FROM media_item s
WHERE s.parent_id = media_item.id),
watch_count = COALESCE((SELECT SUM(s.watch_count) FROM media_item s
WHERE s.parent_id = media_item.id), 0),
abandoned_count = COALESCE((SELECT SUM(s.abandoned_count) FROM media_item s
WHERE s.parent_id = media_item.id), 0),
last_watched_at = (SELECT MAX(s.last_watched_at) FROM media_item s
WHERE s.parent_id = media_item.id),
last_touched_at = (SELECT MAX(s.last_touched_at) FROM media_item s
WHERE s.parent_id = media_item.id)
WHERE kind = 'show' AND provider_id = ?
""", (provider_id,))
def _apply_pre_history(self, provider_id: int, coverage: Coverage | None) -> None:
"""Flag items added before watch history began (§4.11).
On the measured library this is the majority state, not an edge case.
"""
self.db.execute("UPDATE media_item SET pre_history = 0 WHERE provider_id = ?",
(provider_id,))
if not coverage or not coverage.earliest_event_at:
return
self.db.execute(
"UPDATE media_item SET pre_history = 1 "
"WHERE provider_id = ? AND added_at IS NOT NULL AND added_at > 0 AND added_at < ?",
(provider_id, coverage.earliest_event_at),
)
def _mark_missing(self, provider_id: int, scan_id: int, result: ScanResult) -> None:
"""Items not seen this full sweep become 'missing', never deleted (§5.4)."""
cur = self.db.execute(
"UPDATE media_item SET status='missing' "
"WHERE provider_id=? AND status='present' "
"AND (last_seen_scan_id IS NULL OR last_seen_scan_id != ?)",
(provider_id, scan_id),
)
result.items_missing = cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
self.db.execute(
"UPDATE episode SET status='missing' "
"WHERE status='present' AND (last_seen_scan_id IS NULL OR last_seen_scan_id != ?)",
(scan_id,),
)

328
mediashelf/keeps.py Normal file
View file

@ -0,0 +1,328 @@
"""Keep marks — human judgements the score must not override (§6.6).
The critical property of this module is that marks survive Plex reassigning
rating keys. They are stored against content identity:
movie / show : (library_id, guid)
season : (library_id, show_guid, season_number)
provider_item_id is stored for convenience and linking, and is allowed to go
stale. It is never what a mark is matched on. If that invariant is ever broken,
the failure mode in v2 is deleting content someone explicitly protected which
is why test_keeps.py reassigns every rating key in the library and asserts the
marks still resolve.
"""
from __future__ import annotations
import json
import time
SCOPES = ("show", "season", "movie")
MODES = ("keep", "exclude")
class KeepError(ValueError):
pass
# ── resolution ───────────────────────────────────────────────────────────
RESOLVE_SQL = """
UPDATE media_item SET
kept = 0,
kept_via = NULL,
kept_mark_id = NULL
"""
def resolve_all(db) -> dict:
"""Recompute kept/kept_via for every item. Most specific mark wins.
Precedence (§6.6):
movie/season explicit 'exclude' -> NOT kept
movie/season explicit 'keep' -> kept
show 'keep' -> kept
library keep_all -> kept
otherwise -> not kept
"""
conn = db.conn
conn.execute("UPDATE media_item SET kept = 0, kept_via = NULL, kept_mark_id = NULL")
# 1. library rules (weakest)
conn.execute("""
UPDATE media_item SET kept = 1, kept_via = 'library'
WHERE library_id IN (SELECT id FROM library WHERE keep_all = 1)
""")
# 2. show marks -> the show row, its seasons
conn.execute("""
UPDATE media_item SET kept = 1, kept_via = 'show',
kept_mark_id = (
SELECT k.id FROM keep_mark k
WHERE k.scope = 'show' AND k.mode = 'keep'
AND k.library_id = media_item.library_id
AND k.guid = media_item.guid)
WHERE kind = 'show' AND guid IS NOT NULL AND EXISTS (
SELECT 1 FROM keep_mark k
WHERE k.scope = 'show' AND k.mode = 'keep'
AND k.library_id = media_item.library_id
AND k.guid = media_item.guid)
""")
conn.execute("""
UPDATE media_item SET kept = 1, kept_via = 'show',
kept_mark_id = (
SELECT k.id FROM keep_mark k
WHERE k.scope = 'show' AND k.mode = 'keep'
AND k.library_id = media_item.library_id
AND k.guid = media_item.show_guid)
WHERE kind = 'season' AND show_guid IS NOT NULL AND EXISTS (
SELECT 1 FROM keep_mark k
WHERE k.scope = 'show' AND k.mode = 'keep'
AND k.library_id = media_item.library_id
AND k.guid = media_item.show_guid)
""")
# 3. explicit season/movie keeps
conn.execute("""
UPDATE media_item SET kept = 1, kept_via = 'season',
kept_mark_id = (
SELECT k.id FROM keep_mark k
WHERE k.scope = 'season' AND k.mode = 'keep'
AND k.library_id = media_item.library_id
AND k.guid = media_item.show_guid
AND k.season_number = media_item.season_number)
WHERE kind = 'season' AND EXISTS (
SELECT 1 FROM keep_mark k
WHERE k.scope = 'season' AND k.mode = 'keep'
AND k.library_id = media_item.library_id
AND k.guid = media_item.show_guid
AND k.season_number = media_item.season_number)
""")
conn.execute("""
UPDATE media_item SET kept = 1, kept_via = 'movie',
kept_mark_id = (
SELECT k.id FROM keep_mark k
WHERE k.scope = 'movie' AND k.mode = 'keep'
AND k.library_id = media_item.library_id
AND k.guid = media_item.guid)
WHERE kind = 'movie' AND guid IS NOT NULL AND EXISTS (
SELECT 1 FROM keep_mark k
WHERE k.scope = 'movie' AND k.mode = 'keep'
AND k.library_id = media_item.library_id
AND k.guid = media_item.guid)
""")
# 4. explicit excludes win over everything above
conn.execute("""
UPDATE media_item SET kept = 0, kept_via = NULL, kept_mark_id = NULL
WHERE kind = 'season' AND EXISTS (
SELECT 1 FROM keep_mark k
WHERE k.scope = 'season' AND k.mode = 'exclude'
AND k.library_id = media_item.library_id
AND k.guid = media_item.show_guid
AND k.season_number = media_item.season_number)
""")
conn.execute("""
UPDATE media_item SET kept = 0, kept_via = NULL, kept_mark_id = NULL
WHERE kind = 'movie' AND guid IS NOT NULL AND EXISTS (
SELECT 1 FROM keep_mark k
WHERE k.scope = 'movie' AND k.mode = 'exclude'
AND k.library_id = media_item.library_id
AND k.guid = media_item.guid)
""")
kept_items = db.scalar("SELECT COUNT(*) FROM media_item WHERE kept = 1 AND kind != 'show'") or 0
kept_bytes = db.scalar(
"SELECT COALESCE(SUM(size_bytes),0) FROM media_item WHERE kept = 1 AND kind != 'show'"
) or 0
return {"kept_items": kept_items, "kept_bytes": kept_bytes}
def mark_matches(db) -> dict[int, int]:
"""How many items each mark currently resolves to. 0 => orphaned."""
counts: dict[int, int] = {}
for row in db.query("SELECT id FROM keep_mark"):
counts[row["id"]] = 0
for row in db.query(
"SELECT kept_mark_id AS mid, COUNT(*) AS n FROM media_item "
"WHERE kept_mark_id IS NOT NULL GROUP BY kept_mark_id"
):
counts[row["mid"]] = row["n"]
# Excludes never set kept_mark_id, so count them separately.
for row in db.query("SELECT * FROM keep_mark WHERE mode = 'exclude'"):
if row["scope"] == "movie":
n = db.scalar(
"SELECT COUNT(*) FROM media_item WHERE kind='movie' "
"AND library_id=? AND guid=?",
(row["library_id"], row["guid"]),
)
else:
n = db.scalar(
"SELECT COUNT(*) FROM media_item WHERE kind='season' "
"AND library_id=? AND show_guid=? AND season_number=?",
(row["library_id"], row["guid"], row["season_number"]),
)
counts[row["id"]] = n or 0
return counts
def stamp_matches(db, scan_id: int | None) -> int:
"""Record which marks matched something this scan. Returns orphan count."""
counts = mark_matches(db)
orphans = 0
for mark_id, n in counts.items():
if n > 0:
db.execute("UPDATE keep_mark SET last_matched_scan_id = ? WHERE id = ?",
(scan_id, mark_id))
else:
orphans += 1
return orphans
# ── mutation ─────────────────────────────────────────────────────────────
def create_from_item(db, item_id: int, mode: str = "keep", note: str | None = None) -> int:
"""Create a mark from a grid row, resolving it to the durable key."""
if mode not in MODES:
raise KeepError("mode must be 'keep' or 'exclude'")
row = db.one(
"SELECT i.*, lib.title AS library_title FROM media_item i "
"JOIN library lib ON lib.id = i.library_id WHERE i.id = ?",
(item_id,),
)
if row is None:
raise KeepError("no such item")
kind = row["kind"]
if kind == "movie":
scope, guid, season = "movie", row["guid"], None
label = row["title"] + (" (%s)" % row["year"] if row["year"] else "")
elif kind == "show":
scope, guid, season = "show", row["guid"], None
label = row["title"]
elif kind == "season":
scope, guid, season = "season", row["show_guid"], row["season_number"]
parent = db.one("SELECT title FROM media_item WHERE id = ?", (row["parent_id"],))
show_title = parent["title"] if parent else "(unknown show)"
label = "%s — Season %s" % (show_title, row["season_number"])
else:
raise KeepError("cannot keep a %s" % kind)
if not guid:
raise KeepError(
"this item has no Plex GUID, so a keep could not survive a library "
"rebuild; refusing to create one that would silently detach"
)
return upsert(db, scope, mode, row["library_id"], guid, season,
label=label, note=note, provider_item_id=row["provider_item_id"])
def upsert(db, scope: str, mode: str, library_id: int, guid: str,
season_number: int | None, *, label: str, note: str | None = None,
provider_item_id: str | None = None) -> int:
if scope not in SCOPES:
raise KeepError("scope must be one of %s" % (SCOPES,))
if mode not in MODES:
raise KeepError("mode must be 'keep' or 'exclude'")
if scope == "season" and season_number is None:
raise KeepError("a season mark needs a season_number")
if scope != "season":
season_number = None
now = int(time.time())
existing = db.one(
"SELECT id FROM keep_mark WHERE scope=? AND library_id=? AND guid=? "
"AND season_number IS ?",
(scope, library_id, guid, season_number),
)
if existing:
db.execute(
"UPDATE keep_mark SET mode=?, note=COALESCE(?, note), label=?, "
"provider_item_id=?, updated_at=? WHERE id=?",
(mode, note, label, provider_item_id, now, existing["id"]),
)
return existing["id"]
cur = db.execute(
"INSERT INTO keep_mark (scope, mode, library_id, guid, season_number, "
"provider_item_id, label, note, created_at, updated_at) "
"VALUES (?,?,?,?,?,?,?,?,?,?)",
(scope, mode, library_id, guid, season_number, provider_item_id,
label, note, now, now),
)
return cur.lastrowid
def delete(db, mark_id: int) -> bool:
cur = db.execute("DELETE FROM keep_mark WHERE id = ?", (mark_id,))
return cur.rowcount > 0
def set_library_keep_all(db, library_id: int, keep_all: bool) -> None:
db.execute("UPDATE library SET keep_all = ? WHERE id = ?",
(1 if keep_all else 0, library_id))
# ── export / import (§11.5) ──────────────────────────────────────────────
def export(db) -> dict:
"""Human-readable, durable-key export. The only irreplaceable data here."""
libs = {r["id"]: r["title"] for r in db.query("SELECT id, title FROM library")}
marks = []
for r in db.query("SELECT * FROM keep_mark ORDER BY id"):
marks.append({
"scope": r["scope"],
"mode": r["mode"],
"library": libs.get(r["library_id"], str(r["library_id"])),
"guid": r["guid"],
"season_number": r["season_number"],
"label": r["label"],
"note": r["note"],
"created_at": r["created_at"],
})
library_rules = [r["title"] for r in
db.query("SELECT title FROM library WHERE keep_all = 1 ORDER BY title")]
return {
"version": 1,
"exported_at": int(time.time()),
"library_keep_all": library_rules,
"marks": marks,
}
def import_(db, payload: dict) -> dict:
"""Restore an export. Matches libraries by title, since ids are local."""
if not isinstance(payload, dict) or payload.get("version") != 1:
raise KeepError("unrecognised keep export")
by_title = {r["title"]: r["id"] for r in db.query("SELECT id, title FROM library")}
added = skipped = 0
for title in payload.get("library_keep_all") or []:
lid = by_title.get(title)
if lid:
set_library_keep_all(db, lid, True)
else:
skipped += 1
for m in payload.get("marks") or []:
lid = by_title.get(m.get("library"))
if not lid or not m.get("guid"):
skipped += 1
continue
try:
upsert(db, m["scope"], m.get("mode", "keep"), lid, m["guid"],
m.get("season_number"), label=m.get("label") or m["guid"],
note=m.get("note"))
added += 1
except KeepError:
skipped += 1
resolve_all(db)
return {"imported": added, "skipped": skipped}
def write_export_file(db, path: str) -> None:
with open(path, "w") as fh:
json.dump(export(db), fh, indent=2)

View file

@ -0,0 +1,245 @@
-- MediaShelf initial schema. See docs/design.md §5.
CREATE TABLE provider (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL,
name TEXT NOT NULL,
base_url TEXT NOT NULL,
server_id TEXT,
version TEXT,
last_scan_id INTEGER,
created_at INTEGER NOT NULL,
UNIQUE (kind, base_url)
);
CREATE TABLE library (
id INTEGER PRIMARY KEY,
provider_id INTEGER NOT NULL REFERENCES provider(id),
provider_key TEXT NOT NULL,
title TEXT NOT NULL,
kind TEXT NOT NULL, -- 'movie' | 'show'
locations TEXT, -- JSON array
keep_all INTEGER NOT NULL DEFAULT 0,
scanned_at INTEGER,
UNIQUE (provider_id, provider_key)
);
CREATE TABLE media_item (
id INTEGER PRIMARY KEY,
provider_id INTEGER NOT NULL REFERENCES provider(id),
library_id INTEGER NOT NULL REFERENCES library(id),
provider_item_id TEXT NOT NULL,
kind TEXT NOT NULL, -- 'movie' | 'show' | 'season'
guid TEXT,
show_guid TEXT, -- seasons: the parent show's guid
title TEXT NOT NULL,
sort_title TEXT,
year INTEGER,
parent_id INTEGER REFERENCES media_item(id),
season_number INTEGER,
added_at INTEGER,
updated_at INTEGER,
episode_count INTEGER NOT NULL DEFAULT 0,
size_bytes INTEGER NOT NULL DEFAULT 0,
duration_ms INTEGER NOT NULL DEFAULT 0,
part_count INTEGER NOT NULL DEFAULT 0,
primary_path TEXT,
resolution TEXT,
video_codec TEXT,
watch_count INTEGER NOT NULL DEFAULT 0,
partial_count INTEGER NOT NULL DEFAULT 0,
abandoned_count INTEGER NOT NULL DEFAULT 0,
last_watched_at INTEGER,
last_touched_at INTEGER,
first_watched_at INTEGER,
distinct_watcher_count INTEGER NOT NULL DEFAULT 0,
avg_percent_complete REAL,
pre_history INTEGER NOT NULL DEFAULT 0,
kept INTEGER NOT NULL DEFAULT 0,
kept_via TEXT,
kept_mark_id INTEGER,
provider_view_count INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'present',
first_seen_scan_id INTEGER,
last_seen_scan_id INTEGER,
UNIQUE (provider_id, provider_item_id)
);
CREATE TABLE episode (
id INTEGER PRIMARY KEY,
season_item_id INTEGER NOT NULL REFERENCES media_item(id) ON DELETE CASCADE,
provider_item_id TEXT NOT NULL,
episode_number INTEGER,
title TEXT,
added_at INTEGER,
duration_ms INTEGER NOT NULL DEFAULT 0,
size_bytes INTEGER NOT NULL DEFAULT 0,
part_count INTEGER NOT NULL DEFAULT 0,
watch_count INTEGER NOT NULL DEFAULT 0,
partial_count INTEGER NOT NULL DEFAULT 0,
abandoned_count INTEGER NOT NULL DEFAULT 0,
last_watched_at INTEGER,
last_touched_at INTEGER,
status TEXT NOT NULL DEFAULT 'present',
last_seen_scan_id INTEGER,
UNIQUE (provider_item_id)
);
CREATE TABLE media_part (
id INTEGER PRIMARY KEY,
media_item_id INTEGER REFERENCES media_item(id) ON DELETE CASCADE,
episode_id INTEGER REFERENCES episode(id) ON DELETE CASCADE,
provider_part_id TEXT,
file_path TEXT NOT NULL,
size_bytes INTEGER NOT NULL DEFAULT 0,
container TEXT,
resolution TEXT,
video_codec TEXT,
audio_codec TEXT,
bitrate INTEGER,
CHECK ((media_item_id IS NULL) != (episode_id IS NULL))
);
CREATE TABLE watch_event (
id INTEGER PRIMARY KEY,
provider_id INTEGER NOT NULL REFERENCES provider(id),
source TEXT NOT NULL, -- 'tautulli' | 'plex'
source_row_id TEXT NOT NULL,
reference_id TEXT,
provider_item_id TEXT NOT NULL,
account_id TEXT,
viewed_at INTEGER NOT NULL,
stopped_at INTEGER,
play_duration_s INTEGER,
paused_counter_s INTEGER,
percent_complete INTEGER,
watched_status REAL,
disposition TEXT NOT NULL, -- completed | partial | abandoned
session_id TEXT,
media_type TEXT,
platform TEXT,
UNIQUE (provider_id, source, source_row_id)
);
CREATE TABLE account (
id INTEGER PRIMARY KEY,
provider_id INTEGER NOT NULL REFERENCES provider(id),
account_id TEXT NOT NULL,
name TEXT,
friendly_name TEXT,
UNIQUE (provider_id, account_id)
);
CREATE TABLE history_coverage (
id INTEGER PRIMARY KEY,
provider_id INTEGER NOT NULL REFERENCES provider(id),
source TEXT NOT NULL,
earliest_event_at INTEGER,
latest_event_at INTEGER,
event_count INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL,
UNIQUE (provider_id, source)
);
-- Keep marks: human judgements the score must not override (§6.6).
-- Keyed on CONTENT identity, never on provider_item_id.
CREATE TABLE keep_mark (
id INTEGER PRIMARY KEY,
scope TEXT NOT NULL, -- 'show' | 'season' | 'movie'
mode TEXT NOT NULL, -- 'keep' | 'exclude'
library_id INTEGER NOT NULL REFERENCES library(id),
guid TEXT NOT NULL,
season_number INTEGER,
provider_item_id TEXT,
label TEXT NOT NULL,
note TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
last_matched_scan_id INTEGER,
UNIQUE (scope, library_id, guid, season_number)
);
CREATE TABLE scan (
id INTEGER PRIMARY KEY,
provider_id INTEGER REFERENCES provider(id),
mode TEXT NOT NULL, -- 'full' | 'incremental'
trigger TEXT NOT NULL, -- 'manual' | 'schedule' | 'startup'
status TEXT NOT NULL, -- 'running' | 'succeeded' | 'failed'
history_source TEXT,
started_at INTEGER NOT NULL,
finished_at INTEGER,
progress TEXT,
items_seen INTEGER NOT NULL DEFAULT 0,
items_added INTEGER NOT NULL DEFAULT 0,
items_updated INTEGER NOT NULL DEFAULT 0,
items_missing INTEGER NOT NULL DEFAULT 0,
events_added INTEGER NOT NULL DEFAULT 0,
warning_count INTEGER NOT NULL DEFAULT 0,
warnings TEXT,
error TEXT
);
CREATE TABLE scan_lock (
id INTEGER PRIMARY KEY CHECK (id = 1),
scan_id INTEGER,
holder TEXT,
acquired_at INTEGER
);
CREATE TABLE saved_view (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
description TEXT,
rules TEXT NOT NULL,
sort TEXT,
columns TEXT,
weights TEXT,
builtin INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE setting (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE INDEX ix_item_library ON media_item(library_id, kind, status);
CREATE INDEX ix_item_added ON media_item(added_at);
CREATE INDEX ix_item_lastwatch ON media_item(last_watched_at);
CREATE INDEX ix_item_size ON media_item(size_bytes);
CREATE INDEX ix_item_watchcount ON media_item(watch_count);
CREATE INDEX ix_item_parent ON media_item(parent_id);
CREATE INDEX ix_item_guid ON media_item(guid);
CREATE INDEX ix_item_showguid ON media_item(show_guid);
CREATE INDEX ix_item_kept ON media_item(kept, library_id);
CREATE INDEX ix_episode_season ON episode(season_item_id);
CREATE INDEX ix_part_item ON media_part(media_item_id);
CREATE INDEX ix_part_episode ON media_part(episode_id);
CREATE INDEX ix_event_item ON watch_event(provider_item_id);
CREATE INDEX ix_event_viewed ON watch_event(viewed_at);
CREATE INDEX ix_event_disp ON watch_event(provider_item_id, disposition);
CREATE INDEX ix_event_account ON watch_event(account_id, viewed_at);
CREATE INDEX ix_keep_lookup ON keep_mark(library_id, guid, season_number);
-- Full-text search over titles, kept in sync by triggers.
CREATE VIRTUAL TABLE media_item_fts USING fts5(
title, sort_title, content='media_item', content_rowid='id'
);
CREATE TRIGGER media_item_ai AFTER INSERT ON media_item BEGIN
INSERT INTO media_item_fts(rowid, title, sort_title)
VALUES (new.id, new.title, new.sort_title);
END;
CREATE TRIGGER media_item_ad AFTER DELETE ON media_item BEGIN
INSERT INTO media_item_fts(media_item_fts, rowid, title, sort_title)
VALUES ('delete', old.id, old.title, old.sort_title);
END;
CREATE TRIGGER media_item_au AFTER UPDATE OF title, sort_title ON media_item BEGIN
INSERT INTO media_item_fts(media_item_fts, rowid, title, sort_title)
VALUES ('delete', old.id, old.title, old.sort_title);
INSERT INTO media_item_fts(rowid, title, sort_title)
VALUES (new.id, new.title, new.sort_title);
END;

View file

@ -0,0 +1,68 @@
"""Provider construction and history-source selection."""
from __future__ import annotations
import logging
from .base import ( # noqa: F401
Account,
AuthError,
Coverage,
HistoryProvider,
Item,
Library,
MediaProvider,
Part,
ProviderError,
ServerInfo,
WatchEvent,
)
from .plex import PlexClient, PlexHistoryProvider, PlexProvider
from .tautulli import TautulliClient, TautulliHistoryProvider
log = logging.getLogger(__name__)
def build_media_provider(cfg) -> PlexProvider:
client = PlexClient(
cfg.plex_base_url, cfg.plex_token,
timeout=cfg.plex_timeout_s,
verify_ssl=cfg.plex_verify_ssl,
page_size=cfg.plex_page_size,
request_delay_ms=cfg.plex_request_delay_ms,
)
return PlexProvider(client)
def build_history_provider(cfg, media: PlexProvider):
"""Pick a history source per HISTORY_SOURCE (§4.11).
'auto' prefers Tautulli and falls back to Plex, reporting the reason rather
than degrading silently a silently degraded score is one that gets trusted
when it shouldn't be. Returns (provider, degraded_reason_or_None).
"""
mode = (cfg.history_source or "auto").lower()
if mode == "plex":
return PlexHistoryProvider(media.client), None
if mode in ("auto", "tautulli"):
if not cfg.tautulli_configured:
if mode == "tautulli":
raise ProviderError("HISTORY_SOURCE=tautulli but Tautulli is not configured")
return PlexHistoryProvider(media.client), "Tautulli is not configured"
client = TautulliClient(
cfg.tautulli_base_url, cfg.tautulli_api_key,
timeout=cfg.tautulli_timeout_s, page_size=cfg.tautulli_page_size,
)
provider = TautulliHistoryProvider(client)
try:
provider.server_info()
return provider, None
except ProviderError as e:
if mode == "tautulli":
raise
log.warning("Tautulli unreachable, falling back to Plex history: %s", e)
return PlexHistoryProvider(media.client), "Tautulli unreachable: %s" % e
raise ProviderError("unknown HISTORY_SOURCE %r" % cfg.history_source)

View file

@ -0,0 +1,139 @@
"""Provider protocols and the normalized types everything above this layer speaks.
Two protocols, deliberately separate (§3.1): MediaProvider knows what exists,
HistoryProvider knows what was watched. On this network those are two different
machines Plex on Loki, Tautulli on Isis and Emby/Jellyfin later will have no
Tautulli equivalent at all.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Iterator, Protocol, runtime_checkable
class ProviderError(RuntimeError):
"""Any failure talking to a source. Carries a message safe to display."""
class AuthError(ProviderError):
"""Credentials rejected. Never retried — a retry loop won't fix a bad token."""
@dataclass
class ServerInfo:
kind: str
name: str
version: str = ""
server_id: str = "" # Plex machineIdentifier / Tautulli pms_identifier
base_url: str = ""
@dataclass
class Library:
provider_key: str
title: str
kind: str # 'movie' | 'show'
locations: list[str] = field(default_factory=list)
@dataclass
class Part:
file_path: str
size_bytes: int = 0
provider_part_id: str | None = None
container: str | None = None
resolution: str | None = None
video_codec: str | None = None
audio_codec: str | None = None
bitrate: int | None = None
@dataclass
class Item:
"""A movie or an episode as the source reports it.
Episodes carry their season and show identity so the ingest can roll them up
without a second pass over the API.
"""
provider_item_id: str
kind: str # 'movie' | 'episode'
title: str
library_key: str
guid: str | None = None
sort_title: str | None = None
year: int | None = None
added_at: int | None = None
updated_at: int | None = None
duration_ms: int = 0
view_count: int = 0
last_viewed_at: int | None = None
resolution: str | None = None
video_codec: str | None = None
parts: list[Part] = field(default_factory=list)
# episode-only
show_id: str | None = None
show_title: str | None = None
show_guid: str | None = None
season_id: str | None = None
season_number: int | None = None
episode_number: int | None = None
@property
def size_bytes(self) -> int:
return sum(p.size_bytes for p in self.parts)
@dataclass
class WatchEvent:
"""One playback event, normalized across sources."""
source: str # 'tautulli' | 'plex'
source_row_id: str
provider_item_id: str
viewed_at: int
account_id: str | None = None
reference_id: str | None = None
stopped_at: int | None = None
play_duration_s: int | None = None
paused_counter_s: int | None = None
percent_complete: int | None = None
watched_status: float | None = None
media_type: str | None = None
platform: str | None = None
@dataclass
class Account:
account_id: str
name: str | None = None
friendly_name: str | None = None
@dataclass
class Coverage:
earliest_event_at: int | None
latest_event_at: int | None
event_count: int
@runtime_checkable
class MediaProvider(Protocol):
def server_info(self) -> ServerInfo: ...
def libraries(self) -> list[Library]: ...
def items(self, library: Library) -> Iterator[Item]: ...
def refresh_library(self, library: Library) -> None: ... # v2 only
@runtime_checkable
class HistoryProvider(Protocol):
name: str
def server_info(self) -> ServerInfo: ...
def accounts(self) -> list[Account]: ...
def watch_events(self, since: int | None = None) -> Iterator[WatchEvent]: ...
def coverage(self) -> Coverage: ...
@property
def has_completion_data(self) -> bool: ...

View file

@ -0,0 +1,294 @@
"""Plex Media Server provider.
Talks directly to the server on the LAN with a server token no plex.tv round
trip, so it works whether or not Plex's cloud is up. JSON throughout via the
Accept header, which avoids XML parsing entirely (§4.1).
"""
from __future__ import annotations
import logging
import time
from typing import Iterator
import requests
from .base import (
Account,
AuthError,
Coverage,
Item,
Library,
Part,
ProviderError,
ServerInfo,
WatchEvent,
)
log = logging.getLogger(__name__)
LIBTYPE_MOVIE = 1
LIBTYPE_SHOW = 2
LIBTYPE_SEASON = 3
LIBTYPE_EPISODE = 4
def _int(v, default=0):
try:
return int(v)
except (TypeError, ValueError):
return default
def _opt_int(v):
try:
return int(v)
except (TypeError, ValueError):
return None
class PlexClient:
def __init__(self, base_url: str, token: str, *, timeout: int = 30,
verify_ssl: bool = True, page_size: int = 500,
request_delay_ms: int = 0, client_id: str = "mediashelf"):
if not base_url or not token:
raise ProviderError("Plex is not configured (PLEX_BASE_URL / PLEX_TOKEN)")
self.base_url = base_url.rstrip("/")
self.timeout = timeout
self.verify_ssl = verify_ssl
self.page_size = page_size
self.delay = request_delay_ms / 1000.0
self.session = requests.Session()
self.session.headers.update({
"X-Plex-Token": token,
"Accept": "application/json",
"X-Plex-Product": "MediaShelf",
"X-Plex-Client-Identifier": client_id,
})
def get(self, path: str, params: dict | None = None, *, headers: dict | None = None) -> dict:
url = self.base_url + path
last: Exception | None = None
for attempt in range(3):
try:
if self.delay:
time.sleep(self.delay)
r = self.session.get(url, params=params, headers=headers,
timeout=self.timeout, verify=self.verify_ssl)
if r.status_code in (401, 403):
raise AuthError("Plex rejected the token (HTTP %s)" % r.status_code)
r.raise_for_status()
return r.json().get("MediaContainer", {}) or {}
except AuthError:
raise # never retried: a retry loop will not fix a bad token
except Exception as e: # noqa: BLE001
last = e
if attempt < 2:
time.sleep(2 ** attempt)
raise ProviderError("Plex request failed: %s (%s)" % (path, last))
def paged(self, path: str, params: dict | None = None) -> Iterator[dict]:
"""Yield Metadata rows, paging with X-Plex-Container-* headers."""
start = 0
while True:
headers = {
"X-Plex-Container-Start": str(start),
"X-Plex-Container-Size": str(self.page_size),
}
mc = self.get(path, params, headers=headers)
batch = mc.get("Metadata") or []
if not batch:
return
for row in batch:
yield row
start += len(batch)
total = _int(mc.get("totalSize") or mc.get("size") or 0)
if start >= total:
return
class PlexProvider:
"""MediaProvider implementation."""
kind = "plex"
def __init__(self, client: PlexClient):
self.client = client
# ── identity ─────────────────────────────────────────────────────────
def server_info(self) -> ServerInfo:
mc = self.client.get("/identity")
return ServerInfo(
kind="plex",
name=mc.get("friendlyName") or "Plex",
version=mc.get("version") or "",
server_id=mc.get("machineIdentifier") or "",
base_url=self.client.base_url,
)
# ── libraries ────────────────────────────────────────────────────────
def libraries(self) -> list[Library]:
mc = self.client.get("/library/sections")
out = []
for d in mc.get("Directory") or []:
kind = d.get("type")
if kind not in ("movie", "show"):
continue # music/photo/other are out of scope (§1.2)
out.append(Library(
provider_key=str(d.get("key")),
title=(d.get("title") or "").strip(),
kind=kind,
locations=[loc.get("path") for loc in (d.get("Location") or [])
if loc.get("path")],
))
return out
# ── items ────────────────────────────────────────────────────────────
def items(self, library: Library) -> Iterator[Item]:
libtype = LIBTYPE_MOVIE if library.kind == "movie" else LIBTYPE_EPISODE
path = "/library/sections/%s/all" % library.provider_key
for row in self.client.paged(path, {"type": libtype}):
item = self._to_item(row, library)
if item is not None:
yield item
def show_guids(self, library: Library) -> dict[str, str]:
"""Map show ratingKey -> show guid.
Episodes carry grandparentRatingKey but not the show's guid, and keep
marks for seasons are keyed on the SHOW's guid (§6.6), so this mapping
has to be fetched separately one cheap request per show library.
"""
out: dict[str, str] = {}
path = "/library/sections/%s/all" % library.provider_key
for row in self.client.paged(path, {"type": LIBTYPE_SHOW}):
rk = row.get("ratingKey")
if rk is not None:
out[str(rk)] = row.get("guid") or ""
return out
@staticmethod
def _to_item(row: dict, library: Library) -> Item | None:
rk = row.get("ratingKey")
if rk is None:
return None
parts: list[Part] = []
resolution = video_codec = None
for media in row.get("Media") or []:
if resolution is None:
resolution = media.get("videoResolution")
video_codec = media.get("videoCodec")
for p in media.get("Part") or []:
if not p.get("file"):
continue
parts.append(Part(
file_path=p["file"],
size_bytes=_int(p.get("size")),
provider_part_id=str(p.get("id")) if p.get("id") is not None else None,
container=p.get("container") or media.get("container"),
resolution=media.get("videoResolution"),
video_codec=media.get("videoCodec"),
audio_codec=media.get("audioCodec"),
bitrate=_opt_int(media.get("bitrate")),
))
kind = "movie" if library.kind == "movie" else "episode"
return Item(
provider_item_id=str(rk),
kind=kind,
title=row.get("title") or "(untitled)",
library_key=library.provider_key,
guid=row.get("guid"),
sort_title=row.get("titleSort"),
year=_opt_int(row.get("year")),
added_at=_opt_int(row.get("addedAt")),
updated_at=_opt_int(row.get("updatedAt")),
duration_ms=_int(row.get("duration")),
view_count=_int(row.get("viewCount")),
last_viewed_at=_opt_int(row.get("lastViewedAt")),
resolution=resolution,
video_codec=video_codec,
parts=parts,
show_id=str(row["grandparentRatingKey"]) if row.get("grandparentRatingKey") is not None else None,
show_title=row.get("grandparentTitle"),
season_id=str(row["parentRatingKey"]) if row.get("parentRatingKey") is not None else None,
season_number=_opt_int(row.get("parentIndex")),
episode_number=_opt_int(row.get("index")),
)
def refresh_library(self, library: Library) -> None:
raise NotImplementedError("v1 never writes to Plex (§1.2)")
class PlexHistoryProvider:
"""Fallback HistoryProvider using /status/sessions/history/all.
Server-wide across accounts, which solves the token-scoping problem but it
records only that a play happened, never how far it got. Everything it
returns is therefore treated as a completed view and the 'rejection' score
component drops out of the weighting (§4.11).
"""
name = "plex"
def __init__(self, client: PlexClient):
self.client = client
@property
def has_completion_data(self) -> bool:
return False
def server_info(self) -> ServerInfo:
mc = self.client.get("/identity")
return ServerInfo(
kind="plex",
name=mc.get("friendlyName") or "Plex",
version=mc.get("version") or "",
server_id=mc.get("machineIdentifier") or "",
base_url=self.client.base_url,
)
def accounts(self) -> list[Account]:
try:
mc = self.client.get("/accounts")
except ProviderError:
return []
out = []
for a in mc.get("Account") or []:
if a.get("id") is None:
continue
out.append(Account(account_id=str(a["id"]), name=a.get("name")))
return out
def watch_events(self, since: int | None = None) -> Iterator[WatchEvent]:
params: dict = {"sort": "viewedAt:desc"}
for row in self.client.paged("/status/sessions/history/all", params):
viewed = _opt_int(row.get("viewedAt"))
rk = row.get("ratingKey")
if viewed is None or rk is None:
continue
if since is not None and viewed <= since:
return # sorted desc: the watermark ends the walk
yield WatchEvent(
source="plex",
source_row_id=str(row.get("historyKey") or f"{rk}:{viewed}"),
provider_item_id=str(rk),
viewed_at=viewed,
account_id=str(row["accountID"]) if row.get("accountID") is not None else None,
media_type=row.get("type"),
)
def coverage(self) -> Coverage:
earliest = latest = None
count = 0
for ev in self.watch_events():
count += 1
if latest is None or ev.viewed_at > latest:
latest = ev.viewed_at
if earliest is None or ev.viewed_at < earliest:
earliest = ev.viewed_at
return Coverage(earliest, latest, count)

View file

@ -0,0 +1,220 @@
"""Tautulli history provider — the primary watch-data source (§4.8).
Tautulli's database is independent of Plex's: clearing or pruning Plex history
does not touch it, and unlike Plex it records how far into an item each play
actually got. That last fact is the whole reason this is the primary source.
Two things this module deliberately does NOT do:
* It never calls get_library_media_info for a show section. Measured against
the live server, that returns show-level rows with file_size 0 regardless of
the section_type parameter, so it cannot supply TV sizes (§4.11). Plex is the
only size authority.
* It never trusts an HTTP 200 as success. Tautulli returns 200 with
result:"error" in the body as its normal failure mode.
"""
from __future__ import annotations
import logging
import time
from typing import Iterator
import requests
from .base import (
Account,
AuthError,
Coverage,
ProviderError,
ServerInfo,
WatchEvent,
)
log = logging.getLogger(__name__)
def _int(v, default=0):
try:
return int(v)
except (TypeError, ValueError):
return default
def _opt_int(v):
try:
return int(v)
except (TypeError, ValueError):
return None
class TautulliClient:
def __init__(self, base_url: str, api_key: str, *, timeout: int = 30,
page_size: int = 1000):
if not base_url or not api_key:
raise ProviderError("Tautulli is not configured")
self.base_url = base_url.rstrip("/") + "/api/v2"
self.api_key = api_key
self.timeout = timeout
self.page_size = page_size
self.session = requests.Session()
def cmd(self, command: str, **params):
params["apikey"] = self.api_key
params["cmd"] = command
last: Exception | None = None
for attempt in range(3):
try:
r = self.session.get(self.base_url, params=params, timeout=self.timeout)
if r.status_code in (401, 403):
raise AuthError("Tautulli rejected the API key (HTTP %s)" % r.status_code)
r.raise_for_status()
body = r.json()
resp = (body or {}).get("response") or {}
result = resp.get("result")
if result != "success":
msg = resp.get("message") or "unknown error"
# An invalid key surfaces here as a 200 with result:error.
if "apikey" in str(msg).lower() or "auth" in str(msg).lower():
raise AuthError("Tautulli: %s" % msg)
raise ProviderError("Tautulli cmd=%s failed: %s" % (command, msg))
return resp.get("data")
except (AuthError, ProviderError):
raise
except Exception as e: # noqa: BLE001
last = e
if attempt < 2:
time.sleep(2 ** attempt)
raise ProviderError("Tautulli request failed: cmd=%s (%s)" % (command, last))
class TautulliHistoryProvider:
"""HistoryProvider implementation."""
name = "tautulli"
def __init__(self, client: TautulliClient):
self.client = client
@property
def has_completion_data(self) -> bool:
return True
def server_info(self) -> ServerInfo:
d = self.client.cmd("get_server_info") or {}
return ServerInfo(
kind="tautulli",
name=d.get("pms_name") or "Tautulli",
version="",
server_id=d.get("pms_identifier") or "",
base_url=self.client.base_url,
)
def accounts(self) -> list[Account]:
data = self.client.cmd("get_users") or []
out = []
for u in data:
uid = u.get("user_id")
if uid is None:
continue
out.append(Account(
account_id=str(uid),
name=u.get("username"),
friendly_name=u.get("friendly_name") or u.get("username"),
))
return out
def _history_page(self, start: int, length: int, order_dir: str = "desc") -> tuple[list[dict], int]:
d = self.client.cmd(
"get_history",
grouping=0, # raw events; MediaShelf does its own merging (§4.9)
order_column="date",
order_dir=order_dir,
start=start,
length=length,
) or {}
rows = d.get("data") or []
total = _int(d.get("recordsFiltered") or d.get("recordsTotal") or 0)
return rows, total
def watch_events(self, since: int | None = None) -> Iterator[WatchEvent]:
"""Walk history newest-first, stopping at the watermark."""
start = 0
while True:
rows, total = self._history_page(start, self.client.page_size)
if not rows:
return
for row in rows:
viewed = _opt_int(row.get("date"))
rk = row.get("rating_key")
if viewed is None or rk in (None, ""):
continue
if since is not None and viewed <= since:
return
yield self._to_event(row, viewed, rk)
start += len(rows)
if start >= total:
return
@staticmethod
def _to_event(row: dict, viewed: int, rk) -> WatchEvent:
row_id = row.get("row_id")
return WatchEvent(
source="tautulli",
source_row_id=str(row_id if row_id is not None else f"{rk}:{viewed}"),
provider_item_id=str(rk),
viewed_at=viewed,
account_id=str(row["user_id"]) if row.get("user_id") is not None else None,
reference_id=str(row["reference_id"]) if row.get("reference_id") is not None else None,
stopped_at=_opt_int(row.get("stopped")),
play_duration_s=_opt_int(row.get("play_duration")),
paused_counter_s=_opt_int(row.get("paused_counter")),
percent_complete=_opt_int(row.get("percent_complete")),
watched_status=_float_or_none(row.get("watched_status")),
media_type=row.get("media_type"),
platform=row.get("platform"),
)
def coverage(self) -> Coverage:
"""Cheap: one row from each end plus the reported total."""
newest, total = self._history_page(0, 1, "desc")
oldest, _ = self._history_page(0, 1, "asc")
latest = _opt_int(newest[0].get("date")) if newest else None
earliest = _opt_int(oldest[0].get("date")) if oldest else None
return Coverage(earliest, latest, total)
# ── movie-only cross-check (§4.11) ───────────────────────────────────
def movie_media_info(self, section_id: str) -> list[dict]:
"""Per-item size/play data for a MOVIE section only.
Never call this for a show section: it returns show-level rows with
file_size 0 whatever section_type is passed.
"""
out: list[dict] = []
start = 0
while True:
d = self.client.cmd(
"get_library_media_info",
section_id=section_id,
start=start,
length=self.client.page_size,
order_column="file_size",
order_dir="desc",
) or {}
rows = d.get("data") or []
if not rows:
break
out.extend(rows)
total = _int(d.get("recordsFiltered") or d.get("recordsTotal") or 0)
start += len(rows)
if start >= total:
break
return out
def _float_or_none(v):
try:
return float(v)
except (TypeError, ValueError):
return None

358
mediashelf/queries.py Normal file
View file

@ -0,0 +1,358 @@
"""The item query: filters + rules + live score + paging + aggregates.
The score is computed in the SELECT rather than stored, because weights change on
every slider drag and storing it would mean rewriting thousands of rows per drag
(§6.4). At this row count the whole query runs in single-digit milliseconds.
"""
from __future__ import annotations
import json
import time
from . import rules as rules_mod
from . import scoring
def _score_ctx(db, cfg, has_completion_data: bool) -> scoring.ScoreContext:
maxsize = db.scalar(
"SELECT MAX(size_bytes) FROM media_item WHERE kind IN ('movie','season')"
) or 1
s = cfg.score
return scoring.ScoreContext(
now=int(time.time()),
max_size_bytes=maxsize,
stale_horizon_days=s.stale_horizon_days,
age_horizon_days=s.age_horizon_days,
popular_at=s.popular_at,
rejected_at=s.rejected_at,
solitude_at=s.solitude_at,
grace_days=s.grace_days,
recent_days=s.recent_days,
has_completion_data=has_completion_data,
)
def history_has_completion(db) -> bool:
"""True when the active history source records percent_complete (§4.11)."""
row = db.one(
"SELECT source FROM history_coverage ORDER BY event_count DESC LIMIT 1"
)
return bool(row and row["source"] == "tautulli")
BASE_COLUMNS = """
i.id, i.kind, i.title, i.sort_title, i.year, i.guid, i.show_guid,
i.library_id, lib.title AS library_title,
i.size_bytes, i.added_at, i.updated_at, i.duration_ms,
i.episode_count, i.part_count, i.primary_path, i.resolution, i.video_codec,
i.watch_count, i.partial_count, i.abandoned_count, i.avg_percent_complete,
i.last_watched_at, i.last_touched_at, i.first_watched_at,
i.distinct_watcher_count, i.pre_history, i.kept, i.kept_via, i.kept_mark_id,
i.provider_view_count, i.status, i.parent_id, i.season_number,
parent.title AS show_title
"""
FROM_CLAUSE = """
FROM media_item i
JOIN library lib ON lib.id = i.library_id
LEFT JOIN media_item parent ON parent.id = i.parent_id
LEFT JOIN (
SELECT guid, COUNT(*) AS dupe_count
FROM media_item WHERE kind = 'movie' AND guid IS NOT NULL AND status = 'present'
GROUP BY guid HAVING COUNT(*) > 1
) d ON d.guid = i.guid
CROSS JOIN (SELECT MAX(size_bytes) AS maxsize FROM media_item
WHERE kind IN ('movie','season')) s
"""
class Query:
def __init__(self, db, cfg):
self.db = db
self.cfg = cfg
self.has_cd = history_has_completion(db)
self.ctx = _score_ctx(db, cfg, self.has_cd)
def build(self, *, library_ids=None, kinds=None, q=None, rule_group=None,
include_missing=False, include_kept=False, include_shows=False,
weights=None, extra_where=None):
where = []
params: dict = {}
if not include_missing:
where.append("i.status = 'present'")
if not include_kept:
where.append("i.kept = 0")
if not include_shows:
# Shows are containers; seasons are the unit of analysis for TV (§5.3)
where.append("i.kind != 'show'")
if kinds:
keys = []
for n, k in enumerate(kinds):
key = "k%d" % n
params[key] = k
keys.append(":" + key)
where.append("i.kind IN (%s)" % ", ".join(keys))
if library_ids:
keys = []
for n, lid in enumerate(library_ids):
key = "lib%d" % n
params[key] = int(lid)
keys.append(":" + key)
where.append("i.library_id IN (%s)" % ", ".join(keys))
if q:
params["q"] = _fts_query(q)
where.append(
"i.id IN (SELECT rowid FROM media_item_fts WHERE media_item_fts MATCH :q)"
)
if rule_group:
frag, rp = rules_mod.compile_rules(rule_group, self.ctx.now)
if frag:
where.append(frag)
params.update(rp)
if extra_where:
where.append(extra_where)
expr, sp = scoring.sql_expression(self.ctx, weights)
params.update(sp)
params["now"] = self.ctx.now
where_sql = " AND ".join(where) if where else "1=1"
return expr, where_sql, params
def page(self, *, sort=None, page=1, page_size=100, **kw):
expr, where_sql, params = self.build(**kw)
order = rules_mod.compile_sort(sort)
total = self.db.scalar(
"SELECT COUNT(*) " + FROM_CLAUSE + " WHERE " + where_sql, params
) or 0
agg = self.db.one(
"SELECT COALESCE(SUM(i.size_bytes),0) AS total_size, COUNT(*) AS n "
+ FROM_CLAUSE + " WHERE " + where_sql, params
)
page = max(1, int(page))
page_size = max(1, min(int(page_size), 500))
params["_limit"] = page_size
params["_offset"] = (page - 1) * page_size
sql = (
"SELECT " + BASE_COLUMNS + ", " + expr + " AS reclaim_score, "
+ scoring.GRACE_SQL + " AS grace, COALESCE(d.dupe_count, 1) AS duplicate_count "
+ FROM_CLAUSE + " WHERE " + where_sql
+ " ORDER BY " + order + " LIMIT :_limit OFFSET :_offset"
)
rows = self.db.query(sql, params)
return {
"total": total,
"page": page,
"page_size": page_size,
"aggregate": {
"total_size_bytes": agg["total_size"] or 0,
"item_count": agg["n"] or 0,
},
"items": [self.serialize(r, weights=kw.get("weights")) for r in rows],
}
def iter_all(self, *, sort=None, **kw):
"""Stream every matching row, for CSV export."""
expr, where_sql, params = self.build(**kw)
order = rules_mod.compile_sort(sort)
sql = (
"SELECT " + BASE_COLUMNS + ", " + expr + " AS reclaim_score, "
+ scoring.GRACE_SQL + " AS grace, COALESCE(d.dupe_count, 1) AS duplicate_count "
+ FROM_CLAUSE + " WHERE " + where_sql + " ORDER BY " + order
)
for row in self.db.conn.execute(sql, params):
yield row
def serialize(self, row, weights=None) -> dict:
d = dict(row)
comps = scoring.components(d, self.ctx)
flags = []
if d.get("pre_history"):
flags.append("pre_history")
if (d.get("abandoned_count") or 0) >= self.ctx.rejected_at and not d.get("watch_count"):
flags.append("rejected")
if (d.get("duplicate_count") or 1) > 1:
flags.append("duplicate")
if (d.get("provider_view_count") or 0) > 0 and not d.get("watch_count"):
flags.append("history_gap")
if (d.get("part_count") or 0) > 1:
flags.append("multi_part")
return {
"id": d["id"],
"kind": d["kind"],
"title": d["title"],
"show_title": d.get("show_title"),
"season_number": d.get("season_number"),
"year": d.get("year"),
"guid": d.get("guid"),
"library": {"id": d["library_id"], "title": d["library_title"]},
"size_bytes": d.get("size_bytes") or 0,
"added_at": d.get("added_at"),
"last_watched_at": d.get("last_watched_at"),
"last_touched_at": d.get("last_touched_at"),
"watch_count": d.get("watch_count") or 0,
"partial_count": d.get("partial_count") or 0,
"abandoned_count": d.get("abandoned_count") or 0,
"avg_percent_complete": d.get("avg_percent_complete"),
"distinct_watcher_count": d.get("distinct_watcher_count") or 0,
"episode_count": d.get("episode_count") if d["kind"] == "season" else None,
"primary_path": d.get("primary_path"),
"part_count": d.get("part_count") or 0,
"resolution": d.get("resolution"),
"duplicate_count": d.get("duplicate_count") or 1,
"pre_history": bool(d.get("pre_history")),
"kept": bool(d.get("kept")),
"kept_via": d.get("kept_via"),
"status": d.get("status"),
"reclaim_score": d.get("reclaim_score"),
"reclaim_components": {k: (None if v is None else round(v, 4))
for k, v in comps.items()},
"grace": d.get("grace"),
"flags": flags,
}
def _fts_query(q: str) -> str:
"""Turn user text into a safe FTS5 prefix query.
FTS5 has its own operator syntax; passing raw user input straight through
lets a stray quote or NEAR() produce an error or a surprising match, so each
token is quoted and turned into a prefix term.
"""
tokens = [t for t in "".join(c if c.isalnum() else " " for c in q).split() if t]
if not tokens:
return '""'
return " ".join('"%s"*' % t for t in tokens[:10])
# ── aggregate stats for the dashboard ────────────────────────────────────
def overview(db, cfg) -> dict:
def s(sql, params=()):
return db.scalar(sql, params) or 0
unit = "kind IN ('movie','season') AND status='present'"
total_bytes = s(f"SELECT SUM(size_bytes) FROM media_item WHERE {unit}")
total_items = s(f"SELECT COUNT(*) FROM media_item WHERE {unit}")
never_bytes = s(f"SELECT SUM(size_bytes) FROM media_item WHERE {unit} AND watch_count=0")
kept_bytes = s(f"SELECT SUM(size_bytes) FROM media_item WHERE {unit} AND kept=1")
# The three-way split that keeps the keep list honest (§6.6)
never_kept = s(f"SELECT SUM(size_bytes) FROM media_item WHERE {unit} AND watch_count=0 AND kept=1")
confident = s(f"SELECT SUM(size_bytes) FROM media_item WHERE {unit} "
"AND watch_count=0 AND kept=0 AND pre_history=0")
uncertain = s(f"SELECT SUM(size_bytes) FROM media_item WHERE {unit} "
"AND watch_count=0 AND kept=0 AND pre_history=1")
cold = s(f"SELECT SUM(size_bytes) FROM media_item WHERE {unit} AND kept=0 AND "
"(last_watched_at IS NULL OR last_watched_at < strftime('%s','now') - 2*365*86400)")
cov = db.one("SELECT * FROM history_coverage ORDER BY event_count DESC LIMIT 1")
last_scan = db.one("SELECT * FROM scan WHERE status='succeeded' ORDER BY id DESC LIMIT 1")
return {
"total_bytes": total_bytes,
"total_items": total_items,
"never_played_bytes": never_bytes,
"kept_bytes": kept_bytes,
"never_played_kept_bytes": never_kept,
"available_bytes": max(never_bytes - never_kept, 0),
"confident_bytes": confident,
"uncertain_bytes": uncertain,
"cold_bytes": cold,
"libraries": s("SELECT COUNT(*) FROM library"),
"episodes": s("SELECT COUNT(*) FROM episode WHERE status='present'"),
"watch_events": s("SELECT COUNT(*) FROM watch_event"),
"accounts": s("SELECT COUNT(*) FROM account"),
"history_source": cov["source"] if cov else None,
"history_since": cov["earliest_event_at"] if cov else None,
"history_until": cov["latest_event_at"] if cov else None,
"has_completion_data": history_has_completion(db),
"last_scan_at": last_scan["finished_at"] if last_scan else None,
"keep_marks": s("SELECT COUNT(*) FROM keep_mark"),
}
def size_by_library(db) -> list[dict]:
return [dict(r) for r in db.query("""
SELECT lib.id, lib.title, lib.kind, lib.keep_all,
COUNT(*) AS items,
COALESCE(SUM(i.size_bytes),0) AS size_bytes,
COALESCE(SUM(CASE WHEN i.watch_count=0 THEN i.size_bytes ELSE 0 END),0) AS never_bytes,
COALESCE(SUM(CASE WHEN i.kept=1 THEN i.size_bytes ELSE 0 END),0) AS kept_bytes
FROM media_item i JOIN library lib ON lib.id = i.library_id
WHERE i.kind IN ('movie','season') AND i.status='present'
GROUP BY lib.id ORDER BY size_bytes DESC
""")]
def added_over_time(db, bucket="month") -> list[dict]:
fmt = "%Y-%m" if bucket == "month" else "%Y"
return [dict(r) for r in db.query(f"""
SELECT strftime('{fmt}', added_at, 'unixepoch') AS period,
COUNT(*) AS items, COALESCE(SUM(size_bytes),0) AS size_bytes
FROM media_item
WHERE kind IN ('movie','season') AND status='present'
AND added_at IS NOT NULL AND added_at > 0
GROUP BY period ORDER BY period
""")]
def completion_split(db) -> dict:
"""finished / abandoned / never-opened by size — invisible without Tautulli."""
row = db.one("""
SELECT
COALESCE(SUM(CASE WHEN watch_count > 0 THEN size_bytes ELSE 0 END),0) AS finished,
COALESCE(SUM(CASE WHEN watch_count = 0 AND (abandoned_count > 0 OR partial_count > 0)
THEN size_bytes ELSE 0 END),0) AS started,
COALESCE(SUM(CASE WHEN watch_count = 0 AND abandoned_count = 0 AND partial_count = 0
THEN size_bytes ELSE 0 END),0) AS never
FROM media_item WHERE kind IN ('movie','season') AND status='present'
""")
return dict(row)
def size_vs_lastwatched(db, limit=3000) -> list[dict]:
return [dict(r) for r in db.query("""
SELECT id, title, kind, size_bytes, last_watched_at, added_at, kept,
watch_count, pre_history
FROM media_item
WHERE kind IN ('movie','season') AND status='present' AND size_bytes > 0
ORDER BY size_bytes DESC LIMIT ?
""", (limit,))]
def duplicate_groups(db) -> list[dict]:
"""Same content held more than once — usually Movies vs 4K Movies (§6.5)."""
rows = db.query("""
SELECT i.guid, i.id, i.title, i.year, i.size_bytes, i.resolution,
i.watch_count, i.last_watched_at, i.kept, lib.title AS library_title
FROM media_item i JOIN library lib ON lib.id = i.library_id
WHERE i.kind='movie' AND i.status='present' AND i.guid IS NOT NULL
AND i.guid IN (SELECT guid FROM media_item WHERE kind='movie'
AND status='present' AND guid IS NOT NULL
GROUP BY guid HAVING COUNT(*) > 1)
ORDER BY i.guid, i.size_bytes DESC
""")
groups: dict[str, dict] = {}
for r in rows:
g = groups.setdefault(r["guid"], {"guid": r["guid"], "title": r["title"],
"year": r["year"], "copies": []})
g["copies"].append(dict(r))
out = []
for g in groups.values():
sizes = [c["size_bytes"] or 0 for c in g["copies"]]
g["total_bytes"] = sum(sizes)
g["redundant_bytes"] = sum(sizes) - max(sizes) if sizes else 0
out.append(g)
out.sort(key=lambda g: -g["redundant_bytes"])
return out

224
mediashelf/rules.py Normal file
View file

@ -0,0 +1,224 @@
"""Saved-view rule grammar → parameterized SQL (§7.1).
Everything here is whitelist-driven. Field names map to columns through a dict;
operators come from a fixed set; values are always bound parameters. There is no
string interpolation of user input anywhere in this module, and anything outside
the whitelist is a RuleError (surfaced as HTTP 400), never a best-effort guess.
"""
from __future__ import annotations
from typing import Any
DAY = 86400
class RuleError(ValueError):
"""Malformed or non-whitelisted rule. Always a 400, never a 500."""
# field name -> (sql column, type)
FIELDS: dict[str, tuple[str, str]] = {
"library_id": ("i.library_id", "int"),
"kind": ("i.kind", "str"),
"title": ("i.title", "str"),
"year": ("i.year", "int"),
"size_bytes": ("i.size_bytes", "int"),
"added_at": ("i.added_at", "ts"),
"updated_at": ("i.updated_at", "ts"),
"episode_count": ("i.episode_count", "int"),
"part_count": ("i.part_count", "int"),
"duration_ms": ("i.duration_ms", "int"),
"primary_path": ("i.primary_path", "str"),
"resolution": ("i.resolution", "str"),
"video_codec": ("i.video_codec", "str"),
"watch_count": ("i.watch_count", "int"),
"partial_count": ("i.partial_count", "int"),
"abandoned_count": ("i.abandoned_count", "int"),
"avg_percent_complete": ("i.avg_percent_complete", "float"),
"last_watched_at": ("i.last_watched_at", "ts"),
"last_touched_at": ("i.last_touched_at", "ts"),
"first_watched_at": ("i.first_watched_at", "ts"),
"distinct_watcher_count": ("i.distinct_watcher_count", "int"),
"pre_history": ("i.pre_history", "bool"),
"kept": ("i.kept", "bool"),
"kept_via": ("i.kept_via", "str"),
"status": ("i.status", "str"),
"guid": ("i.guid", "str"),
# derived, provided by the query builder as a correlated expression
"watch_ratio": ("(CASE WHEN i.episode_count > 0 "
"THEN CAST(i.watch_count AS REAL) / i.episode_count "
"ELSE CAST(i.watch_count AS REAL) END)", "float"),
"duplicate_count": ("COALESCE(d.dupe_count, 1)", "int"),
}
SIMPLE_OPS = {
"eq": "=", "ne": "!=", "lt": "<", "lte": "<=", "gt": ">", "gte": ">=",
}
LIST_OPS = {"in": "IN", "not_in": "NOT IN"}
LIKE_OPS = {"contains": "%{}%", "starts_with": "{}%", "ends_with": "%{}"}
NULL_OPS = {"is_null": "IS NULL", "is_not_null": "IS NOT NULL"}
REL_OPS = {"older_than_days", "newer_than_days"}
SPECIAL_OPS = {"never"}
ALL_OPS = (set(SIMPLE_OPS) | set(LIST_OPS) | set(LIKE_OPS)
| set(NULL_OPS) | REL_OPS | SPECIAL_OPS)
class _Builder:
def __init__(self, now: int):
self.now = now
self.params: dict[str, Any] = {}
self._n = 0
def bind(self, value) -> str:
self._n += 1
key = "r%d" % self._n
self.params[key] = value
return ":" + key
def coerce(self, value, ftype: str):
if ftype == "int":
try:
return int(value)
except (TypeError, ValueError):
raise RuleError("expected an integer, got %r" % (value,))
if ftype == "float":
try:
return float(value)
except (TypeError, ValueError):
raise RuleError("expected a number, got %r" % (value,))
if ftype == "bool":
if isinstance(value, bool):
return 1 if value else 0
if str(value).lower() in ("1", "true", "yes"):
return 1
if str(value).lower() in ("0", "false", "no"):
return 0
raise RuleError("expected a boolean, got %r" % (value,))
if ftype == "ts":
try:
return int(value)
except (TypeError, ValueError):
raise RuleError("expected a timestamp, got %r" % (value,))
return str(value)
def group(self, node: dict, depth: int = 0) -> str:
if depth > 8:
raise RuleError("rule nesting too deep")
if not isinstance(node, dict):
raise RuleError("rule node must be an object")
if "rules" in node:
op = str(node.get("op", "and")).lower()
if op not in ("and", "or"):
raise RuleError("group operator must be 'and' or 'or'")
rules = node.get("rules") or []
if not isinstance(rules, list):
raise RuleError("'rules' must be a list")
parts = [self.group(r, depth + 1) for r in rules]
parts = [p for p in parts if p]
if not parts:
return ""
joiner = " AND " if op == "and" else " OR "
return "(" + joiner.join(parts) + ")"
return self.condition(node)
def condition(self, node: dict) -> str:
field = node.get("field")
op = str(node.get("op", "")).lower()
if field not in FIELDS:
raise RuleError("unknown field %r" % (field,))
if op not in ALL_OPS:
raise RuleError("unknown operator %r" % (op,))
col, ftype = FIELDS[field]
value = node.get("value")
if op in NULL_OPS:
return "%s %s" % (col, NULL_OPS[op])
if op == "never":
# "never watched" is null-or-zero, which is not the same as IS NULL
return "(%s IS NULL OR %s = 0)" % (col, col)
if op in REL_OPS:
days = self.coerce(value, "int")
cutoff = self.bind(self.now - days * DAY)
if op == "older_than_days":
return "(%s IS NOT NULL AND %s > 0 AND %s < %s)" % (col, col, col, cutoff)
return "(%s IS NOT NULL AND %s >= %s)" % (col, col, cutoff)
if op in LIST_OPS:
if not isinstance(value, (list, tuple)) or not value:
raise RuleError("operator %r needs a non-empty list" % op)
if len(value) > 500:
raise RuleError("list too long")
placeholders = ", ".join(self.bind(self.coerce(v, ftype)) for v in value)
return "%s %s (%s)" % (col, LIST_OPS[op], placeholders)
if op in LIKE_OPS:
if ftype != "str":
raise RuleError("operator %r only applies to text fields" % op)
pattern = LIKE_OPS[op].format(_escape_like(str(value)))
return "%s LIKE %s ESCAPE '\\'" % (col, self.bind(pattern))
return "%s %s %s" % (col, SIMPLE_OPS[op], self.bind(self.coerce(value, ftype)))
def _escape_like(s: str) -> str:
return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
def compile_rules(rules: dict | None, now: int) -> tuple[str, dict]:
"""Compile a rule group to (sql_fragment, params). Empty rules -> ('', {})."""
if not rules:
return "", {}
b = _Builder(now)
sql = b.group(rules)
return sql, b.params
SORTABLE = {
"title": "i.sort_title, i.title",
"year": "i.year",
"size_bytes": "i.size_bytes",
"added_at": "i.added_at",
"updated_at": "i.updated_at",
"last_watched_at": "i.last_watched_at",
"last_touched_at": "i.last_touched_at",
"watch_count": "i.watch_count",
"partial_count": "i.partial_count",
"abandoned_count": "i.abandoned_count",
"distinct_watcher_count": "i.distinct_watcher_count",
"episode_count": "i.episode_count",
"avg_percent_complete": "i.avg_percent_complete",
"reclaim_score": "reclaim_score",
"library": "lib.title",
"kind": "i.kind",
}
def compile_sort(spec: str | None) -> str:
"""'reclaim_score:desc,size_bytes:desc' -> ORDER BY clause. Whitelisted."""
if not spec:
return "reclaim_score DESC, i.size_bytes DESC"
out = []
for part in str(spec).split(","):
part = part.strip()
if not part:
continue
if ":" in part:
name, direction = part.split(":", 1)
else:
name, direction = part, "asc"
name = name.strip()
direction = "DESC" if direction.strip().lower() == "desc" else "ASC"
if name not in SORTABLE:
raise RuleError("cannot sort by %r" % name)
out.append("%s %s" % (SORTABLE[name], direction))
if not out:
return "reclaim_score DESC, i.size_bytes DESC"
out.append("i.id ASC") # stable tiebreak so paging never repeats a row
return ", ".join(out)

152
mediashelf/scanner.py Normal file
View file

@ -0,0 +1,152 @@
"""Background scan runner and the scheduler.
The scheduler runs in-process. With gunicorn --workers 2 both workers would
start one and two scans would race for the SQLite write lock, so the scheduler
only starts in the worker that wins an exclusive flock on /data. The scan_lock
table is the second line of defence; both are needed, since either alone leaves
a race window (§11.4).
"""
from __future__ import annotations
import json
import logging
import os
import threading
import time
log = logging.getLogger(__name__)
_scan_thread: threading.Thread | None = None
_scan_lock = threading.Lock()
_scheduler = None
_flock_handle = None
def _run_scan(app, mode: str, trigger: str) -> None:
from . import ingest, keeps, providers
with app.app_context():
ext = app.extensions["mediashelf"]
cfg = ext["config"]
db = ext["db"]
try:
media = providers.build_media_provider(cfg)
history, degraded = providers.build_history_provider(cfg, media)
if degraded:
log.warning("history running degraded: %s", degraded)
ext["degraded_reason"] = degraded
else:
ext["degraded_reason"] = None
result = ingest.Ingest(db, cfg, media, history).run(mode, trigger)
log.info("scan %s %s: seen=%d added=%d events=%d",
result.scan_id, result.status, result.items_seen,
result.items_added, result.events_added)
if result.status == "succeeded":
_export_backups(db, cfg)
except Exception: # noqa: BLE001
log.exception("background scan failed")
finally:
db.close()
def _export_backups(db, cfg) -> None:
"""Keeps and saved views are the only data not reconstructible (§11.5)."""
from . import keeps
try:
data_dir = os.path.dirname(os.path.abspath(cfg.database_path)) or "."
keeps.write_export_file(db, os.path.join(data_dir, "keeps.json"))
views = [dict(r) for r in db.query("SELECT * FROM saved_view WHERE builtin = 0")]
with open(os.path.join(data_dir, "views.json"), "w") as fh:
json.dump({"version": 1, "exported_at": int(time.time()), "views": views},
fh, indent=2)
except OSError as e:
log.warning("could not write keep/view export: %s", e)
def start_background_scan(app, mode: str = "incremental", trigger: str = "manual") -> bool:
"""Returns False if a scan is already running in this process."""
global _scan_thread
with _scan_lock:
if _scan_thread is not None and _scan_thread.is_alive():
return False
real_app = app._get_current_object() if hasattr(app, "_get_current_object") else app
_scan_thread = threading.Thread(
target=_run_scan, args=(real_app, mode, trigger),
name="mediashelf-scan", daemon=True)
_scan_thread.start()
return True
def scan_running() -> bool:
return _scan_thread is not None and _scan_thread.is_alive()
# ── scheduler ────────────────────────────────────────────────────────────
def _acquire_process_lock(data_dir: str) -> bool:
"""Exclusive flock, so only one gunicorn worker schedules anything."""
global _flock_handle
import fcntl
try:
os.makedirs(data_dir, exist_ok=True)
fh = open(os.path.join(data_dir, ".scheduler.lock"), "w")
fcntl.flock(fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
fh.write(str(os.getpid()))
fh.flush()
_flock_handle = fh # held for the life of the process
return True
except (OSError, BlockingIOError):
return False
def _cron_kwargs(expr: str) -> dict:
parts = (expr or "").split()
if len(parts) != 5:
raise ValueError("cron expression must have 5 fields, got %r" % expr)
minute, hour, dom, month, dow = parts
return {"minute": minute, "hour": hour, "day": dom,
"month": month, "day_of_week": dow}
def start_scheduler(app) -> bool:
"""Start the nightly jobs in exactly one process. Returns True if started."""
global _scheduler
cfg = app.extensions["mediashelf"]["config"]
if not cfg.scheduler_enabled:
log.info("scheduler disabled by config")
return False
data_dir = os.path.dirname(os.path.abspath(cfg.database_path)) or "."
if not _acquire_process_lock(data_dir):
log.info("another worker holds the scheduler lock; not scheduling here")
return False
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
real_app = app._get_current_object() if hasattr(app, "_get_current_object") else app
_scheduler = BackgroundScheduler(timezone=cfg.tz)
try:
_scheduler.add_job(
lambda: start_background_scan(real_app, "incremental", "schedule"),
CronTrigger(**_cron_kwargs(cfg.scan_schedule_cron), timezone=cfg.tz),
id="incremental", replace_existing=True, max_instances=1)
_scheduler.add_job(
lambda: start_background_scan(real_app, "full", "schedule"),
CronTrigger(**_cron_kwargs(cfg.scan_full_sweep_cron), timezone=cfg.tz),
id="full-sweep", replace_existing=True, max_instances=1)
except ValueError as e:
log.error("bad cron configuration, scheduler not started: %s", e)
return False
_scheduler.start()
log.info("scheduler started (incremental %r, full sweep %r, tz %s)",
cfg.scan_schedule_cron, cfg.scan_full_sweep_cron, cfg.tz)
if cfg.scan_on_startup:
start_background_scan(real_app, "full", "startup")
return True

295
mediashelf/scoring.py Normal file
View file

@ -0,0 +1,295 @@
"""The reclaim score (§6).
Implemented twice on purpose: once as a SQL expression, because weights change
on every slider drag and storing the score would mean recomputing 5,700 rows per
drag; and once in Python, for CSV export and for tests. Two implementations of
one formula is a real risk, so test_scoring.py asserts they agree to within 0.01
on a generated corpus.
Components are each normalized to [0, 1] where 1 means "better deletion
candidate". The weighted mean uses only the components AVAILABLE for a row, so a
missing component renormalizes instead of dragging every score down.
"""
from __future__ import annotations
import math
from dataclasses import dataclass
DAY = 86400.0
COMPONENTS = ("size", "staleness", "unpopularity", "solitude", "age", "rejection")
DEFAULT_WEIGHTS = {
"size": 0.28,
"staleness": 0.24,
"unpopularity": 0.22,
"solitude": 0.10,
"age": 0.10,
"rejection": 0.06,
}
@dataclass
class ScoreContext:
"""Per-request constants. max_size is computed once in a CTE, not per row."""
now: int
max_size_bytes: int
stale_horizon_days: int = 730
age_horizon_days: int = 1095
popular_at: int = 3
rejected_at: int = 2
solitude_at: int = 3
grace_days: int = 30
recent_days: int = 90
has_completion_data: bool = True
def normalize_weights(weights: dict | None, *, has_completion_data: bool = True) -> dict:
w = dict(DEFAULT_WEIGHTS)
if weights:
for k, v in weights.items():
if k in DEFAULT_WEIGHTS:
try:
w[k] = max(0.0, float(v))
except (TypeError, ValueError):
pass
if not has_completion_data:
# No percent_complete anywhere => rejection is not merely zero, it is
# unavailable, and must not dilute the remaining components (§4.11).
w["rejection"] = 0.0
if sum(w.values()) <= 0:
w = dict(DEFAULT_WEIGHTS)
return w
# ─────────────────────────────── python ──────────────────────────────────
def components(row: dict, ctx: ScoreContext) -> dict[str, float | None]:
"""Component values for one row. None means 'not available for this row'."""
size = int(row.get("size_bytes") or 0)
added_at = row.get("added_at")
last_watched = row.get("last_watched_at")
watch_count = int(row.get("watch_count") or 0)
abandoned = int(row.get("abandoned_count") or 0)
watchers = int(row.get("distinct_watcher_count") or 0)
episodes = int(row.get("episode_count") or 0)
kind = row.get("kind") or "movie"
pre_history = bool(row.get("pre_history"))
# size — log scale: 2GB vs 4GB matters more than 60GB vs 62GB
max_size = max(int(ctx.max_size_bytes or 0), 1)
c_size = math.log10(1 + size) / math.log10(1 + max_size) if size > 0 else 0.0
c_size = min(max(c_size, 0.0), 1.0)
# staleness — never watched scores 1.0
if last_watched:
days = max(0.0, (ctx.now - last_watched) / DAY)
c_stale = min(days / max(ctx.stale_horizon_days, 1), 1.0)
else:
c_stale = 1.0
if pre_history and not last_watched:
# Might have been watched before history coverage began; capping keeps a
# 2009 film that was watched in 2015 from scoring as never-watched (§4.11)
c_stale = min(c_stale, 0.75)
# unpopularity — TV normalized per-episode so it compares to a movie
if kind == "season" and episodes > 0:
normalized_watches = watch_count / episodes
else:
normalized_watches = float(watch_count)
c_unpop = 1.0 - min(normalized_watches / max(ctx.popular_at, 1), 1.0)
# age
if added_at:
days = max(0.0, (ctx.now - added_at) / DAY)
c_age = min(days / max(ctx.age_horizon_days, 1), 1.0)
else:
c_age = 0.0
# solitude
c_sol = 1.0 - min(watchers / max(ctx.solitude_at, 1), 1.0)
# rejection — zeroed the moment anyone completes a view
if not ctx.has_completion_data:
c_rej = None
elif watch_count > 0:
c_rej = 0.0
else:
c_rej = min(abandoned / max(ctx.rejected_at, 1), 1.0)
return {
"size": c_size,
"staleness": c_stale,
"unpopularity": c_unpop,
"solitude": c_sol,
"age": c_age,
"rejection": c_rej,
}
def round_half_up(x: float, places: int = 2) -> float:
"""Match SQLite's ROUND(), which rounds half away from zero.
Python's built-in round() uses banker's rounding, so the two disagree at
exactly x.xx5 the SQL grid would show 53.13 where the CSV export showed
53.12. Harmless, but the kind of inconsistency that costs an afternoon when
someone notices the two disagree and assumes the formula differs.
"""
factor = 10 ** places
return math.floor(abs(x) * factor + 0.5) / factor * (1 if x >= 0 else -1)
def score_row(row: dict, ctx: ScoreContext, weights: dict | None = None) -> dict:
"""Returns {score, score_raw, components, grace}.
score_raw is unrounded tests compare it against the SQL expression so a
real formula divergence is not masked by rounding, and vice versa.
"""
w = normalize_weights(weights, has_completion_data=ctx.has_completion_data)
comps = components(row, ctx)
num = 0.0
den = 0.0
for name in COMPONENTS:
value = comps.get(name)
weight = w.get(name, 0.0)
if value is None or weight <= 0:
continue
num += weight * value
den += weight
score = 100.0 * (num / den) if den > 0 else 0.0
grace = None
now = ctx.now
added_at = row.get("added_at")
last_watched = row.get("last_watched_at")
if added_at and (now - added_at) < ctx.grace_days * DAY:
score = 0.0
grace = "new"
elif last_watched and (now - last_watched) < ctx.recent_days * DAY:
score = min(score, 25.0)
grace = "recent"
return {
"score": round_half_up(score, 2),
"score_raw": score,
"components": {k: (None if v is None else round(v, 4)) for k, v in comps.items()},
"grace": grace,
}
# ──────────────────────────────── sql ────────────────────────────────────
def sql_expression(ctx: ScoreContext, weights: dict | None = None,
*, rounded: bool = True) -> tuple[str, dict]:
"""Return (expression, params) computing the score for media_item rows.
Expects a CTE or join providing `maxsize` as the library-wide max size.
Mirrors components()/score_row() exactly see test_scoring.py. Pass
rounded=False to compare the raw formula without rounding in the way.
"""
w = normalize_weights(weights, has_completion_data=ctx.has_completion_data)
p = {
"now": ctx.now,
"stale_h": max(ctx.stale_horizon_days, 1),
"age_h": max(ctx.age_horizon_days, 1),
"popular_at": max(ctx.popular_at, 1),
"rejected_at": max(ctx.rejected_at, 1),
"solitude_at": max(ctx.solitude_at, 1),
"grace_s": ctx.grace_days * 86400,
"recent_s": ctx.recent_days * 86400,
"w_size": w["size"],
"w_stale": w["staleness"],
"w_unpop": w["unpopularity"],
"w_sol": w["solitude"],
"w_age": w["age"],
"w_rej": w["rejection"],
}
c_size = (
"MIN(MAX(CASE WHEN i.size_bytes > 0 THEN "
" (LOG(1 + i.size_bytes) / LOG(1 + MAX(s.maxsize, 1))) ELSE 0.0 END, 0.0), 1.0)"
)
c_stale_raw = (
"CASE WHEN i.last_watched_at IS NOT NULL AND i.last_watched_at > 0"
" THEN MIN(MAX(CAST(:now - i.last_watched_at AS REAL) / 86400.0, 0.0)"
" / :stale_h, 1.0)"
" ELSE 1.0 END"
)
# pre_history cap applies only when there is no recorded watch at all
c_stale = (
"CASE WHEN i.pre_history = 1 AND (i.last_watched_at IS NULL OR i.last_watched_at = 0)"
" THEN MIN(%s, 0.75) ELSE %s END" % (c_stale_raw, c_stale_raw)
)
normalized_watches = (
"CASE WHEN i.kind = 'season' AND i.episode_count > 0"
" THEN CAST(i.watch_count AS REAL) / i.episode_count"
" ELSE CAST(i.watch_count AS REAL) END"
)
c_unpop = "(1.0 - MIN((%s) / :popular_at, 1.0))" % normalized_watches
c_age = (
"CASE WHEN i.added_at IS NOT NULL AND i.added_at > 0"
" THEN MIN(MAX(CAST(:now - i.added_at AS REAL) / 86400.0, 0.0) / :age_h, 1.0)"
" ELSE 0.0 END"
)
c_sol = "(1.0 - MIN(CAST(i.distinct_watcher_count AS REAL) / :solitude_at, 1.0))"
if ctx.has_completion_data:
c_rej = (
"CASE WHEN i.watch_count > 0 THEN 0.0"
" ELSE MIN(CAST(i.abandoned_count AS REAL) / :rejected_at, 1.0) END"
)
rej_num = ":w_rej * (%s)" % c_rej
rej_den = ":w_rej"
else:
rej_num = "0.0"
rej_den = "0.0"
numerator = (
f":w_size * ({c_size}) + :w_stale * ({c_stale}) + :w_unpop * ({c_unpop})"
f" + :w_sol * ({c_sol}) + :w_age * ({c_age}) + {rej_num}"
)
denominator = f"(:w_size + :w_stale + :w_unpop + :w_sol + :w_age + {rej_den})"
raw = f"CASE WHEN {denominator} > 0 THEN 100.0 * ({numerator}) / {denominator} ELSE 0.0 END"
graced = (
"CASE"
" WHEN i.added_at IS NOT NULL AND i.added_at > 0"
" AND (:now - i.added_at) < :grace_s THEN 0.0"
" WHEN i.last_watched_at IS NOT NULL AND i.last_watched_at > 0"
" AND (:now - i.last_watched_at) < :recent_s THEN MIN(%s, 25.0)"
" ELSE %s END" % (raw, raw)
)
return ("ROUND(%s, 2)" % graced if rounded else graced), p
GRACE_SQL = (
"CASE"
" WHEN i.added_at IS NOT NULL AND i.added_at > 0"
" AND (:now - i.added_at) < :grace_s THEN 'new'"
" WHEN i.last_watched_at IS NOT NULL AND i.last_watched_at > 0"
" AND (:now - i.last_watched_at) < :recent_s THEN 'recent'"
" ELSE NULL END"
)
def register_sqlite_functions(conn) -> None:
"""SQLite has no LOG() by default; add it (and a MIN/MAX-safe guard)."""
def _log10(x):
try:
x = float(x)
except (TypeError, ValueError):
return None
return math.log10(x) if x > 0 else 0.0
conn.create_function("LOG", 1, _log10, deterministic=True)

194
mediashelf/static/app.css Normal file
View file

@ -0,0 +1,194 @@
:root {
--bg: #14161a;
--panel: #1b1e24;
--panel-2: #22262e;
--line: #2e333c;
--text: #e6e9ef;
--dim: #99a1b0;
--faint: #6b7280;
--accent: #6ea8fe;
--warm: #f0b357;
--hot: #e8705f;
--good: #5dc98a;
--kept: #b48ce8;
--radius: 8px;
}
@media (prefers-color-scheme: light) {
:root {
--bg: #f6f7f9; --panel: #ffffff; --panel-2: #f0f2f5; --line: #dfe3e9;
--text: #1a1d23; --dim: #5b6472; --faint: #8b93a1; --accent: #2f6fd0;
}
}
* { box-sizing: border-box; }
html, body { margin: 0; padding: 0; }
body {
background: var(--bg); color: var(--text);
font: 14px/1.5 ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
}
h2, h3 { margin: 0 0 .5rem; font-weight: 600; }
h3 { font-size: .95rem; }
a { color: var(--accent); }
/* ── topbar ─────────────────────────────────────────────────────── */
.topbar {
display: flex; align-items: center; gap: 1.5rem;
padding: .6rem 1rem; background: var(--panel);
border-bottom: 1px solid var(--line); position: sticky; top: 0; z-index: 20;
}
.brand { display: flex; align-items: center; gap: .5rem; font-weight: 650; }
.logo { color: var(--accent); font-size: 1.2rem; }
.readonly-badge {
font-size: .68rem; font-weight: 600; letter-spacing: .04em; text-transform: uppercase;
color: var(--warm); border: 1px solid var(--warm); border-radius: 999px;
padding: .1rem .5rem; opacity: .9;
}
.tabs { display: flex; gap: .15rem; flex: 1; }
.tab {
background: none; border: 0; color: var(--dim); padding: .4rem .75rem;
border-radius: var(--radius); cursor: pointer; font: inherit;
}
.tab:hover { background: var(--panel-2); color: var(--text); }
.tab.active { background: var(--panel-2); color: var(--text); font-weight: 600; }
.topbar-right { display: flex; align-items: center; gap: .6rem; }
.source-badge { font-size: .78rem; color: var(--dim); }
/* ── buttons ────────────────────────────────────────────────────── */
.btn {
background: var(--panel-2); color: var(--text); border: 1px solid var(--line);
border-radius: var(--radius); padding: .38rem .75rem; cursor: pointer; font: inherit;
}
.btn:hover:not(:disabled) { border-color: var(--accent); }
.btn:disabled { opacity: .4; cursor: not-allowed; }
.btn.small { padding: .22rem .5rem; font-size: .82rem; }
.btn.ghost { background: none; }
.btn.danger:not(:disabled) { color: var(--hot); border-color: var(--hot); }
/* ── layout ─────────────────────────────────────────────────────── */
main { padding: 1rem; max-width: 1800px; margin: 0 auto; }
.view { display: none; }
.view.active { display: block; }
.card {
background: var(--panel); border: 1px solid var(--line);
border-radius: var(--radius); padding: .9rem; margin-bottom: 1rem;
}
.hint { color: var(--dim); font-size: .82rem; margin: .1rem 0 .7rem; }
/* ── dashboard ──────────────────────────────────────────────────── */
.split-hero {
display: grid; grid-template-columns: repeat(3, 1fr); gap: 1px;
background: var(--line); border: 1px solid var(--line);
border-radius: var(--radius); overflow: hidden; margin-bottom: 1rem;
}
.split-cell { background: var(--panel); padding: 1rem 1.1rem; }
.split-cell .label {
font-size: .7rem; letter-spacing: .07em; text-transform: uppercase; color: var(--faint);
}
.split-cell .value { font-size: 1.9rem; font-weight: 650; margin-top: .2rem; }
.split-cell .sub { font-size: .78rem; color: var(--dim); }
.split-cell.kept .value { color: var(--kept); }
.split-cell.available .value { color: var(--good); }
.tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: .6rem; margin-bottom: 1rem; }
.tile { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); padding: .7rem .8rem; }
.tile .label { font-size: .7rem; letter-spacing: .06em; text-transform: uppercase; color: var(--faint); }
.tile .value { font-size: 1.2rem; font-weight: 600; margin-top: .15rem; }
.tile .sub { font-size: .75rem; color: var(--dim); }
.chart-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; }
.card.wide { grid-column: 1 / -1; }
.chart { min-height: 40px; }
.bar-row { display: grid; grid-template-columns: 150px 1fr 90px; gap: .5rem; align-items: center; margin-bottom: .3rem; font-size: .82rem; }
.bar-track { background: var(--panel-2); border-radius: 3px; height: 15px; overflow: hidden; display: flex; }
.bar-fill { height: 100%; background: var(--accent); }
.bar-fill.never { background: var(--warm); }
.bar-fill.kept { background: var(--kept); }
.bar-num { text-align: right; color: var(--dim); font-variant-numeric: tabular-nums; }
.bar-label { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.stack { display: flex; height: 30px; border-radius: 4px; overflow: hidden; margin-bottom: .5rem; }
.stack > div { display: flex; align-items: center; justify-content: center; font-size: .72rem; color: #10131a; font-weight: 600; }
.legend { display: flex; gap: 1rem; font-size: .8rem; color: var(--dim); flex-wrap: wrap; }
.legend i { display: inline-block; width: 10px; height: 10px; border-radius: 2px; margin-right: .3rem; }
/* ── grid ───────────────────────────────────────────────────────── */
.grid-layout { display: grid; grid-template-columns: 250px 1fr; gap: 1rem; align-items: start; }
.filter-rail {
background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius);
padding: .8rem; position: sticky; top: 60px; max-height: calc(100vh - 76px); overflow: auto;
}
.rail-section { margin-bottom: .9rem; }
.rail-label { display: block; font-size: .7rem; letter-spacing: .06em; text-transform: uppercase; color: var(--faint); margin-bottom: .3rem; }
.filter-rail input[type=search], .filter-rail select {
width: 100%; background: var(--panel-2); color: var(--text);
border: 1px solid var(--line); border-radius: 6px; padding: .32rem .45rem; font: inherit;
}
.checklist { max-height: 190px; overflow: auto; font-size: .84rem; }
.checklist label, .switch { display: flex; align-items: center; gap: .4rem; padding: .1rem 0; cursor: pointer; }
.weights label { display: block; font-size: .78rem; color: var(--dim); margin-top: .35rem; }
.weights input[type=range] { width: 100%; }
.grid-main { min-width: 0; }
.grid-toolbar { display: flex; justify-content: space-between; align-items: center; margin-bottom: .5rem; gap: 1rem; flex-wrap: wrap; }
.grid-summary { font-size: .85rem; color: var(--dim); }
.grid-summary b { color: var(--text); }
.grid-actions { display: flex; gap: .4rem; }
.table-wrap { overflow-x: auto; background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); }
table { border-collapse: collapse; width: 100%; font-size: .84rem; }
th, td { padding: .38rem .55rem; text-align: left; white-space: nowrap; border-bottom: 1px solid var(--line); }
th { position: sticky; top: 0; background: var(--panel-2); font-weight: 600; font-size: .76rem; letter-spacing: .03em; text-transform: uppercase; color: var(--dim); cursor: pointer; user-select: none; z-index: 1; }
th.sorted { color: var(--accent); }
tbody tr:hover { background: var(--panel-2); }
tbody tr.kept-row { opacity: .72; }
td.num { text-align: right; font-variant-numeric: tabular-nums; }
td.title-cell { max-width: 380px; overflow: hidden; text-overflow: ellipsis; }
.title-link { cursor: pointer; }
.title-link:hover { text-decoration: underline; }
.sub-title { color: var(--faint); font-size: .78rem; }
.score-pill { display: inline-block; min-width: 44px; text-align: center; padding: .08rem .35rem; border-radius: 4px; font-variant-numeric: tabular-nums; font-weight: 600; }
.flag { font-size: .66rem; padding: .04rem .28rem; border-radius: 3px; margin-left: .25rem; border: 1px solid; }
.flag.pre_history { color: var(--warm); border-color: var(--warm); }
.flag.rejected { color: var(--hot); border-color: var(--hot); }
.flag.duplicate { color: var(--accent); border-color: var(--accent); }
.flag.kept { color: var(--kept); border-color: var(--kept); }
.flag.multi_part, .flag.history_gap { color: var(--faint); border-color: var(--faint); }
.pager { display: flex; gap: .6rem; align-items: center; justify-content: center; padding: .7rem; color: var(--dim); font-size: .85rem; }
/* ── drawer ─────────────────────────────────────────────────────── */
.scrim { position: fixed; inset: 0; background: rgba(0,0,0,.45); z-index: 40; }
.drawer {
position: fixed; top: 0; right: 0; bottom: 0; width: min(680px, 92vw);
background: var(--panel); border-left: 1px solid var(--line);
z-index: 50; overflow: auto; padding: 1rem;
}
.drawer-head { display: flex; justify-content: space-between; align-items: flex-start; gap: 1rem; margin-bottom: .8rem; }
.kv { display: grid; grid-template-columns: 190px 1fr; gap: .2rem .8rem; font-size: .85rem; margin-bottom: 1rem; }
.kv dt { color: var(--dim); }
.kv dd { margin: 0; word-break: break-all; }
.comp-row { display: grid; grid-template-columns: 110px 1fr 46px; gap: .5rem; align-items: center; font-size: .8rem; margin-bottom: .2rem; }
.comp-track { background: var(--panel-2); height: 8px; border-radius: 3px; overflow: hidden; }
.comp-fill { height: 100%; background: var(--accent); }
/* ── misc ───────────────────────────────────────────────────────── */
.banner { padding: .55rem 1rem; font-size: .85rem; border-bottom: 1px solid; }
.banner.warn { background: rgba(240,179,87,.12); border-color: var(--warm); color: var(--warm); }
.banner.info { background: rgba(110,168,254,.1); border-color: var(--accent); color: var(--accent); }
.lib-rules label { display: flex; align-items: center; gap: .5rem; padding: .25rem 0; font-size: .87rem; }
.mark { display: grid; grid-template-columns: 1fr auto; gap: .5rem; padding: .5rem 0; border-bottom: 1px solid var(--line); align-items: center; }
.mark .meta { font-size: .78rem; color: var(--dim); }
.empty { color: var(--faint); font-size: .87rem; padding: .6rem 0; }
.dupe-group { border-bottom: 1px solid var(--line); padding: .55rem 0; }
.dupe-copies { display: flex; gap: 1rem; flex-wrap: wrap; font-size: .82rem; color: var(--dim); margin-top: .2rem; }
.scatter { width: 100%; height: 260px; }
.scatter circle { cursor: pointer; }
@media (max-width: 1000px) {
.chart-grid { grid-template-columns: 1fr; }
.grid-layout { grid-template-columns: 1fr; }
.filter-rail { position: static; max-height: none; }
.split-hero { grid-template-columns: 1fr; }
}

753
mediashelf/static/app.js Normal file
View file

@ -0,0 +1,753 @@
'use strict';
/* MediaShelf UI. Vanilla JS, no build step, no vendored framework (§9). */
const $ = (s, r = document) => r.querySelector(s);
const $$ = (s, r = document) => Array.from(r.querySelectorAll(s));
const state = {
view: 'dashboard',
page: 1,
pageSize: 100,
sort: 'reclaim_score:desc',
selected: new Set(),
weights: null,
defaultWeights: {
size: 0.28, staleness: 0.24, unpopularity: 0.22,
solitude: 0.10, age: 0.10, rejection: 0.06,
},
libraries: [],
views: [],
lastPage: null,
hasCompletion: true,
};
/* ── helpers ──────────────────────────────────────────────────────── */
function bytes(n) {
n = Number(n || 0);
const u = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];
let i = 0;
while (Math.abs(n) >= 1024 && i < u.length - 1) { n /= 1024; i++; }
return `${n.toFixed(n >= 100 || i === 0 ? 0 : 1)} ${u[i]}`;
}
function exact(n) { return Number(n || 0).toLocaleString() + ' bytes'; }
function date(ts) {
if (!ts) return '—';
return new Date(ts * 1000).toISOString().slice(0, 10);
}
function ago(ts) {
if (!ts) return 'never';
const d = Math.floor((Date.now() / 1000 - ts) / 86400);
if (d < 1) return 'today';
if (d < 60) return `${d}d`;
if (d < 730) return `${Math.floor(d / 30)}mo`;
return `${(d / 365).toFixed(1)}y`;
}
/* "today" and "3 days ago" read differently — don't blindly suffix " ago". */
function agoPhrase(ts) {
if (!ts) return 'never';
const a = ago(ts);
return a === 'today' ? 'today' : a + ' ago';
}
function el(tag, attrs = {}, ...kids) {
const n = document.createElement(tag);
for (const [k, v] of Object.entries(attrs)) {
if (v === null || v === undefined || v === false) continue;
if (k === 'class') n.className = v;
else if (k === 'text') n.textContent = v;
else if (k === 'html') n.innerHTML = v;
else if (k.startsWith('on')) n.addEventListener(k.slice(2), v);
else n.setAttribute(k, v);
}
for (const kid of kids.flat()) {
if (kid === null || kid === undefined || kid === false) continue;
n.append(kid.nodeType ? kid : document.createTextNode(kid));
}
return n;
}
async function api(path, opts) {
const r = await fetch('/api/v1' + path, opts);
if (!r.ok) {
let msg = r.statusText;
try { msg = (await r.json()).message || msg; } catch (_) {}
throw new Error(msg);
}
return r.json();
}
function scoreColor(s) {
if (s === null || s === undefined) return 'var(--faint)';
if (s >= 75) return 'var(--hot)';
if (s >= 50) return 'var(--warm)';
if (s >= 25) return 'var(--dim)';
return 'var(--good)';
}
function banner(kind, text) {
$('#banner-area').append(el('div', { class: `banner ${kind}`, text }));
}
/* ── navigation ───────────────────────────────────────────────────── */
$$('.tab').forEach(t => t.addEventListener('click', () => show(t.dataset.view)));
const VIEWS = ['dashboard', 'grid', 'keeps', 'views', 'duplicates', 'scans'];
function show(name) {
if (!VIEWS.includes(name)) name = 'dashboard';
state.view = name;
$$('.tab').forEach(t => t.classList.toggle('active', t.dataset.view === name));
$$('.view').forEach(v => v.classList.toggle('active', v.id === 'view-' + name));
const loader = { dashboard: loadDashboard, grid: loadGrid, keeps: loadKeeps,
views: loadViews, duplicates: loadDuplicates, scans: loadScans }[name];
if (loader) loader();
if (location.hash.replace('#', '') !== name) {
const url = new URL(location);
url.hash = name;
history.replaceState(null, '', url);
}
}
/* A hash-only change does not reload the document, so without this a pasted
#grid link (or the back button) leaves the dashboard on screen. */
window.addEventListener('hashchange', () => {
const name = location.hash.replace('#', '') || 'dashboard';
if (name !== state.view) show(name);
});
/* ── dashboard ────────────────────────────────────────────────────── */
async function loadDashboard() {
const [ov, libs, comp, added, scatter] = await Promise.all([
api('/stats/overview'), api('/stats/size-by-library'),
api('/stats/completion'), api('/stats/added-over-time'),
api('/stats/size-vs-lastwatched'),
]);
state.hasCompletion = ov.has_completion_data;
// The three-way split that keeps the keep list honest (§6.6)
$('#reclaim-split').replaceChildren(
el('div', { class: 'split-cell' },
el('div', { class: 'label', text: 'Never played' }),
el('div', { class: 'value', text: bytes(ov.never_played_bytes) }),
el('div', { class: 'sub', text: `${bytes(ov.confident_bytes)} confident · ${bytes(ov.uncertain_bytes)} uncertain` })),
el('div', { class: 'split-cell kept' },
el('div', { class: 'label', text: 'Kept' }),
el('div', { class: 'value', text: bytes(ov.never_played_kept_bytes) }),
el('div', { class: 'sub', text: `${ov.keep_marks} mark(s) · ${bytes(ov.kept_bytes)} kept overall` })),
el('div', { class: 'split-cell available' },
el('div', { class: 'label', text: 'Available' }),
el('div', { class: 'value', text: bytes(ov.available_bytes) }),
el('div', { class: 'sub', text: 'never played and not kept' })),
);
const pct = ov.never_played_bytes ? (ov.never_played_kept_bytes / ov.never_played_bytes) : 0;
if (pct > 0.5) {
banner('warn', `${Math.round(pct * 100)}% of never-played content is marked keep — ` +
`the reclaim report is mostly reporting on things you have decided to keep.`);
}
$('#tiles').replaceChildren(
tile('Library size', bytes(ov.total_bytes), `${ov.total_items.toLocaleString()} rows · ${ov.libraries} libraries`),
tile('Episodes', ov.episodes.toLocaleString(), 'rolled up into seasons'),
tile('Cold 2+ years', bytes(ov.cold_bytes), 'not played in two years'),
tile('Watch history', ov.watch_events.toLocaleString() + ' plays',
ov.history_since ? `${ov.accounts} users since ${date(ov.history_since)}` : 'no history'),
tile('Last scan', agoPhrase(ov.last_scan_at),
ov.history_source ? 'via ' + ov.history_source : ''),
);
if (ov.history_since) {
const badge = $('#source-badge');
badge.textContent = `${ov.history_source || 'no history'} · since ${date(ov.history_since)}`;
badge.title = ov.has_completion_data
? 'Completion data available — the rejection component is active'
: 'No completion data — running degraded, rejection component disabled';
}
if (!ov.has_completion_data && ov.watch_events > 0) {
banner('warn', 'Watch history has no completion data (Plex fallback). ' +
'The score is running degraded: the “rejection” component is disabled.');
}
// size by library
const maxL = Math.max(...libs.libraries.map(l => l.size_bytes), 1);
$('#chart-libraries').replaceChildren(...libs.libraries.map(l =>
el('div', { class: 'bar-row' },
el('div', { class: 'bar-label', text: l.title, title: l.title }),
el('div', { class: 'bar-track' },
el('div', { class: 'bar-fill', style: `width:${(l.size_bytes - l.never_bytes) / maxL * 100}%` }),
el('div', { class: 'bar-fill never', style: `width:${l.never_bytes / maxL * 100}%` })),
el('div', { class: 'bar-num', text: bytes(l.size_bytes), title: exact(l.size_bytes) }))));
$('#chart-libraries').append(el('div', { class: 'legend' },
el('span', {}, el('i', { style: 'background:var(--accent)' }), 'played'),
el('span', {}, el('i', { style: 'background:var(--warm)' }), 'never played')));
// completion split
const total = comp.finished + comp.started + comp.never || 1;
const seg = (v, c, label) => v / total > 0.04
? el('div', { style: `width:${v / total * 100}%;background:${c}`, text: label, title: bytes(v) })
: el('div', { style: `width:${v / total * 100}%;background:${c}`, title: `${label}: ${bytes(v)}` });
$('#chart-completion').replaceChildren(
el('div', { class: 'stack' },
seg(comp.finished, 'var(--good)', bytes(comp.finished)),
seg(comp.started, 'var(--warm)', bytes(comp.started)),
seg(comp.never, 'var(--hot)', bytes(comp.never))),
el('div', { class: 'legend' },
el('span', {}, el('i', { style: 'background:var(--good)' }), 'finished'),
el('span', {}, el('i', { style: 'background:var(--warm)' }), 'started, never finished'),
el('span', {}, el('i', { style: 'background:var(--hot)' }), 'never opened')));
// added over time
const maxA = Math.max(...added.buckets.map(b => b.size_bytes), 1);
const recent = added.buckets.slice(-48);
$('#chart-added').replaceChildren(el('div', {
style: 'display:flex;align-items:flex-end;gap:2px;height:120px',
}, ...recent.map(b => el('div', {
style: `flex:1;min-width:3px;background:var(--accent);height:${Math.max(b.size_bytes / maxA * 100, 1)}%`,
title: `${b.period}: ${bytes(b.size_bytes)} (${b.items} items)`,
}))));
if (recent.length) {
$('#chart-added').append(el('div', { class: 'legend' },
el('span', { text: recent[0].period }), el('span', { text: '→' }),
el('span', { text: recent[recent.length - 1].period })));
}
drawScatter(scatter.points);
}
function tile(label, value, sub) {
return el('div', { class: 'tile' },
el('div', { class: 'label', text: label }),
el('div', { class: 'value', text: value }),
sub ? el('div', { class: 'sub', text: sub }) : null);
}
function drawScatter(points) {
const W = 900, H = 260, PAD = 34;
const now = Date.now() / 1000;
const maxSize = Math.max(...points.map(p => p.size_bytes), 1);
const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
svg.setAttribute('viewBox', `0 0 ${W} ${H}`);
svg.setAttribute('class', 'scatter');
svg.setAttribute('preserveAspectRatio', 'none');
const mk = (t, a) => {
const n = document.createElementNS('http://www.w3.org/2000/svg', t);
for (const [k, v] of Object.entries(a)) n.setAttribute(k, v);
return n;
};
// axes
svg.append(mk('line', { x1: PAD, y1: H - PAD, x2: W - 4, y2: H - PAD, stroke: 'var(--line)' }));
svg.append(mk('line', { x1: PAD, y1: 4, x2: PAD, y2: H - PAD, stroke: 'var(--line)' }));
const maxDays = 3650;
for (const p of points) {
const days = p.last_watched_at ? Math.min((now - p.last_watched_at) / 86400, maxDays) : maxDays;
const x = PAD + (days / maxDays) * (W - PAD - 8);
const y = (H - PAD) - (Math.log10(1 + p.size_bytes) / Math.log10(1 + maxSize)) * (H - PAD - 8);
const c = mk('circle', {
cx: x.toFixed(1), cy: y.toFixed(1), r: 2.4,
fill: p.kept ? 'var(--kept)' : (p.watch_count ? 'var(--accent)' : 'var(--warm)'),
'fill-opacity': .55,
});
const title = document.createElementNS('http://www.w3.org/2000/svg', 'title');
title.textContent = `${p.title}${bytes(p.size_bytes)}, ` +
(p.last_watched_at ? `last played ${agoPhrase(p.last_watched_at)}` : 'never played');
c.append(title);
c.addEventListener('click', () => openItem(p.id));
svg.append(c);
}
const t1 = mk('text', { x: PAD, y: H - 8, fill: 'var(--faint)', 'font-size': 10 });
t1.textContent = 'recently played';
const t2 = mk('text', { x: W - 90, y: H - 8, fill: 'var(--faint)', 'font-size': 10 });
t2.textContent = 'never / 10y+';
svg.append(t1, t2);
$('#chart-scatter').replaceChildren(svg);
}
/* ── grid ─────────────────────────────────────────────────────────── */
const COLUMNS = [
{ key: '_sel', label: '', sortable: false },
{ key: 'title', label: 'Title' },
{ key: 'library', label: 'Library' },
{ key: 'size_bytes', label: 'Size', num: true },
{ key: 'added_at', label: 'Added', num: true },
{ key: 'last_watched_at', label: 'Last played', num: true },
{ key: 'watch_count', label: 'Plays', num: true },
{ key: 'abandoned_count', label: 'Bailed', num: true },
{ key: 'distinct_watcher_count', label: 'Viewers', num: true },
{ key: 'reclaim_score', label: 'Reclaim', num: true },
{ key: '_keep', label: '', sortable: false },
];
function gridParams() {
const p = new URLSearchParams();
const q = $('#f-q').value.trim();
if (q) p.set('q', q);
$$('#f-libraries input:checked').forEach(i => p.append('library_id', i.value));
$$('.f-kind:checked').forEach(i => p.append('kind', i.value));
if ($('#f-kept').checked) p.set('include_kept', '1');
if ($('#f-missing').checked) p.set('include_missing', '1');
const rules = { op: 'and', rules: [] };
const minSize = Number($('#f-minsize').value);
if (minSize > 0) rules.rules.push({ field: 'size_bytes', op: 'gte', value: minSize });
switch ($('#f-watch').value) {
case 'never': rules.rules.push({ field: 'watch_count', op: 'eq', value: 0 }); break;
case 'confident':
rules.rules.push({ field: 'watch_count', op: 'eq', value: 0 },
{ field: 'pre_history', op: 'eq', value: false }); break;
case 'uncertain':
rules.rules.push({ field: 'watch_count', op: 'eq', value: 0 },
{ field: 'pre_history', op: 'eq', value: true }); break;
case 'rejected':
rules.rules.push({ field: 'abandoned_count', op: 'gte', value: 1 },
{ field: 'watch_count', op: 'eq', value: 0 }); break;
case 'watched': rules.rules.push({ field: 'watch_count', op: 'gte', value: 1 }); break;
}
const viewId = $('#f-view').value;
if (viewId) p.set('view_id', viewId);
if (rules.rules.length) p.set('rules', JSON.stringify(rules));
if (state.weights) p.set('weights', JSON.stringify(state.weights));
p.set('sort', state.sort);
p.set('page', state.page);
p.set('page_size', state.pageSize);
return p;
}
async function loadGrid() {
if (!state.libraries.length) await loadFilterOptions();
let data;
try {
data = await api('/items?' + gridParams());
} catch (e) {
$('#grid-body').replaceChildren(el('tr', {}, el('td', { colspan: COLUMNS.length, class: 'empty', text: 'Query failed: ' + e.message })));
return;
}
state.lastPage = data;
renderHead();
renderRows(data);
renderSummary(data);
$('#page-info').textContent =
`page ${data.page} of ${Math.max(1, Math.ceil(data.total / data.page_size))}`;
$('#page-prev').disabled = data.page <= 1;
$('#page-next').disabled = data.page * data.page_size >= data.total;
}
function selectedBytes() {
if (!state.lastPage) return 0;
return state.lastPage.items
.filter(i => state.selected.has(i.id))
.reduce((s, i) => s + (i.size_bytes || 0), 0);
}
function renderHead() {
$('#grid-head').replaceChildren(...COLUMNS.map(c => {
if (c.key === '_sel') {
return el('th', {}, el('input', {
type: 'checkbox', title: 'select all on this page',
onchange: (e) => {
(state.lastPage?.items || []).forEach(i =>
e.target.checked ? state.selected.add(i.id) : state.selected.delete(i.id));
renderRows(state.lastPage); loadGridSummaryOnly();
},
}));
}
const active = state.sort.startsWith(c.key + ':');
const th = el('th', {
class: active ? 'sorted' : '',
text: c.label + (active ? (state.sort.endsWith('desc') ? ' ↓' : ' ↑') : ''),
});
if (c.sortable !== false) {
th.addEventListener('click', () => {
const dir = state.sort === c.key + ':desc' ? 'asc' : 'desc';
state.sort = c.key + ':' + dir;
state.page = 1;
loadGrid();
});
}
return th;
}));
}
/* replaceChildren() stringifies null into the literal text "null" rather than
skipping it, so the optional selection span is filtered out, not passed in. */
function renderSummary(data) {
const parts = [el('span', {
html: `<b>${data.total.toLocaleString()}</b> rows · <b>${bytes(data.aggregate.total_size_bytes)}</b>`,
})];
if (state.selected.size) {
parts.push(el('span', {
html: ` — <b>${state.selected.size}</b> selected · <b>${bytes(selectedBytes())}</b>`,
}));
}
$('#grid-summary').replaceChildren(...parts);
$('#btn-keep').disabled = state.selected.size === 0;
}
function loadGridSummaryOnly() {
renderSummary(state.lastPage);
}
function renderRows(data) {
if (!data.items.length) {
$('#grid-body').replaceChildren(el('tr', {}, el('td', {
colspan: COLUMNS.length, class: 'empty',
text: 'Nothing matches. If everything here is kept, tick “Kept items” to see it.',
})));
return;
}
$('#grid-body').replaceChildren(...data.items.map(it => {
const label = it.kind === 'season'
? `${it.show_title || '?'} — Season ${it.season_number ?? '?'}`
: it.title + (it.year ? ` (${it.year})` : '');
const flags = it.flags.map(f => el('span', { class: 'flag ' + f, text: f.replace('_', ' ') }));
if (it.kept) flags.unshift(el('span', { class: 'flag kept', text: 'kept · ' + it.kept_via }));
return el('tr', { class: it.kept ? 'kept-row' : '' },
el('td', {}, el('input', {
type: 'checkbox', checked: state.selected.has(it.id),
onchange: (e) => {
e.target.checked ? state.selected.add(it.id) : state.selected.delete(it.id);
loadGridSummaryOnly();
},
})),
el('td', { class: 'title-cell' },
el('span', { class: 'title-link', text: label, onclick: () => openItem(it.id) }),
it.kind === 'season' ? el('span', { class: 'sub-title', text: ` · ${it.episode_count} eps` }) : null,
...flags),
el('td', { text: it.library.title }),
el('td', { class: 'num', text: bytes(it.size_bytes), title: exact(it.size_bytes) }),
el('td', { class: 'num', text: date(it.added_at) }),
el('td', { class: 'num', text: it.last_watched_at ? ago(it.last_watched_at) : 'never' }),
el('td', { class: 'num', text: it.watch_count }),
el('td', { class: 'num', text: it.abandoned_count || '' }),
el('td', { class: 'num', text: it.distinct_watcher_count || '' }),
el('td', { class: 'num' }, el('span', {
class: 'score-pill',
style: `color:${scoreColor(it.reclaim_score)}`,
text: it.kept ? '—' : (it.reclaim_score ?? '—'),
title: componentTooltip(it),
})),
el('td', {}, el('button', {
class: 'btn small ghost', text: it.kept ? 'Un-keep' : 'Keep',
onclick: () => toggleKeep(it),
})));
}));
}
function componentTooltip(it) {
const c = it.reclaim_components || {};
const lines = Object.entries(c)
.map(([k, v]) => `${k}: ${v === null ? 'n/a' : v.toFixed(2)}`);
if (it.grace) lines.push(`grace: ${it.grace}`);
return lines.join('\n');
}
async function toggleKeep(it) {
try {
if (it.kept) {
const marks = await api('/keeps');
const mine = marks.marks.find(m => m.id === it.kept_mark_id);
if (!mine) {
alert('This is kept by a library rule — turn that off on the Kept tab.');
return;
}
await api('/keeps/' + mine.id, { method: 'DELETE' });
} else {
await api('/keeps', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ item_id: it.id, mode: 'keep' }),
});
}
loadGrid();
} catch (e) { alert(e.message); }
}
$('#btn-keep').addEventListener('click', async () => {
const ids = Array.from(state.selected);
const note = prompt(`Keep ${ids.length} item(s). Optional note — why are you keeping these?`, '');
if (note === null) return;
try {
const r = await api('/keeps/bulk', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ item_ids: ids, mode: 'keep', note: note || null }),
});
state.selected.clear();
if (r.failed.length) {
alert(`${r.created} kept. ${r.failed.length} could not be: ${r.failed[0].reason}`);
}
loadGrid();
} catch (e) { alert(e.message); }
});
$('#btn-export').addEventListener('click', () => {
location.href = '/api/v1/export.csv?' + gridParams();
});
$('#page-prev').addEventListener('click', () => { state.page--; loadGrid(); });
$('#page-next').addEventListener('click', () => { state.page++; loadGrid(); });
['#f-q', '#f-minsize', '#f-watch', '#f-view', '#f-kept', '#f-missing'].forEach(sel => {
const node = $(sel);
const ev = node.tagName === 'INPUT' && node.type === 'search' ? 'input' : 'change';
let t;
node.addEventListener(ev, () => {
clearTimeout(t);
t = setTimeout(() => { state.page = 1; loadGrid(); }, ev === 'input' ? 300 : 0);
});
});
async function loadFilterOptions() {
const [libs, views] = await Promise.all([api('/libraries'), api('/views')]);
state.libraries = libs.libraries;
state.views = views.views;
$('#f-libraries').replaceChildren(...libs.libraries.map(l =>
el('label', {}, el('input', {
type: 'checkbox', value: l.id,
onchange: () => { state.page = 1; loadGrid(); },
}), `${l.title} (${bytes(l.size_bytes)})`)));
$('#f-view').replaceChildren(el('option', { value: '', text: '— none —' }),
...views.views.map(v => el('option', { value: v.id, text: v.name, title: v.description || '' })));
buildWeightSliders();
$$('.f-kind').forEach(i => i.addEventListener('change', () => { state.page = 1; loadGrid(); }));
}
function buildWeightSliders() {
const w = state.weights || state.defaultWeights;
$('#weight-sliders').replaceChildren(...Object.keys(state.defaultWeights).map(k =>
el('label', {},
`${k} `, el('span', { id: 'wv-' + k, text: (w[k] ?? 0).toFixed(2) }),
el('input', {
type: 'range', min: 0, max: 1, step: 0.02, value: w[k] ?? 0,
oninput: (e) => {
state.weights = { ...(state.weights || state.defaultWeights) };
state.weights[k] = Number(e.target.value);
$('#wv-' + k).textContent = Number(e.target.value).toFixed(2);
clearTimeout(buildWeightSliders._t);
buildWeightSliders._t = setTimeout(loadGrid, 250);
},
}))));
}
$('#weights-reset').addEventListener('click', () => {
state.weights = null; buildWeightSliders(); loadGrid();
});
/* ── item drawer ──────────────────────────────────────────────────── */
async function openItem(id) {
const it = await api('/items/' + id);
$('#drawer-title').textContent = it.kind === 'season'
? `${it.show_title} — Season ${it.season_number}`
: it.title + (it.year ? ` (${it.year})` : '');
const body = $('#drawer-body');
body.replaceChildren();
const dl = el('dl', { class: 'kv' });
const add = (k, v) => { dl.append(el('dt', { text: k }), el('dd', { text: v })); };
add('Library', it.library.title);
add('Size', `${bytes(it.size_bytes)} (${exact(it.size_bytes)})`);
add('Added', date(it.added_at));
add('Last played', it.last_watched_at ? `${date(it.last_watched_at)} (${agoPhrase(it.last_watched_at)})` : 'never');
add('Plays', `${it.watch_count} finished · ${it.partial_count} partial · ${it.abandoned_count} abandoned`);
add('Distinct viewers', it.distinct_watcher_count);
if (it.avg_percent_complete !== null && it.avg_percent_complete !== undefined) {
add('Average completion', it.avg_percent_complete.toFixed(0) + '%');
}
if (it.kind === 'season') add('Episodes', it.episode_count);
add('Files', it.part_count);
if (it.pre_history) {
add('Note', 'Added before watch history began — “never played” is unproven here.');
}
add('Kept', it.kept ? `yes (via ${it.kept_via})` : 'no');
body.append(dl);
body.append(el('h3', { text: 'Reclaim score' }));
body.append(el('div', { class: 'hint', text: it.kept
? 'Not scored for deletion: this item is kept.'
: `Score ${it.reclaim_score}${it.grace ? ` (clamped: ${it.grace})` : ''}` }));
for (const [k, v] of Object.entries(it.reclaim_components || {})) {
body.append(el('div', { class: 'comp-row' },
el('span', { text: k }),
el('div', { class: 'comp-track' },
el('div', { class: 'comp-fill', style: `width:${(v ?? 0) * 100}%` })),
el('span', { text: v === null ? 'n/a' : v.toFixed(2) })));
}
if (it.duplicates?.length) {
body.append(el('h3', { text: 'Other copies' }));
it.duplicates.forEach(d => body.append(el('div', { class: 'hint' },
`${d.library_title}: ${bytes(d.size_bytes)} ${d.resolution || ''} · ${d.watch_count} plays`)));
}
body.append(el('h3', { text: 'Files' }));
if (it.parts?.length) {
it.parts.forEach(p => body.append(el('div', { class: 'hint', text: `${bytes(p.size_bytes)}${p.file_path}` })));
} else if (it.episodes?.length) {
body.append(el('div', { class: 'hint', text: `${it.episodes.length} episodes` }));
it.episodes.forEach(e => body.append(el('div', { class: 'hint' },
`E${String(e.episode_number).padStart(2, '0')} · ${bytes(e.size_bytes)} · ` +
`${e.watch_count} plays · added ${date(e.added_at)}`)));
}
if (it.watch_history?.length) {
body.append(el('h3', { text: 'Watch history' }));
it.watch_history.slice(0, 40).forEach(h => body.append(el('div', { class: 'hint' },
`${date(h.viewed_at)} · ${h.who || 'unknown'} · ` +
`${h.percent_complete === null ? 'completion unknown' : h.percent_complete + '%'} (${h.disposition})`)));
}
$('#drawer').hidden = false;
$('#drawer-scrim').hidden = false;
}
const closeDrawer = () => { $('#drawer').hidden = true; $('#drawer-scrim').hidden = true; };
$('#drawer-close').addEventListener('click', closeDrawer);
$('#drawer-scrim').addEventListener('click', closeDrawer);
document.addEventListener('keydown', e => { if (e.key === 'Escape') closeDrawer(); });
/* ── kept ─────────────────────────────────────────────────────────── */
async function loadKeeps() {
const [data, libs] = await Promise.all([api('/keeps'), api('/libraries')]);
$('#keep-libraries').replaceChildren(...libs.libraries.map(l =>
el('label', {}, el('input', {
type: 'checkbox', checked: !!l.keep_all,
onchange: async (e) => {
await api(`/libraries/${l.id}/keep_all`, {
method: 'PUT', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ keep_all: e.target.checked }),
});
loadKeeps();
},
}), `${l.title}${bytes(l.size_bytes)}`)));
const live = data.marks.filter(m => !m.orphaned);
const orphans = data.marks.filter(m => m.orphaned);
$('#keep-marks').replaceChildren(
el('div', { class: 'hint', text:
`${data.kept_items.toLocaleString()} items kept, totalling ${bytes(data.kept_bytes)}.` }),
...(live.length ? live.map(markRow) : [el('div', { class: 'empty',
text: 'Nothing marked yet. Keep something from the Library tab.' })]));
$('#keep-orphans-card').hidden = orphans.length === 0;
$('#keep-orphans').replaceChildren(...orphans.map(markRow));
}
function markRow(m) {
return el('div', { class: 'mark' },
el('div', {},
el('div', { text: m.label }),
el('div', { class: 'meta', text:
`${m.scope} · ${m.library_title} · ${m.mode}` +
` · ${m.resolved_items} item(s), ${bytes(m.resolved_bytes)}` +
(m.note ? ` · “${m.note}` : '') +
` · marked ${date(m.created_at)}` })),
el('button', {
class: 'btn small ghost', text: 'Remove',
onclick: async () => {
if (!confirm(`Remove the keep on “${m.label}”?`)) return;
await api('/keeps/' + m.id, { method: 'DELETE' });
loadKeeps();
},
}));
}
/* ── views / duplicates / scans ───────────────────────────────────── */
async function loadViews() {
const data = await api('/views');
$('#views-list').replaceChildren(...data.views.map(v =>
el('div', { class: 'mark' },
el('div', {},
el('div', { text: v.name + (v.builtin ? ' · built in' : '') }),
el('div', { class: 'meta', text: v.description || '' })),
el('button', {
class: 'btn small', text: 'Open',
onclick: () => { $('#f-view').value = v.id; state.page = 1; show('grid'); },
}))));
}
async function loadDuplicates() {
const data = await api('/duplicates');
$('#dupes-summary').textContent =
`${data.group_count} group(s) · ${bytes(data.total_redundant_bytes)} redundant.`;
$('#dupes-list').replaceChildren(...(data.groups.length ? data.groups.slice(0, 200).map(g =>
el('div', { class: 'dupe-group' },
el('div', {}, el('b', { text: g.title }), g.year ? ` (${g.year})` : '',
`${bytes(g.redundant_bytes)} redundant`),
el('div', { class: 'dupe-copies' }, ...g.copies.map(c =>
el('span', { class: 'title-link', onclick: () => openItem(c.id) },
`${c.library_title}: ${bytes(c.size_bytes)} ${c.resolution || ''} · ${c.watch_count} plays`)))))
: [el('div', { class: 'empty', text: 'No duplicate GUIDs across libraries.' })]));
}
async function loadScans() {
const [scans, sources] = await Promise.all([api('/scans'), api('/sources')]);
const cov = sources.coverage[0];
const info = [
el('div', { class: 'hint', text: `Plex: ${sources.plex.base_url || 'not configured'}` }),
el('div', { class: 'hint', text: `Tautulli: ${sources.tautulli.base_url || 'not configured'}` }),
el('div', { class: 'hint', text: `History source: ${sources.history_source || 'none'}` +
(sources.has_completion_data ? ' (completion data available)' : ' (no completion data — degraded)') }),
];
if (cov) {
info.push(el('div', { class: 'hint', text:
`Coverage: ${date(cov.earliest_event_at)}${date(cov.latest_event_at)}, ` +
`${cov.event_count.toLocaleString()} events` }));
}
$('#sources-info').replaceChildren(...info);
$('#scans-list').replaceChildren(...(scans.scans.length ? scans.scans.map(s =>
el('div', { class: 'mark' },
el('div', {},
el('div', { text: `${s.mode} · ${s.status}` + (s.error ? `${s.error}` : '') }),
el('div', { class: 'meta', text:
`${date(s.started_at)} · seen ${s.items_seen} · added ${s.items_added} · ` +
`events ${s.events_added} · missing ${s.items_missing} · warnings ${s.warning_count}` +
(s.finished_at ? ` · took ${s.finished_at - s.started_at}s` : '') })),
el('span', { class: 'meta', text: s.history_source || '' })))
: [el('div', { class: 'empty', text: 'No scans yet.' })]));
}
/* ── scan control ─────────────────────────────────────────────────── */
$('#scan-now').addEventListener('click', async () => {
try {
await api('/scans', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ mode: 'incremental' }),
});
pollScan();
} catch (e) { alert(e.message); }
});
async function pollScan() {
const btn = $('#scan-now');
btn.disabled = true;
const tick = async () => {
const s = await api('/scans/current');
if (s) {
btn.textContent = s.progress ? `Scanning: ${s.progress}` : 'Scanning…';
setTimeout(tick, 2000);
} else {
btn.textContent = 'Scan now';
btn.disabled = false;
$('#banner-area').replaceChildren();
show(state.view);
}
};
tick();
}
/* ── boot ─────────────────────────────────────────────────────────── */
(async function boot() {
show(location.hash.replace('#', '') || 'dashboard');
try {
const s = await api('/scans/current');
if (s) pollScan();
} catch (_) {}
})();

View file

@ -0,0 +1,200 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>MediaShelf</title>
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>📚</text></svg>">
<link rel="stylesheet" href="/static/app.css">
</head>
<body>
<header class="topbar">
<div class="brand">
<span class="logo"></span>
<span>MediaShelf</span>
<span class="readonly-badge" title="v1 never deletes, moves, or modifies anything">report&nbsp;only</span>
</div>
<nav class="tabs">
<button class="tab active" data-view="dashboard">Dashboard</button>
<button class="tab" data-view="grid">Library</button>
<button class="tab" data-view="keeps">Kept</button>
<button class="tab" data-view="views">Views</button>
<button class="tab" data-view="duplicates">Duplicates</button>
<button class="tab" data-view="scans">Scans</button>
</nav>
<div class="topbar-right">
<span id="source-badge" class="source-badge"></span>
<button id="scan-now" class="btn">Scan now</button>
</div>
</header>
<div id="banner-area"></div>
<main>
<!-- ────────────────────────── dashboard ────────────────────────── -->
<section id="view-dashboard" class="view active">
<div class="split-hero" id="reclaim-split"></div>
<div class="tiles" id="tiles"></div>
<div class="chart-grid">
<div class="card">
<h3>Size by library</h3>
<div id="chart-libraries" class="chart"></div>
</div>
<div class="card">
<h3>Finished · started &amp; abandoned · never opened</h3>
<p class="hint">The middle band is invisible without completion data.</p>
<div id="chart-completion" class="chart"></div>
</div>
<div class="card wide">
<h3>Added over time</h3>
<div id="chart-added" class="chart"></div>
</div>
<div class="card wide">
<h3>Size vs. time since last watched</h3>
<p class="hint">Upper right is the reclaim target. Click a point to open it.</p>
<div id="chart-scatter" class="chart"></div>
</div>
</div>
</section>
<!-- ──────────────────────────── grid ───────────────────────────── -->
<section id="view-grid" class="view">
<div class="grid-layout">
<aside class="filter-rail">
<div class="rail-section">
<label class="rail-label">Search</label>
<input type="search" id="f-q" placeholder="title…">
</div>
<div class="rail-section">
<label class="rail-label">Saved view</label>
<select id="f-view"><option value="">— none —</option></select>
</div>
<div class="rail-section">
<label class="rail-label">Libraries</label>
<div id="f-libraries" class="checklist"></div>
</div>
<div class="rail-section">
<label class="rail-label">Type</label>
<div class="checklist">
<label><input type="checkbox" class="f-kind" value="movie" checked> Movies</label>
<label><input type="checkbox" class="f-kind" value="season" checked> Seasons</label>
</div>
</div>
<div class="rail-section">
<label class="rail-label">Show</label>
<label class="switch"><input type="checkbox" id="f-kept"> Kept items</label>
<label class="switch"><input type="checkbox" id="f-missing"> Missing items</label>
</div>
<div class="rail-section">
<label class="rail-label">Minimum size</label>
<select id="f-minsize">
<option value="0">any</option>
<option value="1073741824">1 GB</option>
<option value="4294967296">4 GB</option>
<option value="8589934592">8 GB</option>
<option value="21474836480">20 GB</option>
<option value="53687091200">50 GB</option>
</select>
</div>
<div class="rail-section">
<label class="rail-label">Watch state</label>
<select id="f-watch">
<option value="">any</option>
<option value="never">never played</option>
<option value="confident">never played · confident</option>
<option value="uncertain">never played · uncertain</option>
<option value="rejected">started and abandoned</option>
<option value="watched">played at least once</option>
</select>
</div>
<div class="rail-section weights">
<label class="rail-label">Reclaim weights</label>
<div id="weight-sliders"></div>
<button id="weights-reset" class="btn small ghost">Reset to defaults</button>
</div>
</aside>
<div class="grid-main">
<div class="grid-toolbar">
<div id="grid-summary" class="grid-summary"></div>
<div class="grid-actions">
<button id="btn-keep" class="btn" disabled>Keep selected</button>
<button id="btn-export" class="btn ghost">Export CSV</button>
<button class="btn danger" disabled
title="v1 is report-only — deletion arrives in v2, after the score has earned trust">Delete</button>
</div>
</div>
<div class="table-wrap">
<table id="grid">
<thead><tr id="grid-head"></tr></thead>
<tbody id="grid-body"></tbody>
</table>
</div>
<div class="pager">
<button id="page-prev" class="btn small">← Prev</button>
<span id="page-info"></span>
<button id="page-next" class="btn small">Next →</button>
</div>
</div>
</div>
</section>
<!-- ──────────────────────────── kept ───────────────────────────── -->
<section id="view-keeps" class="view">
<div class="card">
<h3>Library rules</h3>
<p class="hint">A whole library kept by rule — anything added to it later is kept too.
Nothing is set by default.</p>
<div id="keep-libraries" class="lib-rules"></div>
</div>
<div class="card">
<h3>Explicit marks</h3>
<div id="keep-marks"></div>
</div>
<div class="card" id="keep-orphans-card" hidden>
<h3>Orphaned marks</h3>
<p class="hint">These matched nothing on the last scan — the content may have been
removed, renamed, or rematched. Nothing is removed automatically.</p>
<div id="keep-orphans"></div>
</div>
</section>
<!-- ─────────────────────────── views ───────────────────────────── -->
<section id="view-views" class="view">
<div class="card"><h3>Saved views</h3><div id="views-list"></div></div>
</section>
<!-- ──────────────────────── duplicates ─────────────────────────── -->
<section id="view-duplicates" class="view">
<div class="card">
<h3>Duplicate groups</h3>
<p class="hint">The same film held more than once, matched on Plex GUID. Neither copy
looks like a candidate on its own because plays are split between them.
This is a report — which copy supersedes which depends on what plays back well
on your devices, which MediaShelf has no way to know.</p>
<div id="dupes-summary" class="hint"></div>
<div id="dupes-list"></div>
</div>
</section>
<!-- ─────────────────────────── scans ───────────────────────────── -->
<section id="view-scans" class="view">
<div class="card"><h3>Sources</h3><div id="sources-info"></div></div>
<div class="card"><h3>Scan history</h3><div id="scans-list"></div></div>
</section>
</main>
<div id="drawer" class="drawer" hidden>
<div class="drawer-head">
<h2 id="drawer-title"></h2>
<button id="drawer-close" class="btn ghost small">Close</button>
</div>
<div id="drawer-body"></div>
</div>
<div id="drawer-scrim" class="scrim" hidden></div>
<script src="/static/app.js"></script>
</body>
</html>

13
mediashelf/web.py Normal file
View file

@ -0,0 +1,13 @@
"""Server-rendered shell. All data comes from /api/v1 (§9)."""
from __future__ import annotations
from flask import Blueprint, current_app, render_template
bp = Blueprint("web", __name__)
@bp.get("/")
def index():
cfg = current_app.extensions["mediashelf"]["config"]
return render_template("index.html", tz=cfg.tz)