Files
marc 34f1f4396e Close diwan session: TODO/DONE instead of checkbox; purge evolutionary residue from all docs
1. 'Close diwan session' now uses TODO/DONE (assign_diwan_id inserts
'** TODO Close diwan session'), matching Reports. No headline anywhere
in diwan's own docs uses a [ ]/[X] checkbox now -- only :chat: list
items do, which is correct org usage (checklist items, not headline
states) and unaffected.

2. Rewrote PROTOCOL.md/TICK.md/DO.md/DESIGN.md to read as a single
coherent design rather than a changelog of how they got here. Removed:
contrastive 'not because X, but because Y' phrasing that only makes
sense if you know the prior rule, references to old tag names/formats
that no longer exist, a Claude-Code-vs-pi rationale that belongs in
DESIGN.md (if anywhere) not the agent-facing PROTOCOL.md, and 'no
:LINK: property' (never describe a convention by naming something it
doesn't have).

3. DESIGN.md got the heaviest pass since it had drifted from reality,
not just tone: it still said 'no flake.nix yet' (there is one, in
production use via samfelag), claimed several things 'not yet
re-verified live' that have since been verified multiple times
(tick's no-op gate, triage, the tag-consistency/escalation decision,
pi against real heading content via tick), and didn't mention
'diwan remote' or the actual flake.nix packaging shape anywhere.
Rewrote the verified-live claims and Known Gaps list to match current
reality, added flake.nix to the doc map and architecture section.

Live content migrated to match: diwan-dev-notes' decorate-home heading
('Close diwan session' checkbox -> TODO).
2026-11-09 23:17:17 +01:00

576 lines
20 KiB
Plaintext
Executable File

#!/usr/bin/env -S nix shell nixpkgs#python3 --command python3
"""diwan CLI — see PROTOCOL.md, TICK.md, DO.md (installed alongside this
script) for what this orchestrates, and the content repo's own AGENTS.md
for what it's orchestrating against.
This script only does the mechanical, deterministic parts (git bookkeeping,
DIWAN_ID property + "Close diwan session" TODO insertion, herdr
dispatch).
All actual triage/research reasoning happens in the dispatched agent, per the
procedure docs.
herdr integration is against the real `herdr` CLI (workspace/tab/pane/agent
over a JSON socket API — see `herdr --skill`), verified live against herdr
0.9.5. When `herdr` isn't on PATH, every herdr_* call prints what it would
have run instead of failing, so the rest of the pipeline stays testable
without herdr installed.
Two separate roots: DIWAN_DIR is where diwan itself (this script, plus
PROTOCOL.md/TICK.md/DO.md) is installed — `diwan` is meant to live on PATH
via a nix flake, independent of any one person's notes; override with the
DIWAN_DIR env var if the default (this script's own directory) doesn't
hold. DIWAN_NOTES_DIR is whatever notes repo it's being invoked against,
discovered from the current directory via `git rev-parse --show-toplevel`
by default (diwan must then be run from inside, or below, that repo) or
set explicitly via the DIWAN_NOTES_DIR env var (e.g. for a cron/systemd
timer that doesn't cd into it first).
"""
import argparse
import contextlib
import fcntl
import json
import os
import re
import shutil
import subprocess
import sys
import uuid
from datetime import datetime, timezone
from pathlib import Path
DIWAN_DIR = Path(os.environ.get("DIWAN_DIR", str(Path(__file__).resolve().parent)))
def find_notes_dir():
res = subprocess.run(
["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True
)
if res.returncode != 0:
print(
"error: not inside a git repo — run diwan from inside the notes repo "
"you want it to act on, or set DIWAN_NOTES_DIR.",
file=sys.stderr,
)
sys.exit(1)
return Path(res.stdout.strip())
DIWAN_NOTES_DIR = Path(os.environ["DIWAN_NOTES_DIR"]) if "DIWAN_NOTES_DIR" in os.environ else find_notes_dir()
STATE_DIR = DIWAN_NOTES_DIR / ".diwan"
STATE_FILE = STATE_DIR / "state.json"
LOCK_FILE = STATE_DIR / ".lock"
MAX_TICK_TABS = int(os.environ.get("DIWAN_MAX_TICK_TABS", "3"))
DIWAN_WORKSPACE_LABEL = "diwan"
# pi is the only supported harness: it has no per-tool-call approval gate at
# all (see its own security docs), unlike Claude Code's prompt-per-edit
# default, so it needs nothing like a --permission-mode flag to run
# unattended. The only thing that can still block a non-interactive dispatch
# is the one-time project-trust prompt for protected .pi/* resources, if any
# ever show up in a content repo. --approve makes that a one-shot yes
# instead of a hang.
PI_APPROVE_FLAG = os.environ.get("DIWAN_PI_APPROVE", "--approve")
HEADLINE_RE = re.compile(r"^(\*+)\s+(.*)$")
HERDR_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]{0,31}$")
TAGS_RE = re.compile(r"\s+:[\w:]+:\s*$")
TODO_KEYWORDS = {"TODO", "IN-PROGRESS", "WAITING", "DONE", "CANCELLED"}
# --- git -----------------------------------------------------------------
def run_git(args, check=True, capture=False):
return subprocess.run(
["git", *args], cwd=DIWAN_NOTES_DIR, check=check, capture_output=capture, text=True
)
def git_is_dirty():
out = run_git(["status", "--porcelain"], capture=True).stdout.strip()
return bool(out)
def git_head():
return run_git(["rev-parse", "HEAD"], capture=True).stdout.strip()
def git_diff_has_changes(since, until, excludes=()):
out = run_git(["diff", "--name-only", since, until], capture=True).stdout
files = [f for f in out.splitlines() if f]
files = [
f
for f in files
if not any(f == ex or f.startswith(ex.rstrip("/") + "/") for ex in excludes)
]
return bool(files)
@contextlib.contextmanager
def locked():
STATE_DIR.mkdir(exist_ok=True)
fd = open(LOCK_FILE, "w")
try:
fcntl.flock(fd, fcntl.LOCK_EX)
yield
finally:
fcntl.flock(fd, fcntl.LOCK_UN)
fd.close()
def git_commit_all(message, diwan_identity=False):
"""Stage and commit everything currently dirty, under a lock. Returns
False if there was nothing to commit."""
with locked():
run_git(["add", "-A"])
if not run_git(["status", "--porcelain"], capture=True).stdout.strip():
return False
args = []
if diwan_identity:
args += ["-c", "user.name=diwan", "-c", "user.email=diwan@local"]
args += ["commit", "-m", message]
run_git(args)
return True
# --- state -----------------------------------------------------------------
def load_state():
if STATE_FILE.exists():
return json.loads(STATE_FILE.read_text())
return {}
def save_state(state):
STATE_DIR.mkdir(exist_ok=True)
STATE_FILE.write_text(json.dumps(state, indent=2) + "\n")
# --- org parsing (just enough for DIWAN_ID + Close-session bookkeeping) ---
def read_lines(path):
return Path(path).read_text(encoding="utf-8").splitlines()
def write_lines(path, lines):
Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8")
def parse_headline(line):
m = HEADLINE_RE.match(line)
if not m:
return None
return len(m.group(1)), m.group(2)
def heading_title_for_slug(rest):
body = TAGS_RE.sub("", rest)
words = body.split()
if words and words[0] in TODO_KEYWORDS:
words = words[1:]
return " ".join(words)
def slugify(text):
"""Produce a herdr agent-name-safe slug: ^[a-z][a-z0-9_-]{0,31}$."""
text = re.sub(r"\[\[.*?\]\[(.*?)\]\]", r"\1", text) # org link -> description
text = text.lower()
text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
text = text[:24] or "task"
if not re.match(r"^[a-z]", text):
text = "d-" + text
return text[:24]
def find_drawer_insert_point(lines, idx):
"""idx is a headline's line index. Returns (insert_idx, drawer) where
drawer is (start, end) of an existing :PROPERTIES:...:END: block (end =
index of the :END: line) right after the headline/planning line, or
None if there isn't one yet — in which case insert_idx is where one
should be created."""
j = idx + 1
if j < len(lines) and re.match(r"^\s*(SCHEDULED|DEADLINE|CLOSED)\b", lines[j]):
j += 1
if j < len(lines) and lines[j].strip() == ":PROPERTIES:":
start = j
k = j + 1
while k < len(lines) and lines[k].strip() != ":END:":
k += 1
return j, (start, k)
return j, None
def get_property(lines, idx, key):
_, drawer = find_drawer_insert_point(lines, idx)
if not drawer:
return None
start, end = drawer
for line in lines[start + 1 : end]:
m = re.match(r"^\s*:([\w_]+):\s*(.*)$", line)
if m and m.group(1).upper() == key.upper():
return m.group(2).strip()
return None
def set_property(lines, idx, key, value):
insert_idx, drawer = find_drawer_insert_point(lines, idx)
if drawer:
start, end = drawer
for i in range(start + 1, end):
m = re.match(r"^\s*:([\w_]+):\s*(.*)$", lines[i])
if m and m.group(1).upper() == key.upper():
lines[i] = f" :{key}: {value}"
return
lines.insert(end, f" :{key}: {value}")
else:
lines[insert_idx:insert_idx] = [" :PROPERTIES:", f" :{key}: {value}", " :END:"]
def assign_diwan_id(lines, idx, level, diwan_id, child_text="Close diwan session"):
set_property(lines, idx, "DIWAN_ID", diwan_id)
_, drawer = find_drawer_insert_point(lines, idx) # recompute: now guaranteed present
insert_at = drawer[1] + 1
stars = "*" * (level + 1)
lines.insert(insert_at, f"{stars} TODO {child_text}")
def id_in_use(diwan_id):
pattern = re.compile(rf":DIWAN_ID:\s*{re.escape(diwan_id)}\s*$", re.MULTILINE)
for path in DIWAN_NOTES_DIR.rglob("*.org"):
if ".diwan" in path.parts:
continue
try:
text = path.read_text(encoding="utf-8", errors="ignore")
except OSError:
continue
if pattern.search(text):
return True
return False
def unique_id(base):
if not id_in_use(base):
return base
for i in range(2, 50):
candidate = f"{base}-{i}"
if not id_in_use(candidate):
return candidate
return f"{base}-{uuid.uuid4().hex[:6]}"
# --- herdr -----------------------------------------------------------------
# Real `herdr` CLI (workspace/tab/pane/agent over a JSON socket API). Verified
# live against herdr 0.9.5 — see `herdr --skill` for the authoritative
# reference if this drifts on a future version.
def herdr_available():
return shutil.which("herdr") is not None
class HerdrError(RuntimeError):
pass
def herdr_json(*args, machine=None):
cmd = ["herdr"]
if machine:
cmd += ["--machine", machine]
cmd += list(args)
res = subprocess.run(cmd, cwd=DIWAN_NOTES_DIR, capture_output=True, text=True)
if res.returncode != 0:
raise HerdrError(f"{' '.join(cmd)} failed: {res.stderr.strip() or res.stdout.strip()}")
try:
return json.loads(res.stdout)
except json.JSONDecodeError:
raise HerdrError(f"{' '.join(cmd)}: non-JSON output: {res.stdout.strip()}")
def herdr_diwan_workspace_id(machine=None):
data = herdr_json("workspace", "list", machine=machine)
for ws in data["result"]["workspaces"]:
if ws.get("label") == DIWAN_WORKSPACE_LABEL:
return ws["workspace_id"]
created = herdr_json("workspace", "create", "--label", DIWAN_WORKSPACE_LABEL, "--no-focus", machine=machine)
return created["result"]["workspace"]["workspace_id"]
def herdr_find_tab_by_label(workspace_id, label):
data = herdr_json("tab", "list", "--workspace", workspace_id)
for t in data["result"]["tabs"]:
if t.get("label") == label:
return t["tab_id"]
return None
def herdr_tab_root_pane(workspace_id, tab_id):
data = herdr_json("pane", "list", "--workspace", workspace_id)
for p in data["result"]["panes"]:
if p.get("tab_id") == tab_id:
return p["pane_id"]
return None
def herdr_agent_alive(name):
try:
herdr_json("agent", "get", name)
return True
except HerdrError:
return False
def herdr_agent_start_args(name, pane_id):
args = ["agent", "start", name, "--kind", "pi", "--pane", pane_id]
if PI_APPROVE_FLAG:
args += ["--", PI_APPROVE_FLAG]
return args
def herdr_start_agent_in_new_tab(workspace_id, name, prompt):
created = herdr_json(
"tab", "create", "--workspace", workspace_id,
"--cwd", str(DIWAN_NOTES_DIR), "--label", name, "--no-focus",
)
tab_id = created["result"]["tab"]["tab_id"]
pane_id = created["result"]["root_pane"]["pane_id"]
try:
herdr_json(*herdr_agent_start_args(name, pane_id))
herdr_json("agent", "prompt", name, prompt)
except HerdrError:
# don't leave an agent-less orphan tab behind on failure (e.g. name
# rejected by herdr's naming rule)
try:
herdr_json("tab", "close", tab_id)
except HerdrError:
pass
raise
def herdr_dispatch_fresh(name, prompt):
"""First dispatch for `name` (new tick run, or a heading's first diwan do)."""
if not herdr_available():
print(f"[dry-run] herdr not found — would create a '{name}' tab in the "
f"'{DIWAN_WORKSPACE_LABEL}' workspace, start a pi agent, and prompt it.")
print(f" prompt: {prompt}")
return
try:
ws_id = herdr_diwan_workspace_id()
herdr_start_agent_in_new_tab(ws_id, name, prompt)
except HerdrError as e:
print(f"warning: herdr dispatch for '{name}' failed: {e}", file=sys.stderr)
def herdr_dispatch_continuation(name, prompt):
"""Resume `name`: prompt it if its agent is still live, otherwise find its
tab by label and restart an agent there. Recreates the tab from scratch
only if that tab itself is gone (e.g. closed by hand)."""
if not herdr_available():
print(f"[dry-run] herdr not found — would prompt/resume '{name}' in the "
f"'{DIWAN_WORKSPACE_LABEL}' workspace.")
print(f" prompt: {prompt}")
return
try:
if herdr_agent_alive(name):
herdr_json("agent", "prompt", name, prompt)
return
ws_id = herdr_diwan_workspace_id()
tab_id = herdr_find_tab_by_label(ws_id, name)
pane_id = herdr_tab_root_pane(ws_id, tab_id) if tab_id else None
if pane_id is None:
herdr_start_agent_in_new_tab(ws_id, name, prompt)
return
herdr_json(*herdr_agent_start_args(name, pane_id))
herdr_json("agent", "prompt", name, prompt)
except HerdrError as e:
print(f"warning: herdr resume for '{name}' failed: {e}", file=sys.stderr)
# --- commands --------------------------------------------------------------
def cmd_bare(args):
if not herdr_available():
print("herdr not found on PATH. Install herdr: https://herdr.dev/")
print("Once installed, `diwan` will attach you to the 'diwan' workspace.")
return
try:
ws_id = herdr_diwan_workspace_id()
herdr_json("workspace", "focus", ws_id)
except HerdrError as e:
print(f"warning: couldn't focus the '{DIWAN_WORKSPACE_LABEL}' workspace: {e}", file=sys.stderr)
os.execvp("herdr", ["herdr"])
def cmd_remote(args):
machine = args.machine
if not herdr_available():
print("herdr not found on PATH. Install herdr: https://herdr.dev/")
return
# `machine` must be both a saved `herdr machine add` profile (for the
# scripted focus call below, over --machine) and a valid --remote SSH
# target for the actual interactive attach -- true by default, since
# `herdr machine add <target>` labels the profile after the SSH target
# unless --label overrides it. If you used a custom --label, pass the
# original SSH target here instead, or just run `herdr --remote
# <target>` yourself and navigate to the 'diwan' workspace by hand.
try:
ws_id = herdr_diwan_workspace_id(machine=machine)
herdr_json("workspace", "focus", ws_id, machine=machine)
except HerdrError as e:
print(f"warning: couldn't focus the '{DIWAN_WORKSPACE_LABEL}' workspace on '{machine}': {e}", file=sys.stderr)
os.execvp("herdr", ["herdr", "--remote", machine])
def cmd_tick(args):
STATE_DIR.mkdir(exist_ok=True)
if git_is_dirty():
if git_commit_all("chore: snapshot manual edits before tick"):
print("Committed pending manual edits before starting tick.")
state = load_state()
last = state.get("last_tick_commit")
head = git_head()
if last is None:
changed = True
range_desc = f"(initial)..{head[:8]}"
range_arg = "(initial import)"
else:
changed = git_diff_has_changes(last, head, excludes=[".diwan"])
range_desc = f"{last[:8]}..{head[:8]}"
range_arg = f"{last}..{head}"
if not changed and not args.force:
print(f"No changes since last tick ({range_desc}); nothing to do. Use --force to run anyway.")
return
tab = "tick-" + datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
prompt = (
f"You are a diwan tick agent for the content repo at {DIWAN_NOTES_DIR}. "
f"Read {DIWAN_DIR}/PROTOCOL.md and AGENTS.md, then follow {DIWAN_DIR}/TICK.md exactly. "
f"Change range since last completed tick: {range_arg}."
)
herdr_dispatch_fresh(tab, prompt)
history = state.get("tick_history", [])
history.append(tab)
dropped, kept = history[:-MAX_TICK_TABS], history[-MAX_TICK_TABS:]
state["tick_history"] = kept
save_state(state)
git_commit_all(f"diwan: record tick dispatch {tab}", diwan_identity=True)
print(f"Dispatched tick agent: herdr terminal '{tab}' ({range_desc}).")
if dropped and herdr_available():
try:
ws_id = herdr_diwan_workspace_id()
for old_tab in dropped:
old_tab_id = herdr_find_tab_by_label(ws_id, old_tab)
if old_tab_id:
herdr_json("tab", "close", old_tab_id)
print(f"Pruned {len(dropped)} tick terminal(s) past the retention window: {', '.join(dropped)}.")
except HerdrError as e:
print(f"warning: couldn't prune old tick terminal(s) {dropped}: {e}", file=sys.stderr)
elif dropped:
print(f"note: {len(dropped)} older tick terminal(s) ({', '.join(dropped)}) "
f"are past the retention window — herdr not on PATH, clean up by hand.")
def cmd_do(args):
target = args.target
diwan_id_arg = args.diwan_id
m = re.match(r"^(.+):(\d+)$", target)
heading_mode = bool(m) and Path(m.group(1)).is_file()
if heading_mode:
path_str, lineno = m.group(1), int(m.group(2))
path = Path(path_str)
lines = read_lines(path)
idx = lineno - 1
if not (0 <= idx < len(lines)):
print(f"error: {target}: line out of range", file=sys.stderr)
sys.exit(1)
parsed = parse_headline(lines[idx])
if not parsed:
print(f"error: {target} is not an org headline", file=sys.stderr)
sys.exit(1)
level, rest = parsed
existing_id = get_property(lines, idx, "DIWAN_ID")
if diwan_id_arg and existing_id and diwan_id_arg != existing_id:
print(
f"error: heading already has DIWAN_ID={existing_id}, "
f"conflicts with --diwan-id={diwan_id_arg}",
file=sys.stderr,
)
sys.exit(1)
is_continuation = existing_id is not None
if is_continuation:
diwan_id = existing_id
if not HERDR_NAME_RE.match(diwan_id):
print(
f"error: existing DIWAN_ID '{diwan_id}' on {target} is not a valid "
f"herdr agent name (^[a-z][a-z0-9_-]{{0,31}}$) — fix the heading's "
f":DIWAN_ID: property by hand before retrying.",
file=sys.stderr,
)
sys.exit(1)
else:
base = diwan_id_arg or slugify(heading_title_for_slug(rest))
diwan_id = base if diwan_id_arg else unique_id(base)
if not HERDR_NAME_RE.match(diwan_id):
print(
f"error: --diwan-id '{diwan_id}' is not a valid herdr agent name "
f"(^[a-z][a-z0-9_-]{{0,31}}$)",
file=sys.stderr,
)
sys.exit(1)
assign_diwan_id(lines, idx, level, diwan_id)
write_lines(path, lines)
git_commit_all(
f"diwan: assign DIWAN_ID={diwan_id} to {path_str}:{lineno}",
diwan_identity=True,
)
title = heading_title_for_slug(rest)
prompt = (
f"You are a diwan do agent, DIWAN_ID={diwan_id}, for the content repo at {DIWAN_NOTES_DIR}. "
f'Your heading is {path_str}:{lineno} ("{title}"). '
f"Read {DIWAN_DIR}/PROTOCOL.md and AGENTS.md, then follow {DIWAN_DIR}/DO.md."
)
if is_continuation:
herdr_dispatch_continuation(diwan_id, prompt)
else:
herdr_dispatch_fresh(diwan_id, prompt)
else:
diwan_id = diwan_id_arg or f"adhoc-{uuid.uuid4().hex[:8]}"
herdr_dispatch_continuation(diwan_id, target)
print(diwan_id)
def main():
parser = argparse.ArgumentParser(prog="diwan")
sub = parser.add_subparsers(dest="command")
p_tick = sub.add_parser("tick", help="run a diwan tick pass")
p_tick.add_argument("--force", action="store_true", help="dispatch even if nothing changed since last tick")
p_do = sub.add_parser("do", help="dispatch/resume a diwan do session")
p_do.add_argument("target", help="file:lineno pointing at a heading, or a free-text prompt")
p_do.add_argument("--diwan-id", dest="diwan_id", default=None)
p_remote = sub.add_parser("remote", help="open the 'diwan' workspace on a remote herdr machine")
p_remote.add_argument("machine", help="a machine saved via 'herdr machine add' (label defaults to its SSH target)")
args = parser.parse_args()
if args.command is None:
cmd_bare(args)
elif args.command == "tick":
cmd_tick(args)
elif args.command == "do":
cmd_do(args)
elif args.command == "remote":
cmd_remote(args)
if __name__ == "__main__":
main()