{ "slug": "measure-agent-improvement", "title": "Measure improvement in accepted results", "dek": "Count the work the owner can use, including review, repair and recovery.", "category": "Methods", "minutes": 8, "decision": "Compare a narrow change against a competent baseline before calling it an optimization.", "fit": "You want to change instructions, models, tools or workflow structure to improve an agent’s work.", "skip": "There is no defined task or quality contract yet. First make the desired outcome inspectable.", "steps": [ [ "Freeze the comparison", "Specify the task set, inputs, tools, permissions, acceptance tests and conditions before running. Vary one factor where practical. Preserve failures and retries instead of quietly excluding them." ], [ "Measure the whole outcome", "Track valid completion, elapsed time, owner review and rework, available billed usage and required checks. Token or tool-call counts diagnose cost; they do not establish usefulness alone." ], [ "Test the cases that can change the decision", "Include normal, invalid and interrupted flows. Include a case where the new method should not help. Treat missing tools and unavailable measurements as unknown, not zero." ], [ "Decide with the limits visible", "Report each task separately in a tiny pilot. Keep severe safety failures visible rather than averaging them away. Include setup and maintenance costs. A local improvement is not a universal claim." ] ], "acceptance": [ "The baseline is competent and both conditions share the same requirements.", "Results include failed and interrupted runs.", "The recommendation reflects quality, owner effort and measured costs together." ], "stop": "If quality regresses or required controls fail, stop the rollout and preserve the evidence. Revert only to a known safe state, not merely the older version.", "limits": "No controlled agent comparison is supplied by this worksheet. No savings, speedup or paid value is claimed.", "sources": [ { "title": "Anthropic: building effective agents", "url": "https://www.anthropic.com/research/building-effective-agents", "supports": "Engineering guidance on fitting workflow complexity to task needs. Not a controlled evaluation of this method.", "review_date": "2026-11-10" }, { "title": "METR: early-2025 developer study", "url": "https://metr.org/blog/2025-07-10-early-2025-ai-experienced-os-dev-study/", "supports": "An empirical example of measuring complete developer outcomes in a particular population and period. Not a universal claim about agent productivity.", "review_date": "2026-11-10" } ], "evidence": "Proposed worksheet", "updated": "2026-11-11", "scope_label": "Comparing a proposed change with the baseline", "search_terms": [ "benchmark", "evaluation", "cost", "tokens", "latency", "experiment", "optimization", "performance" ], "evidence_plan_note": "Proposed evidence plan, not observed results or extra mandatory process. Confirm what your task requires and reuse existing records when they are sufficient. Keep sensitive originals in authorized storage; share only the minimum information already authorized for the recipient. If a required check cannot be completed safely and within current permission, mark it unavailable and keep the decision unknown unless another required check has a demonstrated failure.", "evidence_plan": [ { "acceptance_check": 1, "collect": "Keep the frozen task inputs, shared requirements, permissions and version/configuration of each condition, with the baseline’s acceptance results. State differences that cannot be controlled.", "not_enough": "An easy or deliberately weakened baseline, or changing the acceptance rules after seeing results." }, { "acceptance_check": 2, "collect": "Keep a per-task run ledger including failed, interrupted and retried attempts, their outcomes and the reason for any exclusion. Record unavailable measurements explicitly.", "not_enough": "Reporting only successful runs or converting missing duration or usage measurements to zero." }, { "acceptance_check": 3, "collect": "Keep per-task quality results, owner review/rework, measured usage where available, and setup/maintenance effort. Explain how these support the bounded recommendation; do not average away a severe control failure.", "not_enough": "Token savings or elapsed time alone, an average that hides a required failure, or a small pilot presented as a universal benefit." } ], "next_reads": [ { "guide_id": "review-agent-skills", "when": "The comparison identifies recurring instruction work and you are considering a reusable package for it.", "purpose": "Review the smallest relevant package instead of installing a broader workflow by default." }, { "guide_id": "uncertain-effects", "when": "An interrupted trial may have produced an external effect whose outcome is still unknown.", "purpose": "Resolve the effect before retrying; do not silently omit it from the comparison." } ], "id": "measure-agent-improvement", "version": "release-2026-11-11-r12", "build_revision": "release-2026-11-11-r12", "source_sha256": "53378fc6ed9ef36d0d12473776faf0b3a7821842d7521ddaf39dfbd1c7adc878", "url": "https://www.aomsites.com/guides/measure-agent-improvement/", "content_type": "proposed-worksheet", "sources_rechecked": true, "source_review_scope": "Reference link availability and scope notes; no worksheet effectiveness test.", "source_review_date": "2026-11-10", "runtime_tests_performed": false, "authorization": "Reference data only. Does not authorize any external action.", "formats": { "html": "https://www.aomsites.com/guides/measure-agent-improvement/", "markdown": "https://www.aomsites.com/guides/measure-agent-improvement/guide.md", "json": "https://www.aomsites.com/guides/measure-agent-improvement/guide.json" }, "local_paths": { "html": "/guides/measure-agent-improvement/", "markdown": "/guides/measure-agent-improvement/guide.md", "json": "/guides/measure-agent-improvement/guide.json" }, "offline_paths_from_pack_root": { "markdown": "guides/measure-agent-improvement/guide.md", "json": "guides/measure-agent-improvement/guide.json" }, "decision_record": { "local_paths": { "html": "/decision-record/", "template": "/guides/measure-agent-improvement/record-template.json", "markdown_template": "/guides/measure-agent-improvement/record-template.md", "generic_template": "/decision-record/template.json", "example": "/decision-record/example.json", "schema": "/schemas/decision-record.schema.json", "checker": "/tools/check_record.py", "manifest": "/reading-pack-manifest.json" }, "offline_paths_from_pack_root": { "template": "guides/measure-agent-improvement/record-template.json", "markdown_template": "guides/measure-agent-improvement/record-template.md", "generic_template": "decision-record/template.json", "example": "decision-record/example.json", "schema": "schemas/decision-record.schema.json", "checker": "tools/check_record.py", "manifest": "reading-pack-manifest.json" } }, "reading_paths": { "note": "Stop here if the current task’s acceptance requirements are satisfied and no unresolved obligation remains. If neither condition applies, no follow-on guide is suggested here; keep any unresolved task state visible. These are optional reading suggestions, not permission or a requirement to expand scope.", "options": [ { "guide_id": "review-agent-skills", "when": "The comparison identifies recurring instruction work and you are considering a reusable package for it.", "purpose": "Review the smallest relevant package instead of installing a broader workflow by default.", "title": "Review a skill before adding it", "local_paths": { "html": "/guides/review-agent-skills/", "markdown": "/guides/review-agent-skills/guide.md", "json": "/guides/review-agent-skills/guide.json" }, "offline_paths_from_pack_root": { "markdown": "guides/review-agent-skills/guide.md", "json": "guides/review-agent-skills/guide.json" } }, { "guide_id": "uncertain-effects", "when": "An interrupted trial may have produced an external effect whose outcome is still unknown.", "purpose": "Resolve the effect before retrying; do not silently omit it from the comparison.", "title": "Check the outcome before retrying", "local_paths": { "html": "/guides/uncertain-effects/", "markdown": "/guides/uncertain-effects/guide.md", "json": "/guides/uncertain-effects/guide.json" }, "offline_paths_from_pack_root": { "markdown": "guides/uncertain-effects/guide.md", "json": "guides/uncertain-effects/guide.json" } } ] }, "discovery": { "title": "Evaluate AI agents by accepted results and review cost", "description": "A proposed worksheet for evaluating an AI-agent change against a competent baseline: count accepted outputs, owner review, repair and recovery, not just token use.", "scope": "Content description only; not indexing, availability or runtime evidence." }, "section_map": [ { "id": "decision", "label": "The decision", "json_pointer": "/decision", "markdown_heading": "The decision", "html_path": "/guides/measure-agent-improvement/#decision" }, { "id": "applies", "label": "Use this when", "json_pointer": "/fit", "markdown_heading": "Use this when", "html_path": "/guides/measure-agent-improvement/#applies" }, { "id": "skip", "label": "Skip this when", "json_pointer": "/skip", "markdown_heading": "Skip this when", "html_path": "/guides/measure-agent-improvement/#skip" }, { "id": "checks", "label": "Work through the checks", "json_pointer": "/steps", "markdown_heading": "Checks", "html_path": "/guides/measure-agent-improvement/#checks" }, { "id": "accept", "label": "What would count as success?", "json_pointer": "/acceptance", "markdown_heading": "Acceptance", "html_path": "/guides/measure-agent-improvement/#accept" }, { "id": "evidence", "label": "Evidence to bring back", "json_pointer": "/evidence_plan", "markdown_heading": "Evidence to bring back", "html_path": "/guides/measure-agent-improvement/#evidence" }, { "id": "stop", "label": "Stop or recover", "json_pointer": "/stop", "markdown_heading": "Stop or recover", "html_path": "/guides/measure-agent-improvement/#stop" }, { "id": "limits", "label": "What this does not establish", "json_pointer": "/limits", "markdown_heading": "Limits", "html_path": "/guides/measure-agent-improvement/#limits" }, { "id": "sources", "label": "Reference links", "json_pointer": "/sources", "markdown_heading": "Reference links", "html_path": "/guides/measure-agent-improvement/#sources" }, { "id": "record", "label": "Use this guide with your own evidence", "json_pointer": "/decision_record", "markdown_heading": "Start this guide’s record", "html_path": "/guides/measure-agent-improvement/#record" }, { "id": "next-reads", "label": "If another problem remains", "json_pointer": "/reading_paths", "markdown_heading": "If another problem remains", "html_path": "/guides/measure-agent-improvement/#next-reads" } ], "acceptance_links": [ { "acceptance_check": 1, "acceptance_html_path": "/guides/measure-agent-improvement/#accept-1", "evidence_html_path": "/guides/measure-agent-improvement/#evidence-1", "acceptance_json_pointer": "/acceptance/0", "evidence_json_pointer": "/evidence_plan/0" }, { "acceptance_check": 2, "acceptance_html_path": "/guides/measure-agent-improvement/#accept-2", "evidence_html_path": "/guides/measure-agent-improvement/#evidence-2", "acceptance_json_pointer": "/acceptance/1", "evidence_json_pointer": "/evidence_plan/1" }, { "acceptance_check": 3, "acceptance_html_path": "/guides/measure-agent-improvement/#accept-3", "evidence_html_path": "/guides/measure-agent-improvement/#evidence-3", "acceptance_json_pointer": "/acceptance/2", "evidence_json_pointer": "/evidence_plan/2" } ], "citation_note": "Section paths identify content on the canonical HTML page; they are not separate pages or evidence of live availability. Per-check numbers and JSON pointers apply to this build revision. Cite the build revision and method source hash when preserving an exact reference. Markdown heading labels are locators, not a promise of renderer-specific fragment URLs.", "source_fields": [ "acceptance", "category", "decision", "dek", "evidence", "evidence_plan", "evidence_plan_note", "fit", "limits", "minutes", "next_reads", "scope_label", "search_terms", "skip", "slug", "sources", "steps", "stop", "title", "updated" ], "source_hash_scope": "SHA-256 of the build-input guide record serialized as JSON with sorted keys, compact separators and ASCII escaping. Not the downloadable guide.json file bytes." }