{
 "schema": "diditwork.verdict-procedure/v1",
 "url": "https://diditwork.org/agents/verdict-procedure.json",
 "page": "https://diditwork.org/agents/verdict-procedure",
 "method_page": "https://diditwork.org/how-we-know#chain",
 "published": "2026-10-25",
 "reviewed": "2026-10-25",
 "status": "Published v1 (build queue T2). Changes are logged at https://diditwork.org/agents/changelog.",
 "purpose": "A procedure an agent runs to answer 'did it work?' about any giving claim, link by link, using only labels that already exist on diditwork.org. It produces per-link records and a named weakest link. It never produces a score.",
 "forbidden": [
  "numeric scores, points, grades or star ratings of any kind",
  "adding, averaging or multiplying rungs or links into one aggregate",
  "combining cost-effectiveness figures across units (deaths averted, DALYs, WELLBYs, income) or across evaluators",
  "computing a cost-effectiveness figure; only quote an evaluator's published figure",
  "assigning a historical verdict word (worked, mixed, didnt, harm) to a live charity",
  "filling an 'unknown' with the agent's own estimate",
  "letting a strong link upgrade the rung of a weaker link",
  "choosing moral weights for the donor"
 ],
 "rungs": [
  {
   "id": "verified",
   "label": "Directly verified outcome",
   "verb": "plain statement of fact, cited"
  },
  {
   "id": "trials",
   "label": "Multiple randomised trials or systematic review",
   "verb": "Trials show / A systematic review found"
  },
  {
   "id": "one_study",
   "label": "One good study or quasi-experiment",
   "verb": "One study found / Economists estimate"
  },
  {
   "id": "own_estimate",
   "label": "An organisation's own estimate",
   "verb": "by its own count"
  },
  {
   "id": "evaluator_model",
   "label": "Evaluator cost-effectiveness model",
   "verb": "[Evaluator]'s estimate ([period]): modeled, not guaranteed"
  },
  {
   "id": "contested",
   "label": "Contested or mixed evidence",
   "verb": "The link is contested"
  },
  {
   "id": "our_view",
   "label": "Our opinion",
   "verb": "In our view"
  },
  {
   "id": "unknown",
   "label": "Unknown",
   "verb": "Not yet rated / Unknown"
  }
 ],
 "rung_source": "docs/brand/voice.md section 5 (confidence ladder). The order is a ladder of verbs, not a numeric scale.",
 "external_labels": {
  "evidence_labels_verbatim": [
   "Exceptionally strong",
   "Very strong",
   "Strong",
   "Moderate",
   "Low to moderate",
   "Not yet rated"
  ],
  "rule": "Quote the evaluator's evidence label verbatim and attributed, alongside the rung. Never translate one into the other.",
  "ce_label": "modeled, not guaranteed",
  "followup_statuses": [
   "pending",
   "confirmed",
   "contradicted",
   "unverifiable"
  ],
  "historical_verdicts_reserved": [
   "worked",
   "mixed",
   "didnt",
   "harm"
  ]
 },
 "links": [
  {
   "id": "L0",
   "name": "Defined",
   "question": "Worked at what, compared with what, for whom, by when?",
   "allowed_rungs": [],
   "gate": true,
   "required_fields": [
    "outcome",
    "comparator"
   ],
   "optional_fields": [
    "population",
    "horizon"
   ],
   "if_missing": "status 'undefined'; output 'Not established: the claim does not say compared with what.' and stop",
   "rules": [
    "No counterfactual, no claim."
   ],
   "tasting_card_questions": [
    "Compared with what?"
   ],
   "url": "https://diditwork.org/how-we-know#chain-l0"
  },
  {
   "id": "L1",
   "name": "Happened",
   "question": "Was it delivered, and did the measured outcome move, counting the missing and checking the measure was not gamed?",
   "allowed_rungs": [
    "verified",
    "trials",
    "own_estimate",
    "unknown"
   ],
   "unknown_phrase": "Reported by the organisation; not independently checked.",
   "rules": [
    "Count the missing."
   ],
   "tasting_card_questions": [
    "Who and what is missing?"
   ],
   "url": "https://diditwork.org/how-we-know#chain-l1"
  },
  {
   "id": "L2",
   "name": "Caused",
   "question": "Would it have happened anyway?",
   "allowed_rungs": [
    "trials",
    "one_study",
    "contested",
    "unknown"
   ],
   "thresholds": {
    "trials": "more than one randomised trial, or a systematic review",
    "one_study": "one trial or credible quasi-experiment with its identifying assumption named; observational evidence only with a sensitivity analysis such as an E-value",
    "unknown": "before-and-after only, or no comparison group chosen the same way"
   },
   "also_record": "evaluator_evidence_label_verbatim",
   "rules": [
    "Design beats adjectives.",
    "Ask who was chosen, and when."
   ],
   "tasting_card_questions": [
    "How was it tested?",
    "Who was chosen, and when?"
   ],
   "url": "https://diditwork.org/how-we-know#chain-l2"
  },
  {
   "id": "L3",
   "name": "Sized",
   "question": "How big is the effect, honestly?",
   "allowed_rungs": [
    "trials",
    "one_study",
    "contested",
    "unknown"
   ],
   "must_include": [
    "interval",
    "number_of_studies"
   ],
   "checks": [
    "publication bias",
    "prediction interval when pooled",
    "subgroups pre-specified",
    "winner's curse for small studies with large effects"
   ],
   "rules": [
    "Read the interval, not the point.",
    "Small, noisy and surprising means shrink it.",
    "Ask how many studies were run.",
    "Ask for the range, not just the average.",
    "Decide the slices before you look."
   ],
   "tasting_card_questions": [
    "What is the range, not just the headline?",
    "Is it too good to be true?",
    "How many studies were run, not just published?",
    "Did the studies agree?",
    "Was the group it “works best for” chosen in advance?"
   ],
   "url": "https://diditwork.org/how-we-know#chain-l3"
  },
  {
   "id": "L4",
   "name": "Travels",
   "question": "Will it work here, run by this organisation, at this scale, now?",
   "allowed_rungs": [
    "evaluator_model",
    "one_study",
    "unknown"
   ],
   "unknown_phrase": "A strong hint, not a promise: tested in [place] by [implementer]; no evidence yet for this setting or scale.",
   "rules": [
    "Ask whether it travels."
   ],
   "tasting_card_questions": [
    "Will it travel to where my money goes?"
   ],
   "url": "https://diditwork.org/how-we-know#chain-l4"
  },
  {
   "id": "L5",
   "name": "Worth it",
   "question": "What good per dollar, in a unit someone values, for how long?",
   "allowed_rungs": [
    "evaluator_model",
    "unknown"
   ],
   "must_include_if_known": [
    "evaluator",
    "figure_verbatim",
    "unit",
    "period",
    "source_url",
    "label 'modeled, not guaranteed'"
   ],
   "allowed_evaluators": [
    "GiveWell",
    "Happier Lives Institute",
    "Animal Charity Evaluators",
    "WHO",
    "IHME"
   ],
   "unknown_phrase": "Not yet rated.",
   "rules": [
    "Count every link.",
    "Mind the clock."
   ],
   "tasting_card_questions": [
    "What does one good outcome cost, and how sure is that?",
    "How long does the benefit last?"
   ],
   "url": "https://diditwork.org/how-we-know#chain-l5"
  },
  {
   "id": "L6",
   "name": "Adds",
   "question": "Will my marginal dollar change what happens?",
   "allowed_rungs": [
    "evaluator_model",
    "own_estimate",
    "unknown"
   ],
   "unknown_phrase": "Whether more money adds now is unknown; an evaluator's fund can direct it to where it does.",
   "rules": [
    "Credit can’t add up to more than the result."
   ],
   "tasting_card_questions": [
    "Would it have happened without my gift?"
   ],
   "url": "https://diditwork.org/how-we-know#chain-l6"
  },
  {
   "id": "L7",
   "name": "Checked",
   "question": "Was a prediction written down, and has anyone looked again?",
   "allowed_values": [
    "pending",
    "confirmed",
    "contradicted",
    "unverifiable",
    "no follow-up scheduled"
   ],
   "rules": [
    "Write it down, then check."
   ],
   "tasting_card_questions": [
    "Will we know in a year?"
   ],
   "url": "https://diditwork.org/how-we-know#chain-l7"
  }
 ],
 "combine": {
  "method": "report, do not aggregate",
  "weakest_link": "the first link in chain order whose rung is unknown, contested or own_estimate; if none, the link whose stated range is widest relative to its estimate",
  "no_borrowing": "no link's rung may be raised by another link's evidence"
 },
 "summary_template_donor_voice": "[L2–L3 in the rung's verb] [the effect, with interval] ([evaluator label, verbatim]). [L4 sentence]. [Evaluator]'s estimate ([period]): [figure], modeled, not guaranteed. [L6 sentence]. Weakest link: [link name]: [open question]. [L7 status].",
 "output_record": {
  "claim": "string",
  "links": [
   {
    "id": "L0..L7",
    "answer": "one sentence",
    "rung": "rung id or null for L0/L7",
    "evaluator_label_verbatim": "string or null",
    "source": {
     "title": "",
     "url": "",
     "retrieved": "YYYY-MM-DD"
    },
    "open_question": "string or null"
   }
  ],
  "weakest_link": "L0..L7",
  "open_questions": [
   "string"
  ],
  "summary_donor_voice": "string following the template in Part 8.3"
 },
 "examples": [
  {
   "id": "chain-ex-nets",
   "title": "Worked example A: treated bed nets (real figures, programme level only; says nothing about any listed charity)",
   "illustrative": false,
   "source_id": "cochrane_itn",
   "source_url": "https://www.cochrane.org/CD000363/INFECTN_insecticide-treated-nets-preventing-malaria",
   "verbatim": [
    "Insecticide-treated nets reduce child mortality from all causes by 17% compared to no nets",
    "rate ratio 0.83, 95% CI 0.77 to 0.89; 5 trials, 200,833 participants, high-certainty evidence",
    "a saving of 5.6 lives (95% CI 3.6 to 7.6) each year for every 1000 children protected with ITNs"
   ],
   "links": [
    {
     "id": "L0",
     "answer": "All-cause child mortality; nets compared with no nets; children in the trial settings; per year of protection.",
     "rung": null
    },
    {
     "id": "L1",
     "answer": "Delivered and measured within randomised trials.",
     "rung": "trials"
    },
    {
     "id": "L2",
     "answer": "A Cochrane review found that insecticide-treated nets reduce child mortality from all causes by 17% compared to no nets (high-certainty evidence, in Cochrane's words).",
     "rung": "trials"
    },
    {
     "id": "L3",
     "answer": "Rate ratio 0.83 (95% CI 0.77 to 0.89), 5 trials, 200,833 participants: a saving of 5.6 lives (95% CI 3.6 to 7.6) each year for every 1000 children protected.",
     "rung": "trials"
    },
    {
     "id": "L4",
     "answer": "The trials come from specific settings and decades; carrying them to a district today needs an external-validity argument, which evaluators supply charity by charity.",
     "rung": "unknown"
    },
    {
     "id": "L5",
     "answer": "Not applicable without a charity and an evaluator's published figure.",
     "rung": "unknown"
    },
    {
     "id": "L6",
     "answer": "Not applicable without a charity.",
     "rung": "unknown"
    },
    {
     "id": "L7",
     "answer": "Not applicable at programme level.",
     "rung": null
    }
   ],
   "weakest_link": "L4",
   "headline": "A Cochrane review of randomised trials found treated nets cut child deaths by 17 percent (high-certainty evidence). Weakest link for any particular gift: whether that effect carries to today's settings and what one more gift adds; that is what evaluators model."
  },
  {
   "id": "chain-ex-before-after",
   "title": "Worked example B: a before-and-after claim",
   "illustrative": true,
   "label": "Illustrative: made-up numbers, not any charity's results",
   "claim": "Our programme improved wellbeing for 80% of participants.",
   "links": [
    {
     "id": "L0",
     "answer": "Outcome stated; comparator missing: before and after is not a comparator.",
     "rung": null
    }
   ],
   "weakest_link": "L0",
   "headline": "Not established: compared with what? People chosen at a hard moment would be expected to improve on their own (regression to the mean), and the 80% is among those who answered. If half answered, the honest range is between 40% and 90% improved, by the charity's own count, and whether the programme caused it stays unknown."
  }
 ],
 "never_do": [
  "Turn rungs into numbers, or count how many links are strong.",
  "Combine L5 figures across units or evaluators.",
  "Assign It worked or It didn't work to a live charity.",
  "Fill an unknown with your own estimate.",
  "Let a strong link upgrade a weak one."
 ]
}