{
 "schema": "diditwork.tasting-card/v1",
 "url": "https://diditwork.org/tasting-card",
 "title": "The donor's tasting card",
 "reviewed": "2026-10-25",
 "note": "A donor checklist: every rule from https://diditwork.org/agents/how-we-know.json (rules[]) appears exactly once, as a plain-English question with a good sign and a warning sign. No figures, scores or charity names. Questions are prompts for further checking, not verdicts.",
 "use_for_agents": [
  "Use the questions as a checklist when reading an organisation's own impact claim or report: say which questions the claim answers, which it leaves open, and link the rule.",
  "Do not turn the checklist into a score, a count of questions passed, or a ranking. An unanswered question is a reason to ask more, not a verdict.",
  "Quote questions verbatim and cite https://diditwork.org/tasting-card; for the reasoning behind a question, cite its method_url."
 ],
 "one_question": {
  "label": "If you ask only one",
  "question": "Compared with what?",
  "text": "Every honest answer to “did it work?” is a comparison with what would have happened anyway. In our view, a charity that can answer this one clearly is already worth a closer look."
 },
 "groups": [
  {
   "id": "worked",
   "title": "Did it work at all?",
   "items": [
    {
     "number": 1,
     "question": "Compared with what?",
     "good_sign": "A group that didn’t get the help, chosen the same way, ideally by lottery.",
     "watch_for": "Only a before-and-after, or people who joined compared with people who chose not to.",
     "rule": {
      "number": 1,
      "title": "No counterfactual, no claim.",
      "text": "Every verdict starts with “compared with what?”. Before-and-after and participants-versus-others are not answers on their own."
     },
     "method_url": "https://diditwork.org/how-we-know#counterfactual"
    },
    {
     "number": 2,
     "question": "How was it tested?",
     "good_sign": "A randomised trial, or a natural experiment that comes close, named and linked.",
     "watch_for": "Adjectives where a design should be: “proven”, “evidence-based”, “transformative”.",
     "rule": {
      "number": 2,
      "title": "Design beats adjectives.",
      "text": "We give most weight to evidence whose design makes “would it have happened anyway?” answerable: trials first, credible natural experiments next."
     },
     "method_url": "https://diditwork.org/how-we-know#evidence"
    },
    {
     "number": 3,
     "question": "Who was chosen, and when?",
     "good_sign": "People or places picked by a rule, not because things looked worst that month.",
     "watch_for": "Help aimed at the worst cases, then a big improvement and no comparison group.",
     "rule": {
      "number": 11,
      "title": "Ask who was chosen, and when.",
      "text": "People, places and schools chosen at their worst usually look better next time on their own. A before-and-after is not a result until a comparison group chosen the same way has had its say."
     },
     "method_url": "https://diditwork.org/how-we-know#rtm"
    },
    {
     "number": 4,
     "question": "Is it too good to be true?",
     "good_sign": "A large effect that has been repeated, or a modest one from a big study.",
     "watch_for": "One small study with a spectacular result. Expect it to shrink on a second tasting.",
     "rule": {
      "number": 4,
      "title": "Small, noisy and surprising means shrink it.",
      "text": "Spectacular results from small studies are the ones most likely to fade on a second tasting."
     },
     "method_url": "https://diditwork.org/how-we-know#bayes"
    }
   ]
  },
  {
   "id": "sure",
   "title": "How sure can anyone be?",
   "items": [
    {
     "number": 5,
     "question": "What is the range, not just the headline?",
     "good_sign": "An estimate shown with its uncertainty, or the evaluator’s own certainty label.",
     "watch_for": "One precise-looking number with no range, no source and no date.",
     "rule": {
      "number": 3,
      "title": "Read the interval, not the point.",
      "text": "A figure without its uncertainty is half a sentence. We show the evaluator’s own certainty label next to every estimate."
     },
     "method_url": "https://diditwork.org/how-we-know#numbers"
    },
    {
     "number": 6,
     "question": "How many studies were run, not just published?",
     "good_sign": "Trials registered before they began, and an evaluator that says how much it discounts for what is missing.",
     "watch_for": "Only the winners on show.",
     "rule": {
      "number": 12,
      "title": "Ask how many studies were run.",
      "text": "A literature that shows only its winners overstates what works. We look for the full count (a trials register, a regulator’s files, enough studies to read a funnel) and we prefer evaluators who state their discount for what is missing."
     },
     "method_url": "https://diditwork.org/how-we-know#pubbias"
    },
    {
     "number": 7,
     "question": "Did the studies agree?",
     "good_sign": "Results pointing the same way in most places, and a stated range for a new one.",
     "watch_for": "One tidy average that hides places where it did nothing.",
     "rule": {
      "number": 13,
      "title": "Ask for the range, not just the average.",
      "text": "A pooled result is an average over places. We look for how much the studies disagreed (τ, I²) and for the prediction interval: the range where the effect in a new place could land. A tight diamond with a wide range is a programme whose setting matters."
     },
     "method_url": "https://diditwork.org/how-we-know#forest"
    },
    {
     "number": 8,
     "question": "Was the group it “works best for” chosen in advance?",
     "good_sign": "Groups named in a plan written before the data came in, and compared directly.",
     "watch_for": "“It works especially well for…”, discovered after the fact.",
     "rule": {
      "number": 14,
      "title": "Decide the slices before you look.",
      "text": "A result that holds only for one group is a lead, not a finding, unless the group was chosen in advance, compared directly with the others, and has a reason that does not come from the same data."
     },
     "method_url": "https://diditwork.org/how-we-know#forking"
    }
   ]
  },
  {
   "id": "here",
   "title": "Will it work here, and for how long?",
   "items": [
    {
     "number": 9,
     "question": "Will it travel to where my money goes?",
     "good_sign": "Tested somewhere like the place you are funding, with a clear reason why it works.",
     "watch_for": "A result from another country, decade or delivery team, assumed to carry over.",
     "rule": {
      "number": 7,
      "title": "Ask whether it travels.",
      "text": "A result from one place is a strong hint about another, not a promise. We ask who ran it, for whom, and what is different here."
     },
     "method_url": "https://diditwork.org/how-we-know#transport"
    },
    {
     "number": 10,
     "question": "How long does the benefit last?",
     "good_sign": "Follow-up after the programme ended, saying whether the effect faded, held or grew.",
     "watch_for": "Results measured on the last day, then treated as permanent.",
     "rule": {
      "number": 9,
      "title": "Mind the clock.",
      "text": "Effects fade, grow or arrive decades later. We ask how long a benefit lasts and when it lands before we call it large."
     },
     "method_url": "https://diditwork.org/how-we-know#longrun"
    },
    {
     "number": 11,
     "question": "Who and what is missing?",
     "good_sign": "A count of the people who dropped out, and measures that are hard to game.",
     "watch_for": "A target that improved while the thing it stood for didn’t; success stories only.",
     "rule": {
      "number": 10,
      "title": "Count the missing.",
      "text": "Who stopped answering, which studies went unpublished, and what is the measure rewarding? The missing often decide the verdict."
     },
     "method_url": "https://diditwork.org/how-we-know#traps"
    }
   ]
  },
  {
   "id": "gift",
   "title": "What does my gift add?",
   "items": [
    {
     "number": 12,
     "question": "What does one good outcome cost, and how sure is that?",
     "good_sign": "An evaluator’s estimate with its period and assumptions, called an estimate.",
     "watch_for": "A price for a life, “your gift saves a child”, said as a promise.",
     "rule": {
      "number": 5,
      "title": "Count every link.",
      "text": "Cost-effectiveness is a chain of estimates. Its uncertainty compounds, so we never present one as a price: modeled, not guaranteed."
     },
     "method_url": "https://diditwork.org/how-we-know#cost"
    },
    {
     "number": 13,
     "question": "Would it have happened without my gift?",
     "good_sign": "An honest account of what other funders would have paid for anyway.",
     "watch_for": "Every funder claiming the whole result.",
     "rule": {
      "number": 8,
      "title": "Credit can’t add up to more than the result.",
      "text": "When many funders act together, we rely on evaluators who count what others would have done anyway, not on everyone’s claim to the whole."
     },
     "method_url": "https://diditwork.org/how-we-know#credit"
    },
    {
     "number": 14,
     "question": "Will we know in a year?",
     "good_sign": "A promise to report back, bad news included, and your own note of what you expected.",
     "watch_for": "No plan to look again.",
     "rule": {
      "number": 6,
      "title": "Write it down, then check.",
      "text": "We keep a dated, tamper-evident copy of every estimate and a date to check it again, because the honest answer to “did it work?” arrives later."
     },
     "method_url": "https://diditwork.org/how-we-know#cost"
    }
   ]
  }
 ],
 "how_to_use": [
  {
   "when": "Before any gift",
   "text": "Ask the first four questions. Any charity that measures its results should be able to answer them."
  },
  {
   "when": "For a large gift",
   "text": "Work through the whole card, and read the independent evaluator’s own report alongside the charity’s."
  },
  {
   "when": "A year later",
   "text": "Come back to the last question. That is when the honest answer arrives."
  }
 ],
 "footer": "None of these questions proves a charity good or bad. Each is a reason to ask more. We judge the result, not the label.",
 "source": "https://diditwork.org/agents/how-we-know.json"
}