{
  "module": "0.3 — The Rubric",
  "course": "Master Course — Harness Engineering",
  "version": "1.0.0",
  "duration_minutes": 25,
  "total_questions": 16,
  "bloom_distribution": {
    "target": "20% recall / 40% application / 40% analysis-design",
    "actual": { "recall": 4, "application": 5, "analysis": 7 }
  },
  "passing_score_percent": 70,
  "questions": [
    {
      "id": "Q01", "bloom": "recall", "type": "multiple_choice",
      "prompt": "In a typical production session, approximately what share of context is tool outputs vs system prompt?",
      "options": ["10% / 50%", "67.6% / 3.4%", "30% / 30%", "3.4% / 67.6%"],
      "answer_index": 1,
      "rationale": "Tool outputs ~67.6%, system prompt ~3.4%. This asymmetry is the foundational anchor and justifies the entire rubric."
    },
    {
      "id": "Q02", "bloom": "recall", "type": "multiple_choice",
      "prompt": "How many modules does the rubric have, and how many phases does the deep-dive methodology have?",
      "options": ["12 modules, 6 phases", "6 modules, 12 phases", "10 modules, 5 phases", "14 modules, 7 phases"],
      "answer_index": 0,
      "rationale": "12 rubric modules (each a 'where does the complexity live' question) and 6 methodology phases (First Contact → Architecture Map → Decision Audit → Security Audit → Benchmark → Score)."
    },
    {
      "id": "Q03", "bloom": "recall", "type": "multiple_choice",
      "prompt": "The deliverable of a deep-dive consists of three parts. What is the format of the Architect's Verdict?",
      "options": [
        "One paragraph of free-form prose",
        "Exactly 3 sentences: optimizes for / sacrifices / who should build on it",
        "A bullet list of features",
        "A numerical score only"
      ],
      "answer_index": 1,
      "rationale": "The format constraint is the point. 3 sentences for the Verdict, 1 sentence for the MLSecOps Relevance note. Forcing identical format makes harnesses comparable."
    },
    {
      "id": "Q04", "bloom": "recall", "type": "multiple_choice",
      "prompt": "What is the 'tell' that you did the rubric wrong?",
      "options": [
        "Your scoring sheet has any score below 3",
        "Your output could be generated by reading the harness's README feature list",
        "You cited fewer than 12 file:line locations",
        "Your verdict mentions security"
      ],
      "answer_index": 1,
      "rationale": "If your rubric output could come from a README, you transcribed marketing, not audited a decision. The decision audit asks what was chosen, what it cost, and where the evidence is."
    },
    {
      "id": "Q05", "bloom": "application", "type": "multiple_choice",
      "prompt": "A team spent 2 weeks polishing their system prompt and ignored tool-output formatting. Per the foundational anchor, what did they optimize and neglect?",
      "options": [
        "Optimized 67.6%, neglected 3.4%",
        "Optimized ~3.4% (system prompt), neglected ~67.6% (tool outputs)",
        "Optimized 50%, neglected 50%",
        "Both were equally important; no mistake"
      ],
      "answer_index": 1,
      "rationale": "The single most common senior-engineer mistake. System prompt is 3.4%; tool outputs are 67.6%. The leverage is in tool-output handling."
    },
    {
      "id": "Q06", "bloom": "application", "type": "multiple_choice",
      "prompt": "You're writing a rubric entry for a harness. Which of these is a DECISION (correct) vs a FEATURE (anti-pattern)?",
      "options": [
        "'Has a sandbox' (feature)",
        "'Supports multi-session' (feature)",
        "'Puts the sandbox outside the agent's reach because credentials cannot live in-sandbox for multi-tenant, accepting API-hop latency for isolation' (decision)",
        "'Uses LangGraph' (feature)"
      ],
      "answer_index": 2,
      "rationale": "Only option C names a pattern + tradeoff + use-case fit. The others are feature-list items a README could produce."
    },
    {
      "id": "Q07", "bloom": "application", "type": "multiple_choice",
      "prompt": "Which rubric module audits the question 'what happens when the model calls the same failing tool 10 times in a row'?",
      "options": ["Module 1 (Execution Loop)", "Module 7 (Error Handling & Recovery)", "Module 9 (Prompt Assembly)", "Module 12 (Observability)"],
      "answer_index": 1,
      "rationale": "Stuck-loop detection and circuit breakers live in Error Handling (Module 7). Tie to Course 1 Module 7.2."
    },
    {
      "id": "Q08", "bloom": "application", "type": "multiple_choice",
      "prompt": "Which rubric module audits the tradeoff 'credentials cannot live in the sandbox if you are multi-tenant'?",
      "options": ["Module 4 (Memory)", "Module 5 (Sandboxing & Isolation)", "Module 6 (Permission)", "Module 8 (State)"],
      "answer_index": 1,
      "rationale": "The inside-sandbox vs outside-sandbox split (Luzzardi framing). Outside trades API-hop latency for credential isolation."
    },
    {
      "id": "Q09", "bloom": "application", "type": "multiple_choice",
      "prompt": "In Phase 3 (Decision Audit), you must find 3 decisions you'd make differently. What is the failure mode if you skip this?",
      "options": [
        "No failure — it's optional",
        "Your verdict becomes pure description (hagiography) with no critical engagement. The '3 disagree' forces you to evaluate tradeoffs, not just transcribe.",
        "You'll fail the benchmark phase",
        "You'll miscount the score"
      ],
      "answer_index": 1,
      "rationale": "Without the '3 disagree' requirement the deep-dive is description, not analysis. The requirement forces actual tradeoff evaluation."
    },
    {
      "id": "Q10", "bloom": "analysis", "type": "multiple_choice",
      "prompt": "A ~100-line Python agent scores 74-76% on SWE-bench Verified, only 4-6 points behind SOTA. A colleague concludes 'the loop doesn't matter.' What does Module 0.3 teach about this?",
      "options": [
        "The colleague is correct",
        "The colleague has misunderstood WORSE than someone who says 'the loop is all that matters.' The loop is necessary and cheap; the last 4-6 points + reliability/security/governance live in the 98.4% around it.",
        "The benchmark is wrong",
        "100 lines is not enough to be competitive"
      ],
      "answer_index": 1,
      "rationale": "The symmetric failure modes. 'Loop is all that matters' misunderstands; 'loop doesn't matter' misunderstands worse. The loop is necessary and cheap; the 98.4% is where the work is."
    },
    {
      "id": "Q11", "bloom": "analysis", "type": "multiple_choice",
      "prompt": "Why does the deliverable enforce a strict format (3-sentence verdict, 1-sentence MLSecOps note)?",
      "options": [
        "To make grading easier",
        "To make harnesses COMPARABLE. Forcing every harness into the same template turns a pile of deep-dives into a mental model of the field.",
        "To save tokens",
        "Because the rubric author preferred brevity"
      ],
      "answer_index": 1,
      "rationale": "The constraint is the point. Comparability is what transforms isolated analyses into a field-wide mental model. A verdict that drifts from the template cannot be compared."
    },
    {
      "id": "Q12", "bloom": "analysis", "type": "multiple_choice",
      "prompt": "NemoClaw inherits ~90% of OpenClaw's code. How can it still be a distinct, scorable harness on the rubric?",
      "options": [
        "It can't — it's the same harness",
        "The rubric scores DECISIONS not FEATURES. NemoClaw's entire score is in the governance decision (enforcement outside the agent's reach); the inherited 90% is OpenClaw's score, not NemoClaw's contribution.",
        "The rubric has a special fork-scoring mode",
        "NemoClaw must be scored as OpenClaw"
      ],
      "answer_index": 1,
      "rationale": "This is exactly why decision-audit beats feature-checklist. A fork's meaning is its diff from the parent. The inherited code is the parent's score; the fork's score is its own architectural decisions only."
    },
    {
      "id": "Q13", "bloom": "analysis", "type": "multiple_choice",
      "prompt": "Why is skipping Phase 4 (Security Audit) particularly catastrophic in this course?",
      "options": [
        "It's not — Phase 6 covers security",
        "Skipping it ships a verdict that misses the harness's central vulnerability — which for a security-focused course defeats the entire exercise. Most deep-dive failures trace to a skipped Phase 4.",
        "Phase 4 is optional for thin harnesses",
        "Security is covered in Module 11 only"
      ],
      "answer_index": 1,
      "rationale": "Phase 4 (credential flow, exec paths, injection test, blast radius) is the most-skipped phase and its omission produces verdicts blind to the harness's defining security property."
    },
    {
      "id": "Q14", "bloom": "analysis", "type": "multiple_choice",
      "prompt": "A 'dumb loop' (delegate everything to the model) sometimes beats a sophisticated plan-then-execute loop. Why?",
      "options": [
        "Dumb loops are faster to write",
        "Co-evolution: a dumb loop improves automatically as the model improves; a rigid architecture constrains capability growth. Tradeoff: harder debugging, heavy model-dependence.",
        "Plan-then-execute is deprecated",
        "Dumb loops use fewer tokens"
      ],
      "answer_index": 1,
      "rationale": "The dumb-loop tradeoff (Anthropic's philosophy, Pi): co-evolution with model improvements vs debuggability/reliability dependence on the model."
    },
    {
      "id": "Q15", "bloom": "analysis", "type": "multiple_choice",
      "prompt": "True or false, with justification: a rubric score without a code location is acceptable if the pattern is well-known.",
      "options": [
        "True — well-known patterns need no citation",
        "False. A score without a code location is an OPINION, not an audit. 'Evidence required' means file:line.",
        "True only for Module 12 (Observability)",
        "True only for the Architect's Verdict"
      ],
      "answer_index": 1,
      "rationale": "Evidence required = file:line citation. Without it, the score is assertion. This is what separates the rubric from impression."
    },
    {
      "id": "Q16", "bloom": "analysis", "type": "multiple_choice",
      "prompt": "The n8n methodology workflow runs Phases 1-5 as grep/cloc nodes and Phase 6 emits templates. Does it score the harness for you?",
      "options": [
        "Yes — that's the point of automating it",
        "No. A human (or LLM judge) fills the templates from the evidence Phases 1-5 produced. The workflow's job is to operationalize the methodology so no phase gets skipped.",
        "Only Phase 6 scores; Phases 1-5 are decorative",
        "It scores only the security phase"
      ],
      "answer_index": 1,
      "rationale": "The workflow produces EVIDENCE; a human/LLM judge produces the DELIVERABLE from that evidence. Its value is enforcing the pipeline, not replacing judgment."
    }
  ]
}