From 8548dd2b849ef5ba086f102f5ab3c0d75bb039e3 Mon Sep 17 00:00:00 2001 From: Denis Morozov Date: Mon, 27 Jul 2026 16:22:24 -0400 Subject: [PATCH 1/3] Verify TLS against the OS trust store #1 Verify TLS against the OS trust store --- .../Developer_Platform.ipynb | 17 ++++++++++++++++- requirements.txt | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/day1/02_developer-platform/Developer_Platform.ipynb b/day1/02_developer-platform/Developer_Platform.ipynb index 2a0de39..22c4d64 100644 --- a/day1/02_developer-platform/Developer_Platform.ipynb +++ b/day1/02_developer-platform/Developer_Platform.ipynb @@ -180,9 +180,24 @@ " f\" (pip said: {tail})\\n\"\n", " )\n", "\n", - "_ensure_packages([(\"anthropic\", \"anthropic\")])\n", + "_ensure_packages([(\"anthropic\", \"anthropic\"), (\"truststore\", \"truststore\")])\n", "print(\"✓ Dependencies ready\")\n", "\n", + "# ── Corporate TLS inspection (Zscaler, Netskope, Palo Alto, …) ────────────\n", + "# On a managed laptop, HTTPS is often intercepted and re-signed by a corporate root\n", + "# CA that lives in the OS trust store. Python doesn't read that store - it verifies\n", + "# against certifi's own bundle - so api.anthropic.com fails with\n", + "# CERTIFICATE_VERIFY_FAILED / APIConnectionError even though the API is perfectly\n", + "# reachable (curl to the same URL succeeds, because curl uses the OS store).\n", + "# truststore points Python at the OS store too. A no-op on unmanaged machines.\n", + "try:\n", + " import truststore\n", + " truststore.inject_into_ssl()\n", + " print(\"✓ TLS: verifying against the OS certificate store\")\n", + "except Exception as _tls_exc: # Python < 3.10, or the install was blocked\n", + " print(\"[!!] truststore unavailable (\" + type(_tls_exc).__name__ + \") - falling back \"\n", + " \"to certifi. If the API check below fails with a certificate error, see SETUP.md.\")\n", + "\n", "import anthropic\n", "import json\n", "import time\n", diff --git a/requirements.txt b/requirements.txt index 1bff818..459c48b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ anthropic>=0.69 # every exercise — structured outputs, adaptive thinking, effort, batches ipykernel>=6 # notebook kernel +truststore>=0.9 # verify TLS against the OS store — corporate proxies (Zscaler et al.) break certifi # Day 1/03 Prompt Rescue · Day 2/02 Inference Optimization · archive/day2-03 matplotlib>=3.7 From e9ceefcb32d85becd92097acd4acec82747cba45 Mon Sep 17 00:00:00 2001 From: Morozov Date: Mon, 27 Jul 2026 18:51:38 -0400 Subject: [PATCH 2/3] Verify TLS against the OS trust store in the remaining notebooks 8548dd2 wired truststore into Developer_Platform.ipynb and added it to requirements.txt. Do the same for the other four exercises, so a corporate TLS-inspecting proxy (Zscaler et al.) doesn't break them either. Each setup cell now calls truststore.inject_into_ssl() before the Anthropic client is constructed, wrapped in a try/except that falls back to certifi with an explanatory message on Python < 3.10 or when the install was blocked. Saving through Jupyter also normalised these four files to canonical nbformat key order and split single-string `source` fields into line lists. That is the bulk of the diff; no cell content changed apart from the TLS block. Co-Authored-By: Claude Opus 5 (1M context) --- .../03_prompt-rescue/Prompt_Rescue_solo.ipynb | 121 ++++++++++----- .../Diagnose_Fix_Brief.ipynb | 37 +++-- day2/01_evals/Building_an_Eval.ipynb | 90 ++++++----- .../Inference_Optimization.ipynb | 140 +++++++++--------- 4 files changed, 235 insertions(+), 153 deletions(-) diff --git a/day1/03_prompt-rescue/Prompt_Rescue_solo.ipynb b/day1/03_prompt-rescue/Prompt_Rescue_solo.ipynb index 0aff85f..a991f61 100644 --- a/day1/03_prompt-rescue/Prompt_Rescue_solo.ipynb +++ b/day1/03_prompt-rescue/Prompt_Rescue_solo.ipynb @@ -1,15 +1,4 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { "cell_type": "markdown", @@ -44,9 +33,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "xdGQsghU0umI" }, + "outputs": [], "source": [ "# Install dependencies into THIS kernel — safe to re-run; survives locked-down (PEP 668) Pythons.\n", "import importlib.util, os, subprocess, sys\n", @@ -158,9 +149,7 @@ "\n", "_ensure_packages([(\"anthropic\", \"anthropic\"), (\"matplotlib\", \"matplotlib\")])\n", "print(\"✓ Dependencies ready\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -179,6 +168,14 @@ "source": [ "import os\n", "\n", + "try:\n", + " import truststore\n", + " truststore.inject_into_ssl()\n", + " print(\"✓ TLS: verifying against the OS certificate store\")\n", + "except Exception as _tls_exc: # Python < 3.10, or the install was blocked\n", + " print(\"[!!] truststore unavailable (\" + type(_tls_exc).__name__ + \") - falling back \"\n", + " \"to certifi. If the API check below fails with a certificate error, see SETUP.md.\")\n", + " \n", "def _status(ok, msg):\n", " \"\"\"Green/red banner in notebooks; plain text when run as a script.\"\"\"\n", " try:\n", @@ -331,9 +328,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "cOhoxjAX0umJ" }, + "outputs": [], "source": [ "# ✏️ YOUR TURN: edit this prompt\n", "# ============================================================\n", @@ -360,9 +359,7 @@ "\n", "print(\"Broken prompt loaded. Run the eval suite cell below to see your baseline score.\")\n", "print(f\"Prompt length: {len(system_prompt)} characters\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -452,10 +449,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "YOn3TQGX0umK" }, + "outputs": [], "source": [ "#@title Eval Harness (click to load -- do not edit this cell)\n", "\n", @@ -1125,24 +1124,22 @@ "print(\" run_eval_suite(prompt_or_prompts, label='...') -- Run full eval, display visual results\")\n", "print(\" compare_prompts(prompt_a, prompt_b, ...) -- A/B comparison of two variants\")\n", "print(\" show_iteration_log() -- Show all runs + score progression chart\")\n" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "aHn2fW1S0umM" }, + "outputs": [], "source": [ "# ============================================================\n", "# RUN YOUR FIRST EVAL -- Establish your baseline score\n", "# ============================================================\n", "\n", "results = run_eval_suite(system_prompt, label=\"Baseline (Broken Prompt)\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1220,7 +1217,17 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "---\n\n### Not sure which technique fits? Ask Claude.\n\nTake the failure pattern you just diagnosed and ask Claude:\n\n> \"I'm fixing a prompt that fails on [describe your failure mode]. Which of these would you recommend: meta-prompting, prompt chaining, few-shot examples, or constraint sharpening? Give me your reasoning for this specific failure pattern.\"\n\nThen make your own call. The point isn't whether Claude is right — it's whether its reasoning holds up against your diagnosis.\n\n---" + "---\n", + "\n", + "### Not sure which technique fits? Ask Claude.\n", + "\n", + "Take the failure pattern you just diagnosed and ask Claude:\n", + "\n", + "> \"I'm fixing a prompt that fails on [describe your failure mode]. Which of these would you recommend: meta-prompting, prompt chaining, few-shot examples, or constraint sharpening? Give me your reasoning for this specific failure pattern.\"\n", + "\n", + "Then make your own call. The point isn't whether Claude is right — it's whether its reasoning holds up against your diagnosis.\n", + "\n", + "---" ] }, { @@ -1272,9 +1279,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "nn2Y3T6i0umO" }, + "outputs": [], "source": [ "# ============================================================\n", "# CHAIN MODE -- Use this if you split into multiple steps\n", @@ -1297,15 +1306,15 @@ "\n", "# Run the chain (uncomment when ready)\n", "# results = run_eval_suite([step1_prompt, step2_prompt, step3_prompt], label=\"Chain v1\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "09BjBx--0umO" }, + "outputs": [], "source": [ "# ============================================================\n", "# A/B COMPARISON -- Compare two prompt variants\n", @@ -1319,15 +1328,15 @@ "\n", "# Run comparison (uncomment when ready)\n", "# result_a, result_b = compare_prompts(prompt_a, prompt_b, label_a=\"Current Best\", label_b=\"New Variant\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "HoYKlgLG0umP" }, + "outputs": [], "source": [ "# ============================================================\n", "# ITERATION LOG -- Track your progress across runs\n", @@ -1337,9 +1346,7 @@ "# Call this to see your iteration history and score progression:\n", "\n", "show_iteration_log()" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1463,15 +1470,57 @@ "metadata": {}, "outputs": [], "source": [ - "# ============================================================\n# PROCESS AUDIT -- Compare your approach vs. Claude's direct fix\n# ============================================================\n\nclaude_fix = \"\"\"\n[Paste Claude's direct rewrite here]\n\"\"\"\n\nresults_audit = run_eval_suite(claude_fix, label=\"Claude direct fix\")" + "# ============================================================\n", + "# PROCESS AUDIT -- Compare your approach vs. Claude's direct fix\n", + "# ============================================================\n", + "\n", + "claude_fix = \"\"\"\n", + "[Paste Claude's direct rewrite here]\n", + "\"\"\"\n", + "\n", + "results_audit = run_eval_suite(claude_fix, label=\"Claude direct fix\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## How to Present This to an Enterprise Buyer\n\nThe goal of this exercise isn't just a better score — it's building the confidence to walk a technical buyer through a real prompt engineering engagement. Here's how.\n\n### The Narrative Arc\n\n| Beat | What to Say | Why It Works |\n|------|-------------|--------------|\n| **The problem** | \"Your prompt does too many things at once, and the tasks interfere with each other.\" | Diagnoses the root cause, not the symptom. |\n| **The diagnosis** | \"I used Claude to analyze its own failure modes — here's what it identified.\" | Shows meta-prompting in action. Buyers respond to this. |\n| **The fix** | \"I decomposed it into a [N]-step chain: classify first, extract second, respond third. Each step is focused and can't pollute the others.\" | Demonstrates architectural thinking, not just prompt tweaking. |\n| **The proof** | \"Before: 19% pass rate. After: 86%. Here's the category breakdown — feature requests improved most because we're now evaluating content, not tone.\" | Quantitative. Specific. Defensible. |\n| **The prevention** | \"Here's a 3-question checklist your team can run on every new prompt before shipping to production.\" | Leaves them with something they can use. |\n\n### What Separates Good from Great\n\n- **Good:** \"I fixed the prompt and the score went up.\"\n- **Great:** \"I diagnosed the structural failure mode, chose a technique that specifically addresses it, and can show you the delta across each failure category.\"\n\nThe second version is what closes enterprise deals.\n\n### One Thing to Never Say\n\nNever say \"I just asked Claude to fix it.\" Even if meta-prompting helped, own the diagnostic judgment. You decided what to ask. You evaluated the result. You made the call on which fix to ship. That's the value you bring." + "## How to Present This to an Enterprise Buyer\n", + "\n", + "The goal of this exercise isn't just a better score — it's building the confidence to walk a technical buyer through a real prompt engineering engagement. Here's how.\n", + "\n", + "### The Narrative Arc\n", + "\n", + "| Beat | What to Say | Why It Works |\n", + "|------|-------------|--------------|\n", + "| **The problem** | \"Your prompt does too many things at once, and the tasks interfere with each other.\" | Diagnoses the root cause, not the symptom. |\n", + "| **The diagnosis** | \"I used Claude to analyze its own failure modes — here's what it identified.\" | Shows meta-prompting in action. Buyers respond to this. |\n", + "| **The fix** | \"I decomposed it into a [N]-step chain: classify first, extract second, respond third. Each step is focused and can't pollute the others.\" | Demonstrates architectural thinking, not just prompt tweaking. |\n", + "| **The proof** | \"Before: 19% pass rate. After: 86%. Here's the category breakdown — feature requests improved most because we're now evaluating content, not tone.\" | Quantitative. Specific. Defensible. |\n", + "| **The prevention** | \"Here's a 3-question checklist your team can run on every new prompt before shipping to production.\" | Leaves them with something they can use. |\n", + "\n", + "### What Separates Good from Great\n", + "\n", + "- **Good:** \"I fixed the prompt and the score went up.\"\n", + "- **Great:** \"I diagnosed the structural failure mode, chose a technique that specifically addresses it, and can show you the delta across each failure category.\"\n", + "\n", + "The second version is what closes enterprise deals.\n", + "\n", + "### One Thing to Never Say\n", + "\n", + "Never say \"I just asked Claude to fix it.\" Even if meta-prompting helped, own the diagnostic judgment. You decided what to ask. You evaluated the result. You made the call on which fix to ship. That's the value you bring." ] } - ] + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/day1/04_diagnosing-ai-problems/Diagnose_Fix_Brief.ipynb b/day1/04_diagnosing-ai-problems/Diagnose_Fix_Brief.ipynb index 725c61c..fa60cc5 100644 --- a/day1/04_diagnosing-ai-problems/Diagnose_Fix_Brief.ipynb +++ b/day1/04_diagnosing-ai-problems/Diagnose_Fix_Brief.ipynb @@ -54,8 +54,8 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Install dependencies into THIS kernel — safe to re-run; survives locked-down (PEP 668) Pythons.\n", @@ -192,6 +192,14 @@ " except Exception:\n", " print((\"[OK] \" if ok else \"[!!] \") + msg)\n", "\n", + "try:\n", + " import truststore\n", + " truststore.inject_into_ssl()\n", + " print(\"✓ TLS: verifying against the OS certificate store\")\n", + "except Exception as _tls_exc: # Python < 3.10, or the install was blocked\n", + " print(\"[!!] truststore unavailable (\" + type(_tls_exc).__name__ + \") - falling back \"\n", + " \"to certifi. If the API check below fails with a certificate error, see SETUP.md.\")\n", + "\n", "import os, pathlib\n", "\n", "# ── API key via .env (gitignored — never committed) ──\n", @@ -290,8 +298,8 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "from IPython.display import Markdown, display\n", @@ -301,12 +309,15 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## 2. Get your baseline\nRun the next cell. You'll get a RESOLVED rate and a cost for ticket T-4471. That rate is the pilot Priya's worried about, and it's your starting line. Write it down. Every scoreboard run is also recorded to `runs.jsonl` and replayed as a RUN HISTORY block under the scoreboard, so your baseline stays on screen for the rest of the exercise. (Give it a minute or two — it runs the ticket five times, and you'll see each trial land as it finishes.)" + "source": [ + "## 2. Get your baseline\n", + "Run the next cell. You'll get a RESOLVED rate and a cost for ticket T-4471. That rate is the pilot Priya's worried about, and it's your starting line. Write it down. Every scoreboard run is also recorded to `runs.jsonl` and replayed as a RUN HISTORY block under the scoreboard, so your baseline stays on screen for the rest of the exercise. (Give it a minute or two — it runs the ticket five times, and you'll see each trial land as it finishes.)" + ] }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "!{sys.executable} Diagnose_Fix_Brief.py --trials 5" @@ -322,8 +333,8 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "!{sys.executable} Diagnose_Fix_Brief.py --trials 5 --model claude-opus-4-8" @@ -341,8 +352,8 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "!{sys.executable} Diagnose_Fix_Brief.py --ticket T-4471" @@ -363,12 +374,15 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## 6. Change one thing, measure again\nFound something? Change it, save the file, and run the next cell. Did the rate move? The RUN HISTORY block under the scoreboard shows every run you've made — baseline included — so the before/after comparison is right there. If it didn't budge, your theory was off or only half-right, so go back to the trace. If it climbed, you found a real lever. Keep pulling until the agent resolves the ticket cleanly, run after run. Same model the whole way." + "source": [ + "## 6. Change one thing, measure again\n", + "Found something? Change it, save the file, and run the next cell. Did the rate move? The RUN HISTORY block under the scoreboard shows every run you've made — baseline included — so the before/after comparison is right there. If it didn't budge, your theory was off or only half-right, so go back to the trace. If it climbed, you found a real lever. Keep pulling until the agent resolves the ticket cleanly, run after run. Same model the whole way." + ] }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "!{sys.executable} Diagnose_Fix_Brief.py --trials 5" @@ -377,12 +391,15 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## 7. Make sure it holds\nIt's easy to fix one ticket by accident. The next cell runs held-out tickets: different customers, different problems. If your fix resolves those too, it's real. If it doesn't, you tuned it to T-4471 and you've got more to do." + "source": [ + "## 7. Make sure it holds\n", + "It's easy to fix one ticket by accident. The next cell runs held-out tickets: different customers, different problems. If your fix resolves those too, it's real. If it doesn't, you tuned it to T-4471 and you've got more to do." + ] }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "!{sys.executable} Diagnose_Fix_Brief.py --holdout --trials 3" diff --git a/day2/01_evals/Building_an_Eval.ipynb b/day2/01_evals/Building_an_Eval.ipynb index 70ef876..3037505 100644 --- a/day2/01_evals/Building_an_Eval.ipynb +++ b/day2/01_evals/Building_an_Eval.ipynb @@ -15,6 +15,11 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cTYbfH1bsitA" + }, + "outputs": [], "source": [ "# Install dependencies into THIS kernel — safe to re-run; survives locked-down (PEP 668) Pythons.\n", "import importlib.util, os, subprocess, sys\n", @@ -126,12 +131,7 @@ "\n", "_ensure_packages([(\"anthropic\", \"anthropic\")])\n", "print(\"✓ Dependencies ready\")" - ], - "metadata": { - "id": "cTYbfH1bsitA" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -144,12 +144,20 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", + "try:\n", + " import truststore\n", + " truststore.inject_into_ssl()\n", + " print(\"✓ TLS: verifying against the OS certificate store\")\n", + "except Exception as _tls_exc: # Python < 3.10, or the install was blocked\n", + " print(\"[!!] truststore unavailable (\" + type(_tls_exc).__name__ + \") - falling back \"\n", + " \"to certifi. If the API check below fails with a certificate error, see SETUP.md.\")\n", + "\n", "def _status(ok, msg):\n", " \"\"\"Green/red banner in notebooks; plain text when run as a script.\"\"\"\n", " try:\n", @@ -259,6 +267,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "BqUkU2rMsitA" + }, "source": [ "## Why Evals?\n", "\n", @@ -275,10 +286,7 @@ "3. **Run the eval** and inspect results systematically\n", "4. **Improve the agent** using eval results, then re-run to verify\n", "5. **Add an LLM-as-judge grader** for queries that can't be checked with simple string matching" - ], - "metadata": { - "id": "BqUkU2rMsitA" - } + ] }, { "cell_type": "markdown", @@ -311,6 +319,12 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8YmHBKf7sitA", + "outputId": "cc937ed4-28a6-4818-bd53-7fd4636860cc" + }, + "outputs": [], "source": [ "import math\n", "from anthropic import Anthropic\n", @@ -460,13 +474,7 @@ "\n", "\n", "print(\"boutique agent ready.\")" - ], - "metadata": { - "id": "8YmHBKf7sitA", - "outputId": "cc937ed4-28a6-4818-bd53-7fd4636860cc" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -489,6 +497,11 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SFHCc_FisitB" + }, + "outputs": [], "source": [ "ANTHROPIC_ORANGE = \"#E07A5F\" # brand accent — keeps the boutique agent from blending into the notebook's black-on-white output\n", "\n", @@ -528,12 +541,7 @@ " print(\"Session ended.\")\n", " break\n", " print(f\"\\nBoutique: {run_agent(query)}\")" - ], - "metadata": { - "id": "SFHCc_FisitB" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -570,10 +578,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "PurW11jPsitB", "outputId": "869b7657-0081-437b-eb09-56c0dc8e896a" }, + "outputs": [], "source": [ "# ── Graders (just run this cell) ──────────────────────────────────────────────\n", "\n", @@ -638,9 +648,7 @@ "}\n", "\n", "print(f\"Graders loaded: {list(GRADER_REGISTRY.keys())}\")" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", @@ -871,6 +879,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "QWjDyJmhsitB" + }, "source": [ "---\n", "\n", @@ -937,10 +948,7 @@ "- Which graders make sense? What checks?\n", "- What's the expected correct answer?\n", "- Is a `tool_use` check needed, or is checking the response enough?" - ], - "metadata": { - "id": "QWjDyJmhsitB" - } + ] }, { "cell_type": "markdown", @@ -953,9 +961,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "nSUg1UDWsitC" }, + "outputs": [], "source": [ "# ✏️ YOUR TURN — write your eval tasks in THIS list.\n", "# Each task needs an id, the query (prompt) to send to the agent, and graders with checks.\n", @@ -998,9 +1008,7 @@ " # ✏️ ADD YOUR TASKS BELOW\n", "\n", "]" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -1275,6 +1283,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "-EndmcG1sitC" + }, "source": [ "---\n", "\n", @@ -1302,19 +1313,16 @@ "- **Harbor.** Open-source eval framework focused on LLM safety.\n", "\n", "Internally, Anthropic uses **eval-toolbox** to add evals to our **eval-registry**. A natural follow-up to this session would be migrating your eval to one of these frameworks." - ], - "metadata": { - "id": "-EndmcG1sitC" - } + ] }, { "cell_type": "code", - "source": [], + "execution_count": null, "metadata": { "id": "_2ZScnO7WtgD" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [] } ], "metadata": { diff --git a/day2/02_inference-optimization/Inference_Optimization.ipynb b/day2/02_inference-optimization/Inference_Optimization.ipynb index 03bd69b..2c5e655 100644 --- a/day2/02_inference-optimization/Inference_Optimization.ipynb +++ b/day2/02_inference-optimization/Inference_Optimization.ipynb @@ -57,8 +57,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-002", "metadata": {}, + "outputs": [], "source": [ "# Install dependencies into THIS kernel — safe to re-run; survives locked-down (PEP 668) Pythons.\n", "import importlib.util, os, subprocess, sys\n", @@ -181,9 +183,7 @@ "from tabulate import tabulate\n", "\n", "import anthropic" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -201,11 +201,21 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-004", "metadata": {}, + "outputs": [], "source": [ "import os\n", "\n", + "try:\n", + " import truststore\n", + " truststore.inject_into_ssl()\n", + " print(\"✓ TLS: verifying against the OS certificate store\")\n", + "except Exception as _tls_exc: # Python < 3.10, or the install was blocked\n", + " print(\"[!!] truststore unavailable (\" + type(_tls_exc).__name__ + \") - falling back \"\n", + " \"to certifi. If the API check below fails with a certificate error, see SETUP.md.\")\n", + "\n", "def _status(ok, msg):\n", " \"\"\"Green/red banner in notebooks; plain text when run as a script.\"\"\"\n", " try:\n", @@ -311,14 +321,14 @@ " else:\n", " os.environ[\"ANTHROPIC_API_KEY\"] = api_key # later cells (and any !python commands) pick it up from here\n", " _status(True, \"API key verified - you're connected to Claude.\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "id": "cell-005", "metadata": {}, + "outputs": [], "source": [ "# The lab client: SDK defaults (long timeout, standard retries). The verification client\n", "# above uses a short 30s timeout — fine for a 1-token ping, fatal for v0's deliberately\n", @@ -332,9 +342,7 @@ "MODEL_OPUS = \"claude-opus-4-8\"\n", "\n", "print(f\"SDK {anthropic.__version__} · portfolio: {MODEL_HAIKU}, {MODEL_SONNET}, {MODEL_OPUS}\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -350,8 +358,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-007", "metadata": {}, + "outputs": [], "source": [ "@dataclass\n", "class BenchmarkResult:\n", @@ -401,9 +411,7 @@ "\n", "suite = BenchmarkSuite()\n", "print(\"BenchmarkSuite ready\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -418,8 +426,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-009", "metadata": {}, + "outputs": [], "source": [ "def _stream_request(messages, model, max_tokens=1024, system=None, **kwargs):\n", " \"\"\"Stream a request; return (ttft_seconds, total_seconds, final_message).\"\"\"\n", @@ -453,9 +463,7 @@ "otps, gen = compute_otps(ttft, total, resp.usage.output_tokens)\n", "print(f\"Response: {text_of(resp).strip()}\")\n", "print(f\"TTFT {ttft*1000:.0f}ms · TTC {total*1000:.0f}ms · OTPS {otps:.1f} tok/s\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -479,8 +487,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-011", "metadata": {}, + "outputs": [], "source": [ "PRICING = {\n", " MODEL_HAIKU: {\"input\": 1.00, \"output\": 5.00},\n", @@ -521,9 +531,7 @@ "\n", "\n", "print(f\"Cost of the 2+2 call above: ${calculate_cost(MODEL_SONNET, resp.usage):.6f}\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -539,8 +547,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-013", "metadata": {}, + "outputs": [], "source": [ "PROBE = \"What is machine learning? Answer in 2 sentences.\"\n", "\n", @@ -555,9 +565,7 @@ "\n", "print()\n", "print(suite.summary())" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -589,8 +597,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-016", "metadata": {}, + "outputs": [], "source": [ "# ── The contract sample (lab-scale stand-ins for the 248K-document estate) ──────────────\n", "CONTRACTS = [\n", @@ -689,9 +699,7 @@ "}\n", "\n", "print(f\"Loaded {len(CONTRACTS)} contracts with gold labels\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -705,8 +713,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-018", "metadata": {}, + "outputs": [], "source": [ "PLAYBOOK_CORE = \"\"\"You are ClauseScan, the contract-diligence reviewer for Project HELVETICA \\\n", "(Aldgate Capital Partners' acquisition of Volta Industrial Group). You review supplier \\\n", @@ -806,9 +816,7 @@ "\n", "PLAYBOOK = _build_playbook()\n", "print(f\"Playbook built: {len(PLAYBOOK):,} chars (~{len(PLAYBOOK)//4:,} tokens) — identical on every call\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -822,8 +830,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-020", "metadata": {}, + "outputs": [], "source": [ "def _normalize_cap(value):\n", " \"\"\"Coerce model output for liability_cap_usd into a float or None.\"\"\"\n", @@ -884,9 +894,7 @@ "\n", "\n", "print(\"Grader ready — 5 audited fields × 6 contracts = 30 checks; gate is ≥ 90% (27/30)\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -899,8 +907,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-022", "metadata": {}, + "outputs": [], "source": [ "EXTRACT_INSTRUCTION = (\n", " \"Determine the five audited fields for the contract below. Respond with a JSON object \"\n", @@ -947,14 +957,14 @@ "\n", "\n", "print(\"ClauseScan v0 loaded — run the portfolio in the next cell\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "id": "cell-023", "metadata": {}, + "outputs": [], "source": [ "def run_portfolio(pipeline, contracts=CONTRACTS, gold=GOLD, workers: int = 0,\n", " warm_first: bool = True) -> dict:\n", @@ -1018,9 +1028,7 @@ "sla_p50 = \"PASS\" if BASELINE[\"p50_s\"] <= 5 else \"FAIL\"\n", "print(f\"\\n SLA check: p50 ≤ 5s → {sla_p50} · accuracy ≥ 90% → \"\n", " f\"{'PASS' if BASELINE['accuracy'] >= 0.9 else 'FAIL'}\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1080,8 +1088,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-026", "metadata": {}, + "outputs": [], "source": [ "CACHED_SYSTEM = [{\"type\": \"text\", \"text\": PLAYBOOK, \"cache_control\": {\"type\": \"ephemeral\"}}]\n", "\n", @@ -1104,9 +1114,7 @@ " f\"cost=${calculate_cost(MODEL_SONNET, u):.5f}\")\n", "\n", "print(\"\\nWatch cache_read jump from 0 to ~the playbook size, and cost drop with it.\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1124,8 +1132,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-028", "metadata": {}, + "outputs": [], "source": [ "TRIAGE_SYSTEM = (\n", " \"You triage supplier contracts for a diligence pipeline. Classify the contract as \"\n", @@ -1166,9 +1176,7 @@ "print(tabulate(rows, headers=[\"ID\", \"Vendor\", \"Triage\", \"Triage cost\"], tablefmt=\"simple\"))\n", "print(\"\\nA few tenths of a cent buys the routing decision. The savings come from what it \"\n", " \"routes AWAY from Opus.\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1191,8 +1199,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-030", "metadata": {}, + "outputs": [], "source": [ "EXTRACTION_SCHEMA = {\n", " \"type\": \"json_schema\",\n", @@ -1243,9 +1253,7 @@ "print(f\"v0 on the same contract: {v0_row['elapsed']:.1f}s · ${v0_row['cost']:.4f} across \"\n", " f\"{v0_row['calls']} calls — most of it prose nobody reads.\")\n", "print(json.dumps(demo[\"fields\"], indent=2))" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1260,8 +1268,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-032", "metadata": {}, + "outputs": [], "source": [ "t0 = time.perf_counter()\n", "ttft, total, sresp = _stream_request(\n", @@ -1272,9 +1282,7 @@ "print(f\"Streamed single pass: TTFT {ttft*1000:.0f}ms · TTC {total*1000:.0f}ms\")\n", "print(f\"v0 two-pass TTC on this contract was {BASELINE['rows'][0]['elapsed']:.1f}s — \"\n", " f\"and its TTFT *was* its TTC, because nothing streamed.\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1294,17 +1302,17 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-034", "metadata": {}, + "outputs": [], "source": [ "seq = run_portfolio(lambda c: extract_structured(c, model=MODEL_SONNET), workers=0)\n", "par = run_portfolio(lambda c: extract_structured(c, model=MODEL_SONNET), workers=4)\n", "print(f\"Sequential portfolio wall-clock: {seq['wall_s']:.0f}s\")\n", "print(f\"Parallel (warm-first, 4 workers): {par['wall_s']:.0f}s\")\n", "print(f\"Accuracy held at {par['accuracy']*100:.0f}% — speed that costs accuracy is not speed.\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1327,8 +1335,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-036", "metadata": {}, + "outputs": [], "source": [ "from anthropic.types.message_create_params import MessageCreateParamsNonStreaming\n", "from anthropic.types.messages.batch_create_params import Request\n", @@ -1363,9 +1373,7 @@ " print(f\"Submitted batch {batch.id} — status: {batch.processing_status}\")\n", " print(\"Poll: client.messages.batches.retrieve(batch.id); \"\n", " \"results: client.messages.batches.results(batch.id)\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1406,8 +1414,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-039", "metadata": {}, + "outputs": [], "source": [ "# ✏️ YOUR TURN: edit this CONFIG\n", "CONFIG = {\n", @@ -1514,19 +1524,17 @@ "\n", "\n", "print(\"Sprint harness ready. Edit CONFIG above, then run the next cell. Repeat until proud.\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "code", + "execution_count": null, "id": "cell-040", "metadata": {}, + "outputs": [], "source": [ "my_report = run_scorecard(clausescan_v1)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1552,8 +1560,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-042", "metadata": {}, + "outputs": [], "source": [ "HOLDOUT_CONTRACTS = [\n", " {\n", @@ -1596,9 +1606,7 @@ "else:\n", " print(\"Holdout armed. Set RUN_HOLDOUT = True when your CONFIG is final — \"\n", " \"no tuning against the holdout; that's the rule.\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -1615,8 +1623,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "cell-044", "metadata": {}, + "outputs": [], "source": [ "ASSUMPTIONS = {\n", " \"contracts_in_estate\": 248_000,\n", @@ -1666,9 +1676,7 @@ "\n", "\n", "steering_committee_slide(BASELINE, my_report)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", From 3a2a7eacde21e082a342579dbb1cd0661e90dce6 Mon Sep 17 00:00:00 2001 From: Morozov Date: Mon, 27 Jul 2026 18:55:08 -0400 Subject: [PATCH 3/3] Install truststore from the notebooks and document the TLS failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps left open by the previous commit. The four notebooks imported truststore but never installed it: only Developer_Platform.ipynb listed it in _ensure_packages. Anyone running them in Colab or a fresh kernel without `pip install -r requirements.txt` landed on the certifi fallback — exactly the corporate-proxy case the shim exists to fix. Add ("truststore", "truststore") to each bootstrap list, keeping the same position after anthropic that Developer_Platform.ipynb uses. The fallback message tells readers to see SETUP.md, which said nothing about certificates. Add a CERTIFICATE_VERIFY_FAILED section covering why TLS inspection breaks certifi, how to read the "[!!] truststore unavailable" diagnostic, and — since it is the first thing people reach for — why not to disable verification, with SSL_CERT_FILE against an IT-supplied root CA as the supported alternative. Co-Authored-By: Claude Opus 5 (1M context) --- SETUP.md | 29 +++++++++++++++++++ .../03_prompt-rescue/Prompt_Rescue_solo.ipynb | 2 +- .../Diagnose_Fix_Brief.ipynb | 2 +- day2/01_evals/Building_an_Eval.ipynb | 2 +- .../Inference_Optimization.ipynb | 2 +- 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/SETUP.md b/SETUP.md index 71f69b5..f55c781 100644 --- a/SETUP.md +++ b/SETUP.md @@ -83,6 +83,35 @@ pip install -r requirements.txt If PyPI itself is blocked, ask IT for your internal package mirror and use it: `pip install -r requirements.txt --index-url https:///simple`. +### `CERTIFICATE_VERIFY_FAILED` when the notebook calls the API +**Why:** your company inspects TLS traffic (Zscaler, Netskope, a corporate firewall), so +the certificate the notebook actually receives is signed by an internal root CA. IT has +already installed that root in your operating system's certificate store — but Python's +HTTP stack ignores the OS store and trusts only the `certifi` bundle shipped with pip, +which has never heard of it. Every API call fails on the handshake. + +**Fix:** usually nothing — the setup cell calls +[`truststore`](https://pypi.org/project/truststore/) before creating the client, which +points Python's TLS verification at the OS store where your corporate root already lives. +You'll see `✓ TLS: verifying against the OS certificate store` and everything just works. + +If instead you see `[!!] truststore unavailable`, the shim didn't load, and the error is +in the parentheses: +- `ModuleNotFoundError` — the install didn't reach it. Run `pip install truststore` in the + same environment as your kernel (see "Wrong kernel selected" above), then re-run the cell. +- `ImportError` / anything else — you're most likely on Python 3.9 or older; `truststore` + needs **3.10+**. See "wrong version" below. + +**Please don't** set `verify=False`, `SSL_CERT_FILE=""`, or `PYTHONHTTPSVERIFY=0`. Those +turn off certificate checking altogether rather than fixing which certificates you trust, +and on a corporate machine that's a security-policy violation, not a workaround. If +`truststore` can't be made to work, ask IT to export the internal root CA as a `.pem` and +point Python at it properly: +```bash +export SSL_CERT_FILE=/path/to/corporate-root.pem +export REQUESTS_CA_BUNDLE=$SSL_CERT_FILE +``` + ### "I don't have Python" / wrong version You need **Python 3.10+**. Install from [python.org](https://www.python.org/downloads/) or your company's software portal, then re-open VS Code so it's detected. Check with diff --git a/day1/03_prompt-rescue/Prompt_Rescue_solo.ipynb b/day1/03_prompt-rescue/Prompt_Rescue_solo.ipynb index a991f61..7e9881c 100644 --- a/day1/03_prompt-rescue/Prompt_Rescue_solo.ipynb +++ b/day1/03_prompt-rescue/Prompt_Rescue_solo.ipynb @@ -147,7 +147,7 @@ " f\" (pip said: {tail})\\n\"\n", " )\n", "\n", - "_ensure_packages([(\"anthropic\", \"anthropic\"), (\"matplotlib\", \"matplotlib\")])\n", + "_ensure_packages([(\"anthropic\", \"anthropic\"), (\"truststore\", \"truststore\"), (\"matplotlib\", \"matplotlib\")])\n", "print(\"✓ Dependencies ready\")" ] }, diff --git a/day1/04_diagnosing-ai-problems/Diagnose_Fix_Brief.ipynb b/day1/04_diagnosing-ai-problems/Diagnose_Fix_Brief.ipynb index fa60cc5..3d8d7d6 100644 --- a/day1/04_diagnosing-ai-problems/Diagnose_Fix_Brief.ipynb +++ b/day1/04_diagnosing-ai-problems/Diagnose_Fix_Brief.ipynb @@ -166,7 +166,7 @@ " f\" (pip said: {tail})\\n\"\n", " )\n", "\n", - "_ensure_packages([(\"anthropic\", \"anthropic\")])\n", + "_ensure_packages([(\"anthropic\", \"anthropic\"), (\"truststore\", \"truststore\")])\n", "print(\"✓ Dependencies ready\")\n", "import os\n", "# Make sure we are in the exercise folder (handles a kernel that starts at the repo root)\n", diff --git a/day2/01_evals/Building_an_Eval.ipynb b/day2/01_evals/Building_an_Eval.ipynb index 3037505..0902cf0 100644 --- a/day2/01_evals/Building_an_Eval.ipynb +++ b/day2/01_evals/Building_an_Eval.ipynb @@ -129,7 +129,7 @@ " f\" (pip said: {tail})\\n\"\n", " )\n", "\n", - "_ensure_packages([(\"anthropic\", \"anthropic\")])\n", + "_ensure_packages([(\"anthropic\", \"anthropic\"), (\"truststore\", \"truststore\")])\n", "print(\"✓ Dependencies ready\")" ] }, diff --git a/day2/02_inference-optimization/Inference_Optimization.ipynb b/day2/02_inference-optimization/Inference_Optimization.ipynb index 2c5e655..4fce77b 100644 --- a/day2/02_inference-optimization/Inference_Optimization.ipynb +++ b/day2/02_inference-optimization/Inference_Optimization.ipynb @@ -170,7 +170,7 @@ " f\" (pip said: {tail})\\n\"\n", " )\n", "\n", - "_ensure_packages([(\"anthropic\", \"anthropic\"), (\"tabulate\", \"tabulate\")])\n", + "_ensure_packages([(\"anthropic\", \"anthropic\"), (\"truststore\", \"truststore\"), (\"tabulate\", \"tabulate\")])\n", "print(\"✓ Dependencies ready\")\n", "\n", "import os\n",