← The evidence ledger
Evidence row 990677
Observed , by probe.mcp/1.
- Kind
- mcp.tools
- About
- endpoint 674aa99f-e86c-46f9-b8c2-5321b2d469b4
- Recorded
- Previous row
- #777768
- Body SHA-256
- 30e381eb09e8bac8d6ec2246cfdf475141f1fbac04ac7298fe722f9dadf54f41
- Leaf
- 0xf5950014be495f743c0b6637eb62ed70a3312cfabfbedc564901610f19ced59b
What was observed
{
"count": 38,
"tools": [
{
"description": "Generate an A/B test report comparing two prompts or model configurations. Accepts arrays of scores and returns mean, median, std deviation, improvement percentage, and a Welch t-test at 95 %: `winner` names a variant only when the difference is significant — otherwise \"no_significant_difference\", or \"insufficient_samples\" under 2 scores per variant.",
"inputSchema": {
"properties": {
"variant_a": {
"description": "First variant configuration with name and score array",
"properties": {
"name": {
"description": "Name/label for variant A",
"type": "string"
},
"scores": {
"description": "Array of scores (0-100)",
"items": {
"type": "number"
},
"type": "array"
}
},
"type": "object"
},
"variant_b": {
"description": "Second variant configuration with name and score array",
"properties": {
"name": {
"description": "Name/label for variant B",
"type": "string"
},
"scores": {
"description": "Array of scores (0-100)",
"items": {
"type": "number"
},
"type": "array"
}
},
"type": "object"
}
},
"required": [
"variant_a",
"variant_b"
],
"type": "object"
},
"name": "ab_test_report"
},
{
"description": "Semantically analyze N already-produced model outputs for the SAME task (the MCP counterpart to the LLM Sandbox). Without a reference: computes consensus — pairwise cosine agreement, the most-representative output, and the outlier. With a `reference` (ground truth): also ranks every output by closeness (token cosine + ROUGE-L composite) and names the closest. No LLM, no key, runs locally. The reference mode is gate-able in CI; the consensus/outlier mode is a signal (no reference to fail against). You bring the outputs (2+). For a 2-way head-to-head with structural JSON diff use compare_responses instead.",
"inputSchema": {
"properties": {
"reference": {
"description": "Optional ground-truth answer. If set, each output is also ranked by closeness to it and the closest one is named.",
"type": "string"
},
"responses": {
"description": "The outputs to analyze (same task, N models/prompts/versions). Each item is a plain string or { \"label\": \"GPT-4o\", \"text\": \"...\" }. At least 2 required.",
"items": {
"properties": {
"label": {
"description": "Human name for this candidate (e.g. model id)",
"type": "string"
},
"text": {
"description": "The produced output",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"minItems": 2,
"type": "array"
}
},
"required": [
"responses"
],
"type": "object"
},
"name": "analyze_responses"
},
{
"description": "Compute BM25 relevance score between a query and one or more documents. BM25 is the industry-standard keyword-based ranking algorithm used in Elasticsearch, OpenSearch, and Weaviate hybrid search. Returns ranked results with normalized scores.",
"inputSchema": {
"properties": {
"b": {
"description": "Length normalization factor (default: 0.75)",
"type": "number"
},
"documents": {
"description": "Array of documents to rank",
"items": {
"type": "string"
},
"type": "array"
},
"k1": {
"description": "Term frequency saturation (default: 1.5)",
"type": "number"
},
"query": {
"description": "The search query",
"type": "string"
},
"top_k": {
"description": "Return top K results (default: all)",
"type": "number"
}
},
"required": [
"query",
"documents"
],
"type": "object"
},
"name": "bm25_score"
},
{
"description": "Assemble a complete RAG (Retrieval-Augmented Generation) prompt from retrieved context chunks and a user query. Handles token budgeting, citation numbering, system instruction injection, and source attribution.",
"inputSchema": {
"properties": {
"chunks": {
"description": "Retrieved context chunks with .text (required), .source (optional), .score (optional)",
"items": {
"properties": {
"score": {
"type": "number"
},
"source": {
"type": "string"
},
"text": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"cite_sources": {
"description": "Add [1], [2] citation numbers (default: true)",
"type": "boolean"
},
"language": {
"description": "Response language instruction (e.g. \"French\", \"Spanish\")",
"type": "string"
},
"max_context_tokens": {
"description": "Max tokens for context section (default: 2000)",
"type": "number"
},
"query": {
"description": "The user question to answer",
"type": "string"
},
"system_instruction": {
"description": "Custom system instruction (default: standard RAG grounding instruction)",
"type": "string"
}
},
"required": [
"query",
"chunks"
],
"type": "object"
},
"name": "build_rag_prompt"
},
{
"description": "Compare 2-5 AI models side by side: context window, pricing, multimodal, reasoning capabilities, and provider. Returns a comparison table with a recommendation based on your use case.",
"inputSchema": {
"properties": {
"models": {
"description": "Array of 2-5 model names (e.g. [\"gpt-4o\",\"claude-3.5-sonnet\",\"gemini-3.5-flash\"])",
"items": {
"type": "string"
},
"type": "array"
},
"use_case": {
"description": "Optimize recommendation for this criterion",
"enum": [
"cost",
"context",
"reasoning",
"multimodal",
"speed"
],
"type": "string"
}
},
"required": [
"models"
],
"type": "object"
},
"name": "compare_models"
},
{
"description": "Compare two ALREADY-PRODUCED outputs (e.g. model A vs model B on the same task) side by side. Returns deterministic metrics (token cosine, ROUGE-L, Jaccard, length/structure deltas, JSON diff) and a verdict. If a `reference` (ground truth) is given, scores each output against it and picks the closer one. If `model` + `api_key` are given, an LLM judge also picks a qualitative winner for the task. No re-execution — you bring the outputs.",
"inputSchema": {
"properties": {
"api_key": {
"description": "Optional API key for the judge model (BYOK). Used only for the judge call; never stored.",
"type": "string"
},
"check_json": {
"description": "Try to parse as JSON and compare structurally (keys, types, values)",
"type": "boolean"
},
"label_a": {
"description": "Label for output A (e.g. \"GPT-4o\", \"v1.0\")",
"type": "string"
},
"label_b": {
"description": "Label for output B (e.g. \"GPT-5-nano\", \"v1.1\")",
"type": "string"
},
"model": {
"description": "Optional judge model id (BYOK). When set with api_key, an LLM judge picks a qualitative winner.",
"type": "string"
},
"reference": {
"description": "Optional ground-truth / expected answer. If set, each output is scored against it and the closer one wins (deterministic).",
"type": "string"
},
"response_a": {
"description": "First output (e.g. model A's answer)",
"type": "string"
},
"response_b": {
"description": "Second output (e.g. model B's answer)",
"type": "string"
},
"task": {
"description": "The task/prompt both outputs were answering — used by the LLM judge for context",
"type": "string"
}
},
"required": [
"response_a",
"response_b"
],
"type": "object"
},
"name": "compare_responses"
},
{
"description": "Compare multiple LLM responses to the same prompt and detect inconsistencies using Jaccard word-overlap similarity and fact drift (number comparison). Fast, deterministic, no API key needed. Limitations: relies on surface-level word matching — \"Paris is the capital of France\" vs \"Paris is the French capital\" may score low despite semantic equivalence. For true semantic consistency, use run_semantic_tests with embedding mode. Essential for determinism testing.",
"inputSchema": {
"properties": {
"check_facts": {
"description": "Check for contradictory numbers/facts across responses (default: true)",
"type": "boolean"
},
"responses": {
"description": "Array of 2+ LLM responses to compare (same prompt, different runs)",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"responses"
],
"type": "object"
},
"name": "consistency_check"
},
{
"description": "Given an array of message objects [{role, content}], estimate total token usage and check if it fits in the target model's context window. Warns about truncation risk.",
"inputSchema": {
"properties": {
"max_output_tokens": {
"description": "Reserved tokens for output (default: 4096)",
"type": "number"
},
"messages": {
"description": "Array of messages (system/user/assistant)",
"items": {
"properties": {
"content": {
"type": "string"
},
"role": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"model": {
"description": "Target model name (e.g. gpt-4o, claude-3.5-sonnet)",
"type": "string"
}
},
"required": [
"messages",
"model"
],
"type": "object"
},
"name": "context_window_check"
},
{
"description": "Analyze a multi-turn conversation for context retention, topic drift, instruction following, and repetition. Accepts messages array [{role, content}]. Essential for chatbot QA.",
"inputSchema": {
"properties": {
"messages": {
"description": "Conversation messages in order",
"items": {
"properties": {
"content": {
"type": "string"
},
"role": {
"enum": [
"system",
"user",
"assistant"
],
"type": "string"
}
},
"type": "object"
},
"type": "array"
}
},
"required": [
"messages"
],
"type": "object"
},
"name": "conversation_analyze"
},
{
"description": "Analyze a Chain-of-Thought (CoT) or reasoning trace from an LLM. Detects step count, logical flow, conclusion presence, backtracking, and estimates reasoning depth. Useful for o1/o3/DeepSeek-R1 evaluation.",
"inputSchema": {
"properties": {
"expected_conclusion": {
"description": "Expected final answer to check against (optional)",
"type": "string"
},
"reasoning": {
"description": "The CoT / reasoning trace text (e.g. from <think> tags or step-by-step output)",
"type": "string"
}
},
"required": [
"reasoning"
],
"type": "object"
},
"name": "cot_analyzer"
},
{
"description": "Estimate the token count of a text string using the cl100k_base approximation (~4 chars/token). Call this BEFORE sending any text to an LLM API to check if it fits within the model context window and to estimate cost. Returns token estimate, character count, and word count.",
"inputSchema": {
"properties": {
"input": {
"description": "Text to count tokens for",
"type": "string"
}
},
"required": [
"input"
],
"type": "object"
},
"name": "count_tokens"
},
{
"description": "Compute text similarity using local algorithms (Bag of Words, TF-IDF, Character N-grams). No API key needed — runs entirely in-process. NOT real embeddings: for true semantic similarity with vector embeddings, use run_semantic_tests with mode=\"embeddings\" and your OpenAI API key. Supports single pair or batch mode with pipe-separated pairs. Useful for RAG retrieval testing, semantic search evaluation, and text deduplication.",
"inputSchema": {
"properties": {
"batch": {
"description": "Batch mode: array of { text_a, text_b } pairs. Overrides text_a/text_b if provided.",
"items": {
"properties": {
"text_a": {
"type": "string"
},
"text_b": {
"type": "string"
}
},
"required": [
"text_a",
"text_b"
],
"type": "object"
},
"type": "array"
},
"methods": {
"description": "Algorithms to use (default: all three). Options: \"bow\", \"tfidf\", \"ngram\"",
"items": {
"enum": [
"bow",
"tfidf",
"ngram"
],
"type": "string"
},
"type": "array"
},
"text_a": {
"description": "First text to compare (single-pair mode)",
"type": "string"
},
"text_b": {
"description": "Second text to compare (single-pair mode)",
"type": "string"
}
},
"type": "object"
},
"name": "embedding_similarity"
},
{
"description": "Estimate the API cost in USD for a given model and token counts. Supports all major 2024–2026 models: GPT-4o, GPT-4.1, o3, o4-mini, Claude Opus 4, Claude Sonnet 4/4.5, Gemini 2.5 Pro/Flash, DeepSeek V3/R1, Grok 3, and legacy models.",
"inputSchema": {
"properties": {
"input_tokens": {
"description": "Number of input/prompt tokens",
"type": "number"
},
"model": {
"description": "Model name, e.g. \"gpt-4o\", \"claude-3.5-sonnet\", \"deepseek-v3\"",
"type": "string"
},
"output_tokens": {
"description": "Number of output/completion tokens (default: 0)",
"type": "number"
}
},
"required": [
"model",
"input_tokens"
],
"type": "object"
},
"name": "estimate_llm_cost"
},
{
"description": "Extract the first valid JSON object or array embedded in chaotic LLM output (surrounded by markdown fences, prose, or explanatory text). Handles ```json blocks and inline JSON. Call this whenever an LLM returns structured data mixed with explanation text instead of raw JSON.",
"inputSchema": {
"properties": {
"input": {
"description": "Raw text (e.g., LLM output) that may contain a JSON object or array",
"type": "string"
}
},
"required": [
"input"
],
"type": "object"
},
"name": "extract_json_from_text"
},
{
"description": "Search available MCP tools by keyword or category before calling them. Returns matching tool names, descriptions, and optionally their inputSchemas. Call this when you are unsure which tool to use or want to explore the catalogue. Categories: data, encoding, text, llm, qa, rag, dev, security, web.",
"inputSchema": {
"properties": {
"category": {
"description": "Optional: filter by category — data | encoding | text | llm | qa | rag | dev | security | web",
"type": "string"
},
"max_results": {
"description": "Maximum tools to return (default 10, max 50). Results are ranked by IDF-weighted relevance, so common words like \"test\" do not inflate the list.",
"type": "number"
},
"query": {
"description": "Keyword(s) to search in tool name and description (e.g. \"cors\", \"token\", \"vector\", \"json\")",
"type": "string"
},
"with_schema": {
"description": "Set true to include inputSchema in results (default: false)",
"type": "boolean"
}
},
"required": [
"query"
],
"type": "object"
},
"name": "find_tool"
},
{
"description": "Generate a ready-to-commit GitHub Actions workflow that gates a build on IA-QA. Two gate types, combinable: \"eval_contract\" runs a .ia-eval.yaml through ia-qa-com/eval-action@v1 (LLM quality gate, needs a provider API key as a repo secret), and \"cli_checks\" runs deterministic primitives via npx @ia-qa/cli (secret scan, prompt-injection scan, security headers…) whose exit code fails the build. Deterministic template — no LLM call, no API key, same inputs give the same file. Returns the YAML, the secrets to create, and the remaining steps. Pair with generate_eval_yaml to produce the contract itself.",
"inputSchema": {
"properties": {
"app_base_url": {
"description": "URL the drift gate waits for before capturing (default: http://127.0.0.1:3000). Must match config.baseUrl in .ia-qa/config.json.",
"type": "string"
},
"app_start_command": {
"description": "Command that boots the app for the selector_drift gate, e.g. \"npm run start:ci\". Cannot be guessed — omitted, the step carries an explicit TODO and the note says so, because a plausible default would silently map nothing.",
"type": "string"
},
"cli_tools": {
"description": "IA-QA tool names to run as deterministic gates, e.g. [\"secret_scan\",\"prompt_injection_scan\"]. Tools with no known CI recipe get a --stdin step flagged in notes.",
"items": {
"type": "string"
},
"type": "array"
},
"contract_path": {
"description": "Path to the .ia-eval.yaml contract, relative to the repo root (default: evals/smoke.ia-eval.yaml). Only used when the gate includes eval_contract.",
"type": "string"
},
"cron": {
"description": "Cron expression when triggers include 'schedule' (default: '0 6 * * 1' — Mondays 06:00 UTC).",
"type": "string"
},
"fail_on_fail": {
"description": "Fail the build on a FAIL/PARTIAL verdict (default: true). Set false to report without gating.",
"type": "boolean"
},
"gate": {
"description": "Which gate to emit. eval_contract = LLM eval via the action (default). cli_checks = deterministic CLI assertions. selector_drift = an E2E selector-drift gate via @ia-qa/self-healing (boots the app, captures, diffs against the committed baseline, branches on exit code 0/1/2). both = CLI checks + eval. all = CLI checks, then drift, then eval.",
"enum": [
"eval_contract",
"cli_checks",
"both",
"selector_drift",
"all"
],
"type": "string"
},
"min_score": {
"description": "Override the contract min_score (0-100). Omit to use the value in the contract.",
"type": "number"
},
"node_version": {
"description": "Node version for the CLI steps (default: \"20\").",
"type": "string"
},
"provider": {
"description": "LLM provider the contract runs against — decides which repository secret the workflow wires (default: groq).",
"enum": [
"groq",
"openai",
"anthropic",
"google"
],
"type": "string"
},
"triggers": {
"description": "Workflow triggers (default: push + pull_request).",
"items": {
"enum": [
"push",
"pull_request",
"workflow_dispatch",
"schedule"
],
"type": "string"
},
"type": "array"
},
"workflow_name": {
"description": "Workflow display name (default: \"IA-QA Quality Gate\").",
"type": "string"
}
},
"type": "object"
},
"name": "generate_ci_workflow"
},
{
"description": "Generate a complete .ia-eval.yaml evaluation contract from a plain-language description of what your LLM should do. Uses Groq openai/gpt-oss-20b (server-side, no API key needed). Returns ready-to-run YAML for the LLM Test Runner (run_eval_contract). Picks appropriate evaluators (cosine_similarity, contains_check, hallucination_check, etc.) based on the task type.",
"inputSchema": {
"properties": {
"description": {
"description": "Plain-language description of what the LLM under test should do. Be specific: describe inputs, expected behaviour, and constraints.",
"type": "string"
},
"scenario_count": {
"description": "Number of scenarios to generate (default: 5). Covers happy path + edge cases + adversarial.",
"enum": [
3,
5,
8
],
"type": "number"
},
"system_prompt": {
"description": "Optional system prompt of the LLM under test. Helps generate more accurate test cases.",
"type": "string"
},
"task_type": {
"description": "Optional task type hint to guide evaluator selection.",
"enum": [
"rag",
"summarization",
"classification",
"safety",
"customer_support",
"code_gen"
],
"type": "string"
}
},
"required": [
"description"
],
"type": "object"
},
"name": "generate_eval_yaml"
},
{
"description": "Query the IA-QA methodology knowledge base. Returns structured testing guidelines, assertion strategies, thresholds, best practices, and relevant MCP tools for a given topic. Call without a topic to list all available topics. Topics: llm-unit-testing, rag-pipeline, prompt-stability, prompt-ab-testing, embedding-quality, eval-framework, semantic-testing, auto-testing, security, api-testing, ci-cd, multimodal, llm-data-security, agent-observability, pro-tips, learning-paths, golden-dataset, selector-drift, qa-recipes, playbooks. Not sure where to start testing an LLM, RAG pipeline or agent? Call without a topic (or with \"start-here\"): it maps what you are testing to the tools to call and the output field to gate CI on. A plain question such as \"how do I test my RAG\" also resolves to the right topic.",
"inputSchema": {
"properties": {
"topic": {
"description": "The testing topic to retrieve guidelines for. Omit to get the start-here map and the full list of available topics.",
"enum": [
"start-here",
"llm-unit-testing",
"rag-pipeline",
"prompt-stability",
"prompt-ab-testing",
"embedding-quality",
"eval-framework",
"semantic-testing",
"auto-testing",
"security",
"api-testing",
"ci-cd",
"multimodal",
"llm-data-security",
"agent-observability",
"pro-tips",
"learning-paths",
"golden-dataset",
"selector-drift",
"qa-recipes",
"playbooks"
],
"type": "string"
}
},
"type": "object"
},
"name": "get_testing_guidelines"
},
{
"description": "Lexical hallucination check: verifies an LLM answer's words, numbers and polarity against the provided source/context. Fast, deterministic, no API key needed. Each answer sentence is aligned to its best-matching source sentence, so a number only counts as support when it sits on the SAME statement (\"founded in 1998\" is not grounded by \"sold 1998 units\"), and a negation or antonym flip against that sentence returns verdict \"contradicted\" — the corrupted-fact hallucination that reuses source vocabulary. Limitations: still lexical — it cannot follow a paraphrase, a synonym, or multi-sentence reasoning, so a \"well_grounded\" verdict means \"nothing lexical found\", never \"verified true\". For entailment use run_semantic_tests (NLI/embedding) or a calibrated judge.",
"inputSchema": {
"properties": {
"answer": {
"description": "The LLM-generated answer to verify",
"type": "string"
},
"context": {
"description": "The source/reference text that should ground the answer",
"type": "string"
},
"strict": {
"description": "If true, every sentence in the answer must be supported (default: false)",
"type": "boolean"
}
},
"required": [
"answer",
"context"
],
"type": "object"
},
"name": "hallucination_check"
},
{
"description": "List all LLM models available on ia-qa.com with their provider, API endpoint, and capabilities. Filter by provider name (e.g. \"Groq\", \"HuggingFace\", \"OpenAI\") or return the full catalog. Use this to discover which models are available before calling an LLM API, or to compare providers.",
"inputSchema": {
"properties": {
"provider": {
"description": "Filter by provider name (case-insensitive). E.g. \"Groq\", \"HuggingFace\", \"OpenAI\", \"Anthropic\", \"Google\", \"DeepSeek\", \"xAI\", \"Ollama\". Omit for full catalog.",
"type": "string"
}
},
"type": "object"
},
"name": "list_llm_models"
},
{
"description": "Validate that an LLM output matches an expected format: JSON, Markdown, code block, bullet list, numbered list, table, YAML, XML, or custom regex. Essential for structured output testing.",
"inputSchema": {
"properties": {
"expected_format": {
"description": "Expected format",
"enum": [
"json",
"markdown_heading",
"code_block",
"bullet_list",
"numbered_list",
"table",
"yaml",
"xml",
"regex"
],
"type": "string"
},
"output": {
"description": "The LLM output to validate",
"type": "string"
},
"regex_pattern": {
"description": "Custom regex pattern (only when expected_format is \"regex\")",
"type": "string"
}
},
"required": [
"output",
"expected_format"
],
"type": "object"
},
"name": "llm_format_check"
},
{
"description": "Generate text with an open-source model hosted on Groq, on the server's key — no API key needed. Models: openai/gpt-oss-20b, openai/gpt-oss-120b (default \"openai/gpt-oss-20b\"). HuggingFace models are not served on the server key. A reasoning model that spends max_tokens thinking is retried once at 4096. Rate-limited to prevent abuse.",
"inputSchema": {
"properties": {
"max_tokens": {
"description": "Maximum tokens to generate (default: 2048, max: 4096)",
"type": "number"
},
"model": {
"description": "Model ID (default: \"openai/gpt-oss-20b\"). Server-keyed whitelist only — Groq: openai/gpt-oss-20b, openai/gpt-oss-120b. Other ids from list_llm_models, HuggingFace ones included, are BYOK-only and will be rejected.",
"type": "string"
},
"prompt": {
"description": "The user prompt / instruction to send to the model",
"type": "string"
},
"system": {
"description": "Optional system prompt to set context or persona",
"type": "string"
},
"temperature": {
"description": "Sampling temperature 0.0–1.5 (default: 0.7)",
"type": "number"
}
},
"required": [
"prompt"
],
"type": "object"
},
"name": "llm_generate"
},
{
"description": "Validate that an LLM JSON output matches a JSON Schema definition. Tests required fields, types, enums, nested objects, and arrays. Critical for function-calling and structured output testing.",
"inputSchema": {
"properties": {
"output": {
"description": "The LLM JSON output (raw string, will be parsed)",
"type": "string"
},
"schema": {
"additionalProperties": true,
"description": "JSON Schema (draft-07 subset) to validate against",
"type": "object"
}
},
"required": [
"output",
"schema"
],
"type": "object"
},
"name": "llm_json_schema_check"
},
{
"description": "Validate an LLM response against QA criteria: format checks (JSON, code, markdown), content rules (must-include, must-not-include), length constraints, language detection, and safety patterns. Essential for QA testing LLM-powered features.",
"inputSchema": {
"properties": {
"check_json_schema": {
"description": "If expected_format is JSON, provide required keys as comma-separated list to validate the structure",
"type": "string"
},
"check_safety": {
"description": "Check for PII patterns (emails, phones, SSN), profanity signals, and prompt leakage",
"type": "boolean"
},
"expected_format": {
"description": "Expected output format",
"enum": [
"json",
"markdown",
"code",
"plain",
"any"
],
"type": "string"
},
"expected_language": {
"description": "Expected language of the output (en, fr, es, de…). Checks for common words.",
"type": "string"
},
"max_length": {
"description": "Maximum character length for the output",
"type": "number"
},
"min_length": {
"description": "Minimum character length for the output",
"type": "number"
},
"must_include": {
"description": "Comma-separated strings that MUST appear in the output",
"type": "string"
},
"must_not_include": {
"description": "Comma-separated strings that must NOT appear (e.g. \"TODO, FIXME, undefined, NaN\")",
"type": "string"
},
"output": {
"description": "The LLM output text to validate",
"type": "string"
}
},
"required": [
"output"
],
"type": "object"
},
"name": "llm_output_validator"
},
{
"description": "Reference-free stability primitive: instead of comparing an answer to a ground truth, it checks that an assistant's answer stays INVARIANT when the QUESTION is transformed (typo, casing, paraphrase, reordering, translation). Catches the failure class no reference answer can expose — an assistant that handles one phrasing well and a trivial variant of it badly. You bring the outputs (no model is called), so it is deterministic and free in tfidf mode. Relations: case (θ .95), typo (.90), paraphrase (.80), reorder (.80), translation (.75, embeddings only), specialization (.60, ADVISORY — directional, never gated). Returns PASS / FAIL / INVALID, where INVALID means the BASE answer was a refusal or too short so invariance was never measurable — an assistant that refuses every variant would otherwise score a perfect 1.0. Use run_semantic_tests alongside it: invariance without a correctness floor is a green light for a broken assistant.",
"inputSchema": {
"properties": {
"api_key": {
"description": "OpenAI API key — required only when mode is embeddings.",
"type": "string"
},
"base": {
"description": "The reference run: the original question and the answer your system produced for it.",
"properties": {
"output": {
"description": "Required — the answer your system produced for the original question.",
"type": "string"
},
"question": {
"description": "Optional — the original question, echoed back in the output for readability.",
"type": "string"
}
},
"type": "object"
},
"baseline_guard": {
"description": "Correctness floor applied to the BASE answer before anything is scored. Failing it returns INVALID, not FAIL.",
"properties": {
"min_length": {
"description": "Minimum base answer length in chars (default 40).",
"type": "number"
},
"must_not_match": {
"description": "Refusal patterns, matched case-insensitively in the FIRST 200 CHARS of the base answer (refusals lead; matching anywhere would flag a long correct answer that merely mentions one). Replaces the FR+EN default list, never merges — pass [] to disable.",
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
},
"mode": {
"description": "tfidf (default): free, lexical, deterministic — but a genuine paraphrase rarely reaches 0.80, so gate on case/typo and treat paraphrase as a trend. embeddings: OpenAI text-embedding-3-small, true semantic similarity, requires api_key. translation requires this mode.",
"enum": [
"tfidf",
"embeddings"
],
"type": "string"
},
"require_all": {
"description": "If true (default), every gated variant must pass. KEEP THE DEFAULT for any run you gate on. Setting it false is not a tolerance dial but an off switch: relations have asymmetric pass rates (a typo variant usually scores ~1.0 because the answer really is identical), so one trivial row is enough to hold the whole run at PASS while a paraphrase fails. When that happens the result carries an explicit warning naming the failed rows.",
"type": "boolean"
},
"thresholds": {
"description": "Per-relation threshold overrides. Calibrate on your own corpus before gating — the defaults are starting points, not measurements.",
"properties": {
"case": {
"maximum": 1,
"minimum": 0,
"type": "number"
},
"paraphrase": {
"maximum": 1,
"minimum": 0,
"type": "number"
},
"reorder": {
"maximum": 1,
"minimum": 0,
"type": "number"
},
"specialization": {
"maximum": 1,
"minimum": 0,
"type": "number"
},
"translation": {
"maximum": 1,
"minimum": 0,
"type": "number"
},
"typo": {
"maximum": 1,
"minimum": 0,
"type": "number"
}
},
"type": "object"
},
"variants": {
"description": "Answers produced for transformed versions of the same question, each tagged with the relation that was applied.",
"items": {
"properties": {
"id": {
"description": "Optional identifier (default: variant_<n>).",
"type": "string"
},
"output": {
"description": "Required — the answer your system produced for the transformed question.",
"type": "string"
},
"question": {
"description": "Optional — the transformed question, echoed back for readability.",
"type": "string"
},
"relation": {
"description": "Transformation applied to the QUESTION. Decides the metric and the default threshold.",
"enum": [
"case",
"typo",
"paraphrase",
"reorder",
"translation",
"specialization"
],
"type": "string"
}
},
"required": [
"relation",
"output"
],
"type": "object"
},
"maxItems": 20,
"type": "array"
}
},
"required": [
"base",
"variants"
],
"type": "object"
},
"name": "metamorphic_check"
},
{
"description": "Get detailed specs for an AI model: context window, pricing per 1K tokens, knowledge cutoff, provider, multimodal support, reasoning capabilities, and feature list. Covers 30+ models from OpenAI, Anthropic, Google, DeepSeek, Meta, Mistral, Cohere, xAI.",
"inputSchema": {
"properties": {
"model": {
"description": "Model name (e.g. \"gpt-4o\", \"claude-3.5-sonnet\", \"gemini-3.1-pro-preview\")",
"type": "string"
}
},
"required": [
"model"
],
"type": "object"
},
"name": "model_info"
},
{
"description": "Generate a \"needle in a haystack\" test: embeds a target fact into a large block of filler text at a specified position. Use this to test LLM context window retrieval accuracy. Returns the full haystack, the question to ask, and metadata. No API key needed.",
"inputSchema": {
"properties": {
"needle": {
"description": "The fact to hide (e.g. \"The secret code is ALPHA-42\")",
"type": "string"
},
"position": {
"default": "middle",
"description": "Where to insert the needle: \"start\", \"middle\", \"end\", \"random\" (default: \"middle\")",
"enum": [
"start",
"middle",
"end",
"random"
],
"type": "string"
},
"question": {
"description": "The question to ask the LLM (e.g. \"What is the secret code?\")",
"type": "string"
},
"tokens": {
"default": 5000,
"description": "Target haystack size in tokens (default: 5000, max: 100000)",
"type": "integer"
}
},
"required": [
"needle",
"question"
],
"type": "object"
},
"name": "needle_haystack_generate"
},
{
"description": "Define a test suite for a prompt: provide the system prompt, user prompt, and expected output criteria. Returns a test plan with scored rubric — use this as input for manual or automated LLM evaluation.",
"inputSchema": {
"properties": {
"adversarial_prompts": {
"description": "Auto-generate adversarial test variants (jailbreak, injection, edge cases)",
"type": "boolean"
},
"check_safety": {
"description": "Include safety/PII checks in the rubric",
"type": "boolean"
},
"expected_behavior": {
"description": "Description of what the LLM should do (free text)",
"type": "string"
},
"expected_format": {
"description": "Expected output format",
"enum": [
"json",
"markdown",
"code",
"plain",
"any"
],
"type": "string"
},
"max_tokens": {
"description": "Max token budget for the test",
"type": "number"
},
"must_include": {
"description": "Required content (comma-separated)",
"type": "string"
},
"must_not_include": {
"description": "Forbidden content (comma-separated)",
"type": "string"
},
"system_prompt": {
"description": "The system prompt under test",
"type": "string"
},
"temperature": {
"description": "Temperature to use",
"type": "number"
},
"user_prompt": {
"description": "The user prompt to send",
"type": "string"
}
},
"required": [
"system_prompt",
"user_prompt"
],
"type": "object"
},
"name": "prompt_test_suite"
},
{
"description": "Rank an array of text chunks by relevance to a query using TF-IDF scoring. Simulates retrieval ranking for RAG testing without needing embeddings or an API.",
"inputSchema": {
"properties": {
"chunks": {
"description": "Array of text chunks to rank",
"items": {
"type": "string"
},
"type": "array"
},
"query": {
"description": "The user query",
"type": "string"
},
"top_k": {
"description": "Return top K results (default: all)",
"type": "number"
}
},
"required": [
"query",
"chunks"
],
"type": "object"
},
"name": "rag_relevance_rank"
},
{
"description": "Give honest usage feedback on an IA-QA MCP tool. Provide a score (1-5) and a comment. Rate low (1-2) if the tool was wrong, irrelevant, or a poor fit; rate high (4-5) only if it genuinely solved your need. Ratings are read by the IA-QA maintainers and are not published, one vote per agent and tool a month. Comments are screened before storing: one carrying a credential is not stored and personal data is replaced by placeholders — describe what happened rather than what a model should do. Skip rating routine successes — we want signal, not praise. Example: rate_tool({ tool_name: \"format_json\", score: 2, comment: \"Tried to pretty-print a JSON5 file, it rejected trailing commas — not usable for my case.\" })",
"inputSchema": {
"properties": {
"comment": {
"description": "Strongly encouraged — explain what you were trying to do and whether the tool got you there. Be specific about what was missing, wrong, or a poor fit. This is the most valuable part of the rating. Up to 2000 chars are stored; go over and the response says so (truncated: true) — send the remainder as a second call rather than assuming it landed.",
"type": "string"
},
"score": {
"description": "Rating from 1 (poor) to 5 (excellent)",
"maximum": 5,
"minimum": 1,
"type": "number"
},
"tool_name": {
"description": "Name of the MCP tool to rate (e.g. \"format_json\", \"shield_analyze\")",
"type": "string"
}
},
"required": [
"tool_name",
"score"
],
"type": "object"
},
"name": "rate_tool"
},
{
"description": "Evaluate RAG retrieval quality: rank passages against a query and compute Precision@k / Recall@k plus a PASS/FAIL CI verdict from ground-truth relevance labels. Three modes, all keyless except the last. (1) BYO scores — give each passage the `score` your own reranker produced (Cohere, Jina, a self-hosted NIM, a cross-encoder): deterministic, offline, and it evaluates YOUR reranker rather than someone else's. This is the mode to gate CI on. (2) Default, no scores and no key — ranks with local BM25, a lexical keyword baseline: it answers \"does a keyword floor already surface my relevant passages?\", never \"is my neural reranker good\". (3) Live NVIDIA reranker — supply `api_key` for an NVIDIA account that still has reranking entitlement; NVIDIA retired its hosted reranking endpoints on 2026-05-18, so this one is for accounts that were grandfathered in.",
"inputSchema": {
"properties": {
"api_key": {
"description": "Your NVIDIA API key (BYOK), used only when no passage carries a score. Transits RAM for the single call, never stored.",
"type": "string"
},
"passages": {
"description": "Array of passage objects to rank (min 2, max 20)",
"items": {
"properties": {
"id": {
"type": "string"
},
"relevant": {
"description": "Ground truth: is this passage relevant?",
"type": "boolean"
},
"score": {
"description": "Relevance score from YOUR reranker. Present on every passage → ranking is done from these (offline, no key). Higher = more relevant. Score all passages or none.",
"type": "number"
},
"text": {
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"type": "array"
},
"query": {
"description": "The search query or question to rank against",
"type": "string"
},
"threshold": {
"description": "Minimum Precision@k to PASS (0-1, default 0.5)",
"maximum": 1,
"minimum": 0,
"type": "number"
},
"top_k": {
"description": "k for Precision@k evaluation (default 3)",
"maximum": 10,
"minimum": 1,
"type": "integer"
}
},
"required": [
"query",
"passages"
],
"type": "object"
},
"name": "rerank_evaluate"
},
{
"description": "Score an LLM response against the criteria you pass: coverage of expected_keywords and compliance with max_length. Returns a 0-100 score over the criteria actually measured — or total_score null with the reason when none is given, because relevance and correctness cannot be read off word overlap (a correct one-word answer shares no words with its question). For meaning against a reference answer, use run_semantic_tests. Also returns unscored signals: question-term overlap, average sentence length, markdown.",
"inputSchema": {
"properties": {
"expected_keywords": {
"description": "Keywords a correct answer must contain (case-insensitive). Scored as coverage.",
"items": {
"type": "string"
},
"type": "array"
},
"max_length": {
"description": "Maximum character length. Scored as a proportional penalty beyond it.",
"type": "number"
},
"question": {
"description": "The original question/prompt",
"type": "string"
},
"response": {
"description": "The LLM response to score",
"type": "string"
}
},
"required": [
"question",
"response"
],
"type": "object"
},
"name": "response_quality_score"
},
{
"description": "Parse a .ia-eval.yaml LLM test suite, call the specified LLM model for each scenario, run all configured scorers, and return a structured JSON report with per-scenario Pass/Fail verdicts and a Markdown summary. Use list_local_tests to discover available test files.",
"inputSchema": {
"properties": {
"api_keys": {
"description": "API keys to use for LLM generation (all optional — falls back to server env vars)",
"properties": {
"anthropic": {
"type": "string"
},
"deepseek": {
"type": "string"
},
"google": {
"type": "string"
},
"groq": {
"type": "string"
},
"hf": {
"type": "string"
},
"openai": {
"type": "string"
},
"xai": {
"type": "string"
}
},
"type": "object"
},
"contract_path": {
"description": "Absolute or relative path to a .ia-eval.yaml file (required unless inline_contract is provided)",
"type": "string"
},
"inline_contract": {
"additionalProperties": true,
"description": "Raw contract object (alternative to contract_path). Must contain top-level \"metadata\" ({name, version, model?, provider?}), \"expectations\" ({min_score?}), and \"scenarios\" ([{id, input, ground_truth?}]) — scenarios alone are rejected. Use generate_eval_yaml to scaffold one.",
"type": "object"
},
"overrides": {
"description": "Override contract defaults",
"properties": {
"model": {
"type": "string"
},
"provider": {
"type": "string"
},
"system_prompt": {
"type": "string"
},
"temperature": {
"type": "number"
}
},
"type": "object"
},
"validate_only": {
"description": "Check the contract against the IA-EVAL schema and stop: no model call, no key, deterministic. Returns valid + issues instead of throwing. Gate a PR that edits a contract on valid.",
"type": "boolean"
}
},
"type": "object"
},
"name": "run_eval_contract"
},
{
"description": "Semantic assertion primitive: compare actual vs expected text pairs using cosine similarity + ROUGE-L. Two modes: tfidf (default, free, no API key) or embeddings (OpenAI text-embedding-3-small, BYOK, true semantic similarity). Returns per-case PASS/FAIL verdicts and an overall verdict. CI-ready: pipe the JSON verdict field to gate a build.",
"inputSchema": {
"properties": {
"api_key": {
"description": "OpenAI API key — required only when mode is embeddings.",
"type": "string"
},
"cases": {
"description": "Array of (actual, expected) pairs to evaluate.",
"items": {
"properties": {
"actual": {
"description": "The text produced by your LLM/system.",
"type": "string"
},
"expected": {
"description": "The reference/ground-truth text.",
"type": "string"
},
"id": {
"description": "Optional identifier for this case.",
"type": "string"
}
},
"required": [
"actual",
"expected"
],
"type": "object"
},
"maxItems": 50,
"type": "array"
},
"mode": {
"description": "tfidf (default): fast, free, lexical. embeddings: OpenAI text-embedding-3-small, true semantic similarity, requires api_key.",
"enum": [
"tfidf",
"embeddings"
],
"type": "string"
},
"require_all": {
"description": "If true (default), all cases must pass for overall PASS. If false, at least one case passing returns PASS.",
"type": "boolean"
},
"thresholds": {
"description": "Pass/fail thresholds (defaults: cosine 0.75, rouge_l 0.5).",
"properties": {
"cosine": {
"description": "Minimum cosine similarity to pass (default: 0.75).",
"maximum": 1,
"minimum": 0,
"type": "number"
},
"rouge_l": {
"description": "Minimum ROUGE-L F1 to pass (default: 0.5).",
"maximum": 1,
"minimum": 0,
"type": "number"
}
},
"type": "object"
}
},
"required": [
"cases"
],
"type": "object"
},
"name": "run_semantic_tests"
},
{
"description": "Run a test suite against a Vision-Language Model (VLM) — send an image (URL or base64) + N test cases (each with a question + assertion) to GPT-4o, Claude 3.5, or Gemini. Returns per-case PASS/FAIL verdicts, a pass rate, an overall PASS/WARNING/FAIL verdict (customizable threshold), and latency stats. Assertion types: contains, not_contains, json_format, min_length, max_length, semantic_contains (TF-IDF cosine similarity ≥ 0.4). BYOK: requires your own API key for the target provider.",
"inputSchema": {
"properties": {
"api_key": {
"description": "API key for the model provider (OpenAI sk-, Anthropic sk-ant-, or Google AIzaSy...).",
"type": "string"
},
"image_base64": {
"description": "Base64-encoded image data (required unless image_url is provided).",
"type": "string"
},
"image_mime_type": {
"description": "MIME type of the image if using image_base64 (default: image/jpeg).",
"type": "string"
},
"image_url": {
"description": "Public URL of the image to evaluate (required unless image_base64 is provided).",
"type": "string"
},
"model": {
"description": "VLM model to use.",
"enum": [
"gpt-4o",
"gpt-4o-mini",
"claude-3-5-sonnet-20241022",
"claude-3-5-haiku-20241022",
"gemini-3.5-flash",
"gemini-2.5-flash"
],
"type": "string"
},
"system_prompt": {
"description": "Optional system prompt sent to the VLM.",
"type": "string"
},
"test_cases": {
"description": "Array of test cases to run.",
"items": {
"properties": {
"assertion_type": {
"description": "Assertion to run on the VLM response. semantic_contains uses TF-IDF cosine similarity ≥ 0.4.",
"enum": [
"contains",
"not_contains",
"json_format",
"min_length",
"max_length",
"semantic_contains"
],
"type": "string"
},
"assertion_value": {
"description": "Expected value for the assertion (not needed for json_format).",
"type": "string"
},
"id": {
"description": "Optional identifier for this case.",
"type": "string"
},
"question": {
"description": "Question to ask the VLM about the image.",
"type": "string"
}
},
"required": [
"question"
],
"type": "object"
},
"maxItems": 10,
"type": "array"
},
"threshold": {
"description": "Pass rate threshold for overall verdict (default: 80, 0–100).",
"type": "number"
}
},
"required": [
"test_cases",
"model",
"api_key"
],
"type": "object"
},
"name": "run_vlm_test_suite"
},
{
"description": "Compute text similarity between reference and hypothesis using multiple metrics: Cosine (BoW, TF-IDF), Jaccard, ROUGE-1, ROUGE-2, ROUGE-L, and BLEU. No API key needed. Ideal for LLM eval (expected vs actual), RAG quality checks, and NLG benchmarking. Supports batch mode.",
"inputSchema": {
"properties": {
"batch": {
"description": "Batch mode: array of {reference, hypothesis} pairs.",
"items": {
"properties": {
"hypothesis": {
"type": "string"
},
"reference": {
"type": "string"
}
},
"required": [
"reference",
"hypothesis"
],
"type": "object"
},
"type": "array"
},
"hypothesis": {
"description": "Hypothesis / actual text (LLM output)",
"type": "string"
},
"metrics": {
"description": "Metrics to compute (default: all). Options: \"cosine_bow\", \"cosine_tfidf\", \"jaccard\", \"rouge1\", \"rouge2\", \"rougeL\", \"bleu\"",
"items": {
"type": "string"
},
"type": "array"
},
"reference": {
"description": "Reference / expected text (ground truth)",
"type": "string"
},
"threshold": {
"description": "Optional pass/fail threshold (0-1). Applies to ROUGE-L F1 score.",
"type": "number"
}
},
"type": "object"
},
"name": "similarity_score"
},
{
"description": "Test a SKILL.md in the open Agent Skills format (https://agentskills.io/specification — read by Claude Code, Codex, GitHub Copilot, Cursor, Gemini CLI, OpenCode and others) in two stages. (1) lint: the spec's frontmatter rules checked without a model (name 1-64 chars, lowercase/digits/single hyphens, matches dir_name when given; description 1-1024 chars; compatibility ≤500; metadata string map) — any error returns BLOCK and calls no model; gate on lint.valid, it is deterministic. (2) triggering, measured the way an agent decides: the model sees ONLY the name + description of this skill and of the other installed skills (other_skills, or six built-in neighbours) and answers which one it would load, at temperature 0. trigger_accuracy = share of requests routed correctly (should load / should not); verdict PASS ≥80, FIX ≥50, else BLOCK. Pass your own scenarios [{prompt, should_trigger}] for a reproducible gate; without them requests are generated (half should-load, half near misses) and the result is a signal, flagged by scenarios_source. step_adherence is keyword overlap on the skill body's run and never enters the verdict. A request the model never answered readably (rate limit, empty output) is not counted as a miss: the run is INCONCLUSIVE instead. Uses the server's default Groq model (server key, rate-limited). Pass api_key + model to use your own provider; the SKILL.md and scenarios are sent to ia-qa.com and to that provider.",
"inputSchema": {
"properties": {
"api_key": {
"description": "API key for the chosen model provider. Not required when using the default Groq model.",
"type": "string"
},
"dir_name": {
"description": "Name of the folder holding the SKILL.md. The spec requires name to match it; checked only when given.",
"type": "string"
},
"lint_only": {
"description": "Run the format check only: no model call, no key, deterministic. The CI gate for a SKILL.md — gate on lint.valid.",
"type": "boolean"
},
"model": {
"description": "LLM model ID to use for both scenario generation and testing (e.g. gpt-4o-mini, claude-3-5-haiku-20241022). Defaults to openai/gpt-oss-20b (Groq, server key).",
"type": "string"
},
"other_skills": {
"description": "The other skills installed next to this one, [{\"name\", \"description\"}], so triggering is tested against the real neighbours. Default: six built-in generic skills.",
"items": {
"properties": {
"description": {
"type": "string"
},
"name": {
"type": "string"
}
},
"required": [
"name",
"description"
],
"type": "object"
},
"type": "array"
},
"scenario_count": {
"description": "Number of requests to generate when scenarios is omitted: half should-load, half near misses. Default: 6.",
"enum": [
4,
6,
8,
10
],
"type": "number"
},
"scenarios": {
"description": "Fixed user requests for a reproducible gate: [{\"prompt\": \"...\", \"should_trigger\": true}]. Include near misses with should_trigger false. Max 20. When omitted, scenario_count requests are generated.",
"items": {
"properties": {
"prompt": {
"type": "string"
},
"should_trigger": {
"type": "boolean"
}
},
"required": [
"prompt",
"should_trigger"
],
"type": "object"
},
"type": "array"
},
"skill_md": {
"description": "Full content of the SKILL.md file to test, YAML frontmatter included.",
"type": "string"
}
},
"required": [
"skill_md"
],
"type": "object"
},
"name": "test_skill"
},
{
"description": "Plan token allocation across system prompt, user input, context/RAG chunks, and expected output. Warns if budget exceeds model context window. Supports 25+ models.",
"inputSchema": {
"properties": {
"context": {
"description": "Actual context text (will estimate tokens)",
"type": "string"
},
"context_tokens": {
"description": "Token count for RAG context / documents",
"type": "number"
},
"expected_output_tokens": {
"description": "Expected max output tokens",
"type": "number"
},
"model": {
"description": "Model name (e.g. gpt-4o, claude-3.5-sonnet, gemini-3.5-flash)",
"type": "string"
},
"system_prompt": {
"description": "Actual system prompt text (will estimate tokens)",
"type": "string"
},
"system_prompt_tokens": {
"description": "Token count for system prompt",
"type": "number"
},
"user_input": {
"description": "Actual user input text (will estimate tokens)",
"type": "string"
},
"user_input_tokens": {
"description": "Token count for user message",
"type": "number"
}
},
"required": [
"model"
],
"type": "object"
},
"name": "token_budget_calculator"
}
]
}What the leaf hashes
{"body":"30e381eb09e8bac8d6ec2246cfdf475141f1fbac04ac7298fe722f9dadf54f41","id":990677,"kind":"mcp.tools","method":"probe.mcp/1","observed_us":1793244426383178,"prev":777768,"subject":"674aa99f-e86c-46f9-b8c2-5321b2d469b4","subject_kind":"endpoint"}Proof
Its day has not closed yet. A day closes ten minutes after midnight UTC; the proof appears here then.