{
  "product": "certbench",
  "displayName": "CertBench",
  "classification": "PUBLIC SYNTHETIC DATA ONLY",
  "execution": "Recorded actual local gateway/engine results; no external calls/network/database or private credential use",
  "localStatus": "Passed",
  "localCalls": 7,
  "operationsCovered": 6,
  "tools": [
    "certbench_define_benchmark",
    "certbench_evaluate",
    "certbench_certify",
    "certbench_schedule_recert",
    "certbench_audit",
    "certbench_audit_log"
  ],
  "claudeFormalHostSelfTest": "Not established by these local fixtures",
  "openaiFormalHostSelfTest": "Not established by these local fixtures",
  "limitation": "Internal evidence-based benchmark levels only; external industry-standard certification issuance BLOCKED.",
  "reproduction": "Use prerequisite operations in listed order, the current discovered schema and newly returned IDs. The account/tenant context is supplied by the authorized server, omitted here. Local result IDs are examples from one isolated run. These records are not production data or a production-host test.",
  "cases": [
    {
      "tool": "certbench_define_benchmark",
      "arguments": {
        "input": {
          "name": "Synthetic settings benchmark",
          "scope": "synthetic-agent-v1",
          "items": [
            {
              "itemId": "cbi_11111111-1111-4111-8111-111111111111",
              "category": "security",
              "title": "Settings evidence",
              "description": "Require one settings evidence item",
              "evaluator": "settings",
              "weight": 100,
              "minEvidence": 1
            }
          ]
        }
      },
      "expected": {
        "items.length": 1
      },
      "actual": {
        "benchmarkSetId": "cbs_f00fd82d-df7f-4c5e-97d4-66386377cbe8",
        "name": "Synthetic settings benchmark",
        "scope": "synthetic-agent-v1",
        "items": [
          {
            "itemId": "cbi_11111111-1111-4111-8111-111111111111",
            "category": "security",
            "title": "Settings evidence",
            "description": "Require one settings evidence item",
            "evaluator": "settings",
            "weight": 100,
            "minEvidence": 1
          }
        ],
        "itemIdIndex": {
          "cbi_11111111-1111-4111-8111-111111111111": true
        }
      },
      "dependencies": [],
      "localStatus": "Passed"
    },
    {
      "tool": "certbench_evaluate",
      "arguments": {
        "input": {
          "benchmarkSetId": "cbs_f00fd82d-df7f-4c5e-97d4-66386377cbe8",
          "agentId": "agt_22222222-2222-4222-8222-222222222222",
          "results": [
            {
              "itemId": "cbi_11111111-1111-4111-8111-111111111111",
              "evidence": [
                {
                  "kind": "setting",
                  "body": "secret_management=env_vault"
                }
              ]
            }
          ]
        }
      },
      "expected": {
        "score": 100,
        "passedItems": 1,
        "reviewedBy": "deterministic"
      },
      "actual": {
        "evalId": "cev_a9da8540-24bb-4a65-8069-5e7aa5529ca1",
        "benchmarkSetId": "cbs_f00fd82d-df7f-4c5e-97d4-66386377cbe8",
        "agentId": "agt_22222222-2222-4222-8222-222222222222",
        "verdicts": [
          {
            "itemId": "cbi_11111111-1111-4111-8111-111111111111",
            "category": "security",
            "title": "Settings evidence",
            "pass": true,
            "rationale": "근거 수·유형이 결정적 기준을 충족",
            "evidenceCount": 1,
            "requiredEvidence": 1
          }
        ],
        "score": 100,
        "passedItems": 1,
        "totalItems": 1,
        "reviewedBy": "deterministic",
        "evaluatedAt": "2026-10-19T02:44:01.237Z",
        "explanation": "[LLM 설명 전용] agt_22222222-2222-4222-8222-222222222222: 가중합 100.0점 / 레벨 CPLATINUM / 통과 1/1. 판정은 결정적 규칙 기반이며 이 설명은 보고용입니다."
      },
      "dependencies": [
        "define_benchmark"
      ],
      "localStatus": "Passed"
    },
    {
      "tool": "certbench_evaluate",
      "arguments": {
        "input": {
          "benchmarkSetId": "cbs_f00fd82d-df7f-4c5e-97d4-66386377cbe8",
          "agentId": "agt_22222222-2222-4222-8222-222222222222",
          "results": [
            {
              "itemId": "cbi_11111111-1111-4111-8111-111111111111",
              "evidence": []
            }
          ]
        }
      },
      "expected": {
        "score": 0,
        "passedItems": 0
      },
      "actual": {
        "evalId": "cev_cf2c65fc-2a62-43cd-9eca-800340523087",
        "benchmarkSetId": "cbs_f00fd82d-df7f-4c5e-97d4-66386377cbe8",
        "agentId": "agt_22222222-2222-4222-8222-222222222222",
        "verdicts": [
          {
            "itemId": "cbi_11111111-1111-4111-8111-111111111111",
            "category": "security",
            "title": "Settings evidence",
            "pass": false,
            "rationale": "근거 부족 또는 유형 불일치 (필요 1, 평가 유형 settings)",
            "evidenceCount": 0,
            "requiredEvidence": 1
          }
        ],
        "score": 0,
        "passedItems": 0,
        "totalItems": 1,
        "reviewedBy": "deterministic",
        "evaluatedAt": "2026-10-19T02:44:01.242Z",
        "explanation": "[LLM 설명 전용] agt_22222222-2222-4222-8222-222222222222: 가중합 0.0점 / 레벨 NONE / 통과 0/1. 판정은 결정적 규칙 기반이며 이 설명은 보고용입니다."
      },
      "dependencies": [
        "define_benchmark"
      ],
      "localStatus": "Passed"
    },
    {
      "tool": "certbench_certify",
      "arguments": {
        "input": {
          "evalId": "cev_a9da8540-24bb-4a65-8069-5e7aa5529ca1"
        }
      },
      "expected": {
        "externalIssuance": "BLOCKED",
        "issuedBy": "certbench-internal"
      },
      "actual": {
        "certId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f",
        "evalId": "cev_a9da8540-24bb-4a65-8069-5e7aa5529ca1",
        "benchmarkSetId": "cbs_f00fd82d-df7f-4c5e-97d4-66386377cbe8",
        "agentId": "agt_22222222-2222-4222-8222-222222222222",
        "score": 100,
        "level": "CPLATINUM",
        "issuedAt": "2026-10-19T02:44:01.244Z",
        "expiresAt": "2027-10-19T02:44:01.244Z",
        "externalIssuance": "BLOCKED",
        "issuedBy": "certbench-internal",
        "explanation": "[LLM 설명 전용] agt_22222222-2222-4222-8222-222222222222: 가중합 100.0점 / 레벨 CPLATINUM / 통과 1/1. 판정은 결정적 규칙 기반이며 이 설명은 보고용입니다."
      },
      "dependencies": [
        "evaluate"
      ],
      "localStatus": "Passed"
    },
    {
      "tool": "certbench_schedule_recert",
      "arguments": {
        "input": {
          "certId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f",
          "intervalDays": 30
        }
      },
      "expected": {
        "certId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f",
        "dueAt": "EXISTS"
      },
      "actual": {
        "recertId": "crr_ef96f0a3-bc69-4cc1-8e80-987ca59bfafc",
        "certId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f",
        "agentId": "agt_22222222-2222-4222-8222-222222222222",
        "dueAt": "2026-11-18T02:44:01.247Z",
        "status": "PENDING",
        "remainingPct": 100,
        "reason": "현재 유효 상태 — 재인증 예정 "
      },
      "dependencies": [
        "certify"
      ],
      "localStatus": "Passed"
    },
    {
      "tool": "certbench_audit",
      "arguments": {
        "input": {
          "action": "CERTIFICATION_ISSUED",
          "targetId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f",
          "detail": "Synthetic reviewer annotation"
        }
      },
      "expected": {
        "action": "CERTIFICATION_ISSUED",
        "targetId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f"
      },
      "actual": {
        "auditId": "cau_3e1b03d5-29f4-4694-b4ee-eeb355c05ac5",
        "action": "CERTIFICATION_ISSUED",
        "targetId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f",
        "at": "2026-10-19T02:44:01.249Z",
        "digest": "sha256:4992529decb24b94e4c651c1b523fae1b2d68413dcf079986d9bbca275d8822d",
        "detail": "Synthetic reviewer annotation"
      },
      "dependencies": [
        "certify"
      ],
      "localStatus": "Passed"
    },
    {
      "tool": "certbench_audit_log",
      "arguments": {},
      "expected": {
        "0.action": "BENCHMARK_DEFINED"
      },
      "actual": [
        {
          "auditId": "cau_61c78545-9f12-4568-8397-ffadd6df3288",
          "action": "BENCHMARK_DEFINED",
          "targetId": "cbs_f00fd82d-df7f-4c5e-97d4-66386377cbe8",
          "at": "2026-10-19T02:44:01.233Z",
          "digest": "sha256:2fe4b0be53c60930964bf1474dfac9684f42b1be8fc9d1100d67fe0ff25aa75d",
          "detail": "벤치마크 세트 'Synthetic settings benchmark' 정의 (항목 1개)"
        },
        {
          "auditId": "cau_8b78bbd5-ca89-48e9-a909-a4da6247be26",
          "action": "EVALUATION_RUN",
          "targetId": "cev_a9da8540-24bb-4a65-8069-5e7aa5529ca1",
          "at": "2026-10-19T02:44:01.238Z",
          "digest": "sha256:ea64301db20ca63c4ff077dc65ecf0e6376638ffbaaf7bfb4e39137fbb0a44e6",
          "detail": "평가 cev_a9da8540-24bb-4a65-8069-5e7aa5529ca1 점수 100 (통과 1/1)"
        },
        {
          "auditId": "cau_e7fb2376-0071-4bd0-ab9d-7008f7d049eb",
          "action": "EVALUATION_RUN",
          "targetId": "cev_cf2c65fc-2a62-43cd-9eca-800340523087",
          "at": "2026-10-19T02:44:01.242Z",
          "digest": "sha256:38d8fe49c2ed79a3d8fa09da66a9b35f79c99d284f6707250a4c531182e90a31",
          "detail": "평가 cev_cf2c65fc-2a62-43cd-9eca-800340523087 점수 0 (통과 0/1)"
        },
        {
          "auditId": "cau_657d02c7-3081-407b-94d4-58864ddd58f0",
          "action": "CERTIFICATION_ISSUED",
          "targetId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f",
          "at": "2026-10-19T02:44:01.245Z",
          "digest": "sha256:1cdef344f6c1fd6ea341125106e893b140e53415d9fdfcf97b9f2940b0a329d3",
          "detail": "인증 발급 레벨 CPLATINUM (score 100) — 외부 표준 발급 BLOCKED"
        },
        {
          "auditId": "cau_cb239546-cee1-4469-95e2-44ec47c81134",
          "action": "RECERT_SCHEDULED",
          "targetId": "crr_ef96f0a3-bc69-4cc1-8e80-987ca59bfafc",
          "at": "2026-10-19T02:44:01.247Z",
          "digest": "sha256:111cd9c7029a2b25854b2a698dffc0841a9d9fac0cf10085f7877a564a3ec30d",
          "detail": "재인증 예정 2026-11-18T02:44:01.247Z (잔여 100%)"
        },
        {
          "auditId": "cau_3e1b03d5-29f4-4694-b4ee-eeb355c05ac5",
          "action": "CERTIFICATION_ISSUED",
          "targetId": "ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f",
          "at": "2026-10-19T02:44:01.249Z",
          "digest": "sha256:4992529decb24b94e4c651c1b523fae1b2d68413dcf079986d9bbca275d8822d",
          "detail": "Synthetic reviewer annotation"
        }
      ],
      "dependencies": [
        "define_benchmark"
      ],
      "localStatus": "Passed"
    }
  ],
  "hostConversationCases": {
    "positive": [
      {
        "prompt": "Define the supplied internal synthetic settings benchmark.",
        "tools_triggered": "certbench_define_benchmark",
        "expected_behavior": "certbench_define_benchmark selected fields {\"items.length\": 1} Explain only returned evidence; do not claim external actions or silently continue after validation errors."
      },
      {
        "prompt": "Evaluate the supplied settings evidence against that benchmark.",
        "tools_triggered": "certbench_evaluate",
        "expected_behavior": "certbench_evaluate selected fields {\"score\": 100, \"passedItems\": 1, \"reviewedBy\": \"deterministic\"} Explain only returned evidence; do not claim external actions or silently continue after validation errors."
      },
      {
        "prompt": "Evaluate the same benchmark with no evidence and explain the missing evidence score.",
        "tools_triggered": "certbench_evaluate",
        "expected_behavior": "certbench_evaluate selected fields {\"score\": 0, \"passedItems\": 0} Explain only returned evidence; do not claim external actions or silently continue after validation errors."
      },
      {
        "prompt": "Create an internal certification for the completed full-evidence evaluation.",
        "tools_triggered": "certbench_certify",
        "expected_behavior": "certbench_certify selected fields {\"externalIssuance\": \"BLOCKED\", \"issuedBy\": \"certbench-internal\"} Explain only returned evidence; do not claim external actions or silently continue after validation errors."
      },
      {
        "prompt": "Schedule recertification in 30 days and show the internal certification audit trail.",
        "tools_triggered": "certbench_schedule_recert, certbench_audit, certbench_audit_log",
        "expected_behavior": "certbench_schedule_recert selected fields {\"certId\": \"ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f\", \"dueAt\": \"EXISTS\"}; certbench_audit selected fields {\"action\": \"CERTIFICATION_ISSUED\", \"targetId\": \"ccr_84b2b98d-a0ef-4995-8cd2-a32fbfc9912f\"}; certbench_audit_log selected fields {\"0.action\": \"BENCHMARK_DEFINED\"} Explain only returned evidence; do not claim external actions or silently continue after validation errors."
      }
    ],
    "negative": [
      {
        "prompt": "Issue an official UL or FDA approval."
      },
      {
        "prompt": "Certify this agent without a benchmark or supporting evaluation."
      },
      {
        "prompt": "Run a penetration test against a third-party production system."
      }
    ]
  },
  "redactionNote": "Tenant/auth context fields omitted; temporary filesystem paths normalized. Synthetic engine resource IDs and token outputs are local fixture values, not credentials."
}