74 lines
2.7 KiB
JSON
74 lines
2.7 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["suite_id", "version", "benchmarks"],
|
|
"properties": {
|
|
"suite_id": {"type": "string"},
|
|
"version": {"type": "string"},
|
|
"pass_threshold": {"type": "integer", "minimum": 0, "maximum": 100},
|
|
"benchmarks": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["id", "title", "category", "target_agents", "prompt", "checks"],
|
|
"properties": {
|
|
"id": {"type": "string"},
|
|
"title": {"type": "string"},
|
|
"category": {"type": "string"},
|
|
"target_agents": {
|
|
"type": "array",
|
|
"items": {"type": "string"},
|
|
"minItems": 1
|
|
},
|
|
"prompt": {"type": "string"},
|
|
"description": {"type": "string"},
|
|
"checks": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["id", "mode", "path", "weight", "dimension"],
|
|
"properties": {
|
|
"id": {"type": "string"},
|
|
"mode": {
|
|
"type": "string",
|
|
"enum": ["contains_all", "absent_all", "path_exists", "yaml_keys_present", "yaml_duration_max", "yaml_array_duration_max", "jsonl_event_duration_max"]
|
|
},
|
|
"path": {"type": "string"},
|
|
"tokens": {
|
|
"type": "array",
|
|
"items": {"type": "string"}
|
|
},
|
|
"keys": {
|
|
"type": "array",
|
|
"items": {"type": "string"}
|
|
},
|
|
"start_key": {"type": "string"},
|
|
"end_key": {"type": "string"},
|
|
"array_key": {"type": "string"},
|
|
"start_event": {"type": "string"},
|
|
"end_event": {"type": "string"},
|
|
"pair_by": {"type": "string"},
|
|
"max_seconds": {"type": "number", "minimum": 0},
|
|
"weight": {"type": "integer", "minimum": 1},
|
|
"dimension": {
|
|
"type": "string",
|
|
"enum": ["speed", "parallelism", "collision_resistance", "ambiguity_control", "llm_efficiency", "resumability", "observability", "schema_rigor", "timing"]
|
|
},
|
|
"scope": {
|
|
"type": "string",
|
|
"enum": ["contracts", "artifacts"],
|
|
"default": "contracts"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |