{"command":{"effect":"write","timeoutSeconds":30,"inputSchema":{"type":"object","description":"To create a model evaluation run, send a POST request to `/v2/genai/model_evaluation_runs`.","properties":{"body":{"$ref":"#/definitions/apiCreateModelEvaluationRunInputPublic"}},"required":[],"additionalProperties":false,"definitions":{"apiCreateModelEvaluationRunInputPublic":{"properties":{"candidate_inference_config":{"$ref":"#/definitions/apiCandidateInferenceConfig"},"candidate_model_name":{"description":"Model slug used to call the candidate model API.\nFor dedicated inference, this is the model slug from the deployment.\nFor serverless, this should match the model's internal name.","type":"string","examples":["example name"]},"candidate_model_source":{"$ref":"#/definitions/apiCandidateModelSource"},"candidate_model_uuid":{"description":"UUID of the candidate model to evaluate.","type":"string","examples":["123e4567-e89b-12d3-a456-426614174000"]},"dataset_uuid":{"description":"UUID of the dataset to use for evaluation.","type":"string","examples":["123e4567-e89b-12d3-a456-426614174000"]},"epochs":{"description":"How many times each dataset row is evaluated. Defaults to 1, maximum 10.\nWhen more than 1, the results show how consistent the model is across\nattempts (avg@k, pass@k, cons@k). Each extra attempt makes the run take\nlonger. Not saved in presets.","type":"integer","examples":[1]},"eval_preset_uuid":{"type":"string","examples":["123e4567-e89b-12d3-a456-426614174000"]},"judge_model_uuid":{"description":"UUID of the judge model used to score responses.","type":"string","examples":["123e4567-e89b-12d3-a456-426614174000"]},"metric_uuids":{"description":"UUIDs of metrics to evaluate (selected from ListModelEvaluationMetrics).","items":{"type":"string","examples":["example string"]},"type":"array","examples":[["example string"]]},"name":{"type":"string","examples":["example name"]},"preset_name":{"type":"string","examples":["example name"]},"preset_save_sections":{"description":"Which sections of this run's resolved configuration to persist as a\nreusable preset. Each selected section saves only its own fields; the\nremaining sections stay empty on the preset and must be supplied inline\non future runs that reference it. Empty means do not save a preset\n(unless the deprecated `save_as_preset` boolean is true, in which case\nall sections are saved). Ignored when `eval_preset_uuid` is set. Use\n`preset_name` to label the saved preset.","items":{"$ref":"#/definitions/apiPresetSaveSection"},"type":"array","examples":[["PRESET_SAVE_SECTION_CANDIDATE","PRESET_SAVE_SECTION_METRICS"]]},"save_as_preset":{"description":"Deprecated: use `preset_save_sections`. When `true` and\n`preset_save_sections` is empty, all five sections of the resolved\nconfiguration are saved as a reusable preset (legacy behavior). Ignored\nwhen `eval_preset_uuid` is set.","type":"boolean","examples":[true]},"source":{"description":"Source of the run creation (api, sdk, cli).","type":"string","examples":["example string"]},"star_metric":{"$ref":"#/definitions/apiStarMetric"}},"type":"object"},"apiCandidateInferenceConfig":{"description":"Inference configuration for the candidate model during evaluation.\nThe output token limit is set automatically and cannot be configured.","properties":{"reasoning_effort":{"description":"Reasoning effort for reasoning-capable models (e.g. \"low\", \"medium\",\n\"high\"). Validated against the candidate model's supported values; a model\nthat advertises none rejects this field.","type":"string","examples":["low"]},"stop_token":{"type":"string","examples":["example string"]},"system_prompt":{"type":"string","examples":["example string"]},"temperature":{"type":"number","examples":[123]}},"type":"object"},"apiCandidateModelSource":{"default":"CANDIDATE_MODEL_SOURCE_SERVERLESS","description":"Whether the candidate is a served model (serverless platform, a dedicated\ndeployment, or a model router) or an OHS-hosted agent config.","enum":["CANDIDATE_MODEL_SOURCE_SERVERLESS","CANDIDATE_MODEL_SOURCE_DEDICATED","CANDIDATE_MODEL_SOURCE_ROUTER","CANDIDATE_MODEL_SOURCE_AGENT"],"type":"string","examples":["CANDIDATE_MODEL_SOURCE_SERVERLESS"]},"apiPresetSaveSection":{"default":"PRESET_SAVE_SECTION_UNSPECIFIED","description":"Sections of an inline evaluation config that can be persisted as a reusable\npreset. Each value names a self-contained group of fields; selecting a\nsection saves exactly the fields it owns and leaves the rest of the preset\nempty so it can be merged with inline values on a future run.\n\n - PRESET_SAVE_SECTION_CANDIDATE: Candidate model identity (`candidate_model_uuid`, `candidate_model_source`,\n`candidate_model_name`) and the non-prompt inference params\n(`temperature`, `stop_token`, `reasoning_effort`).\n - PRESET_SAVE_SECTION_METRICS: The selected `metric_uuids` and the optional `star_metric`.\n - PRESET_SAVE_SECTION_JUDGE: The `judge_model_uuid`.\n - PRESET_SAVE_SECTION_DATASET: The `dataset_uuid`.\n - PRESET_SAVE_SECTION_SYSTEM_PROMPT: The candidate's `system_prompt` only. Independent of CANDIDATE so the\nmodel + params and the prompt can be saved/replayed separately.","enum":["PRESET_SAVE_SECTION_UNSPECIFIED","PRESET_SAVE_SECTION_CANDIDATE","PRESET_SAVE_SECTION_METRICS","PRESET_SAVE_SECTION_JUDGE","PRESET_SAVE_SECTION_DATASET","PRESET_SAVE_SECTION_SYSTEM_PROMPT"],"type":"string","examples":["PRESET_SAVE_SECTION_UNSPECIFIED"]},"apiStarMetric":{"properties":{"metric_uuid":{"type":"string","examples":["123e4567-e89b-12d3-a456-426614174000"]},"name":{"type":"string","examples":["example name"]},"success_threshold":{"description":"The success threshold for the star metric.\nThis is a value that the metric must reach to be considered successful.","type":"number","examples":[123]},"success_threshold_pct":{"description":"The success threshold for the star metric.\nThis is a percentage value between 0 and 100.","type":"integer","examples":[123]}},"type":"object"}}},"name":"Create Model Evaluation Run","id":"digitalocean.genai_create_model_evaluation_run","example":{},"maxRequests":1,"summary":false}}