{"command":{"effect":"write","timeoutSeconds":120,"inputSchema":{"type":"object","description":"Validate a grader.\n","properties":{"body":{"$ref":"#/definitions/ValidateGraderRequest"}},"required":["body"],"additionalProperties":false,"definitions":{"ValidateGraderRequest":{"type":"object","title":"ValidateGraderRequest","properties":{"grader":{"type":"object","description":"The grader used for the fine-tuning job.","oneOf":[{"$ref":"#/definitions/GraderStringCheck"},{"$ref":"#/definitions/GraderTextSimilarity"},{"$ref":"#/definitions/GraderPython"},{"$ref":"#/definitions/GraderScoreModel"},{"$ref":"#/definitions/GraderMulti"}]}},"required":["grader"]},"GraderStringCheck":{"type":"object","title":"StringCheckGrader","description":"A StringCheckGrader object that performs a string comparison between input and reference using a specified operation.\n","properties":{"type":{"type":"string","enum":["string_check"],"description":"The object type, which is always `string_check`."},"name":{"type":"string","description":"The name of the grader."},"input":{"type":"string","description":"The input text. This may include template strings."},"reference":{"type":"string","description":"The reference text. This may include template strings."},"operation":{"type":"string","enum":["eq","ne","like","ilike"],"description":"The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`."}},"required":["type","name","input","reference","operation"]},"GraderTextSimilarity":{"type":"object","title":"TextSimilarityGrader","description":"A TextSimilarityGrader object which grades text based on similarity metrics.\n","properties":{"type":{"type":"string","enum":["text_similarity"],"default":"text_similarity","description":"The type of grader."},"name":{"type":"string","description":"The name of the grader."},"input":{"type":"string","description":"The text being graded."},"reference":{"type":"string","description":"The text being graded against."},"evaluation_metric":{"type":"string","enum":["cosine","fuzzy_match","bleu","gleu","meteor","rouge_1","rouge_2","rouge_3","rouge_4","rouge_5","rouge_l"],"description":"The evaluation metric to use. One of `cosine`, `fuzzy_match`, `bleu`, \n`gleu`, `meteor`, `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, \nor `rouge_l`.\n"}},"required":["type","name","input","reference","evaluation_metric"]},"GraderPython":{"type":"object","title":"PythonGrader","description":"A PythonGrader object that runs a python script on the input.\n","properties":{"type":{"type":"string","enum":["python"],"description":"The object type, which is always `python`."},"name":{"type":"string","description":"The name of the grader."},"source":{"type":"string","description":"The source code of the python script."},"image_tag":{"type":"string","description":"The image tag to use for the python script."}},"required":["type","name","source"]},"GraderScoreModel":{"type":"object","title":"ScoreModelGrader","description":"A ScoreModelGrader object that uses a model to assign a score to the input.\n","properties":{"type":{"type":"string","enum":["score_model"],"description":"The object type, which is always `score_model`."},"name":{"type":"string","description":"The name of the grader."},"model":{"type":"string","description":"The model to use for the evaluation."},"sampling_params":{"type":"object","description":"The sampling parameters for the model.","properties":{"seed":{"anyOf":[{"type":"integer","description":"A seed value to initialize the randomness, during sampling.\n"},{"type":"null"}]},"top_p":{"anyOf":[{"type":"number","default":1,"description":"An alternative to temperature for nucleus sampling; 1.0 includes all tokens.\n","examples":[1]},{"type":"null"}]},"temperature":{"anyOf":[{"type":"number","description":"A higher temperature increases randomness in the outputs.\n"},{"type":"null"}]},"max_completions_tokens":{"anyOf":[{"type":"integer","minimum":1,"description":"The maximum number of tokens the grader model may generate in its response.\n"},{"type":"null"}]},"reasoning_effort":{"$ref":"#/definitions/ReasoningEffort"}}},"input":{"type":"array","items":{"$ref":"#/definitions/EvalItem"},"description":"The input messages evaluated by the grader. Supports text, output text, input image, and input audio content blocks, and may include template strings.\n"},"range":{"type":"array","items":{"type":"number"},"description":"The service requires two numbers for the score range. Defaults to `[0, 1]`.\n"}},"required":["type","name","input","model"]},"GraderMulti":{"type":"object","title":"MultiGrader","description":"A MultiGrader object combines the output of multiple graders to produce a single score.","properties":{"type":{"type":"string","enum":["multi"],"default":"multi","description":"The object type, which is always `multi`."},"name":{"type":"string","description":"The name of the grader."},"graders":{"type":"object","minProperties":1,"additionalProperties":{"oneOf":[{"$ref":"#/definitions/GraderStringCheck"},{"$ref":"#/definitions/GraderTextSimilarity"},{"$ref":"#/definitions/GraderPython"},{"$ref":"#/definitions/GraderScoreModel"},{"$ref":"#/definitions/GraderLabelModel"}]}},"calculate_output":{"type":"string","description":"A formula to calculate the output based on grader results."}},"required":["name","type","graders","calculate_output"]},"ReasoningEffort":{"anyOf":[{"type":"string","enum":["none","minimal","low","medium","high","xhigh","max"],"default":"medium","description":"Constrains effort on reasoning for reasoning models. Currently supported\nvalues are `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`.\nReducing reasoning effort can result in faster responses and fewer tokens\nused on reasoning in a response. Not all reasoning models support every\nvalue. See the\n[reasoning guide](https://developers.openai.com/api/docs/guides/reasoning)\nfor model-specific support.\n"},{"type":"null"}]},"EvalItem":{"type":"object","title":"Eval message object","description":"A message input to the model with a role indicating instruction following\nhierarchy. Instructions given with the `developer` or `system` role take\nprecedence over instructions given with the `user` role. Messages with the\n`assistant` role are presumed to have been generated by the model in previous\ninteractions.\n","properties":{"role":{"type":"string","description":"The role of the message input. One of `user`, `assistant`, `system`, or\n`developer`.\n","enum":["user","assistant","system","developer"]},"content":{"$ref":"#/definitions/EvalItemContent"},"type":{"type":"string","description":"The type of the message input. Always `message`.\n","enum":["message"]}},"required":["role","content"]},"GraderLabelModel":{"type":"object","title":"LabelModelGrader","description":"A LabelModelGrader object which uses a model to assign labels to each item\nin the evaluation.\n","properties":{"type":{"description":"The object type, which is always `label_model`.","type":"string","enum":["label_model"]},"name":{"type":"string","description":"The name of the grader."},"model":{"type":"string","description":"The model to use for the evaluation. Must support structured outputs."},"input":{"type":"array","items":{"$ref":"#/definitions/EvalItem"}},"labels":{"type":"array","items":{"type":"string"},"description":"The labels to assign to each item in the evaluation."},"passing_labels":{"type":"array","items":{"type":"string"},"description":"The labels that indicate a passing result. Must be a subset of labels."}},"required":["type","model","input","passing_labels","labels","name"]},"EvalItemContent":{"title":"Eval content","description":"Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items.\n","oneOf":[{"$ref":"#/definitions/EvalItemContentItem"},{"$ref":"#/definitions/EvalItemContentArray"}]},"EvalItemContentItem":{"title":"Eval content item","description":"A single content item: input text, output text, input image, or input audio.\n","oneOf":[{"$ref":"#/definitions/EvalItemContentText"},{"$ref":"#/definitions/InputTextContent"},{"$ref":"#/definitions/EvalItemContentOutputText"},{"$ref":"#/definitions/EvalItemInputImage"},{"$ref":"#/definitions/InputAudio"}]},"EvalItemContentArray":{"type":"array","title":"An array of Input text, Output text, Input image, and Input audio","description":"A list of inputs, each of which may be either an input text, output text, input\nimage, or input audio object.\n","items":{"$ref":"#/definitions/EvalItemContentItem"}},"EvalItemContentText":{"type":"string","title":"Text input","description":"A text input to the model.\n"},"InputTextContent":{"properties":{"type":{"type":"string","enum":["input_text"],"description":"The type of the input item. Always `input_text`.","default":"input_text"},"text":{"type":"string","description":"The text input to the model."},"prompt_cache_breakpoint":{"$ref":"#/definitions/PromptCacheBreakpointConfig"}},"type":"object","required":["type","text"],"title":"Input text","description":"A text input to the model."},"EvalItemContentOutputText":{"type":"object","title":"Output text","description":"A text output from the model.\n","properties":{"type":{"type":"string","description":"The type of the output text. Always `output_text`.\n","enum":["output_text"]},"text":{"type":"string","description":"The text output from the model.\n"}},"required":["type","text"]},"EvalItemInputImage":{"title":"Input image","description":"An image input block used within EvalItem content arrays.","type":"object","properties":{"type":{"type":"string","description":"The type of the image input. Always `input_image`.\n","enum":["input_image"]},"image_url":{"type":"string","format":"uri","description":"The URL of the image input.\n"},"detail":{"type":"string","description":"The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`.\n"}},"required":["type","image_url"]},"InputAudio":{"type":"object","title":"Input audio","description":"An audio input to the model.\n","properties":{"type":{"type":"string","description":"The type of the input item. Always `input_audio`.\n","enum":["input_audio"]},"input_audio":{"type":"object","properties":{"data":{"type":"string","description":"Base64-encoded audio data.\n"},"format":{"type":"string","description":"The format of the audio data. Currently supported formats are `mp3` and\n`wav`.\n","enum":["mp3","wav"]}},"required":["data","format"]}},"required":["type","input_audio"]},"PromptCacheBreakpointConfig":{"properties":{"mode":{"type":"string","enum":["explicit"],"description":"The breakpoint mode. Always `explicit`.","default":"explicit"},"ttl":{"type":"string","enum":["30m","12h"],"description":"The minimum lifetime of this cache breakpoint. Defaults to the request TTL."}},"type":"object","required":["mode"],"title":"Prompt cache breakpoint","description":"Marks the exact end of a reusable prompt prefix. When `ttl` is omitted, the breakpoint inherits the request's `prompt_cache_options.ttl`; the boundary is not rounded to a token block."}}},"name":"Validate grader","id":"openai.validategrader","example":{"body":{"grader":{"type":"string_check","name":"example","input":"example","reference":"example","operation":"eq"}}},"maxRequests":1,"summary":false}}