Documentation | Evaluations

Evaluations

EvaluationsResource

Methods

client.evaluations.create() ->

Evaluation

post/v5/evaluations

Create Evaluation

client.evaluations.list() -> SyncCursorPage[

Evaluation

]

get/v5/evaluations

List Evaluations

client.evaluations.retrieve(, ) ->

Evaluation

get/v5/evaluations/{evaluation_id}

Get Evaluation

client.evaluations.archive() ->

Evaluation

delete/v5/evaluations/{evaluation_id}

Archive Evaluation

client.evaluations.update(, ) ->

Evaluation

patch/v5/evaluations/{evaluation_id}

Update or Restore Evaluation

client.evaluations.retrieve_schema(, ) ->

EvaluationSchemaResponse

get/v5/evaluations/{evaluation_id}/schema

Get schema information for evaluation item data, including field names, types, and occurrence counts.

client.evaluations.filter() -> SyncCursorPage[

Evaluation

]

post/v5/evaluations/filter

Filter evaluations using metadata and other criteria. Supports up to 10 filters with AND logic.

Parameters

filters: Iterable[

Filter

]

List of metadata filters to apply (maximum 10)

key: str

The metadata key to filter on

operator: Literal["==", "!=", ">=", "<=", "IN", "NOT_IN"]

The comparison operator to use

value: str

The value to compare against (string for all types)

object: Optional[Literal["metadata_filter"]]

(default: "metadata_filter")

ending_before: Optional[str]

include_archived: Optional[

bool

]

limit: Optional[int]

(maximum: 10000, minimum: 1, default: 100)

sort_by: Optional[str]

sort_order: Optional[

SortOrder

]

starting_after: Optional[str]

views: Optional[List[

EvaluationViews

]]

(default: [])

"tasks"

Returns

Evaluation

id: str

created_at:

datetime

(format: date-time)

created_by:

Identity

The identity that created the entity.

datasets: List[

Dataset

]

name: str

status: Literal["failed", "completed", "running"]

tags: List[str]

The tags associated with the entity

archived_at: Optional[datetime]

(format: date-time)

description: Optional[str]

error_count: Optional[int]

Number of task errors across all items in this evaluation.

metadata: Optional[Dict[str,

object

]]

Metadata key-value pairs for the evaluation

object: Optional[Literal["evaluation"]]

(default: "evaluation")

progress: Optional[EvaluationTasksProgressSchema]

Progress of the evaluation's underlying async job

status_reason: Optional[str]

Reason for evaluation status

tasks: Optional[List[

EvaluationTask

]]

Tasks executed during evaluation. Populated with optional task view.

Request example

import os
from scale_gp_beta import SGPClient

client = SGPClient(
    api_key=os.environ.get("SGP_API_KEY"),  # This is the default and can be omitted
)
page = client.evaluations.filter(
    filters=[{
        "key": "key",
        "operator": "==",
        "value": "value",
    }],
)
page = page.items[0]
print(page.id)

200Example

{
  "has_more": true,
  "items": [
    {
      "id": "id",
      "created_at": "2019-12-27T18:11:19.117Z",
      "created_by": {
        "id": "id",
        "type": "user",
        "object": "identity"
      },
      "datasets": [
        {
          "id": "id",
          "created_at": "2019-12-27T18:11:19.117Z",
          "created_by": {
            "id": "id",
            "type": "user",
            "object": "identity"
          },
          "current_version_num": 0,
          "name": "name",
          "tags": [
            "string"
          ],
          "archived_at": "2019-12-27T18:11:19.117Z",
          "description": "description",
          "object": "dataset"
        }
      ],
      "name": "name",
      "status": "failed",
      "tags": [
        "string"
      ],
      "archived_at": "2019-12-27T18:11:19.117Z",
      "description": "description",
      "error_count": 0,
      "metadata": {
        "foo": "bar"
      },
      "object": "evaluation",
      "progress": {
        "items": {
          "failed": 0,
          "pending": 0,
          "successful": 0,
          "total": 0,
          "failed_items": [
            {
              "item_id": "item_id",
              "error": "error",
              "error_type": "error_type"
            }
          ]
        },
        "workflows": {
          "completed": 0,
          "failed": 0,
          "pending": 0,
          "total": 0
        }
      },
      "status_reason": "status_reason",
      "tasks": [
        {
          "configuration": {
            "messages": [
              {
                "foo": "bar"
              }
            ],
            "model": "model",
            "audio": {
              "foo": "bar"
            },
            "frequency_penalty": 0,
            "function_call": {
              "foo": "bar"
            },
            "functions": [
              {
                "foo": "bar"
              }
            ],
            "logit_bias": {
              "foo": 0
            },
            "logprobs": true,
            "max_completion_tokens": 0,
            "max_tokens": 0,
            "metadata": {
              "foo": "string"
            },
            "modalities": [
              "string"
            ],
            "n": 0,
            "parallel_tool_calls": true,
            "prediction": {
              "foo": "bar"
            },
            "presence_penalty": 0,
            "reasoning_effort": "reasoning_effort",
            "response_format": {
              "foo": "bar"
            },
            "seed": 0,
            "stop": "stop",
            "store": true,
            "temperature": 0,
            "tool_choice": "tool_choice",
            "tools": [
              {
                "foo": "bar"
              }
            ],
            "top_k": 0,
            "top_logprobs": 0,
            "top_p": 0
          },
          "alias": "alias",
          "task_type": "chat_completion"
        }
      ]
    }
  ],
  "total": 0,
  "limit": 0,
  "object": "list"
}

client.evaluations.retrieve_taxonomy() ->

EvaluationRetrieveTaxonomyResponse

get/v5/evaluations/{evaluation_id}/taxonomy

Get taxonomy JSON for contributor evaluation question tasks.

Domain types

class AutoEvaluationAgentTaskRequestWithItemLocator: ...

class Evaluation: ...

Dict[str,

object

]

class EvaluationSchemaResponse: ...

Schema information for an evaluation's item data structure

EvaluationTask

class EvaluationTasksProgressSchema: ...

Literal["tasks"]

str

class PaginatedListEvaluation: ...

Evaluations

Tasks

EvaluationsResource.TasksResource

Methods

client.evaluations.tasks.add(, ) ->

Evaluation

post/v5/evaluations/{evaluation_id}/tasks

Add a new test criteria (LLM judge, contributor question, etc.) to an existing evaluation. Gated: rejected if any contributor annotation task has been claimed or completed. Kicks off the evaluation workflow so the new task runs against existing items.

client.evaluations.tasks.update(, ) ->

Evaluation

patch/v5/evaluations/{evaluation_id}/tasks/{alias}

Replace a single test criteria's configuration, identified by its alias. Gated: rejected if any contributor annotation task for the evaluation has been claimed or completed.