Documentation | Models

Models

ModelsResource

Methods

client.models.create() ->

InferenceModel

post/v5/models

Create Model

client.models.list() -> SyncCursorPage[

InferenceModel

]

get/v5/models

List Models

Parameters

ending_before: Optional[str]

limit: Optional[int]

(maximum: 10000, minimum: 1, default: 100)

model_vendor: Optional[

InferenceModelVendor

]

name: Optional[str]

sort_by: Optional[str]

sort_order: Optional[

SortOrder

]

starting_after: Optional[str]

Returns

InferenceModel

id: str

created_at:

datetime

(format: date-time)

created_by_identity_type: Literal["user", "service_account"]

created_by_user_id: str

model_type:

InferenceModelType

model_vendor:

InferenceModelVendor

name: str

status: Literal["failed", "ready", "deploying", "deployment_timeout"]

model_availability: Optional[InferenceModelAvailability]

model_metadata: Optional[Dict[str,

object

]]

object: Optional[Literal["model"]]

(default: "model")

vendor_configuration: Optional[

VendorConfiguration

]

Request example

import os
from scale_gp_beta import SGPClient

client = SGPClient(
    api_key=os.environ.get("SGP_API_KEY"),  # This is the default and can be omitted
)
page = client.models.list()
page = page.items[0]
print(page.id)

200Example

{
  "has_more": true,
  "items": [
    {
      "id": "id",
      "created_at": "2019-12-27T18:11:19.117Z",
      "created_by_identity_type": "user",
      "created_by_user_id": "created_by_user_id",
      "model_type": "generic",
      "model_vendor": "openai",
      "name": "name",
      "status": "failed",
      "model_availability": "unknown",
      "model_metadata": {
        "foo": "bar"
      },
      "object": "model",
      "vendor_configuration": {
        "model_image": {
          "command": [
            "string"
          ],
          "registry": "registry",
          "repository": "repository",
          "tag": "tag",
          "env_vars": {
            "foo": "bar"
          },
          "healthcheck_route": "healthcheck_route",
          "predict_route": "predict_route",
          "readiness_delay": 0,
          "request_schema": {
            "foo": "bar"
          },
          "response_schema": {
            "foo": "bar"
          },
          "streaming_command": [
            "string"
          ],
          "streaming_predict_route": "streaming_predict_route"
        },
        "model_infra": {
          "cpus": "string",
          "endpoint_type": "async",
          "gpu_type": "nvidia-tesla-t4",
          "gpus": 0,
          "high_priority": true,
          "labels": {
            "foo": "string"
          },
          "max_workers": 0,
          "memory": "memory",
          "min_workers": 0,
          "per_worker": 0,
          "public_inference": true,
          "storage": "storage"
        }
      }
    }
  ],
  "total": 0,
  "limit": 0,
  "object": "list"
}

client.models.update(, ) ->

InferenceModel

patch/v5/models/{model_id}

Update Model

client.models.delete() ->

ModelDeleteResponse

delete/v5/models/{model_id}

Delete Model

client.models.retrieve() ->

InferenceModel

get/v5/models/{model_id}

Get Model

Domain types

class InferenceModel: ...

Literal["unknown", "available", "unavailable"]

Literal["generic", "completion", "chat_completion"]

class LaunchVendorConfiguration: ...

class LlmEngineVendorConfiguration: ...

class ModelDeleteResponse: ...