Documentation | Models

Models

ModelsResource

Methods

client.models.create() ->

InferenceModel

post/v5/models

Create Model

client.models.list() -> SyncCursorPage[

InferenceModel

]

get/v5/models

List Models

client.models.update(, ) ->

InferenceModel

patch/v5/models/{model_id}

Update Model

Parameters

model_id: str

model:

Model

class ModelDefaultModelPatchRequest: ...

class ModelModelConfigurationPatchRequest: ...

class ModelSwapNamesModelPatchRequest: ...

Returns

InferenceModel

id: str

created_at:

datetime

(format: date-time)

created_by_identity_type: Literal["user", "service_account"]

created_by_user_id: str

model_type:

InferenceModelType

model_vendor:

InferenceModelVendor

name: str

status: Literal["failed", "ready", "deploying", "deployment_timeout"]

model_availability: Optional[InferenceModelAvailability]

model_metadata: Optional[Dict[str,

object

]]

object: Optional[Literal["model"]]

(default: "model")

vendor_configuration: Optional[

VendorConfiguration

]

Request example

import os
from scale_gp_beta import SGPClient

client = SGPClient(
    api_key=os.environ.get("SGP_API_KEY"),  # This is the default and can be omitted
)
inference_model = client.models.update(
    model_id="model_id",
    model={},
)
print(inference_model.id)

200Example

{
  "id": "id",
  "created_at": "2019-12-27T18:11:19.117Z",
  "created_by_identity_type": "user",
  "created_by_user_id": "created_by_user_id",
  "model_type": "generic",
  "model_vendor": "openai",
  "name": "name",
  "status": "failed",
  "model_availability": "unknown",
  "model_metadata": {
    "foo": "bar"
  },
  "object": "model",
  "vendor_configuration": {
    "model_image": {
      "command": [
        "string"
      ],
      "registry": "registry",
      "repository": "repository",
      "tag": "tag",
      "env_vars": {
        "foo": "bar"
      },
      "healthcheck_route": "healthcheck_route",
      "predict_route": "predict_route",
      "readiness_delay": 0,
      "request_schema": {
        "foo": "bar"
      },
      "response_schema": {
        "foo": "bar"
      },
      "streaming_command": [
        "string"
      ],
      "streaming_predict_route": "streaming_predict_route"
    },
    "model_infra": {
      "cpus": "string",
      "endpoint_type": "async",
      "gpu_type": "nvidia-tesla-t4",
      "gpus": 0,
      "high_priority": true,
      "labels": {
        "foo": "string"
      },
      "max_workers": 0,
      "memory": "memory",
      "min_workers": 0,
      "per_worker": 0,
      "public_inference": true,
      "storage": "storage"
    }
  }
}

client.models.delete() ->

ModelDeleteResponse

delete/v5/models/{model_id}

Delete Model

client.models.retrieve() ->

InferenceModel

get/v5/models/{model_id}

Get Model

Domain types

class InferenceModel: ...

Literal["unknown", "available", "unavailable"]

Literal["generic", "completion", "chat_completion"]

class LaunchVendorConfiguration: ...

class LlmEngineVendorConfiguration: ...

class ModelDeleteResponse: ...