Private Deployments

pip install -U predibase

from predibase import Predibase, DeploymentConfig

pb = Predibase(api_token="<PREDIBASE_API_TOKEN>")

# Create a serverless deployment with auto-scaling
deployment = pb.deployments.create(
    name="my-qwen3-8b",
    config=DeploymentConfig(
        base_model="qwen3-8b",
        min_replicas=0,  # Scale to zero when idle (serverless)
        max_replicas=1,  # Scale up to 1 replica on demand
        cooldown_time=3600  # Scale down after 1 hour idle
    )
)

# Use the deployment - it will automatically scale up when needed
client = pb.deployments.client("my-qwen3-8b")
response = client.generate("What is machine learning?")
print(response.generated_text)

config = DeploymentConfig(
    # Required
    base_model="qwen3-8b",  # Name of the base model to deploy

    # Optional parameters
    accelerator=None,  # Specific GPU accelerator to use (e.g., "a10_24gb_100")
    custom_args=None,  # List of additional arguments passed to the model
    cooldown_time=None,  # Time in seconds before scaling down (default: 3600)
    hf_token=None,  # Hugging Face token for private models
    min_replicas=None,  # Minimum number of replicas (default: 0)
    max_replicas=None,  # Maximum number of replicas (default: 1)
    scale_up_threshold=None,  # Request threshold for scaling up
    quantization=None,  # Quantization method to use
    uses_guaranteed_capacity=None,  # Whether to use guaranteed GPU capacity
    max_total_tokens=None,  # Maximum total tokens for context + generation
    lorax_image_tag=None,  # Specific LoRAX image version
    request_logging_enabled=None,  # Enable/disable request logging
    direct_ingress=None,  # Enable/disable direct ingress
    preloaded_adapters=None,  # List of adapter IDs to preload
    speculator=None,  # Single adapter ID or list of adapter IDs to preload
    prefix_caching=None,  # Enable/disable KV cache prefix optimization
    disable_adapters=None   # Disables adapters on the deployment. Defaults to `False` if not explicitly set.
)

# Basic serverless deployment that scales to zero
config = DeploymentConfig(
    base_model="qwen3-8b",
    min_replicas=0,                 # Scale to zero when idle (minimum cost)
    max_replicas=1,                 # Scale up to 1 replica under load
    cooldown_time=3600              # Scale down after 1 hour idle
)

# Always-on deployment with specific hardware and performance optimizations
config = DeploymentConfig(
    base_model="qwen3-8b",
    min_replicas=1,                 # Always keep 1 replica running
    accelerator="l40s_48gb_100",    # Use A10 GPU
    quantization="fp8",             # Use FP8 quantization for higher throughput
)

# Autoscaling deployment with multiple pre-loaded adapters
config = DeploymentConfig(
    base_model="qwen3-8b",
    min_replicas=1,
    max_replicas=4,
    preloaded_adapters=["my-adapter/1", "my-adapter/2"],  # Pre-loaded adapter
)

# List all deployments
deployments = pb.deployments.list()

# Get specific deployment
deployment = pb.deployments.get("my-qwen3-8b")
print(f"Status: {deployment.status}")
print(f"Current Replicas: {deployment.current_replicas}")  # 0 when scaled to zero

from predibase import UpdateDeploymentConfig

# Update scaling configuration
pb.deployments.update(
    deployment_ref="my-qwen3-8b",
    config=UpdateDeploymentConfig(
        min_replicas=1,    # Change to always-on (disable scale-to-zero)
        max_replicas=2,    # Allow scaling to 2 replicas for higher load
        cooldown_time=1800 # Change cooldown time
    )
)

pb.deployments.delete("my-qwen3-8b")

# Deploy with Turbo LoRA or Turbo adapter
pb.deployments.create(
    name="solar-pro-preview-instruct-deployment",
    config=DeploymentConfig(
        base_model="solar-pro-preview-instruct",
        preloaded_adapters=["my-repo/1"], # Where "my-repo/1" is the adapter ID of a Turbo or Turbo LoRA
    )
)

# Since the adapter is already loaded, prompt without needing to specify the adapter dynamically
client = pb.deployments.client("solar-pro-preview-instruct-deployment")
print(client.generate("Where is the best slice shop in NYC?", max_new_tokens=100).generated_text)

config = DeploymentConfig(
    base_model="qwen3-8b",
    prefix_caching=True  # Enable caching to reduce cold start impact
)

config = DeploymentConfig(
    base_model="qwen3-8b",
    requests_logging_enabled=True  # Enable logging for usage analysis
)

config = DeploymentConfig(
    base_model="qwen3-8b",
    uses_guaranteed_capacity=True,  # Enable reserved capacity
    ...
)

pb.deployments.update(
    deployment_ref="my-qwen3-8b",
    config=UpdateDeploymentConfig(
        uses_guaranteed_capacity=True,  # Enable reserved capacity
        ...
    )
)

from predibase import Predibase

pb = Predibase()
client = pb.deployments.client('deployment-name', serving_url_override='<direct_ingress_endpoint>')
response = client.generate(prompt='hello', max_new_tokens=16)

curl --request POST \
  --url 'http://<direct_ingress_endpoint>/<tenant_id>/deployments/v2/llms/<deployment_name>/generate' \
  --header 'Content-Type: application/json' \
  --header 'User-Agent: <user_agent>' \
  --data '{
    "inputs": "hello",
    "parameters": {
        "max_new_tokens": 16,
        "details": true
    }
}'