Deploy Vertex AI Gemini 2.0 at Scale on Google Cloud: 2026 Guide

# Set your project once — used in every command below
export PROJECT_ID="your-project-id"
export REGION="us-central1"   # Gemini 2.0 is available in us-central1, europe-west4, asia-northeast1

gcloud config set project $PROJECT_ID

# Enable required APIs
gcloud services enable \
  aiplatform.googleapis.com \
  compute.googleapis.com \
  container.googleapis.com \
  cloudresourcemanager.googleapis.com \
  --project=$PROJECT_ID

# Dedicated endpoints give you reserved capacity and a private DNS name
gcloud ai endpoints create \
  --region=$REGION \
  --display-name="gemini-prod-endpoint" \
  --network="projects/$PROJECT_ID/global/networks/default" \
  --project=$PROJECT_ID

# Capture the endpoint ID for the next steps
export ENDPOINT_ID=$(gcloud ai endpoints list \
  --region=$REGION \
  --filter="displayName=gemini-prod-endpoint" \
  --format="value(name)" | awk -F'/' '{print $NF}')

echo "Endpoint ID: $ENDPOINT_ID"

# Deploy Flash as primary (95% traffic) — faster, cheaper, handles most requests
gcloud ai endpoints deploy-model $ENDPOINT_ID \
  --region=$REGION \
  --model="publishers/google/models/gemini-2.0-flash-001" \
  --display-name="gemini-flash-primary" \
  --traffic-split="0=95" \
  --min-replica-count=2 \
  --max-replica-count=20 \
  --machine-type="n1-standard-4" \
  --project=$PROJECT_ID

export FLASH_MODEL_ID="<deployedModelId from output>"

# Pro handles the 5% of requests needing deeper reasoning
gcloud ai endpoints deploy-model $ENDPOINT_ID \
  --region=$REGION \
  --model="publishers/google/models/gemini-2.0-pro-exp-02-05" \
  --display-name="gemini-pro-fallback" \
  --traffic-split="$FLASH_MODEL_ID=95,0=5" \
  --min-replica-count=1 \
  --max-replica-count=5 \
  --machine-type="n1-standard-8" \
  --project=$PROJECT_ID

# Create a dedicated GCP service account for Gemini calls
gcloud iam service-accounts create gemini-inference-sa \
  --display-name="Gemini Inference Service Account" \
  --project=$PROJECT_ID

# Grant it the Vertex AI User role (predict only — no admin)
gcloud projects add-iam-policy-binding $PROJECT_ID \
  --member="serviceAccount:gemini-inference-sa@$PROJECT_ID.iam.gserviceaccount.com" \
  --role="roles/aiplatform.user"

# Allow your Kubernetes service account to impersonate it
# Replace NAMESPACE and KSA_NAME with your actual values
export NAMESPACE="production"
export KSA_NAME="gemini-app"

gcloud iam service-accounts add-iam-policy-binding \
  gemini-inference-sa@$PROJECT_ID.iam.gserviceaccount.com \
  --role="roles/iam.workloadIdentityUser" \
  --member="serviceAccount:$PROJECT_ID.svc.id.goog[$NAMESPACE/$KSA_NAME]"

# k8s/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gemini-app
  namespace: production
  annotations:
    # This is what links the K8s SA to the GCP SA
    iam.gke.io/gcp-service-account: gemini-inference-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com

# requirements: google-cloud-aiplatform>=1.50.0, tenacity

import vertexai
from vertexai.generative_models import GenerativeModel, GenerationConfig
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
from google.api_core.exceptions import ResourceExhausted, ServiceUnavailable
import os

PROJECT_ID = os.environ["GCP_PROJECT_ID"]
REGION = os.environ.get("GCP_REGION", "us-central1")
ENDPOINT_ID = os.environ["VERTEX_ENDPOINT_ID"]

vertexai.init(project=PROJECT_ID, location=REGION)

# Retry on rate limits and transient 5xx — exponential backoff caps at 60s
@retry(
    retry=retry_if_exception_type((ResourceExhausted, ServiceUnavailable)),
    wait=wait_exponential(multiplier=1, min=2, max=60),
    stop=stop_after_attempt(5),
)
def generate_streaming(prompt: str, max_tokens: int = 2048) -> str:
    model = GenerativeModel(
        # Use endpoint ID to route through your dedicated endpoint
        model_name=f"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{ENDPOINT_ID}",
    )

    config = GenerationConfig(
        max_output_tokens=max_tokens,
        temperature=0.2,      # Lower = more deterministic; good for code and structured output
        top_p=0.95,
    )

    full_response = []

    # Stream chunks as they arrive — reduces time-to-first-token for users
    for chunk in model.generate_content(
        prompt,
        generation_config=config,
        stream=True,
    ):
        if chunk.text:
            full_response.append(chunk.text)
            print(chunk.text, end="", flush=True)

    return "".join(full_response)


if __name__ == "__main__":
    result = generate_streaming(
        "Explain the difference between Gemini 2.0 Flash and Pro in 3 bullet points."
    )
    print(f"\n\nTotal tokens used: {len(result.split())}")

# Create an access policy (org-level, do once)
gcloud access-context-manager policies create \
  --organization=$ORG_ID \
  --title="prod-access-policy"

export POLICY_NAME=$(gcloud access-context-manager policies list \
  --organization=$ORG_ID --format="value(name)")

# Create a service perimeter around your AI project
gcloud access-context-manager perimeters create gemini-prod-perimeter \
  --policy=$POLICY_NAME \
  --title="Gemini Production Perimeter" \
  --resources="projects/$PROJECT_NUMBER" \
  --restricted-services="aiplatform.googleapis.com" \
  --access-levels=""

# Test a prediction directly from gcloud
gcloud ai endpoints predict $ENDPOINT_ID \
  --region=$REGION \
  --json-request='{
    "instances": [{
      "content": "What is 2 + 2? Reply with just the number."
    }]
  }' \
  --project=$PROJECT_ID

# Watch replica count respond to load
gcloud ai operations list \
  --region=$REGION \
  --filter="metadata.endpointId=$ENDPOINT_ID" \
  --sort-by="~createTime" \
  --limit=10

gcloud ai endpoints describe $ENDPOINT_ID \
  --region=$REGION \
  --format="yaml(trafficSplit)"