Generate 3D Scenes from Text with Three.js + AI in 25 Minutes

// scene-manager.ts
import * as THREE from 'three';
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls';

export class SceneManager {
  scene: THREE.Scene;
  camera: THREE.PerspectiveCamera;
  renderer: THREE.WebGLRenderer;
  controls: OrbitControls;

  constructor(container: HTMLElement) {
    // Scene with neutral background
    this.scene = new THREE.Scene();
    this.scene.background = new THREE.Color(0x1a1a1a);

    // Camera positioned for general viewing
    this.camera = new THREE.PerspectiveCamera(
      75,
      container.clientWidth / container.clientHeight,
      0.1,
      1000
    );
    this.camera.position.set(5, 5, 5);
    this.camera.lookAt(0, 0, 0);

    // Renderer with antialiasing
    this.renderer = new THREE.WebGLRenderer({ antialias: true });
    this.renderer.setSize(container.clientWidth, container.clientHeight);
    this.renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); // Limit for performance
    container.appendChild(this.renderer.domElement);

    // Controls for user interaction
    this.controls = new OrbitControls(this.camera, this.renderer.domElement);
    this.controls.enableDamping = true; // Smooth camera movement

    // Basic lighting - hemisphere simulates natural light
    const hemiLight = new THREE.HemisphereLight(0xffffff, 0x444444, 1.5);
    hemiLight.position.set(0, 20, 0);
    this.scene.add(hemiLight);

    // Directional light for shadows
    const dirLight = new THREE.DirectionalLight(0xffffff, 1);
    dirLight.position.set(3, 10, 5);
    this.scene.add(dirLight);

    this.animate();
  }

  animate = () => {
    requestAnimationFrame(this.animate);
    this.controls.update(); // Required when damping enabled
    this.renderer.render(this.scene, this.camera);
  };

  clearScene() {
    // Remove all meshes but keep lights
    this.scene.children
      .filter(child => child instanceof THREE.Mesh)
      .forEach(mesh => {
        this.scene.remove(mesh);
        mesh.geometry.dispose();
        (mesh.material as THREE.Material).dispose();
      });
  }
}

// scene-schema.ts
export interface SceneObject {
  type: 'box' | 'sphere' | 'cylinder' | 'cone' | 'torus' | 'plane';
  position: [number, number, number];
  rotation?: [number, number, number];
  scale?: [number, number, number];
  color: string; // Hex color
  name?: string;
}

export interface SceneDescription {
  objects: SceneObject[];
  cameraPosition?: [number, number, number];
  backgroundColor?: string;
}

// This schema tells Claude exactly what format we need
export const SCENE_SCHEMA = {
  name: "generate_3d_scene",
  description: "Generate a Three.js scene from natural language description",
  parameters: {
    type: "object",
    properties: {
      objects: {
        type: "array",
        items: {
          type: "object",
          properties: {
            type: {
              type: "string",
              enum: ["box", "sphere", "cylinder", "cone", "torus", "plane"]
            },
            position: {
              type: "array",
              items: { type: "number" },
              minItems: 3,
              maxItems: 3,
              description: "[x, y, z] coordinates"
            },
            rotation: {
              type: "array",
              items: { type: "number" },
              minItems: 3,
              maxItems: 3,
              description: "[x, y, z] rotation in radians"
            },
            scale: {
              type: "array",
              items: { type: "number" },
              minItems: 3,
              maxItems: 3,
              description: "[x, y, z] scale factors"
            },
            color: {
              type: "string",
              pattern: "^#[0-9A-Fa-f]{6}$",
              description: "Hex color code"
            },
            name: {
              type: "string",
              description: "Optional label for the object"
            }
          },
          required: ["type", "position", "color"]
        }
      },
      cameraPosition: {
        type: "array",
        items: { type: "number" },
        minItems: 3,
        maxItems: 3
      },
      backgroundColor: {
        type: "string",
        pattern: "^#[0-9A-Fa-f]{6}$"
      }
    },
    required: ["objects"]
  }
};

// ai-scene-generator.ts
import Anthropic from '@anthropic-ai/sdk';
import { SCENE_SCHEMA, SceneDescription } from './scene-schema';

export class AISceneGenerator {
  private client: Anthropic;

  constructor(apiKey: string) {
    this.client = new Anthropic({ apiKey });
  }

  async generateScene(prompt: string): Promise<SceneDescription> {
    const response = await this.client.messages.create({
      model: 'claude-sonnet-4-20250514',
      max_tokens: 2048,
      tools: [SCENE_SCHEMA],
      messages: [{
        role: 'user',
        content: `Create a 3D scene based on this description: "${prompt}". 
                  
                  Guidelines:
                  - Use realistic proportions (typical room is 10x10 units)
                  - Place objects at sensible heights (floor is y=0)
                  - Choose colors that match the description
                  - Position camera to show the whole scene
                  
                  Example: "a table with a lamp" would have:
                  - Box for table at y=0, scale [2, 1, 1]
                  - Cylinder for lamp on table at y=1.5, scale [0.2, 0.5, 0.2]`
      }]
    });

    // Extract tool use from response
    const toolUse = response.content.find(
      block => block.type === 'tool_use'
    );

    if (!toolUse || toolUse.type !== 'tool_use') {
      throw new Error('AI did not generate scene data');
    }

    // Claude returns the scene matching our schema
    return toolUse.input as SceneDescription;
  }
}

// scene-builder.ts
import * as THREE from 'three';
import { SceneManager } from './scene-manager';
import { SceneDescription, SceneObject } from './scene-schema';

export class SceneBuilder {
  constructor(private manager: SceneManager) {}

  buildFromDescription(description: SceneDescription) {
    this.manager.clearScene();

    // Apply background if specified
    if (description.backgroundColor) {
      this.manager.scene.background = new THREE.Color(description.backgroundColor);
    }

    // Create each object
    description.objects.forEach(obj => {
      const mesh = this.createMesh(obj);
      this.manager.scene.add(mesh);
    });

    // Position camera if specified
    if (description.cameraPosition) {
      const [x, y, z] = description.cameraPosition;
      this.manager.camera.position.set(x, y, z);
      this.manager.camera.lookAt(0, 0, 0);
    }
  }

  private createMesh(obj: SceneObject): THREE.Mesh {
    const geometry = this.createGeometry(obj.type, obj.scale);
    const material = new THREE.MeshStandardMaterial({
      color: obj.color,
      metalness: 0.3, // Slight shine
      roughness: 0.7  // Not too glossy
    });

    const mesh = new THREE.Mesh(geometry, material);
    
    // Apply transform
    mesh.position.set(...obj.position);
    if (obj.rotation) {
      mesh.rotation.set(...obj.rotation);
    }
    if (obj.name) {
      mesh.name = obj.name;
    }

    return mesh;
  }

  private createGeometry(
    type: SceneObject['type'],
    scale?: [number, number, number]
  ): THREE.BufferGeometry {
    const [sx = 1, sy = 1, sz = 1] = scale || [1, 1, 1];

    switch (type) {
      case 'box':
        return new THREE.BoxGeometry(sx, sy, sz);
      case 'sphere':
        return new THREE.SphereGeometry(sx, 32, 32); // Higher segments for smoothness
      case 'cylinder':
        return new THREE.CylinderGeometry(sx, sx, sy, 32);
      case 'cone':
        return new THREE.ConeGeometry(sx, sy, 32);
      case 'torus':
        return new THREE.TorusGeometry(sx, sx * 0.3, 16, 100);
      case 'plane':
        return new THREE.PlaneGeometry(sx, sz);
      default:
        return new THREE.BoxGeometry(1, 1, 1); // Fallback
    }
  }
}

// main.ts
import { SceneManager } from './scene-manager';
import { AISceneGenerator } from './ai-scene-generator';
import { SceneBuilder } from './scene-builder';

const container = document.getElementById('canvas-container')!;
const input = document.getElementById('prompt-input') as HTMLInputElement;
const button = document.getElementById('generate-btn') as HTMLButtonElement;
const status = document.getElementById('status') as HTMLDivElement;

// Initialize
const sceneManager = new SceneManager(container);
const aiGenerator = new AISceneGenerator(import.meta.env.VITE_ANTHROPIC_KEY);
const sceneBuilder = new SceneBuilder(sceneManager);

button.addEventListener('click', async () => {
  const prompt = input.value.trim();
  if (!prompt) return;

  button.disabled = true;
  status.textContent = 'Generating scene...';

  try {
    const description = await aiGenerator.generateScene(prompt);
    sceneBuilder.buildFromDescription(description);
    status.textContent = `✓ Created ${description.objects.length} objects`;
  } catch (error) {
    status.textContent = `✗ Error: ${error.message}`;
    console.error(error);
  } finally {
    button.disabled = false;
  }
});

// Handle window resize
window.addEventListener('resize', () => {
  const width = container.clientWidth;
  const height = container.clientHeight;
  
  sceneManager.camera.aspect = width / height;
  sceneManager.camera.updateProjectionMatrix();
  sceneManager.renderer.setSize(width, height);
});

<!-- index.html -->
<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>AI 3D Scene Generator</title>
  <style>
    * { margin: 0; padding: 0; box-sizing: border-box; }
    body {
      font-family: system-ui, -apple-system, sans-serif;
      background: #0a0a0a;
      color: #fff;
      height: 100vh;
      display: flex;
      flex-direction: column;
    }
    #controls {
      padding: 1rem;
      background: #1a1a1a;
      border-bottom: 1px solid #333;
      display: flex;
      gap: 1rem;
      align-items: center;
    }
    #prompt-input {
      flex: 1;
      padding: 0.75rem;
      background: #2a2a2a;
      border: 1px solid #444;
      border-radius: 6px;
      color: #fff;
      font-size: 1rem;
    }
    #generate-btn {
      padding: 0.75rem 1.5rem;
      background: #0066ff;
      border: none;
      border-radius: 6px;
      color: white;
      font-weight: 600;
      cursor: pointer;
      transition: background 0.2s;
    }
    #generate-btn:hover:not(:disabled) {
      background: #0052cc;
    }
    #generate-btn:disabled {
      opacity: 0.5;
      cursor: not-allowed;
    }
    #status {
      padding: 0.5rem 1rem;
      background: #2a2a2a;
      font-size: 0.875rem;
      color: #888;
    }
    #canvas-container {
      flex: 1;
      position: relative;
    }
  </style>
</head>
<body>
  <div id="controls">
    <input 
      type="text" 
      id="prompt-input" 
      placeholder="Describe a 3D scene (e.g., 'a modern living room with a couch and coffee table')"
    />
    <button id="generate-btn">Generate</button>
  </div>
  <div id="status">Ready</div>
  <div id="canvas-container"></div>
  
  <script type="module" src="/src/main.ts"></script>
</body>
</html>

# Simple shapes
"a green sphere floating above a blue box"

# Complex scene
"a minimalist desk setup with a monitor, keyboard, and coffee mug"

# Abstract composition  
"three red cylinders arranged in a triangle with a golden sphere in the center"

// Robust error handling
try {
  const description = await aiGenerator.generateScene(prompt);
  
  // Validate before rendering
  if (description.objects.length === 0) {
    throw new Error('Scene contains no objects');
  }
  
  sceneBuilder.buildFromDescription(description);
} catch (error) {
  if (error.status === 429) {
    status.textContent = 'Rate limited - try again in 60s';
  } else if (error.status === 401) {
    status.textContent = 'Invalid API key';
  } else {
    status.textContent = `Error: ${error.message}`;
  }
}

// Cache common prompts to reduce API calls
const cache = new Map<string, SceneDescription>();

async function getCachedScene(prompt: string) {
  const normalized = prompt.toLowerCase().trim();
  
  if (cache.has(normalized)) {
    return cache.get(normalized)!;
  }
  
  const scene = await aiGenerator.generateScene(prompt);
  cache.set(normalized, scene);
  return scene;
}

// Extend SceneObject interface
interface AnimatedSceneObject extends SceneObject {
  animation?: {
    type: 'rotate' | 'bounce' | 'orbit';
    speed: number;
  };
}

// In SceneBuilder
private animatedMeshes: Array<{
  mesh: THREE.Mesh;
  animation: AnimatedSceneObject['animation'];
}> = [];

buildFromDescription(description: SceneDescription) {
  // ... existing code
  
  // Start animation loop if any objects animate
  if (this.animatedMeshes.length > 0) {
    this.startAnimations();
  }
}

private startAnimations() {
  const animate = () => {
    this.animatedMeshes.forEach(({ mesh, animation }) => {
      if (animation.type === 'rotate') {
        mesh.rotation.y += animation.speed;
      }
      // ... other animation types
    });
    requestAnimationFrame(animate);
  };
  animate();
}