LangSmith CI/CD Integration: Automated Regression Testing 2026

# Use uv for fast installs (Python 3.11+)
uv add langsmith langchain-openai python-dotenv

# Or pip
pip install langsmith langchain-openai python-dotenv

export LANGCHAIN_API_KEY="ls__your_key_here"
export LANGCHAIN_TRACING_V2=true
export OPENAI_API_KEY="sk-your_key_here"

# scripts/create_dataset.py
from langsmith import Client

client = Client()

# Define your ground-truth examples
# Each example: input your chain receives + reference output to score against
examples = [
    {
        "input": {"question": "What is the capital of France?"},
        "output": {"answer": "Paris"},
    },
    {
        "input": {"question": "Summarize: 'The cat sat on the mat.'"},
        "output": {"answer": "A cat rested on a mat."},
    },
    {
        "input": {"question": "Translate to Spanish: 'Good morning'"},
        "output": {"answer": "Buenos días"},
    },
    # Add 20–50 examples covering your real use cases
]

dataset_name = "qa-regression-v1"

# Idempotent: skip if dataset already exists
existing = [d.name for d in client.list_datasets()]
if dataset_name not in existing:
    dataset = client.create_dataset(dataset_name, description="QA regression suite")
    client.create_examples(
        inputs=[e["input"] for e in examples],
        outputs=[e["output"] for e in examples],
        dataset_id=dataset.id,
    )
    print(f"Created dataset '{dataset_name}' with {len(examples)} examples")
else:
    print(f"Dataset '{dataset_name}' already exists — skipping creation")

python scripts/create_dataset.py

# src/chain.py
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser

def build_chain():
    # Pull prompt from LangSmith Hub in production, or define inline for testing
    prompt = ChatPromptTemplate.from_messages([
        ("system", "You are a helpful assistant. Answer concisely and accurately."),
        ("human", "{question}"),
    ])

    model = ChatOpenAI(
        model="gpt-4o-mini",
        temperature=0,  # temperature=0 for reproducible evals
    )

    return prompt | model | StrOutputParser()


def predict(inputs: dict) -> dict:
    """Adapter function — LangSmith calls this with each dataset input."""
    chain = build_chain()
    answer = chain.invoke({"question": inputs["question"]})
    return {"answer": answer}

# src/evaluators.py
from langsmith.schemas import Run, Example
from langchain_openai import ChatOpenAI


def semantic_accuracy(run: Run, example: Example) -> dict:
    """
    LLM-as-judge: scores whether the prediction is semantically correct
    relative to the reference. Returns a score between 0.0 and 1.0.
    """
    prediction = run.outputs.get("answer", "")
    reference = example.outputs.get("answer", "")

    if not prediction:
        return {"key": "semantic_accuracy", "score": 0.0}

    judge = ChatOpenAI(model="gpt-4o-mini", temperature=0)

    verdict = judge.invoke(
        f"""You are an evaluation judge. Score whether the prediction matches
the reference answer semantically (not necessarily word-for-word).

Reference: {reference}
Prediction: {prediction}

Reply with ONLY a number between 0.0 (completely wrong) and 1.0 (correct).
No explanation."""
    )

    try:
        score = float(verdict.content.strip())
        score = max(0.0, min(1.0, score))  # clamp to [0, 1]
    except ValueError:
        score = 0.0

    return {"key": "semantic_accuracy", "score": score}


def exact_match(run: Run, example: Example) -> dict:
    """
    Deterministic evaluator for cases where exact output matters
    (structured data, SQL, code snippets).
    """
    prediction = run.outputs.get("answer", "").strip().lower()
    reference = example.outputs.get("answer", "").strip().lower()
    return {"key": "exact_match", "score": 1.0 if prediction == reference else 0.0}

# scripts/run_eval.py
import sys
from langsmith import Client
from langsmith.evaluation import evaluate
from src.chain import predict
from src.evaluators import semantic_accuracy

DATASET_NAME = "qa-regression-v1"
PASS_THRESHOLD = 0.85  # fail CI if average semantic_accuracy drops below this

client = Client()

print(f"Running eval against dataset: {DATASET_NAME}")
print(f"Pass threshold: {PASS_THRESHOLD}")

results = evaluate(
    predict,
    data=DATASET_NAME,
    evaluators=[semantic_accuracy],
    experiment_prefix="ci-regression",  # groups runs in LangSmith UI by prefix
    metadata={"trigger": "github-actions"},
)

# Compute average score across all examples
scores = [
    r["evaluation_results"]["results"][0].score
    for r in results
    if r.get("evaluation_results")
]

if not scores:
    print("ERROR: No scores returned — check evaluator output format")
    sys.exit(1)

avg_score = sum(scores) / len(scores)
print(f"\nResults: {len(scores)} examples evaluated")
print(f"Average semantic_accuracy: {avg_score:.3f}")

if avg_score < PASS_THRESHOLD:
    print(f"\nFAIL: {avg_score:.3f} is below threshold {PASS_THRESHOLD}")
    print("Review regressions at: https://smith.langchain.com")
    sys.exit(1)  # non-zero exit fails the CI job

print(f"\nPASS: {avg_score:.3f} meets threshold {PASS_THRESHOLD}")
sys.exit(0)

# .github/workflows/llm-regression.yml
name: LLM Regression Tests

on:
  pull_request:
    branches: [main]
    paths:
      # Only run evals when prompt/chain code changes — not on README edits
      - "src/**"
      - "prompts/**"
      - "scripts/run_eval.py"

jobs:
  regression:
    runs-on: ubuntu-latest
    timeout-minutes: 15  # prevent runaway eval jobs from burning API budget

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"

      - name: Install dependencies
        run: pip install langsmith langchain-openai python-dotenv

      - name: Run LangSmith regression eval
        env:
          LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
          LANGCHAIN_TRACING_V2: "true"
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: python scripts/run_eval.py

# Simulate a regression locally before pushing
python scripts/run_eval.py

# scripts/run_eval.py (extended version)
import json
import hashlib
from pathlib import Path

CACHE_PATH = Path(".langsmith_cache/dataset.json")

def load_or_fetch_dataset(client, dataset_name: str) -> list:
    dataset = client.read_dataset(dataset_name=dataset_name)
    cache_key = str(dataset.modified_at)  # invalidate cache on dataset update

    if CACHE_PATH.exists():
        cached = json.loads(CACHE_PATH.read_text())
        if cached.get("cache_key") == cache_key:
            print("Using cached dataset")
            return cached["examples"]

    examples = list(client.list_examples(dataset_name=dataset_name))
    CACHE_PATH.parent.mkdir(exist_ok=True)
    CACHE_PATH.write_text(json.dumps({
        "cache_key": cache_key,
        "examples": [e.dict() for e in examples],
    }))
    return examples

- uses: actions/cache@v4
  with:
    path: .langsmith_cache
    key: langsmith-dataset-${{ hashFiles('scripts/create_dataset.py') }}