Ollama Error Pattern Recognition: Master Log Analysis Techniques for Fast Debugging

# Memory pattern example
ERROR: failed to allocate 8.5GB for model weights
FATAL: insufficient GPU memory (required: 8192MB, available: 4096MB)

# Network pattern example
ERROR: connection timeout after 30s
WARN: retrying connection to localhost:11434 (attempt 3/5)

# Model loading pattern example
ERROR: model file corrupted at offset 1024
FATAL: unsupported model format version 2.1

import re
from datetime import datetime

def parse_ollama_log(log_line):
    """Extract timestamp, level, and message from Ollama log entries"""
    
    # Pattern for Ollama log format
    pattern = r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\s+(\w+):\s+(.+)'
    
    match = re.match(pattern, log_line)
    if match:
        return {
            'timestamp': datetime.fromisoformat(match.group(1)),
            'level': match.group(2),
            'message': match.group(3)
        }
    return None

# Example usage
log_entry = "2024-01-15T14:30:45 ERROR: failed to load model llama2:7b"
parsed = parse_ollama_log(log_entry)
print(f"Level: {parsed['level']}, Message: {parsed['message']}")

from collections import defaultdict, Counter

def analyze_error_frequency(log_entries):
    """Analyze error frequency patterns in Ollama logs"""
    
    error_counts = defaultdict(int)
    hourly_errors = defaultdict(int)
    
    for entry in log_entries:
        if entry['level'] == 'ERROR':
            # Count specific error types
            error_type = extract_error_type(entry['message'])
            error_counts[error_type] += 1
            
            # Track hourly distribution
            hour = entry['timestamp'].hour
            hourly_errors[hour] += 1
    
    return {
        'error_types': dict(error_counts),
        'hourly_distribution': dict(hourly_errors)
    }

def extract_error_type(message):
    """Classify error messages into categories"""
    
    if 'memory' in message.lower():
        return 'memory_error'
    elif 'connection' in message.lower():
        return 'network_error'
    elif 'model' in message.lower():
        return 'model_error'
    else:
        return 'unknown_error'

import pandas as pd
import numpy as np

def detect_error_trends(log_data):
    """Detect trending error patterns using time-series analysis"""
    
    # Convert to DataFrame for easier manipulation
    df = pd.DataFrame(log_data)
    df['timestamp'] = pd.to_datetime(df['timestamp'])
    
    # Group errors by hour
    hourly_errors = df[df['level'] == 'ERROR'].groupby(
        df['timestamp'].dt.floor('H')
    ).size()
    
    # Calculate rolling average for trend detection
    hourly_errors['rolling_avg'] = hourly_errors.rolling(window=6).mean()
    
    # Identify anomalies (errors > 2 standard deviations above mean)
    threshold = hourly_errors.mean() + (2 * hourly_errors.std())
    anomalies = hourly_errors[hourly_errors > threshold]
    
    return {
        'hourly_errors': hourly_errors,
        'anomalies': anomalies,
        'threshold': threshold
    }

import re

class OllamaErrorPatterns:
    """Collection of regex patterns for Ollama error recognition"""
    
    MEMORY_PATTERNS = [
        r'failed to allocate (\d+\.?\d*)(GB|MB) for model',
        r'insufficient (GPU|CPU) memory',
        r'out of memory.*required: (\d+)MB'
    ]
    
    NETWORK_PATTERNS = [
        r'connection timeout after (\d+)s',
        r'failed to connect to ([^:]+):(\d+)',
        r'network unreachable'
    ]
    
    MODEL_PATTERNS = [
        r'model file corrupted at offset (\d+)',
        r'unsupported model format version ([\d.]+)',
        r'model ([^:]+):([^\s]+) not found'
    ]
    
    def classify_error(self, message):
        """Classify error message using pattern matching"""
        
        for pattern in self.MEMORY_PATTERNS:
            if re.search(pattern, message, re.IGNORECASE):
                return 'memory_error'
        
        for pattern in self.NETWORK_PATTERNS:
            if re.search(pattern, message, re.IGNORECASE):
                return 'network_error'
        
        for pattern in self.MODEL_PATTERNS:
            if re.search(pattern, message, re.IGNORECASE):
                return 'model_error'
        
        return 'unknown_error'

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import numpy as np

def discover_error_patterns(error_messages):
    """Use ML clustering to discover error patterns automatically"""
    
    # Convert error messages to numerical features
    vectorizer = TfidfVectorizer(
        max_features=100,
        stop_words='english',
        ngram_range=(1, 2)
    )
    
    # Transform messages to TF-IDF vectors
    message_vectors = vectorizer.fit_transform(error_messages)
    
    # Cluster similar error messages
    n_clusters = min(10, len(error_messages) // 5)  # Adaptive cluster count
    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
    clusters = kmeans.fit_predict(message_vectors)
    
    # Analyze each cluster
    cluster_analysis = {}
    for i in range(n_clusters):
        cluster_messages = [msg for j, msg in enumerate(error_messages) if clusters[j] == i]
        cluster_analysis[i] = {
            'count': len(cluster_messages),
            'sample_messages': cluster_messages[:3],
            'representative_terms': get_cluster_terms(vectorizer, kmeans.cluster_centers_[i])
        }
    
    return cluster_analysis

def get_cluster_terms(vectorizer, cluster_center):
    """Extract most representative terms for a cluster"""
    
    feature_names = vectorizer.get_feature_names_out()
    top_indices = cluster_center.argsort()[-10:][::-1]
    
    return [feature_names[i] for i in top_indices]

import asyncio
import aiofiles
from datetime import datetime

class OllamaLogMonitor:
    """Real-time Ollama log monitoring system"""
    
    def __init__(self, log_file_path, alert_callback):
        self.log_file = log_file_path
        self.alert_callback = alert_callback
        self.error_patterns = OllamaErrorPatterns()
        self.last_position = 0
    
    async def monitor_logs(self):
        """Monitor log file for new entries"""
        
        while True:
            try:
                async with aiofiles.open(self.log_file, 'r') as f:
                    await f.seek(self.last_position)
                    new_lines = await f.readlines()
                    
                    for line in new_lines:
                        await self.process_log_line(line.strip())
                    
                    self.last_position = await f.tell()
                
                await asyncio.sleep(1)  # Check every second
                
            except FileNotFoundError:
                await asyncio.sleep(5)  # Wait for log file creation
    
    async def process_log_line(self, line):
        """Process individual log line for errors"""
        
        parsed = parse_ollama_log(line)
        if parsed and parsed['level'] == 'ERROR':
            error_type = self.error_patterns.classify_error(parsed['message'])
            
            alert_data = {
                'timestamp': parsed['timestamp'],
                'error_type': error_type,
                'message': parsed['message']
            }
            
            await self.alert_callback(alert_data)

# Usage example
async def error_alert_handler(alert_data):
    """Handle error alerts"""
    print(f"ALERT: {alert_data['error_type']} at {alert_data['timestamp']}")
    print(f"Message: {alert_data['message']}")
    
    # Send to monitoring system, email, Slack, etc.
    await send_to_monitoring_system(alert_data)

async def send_to_monitoring_system(alert_data):
    """Send alert to external monitoring system"""
    # Implementation depends on your monitoring stack
    pass

from datetime import datetime, timedelta
from collections import deque

class ErrorThresholdManager:
    """Manage error thresholds and alerting logic"""
    
    def __init__(self):
        self.error_history = deque(maxlen=1000)  # Keep last 1000 errors
        self.alert_thresholds = {
            'memory_error': {'count': 5, 'window': 300},      # 5 errors in 5 minutes
            'network_error': {'count': 10, 'window': 600},    # 10 errors in 10 minutes
            'model_error': {'count': 3, 'window': 180}        # 3 errors in 3 minutes
        }
        self.alert_cooldown = {}  # Prevent spam alerts
    
    def should_alert(self, error_type, timestamp):
        """Determine if an alert should be triggered"""
        
        # Add error to history
        self.error_history.append({
            'type': error_type,
            'timestamp': timestamp
        })
        
        # Check if we're in cooldown period
        if self.is_in_cooldown(error_type, timestamp):
            return False
        
        # Check if threshold is exceeded
        if self.check_threshold(error_type, timestamp):
            self.set_cooldown(error_type, timestamp)
            return True
        
        return False
    
    def check_threshold(self, error_type, timestamp):
        """Check if error threshold is exceeded"""
        
        if error_type not in self.alert_thresholds:
            return False
        
        threshold = self.alert_thresholds[error_type]
        window_start = timestamp - timedelta(seconds=threshold['window'])
        
        # Count errors of this type in the time window
        recent_errors = [
            e for e in self.error_history
            if e['type'] == error_type and e['timestamp'] >= window_start
        ]
        
        return len(recent_errors) >= threshold['count']
    
    def is_in_cooldown(self, error_type, timestamp):
        """Check if error type is in cooldown period"""
        
        if error_type not in self.alert_cooldown:
            return False
        
        cooldown_end = self.alert_cooldown[error_type]
        return timestamp < cooldown_end
    
    def set_cooldown(self, error_type, timestamp):
        """Set cooldown period for error type"""
        
        # 30-minute cooldown to prevent spam
        self.alert_cooldown[error_type] = timestamp + timedelta(minutes=30)

# Check available memory
free -h

# Monitor GPU memory (NVIDIA)
nvidia-smi --query-gpu=memory.used,memory.total --format=csv

# Optimize Ollama memory usage
export OLLAMA_NUM_PARALLEL=1
export OLLAMA_MAX_LOADED_MODELS=1

# Check Ollama service status
systemctl status ollama

# Test connectivity
curl -f http://localhost:11434/api/version

# Check listening ports
netstat -tlnp | grep 11434

# Re-download model
ollama pull llama2:7b --force

# Check model files
ls -la ~/.ollama/models/

# Verify disk health
fsck /dev/sda1

# docker-compose.yml for structured logging
version: '3.8'
services:
  ollama:
    image: ollama/ollama
    ports:
      - "11434:11434"
    environment:
      - OLLAMA_DEBUG=1
      - OLLAMA_LOG_LEVEL=info
    volumes:
      - ./ollama-data:/root/.ollama
      - ./logs:/var/log/ollama
    logging:
      driver: json-file
      options:
        max-size: "10m"
        max-file: "3"

# /etc/logrotate.d/ollama
/var/log/ollama/*.log {
    daily
    rotate 7
    compress
    delaycompress
    missingok
    notifempty
    create 0644 ollama ollama
    postrotate
        systemctl reload ollama
    endscript
}

import psutil
import time

class PerformanceCorrelator:
    """Correlate errors with system performance metrics"""
    
    def __init__(self):
        self.metrics_history = []
    
    def collect_metrics(self):
        """Collect system performance metrics"""
        
        return {
            'timestamp': time.time(),
            'cpu_percent': psutil.cpu_percent(interval=1),
            'memory_percent': psutil.virtual_memory().percent,
            'disk_usage': psutil.disk_usage('/').percent,
            'network_io': psutil.net_io_counters()._asdict()
        }
    
    def correlate_with_errors(self, error_timestamp, error_type):
        """Find performance correlations with errors"""
        
        # Find metrics around error time (±5 minutes)
        error_time = error_timestamp.timestamp()
        relevant_metrics = [
            m for m in self.metrics_history
            if abs(m['timestamp'] - error_time) <= 300
        ]
        
        if not relevant_metrics:
            return None
        
        # Calculate averages
        avg_cpu = sum(m['cpu_percent'] for m in relevant_metrics) / len(relevant_metrics)
        avg_memory = sum(m['memory_percent'] for m in relevant_metrics) / len(relevant_metrics)
        
        return {
            'error_type': error_type,
            'avg_cpu_usage': avg_cpu,
            'avg_memory_usage': avg_memory,
            'sample_count': len(relevant_metrics)
        }