Commodity Price Forecasting with Ollama: Gold, Oil, and Agriculture Analysis

# Install Ollama (macOS/Linux)
curl -fsSL https://ollama.ai/install.sh | sh

# Download models for financial analysis
ollama pull llama2:13b
ollama pull codellama:7b
ollama pull mistral:7b

# Create virtual environment
python -m venv commodity_forecast
source commodity_forecast/bin/activate  # Linux/Mac
# commodity_forecast\Scripts\activate  # Windows

# Install dependencies
pip install pandas numpy yfinance requests ollama matplotlib seaborn scikit-learn ta-lib

import ollama
import pandas as pd
import numpy as np
import yfinance as yf
from datetime import datetime, timedelta
import json
import warnings
warnings.filterwarnings('ignore')

class CommodityForecaster:
    def __init__(self, model_name="llama2:13b"):
        self.model = model_name
        self.client = ollama.Client()
    
    def query_model(self, prompt, context=None):
        """Send query to local Ollama model"""
        try:
            response = self.client.chat(
                model=self.model,
                messages=[
                    {"role": "system", "content": context or "You are a commodity trading expert."},
                    {"role": "user", "content": prompt}
                ]
            )
            return response['message']['content']
        except Exception as e:
            print(f"Error querying model: {e}")
            return None

def fetch_gold_data(period="2y"):
    """Fetch gold price data with technical indicators"""
    
    # Primary gold data
    gold = yf.download("GC=F", period=period)  # Gold futures
    
    # Supporting indicators
    dxy = yf.download("DX-Y.NYB", period=period)  # Dollar Index
    vix = yf.download("^VIX", period=period)      # Volatility Index
    treasury = yf.download("^TNX", period=period)  # 10-year Treasury
    
    # Combine datasets
    gold_data = pd.DataFrame({
        'gold_close': gold['Close'],
        'gold_volume': gold['Volume'],
        'dxy_close': dxy['Close'],
        'vix_close': vix['Close'],
        'treasury_yield': treasury['Close']
    })
    
    # Calculate technical indicators
    gold_data['sma_20'] = gold_data['gold_close'].rolling(20).mean()
    gold_data['sma_50'] = gold_data['gold_close'].rolling(50).mean()
    gold_data['rsi'] = calculate_rsi(gold_data['gold_close'])
    gold_data['price_change'] = gold_data['gold_close'].pct_change()
    
    return gold_data.dropna()

def calculate_rsi(prices, window=14):
    """Calculate Relative Strength Index"""
    delta = prices.diff()
    gain = (delta.where(delta > 0, 0)).rolling(window=window).mean()
    loss = (-delta.where(delta < 0, 0)).rolling(window=window).mean()
    rs = gain / loss
    return 100 - (100 / (1 + rs))

class GoldForecaster(CommodityForecaster):
    def __init__(self):
        super().__init__()
        self.gold_data = None
    
    def analyze_gold_patterns(self, data):
        """Use Ollama to identify market patterns"""
        
        # Prepare market summary
        latest_data = data.tail(10)
        market_summary = self.create_market_summary(latest_data)
        
        prompt = f"""
        Analyze this gold market data and identify key patterns:
        
        {market_summary}
        
        Consider:
        1. Technical indicator signals (RSI, moving averages)
        2. Dollar strength impact on gold prices
        3. Volatility patterns and market sentiment
        4. Historical price patterns and support/resistance levels
        
        Provide specific price targets and key levels to watch.
        Format as JSON with: analysis, bullish_factors, bearish_factors, price_targets
        """
        
        response = self.query_model(prompt)
        return self.parse_analysis(response)
    
    def create_market_summary(self, data):
        """Create formatted market summary for analysis"""
        summary = []
        for idx, row in data.iterrows():
            summary.append(f"Date: {idx.strftime('%Y-%m-%d')}")
            summary.append(f"Gold: ${row['gold_close']:.2f}")
            summary.append(f"DXY: {row['dxy_close']:.2f}")
            summary.append(f"VIX: {row['vix_close']:.2f}")
            summary.append(f"RSI: {row['rsi']:.1f}")
            summary.append("---")
        
        return "\n".join(summary)
    
    def generate_forecast(self, horizon_days=30):
        """Generate comprehensive gold price forecast"""
        
        if self.gold_data is None:
            self.gold_data = fetch_gold_data()
        
        # Get pattern analysis
        pattern_analysis = self.analyze_gold_patterns(self.gold_data)
        
        # Calculate statistical forecasts
        recent_prices = self.gold_data['gold_close'].tail(30)
        volatility = recent_prices.std()
        trend = recent_prices.pct_change().mean()
        
        # Generate forecast scenarios
        forecast_prompt = f"""
        Based on this gold market analysis:
        {pattern_analysis}
        
        Current price: ${recent_prices.iloc[-1]:.2f}
        30-day volatility: {volatility:.2f}
        Recent trend: {trend:.4f}
        
        Generate {horizon_days}-day price forecasts with:
        1. Bull case scenario (probability and target)
        2. Base case scenario (most likely outcome)
        3. Bear case scenario (downside risk)
        
        Include specific price levels and timeframes.
        Format as JSON with scenarios and probabilities.
        """
        
        forecast_response = self.query_model(forecast_prompt)
        return self.parse_forecast(forecast_response)
    
    def parse_analysis(self, response):
        """Parse model analysis response"""
        try:
            # Extract JSON from response
            json_start = response.find('{')
            json_end = response.rfind('}') + 1
            if json_start != -1 and json_end != -1:
                return json.loads(response[json_start:json_end])
        except:
            pass
        
        # Fallback parsing
        return {
            "analysis": response,
            "bullish_factors": [],
            "bearish_factors": [],
            "price_targets": {}
        }
    
    def parse_forecast(self, response):
        """Parse forecast response"""
        try:
            json_start = response.find('{')
            json_end = response.rfind('}') + 1
            if json_start != -1 and json_end != -1:
                return json.loads(response[json_start:json_end])
        except:
            pass
        
        return {
            "bull_case": {"probability": 0.3, "target": 0},
            "base_case": {"probability": 0.4, "target": 0},
            "bear_case": {"probability": 0.3, "target": 0}
        }

# Usage example
gold_forecaster = GoldForecaster()
gold_forecast = gold_forecaster.generate_forecast(horizon_days=30)
print(json.dumps(gold_forecast, indent=2))

def fetch_oil_data(period="2y"):
    """Fetch comprehensive oil market data"""
    
    # Oil prices and related instruments
    wti = yf.download("CL=F", period=period)      # WTI Crude
    brent = yf.download("BZ=F", period=period)    # Brent Crude
    gasoline = yf.download("RB=F", period=period) # Gasoline futures
    
    # Economic indicators
    dollar_index = yf.download("DX-Y.NYB", period=period)
    energy_etf = yf.download("XLE", period=period)  # Energy sector ETF
    
    # Combine datasets
    oil_data = pd.DataFrame({
        'wti_close': wti['Close'],
        'wti_volume': wti['Volume'],
        'brent_close': brent['Close'],
        'gasoline_close': gasoline['Close'],
        'dxy_close': dollar_index['Close'],
        'energy_sector': energy_etf['Close']
    })
    
    # Calculate oil-specific indicators
    oil_data['wti_brent_spread'] = oil_data['wti_close'] - oil_data['brent_close']
    oil_data['crack_spread'] = oil_data['gasoline_close'] - oil_data['wti_close']
    oil_data['oil_momentum'] = oil_data['wti_close'].pct_change(10)
    oil_data['volatility_10d'] = oil_data['wti_close'].rolling(10).std()
    
    return oil_data.dropna()

class OilForecaster(CommodityForecaster):
    def __init__(self):
        super().__init__()
        self.oil_data = None
    
    def analyze_oil_fundamentals(self, data):
        """Analyze oil market fundamentals using Ollama"""
        
        latest_data = data.tail(5)
        current_price = latest_data['wti_close'].iloc[-1]
        spread = latest_data['wti_brent_spread'].iloc[-1]
        crack_spread = latest_data['crack_spread'].iloc[-1]
        
        prompt = f"""
        Analyze current oil market conditions:
        
        WTI Price: ${current_price:.2f}
        WTI-Brent Spread: ${spread:.2f}
        Crack Spread: ${crack_spread:.2f}
        10-day Volatility: {latest_data['volatility_10d'].iloc[-1]:.2f}
        
        Recent price action:
        {self.format_price_action(latest_data)}
        
        Analyze:
        1. Supply/demand dynamics indicated by spreads
        2. Refining margins and downstream demand
        3. Geopolitical risk factors
        4. Seasonal patterns and inventory cycles
        
        Provide specific trading levels and market outlook.
        """
        
        return self.query_model(prompt)
    
    def format_price_action(self, data):
        """Format recent price action for analysis"""
        action_summary = []
        for idx, row in data.iterrows():
            change = row['wti_close'] - data['wti_close'].iloc[0]
            action_summary.append(f"{idx.strftime('%m/%d')}: ${row['wti_close']:.2f} ({change:+.2f})")
        return "\n".join(action_summary)
    
    def generate_oil_forecast(self, horizon_days=21):
        """Generate comprehensive oil price forecast"""
        
        if self.oil_data is None:
            self.oil_data = fetch_oil_data()
        
        # Get fundamental analysis
        fundamentals = self.analyze_oil_fundamentals(self.oil_data)
        
        # Calculate technical levels
        current_price = self.oil_data['wti_close'].iloc[-1]
        support_levels = self.calculate_support_resistance(self.oil_data['wti_close'])
        
        forecast_prompt = f"""
        Oil Market Forecast Request:
        
        Current WTI: ${current_price:.2f}
        
        Fundamental Analysis:
        {fundamentals}
        
        Technical Levels:
        Support: ${support_levels['support']:.2f}
        Resistance: ${support_levels['resistance']:.2f}
        
        Generate {horizon_days}-day oil price forecast considering:
        1. OPEC+ production decisions
        2. US shale production trends
        3. Global demand patterns
        4. Inventory levels and seasonal factors
        
        Provide specific price targets and risk scenarios.
        """
        
        return self.query_model(forecast_prompt)
    
    def calculate_support_resistance(self, prices):
        """Calculate key support and resistance levels"""
        recent_prices = prices.tail(50)
        
        # Simple support/resistance calculation
        support = recent_prices.min()
        resistance = recent_prices.max()
        
        return {
            'support': support,
            'resistance': resistance,
            'current': prices.iloc[-1]
        }

def fetch_agricultural_data(period="2y"):
    """Fetch comprehensive agricultural commodity data"""
    
    # Major agricultural commodities
    corn = yf.download("ZC=F", period=period)      # Corn futures
    wheat = yf.download("ZW=F", period=period)     # Wheat futures
    soybeans = yf.download("ZS=F", period=period)  # Soybean futures
    
    # Agricultural sector indicators
    agriculture_etf = yf.download("DBA", period=period)  # Agriculture ETF
    
    # Weather and economic factors
    dollar_index = yf.download("DX-Y.NYB", period=period)
    
    ag_data = pd.DataFrame({
        'corn_close': corn['Close'],
        'wheat_close': wheat['Close'],
        'soybeans_close': soybeans['Close'],
        'ag_sector': agriculture_etf['Close'],
        'dxy_close': dollar_index['Close']
    })
    
    # Calculate agricultural indicators
    ag_data['corn_wheat_ratio'] = ag_data['corn_close'] / ag_data['wheat_close']
    ag_data['soy_corn_ratio'] = ag_data['soybeans_close'] / ag_data['corn_close']
    ag_data['ag_momentum'] = ag_data['ag_sector'].pct_change(20)
    
    return ag_data.dropna()

class AgriculturalForecaster(CommodityForecaster):
    def __init__(self):
        super().__init__()
        self.ag_data = None
    
    def analyze_crop_fundamentals(self, data, crop_type="corn"):
        """Analyze crop-specific fundamentals"""
        
        latest_data = data.tail(7)
        current_month = datetime.now().month
        
        # Determine growing season context
        season_context = self.get_seasonal_context(current_month, crop_type)
        
        prompt = f"""
        Analyze {crop_type} market fundamentals:
        
        Current Seasonal Context: {season_context}
        
        Recent Price Data:
        {self.format_crop_data(latest_data, crop_type)}
        
        Ratios and Indicators:
        Corn/Wheat Ratio: {latest_data['corn_wheat_ratio'].iloc[-1]:.3f}
        Soy/Corn Ratio: {latest_data['soy_corn_ratio'].iloc[-1]:.3f}
        
        Analyze:
        1. Seasonal planting/harvest impacts
        2. Weather patterns and crop conditions
        3. Global supply/demand dynamics
        4. Export demand and trade policies
        5. Substitution effects between crops
        
        Provide specific price outlook and key risk factors.
        """
        
        return self.query_model(prompt)
    
    def get_seasonal_context(self, month, crop_type):
        """Get seasonal context for crop analysis"""
        seasons = {
            "corn": {
                "3-5": "Spring planting season - weather critical",
                "6-8": "Growing season - drought/flood risks",
                "9-11": "Harvest season - yield determination",
                "12-2": "Post-harvest - storage and marketing"
            },
            "wheat": {
                "9-11": "Winter wheat planting",
                "12-2": "Dormant period - weather impacts",
                "3-5": "Spring growth - yield potential",
                "6-8": "Harvest season - quality concerns"
            },
            "soybeans": {
                "4-6": "Planting season - acreage decisions",
                "7-9": "Pod filling - critical growth period",
                "10-12": "Harvest and marketing",
                "1-3": "Export season - demand patterns"
            }
        }
        
        for period, description in seasons.get(crop_type, {}).items():
            start, end = map(int, period.split('-'))
            if start <= month <= end:
                return description
        
        return "Transition period"
    
    def format_crop_data(self, data, crop_type):
        """Format crop data for analysis"""
        crop_column = f"{crop_type}_close"
        formatted = []
        
        for idx, row in data.iterrows():
            formatted.append(f"{idx.strftime('%m/%d')}: ${row[crop_column]:.2f}")
        
        return "\n".join(formatted)
    
    def generate_agricultural_forecast(self, crops=["corn", "wheat", "soybeans"]):
        """Generate multi-crop agricultural forecast"""
        
        if self.ag_data is None:
            self.ag_data = fetch_agricultural_data()
        
        forecasts = {}
        
        for crop in crops:
            # Get crop-specific analysis
            crop_analysis = self.analyze_crop_fundamentals(self.ag_data, crop)
            
            # Generate forecast
            current_price = self.ag_data[f"{crop}_close"].iloc[-1]
            
            forecast_prompt = f"""
            Agricultural Forecast for {crop.upper()}:
            
            Current Price: ${current_price:.2f}
            
            Fundamental Analysis:
            {crop_analysis}
            
            Generate 60-day price forecast considering:
            1. Seasonal price patterns
            2. Weather forecasts and climate impacts
            3. Global production estimates
            4. Trade and policy factors
            5. Demand from food vs. fuel sectors
            
            Provide bullish/bearish scenarios with specific targets.
            """
            
            forecasts[crop] = self.query_model(forecast_prompt)
        
        return forecasts

class AdvancedCommodityForecaster(CommodityForecaster):
    def __init__(self):
        super().__init__()
        self.forecasting_models = {}
    
    def multi_factor_analysis(self, commodity_data, commodity_type):
        """Perform comprehensive multi-factor analysis"""
        
        # Technical analysis
        technical_signals = self.analyze_technical_patterns(commodity_data)
        
        # Fundamental analysis
        fundamental_outlook = self.analyze_fundamentals(commodity_data, commodity_type)
        
        # Sentiment analysis
        market_sentiment = self.analyze_market_sentiment(commodity_data)
        
        # Integrate all factors
        integration_prompt = f"""
        Integrate these analytical perspectives for {commodity_type}:
        
        TECHNICAL ANALYSIS:
        {technical_signals}
        
        FUNDAMENTAL OUTLOOK:
        {fundamental_outlook}
        
        MARKET SENTIMENT:
        {market_sentiment}
        
        Synthesize these inputs into:
        1. Consensus forecast (most probable outcome)
        2. Risk assessment (potential disruptions)
        3. Trading recommendations (entry/exit levels)
        4. Confidence intervals (forecast reliability)
        
        Provide specific price targets and timeframes.
        """
        
        return self.query_model(integration_prompt)
    
    def analyze_technical_patterns(self, data):
        """Analyze technical patterns and indicators"""
        
        # Calculate multiple technical indicators
        price_col = [col for col in data.columns if 'close' in col.lower()][0]
        prices = data[price_col]
        
        # Moving averages
        sma_20 = prices.rolling(20).mean()
        sma_50 = prices.rolling(50).mean()
        
        # RSI
        rsi = calculate_rsi(prices)
        
        # MACD
        macd_line, macd_signal = self.calculate_macd(prices)
        
        # Bollinger Bands
        bb_upper, bb_lower = self.calculate_bollinger_bands(prices)
        
        current_price = prices.iloc[-1]
        
        technical_summary = f"""
        Current Price: ${current_price:.2f}
        SMA 20: ${sma_20.iloc[-1]:.2f}
        SMA 50: ${sma_50.iloc[-1]:.2f}
        RSI: {rsi.iloc[-1]:.1f}
        MACD: {macd_line.iloc[-1]:.3f}
        Bollinger Upper: ${bb_upper.iloc[-1]:.2f}
        Bollinger Lower: ${bb_lower.iloc[-1]:.2f}
        
        Price vs SMA20: {((current_price / sma_20.iloc[-1]) - 1) * 100:.1f}%
        Price vs SMA50: {((current_price / sma_50.iloc[-1]) - 1) * 100:.1f}%
        """
        
        return technical_summary
    
    def calculate_macd(self, prices, fast=12, slow=26, signal=9):
        """Calculate MACD indicator"""
        ema_fast = prices.ewm(span=fast).mean()
        ema_slow = prices.ewm(span=slow).mean()
        macd_line = ema_fast - ema_slow
        signal_line = macd_line.ewm(span=signal).mean()
        
        return macd_line, signal_line
    
    def calculate_bollinger_bands(self, prices, window=20, num_std=2):
        """Calculate Bollinger Bands"""
        sma = prices.rolling(window).mean()
        std = prices.rolling(window).std()
        
        upper_band = sma + (std * num_std)
        lower_band = sma - (std * num_std)
        
        return upper_band, lower_band
    
    def analyze_market_sentiment(self, data):
        """Analyze market sentiment indicators"""
        
        # Volume analysis
        volume_cols = [col for col in data.columns if 'volume' in col.lower()]
        if volume_cols:
            volume = data[volume_cols[0]]
            avg_volume = volume.rolling(20).mean()
            volume_ratio = volume.iloc[-1] / avg_volume.iloc[-1]
            
            sentiment_summary = f"""
            Volume Analysis:
            Recent Volume: {volume.iloc[-1]:,.0f}
            20-day Average: {avg_volume.iloc[-1]:,.0f}
            Volume Ratio: {volume_ratio:.2f}
            
            Volume Trend: {'Above average' if volume_ratio > 1.2 else 'Below average' if volume_ratio < 0.8 else 'Normal'}
            """
        else:
            sentiment_summary = "Volume data not available for sentiment analysis"
        
        return sentiment_summary
    
    def generate_integrated_forecast(self, commodity_type, horizon_days=30):
        """Generate integrated forecast using all analytical methods"""
        
        # Get appropriate data based on commodity type
        if commodity_type == "gold":
            data = fetch_gold_data()
        elif commodity_type == "oil":
            data = fetch_oil_data()
        elif commodity_type in ["corn", "wheat", "soybeans"]:
            data = fetch_agricultural_data()
        else:
            raise ValueError(f"Unsupported commodity type: {commodity_type}")
        
        # Perform multi-factor analysis
        integrated_analysis = self.multi_factor_analysis(data, commodity_type)
        
        # Generate final forecast
        final_forecast_prompt = f"""
        Final Forecast Generation for {commodity_type.upper()}:
        
        Integrated Analysis:
        {integrated_analysis}
        
        Generate a comprehensive {horizon_days}-day forecast including:
        
        1. PRIMARY FORECAST:
           - Most likely price target
           - Confidence level (1-10)
           - Key support/resistance levels
        
        2. SCENARIO ANALYSIS:
           - Bull case (probability and target)
           - Base case (most likely outcome)
           - Bear case (downside risk)
        
        3. RISK FACTORS:
           - Top 3 upside catalysts
           - Top 3 downside risks
           - Black swan events to monitor
        
        4. TRADING STRATEGY:
           - Recommended entry levels
           - Stop loss levels
           - Profit taking targets
        
        Format as structured analysis with specific numbers and dates.
        """
        
        return self.query_model(final_forecast_prompt)

import schedule
import time
from datetime import datetime
import logging

class RealTimeForecastSystem:
    def __init__(self):
        self.forecasters = {
            'gold': GoldForecaster(),
            'oil': OilForecaster(),
            'agriculture': AgriculturalForecaster()
        }
        self.last_forecasts = {}
        
        # Setup logging
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('commodity_forecasts.log'),
                logging.StreamHandler()
            ]
        )
    
    def update_all_forecasts(self):
        """Update all commodity forecasts"""
        logging.info("Starting forecast update cycle")
        
        for commodity, forecaster in self.forecasters.items():
            try:
                if commodity == 'gold':
                    forecast = forecaster.generate_forecast()
                elif commodity == 'oil':
                    forecast = forecaster.generate_oil_forecast()
                elif commodity == 'agriculture':
                    forecast = forecaster.generate_agricultural_forecast()
                
                self.last_forecasts[commodity] = {
                    'forecast': forecast,
                    'timestamp': datetime.now(),
                    'status': 'success'
                }
                
                logging.info(f"Updated {commodity} forecast successfully")
                
            except Exception as e:
                logging.error(f"Error updating {commodity} forecast: {e}")
                self.last_forecasts[commodity] = {
                    'forecast': None,
                    'timestamp': datetime.now(),
                    'status': 'error',
                    'error': str(e)
                }
    
    def get_latest_forecast(self, commodity):
        """Get the latest forecast for a commodity"""
        return self.last_forecasts.get(commodity, {})
    
    def start_automated_forecasting(self):
        """Start automated forecasting system"""
        
        # Schedule forecast updates
        schedule.every().day.at("09:00").do(self.update_all_forecasts)  # Market open
        schedule.every().day.at("16:00").do(self.update_all_forecasts)  # Market close
        
        # Initial forecast generation
        self.update_all_forecasts()
        
        logging.info("Automated forecasting system started")
        
        # Keep the system running
        while True:
            schedule.run_pending()
            time.sleep(60)  # Check every minute
    
    def export_forecasts(self, filename=None):
        """Export current forecasts to file"""
        if filename is None:
            filename = f"commodity_forecasts_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
        
        export_data = {
            'export_timestamp': datetime.now().isoformat(),
            'forecasts': {}
        }
        
        for commodity, data in self.last_forecasts.items():
            export_data['forecasts'][commodity] = {
                'forecast': data.get('forecast'),
                'timestamp': data.get('timestamp').isoformat() if data.get('timestamp') else None,
                'status': data.get('status')
            }
        
        with open(filename, 'w') as f:
            json.dump(export_data, f, indent=2)
        
        logging.info(f"Forecasts exported to {filename}")
        return filename

# Usage example
forecast_system = RealTimeForecastSystem()

# Get individual forecasts
gold_forecast = forecast_system.get_latest_forecast('gold')
oil_forecast = forecast_system.get_latest_forecast('oil')

# Start automated system (runs continuously)
# forecast_system.start_automated_forecasting()

class ForecastValidator:
    def __init__(self):
        self.historical_forecasts = []
        self.accuracy_metrics = {}
    
    def record_forecast(self, commodity, forecast_data, actual_price):
        """Record forecast for later validation"""
        record = {
            'commodity': commodity,
            'forecast_date': datetime.now(),
            'forecast_data': forecast_data,
            'actual_price': actual_price,
            'horizon_days': forecast_data.get('horizon_days', 30)
        }
        
        self.historical_forecasts.append(record)
    
    def validate_forecasts(self, commodity=None):
        """Validate historical forecasts against actual outcomes"""
        
        # Filter forecasts if commodity specified
        forecasts_to_validate = self.historical_forecasts
        if commodity:
            forecasts_to_validate = [f for f in forecasts_to_validate if f['commodity'] == commodity]
        
        validation_results = []
        
        for forecast in forecasts_to_validate:
            # Calculate forecast accuracy
            predicted_price = self.extract_predicted_price(forecast['forecast_data'])
            actual_price = forecast['actual_price']
            
            if predicted_price and actual_price:
                accuracy = 1 - abs(predicted_price - actual_price) / actual_price
                
                validation_results.append({
                    'commodity': forecast['commodity'],
                    'predicted_price': predicted_price,
                    'actual_price': actual_price,
                    'accuracy': accuracy,
                    'forecast_date': forecast['forecast_date']
                })
        
        return validation_results
    
    def extract_predicted_price(self, forecast_data):
        """Extract predicted price from forecast data"""
        # This would need to be customized based on your forecast format
        if isinstance(forecast_data, dict):
            return forecast_data.get('base_case', {}).get('target', 0)
        return 0
    
    def generate_accuracy_report(self):
        """Generate comprehensive accuracy report"""
        
        validation_results = self.validate_forecasts()
        
        if not validation_results:
            return "No forecasts available for validation"
        
        # Calculate metrics by commodity
        commodity_metrics = {}
        for result in validation_results:
            commodity = result['commodity']
            if commodity not in commodity_metrics:
                commodity_metrics[commodity] = []
            commodity_metrics[commodity].append(result['accuracy'])
        
        # Generate report
        report = ["FORECAST ACCURACY REPORT", "=" * 50, ""]
        
        for commodity, accuracies in commodity_metrics.items():
            avg_accuracy = np.mean(accuracies)
            std_accuracy = np.std(accuracies)
            
            report.extend([
                f"{commodity.upper()} FORECASTS:",
                f"  Average Accuracy: {avg_accuracy:.2%}",
                f"  Standard Deviation: {std_accuracy:.2%}",
                f"  Number of Forecasts: {len(accuracies)}",
                f"  Best Accuracy: {max(accuracies):.2%}",
                f"  Worst Accuracy: {min(accuracies):.2%}",
                ""
            ])
        
        return "\n".join(report)

class ModelOptimizer:
    def __init__(self):
        self.available_models = [
            "llama2:7b",
            "llama2:13b",
            "mistral:7b",
            "codellama:7b",
            "neural-chat:7b"
        ]
        self.model_performance = {}
    
    def benchmark_models(self, test_data, commodity_type):
        """Benchmark different models on historical data"""
        
        results = {}
        
        for model_name in self.available_models:
            try:
                # Create forecaster with specific model
                forecaster = CommodityForecaster(model_name=model_name)
                
                # Test on historical data
                test_results = self.test_model_accuracy(forecaster, test_data, commodity_type)
                
                results[model_name] = {
                    'accuracy': test_results['accuracy'],
                    'response_time': test_results['response_time'],
                    'consistency': test_results['consistency']
                }
                
            except Exception as e:
                results[model_name] = {'error': str(e)}
        
        return results
    
    def test_model_accuracy(self, forecaster, test_data, commodity_type):
        """Test model accuracy on historical data"""
        
        # Implementation would depend on your specific testing methodology
        # This is a simplified example
        
        start_time = time.time()
        
        # Generate forecast
        forecast = forecaster.query_model(
            f"Predict {commodity_type} price movement based on this data: {test_data[:500]}"
        )
        
        response_time = time.time() - start_time
        
        return {
            'accuracy': 0.75,  # Placeholder - implement actual accuracy calculation
            'response_time': response_time,
            'consistency': 0.8   # Placeholder - implement consistency measurement
        }
    
    def recommend_model(self, commodity_type, priority='accuracy'):
        """Recommend best model based on performance criteria"""
        
        if commodity_type not in self.model_performance:
            return "llama2:13b"  # Default recommendation
        
        performance_data = self.model_performance[commodity_type]
        
        if priority == 'accuracy':
            return max(performance_data.items(), key=lambda x: x[1].get('accuracy', 0))[0]
        elif priority == 'speed':
            return min(performance_data.items(), key=lambda x: x[1].get('response_time', float('inf')))[0]
        elif priority == 'consistency':
            return max(performance_data.items(), key=lambda x: x[1].get('consistency', 0))[0]
        
        return "llama2:13b"

class PromptOptimizer:
    def __init__(self):
        self.prompt_templates = {
            'technical_analysis': """
            You are a quantitative analyst specializing in {commodity} markets.
            
            MARKET DATA:
            {market_data}
            
            TECHNICAL INDICATORS:
            {technical_indicators}
            
            ANALYSIS FRAMEWORK:
            1. Identify current trend direction and strength
            2. Assess momentum indicators for potential reversals
            3. Evaluate support/resistance levels
            4. Consider volume patterns and market structure
            
            REQUIRED OUTPUT:
            - Specific price targets with confidence levels
            - Key technical levels to monitor
            - Risk assessment and stop-loss recommendations
            - Time horizon for each prediction
            
            Be precise with numbers and avoid vague statements.
            """,
            
            'fundamental_analysis': """
            You are a commodity fundamental analyst with expertise in {commodity} markets.
            
            CURRENT MARKET CONDITIONS:
            {market_conditions}
            
            FUNDAMENTAL FACTORS:
            {fundamental_factors}
            
            ANALYSIS REQUIREMENTS:
            1. Assess supply/demand dynamics
            2. Evaluate macroeconomic influences
            3. Consider geopolitical and regulatory factors
            4. Analyze seasonal patterns and cyclical trends
            
            DELIVERABLES:
            - Fundamental price drivers (bullish/bearish)
            - Supply/demand balance assessment
            - Risk scenarios and probability estimates
            - Long-term structural outlook
            
            Quantify impacts where possible and provide specific forecasts.
            """,
            
            'integrated_forecast': """
            You are a senior commodity strategist creating an integrated forecast for {commodity}.
            
            TECHNICAL ANALYSIS SUMMARY:
            {technical_summary}
            
            FUNDAMENTAL ANALYSIS SUMMARY:
            {fundamental_summary}
            
            MARKET SENTIMENT INDICATORS:
            {sentiment_indicators}
            
            INTEGRATION TASK:
            Synthesize all analytical inputs into a cohesive forecast that addresses:
            
            1. CONSENSUS FORECAST:
               - Most probable price target
               - Confidence interval
               - Key catalysts and timeline
            
            2. SCENARIO ANALYSIS:
               - Bull case: probability and price target
               - Base case: most likely outcome
               - Bear case: downside risk assessment
            
            3. RISK MANAGEMENT:
               - Position sizing recommendations
               - Stop-loss levels
               - Hedge considerations
            
            4. MONITORING PLAN:
               - Key indicators to watch
               - Trigger points for forecast revision
               - Early warning signals
            
            FORMAT: Structured analysis with specific numbers, dates, and actionable recommendations.
            """
        }
    
    def optimize_prompt(self, prompt_type, commodity, **kwargs):
        """Generate optimized prompt for specific analysis type"""
        
        if prompt_type not in self.prompt_templates:
            raise ValueError(f"Unknown prompt type: {prompt_type}")
        
        template = self.prompt_templates[prompt_type]
        
        # Format the template with provided parameters
        optimized_prompt = template.format(
            commodity=commodity,
            **kwargs
        )
        
        return optimized_prompt
    
    def test_prompt_effectiveness(self, prompt_variations, test_data):
        """Test different prompt variations for effectiveness"""
        
        results = {}
        
        for variation_name, prompt in prompt_variations.items():
            # Test the prompt with sample data
            forecaster = CommodityForecaster()
            
            start_time = time.time()
            response = forecaster.query_model(prompt)
            response_time = time.time() - start_time
            
            # Evaluate response quality (simplified)
            quality_score = self.evaluate_response_quality(response)
            
            results[variation_name] = {
                'response_time': response_time,
                'quality_score': quality_score,
                'response_length': len(response),
                'contains_numbers': bool(re.search(r'\d+\.?\d*', response))
            }
        
        return results
    
    def evaluate_response_quality(self, response):
        """Evaluate the quality of a model response"""
        
        quality_indicators = {
            'specific_numbers': len(re.findall(r'\$?\d+\.?\d*', response)),
            'dates_mentioned': len(re.findall(r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b', response)),
            'structured_format': response.count('\n') > 5,
            'actionable_language': any(word in response.lower() for word in ['buy', 'sell', 'target', 'stop', 'support', 'resistance']),
            'length_appropriate': 200 <= len(response) <= 2000
        }
        
        # Simple scoring system
        score = sum(quality_indicators.values()) / len(quality_indicators)
        return score

# Dockerfile for commodity forecasting system
FROM python:3.9-slim

# Install system dependencies
RUN apt-get update && apt-get install -y \
    curl \
    build-essential \
    && rm -rf /var/lib/apt/lists/*

# Install Ollama
RUN curl -fsSL https://ollama.ai/install.sh | sh

# Set working directory
WORKDIR /app

# Copy requirements
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy application code
COPY src/ ./src/
COPY config/ ./config/

# Expose port for API
EXPOSE 8000

# Start script
COPY start.sh .
RUN chmod +x start.sh

CMD ["./start.sh"]

from flask import Flask, jsonify, request
from datetime import datetime
import logging

app = Flask(__name__)

# Initialize forecasting system
forecast_system = RealTimeForecastSystem()

@app.route('/api/forecast/<commodity>', methods=['GET'])
def get_forecast(commodity):
    """Get latest forecast for a commodity"""
    
    try:
        # Get query parameters
        horizon = request.args.get('horizon', 30, type=int)
        include_analysis = request.args.get('analysis', 'false').lower() == 'true'
        
        # Validate commodity
        if commodity not in ['gold', 'oil', 'corn', 'wheat', 'soybeans']:
            return jsonify({'error': 'Unsupported commodity'}), 400
        
        # Get forecast
        forecast_data = forecast_system.get_latest_forecast(commodity)
        
        if not forecast_data:
            return jsonify({'error': 'No forecast available'}), 404
        
        response = {
            'commodity': commodity,
            'forecast': forecast_data['forecast'],
            'timestamp': forecast_data['timestamp'].isoformat(),
            'status': forecast_data['status'],
            'horizon_days': horizon
        }
        
        return jsonify(response)
        
    except Exception as e:
        logging.error(f"Error getting forecast for {commodity}: {e}")
        return jsonify({'error': 'Internal server error'}), 500

@app.route('/api/forecast/<commodity>', methods=['POST'])
def generate_forecast(commodity):
    """Generate new forecast for a commodity"""
    
    try:
        # Get request data
        data = request.get_json()
        horizon = data.get('horizon', 30)
        force_update = data.get('force_update', False)
        
        # Generate forecast based on commodity type
        if commodity == 'gold':
            forecaster = GoldForecaster()
            forecast = forecaster.generate_forecast(horizon_days=horizon)
        elif commodity == 'oil':
            forecaster = OilForecaster()
            forecast = forecaster.generate_oil_forecast(horizon_days=horizon)
        elif commodity in ['corn', 'wheat', 'soybeans']:
            forecaster = AgriculturalForecaster()
            forecast = forecaster.generate_agricultural_forecast([commodity])
        else:
            return jsonify({'error': 'Unsupported commodity'}), 400
        
        # Update system cache
        forecast_system.last_forecasts[commodity] = {
            'forecast': forecast,
            'timestamp': datetime.now(),
            'status': 'success'
        }
        
        return jsonify({
            'commodity': commodity,
            'forecast': forecast,
            'timestamp': datetime.now().isoformat(),
            'horizon_days': horizon
        })
        
    except Exception as e:
        logging.error(f"Error generating forecast for {commodity}: {e}")
        return jsonify({'error': 'Internal server error'}), 500

@app.route('/api/health', methods=['GET'])
def health_check():
    """Health check endpoint"""
    return jsonify({
        'status': 'healthy',
        'timestamp': datetime.now().isoformat(),
        'available_commodities': ['gold', 'oil', 'corn', 'wheat', 'soybeans']
    })

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8000, debug=False)

class DataQualityManager:
    def __init__(self):
        self.quality_thresholds = {
            'missing_data_threshold': 0.05,  # Max 5% missing data
            'outlier_threshold': 3.0,        # 3 standard deviations
            'minimum_history': 252           # Trading days (1 year)
        }
    
    def validate_data_quality(self, data, commodity_type):
        """Validate data quality for forecasting"""
        
        quality_report = {
            'valid': True,
            'issues': [],
            'warnings': [],
            'recommendations': []
        }
        
        # Check for missing data
        missing_ratio = data.isnull().sum().sum() / (data.shape[0] * data.shape[1])
        if missing_ratio > self.quality_thresholds['missing_data_threshold']:
            quality_report['valid'] = False
            quality_report['issues'].append(f"High missing data ratio: {missing_ratio:.2%}")
        
        # Check data history length
        if len(data) < self.quality_thresholds['minimum_history']:
            quality_report['warnings'].append(f"Limited history: {len(data)} days")
            quality_report['recommendations'].append("Consider extending data period for better accuracy")
        
        # Check for outliers
        numeric_columns = data.select_dtypes(include=[np.number]).columns
        for col in numeric_columns:
            z_scores = np.abs((data[col] - data[col].mean()) / data[col].std())
            outliers = z_scores > self.quality_thresholds['outlier_threshold']
            if outliers.sum() > 0:
                quality_report['warnings'].append(f"Outliers detected in {col}: {outliers.sum()} points")
        
        return quality_report
    
    def clean_data(self, data):
        """Clean and preprocess data for forecasting"""
        
        cleaned_data = data.copy()
        
        # Handle missing values
        numeric_columns = cleaned_data.select_dtypes(include=[np.number]).columns
        for col in numeric_columns:
            # Forward fill then backward fill
            cleaned_data[col] = cleaned_data[col].fillna(method='ffill').fillna(method='bfill')
        
        # Remove extreme outliers
        for col in numeric_columns:
            q1 = cleaned_data[col].quantile(0.01)
            q99 = cleaned_data[col].quantile(0.99)
            cleaned_data[col] = cleaned_data[col].clip(lower=q1, upper=q99)
        
        # Ensure data is sorted by date
        if isinstance(cleaned_data.index, pd.DatetimeIndex):
            cleaned_data = cleaned_data.sort_index()
        
        return cleaned_data

class ResilientForecaster:
    def __init__(self):
        self.max_retries = 3
        self.retry_delay = 2  # seconds
        self.fallback_models = ["llama2:7b", "mistral:7b"]
    
    def generate_forecast_with_fallback(self, commodity, primary_model="llama2:13b"):
        """Generate forecast with fallback mechanisms"""
        
        models_to_try = [primary_model] + self.fallback_models
        
        for attempt, model in enumerate(models_to_try):
            try:
                forecaster = CommodityForecaster(model_name=model)
                
                # Attempt forecast generation
                forecast = self.generate_forecast_with_retry(forecaster, commodity)
                
                if forecast:
                    return {
                        'forecast': forecast,
                        'model_used': model,
                        'attempt': attempt + 1,
                        'status': 'success'
                    }
                    
            except Exception as e:
                logging.warning(f"Model {model} failed on attempt {attempt + 1}: {e}")
                
                if attempt < len(models_to_try) - 1:
                    time.sleep(self.retry_delay)
                    continue
                else:
                    return {
                        'forecast': None,
                        'model_used': None,
                        'attempt': attempt + 1,
                        'status': 'failed',
                        'error': str(e)
                    }
        
        return {
            'forecast': None,
            'model_used': None,
            'status': 'all_models_failed'
        }
    
    def generate_forecast_with_retry(self, forecaster, commodity):
        """Generate forecast with retry logic"""
        
        for attempt in range(self.max_retries):
            try:
                if commodity == 'gold':
                    return forecaster.generate_forecast()
                elif commodity == 'oil':
                    return forecaster.generate_oil_forecast()
                elif commodity in ['corn', 'wheat', 'soybeans']:
                    return forecaster.generate_agricultural_forecast([commodity])
                
            except Exception as e:
                if attempt < self.max_retries - 1:
                    logging.info(f"Retry {attempt + 1} for {commodity} forecast")
                    time.sleep(self.retry_delay * (attempt + 1))  # Exponential backoff
                else:
                    raise e
        
        return None