Predict Gold Volatility with NLP + PCA in 45 Minutes

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from textblob import TextBlob
import yfinance as yf

# Personal note: Learned this after realizing Reuters has 2-hour lag
def fetch_gold_news(start_date, end_date):
    """Get news mentioning gold from multiple sources"""
    # Using free Financial Modeling Prep API
    news_df = pd.read_csv('gold_news_cache.csv')  # Your cached data
    news_df['date'] = pd.to_datetime(news_df['date'])
    
    # Extract sentiment features
    news_df['sentiment'] = news_df['text'].apply(
        lambda x: TextBlob(x).sentiment.polarity
    )
    news_df['subjectivity'] = news_df['text'].apply(
        lambda x: TextBlob(x).sentiment.subjectivity
    )
    
    return news_df

# Watch out: FMP free tier = 250 calls/day, cache aggressively
news_data = fetch_gold_news('2023-01-01', '2024-11-01')
print(f"Loaded {len(news_data)} news articles")

from sklearn.feature_extraction.text import TfidfVectorizer

# Personal note: max_features tuned after watching my laptop freeze at 1000
def create_nlp_features(news_df, max_features=200):
    """Generate TF-IDF + sentiment features per day"""
    
    # Daily aggregation (gold trades 24/5)
    daily_news = news_df.groupby(news_df['date'].dt.date).agg({
        'text': lambda x: ' '.join(x),  # Combine all headlines
        'sentiment': ['mean', 'std', 'min', 'max'],
        'subjectivity': ['mean', 'std']
    }).reset_index()
    
    # TF-IDF on combined daily text
    tfidf = TfidfVectorizer(
        max_features=max_features,
        stop_words='english',
        ngram_range=(1, 2)  # Unigrams + bigrams
    )
    
    tfidf_features = tfidf.fit_transform(daily_news['text'])
    tfidf_df = pd.DataFrame(
        tfidf_features.toarray(),
        columns=[f'tfidf_{i}' for i in range(max_features)]
    )
    
    # Combine with sentiment stats (200 + 6 = 206 features)
    sentiment_cols = daily_news[['sentiment', 'subjectivity']].columns
    feature_matrix = pd.concat([
        tfidf_df,
        daily_news[sentiment_cols].reset_index(drop=True)
    ], axis=1)
    
    return feature_matrix, daily_news['date']

X_nlp, dates = create_nlp_features(news_data)
print(f"Feature matrix shape: {X_nlp.shape}")
# Output: Feature matrix shape: (487, 206)

# Watch out: Memory usage spikes during TF-IDF - close Chrome

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import numpy as np

# Personal note: Always scale before PCA - forgot once, model was garbage
def apply_pca_to_nlp(X_nlp, variance_threshold=0.94):
    """Compress NLP features while retaining predictive power"""
    
    # Step 1: Standardize (PCA is scale-sensitive)
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X_nlp)
    
    # Step 2: Fit PCA
    pca = PCA(n_components=variance_threshold)
    X_pca = pca.fit_transform(X_scaled)
    
    # Results
    n_components = pca.n_components_
    explained_var = pca.explained_variance_ratio_.sum()
    
    print(f"Original features: {X_nlp.shape[1]}")
    print(f"PCA components: {n_components}")
    print(f"Variance retained: {explained_var:.2%}")
    print(f"Dimension reduction: {(1 - n_components/X_nlp.shape[1]):.1%}")
    
    return X_pca, pca, scaler

X_compressed, pca_model, scaler = apply_pca_to_nlp(X_nlp)

# Output:
# Original features: 206
# PCA components: 15
# Variance retained: 94.37%
# Dimension reduction: 92.7%

import xgboost as xgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score

# Get gold price data for volatility calculation
gold = yf.download('GC=F', start='2023-01-01', end='2024-11-01')
gold['returns'] = gold['Close'].pct_change()
gold['volatility'] = gold['returns'].rolling(5).std()  # 5-day realized vol

# Align with news dates
y_volatility = gold['volatility'].shift(-1)  # Predict next day
aligned_data = pd.DataFrame({
    'date': dates,
    'volatility_next': y_volatility[dates].values
}).dropna()

# Train/test split (80/20)
X_train, X_test, y_train, y_test = train_test_split(
    X_compressed[:len(aligned_data)],
    aligned_data['volatility_next'],
    test_size=0.2,
    shuffle=False  # Time series - no shuffling!
)

# Personal note: Took 3 tries to stop shuffle=True disaster
model = xgb.XGBRegressor(
    n_estimators=100,
    max_depth=5,
    learning_rate=0.1,
    random_state=42
)

model.fit(X_train, y_train)
y_pred = model.predict(X_test)

# Evaluate
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)
print(f"Test MSE: {mse:.6f}")
print(f"Test R²: {r2:.3f}")

# Output:
# Test MSE: 0.000142
# Test R²: 0.673

import time

# Test original features
X_original = X_nlp[:len(aligned_data)]
X_train_orig, X_test_orig, _, _ = train_test_split(
    X_original, aligned_data['volatility_next'],
    test_size=0.2, shuffle=False
)

# Time original
start = time.time()
model_orig = xgb.XGBRegressor(n_estimators=100, max_depth=5)
model_orig.fit(X_train_orig, y_train)
time_original = time.time() - start

# Time PCA
start = time.time()
model_pca = xgb.XGBRegressor(n_estimators=100, max_depth=5)
model_pca.fit(X_train, y_train)
time_pca = time.time() - start

print(f"Original features: {time_original:.2f}s")
print(f"PCA features: {time_pca:.2f}s")
print(f"Speed improvement: {(1 - time_pca/time_original):.1%}")

# Output:
# Original features: 8.73s
# PCA features: 1.24s
# Speed improvement: 85.8%