AI Yield Farming Pattern Analysis: Machine Learning User Behavior Detection That Actually Works

pip install pandas numpy scikit-learn web3 requests matplotlib seaborn
pip install plotly dash streamlit

import pandas as pd
import numpy as np
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
import matplotlib.pyplot as plt
import seaborn as sns
from web3 import Web3
import requests
import json

# Connect to Ethereum node (use Infura, Alchemy, or local node)
w3 = Web3(Web3.HTTPProvider('https://mainnet.infura.io/v3/YOUR_PROJECT_ID'))

def get_transaction_data(address, start_block, end_block):
    """
    Fetch transaction data for yield farming analysis
    Returns transaction history with timestamps and values
    """
    transactions = []
    
    for block_num in range(start_block, end_block + 1):
        block = w3.eth.get_block(block_num, full_transactions=True)
        
        for tx in block.transactions:
            if tx['from'] == address or tx['to'] == address:
                transactions.append({
                    'hash': tx['hash'].hex(),
                    'from': tx['from'],
                    'to': tx['to'],
                    'value': tx['value'],
                    'gas_price': tx['gasPrice'],
                    'timestamp': block['timestamp'],
                    'block_number': block_num
                })
    
    return pd.DataFrame(transactions)

def fetch_defi_protocol_data(protocol_name, user_address):
    """
    Fetch user interaction data from major DeFi protocols
    Supports Uniswap, Compound, Aave, and Curve
    """
    protocol_apis = {
        'uniswap': 'https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v3',
        'compound': 'https://api.compound.finance/api/v2/account',
        'aave': 'https://api.aave.com/data/users',
        'curve': 'https://api.curve.fi/api/getPools'
    }
    
    if protocol_name not in protocol_apis:
        raise ValueError(f"Protocol {protocol_name} not supported")
    
    # Example query for Uniswap data
    if protocol_name == 'uniswap':
        query = f"""
        {{
          user(id: "{user_address.lower()}") {{
            liquidityPositions {{
              liquidityTokenBalance
              pair {{
                token0 {{ symbol }}
                token1 {{ symbol }}
              }}
            }}
          }}
        }}
        """
        
        response = requests.post(
            protocol_apis[protocol_name],
            json={'query': query}
        )
        
        return response.json()['data']
    
    return None

def create_behavioral_features(df):
    """
    Extract behavioral features from transaction data
    Returns DataFrame with engineered features for ML analysis
    """
    # Sort by timestamp
    df = df.sort_values('timestamp')
    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')
    
    # Calculate time-based features
    df['hour'] = df['datetime'].dt.hour
    df['day_of_week'] = df['datetime'].dt.dayofweek
    df['time_between_tx'] = df['timestamp'].diff()
    
    # Value-based features
    df['value_eth'] = df['value'] / 1e18  # Convert Wei to ETH
    df['gas_cost'] = df['gas_price'] * 21000 / 1e18  # Estimate gas cost
    df['value_to_gas_ratio'] = df['value_eth'] / df['gas_cost']
    
    # Behavioral patterns
    df['tx_frequency'] = df.groupby(df['datetime'].dt.date)['hash'].transform('count')
    df['daily_volume'] = df.groupby(df['datetime'].dt.date)['value_eth'].transform('sum')
    df['avg_tx_size'] = df['value_eth'].rolling(window=10).mean()
    
    # Risk indicators
    df['large_tx_indicator'] = (df['value_eth'] > df['value_eth'].quantile(0.9)).astype(int)
    df['night_trading'] = ((df['hour'] >= 22) | (df['hour'] <= 6)).astype(int)
    df['weekend_trading'] = (df['day_of_week'] >= 5).astype(int)
    
    return df

# Apply feature engineering
user_address = "0x742D35Cc6634C0532925a3b8D29Ad0473AF3F1E1"  # Example address
tx_data = get_transaction_data(user_address, 18000000, 18001000)
behavioral_features = create_behavioral_features(tx_data)

def cluster_yield_farmers(features_df):
    """
    Cluster yield farmers based on behavioral patterns
    Returns cluster labels and cluster centers
    """
    # Select relevant features for clustering
    clustering_features = [
        'tx_frequency', 'daily_volume', 'avg_tx_size',
        'value_to_gas_ratio', 'large_tx_indicator',
        'night_trading', 'weekend_trading'
    ]
    
    # Prepare data
    X = features_df[clustering_features].fillna(0)
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)
    
    # Apply K-means clustering
    kmeans = KMeans(n_clusters=5, random_state=42)
    cluster_labels = kmeans.fit_predict(X_scaled)
    
    # Add cluster labels to dataframe
    features_df['cluster'] = cluster_labels
    
    # Analyze cluster characteristics
    cluster_summary = features_df.groupby('cluster')[clustering_features].mean()
    
    return cluster_labels, cluster_summary, kmeans

# Perform clustering analysis
clusters, summary, model = cluster_yield_farmers(behavioral_features)
print("Cluster Analysis Summary:")
print(summary)

def build_behavior_predictor(features_df):
    """
    Train a Random Forest model to predict yield farming behavior
    Returns trained model and feature importance
    """
    # Create target variable (next action prediction)
    features_df = features_df.sort_values(['from', 'timestamp'])
    features_df['next_tx_large'] = features_df.groupby('from')['large_tx_indicator'].shift(-1)
    
    # Prepare features
    prediction_features = [
        'hour', 'day_of_week', 'tx_frequency', 'daily_volume',
        'avg_tx_size', 'value_to_gas_ratio', 'time_between_tx'
    ]
    
    # Remove rows with missing target values
    model_data = features_df.dropna(subset=['next_tx_large'])
    
    X = model_data[prediction_features].fillna(0)
    y = model_data['next_tx_large']
    
    # Train Random Forest classifier
    rf_model = RandomForestClassifier(n_estimators=100, random_state=42)
    rf_model.fit(X, y)
    
    # Feature importance analysis
    feature_importance = pd.DataFrame({
        'feature': prediction_features,
        'importance': rf_model.feature_importances_
    }).sort_values('importance', ascending=False)
    
    return rf_model, feature_importance

# Train the predictive model
predictor, importance = build_behavior_predictor(behavioral_features)
print("Feature Importance:")
print(importance)

from sklearn.ensemble import IsolationForest

def detect_anomalous_behavior(features_df):
    """
    Detect anomalous yield farming behavior using Isolation Forest
    Returns anomaly scores and flagged transactions
    """
    # Select features for anomaly detection
    anomaly_features = [
        'value_eth', 'gas_cost', 'tx_frequency',
        'time_between_tx', 'value_to_gas_ratio'
    ]
    
    X = features_df[anomaly_features].fillna(0)
    
    # Apply Isolation Forest
    isolation_forest = IsolationForest(
        contamination=0.1,  # Expect 10% anomalies
        random_state=42
    )
    
    anomaly_scores = isolation_forest.fit_predict(X)
    features_df['anomaly_score'] = anomaly_scores
    
    # Flag suspicious transactions
    suspicious_tx = features_df[features_df['anomaly_score'] == -1]
    
    return suspicious_tx, isolation_forest

# Detect anomalies
anomalies, anomaly_model = detect_anomalous_behavior(behavioral_features)
print(f"Found {len(anomalies)} suspicious transactions")

import time
from datetime import datetime

class YieldFarmingMonitor:
    """
    Real-time yield farming pattern monitoring system
    Tracks user behavior and alerts on significant changes
    """
    
    def __init__(self, model, scaler):
        self.model = model
        self.scaler = scaler
        self.alerts = []
        
    def analyze_new_transaction(self, tx_data):
        """
        Analyze a new transaction and check for pattern changes
        """
        # Extract features from new transaction
        features = self.extract_features(tx_data)
        
        # Make prediction
        prediction = self.model.predict([features])[0]
        confidence = max(self.model.predict_proba([features])[0])
        
        # Check for alerts
        if confidence < 0.6:  # Low confidence indicates unusual pattern
            alert = {
                'timestamp': datetime.now(),
                'tx_hash': tx_data['hash'],
                'pattern': 'unusual_behavior',
                'confidence': confidence
            }
            self.alerts.append(alert)
            
        return prediction, confidence
    
    def extract_features(self, tx_data):
        """Extract features from transaction data"""
        # Implement feature extraction logic
        # This would mirror the feature engineering from above
        pass
    
    def get_recent_alerts(self, hours=24):
        """Get alerts from the last N hours"""
        cutoff = datetime.now() - pd.Timedelta(hours=hours)
        return [alert for alert in self.alerts if alert['timestamp'] > cutoff]

# Initialize monitoring system
monitor = YieldFarmingMonitor(predictor, StandardScaler())

import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots

def create_pattern_dashboard(features_df, clusters):
    """
    Create interactive dashboard for yield farming pattern analysis
    """
    # Cluster distribution plot
    cluster_fig = px.scatter(
        features_df, 
        x='daily_volume', 
        y='tx_frequency',
        color='cluster',
        title="Yield Farmer Behavior Clusters",
        labels={'daily_volume': 'Daily Volume (ETH)', 'tx_frequency': 'Transaction Frequency'}
    )
    
    # Time series of trading activity
    daily_activity = features_df.groupby(features_df['datetime'].dt.date).agg({
        'value_eth': 'sum',
        'hash': 'count'
    }).reset_index()
    daily_activity.columns = ['date', 'total_volume', 'tx_count']
    
    activity_fig = make_subplots(
        rows=2, cols=1,
        subplot_titles=['Daily Volume', 'Transaction Count']
    )
    
    activity_fig.add_trace(
        go.Scatter(x=daily_activity['date'], y=daily_activity['total_volume'], name='Volume'),
        row=1, col=1
    )
    
    activity_fig.add_trace(
        go.Scatter(x=daily_activity['date'], y=daily_activity['tx_count'], name='Transactions'),
        row=2, col=1
    )
    
    return cluster_fig, activity_fig

# Generate visualizations
cluster_plot, activity_plot = create_pattern_dashboard(behavioral_features, clusters)

# Display plots (in a Jupyter notebook or save as HTML)
cluster_plot.show()
activity_plot.show()

# Optimize data processing for large datasets
def optimize_data_processing(df):
    """
    Optimize DataFrame operations for better performance
    """
    # Use categorical data types for string columns
    df['from'] = df['from'].astype('category')
    df['to'] = df['to'].astype('category')
    
    # Use appropriate numeric types
    df['value'] = pd.to_numeric(df['value'], downcast='unsigned')
    df['gas_price'] = pd.to_numeric(df['gas_price'], downcast='unsigned')
    
    # Chunk processing for large datasets
    chunk_size = 10000
    processed_chunks = []
    
    for i in range(0, len(df), chunk_size):
        chunk = df.iloc[i:i+chunk_size]
        processed_chunk = create_behavioral_features(chunk)
        processed_chunks.append(processed_chunk)
    
    return pd.concat(processed_chunks, ignore_index=True)

def safe_analyze_patterns(address, start_block, end_block):
    """
    Safely analyze yield farming patterns with error handling
    """
    try:
        # Validate input parameters
        if not Web3.isAddress(address):
            raise ValueError(f"Invalid Ethereum address: {address}")
        
        if start_block >= end_block:
            raise ValueError("start_block must be less than end_block")
        
        # Fetch and process data
        tx_data = get_transaction_data(address, start_block, end_block)
        
        if tx_data.empty:
            return {"error": "No transactions found for the specified range"}
        
        features = create_behavioral_features(tx_data)
        clusters, summary, model = cluster_yield_farmers(features)
        
        return {
            "success": True,
            "cluster_summary": summary.to_dict(),
            "total_transactions": len(tx_data),
            "analysis_timestamp": datetime.now().isoformat()
        }
        
    except Exception as e:
        return {
            "error": str(e),
            "timestamp": datetime.now().isoformat()
        }

# Example usage with error handling
result = safe_analyze_patterns(
    "0x742D35Cc6634C0532925a3b8D29Ad0473AF3F1E1",
    18000000,
    18001000
)

if "error" in result:
    print(f"Analysis failed: {result['error']}")
else:
    print("Analysis completed successfully")
    print(f"Processed {result['total_transactions']} transactions")

def recommend_farming_strategies(user_cluster, market_conditions):
    """
    Recommend yield farming strategies based on user behavior cluster
    """
    strategies = {
        0: {  # Conservative farmers
            "recommended_protocols": ["Compound", "Aave"],
            "risk_level": "Low",
            "expected_apy": "5-8%",
            "strategy": "Focus on blue-chip tokens and established protocols"
        },
        1: {  # Active traders
            "recommended_protocols": ["Uniswap V3", "Curve"],
            "risk_level": "Medium",
            "expected_apy": "10-15%",
            "strategy": "Liquidity provision with active management"
        },
        2: {  # High-risk farmers
            "recommended_protocols": ["Balancer", "SushiSwap"],
            "risk_level": "High",
            "expected_apy": "15-30%",
            "strategy": "New protocol farming with higher rewards"
        }
    }
    
    return strategies.get(user_cluster, strategies[0])

# Get strategy recommendation
user_cluster = 1  # From previous clustering analysis
recommendation = recommend_farming_strategies(user_cluster, "bullish")
print("Recommended Strategy:")
for key, value in recommendation.items():
    print(f"{key}: {value}")

def backtest_strategy(features_df, strategy_rules, start_date, end_date):
    """
    Backtest yield farming strategy based on ML predictions
    """
    # Filter data by date range
    mask = (features_df['datetime'] >= start_date) & (features_df['datetime'] <= end_date)
    test_data = features_df.loc[mask].copy()
    
    # Simulate trading based on predictions
    initial_balance = 10000  # Starting with 10,000 USD equivalent
    current_balance = initial_balance
    trades = []
    
    for index, row in test_data.iterrows():
        # Make prediction
        prediction = predictor.predict([[
            row['hour'], row['day_of_week'], row['tx_frequency'],
            row['daily_volume'], row['avg_tx_size'], row['value_to_gas_ratio'],
            row['time_between_tx']
        ]])[0]
        
        # Execute trade based on strategy rules
        if prediction == 1 and strategy_rules['enter_on_signal']:
            trade_amount = current_balance * strategy_rules['position_size']
            # Simulate yield farming returns
            daily_return = np.random.normal(0.0001, 0.005)  # Example return distribution
            current_balance += trade_amount * daily_return
            
            trades.append({
                'date': row['datetime'],
                'action': 'enter',
                'amount': trade_amount,
                'balance': current_balance
            })
    
    total_return = (current_balance - initial_balance) / initial_balance * 100
    
    return {
        'total_return': total_return,
        'final_balance': current_balance,
        'number_of_trades': len(trades),
        'trades': trades
    }

# Run backtest
strategy = {
    'enter_on_signal': True,
    'position_size': 0.1  # Use 10% of balance per trade
}

backtest_results = backtest_strategy(
    behavioral_features,
    strategy,
    pd.Timestamp('2024-01-01'),
    pd.Timestamp('2024-06-01')
)

print(f"Backtest Results:")
print(f"Total Return: {backtest_results['total_return']:.2f}%")
print(f"Number of Trades: {backtest_results['number_of_trades']}")

def anonymize_user_data(df):
    """
    Anonymize user data while preserving analytical value
    """
    # Hash addresses for privacy
    import hashlib
    
    df['from_hash'] = df['from'].apply(
        lambda x: hashlib.sha256(x.encode()).hexdigest()[:16]
    )
    df['to_hash'] = df['to'].apply(
        lambda x: hashlib.sha256(x.encode()).hexdigest()[:16]
    )
    
    # Remove original addresses
    df = df.drop(['from', 'to'], axis=1)
    
    # Add noise to sensitive numerical features
    noise_level = 0.01  # 1% noise
    numerical_cols = ['value_eth', 'gas_cost']
    
    for col in numerical_cols:
        noise = np.random.normal(0, df[col].std() * noise_level, len(df))
        df[col] += noise
    
    return df

# Apply anonymization
anonymized_data = anonymize_user_data(behavioral_features.copy())