Overview of Ensemble Methods

from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC

def create_voting_ensemble(voting='hard'):
    """Create a simple voting ensemble"""
    # Define base models
    models = [
        ('lr', LogisticRegression()),
        ('dt', DecisionTreeClassifier()),
        ('svm', SVC(probability=True))
    ]
    
    # Create voting classifier
    ensemble = VotingClassifier(
        estimators=models,
        voting=voting
    )
    
    return ensemble

import numpy as np

class ModelAveraging:
    def __init__(self, models, weights=None):
        """Initialize model averaging ensemble"""
        self.models = models
        self.weights = weights if weights is not None else [1/len(models)] * len(models)
    
    def fit(self, X, y):
        """Fit all models"""
        for model in self.models:
            model.fit(X, y)
        return self
    
    def predict(self, X):
        """Make weighted predictions"""
        predictions = np.array([model.predict(X) for model in self.models])
        return np.average(predictions, axis=0, weights=self.weights)

def compute_diversity(predictions1, predictions2):
    """Compute diversity between two models"""
    # Disagreement measure
    disagreement = np.mean(predictions1 != predictions2)
    
    # Q-statistic
    n11 = np.sum((predictions1 == 1) & (predictions2 == 1))
    n00 = np.sum((predictions1 == 0) & (predictions2 == 0))
    n10 = np.sum((predictions1 == 1) & (predictions2 == 0))
    n01 = np.sum((predictions1 == 0) & (predictions2 == 1))
    
    q_statistic = (n11 * n00 - n10 * n01) / (n11 * n00 + n10 * n01)
    
    return {
        'disagreement': disagreement,
        'q_statistic': q_statistic
    }

def forward_ensemble_selection(models, X, y, max_size=10):
    """Select models for ensemble using forward selection"""
    selected = []
    remaining = models.copy()
    scores = []
    
    while len(selected) < max_size and remaining:
        best_score = float('-inf')
        best_model = None
        
        # Try each remaining model
        for model in remaining:
            current = selected + [model]
            ensemble = ModelAveraging(current)
            ensemble.fit(X, y)
            score = ensemble.score(X, y)
            
            if score > best_score:
                best_score = score
                best_model = model
        
        if best_model is not None:
            selected.append(best_model)
            remaining.remove(best_model)
            scores.append(best_score)
        
    return selected, scores

def analyze_ensemble(ensemble, X, y):
    """Analyze ensemble performance and characteristics"""
    # Individual model performance
    individual_scores = []
    for name, model in ensemble.named_estimators_.items():
        score = model.score(X, y)
        individual_scores.append((name, score))
    
    # Ensemble performance
    ensemble_score = ensemble.score(X, y)
    
    # Diversity analysis
    predictions = []
    for name, model in ensemble.named_estimators_.items():
        pred = model.predict(X)
        predictions.append(pred)
    
    diversity_matrix = np.zeros((len(predictions), len(predictions)))
    for i in range(len(predictions)):
        for j in range(len(predictions)):
            if i != j:
                diversity = compute_diversity(predictions[i], predictions[j])
                diversity_matrix[i, j] = diversity['disagreement']
    
    return {
        'individual_scores': individual_scores,
        'ensemble_score': ensemble_score,
        'diversity_matrix': diversity_matrix
    }