Named Entity Recognition (NER)

import spacy

def rule_based_ner(text):
    nlp = spacy.load('en_core_web_sm')
    doc = nlp(text)
    entities = [(ent.text, ent.label_) for ent in doc.ents]
    return entities

from sklearn_crfsuite import CRF

class CRFEntityRecognizer:
    def __init__(self):
        self.model = CRF(
            algorithm='lbfgs',
            c1=0.1,
            c2=0.1,
            max_iterations=100
        )
    
    def extract_features(self, sentence):
        features = []
        for i, word in enumerate(sentence):
            word_features = {
                'word': word,
                'is_first': i == 0,
                'is_last': i == len(sentence) - 1,
                'is_capitalized': word[0].upper() == word[0],
                'is_all_caps': word.upper() == word,
                'prefix-1': word[0],
                'prefix-2': word[:2],
                'suffix-1': word[-1],
                'suffix-2': word[-2:],
            }
            features.append(word_features)
        return features

import torch.nn as nn

class BiLSTM_CRF(nn.Module):
    def __init__(self, vocab_size, tag_to_ix, embedding_dim, hidden_dim):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.lstm = nn.LSTM(embedding_dim, 
                           hidden_dim // 2,
                           num_layers=1, 
                           bidirectional=True)
        self.hidden2tag = nn.Linear(hidden_dim, len(tag_to_ix))
        self.crf = CRF(len(tag_to_ix))
        
    def forward(self, sentence):
        embeds = self.embedding(sentence)
        lstm_out, _ = self.lstm(embeds)
        tag_space = self.hidden2tag(lstm_out)
        return self.crf.decode(tag_space)

def create_custom_entity_patterns(nlp):
    ruler = nlp.add_pipe("entity_ruler")
    patterns = [
        {"label": "PRODUCT", "pattern": "iPhone"},
        {"label": "PRODUCT", "pattern": [{"LOWER": "mac"}, {"LOWER": "book"}]},
    ]
    ruler.add_patterns(patterns)
    return nlp

def find_nested_entities(text, nlp):
    doc = nlp(text)
    entities = []
    for ent in doc.ents:
        # Find main entity
        entities.append((ent.text, ent.label_))
        # Look for nested entities
        doc_nested = nlp(ent.text)
        for nested_ent in doc_nested.ents:
            if nested_ent.text != ent.text:
                entities.append((nested_ent.text, nested_ent.label_))
    return entities

class NamedEntityRecognizer:
    def __init__(self, model_type='spacy'):
        self.model_type = model_type
        if model_type == 'spacy':
            self.nlp = spacy.load('en_core_web_sm')
        else:
            self.model = self.initialize_custom_model()
    
    def process_text(self, text):
        if self.model_type == 'spacy':
            doc = self.nlp(text)
            return [(ent.text, ent.label_, ent.start_char, ent.end_char)
                    for ent in doc.ents]
        else:
            return self.custom_process(text)
    
    def add_custom_entities(self, patterns):
        if self.model_type == 'spacy':
            ruler = self.nlp.get_pipe('entity_ruler')
            ruler.add_patterns(patterns)
    
    def evaluate(self, texts, gold_entities):
        predictions = [self.process_text(text) for text in texts]
        return self.calculate_metrics(predictions, gold_entities)