Word Embedding揭秘：如何用词嵌入提升NLP模型表现

import nltkfrom nltk.corpus import stopwordsfrom nltk.tokenize import word_tokenizefrom nltk.stem import WordNetLemmatizernltk.download('stopwords')
def preprocess_text(text):tokens = word_tokenize(text)tokens = [word.lower() for word in tokens]tokens = [word for word in tokens if word.isalnum()]# remove stop wordsstop_words = set(stopwords.words('english'))tokens = [word for word in tokens if word not in stop_words]# Lemmatize lemmatizer = WordNetLemmatizer()tokens = [lemmatizer.lemmatize(word) for word in tokens]
return ' '.join(tokens)

from tensorflow.keras.preprocessing.text import one_hotfrom tensorflow.keras.preprocessing.sequence import pad_sequences
documents = ["The quick brown fox jumps over the lazy dog","A journey of a thousand miles begins with a single step","To be or not to be that is the question"]
vocab_size=50 integer_encoded=[]for i,doc in enumerate(documents):integer_encoded.append(one_hot(doc,50))print("The encoding for text",i+1," is : ",one_hot(doc,50))

# length of maximum documentmaxlen = max([len(word_tokenize(text)) for text in documents])maxlen
# OUTPUT11

# To create embeddings, all documents must have the same length. hence we can pad to ensure uniformity.pad_corp = pad_sequences(integer_encoded, maxlen=maxlen,padding='post',value=0.0)print("No of padded documents: ",len(pad_corp))
# OUTPUTNo of padded documents:3

for i,doc in enumerate(pad_corp): print("The padded encoding for text",i+1," is : ",doc)

from sklearn.feature_extraction.text import TfidfVectorizer
preprocessed_text = [preprocess_text(doc) for doc in documents]
tfidf = TfidfVectorizer()tfidf_matrix = tfidf.fit_transform(preprocessed_text)print("Vocabulary:", tfidf.vocabulary_)tfidf_matrix.toarray()

from gensim.models import Word2Vecfrom nltk.tokenize import word_tokenizeimport numpy as np
tokenized_docs = [word_tokenize(doc.lower()) for doc in documents]w2v_model = Word2Vec(sentences=tokenized_docs, vector_size=100, window=5, min_count=1, workers=4)
def get_sentence_embedding(sentence, model):words = word_tokenize(sentence.lower())word_vectors = [model.wv[word] for word in words if word in model.wv]return np.mean(word_vectors, axis=0) if word_vectors else np.zeros(model.vector_size)
print("Word2Vec Sentence Embeddings:")for doc in documents:sentence_embedding = get_sentence_embedding(doc, w2v_model)print(f"Sentence: {doc}.\n Embedding: {sentence_embedding}")

from gensim.models import FastText
fasttext_model = FastText(sentences=tokenized_docs, vector_size=100, window=5, min_count=1, workers=4)
def get_sentence_embedding_fasttext(sentence, model):words = word_tokenize(sentence.lower())word_vectors = [model.wv[word] for word in words if word in model.wv]return np.mean(word_vectors, axis=0) if word_vectors else np.zeros(model.vector_size)
print("FastText Sentence Embeddings:")for doc in documents:sentence_embedding = get_sentence_embedding_fasttext(doc, fasttext_model)print(f"Sentence: {doc}. \nEmbedding: {sentence_embedding}")

from transformers import BertTokenizer, BertModelimport torch
# Load pre-trained BERT tokenizer and modeltokenizer = BertTokenizer.from_pretrained('bert-base-uncased')model = BertModel.from_pretrained('bert-base-uncased')
def get_sentence_embedding_bert(sentence, model, tokenizer):inputs = tokenizer(sentence, return_tensors='pt')outputs = model(**inputs)# Get sentence embedding by averaging token embeddingssentence_embedding = torch.mean(outputs.last_hidden_state, dim=1)return sentence_embedding.squeeze().detach().numpy()
print("BERT Sentence Embeddings:")for doc in documents:sentence_embedding = get_sentence_embedding_bert(doc, model, tokenizer)print(f"Sentence: {doc}. \nEmbedding: {sentence_embedding}")