from sklearn.feature_extraction.text import TfidfVectorizer
from collections import defaultdict
import numpy as np
TfidfEmbeddingVectorizer
class TfidfEmbeddingVectorizer(object):
def __init__(self, word2vec):
self.word2vec = word2vec
def transform(self, X):
tfidf = TfidfVectorizer(analyzer = lambda x : x)
tfidf.fit(X)
max_idf = max(tfidf.idf_)
word2weight = defaultdict(lambda : max_idf, [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()])
array_list =[]
for words in X:
array_list.append(np.array(np.mean([self.word2vec[w]*word2weight[w] for w in words if w in self.word2vec] or [np.zeros(100)], axis = 0)))
return(array_list)
vec_tf_skip_gram = TfidfEmbeddingVectorizer(w2v_skip_gram)
vec_tf_CBOW = TfidfEmbeddingVectorizer(w2v_CBOW)
train_tf_s = vec_tf_skip_gram.transform(text_train)
test_tf_s = vec_tf_skip_gram.transform(text_test)
train_tf_c = vec_tf_CBOW.transform(text_train)
test_tf_c = vec_tf_CBOW.transform(text_test)
NameError Traceback (most recent call last)
<ipython-input-8-58d8af0cf2a6> in <module>
67 return(array_list)
68
---> 69 vec_tf_skip_gram = {TfidfEmbeddingVectorizer}'(w2v_skip_gram)
70 vec_tf_CBOW = 'TfidfEmbeddingVectorizer'(w2v_CBOW)
71 # skip-gram
NameError: name 'w2v_skip_gram' is not defined
and also error on w2v_CBOW too how can i fix this?
As you can see,
vec_tf_skip_gram = {TfidfEmbeddingVectorizer}' (w2v_skip_gram) displays name error
Of course, the next person
70 vec_tf_CBOW = 'TfidfEmbeddingVectorizer'(w2v_CBOW)
also shows an error, how can I resolve it?
I modified the part declared model1 and model2 below in the document.
# Word2Vec(CBOW / skip-gram)
# The process of representing a word as a vector while preserving the contextual meaning of the word
- CBOW: Predict target words with N surrounding words
- skip-gram : predict whether N surrounding words appear with target words
from gensim.models import Word2Vec
# # Skip-gram (sg=1)
w2v_skip_gram = Word2Vec(text_train, size=100, window=10, min_count=10, workers=4, sg=1)
# # CBOW (sg=0)
w2v_CBOW = Word2Vec(text_train, size=100, window=10, min_count=10, workers=4, sg=0)
# size : how many dimensions of a vector per word
# window: how many words to use before and after
# min_count : Minimum number of appearances (excluding words below min_count)
# sg : CBOW (=0) or skip-gram (=1)
572 rails db:create error: Could not find mysql2-0.5.4 in any of the sources
581 PHP ssh2_scp_send fails to send files as intended
617 Uncaught (inpromise) Error on Electron: An object could not be cloned
610 GDB gets version error when attempting to debug with the Presense SDK (IDE)
912 When building Fast API+Uvicorn environment with PyInstaller, console=False results in an error
© 2024 OneMinuteCode. All rights reserved.