Я хочу объединить множество групп объектов: лексические, семантические и смычковые функции, извлеченные из твитов, в классификатор
я работаю над проблемой проверки авторства в твиттере, код выглядит следующим образом
в следующем: мой код:
train = pd.read_csv("./av/av1/train.csv")
test = pd.read_csv("./av/av1/test.csv")
num_chapters = len('train.csv')
fvs_lexical = np.zeros((len(train['text']), 3), np.float64)
fvs_punct = np.zeros((len(train['text']), 3), np.float64)
for e, ch_text in enumerate(train['text']):
# note: the nltk.word_tokenize includes punctuation
tokens = nltk.word_tokenize(ch_text.lower())
words = word_tokenizer.tokenize(ch_text.lower())
sentences = sentence_tokenizer.tokenize(ch_text)
vocab = set(words)
words_per_sentence = np.array([len(word_tokenizer.tokenize(s))
for s in sentences])
# average number of words per sentence
fvs_lexical[e, 0] = words_per_sentence.mean()
# sentence length variation
fvs_lexical[e, 1] = words_per_sentence.std()
# apply whitening to decorrelate the features
fvs_lexical = whiten(fvs_lexical)
#bag of wrods features
bow_vectorizer = CountVectorizer(max_df=0.90, min_df=2, max_features=1000, stop_words='english')
vectorizer = FeatureUnion([ ("baw", bow_vectorizer), ("fvs_lexical",fvs_lexical)])
matrix = vectorizer.fit_transform(train['text'].values.astype('U'))
print "num of features: " , len(vectorizer.get_feature_names())
X =matrix.toarray()
y = np.asarray(train['label'].values.astype('U'))
model=LogisticRegression()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
scores = cross_val_score(model,X_train,y_train,cv=3,
scoring='f1_micro')
y_pred = model.fit(X_train, y_train).predict(X_test)
print 'F1 score:',f1_score(y_test, y_pred, average=None) # calculating
прогнозируемый результат - оценка F1, но я получил следующую ошибку:
TypeError Traceback (most recent call last)
<ipython-input-87-1a69ca9a65a2> in <module>()
24 bow_vectorizer = CountVectorizer(max_df=0.90, min_df=2, max_features=1000, stop_words='english')
25
---> 26 vectorizer = FeatureUnion([ ("baw", bow_vectorizer), ("fvs_lexical",fvs_lexical)])
27 matrix = vectorizer.fit_transform(train['text'].values.astype('U'))
28 print "num of features: " , len(vectorizer.get_feature_names())
C:\Users\AsusPc\Anaconda2\lib\site-packages\sklearn\pipeline.pyc in __init__(self, transformer_list, n_jobs, transformer_weights)
616 self.n_jobs = n_jobs
617 self.transformer_weights = transformer_weights
--> 618 self._validate_transformers()
619
620 def get_params(self, deep=True):
C:\Users\AsusPc\Anaconda2\lib\site-packages\sklearn\pipeline.pyc in _validate_transformers(self)
660 raise TypeError("All estimators should implement fit and "
661 "transform. '%s' (type %s) doesn't" %
--> 662 (t, type(t)))
663
664 def _iter(self):
TypeError: All estimators should implement fit and transform. '[[1.29995156 0. 0. ]
[5.38551361 0. 0. ]
[0.37141473 0. 0. ]
...
[0.92853683 0. 0. ]
[1.1142442 3.52964785 0. ]
[1.85707366 0. 0. ]]' (type <type 'numpy.ndarray'>) doesn't