Ошибка TypeError при кодировании данных с использованием кодировщика меток в Scikit Learn - PullRequest
Новая
       4

Ошибка TypeError при кодировании данных с использованием кодировщика меток в Scikit Learn

1 голос
/ 09 февраля 2020

Я не могу кодировать данные, используя кодировщик меток в scikit learn.

dataset.csv имеет два столбца текста и метку. Я пытаюсь прочитать текст из набора данных в список и метки в другой список и добавление этих списков в фрейм данных , но, похоже, он не работает. 

from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn import decomposition, ensemble
import pandas, xgboost, numpy, string

data = open('dataset.csv').read()
labels = []
texts = []

for i ,line in enumerate(data.split("\n")):
    content = line.split("\",")
    texts.append(content[0])
    labels.append(content[1:])

trainDF = pandas.DataFrame()
trainDF['text'] = texts
trainDF['label'] = labels

train_x, valid_x, train_y, valid_y = model_selection.train_test_split(trainDF['text'],trainDF['label'],test_size = 0.2,random_state = 0)
encoder = preprocessing.LabelEncoder()
train_y = encoder.fit_transform(train_y)
valid_y = encoder.fit_transform(valid_y)

count_vect = CountVectorizer(analyzer='word', token_pattern=r'\w{1,}')
count_vect.fit(trainDF['texts'])

xtrain_count =  count_vect.transform(train_x)
xvalid_count =  count_vect.transform(valid_x)

tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', max_features=5000)
tfidf_vect.fit(trainDF['texts'])
xtrain_tfidf =  tfidf_vect.transform(train_x)
xvalid_tfidf =  tfidf_vect.transform(valid_x)

accuracy = train_model(svm.SVC(), xtrain_tfidf, train_y, xvalid_tfidf)

print(accuracy)

Ошибка: 

Traceback (most recent call last):
  File "/home/crackthumb/environments/my_env/lib/python3.6/site-packages/sklearn/preprocessing/label.py", line 105, in _encode
    res = _encode_python(values, uniques, encode)
  File "/home/crackthumb/environments/my_env/lib/python3.6/site-packages/sklearn/preprocessing/label.py", line 59, in _encode_python
    uniques = sorted(set(values))
TypeError: unhashable type: 'list'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "Classifier.py", line 21, in <module>
    train_y = encoder.fit_transform(train_y)
  File "/home/crackthumb/environments/my_env/lib/python3.6/site-packages/sklearn/preprocessing/label.py", line 236, in fit_transform
    self.classes_, y = _encode(y, encode=True)
  File "/home/crackthumb/environments/my_env/lib/python3.6/site-packages/sklearn/preprocessing/label.py", line 107, in _encode
    raise TypeError("argument must be a string or number")
TypeError: argument must be a string or number

1 Ответ

0 голосов
/ 09 февраля 2020 
from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn import decomposition, ensemble
import pandas, xgboost, numpy, string
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.svm import SVC

data = open('dataset.csv').read()
labels = []
texts = []

for i ,line in enumerate(data.split("\n")):
    content = line.split("\",")
    texts.append(str(content[0]))
    labels.append(str(content[1:]))

trainDF = pandas.DataFrame()
trainDF['text'] = texts
trainDF['label'] = labels

train_x, valid_x, train_y, valid_y = model_selection.train_test_split(trainDF['text'],trainDF['label'],test_size = 0.2,random_state = 0)
encoder = preprocessing.LabelEncoder()
train_y = encoder.fit_transform(train_y)
valid_y = encoder.fit_transform(valid_y)

from sklearn.pipeline import Pipeline
text_clf = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', SVC(kernel='rbf'))])
text_clf.fit(train_x, train_y)

predicted = text_clf.predict(valid_x)

from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

print(confusion_matrix(valid_y,predicted))
print(classification_report(valid_y,predicted))
print(accuracy_score(valid_y,predicted))
...