Я пытаюсь применить код, указанный на https://towardsdatascience.com/3-basic-distance-measurement-in-text-mining-5852becff1d7. Когда я использую это с моими собственными данными, я, кажется, получаю доступ к части списка, которая не существует, и просто не в состоянии определить, где я делаю эту ошибку:
File "D:/Proj/projects/eucledian_1/nltk_headline_1.py", line 190, in eucledian
score = sklearn.metrics.pairwise.euclidean_distances([transformed_results[i]], [transformed_results[0]])[0][0]
IndexError: list index out of range
Я не думаю, что я превышение длины only_event или «transformed_results». Вот код, который у меня есть:
def eucledian(self):
print('inside eucledian', only_event)
for i, news_headline in enumerate(only_event): # range(len(only_event))
print('*******', only_event[i])
print('this is transformed results: ', transformed_results[i]) # prints
score = sklearn.metrics.pairwise.euclidean_distances([transformed_results[i]], [transformed_results[0]])[0][0]
print('-----', only_event[i]) # prints
print('Score: %.2f, Comparing Sentence: %s' % (score, news_headline)) # prints
Данные, которые я могу прочитать из БД и сохранить в списке only_event (length = 2), выглядят так: ['Perhaps this code is incomplete or mistyped in some way.', 'Use one of the following methods:\n* Ensure that the power is turned on.\n* Only concatenate a user-supplied value into a query, or if it must be a Boolean or numeric type.\n'].
Операторы print выдают результат, но строка, вызывающая euclidean_distances, выдает ошибку IndexError: list index out of range. transformed_results (длина = 1) выглядит следующим образом:
[array([329., 2., 57., 44., 44., 44., 88., 57., 44., 44., 44.,
57., 13., 13., 88., 1., 2., 13., 136., 13., 13., 13.,
220., 44., 44., 44., 88., 88., 44., 44., 89., 2., 13.,
88., 13., 44., 132., 26., 4., 4., 132., 44., 1., 13.,
48., 27., 88., 132., 88., 44., 44., 132., 13., 4., 13.,
44., 13., 158., 15., 13., 162., 4., 44., 44., 26., 13.,
1., 44., 1., 57., 13., 1., 44., 44., 45., 44., 44.,
4., 13., 44., 1., 13., 44., 44., 44., 44., 336., 44.,
51., 2., 235., 13., 132., 132., 70., 26., 44., 13., 13.,
13., 44., 4., 1., 57., 44., 44., 2., 44.])]
Заранее благодарим вас за просмотр этого
Обновлен, чтобы включить воспроизводимый код @ dzang
import numpy as np
import sklearn.preprocessing
import sklearn.metrics
token_event_obj = ['perhaps', 'this', 'code', 'is', 'incomplete', 'or', 'mistyped', 'in', 'some', 'way', 'use', 'one', 'of', 'the', 'following', 'methodsn', 'use', 'a', 'querypreparation', 'api', 'to', 'safely', 'construct', 'the', 'sql', 'query', 'containing', 'usersupplied', 'valuesn', 'only', 'concatenate', 'a', 'usersupplied', 'value', 'into', 'a', 'query', 'if', 'it', 'has', 'been', 'checked', 'against', 'a', 'whitelist', 'of', 'safe', 'string', 'values', 'or', 'if', 'it', 'must', 'be', 'a', 'boolean', 'or', 'numeric', 'typen']
only_event = ['Perhaps this code is incomplete or mistyped in some way.', 'Use one of the following methods:\n* Use a query-preparation API to safely construct the SQL query containing user-supplied values.\n* Only concatenate a user-supplied value into a query if it has been checked against a whitelist of safe string values, or if it must be a Boolean or numeric type.\n']
def transform(headlines):
print('inside transform', headlines)
tokens = [w for s in headlines for w in s]
print()
print('All Tokens:')
print(tokens)
results = []
label_enc = sklearn.preprocessing.LabelEncoder()
onehot_enc = sklearn.preprocessing.OneHotEncoder()
encoded_all_tokens = label_enc.fit_transform(list(set(tokens)))
encoded_all_tokens = encoded_all_tokens.reshape(len(encoded_all_tokens), 1)
onehot_enc.fit(encoded_all_tokens)
for headline_tokens in headlines:
print()
print(headline_tokens)
print('Original Input:', headline_tokens)
encoded_words = label_enc.transform(headline_tokens)
print('Encoded by Label Encoder:', encoded_words)
encoded_words = onehot_enc.transform(encoded_words.reshape(len(encoded_words), 1))
print('Encoded by OneHot Encoder:')
# print(encoded_words)
results.append(np.sum(encoded_words.toarray(), axis=0))
print('Transform results:', results)
return results
def eucledian():
print('inside eucledian', len(only_event))
for i, news_headline in enumerate(only_event): # range(len(only_event))
print('*******', only_event[i])
print('this is transformed results: ', transformed_results)
# print('len', len(sklearn.metrics.pairwise.euclidean_distances([transformed_results[i]], [transformed_results[0]])[0]))
print(type(transformed_results), len(transformed_results))
score = sklearn.metrics.pairwise.euclidean_distances([transformed_results[i]], [transformed_results[0]])[0]
print('-----', only_event[i])
print('Score: %.2f, Comparing Sentence: %s' % (score, news_headline))
transformed_results = transform([token_event_obj])
eucledian()