Я строю конвейер преобразования объектов в первый раз, но отметил, что использование FeatureUnion для объединения моих конвейеров возвращает массивный массив, но я бы предпочел вместо этого возвращать фрейм данных pandas, чтобы можно было видеть имена компонентов.
Я создал этот класс, который возвращает фрейм данных pandas для объектов, и он прекрасно работает, но этот пользовательский класс featureunion работает в два раза медленнее, чем встроенный класс featureleion sklearn. Как мне распараллелить это так, чтобы время выполнения было на одном уровне? Любая помощь будет принята с благодарностью!
Чтобы ввести многопроцессорность, я пробовал это, но я получаю сообщение об ошибке.
class PandasFeatureUnion(BaseEstimator, TransformerMixin):
def __init__(self, transformer_list, n_jobs=cpu_count()):
self.transformer_list = transformer_list
self.n_jobs = min(n_jobs, len(transformer_list))
def one_fit(self, transformer, X, y):
return transformer.fit(X, y)
def one_transform(self, transformer, X):
return transformer.transform(X)
def fit(self, X, y=None):
Parallel(n_jobs=self.n_jobs)(
delayed(self.one_fit)(trans, X, y)
for _, trans in self.transformer_list)
return self
def transform(self, X):
Xts = Parallel(n_jobs=self.n_jobs)(
delayed(self.one_transform)(trans, X)
for name, trans in self.transformer_list)
Xunion = reduce(lambda X1, X2:
pd.merge(X1, X2,
left_index=True, right_index=True), Xts)
return Xunion
Я получаю следующую ошибку:
sklearn.externals.joblib.externals.loky.process_executor._RemoteTraceback:
"""
Traceback (most recent call last):
File "pandas/_libs/index.pyx", line 132, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 382, in pandas._libs.hashtable.Float64HashTable.get_item
TypeError: must be real number, not NoneType
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/pandas/core/indexes/base.py", line 2657, in get_loc
return self._engine.get_loc(key)
File "pandas/_libs/index.pyx", line 108, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 134, in pandas._libs.index.IndexEngine.get_loc
KeyError: None
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "pandas/_libs/index.pyx", line 132, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 382, in pandas._libs.hashtable.Float64HashTable.get_item
TypeError: must be real number, not NoneType
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/externals/joblib/externals/loky/process_executor.py", line 418, in _process_worker
r = call_item()
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/externals/joblib/externals/loky/process_executor.py", line 272, in __call__
return self.fn(*self.args, **self.kwargs)
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py", line 567, in __call__
return self.func(*args, **kwargs)
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py", line 225, in __call__
for func, args, kwargs in self.items]
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py", line 225, in <listcomp>
for func, args, kwargs in self.items]
File "/Users/rihun/PycharmProjects/cross_sell/code/cross_sell_features.py", line 386, in one_transform
def one_transform(self, transformer, X):
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/pipeline.py", line 451, in _transform
Xt = transform.transform(Xt)
File "/Users/rihun/PycharmProjects/cross_sell/code/cross_sell_features.py", line 126, in transform
X = X.copy()
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/pandas/core/indexing.py", line 190, in __setitem__
self._setitem_with_indexer(indexer, value)
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/pandas/core/indexing.py", line 380, in _setitem_with_indexer
nindexer.append(labels.get_loc(key))
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/pandas/core/indexes/numeric.py", line 436, in get_loc
tolerance=tolerance)
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/pandas/core/indexes/base.py", line 2659, in get_loc
return self._engine.get_loc(self._maybe_cast_indexer(key))
File "pandas/_libs/index.pyx", line 108, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 134, in pandas._libs.index.IndexEngine.get_loc
KeyError: None
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/rihun/PycharmProjects/cross_sell/code/savm_features_pipeline.py", line 75, in <module>
data_tr = full_pipeline.fit_transform(data)
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/pipeline.py", line 300, in fit_transform
return last_step.fit_transform(Xt, y, **fit_params)
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/base.py", line 464, in fit_transform
return self.fit(X, **fit_params).transform(X)
File "/Users/rihun/PycharmProjects/cross_sell/code/cross_sell_features.py", line 397, in transform
for _, trans in self.transformer_list)
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py", line 930, in __call__
self.retrieve()
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py", line 833, in retrieve
self._output.extend(job.get(timeout=self.timeout))
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py", line 521, in wrap_future_result
return future.result(timeout=timeout)
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/concurrent/futures/_base.py", line 425, in result
return self.__get_result()
File "/Users/rihun/anaconda3/envs/CrossSell/lib/python3.6/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
KeyError: None