Я использую логистическую регрессию для обучения модели на некоторых текстовых данных. Вот код, который я использую:
from fonduer.learning import LogisticRegression
disc_model = LogisticRegression()
%time disc_model.train((train_cands[0], F_train[0]), train_marginals, n_epochs=50, lr=0.001)
Код не вызывает проблем, когда я запускаю его на 20 документах, но когда я увеличиваю количество документов до 40, я получаю эту ошибку:
[INFO] fonduer.learning.disc_learning - Load defalut parameters for Logistic Regression
---------------------------------------------------------------------------
MemoryError Traceback (most recent call last)
<timed eval> in <module>
~/.venv/lib/python3.6/site-packages/fonduer/learning/disc_learning.py in train(self, X_train, Y_train, n_epochs, lr, batch_size, rebalance, X_dev, Y_dev, print_freq, dev_ckpt, dev_ckpt_delay, save_dir, seed, host_device)
169
170 _X_train, _Y_train = self._preprocess_data(
--> 171 X_train, Y_train, idxs=train_idxs, train=True
172 )
173 if X_dev is not None:
~/.venv/lib/python3.6/site-packages/fonduer/learning/disc_models/logistic_regression.py in _preprocess_data(self, X, Y, idxs, train)
59 C, F = X
60 if issparse(F):
---> 61 F = F.todense()
62
63 if idxs is None:
~/.venv/lib/python3.6/site-packages/scipy/sparse/base.py in todense(self, order, out)
844 `numpy.matrix` object that shares the same memory.
845 """
--> 846 return np.asmatrix(self.toarray(order=order, out=out))
847
848 def toarray(self, order=None, out=None):
~/.venv/lib/python3.6/site-packages/scipy/sparse/compressed.py in toarray(self, order, out)
945 if out is None and order is None:
946 order = self._swap('cf')[0]
--> 947 out = self._process_toarray_args(order, out)
948 if not (out.flags.c_contiguous or out.flags.f_contiguous):
949 raise ValueError('Output array must be C or F contiguous')
~/.venv/lib/python3.6/site-packages/scipy/sparse/base.py in _process_toarray_args(self, order, out)
1182 return out
1183 else:
-> 1184 return np.zeros(self.shape, dtype=self.dtype, order=order)
1185
1186
MemoryError: