Как передать параметры через Keras Wrapper (или вообще)?

Question

У меня есть следующая модель, и я хочу настроить несколько параметров через GridSearchCV(). Пока это работает, но я не могу настроить параметры compile() таким же образом:

def create_model(activation='relu'
                 , init='he_normal' 
                 , loss = 'mean_squared_error' 
                 , optimizer = 'adam' 
                 , metrics=['accuracy'] 
                 , dropout_rate= 0
                 , learning_rate = 0.1
                 , decay_rate = 0.005
                 , momentum = 0.8
                ):

    # Model definition
    model = Sequential()
    model.add(Dense(16, input_shape=(2, ), 
                    activation = activation,
                    kernel_initializer = init))         
                 ....

    model.add(Dropout(dropout_rate))
    model.add(Dense(2, 
                    activation = "tanh"))

    # Compile model 
    model.compile(loss = 'mean_squared_error'
                  , optimizer = 'adam'
                                      # (
                                      #  learning_rate = learning_rate
                                      # , decay_rate = decay_rate
                                      # , momentum =momentum
                                      # )
                  , metrics = ['accuracy']
                 )

    return model

# define the grid search parameters
batch_size = [2, 6]
epochs = [5, 10]
activation = ['relu', 'tanh']    
optimizer = ['Adam', 'sgd']    
metrics = ['mse', 'acc']
loss = ['mse', 'mae'] 
dropout_rate = [0.1, 0.25]
learning_rate = [0.2, 0.3]
decay_rate = [0.001, 0.005]
momentum = [0.4, 0.7]
kernel_initializer = ['init', 'normal']

param_grid = dict(batch_size=batch_size
                      , epochs=epochs
                      , activation = activation
                      , optimizer = optimizer
                      , metrics = metrics
                      , loss = loss        
                      , dropout_rate = dropout_rate 
                      , learning_rate = learning_rate
                      , decay_rate = decay_rate
                      , momentum = momentum
                      , kernel_initializer = kernel_initializer
                     )

hypparas = param_grid

model = KerasRegressor(build_fn=create_model, verbose=0)

model_cv = GridSearchCV(estimator=model, 
                    param_grid=hypparas, 
                    n_jobs=1,    # -1 uses all cores
                    cv=5
                    )

model_cv_result = model_cv.fit(X, y)

, и я хотел бы добавить learning_rate, decay_rate и momentum также кгиперпараметры для настройки. Но это не работает, как описано выше (поэтому я раскомментировал определенные строки в compile(). Что я должен изменить соответственно, как я могу передать эти параметры в create_model()?

edit:

> model_cv_result = model_cv.fit(X, y)

:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py:266: UserWarning: The total space of parameters 4 is smaller than n_iter=20. Running 4 iterations. For exhaustive searches, use GridSearchCV.
  % (grid_size, self.n_iter, grid_size), UserWarning)
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_validation.py:528: FutureWarning: From version 0.22, errors during fit will result in a cross validation score of NaN by default. Use error_score='raise' if you want an exception raised or error_score=np.nan to adopt the behavior from version 0.22.
  FutureWarning)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-256-703cbc43fc8b> in <module>
----> 5 model_cv_result = model_cv.fit(X, y)

C:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in fit(self, X, y, groups, **fit_params)
    685                 return results
    686 
--> 687             self._run_search(evaluate_candidates)
    688 
    689         # For multi-metric evaluation, store the best_index_, best_params_ and

C:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in _run_search(self, evaluate_candidates)
   1466         evaluate_candidates(ParameterSampler(
   1467             self.param_distributions, self.n_iter,
-> 1468             random_state=self.random_state))

C:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in evaluate_candidates(candidate_params)
    664                                for parameters, (train, test)
    665                                in product(candidate_params,
--> 666                                           cv.split(X, y, groups)))
    667 
    668                 if len(out) < 1:

C:\ProgramData\Anaconda3\lib\site-packages\joblib\parallel.py in __call__(self, iterable)
    919             # remaining jobs.
    920             self._iterating = False
--> 921             if self.dispatch_one_batch(iterator):
    922                 self._iterating = self._original_iterator is not None
    923 

C:\ProgramData\Anaconda3\lib\site-packages\joblib\parallel.py in dispatch_one_batch(self, iterator)
    757                 return False
    758             else:
--> 759                 self._dispatch(tasks)
    760                 return True
    761 

C:\ProgramData\Anaconda3\lib\site-packages\joblib\parallel.py in _dispatch(self, batch)
    714         with self._lock:
    715             job_idx = len(self._jobs)
--> 716             job = self._backend.apply_async(batch, callback=cb)
    717             # A job can complete so quickly than its callback is
    718             # called before we get here, causing self._jobs to

C:\ProgramData\Anaconda3\lib\site-packages\joblib\_parallel_backends.py in apply_async(self, func, callback)
    180     def apply_async(self, func, callback=None):
    181         """Schedule a func to be run"""
--> 182         result = ImmediateResult(func)
    183         if callback:
    184             callback(result)

C:\ProgramData\Anaconda3\lib\site-packages\joblib\_parallel_backends.py in __init__(self, batch)
    547         # Don't delay the application, to avoid keeping the input
    548         # arguments in memory
--> 549         self.results = batch()
    550 
    551     def get(self):

C:\ProgramData\Anaconda3\lib\site-packages\joblib\parallel.py in __call__(self)
    223         with parallel_backend(self._backend, n_jobs=self._n_jobs):
    224             return [func(*args, **kwargs)
--> 225                     for func, args, kwargs in self.items]
    226 
    227     def __len__(self):

C:\ProgramData\Anaconda3\lib\site-packages\joblib\parallel.py in <listcomp>(.0)
    223         with parallel_backend(self._backend, n_jobs=self._n_jobs):
    224             return [func(*args, **kwargs)
--> 225                     for func, args, kwargs in self.items]
    226 
    227     def __len__(self):

C:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_validation.py in _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score)
    512             estimator.fit(X_train, **fit_params)
    513         else:
--> 514             estimator.fit(X_train, y_train, **fit_params)
    515 
    516     except Exception as e:

C:\ProgramData\Anaconda3\lib\site-packages\keras\wrappers\scikit_learn.py in fit(self, x, y, **kwargs)
    140                 **self.filter_sk_params(self.build_fn.__call__))
    141         else:
--> 142             self.model = self.build_fn(**self.filter_sk_params(self.build_fn))
    143 
    144         if (losses.is_categorical_crossentropy(self.model.loss) and

<ipython-input-249-c3eb05e754ec> in create_model(activation, init, loss, optimizer, metrics, dropout_rate, learning_rate, decay_ratio, moment)
     60     # Compile model ### Resets weights
     61     opt = return_optimizer(type=optimizer, learning_rate=learning_rate)
---> 62     model.compile(loss='mean_squared_error', optimizer=opt, metrics=['accuracy'])
     63 
     64     return model

C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, weighted_metrics, target_tensors, **kwargs)
     93                 `optimizer`, `loss`, `metrics` or `sample_weight_mode`.
     94         """
---> 95         self.optimizer = optimizers.get(optimizer)
     96         self.loss = loss or {}
     97         self._compile_metrics = metrics or []

C:\ProgramData\Anaconda3\lib\site-packages\keras\optimizers.py in get(identifier)
    871     else:
    872         raise ValueError('Could not interpret optimizer identifier: ' +
--> 873                          str(identifier))

ValueError: Could not interpret optimizer identifier: None

Answer 1

Возможно, это не самое элегантное решение, но самым очевидным для меня молотком является определение другой функции, которая возвращает оптимизатор. Я немного упростил ваш пример.

from keras.layers import Dense
from keras.models import Sequential
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.model_selection import GridSearchCV
from keras.optimizers import Adam, SGD

def return_optimizer(type, learning_rate):
    if type == 'Adam':
        return Adam(lr=learning_rate)

    elif type == 'SGD':
        return SGD(lr=learning_rate)

Затем добавьте строку для вашего create_model() следующим образом:

def create_model(optimizer='adam', activation = 'sigmoid', 
learning_rate=0.1):

  model = Sequential()

  model.add(Dense(1, activation=activation, input_shape=(1,)))
  model.add(Dense(1, activation=activation))

  opt = return_optimizer(type=optimizer, learning_rate=learning_rate)
  model.compile(loss='mean_squared_error', optimizer=opt, metrics=['accuracy'])
  return model

Затем ваша сетка GridSerachCV() будет равна

param_grid = {
    'epochs': [2, 5],
    'optimizer': ['Adam', 'SGD'],
    'learning_rate': [0.1, 0.2]
}

Наконец,

model = KerasRegressor(build_fn=create_model, verbose=0)

model_cv = GridSearchCV(estimator=model,
                    param_grid=param_grid,
                    n_jobs=1,   
                    cv=5
                    )

Пожалуйста, дайте мне знать, как это работает.