Как изменить искровой фрейм данных для слоя LSTM с помощью dist-keras - PullRequest
Новая
       22

Как изменить искровой фрейм данных для слоя LSTM с помощью dist-keras

0 голосов
/ 06 июня 2019

Я использую модель LSTM для классификации текста. Я пытаюсь обучить свою модель в кластерной системе, используя pyspark и dist-keras.когда я пытался оценить обученную модель с тестовыми данными, она выдает ошибку

Это код, с которым я работаю 


# import necessary libraries

import numpy as np

import time

import requests

from keras.optimizers import *
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.layers import LSTM,Flatten

from pyspark import SparkContext
from pyspark import SparkConf

from pyspark.ml.feature import StandardScaler
from pyspark.ml.feature import MinMaxScaler
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.feature import StringIndexer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.mllib.evaluation import BinaryClassificationMetrics

from distkeras.trainers import *
from distkeras.predictors import *
from distkeras.transformers import *
from distkeras.evaluators import *
from distkeras.utils import *
import keras
import h5py

# Read the dataset.
raw_dataset = spark.read.format('csv') \
                   .options(header='true', inferSchema='true') \
                   .load("training_dataset.csv")

# Extract the desired features from the raw dataset.by constructing a list with all desired columns.
features = raw_dataset.columns
features.remove('word7')

# Spark's VectorAssembler to "assemble" (create) a vector of all desired features.
vector_assembler = VectorAssembler(inputCols=features, outputCol="features")
# This transformer will take all columns specified in features, and create an additional column "features" which will contain all the desired features aggregated into a single vector.
dataset = vector_assembler.transform(raw_dataset)

# Show what happened after applying the vector assembler.
# Note: "features" column got appended to the end.
dataset.select("features").take(1)

# scale the datset
standard_scaler = StandardScaler(inputCol="features", outputCol="features_normalized", withStd=True, withMean=True)
standard_scaler_model = standard_scaler.fit(dataset)
dataset = standard_scaler_model.transform(dataset)


nb_classes = dataset.select("word7").distinct().count() # Number of output classes (signal and background)
nb_features = len(features)
batch_size = 32

dataset = shuffle(dataset)

#split the dtaset to 60 and 40 percent
(training_set, test_set) = dataset.randomSplit([0.6, 0.4])

X_train = np.array(training_set.select('features_normalized').collect())

input_shape=X_train.shape[1:]

model = Sequential()
hidden_layer_size = 25
model.add(LSTM(hidden_layer_size, input_shape=input_shape,return_sequences=True))
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(Flatten())
model.add(Dense(hidden_layer_size))
model.add(Activation('relu'))

model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.summary()

optimizer = 'adagrad'
loss = 'categorical_crossentropy'

# a mapping from distributed trainer instance --> metric name --> metric value
results = {}
# Number of training epochs to run for each Trainer
TRAIN_EPOCHS = 1

trainer = SingleTrainer(keras_model=model, worker_optimizer=optimizer,
                        loss=loss, features_col="features_normalized",
                        label_col="word7", num_epoch=TRAIN_EPOCHS, batch_size=32)
trained_model = trainer.train(training_set)

# evaluate trained model using test set

evaluator = AccuracyEvaluator(prediction_col="prediction_index", label_col="word7")
# Clear the prediction column from the testset.
test_set = test_set.select("features_normalized", "word7")
# Apply a prediction from a trained model.
predictor = ModelPredictor(keras_model=trained_model, features_col="features_normalized")
print(test_set)
test_set = predictor.predict(test_set)
# Allocate an index transformer.
index_transformer = LabelIndexTransformer(output_dim=nb_classes)
# Transform the prediction vector to an indexed label.
test_set = index_transformer.transform(test_set)
# Fetch the score.
score = evaluator.evaluate(test_set)

, когда я предсказываю с обученной моделью, она показывает ошибку, подобную этой

'with shape' + str (data_shape)) ValueError: Ошибка при проверке ввода: ожидалось, что lstm_8_input будет иметь 3 измерения, но получил массив с формой (1, 6)

...