Для жанровой классификации настраиваемая функция потерь обычно не требуется.Комбинированная модель, в которой песня разбита на несколько окон прогнозирования, может быть настроена с помощью Multiple Instance Learning (MIL).
MIL - это контролируемый подход к обучению, при котором метка присутствует не на каждой независимой выборке (экземплярах), но вместо "мешка" (неупорядоченного набора) экземпляров.В вашем случае экземпляр - это каждое 5-секундное окно функций MFCC, а сумка - вся песня.
В Keras мы используем слой TimeDistributed, чтобы выполнить нашу модель для всех окон.Затем мы объединяем результат, используя GlobalAveragePooling1D, эффективно реализуя среднее голосование по окнам.Это легче дифференцировать, чем голосование большинством.
Ниже приведен работоспособный пример:
import math
import keras
import librosa
import pandas
import numpy
import sklearn
def window_model(n_bands, n_frames, n_classes, hidden=32):
from keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D
out_units = 1 if n_classes == 2 else n_classes
out_activation = 'sigmoid' if n_classes == 2 else 'softmax'
shape = (n_bands, n_frames, 1)
# Basic CNN model
# An MLP could also be used, but may need to reshape on input and output
model = keras.Sequential([
Conv2D(16, (3,3), input_shape=shape),
MaxPooling2D((2,3)),
Conv2D(16, (3,3)),
MaxPooling2D((2,2)),
Flatten(),
Dense(hidden, activation='relu'),
Dense(hidden, activation='relu'),
Dense(out_units, activation=out_activation),
])
return model
def song_model(n_bands, n_frames, n_windows, n_classes=3):
from keras.layers import Input, TimeDistributed, GlobalAveragePooling1D
# Create the frame-wise model, will be reused across all frames
base = window_model(n_bands, n_frames, n_classes)
# GlobalAveragePooling1D expects a 'channel' dimension at end
shape = (n_windows, n_bands, n_frames, 1)
print('Frame model')
base.summary()
model = keras.Sequential([
TimeDistributed(base, input_shape=shape),
GlobalAveragePooling1D(),
])
print('Song model')
model.summary()
model.compile(loss='categorical_crossentropy', optimizer='SGD', metrics=['acc'])
return model
def extract_features(path, sample_rate, n_bands, hop_length, n_frames, window_length, song_length):
# melspectrogram might perform better with CNNs
from librosa.feature import mfcc
# Load a fixed length section of sound
# Might need to pad if some songs are too short
y, sr = librosa.load(path, sr=sample_rate, offset=0, duration=song_length)
assert sr == sample_rate, sr
_song_length = len(y)/sample_rate
assert _song_length == song_length, _song_length
# Split into windows
window_samples = int(sample_rate * window_length)
window_hop = window_samples//2 # use 50% overlap
windows = librosa.util.frame(y, frame_length=window_samples, hop_length=window_hop)
# Calculate features for each window
features = []
for w in range(windows.shape[1]):
win = windows[:, w]
f = mfcc(y=win, sr=sample_rate, n_mfcc=n_bands,
hop_length=hop_length, n_fft=2*hop_length)
f = numpy.expand_dims(f, -1) # add channels dimension
features.append(f)
features = numpy.stack(features)
return features
def main():
# Settings for our model
n_bands = 13 # MFCCs
sample_rate = 22050
hop_length = 512
window_length = 5.0
song_length_max = 1.0*60
n_frames = math.ceil(window_length / (hop_length/sample_rate))
n_windows = math.floor(song_length_max / (window_length/2))-1
model = song_model(n_bands, n_frames, n_windows)
# Generate some example data
ex = librosa.util.example_audio_file()
examples = 8
numpy.random.seed(2)
songs = pandas.DataFrame({
'path': [ex] * examples,
'genre': numpy.random.choice([ 'rock', 'metal', 'blues' ], size=examples),
})
assert len(songs.genre.unique() == 3)
print('Song data')
print(songs)
def get_features(path):
f = extract_features(path, sample_rate, n_bands,
hop_length, n_frames, window_length, song_length_max)
return f
from sklearn.preprocessing import LabelBinarizer
binarizer = LabelBinarizer()
y = binarizer.fit_transform(songs.genre.values)
print('y', y.shape, y)
features = numpy.stack([ get_features(p) for p in songs.path ])
print('features', features.shape)
model.fit(features, y)
if __name__ == '__main__':
main()
В примере выводятся внутренние и комбинированные сводные данные модели:
Frame model
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_1 (Conv2D) (None, 11, 214, 16) 160
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 5, 71, 16) 0
_________________________________________________________________
conv2d_2 (Conv2D) (None, 3, 69, 16) 2320
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 1, 34, 16) 0
_________________________________________________________________
flatten_1 (Flatten) (None, 544) 0
_________________________________________________________________
dense_1 (Dense) (None, 32) 17440
_________________________________________________________________
dense_2 (Dense) (None, 32) 1056
_________________________________________________________________
dense_3 (Dense) (None, 3) 99
=================================================================
Total params: 21,075
Trainable params: 21,075
Non-trainable params: 0
_________________________________________________________________
Song model
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
time_distributed_1 (TimeDist (None, 23, 3) 21075
_________________________________________________________________
global_average_pooling1d_1 ( (None, 3) 0
=================================================================
Total params: 21,075
Trainable params: 21,075
Non-trainable params: 0
_________________________________________________________________
И форма векторного элемента, подаваемого в модель:
features (8, 23, 13, 216, 1)
8 песен, 23 окна в каждой, с 13 полосами MFCC, 216 кадров в каждом окне.И пятое измерение размером 1, чтобы сделать Кераса счастливым ...