Я работаю над проектом по распознаванию языка жестов. Просто чтобы проверить тензорный поток в android, я обучил простую модель, состоящую из 10 цифр, и я достиг точности 82%. Модель корректно классифицирует почти все примеры, когда я тестирую ее на python (Spyder IDE). Я развернул модель в android, используя код, приведенный на сайте тензорного потока. Код загружает модель в байтовый буфер, а также преобразует изображение в байтовый буфер, чтобы классифицировать его по интерпретатору. Моя преобразованная модель не классифицирует даже один пример правильно. Я приложил коды ниже.
Код для обучения моей модели и затем преобразование его в тензорный поток lite
import numpy as np
import cv2
import matplotlib.pyplot as plt
import os
from tqdm import tqdm
DATADIR="D:\Computer Science\Final Year Project\Sign-Language-Digits-Dataset-master\Dataset"
CATEGORIES=["0","1","2","3","4","5","6","7","8","9"]
IMG_SIZE=100
training_data = []
def create_training_data():
for category in CATEGORIES: # do dogs and cats
path = os.path.join(DATADIR,category) # create path to dogs and cats
class_num = CATEGORIES.index(category) # get the classification (0 or a 1). 0=dog 1=cat
for img in tqdm(os.listdir(path)): # iterate over each image per dogs and cats
try:
img_array = cv2.imread(os.path.join(path,img) ,cv2.IMREAD_GRAYSCALE) # convert to array
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE)) # resize to normalize data size
training_data.append([new_array, class_num]) # add this to our training_data
except Exception as e: # in the interest in keeping the output clean...
pass
print(e)
create_training_data()
import random
random.shuffle(training_data)
for sample in training_data[:10]:
print(sample[1])
X = []
y = []
for features,label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
y=np.array(y)
import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Activation, Flatten
from keras.optimizers import SGD
import cv2
import matplotlib.pyplot as plt
import pickle
from keras.utils import plot_model
X=X/255.0
model = Sequential()
model.add(Conv2D(32, (7, 7), input_shape=(100, 100, 1),activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (7, 7),activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (5, 5),activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (5, 5),activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(128,activation='relu'))
model.add(Dense(10,activation='softmax'))
opt = SGD(lr=0.01)
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['acc']*100)
history=model.fit(X, y,batch_size=32, epochs=30, validation_split=0.85)
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()
# "Loss"
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()
img=cv2.imread('D:\Computer Science\Final Year Project\Sign-Language-Digits-Dataset-master\Examples\example_1.jpg',cv2.IMREAD_GRAYSCALE)
img=cv2.resize(img,(100,100))
plt.imshow(img)
z=np.array(img).reshape(-1,100,100,1)
z=z/255.0
q=model.predict([z])[0]
print(np.argmax(q))
converter = tf.lite.TFLiteConverter.from_keras_model(model)
tflite_model = converter.convert()
open("82.tflite", "wb").write(tflite_model)
Этот код предназначен для классификации изображений. Я думаю, что есть проблема при преобразовании растрового изображения в байтовый буфер
package com.example.awaaz.Class;
import android.annotation.SuppressLint;
import android.content.res.AssetFileDescriptor;
import android.content.res.AssetManager;
import android.graphics.Bitmap;
import android.util.Log;
import org.tensorflow.lite.Interpreter;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
public class TensorFlowImageClassifier implements Classifier {
private static final int MAX_RESULTS = 3;
private static final int BATCH_SIZE = 1;
private static final int PIXEL_SIZE = 1;
private static final float THRESHOLD = 0.01f;
private static final int IMAGE_MEAN = 100;
private static final float IMAGE_STD = 255.0f;
private Interpreter interpreter;
private int inputSize;
private List<String> labelList;
private boolean quant;
private TensorFlowImageClassifier() {
}
static Classifier create(AssetManager assetManager,
String modelPath,
String labelPath,
int inputSize,
boolean quant) throws IOException {
TensorFlowImageClassifier classifier = new TensorFlowImageClassifier();
classifier.interpreter = new Interpreter(classifier.loadModelFile(assetManager, modelPath), new Interpreter.Options());
classifier.labelList = classifier.loadLabelList(assetManager, labelPath);
classifier.inputSize = inputSize;
classifier.quant = quant;
return classifier;
}
@Override
public List<Recognition> recognizeImage(Bitmap bitmap) {
ByteBuffer byteBuffer = convertBitmapToByteBuffer(bitmap);
Log.e("TAG","After Conversion to Byte Buffer");
if(quant){
byte[][] result = new byte[1][labelList.size()];
interpreter.run(byteBuffer, result);
return getSortedResultByte(result);
} else {
float [][] result = new float[1][labelList.size()];
interpreter.run(byteBuffer, result);
return getSortedResultFloat(result);
}
}
@Override
public void close() {
interpreter.close();
interpreter = null;
}
private MappedByteBuffer loadModelFile(AssetManager assetManager, String modelPath) throws IOException {
AssetFileDescriptor fileDescriptor = assetManager.openFd(modelPath);
FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
FileChannel fileChannel = inputStream.getChannel();
long startOffset = fileDescriptor.getStartOffset();
long declaredLength = fileDescriptor.getDeclaredLength();
return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
}
private List<String> loadLabelList(AssetManager assetManager, String labelPath) throws IOException {
List<String> labelList = new ArrayList<>();
BufferedReader reader = new BufferedReader(new InputStreamReader(assetManager.open(labelPath)));
String line;
while ((line = reader.readLine()) != null) {
labelList.add(line);
}
reader.close();
return labelList;
}
private ByteBuffer convertBitmapToByteBuffer(Bitmap bitmap) {
ByteBuffer byteBuffer;
if(quant) {
byteBuffer = ByteBuffer.allocateDirect(4*BATCH_SIZE * inputSize * inputSize * PIXEL_SIZE);
} else {
byteBuffer = ByteBuffer.allocateDirect(4 * BATCH_SIZE * inputSize * inputSize * PIXEL_SIZE);
}
byteBuffer.order(ByteOrder.nativeOrder());
int[] intValues = new int[inputSize * inputSize];
bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
int pixel = 0;
for (int i = 0; i < inputSize; ++i) {
for (int j = 0; j < inputSize; ++j) {
final int val = intValues[pixel++];
try{
if(quant){
byteBuffer.put((byte) ((val >> 16) & 0xFF));
byteBuffer.put((byte) ((val >> 8) & 0xFF));
byteBuffer.put((byte) (val & 0xFF));
} else {
byteBuffer.putFloat((((val >> 16) & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
byteBuffer.putFloat((((val >> 8) & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
byteBuffer.putFloat((((val) & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
}
}
catch (Exception e){
Log.e("TAG",i+"-"+j);
}
}
}
return byteBuffer;
}
@SuppressLint("DefaultLocale")
private List<Recognition> getSortedResultByte(byte[][] labelProbArray) {
PriorityQueue<Recognition> pq =
new PriorityQueue<>(
MAX_RESULTS,
new Comparator<Recognition>() {
@Override
public int compare(Recognition lhs, Recognition rhs) {
return Float.compare(rhs.getConfidence(), lhs.getConfidence());
}
});
for (int i = 0; i < labelList.size(); ++i) {
float confidence = (labelProbArray[0][i] & 0xff) / 255.0f;
if (confidence > THRESHOLD) {
pq.add(new Recognition("" + i,
labelList.size() > i ? labelList.get(i) : "unknown",
confidence, quant));
}
}
final ArrayList<Recognition> recognitions = new ArrayList<>();
int recognitionsSize = Math.min(pq.size(), MAX_RESULTS);
for (int i = 0; i < recognitionsSize; ++i) {
recognitions.add(pq.poll());
}
return recognitions;
}
@SuppressLint("DefaultLocale")
private List<Recognition> getSortedResultFloat(float[][] labelProbArray) {
PriorityQueue<Recognition> pq =
new PriorityQueue<>(
MAX_RESULTS,
new Comparator<Recognition>() {
@Override
public int compare(Recognition lhs, Recognition rhs) {
return Float.compare(rhs.getConfidence(), lhs.getConfidence());
}
});
for (int i = 0; i < labelList.size(); ++i) {
float confidence = labelProbArray[0][i];
if (confidence > THRESHOLD) {
pq.add(new Recognition("" + i,
labelList.size() > i ? labelList.get(i) : "unknown",
confidence, quant));
}
}
final ArrayList<Recognition> recognitions = new ArrayList<>();
int recognitionsSize = Math.min(pq.size(), MAX_RESULTS);
for (int i = 0; i < recognitionsSize; ++i) {
recognitions.add(pq.poll());
}
return recognitions;
}
}