Как заставить Tensorflow PoseNet работать с формами Xamarin / Xam.Android - PullRequest
Новая
       5

Как заставить Tensorflow PoseNet работать с формами Xamarin / Xam.Android

0 голосов
/ 30 октября 2019

Я пробовал с java / Kotlin android и затем использовал его как .jar, но не могу заставить его работать. У меня есть этот код на Java, и мне нужно попробовать скопировать его, используя C # для Xamarin / Xamarin.Android

JAVA METHOD 

fun estimateSinglePose(bitmap: Bitmap): Person {
var t1: Long = SystemClock.elapsedRealtimeNanos()
val inputArray = arrayOf(initInputArray(bitmap))
var t2: Long = SystemClock.elapsedRealtimeNanos()
Log.i("posenet", String.format("Scaling to [-1,1] took %.2f ms", 1.0f * 
(t2 - t1) / 1_000_000))

val outputMap = initOutputMap(interpreter!!)

t1 = SystemClock.elapsedRealtimeNanos()
interpreter!!.runForMultipleInputsOutputs(inputArray, outputMap)
t2 = SystemClock.elapsedRealtimeNanos()
Log.i("posenet", String.format("Interpreter took %.2f ms", 1.0f * (t2 - t1) / 1_000_000))

val heatmaps = outputMap[0] as Array<Array<Array<FloatArray>>>
val offsets = outputMap[1] as Array<Array<Array<FloatArray>>>

val height = heatmaps[0].size
val width = heatmaps[0][0].size
val numKeypoints = heatmaps[0][0][0].size

// Finds the (row, col) locations of where the keypoints are most likely to be.
val keypointPositions = Array(numKeypoints) { Pair(0, 0) }
for (keypoint in 0 until numKeypoints) {
  var maxVal = heatmaps[0][0][0][keypoint]
  var maxRow = 0
  var maxCol = 0
  for (row in 0 until height) {
    for (col in 0 until width) {
      heatmaps[0][row][col][keypoint] = sigmoid(heatmaps[0][row][col][keypoint])
      if (heatmaps[0][row][col][keypoint] > maxVal) {
        maxVal = heatmaps[0][row][col][keypoint]
        maxRow = row
        maxCol = col
      }
    }
  }
  keypointPositions[keypoint] = Pair(maxRow, maxCol)
}

// Calculating the x and y coordinates of the keypoints with offset adjustment.
val xCoords = IntArray(numKeypoints)
val yCoords = IntArray(numKeypoints)
val confidenceScores = FloatArray(numKeypoints)
keypointPositions.forEachIndexed { idx, position ->
  val positionY = keypointPositions[idx].first
  val positionX = keypointPositions[idx].second
  yCoords[idx] = (
    position.first / (height - 1).toFloat() * bitmap.height +
      offsets[0][positionY][positionX][idx]
    ).toInt()
  xCoords[idx] = (
    position.second / (width - 1).toFloat() * bitmap.width +
      offsets[0][positionY]
      [positionX][idx + numKeypoints]
    ).toInt()
  confidenceScores[idx] = heatmaps[0][positionY][positionX][idx]
}

val person = Person()
val keypointList = Array(numKeypoints) { KeyPoint() }
var totalScore = 0.0f
enumValues<BodyPart>().forEachIndexed { idx, it ->
  keypointList[idx].bodyPart = it
  keypointList[idx].position.x = xCoords[idx]
  keypointList[idx].position.y = yCoords[idx]
  keypointList[idx].score = confidenceScores[idx]
  totalScore += confidenceScores[idx]
}

person.keyPoints = keypointList.toList()
person.score = totalScore / numKeypoints

return person

}

Я получилмодель PoseNet работает в методе C #, но этот метод написан для обнаружения объектов и не возвращает ключевые точки, я не знаю, как адаптировать его для получения ключевых точек

C # метод 

    public void Recognize(int[] colors)
    {
        if (!initialized)
        {
            throw new Exception("Initialize TensorflowLiteService first");
        }

        MessagingCenter.Instance.Send(this, nameof(AR.InputTensorMessage), new InputTensorMessage()
        {
            Colors = colors,
        });

        using (var op = new BuildinOpResolver())
        {
            using (var interpreter = new Interpreter(model, op))
            {
                InvokeInterpreter(colors, interpreter);
            }
        }
    }

    private void InvokeInterpreter(int[] colors, Interpreter interpreter)
    {
        if (useNumThreads)
        {
            interpreter.SetNumThreads(Environment.ProcessorCount);
        }

        var allocateTensorStatus = interpreter.AllocateTensors();
        if (allocateTensorStatus == Status.Error)
        {
            throw new Exception("Failed to allocate tensor");
        }



        var input = interpreter.GetInput();
        using (var inputTensor = interpreter.GetTensor(input[0]))
        {
            CopyColorsToTensor(inputTensor.DataPointer, colors);

            var watchInvoke = Stopwatch.StartNew();
            interpreter.Invoke();
            watchInvoke.Stop();

            Console.WriteLine($"InterpreterInvoke: {watchInvoke.ElapsedMilliseconds}ms");
        }

        var output = interpreter.GetOutput();
        var outputIndex = output[0];

        var outputTensors = new Tensor[output.Length];

        for (var i = 0; i < output.Length; i++)
        {
            outputTensors[i] = interpreter.GetTensor(outputIndex + i);
        }

        var detection_boxes_out = outputTensors[0].GetData() as float[];
        var detection_classes_out = outputTensors[1].GetData() as float[];
        var detection_scores_out = outputTensors[2].GetData() as float[];
        var num_detections_out = outputTensors[3].GetData() as float[];

        var numDetections = num_detections_out[0];

        LogDetectionResults(detection_classes_out, detection_scores_out, detection_boxes_out, (int)numDetections);
    }
...