Мне нужно руководство о том, как преобразовать создание матрицы через python / numpy в javascript

Question

Мне нужно выровнять 2 изображения с помощью opencv. js

Я пытаюсь найти способ обойти обнаруженную проблему здесь . Поэтому я пробую этот подход .

У меня проблемы с попыткой воссоздать (и вычислить) матрицу преобразования python numpy в javascript.

Вот python:

image_baseline_keypoints = np.float32([kpv_image_baseline[m.queryIdx].pt for m in matches[:number_of_matches]]).reshape(-1, 1, 2);
image_keypoints = n.float32([kpv_image[m.trainIdx].pt for m in matches[:number_of_matches]]).reshape(-1, 1, 2);

(note: the article [linked above][2] has another way of creating the matrix via numpy as well. )

каков подходящий подход к выполнению того же самого в javascript? использовать num js (следующая попытка но если возможно, я не хочу приносить другую библиотеку)?

вот общий JS код:

    function Align_img2() {

        let image_baseline = cv.imread(imgElement_Baseline);
        let image = cv.imread('imageChangeup');

        let image_baseline_gray = new cv.Mat();
        let image_gray = new cv.Mat();

        //get size of baseline image
        var image_baseline_width = image_baseline.cols;
        var image_baseline_height = image_baseline.rows;

        //resize image to baseline image
        let image_baseline_dimensions = new cv.Size(image_baseline_width, image_baseline_height);
        cv.resize(image, image, image_baseline_dimensions, cv.INTER_AREA);

        // Convert images to grayscale
        cv.cvtColor(image_baseline, image_baseline_gray, cv.COLOR_BGR2GRAY);
        cv.cvtColor(image, image_gray, cv.COLOR_BGR2GRAY);

        // Initiate detector
        var orb = new cv.ORB(10000);
        var kpv_image_baseline = new cv.KeyPointVector();
        var kpv_image = new cv.KeyPointVector();
        var descriptors_image_baseline =new cv.Mat();
        var descriptors_image =new cv.Mat();
        var image_baseline_keypoints=new cv.Mat();
        var image_keypoints =new cv.Mat();

        // find the keypoints with ORB
        orb.detect(image_baseline_gray, kpv_image_baseline);
        orb.detect(image_gray, kpv_image);

        // compute the descriptors with ORB
        orb.compute(image_baseline_gray, kpv_image_baseline, descriptors_image_baseline);
        orb.compute(image_gray, kpv_image, descriptors_image);

        // Debug to verify key points found
        //let color = new cv.Scalar(0,255,0, 255);
        //cv.drawKeypoints(image_baseline_gray, kpv_image_baseline, image_baseline_keypoints, color);
        //cv.drawKeypoints(image_gray, kpv_image, image_keypoints, color);
        //console.log(image_baseline_keypoints);
        //console.log(image_keypoints);

        // find matches
        let bf = new cv.BFMatcher(cv.NORM_HAMMING, 1);
        // Match descriptors
        let matches = new cv.DMatchVector();
        bf.match(descriptors_image_baseline, descriptors_image, matches);

        // Debug to verify matches found
        //var matches_img = new cv.Mat();
        //cv.drawMatches(image_baseline_gray, kpv_image_baseline, image_gray, kpv_image, matches, matches_img, color);
        //console.log(matches_img);

        // calculate transformation matrix
        number_of_matches = 10;
        /* 
        image_baseline_keypoints = np.float32([kpv_image_baseline[m.queryIdx].pt for m in matches[:number_of_matches]]).reshape(-1, 1, 2);
        image_keypoints = np.float32([kpv_image[m.trainIdx].pt for m in matches[:number_of_matches]]).reshape(-1, 1, 2);
        */

        //note: np => nj in numjs
        image_baseline_keypoints = nj.float32([kpv_image_baseline[m.queryIdx].pt for (m in matches)]).reshape(-1, 1, 2);
        image_keypoints = nj.float32([kpv_image[m.trainIdx].pt for (m in matches)]).reshape(-1, 1, 2);

        // Calculate Homography
        var h = new cv.Mat();
        h = cv.findHomography(image_baseline_keypoints, image_keypoints);

        // Warp image to baseline_image based on homography
        var dst = new cv.Mat();
        cv.warpPerspective(image_gray, dst, h, (image_baseline_gray.shape[1], image_baseline_gray.shape[0]));
        cv.imshow('imageChangeup', dst);

        matches_img.delete();
        matches.delete();
        bf.delete();
        orb.delete();
        kpv_image_baseline.delete();
        kpv_image.delete();
        descriptors_image_baseline.delete();
        descriptors_image.delete();
        image_baseline_keypoints.delete();
        image_keypoints.delete();
        image_baseline_gray.delete();
        image_gray.delete();
        h.delete();
        dst.delete();
    };

Answer 1

К сожалению, opencv js do c УЖАСНО! на самом деле здесь нечего назвать, как do c! Метод cv.findHomography имеет 3 аргумента. где два первых должны быть matArray. вы должны преобразовать точки pointA и pointB в matArray, вызвав: cv.matFromArray(number_of_cols_pointsA,2,cv.CV_32F,pointsA), где «2» - это размер массива. а CV_32F относится к мату типа float32. и для третьего аргумента findHomography вы можете использовать cv.RANSAC

Answer 2

Close, я собираюсь выяснить неперехваченное исключение в warpPerspective ()

opencv_4_3_0. js: 30 Uncaught 6533712 ___cxa_throw @ opencv_4_3_0. js: 30 dynCall_vii @ 0167cb82: 1 Module.dynCall_viiiii @ opencv_4_3_0. js: 30 dynCall_viiiii_966 @ VM513794: 4 warpPerspective @ VM514734: 11 прото. @ opencv_4_3_0. js: 30 Align_img2 @ opencv: 1932 onclick @ VM515781 opencv: 190

Должно быть, гомография еще не верна для отправки в warpPerspective (). Но выглядит достоверно, но трудно сказать:

homography : 
Mat {$$: {…}}
cols: (...)
data: (...)
data8S: (...)
data16S: (...)
data16U: (...)
data32F: Float32Array(0)
buffer: ArrayBuffer(134217728)
[[Int8Array]]: Int8Array(134217728) [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …]
[[Int16Array]]: Int16Array(67108864)
[0 … 999999]
[0 … 9999]
[10000 … 19999]
[20000 … 29999]
[30000 … 39999]
[30000 … 30099]
30000: 0
30001: 10800
30002: 12768
30003: 10812
30004: 0
30005: 10935
30006: 21504
30007: 10986
30008: -20480
30009: 11071
30010: -22464
30011: 11123
30012: -8461
30013: 11175

код:

function Align_img2() {

    //image_A is the original image we are trying to align to
    //image_B is the image we are trying to line up correctly

    let image_A = cv.imread(imgElement_Baseline);
    let image_B = cv.imread('imageChangeup');

    let image_A_gray = new cv.Mat();
    let image_B_gray = new cv.Mat();

    //get size of baseline image (image A)
    var image_A_width = image_A.cols;
    var image_A_height = image_A.rows;

    //resize image B to the baseline (image A) image
    let image_A_dimensions = new cv.Size(image_A_width, image_A_height);
    cv.resize(image_B, image_B, image_A_dimensions, cv.INTER_AREA);

    // Convert both images to grayscale
    cv.cvtColor(image_A, image_A_gray, cv.COLOR_BGRA2GRAY);
    cv.cvtColor(image_B, image_B_gray, cv.COLOR_BGRA2GRAY);

    // Initiate detector
    var orb = new cv.ORB(1000);
    var kpv_image_A = new cv.KeyPointVector();
    var kpv_image_B = new cv.KeyPointVector();
    var descriptors_image_A =new cv.Mat();
    var descriptors_image_B =new cv.Mat();
    var image_A_keypoints=new cv.Mat();
    var image_B_keypoints =new cv.Mat();

    mask = new cv.Mat();

    orb.detectAndCompute(image_A_gray, new cv.Mat(), kpv_image_A, descriptors_image_A);
    orb.detectAndCompute(image_B_gray, new cv.Mat(), kpv_image_B, descriptors_image_B);

    let color = new cv.Scalar(0,255,0, 255);

    // find matches
    let bf = new cv.BFMatcher(cv.NORM_HAMMING, true);

    // Match descriptors
    let matches = new cv.DMatchVector();
    bf.match(descriptors_image_A, descriptors_image_B, matches);

    var good_matches = new cv.DMatchVector();
    for (let i = 0; i < matches.size(); i++) {
        if (matches.get(i).distance < 30) {
            good_matches.push_back(matches.get(i));
            console.log("Good match, distance: ", matches.get(i).distance, "queryIdx: ", matches.get(i).queryIdx);
        }
    }

    // Debug to verify matches found
    //var matches_img = new cv.Mat();
    //cv.drawMatches(image_A_gray, kpv_image_A, image_B_gray, kpv_image_B, good_matches, matches_img, color);
    //cv.imshow('imageChangeup', matches_img);

    var points_A = [];
    var points_B = [];

    for (let i = 0; i < good_matches.size(); i++) {
        console.log("good_matches.get(i).queryIdx", good_matches.get(i).queryIdx);
        console.log("kpv_image_A.get(good_matches.get(i).queryIdx ).pt", kpv_image_A.get(good_matches.get(i).queryIdx ).pt);
        console.log("points_A", points_A);
        points_A.push(kpv_image_A.get(good_matches.get(i).queryIdx ).pt );
        points_B.push(kpv_image_B.get(good_matches.get(i).trainIdx ).pt );
    }

    let mat_A = cv.matFromArray(points_A.length, 2, cv.CV_32F,points_A);
    let mat_B = cv.matFromArray(points_B.length, 2, cv.CV_32F,points_B);

    // Calculate Homography points_A and B need to be CV_32FC2 - a 32-bit, floating-point, and 2-channels structure
    var homography = new cv.Mat();
    homography = cv.findHomography(mat_A, mat_B, cv.RANSAC);

    // Warp image to baseline_image based on homography
    var image_B_final_result = new cv.Mat();
    
    let dsize = new cv.Size(image_A.rows, image_A.cols);
    //console.log("dsize", dsize); //this is Size {width: 1293, height: 1000} for our test image

    cv.warpPerspective(image_B, image_B_final_result, homography, dsize);

    cv.imshow('imageChangeup', image_B_final_result);

    //matches_img.delete();
    matches.delete();
    bf.delete();
    orb.delete();
    kpv_image_A.delete();
    kpv_image_B.delete();
    descriptors_image_A.delete();
    descriptors_image_B.delete();
    image_A_keypoints.delete();
    image_B_keypoints.delete();
    image_A_gray.delete();
    image_B_gray.delete();
    homography.delete();
    image_B_final_result.delete();
    mat_A.delete();
    mat_B.delete();
};