java - пытается вытащить изображение с помощью opencv

Question

У меня есть следующий код для распознавания теста с openCV

package app;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
import org.opencv.core.Core;
import org.opencv.core.*;
import org.opencv.core.MatOfFloat;
import org.opencv.core.MatOfByte;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.*;
import org.opencv.dnn.Dnn;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import org.opencv.utils.*;

public class eastSample {
    static {
        try {

            String model = System.getProperty("sun.arch.data.model");
            // the path the .dll lib location
            String libraryPath = "D:/work/java/opencv/lib/x64";
            // check for if system is 64 or 32
            if (model.equals("64")) {
                libraryPath = "D:/work/java/opencv/lib/x64";
            }
            // set the path
            System.setProperty("java.library.path", libraryPath);
            Field sysPath = ClassLoader.class.getDeclaredField("sys_paths");
            sysPath.setAccessible(true);
            sysPath.set(null, null);
            // load the lib
            // System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
            System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    public static void main(String[] args) {
        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);

        float scoreThresh = 0.5f;
        float nmsThresh = 0.4f;
        // Model from https://github.com/argman/EAST
        // You can find it here :
        // https://github.com/opencv/opencv_extra/blob/master/testdata/dnn/download_models.py#L309
        Net net = Dnn.readNetFromTensorflow("D:/opencv/frozen_east_text_detection.pb");
        // input image
        Mat frame = Imgcodecs.imread("D:/tst_texture2.jpg");
        Imgproc.cvtColor(frame, frame, Imgproc.COLOR_RGBA2RGB);

        Size siz = new Size(320, 320);
        int W = (int) (siz.width / 4); // width of the output geometry / score maps
        int H = (int) (siz.height / 4); // height of those. the geometry has 4, vertically stacked maps, the score one 1
        Mat blob = Dnn.blobFromImage(frame, 1.0, siz, new Scalar(123.68, 116.78, 103.94), true, false);
        net.setInput(blob);
        List<Mat> outs = new ArrayList<>(2);
        List<String> outNames = new ArrayList<String>();
        outNames.add("feature_fusion/Conv_7/Sigmoid");
        outNames.add("feature_fusion/concat_3");
        net.forward(outs, outNames);

        // Decode predicted bounding boxes.
        Mat scores = outs.get(0).reshape(1, H);
        // My lord and savior :
        // http://answers.opencv.org/question/175676/javaandroid-access-4-dim-mat-planes/
        Mat geometry = outs.get(1).reshape(1, 5 * H); // don't hardcode it !
        List<Float> confidencesList = new ArrayList<>();
        List<RotatedRect> boxesList = decode(scores, geometry, confidencesList, scoreThresh);

        // Apply non-maximum suppression procedure.
        MatOfFloat confidences = new MatOfFloat(Converters.vector_float_to_Mat(confidencesList));
        RotatedRect[] boxesArray = boxesList.toArray(new RotatedRect[0]);
        MatOfRotatedRect boxes = new MatOfRotatedRect(boxesArray);
        MatOfInt indices = new MatOfInt();
        Dnn.NMSBoxesRotated(boxes, confidences, scoreThresh, nmsThresh, indices);

        // Render detections
        Point ratio = new Point((float) frame.cols() / siz.width, (float) frame.rows() / siz.height);
        int[] indexes = indices.toArray();
        for (int i = 0; i < indexes.length; ++i) {
            RotatedRect rot = boxesArray[indexes[i]];
            Point[] vertices = new Point[4];
            rot.points(vertices);
            for (int j = 0; j < 4; ++j) {
                vertices[j].x *= ratio.x;
                vertices[j].y *= ratio.y;
                System.out.println("vertices "+j+" x : "+vertices[j].x);
                System.out.println("vertices "+j+" y : "+vertices[j].y);

            }
            for (int j = 0; j < 4; ++j) {
                Imgproc.line(frame, vertices[j], vertices[(j + 1) % 4], new Scalar(0, 0, 255), 1);
            }
        }
        Imgcodecs.imwrite("D:/res_tst_texture.jpg", frame);
    }

    private static List<RotatedRect> decode(Mat scores, Mat geometry, List<Float> confidences, float scoreThresh) {
        // size of 1 geometry plane
        int W = geometry.cols();
        int H = geometry.rows() / 5;
        // System.out.println(geometry);
        // System.out.println(scores);

        List<RotatedRect> detections = new ArrayList<>();
        for (int y = 0; y < H; ++y) {
            Mat scoresData = scores.row(y);
            Mat x0Data = geometry.submat(0, H, 0, W).row(y);
            Mat x1Data = geometry.submat(H, 2 * H, 0, W).row(y);
            Mat x2Data = geometry.submat(2 * H, 3 * H, 0, W).row(y);
            Mat x3Data = geometry.submat(3 * H, 4 * H, 0, W).row(y);
            Mat anglesData = geometry.submat(4 * H, 5 * H, 0, W).row(y);

            for (int x = 0; x < W; ++x) {
                double score = scoresData.get(0, x)[0];
                if (score >= scoreThresh) {
                    double offsetX = x * 4.0;
                    double offsetY = y * 4.0;
                    double angle = anglesData.get(0, x)[0];
                    double cosA = Math.cos(angle);
                    double sinA = Math.sin(angle);
                    double x0 = x0Data.get(0, x)[0];
                    double x1 = x1Data.get(0, x)[0];
                    double x2 = x2Data.get(0, x)[0];
                    double x3 = x3Data.get(0, x)[0];
                    double h = x0 + x2;
                    double w = x1 + x3;
                    Point offset = new Point(offsetX + cosA * x1 + sinA * x2, offsetY - sinA * x1 + cosA * x2);
                    Point p1 = new Point(-1 * sinA * h + offset.x, -1 * cosA * h + offset.y);
                    Point p3 = new Point(-1 * cosA * w + offset.x, sinA * w + offset.y); // original trouble here !
                    RotatedRect r = new RotatedRect(new Point(0.5 * (p1.x + p3.x), 0.5 * (p1.y + p3.y)), new Size(w, h),
                            -1 * angle * 180 / Math.PI);
                    detections.add(r);
                    confidences.add((float) score);
                }
            }
        }
        return detections;
    }
}

Вот мое изображение

, и это результат:

Этот код работает с выровненным текстом, но, как вы можете видеть, он не работает при перекосе текста.

Есть ли способ получить правильная ограничительная рамка наклонного текста?

Чтобы выровнять, мне нужно разделить BG для текста, то есть бинаризацию (remove_noise_and_smooth)

Я могу использовать следующий код:

Mat remove_noise_and_smooth_Mat(Mat source) {
    int morph_elem = 1;
    int morph_size = 1;
    Imgproc.cvtColor(source, source, Imgproc.COLOR_RGB2GRAY);

    // Applying Bilateral filter on the Image
    Mat dest = new Mat();
    Imgproc.bilateralFilter(source, dest, 3, 10, 10, Core.BORDER_DEFAULT);
    source = dest;

    // Core.bitwise_not(source, source);
    Mat filtered = new Mat();
    Imgproc.adaptiveThreshold(source, filtered, 255, Imgproc.ADAPTIVE_THRESH_MEAN_C, Imgproc.THRESH_BINARY, 41, 10);

    Mat element = Imgproc.getStructuringElement(morph_elem, new Size(2 * morph_size + 1, 2 * morph_size + 1),
            new Point(morph_size, morph_size));
    Mat opening = new Mat();

    Imgproc.morphologyEx(filtered, opening, Imgproc.MORPH_OPEN, element);
    Mat closing = new Mat();
    Imgproc.morphologyEx(opening, closing, Imgproc.MORPH_CLOSE, element);

    source = image_smoothening(source);
    Core.bitwise_or(source, closing, source);
    return source;
}

Mat image_smoothening(Mat source) {
    int BINARY_THREHOLD = 180;
    Imgproc.threshold(source, source, BINARY_THREHOLD, 255, Imgproc.THRESH_BINARY);
    Imgproc.threshold(source, source, 0, 255, Imgproc.THRESH_BINARY + Imgproc.THRESH_OTSU);
    Imgproc.GaussianBlur(source, source, new Size(1, 1), 0);
    Imgproc.threshold(source, source, 0, 255, Imgproc.THRESH_BINARY + Imgproc.THRESH_OTSU);
    return source;
}

это результат этого кода на изображении:

Проблема в том, что я не могу исправить перекос, потому что с одной стороны: BG слишком шумный Для этого используйте opencv Deskew (код, прикрепленный в конце). и с другой стороны: EAST не дает мне правильную ограничивающую рамку, если бы я мог просто удалить все, что не является текстом.

public Mat deskew(Mat src, double angle) {
    Point center = new Point(src.width() / 2, src.height() / 2);
    Mat rotImage = Imgproc.getRotationMatrix2D(center, angle, 1.0);
    //1.0 means 100 % scale
    Size size = new Size(src.width(), src.height());
    Imgproc.warpAffine(src, src, rotImage, size, Imgproc.INTER_LINEAR + Imgproc.CV_WARP_FILL_OUTLIERS);
    return src;
}

public void computeSkew(String inFile) {
    //Load this image in grayscale
    Mat img = Imgcodecs.imread(inFile, Imgcodecs.IMREAD_GRAYSCALE);

    //Binarize it
    //Use adaptive threshold if necessary
    //Imgproc.adaptiveThreshold(img, img, 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY, 15, 40);
    Imgproc.threshold(img, img, 200, 255, Imgproc.THRESH_BINARY);

    //Invert the colors (because objects are represented as white pixels, and the background is represented by black pixels)
    Core.bitwise_not(img, img);
    Mat element = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));

    //We can now perform our erosion, we must declare our rectangle-shaped structuring element and call the erode function
    Imgproc.erode(img, img, element);

    //Find all white pixels
    Mat wLocMat = Mat.zeros(img.size(), img.type());
    Core.findNonZero(img, wLocMat);

    //Create an empty Mat and pass it to the function
    MatOfPoint matOfPoint = new MatOfPoint(wLocMat);

    //Translate MatOfPoint to MatOfPoint2f in order to user at a next step
    MatOfPoint2f mat2f = new MatOfPoint2f();
    matOfPoint.convertTo(mat2f, CvType.CV_32FC2);

    //Get rotated rect of white pixels
    RotatedRect rotatedRect = Imgproc.minAreaRect(mat2f);

    Point[] vertices = new Point[4];
    rotatedRect.points(vertices);
    List<MatOfPoint> boxContours = new ArrayList<>();
    boxContours.add(new MatOfPoint(vertices));
    Imgproc.drawContours(img, boxContours, 0, new Scalar(128, 128, 128), -1);

    double resultAngle = rotatedRect.angle;
    if (rotatedRect.size.width > rotatedRect.size.height) {
        rotatedRect.angle += 90.f;
    }

    //Or
    //rotatedRect.angle = rotatedRect.angle < -45 ? rotatedRect.angle + 90.f : rotatedRect.angle;

    Mat result = deskew(Imgcodecs.imread(inFile), rotatedRect.angle);
    Imgcodecs.imwrite(inFile, result);

}