Kotlin / Python code

for(cnt in contours){
            val boundRect: Rect = Imgproc.boundingRect(cnt)
            if (boundRect.width > 90 && boundRect.height > 90 && boundRect.x != 0 && boundRect.y != 0){
                // Draw rectangles
                val x = boundRect.x.toDouble() - 30
                val y = boundRect.y.toDouble() - 30
                val w = boundRect.width.toDouble() + 30
                val h = boundRect.height.toDouble() + 30

                Imgproc.rectangle(tmp, Point(x, y),
                    Point(x + w, y + h),
                    Scalar(0.0,255.0,0.0), 3)

                // Extract bounding images, resize, and normalise
                val image_roi = Mat(tmp, boundRect)
                Imgproc.cvtColor(image_roi, image_roi, Imgproc.COLOR_RGB2GRAY)
                Imgproc.GaussianBlur(image_roi, image_roi, Size(3.0,3.0), 0.0)
                Imgproc.dilate(image_roi, image_roi, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, Size(4.0, 4.0)))
                Imgproc.threshold(image_roi, image_roi, 90.0, 255.0, Imgproc.THRESH_BINARY);
                Imgproc.resize(image_roi, image_roi, Size(28.0,28.0))
                Core.normalize(image_roi, image_roi, 0.0, 255.0, Core.NORM_MINMAX);
                image_roi.convertTo(image_roi, CvType.CV_8UC1)
                extracted.add(image_roi)
            }

}

This extracts and process the images. It's a bit messy. It extracts correctly, and seems to process it decently.

for (img in extracted) {
       val tensorBuffer = extractBytes(img)
       val outputs = model.process(tensorBuffer)
       val outputFeature0 = outputs.outputFeature0AsTensorBuffer
       val conf = outputFeature0.floatArray
       out += getLanguageText(conf, numbers)

}


In another function, images are processed.

 private fun extractBytes(img: Mat): TensorBuffer{
        val inputFeature = TensorBuffer.createFixedSize(intArrayOf(1, 28, 28, 1), DataType.FLOAT32)
        val byteBuffer = ByteBuffer.allocateDirect(28 * 28 * 4) // 4 bytes per float
        byteBuffer.order(ByteOrder.nativeOrder())
        byteBuffer.rewind()

        for (i in 0 until 28) {
            for (j in 0 until 28) {
                val temp = img.get(i, j)[0].toFloat() // Assuming single-channel (gray)
                byteBuffer.putFloat(temp)
            }
        }

        inputFeature.loadBuffer(byteBuffer)
        return inputFeature
    }
Changing from opencv Mat to Tensorbuffer.

private fun getLanguageText(conf: FloatArray, numbers: Array<String>): String {
        return numbers[conf.indices.maxBy {conf[it]}]
}

This just gets the maximum value of the conf array.


Python code:
def extractText(dirs):
    images = []
    # Read the original image
    img = cv2.imread(dirs[0])

    # Convert to grayscale, blur, and get threshold
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    blur = cv2.GaussianBlur(img_gray, (3,3), 0)
    thresh = cv2.threshold(blur, 100, 255, cv2.THRESH_BINARY)[1]


    # Find contours
    cnts, hierarchy = cv2.findContours(image=thresh, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_SIMPLE)
    #img_copy = img.copy()

    # From contours, find bounding boxes with ROI (region of interest)
    for contour in cnts:
        (x,y,w,h) = cv2.boundingRect(contour)
        if w > 90 and h > 90 and x != 0 and y != 0:
            # Pre-processing
            img_ = img[y-20:y+h+20, x-15:x+w+20]
            #img_copy = cv2.rectangle(img_copy, (x-5, y-5), (x+w+5, y+h+5), (0, 255, 0), 2)

            # Blur the image for better edge detection
            img_gray = cv2.cvtColor(img_, cv2.COLOR_BGR2GRAY)
            img_invert = cv2.bitwise_not(img_gray)
            img_ = cv2.resize(img_invert, (28, 28))
            img_blur = cv2.GaussianBlur(img_, (3,3), 0)
            _, img_ts = cv2.threshold(img_blur, 90, 255, cv2.THRESH_BINARY)


            # Add to array
            img_ = np.asarray(img_ts, dtype=np.float32)/255.0
            images.append(img_)
    return np.array(images)

def predictExtractedText(data, model):
    # Load model
    model = tf.keras.models.load_model(model)


    # run prediction
    prediction = model.predict(data)

    # Plot data
    plt.figure(1)
    #nko_numbers = "0߁߂߃߄߅߆߇߈߉"
    nko_numbers = "0123456789.-+wxyz/="
    for index in range(0, len(prediction)):
        plt.subplot(5,5,index+1)
        plt.title(f"Number: {nko_numbers[np.argmax(prediction[index])]}")
        img = np.asarray(images[index]).squeeze()
        plt.axis('off')
        plt.imshow(img)
    plt.show()