From fdc376a48de609fb2e87a9a13d20a3431467c702 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sat, 25 Jul 2026 20:15:51 -0700 Subject: [PATCH 1/9] update PaddleOCR pipeline --- .../ML/Inference/ML_PaddleOCRPipeline.cpp | 85 +++++++++++++------ 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index fbe0f98797..459941f34e 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -103,44 +103,79 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ // Invert image for findNonZero: text becomes white (255), background black (0) cv::Mat binary_inv; - cv::threshold(gray, binary_inv, 254, 255, cv::THRESH_BINARY_INV); + cv::threshold(gray, binary_inv, 250, 255, cv::THRESH_BINARY_INV); // Find coordinates of all text pixels std::vector nonZeroCoords; cv::findNonZero(binary_inv, nonZeroCoords); cv::Mat cropped_image; - if (!nonZeroCoords.empty()){ - // Get the minimal bounding rectangle for the text - cv::Rect boundingBox = cv::boundingRect(nonZeroCoords); - - // Add a small buffer/padding around the box if needed (optional) - boundingBox.x = std::max(0, boundingBox.x - 2); - boundingBox.y = std::max(0, boundingBox.y - 2); - boundingBox.width = std::min(cv_image_rgb.cols - boundingBox.x, boundingBox.width + 4); - boundingBox.height = std::min(cv_image_rgb.rows - boundingBox.y, boundingBox.height + 4); - - // Crop the original color image - cropped_image = cv_image_rgb(boundingBox); - // static int i = 0; - // cv::imwrite("output" + std::to_string(i) + ".png", cropped_image); - // i++; - }else{ - return ""; // Return empty string if no text is detected in the region + if (nonZeroCoords.empty()){ + return ""; } - + cv::Rect bbox = cv::boundingRect(nonZeroCoords); + + // Small safety margin (NOT 20 pixels) + int pad_x = std::max(1, bbox.width / 20); // ~5% + int pad_y = std::max(1, bbox.height / 20); // ~5% + + bbox.x = std::max(0, bbox.x - pad_x); + bbox.y = std::max(0, bbox.y - pad_y); + + bbox.width = std::min( + cv_image_rgb.cols - bbox.x, + bbox.width + 2 * pad_x + ); + + bbox.height = std::min( + cv_image_rgb.rows - bbox.y, + bbox.height + 2 * pad_y + ); + + cropped_image = cv_image_rgb(bbox).clone(); + + int h = cropped_image.rows; + int w = cropped_image.cols; + + constexpr float min_ratio = 0.5f; + + if ((float)w / h < min_ratio) { // handle tall/narrow characters + int target_w = static_cast(std::ceil(min_ratio * h)); + + int left = (target_w - w) / 2; + int right = target_w - w - left; + + cv::copyMakeBorder( + cropped_image, + cropped_image, + 0, 0, // no vertical padding + left, right, + cv::BORDER_CONSTANT, + cv::Scalar(255,255,255) + ); + } + // 2a. Calculate dynamic width (maintain aspect ratio) // the model shape is {1, 3, 48, dynamic_width}. Note that the height is fixed at 48 pixels // the input image must be scaled to match the height of 48, for the neural network int target_h = 48; - float aspect_ratio = (float)cropped_image.cols / (float)cropped_image.rows; - int target_w = static_cast(target_h * aspect_ratio); + float aspect_ratio = (float)cropped_image.cols / cropped_image.rows; + + int target_w = std::max( + 1, + (int)std::round(target_h * aspect_ratio) + ); - if (target_w <= 0) return ""; - - // 2b. Resize cv::Mat resized; - cv::resize(cropped_image, resized, cv::Size(target_w, target_h)); + cv::resize( + cropped_image, + resized, + cv::Size(target_w, target_h), + 0, + 0, + cv::INTER_LINEAR + ); + // 3. Normalize // convert UC3 8-bit [0,255] to 32FC3 float [0,1], then use ImageNet Normalization From 5c82dd1b80e4c62954bc4256a22db9472c4ed5f2 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 26 Jul 2026 10:44:22 -0700 Subject: [PATCH 2/9] update SandwichRecipeNumberDetector --- .../Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp b/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp index 4127f5b8ad..ba1bd702d4 100644 --- a/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp +++ b/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp @@ -7,6 +7,7 @@ #include #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/Containers/FixedLimitVector.tpp" +#include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" @@ -56,7 +57,7 @@ void SandwichRecipeNumberDetector::detect_recipes(const ImageViewRGB32& screen, ImageRGB32 dilated_image(filterd_image.width(), filterd_image.height()); - if (screen.width() >= 1280){ + if (screen.width() >= 1280 && GlobalSettings::instance().OCR_LIBRARY == OcrLibrary::TESSERACT){ const int dilation_type = cv::MORPH_ELLIPSE; const int dilation_size = 1; From c908989d45bb999f7388c0d46d996557fa4ceb9f Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 26 Jul 2026 11:02:39 -0700 Subject: [PATCH 3/9] update paddleOCR --- .../Source/ML/Inference/ML_PaddleOCRPipeline.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 459941f34e..c0b094ba72 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -98,9 +98,12 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ cv::Mat cv_image_rgb = imageviewrgb32_to_cv_mat_rgb(image); // 1b. Whitespace Trimming Logic + // crop tightly around the text, with small safety margin cv::Mat gray; cv::cvtColor(cv_image_rgb, gray, cv::COLOR_BGR2GRAY); + // we are assuming that pre-processing already done on image so the text is black, + // and background is white. // Invert image for findNonZero: text becomes white (255), background black (0) cv::Mat binary_inv; cv::threshold(gray, binary_inv, 250, 255, cv::THRESH_BINARY_INV); @@ -115,9 +118,9 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ } cv::Rect bbox = cv::boundingRect(nonZeroCoords); - // Small safety margin (NOT 20 pixels) - int pad_x = std::max(1, bbox.width / 20); // ~5% - int pad_y = std::max(1, bbox.height / 20); // ~5% + // Small safety margin around the crop + int pad_x = std::max(4, bbox.height / 10); // ~10% of text height. use height to account for cases of a single narrow character + int pad_y = std::max(2, bbox.height / 20); // ~5% bbox.x = std::max(0, bbox.x - pad_x); bbox.y = std::max(0, bbox.y - pad_y); @@ -139,7 +142,8 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ constexpr float min_ratio = 0.5f; - if ((float)w / h < min_ratio) { // handle tall/narrow characters + // add more horizontal padding to tall/narrow characters + if ((float)w / h < min_ratio) { int target_w = static_cast(std::ceil(min_ratio * h)); int left = (target_w - w) / 2; From 6567f68934f4530d1d4926e80c3e45f2e95170f1 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 26 Jul 2026 11:06:03 -0700 Subject: [PATCH 4/9] undo sandwich changes --- .../Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp b/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp index ba1bd702d4..4127f5b8ad 100644 --- a/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp +++ b/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichRecipeDetector.cpp @@ -7,7 +7,6 @@ #include #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/Containers/FixedLimitVector.tpp" -#include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" @@ -57,7 +56,7 @@ void SandwichRecipeNumberDetector::detect_recipes(const ImageViewRGB32& screen, ImageRGB32 dilated_image(filterd_image.width(), filterd_image.height()); - if (screen.width() >= 1280 && GlobalSettings::instance().OCR_LIBRARY == OcrLibrary::TESSERACT){ + if (screen.width() >= 1280){ const int dilation_type = cv::MORPH_ELLIPSE; const int dilation_size = 1; From 841ce76b4555d549e056f2b8932592d13ce6d308 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 26 Jul 2026 18:40:12 -0700 Subject: [PATCH 5/9] update PaddleOCR cropping --- .../ML/Inference/ML_PaddleOCRPipeline.cpp | 68 +++++++++++++------ 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index c0b094ba72..13bf2aac48 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -15,6 +15,10 @@ #include "ML/Models/ML_ONNXRuntimeHelpers.h" #include "ML_PaddleOCRPipeline.h" +#include +using std::cout; +using std::endl; + namespace PokemonAutomation{ namespace ML{ @@ -96,30 +100,42 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ // 1. Convert Image to OpenCV image (cv::mat) cv::Mat cv_image_rgb = imageviewrgb32_to_cv_mat_rgb(image); + if (cv_image_rgb.empty()) { + return ""; + } + - // 1b. Whitespace Trimming Logic - // crop tightly around the text, with small safety margin + // 2. Crop tightly around the text, with small safety margin + // first convert to grayscale cv::Mat gray; cv::cvtColor(cv_image_rgb, gray, cv::COLOR_BGR2GRAY); - // we are assuming that pre-processing already done on image so the text is black, - // and background is white. - // Invert image for findNonZero: text becomes white (255), background black (0) - cv::Mat binary_inv; - cv::threshold(gray, binary_inv, 250, 255, cv::THRESH_BINARY_INV); + // get a binary image, for cropping purposes + cv::Mat binary; + cv::threshold(gray, binary, 0, 255, + cv::THRESH_BINARY_INV | cv::THRESH_OTSU); - // Find coordinates of all text pixels - std::vector nonZeroCoords; - cv::findNonZero(binary_inv, nonZeroCoords); + double ratio = cv::countNonZero(binary) / + static_cast(binary.total()); - cv::Mat cropped_image; + // If most pixels are white, we actually segmented the background. + // Flip it so text becomes white again. + if (ratio > 0.5){ + cv::bitwise_not(binary, binary); + } + + // Find coordinates of all non-zero pixels (the text) + std::vector nonZeroCoords; + cv::findNonZero(binary, nonZeroCoords); if (nonZeroCoords.empty()){ return ""; } + + // create bounding box for crop cv::Rect bbox = cv::boundingRect(nonZeroCoords); - // Small safety margin around the crop - int pad_x = std::max(4, bbox.height / 10); // ~10% of text height. use height to account for cases of a single narrow character + // increase bounding box slightly to add small safety margin + int pad_x = std::max(4, bbox.width / 20); // ~5% int pad_y = std::max(2, bbox.height / 20); // ~5% bbox.x = std::max(0, bbox.x - pad_x); @@ -135,6 +151,8 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ bbox.height + 2 * pad_y ); + // crop the original image based on the bounding box + cv::Mat cropped_image; cropped_image = cv_image_rgb(bbox).clone(); int h = cropped_image.rows; @@ -142,7 +160,7 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ constexpr float min_ratio = 0.5f; - // add more horizontal padding to tall/narrow characters + // add horizontal padding (white pixels) to tall/narrow characters if ((float)w / h < min_ratio) { int target_w = static_cast(std::ceil(min_ratio * h)); @@ -155,11 +173,17 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ 0, 0, // no vertical padding left, right, cv::BORDER_CONSTANT, - cv::Scalar(255,255,255) + cv::Scalar(255,255,255) // add white pixels ); } - // 2a. Calculate dynamic width (maintain aspect ratio) + // static int i = 0; + // i++; + // cv::imwrite("aabinary" + std::to_string(i) + ".png", binary); + // cv::imwrite("aacropped_image" + std::to_string(i) + ".png", cropped_image); + + + // 3. Calculate dynamic width (maintain aspect ratio) // the model shape is {1, 3, 48, dynamic_width}. Note that the height is fixed at 48 pixels // the input image must be scaled to match the height of 48, for the neural network int target_h = 48; @@ -181,13 +205,13 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ ); - // 3. Normalize + // 4. Normalize // convert UC3 8-bit [0,255] to 32FC3 float [0,1], then use ImageNet Normalization // output = (Input * Scale) = (old_pixel * 1/255). This transforms [0,255] to range [0, 1] // TODO: determine if normalizing to [-1,1] is preferred or to perform ImageNet normalization (mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225]) resized.convertTo(resized, CV_32FC3, 1.0 / 255.0); - // 3b. Apply Mean/Std (Standard for PaddleOCR). except for Chinese + // 4b. Apply Mean/Std (Standard for PaddleOCR). except for Chinese // Mean: [0.485, 0.456, 0.406], Std: [0.229, 0.224, 0.225] if (!(m_language == Language::ChineseSimplified || m_language == Language::ChineseTraditional || @@ -203,13 +227,13 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ } - // 3. Convert HWC to NCHW + // 5. Convert HWC to NCHW std::vector input_tensor_values = preprocess_NCHW(resized); - // 4. Define Dynamic Shape + // 6. Define Dynamic Shape std::vector input_shape = {1, 3, target_h, target_w}; - // 5. Create tensor with its own managed memory + // 7. Create tensor with its own managed memory Ort::AllocatorWithDefaultOptions allocator; auto input_tensor = Ort::Value::CreateTensor( allocator, input_shape.data(), input_shape.size() @@ -224,7 +248,7 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ const char* output_names[] = {m_output_name.c_str()}; try{ - // 7. Run the recognition session + // 8. Run the recognition session auto outputs = m_rec_session.Run( Ort::RunOptions{nullptr}, input_names, // char** From a5dc782a1acbc079ccd52081feaf4ca33dab5cd1 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 26 Jul 2026 19:11:08 -0700 Subject: [PATCH 6/9] refactor --- .../ML/Inference/ML_PaddleOCRPipeline.cpp | 162 ++++++++++-------- .../ML/Inference/ML_PaddleOCRPipeline.h | 6 + 2 files changed, 95 insertions(+), 73 deletions(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 13bf2aac48..5b173fcdcc 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -106,82 +106,13 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ // 2. Crop tightly around the text, with small safety margin - // first convert to grayscale - cv::Mat gray; - cv::cvtColor(cv_image_rgb, gray, cv::COLOR_BGR2GRAY); - - // get a binary image, for cropping purposes - cv::Mat binary; - cv::threshold(gray, binary, 0, 255, - cv::THRESH_BINARY_INV | cv::THRESH_OTSU); - - double ratio = cv::countNonZero(binary) / - static_cast(binary.total()); - - // If most pixels are white, we actually segmented the background. - // Flip it so text becomes white again. - if (ratio > 0.5){ - cv::bitwise_not(binary, binary); - } - - // Find coordinates of all non-zero pixels (the text) - std::vector nonZeroCoords; - cv::findNonZero(binary, nonZeroCoords); - if (nonZeroCoords.empty()){ + cv::Mat cropped_image = crop_to_text_region(cv_image_rgb); + if (cropped_image.empty()){ return ""; } - // create bounding box for crop - cv::Rect bbox = cv::boundingRect(nonZeroCoords); - - // increase bounding box slightly to add small safety margin - int pad_x = std::max(4, bbox.width / 20); // ~5% - int pad_y = std::max(2, bbox.height / 20); // ~5% - - bbox.x = std::max(0, bbox.x - pad_x); - bbox.y = std::max(0, bbox.y - pad_y); - - bbox.width = std::min( - cv_image_rgb.cols - bbox.x, - bbox.width + 2 * pad_x - ); - - bbox.height = std::min( - cv_image_rgb.rows - bbox.y, - bbox.height + 2 * pad_y - ); - - // crop the original image based on the bounding box - cv::Mat cropped_image; - cropped_image = cv_image_rgb(bbox).clone(); - - int h = cropped_image.rows; - int w = cropped_image.cols; - - constexpr float min_ratio = 0.5f; - - // add horizontal padding (white pixels) to tall/narrow characters - if ((float)w / h < min_ratio) { - int target_w = static_cast(std::ceil(min_ratio * h)); - - int left = (target_w - w) / 2; - int right = target_w - w - left; - - cv::copyMakeBorder( - cropped_image, - cropped_image, - 0, 0, // no vertical padding - left, right, - cv::BORDER_CONSTANT, - cv::Scalar(255,255,255) // add white pixels - ); - } - - // static int i = 0; - // i++; - // cv::imwrite("aabinary" + std::to_string(i) + ".png", binary); - // cv::imwrite("aacropped_image" + std::to_string(i) + ".png", cropped_image); - + // add horizontal padding to tall/narrow characters + add_horizontal_padding(cropped_image); // 3. Calculate dynamic width (maintain aspect ratio) // the model shape is {1, 3, 48, dynamic_width}. Note that the height is fixed at 48 pixels @@ -264,6 +195,91 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ } +cv::Mat crop_to_text_region(const cv::Mat& image) { + // first convert to grayscale + cv::Mat gray; + cv::cvtColor(image, gray, cv::COLOR_BGR2GRAY); + + // get a binary image, for cropping purposes + cv::Mat binary; + cv::threshold(gray, binary, 0, 255, + cv::THRESH_BINARY_INV | cv::THRESH_OTSU); + + double ratio = cv::countNonZero(binary) / + static_cast(binary.total()); + + // we want text pixels to be white for the next step + // If most pixels are white, Otsu likely classified the background as foreground. + // Flip it so text becomes white again. + if (ratio > 0.5){ + cv::bitwise_not(binary, binary); + } + + // Find coordinates of all non-zero pixels (the text) + std::vector nonZeroCoords; + cv::findNonZero(binary, nonZeroCoords); + if (nonZeroCoords.empty()){ + return {}; + } + + // create bounding box for crop + cv::Rect bbox = cv::boundingRect(nonZeroCoords); + + // increase bounding box slightly to add small safety margin + int pad_x = std::max(4, bbox.width / 20); // ~5% + int pad_y = std::max(2, bbox.height / 20); // ~5% + + bbox.x = std::max(0, bbox.x - pad_x); + bbox.y = std::max(0, bbox.y - pad_y); + + bbox.width = std::min( + image.cols - bbox.x, + bbox.width + 2 * pad_x + ); + + bbox.height = std::min( + image.rows - bbox.y, + bbox.height + 2 * pad_y + ); + + // crop the original image based on the bounding box + // the crop should be within bounds. + cv::Mat cropped_image; + cropped_image = image(bbox).clone(); + + // static int i = 0; + // i++; + // cv::imwrite("aabinary" + std::to_string(i) + ".png", binary); + // cv::imwrite("aacropped_image" + std::to_string(i) + ".png", cropped_image); + + return cropped_image; +} + +void add_horizontal_padding(cv::Mat& image){ + + int h = image.rows; + int w = image.cols; + constexpr float min_ratio = 0.5f; + + // add horizontal padding (white pixels) to tall/narrow characters + if (h > 0 && (float)w / h < min_ratio) { + int target_w = static_cast(std::ceil(min_ratio * h)); + + int left = (target_w - w) / 2; + int right = target_w - w - left; + + cv::copyMakeBorder( + image, + image, + 0, 0, // no vertical padding + left, right, + cv::BORDER_CONSTANT, + cv::Scalar(255,255,255) // add white pixels + ); + } + +} + std::vector preprocess_NCHW(cv::Mat& img){ std::vector dst(img.rows * img.cols * 3); diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h index 9fe2bbbdf2..90dc5daf83 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h @@ -48,6 +48,12 @@ class PaddleOCRPipeline { }; +cv::Mat crop_to_text_region(const cv::Mat& image); + +// if the image is narrow/tall, add horizontal padding +// modifies the input image +void add_horizontal_padding(cv::Mat& image); + // convert HCW (height, width, channels) to NCHW (batch N, channels C, height H, width W) std::vector preprocess_NCHW(cv::Mat& img); From 83660063c7969780c8371d665e1d1b5f3b02f971 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 26 Jul 2026 19:12:44 -0700 Subject: [PATCH 7/9] fix BGR to RGB --- SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp | 2 +- SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 5b173fcdcc..a00a46dade 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -198,7 +198,7 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ cv::Mat crop_to_text_region(const cv::Mat& image) { // first convert to grayscale cv::Mat gray; - cv::cvtColor(image, gray, cv::COLOR_BGR2GRAY); + cv::cvtColor(image, gray, cv::COLOR_RGB2GRAY); // get a binary image, for cropping purposes cv::Mat binary; diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h index 90dc5daf83..c6ab45bc78 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h @@ -48,6 +48,7 @@ class PaddleOCRPipeline { }; +// assumes the input image is RGB cv::Mat crop_to_text_region(const cv::Mat& image); // if the image is narrow/tall, add horizontal padding From c1c1e2c94c29d31c451b845857b6362633f3bb48 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 26 Jul 2026 19:52:16 -0700 Subject: [PATCH 8/9] update add_horizontal_padding() --- .../ML/Inference/ML_PaddleOCRPipeline.cpp | 43 +++++++++++++++++-- .../ML/Inference/ML_PaddleOCRPipeline.h | 4 ++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index a00a46dade..23f97a6e54 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -256,30 +256,67 @@ cv::Mat crop_to_text_region(const cv::Mat& image) { } void add_horizontal_padding(cv::Mat& image){ + if (image.empty()) { + return; + } int h = image.rows; int w = image.cols; constexpr float min_ratio = 0.5f; - // add horizontal padding (white pixels) to tall/narrow characters + // add horizontal padding to tall/narrow characters if (h > 0 && (float)w / h < min_ratio) { int target_w = static_cast(std::ceil(min_ratio * h)); int left = (target_w - w) / 2; int right = target_w - w - left; + cv::Scalar bg = estimate_background_color(image); + cv::Mat padded_image; cv::copyMakeBorder( image, - image, + padded_image, 0, 0, // no vertical padding left, right, cv::BORDER_CONSTANT, - cv::Scalar(255,255,255) // add white pixels + bg ); + image = padded_image; } + // static int i = 0; + // i++; + // cv::imwrite("aapadded" + std::to_string(i) + ".png", image); + } +cv::Scalar estimate_background_color(const cv::Mat& image) { + if (image.empty() || image.type() != CV_8UC3) { + return cv::Scalar(255, 255, 255); + } + + cv::Vec3b tl = image.at(0, 0); + cv::Vec3b tr = image.at(0, image.cols - 1); + cv::Vec3b bl = image.at(image.rows - 1, 0); + cv::Vec3b br = image.at(image.rows - 1, image.cols - 1); + + // optimization due to the fact that most input images have a white background + if (tl[0] >= 240 && tl[1] >= 240 && tl[2] >= 240 && + tr[0] >= 240 && tr[1] >= 240 && tr[2] >= 240 && + bl[0] >= 240 && bl[1] >= 240 && bl[2] >= 240 && + br[0] >= 240 && br[1] >= 240 && br[2] >= 240) { + return cv::Scalar(255, 255, 255); + } + + return cv::Scalar( + (static_cast(tl[0]) + tr[0] + bl[0] + br[0]) * 0.25, // Blue (or Red if RGB). cast to int to avoid overflow + (static_cast(tl[1]) + tr[1] + bl[1] + br[1]) * 0.25, // Green + (static_cast(tl[2]) + tr[2] + bl[2] + br[2]) * 0.25 // Red (or Blue if RGB) + ); +} + + + std::vector preprocess_NCHW(cv::Mat& img){ std::vector dst(img.rows * img.cols * 3); diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h index c6ab45bc78..3d7601ecf2 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h @@ -53,8 +53,12 @@ cv::Mat crop_to_text_region(const cv::Mat& image); // if the image is narrow/tall, add horizontal padding // modifies the input image +// assumes input image is RGB void add_horizontal_padding(cv::Mat& image); +// assumes input image is RGB +cv::Scalar estimate_background_color(const cv::Mat& image); + // convert HCW (height, width, channels) to NCHW (batch N, channels C, height H, width W) std::vector preprocess_NCHW(cv::Mat& img); From 29fc56f8bd69c4de75161cea0f664bdd805839e9 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 26 Jul 2026 20:16:59 -0700 Subject: [PATCH 9/9] update estimate_background_color --- .../ML/Inference/ML_PaddleOCRPipeline.cpp | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 23f97a6e54..aa83363958 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -295,24 +295,16 @@ cv::Scalar estimate_background_color(const cv::Mat& image) { return cv::Scalar(255, 255, 255); } - cv::Vec3b tl = image.at(0, 0); - cv::Vec3b tr = image.at(0, image.cols - 1); - cv::Vec3b bl = image.at(image.rows - 1, 0); - cv::Vec3b br = image.at(image.rows - 1, image.cols - 1); - - // optimization due to the fact that most input images have a white background - if (tl[0] >= 240 && tl[1] >= 240 && tl[2] >= 240 && - tr[0] >= 240 && tr[1] >= 240 && tr[2] >= 240 && - bl[0] >= 240 && bl[1] >= 240 && bl[2] >= 240 && - br[0] >= 240 && br[1] >= 240 && br[2] >= 240) { - return cv::Scalar(255, 255, 255); - } + // use the corner pixels to estimate the background color + const cv::Vec3b* top = image.ptr(0); + const cv::Vec3b* bottom = image.ptr(image.rows - 1); - return cv::Scalar( - (static_cast(tl[0]) + tr[0] + bl[0] + br[0]) * 0.25, // Blue (or Red if RGB). cast to int to avoid overflow - (static_cast(tl[1]) + tr[1] + bl[1] + br[1]) * 0.25, // Green - (static_cast(tl[2]) + tr[2] + bl[2] + br[2]) * 0.25 // Red (or Blue if RGB) - ); + cv::Scalar tl(top[0]); + cv::Scalar tr(top[image.cols - 1]); + cv::Scalar bl(bottom[0]); + cv::Scalar br(bottom[image.cols - 1]); + + return (tl + tr + bl + br) * 0.25; }