diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index c969934ca7..883de554ad 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -14,6 +14,7 @@ #include "Common/Cpp/Containers/FixedLimitVector.tpp" #include "Common/Cpp/Concurrency/BusyPeriodicRunner.h" #include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/GlobalSettingsPanel.h" #include "CommonTools/Async/InferenceRoutines.h" #include "PokemonLA/Inference/PokemonLA_MountDetector.h" #include "Pokemon/Pokemon_Strings.h" @@ -173,11 +174,12 @@ #include "PokemonFRLG/Inference/PokemonFRLG_BattleSelectionArrowDetector.h" #include "Controllers/RumbleListener.h" #include "PokemonSwSh/Inference/PokemonSwSh_SelectionArrowFinder.h" +#include "PokemonLZA/Inference/PokemonLZA_WeatherDetector.h" +#include "PokemonLZA/Inference/PokemonLZA_WeatherDetector.h" #include "PokemonSwSh/Inference/PokemonSwSh_MainMenuDetector.h" #include "PokemonSwSh/Programs/PokemonSwSh_MenuNavigation.h" - #include #include @@ -334,6 +336,69 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& VideoOverlaySet overlays(overlay); +#if 0 + YCommIconDetector detector(COLOR_RED, true); + detector.make_overlays(overlays); + + auto snapshot = feed.snapshot(); + cout << detector.detect(snapshot) << endl; +#endif + auto snapshot = feed.snapshot(); +#if 1 + + using namespace PokemonLZA; + + const bool old_image_template_matching = PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING; + PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = true; + cout << "Weather detector waterfill debug enabled (dump + stats)." << endl; + + const struct { + WeatherIconType type; + const char* name; + } weather[] = { + {WeatherIconType::Clear, "Clear"}, + {WeatherIconType::Sunny, "Sunny"}, + {WeatherIconType::Rain, "Rain"}, + {WeatherIconType::Cloudy, "Cloudy"}, + {WeatherIconType::Foggy, "Foggy"}, + {WeatherIconType::Rainbow, "Rainbow"}, + }; + bool weather_detected[sizeof(weather)/sizeof(weather[0])] = {}; + + for (size_t i = 0; i < sizeof(weather)/sizeof(weather[0]); i++) { + const auto& entry = weather[i]; + WeatherIconDetector detector(entry.type, &overlay); + + detector.make_overlays(overlays); + + bool detected = detector.detect(snapshot); + weather_detected[i] = detected; + + cout << entry.name + << ": " + << (detected ? "MATCH" : "NO MATCH") + << endl; + } + + PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = old_image_template_matching; + cout << "Weather detector waterfill debug restored." << endl; + + cout << endl; + cout << "Weather Summary:" << endl; + for (size_t i = 0; i < sizeof(weather)/sizeof(weather[0]); i++) { + cout << weather[i].name + << ": " + << (weather_detected[i] ? "MATCH" : "NO MATCH") + << endl; + } + + scope.wait_for(std::chrono::seconds(30)); +#endif + + + +// context->issue_gyro_accel_x(&scope, 1000ms, 1000ms, 0ms, 123); + // SinglesAIOption ai(false); // run_singles_battle(console, context, ai, false); diff --git a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp index 62da2d5227..8066b3103d 100644 --- a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp +++ b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp @@ -1,83 +1,119 @@ #include "CommonFramework/Globals.h" #include "CommonFramework/ImageTools/ImageDiff.h" -#include "CommonTools/Images/ImageFilter.h" +#include "CommonTools/Images/WaterfillUtilities.h" +#include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h" #include "PokemonLZA_WeatherDetector.h" +#include +#include +#include +#include namespace PokemonAutomation { namespace NintendoSwitch { namespace PokemonLZA { +namespace{ +const ImageFloatBox WEATHER_ICON_ROI(0.880000, 0.010000, 0.035800, 0.068000); -//----------------------------------------------------- -// Weather Info Table (two images per weather) -//----------------------------------------------------- +struct SupplementalTemplateCheck{ + const char* path; + ImageFloatBox box; + Color color; + double rmsd_threshold; +}; -static const WeatherTemplateInfo WEATHER_TABLE[] = { +const std::vector& supplemental_template_checks(WeatherIconType type){ + static const std::vector NONE = {}; + + static const std::vector RAIN = { + {"PokemonLZA/Weather/rain_cloud.png", ImageFloatBox(0.8865, 0.0265, 0.0210, 0.0270), COLOR_GREEN, 90.0}, + {"PokemonLZA/Weather/rain_drop.png", ImageFloatBox(0.8885, 0.0525, 0.0050, 0.0150), COLOR_BLUE, 90.0}, + }; + static const std::vector CLOUDY = { + {"PokemonLZA/Weather/cloudy_cloud.png", ImageFloatBox(0.8895, 0.0430, 0.0210, 0.0265), COLOR_GREEN, 90.0}, + {"PokemonLZA/Weather/cloudy_drop.png", ImageFloatBox(0.8915, 0.0255, 0.0050, 0.0100), COLOR_BLUE, 90.0}, + }; + static const std::vector RAINBOW = { + {"PokemonLZA/Weather/rainbow_cloud.png", ImageFloatBox(0.8840, 0.0465, 0.0140, 0.0165), COLOR_GREEN, 90.0}, + {"PokemonLZA/Weather/rainbow_arch.png", ImageFloatBox(0.8930, 0.0420, 0.0120, 0.0100), COLOR_BLUE, 90.0}, + }; + static const std::vector FOGGY = { + {"PokemonLZA/Weather/foggy_tray_1.png", ImageFloatBox(0.8893, 0.0487, 0.0218, 0.0080), COLOR_GREEN, 90.0}, + {"PokemonLZA/Weather/foggy_tray_2.png", ImageFloatBox(0.8880, 0.0555, 0.0225, 0.0080), COLOR_BLUE, 90.0}, + }; + + switch (type){ + case WeatherIconType::Rain: + return RAIN; + case WeatherIconType::Cloudy: + return CLOUDY; + case WeatherIconType::Rainbow: + return RAINBOW; + case WeatherIconType::Foggy: + return FOGGY; + default: + return NONE; + } +} - // Clear - { - "PokemonLZA/Weather/sun_drop.png", - "PokemonLZA/Weather/clear_core.png", - ImageFloatBox(0.8945, 0.0570, 0.0050, 0.0100), - ImageFloatBox(0.8910, 0.0360, 0.0110, 0.0210), - COLOR_RED, - COLOR_GREEN - }, - - // Sunny - { - "PokemonLZA/Weather/sun_drop.png", - "PokemonLZA/Weather/sunny_core.png", - ImageFloatBox(0.8945, 0.0570, 0.0050, 0.0100), - ImageFloatBox(0.8910, 0.0360, 0.0110, 0.0210), - COLOR_RED, - COLOR_GREEN - }, - - // Rain +class WeatherFullMatcher : public ImageMatch::WaterfillTemplateMatcher{ +public: + WeatherFullMatcher(const char* path, double max_rmsd) + : WaterfillTemplateMatcher(path, Color(0xff707070), Color(0xffffffff), 50) + , m_max_rmsd(max_rmsd) { - "PokemonLZA/Weather/rain_cloud.png", - "PokemonLZA/Weather/rain_drop.png", - ImageFloatBox(0.8865, 0.0265, 0.0210, 0.0270), - ImageFloatBox(0.8885, 0.0525, 0.0050, 0.0150), - COLOR_RED, - COLOR_GREEN - }, - - // Cloudy - { - "PokemonLZA/Weather/cloudy_cloud.png", - "PokemonLZA/Weather/cloudy_drop.png", - ImageFloatBox(0.8895, 0.0430, 0.0210, 0.0265), - ImageFloatBox(0.8915, 0.0255, 0.0050, 0.0100), - COLOR_RED, - COLOR_GREEN - }, - - // Foggy - { - "PokemonLZA/Weather/foggy_tray_1.png", - "PokemonLZA/Weather/foggy_tray_2.png", - ImageFloatBox(0.8893, 0.0487, 0.0218, 0.0080), - ImageFloatBox(0.8880, 0.0555, 0.0225, 0.0080), - COLOR_RED, - COLOR_GREEN - }, - - // Rainbow - { - "PokemonLZA/Weather/rainbow_cloud.png", - "PokemonLZA/Weather/rainbow_arch.png", - ImageFloatBox(0.8840, 0.0465, 0.0140, 0.0165), - ImageFloatBox(0.8930, 0.0420, 0.0120, 0.0100), - COLOR_RED, - COLOR_GREEN - }, -}; + m_aspect_ratio_lower = 0.60; + m_aspect_ratio_upper = 1.40; + m_area_ratio_lower = 0.55; + m_area_ratio_upper = 1.45; + } + + static const WeatherFullMatcher& clear(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/clear_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& sunny(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/sunny_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& rain(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/rain_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& cloudy(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/cloudy_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& foggy(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/foggy_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& rainbow(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/rainbow_full.png", 100.0); + return matcher; + } + double m_max_rmsd; +}; -const WeatherTemplateInfo& weather_template_info(WeatherIconType icon){ - return WEATHER_TABLE[(int)icon]; +const WeatherFullMatcher& weather_full_matcher(WeatherIconType type){ + switch (type){ + case WeatherIconType::Clear: + return WeatherFullMatcher::clear(); + case WeatherIconType::Sunny: + return WeatherFullMatcher::sunny(); + case WeatherIconType::Rain: + return WeatherFullMatcher::rain(); + case WeatherIconType::Cloudy: + return WeatherFullMatcher::cloudy(); + case WeatherIconType::Foggy: + return WeatherFullMatcher::foggy(); + case WeatherIconType::Rainbow: + return WeatherFullMatcher::rainbow(); + default: + throw std::runtime_error("No weather full matcher for requested WeatherIconType"); + } +} } //----------------------------------------------------- @@ -86,37 +122,83 @@ const WeatherTemplateInfo& weather_template_info(WeatherIconType icon){ WeatherIconDetector::WeatherIconDetector(WeatherIconType type, VideoOverlay* overlay) { - m_info = &weather_template_info(type); - + m_type = type; if (overlay){ - m_overlay1.emplace(*overlay, m_info->box1, m_info->color1); - m_overlay2.emplace(*overlay, m_info->box2, m_info->color2); + m_overlay1.emplace(*overlay, WEATHER_ICON_ROI, COLOR_RED); } } void WeatherIconDetector::make_overlays(VideoOverlaySet& items) const { - items.add(m_info->color1, m_info->box1); - items.add(m_info->color2, m_info->box2); + items.add(COLOR_RED, WEATHER_ICON_ROI); + for (const auto& check : supplemental_template_checks(m_type)){ + items.add(check.color, check.box); + } } bool WeatherIconDetector::detect(const ImageViewRGB32& screen){ - - // Extract screen patches - ImageRGB32 c1 = extract_box_reference(screen, m_info->box1).copy(); - ImageRGB32 c2 = extract_box_reference(screen, m_info->box2).copy(); - - // Load templates - ImageRGB32 t1(RESOURCE_PATH() + m_info->path1); - ImageRGB32 t2(RESOURCE_PATH() + m_info->path2); - - if (t1.width() == 0 || t2.width() == 0){ + const WeatherFullMatcher& matcher = weather_full_matcher(m_type); + + const double scale = screen.height() / 1080.0; + const size_t min_area = (size_t)(scale * scale * 120.0); + + static const std::vector> FILTERS = { + {0xff707070, 0xffffffff}, + }; + + const bool full_match = match_template_by_waterfill( + screen.size(), + extract_box_reference(screen, WEATHER_ICON_ROI), + matcher, + FILTERS, + {min_area, SIZE_MAX}, + matcher.m_max_rmsd, + [](Kernels::Waterfill::WaterfillObject& object) -> bool { + (void)object; + return true; + } + ); + + if (!full_match){ return false; } - double rms1 = ImageMatch::pixel_RMSD(c1, t1); - double rms2 = ImageMatch::pixel_RMSD(c2, t2); + for (const auto& check : supplemental_template_checks(m_type)){ + ImageViewRGB32 candidate = extract_box_reference(screen, check.box); + ImageRGB32 templ(RESOURCE_PATH() + check.path); + if (templ.width() == 0 || templ.height() == 0){ + return false; + } + + auto compute_rmsd = [&](const ImageViewRGB32& image) -> double{ + return image.width() == templ.width() && image.height() == templ.height() + ? ImageMatch::pixel_RMSD(image, templ) + : ImageMatch::pixel_RMSD(image, templ.scale_to(image.width(), image.height())); + }; + + double rmsd = compute_rmsd(candidate); + + if (screen.height() < 1080){ + const int search_radius = std::min(candidate.width(), candidate.height()) <= 12 ? 2 : 1; + for (int dy = -search_radius; dy <= search_radius; dy++){ + for (int dx = -search_radius; dx <= search_radius; dx++){ + if (dx == 0 && dy == 0){ + continue; + } + ImageViewRGB32 shifted = extract_box_reference(screen, check.box, dx, dy); + if (shifted.width() != candidate.width() || shifted.height() != candidate.height()){ + continue; + } + rmsd = std::min(rmsd, compute_rmsd(shifted)); + } + } + } + + if (rmsd >= check.rmsd_threshold){ + return false; + } + } - return rms1 < 90 && rms2 < 90; + return true; } diff --git a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h index 0d9bd10e89..30c9a8826c 100644 --- a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h +++ b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h @@ -2,11 +2,8 @@ #define PokemonAutomation_PokemonLZA_WeatherDetector_H #include -#include "Common/Cpp/Color.h" -#include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "CommonTools/VisualDetector.h" -#include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h" namespace PokemonAutomation { namespace NintendoSwitch { @@ -23,23 +20,6 @@ enum class WeatherIconType { Unknown, }; - -//----------------------------------------------------- -// Template Info Struct -//----------------------------------------------------- -struct WeatherTemplateInfo{ - const char* path1; // template for box1 - const char* path2; // template for box2 - ImageFloatBox box1; - ImageFloatBox box2; - Color color1; - Color color2; -}; - - -const WeatherTemplateInfo& weather_template_info(WeatherIconType icon); - - //----------------------------------------------------- // Detector //----------------------------------------------------- @@ -52,10 +32,9 @@ class WeatherIconDetector : public StaticScreenDetector { virtual bool detect(const ImageViewRGB32& screen) override; private: - const WeatherTemplateInfo* m_info; + WeatherIconType m_type; std::optional m_overlay1; - std::optional m_overlay2; };