feat: features2d (#66)

fix #56
MaaXYZ · Oct 13, 2023 · 9375442 · 9375442
1 parent 8f1e674
commit 9375442
Show file tree

Hide file tree

Showing 16 changed files with 437 additions and 13 deletions.
diff --git a/MaaDeps b/MaaDeps
diff --git a/docs/zh_cn/3.1-任务流水线协议.md b/docs/zh_cn/3.1-任务流水线协议.md
@@ -238,6 +238,33 @@ graph LR;
     是否进行绿色掩码。可选，默认 false。  
     若为 true，可以将图片中不希望匹配的部分涂绿 RGB: (0, 255, 0)，则不对绿色部分进行匹配。
 
+### `FeatureMatch`
+
+特征匹配，泛化能力更强的“找图”，具有抗透视、抗尺寸变化等特点。  
+
+该任务属性需额外部分字段：
+
+- `roi`: *array<int, 4>* | *list<array<int, 4>>*  
+    识别区域坐标。可选，默认 [0, 0, 0, 0]，即全屏。  
+    四个值分别为 [x, y, w, h]。
+
+- `template`: *string*  
+    模板图片路径，需要 `image` 文件夹的相对路径。必选。  
+    目前仅支持单张图片。
+
+- `count`: *int*  
+    匹配的特征点数量，默认 4.
+
+- `green_mask`: *bool*  
+    是否进行绿色掩码。可选，默认 false。  
+    若为 true，可以将图片中不希望匹配的部分涂绿 RGB: (0, 255, 0)，则不对绿色部分进行匹配。
+
+- `hessian`: *int*  
+    SURF 检测算法的 hessian 阈值，越大特征点的数量将越多。可选，默认 100。
+
+- `ratio`: *double*  
+    KNN 匹配算法的距离比值，[0 - 1.0], 越大则匹配越宽松（更容易连线）。可选，默认 0.6。
+
 ### `ColorMatch`
 
 颜色匹配，即“找色”。  
@@ -328,7 +355,7 @@ graph LR;
 
 ### `NeuralNetworkDetect`
 
-深度学习检测，泛化版“找图”。  
+深度学习目标检测，高级版“找图”。  
 
 与分类器主要区别在于“找”，即支持任意位置。但通常来说模型复杂度会更高，需要更多的训练集、训练时间，使用时的资源占用（推理开销）也会成倍上涨。  
 

diff --git a/sample/cpp/main.cpp b/sample/cpp/main.cpp
@@ -51,7 +51,7 @@ int main([[maybe_unused]] int argc, char** argv)
 
     register_my_recognizer(maa_handle);
 
-    auto task_id = MaaPostTask(maa_handle, "MyTask", MaaTaskParam_Empty);
+    auto task_id = MaaPostTask(maa_handle, "VisonTest", MaaTaskParam_Empty);
     MaaWaitTask(maa_handle, task_id);
 
     MaaDestroy(maa_handle);

diff --git a/source/MaaFramework/MaaFramework.vcxproj b/source/MaaFramework/MaaFramework.vcxproj
@@ -71,6 +71,7 @@
     <ClInclude Include="Utils\StringMisc.hpp" />
     <ClInclude Include="Utils\TempPath.hpp" />
     <ClInclude Include="Utils\Time.hpp" />
+    <ClInclude Include="Vision\FeatureMatcher.h" />
     <ClInclude Include="Vision\NeuralNetworkClassifier.h" />
     <ClInclude Include="Vision\ColorMatcher.h" />
     <ClInclude Include="Vision\TemplateComparator.h" />
@@ -107,6 +108,7 @@
     <ClCompile Include="Task\Recognizer.cpp" />
     <ClCompile Include="Task\SyncContext.cpp" />
     <ClCompile Include="Task\PipelineTask.cpp" />
+    <ClCompile Include="Vision\FeatureMatcher.cpp" />
     <ClCompile Include="Vision\NeuralNetworkClassifier.cpp" />
     <ClCompile Include="Vision\ColorMatcher.cpp" />
     <ClCompile Include="Vision\TemplateComparator.cpp" />

diff --git a/source/MaaFramework/Resource/PipelineResMgr.cpp b/source/MaaFramework/Resource/PipelineResMgr.cpp
@@ -340,6 +340,7 @@ bool PipelineResMgr::parse_recognition(const json::value& input, Recognition::Ty
         { kDefaultRecognitionFlag, default_type },
         { "DirectHit", Type::DirectHit },
         { "TemplateMatch", Type::TemplateMatch },
+        { "FeatureMatch", Type::FeatureMatch },
         { "OCR", Type::OCR },
         { "NeuralNetworkClassify", Type::NeuralNetworkClassify },
         { "NeuralNetworkDetect", Type::NeuralNetworkDetect },
@@ -367,6 +368,12 @@ bool PipelineResMgr::parse_recognition(const json::value& input, Recognition::Ty
                                             same_type ? std::get<TemplateMatcherParam>(default_param)
                                                       : TemplateMatcherParam {});
 
+    case Type::FeatureMatch:
+        out_param = FeatureMatcherParam {};
+        return parse_feature_matcher_param(input, std::get<FeatureMatcherParam>(out_param),
+                                           same_type ? std::get<FeatureMatcherParam>(default_param)
+                                                     : FeatureMatcherParam {});
+
     case Type::NeuralNetworkClassify:
         out_param = NeuralNetworkClassifierParam {};
         return parse_nn_classifier_param(input, std::get<NeuralNetworkClassifierParam>(out_param),
@@ -457,6 +464,42 @@ bool PipelineResMgr::parse_template_matcher_param(const json::value& input, MAA_
     return true;
 }
 
+bool PipelineResMgr::parse_feature_matcher_param(const json::value& input, MAA_VISION_NS::FeatureMatcherParam& output,
+                                                 const MAA_VISION_NS::FeatureMatcherParam& default_value)
+{
+    if (!parse_roi(input, output.roi, default_value.roi)) {
+        LogError << "failed to parse_roi" << VAR(input);
+        return false;
+    }
+
+    if (!get_and_check_value(input, "template", output.template_path, default_value.template_path)) {
+        LogError << "failed to get_and_check_value template_path" << VAR(input);
+        return false;
+    }
+
+    if (!get_and_check_value(input, "green_mask", output.green_mask, default_value.green_mask)) {
+        LogError << "failed to get_and_check_value green_mask" << VAR(input);
+        return false;
+    }
+
+    if (!get_and_check_value(input, "hessian", output.hessian, default_value.hessian)) {
+        LogError << "failed to get_and_check_value hessian" << VAR(input);
+        return false;
+    }
+
+    if (!get_and_check_value(input, "distance_ratio", output.distance_ratio, default_value.distance_ratio)) {
+        LogError << "failed to get_and_check_value hessian" << VAR(input);
+        return false;
+    }
+
+    if (!get_and_check_value(input, "count", output.count, default_value.count)) {
+        LogError << "failed to get_and_check_value hessian" << VAR(input);
+        return false;
+    }
+
+    return true;
+}
+
 bool PipelineResMgr::parse_ocrer_param(const json::value& input, MAA_VISION_NS::OCRerParam& output,
                                        const MAA_VISION_NS::OCRerParam& default_value)
 {

diff --git a/source/MaaFramework/Resource/PipelineResMgr.h b/source/MaaFramework/Resource/PipelineResMgr.h
@@ -39,6 +39,8 @@ class PipelineResMgr : public NonCopyable
     //                                    const MAA_VISION_NS::DirectHitParam& default_value);
     static bool parse_template_matcher_param(const json::value& input, MAA_VISION_NS::TemplateMatcherParam& output,
                                              const MAA_VISION_NS::TemplateMatcherParam& default_value);
+    static bool parse_feature_matcher_param(const json::value& input, MAA_VISION_NS::FeatureMatcherParam& output,
+                                            const MAA_VISION_NS::FeatureMatcherParam& default_value);
     static bool parse_ocrer_param(const json::value& input, MAA_VISION_NS::OCRerParam& output,
                                   const MAA_VISION_NS::OCRerParam& default_value);
     static bool parse_custom_recognizer_param(const json::value& input, MAA_VISION_NS::CustomRecognizerParam& output,

diff --git a/source/MaaFramework/Resource/PipelineTypes.h b/source/MaaFramework/Resource/PipelineTypes.h
@@ -23,6 +23,7 @@ enum class Type
     Invalid = 0,
     DirectHit,
     TemplateMatch,
+    FeatureMatch,
     OCR,
     NeuralNetworkClassify,
     NeuralNetworkDetect,
@@ -31,9 +32,9 @@ enum class Type
 };
 
 using Param = std::variant<std::monostate, MAA_VISION_NS::DirectHitParam, MAA_VISION_NS::TemplateMatcherParam,
-                           MAA_VISION_NS::OCRerParam, MAA_VISION_NS::NeuralNetworkClassifierParam,
-                           MAA_VISION_NS::NeuralNetworkDetectorParam, MAA_VISION_NS::ColorMatcherParam,
-                           MAA_VISION_NS::CustomRecognizerParam>;
+                           MAA_VISION_NS::FeatureMatcherParam, MAA_VISION_NS::OCRerParam,
+                           MAA_VISION_NS::NeuralNetworkClassifierParam, MAA_VISION_NS::NeuralNetworkDetectorParam,
+                           MAA_VISION_NS::ColorMatcherParam, MAA_VISION_NS::CustomRecognizerParam>;
 } // namespace Recognition
 
 namespace Action

diff --git a/source/MaaFramework/Task/Recognizer.cpp b/source/MaaFramework/Task/Recognizer.cpp
@@ -5,6 +5,7 @@
 #include "Utils/Logger.h"
 #include "Vision/ColorMatcher.h"
 #include "Vision/CustomRecognizer.h"
+#include "Vision/FeatureMatcher.h"
 #include "Vision/NeuralNetworkClassifier.h"
 #include "Vision/NeuralNetworkDetector.h"
 #include "Vision/OCRer.h"
@@ -35,6 +36,10 @@ std::optional<Recognizer::Result> Recognizer::recognize(const cv::Mat& image, co
         result = template_match(image, std::get<TemplateMatcherParam>(task_data.rec_param), task_data.name);
         break;
 
+    case Type::FeatureMatch:
+        result = feature_match(image, std::get<FeatureMatcherParam>(task_data.rec_param), task_data.name);
+        break;
+
     case Type::ColorMatch:
         result = color_match(image, std::get<ColorMatcherParam>(task_data.rec_param), task_data.name);
         break;
@@ -117,6 +122,38 @@ std::optional<Recognizer::Result> Recognizer::template_match(const cv::Mat& imag
     return Result { .box = box, .detail = detail.to_string() };
 }
 
+std::optional<Recognizer::Result> Recognizer::feature_match(const cv::Mat& image,
+                                                            const MAA_VISION_NS::FeatureMatcherParam& param,
+                                                            const std::string& name)
+{
+    using namespace MAA_VISION_NS;
+
+    if (!resource()) {
+        LogError << "Resource not binded";
+        return std::nullopt;
+    }
+
+    FeatureMatcher matcher;
+    matcher.set_image(image);
+    matcher.set_name(name);
+    matcher.set_param(param);
+
+    std::shared_ptr<cv::Mat> templ = resource()->template_res().image(param.template_path);
+    matcher.set_template(std::move(templ));
+
+    auto ret = matcher.analyze();
+    if (ret.empty()) {
+        return std::nullopt;
+    }
+
+    const cv::Rect& box = ret.front().box;
+    json::array detail;
+    for (const auto& res : ret) {
+        detail.emplace_back(res.to_json());
+    }
+    return Result { .box = box, .detail = detail.to_string() };
+}
+
 std::optional<Recognizer::Result> Recognizer::color_match(const cv::Mat& image,
                                                           const MAA_VISION_NS::ColorMatcherParam& param,
                                                           const std::string& name)

diff --git a/source/MaaFramework/Task/Recognizer.h b/source/MaaFramework/Task/Recognizer.h
@@ -35,6 +35,8 @@ class Recognizer
     std::optional<Result> direct_hit();
     std::optional<Result> template_match(const cv::Mat& image, const MAA_VISION_NS::TemplateMatcherParam& param,
                                          const std::string& name);
+    std::optional<Result> feature_match(const cv::Mat& image, const MAA_VISION_NS::FeatureMatcherParam& param,
+                                        const std::string& name);
     std::optional<Result> color_match(const cv::Mat& image, const MAA_VISION_NS::ColorMatcherParam& param,
                                       const std::string& name);
     std::optional<Result> ocr(const cv::Mat& image, const MAA_VISION_NS::OCRerParam& param, const std::string& name);