Merge pull request #1 from khoben/dev

Merge dev branch into master

.gitignore

@@ -2,5 +2,4 @@
 .idea
 bin/
 lib/
-cmake-build-debug/
-cmake-build-release/
+cmake-build-*/

CMakeLists.txt

@@ -8,8 +8,6 @@ file(GLOB_RECURSE SOURCE_FILES
         "src/*.cpp"
         )
 
-list(FILTER SOURCE_FILES EXCLUDE REGEX ".*android\\.[cpp|hpp]")
-
 set(OpenCV_DIR "D:\\opencv-4.1.1\\build\\install")
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
 

src/AR/Ar.cpp

@@ -4,7 +4,7 @@ AR::~AR() = default;
 
 AR::AR() {
     recognitionInstance = new Recognition();
-    trackingInstance = Tracking::create();
+    trackingInstance = new Tracking();
 }
 
 
@@ -13,24 +13,18 @@ std::vector<QueryItem> AR::process(cv::Mat frame) {
 
 }
 
-int AR::addAndCreate(std::vector<cv::Mat> imgs) {
-    recognitionInstance->addAndCreateBagOfVisualWords(imgs);
+int AR::addAll(std::vector<cv::Mat> imgs) {
     for (const auto &img: imgs) {
         recognitionInstance->addTrackImage(img);
     }
     return 0;
 }
 
 int AR::add(cv::Mat img) {
-    recognitionInstance->addVisualWord(img);
     recognitionInstance->addTrackImage(img);
     return 0;
 }
 
-int AR::create() {
-    recognitionInstance->createBagOfVisualWords();
-    return 0;
-}
 
 bool AR::keepTracking(const cv::Mat &frame) {
     return trackingInstance->keepTracking(frame);

src/AR/Ar.hpp

@@ -51,12 +51,12 @@ class AR {
     Tracking *getTrackingInstance() { return trackingInstance; }
 
     /**
-     * @brief Add and create marker image objects
+     * @brief Add marker image objects
      * 
      * @param imgs marker images
      * @return int 
      */
-    int addAndCreate(std::vector<cv::Mat> imgs);
+    int addAll(std::vector<cv::Mat> imgs);
 
     /**
      * @brief Add marker image
@@ -66,13 +66,6 @@ class AR {
      */
     int add(cv::Mat img);
 
-    /**
-     * @brief Create tracking markers
-     * 
-     * @return int 
-     */
-    int create();
-
     ~AR();
 };
 

src/Recognition/MarkerlessDB.cpp

@@ -0,0 +1,126 @@
+#include "MarkerlessDB.hpp"
+
+MarkerlessDB::MarkerlessDB() {
+    featureDetector = cv::AKAZE::create();
+    (featureDetector.dynamicCast<cv::AKAZE>())->setThreshold(3e-3);
+    descriptorMatcher = new cv::BFMatcher(cv::NORM_HAMMING);
+    markerAmount = 0;
+    featureAmount = 0;
+    RADIUS = 0.8f;
+}
+
+void MarkerlessDB::addFeatures(const cv::Mat &feature) {
+    descriptorMatcher->add(std::vector<cv::Mat>{feature});
+}
+
+MarkerlessDB::~MarkerlessDB() {
+}
+
+
+int MarkerlessDB::size() const {
+    int num = 0;
+    std::for_each(markers.begin(), markers.end(),
+                  [&num](MarkerlessTrackable *e) {
+                      num += e->numFeatures;
+                  }
+    );
+    return num;
+}
+
+void MarkerlessDB::add(const cv::Mat &image) {
+    std::vector<cv::KeyPoint> keyPoints;
+    cv::Mat descriptor;
+    extractFeatures(image, keyPoints, descriptor);
+    addFeatures(descriptor);
+    int id = markerAmount;
+    markers.push_back(new MarkerlessTrackable(id, descriptor, keyPoints, image.size()));
+    markerAmount++;
+}
+
+void MarkerlessDB::extractFeatures(const cv::Mat &img, std::vector<cv::KeyPoint> &keyPoints, cv::Mat &descriptor) {
+    featureDetector->detectAndCompute(img, cv::noArray(), keyPoints, descriptor);
+}
+
+std::vector<QueryItem>
+MarkerlessDB::match(cv::Mat queryImage, int minNumMatch, float minProbability) {
+    std::vector<QueryItem> results;
+    QueryItem result;
+    // Extract features from query image
+    cv::Mat descriptors;
+    std::vector<cv::KeyPoint> keyPoints;
+    extractFeatures(queryImage, keyPoints, descriptors);
+    // Search matches
+    if (markerAmount < 1)
+        return results;
+
+    int size = descriptors.size().height; // number of query image features
+
+    std::vector<std::vector<std::pair<int, int>>> descriptorMatches(markerAmount);
+
+    // iterate over all available markers
+    // and find matches
+    for (auto marker: markers) {
+        std::vector<std::vector<cv::DMatch>> matchId;
+        descriptorMatcher->knnMatch(descriptors, marker->descriptors, matchId, KNN_SIZE);
+        for (int i = 0; i < size; ++i) {
+            cv::DMatch matchM = matchId[i][0];
+            cv::DMatch matchN = matchId[i][1];
+            if (matchM.distance < matchN.distance * RADIUS) {
+                descriptorMatches[marker->id].push_back(std::make_pair(matchM.trainIdx, matchN.queryIdx));
+            }
+        }
+    }
+
+    int thresholdDist = (int) round(sqrt(5e-4f * queryImage.size().width * queryImage.size().height / M_PI));
+    float pp, prob;
+    int markerIdx, numMatches;
+    cv::Mat homography;
+    std::vector<cv::Point2f> r, q;
+    const int amountFeatures = this->size();
+    auto markerIt = descriptorMatches.begin();
+    while (markerIt != descriptorMatches.end()) {
+        markerIdx = (markerIt - descriptorMatches.begin());
+        numMatches = markerIt->size();
+        if (numMatches > minNumMatch) {
+            pp = std::min(0.05f * markers[markerIdx]->numFeatures / amountFeatures, 1.f);
+            prob = CvUtils::binomialCDF(numMatches, markers[markerIdx]->numFeatures, pp);
+
+            if (prob >= minProbability) {
+
+                auto matchIt = (*markerIt).begin();
+                while (matchIt != (*markerIt).end()) {
+                    q.push_back(keyPoints[matchIt->second].pt);
+                    r.push_back(markers[markerIdx]->keyPoints[matchIt->first].pt);
+                    matchIt++;
+                }
+
+                homography = cv::findHomography(cv::Mat(r), cv::Mat(q), cv::RANSAC, thresholdDist);
+
+                if (!homography.empty()) {
+                    std::vector<cv::Point2f> posePoint = CvUtils::transformMarkerCoordToObjectCoord(
+                            markers[markerIdx]->size,
+                            homography);
+                    if (CvUtils::_proveRect(posePoint)) {
+                        result.imgId = markerIdx;
+                        result.amountMatched = numMatches;
+                        result.imgSize = markers[markerIdx]->size;
+                        result.probability = prob;
+                        result.homography = homography;
+                        result.objPose = posePoint;
+                        results.push_back(result);
+                    }
+                }
+            }
+        }
+        r.clear();
+        q.clear();
+        markerIt++;
+    }
+    std::sort(results.begin(), results.end(), [](const QueryItem &a, const QueryItem &b) -> bool {
+        return a.probability > b.probability;
+    });
+
+    return results;
+}
+
+

src/Recognition/MarkerlessDB.hpp

@@ -0,0 +1,57 @@
+#pragma once
+#ifndef AR_CORE_MARKERLESSDB_HPP
+#define AR_CORE_MARKERLESSDB_HPP
+
+#include <opencv2/opencv.hpp>
+#include <opencv2/features2d/features2d.hpp>
+#include "../Utils/CvUtils.hpp"
+#include "MarkerlessTrackable.hpp"
+
+/**
+ * @brief Class provides operations with features
+ * 
+ */
+class MarkerlessDB {
+private:
+    cv::Ptr<cv::FeatureDetector> featureDetector;      // Feature detector
+    std::vector<MarkerlessTrackable *> markers;
+    cv::Ptr<cv::DescriptorMatcher> descriptorMatcher; // Descriptor matcher
+    float RADIUS;                                     // Descriptor matcher nn ratio
+    const int KNN_SIZE = 2;                           // knn size
+    int markerAmount;                                  // Number of markers
+    int featureAmount;                                  // Number of features
+public:
+    MarkerlessDB();
+
+    void add(const cv::Mat &image);
+
+    std::vector<QueryItem>
+    match(cv::Mat queryImage, int minNumMatch = 10, float minProbability = 0.75f);
+
+    /**
+     * @brief Extract features from image
+     *
+     * @param img Image
+     * @param keyPoints [out] KeyPoints
+     * @param descriptor [out] Descriptors
+     */
+    void extractFeatures(const cv::Mat &img, std::vector<cv::KeyPoint> &keyPoints, cv::Mat &descriptor);
+
+    /**
+     * @brief Add marker image feature
+     * 
+     * @param feature 
+     */
+    void addFeatures(const cv::Mat &feature);
+
+    /**
+     * @brief Get amount of features
+     * 
+     * @return int 
+     */
+    int size() const;
+
+    ~MarkerlessDB();
+};
+
+#endif //AR_CORE_MARKERLESSDB_HPP