/**
* @file Irregularity.hpp
* @author Matteo Spanio (dev2@audioinnova.com)
* @brief Header file containing the Irregularity class
* @version 1.0
* @date 2023-05-14
*
* @copyright Copyright (c) 2023
*
*/
#ifndef IRREGULARITY_H
#define IRREGULARITY_H
#include <boost/lexical_cast.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include <nlohmann/json.hpp>
#include "enums.hpp"
using std::string;
using json = nlohmann::json;
/**
* @class Irregularity
* @brief an irregularity of the tape detected by the system
*
*/
class Irregularity {
private:
boost::uuids::uuid id;
Source source;
string time_label;
std::optional<IrregularityType> type;
std::optional<string> image_URI;
std::optional<string> audio_URI;
public:
Irregularity(const Irregularity& other);
Irregularity(Irregularity&& other) noexcept;
Irregularity(Source source, string time_label);
Irregularity(Source source, string time_label, IrregularityType type);
~Irregularity() = default;
/**
* @brief Convert the Irregularity to a JSON object
*
* @return json
*/
json to_JSON() const;
/**
* @brief Create an Irregularity object from a JSON object
*
* @param j the JSON object
* @return Irregularity
*/
static Irregularity from_JSON(const json& j);
/**
* @brief Get the source object
*
* @return Source
*/
Source get_source() const;
/**
* @brief Get the time label object
*
* @return string
*/
string get_time_label() const;
/**
* @brief Get the type object
*
* @return IrregularityType
*/
std::optional<IrregularityType> get_type() const;
/**
* @brief Get the id object
*
* @return boost::uuids::uuid
*/
boost::uuids::uuid get_id() const;
/**
* @brief Get the audio URI object
*
* @return std::optional<string>
*/
std::optional<string> get_audio_URI() const;
/**
* @brief Get the image URI object
*
* @return std::optional<string>
*/
std::optional<string> get_image_URI() const;
/**
* @brief Set the audio URI object
*
* @param audio_URI
* @return Irregularity&
*/
Irregularity& set_audio_URI(string audio_URI);
/**
* @brief Set the image URI object
*
* @param image_URI
* @return Irregularity&
*/
Irregularity& set_image_URI(string image_URI);
};
#endif // IRREGULARITY_H
\ No newline at end of file
#include <exception> #include "IrregularityFile.hpp"
#include "IrregularityFile.h"
IrregularityFile::IrregularityFile(uint16_t offset, std::list<Irregularity> irregularities) IrregularityFile::IrregularityFile(std::optional<uint16_t> offset) : offset_(offset) {}
: offset(offset), irregularities(irregularities) {}
IrregularityFile::~IrregularityFile() {} IrregularityFile::IrregularityFile(const IrregularityFile& rhs) {
std::transform(rhs.irregularities_.begin(), rhs.irregularities_.end(), std::back_inserter(irregularities_),
json IrregularityFile::toJSON() { [](const std::unique_ptr<Irregularity>& ptr) { return std::make_unique<Irregularity>(*ptr); });
json j;
j["Offset"] = this->offset;
return j;
} }
IrregularityFile IrregularityFile::fromJSON(json j) { IrregularityFile& IrregularityFile::add(std::unique_ptr<Irregularity> irregularity) {
throw "Not implemented"; irregularities_.push_back(std::move(irregularity));
}
uint16_t IrregularityFile::getOffset() const {
return this->offset;
}
std::list<Irregularity> IrregularityFile::getIrregularities() const {
return this->irregularities;
}
IrregularityFile& IrregularityFile::add(const Irregularity irregularity) {
this->irregularities.push_back(irregularity);
return *this; return *this;
} }
IrregularityFile& IrregularityFile::remove_by_id(const boost::uuids::uuid id) { IrregularityFile& IrregularityFile::remove_by_id(const boost::uuids::uuid id) {
for (auto it = this->irregularities.begin(); it != this->irregularities.end(); ++it) { auto it =
if (it->id == id) { std::find_if(irregularities_.begin(), irregularities_.end(),
this->irregularities.erase(it); [&id](const std::unique_ptr<Irregularity>& irregularity) { return irregularity->get_id() == id; });
break; if (it != irregularities_.end()) {
} irregularities_.erase(it);
} }
return *this; return *this;
} }
IrregularityFile& IrregularityFile::sort() { IrregularityFile& IrregularityFile::sort() {
this->irregularities.sort([](const Irregularity& a, const Irregularity& b) { std::sort(irregularities_.begin(), irregularities_.end(),
return a.time_label < b.time_label; [](const std::unique_ptr<Irregularity>& a, const std::unique_ptr<Irregularity>& b) {
}); return a->get_time_label() < b->get_time_label();
});
return *this; return *this;
} }
\ No newline at end of file
std::optional<uint16_t> IrregularityFile::get_offset() const { return offset_; }
std::vector<std::unique_ptr<Irregularity>>::iterator IrregularityFile::begin() { return irregularities_.begin(); }
std::vector<std::unique_ptr<Irregularity>>::iterator IrregularityFile::end() { return irregularities_.end(); }
json IrregularityFile::toJSON() const {
json j;
j["Offset"] = offset_.value_or(0);
j["Irregularities"] = json::array();
for (const auto& irregularity : irregularities_) {
j["Irregularities"].push_back(irregularity->to_JSON());
}
return j;
}
IrregularityFile IrregularityFile::fromJSON(const json j) {
if (!j.contains("Offset") || !j.contains("Irregularities")) {
throw std::invalid_argument("Invalid JSON");
}
IrregularityFile irregularity_file(j["Offset"].get<uint16_t>());
for (const auto& irregularity : j["Irregularities"]) {
irregularity_file.add(std::make_unique<Irregularity>(Irregularity::from_JSON(irregularity)));
}
return irregularity_file;
}
#ifndef IRREGULARITY_FILE_H
#define IRREGULARITY_FILE_H
#include <list>
#include <nlohmann/json.hpp>
#include "Irregularity.h"
using json = nlohmann::json;
class IrregularityFile
{
private:
/* data */
uint16_t offset;
std::list<Irregularity> irregularities;
public:
IrregularityFile(uint16_t offset, std::list<Irregularity> irregularities);
~IrregularityFile();
static IrregularityFile fromJSON(const json j);
json toJSON();
uint16_t getOffset() const;
std::list<Irregularity> getIrregularities() const;
IrregularityFile& add(const Irregularity irregularity);
IrregularityFile& remove_by_id(const boost::uuids::uuid id);
IrregularityFile& sort();
};
#endif // IRREGULARITY_FILE_H
\ No newline at end of file
/**
* @file IrregularityFile.hpp
* @author Matteo Spanio (dev2@audioinnova.com)
* @brief Header file containing the IrregularityFile class
* @version 1.0
* @date 2023-05-14
*
* @copyright Copyright (c) 2023
*
*/
#ifndef IRREGULARITY_FILE_H
#define IRREGULARITY_FILE_H
#include <algorithm>
#include <exception>
#include <iterator>
#include <memory>
#include <nlohmann/json.hpp>
#include <optional>
#include <vector>
#include "Irregularity.hpp"
using json = nlohmann::json;
/**
* @class IrregularityFile
* @brief An IrregularityFile is a collection of Irregularities detected on a
* tape.
*
*/
class IrregularityFile {
public:
/**
* @brief Create an IrregularityFile object from a JSON object
*
* @param j
* @return IrregularityFile
*/
static IrregularityFile fromJSON(const json j);
/**
* @brief Convert the IrregularityFile to a JSON object
*
* @return json
*/
json toJSON() const;
IrregularityFile(std::optional<uint16_t> offset = std::nullopt);
/**
* @brief Copy constructor
*
* @param rhs
*/
IrregularityFile(const IrregularityFile& rhs);
~IrregularityFile(){};
/**
* @brief Add an Irregularity to the IrregularityFile
*
* @param irregularity
* @return IrregularityFile&
*/
IrregularityFile& add(std::unique_ptr<Irregularity> irregularity);
/**
* @brief Remove an Irregularity from the IrregularityFile
*
* @param id
* @return IrregularityFile&
*/
IrregularityFile& remove_by_id(const boost::uuids::uuid id);
/**
* @brief Sort the IrregularityFile by time_label
*
* @return IrregularityFile&
*/
IrregularityFile& sort();
/**
* @brief Get the offset object
*
* @return std::optional<uint16_t>
*/
std::optional<uint16_t> get_offset() const;
/**
* @brief Get an iterator to the beginning of the IrregularityFile
*
* @return std::vector<std::unique_ptr<Irregularity>>::iterator
*/
std::vector<std::unique_ptr<Irregularity>>::iterator begin();
/**
* @brief Get an iterator to the end of the IrregularityFile
*
* @return std::vector<std::unique_ptr<Irregularity>>::iterator
*/
std::vector<std::unique_ptr<Irregularity>>::iterator end();
private:
std::optional<uint16_t> offset_;
std::vector<std::unique_ptr<Irregularity>> irregularities_;
};
#endif // IRREGULARITY_FILE_HPP
#include "TimeLabel.h" #include "TimeLabel.hpp"
TimeLabel::TimeLabel(/* args */) TimeLabel::TimeLabel(/* args */) {}
{
}
TimeLabel::~TimeLabel() TimeLabel::~TimeLabel() {}
{ \ No newline at end of file
}
\ No newline at end of file
#ifndef TIME_LABEL_H
#define TIME_LABEL_H
class TimeLabel
{
private:
/* data */
public:
TimeLabel(/* args */);
~TimeLabel();
};
#endif // TIME_LABEL_H
\ No newline at end of file
#ifndef TIME_LABEL_H
#define TIME_LABEL_H
/**
* @brief A label that displays the current time in the format hh:mm:ss.mms
* @todo Implement this class:
* - Implement the constructor
* - Implement the destructor
* - Move functions from time.h to here
*/
class TimeLabel {
private:
/* data */
public:
TimeLabel(/* args */);
~TimeLabel();
};
#endif // TIME_LABEL_H
\ No newline at end of file
#include <stdlib.h>
#include <string>
using std::string;
string PURPLE = "\033[95m";
string CYAN = "\033[96m";
string DARK_CYAN = "\033[36m";
string BLUE = "\033[94m";
string GREEN = "\033[92m";
string YELLOW = "\033[93m";
string RED = "\033[91m";
string BOLD = "\033[1m";
string UNDERLINE = "\033[4m";
string END = "\033[0m";
\ No newline at end of file
/**
* @file colors.hpp
* @author Matteo Spanio (dev2@audioinnova.com)
* @brief Header file containing a set of ANSI escape codes to print colored
* text in the terminal.
*
* When printing text in the terminal, it is possible to use ANSI escape codes
* to change the color of the text. This header file contains a set of
* pre-defined ANSI escape codes to print colored text in the terminal.
*
* @see https://en.wikipedia.org/wiki/ANSI_escape_code
* @version 1.0
* @date 2023-05-13
* @copyright Copyright (c) 2023
*
*/
#ifndef COLORS_H
#define COLORS_H
namespace colors {
using Color = const char*;
constexpr Color PURPLE = "\033[95m";
constexpr Color CYAN = "\033[96m";
constexpr Color DARK_CYAN = "\033[36m";
constexpr Color BLUE = "\033[94m";
constexpr Color GREEN = "\033[92m";
constexpr Color YELLOW = "\033[93m";
constexpr Color RED = "\033[91m";
constexpr Color WHITE = "\033[97m";
constexpr Color BOLD = "\033[1m";
constexpr Color UNDERLINE = "\033[4m";
constexpr Color END = "\033[0m";
} // namespace colors
#endif // COLORS_H
\ No newline at end of file
#include "core.hpp"
namespace videoanalyser {
namespace core {
Frame::Frame() : cv::Mat() {}
Frame::Frame(const cv::Mat& m) : cv::Mat(m) {}
Frame::Frame(const Frame& f) : cv::Mat(f) {}
Frame& Frame::operator=(const Mat& m) {
Mat::operator=(m);
return *this;
}
Frame& Frame::operator=(const Frame& f) {
Mat::operator=(f);
return *this;
}
Frame Frame::clone() const { return Frame(cv::Mat::clone()); }
Frame& Frame::downsample(int factor) {
cv::pyrDown(*this, *this, cv::Size(size().width / factor, size().height / factor));
return *this;
}
Frame& Frame::convert_color(int code) {
cv::cvtColor(*this, *this, code);
return *this;
}
Frame Frame::difference(Frame& f) {
Frame diff = this->clone();
for (int i = 0; i < this->rows; i++) {
for (int j = 0; j < this->cols; j++) {
if (f.at<cv::Vec3b>(i, j)[0] != this->at<cv::Vec3b>(i, j)[0] ||
f.at<cv::Vec3b>(i, j)[1] != this->at<cv::Vec3b>(i, j)[1] ||
f.at<cv::Vec3b>(i, j)[2] != this->at<cv::Vec3b>(i, j)[2]) {
// Different pixels
diff.at<cv::Vec3b>(i, j)[0] = 0;
diff.at<cv::Vec3b>(i, j)[1] = 0;
diff.at<cv::Vec3b>(i, j)[2] = 0;
} else {
// Identical pixels
diff.at<cv::Vec3b>(i, j)[0] = 255;
diff.at<cv::Vec3b>(i, j)[1] = 255;
diff.at<cv::Vec3b>(i, j)[2] = 255;
}
}
}
return diff;
}
Frame& Frame::crop(cv::Size rect_size, cv::Point2f center) {
cv::getRectSubPix(*this, rect_size, center, *this);
return *this;
}
Frame& Frame::warp(cv::Mat rotationMatrix) {
cv::warpAffine(*this, *this, rotationMatrix, this->size(), cv::INTER_CUBIC);
return *this;
}
std::pair<Frame, Frame> Frame::deinterlace() const {
Frame odd_frame(cv::Mat(this->rows / 2, this->cols, CV_8UC3));
Frame even_frame(cv::Mat(this->rows / 2, this->cols, CV_8UC3));
int i_odd_frame = 0;
int i_even_frame = 0;
for (int i = 0; i < this->rows; i++) {
for (int j = 0; j < this->cols; j++) {
if (i % 2 == 0) {
even_frame.at<cv::Vec3b>(i_even_frame, j)[0] = this->at<cv::Vec3b>(i, j)[0];
even_frame.at<cv::Vec3b>(i_even_frame, j)[1] = this->at<cv::Vec3b>(i, j)[1];
even_frame.at<cv::Vec3b>(i_even_frame, j)[2] = this->at<cv::Vec3b>(i, j)[2];
} else {
odd_frame.at<cv::Vec3b>(i_odd_frame, j)[0] = this->at<cv::Vec3b>(i, j)[0];
odd_frame.at<cv::Vec3b>(i_odd_frame, j)[1] = this->at<cv::Vec3b>(i, j)[1];
odd_frame.at<cv::Vec3b>(i_odd_frame, j)[2] = this->at<cv::Vec3b>(i, j)[2];
}
}
if (i % 2 == 0) {
i_even_frame++;
} else {
i_odd_frame++;
}
}
return std::make_pair(odd_frame, even_frame);
}
} // namespace core
} // namespace videoanalyser
\ No newline at end of file
/**
* @file core.hpp
* @author Matteo Spanio (dev2@audioinnova.com)
* @brief This file contains the core functionalities of the project.
* @version 1.2
* @date 2023-06-03
*
* @copyright Copyright (c) 2023
*
*/
#ifndef CORE_H
#define CORE_H
#include <opencv2/calib3d.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/xfeatures2d.hpp>
#include <optional>
#include <string>
#include <tuple>
#include <variant>
#include <vector>
namespace videoanalyser {
using Error = std::string;
template <typename T>
using Result = std::variant<T, Error>;
namespace core {
/**
* @class Frame
* @brief Class that extends the OpenCV Mat class, adding some useful methods
* frequently used in the project.
*
*/
class Frame : public cv::Mat {
public:
Frame();
Frame(const cv::Mat& m);
Frame(const Frame& f);
Frame& operator=(const cv::Mat& m);
Frame& operator=(const Frame& f);
Frame clone() const;
/**
* @brief Downsample the image by a given factor.
*
* @param factor The factor by which the image will be downsampled.
* @return Frame& The downsampled image.
*/
Frame& downsample(int factor);
/**
* @brief Convert the image to a given color space.
*
* @param code The code of the color space to which the image will be
* converted.
* @return Frame& The converted image.
*/
Frame& convert_color(int code);
/**
* @brief Compute the number of different pixels between two frames.
*
* @param f The frame to compare with.
* @return A black and white frame, where black pixels represent a
* difference, while white pixels represent an equality.
*/
Frame difference(Frame& f);
/**
* @brief Crop the image to a given size, centered in a given point.
*
* @param rect_size The size of the cropped image.
* @param center The center of the cropped image.
* @return Frame& The cropped image.
*/
Frame& crop(cv::Size rect_size, cv::Point2f center);
/**
* @brief Warp the image using a given rotation matrix.
*
* @param rotationMatrix The rotation matrix used to warp the image.
* @return Frame& The warped image.
*/
Frame& warp(cv::Mat rotationMatrix);
/**
* @brief Deinterlace the image, returning two images, one containing the
* odd lines and the other containing the even lines.
*
* @return std::pair<Frame, Frame> The two images containing the odd and
* even lines.
*/
std::pair<Frame, Frame> deinterlace() const;
};
} // namespace core
} // namespace videoanalyser
#endif // CORE_H
\ No newline at end of file
#include "detection.hpp"
namespace videoanalyser {
namespace detection {
using namespace cv;
namespace {
cv::RotatedRect get_rectangle_from_match(const cv::Vec4f& positions, int width, int height, int offsetX, int offsetY,
float processingScale) {
cv::RotatedRect rr;
cv::Point2f rrpts[4];
cv::Point2f pos(positions[0] + offsetX, positions[1] + offsetY);
float scale = positions[2];
float angle = positions[3];
rr.center = pos * processingScale;
rr.size = cv::Size2f(width * scale * processingScale, height * scale * processingScale);
rr.angle = angle;
rr.points(rrpts);
return rr;
}
using ShapeMatch = std::tuple<std::vector<cv::Vec4f>, std::vector<cv::Vec4f>, cv::Mat, cv::Mat>;
ShapeMatch detect_shape(cv::Ptr<cv::GeneralizedHoughGuil> alg, int pos_thresh, cv::Mat processing_area) {
cv::Mat positive_votes, negative_votes;
std::vector<cv::Vec4f> positive_positions, negative_positions;
alg->setPosThresh(pos_thresh);
int num_prev_matches = 0;
int threshold_increment = 0;
int max_match_score = 0;
// Process shapes with positive angles
alg->setMinAngle(0);
alg->setMaxAngle(3);
while (true) {
alg->detect(processing_area, positive_positions, positive_votes);
int current_matches = positive_positions.size();
if (current_matches == 1 || (current_matches == 0 && num_prev_matches == 0)) {
// We detected the most interesting shape
// Impossible to find with these parameters
break;
} else if (current_matches == 0 && num_prev_matches > 0) {
// It is not possible to detect only one shape with the current
// parameters
alg->setPosThresh(pos_thresh + threshold_increment - 1); // Decrease position value
alg->detect(processing_area, positive_positions,
positive_votes); // Detect all available shapes
break;
}
num_prev_matches = current_matches;
// Find maximum vote
for (int j = 0; j < positive_votes.cols / 3; j++) {
if (positive_votes.at<int>(3 * j) > max_match_score) max_match_score = positive_votes.at<int>(3 * j);
}
if (current_matches > 10) {
threshold_increment += 5; // To speed up computation when there are too many matches
} else if (max_match_score - (pos_thresh + threshold_increment) > 100) {
threshold_increment += 100; // To speed up computation when there are few super high
// matches
} else {
threshold_increment++;
}
alg->setPosThresh(pos_thresh + threshold_increment);
}
// Reset incremental position value
threshold_increment = 0;
num_prev_matches = 0;
max_match_score = 0;
// Process shapes with negative angles
alg->setMinAngle(357);
alg->setMaxAngle(360);
while (true) {
alg->detect(processing_area, negative_positions, negative_votes);
int current_matches = negative_positions.size();
if (current_matches == 1 || (current_matches == 0 && num_prev_matches == 0)) {
// We detected the most interesting shape
// Impossible to found with these parameters
break;
} else if (current_matches == 0 && num_prev_matches > 0) {
// It is not possible to detect only one shape with the current
// parameters
alg->setPosThresh(pos_thresh + threshold_increment - 1); // Decrease position value
alg->detect(processing_area, negative_positions,
negative_votes); // Detect all available shapes
break;
}
num_prev_matches = current_matches;
// Find maximum vote
for (int j = 0; j < positive_votes.cols / 3; j++) {
if (positive_votes.at<int>(3 * j) > max_match_score) max_match_score = positive_votes.at<int>(3 * j);
}
if (current_matches > 10) {
threshold_increment += 5; // To speed up computation when there are too many matches
} else if (max_match_score - (pos_thresh + threshold_increment) > 100) {
threshold_increment += 100; // To speed up computation when there are few super high
// matches
} else {
threshold_increment++;
}
alg->setPosThresh(pos_thresh + threshold_increment);
}
return std::make_tuple(positive_positions, negative_positions, positive_votes, negative_votes);
}
Result<core::Frame> get_template_image(ElementType element_type) {
switch (element_type) {
case ElementType::TAPE:
return core::Frame(cv::imread("input/readingHead.png", cv::IMREAD_GRAYSCALE));
case ElementType::CAPSTAN:
return core::Frame(cv::imread("input/capstanBERIO058prova.png", cv::IMREAD_GRAYSCALE));
default:
return Error("Invalid element type");
}
}
/**
* @fn std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>>
* find_object(Mat model, SceneObject object)
* @brief Find the model in the scene using the Generalized Hough Transform.
* It returns the best matches. Find the best matches for positive and negative
* angles. If there are more than one shape, then choose the one with the
* highest score. If there are more than one with the same highest score, then
* arbitrarily choose the latest.
*
* For informations about the Generalized Hough Guild usage see the tutorial
* at https://docs.opencv.org/4.7.0/da/ddc/tutorial_generalized_hough_ballard_guil.html
*
* @param model the template image to be searched with the Generalized Hough
* Transform
* @param object the sceneObject struct containing the parameters for the
* Generalized Hough Transform
* @return std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>> a
* tuple containing the best matches for positive and negative angles
*/
Result<Roi> find_roi_ght(core::Frame image, SceneElement element_to_find) {
// Save a grayscale version of image in gray_image
core::Frame gray_image = core::Frame(image).convert_color(cv::COLOR_BGR2GRAY);
// downsample the frame in half pixels for performance reasons
core::Frame halved_gray_image = core::Frame(gray_image).clone().downsample(2);
// Get input shape in grayscale and downsample it in half pixels
Result<core::Frame> template_image_result = get_template_image(element_to_find.type);
if (std::holds_alternative<Error>(template_image_result)) {
return Error("Error while loading template image:" + std::get<Error>(template_image_result));
}
core::Frame template_image = std::get<core::Frame>(template_image_result).downsample(2);
cv::Ptr<cv::GeneralizedHoughGuil> ght = cv::createGeneralizedHoughGuil();
ght->setMinDist(element_to_find.min_dist);
ght->setLevels(360);
ght->setDp(2);
ght->setMaxBufferSize(1000);
ght->setAngleStep(1);
ght->setAngleThresh(element_to_find.threshold.angle);
ght->setMinScale(0.9);
ght->setMaxScale(1.1);
ght->setScaleStep(0.01);
ght->setScaleThresh(element_to_find.threshold.scale);
ght->setCannyLowThresh(150);
ght->setCannyHighThresh(240);
ght->setTemplate(template_image);
cv::Rect processing_area;
cv::Mat processing_image;
if (element_to_find.type == ElementType::TAPE) {
processing_area = cv::Rect(halved_gray_image.cols / 4, halved_gray_image.rows / 2, halved_gray_image.cols / 2,
halved_gray_image.rows / 2);
processing_image = halved_gray_image(processing_area);
} else if (element_to_find.type == ElementType::CAPSTAN) {
processing_area = cv::Rect(image.cols * 3 / 4, image.rows / 2, image.cols / 4, image.rows / 2);
processing_image = gray_image(processing_area);
}
auto [positive_positions, negative_positions, posPos, posNeg] =
detect_shape(ght, element_to_find.threshold.pos, processing_image);
double max_score_for_positive_match = 0, max_score_for_negative_match = 0;
int index_max_positive_score = 0, index_max_negative_score = 0;
cv::Mat positive_matches_scores = posPos;
cv::Mat negative_matches_scores = posNeg;
for (int i = 0; i < positive_matches_scores.size().width; i++) {
if (positive_matches_scores.at<int>(i) >= max_score_for_positive_match) {
max_score_for_positive_match = positive_matches_scores.at<int>(i);
index_max_positive_score = i;
}
}
for (int i = 0; i < negative_matches_scores.size().width; i++) {
if (negative_matches_scores.at<int>(i) >= max_score_for_negative_match) {
max_score_for_negative_match = negative_matches_scores.at<int>(i);
index_max_negative_score = i;
}
}
cv::RotatedRect roi_pos;
cv::RotatedRect roi_neg;
if (element_to_find.type == ElementType::TAPE) {
if (positive_positions.size() > 0) {
roi_pos = get_rectangle_from_match(positive_positions[index_max_positive_score], template_image.cols,
template_image.rows, halved_gray_image.cols / 4,
halved_gray_image.rows / 2, 2);
}
if (negative_positions.size() > 0) {
roi_neg = get_rectangle_from_match(negative_positions[index_max_negative_score], template_image.cols,
template_image.rows, halved_gray_image.cols / 4,
halved_gray_image.rows / 2, 2);
}
} else if (element_to_find.type == ElementType::CAPSTAN) {
if (positive_positions.size() > 0) {
roi_pos =
get_rectangle_from_match(positive_positions[index_max_positive_score], template_image.cols - 22,
template_image.rows - 92, image.cols * 3 / 4 + 11, image.rows / 2 + 46, 1);
}
if (negative_positions.size() > 0) {
roi_neg =
get_rectangle_from_match(negative_positions[index_max_negative_score], template_image.cols - 22,
template_image.rows - 92, image.cols * 3 / 4 + 11, image.rows / 2 + 46, 1);
}
}
cv::RotatedRect result;
if (max_score_for_positive_match > 0) {
if (max_score_for_negative_match > 0) {
result = max_score_for_positive_match > max_score_for_negative_match ? roi_pos : roi_neg;
} else {
result = roi_pos;
}
} else if (max_score_for_negative_match > 0) {
result = roi_neg;
} else {
return Error("No match found");
}
if (element_to_find.type == ElementType::TAPE) {
cv::Vec4f tape_position(result.center.x,
result.center.y + result.size.height / 2 + 20 * (result.size.width / 200), 1,
result.angle);
result = get_rectangle_from_match(tape_position, result.size.width, 50 * (result.size.width / 200), 0, 0, 1);
}
return result;
}
Result<Roi> find_roi_surf(core::Frame image, SceneElement element_to_find) {
// Step 1: Detect the keypoints using SURF Detector, compute the
// descriptors
int min_hessian = 100;
Ptr<xfeatures2d::SURF> detector = xfeatures2d::SURF::create(min_hessian);
std::vector<cv::KeyPoint> keypoints_object, keypoints_scene;
cv::Mat descriptors_object, descriptors_scene;
// Save a grayscale version of image in gray_image
core::Frame gray_image = core::Frame(image).convert_color(cv::COLOR_BGR2GRAY);
// downsample the frame in half pixels for performance reasons
core::Frame halved_gray_image = core::Frame(gray_image).clone().downsample(2);
Result<core::Frame> template_image_result = get_template_image(element_to_find.type);
if (std::holds_alternative<Error>(template_image_result)) {
return Error("Error while loading template image:" + std::get<Error>(template_image_result));
}
core::Frame template_image = std::get<core::Frame>(template_image_result);
detector->detectAndCompute(template_image, cv::noArray(), keypoints_object, descriptors_object);
detector->detectAndCompute(gray_image, cv::noArray(), keypoints_scene, descriptors_scene);
// Step 2: Matching descriptor vectors with a FLANN based matcher
// Since SURF is a floating-point descriptor NORM_L2 is used
cv::Ptr<cv::DescriptorMatcher> matcher = cv::DescriptorMatcher::create(cv::DescriptorMatcher::FLANNBASED);
std::vector<std::vector<cv::DMatch>> knn_matches;
matcher->knnMatch(descriptors_object, descriptors_scene, knn_matches, 2);
//-- Filter matches using the Lowe's ratio test
const float RATIO_THRESH = 0.75f;
std::vector<cv::DMatch> good_matches;
for (size_t i = 0; i < knn_matches.size(); i++) {
if (knn_matches[i][0].distance < RATIO_THRESH * knn_matches[i][1].distance) {
good_matches.push_back(knn_matches[i][0]);
}
}
// Draw matches
cv::Mat img_matches;
cv::drawMatches(template_image, keypoints_object, halved_gray_image, keypoints_scene, good_matches, img_matches,
cv::Scalar::all(-1), cv::Scalar::all(-1), std::vector<char>(),
cv::DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
// Localize the object
std::vector<cv::Point2f> obj;
std::vector<cv::Point2f> scene;
for (size_t i = 0; i < good_matches.size(); i++) {
// Get the keypoints from the good matches
obj.push_back(keypoints_object[good_matches[i].queryIdx].pt);
scene.push_back(keypoints_scene[good_matches[i].trainIdx].pt);
}
cv::Mat H = cv::findHomography(obj, scene, cv::RANSAC);
// Get the corners from the image_1 ( the object to be "detected" )
std::vector<cv::Point2f> obj_corners(4);
obj_corners[0] = cv::Point2f(0, 0);
obj_corners[1] = cv::Point2f((float)template_image.cols, 0);
obj_corners[2] = cv::Point2f((float)template_image.cols, (float)template_image.rows);
obj_corners[3] = cv::Point2f(0, (float)template_image.rows);
std::vector<cv::Point2f> scene_corners(4);
cv::perspectiveTransform(obj_corners, scene_corners, H);
// Find average
float capstanX = (scene_corners[0].x + scene_corners[1].x + scene_corners[2].x + scene_corners[3].x) / 4;
float capstanY = (scene_corners[0].y + scene_corners[1].y + scene_corners[2].y + scene_corners[3].y) / 4;
// In the following there are two alterations to cut the first 20
// horizontal pixels and the first 90 vertical pixels from the found
// rectangle: +10 in X for centering and -20 in width +45 in Y for
// centering and -90 in height
cv::Vec4f positionCapstan(capstanX + 10, capstanY + 45, 1, 0);
return get_rectangle_from_match(positionCapstan, template_image.cols - 20, template_image.rows - 90, 0, 0, 1);
}
} // anonymous namespace
Result<Roi> find_roi(core::Frame image, Algorithm algorithm, SceneElement element_to_find) {
switch (algorithm) {
case Algorithm::GHT:
return find_roi_ght(image, element_to_find);
case Algorithm::SURF:
return find_roi_surf(image, element_to_find);
default:
return Error("Invalid algorithm");
}
}
} // namespace detection
} // namespace videoanalyser
\ No newline at end of file
/**
* @file detection.hpp
* @author Matteo Spanio (dev2@audioinnova.com)
* @brief This file contains the functions used to detect the objects in a scene.
* @version 1.2
* @date 2023-06-04
*
* @copyright Copyright (c) 2023
*
*/
#ifndef VIDEOANALYSER_DETECTION_H
#define VIDEOANALYSER_DETECTION_H
#include <opencv2/calib3d.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/xfeatures2d.hpp>
#include <variant>
#include <vector>
#include "core.hpp"
namespace videoanalyser {
namespace detection {
enum class ElementType { TAPE, CAPSTAN, READING_HEAD };
enum class Algorithm { GHT, SURF };
/**
* @struct DetectionThreshold
* @brief Struct containing the threshold values used to detect a shape.
*/
struct DetectionThreshold {
float percentual; /**< The minimum percentage of different pixels for
considering the current frame under the ROI as a
potential Irregularity */
int angle; /**< The angle votes threshold for the detection of the object */
int scale; /**< The scale votes threshold for the detection of the object */
int pos; /**< The position votes threshold for the detection of the object
*/
};
/**
* @typedef Roi
* @brief The region of interest of a scene. It is a pair containing the
* coordinates of the top-left and bottom-right corners of the ROI and the
* type of the object in the ROI.
*/
using Roi = cv::RotatedRect;
/**
* @struct SceneElement
* @brief A scene element is an object that can be detected in a scene, such as a
* tape or a capstan.
*
*/
struct SceneElement {
ElementType type; /**< The type of the object */
int min_dist; /**< The minimum distance between the centers of the detected
objects for the detection of the reading head */
DetectionThreshold threshold; /**< the threshold values used to detect the object */
};
/**
* @fn Result<Roi> find_roi(core::Frame image, Algorithm algorithm, SceneElement element_to_find)
* @brief Looks for a shape in a frame.
*
* Find a shape in a frame using the specified algorithm.
*
* @param image The frame in which the object will be searched.
* @param algorithm The algorithm to use for the detection.
* @param element_to_find The object to find.
* @return Result<Roi<T>> A Result object containing the ROI of the object if found, otherwise an error.
*/
Result<Roi> find_roi(core::Frame image, Algorithm algorithm, SceneElement element_to_find);
} // namespace detection
} // namespace videoanalyser
#endif // VIDEOANALYSER_DETECTION_H
\ No newline at end of file
#include <stdexcept> #include "enums.hpp"
#include "enums.h"
std::string sourceToString(Source source) std::string sourceToString(Source source) {
{
switch (source) { switch (source) {
case Audio: case Source::Audio:
return "a"; return "a";
case Video: case Source::Video:
return "v"; return "v";
case Both: case Source::Both:
return "b"; return "b";
default: default:
throw std::invalid_argument("Invalid Source"); throw std::invalid_argument("Invalid Source");
} }
} }
Source sourceFromString(std::string source) Source sourceFromString(std::string source) {
{
if (source == "a") if (source == "a")
return Audio; return Source::Audio;
else if (source == "v") else if (source == "v")
return Video; return Source::Video;
else if (source == "b") else if (source == "b")
return Both; return Source::Both;
else else
throw std::invalid_argument("Invalid Source"); throw std::invalid_argument("Invalid Source");
} }
std::string irregularityTypeToString(IrregularityType type) std::string irregularityTypeToString(IrregularityType type) {
{
switch (type) { switch (type) {
case BRANDS_ON_TAPE: case IrregularityType::BRANDS_ON_TAPE:
return "b"; return "b";
case SPLICE: case IrregularityType::SPLICE:
return "sp"; return "sp";
case START_OF_TAPE: case IrregularityType::START_OF_TAPE:
return "sot"; return "sot";
case ENDS_OF_TAPE: case IrregularityType::ENDS_OF_TAPE:
return "eot"; return "eot";
case DAMAGED_TAPE: case IrregularityType::DAMAGED_TAPE:
return "da"; return "da";
case DIRT: case IrregularityType::DIRT:
return "di"; return "di";
case MARKS: case IrregularityType::MARKS:
return "m"; return "m";
case SHADOWS: case IrregularityType::SHADOWS:
return "s"; return "s";
case WOW_AND_FLUTTER: case IrregularityType::WOW_AND_FLUTTER:
return "wf"; return "wf";
case PLAY_PAUSE_STOP: case IrregularityType::PLAY_PAUSE_STOP:
return "pps"; return "pps";
case SPEED: case IrregularityType::SPEED:
return "ssv"; return "ssv";
case EQUALIZATION: case IrregularityType::EQUALIZATION:
return "esv"; return "esv";
case SPEED_AND_EQUALIZATION: case IrregularityType::SPEED_AND_EQUALIZATION:
return "ssv"; return "ssv";
case BACKWARD: case IrregularityType::BACKWARD:
return "sb"; return "sb";
default: default:
throw std::invalid_argument("Invalid IrregularityType"); throw std::invalid_argument("Invalid IrregularityType");
} }
} }
IrregularityType irregularityTypeFromString(std::string type) IrregularityType irregularityTypeFromString(std::string type) {
{
if (type == "b") if (type == "b")
return BRANDS_ON_TAPE; return IrregularityType::BRANDS_ON_TAPE;
else if (type == "sp") else if (type == "sp")
return SPLICE; return IrregularityType::SPLICE;
else if (type == "sot") else if (type == "sot")
return START_OF_TAPE; return IrregularityType::START_OF_TAPE;
else if (type == "eot") else if (type == "eot")
return ENDS_OF_TAPE; return IrregularityType::ENDS_OF_TAPE;
else if (type == "da") else if (type == "da")
return DAMAGED_TAPE; return IrregularityType::DAMAGED_TAPE;
else if (type == "di") else if (type == "di")
return DIRT; return IrregularityType::DIRT;
else if (type == "m") else if (type == "m")
return MARKS; return IrregularityType::MARKS;
else if (type == "s") else if (type == "s")
return SHADOWS; return IrregularityType::SHADOWS;
else if (type == "wf") else if (type == "wf")
return WOW_AND_FLUTTER; return IrregularityType::WOW_AND_FLUTTER;
else if (type == "pps") else if (type == "pps")
return PLAY_PAUSE_STOP; return IrregularityType::PLAY_PAUSE_STOP;
else if (type == "ssv") else if (type == "ssv")
return SPEED; return IrregularityType::SPEED;
else if (type == "esv") else if (type == "esv")
return EQUALIZATION; return IrregularityType::EQUALIZATION;
else if (type == "ssv") else if (type == "ssv")
return SPEED_AND_EQUALIZATION; return IrregularityType::SPEED_AND_EQUALIZATION;
else if (type == "sb") else if (type == "sb")
return BACKWARD; return IrregularityType::BACKWARD;
else else
throw std::invalid_argument("Invalid IrregularityType"); throw std::invalid_argument("Invalid IrregularityType");
} }
\ No newline at end of file
#include<string>
enum Source{
Audio,
Video,
Both
};
enum IrregularityType {
BRANDS_ON_TAPE,
SPLICE,
START_OF_TAPE,
ENDS_OF_TAPE,
DAMAGED_TAPE,
DIRT,
MARKS,
SHADOWS,
WOW_AND_FLUTTER,
PLAY_PAUSE_STOP,
SPEED,
EQUALIZATION,
SPEED_AND_EQUALIZATION,
BACKWARD
};
std::string sourceToString(Source source);
Source sourceFromString(std::string source);
std::string irregularityTypeToString(IrregularityType type);
IrregularityType irregularityTypeFromString(std::string type);
/**
* @file enums.hpp
* @author Matteo Spanio (dev2@audioinnova.com)
* @brief A collection of enums and functions to handle them.
* @version 1.0
* @date 2023-05-13
*
* @copyright Copyright (c) 2023
*
* This file contains a set of enums to define properties of an Irregularity.
* The enums are:
* - Source: the source of the Irregularity (Audio, Video or Both)
* - IrregularityType: the type of Irregularity (Brands on tape, Splice, etc.)
* that can be present on the tape.
*
* The file also contains functions to convert from enum to string and viceversa
* (useful when saving the Irregularity to a file).
*
*/
#ifndef ENUMS_H
#define ENUMS_H
#include <stdexcept>
#include <string>
/**
* @enum Source
* @brief The source of the Irregularity (Audio, Video or Both)
*
* An Irregularity can be detected by the Audio analyser, the Video analyser or
* both.
*
*/
enum class Source { Audio, Video, Both };
/**
* @enum IrregularityType
* @brief The type of Irregularity (Brands on tape, Splice, etc.) that can be
* present on the tape.
*
* The types of Irregularities are:
* - BRANDS_ON_TAPE: Brands on tape
* - SPLICE: Splice
* - START_OF_TAPE: Start of tape
* - ENDS_OF_TAPE: End of tape
* - DAMAGED_TAPE: Damaged tape
* - DIRT: Dirt
* - MARKS: Marks
* - SHADOWS: Shadows
* - WOW_AND_FLUTTER: Wow and flutter
* - PLAY_PAUSE_STOP: Play, pause, stop
* - SPEED: Speed
* - EQUALIZATION: Equalization
* - SPEED_AND_EQUALIZATION: Speed and equalization
* - BACKWARD: Backward
*
* @note Speed, Equalization and Speed and equalization are detected only by the
* Audio analyser, while the other Irregularities are detected only by the Video
* analyser.
*
*/
enum class IrregularityType {
BRANDS_ON_TAPE,
SPLICE,
START_OF_TAPE,
ENDS_OF_TAPE,
DAMAGED_TAPE,
DIRT,
MARKS,
SHADOWS,
WOW_AND_FLUTTER,
PLAY_PAUSE_STOP,
SPEED,
EQUALIZATION,
SPEED_AND_EQUALIZATION,
BACKWARD
};
std::string sourceToString(Source source);
Source sourceFromString(std::string source);
std::string irregularityTypeToString(IrregularityType type);
IrregularityType irregularityTypeFromString(std::string type);
#endif // ENUMS_H
\ No newline at end of file
#include <iostream> #include "files.hpp"
#include "files.h"
using std::cout, std::endl, std::cerr, std::ofstream, std::ios; using std::cout, std::endl, std::cerr, std::ofstream, std::ios;
void files::saveFile(std::filesystem::path fileName, std::string content, bool append) { void files::save_file(std::filesystem::path fileName, std::string content) {
ofstream outputFile; ofstream outputFile;
if (append) { outputFile.open(fileName);
outputFile.open(fileName, ios::app); outputFile << content << endl;
} else { outputFile.close();
outputFile.open(fileName);
}
outputFile << content << endl;
outputFile.close();
} }
void files::findFileNameFromPath(std::string* path, std::string* fileName, std::string* extension) { void files::findFileNameFromPath(std::string* path, std::string* fileName, std::string* extension) {
*path = path->substr(path->find_last_of("'") + 1, path->size()); *path = path->substr(path->find_last_of("'") + 1, path->size());
std::string path_without_extension = path->substr(0, path->find_last_of(".")); std::string path_without_extension = path->substr(0, path->find_last_of("."));
*fileName = path_without_extension.substr(path_without_extension.find_last_of("/") + 1, path_without_extension.size()); *fileName =
*extension = path->substr(path->find_last_of(".") + 1, path->size()); path_without_extension.substr(path_without_extension.find_last_of("/") + 1, path_without_extension.size());
*extension = path->substr(path->find_last_of(".") + 1, path->size());
} }
int files::findFileName(std::string videoPath, std::string &fileName, std::string &extension) { std::pair<std::string, std::string> files::get_filename_and_extension(std::string videoPath) {
std::string fileName, extension;
files::findFileNameFromPath(&videoPath, &fileName, &extension); files::findFileNameFromPath(&videoPath, &fileName, &extension);
if (extension.compare("avi") != 0 && extension.compare("mp4") != 0 && extension.compare("mov") != 0) { return std::pair<std::string, std::string>(fileName, extension);
cerr << "Input file extension must be \"avi\", \"mp4\" or \"mov\"." << endl;
return -1;
} else {
cout << "Video to be analysed: " << endl;
cout << " File name: " << fileName << endl;
cout << " Extension: " << extension << endl;
}
return 0;
} }
#include <filesystem>
#include <fstream>
#include <string>
#include <iostream>
#include <stdlib.h>
namespace files {
/**
* @brief Save content to a file
*
* @param fileName the name of the file
* @param content the content to be saved
* @param append if true, the content will be appended to the file, otherwise the file will be overwritten
*/
void saveFile(std::filesystem::path fileName, std::string content, bool append);
/**
* @brief Separates video file name from its extension.
*
* @param[in] path Full video path;
* @param[out] fileName Video file name;
* @param[out] extension Video extension.
*/
void findFileNameFromPath(std::string* path, std::string* fileName, std::string* extension);
/**
* @brief Check if the specified input video file exists and is supported.
*
* @param[in] videoPath Full video path;
* @param[out] fileName Video file name;
* @param[out] extension Video extension.
* @return int -1 if the format is not supported, 0 otherwise.
*/
int findFileName(std::string videoPath, std::string &fileName, std::string &extension);
}
\ No newline at end of file
/**
* @file files.hpp
* @author Matteo Spanio (dev2@audioinnova.com)
* @brief A collection of functions to handle files.
* @version 1.0
* @date 2023-05-13
*
* @copyright Copyright (c) 2023
*
*/
#ifndef FILES_H
#define FILES_H
#include <stdlib.h>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <string>
/**
* @namespace files
* @brief A collection of functions to handle files.
*
*/
namespace files {
/**
* @fn void save_file(std::filesystem::path fileName, std::string content)
* @brief Save content to a file
*
* @param fileName the name of the file
* @param content the content to be saved
*/
void save_file(std::filesystem::path fileName, std::string content);
/**
* @fn void findFileNameFromPath(std::string* path, std::string* fileName,
* std::string* extension)
* @brief Separates video file name from its extension.
*
* @param[in] path Full video path;
* @param[out] fileName Video file name;
* @param[out] extension Video extension.
*/
void findFileNameFromPath(std::string* path, std::string* fileName, std::string* extension);
/**
* @fn std::pair<std::string, std::string> findFileName(std::string videoPath)
* @brief Check if the specified input video file exists and is supported.
*
* @param videoPath Full video path;
* @return a pair of strings containing the video file name and its extension.
*/
std::pair<std::string, std::string> get_filename_and_extension(std::string videoPath);
} // namespace files
#endif // FILES_H
\ No newline at end of file
#include "io.hpp"
namespace videoanalyser {
namespace io {
void pprint(std::string msg, Color color) { std::cout << color << msg << END << std::endl; }
void print_error_and_exit(std::string msg) {
std::cerr << RED << msg << END << std::endl;
exit(EXIT_FAILURE);
}
} // namespace io
} // namespace videoanalyser
\ No newline at end of file