/**
* @file io.hpp
* @author Matteo Spanio (dev2@audioinnova.com)
* @brief Header file containing a set of functions related to the input/output
* @version 1.2
* @date 2023-06-03
*
* @copyright Copyright (c) 2023
*
*/
#ifndef PRETTYIO_H
#define PRETTYIO_H
#include <iostream>
#include <string>
#include "colors.hpp"
using namespace colors;
namespace videoanalyser {
/**
* @namespace io
* @brief Namespace containing all the functions related to the input/output
* operations.
*/
namespace io {
/**
* @fn void pprint(std::string msg, Color color = WHITE)
* @brief Print a colored message in the terminal.
*
* @param msg The message to print.
* @param color The color of the message. Default is WHITE.
*/
void pprint(std::string msg, Color color = WHITE);
/**
* @fn void print_error_and_exit(std::string msg)
* @brief Print a colored message in the terminal and exit the program.
*
* @param msg The message to print.
*/
void print_error_and_exit(std::string msg);
} // namespace io
} // namespace videoanalyser
#endif // PRETTYIO_H
\ No newline at end of file
#include "time.h"
#include "time.hpp"
/**
* @brief Convert an int representing milliseconds to the corresponding Time Label string.
*
* @param ms the number of milliseconds.
* @param delim the time separator
* @return string the corresponding Time Label string.
*/
std::string getTimeLabel(int ms, std::string delim) {
int mil = ms % 1000;
int sec = ms / 1000;
int min = (sec / 60) % 60;
int hours = sec / 3600;
sec = sec % 60;
int mil = ms % 1000;
int sec = ms / 1000;
int min = (sec / 60) % 60;
int hours = sec / 3600;
sec = sec % 60;
std::string hoursStr = std::to_string(hours), minStr = std::to_string(min), secStr = std::to_string(sec), milStr = std::to_string(mil);
if (hours < 10)
hoursStr = "0" + hoursStr;
if (min < 10)
minStr = "0" + minStr;
if (sec < 10)
secStr = "0" + secStr;
if (mil < 100) {
if (mil < 10) {
milStr = "00" + milStr;
} else {
milStr = "0" + milStr;
}
}
std::string hoursStr = std::to_string(hours), minStr = std::to_string(min), secStr = std::to_string(sec),
milStr = std::to_string(mil);
if (hours < 10) hoursStr = "0" + hoursStr;
if (min < 10) minStr = "0" + minStr;
if (sec < 10) secStr = "0" + secStr;
if (mil < 100) {
if (mil < 10) {
milStr = "00" + milStr;
} else {
milStr = "0" + milStr;
}
}
std::string timeLabel = hoursStr + delim + minStr + delim + secStr + delim + milStr;
return timeLabel;
return timeLabel;
}
int time_label_to_ms(std::string time_label) {
// Obtain time measures from JSON
int h = stoi(time_label.substr(0, 2));
int min = stoi(time_label.substr(3, 2));
int sec = stoi(time_label.substr(6, 2));
int ms = stoi(time_label.substr(9, 3));
// Compute the Irregularity instant in milliseconds
return ms + sec * 1000 + min * 60000 + h * 3600000;
}
\ No newline at end of file
#ifndef GETTIMELABEL_H
#define GETTIMELABEL_H
#include <stdlib.h>
#include <string>
std::string getTimeLabel(int ms, std::string delim = ":");
#endif
\ No newline at end of file
#ifndef GETTIMELABEL_H
#define GETTIMELABEL_H
#include <stdlib.h>
#include <string>
/**
* @fn std::string getTimeLabel(int ms, std::string delim = ":")
* @brief Convert an int representing milliseconds to the corresponding Time
* Label string.
*
* @param ms the number of milliseconds.
* @param delim the time separator
* @return string the corresponding Time Label string.
*/
std::string getTimeLabel(int ms, std::string delim = ":");
int time_label_to_ms(std::string time_label);
#endif // GETTIMELABEL_H
\ No newline at end of file
/**
* @mainpage MPAI CAE-ARP Video Analyser
* @file main.cpp
* MPAI CAE-ARP Video Analyser.
*
* Implements MPAI CAE-ARP Video Analyser Technical Specification.
* It identifies Irregularities on the Preservation Audio-Visual File, providing:
* It identifies Irregularities on the Preservation Audio-Visual File,
*providing:
* - Irregularity Files;
* - Irregularity Images.
*
* WARNING:
* Currently, this program is only compatible with the Studer A810 and videos recorded in PAL standard.
* @warning Currently, this program is only compatible with the Studer A810
*and videos recorded in PAL standard.
*
* @authors Nadir Dalla Pozza <nadir.dallapozza@unipd.it>, Matteo Spanio <dev2@audioinnova.com>
* @todo
* - A resize function of the entire video should be implemented if it does not
*conform to the PAL standard (currently taken for granted).
* - Progressive videos, which do not require deinterlacing, should be managed
*(in the code there are several steps that operate considering this property).
*
* @author Nadir Dalla Pozza <nadir.dallapozza@unipd.it>
* @author Matteo Spanio <dev2@audioinnova.com>
* @copyright 2023, Audio Innova S.r.l.
* @credits Niccolò Pretto, Nadir Dalla Pozza, Sergio Canazza
* @license GPL v3.0
* @version 1.1.1
* @version 1.2
* @status Production
*/
#include <filesystem>
#include <fstream>
#include <iostream>
#include <stdlib.h>
#include <sys/timeb.h>
#include <ranges>
#include <boost/program_options.hpp>
#include <boost/uuid/uuid.hpp> // uuid class
#include <boost/uuid/uuid_generators.hpp> // generators
#include <boost/uuid/uuid_io.hpp> // streaming operators etc.
#include <boost/lexical_cast.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui.hpp>
#include <variant>
#include "opencv2/core.hpp"
#include "opencv2/calib3d.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/features2d.hpp"
#include "opencv2/xfeatures2d.hpp"
#include "utility.h"
#include "forAudioAnalyser.h"
#include "lib/Irregularity.hpp"
#include "lib/IrregularityFile.hpp"
#include "lib/colors.hpp"
#include "lib/core.hpp"
#include "lib/detection.hpp"
#include "lib/files.hpp"
#include "lib/io.hpp"
#include "lib/time.hpp"
#include "utility.hpp"
#define BLACK_PIXEL 0
#include "lib/files.h"
#include "lib/colors.h"
#include "lib/time.h"
using namespace cv;
using namespace std;
using utility::Frame;
using namespace colors;
using json = nlohmann::json;
using videoanalyser::core::Frame;
using videoanalyser::io::pprint;
using videoanalyser::io::print_error_and_exit;
namespace fs = std::filesystem;
namespace po = boost::program_options;
namespace va = videoanalyser;
bool g_end_tape_saved = false;
bool g_first_brand = true; // The first frame containing brands on tape must be saved
float g_first_instant = 0;
float g_mean_prev_frame_color = 0; // Average frame color
static fs::path g_output_path{};
static fs::path g_irregularity_images_path{};
static json g_irregularity_file_1{};
static json g_irregularity_file_2{};
struct Args {
fs::path
working_path; /**< The working path where all input files are stored and where all output files will be saved */
string files_name; /**< The name of the preservation files to be considered */
bool brands; /**< True if tape presents brands on its surface */
float speed; /**< The speed at which the tape was read */
Args(fs::path working_path, string files_name, bool brands, float speed) {
if (speed != 7.5 && speed != 15) throw invalid_argument("Speed must be 7.5 or 15");
this->working_path = working_path;
this->files_name = files_name;
this->brands = brands;
this->speed = speed;
}
~Args() {}
// For capstan detection, there are two alternative approaches:
// Generalized Hough Transform and SURF.
bool useSURF = true;
bool savingPinchRoller = false;
bool pinchRollerRect = false;
bool savingBrand = false;
bool endTapeSaved = false;
float mediaPrevFrame = 0;
bool firstBrand = true; // The first frame containing brands on tape must be saved
float firstInstant = 0;
string fileName, extension;
// Path variables
static fs::path outputPath {};
static fs::path irregularityImagesPath {};
// JSON files
static json configurationFile {};
static json irregularityFileOutput1 {};
static json irregularityFileOutput2 {};
// RotatedRect identifying the processing area
RotatedRect rect, rectTape, rectCapstan;
// config.json parameters
struct Config {
fs::path workingPath;
string filesName;
bool brands;
float speed;
};
struct Threshold {
float percentual;
int angle;
int scale;
int pos;
};
static Args from_file(fs::path path) {
ifstream iConfig(path);
json j;
iConfig >> j;
return Args(fs::path(string(j["WorkingPath"])), j["FilesName"], j["Brands"], j["Speed"]);
}
struct SceneObject {
int minDist;
Threshold threshold;
static Args from_cli(int argc, char** argv) {
po::variables_map vm;
try {
po::options_description desc(
"A tool that implements MPAI CAE-ARP Video Analyser Technical "
"Specification.\n"
"By default, the configuartion parameters are loaded from "
"config/config.json file,\n"
"but, alternately, you can pass command line arguments to "
"replace them");
desc.add_options()("help,h", "Display this help message")(
"working-path,w", po::value<string>()->required(),
"Specify the Working Path, where all input files are stored")(
"files-name,f", po::value<string>()->required(),
"Specify the name of the Preservation files (without "
"extension)")("brands,b", po::value<bool>()->required(),
"Specify if the tape presents brands on its surface")(
"speed,s", po::value<float>()->required(), "Specify the speed at which the tape was read");
po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
if (vm.count("help")) {
std::cout << desc << "\n";
std::exit(EXIT_SUCCESS);
}
po::notify(vm);
} catch (po::invalid_command_line_syntax& e) {
print_error_and_exit("Invalid command line syntax: " + string(e.what()));
} catch (po::required_option& e) {
print_error_and_exit("Missing required option: " + string(e.what()));
} catch (nlohmann::detail::type_error e) {
print_error_and_exit("config.json error: " + string(e.what()));
}
return Args(fs::path(vm["working-path"].as<string>()), vm["files-name"].as<string>(), vm["brands"].as<bool>(),
vm["speed"].as<float>());
}
};
static Config config;
static SceneObject tape;
static SceneObject capstan;
// Constants Paths
static const string READING_HEAD_IMG = "input/readingHead.png";
static const string CAPSTAN_TEMPLATE_IMG = "input/capstanBERIO058prova.png";
static const string CONFIG_FILE = "config/config.json";
double rotatedRectArea(RotatedRect rect) {
return rect.size.width * rect.size.height;
}
/**
* @brief Prints a text in a given color.
*
* @param text
* @param color
*/
void pprint(string text, string color) {
cout << color << text << END << endl;
}
/**
* @brief Get operation arguments from command line or config.json file.
* @brief Get the next frame object.
*
* @param argc Command line arguments count;
* @param argv Command line arguments.
* @return true if input configuration is valid;
* @return false otherwise.
* Whenever we find an Irregularity, we want to skip a lenght equal to the
* Studer reading head (3 cm = 1.18 inches).
*
* Note the following considerations:
* - since we are analysing video at 25 fps a frame occurs every 40 ms
* - at 15 ips we cross 3 cm of tape in 79 ms (2 frames)
* - at 7.5 ips we cross 3 cm of tape in 157 ms (4 frames)
*
* The considered lengths are the widths of the tape areas.
* The following condition constitutes a valid approach if the tape areas
* have widths always equal to the reading head
*
* @param cap VideoCapture object
* @param speed tape reading speed
* @return Frame
*/
bool getArguments(int argc, char** argv) {
// Read configuration file
ifstream iConfig(CONFIG_FILE);
iConfig >> configurationFile;
if (argc == 1) {
// Read from JSON file
string working_path = configurationFile["WorkingPath"];
config = {
fs::path(working_path),
configurationFile["FilesName"],
configurationFile["Brands"],
configurationFile["Speed"]
};
} else {
// Get from command line
try {
po::options_description desc(
"A tool that implements MPAI CAE-ARP Video Analyser Technical Specification.\n"
"By default, the configuartion parameters are loaded from config/config.json file,\n"
"but, alternately, you can pass command line arguments to replace them"
);
desc.add_options()
("help,h", "Display this help message")
("working-path,w", po::value<string>()->required(), "Specify the Working Path, where all input files are stored")
("files-name,f", po::value<string>()->required(), "Specify the name of the Preservation files (without extension)")
("brands,b", po::value<bool>()->required(), "Specify if the tape presents brands on its surface")
("speed,s", po::value<float>()->required(), "Specify the speed at which the tape was read");
po::variables_map vm;
po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
if (vm.count("help")) {
cout << desc << "\n";
return false;
}
po::notify(vm);
// Access the stored options
config.workingPath = fs::path(vm["working-path"].as<string>());
config.filesName = vm["files-name"].as<string>();
config.brands = vm["brands"].as<bool>();
config.speed = vm["speed"].as<float>();
} catch (po::invalid_command_line_syntax& e) {
cerr << RED << BOLD << "The command line syntax is invalid: " << END << RED << e.what() << END << endl;
return false;
} catch (po::required_option& e) {
cerr << "Error: " << e.what() << endl;
return false;
}
}
capstan = {
configurationFile["MinDistCapstan"],
{
configurationFile["CapstanThresholdPercentual"],
configurationFile["AngleThreshCapstan"],
configurationFile["ScaleThreshCapstan"],
configurationFile["PosThreshCapstan"]
}
};
tape = {
configurationFile["MinDist"],
{
configurationFile["TapeThresholdPercentual"],
configurationFile["AngleThresh"],
configurationFile["ScaleThresh"],
configurationFile["PosThresh"]
}
};
return true;
Frame get_next_frame(VideoCapture& cap, float speed, bool skip = false) {
if (skip) {
int ms_to_skip = speed == 15 ? 79 : 157;
cap.set(CAP_PROP_POS_MSEC, cap.get(CAP_PROP_POS_MSEC) + ms_to_skip);
}
Frame frame;
cap >> frame;
return frame;
}
float rotated_rect_area(RotatedRect rect) { return rect.size.width * rect.size.height; }
/**
* @brief Find the model in the scene using the Generalized Hough Transform. It returns the best matches.
* Find the best matches for positive and negative angles.
* If there are more than one shapes, then choose the one with the highest score.
* If there are more than one with the same highest score, then arbitrarily choose the latest
*
* @param model the template image to be searched with the Generalized Hough Transform
* @param object the sceneObject struct containing the parameters for the Generalized Hough Transform
* @return std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>> a tuple containing the best matches for positive and negative angles
*/
std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>> findObject(Mat model, SceneObject object, Mat processing_area) {
// Algorithm and parameters
// for informations about the Generalized Hough Guild usage see the tutorial at https://docs.opencv.org/4.7.0/da/ddc/tutorial_generalized_hough_ballard_guil.html
Ptr<GeneralizedHoughGuil> alg = createGeneralizedHoughGuil();
vector<Vec4f> positionsPos, positionsNeg;
Mat votesPos, votesNeg;
double maxValPos = 0, maxValNeg = 0;
int indexPos = 0, indexNeg = 0;
alg->setMinDist(object.minDist);
alg->setLevels(360);
alg->setDp(2);
alg->setMaxBufferSize(1000);
alg->setAngleStep(1);
alg->setAngleThresh(object.threshold.angle);
alg->setMinScale(0.9);
alg->setMaxScale(1.1);
alg->setScaleStep(0.01);
alg->setScaleThresh(object.threshold.scale);
alg->setPosThresh(object.threshold.pos);
alg->setCannyLowThresh(150); // Old: 100
alg->setCannyHighThresh(240); // Old: 300
alg->setTemplate(model);
utility::detectShape(alg, model, object.threshold.pos, positionsPos, votesPos, positionsNeg, votesNeg, processing_area);
for (int i = 0; i < votesPos.size().width; i++) {
if (votesPos.at<int>(i) >= maxValPos) {
maxValPos = votesPos.at<int>(i);
indexPos = i;
}
}
for (int i = 0; i < votesNeg.size().width; i++) {
if (votesNeg.at<int>(i) >= maxValNeg) {
maxValNeg = votesNeg.at<int>(i);
indexNeg = i;
}
}
return { indexPos, indexNeg, maxValPos, maxValNeg, positionsPos, positionsNeg };
}
/**
* @fn bool find_processing_areas(Frame frame, SceneObject tape, SceneObject capstan)
* @brief Identifies the Regions Of Interest (ROIs) on the video,
* which are:
* - The reading head;
* - The tape area under the tape head (computed on the basis of the detected reading head);
* - The tape area under the tape head (computed on the basis of the detected
* reading head);
* - The capstan.
* @param myFrame The current frame of the video.
*
*
*
* @param frame the frame to be analysed
* @param tape the tape object
* @param capstan the capstan object
* @return true if some areas have been detected;
* @return false otherwise.
*/
bool findProcessingAreas(Mat myFrame) {
/*********************************************************************************************/
/*********************************** READING HEAD DETECTION **********************************/
/*********************************************************************************************/
// Save a grayscale version of myFrame in myFrameGrayscale and downsample it in half pixels for performance reasons
Frame gray_current_frame = Frame(myFrame)
.convertColor(COLOR_BGR2GRAY);
Frame halved_gray_current_frame = gray_current_frame
.clone()
.downsample(2);
// Get input shape in grayscale and downsample it in half pixels
Frame reading_head_template = Frame(cv::imread(READING_HEAD_IMG, IMREAD_GRAYSCALE)).downsample(2);
// Process only the bottom-central portion of the input video -> best results with our videos
Rect readingHeadProcessingAreaRect(
halved_gray_current_frame.cols/4,
halved_gray_current_frame.rows/2,
halved_gray_current_frame.cols/2,
halved_gray_current_frame.rows/2
);
Mat processingImage = halved_gray_current_frame(readingHeadProcessingAreaRect);
RotatedRect rectPos, rectNeg;
auto [indexPos, indexNeg, maxValPos, maxValNeg, positionsPos, positionsNeg] = findObject(reading_head_template, tape, processingImage);
// The color is progressively darkened to emphasize that the algorithm found more than one shape
if (positionsPos.size() > 0)
rectPos = utility::drawShapes(myFrame, positionsPos[indexPos], Scalar(0, 0, 255-indexPos*64), reading_head_template.cols, reading_head_template.rows, halved_gray_current_frame.cols/4, halved_gray_current_frame.rows/2, 2);
if (positionsNeg.size() > 0)
rectNeg = utility::drawShapes(myFrame, positionsNeg[indexNeg], Scalar(128, 128, 255-indexNeg*64), reading_head_template.cols, reading_head_template.rows, halved_gray_current_frame.cols/4, halved_gray_current_frame.rows/2, 2);
if (maxValPos > 0)
if (maxValNeg > 0)
if (maxValPos > maxValNeg) {
rect = rectPos;
} else {
rect = rectNeg;
}
else {
rect = rectPos;
}
else if (maxValNeg > 0) {
rect = rectNeg;
} else {
return false;
}
/*********************************************************************************************/
/************************************ TAPE AREA DETECTION ************************************/
/*********************************************************************************************/
// Compute area basing on reading head detection
Vec4f positionTape( rect.center.x, rect.center.y + rect.size.height / 2 + 20 * (rect.size.width / 200), 1, rect.angle );
rectTape = utility::drawShapes(myFrame, positionTape, Scalar(0, 255-indexPos*64, 0), rect.size.width, 50 * (rect.size.width / 200), 0, 0, 1);
/*********************************************************************************************/
/************************************* CAPSTAN DETECTION *************************************/
/*********************************************************************************************/
// Read template image - it is smaller than before, therefore there is no need to downsample
Mat templateShape = imread(CAPSTAN_TEMPLATE_IMG, IMREAD_GRAYSCALE);
// templateShape = imread("../input/capstanBERIO058.png", IMREAD_GRAYSCALE);
if (useSURF) {
// Step 1: Detect the keypoints using SURF Detector, compute the descriptors
int minHessian = 100;
Ptr<xfeatures2d::SURF> detector = xfeatures2d::SURF::create(minHessian);
vector<KeyPoint> keypoints_object, keypoints_scene;
Mat descriptors_object, descriptors_scene;
detector->detectAndCompute(templateShape, noArray(), keypoints_object, descriptors_object);
detector->detectAndCompute(gray_current_frame, noArray(), keypoints_scene, descriptors_scene);
// Step 2: Matching descriptor vectors with a FLANN based matcher
// Since SURF is a floating-point descriptor NORM_L2 is used
Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create(DescriptorMatcher::FLANNBASED);
vector<vector<DMatch>> knn_matches;
matcher->knnMatch(descriptors_object, descriptors_scene, knn_matches, 2);
//-- Filter matches using the Lowe's ratio test
const float ratio_thresh = 0.75f;
vector<DMatch> good_matches;
for (size_t i = 0; i < knn_matches.size(); i++) {
if (knn_matches[i][0].distance < ratio_thresh * knn_matches[i][1].distance) {
good_matches.push_back(knn_matches[i][0]);
}
}
// Draw matches
Mat img_matches;
drawMatches(templateShape, keypoints_object, halved_gray_current_frame, keypoints_scene, good_matches, img_matches, Scalar::all(-1), Scalar::all(-1), vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
// Localize the object
vector<Point2f> obj;
vector<Point2f> scene;
for (size_t i = 0; i < good_matches.size(); i++) {
// Get the keypoints from the good matches
obj.push_back(keypoints_object[good_matches[i].queryIdx].pt);
scene.push_back(keypoints_scene[good_matches[i].trainIdx].pt);
}
Mat H = findHomography(obj, scene, RANSAC);
// Get the corners from the image_1 ( the object to be "detected" )
vector<Point2f> obj_corners(4);
obj_corners[0] = Point2f(0, 0);
obj_corners[1] = Point2f((float)templateShape.cols, 0);
obj_corners[2] = Point2f((float)templateShape.cols, (float)templateShape.rows);
obj_corners[3] = Point2f(0, (float)templateShape.rows);
vector<Point2f> scene_corners(4);
perspectiveTransform( obj_corners, scene_corners, H);
// Find average
float capstanX = (scene_corners[0].x + scene_corners[1].x + scene_corners[2].x + scene_corners[3].x) / 4;
float capstanY = (scene_corners[0].y + scene_corners[1].y + scene_corners[2].y + scene_corners[3].y) / 4;
// In the following there are two alterations to cut the first 20 horizontal pixels and the first 90 vertical pixels from the found rectangle:
// +10 in X for centering and -20 in width
// +45 in Y for centering and -90 in height
Vec4f positionCapstan(capstanX + 10, capstanY + 45, 1, 0);
rectCapstan = utility::drawShapes(myFrame, positionCapstan, Scalar(255-indexPos*64, 0, 0), templateShape.cols - 20, templateShape.rows - 90, 0, 0, 1);
} else {
// Process only right portion of the image, where the capstain always appears
int capstanProcessingAreaRectX = myFrame.cols*3/4;
int capstanProcessingAreaRectY = myFrame.rows/2;
int capstanProcessingAreaRectWidth = myFrame.cols/4;
int capstanProcessingAreaRectHeight = myFrame.rows/2;
Rect capstanProcessingAreaRect(capstanProcessingAreaRectX, capstanProcessingAreaRectY, capstanProcessingAreaRectWidth, capstanProcessingAreaRectHeight);
Mat capstanProcessingAreaGrayscale = gray_current_frame(capstanProcessingAreaRect);
// Reset algorithm and set parameters
auto [indexPos, indexNeg, maxValPos, maxValNeg, positionsC1Pos, positionsC1Neg] = findObject(templateShape, capstan, capstanProcessingAreaGrayscale);
RotatedRect rectCapstanPos, rectCapstanNeg;
if (positionsC1Pos.size() > 0)
rectCapstanPos = utility::drawShapes(myFrame, positionsC1Pos[indexPos], Scalar(255-indexPos*64, 0, 0), templateShape.cols-22, templateShape.rows-92, capstanProcessingAreaRectX+11, capstanProcessingAreaRectY+46, 1);
if (positionsC1Neg.size() > 0)
rectCapstanNeg = utility::drawShapes(myFrame, positionsC1Neg[indexNeg], Scalar(255-indexNeg*64, 128, 0), templateShape.cols-22, templateShape.rows-92, capstanProcessingAreaRectX+11, capstanProcessingAreaRectY+46, 1);
if (maxValPos > 0)
if (maxValNeg > 0)
if (maxValPos > maxValNeg) {
rectCapstan = rectCapstanPos;
} else {
rectCapstan = rectCapstanNeg;
}
else {
rectCapstan = rectCapstanPos;
}
else if (maxValNeg > 0) {
rectCapstan = rectCapstanNeg;
} else {
return false;
}
}
cout << endl;
// Save the image containing the detected areas
cv::imwrite(outputPath.string() + "/tapeAreas.jpg", myFrame);
return true;
va::Result<pair<va::detection::Roi, va::detection::Roi>> find_processing_areas(Frame frame, SceneObject tape,
SceneObject capstan) {
va::detection::SceneElement tape_element{
va::detection::ElementType::TAPE,
tape.minDist,
{tape.threshold.percentual, tape.threshold.angle, tape.threshold.scale, tape.threshold.pos}};
auto tape_roi_result = va::detection::find_roi(frame, va::detection::Algorithm::GHT, tape_element);
if (std::holds_alternative<va::Error>(tape_roi_result))
return va::Error("Error while finding tape roi: " + std::get<va::Error>(tape_roi_result));
auto rect_tape = std::get<va::detection::Roi>(tape_roi_result);
va::detection::SceneElement capstan_element{
va::detection::ElementType::CAPSTAN,
capstan.minDist,
{capstan.threshold.percentual, capstan.threshold.angle, capstan.threshold.scale, capstan.threshold.pos}};
auto capstan_roi_result = va::detection::find_roi(frame, va::detection::Algorithm::SURF, capstan_element);
if (std::holds_alternative<va::Error>(capstan_roi_result))
return va::Error("Error while finding capstan roi: " + std::get<va::Error>(capstan_roi_result));
auto rect_capstan = std::get<va::detection::Roi>(capstan_roi_result);
// save detected areas to file
cv::rectangle(frame, rect_tape.boundingRect(), cv::Scalar(0, 255, 0), 2);
cv::rectangle(frame, rect_capstan.boundingRect(), cv::Scalar(255, 0, 0), 2);
cv::imwrite(g_output_path.string() + "/my_tape_areas.jpg", frame);
return make_pair(rect_tape, rect_capstan);
}
/**
* @fn RotatedRect check_skew(RotatedRect roi)
* @brief Check if the region of interest is skewed and correct it
*
* @param roi the region of interest
* @return RotatedRect the corrected region of interest
*/
RotatedRect check_skew(RotatedRect roi) {
// get angle and size from the bounding box
// thanks to http://felix.abecassis.me/2011/10/opencv-rotation-deskewing/
cv::Size rect_size = roi.size;
float angle = roi.angle;
if (roi.angle < -45.) {
angle += 90.0;
swap(rect_size.width, rect_size.height);
}
return RotatedRect(roi.center, rect_size, angle);
// get angle and size from the bounding box
// thanks to http://felix.abecassis.me/2011/10/opencv-rotation-deskewing/
cv::Size rect_size = roi.size;
float angle = roi.angle;
if (roi.angle < -45.) {
angle += 90.0;
std::swap(rect_size.width, rect_size.height);
}
return RotatedRect(roi.center, rect_size, angle);
}
/**
* @brief Look for differences in two consecutive frames in a specific region of interest
*
* @fn Frame get_difference_for_roi(Frame previous, Frame current, RotatedRect
* roi)
* @brief Look for differences in two consecutive frames in a specific region of
* interest
*
* @param previous the reference frame
* @param current the frame to compare with the reference
* @param roi the region of interest
* @return Frame the difference matrix between the two frames
*/
Frame get_difference_for_roi(Frame previous, Frame current, RotatedRect roi) {
cv::Mat rotation_matrix = getRotationMatrix2D(roi.center, roi.angle, 1.0);
return previous
.warp(rotation_matrix)
.crop(roi.size, roi.center)
.difference(current
.warp(rotation_matrix)
.crop(roi.size, roi.center));
cv::Mat rotation_matrix = cv::getRotationMatrix2D(roi.center, roi.angle, 1.0);
return previous.warp(rotation_matrix)
.crop(roi.size, roi.center)
.difference(current.warp(rotation_matrix).crop(roi.size, roi.center));
}
/**
* @brief Compares two consecutive video frames and establish if there potentially is an Irregularity.
* The comparison is pixel-wise and based on threshold values set on config.json file.
* @fn bool is_frame_different(cv::Mat prev_frame, cv::Mat current_frame, int
* ms_to_end, SceneObject capstan, SceneObject tape, Args args, va::detection::Roi rect_tape, va::detection::Roi
* rect_capstan)
* @brief Compares two consecutive video frames and establish if there
* potentially is an Irregularity. The comparison is pixel-wise and based on
* threshold values set on config.json file.
*
* @param prevFrame the frame before the current one;
* @param currentFrame the current frame;
* @param msToEnd the number of milliseconds left before the end of the video. Useful for capstan analysis.
* @param prev_frame the frame before the current one;
* @param current_frame the current frame;
* @param ms_to_end the number of milliseconds left before the end of the video.
* Useful for capstan analysis.
* @param capstan the capstan object;
* @param tape the tape object;
* @param args the command line arguments;
* @param rect_tape the tape area under the tape head;
* @param rect_capstan the capstan area.
* @return true if a potential Irregularity has been found;
* @return false otherwise.
*/
bool frameDifference(cv::Mat prevFrame, cv::Mat currentFrame, int msToEnd) {
/********************************** Capstan analysis *****************************************/
// In the last minute of the video, check for pinchRoller position for endTape event
if (!endTapeSaved && msToEnd < 60000) {
// Capstan area
int capstanAreaPixels = rectCapstan.size.width * rectCapstan.size.height;
float capstanDifferentPixelsThreshold = capstanAreaPixels * capstan.threshold.percentual / 100;
RotatedRect corrected_capstan_roi = check_skew(rectCapstan);
Frame difference_frame = get_difference_for_roi(Frame(prevFrame), Frame(currentFrame), corrected_capstan_roi);
int blackPixelsCapstan = 0;
for (int i = 0; i < difference_frame.rows; i++) {
for (int j = 0; j < difference_frame.cols; j++) {
if (difference_frame.at<cv::Vec3b>(i, j)[0] == 0) {
// There is a black pixel, then there is a difference between previous and current frames
blackPixelsCapstan++;
}
}
}
if (blackPixelsCapstan > capstanDifferentPixelsThreshold) {
savingPinchRoller = true;
endTapeSaved = true; // Never check again for end tape instant
return true;
}
}
savingPinchRoller = false; // It will already be false before the last minute of the video. After having saved the capstan, the next time reset the variable to not save again
/************************************ Tape analysis ******************************************/
// Tape area
int tapeAreaPixels = rotatedRectArea(rectTape);
float tapeDifferentPixelsThreshold = tapeAreaPixels * tape.threshold.percentual / 100;
RotatedRect corrected_tape_roi = check_skew(rectTape);
Frame croppedCurrentFrame = Frame(currentFrame)
.warp(getRotationMatrix2D(corrected_tape_roi.center, corrected_tape_roi.angle, 1.0))
.crop(corrected_tape_roi.size, corrected_tape_roi.center);
Frame difference_frame = get_difference_for_roi(Frame(prevFrame), Frame(currentFrame), corrected_tape_roi);
int decEnd = (msToEnd % 1000) / 100;
int secEnd = (msToEnd - (msToEnd % 1000)) / 1000;
int minEnd = secEnd / 60;
secEnd = secEnd % 60;
/************************************* Segment analysis **************************************/
int blackPixels = 0;
float mediaCurrFrame;
int totColoreCF = 0;
for (int i = 0; i < croppedCurrentFrame.rows; i++) {
for (int j = 0; j < croppedCurrentFrame.cols; j++) {
totColoreCF += croppedCurrentFrame.at<cv::Vec3b>(i, j)[0] + croppedCurrentFrame.at<cv::Vec3b>(i, j)[1] + croppedCurrentFrame.at<cv::Vec3b>(i, j)[2];
if (difference_frame.at<cv::Vec3b>(i, j)[0] == 0) {
blackPixels++;
}
}
}
mediaCurrFrame = totColoreCF/tapeAreaPixels;
/************************************* Decision stage ****************************************/
bool isIrregularity = false;
if (blackPixels > tapeDifferentPixelsThreshold) { // The threshold must be passed
/***** AVERAGE_COLOR-BASED DECISION *****/
if (mediaPrevFrame > (mediaCurrFrame + 7) || mediaPrevFrame < (mediaCurrFrame - 7)) { // They are not similar for color average
isIrregularity = true;
}
/***** BRANDS MANAGEMENT *****/
if (config.brands) {
// At the beginning of the video, wait at least 5 seconds before the next Irregularity to consider it as a brand.
// It is not guaranteed that it will be the first brand, but it is generally a safe approach to have a correct image
if (firstBrand) {
if (firstInstant - msToEnd > 5000) {
firstBrand = false;
savingBrand = true;
isIrregularity = true;
}
// In the following iterations reset savingBrand, since we are no longer interested in brands.
} else
savingBrand = false;
}
bool is_frame_different(cv::Mat prev_frame, cv::Mat current_frame, int ms_to_end, SceneObject capstan, SceneObject tape,
Args args, va::detection::Roi rect_tape, va::detection::Roi rect_capstan) {
bool result = false;
int num_different_pixels = 0;
/*********************** Capstan analysis ************************/
// In the last minute of the video, check for pinchRoller position for
// endTape event
if (!g_end_tape_saved && ms_to_end < 60000) {
RotatedRect corrected_capstan_roi = check_skew(rect_capstan);
Frame difference_frame = get_difference_for_roi(Frame(prev_frame), Frame(current_frame), corrected_capstan_roi);
for (int i = 0; i < difference_frame.rows; i++) {
for (int j = 0; j < difference_frame.cols; j++) {
if (difference_frame.at<cv::Vec3b>(i, j)[0] == BLACK_PIXEL) {
// There is a black pixel, then there is a difference
// between previous and current frames
num_different_pixels++;
}
}
}
float capstan_pixel_threshold = rotated_rect_area(rect_capstan) * capstan.threshold.percentual / 100;
if (num_different_pixels > capstan_pixel_threshold) {
g_end_tape_saved = true; // Never check again for end tape instant
return true;
}
}
}
/********************* Tape analysis *********************/
RotatedRect corrected_tape_roi = check_skew(rect_tape);
Frame difference_frame = get_difference_for_roi(Frame(prev_frame), Frame(current_frame), corrected_tape_roi);
/********************** Segment analysis ************************/
Frame cropped_current_frame =
Frame(current_frame)
.warp(cv::getRotationMatrix2D(corrected_tape_roi.center, corrected_tape_roi.angle, 1.0))
.crop(corrected_tape_roi.size, corrected_tape_roi.center);
num_different_pixels = 0;
float mean_current_frame_color;
int current_frame_color_sum = 0;
for (int i = 0; i < cropped_current_frame.rows; i++) {
for (int j = 0; j < cropped_current_frame.cols; j++) {
current_frame_color_sum += cropped_current_frame.at<cv::Vec3b>(i, j)[0] +
cropped_current_frame.at<cv::Vec3b>(i, j)[1] +
cropped_current_frame.at<cv::Vec3b>(i, j)[2];
if (difference_frame.at<cv::Vec3b>(i, j)[0] == BLACK_PIXEL) {
num_different_pixels++;
}
}
}
float tape_area_pixels_sq = rotated_rect_area(rect_tape);
mean_current_frame_color = current_frame_color_sum / tape_area_pixels_sq;
/*********************** Decision stage ************************/
float tape_pixel_threshold = tape_area_pixels_sq * tape.threshold.percentual / 100;
if (num_different_pixels > tape_pixel_threshold) { // The threshold must be passed
/***** AVERAGE_COLOR-BASED DECISION *****/
if (g_mean_prev_frame_color > (mean_current_frame_color + 7) ||
g_mean_prev_frame_color < (mean_current_frame_color - 7)) { // They are not similar for color average
result = true;
}
/***** BRANDS MANAGEMENT *****/
// At the beginning of the video, wait at least 5 seconds before the
// next Irregularity to consider it as a brand. It is not guaranteed
// that it will be the first brand, but it is generally a safe
// approach to have a correct image
if (args.brands && g_first_brand && g_first_instant - ms_to_end > 5000) {
g_first_brand = false;
result = true;
}
}
// Update mediaPrevFrame
mediaPrevFrame = mediaCurrFrame;
g_mean_prev_frame_color = mean_current_frame_color;
return isIrregularity;
return result;
}
/**
* @fn void processing(cv::VideoCapture video_capture, SceneObject capstan,
* SceneObject tape, Args args, va::detection::Roi rect_tape, va::detection::Roi rect_capstan)
* @brief video processing phase, where each frame is analysed.
* It saves the IrregularityImages and updates the IrregularityFiles if an Irregularity is found
* It saves the IrregularityImages and updates the IrregularityFiles if an
* Irregularity is found
*
* @note To be able to work with the "old" neural network (by Ilenya),
* the output images should correspond to the old "whole tape" where, from the
* frame judged as interesting, an area corresponding to the height of the tape
* was extracted (so about the height of the current rectangle) and as wide as
* the original frame (so 720px). This area will then have to be resized to
* 224x224 as in the past. If instead you decide to use the new neural network,
* no changes are needed.
*
* @param videoCapture the input Preservation Audio-Visual File;
* @param video_capture the input Preservation Audio-Visual File;
* @param capstan the capstan SceneObject;
* @param tape the tape SceneObject;
* @param args the command line arguments.
* @param rect_tape the tape Roi;
* @param rect_capstan the capstan Roi.
*/
void processing(cv::VideoCapture videoCapture) {
const int video_length_ms = ((float) videoCapture.get(CAP_PROP_FRAME_COUNT) / videoCapture.get(CAP_PROP_FPS)) * 1000;
int video_current_ms = videoCapture.get(CAP_PROP_POS_MSEC);
// counters
int savedFrames = 0;
int unsavedFrames = 0;
float lastSaved = -160;
// Whenever we find an Irregularity, we want to skip a lenght equal to the Studer reading head (3 cm = 1.18 inches).
int savingRate = 79; // [ms]. Time taken to cross 3 cm at 15 ips, or 1.5 cm at 7.5 ips. The considered lengths are the widths of the tape areas.
// The following condition constitutes a valid approach if the tape areas have widths always equal to the reading head
if (config.speed == 7.5)
savingRate = 157; // Time taken to cross 3 cm at 7.5 ips
// The first frame of the video won't be processed
cv::Mat prevFrame;
videoCapture >> prevFrame;
firstInstant = video_length_ms - video_current_ms;
while (videoCapture.isOpened()) {
cv::Mat frame;
videoCapture >> frame;
video_current_ms = videoCapture.get(CAP_PROP_POS_MSEC);
if (frame.empty()) {
cout << endl << "Empty frame!" << endl;
videoCapture.release();
break;
}
int msToEnd = video_length_ms - video_current_ms;
if (video_current_ms == 0) // With OpenCV library, this happens at the last few frames of the video before realising that "frame" is empty.
break;
// Variables to display program status
int secToEnd = msToEnd / 1000;
int minToEnd = (secToEnd / 60) % 60;
secToEnd = secToEnd % 60;
string secStrToEnd = to_string(secToEnd), minStrToEnd = to_string(minToEnd);
if (minToEnd < 10)
minStrToEnd = "0" + minStrToEnd;
if (secToEnd < 10)
secStrToEnd = "0" + secStrToEnd;
// Display program status
cout << "\rIrregularities: " << savedFrames << ". ";
cout << "Remaining video time [mm:ss]: " << minStrToEnd << ":" << secStrToEnd << flush;
if ((video_current_ms - lastSaved > savingRate) && frameDifference(prevFrame, frame, msToEnd)) {
// An Irregularity has been found!
// De-interlacing frame
cv::Mat oddFrame(frame.rows/2, frame.cols, CV_8UC3);
cv::Mat evenFrame(frame.rows/2, frame.cols, CV_8UC3);
utility::separateFrame(frame, oddFrame, evenFrame);
// Extract the image corresponding to the ROIs
Point2f pts[4];
if (savingPinchRoller)
rectCapstan.points(pts);
else
rectTape.points(pts);
cv::Mat subImage(frame, cv::Rect(100, min(pts[1].y, pts[2].y), frame.cols - 100, static_cast<int>(rectTape.size.height)));
// De-interlacing
cv::Mat oddSubImage(subImage.rows/2, subImage.cols, CV_8UC3);
int evenSubImageRows = subImage.rows/2;
if (subImage.rows % 2 != 0) // If the found rectangle is of odd height, we must increase evenSubImage height by 1, otherwise we have segmentation_fault!!!
evenSubImageRows += 1;
cv::Mat evenSubImage(evenSubImageRows, subImage.cols, CV_8UC3);
utility::separateFrame(subImage, oddSubImage, evenSubImage);
string timeLabel = getTimeLabel(video_current_ms, ":");
string safeTimeLabel = getTimeLabel(video_current_ms, "-");
string irregularityImageFilename = to_string(savedFrames) + "_" + safeTimeLabel + ".jpg";
cv::imwrite(irregularityImagesPath / irregularityImageFilename, oddFrame);
// Append Irregularity information to JSON
boost::uuids::uuid uuid = boost::uuids::random_generator()();
irregularityFileOutput1["Irregularities"] += {
{
"IrregularityID", boost::lexical_cast<string>(uuid)
}, {
"Source", "v"
}, {
"TimeLabel", timeLabel
}
};
irregularityFileOutput2["Irregularities"] += {
{
"IrregularityID", boost::lexical_cast<string>(uuid)
}, {
"Source", "v"
}, {
"TimeLabel", timeLabel
}, {
"ImageURI", irregularityImagesPath.string() + "/" + irregularityImageFilename
}
};
lastSaved = video_current_ms;
savedFrames++;
} else {
unsavedFrames++;
}
prevFrame = frame;
}
void processing(cv::VideoCapture video_capture, SceneObject capstan, SceneObject tape, Args args,
va::detection::Roi rect_tape, va::detection::Roi rect_capstan) {
const int video_length_ms =
((float)video_capture.get(CAP_PROP_FRAME_COUNT) / video_capture.get(CAP_PROP_FPS)) * 1000;
int video_current_ms = video_capture.get(CAP_PROP_POS_MSEC);
int num_saved_frames = 0;
bool irregularity_found = false;
// The first frame of the video won't be processed
cv::Mat prev_frame = get_next_frame(video_capture, args.speed, irregularity_found);
g_first_instant = video_length_ms - video_current_ms;
while (video_capture.isOpened()) {
Frame frame = get_next_frame(video_capture, args.speed, irregularity_found);
video_current_ms = video_capture.get(CAP_PROP_POS_MSEC);
if (frame.empty()) {
std::cout << endl << "Empty frame!" << endl;
video_capture.release();
return;
}
int ms_to_end = video_length_ms - video_current_ms;
if (video_current_ms == 0) // With OpenCV library, this happens at the last few frames of
// the video before realising that "frame" is empty.
return;
// Display program status
int sec_to_end = ms_to_end / 1000;
int min_to_end = (sec_to_end / 60) % 60;
sec_to_end = sec_to_end % 60;
string sec_str_to_end = (sec_to_end < 10 ? "0" : "") + to_string(sec_to_end);
string min_str_to_end = (min_to_end < 10 ? "0" : "") + to_string(min_to_end);
std::cout << "\rIrregularities: " << num_saved_frames << ". ";
std::cout << "Remaining video time [mm:ss]: " << min_str_to_end << ":" << sec_str_to_end << flush;
irregularity_found =
is_frame_different(prev_frame, frame, ms_to_end, capstan, tape, args, rect_tape, rect_capstan);
if (irregularity_found) {
auto [odd_frame, _] = frame.deinterlace();
string irregularityImageFilename =
to_string(num_saved_frames) + "_" + getTimeLabel(video_current_ms, "-") + ".jpg";
cv::imwrite(g_irregularity_images_path / irregularityImageFilename, odd_frame);
// Append Irregularity information to JSON
Irregularity irreg = Irregularity(Source::Video, getTimeLabel(video_current_ms, ":"));
g_irregularity_file_1["Irregularities"] += irreg.to_JSON();
g_irregularity_file_2["Irregularities"] +=
irreg.set_image_URI(g_irregularity_images_path.string() + "/" + irregularityImageFilename).to_JSON();
num_saved_frames++;
}
prev_frame = frame;
}
}
/**
* @fn int main(int argc, char** argv)
* @brief main program, organised as:
* - Get input from command line or config.json file;
* - Check input parameters;
......@@ -719,151 +443,79 @@ void processing(cv::VideoCapture videoCapture) {
* - Irregularities detection;
* - Saving of output IrregularityFiles.
*
* @todo The main function should be splitted into 2 steps, and each step should be callable from the command line, so
* that the user can choose to run only the first step, only the second step, or both:
* - First step: generate irregularity file output 1;
* - Second step: generate irregularity file output 2.
*
* @param argc Command line arguments count;
* @param argv Command line arguments.
* @return int program status.
*/
int main(int argc, char** argv) {
const string CONFIG_FILE = "config/config.json";
const string A_IRREG_FILE_1 = "AudioAnalyser_IrregularityFileOutput1.json";
const string V_IRREG_FILE_1 = "VideoAnalyser_IrregularityFileOutput1.json";
const string V_IRREG_FILE_2 = "VideoAnalyser_IrregularityFileOutput2.json";
json irregularityFileInput;
fs::path irregularityFileInputPath;
cv::Mat myFrame;
/*********************************************************************************************/
/*************************************** CONFIGURATION ***************************************/
/*********************************************************************************************/
// Get the input from config.json or command line
try {
bool continueExecution = getArguments(argc, argv);
if (!continueExecution) {
return 0;
}
} catch (nlohmann::detail::type_error e) {
cerr << RED << "config.json error!" << endl << e.what() << END << endl;
return -1;
}
const fs::path VIDEO_PATH = config.workingPath / "PreservationAudioVisualFile" / config.filesName;
if (files::findFileName(VIDEO_PATH, fileName, extension) == -1) {
cerr << RED << BOLD << "config.json error!" << END << endl << RED << VIDEO_PATH.string() << " cannot be found or opened." << END << endl;
return -1;
}
irregularityFileInputPath = config.workingPath / "temp" / fileName / "AudioAnalyser_IrregularityFileOutput1.json";
// Input JSON check
ifstream iJSON(irregularityFileInputPath);
if (iJSON.fail()) {
cerr << RED << BOLD << "config.json error!" << END << endl << RED << irregularityFileInputPath.string() << " cannot be found or opened." << END << endl;
return -1;
}
if (config.speed != 7.5 && config.speed != 15) {
cerr << RED << BOLD << "config.json error!" << END << endl << RED << "Speed parameter must be 7.5 or 15 ips." << END << endl;
return -1;
}
if (tape.threshold.percentual < 0 || tape.threshold.percentual > 100) {
cerr << RED << BOLD << "config.json error!" << END << endl << RED << "TapeThresholdPercentual parameter must be a percentage value." << END << endl;
return -1;
}
if (capstan.threshold.percentual < 0 || capstan.threshold.percentual > 100) {
cerr << RED << BOLD << "config.json error!" << END << endl << RED << "CapstanThresholdPercentual parameter must be a percentage value." << END << endl;
return -1;
}
// Adjust input paramenters (considering given ones as pertinent to a speed reference = 7.5)
if (config.brands) {
if (config.speed == 15)
tape.threshold.percentual += 6;
} else
if (config.speed == 15)
tape.threshold.percentual += 20;
else
tape.threshold.percentual += 21;
cout << endl;
cout << "Parameters:" << endl;
cout << " Brands: " << config.brands << endl;
cout << " Speed: " << config.speed << endl;
cout << " ThresholdPercentual: " << tape.threshold.percentual << endl;
cout << " ThresholdPercentualCapstan: " << capstan.threshold.percentual << endl;
cout << endl;
// Read input JSON
iJSON >> irregularityFileInput;
/*********************************************************************************************/
/*********************************** MAKE OUTPUT DIRECTORY ***********************************/
/*********************************************************************************************/
// Make directory with fileName name
outputPath = config.workingPath / "temp" / fileName;
int outputFileNameDirectory = create_directory(outputPath);
irregularityImagesPath = outputPath / "IrregularityImages";
int fullFrameDirectory = fs::create_directory(irregularityImagesPath);
/*********************************************************************************************/
/************************************** AREAS DETECTION **************************************/
/*********************************************************************************************/
cv::VideoCapture videoCapture(VIDEO_PATH);
if (!videoCapture.isOpened()) {
cerr << RED << BOLD << "Video unreadable." << END << endl;
return -1;
}
int frames_number = videoCapture.get(CAP_PROP_FRAME_COUNT);
// Set frame position to half video length
videoCapture.set(CAP_PROP_POS_FRAMES, frames_number/2);
// Get frame
videoCapture >> myFrame;
Args args = argc > 1 ? Args::from_cli(argc, argv) : Args::from_file(CONFIG_FILE);
SceneObject capstan = SceneObject::from_file(CONFIG_FILE, ROI::CAPSTAN);
SceneObject tape = SceneObject::from_file(CONFIG_FILE, ROI::TAPE);
cout << "Video resolution: " << myFrame.cols << "x" << myFrame.rows << endl;
const fs::path VIDEO_PATH = args.working_path / "PreservationAudioVisualFile" / args.files_name;
const auto [FILE_NAME, FILE_FORMAT] = files::get_filename_and_extension(VIDEO_PATH);
const fs::path AUDIO_IRR_FILE_PATH = args.working_path / "temp" / FILE_NAME / A_IRREG_FILE_1;
bool found = findProcessingAreas(myFrame);
std::cout << "Video to be analysed: " << endl;
std::cout << "\tFile name: " << FILE_NAME << endl;
std::cout << "\tExtension: " << FILE_FORMAT << endl;
// Reset frame position
videoCapture.set(CAP_PROP_POS_FRAMES, 0);
if (FILE_FORMAT.compare("avi") != 0 && FILE_FORMAT.compare("mp4") != 0 && FILE_FORMAT.compare("mov") != 0)
print_error_and_exit("Input error: The input file must be an AVI, MP4 or MOV file.");
ifstream iJSON(AUDIO_IRR_FILE_PATH);
if (iJSON.fail())
print_error_and_exit("config.json error" + AUDIO_IRR_FILE_PATH.string() + " cannot be found or opened.");
if (!found) {
pprint("Processing area not found. Try changing JSON parameters.", RED);
return -1; // Program terminated early
}
json audio_irr_file;
iJSON >> audio_irr_file;
/*********************************************************************************************/
/**************************************** PROCESSING *****************************************/
/*********************************************************************************************/
// Adjust input paramenters (considering given ones as pertinent to a speed reference = 7.5)
if (args.speed == 15) {
tape.threshold.percentual += args.brands ? 6 : 20;
} else if (!args.brands)
tape.threshold.percentual += 21;
pprint("\nProcessing...", CYAN);
g_output_path = args.working_path / "temp" / FILE_NAME;
fs::create_directory(g_output_path);
// Processing timer
time_t startTimer, endTimer;
startTimer = time(NULL);
g_irregularity_images_path = g_output_path / "IrregularityImages";
fs::create_directory(g_irregularity_images_path);
processing(videoCapture);
cv::VideoCapture video_capture(VIDEO_PATH); // Open video file
if (!video_capture.isOpened()) print_error_and_exit("Video error: Video file cannot be opened.");
endTimer = time(NULL);
float min = (endTimer - startTimer) / 60;
float sec = (endTimer - startTimer) % 60;
video_capture.set(CAP_PROP_POS_FRAMES,
video_capture.get(CAP_PROP_FRAME_COUNT) / 2); // Set frame position to half video length
cv::Mat middle_frame = get_next_frame(video_capture, args.speed);
video_capture.set(CAP_PROP_POS_FRAMES, 0); // Reset frame position
string result("Processing elapsed time: " + to_string((int)min) + ":" + to_string((int)sec));
cout << endl << result << endl;
std::cout << "\tResolution: " << middle_frame.cols << "x" << middle_frame.rows << "\n\n";
/*********************************************************************************************/
/************************************* IRREGULARITY FILES ************************************/
/*********************************************************************************************/
auto processing_areas = find_processing_areas(middle_frame, tape, capstan);
if (std::holds_alternative<va::Error>(processing_areas))
print_error_and_exit("Processing area not found: Try changing JSON parameters.");
auto [rect_tape, rect_capstan] = std::get<pair<va::detection::Roi, va::detection::Roi>>(processing_areas);
fs::path outputFile1Name = outputPath / "VideoAnalyser_IrregularityFileOutput1.json";
files::saveFile(outputFile1Name, irregularityFileOutput1.dump(4), false);
pprint("Processing...", CYAN);
processing(video_capture, capstan, tape, args, rect_tape, rect_capstan);
// Irregularities to extract for the AudioAnalyser and to the TapeIrregularityClassifier
extractIrregularityImagesForAudio(outputPath, VIDEO_PATH, irregularityFileInput, irregularityFileOutput2);
files::save_file(g_output_path / V_IRREG_FILE_1, g_irregularity_file_1.dump(4));
fs::path outputFile2Name = outputPath / "VideoAnalyser_IrregularityFileOutput2.json";
files::saveFile(outputFile2Name, irregularityFileOutput2.dump(4), false);
// Irregularities to extract for the AudioAnalyser and to the TapeIrregularityClassifier
auto audio_irreg = extract_irregularity_images_for_audio(g_output_path, VIDEO_PATH, audio_irr_file, g_irregularity_file_2);
if (std::holds_alternative<va::Error>(audio_irreg))
print_error_and_exit(std::get<va::Error>(audio_irreg));
files::save_file(g_output_path / V_IRREG_FILE_2, g_irregularity_file_2.dump(4));
return 0;
return EXIT_SUCCESS;
}
#include "utility.h"
#include "utility.hpp"
using namespace cv;
using namespace std;
namespace fs = std::filesystem;
using json = nlohmann::json;
// Constructors
utility::Frame::Frame() : Mat() {}
utility::Frame::Frame(const Mat& m) : Mat(m) {}
utility::Frame::Frame(const Frame& f) : Mat(f) {}
// Operators
utility::Frame& utility::Frame::operator=(const Mat& m) {
Mat::operator=(m);
return *this;
}
utility::Frame& utility::Frame::operator=(const Frame& f) {
Mat::operator=(f);
return *this;
}
// Methods
utility::Frame utility::Frame::clone() const {
return utility::Frame(Mat::clone());
}
utility::Frame& utility::Frame::downsample(int factor) {
pyrDown(*this, *this, Size(size().width / factor, size().height / factor));
return *this;
}
utility::Frame& utility::Frame::convertColor(int code) {
cvtColor(*this, *this, code);
return *this;
}
utility::Frame utility::Frame::difference(Frame& f) {
return Frame(utility::difference(*this, f));
}
utility::Frame& utility::Frame::crop(Size rect_size, Point2f center) {
cv::getRectSubPix(*this, rect_size, center, *this);
return *this;
}
utility::Frame& utility::Frame::warp(cv::Mat rotationMatrix) {
cv::warpAffine(*this, *this, rotationMatrix, this->size(), INTER_CUBIC);
return *this;
}
void utility::detectShape(Ptr<GeneralizedHoughGuil> alg, Mat templateShape, int posThresh, vector<Vec4f> &positivePositions, Mat &positiveVotes, vector<Vec4f> &negativePositions, Mat &negativeVotes, Mat processingArea) {
alg -> setPosThresh(posThresh);
alg -> setTemplate(templateShape);
int oldSizePositive = 0;
int i = 0;
int maxVote = 0;
// Process shapes with positive angles
alg -> setMinAngle(0);
alg -> setMaxAngle(3);
while (true) {
alg -> detect(processingArea, positivePositions, positiveVotes);
int currentSize = positivePositions.size();
if (currentSize == 1) {
// We detected the most interesting shape
break;
} else if (currentSize == 0 && oldSizePositive > 0) {
// It is not possible to detect only one shape with the current parameters
alg -> setPosThresh(posThresh+i-1); // Decrease position value
alg -> detect(processingArea, positivePositions, positiveVotes); // Detect all available shapes
break;
} else if (currentSize == 0 && oldSizePositive == 0) {
// Impossible to find with these parameters
break;
}
oldSizePositive = currentSize;
// Find maximum vote
for (int j = 0; j < positiveVotes.cols / 3; j++) {
if (positiveVotes.at<int>(3*j) > maxVote)
maxVote = positiveVotes.at<int>(3*j);
}
if (currentSize > 10) {
i += 5; // To speed up computation when there are too many matches
} else if (maxVote - (posThresh + i) > 100) {
i += 100; // To speed up computation when there are few super high matches
} else {
i++;
}
alg -> setPosThresh(posThresh+i);
}
int oldSizeNegative = 0;
// Reset incremental position value
i = 0;
maxVote = 0;
// Process shapes with negative angles
alg -> setMinAngle(357);
alg -> setMaxAngle(360);
while (true) {
alg -> detect(processingArea, negativePositions, negativeVotes);
int currentSize = negativePositions.size();
if (currentSize == 1) {
// We detected the most interesting shape
break;
} else if (currentSize == 0 && oldSizeNegative > 0) {
// It is not possible to detect only one shape with the current parameters
alg -> setPosThresh(posThresh+i-1); // Decrease position value
alg -> detect(processingArea, negativePositions, negativeVotes); // Detect all available shapes
break;
} else if (currentSize == 0 && oldSizeNegative == 0) {
// Impossible to found with these parameters
break;
}
oldSizeNegative = currentSize;
Threshold::Threshold(float percentual, int angle, int scale, int pos) {
if (percentual < 0 || percentual > 100) throw std::invalid_argument("Percentual must be between 0 and 100");
// Find maximum vote
for (int j = 0; j < positiveVotes.cols / 3; j++) {
if (positiveVotes.at<int>(3*j) > maxVote)
maxVote = positiveVotes.at<int>(3*j);
}
if (currentSize > 10) {
i += 5; // To speed up computation when there are too many matches
} else if (maxVote - (posThresh + i) > 100) {
i += 100; // To speed up computation when there are few super high matches
} else {
i++;
}
alg -> setPosThresh(posThresh+i);
}
this->percentual = percentual;
this->angle = angle;
this->scale = scale;
this->pos = pos;
}
RotatedRect utility::drawShapes(Mat frame, Vec4f &positions, Scalar color, int width, int height, int offsetX, int offsetY, float processingScale) {
RotatedRect rr;
Point2f rrpts[4];
Point2f pos(positions[0]+offsetX, positions[1]+offsetY);
float scale = positions[2];
float angle = positions[3];
rr.center = pos * processingScale;
rr.size = Size2f(width * scale * processingScale, height * scale * processingScale);
rr.angle = angle;
rr.points(rrpts);
line(frame, rrpts[0], rrpts[1], color, 2);
line(frame, rrpts[1], rrpts[2], color, 2);
line(frame, rrpts[2], rrpts[3], color, 2);
line(frame, rrpts[3], rrpts[0], color, 2);
return rr;
SceneObject::SceneObject(int minDist, Threshold threshold) {
this->minDist = minDist;
this->threshold = threshold;
}
void utility::separateFrame(cv::Mat frame, cv::Mat &odd_frame, cv::Mat &even_frame) {
int i_odd_frame = 0;
int i_even_frame = 0;
for (int i = 0; i < frame.rows; i++) {
for (int j = 0; j < frame.cols; j++) {
if (i % 2 == 0) {
even_frame.at<cv::Vec3b>( i_even_frame, j )[0] = frame.at<cv::Vec3b>(i, j)[0];
even_frame.at<cv::Vec3b>( i_even_frame, j )[1] = frame.at<cv::Vec3b>(i, j)[1];
even_frame.at<cv::Vec3b>( i_even_frame, j )[2] = frame.at<cv::Vec3b>(i, j)[2];
} else {
odd_frame.at<cv::Vec3b>( i_odd_frame, j )[0] = frame.at<cv::Vec3b>(i, j)[0];
odd_frame.at<cv::Vec3b>( i_odd_frame, j )[1] = frame.at<cv::Vec3b>(i, j)[1];
odd_frame.at<cv::Vec3b>( i_odd_frame, j )[2] = frame.at<cv::Vec3b>(i, j)[2];
}
}
if (i % 2 == 0) {
i_even_frame++;
} else {
i_odd_frame++;
}
}
return;
}
SceneObject SceneObject::from_file(fs::path path, ROI obj) {
ifstream iConfig(path);
json j;
iConfig >> j;
cv::Mat utility::difference(cv::Mat &prevFrame, cv::Mat &currentFrame) {
cv::Mat diff = currentFrame.clone();
for (int i = 0; i < currentFrame.rows; i++) {
for (int j = 0; j < currentFrame.cols; j++) {
if (prevFrame.at<cv::Vec3b>(i, j)[0] != currentFrame.at<cv::Vec3b>(i, j)[0] || prevFrame.at<cv::Vec3b>(i, j)[1] != currentFrame.at<cv::Vec3b>(i, j)[1] || prevFrame.at<cv::Vec3b>(i, j)[2] != currentFrame.at<cv::Vec3b>(i, j)[2]) {
// Different pixels
diff.at<cv::Vec3b>(i, j)[0] = 0;
} else {
// Identical pixels
diff.at<cv::Vec3b>(i, j)[0] = 255;
}
}
if (obj == ROI::TAPE) {
return SceneObject(j["MinDist"],
Threshold(j["TapeThresholdPercentual"], j["AngleThresh"], j["ScaleThresh"], j["PosThresh"]));
} else {
return SceneObject(j["MinDistCapstan"], Threshold(j["CapstanThresholdPercentual"], j["AngleThreshCapstan"],
j["ScaleThreshCapstan"], j["PosThreshCapstan"]));
}
return diff;
}
\ No newline at end of file
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
using namespace cv;
using namespace std;
namespace fs = std::filesystem;
namespace utility {
/**
* @brief Class that extends the OpenCV Mat class, adding some useful methods frequently used in the project.
*
*/
class Frame : public Mat {
public:
Frame();
Frame(const Mat& m);
Frame(const Frame& f);
Frame& operator=(const Mat& m);
Frame& operator=(const Frame& f);
Frame clone() const;
Frame& downsample(int factor);
Frame& convertColor(int code);
Frame difference(Frame& f);
Frame& crop(Size rect_size, Point2f center);
Frame& warp(cv::Mat rotationMatrix);
};
/**
* @brief Detects a given shape in an image, using a the OpenCV algorithm GeneralizedHoughGuil.
*
* @param[in] alg the algorithm instance;
* @param[in] templateShape the shape to detect;
* @param[in] posThresh the position votes threshold;
* @param[out] positivePositions vector representing the position assigned to each found rectangle for positive angles;
* @param[out] positiveVotes vector representing the vote assigned to each found rectangle for positive angles;
* @param[out] negativePositions vector representing the position assigned to each found rectangle for negative angles;
* @param[out] negativeVotes vector representing the vote assigned to each found rectangle for negative angles;
* @param[in] processingArea the image to be processed.
*/
void detectShape(Ptr<GeneralizedHoughGuil> alg, Mat templateShape, int posThresh, vector<Vec4f> &positivePositions, Mat &positiveVotes, vector<Vec4f> &negativePositions, Mat &negativeVotes, Mat processingArea);
/**
* @brief Draw rectangles on an image.
*
* @param frame Frame on which the rectangles will be drawn;
* @param positions The position of the rectangle;
* @param color The color of the rectangle;
* @param width The width of the rectangle;
* @param height The height of the rectangle;
* @param offsetX X offset on the position of the rectangle;
* @param offsetY Y offset on the position of the rectangle;
* @param processingScale Scaling factor, useful for downsizing.
* @return RotatedRect Object representing the drawn rectangle.
*/
RotatedRect drawShapes(Mat frame, Vec4f &positions, Scalar color, int width, int height, int offsetX, int offsetY, float processingScale);
/**
* @brief Function to deinterlace the current image.
*
* @param[in] frame image to be processed;
* @param[out] odd_frame odd plane;
* @param[out] even_frame even plane.
*/
void separateFrame(cv::Mat frame, cv::Mat &odd_frame, cv::Mat &even_frame);
/**
* @brief Compute the number of different pixels between two frames.
*
* @param prevFrame the first frame;
* @param currentFrame the second frame.
* @return cv::Mat A black and white frame, where black pixels represent a difference, while white pixels represent an equality.
*/
cv::Mat difference(cv::Mat &prevFrame, cv::Mat &currentFrame);
}
#ifndef UTILITY_H
#define UTILITY_H
#include <filesystem>
#include <fstream>
#include <nlohmann/json.hpp>
using namespace std;
namespace fs = std::filesystem;
/**
* @struct Threshold
* @brief Struct containing the threshold values used to detect a shape.
*
*/
struct Threshold {
float percentual; /**< The minimum percentage of different pixels for
considering the current frame under the ROI as a
potential Irregularity */
int angle; /**< The angle votes threshold for the detection of the object */
int scale; /**< The scale votes threshold for the detection of the object */
int pos; /**< The position votes threshold for the detection of the object
*/
Threshold(){};
/**
* @brief Construct a new Threshold object
* @throws std::invalid_argument if the percentual is not in the range [0, 1]
*/
Threshold(float percentual, int angle, int scale, int pos);
};
/**
* @enum ROI
* @brief Enum containing the possible objects to detect.
*
*/
enum class ROI { TAPE, CAPSTAN };
/**
* @struct SceneObject
* @brief A scene object is an object that can be detected in a scene, such as a
* tape or a capstan.
*
*/
struct SceneObject {
int minDist; /**< The minimum distance between the centers of the detected
objects for the detection of the reading head */
Threshold threshold; /**< the threshold values used to detect the object */
SceneObject(){};
SceneObject(int minDist, Threshold threshold);
~SceneObject() = default;
/**
* @fn static SceneObject from_file(fs::path path, Object obj)
* @brief Create a SceneObject from a given file.
*
* @param path The path of the file containing the object.
* @note The file must be a JSON file.
* @param obj The object to detect.
* @return SceneObject The SceneObject created from the file.
*/
static SceneObject from_file(fs::path path, ROI obj);
};
#endif // UTILITY_H
\ No newline at end of file
#include "../src/lib/core.hpp"
#include <gtest/gtest.h>
#include <filesystem>
#include <opencv2/core/core.hpp>
using namespace videoanalyser;
using namespace cv;
namespace fs = std::filesystem;
std::string test_image_path = std::string(fs::absolute("tests/images/test_scene_01.jpg"));
core::Frame f = core::Frame(cv::imread(test_image_path));
TEST(Core, Frame) {
EXPECT_EQ(f.size().width, 720);
EXPECT_EQ(f.size().height, 576);
EXPECT_EQ(f.channels(), 3);
}
TEST(Core, FrameClone) {
core::Frame f2 = f.clone();
EXPECT_EQ(f.size().width, f2.size().width);
EXPECT_EQ(f.size().height, f2.size().height);
}
TEST(Core, FrameDownsample) {
core::Frame f2 = f.clone();
f2.downsample(2);
EXPECT_EQ(f.size().width / 2, f2.size().width);
EXPECT_EQ(f.size().height / 2, f2.size().height);
}
TEST(Core, FrameConvertColor) {
core::Frame f2 = f.clone();
f2.convert_color(cv::COLOR_BGR2GRAY);
EXPECT_EQ(f.size().width, f2.size().width);
EXPECT_EQ(f.size().height, f2.size().height);
EXPECT_EQ(f2.channels(), 1);
}
TEST(Core, FrameCrop) {
core::Frame f2 = f.clone();
f2.crop(cv::Size(100, 100), cv::Point2f(100, 100));
EXPECT_EQ(f2.size().width, 100);
EXPECT_EQ(f2.size().height, 100);
}
TEST(Core, FrameWarp) {
core::Frame f2 = f.clone();
cv::Mat rotationMatrix = cv::getRotationMatrix2D(cv::Point2f(f.size().width / 2, f.size().height / 2), 45, 1);
f2.warp(rotationMatrix);
EXPECT_EQ(f2.size().width, f.size().width);
EXPECT_EQ(f2.size().height, f.size().height);
}
TEST(Core, FrameDeinterlace) {
std::pair<core::Frame, core::Frame> f2 = f.deinterlace();
EXPECT_EQ(f2.first.size().width, f2.second.size().width);
EXPECT_EQ(f2.first.size().height, f2.second.size().height);
EXPECT_EQ(f2.first.size().width, f.size().width);
EXPECT_EQ(f2.first.size().height, f.size().height / 2);
}
#include "../src/lib/enums.hpp"
#include <gtest/gtest.h>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include "../src/lib/enums.h"
TEST(IrregularityType, toString) {
EXPECT_EQ(irregularityTypeToString(IrregularityType::WOW_AND_FLUTTER), "wf");
}
TEST(IrregularityType, toString) { EXPECT_EQ(irregularityTypeToString(IrregularityType::WOW_AND_FLUTTER), "wf"); }
TEST(IrregularityType, fromString) {
EXPECT_EQ(irregularityTypeFromString("wf"), IrregularityType::WOW_AND_FLUTTER);
......
#include "../src/lib/files.hpp"
#include <gtest/gtest.h>
using namespace files;
namespace fs = std::filesystem;
TEST(Files, SaveFile) {
std::string content = "test";
std::filesystem::path fileName = fs::absolute("tests/test.txt");
save_file(fileName, content);
std::ifstream file(fileName);
std::string line;
std::getline(file, line);
EXPECT_EQ(line, content);
file.close();
std::filesystem::remove(fileName);
}
TEST(Files, GetFilenameAndExtension) {
std::string videoPath = std::string(fs::absolute("tests/images/test_scene_01.jpg"));
std::pair<std::string, std::string> result = get_filename_and_extension(videoPath);
EXPECT_EQ(result.first, "test_scene_01");
EXPECT_EQ(result.second, "jpg");
}
\ No newline at end of file
#include "../src/lib/Irregularity.hpp"
#include <gtest/gtest.h>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include "../src/lib/Irregularity.h"
TEST(Irregularity, Init) {
Irregularity irreg = Irregularity(
Source::Video,
"00:00:00.000",
IrregularityType::WOW_AND_FLUTTER,
"https://example.com/image.png"
);
Irregularity irreg = Irregularity(Source::Video, "00:00:00.000", IrregularityType::WOW_AND_FLUTTER)
.set_image_URI("https://example.com/image.png");
Irregularity irreg2 = Irregularity(
Source::Video,
"00:00:00.000",
IrregularityType::WOW_AND_FLUTTER,
"https://example.com/image.png"
);
Irregularity irreg2 = Irregularity(Source::Video, "00:00:00.000", IrregularityType::WOW_AND_FLUTTER)
.set_image_URI("https://example.com/image.png");
EXPECT_NE(irreg.id, irreg2.id);
EXPECT_EQ(irreg.source, irreg2.source);
EXPECT_EQ(irreg.time_label, irreg2.time_label);
EXPECT_EQ(irreg.type, irreg2.type);
EXPECT_EQ(irreg.image_URI, irreg2.image_URI);
EXPECT_NE(irreg.get_id(), irreg2.get_id());
EXPECT_EQ(irreg.get_source(), irreg2.get_source());
EXPECT_EQ(irreg.get_time_label(), irreg2.get_time_label());
EXPECT_EQ(irreg.get_type(), irreg2.get_type());
EXPECT_EQ(irreg.get_image_URI(), irreg2.get_image_URI());
EXPECT_EQ(irreg.get_audio_URI(), irreg2.get_audio_URI());
}