main.cpp 31.2 KB
Newer Older
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
1
2
3
4
5
6
7
8
9
10
11
/**
 *  MPAI CAE-ARP Video Analyser.
 *
 *	Implements MPAI CAE-ARP Video Analyser Technical Specification.
 *	It identifies Irregularities on the Preservation Audio-Visual File, providing:
 *	- Irregularity Files;
 *	- Irregularity Images.
 *
 *	WARNING:
 *	Currently, this program is only compatible with the Studer A810 and videos recorded in PAL standard.
 *
Matteo's avatar
update    
Matteo committed
12
 *  @authors Nadir Dalla Pozza <nadir.dallapozza@unipd.it>, Matteo Spanio <dev2@audioinnova.com>
Matteo's avatar
Matteo committed
13
 *	@copyright 2023, Audio Innova S.r.l.
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
14
15
 *	@credits Niccolò Pretto, Nadir Dalla Pozza, Sergio Canazza
 *	@license GPL v3.0
Matteo's avatar
Matteo committed
16
 *	@version 1.1.1
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
17
18
 *	@status Production
 */
19
#include <filesystem>
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
20
#include <fstream>
21
#include <iostream>
22
#include <stdlib.h>
23
#include <sys/timeb.h>
Matteo's avatar
update    
Matteo committed
24
#include <ranges>
25

26
#include <boost/program_options.hpp>
27
28
29
30
31
32
33
34
35
36
37
38
#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_generators.hpp> // generators
#include <boost/uuid/uuid_io.hpp>         // streaming operators etc.
#include <boost/lexical_cast.hpp>

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>

#include <nlohmann/json.hpp>

39
40
41
42
43
44
45
#include "opencv2/core.hpp"
#include "opencv2/calib3d.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/features2d.hpp"
#include "opencv2/xfeatures2d.hpp"

46
47
48
#include "utility.h"
#include "forAudioAnalyser.h"

Matteo's avatar
update    
Matteo committed
49
#include "lib/files.h"
Matteo's avatar
update    
Matteo committed
50
51
52
#include "lib/colors.h"
#include "lib/time.h"

53
54
using namespace cv;
using namespace std;
Matteo's avatar
update    
Matteo committed
55
using utility::Frame;
56
using json = nlohmann::json;
57
58
namespace fs = std::filesystem;
namespace po = boost::program_options;
59

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
60
// For capstan detection, there are two alternative approaches:
61
62
63
// Generalized Hough Transform and SURF.
bool useSURF = true;

Matteo's avatar
update    
Matteo committed
64
65
bool savingPinchRoller = false;
bool pinchRollerRect = false;
66
bool savingBrand = false;
67
bool endTapeSaved = false;
68
69
float mediaPrevFrame = 0;
bool firstBrand = true;	// The first frame containing brands on tape must be saved
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
70
float firstInstant = 0;
71
string fileName, extension;
72

73
// Path variables
Matteo's avatar
update    
Matteo committed
74
75
static fs::path outputPath {};
static fs::path irregularityImagesPath {};
76
// JSON files
Matteo's avatar
update    
Matteo committed
77
78
79
static json configurationFile {};
static json irregularityFileOutput1 {};
static json irregularityFileOutput2 {};
80
// RotatedRect identifying the processing area
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
81
RotatedRect rect, rectTape, rectCapstan;
82

Matteo's avatar
update    
Matteo committed
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
// config.json parameters
struct Config {
	fs::path workingPath;
	string filesName;
	bool brands;
	float speed;
};

struct Threshold {
	float percentual;
	int angle;
	int scale;
	int pos;
};

struct SceneObject {
	int minDist;
	Threshold threshold;
};

Matteo's avatar
update    
Matteo committed
103
104
105
static Config config;
static SceneObject tape;
static SceneObject capstan;
Matteo's avatar
update    
Matteo committed
106

Matteo's avatar
update    
Matteo committed
107
// Constants Paths
Matteo's avatar
update    
Matteo committed
108
static const string READING_HEAD_IMG = "input/readingHead.png";
Matteo's avatar
update    
Matteo committed
109
110
static const string CAPSTAN_TEMPLATE_IMG = "input/capstanBERIO058prova.png";
static const string CONFIG_FILE = "config/config.json";
111

Matteo's avatar
update    
Matteo committed
112
113
114
115
double rotatedRectArea(RotatedRect rect) {
	return rect.size.width * rect.size.height;
}

Matteo's avatar
Matteo committed
116
117
118
119
120
121
122
123
124
125
/**
 * @brief Prints a text in a given color.
 * 
 * @param text
 * @param color 
 */
void pprint(string text, string color) {
	cout << color << text << END << endl;
}

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
126
127
128
129
130
131
132
133
/**
 * @brief Get operation arguments from command line or config.json file.
 *
 * @param argc Command line arguments count;
 * @param argv Command line arguments.
 * @return true if input configuration is valid;
 * @return false otherwise.
 */
Matteo's avatar
update    
Matteo committed
134
bool getArguments(int argc, char** argv) {
135
	// Read configuration file
Matteo's avatar
update    
Matteo committed
136
	ifstream iConfig(CONFIG_FILE);
137
138
139
140
	iConfig >> configurationFile;

	if (argc == 1) {
		// Read from JSON file
Matteo's avatar
update    
Matteo committed
141
		string working_path = configurationFile["WorkingPath"];
Matteo's avatar
update    
Matteo committed
142
143

		config = {
Matteo's avatar
update    
Matteo committed
144
			fs::path(working_path),
Matteo's avatar
update    
Matteo committed
145
146
147
148
149
			configurationFile["FilesName"],
			configurationFile["Brands"],
			configurationFile["Speed"]
		};

150
151
152
153
154
	} else {
		// Get from command line
		try {
			po::options_description desc(
				"A tool that implements MPAI CAE-ARP Video Analyser Technical Specification.\n"
Matteo's avatar
update    
Matteo committed
155
				"By default, the configuartion parameters are loaded from config/config.json file,\n"
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
				"but, alternately, you can pass command line arguments to replace them"
			);
			desc.add_options()
				("help,h", "Display this help message")
				("working-path,w", po::value<string>()->required(), "Specify the Working Path, where all input files are stored")
				("files-name,f", po::value<string>()->required(), "Specify the name of the Preservation files (without extension)")
				("brands,b", po::value<bool>()->required(), "Specify if the tape presents brands on its surface")
				("speed,s", po::value<float>()->required(), "Specify the speed at which the tape was read");
			po::variables_map vm;
			po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
			if (vm.count("help")) {
				cout << desc << "\n";
				return false;
			}
			po::notify(vm);
Matteo's avatar
update    
Matteo committed
171
172
173
174
175
176
177

			// Access the stored options
			config.workingPath = fs::path(vm["working-path"].as<string>());
			config.filesName = vm["files-name"].as<string>();
			config.brands = vm["brands"].as<bool>();
			config.speed = vm["speed"].as<float>();

178
179
180
181
182
183
184
185
186
		} catch (po::invalid_command_line_syntax& e) {
			cerr << RED << BOLD << "The command line syntax is invalid: " << END << RED << e.what() << END << endl;
			return false;
		} catch (po::required_option& e) {
			cerr << "Error: " << e.what() << endl;
			return false;
		}
	}

Matteo's avatar
update    
Matteo committed
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
	capstan = {
		configurationFile["MinDistCapstan"],
		{
			configurationFile["CapstanThresholdPercentual"],
			configurationFile["AngleThreshCapstan"],
			configurationFile["ScaleThreshCapstan"],
			configurationFile["PosThreshCapstan"]
		}
	};

	tape = {
		configurationFile["MinDist"],
		{
			configurationFile["TapeThresholdPercentual"],
			configurationFile["AngleThresh"],
			configurationFile["ScaleThresh"],
			configurationFile["PosThresh"]
		}
	};
206
207
208
209
210

	return true;
}


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
211
/**
Matteo's avatar
Matteo committed
212
213
214
215
216
217
218
219
 * @brief Find the model in the scene using the Generalized Hough Transform. It returns the best matches.
 * Find the best matches for positive and negative angles.
 * If there are more than one shapes, then choose the one with the highest score.
 * If there are more than one with the same highest score, then arbitrarily choose the latest
 * 
 * @param model the template image to be searched with the Generalized Hough Transform
 * @param object the sceneObject struct containing the parameters for the Generalized Hough Transform
 * @return std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>> a tuple containing the best matches for positive and negative angles
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
220
 */
Matteo's avatar
Matteo committed
221
std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>> findObject(Mat model, SceneObject object, Mat processing_area) {
222
223

	// Algorithm and parameters
Matteo's avatar
Matteo committed
224
	// for informations about the Generalized Hough Guild usage see the tutorial at https://docs.opencv.org/4.7.0/da/ddc/tutorial_generalized_hough_ballard_guil.html
225
226
227
228
229
230
231
232
	Ptr<GeneralizedHoughGuil> alg = createGeneralizedHoughGuil();

	vector<Vec4f> positionsPos, positionsNeg;
	Mat votesPos, votesNeg;

	double maxValPos = 0, maxValNeg = 0;
	int indexPos = 0, indexNeg = 0;

Matteo's avatar
Matteo committed
233
	alg->setMinDist(object.minDist);
Matteo's avatar
update    
Matteo committed
234
235
236
	alg->setLevels(360);
	alg->setDp(2);
	alg->setMaxBufferSize(1000);
237

Matteo's avatar
update    
Matteo committed
238
	alg->setAngleStep(1);
Matteo's avatar
Matteo committed
239
	alg->setAngleThresh(object.threshold.angle);
240

Matteo's avatar
update    
Matteo committed
241
242
243
	alg->setMinScale(0.9);
	alg->setMaxScale(1.1);
	alg->setScaleStep(0.01);
Matteo's avatar
Matteo committed
244
	alg->setScaleThresh(object.threshold.scale);
245

Matteo's avatar
Matteo committed
246
	alg->setPosThresh(object.threshold.pos);
247

Matteo's avatar
update    
Matteo committed
248
249
	alg->setCannyLowThresh(150); // Old: 100
	alg->setCannyHighThresh(240); // Old: 300
250

Matteo's avatar
Matteo committed
251
	alg->setTemplate(model);
252

Matteo's avatar
Matteo committed
253
	utility::detectShape(alg, model, object.threshold.pos, positionsPos, votesPos, positionsNeg, votesNeg, processing_area);
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268

	for (int i = 0; i < votesPos.size().width; i++) {
		if (votesPos.at<int>(i) >= maxValPos) {
			maxValPos = votesPos.at<int>(i);
			indexPos = i;
		}
	}

	for (int i = 0; i < votesNeg.size().width; i++) {
		if (votesNeg.at<int>(i) >= maxValNeg) {
			maxValNeg = votesNeg.at<int>(i);
			indexNeg = i;
		}
	}

Matteo's avatar
Matteo committed
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
	return { indexPos, indexNeg, maxValPos, maxValNeg, positionsPos, positionsNeg };
}

/**
 * @brief Identifies the Regions Of Interest (ROIs) on the video,
 * which are:
 * - The reading head;
 * - The tape area under the tape head (computed on the basis of the detected reading head);
 * - The capstan.
 * @param myFrame The current frame of the video.
 * @return true if some areas have been detected;
 * @return false otherwise.
 */
bool findProcessingAreas(Mat myFrame) {

	/*********************************************************************************************/
	/*********************************** READING HEAD DETECTION **********************************/
	/*********************************************************************************************/

	// Save a grayscale version of myFrame in myFrameGrayscale and downsample it in half pixels for performance reasons
	Frame gray_current_frame = Frame(myFrame)
		.convertColor(COLOR_BGR2GRAY);

	Frame halved_gray_current_frame = gray_current_frame
		.clone()
		.downsample(2);

	// Get input shape in grayscale and downsample it in half pixels
	Frame reading_head_template = Frame(cv::imread(READING_HEAD_IMG, IMREAD_GRAYSCALE)).downsample(2);

	// Process only the bottom-central portion of the input video -> best results with our videos
	Rect readingHeadProcessingAreaRect(
		halved_gray_current_frame.cols/4,
		halved_gray_current_frame.rows/2,
		halved_gray_current_frame.cols/2,
		halved_gray_current_frame.rows/2
	);
	Mat processingImage = halved_gray_current_frame(readingHeadProcessingAreaRect);

	RotatedRect rectPos, rectNeg;
	auto [indexPos, indexNeg, maxValPos, maxValNeg, positionsPos, positionsNeg] = findObject(reading_head_template, tape, processingImage);

311
312
	// The color is progressively darkened to emphasize that the algorithm found more than one shape
	if (positionsPos.size() > 0)
Matteo's avatar
update    
Matteo committed
313
		rectPos = utility::drawShapes(myFrame, positionsPos[indexPos], Scalar(0, 0, 255-indexPos*64), reading_head_template.cols, reading_head_template.rows, halved_gray_current_frame.cols/4, halved_gray_current_frame.rows/2, 2);
314
	if (positionsNeg.size() > 0)
Matteo's avatar
update    
Matteo committed
315
		rectNeg = utility::drawShapes(myFrame, positionsNeg[indexNeg], Scalar(128, 128, 255-indexNeg*64), reading_head_template.cols, reading_head_template.rows, halved_gray_current_frame.cols/4, halved_gray_current_frame.rows/2, 2);
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338

	if (maxValPos > 0)
		if (maxValNeg > 0)
			if (maxValPos > maxValNeg) {
				rect = rectPos;
			} else {
				rect = rectNeg;
			}
		else {
			rect = rectPos;
		}
	else if (maxValNeg > 0) {
		rect = rectNeg;
	} else {
		return false;
	}

	/*********************************************************************************************/
	/************************************ TAPE AREA DETECTION ************************************/
	/*********************************************************************************************/

	// Compute area basing on reading head detection
	Vec4f positionTape( rect.center.x, rect.center.y + rect.size.height / 2 + 20 * (rect.size.width / 200), 1, rect.angle );
Matteo's avatar
update    
Matteo committed
339
	rectTape = utility::drawShapes(myFrame, positionTape, Scalar(0, 255-indexPos*64, 0), rect.size.width, 50 * (rect.size.width / 200), 0, 0, 1);
340
341
342
343
344
345

	/*********************************************************************************************/
	/************************************* CAPSTAN DETECTION *************************************/
	/*********************************************************************************************/

	// Read template image - it is smaller than before, therefore there is no need to downsample
Matteo's avatar
update    
Matteo committed
346
	Mat templateShape = imread(CAPSTAN_TEMPLATE_IMG, IMREAD_GRAYSCALE);
347
348
349
350
351
352
353
354
355
356
357
	// templateShape = imread("../input/capstanBERIO058.png", IMREAD_GRAYSCALE);

	if (useSURF) {

		// Step 1: Detect the keypoints using SURF Detector, compute the descriptors
		int minHessian = 100;
		Ptr<xfeatures2d::SURF> detector = xfeatures2d::SURF::create(minHessian);
		vector<KeyPoint> keypoints_object, keypoints_scene;
		Mat descriptors_object, descriptors_scene;

		detector->detectAndCompute(templateShape, noArray(), keypoints_object, descriptors_object);
Matteo's avatar
update    
Matteo committed
358
		detector->detectAndCompute(gray_current_frame, noArray(), keypoints_scene, descriptors_scene);
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374

		// Step 2: Matching descriptor vectors with a FLANN based matcher
		// Since SURF is a floating-point descriptor NORM_L2 is used
		Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create(DescriptorMatcher::FLANNBASED);
		vector<vector<DMatch>> knn_matches;
		matcher->knnMatch(descriptors_object, descriptors_scene, knn_matches, 2);
		//-- Filter matches using the Lowe's ratio test
		const float ratio_thresh = 0.75f;
		vector<DMatch> good_matches;
		for (size_t i = 0; i < knn_matches.size(); i++) {
			if (knn_matches[i][0].distance < ratio_thresh * knn_matches[i][1].distance) {
				good_matches.push_back(knn_matches[i][0]);
			}
		}
		// Draw matches
		Mat img_matches;
Matteo's avatar
update    
Matteo committed
375
		drawMatches(templateShape, keypoints_object, halved_gray_current_frame, keypoints_scene, good_matches, img_matches, Scalar::all(-1), Scalar::all(-1), vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
376
377
378
		// Localize the object
		vector<Point2f> obj;
		vector<Point2f> scene;
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
379
		for (size_t i = 0; i < good_matches.size(); i++) {
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
			// Get the keypoints from the good matches
			obj.push_back(keypoints_object[good_matches[i].queryIdx].pt);
			scene.push_back(keypoints_scene[good_matches[i].trainIdx].pt);
		}
		Mat H = findHomography(obj, scene, RANSAC);
		// Get the corners from the image_1 ( the object to be "detected" )
		vector<Point2f> obj_corners(4);
		obj_corners[0] = Point2f(0, 0);
		obj_corners[1] = Point2f((float)templateShape.cols, 0);
		obj_corners[2] = Point2f((float)templateShape.cols, (float)templateShape.rows);
		obj_corners[3] = Point2f(0, (float)templateShape.rows);
		vector<Point2f> scene_corners(4);
		perspectiveTransform( obj_corners, scene_corners, H);

		// Find average
		float capstanX = (scene_corners[0].x + scene_corners[1].x + scene_corners[2].x + scene_corners[3].x) / 4;
		float capstanY = (scene_corners[0].y + scene_corners[1].y + scene_corners[2].y + scene_corners[3].y) / 4;

		// In the following there are two alterations to cut the first 20 horizontal pixels and the first 90 vertical pixels from the found rectangle:
		// +10 in X for centering and -20 in width
		// +45 in Y for centering and -90 in height
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
401
		Vec4f positionCapstan(capstanX + 10, capstanY + 45, 1, 0);
Matteo's avatar
update    
Matteo committed
402
		rectCapstan = utility::drawShapes(myFrame, positionCapstan, Scalar(255-indexPos*64, 0, 0), templateShape.cols - 20, templateShape.rows - 90, 0, 0, 1);
403
404
405

	} else {

Matteo's avatar
Matteo committed
406
		// Process only right portion of the image, where the capstain always appears
407
408
409
410
411
		int capstanProcessingAreaRectX = myFrame.cols*3/4;
		int capstanProcessingAreaRectY = myFrame.rows/2;
		int capstanProcessingAreaRectWidth = myFrame.cols/4;
		int capstanProcessingAreaRectHeight = myFrame.rows/2;
		Rect capstanProcessingAreaRect(capstanProcessingAreaRectX, capstanProcessingAreaRectY, capstanProcessingAreaRectWidth, capstanProcessingAreaRectHeight);
Matteo's avatar
update    
Matteo committed
412
		Mat capstanProcessingAreaGrayscale = gray_current_frame(capstanProcessingAreaRect);
413
414
		// Reset algorithm and set parameters

Matteo's avatar
Matteo committed
415
		auto [indexPos, indexNeg, maxValPos, maxValNeg, positionsC1Pos, positionsC1Neg] = findObject(templateShape, capstan, capstanProcessingAreaGrayscale);
416
417
418

		RotatedRect rectCapstanPos, rectCapstanNeg;
		if (positionsC1Pos.size() > 0)
Matteo's avatar
update    
Matteo committed
419
			rectCapstanPos = utility::drawShapes(myFrame, positionsC1Pos[indexPos], Scalar(255-indexPos*64, 0, 0), templateShape.cols-22, templateShape.rows-92, capstanProcessingAreaRectX+11, capstanProcessingAreaRectY+46, 1);
420
		if (positionsC1Neg.size() > 0)
Matteo's avatar
update    
Matteo committed
421
			rectCapstanNeg = utility::drawShapes(myFrame, positionsC1Neg[indexNeg], Scalar(255-indexNeg*64, 128, 0), templateShape.cols-22, templateShape.rows-92, capstanProcessingAreaRectX+11, capstanProcessingAreaRectY+46, 1);
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447

		if (maxValPos > 0)
			if (maxValNeg > 0)
				if (maxValPos > maxValNeg) {
					rectCapstan = rectCapstanPos;
				} else {
					rectCapstan = rectCapstanNeg;
				}
			else {
				rectCapstan = rectCapstanPos;
			}
		else if (maxValNeg > 0) {
			rectCapstan = rectCapstanNeg;
		} else {
			return false;
		}
	}

	cout << endl;

	// Save the image containing the detected areas
	cv::imwrite(outputPath.string() + "/tapeAreas.jpg", myFrame);

	return true;
}

Matteo's avatar
Matteo committed
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
RotatedRect check_skew(RotatedRect roi) {
	// get angle and size from the bounding box
	// thanks to http://felix.abecassis.me/2011/10/opencv-rotation-deskewing/
	cv::Size rect_size = roi.size;
	float angle = roi.angle;

	if (roi.angle < -45.) {
		angle += 90.0;
		swap(rect_size.width, rect_size.height);
	}

	return RotatedRect(roi.center, rect_size, angle);
}

/**
 * @brief Look for differences in two consecutive frames in a specific region of interest
 * 
 * @param previous the reference frame
 * @param current the frame to compare with the reference
 * @param roi the region of interest
 * @return Frame the difference matrix between the two frames
 */
Frame get_difference_for_roi(Frame previous, Frame current, RotatedRect roi) {
	cv::Mat rotation_matrix = getRotationMatrix2D(roi.center, roi.angle, 1.0);

	return previous
		.warp(rotation_matrix)
		.crop(roi.size, roi.center)
		.difference(current
			.warp(rotation_matrix)
			.crop(roi.size, roi.center));
}
480

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
481
482
483
484
485
486
487
488
489
490
/**
 * @brief Compares two consecutive video frames and establish if there potentially is an Irregularity.
 * The comparison is pixel-wise and based on threshold values set on config.json file.
 *
 * @param prevFrame the frame before the current one;
 * @param currentFrame the current frame;
 * @param msToEnd the number of milliseconds left before the end of the video. Useful for capstan analysis.
 * @return true if a potential Irregularity has been found;
 * @return false otherwise.
 */
491
492
bool frameDifference(cv::Mat prevFrame, cv::Mat currentFrame, int msToEnd) {

493
494
495
	/********************************** Capstan analysis *****************************************/

	// In the last minute of the video, check for pinchRoller position for endTape event
496
	if (!endTapeSaved && msToEnd < 60000) {
497
498
499

		// Capstan area
		int capstanAreaPixels = rectCapstan.size.width * rectCapstan.size.height;
Matteo's avatar
update    
Matteo committed
500
		float capstanDifferentPixelsThreshold = capstanAreaPixels * capstan.threshold.percentual / 100;
501

Matteo's avatar
Matteo committed
502
503
		RotatedRect corrected_capstan_roi = check_skew(rectCapstan);
		Frame difference_frame = get_difference_for_roi(Frame(prevFrame), Frame(currentFrame), corrected_capstan_roi);
504
505
506

		int blackPixelsCapstan = 0;

Matteo's avatar
Matteo committed
507
508
509
		for (int i = 0; i < difference_frame.rows; i++) {
			for (int j = 0; j < difference_frame.cols; j++) {
				if (difference_frame.at<cv::Vec3b>(i, j)[0] == 0) {
510
511
512
513
514
515
516
517
					// There is a black pixel, then there is a difference between previous and current frames
					blackPixelsCapstan++;
				}
			}
		}

		if (blackPixelsCapstan > capstanDifferentPixelsThreshold) {
			savingPinchRoller = true;
518
			endTapeSaved = true; // Never check again for end tape instant
519
			return true;
Matteo's avatar
Matteo committed
520
		} 
521
	}
522

Matteo's avatar
Matteo committed
523
524
	savingPinchRoller = false; // It will already be false before the last minute of the video. After having saved the capstan, the next time reset the variable to not save again

525
526
527
	/************************************ Tape analysis ******************************************/

	// Tape area
Matteo's avatar
update    
Matteo committed
528
    int tapeAreaPixels = rotatedRectArea(rectTape);
Matteo's avatar
update    
Matteo committed
529
	float tapeDifferentPixelsThreshold = tapeAreaPixels * tape.threshold.percentual / 100;
530

Matteo's avatar
Matteo committed
531
	RotatedRect corrected_tape_roi = check_skew(rectTape);
532

Matteo's avatar
Matteo committed
533
534
535
	Frame croppedCurrentFrame = Frame(currentFrame)
		.warp(getRotationMatrix2D(corrected_tape_roi.center, corrected_tape_roi.angle, 1.0))
		.crop(corrected_tape_roi.size, corrected_tape_roi.center);
536

Matteo's avatar
Matteo committed
537
	Frame difference_frame = get_difference_for_roi(Frame(prevFrame), Frame(currentFrame), corrected_tape_roi);
538
539
540
541
542
543

	int decEnd = (msToEnd % 1000) / 100;
	int secEnd = (msToEnd - (msToEnd % 1000)) / 1000;
	int minEnd = secEnd / 60;
	secEnd = secEnd % 60;

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
544
	/************************************* Segment analysis **************************************/
545

546
547
548
549
550
551
552
  	int blackPixels = 0;
	float mediaCurrFrame;
	int totColoreCF = 0;

	for (int i = 0; i < croppedCurrentFrame.rows; i++) {
		for (int j = 0; j < croppedCurrentFrame.cols; j++) {
			totColoreCF += croppedCurrentFrame.at<cv::Vec3b>(i, j)[0] + croppedCurrentFrame.at<cv::Vec3b>(i, j)[1] + croppedCurrentFrame.at<cv::Vec3b>(i, j)[2];
Matteo's avatar
Matteo committed
553
			if (difference_frame.at<cv::Vec3b>(i, j)[0] == 0) {
554
555
556
557
				blackPixels++;
			}
		}
	}
558
	mediaCurrFrame = totColoreCF/tapeAreaPixels;
559

Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
560
561
	/************************************* Decision stage ****************************************/

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
562
	bool isIrregularity = false;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
563

564
	if (blackPixels > tapeDifferentPixelsThreshold) { // The threshold must be passed
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
565

566
567
		/***** AVERAGE_COLOR-BASED DECISION *****/
		if (mediaPrevFrame > (mediaCurrFrame + 7) || mediaPrevFrame < (mediaCurrFrame - 7)) { // They are not similar for color average
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
568
			isIrregularity = true;
569
		}
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
570

571
		/***** BRANDS MANAGEMENT *****/
Matteo's avatar
update    
Matteo committed
572
		if (config.brands) {
573
			// At the beginning of the video, wait at least 5 seconds before the next Irregularity to consider it as a brand.
574
			// It is not guaranteed that it will be the first brand, but it is generally a safe approach to have a correct image
575
576
577
578
			if (firstBrand) {
				if (firstInstant - msToEnd > 5000) {
					firstBrand = false;
					savingBrand = true;
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
579
					isIrregularity = true;
580
				}
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
581
			// In the following iterations reset savingBrand, since we are no longer interested in brands.
582
			} else
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
583
				savingBrand = false;
584
		}
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
585

586
	}
587
588
589
590

	// Update mediaPrevFrame
	mediaPrevFrame = mediaCurrFrame;

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
591
	return isIrregularity;
592
593
594
}


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
595
596
597
598
599
600
601
/**
 * @brief video processing phase, where each frame is analysed.
 * It saves the IrregularityImages and updates the IrregularityFiles if an Irregularity is found
 *
 * @param videoCapture the input Preservation Audio-Visual File;
 */
void processing(cv::VideoCapture videoCapture) {
602

Matteo's avatar
update    
Matteo committed
603
604
605
606
607
	const int video_length_ms = ((float) videoCapture.get(CAP_PROP_FRAME_COUNT) / videoCapture.get(CAP_PROP_FPS)) * 1000;
	int video_current_ms = videoCapture.get(CAP_PROP_POS_MSEC);
	// counters
    int savedFrames = 0;
	int unsavedFrames = 0;
608
	float lastSaved = -160;
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
609
	// Whenever we find an Irregularity, we want to skip a lenght equal to the Studer reading head (3 cm = 1.18 inches).
610
611
	int savingRate = 79; // [ms]. Time taken to cross 3 cm at 15 ips, or 1.5 cm at 7.5 ips. The considered lengths are the widths of the tape areas.
	// The following condition constitutes a valid approach if the tape areas have widths always equal to the reading head
Matteo's avatar
update    
Matteo committed
612
	if (config.speed == 7.5)
613
614
615
616
617
		savingRate = 157; // Time taken to cross 3 cm at 7.5 ips

	// The first frame of the video won't be processed
    cv::Mat prevFrame;
	videoCapture >> prevFrame;
Matteo's avatar
update    
Matteo committed
618
	firstInstant = video_length_ms - video_current_ms;
619
620
621
622

    while (videoCapture.isOpened()) {
		cv::Mat frame;
        videoCapture >> frame;
Matteo's avatar
update    
Matteo committed
623
		video_current_ms = videoCapture.get(CAP_PROP_POS_MSEC);
624

Matteo's avatar
update    
Matteo committed
625
		if (frame.empty()) {
626
			cout << endl << "Empty frame!" << endl;
627
628
	    	videoCapture.release();
	    	break;
Matteo's avatar
update    
Matteo committed
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
		}

		int msToEnd = video_length_ms - video_current_ms;
		if (video_current_ms == 0) // With OpenCV library, this happens at the last few frames of the video before realising that "frame" is empty.
			break;

		// Variables to display program status
		int secToEnd = msToEnd / 1000;
		int minToEnd = (secToEnd / 60) % 60;
		secToEnd = secToEnd % 60;

		string secStrToEnd = to_string(secToEnd), minStrToEnd = to_string(minToEnd);
		if (minToEnd < 10)
			minStrToEnd = "0" + minStrToEnd;
		if (secToEnd < 10)
			secStrToEnd = "0" + secStrToEnd;

		// Display program status
		cout << "\rIrregularities: " << savedFrames << ".   ";
		cout << "Remaining video time [mm:ss]: " << minStrToEnd << ":" << secStrToEnd << flush;

		if ((video_current_ms - lastSaved > savingRate) && frameDifference(prevFrame, frame, msToEnd)) {
			// An Irregularity has been found!
			// De-interlacing frame
			cv::Mat oddFrame(frame.rows/2, frame.cols, CV_8UC3);
			cv::Mat evenFrame(frame.rows/2, frame.cols, CV_8UC3);
			utility::separateFrame(frame, oddFrame, evenFrame);

			// Extract the image corresponding to the ROIs
			Point2f pts[4];
			if (savingPinchRoller)
				rectCapstan.points(pts);
			else
				rectTape.points(pts);
			cv::Mat subImage(frame, cv::Rect(100, min(pts[1].y, pts[2].y), frame.cols - 100, static_cast<int>(rectTape.size.height)));

			// De-interlacing
			cv::Mat oddSubImage(subImage.rows/2, subImage.cols, CV_8UC3);
			int evenSubImageRows = subImage.rows/2;
			if (subImage.rows % 2 != 0) // If the found rectangle is of odd height, we must increase evenSubImage height by 1, otherwise we have segmentation_fault!!!
				evenSubImageRows += 1;
			cv::Mat evenSubImage(evenSubImageRows, subImage.cols, CV_8UC3);
			utility::separateFrame(subImage, oddSubImage, evenSubImage);

			string timeLabel = getTimeLabel(video_current_ms, ":");
			string safeTimeLabel = getTimeLabel(video_current_ms, "-");

			string irregularityImageFilename = to_string(savedFrames) + "_" + safeTimeLabel + ".jpg";
			cv::imwrite(irregularityImagesPath / irregularityImageFilename, oddFrame);

			// Append Irregularity information to JSON
			boost::uuids::uuid uuid = boost::uuids::random_generator()();
			irregularityFileOutput1["Irregularities"] += {
				{
					"IrregularityID", boost::lexical_cast<string>(uuid)
				}, {
					"Source", "v"
				}, {
					"TimeLabel", timeLabel
				}
			};
			irregularityFileOutput2["Irregularities"] += {
				{
					"IrregularityID", boost::lexical_cast<string>(uuid)
				}, {
					"Source", "v"
				}, {
					"TimeLabel", timeLabel
				}, {
					"ImageURI", irregularityImagesPath.string() + "/" + irregularityImageFilename
				}
			};
701

Matteo's avatar
update    
Matteo committed
702
703
			lastSaved = video_current_ms;
			savedFrames++;
704

Matteo's avatar
update    
Matteo committed
705
706
707
708
709
		} else {
			unsavedFrames++;
		}
		prevFrame = frame;
	}
710
711
712
}


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
713
714
715
716
717
718
719
720
721
722
723
724
725
/**
 * @brief main program, organised as:
 * - Get input from command line or config.json file;
 * - Check input parameters;
 * - Creation of output directories;
 * - Regions Of Interest (ROIs) detection;
 * - Irregularities detection;
 * - Saving of output IrregularityFiles.
 *
 * @param argc Command line arguments count;
 * @param argv Command line arguments.
 * @return int program status.
 */
726
727
int main(int argc, char** argv) {

Matteo's avatar
update    
Matteo committed
728
729
	json irregularityFileInput;
	fs::path irregularityFileInputPath;
Matteo's avatar
update    
Matteo committed
730
731
	cv::Mat myFrame;

732
733
734
	/*********************************************************************************************/
	/*************************************** CONFIGURATION ***************************************/
	/*********************************************************************************************/
735

736
	// Get the input from config.json or command line
737
	try {
738
739
740
		bool continueExecution = getArguments(argc, argv);
		if (!continueExecution) {
			return 0;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
741
		}
742
	} catch (nlohmann::detail::type_error e) {
743
		cerr << RED << "config.json error!" << endl << e.what() << END << endl;
744
745
		return -1;
	}
746

Matteo's avatar
update    
Matteo committed
747
	const fs::path VIDEO_PATH = config.workingPath / "PreservationAudioVisualFile" / config.filesName;
Matteo's avatar
update    
Matteo committed
748
749
750

    if (files::findFileName(VIDEO_PATH, fileName, extension) == -1) {
        cerr << RED << BOLD << "config.json error!" << END << endl << RED << VIDEO_PATH.string() << " cannot be found or opened." << END << endl;
751
752
753
        return -1;
    }

Matteo's avatar
update    
Matteo committed
754
755
	irregularityFileInputPath = config.workingPath / "temp" / fileName / "AudioAnalyser_IrregularityFileOutput1.json";

756

757
	// Input JSON check
758
	ifstream iJSON(irregularityFileInputPath);
759
	if (iJSON.fail()) {
760
		cerr << RED << BOLD << "config.json error!" << END << endl << RED << irregularityFileInputPath.string() << " cannot be found or opened." << END << endl;
761
762
		return -1;
	}
Matteo's avatar
update    
Matteo committed
763
	if (config.speed != 7.5 && config.speed != 15) {
764
		cerr << RED << BOLD << "config.json error!" << END << endl << RED << "Speed parameter must be 7.5 or 15 ips." << END << endl;
765
766
		return -1;
	}
Matteo's avatar
update    
Matteo committed
767
	if (tape.threshold.percentual < 0 || tape.threshold.percentual > 100) {
768
		cerr << RED << BOLD << "config.json error!" << END << endl << RED << "TapeThresholdPercentual parameter must be a percentage value." << END << endl;
769
770
		return -1;
	}
Matteo's avatar
update    
Matteo committed
771
	if (capstan.threshold.percentual < 0 || capstan.threshold.percentual > 100) {
772
		cerr << RED << BOLD << "config.json error!" << END << endl << RED << "CapstanThresholdPercentual parameter must be a percentage value." << END << endl;
773
774
775
		return -1;
	}

776
	// Adjust input paramenters (considering given ones as pertinent to a speed reference = 7.5)
Matteo's avatar
update    
Matteo committed
777
778
779
	if (config.brands) {
		if (config.speed == 15)
			tape.threshold.percentual += 6;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
780
	} else
Matteo's avatar
update    
Matteo committed
781
782
		if (config.speed == 15)
			tape.threshold.percentual += 20;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
783
		else
Matteo's avatar
update    
Matteo committed
784
			tape.threshold.percentual += 21;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
785

786
787
    cout << endl;
	cout << "Parameters:" << endl;
Matteo's avatar
update    
Matteo committed
788
789
790
791
	cout << "    Brands: " << config.brands << endl;
	cout << "    Speed: " << config.speed << endl;
    cout << "    ThresholdPercentual: " << tape.threshold.percentual << endl;
	cout << "    ThresholdPercentualCapstan: " << capstan.threshold.percentual << endl;
792
	cout << endl;
793
794
795
796

	// Read input JSON
	iJSON >> irregularityFileInput;

797
798
799
	/*********************************************************************************************/
	/*********************************** MAKE OUTPUT DIRECTORY ***********************************/
	/*********************************************************************************************/
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
800
801

	// Make directory with fileName name
Matteo's avatar
update    
Matteo committed
802
	outputPath = config.workingPath / "temp" / fileName;
803
	int outputFileNameDirectory = create_directory(outputPath);
Matteo's avatar
update    
Matteo committed
804

Matteo's avatar
update    
Matteo committed
805
806
	irregularityImagesPath = outputPath / "IrregularityImages";
	int fullFrameDirectory = fs::create_directory(irregularityImagesPath);
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
807

808
809
810
	/*********************************************************************************************/
	/************************************** AREAS DETECTION **************************************/
	/*********************************************************************************************/
811

Matteo's avatar
update    
Matteo committed
812
	cv::VideoCapture videoCapture(VIDEO_PATH);
813
    if (!videoCapture.isOpened()) {
814
        cerr << RED << BOLD << "Video unreadable." << END << endl;
815
816
817
        return -1;
    }

Matteo's avatar
update    
Matteo committed
818
	int frames_number = videoCapture.get(CAP_PROP_FRAME_COUNT);
819
	// Set frame position to half video length
Matteo's avatar
update    
Matteo committed
820
	videoCapture.set(CAP_PROP_POS_FRAMES, frames_number/2);
821
	// Get frame
822
	videoCapture >> myFrame;
823

Matteo's avatar
update    
Matteo committed
824
	cout << "Video resolution: " << myFrame.cols << "x" << myFrame.rows << endl;
825

Matteo's avatar
update    
Matteo committed
826
	bool found = findProcessingAreas(myFrame);
827
828
829
830

	// Reset frame position
	videoCapture.set(CAP_PROP_POS_FRAMES, 0);

Matteo's avatar
update    
Matteo committed
831
	if (!found) {
Matteo's avatar
Matteo committed
832
		pprint("Processing area not found. Try changing JSON parameters.", RED);
833
		return -1; // Program terminated early
834
835
	}

836
	/*********************************************************************************************/
837
	/**************************************** PROCESSING *****************************************/
838
	/*********************************************************************************************/
839

Matteo's avatar
Matteo committed
840
	pprint("\nProcessing...", CYAN);
841
842
843
844
845

	// Processing timer
	time_t startTimer, endTimer;
	startTimer = time(NULL);

846
	processing(videoCapture);
847
848
849
850
851

	endTimer = time(NULL);
	float min = (endTimer - startTimer) / 60;
	float sec = (endTimer - startTimer) % 60;

852
853
	string result("Processing elapsed time: " + to_string((int)min) + ":" + to_string((int)sec));
	cout << endl << result << endl;
854

855
856
857
	/*********************************************************************************************/
	/************************************* IRREGULARITY FILES ************************************/
	/*********************************************************************************************/
858

859
	fs::path outputFile1Name = outputPath / "VideoAnalyser_IrregularityFileOutput1.json";
Matteo's avatar
update    
Matteo committed
860
	files::saveFile(outputFile1Name, irregularityFileOutput1.dump(4), false);
861
862

	// Irregularities to extract for the AudioAnalyser and to the TapeIrregularityClassifier
Matteo's avatar
update    
Matteo committed
863
	extractIrregularityImagesForAudio(outputPath, VIDEO_PATH, irregularityFileInput, irregularityFileOutput2);
864

865
	fs::path outputFile2Name = outputPath / "VideoAnalyser_IrregularityFileOutput2.json";
Matteo's avatar
update    
Matteo committed
866
	files::saveFile(outputFile2Name, irregularityFileOutput2.dump(4), false);
867

868
869
    return 0;
}