main.cpp 36.4 KB
Newer Older
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/**
 *  MPAI CAE-ARP Video Analyser.
 *
 *	Implements MPAI CAE-ARP Video Analyser Technical Specification.
 *	It identifies Irregularities on the Preservation Audio-Visual File, providing:
 *	- Irregularity Files;
 *	- Irregularity Images.
 *
 *	WARNING:
 *	Currently, this program is only compatible with the Studer A810 and videos recorded in PAL standard.
 *
 *  @author Nadir Dalla Pozza
 *	@copyright 2022, Audio Innova S.r.l.
 *	@credits Niccolò Pretto, Nadir Dalla Pozza, Sergio Canazza
 *	@license GPL v3.0
 *	@version 1.0.1
 *	@maintainer Nadir Dalla Pozza
 *	@email nadir.dallapozza@unipd.it
 *	@status Production
 */
21
#include <filesystem>
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
22
#include <fstream>
23
#include <iostream>
24
#include <stdlib.h>
25
26
#include <sys/timeb.h>

27
#include <boost/program_options.hpp>
28
29
30
31
32
33
34
35
36
37
38
39
#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_generators.hpp> // generators
#include <boost/uuid/uuid_io.hpp>         // streaming operators etc.
#include <boost/lexical_cast.hpp>

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>

#include <nlohmann/json.hpp>

40
41
42
43
44
45
46
#include "opencv2/core.hpp"
#include "opencv2/calib3d.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/features2d.hpp"
#include "opencv2/xfeatures2d.hpp"

47
48
49
#include "utility.h"
#include "forAudioAnalyser.h"

Matteo's avatar
update    
Matteo committed
50
51
52
53
#include "lib/colors.h"
#include "lib/time.h"
#include "lib/parser.h"

54
55
56
using namespace cv;
using namespace std;
using json = nlohmann::json;
57
58
namespace fs = std::filesystem;
namespace po = boost::program_options;
59
60


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
61
// For capstan detection, there are two alternative approaches:
62
63
64
// Generalized Hough Transform and SURF.
bool useSURF = true;

Matteo's avatar
update    
Matteo committed
65
66
bool savingPinchRoller = false;
bool pinchRollerRect = false;
67
bool savingBrand = false;
68
bool endTapeSaved = false;
69
70
71
cv::Mat myFrame;
float mediaPrevFrame = 0;
bool firstBrand = true;	// The first frame containing brands on tape must be saved
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
72
float firstInstant = 0;
73
string fileName, extension;
74

75
76
77
78
// Path variables
fs::path outputPath;
fs::path irregularityImagesPath;
fs::path videoPath;
79
80
81
82
83
// JSON files
json configurationFile;
json irregularityFileOutput1;
json irregularityFileOutput2;
// RotatedRect identifying the processing area
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
84
RotatedRect rect, rectTape, rectCapstan;
85

Matteo's avatar
update    
Matteo committed
86
// Structs
87

Matteo's avatar
update    
Matteo committed
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
// config.json parameters
struct Config {
	fs::path workingPath;
	string filesName;
	bool brands;
	float speed;
};

struct Threshold {
	float percentual;
	int angle;
	int scale;
	int pos;
};

struct SceneObject {
	int minDist;
	Threshold threshold;
};

Config config;
SceneObject tape;
SceneObject capstan;

void saveFile(fs::path fileName, auto content, bool append) {
	ofstream outputFile;
	if (append) {
		outputFile.open(fileName, ios::app);
	} else {
		outputFile.open(fileName);
	}
	outputFile << content << endl;
	outputFile.close();
}
122

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
123
124
125
126
127
128
129
130
/**
 * @brief Get operation arguments from command line or config.json file.
 *
 * @param argc Command line arguments count;
 * @param argv Command line arguments.
 * @return true if input configuration is valid;
 * @return false otherwise.
 */
131
132
bool getArguments(int argc, char** argv) {
	// Read configuration file
Matteo's avatar
Matteo committed
133
	ifstream iConfig("config/config.json");
134
135
136
137
138
	iConfig >> configurationFile;

	if (argc == 1) {
		// Read from JSON file
		string wp = configurationFile["WorkingPath"];
Matteo's avatar
update    
Matteo committed
139
140
141
142
143
144
145
146

		config = {
			fs::path(wp),
			configurationFile["FilesName"],
			configurationFile["Brands"],
			configurationFile["Speed"]
		};

147
148
149
150
151
	} else {
		// Get from command line
		try {
			po::options_description desc(
				"A tool that implements MPAI CAE-ARP Video Analyser Technical Specification.\n"
Matteo's avatar
update    
Matteo committed
152
				"By default, the configuartion parameters are loaded from config/config.json file,\n"
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
				"but, alternately, you can pass command line arguments to replace them"
			);
			desc.add_options()
				("help,h", "Display this help message")
				("working-path,w", po::value<string>()->required(), "Specify the Working Path, where all input files are stored")
				("files-name,f", po::value<string>()->required(), "Specify the name of the Preservation files (without extension)")
				("brands,b", po::value<bool>()->required(), "Specify if the tape presents brands on its surface")
				("speed,s", po::value<float>()->required(), "Specify the speed at which the tape was read");
			po::variables_map vm;
			po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
			if (vm.count("help")) {
				cout << desc << "\n";
				return false;
			}
			po::notify(vm);
Matteo's avatar
update    
Matteo committed
168
169
170
171
172
173
174

			// Access the stored options
			config.workingPath = fs::path(vm["working-path"].as<string>());
			config.filesName = vm["files-name"].as<string>();
			config.brands = vm["brands"].as<bool>();
			config.speed = vm["speed"].as<float>();

175
176
177
178
179
180
181
182
183
		} catch (po::invalid_command_line_syntax& e) {
			cerr << RED << BOLD << "The command line syntax is invalid: " << END << RED << e.what() << END << endl;
			return false;
		} catch (po::required_option& e) {
			cerr << "Error: " << e.what() << endl;
			return false;
		}
	}

Matteo's avatar
update    
Matteo committed
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
	capstan = {
		configurationFile["MinDistCapstan"],
		{
			configurationFile["CapstanThresholdPercentual"],
			configurationFile["AngleThreshCapstan"],
			configurationFile["ScaleThreshCapstan"],
			configurationFile["PosThreshCapstan"]
		}
	};

	tape = {
		configurationFile["MinDist"],
		{
			configurationFile["TapeThresholdPercentual"],
			configurationFile["AngleThresh"],
			configurationFile["ScaleThresh"],
			configurationFile["PosThresh"]
		}
	};
203
204
205
206
207

	return true;
}


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
208
209
210
211
212
213
214
215
216
217
/**
 * @brief Identifies the Regions Of Interest (ROIs) on the video,
 * which are:
 * - The reading head;
 * - The tape area under the tape head (computed on the basis of the detected reading head);
 * - The capstan.
 *
 * @return true if some areas have been detected;
 * @return false otherwise.
 */
Matteo's avatar
update    
Matteo committed
218
bool findProcessingAreas() {
219
220
221
222
223
224
225
226
227

	/*********************************************************************************************/
	/*********************************** READING HEAD DETECTION **********************************/
	/*********************************************************************************************/

	// Obtain grayscale version of myFrame
	Mat myFrameGrayscale;
	cvtColor(myFrame, myFrameGrayscale, COLOR_BGR2GRAY);
	// Get input shape in grayscale
Matteo's avatar
update    
Matteo committed
228
	Mat templateImage = imread("input/readingHead.png", IMREAD_GRAYSCALE);
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
	// Downsample myFrameGrayscale in half pixels for performance reasons
	Mat myFrameGrayscaleHalf;
	pyrDown(myFrameGrayscale, myFrameGrayscaleHalf, Size(myFrame.cols/2, myFrame.rows/2));
	// Downsample tapeShape in half pixels
	Mat templateImageHalf;
	pyrDown(templateImage, templateImageHalf, Size(templateImage.cols/2, templateImage.rows/2));

	// Process only the bottom-central portion of the input video -> best results with our videos
	Rect readingHeadProcessingAreaRect(myFrameGrayscaleHalf.cols/4, myFrameGrayscaleHalf.rows/2, myFrameGrayscaleHalf.cols/2, myFrameGrayscaleHalf.rows/2);
	Mat processingImage = myFrameGrayscaleHalf(readingHeadProcessingAreaRect);
	// Select the template to be detected
	Mat templateShape = templateImageHalf;

	// Algorithm and parameters
	Ptr<GeneralizedHoughGuil> alg = createGeneralizedHoughGuil();

	vector<Vec4f> positionsPos, positionsNeg;
	Mat votesPos, votesNeg;
	TickMeter tm;
Matteo's avatar
update    
Matteo committed
248
	int oldPosThresh = tape.threshold.pos;
249
250
251
252
253
254
255
256
257
258
	RotatedRect rectPos, rectNeg;
	ofstream myFile;
	Point2f pts[4];

	// Find the best matches for positive and negative angles
	// If there are more than one shapes, then choose the one with the highest score
	// If there are more than one with the same highest score, then arbitrarily choose the latest
	double maxValPos = 0, maxValNeg = 0;
	int indexPos = 0, indexNeg = 0;

Matteo's avatar
update    
Matteo committed
259
	alg -> setMinDist(tape.minDist);
260
261
262
263
264
	alg -> setLevels(360);
	alg -> setDp(2);
	alg -> setMaxBufferSize(1000);

	alg -> setAngleStep(1);
Matteo's avatar
update    
Matteo committed
265
	alg -> setAngleThresh(tape.threshold.angle);
266
267
268
269

	alg -> setMinScale(0.9);
	alg -> setMaxScale(1.1);
	alg -> setScaleStep(0.01);
Matteo's avatar
update    
Matteo committed
270
	alg -> setScaleThresh(tape.threshold.scale);
271

Matteo's avatar
update    
Matteo committed
272
	alg -> setPosThresh(tape.threshold.pos);
273
274
275
276
277
278
279
280
281

	alg -> setCannyLowThresh(150); // Old: 100
	alg -> setCannyHighThresh(240); // Old: 300

	alg -> setTemplate(templateShape);

	cout << DARK_CYAN << "Reading head" << END << endl;
	tm.start();
	// Invoke utility.h function
Matteo's avatar
update    
Matteo committed
282
283
	
	detectShape(alg, templateShape, tape.threshold.pos, positionsPos, votesPos, positionsNeg, votesNeg, processingImage);
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
	tm.stop();
	cout << "Reading head detection time: " << tm.getTimeMilli() << " ms" << endl;

	for (int i = 0; i < votesPos.size().width; i++) {
		if (votesPos.at<int>(i) >= maxValPos) {
			maxValPos = votesPos.at<int>(i);
			indexPos = i;
		}
	}

	for (int i = 0; i < votesNeg.size().width; i++) {
		if (votesNeg.at<int>(i) >= maxValNeg) {
			maxValNeg = votesNeg.at<int>(i);
			indexNeg = i;
		}
	}

	// The color is progressively darkened to emphasize that the algorithm found more than one shape
	if (positionsPos.size() > 0)
		rectPos = drawShapes(myFrame, positionsPos[indexPos], Scalar(0, 0, 255-indexPos*64), templateImageHalf.cols, templateImageHalf.rows, myFrameGrayscaleHalf.cols/4, myFrameGrayscaleHalf.rows/2, 2);
	if (positionsNeg.size() > 0)
		rectNeg = drawShapes(myFrame, positionsNeg[indexNeg], Scalar(128, 128, 255-indexNeg*64), templateImageHalf.cols, templateImageHalf.rows, myFrameGrayscaleHalf.cols/4, myFrameGrayscaleHalf.rows/2, 2);

	myFile.open("log.txt", ios::app);

	if (maxValPos > 0)
		if (maxValNeg > 0)
			if (maxValPos > maxValNeg) {
				myFile << "READING HEAD: Positive angle is best, match number: " << indexPos << endl;
				rect = rectPos;
			} else {
				myFile << "READING HEAD: Negative angle is best, match number: " << indexNeg << endl;
				rect = rectNeg;
			}
		else {
			myFile << "READING HEAD: Positive angle is the only choice, match number: " << indexPos << endl;
			rect = rectPos;
		}
	else if (maxValNeg > 0) {
		myFile << "READING HEAD: Negative angle is the only choice, match number: " << indexNeg << endl;
		rect = rectNeg;
	} else {
		myFile.close();
		return false;
	}
	cout << endl;

	rect.points(pts);

	/*********************************************************************************************/
	/************************************ TAPE AREA DETECTION ************************************/
	/*********************************************************************************************/

	// Compute area basing on reading head detection
	Vec4f positionTape( rect.center.x, rect.center.y + rect.size.height / 2 + 20 * (rect.size.width / 200), 1, rect.angle );
	rectTape = drawShapes(myFrame, positionTape, Scalar(0, 255-indexPos*64, 0), rect.size.width, 50 * (rect.size.width / 200), 0, 0, 1);

	myFile << "Tape area:" << endl;
	myFile << "  Center (x, y): (" << rectTape.center.x << ", " << rectTape.center.y << ")" << endl;
	myFile << "  Size (w, h): (" << rectTape.size.width << ", " << rectTape.size.height << ")" << endl;
	myFile << "  Angle (deg): (" << rectTape.angle << ")" << endl;

	json autoJSON;
	autoJSON["PreservationAudioVisualFile"] = fileName;
	autoJSON["RotatedRect"] = {
		{
			"CenterX", rectTape.center.x
		}, {
			"CenterY", rectTape.center.y
		}, {
			"Width", rectTape.size.width
		}, {
			"Height", rectTape.size.height
		}, {
			"Angle", rectTape.angle
		}
	};

Matteo's avatar
update    
Matteo committed
362
	saveFile(fs::path("./" + fileName + ".json"), autoJSON.dump(4), false);
363
364
365
366
367
368

	/*********************************************************************************************/
	/************************************* CAPSTAN DETECTION *************************************/
	/*********************************************************************************************/

	// Read template image - it is smaller than before, therefore there is no need to downsample
Matteo's avatar
update    
Matteo committed
369
	templateShape = imread("input/capstanBERIO058prova.png", IMREAD_GRAYSCALE); // WORKING
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
	// templateShape = imread("../input/capstanBERIO058.png", IMREAD_GRAYSCALE);

	cout << DARK_CYAN << "Capstan" << END << endl;

	if (useSURF) {

		// Step 1: Detect the keypoints using SURF Detector, compute the descriptors
		int minHessian = 100;
		Ptr<xfeatures2d::SURF> detector = xfeatures2d::SURF::create(minHessian);
		vector<KeyPoint> keypoints_object, keypoints_scene;
		Mat descriptors_object, descriptors_scene;

		tm.reset();
		tm.start();
		detector->detectAndCompute(templateShape, noArray(), keypoints_object, descriptors_object);
		detector->detectAndCompute(myFrameGrayscale, noArray(), keypoints_scene, descriptors_scene);
		tm.stop();
		cout << "Capstan detection time: " << tm.getTimeMilli() << " ms" << endl;

		// Step 2: Matching descriptor vectors with a FLANN based matcher
		// Since SURF is a floating-point descriptor NORM_L2 is used
		Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create(DescriptorMatcher::FLANNBASED);
		vector<vector<DMatch>> knn_matches;
		matcher->knnMatch(descriptors_object, descriptors_scene, knn_matches, 2);
		//-- Filter matches using the Lowe's ratio test
		const float ratio_thresh = 0.75f;
		vector<DMatch> good_matches;
		for (size_t i = 0; i < knn_matches.size(); i++) {
			if (knn_matches[i][0].distance < ratio_thresh * knn_matches[i][1].distance) {
				good_matches.push_back(knn_matches[i][0]);
			}
		}
		// Draw matches
		Mat img_matches;
		drawMatches(templateShape, keypoints_object, myFrameGrayscale, keypoints_scene, good_matches, img_matches, Scalar::all(-1), Scalar::all(-1), vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
		// Localize the object
		vector<Point2f> obj;
		vector<Point2f> scene;
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
408
		for (size_t i = 0; i < good_matches.size(); i++) {
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
			// Get the keypoints from the good matches
			obj.push_back(keypoints_object[good_matches[i].queryIdx].pt);
			scene.push_back(keypoints_scene[good_matches[i].trainIdx].pt);
		}
		Mat H = findHomography(obj, scene, RANSAC);
		// Get the corners from the image_1 ( the object to be "detected" )
		vector<Point2f> obj_corners(4);
		obj_corners[0] = Point2f(0, 0);
		obj_corners[1] = Point2f((float)templateShape.cols, 0);
		obj_corners[2] = Point2f((float)templateShape.cols, (float)templateShape.rows);
		obj_corners[3] = Point2f(0, (float)templateShape.rows);
		vector<Point2f> scene_corners(4);
		perspectiveTransform( obj_corners, scene_corners, H);

		// Find average
		float capstanX = (scene_corners[0].x + scene_corners[1].x + scene_corners[2].x + scene_corners[3].x) / 4;
		float capstanY = (scene_corners[0].y + scene_corners[1].y + scene_corners[2].y + scene_corners[3].y) / 4;

		// In the following there are two alterations to cut the first 20 horizontal pixels and the first 90 vertical pixels from the found rectangle:
		// +10 in X for centering and -20 in width
		// +45 in Y for centering and -90 in height
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
430
		Vec4f positionCapstan(capstanX + 10, capstanY + 45, 1, 0);
431
432
433
434
435
436
437
438
439
440
441
442
443
444
		rectCapstan = drawShapes(myFrame, positionCapstan, Scalar(255-indexPos*64, 0, 0), templateShape.cols - 20, templateShape.rows - 90, 0, 0, 1);

	} else {

		// Process only right portion of the image, wherw the capstain always appears
		int capstanProcessingAreaRectX = myFrame.cols*3/4;
		int capstanProcessingAreaRectY = myFrame.rows/2;
		int capstanProcessingAreaRectWidth = myFrame.cols/4;
		int capstanProcessingAreaRectHeight = myFrame.rows/2;
		Rect capstanProcessingAreaRect(capstanProcessingAreaRectX, capstanProcessingAreaRectY, capstanProcessingAreaRectWidth, capstanProcessingAreaRectHeight);
		Mat capstanProcessingAreaGrayscale = myFrameGrayscale(capstanProcessingAreaRect);
		// Reset algorithm and set parameters
		alg = createGeneralizedHoughGuil();

Matteo's avatar
update    
Matteo committed
445
		alg -> setMinDist(capstan.minDist);
446
447
448
449
450
		alg -> setLevels(360);
		alg -> setDp(2);
		alg -> setMaxBufferSize(1000);

		alg -> setAngleStep(1);
Matteo's avatar
update    
Matteo committed
451
		alg -> setAngleThresh(capstan.threshold.angle);
452
453
454
455

		alg -> setMinScale(0.9);
		alg -> setMaxScale(1.1);
		alg -> setScaleStep(0.01);
Matteo's avatar
update    
Matteo committed
456
		alg -> setScaleThresh(capstan.threshold.scale);
457

Matteo's avatar
update    
Matteo committed
458
		alg -> setPosThresh(capstan.threshold.pos);
459
460
461
462
463
464

		alg -> setCannyLowThresh(150);
		alg -> setCannyHighThresh(240);

		alg -> setTemplate(templateShape);

Matteo's avatar
update    
Matteo committed
465
		oldPosThresh = capstan.threshold.pos;
466
467
468
469
470
471

		vector<Vec4f> positionsC1Pos, positionsC1Neg;
		Mat votesC1Pos, votesC1Neg;

		tm.reset();
		tm.start();
Matteo's avatar
update    
Matteo committed
472
		detectShape(alg, templateShape, capstan.threshold.pos, positionsC1Pos, votesC1Pos, positionsC1Neg, votesC1Neg, capstanProcessingAreaGrayscale);
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
		tm.stop();
		cout << "Capstan detection time: " << tm.getTimeMilli() << " ms" << endl;

		// Find the best matches for positive and negative angles
		// If there are more than one shapes, then choose the one with the highest score
		// If there are more than one with the same highest score, then choose the latest
		maxValPos = 0, maxValNeg = 0, indexPos = 0, indexNeg = 0;

		for (int i = 0; i < votesC1Pos.size().width; i++) {
			if (votesC1Pos.at<int>(i) >= maxValPos) {
				maxValPos = votesC1Pos.at<int>(i);
				indexPos = i;
			}
		}

		for (int i = 0; i < votesC1Neg.size().width; i++) {
			if (votesC1Neg.at<int>(i) >= maxValNeg) {
				maxValNeg = votesC1Neg.at<int>(i);
				indexNeg = i;
			}
		}

		RotatedRect rectCapstanPos, rectCapstanNeg;
		if (positionsC1Pos.size() > 0)
			rectCapstanPos = drawShapes(myFrame, positionsC1Pos[indexPos], Scalar(255-indexPos*64, 0, 0), templateShape.cols-22, templateShape.rows-92, capstanProcessingAreaRectX+11, capstanProcessingAreaRectY+46, 1);
		if (positionsC1Neg.size() > 0)
			rectCapstanNeg = drawShapes(myFrame, positionsC1Neg[indexNeg], Scalar(255-indexNeg*64, 128, 0), templateShape.cols-22, templateShape.rows-92, capstanProcessingAreaRectX+11, capstanProcessingAreaRectY+46, 1);

		if (maxValPos > 0)
			if (maxValNeg > 0)
				if (maxValPos > maxValNeg) {
					myFile << "CAPSTAN: Positive is best, match number: " << indexPos << endl;
					rectCapstan = rectCapstanPos;
				} else {
					myFile << "CAPSTAN: Negative is best, match number: " << indexNeg << endl;
					rectCapstan = rectCapstanNeg;
				}
			else {
				myFile << "CAPSTAN: Positive is the only choice, match number: " << indexPos << endl;
				rectCapstan = rectCapstanPos;
			}
		else if (maxValNeg > 0) {
			myFile << "CAPSTAN: Negative is the only choice, match number: " << indexNeg << endl;
			rectCapstan = rectCapstanNeg;
		} else {
			myFile.close();
			return false;
		}

	}

	myFile << "Capstan ROI:" << endl;
	myFile << "  Center (x, y): (" << rectCapstan.center.x << ", " << rectCapstan.center.y << ")" << endl;
	myFile << "  Size (w, h): (" << rectCapstan.size.width << ", " << rectCapstan.size.height << ")" << endl;
	myFile << "  Angle (deg): (" << rectCapstan.angle << ")" << endl;
	myFile.close();
	cout << endl;

	// Save the image containing the detected areas
	cv::imwrite(outputPath.string() + "/tapeAreas.jpg", myFrame);

	return true;
}


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
538
539
540
541
542
543
544
545
546
547
/**
 * @brief Compares two consecutive video frames and establish if there potentially is an Irregularity.
 * The comparison is pixel-wise and based on threshold values set on config.json file.
 *
 * @param prevFrame the frame before the current one;
 * @param currentFrame the current frame;
 * @param msToEnd the number of milliseconds left before the end of the video. Useful for capstan analysis.
 * @return true if a potential Irregularity has been found;
 * @return false otherwise.
 */
548
549
bool frameDifference(cv::Mat prevFrame, cv::Mat currentFrame, int msToEnd) {

550
	/*********************************************************************************************/
551
	/********************************** Capstan analysis *****************************************/
552
	/*********************************************************************************************/
553
554

	// In the last minute of the video, check for pinchRoller position for endTape event
555
	if (!endTapeSaved && msToEnd < 60000) {
556
557
558

		// Capstan area
		int capstanAreaPixels = rectCapstan.size.width * rectCapstan.size.height;
Matteo's avatar
update    
Matteo committed
559
		float capstanDifferentPixelsThreshold = capstanAreaPixels * capstan.threshold.percentual / 100;
560

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
561
		// Extract matrices corresponding to the processing area
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
		// CODE FROM https://answers.opencv.org/question/497/extract-a-rotatedrect-area/

		// matrices we'll use
		Mat M, rotatedPrevFrame, croppedPrevFrame, rotatedCurrentFrame, croppedCurrentFrame;
		// get angle and size from the bounding box
		float angle = rectCapstan.angle;
		Size rect_size = rectCapstan.size;
		// thanks to http://felix.abecassis.me/2011/10/opencv-rotation-deskewing/
		if (rectCapstan.angle < -45.) {
			angle += 90.0;
			swap(rect_size.width, rect_size.height);
		}
		// get the rotation matrix
		M = getRotationMatrix2D(rectCapstan.center, angle, 1.0);
		// perform the affine transformation
577
578
		cv::warpAffine(prevFrame, rotatedPrevFrame, M, prevFrame.size(), INTER_CUBIC);
		cv::warpAffine(currentFrame, rotatedCurrentFrame, M, currentFrame.size(), INTER_CUBIC);
579
		// crop the resulting image
580
581
		cv::getRectSubPix(rotatedPrevFrame, rect_size, rectCapstan.center, croppedPrevFrame);
		cv::getRectSubPix(rotatedCurrentFrame, rect_size, rectCapstan.center, croppedCurrentFrame);
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599

		// END CODE FROM https://answers.opencv.org/question/497/extract-a-rotatedrect-area/

		cv::Mat differenceFrame = difference(croppedPrevFrame, croppedCurrentFrame);

		int blackPixelsCapstan = 0;

		for (int i = 0; i < croppedCurrentFrame.rows; i++) {
			for (int j = 0; j < croppedCurrentFrame.cols; j++) {
				if (differenceFrame.at<cv::Vec3b>(i, j)[0] == 0) {
					// There is a black pixel, then there is a difference between previous and current frames
					blackPixelsCapstan++;
				}
			}
		}

		if (blackPixelsCapstan > capstanDifferentPixelsThreshold) {
			savingPinchRoller = true;
600
			endTapeSaved = true; // Never check again for end tape instant
601
602
603
604
			return true;
		} else {
			savingPinchRoller = false;
		}
605
606
	} else {
		savingPinchRoller = false; // It will already be false before the last minute of the video. After having saved the capstan, the next time reset the variable to not save again
607
	}
608
609

	/*********************************************************************************************/
610
	/************************************ Tape analysis ******************************************/
611
	/*********************************************************************************************/
612
613
614

	// Tape area
    int tapeAreaPixels = rectTape.size.width * rectTape.size.height;
Matteo's avatar
update    
Matteo committed
615
	float tapeDifferentPixelsThreshold = tapeAreaPixels * tape.threshold.percentual / 100;
616

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
617
	// Extract matrices corresponding to the processing area
618
619
620
621
622
	// CODE FROM https://answers.opencv.org/question/497/extract-a-rotatedrect-area/

	// matrices we'll use
	Mat M, rotatedPrevFrame, croppedPrevFrame, rotatedCurrentFrame, croppedCurrentFrame;
	// get angle and size from the bounding box
623
624
	float angle = rectTape.angle;
	Size rect_size = rectTape.size;
625
	// thanks to http://felix.abecassis.me/2011/10/opencv-rotation-deskewing/
626
	if (rectTape.angle < -45.) {
627
628
629
630
		angle += 90.0;
		swap(rect_size.width, rect_size.height);
	}
	// get the rotation matrix
631
	M = getRotationMatrix2D(rectTape.center, angle, 1.0);
632
	// perform the affine transformation
633
634
	cv::warpAffine(prevFrame, rotatedPrevFrame, M, prevFrame.size(), INTER_CUBIC);
	cv::warpAffine(currentFrame, rotatedCurrentFrame, M, currentFrame.size(), INTER_CUBIC);
635
	// crop the resulting image
636
637
	cv::getRectSubPix(rotatedPrevFrame, rect_size, rectTape.center, croppedPrevFrame);
	cv::getRectSubPix(rotatedCurrentFrame, rect_size, rectTape.center, croppedCurrentFrame);
638
639
640
641
642
643
644
645
646
647
648

	// END CODE FROM https://answers.opencv.org/question/497/extract-a-rotatedrect-area/

	cv::Mat differenceFrame = difference(croppedPrevFrame, croppedCurrentFrame);

	int decEnd = (msToEnd % 1000) / 100;
	int secEnd = (msToEnd - (msToEnd % 1000)) / 1000;
	int minEnd = secEnd / 60;
	secEnd = secEnd % 60;


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
649
	/************************************* Segment analysis **************************************/
650

651
652
653
654
655
656
657
658
659
660
661
662
  	int blackPixels = 0;
	float mediaCurrFrame;
	int totColoreCF = 0;

	for (int i = 0; i < croppedCurrentFrame.rows; i++) {
		for (int j = 0; j < croppedCurrentFrame.cols; j++) {
			totColoreCF += croppedCurrentFrame.at<cv::Vec3b>(i, j)[0] + croppedCurrentFrame.at<cv::Vec3b>(i, j)[1] + croppedCurrentFrame.at<cv::Vec3b>(i, j)[2];
			if (differenceFrame.at<cv::Vec3b>(i, j)[0] == 0) {
				blackPixels++;
			}
		}
	}
663
	mediaCurrFrame = totColoreCF/tapeAreaPixels;
664

Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
665
666
	/************************************* Decision stage ****************************************/

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
667
	bool isIrregularity = false;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
668

669
	if (blackPixels > tapeDifferentPixelsThreshold) { // The threshold must be passed
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
670

671
672
		/***** AVERAGE_COLOR-BASED DECISION *****/
		if (mediaPrevFrame > (mediaCurrFrame + 7) || mediaPrevFrame < (mediaCurrFrame - 7)) { // They are not similar for color average
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
673
			isIrregularity = true;
674
		}
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
675

676
		/***** BRANDS MANAGEMENT *****/
Matteo's avatar
update    
Matteo committed
677
		if (config.brands) {
678
			// At the beginning of the video, wait at least 5 seconds before the next Irregularity to consider it as a brand.
679
			// It is not guaranteed that it will be the first brand, but it is generally a safe approach to have a correct image
680
681
682
683
			if (firstBrand) {
				if (firstInstant - msToEnd > 5000) {
					firstBrand = false;
					savingBrand = true;
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
684
					isIrregularity = true;
685
				}
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
686
			// In the following iterations reset savingBrand, since we are no longer interested in brands.
687
			} else
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
688
				savingBrand = false;
689
		}
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
690

691
	}
692
693
694
695

	// Update mediaPrevFrame
	mediaPrevFrame = mediaCurrFrame;

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
696
	return isIrregularity;
697
698
699
}


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
700
701
702
703
704
705
706
/**
 * @brief video processing phase, where each frame is analysed.
 * It saves the IrregularityImages and updates the IrregularityFiles if an Irregularity is found
 *
 * @param videoCapture the input Preservation Audio-Visual File;
 */
void processing(cv::VideoCapture videoCapture) {
707
708
709
710
711
712

	// Video duration
	int frameNumbers_v = videoCapture.get(CAP_PROP_FRAME_COUNT);
	float fps_v = videoCapture.get(CAP_PROP_FPS); // FPS can be non-integers!!!
	float videoLength = (float) frameNumbers_v / fps_v; // [s]
	int videoLength_ms = videoLength * 1000;
713

714
715
    int savedFrames = 0, unsavedFrames = 0;
	float lastSaved = -160;
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
716
	// Whenever we find an Irregularity, we want to skip a lenght equal to the Studer reading head (3 cm = 1.18 inches).
717
718
	int savingRate = 79; // [ms]. Time taken to cross 3 cm at 15 ips, or 1.5 cm at 7.5 ips. The considered lengths are the widths of the tape areas.
	// The following condition constitutes a valid approach if the tape areas have widths always equal to the reading head
Matteo's avatar
update    
Matteo committed
719
	if (config.speed == 7.5)
720
721
722
723
724
		savingRate = 157; // Time taken to cross 3 cm at 7.5 ips

	// The first frame of the video won't be processed
    cv::Mat prevFrame;
	videoCapture >> prevFrame;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
725
	firstInstant = videoLength_ms - videoCapture.get(CAP_PROP_POS_MSEC);
726
727
728
729
730
731
732
733
734
735
736
737

    while (videoCapture.isOpened()) {

		cv::Mat frame;
        videoCapture >> frame;

        if (!frame.empty()) {

			int ms = videoCapture.get(CAP_PROP_POS_MSEC);
			int msToEnd = videoLength_ms - ms;
			if (ms == 0) // With OpenCV library, this happens at the last few frames of the video before realising that "frame" is empty.
				break;
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
738
739

			// Variables to display program status
740
741
742
743
			int secToEnd = msToEnd / 1000;
			int minToEnd = (secToEnd / 60) % 60;
			secToEnd = secToEnd % 60;

744
			string secStrToEnd = to_string(secToEnd), minStrToEnd = to_string(minToEnd);
745
746
747
748
749
			if (minToEnd < 10)
				minStrToEnd = "0" + minStrToEnd;
			if (secToEnd < 10)
				secStrToEnd = "0" + secStrToEnd;

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
750
			// Display program status
751
752
			cout << "\rIrregularities: " << savedFrames << ".   ";
			cout << "Remaining video time [mm:ss]: " << minStrToEnd << ":" << secStrToEnd << flush;
753
754

			if ((ms - lastSaved > savingRate) && frameDifference(prevFrame, frame, msToEnd)) {
755

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
756
				// An Irregularity has been found!
757
758
759
760
761
762

				// De-interlacing frame
				cv::Mat oddFrame(frame.rows/2, frame.cols, CV_8UC3);
				cv::Mat evenFrame(frame.rows/2, frame.cols, CV_8UC3);
				separateFrame(frame, oddFrame, evenFrame);

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
763
				// Extract the image corresponding to the ROIs
764
				Point2f pts[4];
765
766
767
768
769
				if (savingPinchRoller)
					rectCapstan.points(pts);
				else
					rectTape.points(pts);
				cv::Mat subImage(frame, cv::Rect(100, min(pts[1].y, pts[2].y), frame.cols - 100, static_cast<int>(rectTape.size.height)));
770

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
771
				// De-interlacing
772
773
774
				cv::Mat oddSubImage(subImage.rows/2, subImage.cols, CV_8UC3);
				int evenSubImageRows = subImage.rows/2;
				if (subImage.rows % 2 != 0) // If the found rectangle is of odd height, we must increase evenSubImage height by 1, otherwise we have segmentation_fault!!!
775
					evenSubImageRows += 1;
776
777
				cv::Mat evenSubImage(evenSubImageRows, subImage.cols, CV_8UC3);
				separateFrame(subImage, oddSubImage, evenSubImage);
778

Matteo's avatar
update    
Matteo committed
779
780
				string timeLabel = getTimeLabel(ms, ":");
				string safeTimeLabel = getTimeLabel(ms, "-");
781

782
783
				string irregularityImageFilename = to_string(savedFrames) + "_" + safeTimeLabel + ".jpg";
				cv::imwrite(irregularityImagesPath / irregularityImageFilename, oddFrame);
784
785
786

				// Append Irregularity information to JSON
				boost::uuids::uuid uuid = boost::uuids::random_generator()();
787
788
789
				irregularityFileOutput1["Irregularities"] += {
					{
						"IrregularityID", boost::lexical_cast<string>(uuid)
790
791
792
793
794
795
					}, {
						"Source", "v"
					}, {
						"TimeLabel", timeLabel
					}
				};
796
797
798
				irregularityFileOutput2["Irregularities"] += {
					{
						"IrregularityID", boost::lexical_cast<string>(uuid)
799
800
801
802
803
					}, {
						"Source", "v"
					}, {
						"TimeLabel", timeLabel
					}, {
804
						"ImageURI", irregularityImagesPath.string() + "/" + irregularityImageFilename
805
806
807
808
809
810
811
812
813
814
815
816
817
					}
				};

				lastSaved = ms;
				savedFrames++;

			} else {
				unsavedFrames++;
			}

			prevFrame = frame;

	    } else {
818
			cout << endl << "Empty frame!" << endl;
819
820
821
822
823
824
825
	    	videoCapture.release();
	    	break;
	    }
	}

	ofstream myFile;
	myFile.open("log.txt", ios::app);
826
	myFile << "Saved frames are: " << savedFrames << endl;
827
828
829
830
831
	myFile.close();

}


Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
/*************************************************************************************************/
/********************************************* MAIN **********************************************/
/*************************************************************************************************/

/**
 * @brief main program, organised as:
 * - Get input from command line or config.json file;
 * - Check input parameters;
 * - Creation of output directories;
 * - Regions Of Interest (ROIs) detection;
 * - Irregularities detection;
 * - Saving of output IrregularityFiles.
 *
 * @param argc Command line arguments count;
 * @param argv Command line arguments.
 * @return int program status.
 */
849
850
int main(int argc, char** argv) {

Matteo's avatar
update    
Matteo committed
851
852
	json irregularityFileInput;
	fs::path irregularityFileInputPath;
853
854
855
	/*********************************************************************************************/
	/*************************************** CONFIGURATION ***************************************/
	/*********************************************************************************************/
856

857
	// Get the input from config.json or command line
858
	try {
859
860
861
		bool continueExecution = getArguments(argc, argv);
		if (!continueExecution) {
			return 0;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
862
		}
863
	} catch (nlohmann::detail::type_error e) {
864
		cerr << RED << "config.json error!" << endl << e.what() << END << endl;
865
866
		return -1;
	}
867

Matteo's avatar
update    
Matteo committed
868
	videoPath = config.workingPath / "PreservationAudioVisualFile" / config.filesName;
869
870
871
872
873
    if (findFileName(videoPath, fileName, extension) == -1) {
        cerr << RED << BOLD << "config.json error!" << END << endl << RED << videoPath.string() << " cannot be found or opened." << END << endl;
        return -1;
    }

Matteo's avatar
update    
Matteo committed
874
875
	irregularityFileInputPath = config.workingPath / "temp" / fileName / "AudioAnalyser_IrregularityFileOutput1.json";

876

877
	// Input JSON check
878
	ifstream iJSON(irregularityFileInputPath);
879
	if (iJSON.fail()) {
880
		cerr << RED << BOLD << "config.json error!" << END << endl << RED << irregularityFileInputPath.string() << " cannot be found or opened." << END << endl;
881
882
		return -1;
	}
Matteo's avatar
update    
Matteo committed
883
	if (config.speed != 7.5 && config.speed != 15) {
884
		cerr << RED << BOLD << "config.json error!" << END << endl << RED << "Speed parameter must be 7.5 or 15 ips." << END << endl;
885
886
		return -1;
	}
Matteo's avatar
update    
Matteo committed
887
	if (tape.threshold.percentual < 0 || tape.threshold.percentual > 100) {
888
		cerr << RED << BOLD << "config.json error!" << END << endl << RED << "TapeThresholdPercentual parameter must be a percentage value." << END << endl;
889
890
		return -1;
	}
Matteo's avatar
update    
Matteo committed
891
	if (capstan.threshold.percentual < 0 || capstan.threshold.percentual > 100) {
892
		cerr << RED << BOLD << "config.json error!" << END << endl << RED << "CapstanThresholdPercentual parameter must be a percentage value." << END << endl;
893
894
895
		return -1;
	}

896
	// Adjust input paramenters (considering given ones as pertinent to a speed reference = 7.5)
Matteo's avatar
update    
Matteo committed
897
898
899
	if (config.brands) {
		if (config.speed == 15)
			tape.threshold.percentual += 6;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
900
	} else
Matteo's avatar
update    
Matteo committed
901
902
		if (config.speed == 15)
			tape.threshold.percentual += 20;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
903
		else
Matteo's avatar
update    
Matteo committed
904
			tape.threshold.percentual += 21;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
905

906
907
    cout << endl;
	cout << "Parameters:" << endl;
Matteo's avatar
update    
Matteo committed
908
909
910
911
	cout << "    Brands: " << config.brands << endl;
	cout << "    Speed: " << config.speed << endl;
    cout << "    ThresholdPercentual: " << tape.threshold.percentual << endl;
	cout << "    ThresholdPercentualCapstan: " << capstan.threshold.percentual << endl;
912
	cout << endl;
913
914
915
916

	// Read input JSON
	iJSON >> irregularityFileInput;

917
918
919
	/*********************************************************************************************/
	/*********************************** MAKE OUTPUT DIRECTORY ***********************************/
	/*********************************************************************************************/
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
920
921

	// Make directory with fileName name
Matteo's avatar
update    
Matteo committed
922
	outputPath = config.workingPath / "temp" / fileName;
923
	int outputFileNameDirectory = create_directory(outputPath);
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
924
	// Get now time
925
926
927
	time_t t = chrono::system_clock::to_time_t(chrono::system_clock::now());
    string ts = ctime(&t);
	// Write useful info to log file
Matteo's avatar
update    
Matteo committed
928
929
930
931
932
	fs::path logPath = outputPath / "log.txt";

	string logInfo = fileName + '\n' + "tsh: " + to_string(tape.threshold.percentual) + "   tshp: " + to_string(capstan.threshold.percentual) + '\n' + ts;
	saveFile(logPath, logInfo, true);

Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
933

934
935
936
	/*********************************************************************************************/
	/************************************** AREAS DETECTION **************************************/
	/*********************************************************************************************/
937
938
939

	cv::VideoCapture videoCapture(videoPath);
    if (!videoCapture.isOpened()) {
940
        cerr << RED << BOLD << "Video unreadable." << END << endl;
941
942
943
944
945
946
947
        return -1;
    }

	// Get total number of frames
	int totalFrames = videoCapture.get(CAP_PROP_FRAME_COUNT);
	// Set frame position to half video length
	videoCapture.set(CAP_PROP_POS_FRAMES, totalFrames/2);
948
	// Get frame
949
	videoCapture >> myFrame;
950
951
952

	cout << "Video resolution: " << myFrame.cols << "x" << myFrame.rows << endl << endl;

953
	// Find the processing area corresponding to the tape area over the reading head
Matteo's avatar
update    
Matteo committed
954
	bool found = findProcessingAreas();
955
956
957
958

	// Reset frame position
	videoCapture.set(CAP_PROP_POS_FRAMES, 0);

959
	// Write useful information to log file
Matteo's avatar
update    
Matteo committed
960
	string message;
961
	if (found) {
Matteo's avatar
update    
Matteo committed
962
963
964
		message = "Processing areas found!\n";
		cout << message;
		saveFile(logPath, message, true);
965
	} else {
Matteo's avatar
update    
Matteo committed
966
967
968
		message = "Processing area not found. Try changing JSON parameters.\n";
		cout << message;
		saveFile(logPath, message, true);
969
		return -1; // Program terminated early
970
971
	}

972
973
974
	/*********************************************************************************************/
	/***************************** MAKE ADDITIONAL OUTPUT DIRECTORIES ****************************/
	/*********************************************************************************************/
975

976
977
978
979
	irregularityImagesPath = outputPath / "IrregularityImages";
	int fullFrameDirectory = fs::create_directory(irregularityImagesPath);

	/*********************************************************************************************/
980
	/**************************************** PROCESSING *****************************************/
981
	/*********************************************************************************************/
982

Matteo's avatar
update    
Matteo committed
983
	cout << '\n' << CYAN << "Starting processing..." << END << '\n';
984
985
986
987
988

	// Processing timer
	time_t startTimer, endTimer;
	startTimer = time(NULL);

989
	processing(videoCapture);
990
991
992
993
994

	endTimer = time(NULL);
	float min = (endTimer - startTimer) / 60;
	float sec = (endTimer - startTimer) % 60;

995
996
	string result("Processing elapsed time: " + to_string((int)min) + ":" + to_string((int)sec));
	cout << endl << result << endl;
997

Matteo's avatar
update    
Matteo committed
998
	saveFile("log.txt", result + '\n', true);
999

1000
	/*********************************************************************************************/
For faster browsing, not all history is shown. View entire blame