main.cpp 31.1 KB
Newer Older
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
1
/**
Matteo's avatar
Matteo committed
2
 * @mainpage MPAI CAE-ARP Video Analyser
Matteo's avatar
Matteo committed
3
 * @file main.cpp
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
4
5
6
 *  MPAI CAE-ARP Video Analyser.
 *
 *	Implements MPAI CAE-ARP Video Analyser Technical Specification.
Matteo's avatar
Matteo committed
7
8
 *	It identifies Irregularities on the Preservation Audio-Visual File,
 *providing:
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
9
10
11
 *	- Irregularity Files;
 *	- Irregularity Images.
 *
Matteo's avatar
Matteo committed
12
13
 * @warning Currently, this program is only compatible with the Studer A810
 *and videos recorded in PAL standard.
Matteo's avatar
update    
Matteo committed
14
 *
Matteo's avatar
Matteo committed
15
16
17
18
19
 * @todo
 *  - A resize function of the entire video should be implemented if it does not
 *conform to the PAL standard (currently taken for granted).
 *  - Progressive videos, which do not require deinterlacing, should be managed
 *(in the code there are several steps that operate considering this property).
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
20
 *
Matteo's avatar
Matteo committed
21
22
 *  @author Nadir Dalla Pozza <nadir.dallapozza@unipd.it>
 *  @author Matteo Spanio <dev2@audioinnova.com>
Matteo's avatar
Matteo committed
23
 *	@copyright 2023, Audio Innova S.r.l.
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
24
25
 *	@credits Niccolò Pretto, Nadir Dalla Pozza, Sergio Canazza
 *	@license GPL v3.0
Matteo's avatar
Matteo committed
26
 *	@version 1.1.3
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
27
28
 *	@status Production
 */
29
#include <stdlib.h>
30
31
32
#include <sys/timeb.h>

#include <boost/lexical_cast.hpp>
Matteo's avatar
Matteo committed
33
34
35
36
37
38
39
40
#include <boost/program_options.hpp>
#include <boost/uuid/uuid.hpp>             // uuid class
#include <boost/uuid/uuid_generators.hpp>  // generators
#include <boost/uuid/uuid_io.hpp>          // streaming operators etc.
#include <filesystem>
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
Matteo's avatar
Matteo committed
41
#include <opencv2/calib3d.hpp>
42
#include <opencv2/core/core.hpp>
Matteo's avatar
Matteo committed
43
44
#include <opencv2/features2d.hpp>
#include <opencv2/highgui.hpp>
45
46
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
Matteo's avatar
Matteo committed
47
#include <opencv2/xfeatures2d.hpp>
48

Matteo's avatar
Matteo committed
49
50
51
52
53
54
#include "forAudioAnalyser.h"
#include "lib/Irregularity.h"
#include "lib/IrregularityFile.h"
#include "lib/colors.h"
#include "lib/files.h"
#include "lib/time.h"
55
#include "utility.h"
Matteo's avatar
update    
Matteo committed
56

Matteo's avatar
Matteo committed
57
58
59
60
#define A_IRREG_FILE_1 "AudioAnalyser_IrregularityFileOutput1.json"
#define V_IRREG_FILE_1 "VideoAnalyser_IrregularityFileOutput1.json"
#define V_IRREG_FILE_2 "VideoAnalyser_IrregularityFileOutput2.json"

61
62
using namespace cv;
using namespace std;
Matteo's avatar
update    
Matteo committed
63
using utility::Frame;
64
using json = nlohmann::json;
65
66
namespace fs = std::filesystem;
namespace po = boost::program_options;
67

Matteo's avatar
Matteo committed
68
/**
Matteo's avatar
Matteo committed
69
 * @const bool g_use_surf
Matteo's avatar
Matteo committed
70
71
72
73
74
75
 * @brief If true, SURF is used for capstan detection, otherwise GHT is used.
 *
 * For capstan detection, there are two alternative approaches:
 * 1. Generalized Hough Transform
 * 2. SURF.
 */
Matteo's avatar
Matteo committed
76
77
78
79
80
81
82
83
84
85
bool g_use_surf = true;
bool g_end_tape_saved = false;
bool g_first_brand = true;          // The first frame containing brands on tape must be saved
float g_first_instant = 0;
float g_mean_prev_frame_color = 0;  // Average frame color

static fs::path g_output_path{};
static fs::path g_irregularity_images_path{};
static json g_irregularity_file_1{};
static json g_irregularity_file_2{};
86
// RotatedRect identifying the processing area
Matteo's avatar
Matteo committed
87
RotatedRect rect, g_rect_tape, g_rect_capstan;
88

Matteo's avatar
Matteo committed
89
/**
Matteo's avatar
Matteo committed
90
 * @fn void pprint(string text, string color)
Matteo's avatar
Matteo committed
91
 * @brief Prints a text in a given color.
Matteo's avatar
Matteo committed
92
 *
Matteo's avatar
Matteo committed
93
 * @param text
Matteo's avatar
Matteo committed
94
 * @param color
Matteo's avatar
Matteo committed
95
 */
Matteo's avatar
Matteo committed
96
97
98
99
100
101
void pprint(string text, string color) { std::cout << color << text << END << endl; }
void print_error_and_exit(string title, string message) {
    std::cerr << RED << BOLD << title << END << endl;
    std::cerr << RED << message << END << endl;
    exit(EXIT_FAILURE);
}
Matteo's avatar
Matteo committed
102

103
struct Args {
Matteo's avatar
Matteo committed
104
105
    fs::path
        workingPath;  /**< The working path where all input files are stored and where all output files will be saved */
Matteo's avatar
Matteo committed
106
    string filesName; /**< The name of the preservation files to be considered */
Matteo's avatar
Matteo committed
107
108
    bool brands;      /**< True if tape presents brands on its surface */
    float speed;      /**< The speed at which the tape was read */
Matteo's avatar
Matteo committed
109
110

    Args(fs::path workingPath, string filesName, bool brands, float speed) {
Matteo's avatar
Matteo committed
111
        if (speed != 7.5 && speed != 15) throw invalid_argument("Speed must be 7.5 or 15");
Matteo's avatar
Matteo committed
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        this->workingPath = workingPath;
        this->filesName = filesName;
        this->brands = brands;
        this->speed = speed;
    }
    ~Args() {}

    static Args from_file(fs::path path) {
        ifstream iConfig(path);
        json j;
        iConfig >> j;
        return Args(fs::path(string(j["WorkingPath"])), j["FilesName"], j["Brands"], j["Speed"]);
    }

    static Args from_cli(int argc, char** argv) {
        po::variables_map vm;
        try {
            po::options_description desc(
                "A tool that implements MPAI CAE-ARP Video Analyser Technical "
                "Specification.\n"
                "By default, the configuartion parameters are loaded from "
                "config/config.json file,\n"
                "but, alternately, you can pass command line arguments to "
                "replace them");
            desc.add_options()("help,h", "Display this help message")(
                "working-path,w", po::value<string>()->required(),
                "Specify the Working Path, where all input files are stored")(
                "files-name,f", po::value<string>()->required(),
                "Specify the name of the Preservation files (without "
                "extension)")("brands,b", po::value<bool>()->required(),
                              "Specify if the tape presents brands on its surface")(
                "speed,s", po::value<float>()->required(), "Specify the speed at which the tape was read");
            po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
            if (vm.count("help")) {
Matteo's avatar
Matteo committed
146
                std::cout << desc << "\n";
Matteo's avatar
Matteo committed
147
148
149
150
                std::exit(EXIT_SUCCESS);
            }
            po::notify(vm);
        } catch (po::invalid_command_line_syntax& e) {
Matteo's avatar
Matteo committed
151
            print_error_and_exit("Invalid command line syntax!", string(e.what()));
Matteo's avatar
Matteo committed
152
        } catch (po::required_option& e) {
Matteo's avatar
Matteo committed
153
            print_error_and_exit("Missing required option!", string(e.what()));
Matteo's avatar
Matteo committed
154
        } catch (nlohmann::detail::type_error e) {
Matteo's avatar
Matteo committed
155
            print_error_and_exit("config.json error!", string(e.what()));
Matteo's avatar
Matteo committed
156
157
158
159
160
        }

        return Args(fs::path(vm["working-path"].as<string>()), vm["files-name"].as<string>(), vm["brands"].as<bool>(),
                    vm["speed"].as<float>());
    }
161
};
162

163
164
165
166
// Constants Paths
static const string READING_HEAD_IMG = "input/readingHead.png";
static const string CAPSTAN_TEMPLATE_IMG = "input/capstanBERIO058prova.png";
static const string CONFIG_FILE = "config/config.json";
167

Matteo's avatar
Matteo committed
168
/**
Matteo's avatar
update    
Matteo committed
169
170
 * @brief Get the next frame object.
 *
Matteo's avatar
Matteo committed
171
172
 * Whenever we find an Irregularity, we want to skip a lenght equal to the
 * Studer reading head (3 cm = 1.18 inches).
Matteo's avatar
update    
Matteo committed
173
 *
Matteo's avatar
Matteo committed
174
175
176
177
 * Note the following considerations:
 * - since we are analysing video at 25 fps a frame occurs every 40 ms
 * - at 15 ips we cross 3 cm of tape in 79 ms (2 frames)
 * - at 7.5 ips we cross 3 cm of tape in 157 ms (4 frames)
Matteo's avatar
update    
Matteo committed
178
 *
Matteo's avatar
Matteo committed
179
180
181
 * The considered lengths are the widths of the tape areas.
 * The following condition constitutes a valid approach if the tape areas
 * have widths always equal to the reading head
Matteo's avatar
update    
Matteo committed
182
 *
Matteo's avatar
Matteo committed
183
184
185
186
187
188
189
190
191
192
193
194
195
196
 * @param cap VideoCapture object
 * @param speed tape reading speed
 * @return Frame
 */
Frame get_next_frame(VideoCapture& cap, float speed, bool skip = false) {
    if (skip) {
        int ms_to_skip = speed == 15 ? 79 : 157;
        cap.set(CAP_PROP_POS_MSEC, cap.get(CAP_PROP_POS_MSEC) + ms_to_skip);
    }
    Frame frame;
    cap >> frame;
    return frame;
}

Matteo's avatar
Matteo committed
197
float rotated_rect_area(RotatedRect rect) { return rect.size.width * rect.size.height; }
198

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
199
/**
Matteo's avatar
Matteo committed
200
 * @fn std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>>
Matteo's avatar
Matteo committed
201
 * find_object(Mat model, SceneObject object)
Matteo's avatar
Matteo committed
202
203
204
 * @brief Find the model in the scene using the Generalized Hough Transform.
 * It returns the best matches. Find the best matches for positive and negative
 * angles. If there are more than one shape, then choose the one with the
Matteo's avatar
Matteo committed
205
 * highest score. If there are more than one with the same highest score, then
Matteo's avatar
Matteo committed
206
207
208
209
 * arbitrarily choose the latest.
 *
 * For informations about the Generalized Hough Guild usage see the tutorial
 * at https://docs.opencv.org/4.7.0/da/ddc/tutorial_generalized_hough_ballard_guil.html
Matteo's avatar
Matteo committed
210
211
212
213
214
215
216
 *
 * @param model the template image to be searched with the Generalized Hough
 * Transform
 * @param object the sceneObject struct containing the parameters for the
 * Generalized Hough Transform
 * @return std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>> a
 * tuple containing the best matches for positive and negative angles
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
217
 */
Matteo's avatar
Matteo committed
218
219
std::tuple<int, int, double, double, vector<Vec4f>, vector<Vec4f>> find_object(Mat model, SceneObject object,
                                                                               Mat processing_area) {
Matteo's avatar
Matteo committed
220
    Ptr<GeneralizedHoughGuil> alg = createGeneralizedHoughGuil();
221

Matteo's avatar
Matteo committed
222
    vector<Vec4f> positive_positions, negative_positions;
Matteo's avatar
Matteo committed
223
    Mat votesPos, votesNeg;
224

Matteo's avatar
Matteo committed
225
226
    double maxValPos = 0, maxValNeg = 0;
    int indexPos = 0, indexNeg = 0;
227

Matteo's avatar
Matteo committed
228
229
230
231
    alg->setMinDist(object.minDist);
    alg->setLevels(360);
    alg->setDp(2);
    alg->setMaxBufferSize(1000);
232

Matteo's avatar
Matteo committed
233
234
    alg->setAngleStep(1);
    alg->setAngleThresh(object.threshold.angle);
235

Matteo's avatar
Matteo committed
236
237
238
239
    alg->setMinScale(0.9);
    alg->setMaxScale(1.1);
    alg->setScaleStep(0.01);
    alg->setScaleThresh(object.threshold.scale);
240

Matteo's avatar
Matteo committed
241
    alg->setPosThresh(object.threshold.pos);
242

Matteo's avatar
Matteo committed
243
244
    alg->setCannyLowThresh(150);   // Old: 100
    alg->setCannyHighThresh(240);  // Old: 300
245

Matteo's avatar
Matteo committed
246
    alg->setTemplate(model);
247

Matteo's avatar
Matteo committed
248
    utility::detectShape(alg, model, object.threshold.pos, positive_positions, votesPos, negative_positions, votesNeg,
Matteo's avatar
Matteo committed
249
                         processing_area);
250

Matteo's avatar
Matteo committed
251
252
253
254
255
256
    for (int i = 0; i < votesPos.size().width; i++) {
        if (votesPos.at<int>(i) >= maxValPos) {
            maxValPos = votesPos.at<int>(i);
            indexPos = i;
        }
    }
257

Matteo's avatar
Matteo committed
258
259
260
261
262
263
    for (int i = 0; i < votesNeg.size().width; i++) {
        if (votesNeg.at<int>(i) >= maxValNeg) {
            maxValNeg = votesNeg.at<int>(i);
            indexNeg = i;
        }
    }
264

Matteo's avatar
Matteo committed
265
    return {indexPos, indexNeg, maxValPos, maxValNeg, positive_positions, negative_positions};
Matteo's avatar
Matteo committed
266
267
268
}

/**
Matteo's avatar
Matteo committed
269
 * @fn bool find_processing_areas(Mat my_frame)
Matteo's avatar
Matteo committed
270
271
272
 * @brief Identifies the Regions Of Interest (ROIs) on the video,
 * which are:
 * - The reading head;
Matteo's avatar
Matteo committed
273
274
 * - The tape area under the tape head (computed on the basis of the detected
 * reading head);
Matteo's avatar
Matteo committed
275
 * - The capstan.
Matteo's avatar
Matteo committed
276
 * @param my_frame The current frame of the video.
Matteo's avatar
Matteo committed
277
278
279
 * @return true if some areas have been detected;
 * @return false otherwise.
 */
Matteo's avatar
Matteo committed
280
bool find_processing_areas(Mat my_frame, SceneObject tape, SceneObject capstan) {
Matteo's avatar
Matteo committed
281
    /************************** READING HEAD DETECTION ***********************/
Matteo's avatar
Matteo committed
282

Matteo's avatar
Matteo committed
283
    // Save a grayscale version of my_frame in myFrameGrayscale and downsample it
Matteo's avatar
Matteo committed
284
    // in half pixels for performance reasons
Matteo's avatar
Matteo committed
285
    Frame gray_current_frame = Frame(my_frame).convertColor(COLOR_BGR2GRAY);
Matteo's avatar
Matteo committed
286
287
288
289
290
291
292
293
294
295
296
297
298
299

    Frame halved_gray_current_frame = gray_current_frame.clone().downsample(2);

    // Get input shape in grayscale and downsample it in half pixels
    Frame reading_head_template = Frame(cv::imread(READING_HEAD_IMG, IMREAD_GRAYSCALE)).downsample(2);

    // Process only the bottom-central portion of the input video -> best
    // results with our videos
    Rect readingHeadProcessingAreaRect(halved_gray_current_frame.cols / 4, halved_gray_current_frame.rows / 2,
                                       halved_gray_current_frame.cols / 2, halved_gray_current_frame.rows / 2);
    Mat processingImage = halved_gray_current_frame(readingHeadProcessingAreaRect);

    RotatedRect rectPos, rectNeg;
    auto [indexPos, indexNeg, maxValPos, maxValNeg, positionsPos, positionsNeg] =
Matteo's avatar
Matteo committed
300
        find_object(reading_head_template, tape, processingImage);
Matteo's avatar
Matteo committed
301
302
303
304

    // The color is progressively darkened to emphasize that the algorithm found
    // more than one shape
    if (positionsPos.size() > 0)
Matteo's avatar
Matteo committed
305
        rectPos = utility::drawShapes(my_frame, positionsPos[indexPos], Scalar(0, 0, 255 - indexPos * 64),
Matteo's avatar
Matteo committed
306
307
308
                                      reading_head_template.cols, reading_head_template.rows,
                                      halved_gray_current_frame.cols / 4, halved_gray_current_frame.rows / 2, 2);
    if (positionsNeg.size() > 0)
Matteo's avatar
Matteo committed
309
        rectNeg = utility::drawShapes(my_frame, positionsNeg[indexNeg], Scalar(128, 128, 255 - indexNeg * 64),
Matteo's avatar
Matteo committed
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
                                      reading_head_template.cols, reading_head_template.rows,
                                      halved_gray_current_frame.cols / 4, halved_gray_current_frame.rows / 2, 2);

    if (maxValPos > 0)
        if (maxValNeg > 0)
            if (maxValPos > maxValNeg) {
                rect = rectPos;
            } else {
                rect = rectNeg;
            }
        else {
            rect = rectPos;
        }
    else if (maxValNeg > 0) {
        rect = rectNeg;
    } else {
        return false;
    }
Matteo's avatar
Matteo committed
328

Matteo's avatar
Matteo committed
329
    /************************************ TAPE AREA DETECTION ****************/
Matteo's avatar
Matteo committed
330
331

    // Compute area basing on reading head detection
Matteo's avatar
Matteo committed
332
333
334
335
    Vec4f tape_position(rect.center.x, rect.center.y + rect.size.height / 2 + 20 * (rect.size.width / 200), 1,
                        rect.angle);
    g_rect_tape = utility::drawShapes(my_frame, tape_position, Scalar(0, 255 - indexPos * 64, 0), rect.size.width,
                                      50 * (rect.size.width / 200), 0, 0, 1);
Matteo's avatar
Matteo committed
336

Matteo's avatar
Matteo committed
337
    /************************************* CAPSTAN DETECTION ******************/
Matteo's avatar
Matteo committed
338
339
340

    // Read template image - it is smaller than before, therefore there is no
    // need to downsample
Matteo's avatar
Matteo committed
341
    Mat capstan_template = cv::imread(CAPSTAN_TEMPLATE_IMG, IMREAD_GRAYSCALE);
Matteo's avatar
Matteo committed
342

Matteo's avatar
Matteo committed
343
    if (g_use_surf) {
Matteo's avatar
Matteo committed
344
345
        // Step 1: Detect the keypoints using SURF Detector, compute the
        // descriptors
Matteo's avatar
Matteo committed
346
347
        int min_hessian = 100;
        Ptr<xfeatures2d::SURF> detector = xfeatures2d::SURF::create(min_hessian);
Matteo's avatar
Matteo committed
348
349
350
        vector<KeyPoint> keypoints_object, keypoints_scene;
        Mat descriptors_object, descriptors_scene;

Matteo's avatar
Matteo committed
351
        detector->detectAndCompute(capstan_template, noArray(), keypoints_object, descriptors_object);
Matteo's avatar
Matteo committed
352
353
354
355
356
357
358
359
        detector->detectAndCompute(gray_current_frame, noArray(), keypoints_scene, descriptors_scene);

        // Step 2: Matching descriptor vectors with a FLANN based matcher
        // Since SURF is a floating-point descriptor NORM_L2 is used
        Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create(DescriptorMatcher::FLANNBASED);
        vector<vector<DMatch>> knn_matches;
        matcher->knnMatch(descriptors_object, descriptors_scene, knn_matches, 2);
        //-- Filter matches using the Lowe's ratio test
Matteo's avatar
Matteo committed
360
        const float RATIO_THRESH = 0.75f;
Matteo's avatar
Matteo committed
361
362
        vector<DMatch> good_matches;
        for (size_t i = 0; i < knn_matches.size(); i++) {
Matteo's avatar
Matteo committed
363
            if (knn_matches[i][0].distance < RATIO_THRESH * knn_matches[i][1].distance) {
Matteo's avatar
Matteo committed
364
365
366
367
368
                good_matches.push_back(knn_matches[i][0]);
            }
        }
        // Draw matches
        Mat img_matches;
Matteo's avatar
Matteo committed
369
        cv::drawMatches(capstan_template, keypoints_object, halved_gray_current_frame, keypoints_scene, good_matches,
Matteo's avatar
update    
Matteo committed
370
371
                        img_matches, Scalar::all(-1), Scalar::all(-1), vector<char>(),
                        DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
Matteo's avatar
Matteo committed
372
373
374
375
376
377
378
379
        // Localize the object
        vector<Point2f> obj;
        vector<Point2f> scene;
        for (size_t i = 0; i < good_matches.size(); i++) {
            // Get the keypoints from the good matches
            obj.push_back(keypoints_object[good_matches[i].queryIdx].pt);
            scene.push_back(keypoints_scene[good_matches[i].trainIdx].pt);
        }
Matteo's avatar
Matteo committed
380
        Mat H = cv::findHomography(obj, scene, RANSAC);
Matteo's avatar
Matteo committed
381
382
383
        // Get the corners from the image_1 ( the object to be "detected" )
        vector<Point2f> obj_corners(4);
        obj_corners[0] = Point2f(0, 0);
Matteo's avatar
Matteo committed
384
385
386
        obj_corners[1] = Point2f((float)capstan_template.cols, 0);
        obj_corners[2] = Point2f((float)capstan_template.cols, (float)capstan_template.rows);
        obj_corners[3] = Point2f(0, (float)capstan_template.rows);
Matteo's avatar
Matteo committed
387
        vector<Point2f> scene_corners(4);
Matteo's avatar
Matteo committed
388
        cv::perspectiveTransform(obj_corners, scene_corners, H);
Matteo's avatar
Matteo committed
389
390
391
392
393
394
395
396
397
398

        // Find average
        float capstanX = (scene_corners[0].x + scene_corners[1].x + scene_corners[2].x + scene_corners[3].x) / 4;
        float capstanY = (scene_corners[0].y + scene_corners[1].y + scene_corners[2].y + scene_corners[3].y) / 4;

        // In the following there are two alterations to cut the first 20
        // horizontal pixels and the first 90 vertical pixels from the found
        // rectangle: +10 in X for centering and -20 in width +45 in Y for
        // centering and -90 in height
        Vec4f positionCapstan(capstanX + 10, capstanY + 45, 1, 0);
Matteo's avatar
Matteo committed
399
400
        g_rect_capstan = utility::drawShapes(my_frame, positionCapstan, Scalar(255 - indexPos * 64, 0, 0),
                                             capstan_template.cols - 20, capstan_template.rows - 90, 0, 0, 1);
Matteo's avatar
Matteo committed
401
402
403
404

    } else {
        // Process only right portion of the image, where the capstain always
        // appears
Matteo's avatar
Matteo committed
405
406
407
408
        int capstanProcessingAreaRectX = my_frame.cols * 3 / 4;
        int capstanProcessingAreaRectY = my_frame.rows / 2;
        int capstanProcessingAreaRectWidth = my_frame.cols / 4;
        int capstanProcessingAreaRectHeight = my_frame.rows / 2;
Matteo's avatar
Matteo committed
409
410
411
412
413
414
        Rect capstanProcessingAreaRect(capstanProcessingAreaRectX, capstanProcessingAreaRectY,
                                       capstanProcessingAreaRectWidth, capstanProcessingAreaRectHeight);
        Mat capstanProcessingAreaGrayscale = gray_current_frame(capstanProcessingAreaRect);
        // Reset algorithm and set parameters

        auto [indexPos, indexNeg, maxValPos, maxValNeg, positionsC1Pos, positionsC1Neg] =
Matteo's avatar
Matteo committed
415
            find_object(capstan_template, capstan, capstanProcessingAreaGrayscale);
Matteo's avatar
Matteo committed
416
417
418

        RotatedRect rectCapstanPos, rectCapstanNeg;
        if (positionsC1Pos.size() > 0)
Matteo's avatar
Matteo committed
419
            rectCapstanPos = utility::drawShapes(my_frame, positionsC1Pos[indexPos], Scalar(255 - indexPos * 64, 0, 0),
Matteo's avatar
Matteo committed
420
                                                 capstan_template.cols - 22, capstan_template.rows - 92,
Matteo's avatar
Matteo committed
421
422
                                                 capstanProcessingAreaRectX + 11, capstanProcessingAreaRectY + 46, 1);
        if (positionsC1Neg.size() > 0)
Matteo's avatar
Matteo committed
423
424
425
            rectCapstanNeg = utility::drawShapes(
                my_frame, positionsC1Neg[indexNeg], Scalar(255 - indexNeg * 64, 128, 0), capstan_template.cols - 22,
                capstan_template.rows - 92, capstanProcessingAreaRectX + 11, capstanProcessingAreaRectY + 46, 1);
Matteo's avatar
Matteo committed
426
427
428
429

        if (maxValPos > 0)
            if (maxValNeg > 0)
                if (maxValPos > maxValNeg) {
Matteo's avatar
Matteo committed
430
                    g_rect_capstan = rectCapstanPos;
Matteo's avatar
Matteo committed
431
                } else {
Matteo's avatar
Matteo committed
432
                    g_rect_capstan = rectCapstanNeg;
Matteo's avatar
Matteo committed
433
434
                }
            else {
Matteo's avatar
Matteo committed
435
                g_rect_capstan = rectCapstanPos;
Matteo's avatar
Matteo committed
436
437
            }
        else if (maxValNeg > 0) {
Matteo's avatar
Matteo committed
438
            g_rect_capstan = rectCapstanNeg;
Matteo's avatar
Matteo committed
439
440
441
442
443
        } else {
            return false;
        }
    }

Matteo's avatar
Matteo committed
444
    // Save the image containing the ROIs
Matteo's avatar
Matteo committed
445
    cv::imwrite(g_output_path.string() + "/tape_areas.jpg", my_frame);
Matteo's avatar
Matteo committed
446
447

    return true;
448
449
}

Matteo's avatar
Matteo committed
450
451
452
/**
 * @fn RotatedRect check_skew(RotatedRect roi)
 * @brief Check if the region of interest is skewed and correct it
Matteo's avatar
Matteo committed
453
 *
Matteo's avatar
Matteo committed
454
455
456
 * @param roi the region of interest
 * @return RotatedRect the corrected region of interest
 */
Matteo's avatar
Matteo committed
457
RotatedRect check_skew(RotatedRect roi) {
Matteo's avatar
Matteo committed
458
459
460
461
462
463
464
465
466
    // get angle and size from the bounding box
    // thanks to http://felix.abecassis.me/2011/10/opencv-rotation-deskewing/
    cv::Size rect_size = roi.size;
    float angle = roi.angle;
    if (roi.angle < -45.) {
        angle += 90.0;
        swap(rect_size.width, rect_size.height);
    }
    return RotatedRect(roi.center, rect_size, angle);
Matteo's avatar
Matteo committed
467
468
469
}

/**
Matteo's avatar
Matteo committed
470
471
472
473
474
 * @fn Frame get_difference_for_roi(Frame previous, Frame current, RotatedRect
 * roi)
 * @brief Look for differences in two consecutive frames in a specific region of
 * interest
 *
Matteo's avatar
Matteo committed
475
476
477
478
479
480
 * @param previous the reference frame
 * @param current the frame to compare with the reference
 * @param roi the region of interest
 * @return Frame the difference matrix between the two frames
 */
Frame get_difference_for_roi(Frame previous, Frame current, RotatedRect roi) {
Matteo's avatar
Matteo committed
481
482
483
484
485
    cv::Mat rotation_matrix = getRotationMatrix2D(roi.center, roi.angle, 1.0);

    return previous.warp(rotation_matrix)
        .crop(roi.size, roi.center)
        .difference(current.warp(rotation_matrix).crop(roi.size, roi.center));
Matteo's avatar
Matteo committed
486
}
487

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
488
/**
Matteo's avatar
Matteo committed
489
490
 * @fn bool is_frame_different(cv::Mat prev_frame, cv::Mat current_frame, int
 * ms_to_end)
Matteo's avatar
Matteo committed
491
492
493
 * @brief Compares two consecutive video frames and establish if there
 * potentially is an Irregularity. The comparison is pixel-wise and based on
 * threshold values set on config.json file.
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
494
 *
Matteo's avatar
Matteo committed
495
496
497
 * @param prev_frame the frame before the current one;
 * @param current_frame the current frame;
 * @param ms_to_end the number of milliseconds left before the end of the video.
Matteo's avatar
Matteo committed
498
 * Useful for capstan analysis.
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
499
500
501
 * @return true if a potential Irregularity has been found;
 * @return false otherwise.
 */
Matteo's avatar
Matteo committed
502
bool is_frame_different(cv::Mat prev_frame, cv::Mat current_frame, int ms_to_end, SceneObject capstan, SceneObject tape,
Matteo's avatar
Matteo committed
503
                        Args args) {
Matteo's avatar
Matteo committed
504
    bool result = false;
Matteo's avatar
Matteo committed
505
    int num_different_pixels = 0;
Matteo's avatar
Matteo committed
506
507
508
509
510

    /*********************** Capstan analysis ************************/

    // In the last minute of the video, check for pinchRoller position for
    // endTape event
Matteo's avatar
Matteo committed
511
512
513
    if (!g_end_tape_saved && ms_to_end < 60000) {
        RotatedRect corrected_capstan_roi = check_skew(g_rect_capstan);
        Frame difference_frame = get_difference_for_roi(Frame(prev_frame), Frame(current_frame), corrected_capstan_roi);
Matteo's avatar
Matteo committed
514
515
516
517
518
519

        for (int i = 0; i < difference_frame.rows; i++) {
            for (int j = 0; j < difference_frame.cols; j++) {
                if (difference_frame.at<cv::Vec3b>(i, j)[0] == 0) {
                    // There is a black pixel, then there is a difference
                    // between previous and current frames
Matteo's avatar
Matteo committed
520
                    num_different_pixels++;
Matteo's avatar
Matteo committed
521
522
523
524
                }
            }
        }

Matteo's avatar
Matteo committed
525
526
527
        float capstan_pixel_threshold = rotated_rect_area(g_rect_capstan) * capstan.threshold.percentual / 100;
        if (num_different_pixels > capstan_pixel_threshold) {
            g_end_tape_saved = true;  // Never check again for end tape instant
Matteo's avatar
Matteo committed
528
529
530
            return true;
        }
    }
531

Matteo's avatar
Matteo committed
532
    /********************* Tape analysis *********************/
533

Matteo's avatar
Matteo committed
534
535
536
537
538
539
    RotatedRect corrected_tape_roi = check_skew(g_rect_tape);
    Frame cropped_current_frame =
        Frame(current_frame)
            .warp(getRotationMatrix2D(corrected_tape_roi.center, corrected_tape_roi.angle, 1.0))
            .crop(corrected_tape_roi.size, corrected_tape_roi.center);
    Frame difference_frame = get_difference_for_roi(Frame(prev_frame), Frame(current_frame), corrected_tape_roi);
540

Matteo's avatar
Matteo committed
541
    /********************** Segment analysis ************************/
542

Matteo's avatar
Matteo committed
543
544
545
    num_different_pixels = 0;
    float mean_current_frame_color;
    int current_frame_color_sum = 0;
Matteo's avatar
Matteo committed
546

Matteo's avatar
Matteo committed
547
548
549
550
551
    for (int i = 0; i < cropped_current_frame.rows; i++) {
        for (int j = 0; j < cropped_current_frame.cols; j++) {
            current_frame_color_sum += cropped_current_frame.at<cv::Vec3b>(i, j)[0] +
                                       cropped_current_frame.at<cv::Vec3b>(i, j)[1] +
                                       cropped_current_frame.at<cv::Vec3b>(i, j)[2];
Matteo's avatar
Matteo committed
552
            if (difference_frame.at<cv::Vec3b>(i, j)[0] == 0) {
Matteo's avatar
Matteo committed
553
                num_different_pixels++;
Matteo's avatar
Matteo committed
554
555
556
            }
        }
    }
Matteo's avatar
Matteo committed
557
558
    float tape_area_pixels_sq = rotated_rect_area(g_rect_tape);
    mean_current_frame_color = current_frame_color_sum / tape_area_pixels_sq;
Matteo's avatar
Matteo committed
559
560
561

    /*********************** Decision stage ************************/

Matteo's avatar
Matteo committed
562
563
    float tape_pixel_threshold = tape_area_pixels_sq * tape.threshold.percentual / 100;
    if (num_different_pixels > tape_pixel_threshold) {  // The threshold must be passed
Matteo's avatar
Matteo committed
564
565

        /***** AVERAGE_COLOR-BASED DECISION *****/
Matteo's avatar
Matteo committed
566
567
        if (g_mean_prev_frame_color > (mean_current_frame_color + 7) ||
            g_mean_prev_frame_color < (mean_current_frame_color - 7)) {  // They are not similar for color average
Matteo's avatar
Matteo committed
568
569
570
571
            result = true;
        }

        /***** BRANDS MANAGEMENT *****/
Matteo's avatar
Matteo committed
572
573
574
575
        // At the beginning of the video, wait at least 5 seconds before the
        // next Irregularity to consider it as a brand. It is not guaranteed
        // that it will be the first brand, but it is generally a safe
        // approach to have a correct image
Matteo's avatar
Matteo committed
576
577
        if (args.brands && g_first_brand && g_first_instant - ms_to_end > 5000) {
            g_first_brand = false;
Matteo's avatar
Matteo committed
578
            result = true;
Matteo's avatar
Matteo committed
579
580
        }
    }
581

Matteo's avatar
Matteo committed
582
    g_mean_prev_frame_color = mean_current_frame_color;
583

Matteo's avatar
Matteo committed
584
585
    return result;
}
586

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
587
/**
Matteo's avatar
Matteo committed
588
 * @fn void processing(cv::VideoCapture video_capture, SceneObject capstan,
Matteo's avatar
Matteo committed
589
 * SceneObject tape, Args args)
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
590
 * @brief video processing phase, where each frame is analysed.
Matteo's avatar
Matteo committed
591
592
593
 * It saves the IrregularityImages and updates the IrregularityFiles if an
 * Irregularity is found
 *
Matteo's avatar
update    
Matteo committed
594
 * @note To be able to work with the "old" neural network (by Ilenya),
Matteo's avatar
Matteo committed
595
596
597
598
599
600
 * the output images should correspond to the old "whole tape" where, from the
 * frame judged as interesting, an area corresponding to the height of the tape
 * was extracted (so about the height of the current rectangle) and as wide as
 * the original frame (so 720px). This area will then have to be resized to
 * 224x224 as in the past. If instead you decide to use the new neural network,
 * no changes are needed.
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
601
 *
Matteo's avatar
Matteo committed
602
 * @param video_capture the input Preservation Audio-Visual File;
Matteo's avatar
update    
Matteo committed
603
604
605
 * @param capstan the capstan SceneObject;
 * @param tape the tape SceneObject;
 * @param args the command line arguments.
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
606
 */
Matteo's avatar
Matteo committed
607
608
609
610
611
void processing(cv::VideoCapture video_capture, SceneObject capstan, SceneObject tape, Args args) {
    const int video_length_ms =
        ((float)video_capture.get(CAP_PROP_FRAME_COUNT) / video_capture.get(CAP_PROP_FPS)) * 1000;
    int video_current_ms = video_capture.get(CAP_PROP_POS_MSEC);
    int num_saved_frames = 0;
Matteo's avatar
update    
Matteo committed
612
    bool irregularity_found = false;
Matteo's avatar
Matteo committed
613
614

    // The first frame of the video won't be processed
Matteo's avatar
Matteo committed
615
616
    cv::Mat prev_frame = get_next_frame(video_capture, args.speed, irregularity_found);
    g_first_instant = video_length_ms - video_current_ms;
617

Matteo's avatar
Matteo committed
618
619
620
    while (video_capture.isOpened()) {
        Frame frame = get_next_frame(video_capture, args.speed, irregularity_found);
        video_current_ms = video_capture.get(CAP_PROP_POS_MSEC);
Matteo's avatar
Matteo committed
621
622

        if (frame.empty()) {
Matteo's avatar
Matteo committed
623
            std::cout << endl << "Empty frame!" << endl;
Matteo's avatar
Matteo committed
624
            video_capture.release();
Matteo's avatar
Matteo committed
625
626
627
            return;
        }

Matteo's avatar
Matteo committed
628
        int ms_to_end = video_length_ms - video_current_ms;
Matteo's avatar
Matteo committed
629
630
631
632
633
        if (video_current_ms == 0)  // With OpenCV library, this happens at the last few frames of
                                    // the video before realising that "frame" is empty.
            return;

        // Display program status
Matteo's avatar
Matteo committed
634
635
636
637
638
        int sec_to_end = ms_to_end / 1000;
        int min_to_end = (sec_to_end / 60) % 60;
        sec_to_end = sec_to_end % 60;
        string secStrToEnd = (sec_to_end < 10 ? "0" : "") + to_string(sec_to_end);
        string minStrToEnd = (min_to_end < 10 ? "0" : "") + to_string(min_to_end);
Matteo's avatar
Matteo committed
639

Matteo's avatar
Matteo committed
640
        std::cout << "\rIrregularities: " << num_saved_frames << ".   ";
Matteo's avatar
Matteo committed
641
        std::cout << "Remaining video time [mm:ss]: " << minStrToEnd << ":" << secStrToEnd << flush;
Matteo's avatar
Matteo committed
642

Matteo's avatar
Matteo committed
643
        irregularity_found = is_frame_different(prev_frame, frame, ms_to_end, capstan, tape, args);
Matteo's avatar
update    
Matteo committed
644
        if (irregularity_found) {
Matteo's avatar
Matteo committed
645
            auto [odd_frame, _] = frame.deinterlace();
Matteo's avatar
Matteo committed
646

Matteo's avatar
update    
Matteo committed
647
            string irregularityImageFilename =
Matteo's avatar
Matteo committed
648
649
                to_string(num_saved_frames) + "_" + getTimeLabel(video_current_ms, "-") + ".jpg";
            cv::imwrite(g_irregularity_images_path / irregularityImageFilename, odd_frame);
Matteo's avatar
Matteo committed
650
651

            // Append Irregularity information to JSON
Matteo's avatar
update    
Matteo committed
652
            Irregularity irreg = Irregularity(Source::Video, getTimeLabel(video_current_ms, ":"));
Matteo's avatar
Matteo committed
653
654
655
            g_irregularity_file_1["Irregularities"] += irreg.to_JSON();
            g_irregularity_file_2["Irregularities"] +=
                irreg.set_image_URI(g_irregularity_images_path.string() + "/" + irregularityImageFilename).to_JSON();
Matteo's avatar
Matteo committed
656

Matteo's avatar
Matteo committed
657
            num_saved_frames++;
Matteo's avatar
Matteo committed
658
        }
Matteo's avatar
Matteo committed
659
        prev_frame = frame;
Matteo's avatar
Matteo committed
660
    }
661
662
}

Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
663
/**
Matteo's avatar
Matteo committed
664
 * @fn int main(int argc, char** argv)
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
665
666
667
668
669
670
671
672
 * @brief main program, organised as:
 * - Get input from command line or config.json file;
 * - Check input parameters;
 * - Creation of output directories;
 * - Regions Of Interest (ROIs) detection;
 * - Irregularities detection;
 * - Saving of output IrregularityFiles.
 *
Matteo's avatar
Matteo committed
673
674
675
676
677
 * @todo The main function should be splitted into 2 steps, and each step should be callable from the command line, so
 * that the user can choose to run only the first step, only the second step, or both:
 * - First step: generate irregularity file output 1;
 * - Second step: generate irregularity file output 2.
 *
Nadir Dalla Pozza's avatar
Nadir Dalla Pozza committed
678
679
680
681
 * @param argc Command line arguments count;
 * @param argv Command line arguments.
 * @return int program status.
 */
682
int main(int argc, char** argv) {
Matteo's avatar
Matteo committed
683
684
    SceneObject capstan = SceneObject::from_file(CONFIG_FILE, ROI::CAPSTAN);
    SceneObject tape = SceneObject::from_file(CONFIG_FILE, ROI::TAPE);
Matteo's avatar
Matteo committed
685
686
    Args args = argc > 1 ? Args::from_cli(argc, argv) : Args::from_file(CONFIG_FILE);

Matteo's avatar
Matteo committed
687
    const fs::path VIDEO_PATH = args.workingPath / "PreservationAudioVisualFile" / args.filesName;
Matteo's avatar
Matteo committed
688
689
    const auto [FILE_NAME, FILE_FORMAT] = files::get_filename_and_extension(VIDEO_PATH);
    const fs::path AUDIO_IRR_FILE_PATH = args.workingPath / "temp" / FILE_NAME / A_IRREG_FILE_1;
Matteo's avatar
Matteo committed
690

Matteo's avatar
Matteo committed
691
    std::cout << "Video to be analysed: " << endl;
Matteo's avatar
Matteo committed
692
693
    std::cout << "\tFile name:  " << FILE_NAME << endl;
    std::cout << "\tExtension:  " << FILE_FORMAT << endl;
Matteo's avatar
Matteo committed
694

Matteo's avatar
Matteo committed
695
    if (FILE_FORMAT.compare("avi") != 0 && FILE_FORMAT.compare("mp4") != 0 && FILE_FORMAT.compare("mov") != 0)
Matteo's avatar
Matteo committed
696
        print_error_and_exit("Input error", "The input file must be an AVI, MP4 or MOV file.");
Matteo's avatar
Matteo committed
697
    ifstream iJSON(AUDIO_IRR_FILE_PATH);
Matteo's avatar
Matteo committed
698
    if (iJSON.fail())
Matteo's avatar
Matteo committed
699
        print_error_and_exit("config.json error", AUDIO_IRR_FILE_PATH.string() + " cannot be found or opened.");
Matteo's avatar
Matteo committed
700

Matteo's avatar
Matteo committed
701
702
    json audio_irr_file;
    iJSON >> audio_irr_file;
703

Matteo's avatar
Matteo committed
704
    // Adjust input paramenters (considering given ones as pertinent to a speed reference = 7.5)
Matteo's avatar
Matteo committed
705
706
707
    if (args.speed == 15) {
        tape.threshold.percentual += args.brands ? 6 : 20;
    } else if (!args.brands)
Matteo's avatar
Matteo committed
708
        tape.threshold.percentual += 21;
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
709

Matteo's avatar
Matteo committed
710
711
    g_output_path = args.workingPath / "temp" / FILE_NAME;
    fs::create_directory(g_output_path);
Matteo's avatar
update    
Matteo committed
712

Matteo's avatar
Matteo committed
713
714
    g_irregularity_images_path = g_output_path / "IrregularityImages";
    fs::create_directory(g_irregularity_images_path);
Nadir Dalla Pozza's avatar
Update.    
Nadir Dalla Pozza committed
715

Matteo's avatar
Matteo committed
716
717
    cv::VideoCapture video_capture(VIDEO_PATH);  // Open video file
    if (!video_capture.isOpened()) print_error_and_exit("Video error", "Video file cannot be opened.");
718

Matteo's avatar
Matteo committed
719
720
721
722
    video_capture.set(CAP_PROP_POS_FRAMES,
                      video_capture.get(CAP_PROP_FRAME_COUNT) / 2);  // Set frame position to half video length
    cv::Mat middle_frame = get_next_frame(video_capture, args.speed);
    video_capture.set(CAP_PROP_POS_FRAMES, 0);  // Reset frame position
723

Matteo's avatar
Matteo committed
724
    std::cout << "\tResolution: " << middle_frame.cols << "x" << middle_frame.rows << "\n\n";
725

Matteo's avatar
Matteo committed
726
    bool found = find_processing_areas(middle_frame, tape, capstan);
Matteo's avatar
Matteo committed
727
    if (!found) print_error_and_exit("Processing area not found", "Try changing JSON parameters.");
728

Matteo's avatar
Matteo committed
729
    pprint("Processing...", CYAN);
Matteo's avatar
Matteo committed
730
    processing(video_capture, capstan, tape, args);
731

Matteo's avatar
Matteo committed
732
    files::save_file(g_output_path / V_IRREG_FILE_1, g_irregularity_file_1.dump(4));
733

Matteo's avatar
Matteo committed
734
    // Irregularities to extract for the AudioAnalyser and to the TapeIrregularityClassifier
Matteo's avatar
Matteo committed
735
736
    extractIrregularityImagesForAudio(g_output_path, VIDEO_PATH, audio_irr_file, g_irregularity_file_2);
    files::save_file(g_output_path / V_IRREG_FILE_2, g_irregularity_file_2.dump(4));
737

Matteo's avatar
Matteo committed
738
    return EXIT_SUCCESS;
739
}