Skip to content
GitLab
Commit eba1ee01 authored by Nadir Dalla Pozza's avatar Nadir Dalla Pozza
Browse files

Comments and beauty update

parent e6b74595
Hide whitespace changes
Inline Side-by-side
README.md
View file @ eba1ee01
......@@ -48,8 +48,8 @@ To execute the script without issues, the inner structure of the `WORKING_PATH`
│ ├── File2.wav
│ └── ...
├── PreservationAudioVisualFile
│ ├── File1.wav
│ ├── File2.wav
│ ├── File1.mp4
│ ├── File2.mp4
│ └── ...
├── PreservationMasterFiles
│ └── ...
......@@ -108,10 +108,20 @@ With this structure, `PRESERVATION_FILES_NAME` parameter could be equal to `File
You can now launch the *Packager* from the command line with:
```
python3 main.py
python3 packager.py
```
Useful log information will be displayed during execution, requiring occasional interaction.
To enable integration in more complex workflows, it is also possible to launch the *Packager* with command line arguments:
```
python3 packager.py [-h] -w WORKING_PATH -f FILES_NAME
```
If you use the `-h` flag:
```
python3 packager.py -h
```
all instructions will be displayed.
## Support
If you require additional information or have any problem, you can contact us at:
* Nadir Dalla Pozza (nadir.dallapozza@unipd.it);
......
config.yaml
View file @ eba1ee01
# Working path
WORKING_PATH: "/Users/nadir/Documents/MPAI-CAE/Workflow"
# Name of the Preservation files (without extension)
PRESERVATION_FILES_NAME: "test"
FILES_NAME: "test"
main.py deleted 100755 → 0
View file @ e6b74595
#!/usr/bin/env python
"""MPAI CAE-ARP Packager.
Implements MPAI CAE-ARP Packager Technical Specification, providing:
- Access Copy Files:
1. Restored Audio Files.
2. Editing List.
3. Set of Irregularity Images in a .zip file.
4. Irregularity File.
- Preservation Master Files:
1. Preservation Audio File.
2. Preservation Audio-Visual File where the audio has been replaced with the Audio of the Preservation Audio File
fully synchronised with the video.
3. Set of Irregularity Images in a .zip file.
4. Irregularity File.
"""
import json
import os
import shutil
import yaml
from moviepy.editor import VideoFileClip, AudioFileClip
__author__ = "Nadir Dalla Pozza"
__copyright__ = "Copyright 2022, Audio Innova S.r.l."
__credits__ = ["Niccolò Pretto", "Nadir Dalla Pozza", "Sergio Canazza"]
__license__ = "GPL v3.0"
__version__ = "1.0.1"
__maintainer__ = "Nadir Dalla Pozza"
__email__ = "nadir.dallapozza@unipd.it"
__status__ = "Production"
# Class for customizing console colors
class ConsoleColors:
PURPLE = '\033[95m'
CYAN = '\033[96m'
DARK_CYAN = '\033[36m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
if __name__ == '__main__':
# Read configuration file
config = object
try:
config = yaml.safe_load(open('./config.yaml', 'r'))
if 'WORKING_PATH' not in config:
print(ConsoleColors.RED + 'WORKING_PATH key not found in config.yaml!' + ConsoleColors.END)
quit(os.EX_CONFIG)
if 'PRESERVATION_FILES_NAME' not in config:
print(ConsoleColors.RED + 'PRESERVATION_FILES_NAME key not found in config.yaml!' + ConsoleColors.END)
quit(os.EX_CONFIG)
except FileNotFoundError:
print(ConsoleColors.RED + 'config.yaml file not found!' + ConsoleColors.END)
quit(os.EX_NOINPUT)
temp_path = os.path.join(config['WORKING_PATH'], 'temp/', config['PRESERVATION_FILES_NAME'])
print(temp_path)
# Access Copy Files
print('\n' + ConsoleColors.BOLD + 'Creation of Access Copy Files...' + ConsoleColors.END)
# By default, AccessCopyFiles directory is created in the output path...
acf_dir = 'AccessCopyFiles/' + config['PRESERVATION_FILES_NAME'] + '/'
acf_path = os.path.join(config['WORKING_PATH'], acf_dir)
# ...however, if it already exists, it is possible to skip its creation
make_acf = False
if not os.path.exists(acf_path):
# Create directory
os.mkdir(acf_path)
make_acf = True
print("Access Copy Files directory '% s' created" % acf_path)
else:
print((ConsoleColors.PURPLE + "Access Copy Files directory '% s' already exists!" + ConsoleColors.END) % acf_path)
overwrite = input('Do you want to overwrite it? [y/n]: ')
if overwrite.casefold() == 'y':
# Overwrite directory
shutil.rmtree(acf_path)
os.mkdir(acf_path)
make_acf = True
print('Access Copy Files directory overwritten')
elif overwrite.casefold() != 'n':
print(ConsoleColors.RED + 'Unknown command, exiting' + ConsoleColors.END)
quit(os.EX_USAGE)
if make_acf:
# Copy RestoredAudioFiles
raf_path = os.path.join(temp_path, 'RestoredAudioFiles')
if not os.path.exists(raf_path):
print((ConsoleColors.RED + "Restored Audio Files directory '% s' not found!" + ConsoleColors.END) % raf_path)
quit(os.EX_NOINPUT)
restored_audio_files = os.listdir(raf_path)
if len(restored_audio_files) == 1:
print((ConsoleColors.YELLOW + "Restored Audio Files directory '% s' is empty" + ConsoleColors.END) % raf_path)
shutil.copytree(raf_path, os.path.join(acf_path, 'RestoredAudioFiles'))
print("Restored Audio Files copied")
# Copy Editing List
el_path = os.path.join(temp_path, 'EditingList.json')
try:
shutil.copy2(el_path, acf_path)
except FileNotFoundError:
print((ConsoleColors.RED + "Editing List file '% s' not found!" + ConsoleColors.END) % el_path)
quit(os.EX_NOINPUT)
print("Editing List copied")
# Create Irregularity Images archive
ii_path = os.path.join(temp_path, 'IrregularityImages')
if not os.path.exists(ii_path):
print((ConsoleColors.RED + "Irregularity Images directory '% s' not found!" + ConsoleColors.END) % ii_path)
quit(os.EX_NOINPUT)
irregularity_images = os.listdir(ii_path)
if len(irregularity_images) == 1:
print((ConsoleColors.YELLOW + "Irregularity Images directory '% s' is empty" + ConsoleColors.END) % ii_path)
shutil.make_archive(acf_path + 'IrregularityImages', 'zip', temp_path, 'IrregularityImages')
print("Irregularity Images archive created")
# Copy Irregularity File
if_path = os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json')
try:
shutil.copy2(if_path, os.path.join(acf_path, 'IrregularityFile.json'))
except FileNotFoundError:
print((ConsoleColors.RED + "Irregularity File file '% s' not found!" + ConsoleColors.END) % if_path)
quit(os.EX_NOINPUT)
print("Irregularity File copied")
# End Access Copy Files
print(ConsoleColors.GREEN + ConsoleColors.BOLD + "Success!" + ConsoleColors.END)
# Preservation Master Files
print('\n' + ConsoleColors.BOLD + 'Creation of Preservation Master Files...' + ConsoleColors.END)
# By default, PreservationMasterFiles directory is created in the output path...
pmf_dir = 'PreservationMasterFiles/' + config['PRESERVATION_FILES_NAME'] + '/'
pmf_path = os.path.join(config['WORKING_PATH'], pmf_dir)
# ...however, if it already exists, it is possible to skip its creation
if not os.path.exists(pmf_path):
os.mkdir(pmf_path)
print("Preservation Master Files directory '% s' created" % pmf_path)
else:
print((ConsoleColors.PURPLE + "Preservation Master Files directory '% s' already exists!" + ConsoleColors.END) % pmf_path)
overwrite = input('Do you want to overwrite it? [y/n]: ')
if overwrite.casefold() == 'y':
shutil.rmtree(pmf_path)
os.mkdir(pmf_path)
print("Preservation Master Files directory overwritten")
elif overwrite.casefold() == 'n':
print('\nExit\n')
quit(os.EX_OK)
else:
print(ConsoleColors.RED + 'Unknown command, exiting\n' + ConsoleColors.END)
quit(os.EX_USAGE)
# Copy Preservation Audio File
audio_file = config['PRESERVATION_FILES_NAME'] + '.wav'
paf_path = os.path.join(config['WORKING_PATH'], 'PreservationAudioFile/', audio_file)
try:
shutil.copy2(paf_path, pmf_path + 'PreservationAudioFile.wav')
except FileNotFoundError:
print((ConsoleColors.RED + "Preservation Audio File file '% s' not found!" + ConsoleColors.END) % paf_path)
quit(os.EX_NOINPUT)
print("Preservation Audio File copied")
# Create Preservation Audio-Visual File with substituted audio
video_file = config['PRESERVATION_FILES_NAME'] + '.mov'
pvf_path = os.path.join(config['WORKING_PATH'], 'PreservationAudioVisualFile/', video_file)
try:
audio = AudioFileClip(paf_path)
video = VideoFileClip(pvf_path)
# Open Irregularity File to get offset
irregularity_file_json = open(os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json'))
irregularity_file = json.load(irregularity_file_json)
offset = irregularity_file['Offset']/1000
if offset > 0:
audio = audio.subclip(t_start=offset)
else:
video = video.subclip(t_start=offset)
video = video.set_audio(audio)
video.write_videofile(pmf_path + 'PreservationAudioVisualFile.mov', bitrate='3000k', codec='mpeg4')
print("Preservation Audio-Visual File created")
except OSError:
print((ConsoleColors.RED + "Preservation Audio-Visual File file '% s' not found!" + ConsoleColors.END) % pvf_path)
quit(os.EX_NOINPUT)
# Create Irregularity Images archive (already checked)
shutil.make_archive(pmf_path + 'IrregularityImages', 'zip', temp_path, 'IrregularityImages')
print("Irregularity Images archive created")
# Copy Irregularity File (already checked)
shutil.copy2(os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json'), os.path.join(pmf_path, 'IrregularityFile.json'))
print("Irregularity File copied")
# End Preservation Master Files
print(ConsoleColors.GREEN + ConsoleColors.BOLD + "Success!" + ConsoleColors.END + '\n')
packager.py 0 → 100755
View file @ eba1ee01
#!./venv/bin/activate
"""
MPAI CAE-ARP Packager.
Implements MPAI CAE-ARP Packager Technical Specification, providing:
- Access Copy Files:
1. Restored Audio Files.
2. Editing List.
3. Set of Irregularity Images in a .zip file.
4. Irregularity File.
- Preservation Master Files:
1. Preservation Audio File.
2. Preservation Audio-Visual File where the audio has been replaced with the Audio of the Preservation Audio File
fully synchronised with the video.
3. Set of Irregularity Images in a .zip file.
4. Irregularity File.
"""
import json
import os
import shutil
import sys
import yaml
from argparse import ArgumentParser, RawTextHelpFormatter
from moviepy.editor import VideoFileClip, AudioFileClip
__author__ = "Nadir Dalla Pozza"
__copyright__ = "Copyright 2022, Audio Innova S.r.l."
__credits__ = ["Niccolò Pretto", "Nadir Dalla Pozza", "Sergio Canazza"]
__license__ = "GPL v3.0"
__version__ = "1.0.1"
__maintainer__ = "Nadir Dalla Pozza"
__email__ = "nadir.dallapozza@unipd.it"
__status__ = "Production"
class CC:
"""
Variables for customizing console colors
"""
PURPLE = '\033[95m'
CYAN = '\033[96m'
DARK_CYAN = '\033[36m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
def get_arguments() -> tuple[str, str]:
"""
Method to obtain arguments from config.yaml file or command line.
Default config.yaml, ignored if a command line argument is passed.
:return: tuple consisting of two strings:
1) the working path;
2) the name of the Preservation files, which is key element to retrieve necessary files.
"""
if len(sys.argv) > 1:
# Read from command line
parser = ArgumentParser(
prog="python3 packager.py",
formatter_class=RawTextHelpFormatter,
description="A tool that implements MPAI CAE-ARP Packager Technical Specification.\n"
"By default, the configuration parameters are loaded from ./config.yaml file,\n"
"but, alternately, you can pass command line arguments to replace them."
)
parser.add_argument(
"-w",
"--working-path",
help="Specify the Working Path, where all input files are stored",
required=True
)
parser.add_argument(
"-f",
"--files-name",
help="Specify the name of the Preservation files (without extension)",
required=True
)
args = parser.parse_args()
working_path = args.working_path
files_name = args.files_name
else:
# Read from configuration file
config = object
try:
config = yaml.safe_load(open('./config.yaml', 'r'))
if 'WORKING_PATH' not in config:
print(CC.RED + 'WORKING_PATH key not found in config.yaml!' + CC.END)
quit(os.EX_CONFIG)
if 'FILES_NAME' not in config:
print(CC.RED + 'FILES_NAME key not found in config.yaml!' + CC.END)
quit(os.EX_CONFIG)
except FileNotFoundError:
print(CC.RED + 'config.yaml file not found!' + CC.END)
quit(os.EX_NOINPUT)
working_path = config['WORKING_PATH']
files_name = config['FILES_NAME']
return working_path, files_name
def check_input(working_path: str, files_name: str) -> str:
"""
Method to check that passed arguments are correct and that the environment is conformant to the standard.
:param working_path: str representing the path where all files resulting from previous AIMs are stored.
:param files_name: str representing the Preservation files name, to identify the input directory.
:return: str representing the path where the files to be processed during the current execution are stored.
"""
if not os.path.exists(working_path):
print(CC.RED + 'The specified WORKING_PATH is non-existent!' + CC.END)
quit(os.EX_CONFIG)
# Check for temp directory existence
temp_path = os.path.join(working_path, 'temp')
if not os.path.exists(temp_path):
print(CC.RED + 'WORKING_PATH structure is not conformant!' + CC.END)
quit(os.EX_NOINPUT)
# Check for input directory existence
temp_path = os.path.join(temp_path, files_name)
if not os.path.exists(temp_path):
print(CC.RED + 'The specified FILES_NAME has no corresponding files!' + CC.END)
quit(os.EX_NOINPUT)
return temp_path
def make_dir(dir_path: str) -> bool:
"""
Decide if the given directory has to be created or should be overridden.
:param dir_path: str representing the path of the directory.
:return: bool specifying if the files within the directory should be created.
"""
if not os.path.exists(dir_path):
# Create directory
os.mkdir(dir_path)
print("Directory '% s' created" % dir_path)
return True
else:
print((CC.PURPLE + "Directory '% s' already exists!" + CC.END) % dir_path)
overwrite = input('Do you want to overwrite it? [y/n]: ')
if overwrite.casefold() == 'y':
# Overwrite directory
shutil.rmtree(dir_path)
os.mkdir(dir_path)
print('%s overwritten' % dir_path)
return True
elif overwrite.casefold() != 'n':
print(CC.RED + 'Unknown command, exiting' + CC.END)
quit(os.EX_USAGE)
return False
def main():
"""
Main execution method.
:return: exit codes corresponding to the execution status
"""
print(CC.BOLD + "\nWelcome to ARP Packager!" + CC.END)
print("You are using Python version: " + sys.version)
# Get the input from config.yaml or command line
working_path, files_name = get_arguments()
# Check if input is correct
temp_path = check_input(working_path, files_name)
# Access Copy Files
print('\n' + CC.BOLD + 'Creation of Access Copy Files...' + CC.END)
# By default, AccessCopyFiles directory is created in the output path...
acf_dir = 'AccessCopyFiles/' + files_name + '/'
acf_path = os.path.join(working_path, acf_dir)
# ...however, if it already exists, the user can decide if it has to be overridden
make_acf = make_dir(acf_path)
if make_acf:
# Copy RestoredAudioFiles
raf_path = os.path.join(temp_path, 'RestoredAudioFiles')
if not os.path.exists(raf_path):
print((CC.RED + "Restored Audio Files directory '% s' not found!" + CC.END) % raf_path)
quit(os.EX_NOINPUT)
restored_audio_files = os.listdir(raf_path)
if len(restored_audio_files) == 1:
print((CC.YELLOW + "Restored Audio Files directory '% s' is empty" + CC.END) % raf_path)
shutil.copytree(raf_path, os.path.join(acf_path, 'RestoredAudioFiles'))
print("Restored Audio Files copied")
# Copy Editing List
el_path = os.path.join(temp_path, 'EditingList.json')
try:
shutil.copy2(el_path, acf_path)
except FileNotFoundError:
print((CC.RED + "Editing List file '% s' not found!" + CC.END) % el_path)
quit(os.EX_NOINPUT)
print("Editing List copied")
# Create Irregularity Images archive
ii_path = os.path.join(temp_path, 'IrregularityImages')
if not os.path.exists(ii_path):
print((CC.RED + "Irregularity Images directory '% s' not found!" + CC.END) % ii_path)
quit(os.EX_NOINPUT)
irregularity_images = os.listdir(ii_path)
if len(irregularity_images) == 1:
print((CC.YELLOW + "Irregularity Images directory '% s' is empty" + CC.END) % ii_path)
shutil.make_archive(acf_path + 'IrregularityImages', 'zip', temp_path, 'IrregularityImages')
print("Irregularity Images archive created")
# Copy Irregularity File
if_path = os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json')
try:
shutil.copy2(if_path, os.path.join(acf_path, 'IrregularityFile.json'))
except FileNotFoundError:
print((CC.RED + "Irregularity File file '% s' not found!" + CC.END) % if_path)
quit(os.EX_NOINPUT)
print("Irregularity File copied")
# End Access Copy Files
print(CC.GREEN + CC.BOLD + "Success!" + CC.END)
# Preservation Master Files
print('\n' + CC.BOLD + 'Creation of Preservation Master Files...' + CC.END)
# By default, PreservationMasterFiles directory is created in the output path...
pmf_dir = 'PreservationMasterFiles/' + files_name + '/'
pmf_path = os.path.join(working_path, pmf_dir)
# ...however, if it already exists, it is possible to skip its creation
make_pmf = make_dir(pmf_path)
if make_pmf:
# Copy Preservation Audio File
audio_file = files_name + '.wav'
paf_path = os.path.join(working_path, 'PreservationAudioFile/', audio_file)
try:
shutil.copy2(paf_path, pmf_path + 'PreservationAudioFile.wav')
except FileNotFoundError:
print((CC.RED + "Preservation Audio File file '% s' not found!" + CC.END) % paf_path)
quit(os.EX_NOINPUT)
print("Preservation Audio File copied")
# Create Preservation Audio-Visual File with substituted audio
video_file = files_name + '.mov'
pvf_path = os.path.join(working_path, 'PreservationAudioVisualFile/', video_file)
try:
audio = AudioFileClip(paf_path)
video = VideoFileClip(pvf_path)
# Open Irregularity File to get offset
irregularity_file_json = open(
os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json')
)
irregularity_file = json.load(irregularity_file_json)
offset = irregularity_file['Offset']/1000
if offset > 0:
audio = audio.subclip(t_start=offset)
else:
video = video.subclip(t_start=offset)
video = video.set_audio(audio)
video.write_videofile(pmf_path + 'PreservationAudioVisualFile.mov', bitrate='3000k', codec='mpeg4')
print("Preservation Audio-Visual File created")
except OSError:
print((CC.RED + "Preservation Audio-Visual File file '% s' not found!" + CC.END) % pvf_path)
quit(os.EX_NOINPUT)
# Create Irregularity Images archive (already checked)
shutil.make_archive(pmf_path + 'IrregularityImages', 'zip', temp_path, 'IrregularityImages')
print("Irregularity Images archive created")
# Copy Irregularity File (already checked)
shutil.copy2(
os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json'),
os.path.join(pmf_path, 'IrregularityFile.json')
)
print("Irregularity File copied")
# End Preservation Master Files
print(CC.GREEN + CC.BOLD + "Success!" + CC.END + '\n')
else:
print("\nExit")
if __name__ == '__main__':
main()
requirements.txt
View file @ eba1ee01
anyio==3.5.0
appnope==0.1.2
argon2-cffi==21.3.0
argon2-cffi-bindings==21.2.0
asttokens==2.0.5
attrs==21.4.0
Babel==2.9.1
backcall==0.2.0
black==21.12b0
bleach==4.1.0
certifi==2021.10.8
cffi==1.15.0
charset-normalizer==2.0.10
click==8.0.3
debugpy==1.5.1
decorator==4.4.2
defusedxml==0.7.1
entrypoints==0.3
executing==0.8.2
GDAL==3.5.0
idna==3.3
imageio==2.22.2
imageio-ffmpeg==0.4.7
ipykernel==6.7.0
ipython==8.0.0
ipython-genutils==0.2.0
jedi==0.18.1
Jinja2==3.0.3
json5==0.9.6
jsonschema==4.4.0
jupyter-client==7.1.1
jupyter-core==4.9.1
jupyter-server==1.13.3
jupyterlab==3.2.8
jupyterlab-pygments==0.1.2
jupyterlab-server==2.10.3
MarkupSafe==2.0.1
matplotlib-inline==0.1.3
mistune==0.8.4
moviepy==1.0.3
mypy-extensions==0.4.3
nbclassic==0.3.5
nbclient==0.5.10
nbconvert==6.4.0
nbformat==5.1.3
nest-asyncio==1.5.4
notebook==6.4.7
numpy==1.23.0
pandocfilters==1.5.0
parso==0.8.3
pathspec==0.9.0
pexpect==4.8.0
pickleshare==0.7.5
Pillow==9.2.0
platformdirs==2.4.1
proglog==0.1.10
prometheus-client==0.12.0
prompt-toolkit==3.0.24
protobuf==3.19.4
ptyprocess==0.7.0
pure-eval==0.2.1
pycparser==2.21
Pygments==2.11.2
PyQt3D==5.15.5
PyQt5==5.15.7
PyQt5-sip==12.11.0
PyQtChart==5.15.6
PyQtDataVisualization==5.15.5
PyQtNetworkAuth==5.15.5
PyQtPurchasing==5.15.5
PyQtWebEngine==5.15.6
pyrsistent==0.18.1
python-dateutil==2.8.2
pytz==2021.3
PyYAML==6.0
pyzmq==22.3.0
QScintilla==2.13.3
requests==2.27.1
Send2Trash==1.8.0
six==1.16.0
sniffio==1.2.0
stack-data==0.1.4
TBB==0.2
terminado==0.12.1
testpath==0.5.0
tomli==1.2.3
tornado==6.1
tqdm==4.64.1
traitlets==5.1.1
typing_extensions==4.0.1
urllib3==1.26.8
wcwidth==0.2.5
webencodings==0.5.1
websocket-client==1.2.3
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment