Skip to content
GitLab
Commit eba1ee01 authored by Nadir Dalla Pozza's avatar Nadir Dalla Pozza
Browse files

Comments and beauty update

parent e6b74595
Hide whitespace changes
Inline Side-by-side
README.md
View file @ eba1ee01
...@@ -48,8 +48,8 @@ To execute the script without issues, the inner structure of the `WORKING_PATH` ...@@ -48,8 +48,8 @@ To execute the script without issues, the inner structure of the `WORKING_PATH`
│ ├── File2.wav │ ├── File2.wav
│ └── ... │ └── ...
├── PreservationAudioVisualFile ├── PreservationAudioVisualFile
│ ├── File1.wav │ ├── File1.mp4
│ ├── File2.wav │ ├── File2.mp4
│ └── ... │ └── ...
├── PreservationMasterFiles ├── PreservationMasterFiles
│ └── ... │ └── ...
...@@ -108,10 +108,20 @@ With this structure, `PRESERVATION_FILES_NAME` parameter could be equal to `File ...@@ -108,10 +108,20 @@ With this structure, `PRESERVATION_FILES_NAME` parameter could be equal to `File
You can now launch the *Packager* from the command line with: You can now launch the *Packager* from the command line with:
``` ```
python3 main.py python3 packager.py
``` ```
Useful log information will be displayed during execution, requiring occasional interaction. Useful log information will be displayed during execution, requiring occasional interaction.
To enable integration in more complex workflows, it is also possible to launch the *Packager* with command line arguments:
```
python3 packager.py [-h] -w WORKING_PATH -f FILES_NAME
```
If you use the `-h` flag:
```
python3 packager.py -h
```
all instructions will be displayed.
## Support ## Support
If you require additional information or have any problem, you can contact us at: If you require additional information or have any problem, you can contact us at:
* Nadir Dalla Pozza (nadir.dallapozza@unipd.it); * Nadir Dalla Pozza (nadir.dallapozza@unipd.it);
......
config.yaml
View file @ eba1ee01
# Working path # Working path
WORKING_PATH: "/Users/nadir/Documents/MPAI-CAE/Workflow" WORKING_PATH: "/Users/nadir/Documents/MPAI-CAE/Workflow"
# Name of the Preservation files (without extension) # Name of the Preservation files (without extension)
PRESERVATION_FILES_NAME: "test" FILES_NAME: "test"
main.py deleted 100755 → 0
View file @ e6b74595
#!/usr/bin/env python
"""MPAI CAE-ARP Packager.
Implements MPAI CAE-ARP Packager Technical Specification, providing:
- Access Copy Files:
1. Restored Audio Files.
2. Editing List.
3. Set of Irregularity Images in a .zip file.
4. Irregularity File.
- Preservation Master Files:
1. Preservation Audio File.
2. Preservation Audio-Visual File where the audio has been replaced with the Audio of the Preservation Audio File
fully synchronised with the video.
3. Set of Irregularity Images in a .zip file.
4. Irregularity File.
"""
import json
import os
import shutil
import yaml
from moviepy.editor import VideoFileClip, AudioFileClip
__author__ = "Nadir Dalla Pozza"
__copyright__ = "Copyright 2022, Audio Innova S.r.l."
__credits__ = ["Niccolò Pretto", "Nadir Dalla Pozza", "Sergio Canazza"]
__license__ = "GPL v3.0"
__version__ = "1.0.1"
__maintainer__ = "Nadir Dalla Pozza"
__email__ = "nadir.dallapozza@unipd.it"
__status__ = "Production"
# Class for customizing console colors
class ConsoleColors:
PURPLE = '\033[95m'
CYAN = '\033[96m'
DARK_CYAN = '\033[36m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
if __name__ == '__main__':
# Read configuration file
config = object
try:
config = yaml.safe_load(open('./config.yaml', 'r'))
if 'WORKING_PATH' not in config:
print(ConsoleColors.RED + 'WORKING_PATH key not found in config.yaml!' + ConsoleColors.END)
quit(os.EX_CONFIG)
if 'PRESERVATION_FILES_NAME' not in config:
print(ConsoleColors.RED + 'PRESERVATION_FILES_NAME key not found in config.yaml!' + ConsoleColors.END)
quit(os.EX_CONFIG)
except FileNotFoundError:
print(ConsoleColors.RED + 'config.yaml file not found!' + ConsoleColors.END)
quit(os.EX_NOINPUT)
temp_path = os.path.join(config['WORKING_PATH'], 'temp/', config['PRESERVATION_FILES_NAME'])
print(temp_path)
# Access Copy Files
print('\n' + ConsoleColors.BOLD + 'Creation of Access Copy Files...' + ConsoleColors.END)
# By default, AccessCopyFiles directory is created in the output path...
acf_dir = 'AccessCopyFiles/' + config['PRESERVATION_FILES_NAME'] + '/'
acf_path = os.path.join(config['WORKING_PATH'], acf_dir)
# ...however, if it already exists, it is possible to skip its creation
make_acf = False
if not os.path.exists(acf_path):
# Create directory
os.mkdir(acf_path)
make_acf = True
print("Access Copy Files directory '% s' created" % acf_path)
else:
print((ConsoleColors.PURPLE + "Access Copy Files directory '% s' already exists!" + ConsoleColors.END) % acf_path)
overwrite = input('Do you want to overwrite it? [y/n]: ')
if overwrite.casefold() == 'y':
# Overwrite directory
shutil.rmtree(acf_path)
os.mkdir(acf_path)
make_acf = True
print('Access Copy Files directory overwritten')
elif overwrite.casefold() != 'n':
print(ConsoleColors.RED + 'Unknown command, exiting' + ConsoleColors.END)
quit(os.EX_USAGE)
if make_acf:
# Copy RestoredAudioFiles
raf_path = os.path.join(temp_path, 'RestoredAudioFiles')
if not os.path.exists(raf_path):
print((ConsoleColors.RED + "Restored Audio Files directory '% s' not found!" + ConsoleColors.END) % raf_path)
quit(os.EX_NOINPUT)
restored_audio_files = os.listdir(raf_path)
if len(restored_audio_files) == 1:
print((ConsoleColors.YELLOW + "Restored Audio Files directory '% s' is empty" + ConsoleColors.END) % raf_path)
shutil.copytree(raf_path, os.path.join(acf_path, 'RestoredAudioFiles'))
print("Restored Audio Files copied")
# Copy Editing List
el_path = os.path.join(temp_path, 'EditingList.json')
try:
shutil.copy2(el_path, acf_path)
except FileNotFoundError:
print((ConsoleColors.RED + "Editing List file '% s' not found!" + ConsoleColors.END) % el_path)
quit(os.EX_NOINPUT)
print("Editing List copied")
# Create Irregularity Images archive
ii_path = os.path.join(temp_path, 'IrregularityImages')
if not os.path.exists(ii_path):
print((ConsoleColors.RED + "Irregularity Images directory '% s' not found!" + ConsoleColors.END) % ii_path)
quit(os.EX_NOINPUT)
irregularity_images = os.listdir(ii_path)
if len(irregularity_images) == 1:
print((ConsoleColors.YELLOW + "Irregularity Images directory '% s' is empty" + ConsoleColors.END) % ii_path)
shutil.make_archive(acf_path + 'IrregularityImages', 'zip', temp_path, 'IrregularityImages')
print("Irregularity Images archive created")
# Copy Irregularity File
if_path = os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json')
try:
shutil.copy2(if_path, os.path.join(acf_path, 'IrregularityFile.json'))
except FileNotFoundError:
print((ConsoleColors.RED + "Irregularity File file '% s' not found!" + ConsoleColors.END) % if_path)
quit(os.EX_NOINPUT)
print("Irregularity File copied")
# End Access Copy Files
print(ConsoleColors.GREEN + ConsoleColors.BOLD + "Success!" + ConsoleColors.END)
# Preservation Master Files
print('\n' + ConsoleColors.BOLD + 'Creation of Preservation Master Files...' + ConsoleColors.END)
# By default, PreservationMasterFiles directory is created in the output path...
pmf_dir = 'PreservationMasterFiles/' + config['PRESERVATION_FILES_NAME'] + '/'
pmf_path = os.path.join(config['WORKING_PATH'], pmf_dir)
# ...however, if it already exists, it is possible to skip its creation
if not os.path.exists(pmf_path):
os.mkdir(pmf_path)
print("Preservation Master Files directory '% s' created" % pmf_path)
else:
print((ConsoleColors.PURPLE + "Preservation Master Files directory '% s' already exists!" + ConsoleColors.END) % pmf_path)
overwrite = input('Do you want to overwrite it? [y/n]: ')
if overwrite.casefold() == 'y':
shutil.rmtree(pmf_path)
os.mkdir(pmf_path)
print("Preservation Master Files directory overwritten")
elif overwrite.casefold() == 'n':
print('\nExit\n')
quit(os.EX_OK)
else:
print(ConsoleColors.RED + 'Unknown command, exiting\n' + ConsoleColors.END)
quit(os.EX_USAGE)
# Copy Preservation Audio File
audio_file = config['PRESERVATION_FILES_NAME'] + '.wav'
paf_path = os.path.join(config['WORKING_PATH'], 'PreservationAudioFile/', audio_file)
try:
shutil.copy2(paf_path, pmf_path + 'PreservationAudioFile.wav')
except FileNotFoundError:
print((ConsoleColors.RED + "Preservation Audio File file '% s' not found!" + ConsoleColors.END) % paf_path)
quit(os.EX_NOINPUT)
print("Preservation Audio File copied")
# Create Preservation Audio-Visual File with substituted audio
video_file = config['PRESERVATION_FILES_NAME'] + '.mov'
pvf_path = os.path.join(config['WORKING_PATH'], 'PreservationAudioVisualFile/', video_file)
try:
audio = AudioFileClip(paf_path)
video = VideoFileClip(pvf_path)
# Open Irregularity File to get offset
irregularity_file_json = open(os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json'))
irregularity_file = json.load(irregularity_file_json)
offset = irregularity_file['Offset']/1000
if offset > 0:
audio = audio.subclip(t_start=offset)
else:
video = video.subclip(t_start=offset)
video = video.set_audio(audio)
video.write_videofile(pmf_path + 'PreservationAudioVisualFile.mov', bitrate='3000k', codec='mpeg4')
print("Preservation Audio-Visual File created")
except OSError:
print((ConsoleColors.RED + "Preservation Audio-Visual File file '% s' not found!" + ConsoleColors.END) % pvf_path)
quit(os.EX_NOINPUT)
# Create Irregularity Images archive (already checked)
shutil.make_archive(pmf_path + 'IrregularityImages', 'zip', temp_path, 'IrregularityImages')
print("Irregularity Images archive created")
# Copy Irregularity File (already checked)
shutil.copy2(os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json'), os.path.join(pmf_path, 'IrregularityFile.json'))
print("Irregularity File copied")
# End Preservation Master Files
print(ConsoleColors.GREEN + ConsoleColors.BOLD + "Success!" + ConsoleColors.END + '\n')
packager.py 0 → 100755
View file @ eba1ee01
#!./venv/bin/activate
"""
MPAI CAE-ARP Packager.
Implements MPAI CAE-ARP Packager Technical Specification, providing:
- Access Copy Files:
1. Restored Audio Files.
2. Editing List.
3. Set of Irregularity Images in a .zip file.
4. Irregularity File.
- Preservation Master Files:
1. Preservation Audio File.
2. Preservation Audio-Visual File where the audio has been replaced with the Audio of the Preservation Audio File
fully synchronised with the video.
3. Set of Irregularity Images in a .zip file.
4. Irregularity File.
"""
import json
import os
import shutil
import sys
import yaml
from argparse import ArgumentParser, RawTextHelpFormatter
from moviepy.editor import VideoFileClip, AudioFileClip
__author__ = "Nadir Dalla Pozza"
__copyright__ = "Copyright 2022, Audio Innova S.r.l."
__credits__ = ["Niccolò Pretto", "Nadir Dalla Pozza", "Sergio Canazza"]
__license__ = "GPL v3.0"
__version__ = "1.0.1"
__maintainer__ = "Nadir Dalla Pozza"
__email__ = "nadir.dallapozza@unipd.it"
__status__ = "Production"
class CC:
"""
Variables for customizing console colors
"""
PURPLE = '\033[95m'
CYAN = '\033[96m'
DARK_CYAN = '\033[36m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
def get_arguments() -> tuple[str, str]:
"""
Method to obtain arguments from config.yaml file or command line.
Default config.yaml, ignored if a command line argument is passed.
:return: tuple consisting of two strings:
1) the working path;
2) the name of the Preservation files, which is key element to retrieve necessary files.
"""
if len(sys.argv) > 1:
# Read from command line
parser = ArgumentParser(
prog="python3 packager.py",
formatter_class=RawTextHelpFormatter,
description="A tool that implements MPAI CAE-ARP Packager Technical Specification.\n"
"By default, the configuration parameters are loaded from ./config.yaml file,\n"
"but, alternately, you can pass command line arguments to replace them."
)
parser.add_argument(
"-w",
"--working-path",
help="Specify the Working Path, where all input files are stored",
required=True
)
parser.add_argument(
"-f",
"--files-name",
help="Specify the name of the Preservation files (without extension)",
required=True
)
args = parser.parse_args()
working_path = args.working_path
files_name = args.files_name
else:
# Read from configuration file
config = object
try:
config = yaml.safe_load(open('./config.yaml', 'r'))
if 'WORKING_PATH' not in config:
print(CC.RED + 'WORKING_PATH key not found in config.yaml!' + CC.END)
quit(os.EX_CONFIG)
if 'FILES_NAME' not in config:
print(CC.RED + 'FILES_NAME key not found in config.yaml!' + CC.END)
quit(os.EX_CONFIG)
except FileNotFoundError:
print(CC.RED + 'config.yaml file not found!' + CC.END)
quit(os.EX_NOINPUT)
working_path = config['WORKING_PATH']
files_name = config['FILES_NAME']
return working_path, files_name
def check_input(working_path: str, files_name: str) -> str:
"""
Method to check that passed arguments are correct and that the environment is conformant to the standard.
:param working_path: str representing the path where all files resulting from previous AIMs are stored.
:param files_name: str representing the Preservation files name, to identify the input directory.
:return: str representing the path where the files to be processed during the current execution are stored.
"""
if not os.path.exists(working_path):
print(CC.RED + 'The specified WORKING_PATH is non-existent!' + CC.END)
quit(os.EX_CONFIG)
# Check for temp directory existence
temp_path = os.path.join(working_path, 'temp')
if not os.path.exists(temp_path):
print(CC.RED + 'WORKING_PATH structure is not conformant!' + CC.END)
quit(os.EX_NOINPUT)
# Check for input directory existence
temp_path = os.path.join(temp_path, files_name)
if not os.path.exists(temp_path):
print(CC.RED + 'The specified FILES_NAME has no corresponding files!' + CC.END)
quit(os.EX_NOINPUT)
return temp_path
def make_dir(dir_path: str) -> bool:
"""
Decide if the given directory has to be created or should be overridden.
:param dir_path: str representing the path of the directory.
:return: bool specifying if the files within the directory should be created.
"""
if not os.path.exists(dir_path):
# Create directory
os.mkdir(dir_path)
print("Directory '% s' created" % dir_path)
return True
else:
print((CC.PURPLE + "Directory '% s' already exists!" + CC.END) % dir_path)
overwrite = input('Do you want to overwrite it? [y/n]: ')
if overwrite.casefold() == 'y':
# Overwrite directory
shutil.rmtree(dir_path)
os.mkdir(dir_path)
print('%s overwritten' % dir_path)
return True
elif overwrite.casefold() != 'n':
print(CC.RED + 'Unknown command, exiting' + CC.END)
quit(os.EX_USAGE)
return False
def main():
"""
Main execution method.
:return: exit codes corresponding to the execution status
"""
print(CC.BOLD + "\nWelcome to ARP Packager!" + CC.END)
print("You are using Python version: " + sys.version)
# Get the input from config.yaml or command line
working_path, files_name = get_arguments()
# Check if input is correct
temp_path = check_input(working_path, files_name)
# Access Copy Files
print('\n' + CC.BOLD + 'Creation of Access Copy Files...' + CC.END)
# By default, AccessCopyFiles directory is created in the output path...
acf_dir = 'AccessCopyFiles/' + files_name + '/'
acf_path = os.path.join(working_path, acf_dir)
# ...however, if it already exists, the user can decide if it has to be overridden
make_acf = make_dir(acf_path)
if make_acf:
# Copy RestoredAudioFiles
raf_path = os.path.join(temp_path, 'RestoredAudioFiles')
if not os.path.exists(raf_path):
print((CC.RED + "Restored Audio Files directory '% s' not found!" + CC.END) % raf_path)
quit(os.EX_NOINPUT)
restored_audio_files = os.listdir(raf_path)
if len(restored_audio_files) == 1:
print((CC.YELLOW + "Restored Audio Files directory '% s' is empty" + CC.END) % raf_path)
shutil.copytree(raf_path, os.path.join(acf_path, 'RestoredAudioFiles'))
print("Restored Audio Files copied")
# Copy Editing List
el_path = os.path.join(temp_path, 'EditingList.json')
try:
shutil.copy2(el_path, acf_path)
except FileNotFoundError:
print((CC.RED + "Editing List file '% s' not found!" + CC.END) % el_path)
quit(os.EX_NOINPUT)
print("Editing List copied")
# Create Irregularity Images archive
ii_path = os.path.join(temp_path, 'IrregularityImages')
if not os.path.exists(ii_path):
print((CC.RED + "Irregularity Images directory '% s' not found!" + CC.END) % ii_path)
quit(os.EX_NOINPUT)
irregularity_images = os.listdir(ii_path)
if len(irregularity_images) == 1:
print((CC.YELLOW + "Irregularity Images directory '% s' is empty" + CC.END) % ii_path)
shutil.make_archive(acf_path + 'IrregularityImages', 'zip', temp_path, 'IrregularityImages')
print("Irregularity Images archive created")
# Copy Irregularity File
if_path = os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json')
try:
shutil.copy2(if_path, os.path.join(acf_path, 'IrregularityFile.json'))
except FileNotFoundError:
print((CC.RED + "Irregularity File file '% s' not found!" + CC.END) % if_path)
quit(os.EX_NOINPUT)
print("Irregularity File copied")
# End Access Copy Files
print(CC.GREEN + CC.BOLD + "Success!" + CC.END)
# Preservation Master Files
print('\n' + CC.BOLD + 'Creation of Preservation Master Files...' + CC.END)
# By default, PreservationMasterFiles directory is created in the output path...
pmf_dir = 'PreservationMasterFiles/' + files_name + '/'
pmf_path = os.path.join(working_path, pmf_dir)
# ...however, if it already exists, it is possible to skip its creation
make_pmf = make_dir(pmf_path)
if make_pmf:
# Copy Preservation Audio File
audio_file = files_name + '.wav'
paf_path = os.path.join(working_path, 'PreservationAudioFile/', audio_file)
try:
shutil.copy2(paf_path, pmf_path + 'PreservationAudioFile.wav')
except FileNotFoundError:
print((CC.RED + "Preservation Audio File file '% s' not found!" + CC.END) % paf_path)
quit(os.EX_NOINPUT)
print("Preservation Audio File copied")
# Create Preservation Audio-Visual File with substituted audio
video_file = files_name + '.mov'
pvf_path = os.path.join(working_path, 'PreservationAudioVisualFile/', video_file)
try:
audio = AudioFileClip(paf_path)
video = VideoFileClip(pvf_path)
# Open Irregularity File to get offset
irregularity_file_json = open(
os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json')
)
irregularity_file = json.load(irregularity_file_json)
offset = irregularity_file['Offset']/1000
if offset > 0:
audio = audio.subclip(t_start=offset)
else:
video = video.subclip(t_start=offset)
video = video.set_audio(audio)
video.write_videofile(pmf_path + 'PreservationAudioVisualFile.mov', bitrate='3000k', codec='mpeg4')
print("Preservation Audio-Visual File created")
except OSError:
print((CC.RED + "Preservation Audio-Visual File file '% s' not found!" + CC.END) % pvf_path)
quit(os.EX_NOINPUT)
# Create Irregularity Images archive (already checked)
shutil.make_archive(pmf_path + 'IrregularityImages', 'zip', temp_path, 'IrregularityImages')
print("Irregularity Images archive created")
# Copy Irregularity File (already checked)
shutil.copy2(
os.path.join(temp_path, 'TapeIrregularityClassifier_IrregularityFileOutput2.json'),
os.path.join(pmf_path, 'IrregularityFile.json')
)
print("Irregularity File copied")
# End Preservation Master Files
print(CC.GREEN + CC.BOLD + "Success!" + CC.END + '\n')
else:
print("\nExit")
if __name__ == '__main__':
main()
requirements.txt
View file @ eba1ee01
anyio==3.5.0
appnope==0.1.2
argon2-cffi==21.3.0
argon2-cffi-bindings==21.2.0
asttokens==2.0.5
attrs==21.4.0
Babel==2.9.1
backcall==0.2.0
black==21.12b0
bleach==4.1.0
certifi==2021.10.8
cffi==1.15.0
charset-normalizer==2.0.10
click==8.0.3
debugpy==1.5.1
decorator==4.4.2
defusedxml==0.7.1
entrypoints==0.3
executing==0.8.2
GDAL==3.5.0
idna==3.3
imageio==2.22.2
imageio-ffmpeg==0.4.7
ipykernel==6.7.0
ipython==8.0.0
ipython-genutils==0.2.0
jedi==0.18.1
Jinja2==3.0.3
json5==0.9.6
jsonschema==4.4.0
jupyter-client==7.1.1
jupyter-core==4.9.1
jupyter-server==1.13.3
jupyterlab==3.2.8
jupyterlab-pygments==0.1.2
jupyterlab-server==2.10.3
MarkupSafe==2.0.1
matplotlib-inline==0.1.3
mistune==0.8.4
moviepy==1.0.3 moviepy==1.0.3
mypy-extensions==0.4.3
nbclassic==0.3.5
nbclient==0.5.10
nbconvert==6.4.0
nbformat==5.1.3
nest-asyncio==1.5.4
notebook==6.4.7
numpy==1.23.0
pandocfilters==1.5.0
parso==0.8.3
pathspec==0.9.0
pexpect==4.8.0
pickleshare==0.7.5
Pillow==9.2.0
platformdirs==2.4.1
proglog==0.1.10
prometheus-client==0.12.0
prompt-toolkit==3.0.24
protobuf==3.19.4
ptyprocess==0.7.0
pure-eval==0.2.1
pycparser==2.21
Pygments==2.11.2
PyQt3D==5.15.5
PyQt5==5.15.7
PyQt5-sip==12.11.0
PyQtChart==5.15.6
PyQtDataVisualization==5.15.5
PyQtNetworkAuth==5.15.5
PyQtPurchasing==5.15.5
PyQtWebEngine==5.15.6
pyrsistent==0.18.1
python-dateutil==2.8.2
pytz==2021.3
PyYAML==6.0 PyYAML==6.0
pyzmq==22.3.0
QScintilla==2.13.3
requests==2.27.1
Send2Trash==1.8.0
six==1.16.0
sniffio==1.2.0
stack-data==0.1.4
TBB==0.2
terminado==0.12.1
testpath==0.5.0
tomli==1.2.3
tornado==6.1
tqdm==4.64.1
traitlets==5.1.1
typing_extensions==4.0.1
urllib3==1.26.8
wcwidth==0.2.5
webencodings==0.5.1
websocket-client==1.2.3
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment