update

.editorconfig

+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = false
+insert_final_newline = false
\ No newline at end of file

.env

+FILES_NAME=BERIO100
+WORKING_PATH=../data
\ No newline at end of file

Dockerfile

@@ -10,4 +10,4 @@ RUN poetry install --no-cache --only main
 
 VOLUME [ "/data" ]
 
-CMD ["poetry", "run", "uvicorn", "src.server:app", "--host", "0.0.0.0", "--port", "80", "--log-config", "config/logger.yaml", "--root-path", "/packager"]
+CMD ["poetry", "run", "python", "src/packager/server.py"]
\ No newline at end of file

config/args.yaml

-FILES_NAME: BERIO100
-WORKING_PATH: ../data

config/logger.yaml

-version: 1
-formatters:
-  brief:
-    format: '%(message)s'
-  precise:
-    format: '[%(levelname)s  %(name)s %(module)s:%(lineno)s - %(funcName)s() - %(asctime)s]\n\t %(message)s \n'
-    datefmt: '%d/%m/%Y %H:%M:%S'
-handlers:
-  file:
-    class: logging.handlers.RotatingFileHandler
-    formatter: precise
-    filename: /var/log/packager/server.log
-    maxBytes: 10485760
-    backupCount: 3
-  console:
-    class: logging.StreamHandler
-    formatter: brief
-loggers:
-  uvicorn:
-    level: INFO
-    handlers: [file, console]
-    propagate: no
\ No newline at end of file

poetry.lock

@@ -497,6 +497,43 @@ files = [
     {file = "llvmlite-0.39.1.tar.gz", hash = "sha256:b43abd7c82e805261c425d50335be9a6c4f84264e34d6d6e475207300005d572"},
 ]
 
+[[package]]
+name = "markdown-it-py"
+version = "2.2.0"
+description = "Python port of markdown-it. Markdown parsing, done right!"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"},
+    {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"},
+]
+
+[package.dependencies]
+mdurl = ">=0.1,<1.0"
+
+[package.extras]
+benchmarking = ["psutil", "pytest", "pytest-benchmark"]
+code-style = ["pre-commit (>=3.0,<4.0)"]
+compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"]
+linkify = ["linkify-it-py (>=1,<3)"]
+plugins = ["mdit-py-plugins"]
+profiling = ["gprof2dot"]
+rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"]
+testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
+
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+description = "Markdown URL utilities"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
+    {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
+]
+
 [[package]]
 name = "moviepy"
 version = "1.0.3"
@@ -527,14 +564,14 @@ test = ["coverage (<5.0)", "coveralls (>=1.1,<2.0)", "pytest (>=3.0.0,<4.0)", "p
 
 [[package]]
 name = "mpai-cae-arp"
-version = "0.3.2"
+version = "0.4.1"
 description = "The MPAI CAE-ARP software API"
 category = "main"
 optional = false
 python-versions = ">=3.10,<4.0"
 files = [
-    {file = "mpai_cae_arp-0.3.2-py3-none-any.whl", hash = "sha256:468db6aaae5a6d54d1460ede6e9a47686d12ab92680a51e14adf14dcba405aac"},
-    {file = "mpai_cae_arp-0.3.2.tar.gz", hash = "sha256:e447ee46f6c4ca18dd87e3b94dbf366d3f4852ad5530886fd577df9b8d32b2cd"},
+    {file = "mpai_cae_arp-0.4.1-py3-none-any.whl", hash = "sha256:9601eb46c4561cae52347f8bdfbd8b575861fb6dad180b82eb0496b9dfa9f041"},
+    {file = "mpai_cae_arp-0.4.1.tar.gz", hash = "sha256:11329c3abc906112a123130ebfb728151979af05d01ee9a1984af06b8c7be74c"},
 ]
 
 [package.dependencies]
@@ -916,6 +953,21 @@ typing-extensions = ">=4.2.0"
 dotenv = ["python-dotenv (>=0.10.4)"]
 email = ["email-validator (>=1.0.3)"]
 
+[[package]]
+name = "pygments"
+version = "2.15.1"
+description = "Pygments is a syntax highlighting package written in Python."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"},
+    {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"},
+]
+
+[package.extras]
+plugins = ["importlib-metadata"]
+
 [[package]]
 name = "pyyaml"
 version = "6.0"
@@ -988,6 +1040,25 @@ urllib3 = ">=1.21.1,<1.27"
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
 use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
+[[package]]
+name = "rich"
+version = "13.3.4"
+description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+category = "main"
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "rich-13.3.4-py3-none-any.whl", hash = "sha256:22b74cae0278fd5086ff44144d3813be1cedc9115bdfabbfefd86400cb88b20a"},
+    {file = "rich-13.3.4.tar.gz", hash = "sha256:b5d573e13605423ec80bdd0cd5f8541f7844a0e71a13f74cf454ccb2f490708b"},
+]
+
+[package.dependencies]
+markdown-it-py = ">=2.2.0,<3.0.0"
+pygments = ">=2.13.0,<3.0.0"
+
+[package.extras]
+jupyter = ["ipywidgets (>=7.5.1,<9)"]
+
 [[package]]
 name = "scikit-learn"
 version = "1.2.2"
@@ -1072,14 +1143,14 @@ test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "sciki
 
 [[package]]
 name = "setuptools"
-version = "67.7.1"
+version = "67.7.2"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
 category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "setuptools-67.7.1-py3-none-any.whl", hash = "sha256:6f0839fbdb7e3cfef1fc38d7954f5c1c26bf4eebb155a55c9bf8faf997b9fb67"},
-    {file = "setuptools-67.7.1.tar.gz", hash = "sha256:bb16732e8eb928922eabaa022f881ae2b7cdcfaf9993ef1f5e841a96d32b8e0c"},
+    {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"},
+    {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"},
 ]
 
 [package.extras]
@@ -1220,4 +1291,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "a551987715c8ce857de09712be1d99220a26f0cf9091ef50cdb97e8221a0e93d"
+content-hash = "452bb4a990359a117b785dc2aba650d39e7cad5cb79cb587e2309712a31342db"

pyproject.toml

 [tool.poetry]
 name = "packager"
-version = "1.0.1"
+version = "1.1.0"
 description = "MPAI CAE-ARP Packager."
 authors = [
     "Matteo Spanio <dev2@audioinnova.com>",
@@ -14,7 +14,8 @@ packages = [{ include = "packager", from = "src" }]
 [tool.poetry.dependencies]
 python = "^3.10"
 moviepy = "1.0.3"
-mpai-cae-arp = "^0.3.2"
+mpai-cae-arp = "^0.4.1"
+rich = "^13.3.4"
 
 [tool.poetry.scripts]
 packager = 'packager.cli:main'

src/packager/packager.py → src/packager/lib.py

@@ -27,20 +27,15 @@ from moviepy.editor import VideoFileClip, AudioFileClip
 from mpai_cae_arp.files import File, FileType
 from mpai_cae_arp.io import Color, Style, pprint
 
-__author__ = "Nadir Dalla Pozza"
 __copyright__ = "Copyright 2022, Audio Innova S.r.l."
 __credits__ = ["Niccolò Pretto", "Nadir Dalla Pozza", "Sergio Canazza"]
-__license__ = "GPL v3.0"
-__version__ = "1.0.1"
-__maintainer__ = "Nadir Dalla Pozza"
-__email__ = "nadir.dallapozza@unipd.it"
 __status__ = "Production"
 
 
 def get_arguments() -> tuple[str, str]:
     """
-    Method to obtain arguments from config/args.yaml file or command line.
-    Default config/args.yaml, ignored if a command line argument is passed.
+    Method to obtain arguments from environment or command line.
+    Default environment, ignored if a command line argument is passed.
     :return: tuple consisting of two strings:
              1) the working path;
              2) the name of the Preservation files, which is key element to retrieve necessary files.
@@ -70,21 +65,13 @@ def get_arguments() -> tuple[str, str]:
         working_path = args.working_path
         files_name = args.files_name
     else:
-        # Read from configuration file
-        config = object
-        try:
-            config = File('./config/args.yaml', FileType.YAML).get_content()
-            if 'WORKING_PATH' not in config:
-                pprint('WORKING_PATH key not found in config/args.yaml!', color=Color.RED)
-                quit(os.EX_CONFIG)
-            if 'FILES_NAME' not in config:
-                pprint('FILES_NAME key not found in config/args.yaml!', color=Color.RED)
-                quit(os.EX_CONFIG)
-        except FileNotFoundError:
-            pprint('config/args.yaml file not found!', color=Color.RED)
-            quit(os.EX_NOINPUT)
-        working_path = config['WORKING_PATH']
-        files_name = config['FILES_NAME']
+        arg_names = ['WORKING_PATH', 'FILES_NAME']
+        config = {argument: os.getenv(argument) for argument in arg_names}
+        working_path, files_name = map(config.get, arg_names)
+        
+        if any(value is None for value in config.values()):
+            raise ValueError("Please, set the environment variables: WORKING_PATH, FILES_NAME")
+
     return working_path, files_name
 
 

src/packager/server.py

-import subprocess
-from fastapi import FastAPI, Response, status, Query
-from typing import Annotated
+import os
+import shutil
+from concurrent import futures
+import grpc
+import json
+from grpc import StatusCode
+from rich.console import Console
+from moviepy.editor import VideoFileClip, AudioFileClip
 
 from mpai_cae_arp.files import File, FileType
-from mpai_cae_arp.types.schema import Info
-
-info = File("config/server.yaml", FileType.YAML).get_content()
-app = FastAPI(**info)
-
-
-@app.get("/")
-async def list_all_entrypoints() -> list[str]:
-    """
-    Get the list of available main routes.
-    """
-    return ["restore", "description", "docs"]
-
-
-@app.get("/description")
-async def get_server_info() -> Info:
-    """
-    Retrieve all the informations about the software running on the server.
-    """
-    return info
-
-
-@app.get("/package", status_code=200)
-async def predict(
-        response: Response,
-        files_name: Annotated[str, Query(
-            description="Name of the audio/video file to be analyzed without the extension.",
-            example="filename"
-        )],
-        working_path: Annotated[str, Query(
-            description="Path to the directory containing the preservation files.",
-            examples={
-                "Absolute path": {"value": "/home/user/preservation"},
-                "Relative path": {"value": "preservation"}
-            }
-        )] = None
-    ):
-    
-    process = [
-        "python",
-        "src/packager.py",
-        ]
-
-    config_file = File("config/args.yaml", FileType.YAML)
-    # Read configuration file
-    config = config_file.get_content()
-
-    # Update configuration file with query parameters
-    config["FilesName"] = files_name
-    if working_path is not None:
-        config["WorkingPath"] = working_path
-    config_file.write_content(config)
-
-
-    cprocess = subprocess.run(process, capture_output=True)
-
-    if cprocess.returncode == 0:
-        return {"message": "Package created successfully"}
-
-    else:
-        response.status_code = status.HTTP_412_PRECONDITION_FAILED
-        return {"error": {
-            "returncode": cprocess.returncode,
-            "stdout": cprocess.stdout.decode("utf-8"),
-            "stderr": cprocess.stderr.decode("utf-8"),
-        }}
+from mpai_cae_arp.network import arp_pb2_grpc
+from mpai_cae_arp.network.arp_pb2 import (
+    JobRequest,
+    JobResponse,
+    Contact,
+    InfoResponse,
+    License,
+)
+from packager import lib
+
+PORT = os.getenv("PORT") or '50051'
+info = File('config.yml', FileType.YAML).get_content()
+
+
+def error_response(context, status, message):
+    context.set_code(status)
+    context.set_details(message)
+    return JobResponse(status="error", message=message)
+
+
+class PackagerServicer(arp_pb2_grpc.AIMServicer):
+
+    def __init__(self, console: Console):
+        self.console = console
+
+    def getInfo(self, request, context) -> InfoResponse:
+        self.console.log('Received request for AIM info')
+
+        context.set_code(StatusCode.OK)
+        context.set_details('Success')
+
+        return InfoResponse(
+            title=info['title'],
+            description=info['description'],
+            version=info['version'],
+            contact=Contact(
+                name=info['contact']['name'],
+                email=info['contact']['email'],
+            ),
+            license=License(
+                name=info['license_info']['name'],
+                url=info['license_info']['url'],
+            )
+        )
+
+    def work(self, request: JobRequest, context):
+        self.console.log('Received request for computation')
+        self.console.log(request)
+
+        working_dir: str = request.working_dir
+        files_name: str = request.files_name
+
+        temp_dir = os.path.join(working_dir, "temp", files_name)
+
+        yield JobResponse(status="success", message="Job started")
+
+        # By default, AccessCopyFiles directory is created in the output path...
+        acf_dir = 'AccessCopyFiles/' + files_name + '/'
+        acf_path = os.path.join(working_dir, acf_dir)
+        # ...however, if it already exists, the user can decide if it has to be overridden
+        make_acf = lib.make_dir(acf_path)
+
+        if make_acf:
+            # Copy RestoredAudioFiles
+            raf_path = os.path.join(temp_dir, 'RestoredAudioFiles')
+            if not os.path.exists(raf_path):
+                yield error_response(context, StatusCode.INVALID_ARGUMENT, "Restored Audio Files directory not found")
+            restored_audio_files = os.listdir(raf_path)
+            if len(restored_audio_files) == 1:
+                yield JobResponse(status="warning", message="Restored Audio Files directory is empty")
+            shutil.copytree(raf_path, os.path.join(acf_path, 'RestoredAudioFiles'))
+            console.log("Restored Audio Files copied")
+            yield JobResponse(status="success", message="Restored Audio Files copied")
+
+            # Copy Editing List
+            el_path = os.path.join(temp_dir, 'EditingList.json')
+            try:
+                shutil.copy2(el_path, acf_path)
+            except FileNotFoundError:
+                yield error_response(context, StatusCode.INVALID_ARGUMENT, "Editing List file not found")
+            console.log("Editing List copied")
+            yield JobResponse(status="success", message="Editing List copied")
+
+            # Create Irregularity Images archive
+            ii_path = os.path.join(temp_dir, 'IrregularityImages')
+            if not os.path.exists(ii_path):
+                yield error_response(context, StatusCode.INVALID_ARGUMENT, "Irregularity Images directory not found")
+            irregularity_images = os.listdir(ii_path)
+            if len(irregularity_images) == 1:
+                yield JobResponse(status="warning", message="Irregularity Images directory is empty")
+            shutil.make_archive(acf_path + 'IrregularityImages', 'zip', temp_dir, 'IrregularityImages')
+            console.log("Irregularity Images archive created")
+            yield JobResponse(status="success", message="Irregularity Images archive created")
+
+            # Copy Irregularity File
+            if_path = os.path.join(temp_dir, 'TapeIrregularityClassifier_IrregularityFileOutput2.json')
+            try:
+                shutil.copy2(if_path, os.path.join(acf_path, 'IrregularityFile.json'))
+            except FileNotFoundError:
+                yield error_response(context, StatusCode.INVALID_ARGUMENT, "Irregularity File file not found")
+            console.log("Irregularity File copied")
+            yield JobResponse(status="success", message="Irregularity File copied")
+
+        # Preservation Master Files
+        console.log('Creation of Preservation Master Files...')
+        yield JobResponse(status="success", message="Creation of Preservation Master Files...")
+
+        # By default, PreservationMasterFiles directory is created in the output path...
+        pmf_dir = 'PreservationMasterFiles/' + files_name + '/'
+        pmf_path = os.path.join(working_dir, pmf_dir)
+        # ...however, if it already exists, it is possible to skip its creation
+        make_pmf = lib.make_dir(pmf_path)
+
+        if make_pmf:
+
+            # Copy Preservation Audio File
+            audio_file = files_name + '.wav'
+            paf_path = os.path.join(working_dir, 'PreservationAudioFile/', audio_file)
+            try:
+                shutil.copy2(paf_path, pmf_path + 'PreservationAudioFile.wav')
+            except FileNotFoundError:
+                yield error_response(context, StatusCode.INVALID_ARGUMENT, "Preservation Audio File file not found")
+            console.log("Preservation Audio File copied")
+            yield JobResponse(status="success", message="Preservation Audio File copied")
+
+            # Create Preservation Audio-Visual File with substituted audio
+            video_file = files_name + '.mov'
+            pvf_path = os.path.join(working_dir, 'PreservationAudioVisualFile/', video_file)
+            try:
+                audio = AudioFileClip(paf_path)
+                video = VideoFileClip(pvf_path)
+                # Open Irregularity File to get offset
+                irregularity_file_json = open(
+                    os.path.join(temp_dir, 'TapeIrregularityClassifier_IrregularityFileOutput2.json')
+                )
+                irregularity_file = json.load(irregularity_file_json)
+                offset = irregularity_file['Offset']/1000
+                if offset > 0:
+                    audio = audio.subclip(t_start=offset)
+                else:
+                    video = video.subclip(t_start=offset)
+                video = video.set_audio(audio)
+                video.write_videofile(pmf_path + 'PreservationAudioVisualFile.mov', bitrate='3000k', codec='mpeg4')
+                console.log("Preservation Audio-Visual File created")
+                yield JobResponse(status="success", message="Preservation Audio-Visual File created")
+            except OSError:
+                yield error_response(context, StatusCode.INVALID_ARGUMENT, "Preservation Audio-Visual File file not found")
+
+            # Create Irregularity Images archive (already checked)
+            shutil.make_archive(pmf_path + 'IrregularityImages', 'zip', temp_dir, 'IrregularityImages')
+            console.log("Irregularity Images archive created")
+            yield JobResponse(status="success", message="Irregularity Images archive created")
+
+            # Copy Irregularity File (already checked)
+            shutil.copy2(
+                os.path.join(temp_dir, 'TapeIrregularityClassifier_IrregularityFileOutput2.json'),
+                os.path.join(pmf_path, 'IrregularityFile.json')
+            )
+            console.log("Irregularity File copied")
+            yield JobResponse(status="success", message="Irregularity File copied")
+
+            # End Preservation Master Files
+            yield JobResponse(status="success", message="Success!")
+
+        else:
+            console.log("\nExit")
+            yield JobResponse(status="success", message="Exit")
+
+
+def serve(console):
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    arp_pb2_grpc.add_AIMServicer_to_server(PackagerServicer(console), server)
+    server.add_insecure_port(f'[::]:{PORT}')
+    server.start()
+    server.wait_for_termination()
+
+
+if __name__ == '__main__':
+    console = Console()
+    console.print(f'Server started at localhost:{PORT} :satellite:')
+    serve(console)