update

poetry.lock

-# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.
+# This file is automatically @generated by Poetry and should not be changed by hand.
 
 [[package]]
 name = "alabaster"
@@ -287,6 +287,35 @@ files = [
 [package.extras]
 testing = ["pre-commit"]
 
+[[package]]
+name = "ffmpeg-python"
+version = "0.2.0"
+description = "Python bindings for FFmpeg - with complex filtering support"
+category = "main"
+optional = false
+python-versions = "*"
+files = [
+    {file = "ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127"},
+    {file = "ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5"},
+]
+
+[package.dependencies]
+future = "*"
+
+[package.extras]
+dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
+
+[[package]]
+name = "future"
+version = "0.18.3"
+description = "Clean single-source support for Python 3 and 2"
+category = "main"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+files = [
+    {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"},
+]
+
 [[package]]
 name = "grpcio"
 version = "1.53.0"
@@ -656,14 +685,14 @@ files = [
 
 [[package]]
 name = "mpai-cae-arp"
-version = "0.3.0"
+version = "0.3.2"
 description = "The MPAI CAE-ARP software API"
 category = "main"
 optional = false
 python-versions = ">=3.10,<4.0"
 files = [
-    {file = "mpai_cae_arp-0.3.0-py3-none-any.whl", hash = "sha256:ccb376ffa3a37ec3d058061f078bf07b1bb15f1de98a7de7be91ae60bc8be703"},
-    {file = "mpai_cae_arp-0.3.0.tar.gz", hash = "sha256:d9970910e507a85232e87648fe3f1e4b07159118d66ca1eb52f0bc402e90f6b4"},
+    {file = "mpai_cae_arp-0.3.2-py3-none-any.whl", hash = "sha256:468db6aaae5a6d54d1460ede6e9a47686d12ab92680a51e14adf14dcba405aac"},
+    {file = "mpai_cae_arp-0.3.2.tar.gz", hash = "sha256:e447ee46f6c4ca18dd87e3b94dbf366d3f4852ad5530886fd577df9b8d32b2cd"},
 ]
 
 [package.dependencies]
@@ -1033,21 +1062,6 @@ typing-extensions = ">=4.2.0"
 dotenv = ["python-dotenv (>=0.10.4)"]
 email = ["email-validator (>=1.0.3)"]
 
-[[package]]
-name = "pyee"
-version = "9.0.4"
-description = "A port of node.js's EventEmitter to python."
-category = "main"
-optional = false
-python-versions = "*"
-files = [
-    {file = "pyee-9.0.4-py2.py3-none-any.whl", hash = "sha256:9f066570130c554e9cc12de5a9d86f57c7ee47fece163bbdaa3e9c933cfbdfa5"},
-    {file = "pyee-9.0.4.tar.gz", hash = "sha256:2770c4928abc721f46b705e6a72b0c59480c4a69c9a83ca0b00bb994f1ea4b32"},
-]
-
-[package.dependencies]
-typing-extensions = "*"
-
 [[package]]
 name = "pygments"
 version = "2.15.0"
@@ -1122,22 +1136,6 @@ files = [
 [package.dependencies]
 six = ">=1.5"
 
-[[package]]
-name = "python-ffmpeg"
-version = "2.0.4"
-description = "A python binding for FFmpeg which provides sync and async APIs"
-category = "main"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "python-ffmpeg-2.0.4.tar.gz", hash = "sha256:4d5275a58da2a82fb550acbf496ff86e76d677e06b781baeefbe05a4cd47ab50"},
-    {file = "python_ffmpeg-2.0.4-py3-none-any.whl", hash = "sha256:89aa9a0a6232a4fda58a9153af7d6c73b4d80d52b08dcde2b3c4ddcceb392b95"},
-]
-
-[package.dependencies]
-pyee = "*"
-typing-extensions = "*"
-
 [[package]]
 name = "pytz"
 version = "2023.3"
@@ -1402,7 +1400,6 @@ files = [
     {file = "soundfile-0.12.1-py2.py3-none-any.whl", hash = "sha256:828a79c2e75abab5359f780c81dccd4953c45a2c4cd4f05ba3e233ddf984b882"},
     {file = "soundfile-0.12.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:d922be1563ce17a69582a352a86f28ed8c9f6a8bc951df63476ffc310c064bfa"},
     {file = "soundfile-0.12.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:bceaab5c4febb11ea0554566784bcf4bc2e3977b53946dda2b12804b4fe524a8"},
-    {file = "soundfile-0.12.1-py2.py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:2dc3685bed7187c072a46ab4ffddd38cef7de9ae5eb05c03df2ad569cf4dacbc"},
     {file = "soundfile-0.12.1-py2.py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:074247b771a181859d2bc1f98b5ebf6d5153d2c397b86ee9e29ba602a8dfe2a6"},
     {file = "soundfile-0.12.1-py2.py3-none-win32.whl", hash = "sha256:59dfd88c79b48f441bbf6994142a19ab1de3b9bb7c12863402c2bc621e49091a"},
     {file = "soundfile-0.12.1-py2.py3-none-win_amd64.whl", hash = "sha256:0d86924c00b62552b650ddd28af426e3ff2d4dc2e9047dae5b3d8452e0a49a77"},
@@ -1658,4 +1655,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "ca35a4cda1d1842a6fae117b5805bf43e963330173bbd66780c666b4ae903aba"
+content-hash = "d870dea06ffbf0b78f7f22a55affd36a16441d79e12a0c390f00384f4b53a903"

pyproject.toml

@@ -14,7 +14,7 @@ pandas = "^2.0.0"
 scikit-learn = "^1.2.2"
 grpcio-tools = "^1.53.0"
 mpai-cae-arp = "^0.3.0"
-python-ffmpeg = "^2.0.4"
+ffmpeg-python = "^0.2.0"
 
 
 [tool.poetry.group.docs.dependencies]

src/classifier.py

+from pandas import DataFrame
+
+from mpai_cae_arp.audio import AudioWave
+from mpai_cae_arp.types.irregularity import Irregularity, IrregularityProperties
+from ml.classification import load_model, ClassificationResult
+
+
+def classification_to_irreg_properties(classification: ClassificationResult) -> IrregularityProperties:
+    return IrregularityProperties(
+        reading_equalisation=classification.reading_equalization,
+        reading_speed=classification.reading_speed,
+        writing_speed=classification.writing_speed,
+        writing_equalisation=classification.writing_equalization,
+    )
+
+
+def classify(audio_blocks: DataFrame) -> list[IrregularityProperties]:
+
+    audio_blocks_classification = []
+
+    # classify the audioBlocks
+    eq_classifier = load_model("pretto_and_berio_nono_classifier")
+    prediction = eq_classifier.predict(audio_blocks)
+
+    for i in range(len(prediction)):
+        classification = classification_to_irreg_properties(prediction.iloc[i].classification)
+        audio_blocks_classification.append(classification)
+
+    return audio_blocks_classification
+
+
+def extract_features(audio_blocks: list[Irregularity]) -> DataFrame:
+
+    features = {f'mfcc{i}': [] for i in range(1, 14)}
+
+    for audio_block in audio_blocks:
+        audio = AudioWave.from_file(audio_block.audio_block_URI)
+        audio_mfcc = audio.get_mfcc()
+        for i, key in enumerate(features.keys()):
+            features[key].append(audio_mfcc[i])
+
+    return DataFrame(features)

src/cli.py

@@ -3,8 +3,9 @@ import os
 import sys
 from rich.console import Console
 import segment_finder as sf
+import classifier as cl
 
-from mpai_cae_arp.types.irregularity import IrregularityFile
+from mpai_cae_arp.types.irregularity import IrregularityFile, Source
 from mpai_cae_arp.files import File, FileType
 from mpai_cae_arp.io import prettify, Style
 
@@ -42,7 +43,10 @@ def main() -> None:
             "Working directory or files name not specified!",
             "Try -h/--help to know more about Audio Analyser usage"), console)
 
-    os.makedirs(os.path.join(working_directory, "temp", files_name))
+    try:
+        os.makedirs(os.path.join(working_directory, "temp", files_name))
+    except:
+        exit_with_error("Unable to create temporary directory, output path already exists", console)
     
     with console.status("[purple]Reading input files", spinner="dots"):
         audio_src = os.path.join(working_directory, "PreservationAudioFile", f"{files_name}.wav")
@@ -91,12 +95,22 @@ def main() -> None:
 
     with console.status("[cyan]Extracting audio irregularities", spinner="dots"):
         irreg2= sf.extract_audio_irregularities(audio_src, irreg2, working_directory + "/temp/" + files_name)
-        File(f"{working_directory}/temp/{files_name}/IrregularityFile2.json", FileType.JSON).write_content(irreg2.to_json())
         console.log("[green]Audio irregularities extracted")
 
     # classify audio irregularities
     with console.status("[cyan bold]Classifying audio irregularities", spinner="monkey"):
-        sf.classify_audio_irregularities(working_directory)
+        irregularities_features = cl.extract_features(irreg2.irregularities)
+        console.log("[green]Audio irregularities features extracted")
+        classification_results = cl.classify(irregularities_features)
+        console.log("[green]Audio irregularities classified")
+
+    with console.status("[purple]Updating irregularity file 2", spinner="dots"):
+        for irreg, classification_result in zip(irreg2.irregularities, classification_results):
+            if irreg.source == Source.AUDIO:
+                irreg.irregularity_type = classification_result.get_irregularity_type()
+                irreg.irregularity_properties = classification_result if classification_result.get_irregularity_type() is not None else None
+
+    File(f"{working_directory}/temp/{files_name}/IrregularityFile2.json", FileType.JSON).write_content(irreg2.to_json())
 
     console.print("[green bold]Success! :tada:")
     quit(os.EX_OK)

src/client.py

+from rich.console import Console
+from rich.markdown import Markdown
+import os
+
+import grpc
+from mpai_cae_arp.network import arp_pb2
+from mpai_cae_arp.network import arp_pb2_grpc
+
+channels = {
+    "AudioAnalyser": grpc.insecure_channel("[::]:50051/audio-analyser"),
+    "VideoAnalyser": grpc.insecure_channel("[::]:50051/video-analyser"),
+    "TapeIrregularityClassifier": grpc.insecure_channel("[::]:50051/tape-irregularity-classifier"),
+    "TapeAudioRestoration": grpc.insecure_channel("[::]:50051/tape-audio-restoration"),
+    "Packager": grpc.insecure_channel("[::]:50051/packager"),
+}
+
+def run(console: Console):
+    with grpc.insecure_channel("[::]:50051") as channel:
+        audio_analyser = arp_pb2_grpc.AIMStub(channel)
+
+        request = arp_pb2.InfoRequest()
+        response = audio_analyser.getInfo(request)
+        console.print("[bold]{}[/], v{}".format(response.title, response.version))
+        console.print(Markdown(response.description))
+
+        request = arp_pb2.JobRequest(
+            working_dir="../data",
+            files_name="BERIO100",
+            index=1,
+        )
+        with console.status("[bold]Computing...", spinner="bouncingBall"):
+            for result in audio_analyser.work(request):
+                if result.status == "error":
+                    console.print("[bold red]Error![/] :boom:")
+                    console.print(f"[italic red]{result.message}")
+                    for channel in channels.values():
+                        channel.close()
+                    exit(os.EX_SOFTWARE)
+                console.print(result.message)
+
+    console.print("[bold green]Success![/] :tada:")
+
+
+if __name__ == '__main__':
+    console = Console()
+    run(console)
\ No newline at end of file

src/segment_finder.py

+from enum import Enum
 import os
 import tempfile
 from uuid import uuid4
 
 import numpy as np
-from ffmpeg import FFmpeg
+import ffmpeg
+import scipy
 
 from mpai_cae_arp.audio import AudioWave, Noise
 from mpai_cae_arp.files import File, FileType
 from mpai_cae_arp.types.irregularity import Irregularity, IrregularityFile, Source
 from mpai_cae_arp.time import frames_to_seconds, seconds_to_frames, seconds_to_string, time_to_seconds
 
-TMP_CHANNELS_MAP = os.path.join(tempfile.gettempdir(), "mpai", "channels_map.json")
+TMP_FOLDER = os.path.join(tempfile.gettempdir(), "mpai")
+os.makedirs(TMP_FOLDER, exist_ok=True)
+TMP_CHANNELS_MAP = os.path.join(TMP_FOLDER, "channels_map.json")
 
-def calculate_offset(audio: AudioWave, video: AudioWave) -> float:
+def calculate_offset(audio: AudioWave, video: AudioWave, interval: int = 10) -> int:
     """
     Calculates the offset between two audio files based on their cross-correlation.
 
@@ -22,30 +26,45 @@ def calculate_offset(audio: AudioWave, video: AudioWave) -> float:
         The audio file to be used as reference.
     video : AudioWave
         The audio file to be used as target.
-    
+    interval : int, optional
+        The interval in seconds to be used for the cross-correlation, by default 10
+
     Returns
     -------
-    float
+    int
+        The offset in milliseconds.
     """
+    audio = audio.get_channel(0)[(audio.samplerate*15):(audio.samplerate*(15+interval))]
+    video = video.get_channel(0)[(audio.samplerate*15):(video.samplerate*(15+interval))]
+
+    resampled_audio = audio.array[::4]
+    resampled_video = video.array[::4]
+
+    corr = scipy.signal.correlate(resampled_audio, resampled_video, mode="full", method="auto")
+    offset = np.argmax(corr) - len(resampled_audio)
+    offset_ms = offset / (audio.samplerate / 4) * 1000
 
-    corr = np.correlate(audio.array, video.array, mode="full")
-    lags = np.arange(-len(audio.array) + 1, len(video.array))
-    lag_idx = np.argmax(np.abs(corr))
+    return round(offset_ms)
 
-    return lags[lag_idx] / audio.samplerate
 
+class BitDepth(Enum):
+    PCM_S8 = "pcm_s8"
+    PCM_S16LE = "pcm_s16le"
+    PCM_S24LE = "pcm_s24le"
+    PCM_S32LE = "pcm_s32le"
+
+def get_audio_from_video(video_src: str, samplerate: int, bit_depth: BitDepth) -> AudioWave:
 
-def get_audio_from_video(video_src: str):
-    # calculate offset with ffmpeg
     # ffmpeg -i video.mov -acodec pcm_s16le -ac 2 audio.wav
-    ffmpeg = (
-        FFmpeg()
-        .input(video_src)
-        .output(
-            "out.wav",
-            {""}
-        )
-    )
+    extracted_audio_path = os.path.join(TMP_FOLDER, 'audio.wav')
+    
+    in_file = ffmpeg.input(video_src)
+    out_file = ffmpeg.output(in_file.audio, extracted_audio_path, ac=2, ar=samplerate, acodec=bit_depth.value)
+    ffmpeg.run(out_file, quiet=True, overwrite_output=True)
+
+    rate, data = scipy.io.wavfile.read(extracted_audio_path)
+
+    return AudioWave(data, 24, 2, rate)
 
 
 def get_irregularities_from_audio(audio_src: AudioWave) -> list[Irregularity]:
@@ -85,10 +104,9 @@ def get_irregularities_from_audio(audio_src: AudioWave) -> list[Irregularity]:
 def create_irreg_file(audio_src: str, video_src: str) -> IrregularityFile:
 
     audio = AudioWave.from_file(audio_src, bufferize=True)
-    
-
+    video = get_audio_from_video(video_src, audio.samplerate, BitDepth.PCM_S24LE)
 
-    offset = calculate_offset(audio, video_src)
+    offset = calculate_offset(audio, video)
     irregularities = get_irregularities_from_audio(audio)
 
     irregularities.sort(key=lambda x: time_to_seconds(x.time_label))
@@ -146,3 +164,19 @@ def extract_audio_irregularities(
     os.remove(TMP_CHANNELS_MAP)
 
     return irreg_file
+
+
+if __name__ == "__main__":
+
+    from rich.console import Console
+    
+    console = Console()
+    
+    with console.status("Reading PreservationAudioFile", spinner="dots"):
+        audio = AudioWave.from_file("../data/PreservationAudioFile/BERIO100.wav", bufferize=True)
+    with console.status("Extracting audio from PreservationAudioVisualFile", spinner="dots"):
+        video = get_audio_from_video("../data/PreservationAudioVisualFile/BERIO100.mov", audio.samplerate, BitDepth.PCM_S24LE)
+    with console.status("Calculating offset", spinner="dots"):
+        offset = calculate_offset(audio, video)
+
+    print(offset)

src/server.py

+import os
 from concurrent import futures
+from typing import Any, Callable
+import grpc
+from grpc import StatusCode
 from rich.console import Console
+
 from mpai_cae_arp.files import File, FileType
-from mpai_cae_arp.types.irregularity import IrregularityFile
-import grpc
+from mpai_cae_arp.types.irregularity import IrregularityFile, Source
 from mpai_cae_arp.network import arp_pb2_grpc as arp_pb2_grpc
 from mpai_cae_arp.network.arp_pb2 import (
-    ComputationRequest,
-    ComputationResult,
+    JobRequest,
+    JobResponse,
     Contact,
-    Info,
+    InfoResponse,
     License,
 )
-import time
+
+import segment_finder as sf
+import classifier as cl
 
 info = File('config/server.yaml', FileType.YAML).get_content()
 
 
-class AudioAnalyserServicer(arp_pb2_grpc.AudioAnalyserServicer):
+def try_or_error_response(
+    context,
+    on_success_message: str,
+    on_error_message: str,
+    func: Callable,
+    args,
+    on_success_status: StatusCode = StatusCode.OK,
+    on_error_status: StatusCode = StatusCode.INTERNAL,
+) -> tuple[JobResponse, Any]:
+    try:
+        result = func(*args)
+        context.set_code(on_success_status)
+        context.set_details(on_success_message)
+        return JobResponse(status="success", message=on_success_message), result
+    except:
+        context.set_code(on_error_status)
+        context.set_details(on_error_message)
+        return JobResponse(status="error", message=on_error_message), None
+
+
+def error_response(context, status, message):
+    context.set_code(status)
+    context.set_details(message)
+    return JobResponse(status="error", message=message)
+
+
+class AudioAnalyserServicer(arp_pb2_grpc.AIMServicer):
 
     def __init__(self, console: Console):
         self.console = console
 
-    def getAimInfo(self, request, context) -> Info:
+    def getInfo(self, request, context) -> InfoResponse:
         self.console.log('Received request for AIM info')
-        return Info(
+
+        context.set_code(StatusCode.OK)
+        context.set_details('Success')
+
+        return InfoResponse(
             title=info['title'],
             description=info['description'],
             version=info['version'],
@@ -37,23 +73,102 @@ class AudioAnalyserServicer(arp_pb2_grpc.AudioAnalyserServicer):
             )
         )
 
-    def analyse(self, request: ComputationRequest, context):
+    def work(self, request: JobRequest, context):
 
+        self.console.log('Received request for computation')
+        self.console.log(request)
+        
         working_dir: str = request.working_dir
         files_name: str = request.files_name
+        index: int = request.index
 
-        self.console.log('Received request for computation')
-        for x in range(10):
-            yield ComputationResult(
-                success=True,
-                message=f'Processing {x * 10}%'
+        audio_src = os.path.join(working_dir, "PreservationAudioFile", f"{files_name}.wav")
+        video_src = os.path.join(working_dir, "PreservationAudioVisualFile", f"{files_name}.mov")
+
+        temp_dir = os.path.join(working_dir, "temp", files_name)
+        audio_irreg_1 = os.path.join(temp_dir, "AudioAnalyser_IrregularityFileOutput_1.json")
+        audio_irreg_2 = os.path.join(temp_dir, "AudioAnalyser_IrregularityFileOutput_2.json")
+        video_irreg_1 = os.path.join(temp_dir, "VideoAnalyser_IrregularityFileOutput_1.json")
+        
+        if index == 1:
+
+            response, _ = try_or_error_response(
+                context,
+                func=os.makedirs,
+                on_success_message="Folders created successfully",
+                on_error_message="Unable to create temporary directory, output path already exists",
+                on_error_status=StatusCode.ALREADY_EXISTS,
+                args=temp_dir
             )
-            time.sleep(2)
+            yield response
+
+
+            response, irreg1 = try_or_error_response(
+                context,
+                func=sf.create_irreg_file,
+                args=(audio_src, video_src),
+                on_success_message=f"Found irregularities in Audio source",
+                on_error_message="Failed to create irregularity file 1",
+            )
+            yield response
+
+            try:
+                File(audio_irreg_1, FileType.JSON).write_content(irreg1.to_json())
+                context.set_code(StatusCode.OK)
+                yield JobResponse(status="success", message="Irregularity file 1 saved to disk")
+            except:
+                yield error_response(context, StatusCode.INTERNAL, "Failed to save irregularity file 1")
+
+        if index == 2:
+            
+            response, irreg2 = try_or_error_response(
+                context,
+                func=sf.merge_irreg_files,
+                args=(irreg1, IrregularityFile.from_json(video_irreg_1)),
+                on_success_message="Irregularity files merged successfully",
+                on_error_message="Failed to merge irregularity files",
+            )
+            yield response
+
+            response, irreg2 = try_or_error_response(
+                context,
+                func=sf.extract_audio_irregularities,
+                args=(audio_src, irreg2, temp_dir),
+                on_success_message="Audio irregularities extracted",
+                on_error_message="Failed to extract audio irregularities",
+            )
+            yield response
+
+            response, irregularities_features = try_or_error_response(
+                context,
+                func=cl.extract_features,
+                args=irreg2.irregularities,
+                on_success_message="Audio irregularities features extracted",
+                on_error_message="Failed to extract audio irregularities features",
+            )
+            yield response
+
+            response, classification_results = try_or_error_response(
+                context,
+                func=cl.classify,
+                args=irregularities_features,
+                on_success_message="Audio irregularities classified",
+                on_error_message="Failed to classify audio irregularities",
+            )
+            yield response
+
+            for irreg, classification_result in zip(irreg2.irregularities, classification_results):
+                if irreg.source == Source.AUDIO:
+                    irreg.irregularity_type = classification_result.get_irregularity_type()
+                    irreg.irregularity_properties = classification_result if classification_result.get_irregularity_type() is not None else None
+
+            File(audio_irreg_2, FileType.JSON).write_content(irreg2.to_json())
+            yield JobResponse(status="success", message="Irregularity file 2 created")
 
 
 def serve(console):
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
-    arp_pb2_grpc.add_AudioAnalyserServicer_to_server(AudioAnalyserServicer(console), server)
+    arp_pb2_grpc.add_AIMServicer_to_server(AudioAnalyserServicer(console), server)
     server.add_insecure_port('[::]:50051')
     server.start()
     server.wait_for_termination()
@@ -62,4 +177,4 @@ def serve(console):
 if __name__ == '__main__':
     console = Console()
     console.print('Server started at localhost:50051 :satellite:')
-    serve(console)
+    serve(console)
\ No newline at end of file