Skip to content
GitLab
Commit dbb29744 authored by Matteo's avatar Matteo
Browse files

update

parent 41fb91b4
Hide whitespace changes
Inline Side-by-side
poetry.lock
View file @ dbb29744
# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.
# This file is automatically @generated by Poetry and should not be changed by hand.
[[package]]
name = "alabaster"
......@@ -287,6 +287,35 @@ files = [
[package.extras]
testing = ["pre-commit"]
[[package]]
name = "ffmpeg-python"
version = "0.2.0"
description = "Python bindings for FFmpeg - with complex filtering support"
category = "main"
optional = false
python-versions = "*"
files = [
{file = "ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127"},
{file = "ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5"},
]
[package.dependencies]
future = "*"
[package.extras]
dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
[[package]]
name = "future"
version = "0.18.3"
description = "Clean single-source support for Python 3 and 2"
category = "main"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
files = [
{file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"},
]
[[package]]
name = "grpcio"
version = "1.53.0"
......@@ -656,14 +685,14 @@ files = [
[[package]]
name = "mpai-cae-arp"
version = "0.3.0"
version = "0.3.2"
description = "The MPAI CAE-ARP software API"
category = "main"
optional = false
python-versions = ">=3.10,<4.0"
files = [
{file = "mpai_cae_arp-0.3.0-py3-none-any.whl", hash = "sha256:ccb376ffa3a37ec3d058061f078bf07b1bb15f1de98a7de7be91ae60bc8be703"},
{file = "mpai_cae_arp-0.3.0.tar.gz", hash = "sha256:d9970910e507a85232e87648fe3f1e4b07159118d66ca1eb52f0bc402e90f6b4"},
{file = "mpai_cae_arp-0.3.2-py3-none-any.whl", hash = "sha256:468db6aaae5a6d54d1460ede6e9a47686d12ab92680a51e14adf14dcba405aac"},
{file = "mpai_cae_arp-0.3.2.tar.gz", hash = "sha256:e447ee46f6c4ca18dd87e3b94dbf366d3f4852ad5530886fd577df9b8d32b2cd"},
]
[package.dependencies]
......@@ -1033,21 +1062,6 @@ typing-extensions = ">=4.2.0"
dotenv = ["python-dotenv (>=0.10.4)"]
email = ["email-validator (>=1.0.3)"]
[[package]]
name = "pyee"
version = "9.0.4"
description = "A port of node.js's EventEmitter to python."
category = "main"
optional = false
python-versions = "*"
files = [
{file = "pyee-9.0.4-py2.py3-none-any.whl", hash = "sha256:9f066570130c554e9cc12de5a9d86f57c7ee47fece163bbdaa3e9c933cfbdfa5"},
{file = "pyee-9.0.4.tar.gz", hash = "sha256:2770c4928abc721f46b705e6a72b0c59480c4a69c9a83ca0b00bb994f1ea4b32"},
]
[package.dependencies]
typing-extensions = "*"
[[package]]
name = "pygments"
version = "2.15.0"
......@@ -1122,22 +1136,6 @@ files = [
[package.dependencies]
six = ">=1.5"
[[package]]
name = "python-ffmpeg"
version = "2.0.4"
description = "A python binding for FFmpeg which provides sync and async APIs"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "python-ffmpeg-2.0.4.tar.gz", hash = "sha256:4d5275a58da2a82fb550acbf496ff86e76d677e06b781baeefbe05a4cd47ab50"},
{file = "python_ffmpeg-2.0.4-py3-none-any.whl", hash = "sha256:89aa9a0a6232a4fda58a9153af7d6c73b4d80d52b08dcde2b3c4ddcceb392b95"},
]
[package.dependencies]
pyee = "*"
typing-extensions = "*"
[[package]]
name = "pytz"
version = "2023.3"
......@@ -1402,7 +1400,6 @@ files = [
{file = "soundfile-0.12.1-py2.py3-none-any.whl", hash = "sha256:828a79c2e75abab5359f780c81dccd4953c45a2c4cd4f05ba3e233ddf984b882"},
{file = "soundfile-0.12.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:d922be1563ce17a69582a352a86f28ed8c9f6a8bc951df63476ffc310c064bfa"},
{file = "soundfile-0.12.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:bceaab5c4febb11ea0554566784bcf4bc2e3977b53946dda2b12804b4fe524a8"},
{file = "soundfile-0.12.1-py2.py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:2dc3685bed7187c072a46ab4ffddd38cef7de9ae5eb05c03df2ad569cf4dacbc"},
{file = "soundfile-0.12.1-py2.py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:074247b771a181859d2bc1f98b5ebf6d5153d2c397b86ee9e29ba602a8dfe2a6"},
{file = "soundfile-0.12.1-py2.py3-none-win32.whl", hash = "sha256:59dfd88c79b48f441bbf6994142a19ab1de3b9bb7c12863402c2bc621e49091a"},
{file = "soundfile-0.12.1-py2.py3-none-win_amd64.whl", hash = "sha256:0d86924c00b62552b650ddd28af426e3ff2d4dc2e9047dae5b3d8452e0a49a77"},
......@@ -1658,4 +1655,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
content-hash = "ca35a4cda1d1842a6fae117b5805bf43e963330173bbd66780c666b4ae903aba"
content-hash = "d870dea06ffbf0b78f7f22a55affd36a16441d79e12a0c390f00384f4b53a903"
pyproject.toml
View file @ dbb29744
......@@ -14,7 +14,7 @@ pandas = "^2.0.0"
scikit-learn = "^1.2.2"
grpcio-tools = "^1.53.0"
mpai-cae-arp = "^0.3.0"
python-ffmpeg = "^2.0.4"
ffmpeg-python = "^0.2.0"
[tool.poetry.group.docs.dependencies]
......
src/classifier.py 0 → 100644
View file @ dbb29744
from pandas import DataFrame
from mpai_cae_arp.audio import AudioWave
from mpai_cae_arp.types.irregularity import Irregularity, IrregularityProperties
from ml.classification import load_model, ClassificationResult
def classification_to_irreg_properties(classification: ClassificationResult) -> IrregularityProperties:
return IrregularityProperties(
reading_equalisation=classification.reading_equalization,
reading_speed=classification.reading_speed,
writing_speed=classification.writing_speed,
writing_equalisation=classification.writing_equalization,
)
def classify(audio_blocks: DataFrame) -> list[IrregularityProperties]:
audio_blocks_classification = []
# classify the audioBlocks
eq_classifier = load_model("pretto_and_berio_nono_classifier")
prediction = eq_classifier.predict(audio_blocks)
for i in range(len(prediction)):
classification = classification_to_irreg_properties(prediction.iloc[i].classification)
audio_blocks_classification.append(classification)
return audio_blocks_classification
def extract_features(audio_blocks: list[Irregularity]) -> DataFrame:
features = {f'mfcc{i}': [] for i in range(1, 14)}
for audio_block in audio_blocks:
audio = AudioWave.from_file(audio_block.audio_block_URI)
audio_mfcc = audio.get_mfcc()
for i, key in enumerate(features.keys()):
features[key].append(audio_mfcc[i])
return DataFrame(features)
src/cli.py
View file @ dbb29744
......@@ -3,8 +3,9 @@ import os
import sys
from rich.console import Console
import segment_finder as sf
import classifier as cl
from mpai_cae_arp.types.irregularity import IrregularityFile
from mpai_cae_arp.types.irregularity import IrregularityFile, Source
from mpai_cae_arp.files import File, FileType
from mpai_cae_arp.io import prettify, Style
......@@ -42,7 +43,10 @@ def main() -> None:
"Working directory or files name not specified!",
"Try -h/--help to know more about Audio Analyser usage"), console)
os.makedirs(os.path.join(working_directory, "temp", files_name))
try:
os.makedirs(os.path.join(working_directory, "temp", files_name))
except:
exit_with_error("Unable to create temporary directory, output path already exists", console)
with console.status("[purple]Reading input files", spinner="dots"):
audio_src = os.path.join(working_directory, "PreservationAudioFile", f"{files_name}.wav")
......@@ -91,12 +95,22 @@ def main() -> None:
with console.status("[cyan]Extracting audio irregularities", spinner="dots"):
irreg2= sf.extract_audio_irregularities(audio_src, irreg2, working_directory + "/temp/" + files_name)
File(f"{working_directory}/temp/{files_name}/IrregularityFile2.json", FileType.JSON).write_content(irreg2.to_json())
console.log("[green]Audio irregularities extracted")
# classify audio irregularities
with console.status("[cyan bold]Classifying audio irregularities", spinner="monkey"):
sf.classify_audio_irregularities(working_directory)
irregularities_features = cl.extract_features(irreg2.irregularities)
console.log("[green]Audio irregularities features extracted")
classification_results = cl.classify(irregularities_features)
console.log("[green]Audio irregularities classified")
with console.status("[purple]Updating irregularity file 2", spinner="dots"):
for irreg, classification_result in zip(irreg2.irregularities, classification_results):
if irreg.source == Source.AUDIO:
irreg.irregularity_type = classification_result.get_irregularity_type()
irreg.irregularity_properties = classification_result if classification_result.get_irregularity_type() is not None else None
File(f"{working_directory}/temp/{files_name}/IrregularityFile2.json", FileType.JSON).write_content(irreg2.to_json())
console.print("[green bold]Success! :tada:")
quit(os.EX_OK)
......
src/client.py 0 → 100644
View file @ dbb29744
from rich.console import Console
from rich.markdown import Markdown
import os
import grpc
from mpai_cae_arp.network import arp_pb2
from mpai_cae_arp.network import arp_pb2_grpc
channels = {
"AudioAnalyser": grpc.insecure_channel("[::]:50051/audio-analyser"),
"VideoAnalyser": grpc.insecure_channel("[::]:50051/video-analyser"),
"TapeIrregularityClassifier": grpc.insecure_channel("[::]:50051/tape-irregularity-classifier"),
"TapeAudioRestoration": grpc.insecure_channel("[::]:50051/tape-audio-restoration"),
"Packager": grpc.insecure_channel("[::]:50051/packager"),
}
def run(console: Console):
with grpc.insecure_channel("[::]:50051") as channel:
audio_analyser = arp_pb2_grpc.AIMStub(channel)
request = arp_pb2.InfoRequest()
response = audio_analyser.getInfo(request)
console.print("[bold]{}[/], v{}".format(response.title, response.version))
console.print(Markdown(response.description))
request = arp_pb2.JobRequest(
working_dir="../data",
files_name="BERIO100",
index=1,
)
with console.status("[bold]Computing...", spinner="bouncingBall"):
for result in audio_analyser.work(request):
if result.status == "error":
console.print("[bold red]Error![/] :boom:")
console.print(f"[italic red]{result.message}")
for channel in channels.values():
channel.close()
exit(os.EX_SOFTWARE)
console.print(result.message)
console.print("[bold green]Success![/] :tada:")
if __name__ == '__main__':
console = Console()
run(console)
\ No newline at end of file
src/segment_finder.py
View file @ dbb29744
from enum import Enum
import os
import tempfile
from uuid import uuid4
import numpy as np
from ffmpeg import FFmpeg
import ffmpeg
import scipy
from mpai_cae_arp.audio import AudioWave, Noise
from mpai_cae_arp.files import File, FileType
from mpai_cae_arp.types.irregularity import Irregularity, IrregularityFile, Source
from mpai_cae_arp.time import frames_to_seconds, seconds_to_frames, seconds_to_string, time_to_seconds
TMP_CHANNELS_MAP = os.path.join(tempfile.gettempdir(), "mpai", "channels_map.json")
TMP_FOLDER = os.path.join(tempfile.gettempdir(), "mpai")
os.makedirs(TMP_FOLDER, exist_ok=True)
TMP_CHANNELS_MAP = os.path.join(TMP_FOLDER, "channels_map.json")
def calculate_offset(audio: AudioWave, video: AudioWave) -> float:
def calculate_offset(audio: AudioWave, video: AudioWave, interval: int = 10) -> int:
"""
Calculates the offset between two audio files based on their cross-correlation.
......@@ -22,30 +26,45 @@ def calculate_offset(audio: AudioWave, video: AudioWave) -> float:
The audio file to be used as reference.
video : AudioWave
The audio file to be used as target.
interval : int, optional
The interval in seconds to be used for the cross-correlation, by default 10
Returns
-------
float
int
The offset in milliseconds.
"""
audio = audio.get_channel(0)[(audio.samplerate*15):(audio.samplerate*(15+interval))]
video = video.get_channel(0)[(audio.samplerate*15):(video.samplerate*(15+interval))]
resampled_audio = audio.array[::4]
resampled_video = video.array[::4]
corr = scipy.signal.correlate(resampled_audio, resampled_video, mode="full", method="auto")
offset = np.argmax(corr) - len(resampled_audio)
offset_ms = offset / (audio.samplerate / 4) * 1000
corr = np.correlate(audio.array, video.array, mode="full")
lags = np.arange(-len(audio.array) + 1, len(video.array))
lag_idx = np.argmax(np.abs(corr))
return round(offset_ms)
return lags[lag_idx] / audio.samplerate
class BitDepth(Enum):
PCM_S8 = "pcm_s8"
PCM_S16LE = "pcm_s16le"
PCM_S24LE = "pcm_s24le"
PCM_S32LE = "pcm_s32le"
def get_audio_from_video(video_src: str, samplerate: int, bit_depth: BitDepth) -> AudioWave:
def get_audio_from_video(video_src: str):
# calculate offset with ffmpeg
# ffmpeg -i video.mov -acodec pcm_s16le -ac 2 audio.wav
ffmpeg = (
FFmpeg()
.input(video_src)
.output(
"out.wav",
{""}
)
)
extracted_audio_path = os.path.join(TMP_FOLDER, 'audio.wav')
in_file = ffmpeg.input(video_src)
out_file = ffmpeg.output(in_file.audio, extracted_audio_path, ac=2, ar=samplerate, acodec=bit_depth.value)
ffmpeg.run(out_file, quiet=True, overwrite_output=True)
rate, data = scipy.io.wavfile.read(extracted_audio_path)
return AudioWave(data, 24, 2, rate)
def get_irregularities_from_audio(audio_src: AudioWave) -> list[Irregularity]:
......@@ -85,10 +104,9 @@ def get_irregularities_from_audio(audio_src: AudioWave) -> list[Irregularity]:
def create_irreg_file(audio_src: str, video_src: str) -> IrregularityFile:
audio = AudioWave.from_file(audio_src, bufferize=True)
video = get_audio_from_video(video_src, audio.samplerate, BitDepth.PCM_S24LE)
offset = calculate_offset(audio, video_src)
offset = calculate_offset(audio, video)
irregularities = get_irregularities_from_audio(audio)
irregularities.sort(key=lambda x: time_to_seconds(x.time_label))
......@@ -146,3 +164,19 @@ def extract_audio_irregularities(
os.remove(TMP_CHANNELS_MAP)
return irreg_file
if __name__ == "__main__":
from rich.console import Console
console = Console()
with console.status("Reading PreservationAudioFile", spinner="dots"):
audio = AudioWave.from_file("../data/PreservationAudioFile/BERIO100.wav", bufferize=True)
with console.status("Extracting audio from PreservationAudioVisualFile", spinner="dots"):
video = get_audio_from_video("../data/PreservationAudioVisualFile/BERIO100.mov", audio.samplerate, BitDepth.PCM_S24LE)
with console.status("Calculating offset", spinner="dots"):
offset = calculate_offset(audio, video)
print(offset)
src/server.py
View file @ dbb29744
import os
from concurrent import futures
from typing import Any, Callable
import grpc
from grpc import StatusCode
from rich.console import Console
from mpai_cae_arp.files import File, FileType
from mpai_cae_arp.types.irregularity import IrregularityFile
import grpc
from mpai_cae_arp.types.irregularity import IrregularityFile, Source
from mpai_cae_arp.network import arp_pb2_grpc as arp_pb2_grpc
from mpai_cae_arp.network.arp_pb2 import (
ComputationRequest,
ComputationResult,
JobRequest,
JobResponse,
Contact,
Info,
InfoResponse,
License,
)
import time
import segment_finder as sf
import classifier as cl
info = File('config/server.yaml', FileType.YAML).get_content()
class AudioAnalyserServicer(arp_pb2_grpc.AudioAnalyserServicer):
def try_or_error_response(
context,
on_success_message: str,
on_error_message: str,
func: Callable,
args,
on_success_status: StatusCode = StatusCode.OK,
on_error_status: StatusCode = StatusCode.INTERNAL,
) -> tuple[JobResponse, Any]:
try:
result = func(*args)
context.set_code(on_success_status)
context.set_details(on_success_message)
return JobResponse(status="success", message=on_success_message), result
except:
context.set_code(on_error_status)
context.set_details(on_error_message)
return JobResponse(status="error", message=on_error_message), None
def error_response(context, status, message):
context.set_code(status)
context.set_details(message)
return JobResponse(status="error", message=message)
class AudioAnalyserServicer(arp_pb2_grpc.AIMServicer):
def __init__(self, console: Console):
self.console = console
def getAimInfo(self, request, context) -> Info:
def getInfo(self, request, context) -> InfoResponse:
self.console.log('Received request for AIM info')
return Info(
context.set_code(StatusCode.OK)
context.set_details('Success')
return InfoResponse(
title=info['title'],
description=info['description'],
version=info['version'],
......@@ -37,23 +73,102 @@ class AudioAnalyserServicer(arp_pb2_grpc.AudioAnalyserServicer):
)
)
def analyse(self, request: ComputationRequest, context):
def work(self, request: JobRequest, context):
self.console.log('Received request for computation')
self.console.log(request)
working_dir: str = request.working_dir
files_name: str = request.files_name
index: int = request.index
self.console.log('Received request for computation')
for x in range(10):
yield ComputationResult(
success=True,
message=f'Processing {x * 10}%'
audio_src = os.path.join(working_dir, "PreservationAudioFile", f"{files_name}.wav")
video_src = os.path.join(working_dir, "PreservationAudioVisualFile", f"{files_name}.mov")
temp_dir = os.path.join(working_dir, "temp", files_name)
audio_irreg_1 = os.path.join(temp_dir, "AudioAnalyser_IrregularityFileOutput_1.json")
audio_irreg_2 = os.path.join(temp_dir, "AudioAnalyser_IrregularityFileOutput_2.json")
video_irreg_1 = os.path.join(temp_dir, "VideoAnalyser_IrregularityFileOutput_1.json")
if index == 1:
response, _ = try_or_error_response(
context,
func=os.makedirs,
on_success_message="Folders created successfully",
on_error_message="Unable to create temporary directory, output path already exists",
on_error_status=StatusCode.ALREADY_EXISTS,
args=temp_dir
)
time.sleep(2)
yield response
response, irreg1 = try_or_error_response(
context,
func=sf.create_irreg_file,
args=(audio_src, video_src),
on_success_message=f"Found irregularities in Audio source",
on_error_message="Failed to create irregularity file 1",
)
yield response
try:
File(audio_irreg_1, FileType.JSON).write_content(irreg1.to_json())
context.set_code(StatusCode.OK)
yield JobResponse(status="success", message="Irregularity file 1 saved to disk")
except:
yield error_response(context, StatusCode.INTERNAL, "Failed to save irregularity file 1")
if index == 2:
response, irreg2 = try_or_error_response(
context,
func=sf.merge_irreg_files,
args=(irreg1, IrregularityFile.from_json(video_irreg_1)),
on_success_message="Irregularity files merged successfully",
on_error_message="Failed to merge irregularity files",
)
yield response
response, irreg2 = try_or_error_response(
context,
func=sf.extract_audio_irregularities,
args=(audio_src, irreg2, temp_dir),
on_success_message="Audio irregularities extracted",
on_error_message="Failed to extract audio irregularities",
)
yield response
response, irregularities_features = try_or_error_response(
context,
func=cl.extract_features,
args=irreg2.irregularities,
on_success_message="Audio irregularities features extracted",
on_error_message="Failed to extract audio irregularities features",
)
yield response
response, classification_results = try_or_error_response(
context,
func=cl.classify,
args=irregularities_features,
on_success_message="Audio irregularities classified",
on_error_message="Failed to classify audio irregularities",
)
yield response
for irreg, classification_result in zip(irreg2.irregularities, classification_results):
if irreg.source == Source.AUDIO:
irreg.irregularity_type = classification_result.get_irregularity_type()
irreg.irregularity_properties = classification_result if classification_result.get_irregularity_type() is not None else None
File(audio_irreg_2, FileType.JSON).write_content(irreg2.to_json())
yield JobResponse(status="success", message="Irregularity file 2 created")
def serve(console):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
arp_pb2_grpc.add_AudioAnalyserServicer_to_server(AudioAnalyserServicer(console), server)
arp_pb2_grpc.add_AIMServicer_to_server(AudioAnalyserServicer(console), server)
server.add_insecure_port('[::]:50051')
server.start()
server.wait_for_termination()
......@@ -62,4 +177,4 @@ def serve(console):
if __name__ == '__main__':
console = Console()
console.print('Server started at localhost:50051 :satellite:')
serve(console)
\ No newline at end of file
serve(console)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment