You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by cd...@apache.org on 2026/07/01 12:38:59 UTC

(systemds) branch main updated: [SYSTEMDS-3835] Fine grained window size parameters

This is an automated email from the ASF dual-hosted git repository.

christinadionysio pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/main by this push:
     new 4ce34d6ba0 [SYSTEMDS-3835] Fine grained window size parameters
4ce34d6ba0 is described below

commit 4ce34d6ba0f4ce5bb8e9e27102d3683763deec0a
Author: Christina Dionysio <di...@tu-berlin.de>
AuthorDate: Wed Jul 1 14:38:54 2026 +0200

    [SYSTEMDS-3835] Fine grained window size parameters
    
    This patch adds a mechanism to evaluate more fine grained context windows in the unimodal optimizer.
---
 .../systemds/scuro/dataloader/timeseries_loader.py | 21 ++++--
 .../systemds/scuro/dataloader/video_loader.py      |  4 ++
 .../scuro/drsearch/hyperparameter_tuner.py         |  2 +
 .../systemds/scuro/drsearch/node_executor.py       | 53 +++++++++++++--
 .../systemds/scuro/drsearch/operator_registry.py   | 78 +++++++++++++++++++++-
 .../systemds/scuro/drsearch/representation_dag.py  |  1 +
 src/main/python/systemds/scuro/drsearch/task.py    |  4 +-
 .../systemds/scuro/drsearch/unimodal_optimizer.py  | 67 +++++++++++++++----
 src/main/python/systemds/scuro/modality/type.py    | 22 ++++++
 .../python/systemds/scuro/representations/lstm.py  |  2 +-
 .../scuro/representations/mel_spectrogram.py       |  9 +++
 .../python/systemds/scuro/representations/mfcc.py  | 41 ++++++++----
 .../scuro/representations/mlp_averaging.py         | 21 ++----
 .../representations/multimodal_attention_fusion.py |  7 ++
 .../systemds/scuro/representations/spectrogram.py  |  8 +++
 .../representations/timeseries_representations.py  | 59 +++++++++++-----
 .../systemds/scuro/representations/wav2vec.py      |  5 +-
 .../scuro/representations/window_aggregation.py    | 27 +++++++-
 src/main/python/tests/scuro/data_generator.py      |  3 +
 19 files changed, 352 insertions(+), 82 deletions(-)

diff --git a/src/main/python/systemds/scuro/dataloader/timeseries_loader.py b/src/main/python/systemds/scuro/dataloader/timeseries_loader.py
index 6b697e6a7a..8e6c11316b 100644
--- a/src/main/python/systemds/scuro/dataloader/timeseries_loader.py
+++ b/src/main/python/systemds/scuro/dataloader/timeseries_loader.py
@@ -36,6 +36,8 @@ class TimeseriesStats:
     num_signals: int
     output_shape: tuple
     output_shape_is_known: bool
+    avg_length: float
+    sampling_rate: int
 
 
 class TimeseriesLoader(BaseLoader):
@@ -49,15 +51,14 @@ class TimeseriesLoader(BaseLoader):
         sampling_rate: Optional[int] = None,
         normalize: bool = True,
         file_format: str = "npy",
+        modality_type: Optional[ModalityType] = ModalityType.TIMESERIES,
     ):
-        super().__init__(
-            source_path, indices, data_type, chunk_size, ModalityType.TIMESERIES
-        )
+        super().__init__(source_path, indices, data_type, chunk_size, modality_type)
         self.signal_names = signal_names
         self.sampling_rate = sampling_rate
         self.normalize = normalize
         self.file_format = file_format.lower()
-        self.stats = self.get_stats(source_path)
+        self.stats = self.get_stats(source_path, sampling_rate)
         if self.file_format not in ["npy", "mat", "hdf5", "txt"]:
             raise ValueError(f"Unsupported file format: {self.file_format}")
 
@@ -148,17 +149,25 @@ class TimeseriesLoader(BaseLoader):
         data = df[selected].to_numpy(dtype=self._data_type)
         return data
 
-    def get_stats(self, source_path: str):
+    def get_stats(self, source_path: str, sampling_rate: int):
         self.file_sanity_check(source_path)
         max_length = 0
         num_instances = 0
         num_signals = 0
+        avg_length = 0
         for file_name in self.indices:
             file = source_path + file_name + "." + self.file_format
             data = self._load_data(file)
             max_length = max(max_length, data.shape[0])
+            avg_length += data.shape[0]
             num_instances += 1
             num_signals = max(num_signals, data.shape[1])
         return TimeseriesStats(
-            max_length, num_instances, num_signals, (max_length,), True
+            max_length,
+            num_instances,
+            num_signals,
+            (max_length,),
+            True,
+            avg_length / num_instances,
+            sampling_rate,
         )
diff --git a/src/main/python/systemds/scuro/dataloader/video_loader.py b/src/main/python/systemds/scuro/dataloader/video_loader.py
index b35a22a8b6..bf7bdd846c 100644
--- a/src/main/python/systemds/scuro/dataloader/video_loader.py
+++ b/src/main/python/systemds/scuro/dataloader/video_loader.py
@@ -33,6 +33,7 @@ from systemds.scuro.modality.type import ModalityType
 class VideoStats:
     fps: int
     max_length: int
+    avg_length: float
     max_width: int
     max_height: int
     max_channels: int
@@ -117,6 +118,7 @@ class VideoLoader(BaseLoader):
         max_height = 0
         max_num_channels = 0
         num_instances = 0
+        avg_length = 0
         for file in os.listdir(source_path):
             file_name = file.split(".")[0]
             if file_name not in self.indices:
@@ -129,6 +131,7 @@ class VideoLoader(BaseLoader):
             height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
             num_channels = 3
             max_length = max(max_length, length)
+            avg_length += length
             max_width = max(max_width, width)
             max_height = max(max_height, height)
             max_num_channels = max(max_num_channels, num_channels)
@@ -142,6 +145,7 @@ class VideoLoader(BaseLoader):
         return VideoStats(
             fps,
             max_length,
+            avg_length / num_instances,
             max_width,
             max_height,
             max_num_channels,
diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py
index 0305f613b6..4f04bffcbe 100644
--- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py
+++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py
@@ -233,6 +233,8 @@ class HyperparamResults:
     def add_result(self, results):
         # TODO: Check if order of best results matters (deterministic)
         for result in results:
+            if result is None:
+                continue
             if result.mm_opt:
                 self.results[result.task_name]["mm_results"].append(result)
             else:
diff --git a/src/main/python/systemds/scuro/drsearch/node_executor.py b/src/main/python/systemds/scuro/drsearch/node_executor.py
index 4b9b2acc65..a6a7ffe2ca 100644
--- a/src/main/python/systemds/scuro/drsearch/node_executor.py
+++ b/src/main/python/systemds/scuro/drsearch/node_executor.py
@@ -234,6 +234,7 @@ def _execute_multiple_reps_for_leaf_dependencies(
 
 
 def _execute_node_worker(node, input_mods, task, rep_cache, gpu_id):
+    start_time = time.perf_counter()
     if gpu_id is not None:
         device = torch.device(f"cuda:{gpu_id}")
         torch.cuda.set_device(device)
@@ -287,23 +288,28 @@ def _execute_node_worker(node, input_mods, task, rep_cache, gpu_id):
         )
     else:
         result = _run_node_op()
-
+    end_time = time.perf_counter()
+    pid = os.getpid()
     return {
         "result": result,
         "peak_bytes": peak_delta_bytes,
         "peak_abs_rss_bytes": peak_abs_rss,
         "gpu_peak_bytes": gpu_peak_bytes,
         "operation_name": operation_name,
+        "start_time": start_time,
+        "end_time": end_time,
+        "pid": pid,
     }
 
 
 def _execute_task_worker(
     task_node_id: str,
     task: Any,
-    data: Any,
+    modality: Any,
     gpu_id: Optional[int],
+    aggregation: AggregatedRepresentation = None,
 ) -> Dict[str, Any]:
-
+    start_time = time.perf_counter()
     if DEBUG:
         print(f"Executing task {task_node_id} on GPU {gpu_id}")
     if gpu_id is not None:
@@ -316,6 +322,14 @@ def _execute_task_worker(
 
     def _run_task():
         start = time.perf_counter()
+        if aggregation is not None:
+            data = (
+                aggregation.operation(params=aggregation.parameters)
+                .transform(modality)
+                .data
+            )
+        else:
+            data = modality.data
         scores = task.run(data)
         end = time.perf_counter()
         return scores, end - start
@@ -336,12 +350,16 @@ def _execute_task_worker(
         )
     else:
         result = _run_task()
-
+    end_time = time.perf_counter()
+    pid = os.getpid()
     return {
         "scores": result[0],
         "task_time": result[1],
         "peak_bytes": peak_delta_bytes,
         "gpu_peak_bytes": gpu_peak_bytes,
+        "start_time": start_time,
+        "end_time": end_time,
+        "pid": pid,
     }
 
 
@@ -385,6 +403,9 @@ class NodeExecutor:
             checkpoint_every=1,
             resume=False,
         )
+        self.statistics = {}
+        self.statistics["worker_stats"] = {}
+        self.statistics["node_stats"] = {}
 
     def _shm_names_for_submit(
         self, parent_node_ids: List[str], payload_data: Any
@@ -451,6 +472,10 @@ class NodeExecutor:
                     ]
 
                 if self._is_task_node(node):
+                    # potentially batch task nodes and then execute them together
+                    # by the the same task type (index, gpu vs cpu)
+                    # either enough nodes to batch or enough time to batch whatever happens first
+
                     task_result = ResultEntry(
                         dag=self._get_dag_from_node_ids(node_id),
                         representation_time=parent_results[0].transform_time,
@@ -458,17 +483,20 @@ class NodeExecutor:
                     task_results[node_id] = task_result
                     task_idx = int(node.parameters.get("_task_idx", 0))
                     payload_data = (
-                        self.modalities[0].data
+                        self.modalities[0]
                         if parent_results is None
-                        else parent_results[0].data
+                        else parent_results[0]
                     )
                     retained = self._retain_for_submit(parent_node_ids, payload_data)
+                    aggregation = node.aggregation
+
                     future = executor.submit(
                         _execute_task_worker,
                         node_id,
                         self.tasks[task_idx],
                         payload_data,
                         gpu_id,
+                        aggregation,
                     )
                 else:
                     payload_data = (
@@ -538,6 +566,14 @@ class NodeExecutor:
                         peak_bytes = result["peak_bytes"]
                         gpu_peak_bytes = result["gpu_peak_bytes"]
                         node = self.scheduler.mapping[node_id]
+                        self.statistics["worker_stats"][result["pid"]] = {
+                            "start_time": result["start_time"],
+                            "end_time": result["end_time"],
+                        }
+                        self.statistics["node_stats"][node_id] = {
+                            "start_time": result["start_time"],
+                            "end_time": result["end_time"],
+                        }
                         if self._is_task_node(node):
                             task_results[node_id].task_time = result["task_time"]
                             task_results[node_id].train_score = result["scores"][
@@ -594,7 +630,10 @@ class NodeExecutor:
 
         self.result_cache.cleanup_all()
         self._cleanup_leaf_shared_memory()
-        return {"task_results": list(task_results.values())}
+        return {
+            "task_results": list(task_results.values()),
+            "statistics": self.statistics,
+        }
 
     def _handle_modality_result(
         self,
diff --git a/src/main/python/systemds/scuro/drsearch/operator_registry.py b/src/main/python/systemds/scuro/drsearch/operator_registry.py
index bc9bd406a8..4c5641fdd9 100644
--- a/src/main/python/systemds/scuro/drsearch/operator_registry.py
+++ b/src/main/python/systemds/scuro/drsearch/operator_registry.py
@@ -19,7 +19,7 @@
 #
 # -------------------------------------------------------------
 from typing import Union, List
-
+import math
 from systemds.scuro.modality.type import ModalityType
 from systemds.scuro.representations.representation import Representation
 
@@ -57,6 +57,24 @@ class Registry:
         else:
             self._representations[modality_type] = [representations]
 
+    def set_context_operators(self, modality_type, context_operators):
+        if isinstance(context_operators, list):
+            self._context_operators[modality_type] = context_operators
+        else:
+            self._context_operators[modality_type] = [context_operators]
+
+    def set_context_representation_operators(
+        self, modality_type, context_representation_operators
+    ):
+        if isinstance(context_representation_operators, list):
+            self._context_representation_operators[modality_type] = (
+                context_representation_operators
+            )
+        else:
+            self._context_representation_operators[modality_type] = [
+                context_representation_operators
+            ]
+
     def add_representation(
         self, representation: Representation, modality: ModalityType
     ):
@@ -139,6 +157,64 @@ class Registry:
     def get_context_representations(self, modality_type):
         return self._context_representation_operators[modality_type]
 
+    def get_context_lenghts_for_modality(self, modality_type, statistics):
+        if modality_type == ModalityType.AUDIO:
+            window_lengths = [
+                0.010,
+                0.020,
+                0.050,
+                0.075,
+                0.100,
+                0.5,
+                1,
+                2,
+                5,
+                10,
+            ]  # seconds
+
+        if (
+            modality_type == ModalityType.TIMESERIES
+            or modality_type == ModalityType.PHYSIOLOGICAL
+        ):
+            window_lengths = [0.05, 0.1, 0.5, 0.75, 1, 2, 5, 10, 30, 60]  # seconds
+
+        if modality_type == ModalityType.VIDEO:
+            window_lengths = [0.5, 1, 2, 5, 10]  # seconds
+
+        if (
+            modality_type == ModalityType.AUDIO
+            or modality_type == ModalityType.TIMESERIES
+            or modality_type == ModalityType.PHYSIOLOGICAL
+        ):
+            max_length_in_seconds = statistics.max_length / statistics.sampling_rate
+            window_lengths = [
+                length for length in window_lengths if length <= max_length_in_seconds
+            ]
+
+            effective_window_lenghts = [
+                statistics.sampling_rate * length for length in window_lengths
+            ]
+            num_windows = [
+                math.ceil(statistics.avg_length / length)
+                for length in effective_window_lenghts
+            ]
+            return effective_window_lenghts, num_windows
+
+        if modality_type == ModalityType.VIDEO:
+            max_length_in_seconds = statistics.max_length / statistics.fps
+            window_lengths = [
+                length for length in window_lengths if length <= max_length_in_seconds
+            ]
+
+            effective_window_lenghts = [
+                statistics.fps * length for length in window_lengths
+            ]
+            num_windows = [
+                math.ceil(statistics.avg_length / length)
+                for length in effective_window_lenghts
+            ]
+            return effective_window_lenghts, num_windows
+
 
 def register_representation(modalities: Union[ModalityType, List[ModalityType]]):
     """
diff --git a/src/main/python/systemds/scuro/drsearch/representation_dag.py b/src/main/python/systemds/scuro/drsearch/representation_dag.py
index b1d5835ad3..a19c44396f 100644
--- a/src/main/python/systemds/scuro/drsearch/representation_dag.py
+++ b/src/main/python/systemds/scuro/drsearch/representation_dag.py
@@ -80,6 +80,7 @@ class RepresentationNode:
     representation_index: int = None
     parameters: Dict[str, Any] = field(default_factory=dict)
     gpu_id: int = None
+    aggregation: AggregatedRepresentation = None
 
 
 @dataclass
diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py
index 404624ec3a..e74e791794 100644
--- a/src/main/python/systemds/scuro/drsearch/task.py
+++ b/src/main/python/systemds/scuro/drsearch/task.py
@@ -259,7 +259,9 @@ class Task:
             val_y = self._gather_by_indices(self.labels, fold_val_indices)
 
             self._run_fold(model, train_X, train_y, val_X, val_y, test_X, test_y)
-
+        if hasattr(model, "clean_up"):
+            model.clean_up()
+            del model
         return [
             self.train_scores.compute_averages(),
             self.val_scores.compute_averages(),
diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py
index 8dc2e1a082..a67cbe1202 100644
--- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py
+++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py
@@ -20,6 +20,8 @@
 # -------------------------------------------------------------
 import copy
 import pickle
+import csv
+from pathlib import Path
 import time
 from concurrent.futures import ProcessPoolExecutor, as_completed
 from dataclasses import dataclass
@@ -39,6 +41,7 @@ from systemds.scuro.representations.aggregated_representation import (
 )
 from systemds.scuro.modality.modality import Modality
 from systemds.scuro.drsearch.operator_registry import Registry
+from systemds.scuro.representations.window_aggregation import WindowAggregation
 from systemds.scuro.utils.checkpointing import CheckpointManager
 from systemds.scuro.drsearch.representation_dag import (
     RepresentationDag,
@@ -63,6 +66,8 @@ class UnimodalOptimizer:
         resume: bool = False,
         max_num_workers: int = -1,
         enable_checkpointing: bool = True,
+        enable_execution_profile: bool = False,
+        execution_profile_path: Optional[str] = None,
     ):
         self.enable_checkpointing = enable_checkpointing
         self.modalities = modalities
@@ -92,7 +97,8 @@ class UnimodalOptimizer:
         self.operator_performance = UnimodalResults(
             modalities, tasks, debug, True, k, self.metric_name
         )
-
+        self.enable_execution_profile = enable_execution_profile
+        self.execution_profile_path = execution_profile_path
         self._tasks_require_same_dims = True
         self.expected_dimensions = tasks[0].expected_dim
 
@@ -242,7 +248,7 @@ class UnimodalOptimizer:
 
         for modality in self.modalities:
             try:
-                local_result = self._process_modality(
+                local_result, execution_time = self._process_modality(
                     modality,
                     (
                         int(
@@ -274,6 +280,7 @@ class UnimodalOptimizer:
                     self.operator_performance.results, {}
                 )
                 raise
+        return execution_time
 
     def _expand_dags_with_task_roots(
         self, dags: List[RepresentationDag]
@@ -281,7 +288,16 @@ class UnimodalOptimizer:
         expanded_dags: List[RepresentationDag] = []
 
         for dag in dags:
-            root_id = dag.root_node_id
+            dag = copy.deepcopy(dag)
+            root_node = dag.get_node_by_id(dag.root_node_id)
+            if root_node and root_node.operation == AggregatedRepresentation:
+                aggregation = root_node
+                dag.nodes = [n for n in dag.nodes if n.node_id != root_node.node_id]
+                root_id = aggregation.inputs[0]
+                dag.root_node_id = root_id
+            else:
+                aggregation = None
+                root_id = dag.root_node_id
             for task_idx, _ in enumerate(self.tasks):
                 task_node_id = f"task_{root_id}_{task_idx}"
 
@@ -294,6 +310,7 @@ class UnimodalOptimizer:
                         "_task_idx": task_idx,
                         "_dag_root_id": root_id,
                     },
+                    aggregation=aggregation,
                 )
 
                 task_root_dag = RepresentationDag(
@@ -326,12 +343,18 @@ class UnimodalOptimizer:
             self.result_path,
             enable_checkpointing=self.enable_checkpointing,
         )
-
+        start_time = time.perf_counter()
         exec_out = node_executor.run()
+        end_time = time.perf_counter()
         task_results = exec_out["task_results"]
 
         for task_result in task_results:
             local_results.add_task_result(task_result, dags)
+        statistics = exec_out["statistics"]
+        for worker_stat in statistics["worker_stats"]:
+            local_results.add_worker_stat(worker_stat, modality.modality_id)
+        for node_stat in statistics["node_stats"]:
+            local_results.add_node_stat(node_stat, modality.modality_id)
 
         if self.save_all_results:
             timestr = time.strftime("%Y%m%d-%H%M%S")
@@ -339,7 +362,7 @@ class UnimodalOptimizer:
             with open(file_name, "wb") as f:
                 pickle.dump(local_results.results, f)
 
-        return local_results
+        return local_results, end_time - start_time
 
     def _build_execution_dags_for_modality(
         self, modality: Modality, skip_remaining: int = 0
@@ -354,6 +377,7 @@ class UnimodalOptimizer:
         if (
             modality.modality_type == ModalityType.TIMESERIES
             or modality.modality_type == ModalityType.AUDIO
+            or modality.modality_type == ModalityType.PHYSIOLOGICAL
         ):
             dags.extend(
                 self.temporal_context_operators(
@@ -509,7 +533,6 @@ class UnimodalOptimizer:
 
     def _aggregation_needed(self, dag: RepresentationDag) -> bool:
         input_stats = {}
-        # TODO: adapt this to the fusion of multiple modalities, list of input stats needed
         for modality in self.modalities:
             if modality.modality_id == dag.nodes[0].modality_id:
                 input_stats[dag.nodes[0].node_id] = modality.stats
@@ -584,16 +607,26 @@ class UnimodalOptimizer:
             modality.modality_type
         )
         context_operators = self._get_context_operators(modality.modality_type)
-
+        window_lengths, num_windows = (
+            self.operator_registry.get_context_lenghts_for_modality(
+                modality.modality_type, modality.stats
+            )
+        )
         dags = []
         for agg in aggregators:
             for context_operator in context_operators:
-                context_node_id = builder.create_operation_node(
-                    context_operator,
-                    [leaf_id],
-                    context_operator(agg()).get_current_parameters(),
-                )
-                dags.append(builder.build(context_node_id))
+                for window_size, num_window in zip(window_lengths, num_windows):
+                    context_operator_instance = context_operator(agg())
+                    if hasattr(context_operator, "num_windows"):
+                        context_operator_instance.num_windows = num_window
+                    elif hasattr(context_operator_instance, "window_size"):
+                        context_operator_instance.window_size = window_size
+                    context_node_id = builder.create_operation_node(
+                        context_operator,
+                        [leaf_id],
+                        context_operator_instance.get_current_parameters(),
+                    )
+                    dags.append(builder.build(context_node_id))
 
         return dags
 
@@ -619,6 +652,8 @@ class UnimodalResults:
         for modality in self.modality_ids:
             self.results[modality] = {task_name: [] for task_name in self.task_names}
             self.cache[modality] = {task_name: [] for task_name in self.task_names}
+        self.worker_stats = {}
+        self.node_stats = {}
 
     def add_task_result(self, task_result: ResultEntry, dags: List[RepresentationDag]):
         dag_id = task_result.dag.dag_id
@@ -750,6 +785,12 @@ class UnimodalResults:
 
         return results, cache
 
+    def add_worker_stat(self, worker_stats, modality_id):
+        self.worker_stats[modality_id] = worker_stats
+
+    def add_node_stat(self, node_stats, modality_id):
+        self.node_stats[modality_id] = node_stats
+
 
 def get_dag_by_id(dags: List[RepresentationDag], dag_id: int) -> RepresentationDag:
     for dag in dags:
diff --git a/src/main/python/systemds/scuro/modality/type.py b/src/main/python/systemds/scuro/modality/type.py
index 0493edf5bd..a648044562 100644
--- a/src/main/python/systemds/scuro/modality/type.py
+++ b/src/main/python/systemds/scuro/modality/type.py
@@ -66,6 +66,11 @@ class ModalitySchemas:
         "num_columns": "integer",
     }
 
+    PHYSIOLOGICAL_SCHEMA = {
+        **TEMPORAL_BASE_SCHEMA,
+        "num_columns": "integer",
+    }
+
     _metadata_handlers = {}
 
     @classmethod
@@ -184,6 +189,22 @@ def handle_timeseries_metadata(md, data):
     return md
 
 
+@ModalitySchemas.register_metadata_handler("PHYSIOLOGICAL")
+def handle_physiological_metadata(md, data):
+    new_frequency = calculate_new_frequency(len(data), md["length"], md["frequency"])
+    md.update(
+        {
+            "length": len(data),
+            "num_columns": (
+                data.shape[1] if isinstance(data, np.ndarray) and data.ndim > 1 else 1
+            ),
+            "frequency": new_frequency,
+            "timestamp": create_timestamps(new_frequency, len(data)),
+        }
+    )
+    return md
+
+
 @ModalitySchemas.register_metadata_handler("TEXT")
 def handle_text_metadata(md, data):
     md.update({"length": len(data)})
@@ -205,6 +226,7 @@ class ModalityType(Flag):
         "VIDEO": "create_video_metadata",
         "IMAGE": "create_image_metadata",
         "TIMESERIES": "create_ts_metadata",
+        "PHYSIOLOGICAL": "create_ts_metadata",
         "EMBEDDING": "create_embedding_metadata",
     }
 
diff --git a/src/main/python/systemds/scuro/representations/lstm.py b/src/main/python/systemds/scuro/representations/lstm.py
index 7243b65966..c15776284c 100644
--- a/src/main/python/systemds/scuro/representations/lstm.py
+++ b/src/main/python/systemds/scuro/representations/lstm.py
@@ -25,7 +25,7 @@ import torch
 from torch import nn
 from torch.utils.data import DataLoader, TensorDataset
 from typing import List, Dict, Any
-from systemds.scuro.utils.static_variables import get_device, get_device_for_model
+from systemds.scuro.utils.static_variables import get_device
 import numpy as np
 
 from systemds.scuro.modality.modality import Modality
diff --git a/src/main/python/systemds/scuro/representations/mel_spectrogram.py b/src/main/python/systemds/scuro/representations/mel_spectrogram.py
index 6d37880647..3cf25b44c4 100644
--- a/src/main/python/systemds/scuro/representations/mel_spectrogram.py
+++ b/src/main/python/systemds/scuro/representations/mel_spectrogram.py
@@ -36,6 +36,14 @@ from systemds.scuro.utils.static_variables import (
     PY_LIST_SLOT_BYTES,
 )
 
+import warnings
+
+warnings.filterwarnings(
+    "ignore",
+    message=r"n_fft=\d+ is too (?:small|large) for input signal",
+    module=r"librosa",
+)
+
 
 @register_representation(ModalityType.AUDIO)
 @register_context_representation_operator(ModalityType.AUDIO)
@@ -56,6 +64,7 @@ class MelSpectrogram(UnimodalRepresentation):
         self.n_mels = int(n_mels)
         self.hop_length = int(hop_length)
         self.n_fft = int(n_fft)
+        self.window_size = self.n_fft
 
     def transform(self, modality, aggregation=None):
         transformed_modality = TransformedModality(
diff --git a/src/main/python/systemds/scuro/representations/mfcc.py b/src/main/python/systemds/scuro/representations/mfcc.py
index 406fc6616c..483ae3eef8 100644
--- a/src/main/python/systemds/scuro/representations/mfcc.py
+++ b/src/main/python/systemds/scuro/representations/mfcc.py
@@ -35,43 +35,54 @@ from systemds.scuro.utils.static_variables import (
     PY_LIST_HEADER_BYTES,
     PY_LIST_SLOT_BYTES,
 )
+import warnings
+
+warnings.filterwarnings(
+    "ignore",
+    message=r"n_fft=\d+ is too (?:small|large) for input signal",
+    module=r"librosa",
+)
 
 
 @register_representation(ModalityType.AUDIO)
 @register_context_representation_operator(ModalityType.AUDIO)
 class MFCC(UnimodalRepresentation):
-    def __init__(self, n_mfcc=12, dct_type=2, n_mels=128, hop_length=512, params=None):
+    def __init__(
+        self, n_mfcc=12, dct_type=2, n_mels=128, hop_length=512, n_fft=2048, params=None
+    ):
         parameters = {
             "n_mfcc": [x for x in range(10, 26)],
             "dct_type": [1, 2, 3],
             "hop_length": [256, 512, 1024, 2048],
             "n_mels": [20, 32, 64, 128],
-        }  # TODO
+            "n_fft": [1024, 2048, 4096],
+        }
+
         super().__init__("MFCC", ModalityType.TIMESERIES, parameters, False)
 
-        # Allow construction from a parameter dict (used by optimizer)
         if params is not None:
             n_mfcc = params.get("n_mfcc", n_mfcc)
             dct_type = params.get("dct_type", dct_type)
             n_mels = params.get("n_mels", n_mels)
             hop_length = params.get("hop_length", hop_length)
+            n_fft = params.get("n_fft", n_fft)
 
         self.n_mfcc = int(n_mfcc)
         self.dct_type = int(dct_type)
         self.n_mels = int(n_mels)
         self.hop_length = int(hop_length)
+        self.n_fft = int(n_fft)
+        self.window_size = self.n_fft
 
     def transform(self, modality, aggregation=None):
         transformed_modality = TransformedModality(
             modality, self, self.output_modality_type
         )
         result = []
-
-        for i, sample in enumerate(modality.data):
-            sr = modality.metadata[i]["frequency"]
-            computed_feature = self.compute_feature(sample, sr)
+        sr = modality.metadata[0]["frequency"] if modality.metadata else 22050
+        for sample in modality.data:
+            computed_feature = self.compute_feature(sample, sr=sr)
             result.append(computed_feature)
-
         transformed_modality.data = result
         return transformed_modality
 
@@ -79,20 +90,22 @@ class MFCC(UnimodalRepresentation):
         if sr is None:
             sr = 22050
         mfcc = librosa.feature.mfcc(
-            y=np.array(instance),
+            y=np.asarray(instance, dtype=np.float32),
             sr=sr,
             n_mfcc=self.n_mfcc,
             dct_type=self.dct_type,
             hop_length=self.hop_length,
             n_mels=self.n_mels,
+            n_fft=self.n_fft,
         )
         if mfcc.ndim == 2:
-            mean = np.mean(mfcc, keepdims=True)
-            std = np.std(mfcc, keepdims=True)
+            mean = mfcc.mean(keepdims=True)
+            std = mfcc.std(keepdims=True)
         else:
-            mean = np.mean(mfcc, axis=(1, 2), keepdims=True)
-            std = np.std(mfcc, axis=(1, 2), keepdims=True)
-        mfcc = (mfcc - mean) / np.maximum(std, 1e-8)
+            mean = mfcc.mean(axis=(1, 2), keepdims=True)
+            std = mfcc.std(axis=(1, 2), keepdims=True)
+        mfcc -= mean
+        mfcc /= np.maximum(std, 1e-8)
 
         if instance.ndim == 1:
             return mfcc.T
diff --git a/src/main/python/systemds/scuro/representations/mlp_averaging.py b/src/main/python/systemds/scuro/representations/mlp_averaging.py
index 46fe04899a..8c8d67a06e 100644
--- a/src/main/python/systemds/scuro/representations/mlp_averaging.py
+++ b/src/main/python/systemds/scuro/representations/mlp_averaging.py
@@ -54,12 +54,8 @@ class MLPAveraging(DimensionalityReduction):
             "batch_size": [8, 16, 32, 64, 128],
         }
         super().__init__("MLPAveraging", parameters)
-        if params is not None:
-            self.output_dim = params.get("output_dim", output_dim)
-            self.batch_size = params.get("batch_size", batch_size)
-        else:
-            self.output_dim = output_dim
-            self.batch_size = batch_size
+        self.output_dim = output_dim
+        self.batch_size = batch_size
         self.device = None
         self.data_type = np.float32
         self.gpu_id = None
@@ -74,10 +70,7 @@ class MLPAveraging(DimensionalityReduction):
         self.device = get_device(gpu_id)
 
     def get_output_stats(self, input_stats: RepresentationStats) -> RepresentationStats:
-        if (
-            len(input_stats.output_shape) > 1
-            and np.prod(input_stats.output_shape) > self.output_dim
-        ):
+        if len(input_stats.output_shape) > 1:
             return RepresentationStats(
                 input_stats.num_instances,
                 (self.output_dim,),
@@ -94,13 +87,7 @@ class MLPAveraging(DimensionalityReduction):
             )
         return RepresentationStats(
             input_stats.num_instances,
-            (
-                (
-                    np.prod(input_stats.output_shape)
-                    if np.prod(input_stats.output_shape) < self.output_dim
-                    else self.output_dim
-                ),
-            ),
+            (self.output_dim,),
             output_shape_is_known=input_stats.output_shape_is_known,
         )
 
diff --git a/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py b/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py
index 066f343215..af5a83c7e5 100644
--- a/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py
+++ b/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py
@@ -60,6 +60,13 @@ class AttentionFusion(Fusion):
         self.batch_size = int(batch_size)
         self.num_epochs = int(num_epochs)
         self.learning_rate = float(learning_rate)
+        if params is not None:
+            self.hidden_dim = int(params.get("hidden_dim", self.hidden_dim))
+            self.num_heads = int(params.get("num_heads", self.num_heads))
+            self.dropout = float(params.get("dropout", self.dropout))
+            self.batch_size = int(params.get("batch_size", self.batch_size))
+            self.num_epochs = int(params.get("num_epochs", self.num_epochs))
+            self.learning_rate = float(params.get("learning_rate", self.learning_rate))
 
         self.needs_training = True
         self.needs_alignment = True
diff --git a/src/main/python/systemds/scuro/representations/spectrogram.py b/src/main/python/systemds/scuro/representations/spectrogram.py
index ab0c8a6c64..9d14931ef0 100644
--- a/src/main/python/systemds/scuro/representations/spectrogram.py
+++ b/src/main/python/systemds/scuro/representations/spectrogram.py
@@ -35,6 +35,13 @@ from systemds.scuro.utils.static_variables import (
     PY_LIST_HEADER_BYTES,
     PY_LIST_SLOT_BYTES,
 )
+import warnings
+
+warnings.filterwarnings(
+    "ignore",
+    message=r"n_fft=\d+ is too (?:small|large) for input signal",
+    module=r"librosa",
+)
 
 
 @register_representation(ModalityType.AUDIO)
@@ -45,6 +52,7 @@ class Spectrogram(UnimodalRepresentation):
         super().__init__("Spectrogram", ModalityType.TIMESERIES, parameters, False)
         self.hop_length = int(hop_length)
         self.n_fft = int(n_fft)
+        self.window_size = self.n_fft
 
     def transform(self, modality, aggregation=None):
         transformed_modality = TransformedModality(
diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py
index 26c8c1a8d9..14fcacf724 100644
--- a/src/main/python/systemds/scuro/representations/timeseries_representations.py
+++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py
@@ -51,9 +51,13 @@ class TimeSeriesRepresentation(UnimodalRepresentation):
             feature = self.compute_feature(signal)
             result.append(feature)
 
-        transformed_modality.data = np.vstack(np.array(result)).astype(
-            modality.metadata[0]["data_layout"]["type"]
-        )
+        maxlen = max(r.size for r in result)
+        padded_result = [
+            np.pad(r, (0, maxlen - r.size), mode="constant", constant_values=0.0)
+            for r in result
+        ]
+        dtype = modality.metadata[0]["data_layout"]["type"]
+        transformed_modality.data = np.vstack(np.asarray(padded_result)).astype(dtype)
         return transformed_modality
 
     def get_output_stats(self, input_stats):
@@ -71,8 +75,10 @@ class TimeSeriesRepresentation(UnimodalRepresentation):
         }
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
+@register_context_representation_operator(ModalityType.AUDIO)
 class Mean(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("Mean")
@@ -81,8 +87,9 @@ class Mean(TimeSeriesRepresentation):
         return np.array(np.mean(signal, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 class Min(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("Min")
@@ -91,8 +98,9 @@ class Min(TimeSeriesRepresentation):
         return np.array(np.min(signal, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 class Max(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("Max")
@@ -101,8 +109,9 @@ class Max(TimeSeriesRepresentation):
         return np.array(np.max(signal, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 class Sum(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("Sum")
@@ -111,8 +120,10 @@ class Sum(TimeSeriesRepresentation):
         return np.array(np.sum(signal, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
+@register_context_representation_operator(ModalityType.AUDIO)
 class Std(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("Std")
@@ -121,8 +132,10 @@ class Std(TimeSeriesRepresentation):
         return np.array(np.std(signal, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
+@register_context_representation_operator(ModalityType.AUDIO)
 class Skew(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("Skew")
@@ -131,12 +144,13 @@ class Skew(TimeSeriesRepresentation):
         return np.array(stats.skew(signal, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 class Quantile(TimeSeriesRepresentation):
     def __init__(self, quantile=0.9, params=None):
         super().__init__(
-            "Qunatile", {"quantile": [0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]}
+            "Qunatile", {"quantile": [0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]}
         )
         self.quantile = quantile
 
@@ -144,8 +158,10 @@ class Quantile(TimeSeriesRepresentation):
         return np.array(np.quantile(signal, self.quantile, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
+@register_context_representation_operator(ModalityType.AUDIO)
 class Kurtosis(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("Kurtosis")
@@ -154,8 +170,10 @@ class Kurtosis(TimeSeriesRepresentation):
         return np.array(stats.kurtosis(signal, fisher=True, bias=True, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
+@register_context_representation_operator(ModalityType.AUDIO)
 class RMS(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("RMS")
@@ -164,8 +182,9 @@ class RMS(TimeSeriesRepresentation):
         return np.array(np.sqrt(np.mean(np.square(signal), axis=axis)))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 class ZeroCrossingRate(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("ZeroCrossingRate")
@@ -174,8 +193,9 @@ class ZeroCrossingRate(TimeSeriesRepresentation):
         return np.array(np.sum(np.diff(np.signbit(signal), axis=axis) != 0, axis=axis))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 class ACF(TimeSeriesRepresentation):
     def __init__(self, k=1, params=None):
         super().__init__("ACF", {"k": [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]})
@@ -209,8 +229,9 @@ class ACF(TimeSeriesRepresentation):
         return k_vals.tolist()
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
 @register_context_representation_operator(ModalityType.TIMESERIES)
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 class FrequencyMagnitude(TimeSeriesRepresentation):
     def __init__(self, params=None):
         super().__init__("FrequencyMagnitude")
@@ -219,7 +240,8 @@ class FrequencyMagnitude(TimeSeriesRepresentation):
         return np.array(np.abs(np.fft.rfft(signal, axis=axis)))
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 @register_context_representation_operator(ModalityType.TIMESERIES)
 class SpectralCentroid(TimeSeriesRepresentation):
     def __init__(self, fs=1.0, params=None):
@@ -240,7 +262,8 @@ class SpectralCentroid(TimeSeriesRepresentation):
         return np.array(num / den)
 
 
-@register_representation([ModalityType.TIMESERIES])
+@register_representation([ModalityType.TIMESERIES, ModalityType.PHYSIOLOGICAL])
+@register_context_representation_operator(ModalityType.PHYSIOLOGICAL)
 @register_context_representation_operator(ModalityType.TIMESERIES)
 class BandpowerFFT(TimeSeriesRepresentation):
     def __init__(self, fs=1.0, f1=0.0, f2=0.5, params=None):
diff --git a/src/main/python/systemds/scuro/representations/wav2vec.py b/src/main/python/systemds/scuro/representations/wav2vec.py
index ece034e099..5e03baf8bc 100644
--- a/src/main/python/systemds/scuro/representations/wav2vec.py
+++ b/src/main/python/systemds/scuro/representations/wav2vec.py
@@ -28,10 +28,11 @@ from systemds.scuro.modality.transformed import TransformedModality
 from systemds.scuro.representations.representation import RepresentationStats
 from systemds.scuro.representations.unimodal import UnimodalRepresentation
 from systemds.scuro.drsearch.operator_registry import register_representation
+from systemds.scuro.utils.memory_utility import get_device
 
-import warnings
+from transformers.utils import logging as transformers_logging
 
-warnings.filterwarnings("ignore", message="Some weights of")
+transformers_logging.set_verbosity_error()
 
 
 @register_representation(ModalityType.AUDIO)
diff --git a/src/main/python/systemds/scuro/representations/window_aggregation.py b/src/main/python/systemds/scuro/representations/window_aggregation.py
index 36e12c0cd5..52a1740195 100644
--- a/src/main/python/systemds/scuro/representations/window_aggregation.py
+++ b/src/main/python/systemds/scuro/representations/window_aggregation.py
@@ -155,11 +155,21 @@ class WindowAggregation(Window):
         pad=True,
         params=None,
     ):
+        window_size_set = False
+        if isinstance(aggregation_function, Representation) and hasattr(
+            aggregation_function, "window_size"
+        ):
+            window_size = aggregation_function.window_size
+            window_size_set = True
+
         if params is not None:
             if isinstance(
                 params.get("aggregation_function"), (Aggregation, Representation)
             ):
                 aggregation_function = params["aggregation_function"]
+                if hasattr(aggregation_function, "window_size"):
+                    window_size = aggregation_function.window_size
+                    window_size_set = True
             else:
                 nested_agg = {
                     key[len("aggregation_function_") :]: value
@@ -177,7 +187,8 @@ class WindowAggregation(Window):
                     aggregation_function = params.get(
                         "aggregation_function", aggregation_function
                     )
-            window_size = params["window_size"]
+
+            window_size = params["window_size"] if not window_size_set else window_size
             pad = params.get("pad", True)
         super().__init__("WindowAggregation", aggregation_function)
         self.parameters["window_size"] = (4, 128)
@@ -234,6 +245,7 @@ class WindowAggregation(Window):
 
         effective_seq_len = in_shape[0]
         in_numel = effective_seq_len * self._rest_numel(in_shape)
+        output_bytes = self.estimate_output_memory_bytes(input_stats)
         one_instance_bytes = in_numel * np.dtype(self.data_type).itemsize
         input_bytes = one_instance_bytes * input_stats.num_instances
 
@@ -269,6 +281,7 @@ class WindowAggregation(Window):
                 )
             original_lengths.append(windowed_instance.shape[0])
             windowed_data.append(windowed_instance)
+
         if self.pad and not isinstance(windowed_data, np.ndarray):
             target_length = max(original_lengths)
 
@@ -322,7 +335,17 @@ class WindowAggregation(Window):
             full_result = self.aggregation_function.compute_feature(full_batches)
             if tail.size:
                 tail_result = self.aggregation_function.compute_feature(tail)
-                full_result = np.concatenate([full_result, tail_result[None, :]])
+                if tail_result.shape == full_result.shape[1:]:
+                    tail_row = tail_result
+                else:
+                    tail_row = np.zeros_like(full_result[0])
+                    slices = tuple(
+                        slice(0, min(d, s))
+                        for d, s in zip(tail_row.shape, tail_result.shape)
+                    )
+                    tail_row[slices] = tail_result[slices]
+                full_result = np.concatenate([full_result, tail_row[None, :]])
+
         return full_result
 
     def window_aggregate_nested_level(self, instance, new_length):
diff --git a/src/main/python/tests/scuro/data_generator.py b/src/main/python/tests/scuro/data_generator.py
index a51ea510ea..937fd622d8 100644
--- a/src/main/python/tests/scuro/data_generator.py
+++ b/src/main/python/tests/scuro/data_generator.py
@@ -75,6 +75,7 @@ class TestDataLoader(BaseLoader):
             self.stats = VideoStats(
                 30,
                 max(d.shape[0] for d in data),
+                sum(d.shape[0] for d in data) / len(data),
                 max(d.shape[1] for d in data),
                 max(d.shape[2] for d in data),
                 max(d.shape[3] for d in data),
@@ -88,6 +89,8 @@ class TestDataLoader(BaseLoader):
                 sum(len(d) for d in data) / len(data),
                 (max(len(d) for d in data),),
                 True,
+                sum(len(d) for d in data) / len(data),
+                16000,
             )
         elif modality_type == ModalityType.IMAGE:
             self.stats = ImageStats(