You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by cd...@apache.org on 2025/10/16 18:18:56 UTC

(systemds) branch main updated: [SYSTEMDS-3887] Rework Scuro Optimizers

This is an automated email from the ASF dual-hosted git repository.

cdionysio pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/main by this push:
     new c8d5460c4c [SYSTEMDS-3887] Rework Scuro Optimizers
c8d5460c4c is described below

commit c8d5460c4cbc5211a306fe45b6932db38c5cb972
Author: Christina Dionysio <di...@tu-berlin.de>
AuthorDate: Thu Oct 16 20:18:52 2025 +0200

    [SYSTEMDS-3887] Rework Scuro Optimizers
    
    This patch adds a reworked version of the unimodal and multimodal optimizer and the hyperparameter tuner.
---
 .github/workflows/python.yml                       |  15 +-
 src/main/python/systemds/scuro/__init__.py         |  14 +-
 .../systemds/scuro/dataloader/base_loader.py       |   4 +-
 .../python/systemds/scuro/drsearch/dr_search.py    | 167 -------
 .../systemds/scuro/drsearch/fusion_optimizer.py    | 295 -----------
 .../scuro/drsearch/hyperparameter_tuner.py         | 333 ++++++++++---
 .../scuro/drsearch/multimodal_optimizer.py         | 554 +++++++++------------
 .../systemds/scuro/drsearch/operator_registry.py   |   6 +
 .../scuro/drsearch/representation_cache.py         | 128 -----
 .../systemds/scuro/drsearch/representation_dag.py  | 232 +++++++++
 .../representation_dag_visualizer.py}              |  43 +-
 .../systemds/scuro/drsearch/unimodal_optimizer.py  | 340 +++++++------
 .../drsearch/unimodal_representation_optimizer.py  | 271 ----------
 .../systemds/scuro/modality/joined_transformed.py  |   2 +-
 .../python/systemds/scuro/modality/modality.py     |  12 +-
 .../python/systemds/scuro/modality/transformed.py  |  24 +-
 src/main/python/systemds/scuro/modality/type.py    |  11 +
 .../systemds/scuro/modality/unimodal_modality.py   |   9 +-
 .../systemds/scuro/representations/aggregate.py    |  30 +-
 .../representations/aggregated_representation.py   |  10 +-
 .../python/systemds/scuro/representations/bow.py   |   4 +-
 .../systemds/scuro/representations/fusion.py       |  72 ++-
 .../python/systemds/scuro/representations/lstm.py  | 230 +++++++--
 .../scuro/representations/mel_spectrogram.py       |   6 +-
 .../python/systemds/scuro/representations/mfcc.py  |   8 +-
 .../representations/multimodal_attention_fusion.py | 336 ++++++++-----
 .../scuro/representations/representation.py        |   3 +-
 .../systemds/scuro/representations/resnet.py       |  11 +-
 .../systemds/scuro/representations/spectrogram.py  |   8 +-
 .../python/systemds/scuro/representations/tfidf.py |   2 +-
 .../scuro/representations/window_aggregation.py    |  13 +-
 .../systemds/scuro/representations/word2vec.py     |   7 +-
 .../modality_identifier.py => utils/identifier.py} |  29 +-
 src/main/python/tests/scuro/test_data_loaders.py   |   1 -
 src/main/python/tests/scuro/test_dr_search.py      | 174 -------
 ...test_unimodal_optimizer.py => test_hp_tuner.py} | 106 ++--
 .../python/tests/scuro/test_multimodal_fusion.py   |  68 +--
 .../python/tests/scuro/test_operator_registry.py   |   6 +-
 .../python/tests/scuro/test_unimodal_optimizer.py  |   8 +-
 .../tests/scuro/test_unimodal_representations.py   |  12 +-
 .../python/tests/scuro/test_window_operations.py   |   4 +-
 41 files changed, 1637 insertions(+), 1971 deletions(-)

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index ac2857cc22..299c50718e 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -155,18 +155,23 @@ jobs:
 
     - name: Run Scuro Python Tests 
       if: ${{ matrix.test_mode == 'scuro' }}
+      env:
+        TORCH_HOME: ${{ github.workspace }}/.torch
       run: |
+        ( while true; do echo "."; sleep 25; done ) &
+        KA=$!
+        pip install --upgrade pip wheel setuptools
+        # Use CUDA 12.1 wheels to avoid slow/source builds
+        pip install --extra-index-url https://download.pytorch.org/whl/cu121 \
+          torch==2.4.1 torchvision==0.19.1
         pip install \
-          torchvision \
           transformers \
           opencv-python \
-          torch \
           librosa \
           h5py \
           gensim \
           opt-einsum \
           nltk
+        kill $KA 
         cd src/main/python
-        python -m unittest discover -s tests/scuro -p 'test_*.py'
-
-
+        python -m unittest discover -s tests/scuro -p 'test_*.py' -v
\ No newline at end of file
diff --git a/src/main/python/systemds/scuro/__init__.py b/src/main/python/systemds/scuro/__init__.py
index 8e83c865a2..b567b30024 100644
--- a/src/main/python/systemds/scuro/__init__.py
+++ b/src/main/python/systemds/scuro/__init__.py
@@ -67,19 +67,13 @@ from systemds.scuro.models.discrete_model import DiscreteModel
 from systemds.scuro.modality.joined import JoinedModality
 from systemds.scuro.modality.joined_transformed import JoinedTransformedModality
 from systemds.scuro.modality.modality import Modality
-from systemds.scuro.modality.modality_identifier import ModalityIdentifier
+from systemds.scuro.utils.identifier import Identifier
 from systemds.scuro.modality.transformed import TransformedModality
 from systemds.scuro.modality.type import ModalityType
 from systemds.scuro.modality.unimodal_modality import UnimodalModality
-from systemds.scuro.drsearch.dr_search import DRSearch
 from systemds.scuro.drsearch.task import Task
-from systemds.scuro.drsearch.fusion_optimizer import FusionOptimizer
 from systemds.scuro.drsearch.operator_registry import Registry
 from systemds.scuro.drsearch.optimization_data import OptimizationData
-from systemds.scuro.drsearch.representation_cache import RepresentationCache
-from systemds.scuro.drsearch.unimodal_representation_optimizer import (
-    UnimodalRepresentationOptimizer,
-)
 from systemds.scuro.representations.covarep_audio_features import (
     RMSE,
     Spectral,
@@ -131,17 +125,13 @@ __all__ = [
     "JoinedModality",
     "JoinedTransformedModality",
     "Modality",
-    "ModalityIdentifier",
+    "Identifier",
     "TransformedModality",
     "ModalityType",
     "UnimodalModality",
-    "DRSearch",
     "Task",
-    "FusionOptimizer",
     "Registry",
     "OptimizationData",
-    "RepresentationCache",
-    "UnimodalRepresentationOptimizer",
     "UnimodalOptimizer",
     "MultimodalOptimizer",
     "ZeroCrossing",
diff --git a/src/main/python/systemds/scuro/dataloader/base_loader.py b/src/main/python/systemds/scuro/dataloader/base_loader.py
index f21f212e7a..33b418efb3 100644
--- a/src/main/python/systemds/scuro/dataloader/base_loader.py
+++ b/src/main/python/systemds/scuro/dataloader/base_loader.py
@@ -127,8 +127,8 @@ class BaseLoader(ABC):
         if isinstance(file_names, str):
             self.extract(file_names, indices)
         else:
-            for file_name in file_names:
-                self.extract(file_name)
+            for i, file_name in enumerate(file_names):
+                self.extract(file_name, indices[i])
 
         return self.data, self.metadata
 
diff --git a/src/main/python/systemds/scuro/drsearch/dr_search.py b/src/main/python/systemds/scuro/drsearch/dr_search.py
deleted file mode 100644
index 601001c742..0000000000
--- a/src/main/python/systemds/scuro/drsearch/dr_search.py
+++ /dev/null
@@ -1,167 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
-import itertools
-import random
-from typing import List
-
-from systemds.scuro.drsearch.task import Task
-from systemds.scuro.modality.modality import Modality
-from systemds.scuro.representations.representation import Representation
-
-import warnings
-
-warnings.filterwarnings("ignore")
-
-
-def get_modalities_by_name(modalities, name):
-    for modality in modalities:
-        if modality.name == name:
-            return modality
-
-    raise "Modality " + name + "not in modalities"
-
-
-class DRSearch:
-    def __init__(
-        self,
-        modalities: List[Modality],
-        task: Task,
-        representations: List[Representation],
-    ):
-        """
-        The DRSearch primitive finds the best uni- or multimodal data representation for the given modalities for
-        a specific task
-        :param modalities: List of uni-modal modalities
-        :param task: custom task
-        :param representations: List of representations to be evaluated
-        """
-        self.modalities = modalities
-        self.task = task
-        self.representations = representations
-        self.scores = {}
-        self.best_modalities = None
-        self.best_representation = None
-        self.best_score = -1
-
-    def set_best_params(
-        self,
-        representation: Representation,
-        scores: List[float],
-        modality_names: List[str],
-    ):
-        """
-        Updates the best parameters for given modalities, representation, and score
-        :param representation: The representation used to retrieve the current score
-        :param scores: achieved train/test scores for the set of modalities and representation
-        :param modality_names: List of modality names used in this setting
-        :return:
-        """
-
-        # check if modality name is already in dictionary
-        if "_".join(modality_names) not in list(self.scores.keys()):
-            # if not add it to dictionary
-            self.scores["_".join(modality_names)] = {}
-
-        # set score for representation
-        self.scores["_".join(modality_names)][representation] = scores
-
-        # compare current score with best score
-        if scores[1] > self.best_score:
-            self.best_score = scores[1]
-            self.best_representation = representation
-            self.best_modalities = modality_names
-
-    def reset_best_params(self):
-        self.best_score = -1
-        self.best_modalities = None
-        self.best_representation = None
-        self.scores = {}
-
-    def fit_random(self, seed=-1):
-        """
-        This method randomly selects a modality or combination of modalities and representation
-        """
-        if seed != -1:
-            random.seed(seed)
-
-        modalities = []
-        for M in range(1, len(self.modalities) + 1):
-            for combination in itertools.combinations(self.modalities, M):
-                modalities.append(combination)
-
-        modality_combination = random.choice(modalities)
-        representation = random.choice(self.representations)
-
-        modality = modality_combination[0].combine(
-            list(modality_combination[1:]), representation
-        )
-
-        scores = self.task.run(modality.data)
-        self.set_best_params(representation, scores, modality.get_modality_names())
-
-        return self.best_representation, self.best_score, self.best_modalities
-
-    def fit_enumerate_all(self):
-        """
-        This method finds the best representation out of a given List of uni-modal modalities and
-        representations
-        :return: The best parameters found in the search procedure
-        """
-
-        for M in range(1, len(self.modalities) + 1):
-            for combination in itertools.combinations(self.modalities, M):
-                for representation in self.representations:
-                    modality = combination[0]
-                    if len(combination) > 1:
-                        modality = combination[0].combine(
-                            list(combination[1:]), representation
-                        )
-
-                    scores = self.task.run(modality.data)
-                    self.set_best_params(
-                        representation,
-                        scores,
-                        modality.get_modality_names(),
-                    )
-
-        return self.best_representation, self.best_score, self.best_modalities
-
-    def transform(self, modalities: List[Modality]):
-        """
-        The transform method takes a list of uni-modal modalities and creates an aligned representation
-        by using the best parameters found during the fitting step
-        :param modalities: List of uni-modal modalities
-        :return: aligned data
-        """
-
-        if self.best_score == -1:
-            raise "Please fit representations first!"
-
-        used_modalities = []
-
-        for modality_name in self.best_modalities:
-            used_modalities.append(get_modalities_by_name(modalities, modality_name))
-
-        modality = used_modalities[0].combine(
-            used_modalities[1:], self.best_representation
-        )
-
-        return modality.data
diff --git a/src/main/python/systemds/scuro/drsearch/fusion_optimizer.py b/src/main/python/systemds/scuro/drsearch/fusion_optimizer.py
deleted file mode 100644
index 7247720f55..0000000000
--- a/src/main/python/systemds/scuro/drsearch/fusion_optimizer.py
+++ /dev/null
@@ -1,295 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
-import time
-import copy
-import pickle
-from systemds.scuro.drsearch.operator_registry import Registry
-from systemds.scuro.drsearch.optimization_data import (
-    OptimizationResult,
-    OptimizationStatistics,
-)
-from systemds.scuro.drsearch.representation_cache import RepresentationCache
-from systemds.scuro.drsearch.task import Task
-from systemds.scuro.representations.aggregate import Aggregation
-from systemds.scuro.representations.context import Context
-
-
-def extract_names(operator_chain):
-    result = []
-    for op in operator_chain:
-        result.append(op.name)
-
-    return result
-
-
-class FusionOptimizer:
-    def __init__(
-        self,
-        modalities,
-        task: Task,
-        unimodal_representations_candidates,
-        representation_cache: RepresentationCache,
-        num_best_candidates=4,
-        max_chain_depth=5,
-        debug=False,
-    ):
-        self.modalities = modalities
-        self.task = task
-        self.unimodal_representations_candidates = unimodal_representations_candidates
-        self.num_best_candidates = num_best_candidates
-        self.k_best_candidates, self.candidates_per_modality = self.get_k_best_results(
-            num_best_candidates
-        )
-        self.operator_registry = Registry()
-        self.max_chain_depth = max_chain_depth
-        self.debug = debug
-        self.evaluated_candidates = set()
-        self.cache = representation_cache
-        self.optimization_statistics = OptimizationStatistics(self.k_best_candidates)
-        self.optimization_results = []
-
-    def optimize(self):
-        """
-        This method finds different ways in how to combine modalities and evaluates the fused representations against
-        the given task. It can fuse different representations from the same modality as well as fuse representations
-        form different modalities.
-        """
-
-        # TODO: add an aligned representation for all modalities with a temporal dimension
-        # TODO: keep a map of operator chains so that we don't evaluate them multiple times in different orders (if it does not make a difference)
-
-        r = []
-
-        for candidate in self.k_best_candidates:
-            modality = self.candidates_per_modality[str(candidate)]
-            cached_representation, representation_ops, used_op_names = (
-                self.cache.load_from_cache(modality, candidate.operator_chain)
-            )
-            if cached_representation is not None:
-                modality = cached_representation
-            store = False
-            for representation in representation_ops:
-                if isinstance(representation, Context):
-                    modality = modality.context(representation)
-                elif representation.name == "RowWiseConcatenation":
-                    modality = modality.flatten(True)
-                else:
-                    modality = modality.apply_representation(representation)
-                store = True
-            if store:
-                self.cache.save_to_cache(modality, used_op_names, representation_ops)
-
-            remaining_candidates = [c for c in self.k_best_candidates if c != candidate]
-            r.append(
-                self._optimize_candidate(modality, candidate, remaining_candidates, 1)
-            )
-
-        if self.debug:
-            with open(
-                f"fusion_statistics_{self.task.model.name}_{self.num_best_candidates}_{self.max_chain_depth}.pkl",
-                "wb",
-            ) as fp:
-                pickle.dump(
-                    self.optimization_statistics,
-                    fp,
-                    protocol=pickle.HIGHEST_PROTOCOL,
-                )
-
-            opt_results = copy.deepcopy(self.optimization_results)
-            for i, opt_res in enumerate(self.optimization_results):
-                op_name = []
-                for op in opt_res.operator_chain:
-                    if isinstance(op, list):
-                        for o in op:
-                            if isinstance(o, list):
-                                for j in o:
-                                    op_name.append(j.name)
-                            elif isinstance(o, str):
-                                op_name.append(o)
-                            else:
-                                op_name.append(o.name)
-                    elif isinstance(op, str):
-                        op_name.append(op)
-                    else:
-                        op_name.append(op.name)
-                opt_results[i].operator_chain = op_name
-            with open(
-                f"fusion_results_{self.task.model.name}_{self.num_best_candidates}_{self.max_chain_depth}.pkl",
-                "wb",
-            ) as fp:
-                pickle.dump(opt_results, fp, protocol=pickle.HIGHEST_PROTOCOL)
-
-            self.optimization_statistics.print_statistics()
-
-    def get_k_best_results(self, k: int):
-        """
-        Get the k best results per modality
-        :param k: number of best results
-        """
-        best_results = []
-        candidate_for_modality = {}
-        for modality in self.modalities:
-            k_results = sorted(
-                self.unimodal_representations_candidates[modality.modality_id][
-                    self.task.model.name
-                ],
-                key=lambda x: x.test_accuracy,
-                reverse=True,
-            )[:k]
-            for k_result in k_results:
-                candidate_for_modality[str(k_result)] = modality
-            best_results.extend(k_results)
-
-        return best_results, candidate_for_modality
-
-    def _optimize_candidate(
-        self, modality, candidate, remaining_candidates, chain_depth
-    ):
-        """
-        Optimize a single candidate by fusing it with others recursively.
-
-        :param candidate: The current candidate representation.
-        :param chain_depth: The current depth of fusion chains.
-        """
-        if chain_depth > self.max_chain_depth:
-            return
-
-        for other_candidate in remaining_candidates:
-            other_modality = self.candidates_per_modality[str(other_candidate)]
-            cached_representation, representation_ops, used_op_names = (
-                self.cache.load_from_cache(
-                    other_modality, other_candidate.operator_chain
-                )
-            )
-            if cached_representation is not None:
-                other_modality = cached_representation
-            store = False
-            for representation in representation_ops:
-                if representation.name == "Aggregation":
-                    params = other_candidate.parameters[representation.name]
-                    representation = Aggregation(
-                        aggregation_function=params["aggregation"]
-                    )
-                if isinstance(representation, Context):
-                    other_modality = other_modality.context(representation)
-                elif isinstance(representation, Aggregation):
-                    other_modality = representation.execute(other_modality)
-                elif representation.name == "RowWiseConcatenation":
-                    other_modality = other_modality.flatten(True)
-                else:
-                    other_modality = other_modality.apply_representation(representation)
-                store = True
-            if store:
-                self.cache.save_to_cache(
-                    other_modality, used_op_names, representation_ops
-                )
-
-            fusion_results = self.operator_registry.get_fusion_operators()
-            fusion_representation = None
-            for fusion_operator in fusion_results:
-                fusion_operator = fusion_operator()
-                chain_key = self.create_identifier(
-                    candidate, fusion_operator, other_candidate
-                )
-                # print(fusion_operator.name)
-                representation_start = time.time()
-                if (
-                    isinstance(fusion_operator, Context)
-                    and fusion_representation is not None
-                ):
-                    fusion_representation.context(fusion_operator)
-                elif isinstance(fusion_operator, Context):
-                    continue
-                else:
-                    fused_representation = modality.combine(
-                        other_modality, fusion_operator
-                    )
-
-                representation_end = time.time()
-                if chain_key not in self.evaluated_candidates:
-                    # Evaluate the fused representation
-
-                    score = self.task.run(fused_representation.data)
-                    fusion_params = {fusion_operator.name: fusion_operator.parameters}
-                    result = OptimizationResult(
-                        operator_chain=[
-                            candidate.operator_chain,
-                            fusion_operator.name,
-                            other_candidate.operator_chain,
-                        ],
-                        parameters=[
-                            candidate.parameters,
-                            fusion_params,
-                            other_candidate.parameters,
-                        ],
-                        train_accuracy=score[0],
-                        test_accuracy=score[1],
-                        # train_min_it_acc=score[2],
-                        # test_min_it_acc=score[3],
-                        training_runtime=self.task.training_time,
-                        inference_runtime=self.task.inference_time,
-                        representation_time=representation_end - representation_start,
-                        output_shape=(1, 1),  # TODO
-                    )
-
-                    # Store the result
-                    self.optimization_results.append(result)
-                    self.optimization_statistics.add_entry(
-                        [
-                            candidate.operator_chain,
-                            [fusion_operator.name],
-                            other_candidate.operator_chain,
-                        ],
-                        score[1],
-                    )
-
-                    # Mark this chain as evaluated
-                    self.evaluated_candidates.add(chain_key)
-
-                    if self.debug:
-                        print(
-                            f"Evaluated chain: {candidate.operator_chain} + {fusion_operator.name} + {other_candidate.operator_chain} -> {score[1]}"
-                        )
-
-                    # Recursively optimize further with this fused representation
-                    self._optimize_candidate(
-                        fused_representation,
-                        result,
-                        [c for c in remaining_candidates if c != other_candidate],
-                        chain_depth + 1,
-                    )
-
-    def create_identifier(self, candidate, fusion, other_candidate):
-        identifier = "".join(flatten_and_join(candidate.operator_chain))
-        identifier += fusion.name
-        identifier += "".join(flatten_and_join(other_candidate.operator_chain))
-
-        return identifier
-
-
-def flatten_and_join(data):
-    flat_list = []
-    for item in data:
-        if isinstance(item, list):
-            flat_list.extend(flatten_and_join(item))
-        else:
-            flat_list.append(item.name if not isinstance(item, str) else item)
-    return flat_list
diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py
index 04a3fa4701..8902bb7d01 100644
--- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py
+++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py
@@ -18,89 +18,276 @@
 # under the License.
 #
 # -------------------------------------------------------------
-import itertools
+from typing import Dict, List, Tuple, Any, Optional
+import numpy as np
+from sklearn.model_selection import ParameterGrid
+import json
+import logging
+from dataclasses import dataclass
 import time
+import copy
+
+from systemds.scuro.modality.modality import Modality
+from systemds.scuro.drsearch.task import Task
 
-import numpy as np
 
-from systemds.scuro.drsearch.optimization_data import OptimizationResult
-from systemds.scuro.representations.context import Context
+@dataclass
+class HyperparamResult:
+
+    representation_name: str
+    best_params: Dict[str, Any]
+    best_score: float
+    all_results: List[Tuple[Dict[str, Any], float]]
+    tuning_time: float
+    modality_id: int
 
 
 class HyperparameterTuner:
-    def __init__(self, task, n_trials=10, early_stopping_patience=5):
-        self.task = task
-        self.n_trials = n_trials
-        self.early_stopping_patience = early_stopping_patience
-
-    def tune_operator_chain(self, modality, operator_chain):
-        best_result = None
-        best_score = -np.inf
-
-        param_grids = {}
-
-        for operator in operator_chain:
-            param_grids[operator.name] = operator.parameters
-
-        param_combinations = self._generate_search_space(param_grids)
-
-        for params in param_combinations:
-            modified_modality = modality
-            current_chain = []
-
-            representation_start = time.time()
-            try:
-                for operator in operator_chain:
-
-                    if operator.name in params:
-                        operator.set_parameters(params[operator.name])
-
-                    if isinstance(operator, Context):
-                        modified_modality = modified_modality.context(operator)
-                    else:
-                        modified_modality = modified_modality.apply_representation(
-                            operator
-                        )
-
-                    current_chain.append(operator)
-
-                representation_end = time.time()
-
-                score = self.task.run(modified_modality.data)
-
-                if score[1] > best_score:
-                    best_score = score[1]
-                    best_params = params
-                    best_result = OptimizationResult(
-                        operator_chain=current_chain,
-                        parameters=params,
-                        train_accuracy=score[0],
-                        test_accuracy=score[1],
-                        training_runtime=self.task.training_time,
-                        inference_runtime=self.task.inference_time,
-                        representation_time=representation_end - representation_start,
-                        output_shape=(1, 1),
+
+    def __init__(
+        self,
+        modalities,
+        tasks,
+        optimization_results,
+        k: int = 2,
+        n_jobs: int = -1,
+        scoring_metric: str = "accuracy",
+        maximize_metric: bool = True,
+        save_results: bool = False,
+        debug: bool = False,
+    ):
+        self.tasks = tasks
+        self.optimization_results = optimization_results
+        self.n_jobs = n_jobs
+        self.scoring_metric = scoring_metric
+        self.maximize_metric = maximize_metric
+        self.save_results = save_results
+        self.results = {}
+        self.k = k
+        self.modalities = modalities
+        self.representations = None
+        self.k_best_cache = None
+        self.k_best_representations = None
+        self.extract_k_best_modalities_per_task()
+        self.debug = debug
+        self.logger = logging.getLogger(__name__)
+        if debug:
+            logging.basicConfig(
+                level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+            )
+
+    def get_modalities_by_id(self, modality_ids: List[int]) -> Modality:
+        modalities = []
+        for mod in self.modalities:
+            if mod.modality_id in modality_ids:
+                modalities.append(mod)
+        return modalities
+
+    def get_modality_by_id_and_instance_id(self, modality_id, instance_id):
+        counter = 0
+        for modality in self.modalities:
+            if modality.modality_id == modality_id:
+                if counter == instance_id or instance_id == -1:
+                    return modality
+                else:
+                    counter += 1
+        return None
+
+    def extract_k_best_modalities_per_task(self):
+        self.k_best_representations = {}
+        self.k_best_cache = {}
+        representations = {}
+        for task in self.tasks:
+            self.k_best_representations[task.model.name] = []
+            self.k_best_cache[task.model.name] = []
+            representations[task.model.name] = {}
+            for modality in self.modalities:
+                k_best_results, cached_data = (
+                    self.optimization_results.get_k_best_results(modality, self.k, task)
+                )
+                representations[task.model.name][modality.modality_id] = k_best_results
+                self.k_best_representations[task.model.name].extend(k_best_results)
+                self.k_best_cache[task.model.name].extend(cached_data)
+        self.representations = representations
+
+    def tune_unimodal_representations(self, max_eval_per_rep: Optional[int] = None):
+        results = {}
+        for task in self.tasks:
+            results[task.model.name] = []
+            for representation in self.k_best_representations[task.model.name]:
+                result = self.tune_dag_representation(
+                    representation.dag,
+                    representation.dag.root_node_id,
+                    task,
+                    max_eval_per_rep,
+                )
+                results[task.model.name].append(result)
+
+        self.results = results
+
+        if self.save_results:
+            self.save_tuning_results()
+
+        return results
+
+    def tune_dag_representation(self, dag, root_node_id, task, max_evals=None):
+        hyperparams = {}
+        reps = []
+        modality_ids = []
+        node_order = []
+
+        visited = set()
+
+        def visit_node(node_id):
+            if node_id in visited:
+                return
+            node = dag.get_node_by_id(node_id)
+            for input_id in node.inputs:
+                visit_node(input_id)
+            visited.add(node_id)
+            if node.operation is not None:
+                if node.parameters:
+                    hyperparams.update(node.parameters)
+                reps.append(node.operation)
+                node_order.append(node_id)
+            if node.modality_id is not None:
+                modality_ids.append(node.modality_id)
+
+        visit_node(root_node_id)
+
+        if not hyperparams:
+            return None
+
+        start_time = time.time()
+        rep_name = "_".join([rep.__name__ for rep in reps])
+
+        param_grid = list(ParameterGrid(hyperparams))
+        if max_evals and len(param_grid) > max_evals:
+            np.random.shuffle(param_grid)
+            param_grid = param_grid[:max_evals]
+
+        all_results = []
+        for params in param_grid:
+            result = self.evaluate_dag_config(
+                dag, params, node_order, modality_ids, task
+            )
+            all_results.append(result)
+
+        if self.maximize_metric:
+            best_params, best_score = max(all_results, key=lambda x: x[1])
+        else:
+            best_params, best_score = min(all_results, key=lambda x: x[1])
+
+        tuning_time = time.time() - start_time
+
+        return HyperparamResult(
+            representation_name=rep_name,
+            best_params=best_params,
+            best_score=best_score,
+            all_results=all_results,
+            tuning_time=tuning_time,
+            modality_id=modality_ids[0] if modality_ids else None,
+        )
+
+    def evaluate_dag_config(self, dag, params, node_order, modality_ids, task):
+        try:
+            dag_copy = copy.deepcopy(dag)
+
+            for node_id in node_order:
+                node = dag_copy.get_node_by_id(node_id)
+                if node.operation is not None and node.parameters:
+                    node_params = {
+                        k: v for k, v in params.items() if k in node.parameters
+                    }
+                    node.parameters = node_params
+
+            modalities = self.get_modalities_by_id(modality_ids)
+            modified_modality = dag_copy.execute(modalities, task)
+            score = task.run(
+                modified_modality[list(modified_modality.keys())[-1]].data
+            )[1]
+
+            return params, score
+        except Exception as e:
+            self.logger.error(f"Error evaluating DAG with params {params}: {e}")
+            return params, float("-inf") if self.maximize_metric else float("inf")
+
+    def tune_multimodal_representations(
+        self,
+        optimization_results,
+        k: int = 1,
+        optimize_unimodal: bool = True,
+        max_eval_per_rep: Optional[int] = None,
+    ):
+        results = {}
+        for task in self.tasks:
+            best_results = sorted(
+                optimization_results[task.model.name],
+                key=lambda x: x.val_score,
+                reverse=True,
+            )[:k]
+            results[task.model.name] = []
+            best_optimization_results = best_results
+
+            for representation in best_optimization_results:
+                if optimize_unimodal:
+                    dag = copy.deepcopy(representation.dag)
+                    index = 0
+                    for i, node in enumerate(representation.dag.nodes):
+                        if not node.inputs:
+                            leaf_node_id = node.node_id
+                            leaf_nodes = self.representations[task.model.name][
+                                node.modality_id
+                            ][node.representation_index].dag.nodes
+                            for leaf_idx, node in enumerate(dag.nodes):
+                                if node.node_id == leaf_node_id:
+                                    dag.nodes[leaf_idx : leaf_idx + 1] = leaf_nodes
+                                    index = leaf_idx + len(leaf_nodes) - 1
+                                    break
+
+                            for node in dag.nodes:
+                                try:
+                                    idx = node.inputs.index(leaf_node_id)
+                                    node.inputs[idx] = dag.nodes[index].node_id
+                                    break
+                                except ValueError:
+                                    continue
+
+                    result = self.tune_dag_representation(
+                        dag, dag.root_node_id, task, max_eval_per_rep
                     )
+                else:
+                    result = self.tune_dag_representation(
+                        representation.dag,
+                        representation.dag.root_node_id,
+                        task,
+                        max_eval_per_rep,
+                    )
+                results[task.model.name].append(result)
+
+        self.results = results
 
-            except Exception as e:
-                print(f"Failed parameter combination {params}: {str(e)}")
-                continue
+        if self.save_results:
+            self.save_tuning_results()
 
-        return best_result
+        return results
 
-    def _generate_search_space(self, param_grids):
-        combinations = {}
-        for operator_name, params in param_grids.items():
-            operator_combinations = [
-                dict(zip(params.keys(), v)) for v in itertools.product(*params.values())
-            ]
-            combinations[operator_name] = operator_combinations
+    def save_tuning_results(self, filepath: str = None):
+        if not filepath:
+            filepath = f"hyperparameter_results_{int(time.time())}.json"
 
-        keys = list(combinations.keys())
-        values = [combinations[key] for key in keys]
+        json_results = {}
+        for task in self.results.keys():
+            for result in self.results[task]:
+                json_results[result.representation_name] = {
+                    "best_params": result.best_params,
+                    "best_score": result.best_score,
+                    "tuning_time": result.tuning_time,
+                    "num_evaluations": len(result.all_results),
+                }
 
-        parameter_grid = [
-            dict(zip(keys, combo)) for combo in itertools.product(*values)
-        ]
+        with open(filepath, "w") as f:
+            json.dump(json_results, f, indent=2)
 
-        return parameter_grid
+        if self.debug:
+            self.logger.info(f"Results saved to {filepath}")
diff --git a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py
index ac4365ed5c..91d569bc59 100644
--- a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py
+++ b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py
@@ -18,368 +18,276 @@
 # under the License.
 #
 # -------------------------------------------------------------
-import itertools
 
+
+import itertools
+import pickle
+import time
+from dataclasses import dataclass
+from typing import List, Dict, Any, Generator
+import copy
+import traceback
+from itertools import chain
+from systemds.scuro.drsearch.task import Task
+from systemds.scuro.modality.type import ModalityType
+from systemds.scuro.drsearch.representation_dag import (
+    RepresentationDag,
+    RepresentationDAGBuilder,
+)
 from systemds.scuro.representations.aggregated_representation import (
     AggregatedRepresentation,
 )
-
 from systemds.scuro.representations.aggregate import Aggregation
-
 from systemds.scuro.drsearch.operator_registry import Registry
-
 from systemds.scuro.utils.schema_helpers import get_shape
-import dataclasses
 
 
 class MultimodalOptimizer:
     def __init__(
-        self, modalities, unimodal_optimization_results, tasks, k=2, debug=True
+        self,
+        modalities: List[Any],
+        unimodal_optimization_results: Any,
+        tasks: List[Any],
+        k: int = 2,
+        debug: bool = True,
+        min_modalities: int = 2,
+        max_modalities: int = None,
     ):
-        self.k_best_cache = None
-        self.k_best_modalities = None
         self.modalities = modalities
-        self.unimodal_optimization_results = unimodal_optimization_results
         self.tasks = tasks
         self.k = k
-        self.extract_k_best_modalities_per_task()
         self.debug = debug
+        self.min_modalities = max(2, min_modalities)
+        self.max_modalities = max_modalities or len(modalities)
 
         self.operator_registry = Registry()
-        self.optimization_results = MultimodalResults(
-            modalities, tasks, debug, self.k_best_modalities
+        self.fusion_operators = self.operator_registry.get_fusion_operators()
+
+        self.k_best_representations = self._extract_k_best_representations(
+            unimodal_optimization_results
         )
-        self.cache = {}
+        self.optimization_results = []
 
-    def optimize(self):
-        for task in self.tasks:
-            self.optimize_intermodal_representations(task)
+    def _extract_k_best_representations(
+        self, unimodal_optimization_results: Any
+    ) -> Dict[str, Dict[str, List[Any]]]:
+        k_best = {}
 
-    def optimize_intramodal_representations(self, task):
-        for modality in self.modalities:
-            representations = self.k_best_modalities[task.model.name][
-                modality.modality_id
-            ]
-            applied_representations = self.extract_representations(
-                representations, modality, task.model.name
-            )
+        for task in self.tasks:
+            k_best[task.model.name] = {}
 
-            for i in range(1, len(applied_representations)):
-                for fusion_method in self.operator_registry.get_fusion_operators():
-                    if fusion_method().needs_alignment and not applied_representations[
-                        i - 1
-                    ].is_aligned(applied_representations[i]):
-                        continue
-                    combined = applied_representations[i - 1].combine(
-                        applied_representations[i], fusion_method()
-                    )
-                    self.evaluate(
-                        task,
-                        combined,
-                        [i - 1, i],
-                        fusion_method,
-                        [
-                            applied_representations[i - 1].modality_id,
-                            applied_representations[i].modality_id,
-                        ],
+            for modality in self.modalities:
+                k_best_results, cached_data = (
+                    unimodal_optimization_results.get_k_best_results(
+                        modality, self.k, task
                     )
-                    if not fusion_method().commutative:
-                        combined_comm = applied_representations[i].combine(
-                            applied_representations[i - 1], fusion_method()
-                        )
-                        self.evaluate(
-                            task,
-                            combined_comm,
-                            [i, i - 1],
-                            fusion_method,
-                            [
-                                applied_representations[i - 1].modality_id,
-                                applied_representations[i].modality_id,
-                            ],
-                        )
-
-    # TODO: check if order matters for reused reps - only compute once - check in cache
-    # TODO: parallelize - whenever an item of len 0 comes along give it to a new thread - merge results
-    # TODO: change the algorithm so that one representation is used until there is no more representations to add - saves a lot of memory
-    def optimize_intermodal_representations(self, task):
-        modality_combos = []
-        n = len(self.k_best_cache[task.model.name])
-        reuse_cache = {}
-
-        def generate_extensions(current_combo, remaining_indices):
-            # Add current combination if it has at least 2 elements
-            if len(current_combo) >= 2:
-                combo_tuple = tuple(i for i in current_combo)
-                modality_combos.append(combo_tuple)
-
-            for i in remaining_indices:
-                new_combo = current_combo + [i]
-                new_remaining = [j for j in remaining_indices if j > i]
-                generate_extensions(new_combo, new_remaining)
-
-        for start_idx in range(n):
-            remaining = list(range(start_idx + 1, n))
-            generate_extensions([start_idx], remaining)
-        fusion_methods = self.operator_registry.get_fusion_operators()
-        fused_representations = []
-        reuse_fused_representations = False
-        for i, modality_combo in enumerate(modality_combos):
-            # clear reuse cache
-            reuse_cache = self.prune_cache(modality_combos[i:], reuse_cache)
-
-            if i != 0:
-                reuse_fused_representations = self.is_prefix_match(
-                    modality_combos[i - 1], modality_combo
-                )
-            if reuse_fused_representations:
-                mods = [
-                    self.k_best_cache[task.model.name][mod_idx]
-                    for mod_idx in modality_combo[len(modality_combos[i - 1]) :]
-                ]
-                fused_representations = reuse_cache[modality_combos[i - 1]]
-            else:
-                prefix_idx = self.compute_equal_prefix_index(
-                    modality_combos[i - 1], modality_combo
-                )
-                if prefix_idx > 1:
-                    fused_representations = reuse_cache[
-                        modality_combos[i - 1][:prefix_idx]
-                    ]
-                    reuse_fused_representations = True
-                    mods = [
-                        self.k_best_cache[task.model.name][mod_idx]
-                        for mod_idx in modality_combo[prefix_idx:]
-                    ]
-            if self.debug:
-                print(
-                    f"New modality combo: {modality_combo} - Reuse: {reuse_fused_representations} - # fused reps: {len(fused_representations)}"
                 )
 
-            all_mods = [
-                self.k_best_cache[task.model.name][mod_idx]
-                for mod_idx in modality_combo
-            ]
-            temp_fused_reps = []
-            for j, fusion_method in enumerate(fusion_methods):
-                # Evaluate all mods
-                fused_rep = all_mods[0].combine(all_mods[1:], fusion_method())
-                temp_fused_reps.append(fused_rep)
-                self.evaluate(
-                    task,
-                    fused_rep,
-                    [
-                        self.k_best_modalities[task.model.name][k].representations
-                        for k in modality_combo
-                    ],
-                    fusion_method,
-                    modality_combo,
-                )
-                if reuse_fused_representations:
-                    for fused_representation in fused_representations:
-                        fused_rep = fused_representation.combine(mods, fusion_method())
-                        temp_fused_reps.append(fused_rep)
-                        self.evaluate(
-                            task,
-                            fused_rep,
-                            [
-                                self.k_best_modalities[task.model.name][
-                                    k
-                                ].representations
-                                for k in modality_combo
-                            ],
-                            fusion_method,
-                            modality_combo,
-                        )
+                k_best[task.model.name][modality.modality_id] = cached_data
 
-            if (
-                len(modality_combo) < len(self.k_best_cache[task.model.name])
-                and i + 1 < len(modality_combos)
-                and self.is_prefix_match(modality_combos[i], modality_combos[i + 1])
-            ):
-                reuse_cache[modality_combo] = temp_fused_reps
-            reuse_fused_representations = False
-
-    def prune_cache(self, sequences, cache):
-        seqs_as_tuples = [tuple(seq) for seq in sequences]
-
-        def still_used(key):
-            return any(self.is_prefix_match(key, seq) for seq in seqs_as_tuples)
-
-        cache = {key: value for key, value in cache.items() if still_used(key)}
-        return cache
-
-    def is_prefix_match(self, seq1, seq2):
-        if len(seq1) > len(seq2):
-            return False
-
-        # Check if seq1 matches the beginning of seq2
-        return seq2[: len(seq1)] == seq1
-
-    def compute_equal_prefix_index(self, seq1, seq2):
-        max_len = min(len(seq1), len(seq2))
-        i = 0
-        while i < max_len and seq1[i] == seq2[i]:
-            i += 1
-
-        return i
-
-    def extract_representations(self, representations, modality, task_name):
-        applied_representations = []
-        for i in range(0, len(representations)):
-            cache_key = (
-                tuple(representations[i].representations),
-                representations[i].task_time,
-                representations[i].representation_time,
+        return k_best
+
+    def _generate_modality_combinations(self) -> Generator[List[str], None, None]:
+        modality_ids = [mod.modality_id for mod in self.modalities]
+
+        for r in range(
+            self.min_modalities, min(self.max_modalities + 1, len(modality_ids) + 1)
+        ):
+            for modality_subset in itertools.combinations(modality_ids, r):
+                yield list(modality_subset)
+
+    def _generate_representation_combinations(
+        self, modality_subset: List[str], task_name: str
+    ) -> Generator[Dict[str, int], None, None]:
+        representation_options = []
+
+        for modality_id in modality_subset:
+            num_representations = len(
+                self.k_best_representations[task_name][modality_id]
             )
-            if (
-                cache_key
-                in self.unimodal_optimization_results.cache[modality.modality_id][
-                    task_name
-                ]
-            ):
-                applied_representations.append(
-                    self.unimodal_optimization_results.cache[modality.modality_id][
-                        task_name
-                    ][cache_key]
+            representation_options.append(list(range(num_representations)))
+
+        for combo in itertools.product(*representation_options):
+            yield {
+                modality_id: repr_idx
+                for modality_id, repr_idx in zip(modality_subset, combo)
+            }
+
+    def _generate_fusion_dags(
+        self, modality_subset: List[str], representation_combo: Dict[str, int]
+    ) -> Generator[RepresentationDag, None, None]:
+        leaf_infos = [(m, representation_combo[m]) for m in modality_subset]
+
+        def gen_trees(indices: List[int]):
+            if len(indices) == 1:
+                yield indices[0]
+                return
+            for split in range(1, len(indices)):
+                for left_idxs in itertools.combinations(indices, split):
+                    left = list(left_idxs)
+                    right = [i for i in indices if i not in left]
+                    for l_tree in gen_trees(left):
+                        for r_tree in gen_trees(right):
+                            yield (l_tree, r_tree)
+
+        def build_variants(
+            subtree, base_builder: RepresentationDAGBuilder, leaf_id_map
+        ):
+            variants = []
+
+            if isinstance(subtree, int):
+                variants.append((base_builder, leaf_id_map[subtree]))
+                return variants
+
+            left_sub, right_sub = subtree
+
+            left_variants = build_variants(
+                left_sub, copy.deepcopy(base_builder), leaf_id_map
+            )
+
+            for left_builder, left_root in left_variants:
+                right_variants = build_variants(
+                    right_sub, copy.deepcopy(left_builder), leaf_id_map
                 )
-            else:
-                applied_representation = modality
-                for j, rep in enumerate(representations[i].representations):
-                    representation, is_context = (
-                        self.operator_registry.get_representation_by_name(
-                            rep, modality.modality_type
+
+                for right_builder, right_root in right_variants:
+                    for fusion_op_class in self.fusion_operators:
+                        new_builder = copy.deepcopy(right_builder)
+                        fusion_op = fusion_op_class()
+                        fusion_id = new_builder.create_operation_node(
+                            fusion_op.__class__,
+                            [left_root, right_root],
+                            fusion_op.parameters,
                         )
+                        variants.append((new_builder, fusion_id))
+
+            return variants
+
+        n = len(leaf_infos)
+
+        for permuted_leaf_infos in itertools.permutations(leaf_infos, n):
+            base_builder = RepresentationDAGBuilder()
+            leaf_id_map = {}
+            for idx, (modality_id, repr_idx) in enumerate(permuted_leaf_infos):
+                nodeid = base_builder.create_leaf_node(modality_id, repr_idx)
+                leaf_id_map[idx] = nodeid
+
+            indices = list(range(n))
+
+            for tree in gen_trees(indices):
+                variants = build_variants(tree, base_builder, leaf_id_map)
+                for builder_variant, root_id in variants:
+                    try:
+                        yield builder_variant.build(root_id)
+                    except ValueError:
+                        if self.debug:
+                            print(f"Skipping invalid DAG for root {root_id}")
+                        continue
+
+    def _evaluate_dag(self, dag: RepresentationDag, task: Task) -> "OptimizationResult":
+        start_time = time.time()
+
+        try:
+            fused_representation = dag.execute(
+                list(
+                    chain.from_iterable(
+                        self.k_best_representations[task.model.name].values()
                     )
-                    if representation is None:
-                        if rep == AggregatedRepresentation.__name__:
-                            representation = AggregatedRepresentation(Aggregation())
-                    else:
-                        representation = representation()
-                    representation.set_parameters(representations[i].params[j])
-                    if is_context:
-                        applied_representation = applied_representation.context(
-                            representation
-                        )
-                    else:
-                        applied_representation = (
-                            applied_representation.apply_representation(representation)
-                        )
-                self.k_best_cache[task_name].append(applied_representation)
-                applied_representations.append(applied_representation)
-        return applied_representations
-
-    def evaluate(self, task, modality, representations, fusion, modality_combo):
-        if task.expected_dim == 1 and get_shape(modality.metadata) > 1:
-            for aggregation in Aggregation().get_aggregation_functions():
-                agg_operator = AggregatedRepresentation(Aggregation(aggregation, False))
-                agg_modality = agg_operator.transform(modality)
-
-                scores = task.run(agg_modality.data)
-                reps = representations.copy()
-                reps.append(agg_operator)
-
-                self.optimization_results.add_result(
-                    scores,
-                    reps,
-                    modality.transformation,
-                    modality_combo,
-                    task.model.name,
-                )
-        else:
-            scores = task.run(modality.data)
-            self.optimization_results.add_result(
-                scores,
-                representations,
-                modality.transformation,
-                modality_combo,
-                task.model.name,
+                ),
+                task,
+            )
+
+            if fused_representation is None:
+                return None
+
+            final_representation = fused_representation[
+                list(fused_representation.keys())[-1]
+            ]
+            if task.expected_dim == 1 and get_shape(final_representation.metadata) > 1:
+                agg_operator = AggregatedRepresentation(Aggregation())
+                final_representation = agg_operator.transform(final_representation)
+
+            eval_start = time.time()
+            scores = task.run(final_representation.data)
+            eval_time = time.time() - eval_start
+
+            total_time = time.time() - start_time
+
+            return OptimizationResult(
+                dag=dag,
+                train_score=scores[0],
+                val_score=scores[1],
+                runtime=total_time,
+                task_name=task.model.name,
+                evaluation_time=eval_time,
             )
 
-    def add_to_cache(self, result_idx, combined_modality):
-        self.cache[result_idx] = combined_modality
+        except Exception as e:
+            print(f"Error evaluating DAG: {e}")
+            traceback.print_exc()
+            return None
+
+    def _get_modality_by_id_and_instance_id(self, modalities, modality_id, instance_id):
+        counter = 0
+        for modality in modalities:
+            if modality.modality_id == modality_id:
+                if counter == instance_id or instance_id == -1:
+                    return modality
+                else:
+                    counter += 1
+        return None
+
+    def optimize(
+        self, max_combinations: int = None
+    ) -> Dict[str, List["OptimizationResult"]]:
+        all_results = {}
 
-    def extract_k_best_modalities_per_task(self):
-        self.k_best_modalities = {}
-        self.k_best_cache = {}
         for task in self.tasks:
-            self.k_best_modalities[task.model.name] = []
-            self.k_best_cache[task.model.name] = []
-            for modality in self.modalities:
-                k_best_results, cached_data = (
-                    self.unimodal_optimization_results.get_k_best_results(
-                        modality, self.k, task
-                    )
-                )
+            if self.debug:
+                print(f"Optimizing multimodal fusion for task: {task.model.name}")
+            all_results[task.model.name] = []
+            evaluated_count = 0
 
-                self.k_best_modalities[task.model.name].extend(k_best_results)
-                self.k_best_cache[task.model.name].extend(cached_data)
+            for modality_subset in self._generate_modality_combinations():
+                if self.debug:
+                    print(f"  Evaluating modality subset: {modality_subset}")
 
+                for repr_combo in self._generate_representation_combinations(
+                    modality_subset, task.model.name
+                ):
 
-class MultimodalResults:
-    def __init__(self, modalities, tasks, debug, k_best_modalities):
-        self.modality_ids = [modality.modality_id for modality in modalities]
-        self.task_names = [task.model.name for task in tasks]
-        self.results = {}
-        self.debug = debug
-        self.k_best_modalities = k_best_modalities
+                    for dag in self._generate_fusion_dags(modality_subset, repr_combo):
+                        if max_combinations and evaluated_count >= max_combinations:
+                            break
 
-        for task in tasks:
-            self.results[task.model.name] = {}
+                        result = self._evaluate_dag(dag, task)
+                        if result is not None:
+                            all_results[task.model.name].append(result)
 
-    def add_result(
-        self, scores, best_representation_idx, fusion_methods, modality_combo, task_name
-    ):
+                        evaluated_count += 1
 
-        entry = MultimodalResultEntry(
-            representations=best_representation_idx,
-            train_score=scores[0],
-            val_score=scores[1],
-            fusion_methods=[
-                fusion_method.__class__.__name__ for fusion_method in fusion_methods
-            ],
-            modality_combo=modality_combo,
-            task=task_name,
-        )
+                        if self.debug and evaluated_count % 100 == 0:
+                            print(f"    Evaluated {evaluated_count} combinations...")
 
-        modality_id_strings = "_".join(list(map(str, modality_combo)))
-        if not modality_id_strings in self.results[task_name]:
-            self.results[task_name][modality_id_strings] = []
+                    if max_combinations and evaluated_count >= max_combinations:
+                        break
 
-        self.results[task_name][modality_id_strings].append(entry)
+                if max_combinations and evaluated_count >= max_combinations:
+                    break
 
-        if self.debug:
-            print(f"{modality_id_strings}_{task_name}: {entry}")
-
-    def print_results(self):
-        for task_name in self.task_names:
-            for modality in self.results[task_name].keys():
-                for entry in self.results[task_name][modality]:
-                    reps = []
-                    for i, mod_idx in enumerate(entry.modality_combo):
-                        reps.append(self.k_best_modalities[task_name][mod_idx])
-
-                    print(
-                        f"{modality}_{task_name}: "
-                        f"Validation score: {entry.val_score} - Training score: {entry.train_score}"
-                    )
-                    for i, rep in enumerate(reps):
-                        print(
-                            f"    Representation: {entry.modality_combo[i]} - {rep.representations}"
-                        )
+            if self.debug:
+                print(
+                    f"  Task completed: {len(all_results[task.model.name])} valid combinations evaluated"
+                )
 
-                    print(f"    Fusion: {entry.fusion_methods[0]} ")
+        self.optimization_results = all_results
 
-    def store_results(self, file_name=None):
-        for task_name in self.task_names:
-            for modality in self.results[task_name].keys():
-                for entry in self.results[task_name][modality]:
-                    reps = []
-                    for i, mod_idx in enumerate(entry.modality_combo):
-                        reps.append(self.k_best_modalities[task_name][mod_idx])
-                    entry.representations = reps
+        if self.debug:
+            print(f"\nOptimization completed")
 
-        import pickle
+        return all_results
 
+    def store_results(self, file_name=None):
         if file_name is None:
             import time
 
@@ -387,14 +295,14 @@ class MultimodalResults:
             file_name = "multimodal_optimizer" + timestr + ".pkl"
 
         with open(file_name, "wb") as f:
-            pickle.dump(self.results, f)
+            pickle.dump(self.optimization_results, f)
 
 
-@dataclasses.dataclass
-class MultimodalResultEntry:
-    val_score: float
-    modality_combo: list
-    representations: list
-    fusion_methods: list
+@dataclass
+class OptimizationResult:
+    dag: RepresentationDag
     train_score: float
-    task: str
+    val_score: float
+    runtime: float
+    task_name: str
+    evaluation_time: float = 0.0
diff --git a/src/main/python/systemds/scuro/drsearch/operator_registry.py b/src/main/python/systemds/scuro/drsearch/operator_registry.py
index 699dcad857..9bc90720f8 100644
--- a/src/main/python/systemds/scuro/drsearch/operator_registry.py
+++ b/src/main/python/systemds/scuro/drsearch/operator_registry.py
@@ -77,6 +77,12 @@ class Registry:
     def get_fusion_operators(self):
         return self._fusion_operators
 
+    def get_fusion_operator_by_name(self, fusion_name):
+        for fusion in self._fusion_operators:
+            if fusion.__name__ == fusion_name:
+                return fusion
+        return None
+
     def get_representation_by_name(self, representation_name, modality_type):
         for representation in self._context_operators:
             if representation.__name__ == representation_name:
diff --git a/src/main/python/systemds/scuro/drsearch/representation_cache.py b/src/main/python/systemds/scuro/drsearch/representation_cache.py
deleted file mode 100644
index 4df478272d..0000000000
--- a/src/main/python/systemds/scuro/drsearch/representation_cache.py
+++ /dev/null
@@ -1,128 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
-import copy
-import os
-import pickle
-import tempfile
-
-from systemds.scuro.modality.transformed import TransformedModality
-
-
-class RepresentationCache:
-    """ """
-
-    _instance = None
-    _cache_dir = None
-    debug = False
-
-    def __new__(cls, debug=False):
-        if not cls._instance:
-            cls.debug = debug
-            cls._instance = super().__new__(cls)
-            cls._cache_dir = tempfile.TemporaryDirectory()
-            # cls._cache_dir = "representation_cache"
-        return cls._instance
-
-    def _generate_cache_filename(self, modality_id, operators):
-        """
-        Generate a unique filename for an operator based on its name.
-
-        :param operator_name: The name of the operator.
-        :return: A full path to the cache file.
-        """
-        op_names = []
-        filename = modality_id
-        for operator in operators:
-            if isinstance(operator, str):
-                op_names.append(operator)
-                filename += operator
-            else:
-                op_names.append(operator.name)
-                filename += operator.name
-
-        return os.path.join(self._cache_dir.name, filename), op_names  # _cache_dir.name
-
-    def save_to_cache(self, modality, used_op_names, operators):
-        """
-        Save data to a cache file.
-
-        :param operator_name: The name of the operator.
-        :param data: The data to save.
-        """
-        filename, op_names = self._generate_cache_filename(
-            str(modality.modality_id) + used_op_names, operators
-        )
-        if not os.path.exists(filename):
-            with open(f"{filename}.pkl", "wb") as f:
-                pickle.dump(modality.data, f)
-
-            with open(f"{filename}.meta", "wb") as f:
-                pickle.dump(modality.metadata, f)
-
-            if self.debug:
-                str_names = ", ".join(op_names)
-                print(
-                    f"Saved data for operator {str(modality.modality_id)}{used_op_names}{str_names} to cache: {filename}"
-                )
-
-    def load_from_cache(self, modality, operators):
-        """
-        Load data from a cache file if it exists.
-
-        :param operator_name: The name of the operator.
-        :return: The cached data or None if not found.
-        """
-        ops = copy.deepcopy(operators)
-        filename, op_names = self._generate_cache_filename(
-            str(modality.modality_id), ops
-        )
-        dropped_ops = []
-        while not os.path.exists(f"{filename}.pkl"):
-            op_names.pop()
-            dropped_ops.append(ops.pop())
-            if len(ops) < 1:
-                break
-            filename, op_names = self._generate_cache_filename(
-                str(modality.modality_id), ops
-            )
-
-        dropped_ops.reverse()
-        op_names = "".join(op_names)
-
-        if os.path.exists(f"{filename}.pkl"):
-            with open(f"{filename}.meta", "rb") as f:
-                metadata = pickle.load(f)
-
-            transformed_modality = TransformedModality(
-                modality,
-                op_names,
-            )
-            data = None
-            with open(f"{filename}.pkl", "rb") as f:
-                if self.debug:
-                    print(
-                        f"Loaded cached data for operator '{str(modality.modality_id) + op_names}' from {filename}"
-                    )
-                data = pickle.load(f)
-            transformed_modality.data = data
-            return transformed_modality, dropped_ops, op_names
-
-        return None, dropped_ops, op_names
diff --git a/src/main/python/systemds/scuro/drsearch/representation_dag.py b/src/main/python/systemds/scuro/drsearch/representation_dag.py
new file mode 100644
index 0000000000..1d5f512eb8
--- /dev/null
+++ b/src/main/python/systemds/scuro/drsearch/representation_dag.py
@@ -0,0 +1,232 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+import copy
+from dataclasses import dataclass, field
+from typing import List, Dict, Any
+from systemds.scuro.modality.modality import Modality
+from systemds.scuro.modality.transformed import TransformedModality
+from systemds.scuro.representations.representation import (
+    Representation as UnimodalRepresentation,
+)
+from systemds.scuro.representations.aggregated_representation import (
+    AggregatedRepresentation,
+)
+from systemds.scuro.representations.context import Context
+from systemds.scuro.utils.identifier import get_op_id, get_node_id
+
+
+@dataclass
+class RepresentationNode:
+    node_id: str
+    operation: Any
+    inputs: List[str]
+    modality_id: str = None
+    representation_index: int = None
+    parameters: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class RepresentationDag:
+
+    def __init__(self, nodes: List[Any], root_node_id):
+        self.root_node_id = root_node_id
+        self.nodes = self.filter_connected_nodes(nodes)
+
+    def filter_connected_nodes(self, nodes):
+        node_map = {node.node_id: node for node in nodes}
+
+        if self.root_node_id not in node_map:
+            return []
+
+        visited = set()
+        stack = [self.root_node_id]
+
+        while stack:
+            current_id = stack.pop()
+            if current_id not in visited:
+                visited.add(current_id)
+
+                current_node = node_map[current_id]
+                for input_id in current_node.inputs:
+                    if input_id in node_map and input_id not in visited:
+                        stack.append(input_id)
+
+        return [node for node in nodes if node.node_id in visited]
+
+    def get_leaf_nodes(self) -> List[str]:
+        leaf_nodes = []
+        for node in self.nodes:
+            if not node.inputs:
+                leaf_nodes.append(node.node_id)
+        return leaf_nodes
+
+    def get_node_by_id(self, node_id: str):
+        for node in self.nodes:
+            if node.node_id == node_id:
+                return node
+        return None
+
+    def get_children(self, node_id: str) -> List[str]:
+        children = []
+        for node in self.nodes:
+            if node_id in node.inputs:
+                children.append(node.node_id)
+        return children
+
+    def validate(self) -> bool:
+        node_ids = {node.node_id for node in self.nodes}
+
+        if self.root_node_id not in node_ids:
+            return False
+
+        for node in self.nodes:
+            for input_id in node.inputs:
+                if input_id not in node_ids:
+                    return False
+
+        visited = set()
+
+        def has_cycle(node_id: str, path: set) -> bool:
+            if node_id in path:
+                return True
+            if node_id in visited:
+                return False
+            path.add(node_id)
+            visited.add(node_id)
+            node = self.get_node_by_id(node_id)
+            for input_id in node.inputs:
+                if has_cycle(input_id, path.copy()):
+                    return True
+            return False
+
+        return not has_cycle(self.root_node_id, set())
+
+    def execute(
+        self, modalities: List[Modality], task=None
+    ) -> Dict[str, TransformedModality]:
+        cache = {}
+
+        def execute_node(node_id: str, task) -> TransformedModality:
+            if node_id in cache:
+                return cache[node_id]
+
+            node = self.get_node_by_id(node_id)
+
+            if not node.inputs:
+                modality = get_modality_by_id_and_instance_id(
+                    modalities, node.modality_id, node.representation_index
+                )
+                cache[node_id] = modality
+                return modality
+
+            input_mods = [execute_node(input_id, task) for input_id in node.inputs]
+
+            node_operation = node.operation()
+            if len(input_mods) == 1:
+                # It's a unimodal operation
+                if isinstance(node_operation, Context):
+                    result = input_mods[0].context(node_operation)
+                elif isinstance(node_operation, AggregatedRepresentation):
+                    result = node_operation.transform(input_mods[0])
+                elif isinstance(node_operation, UnimodalRepresentation):
+                    if (
+                        isinstance(input_mods[0], TransformedModality)
+                        and input_mods[0].transformation[0].__class__ == node.operation
+                    ):
+                        # Avoid duplicate transformations
+                        result = input_mods[0]
+                    else:
+                        # Compute the representation
+                        result = input_mods[0].apply_representation(node_operation)
+            else:
+                # It's a fusion operation
+                fusion_op = node_operation
+                if hasattr(fusion_op, "needs_training") and fusion_op.needs_training:
+                    result = input_mods[0].combine_with_training(
+                        input_mods[1:], fusion_op, task
+                    )
+                else:
+                    result = input_mods[0].combine(input_mods[1:], fusion_op)
+
+            cache[node_id] = result
+            return result
+
+        execute_node(self.root_node_id, task)
+
+        return cache
+
+
+def get_modality_by_id_and_instance_id(
+    modalities: List[Modality], modality_id: int, instance_id: int
+):
+    counter = 0
+    for modality in modalities:
+        if modality.modality_id == modality_id:
+            if counter == instance_id or instance_id == -1:
+                return modality
+            else:
+                counter += 1
+    return None
+
+
+class RepresentationDAGBuilder:
+    def __init__(self):
+        self.nodes = []
+        self.node_counter = 0
+
+    def create_leaf_node(
+        self, modality_id: str, representation_index: int = -1, operation=None
+    ) -> str:
+        if representation_index != -1:
+            node_id = f"leaf_{modality_id}_{representation_index}"
+        else:
+            node_id = f"leaf_{get_node_id()}"
+        node = RepresentationNode(
+            node_id=node_id,
+            inputs=[],
+            operation=operation,
+            modality_id=modality_id,
+            representation_index=representation_index,
+        )
+        self.nodes.append(node)
+        return node_id
+
+    def create_operation_node(
+        self, operation: Any, inputs: List[str], parameters: Dict[str, Any] = None
+    ) -> str:
+        node_id = f"op_{get_op_id()}"
+        self.node_counter += 1
+        node = RepresentationNode(
+            node_id=node_id,
+            inputs=inputs,
+            operation=operation,
+            parameters=parameters or {},
+        )
+        self.nodes.append(node)
+        return node_id
+
+    def build(self, root_node_id: str) -> RepresentationDag:
+        dag = RepresentationDag(
+            nodes=copy.deepcopy(self.nodes), root_node_id=root_node_id
+        )
+        if not dag.validate():
+            raise ValueError("Invalid DAG construction")
+        return dag
diff --git a/src/main/python/systemds/scuro/representations/aggregated_representation.py b/src/main/python/systemds/scuro/drsearch/representation_dag_visualizer.py
similarity index 50%
copy from src/main/python/systemds/scuro/representations/aggregated_representation.py
copy to src/main/python/systemds/scuro/drsearch/representation_dag_visualizer.py
index 9119070a02..b0a1bbe285 100644
--- a/src/main/python/systemds/scuro/representations/aggregated_representation.py
+++ b/src/main/python/systemds/scuro/drsearch/representation_dag_visualizer.py
@@ -18,19 +18,38 @@
 # under the License.
 #
 # -------------------------------------------------------------
-from systemds.scuro.modality.transformed import TransformedModality
-from systemds.scuro.representations.representation import Representation
 
+from typing import Dict, Any
+from systemds.scuro.drsearch.representation_dag import RepresentationDag
 
-class AggregatedRepresentation(Representation):
-    def __init__(self, aggregation):
-        super().__init__("AggregatedRepresentation", aggregation.parameters)
-        self.aggregation = aggregation
-        self.self_contained = True
 
-    def transform(self, modality):
-        aggregated_modality = TransformedModality(
-            modality, self, self_contained=modality.self_contained
+def visualize_dag(dag: RepresentationDag) -> Dict[str, Any]:
+    nodes = []
+    edges = []
+
+    for i, node in enumerate(dag.nodes):
+        # Create node entry
+        node_type = "operation" if node.operation else "modality"
+        label = node.operation if node.operation else f"Modality: {node.modality_id}"
+
+        nodes.append(
+            {
+                "id": node.node_id,
+                "label": label,
+                "type": node_type,
+                "parameters": node.parameters,
+            }
         )
-        aggregated_modality.data = self.aggregation.execute(modality)
-        return aggregated_modality
+
+        print(nodes[i])
+
+        # Create edges
+        for input_id in node.inputs:
+            edges.append({"from": input_id, "to": node.node_id})
+
+    for edge in edges:
+        print(edge)
+
+    print(f"Root Node ID: {dag.root_node_id}")
+
+    return {"nodes": nodes, "edges": edges, "root": dag.root_node_id}
diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py
index 86c7ce1e63..10b127f5b6 100644
--- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py
+++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py
@@ -10,7 +10,7 @@
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing,
+# Unless required by applicable law or agreed in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
@@ -20,37 +20,46 @@
 # -------------------------------------------------------------
 import pickle
 import time
-import copy
 from concurrent.futures import ProcessPoolExecutor, as_completed
-from dataclasses import dataclass, field, asdict
-
+from dataclasses import dataclass
 import multiprocessing as mp
-from typing import Union
+from typing import List, Any
+from functools import lru_cache
 
-import numpy as np
-from systemds.scuro.representations.window_aggregation import WindowAggregation
+from systemds.scuro import ModalityType
+from systemds.scuro.representations.fusion import Fusion
 from systemds.scuro.representations.concatenation import Concatenation
 from systemds.scuro.representations.hadamard import Hadamard
 from systemds.scuro.representations.sum import Sum
-
 from systemds.scuro.representations.aggregated_representation import (
     AggregatedRepresentation,
 )
-from systemds.scuro.modality.type import ModalityType
 from systemds.scuro.modality.modality import Modality
-from systemds.scuro.modality.transformed import TransformedModality
 from systemds.scuro.representations.aggregate import Aggregation
 from systemds.scuro.drsearch.operator_registry import Registry
 from systemds.scuro.utils.schema_helpers import get_shape
+from systemds.scuro.drsearch.representation_dag import (
+    RepresentationDag,
+    RepresentationNode,
+    RepresentationDAGBuilder,
+)
+from systemds.scuro.drsearch.representation_dag_visualizer import visualize_dag
 
 
 class UnimodalOptimizer:
     def __init__(self, modalities, tasks, debug=True):
         self.modalities = modalities
         self.tasks = tasks
+        self.run = None
+
+        self.builders = {
+            modality.modality_id: RepresentationDAGBuilder() for modality in modalities
+        }
+
+        self.debug = debug
 
         self.operator_registry = Registry()
-        self.operator_performance = UnimodalResults(modalities, tasks, debug)
+        self.operator_performance = UnimodalResults(modalities, tasks, debug, self.run)
 
         self._tasks_require_same_dims = True
         self.expected_dimensions = tasks[0].expected_dim
@@ -60,6 +69,22 @@ class UnimodalOptimizer:
             if tasks[i - 1].expected_dim != tasks[i].expected_dim:
                 self._tasks_require_same_dims = False
 
+        self._combination_operators = [Concatenation(), Hadamard(), Sum()]
+
+    @lru_cache(maxsize=128)
+    def _get_modality_operators(self, modality_type):
+        return self.operator_registry.get_representations(modality_type)
+
+    @lru_cache(maxsize=128)
+    def _get_not_self_contained_reps(self, modality_type):
+        return self.operator_registry.get_not_self_contained_representations(
+            modality_type
+        )
+
+    @lru_cache(maxsize=32)
+    def _get_context_operators(self):
+        return self.operator_registry.get_context_operators()
+
     def store_results(self, file_name=None):
         if file_name is None:
             import time
@@ -82,80 +107,59 @@ class UnimodalOptimizer:
 
             for future in as_completed(future_to_modality):
                 modality = future_to_modality[future]
-                # try:
                 results = future.result()
                 self._merge_results(results)
-                # except Exception as exc:
-                #     print(f'Modality {modality.modality_id} generated an exception: {exc}')
 
     def optimize(self):
+        """Optimize representations for each modality"""
         for modality in self.modalities:
             local_result = self._process_modality(modality, False)
 
     def _process_modality(self, modality, parallel):
         if parallel:
-            local_results = UnimodalResults(
-                modalities=[modality], tasks=self.tasks, debug=False
-            )
+            local_results = UnimodalResults([modality], self.tasks, debug=False)
         else:
             local_results = self.operator_performance
 
-        context_operators = self.operator_registry.get_context_operators()
-        not_self_contained_reps = (
-            self.operator_registry.get_not_self_contained_representations(
-                modality.modality_type
-            )
-        )
-        modality_specific_operators = self.operator_registry.get_representations(
+        modality_specific_operators = self._get_modality_operators(
             modality.modality_type
         )
-        for modality_specific_operator in modality_specific_operators:
-            mod_op = modality_specific_operator()
 
-            mod = modality.apply_representation(mod_op)
-            self._evaluate_local(mod, [mod_op], local_results)
+        for operator in modality_specific_operators:
+            dags = self._build_modality_dag(modality, operator())
 
-            if not mod_op.self_contained:
-                self._combine_non_self_contained_representations(
-                    modality, mod, not_self_contained_reps, local_results
-                )
+            for dag in dags:
+                representations = dag.execute([modality])
+                node_id = list(representations.keys())[-1]
+                node = dag.get_node_by_id(node_id)
+                if node.operation is None:
+                    continue
 
-            for context_operator_after in context_operators:
-                con_op_after = context_operator_after()
-                mod_con = mod.context(con_op_after)
-                self._evaluate_local(mod_con, [mod_op, con_op_after], local_results)
-
-            return local_results
-
-    def _combine_non_self_contained_representations(
-        self,
-        modality: Modality,
-        representation: TransformedModality,
-        other_representations,
-        local_results,
-    ):
-        combined = representation
-        context_operators = self.operator_registry.get_context_operators()
-        used_representations = representation.transformation
-        for other_representation in other_representations:
-            used_representations.append(other_representation())
-            for combination in [Concatenation(), Hadamard(), Sum()]:
-                combined = combined.combine(
-                    modality.apply_representation(other_representation()), combination
-                )
+                reps = self._get_representation_chain(node, dag)
+                combination = next((op for op in reps if isinstance(op, Fusion)), None)
                 self._evaluate_local(
-                    combined, used_representations, local_results, combination
+                    representations[node_id], local_results, dag, combination
                 )
+                if self.debug:
+                    visualize_dag(dag)
+
+        return local_results
 
-                for context_op in context_operators:
-                    con_op = context_op()
-                    mod = combined.context(con_op)
-                    c_t = copy.deepcopy(used_representations)
-                    c_t.append(con_op)
-                    self._evaluate_local(mod, c_t, local_results, combination)
+    def _get_representation_chain(
+        self, node: "RepresentationNode", dag: RepresentationDag
+    ) -> List[Any]:
+        representations = []
+        if node.operation:
+            representations.append(node.operation)
+
+        for input_id in node.inputs:
+            input_node = dag.get_node_by_id(input_id)
+            if input_node.operation:
+                representations.extend(self._get_representation_chain(input_node, dag))
+
+        return representations
 
     def _merge_results(self, local_results):
-        """Merge local results into the main results"""
         for modality_id in local_results.results:
             for task_name in local_results.results[modality_id]:
                 self.operator_performance.results[modality_id][task_name].extend(
@@ -167,82 +171,121 @@ class UnimodalOptimizer:
                 for key, value in local_results.cache[modality][task_name].items():
                     self.operator_performance.cache[modality][task_name][key] = value
 
-    def _evaluate_local(
-        self, modality, representations, local_results, combination=None
-    ):
+    def _evaluate_local(self, modality, local_results, dag, combination=None):
         if self._tasks_require_same_dims:
             if self.expected_dimensions == 1 and get_shape(modality.metadata) > 1:
-                # for aggregation in Aggregation().get_aggregation_functions():
-                agg_operator = AggregatedRepresentation(Aggregation())
-                agg_modality = agg_operator.transform(modality)
-                reps = representations.copy()
-                reps.append(agg_operator)
-                # agg_modality.pad()
+                builder = self.builders[modality.modality_id]
+                agg_operator = AggregatedRepresentation()
+                rep_node_id = builder.create_operation_node(
+                    agg_operator.__class__, [dag.root_node_id], agg_operator.parameters
+                )
+                dag = builder.build(rep_node_id)
+                representations = dag.execute([modality])
+                node_id = list(representations.keys())[-1]
                 for task in self.tasks:
-                    start = time.time()
-                    scores = task.run(agg_modality.data)
-                    end = time.time()
+                    start = time.perf_counter()
+                    scores = task.run(representations[node_id].data)
+                    end = time.perf_counter()
 
                     local_results.add_result(
-                        scores,
-                        reps,
-                        modality,
-                        task.model.name,
-                        end - start,
-                        combination,
+                        scores, modality, task.model.name, end - start, combination, dag
                     )
             else:
                 modality.pad()
                 for task in self.tasks:
-                    start = time.time()
+                    start = time.perf_counter()
                     scores = task.run(modality.data)
-                    end = time.time()
+                    end = time.perf_counter()
                     local_results.add_result(
-                        scores,
-                        representations,
-                        modality,
-                        task.model.name,
-                        end - start,
-                        combination,
+                        scores, modality, task.model.name, end - start, combination, dag
                     )
         else:
             for task in self.tasks:
                 if task.expected_dim == 1 and get_shape(modality.metadata) > 1:
-                    # for aggregation in Aggregation().get_aggregation_functions():
+                    builder = self.builders[modality.modality_id]
                     agg_operator = AggregatedRepresentation(Aggregation())
-                    agg_modality = agg_operator.transform(modality)
-
-                    reps = representations.copy()
-                    reps.append(agg_operator)
-                    # modality.pad()
-                    start = time.time()
-                    scores = task.run(agg_modality.data)
-                    end = time.time()
+                    rep_node_id = builder.create_operation_node(
+                        agg_operator.__class__,
+                        [dag.root_node_id],
+                        agg_operator.parameters,
+                    )
+                    dag = builder.build(rep_node_id)
+                    representations = dag.execute([modality])
+                    node_id = list(representations.keys())[-1]
+
+                    start = time.perf_counter()
+                    scores = task.run(representations[node_id].data)
+                    end = time.perf_counter()
                     local_results.add_result(
-                        scores,
-                        reps,
-                        modality,
-                        task.model.name,
-                        end - start,
-                        combination,
+                        scores, modality, task.model.name, end - start, combination, dag
                     )
                 else:
-                    # modality.pad()
-                    start = time.time()
+                    start = time.perf_counter()
                     scores = task.run(modality.data)
-                    end = time.time()
+                    end = time.perf_counter()
                     local_results.add_result(
-                        scores,
-                        representations,
-                        modality,
-                        task.model.name,
-                        end - start,
-                        combination,
+                        scores, modality, task.model.name, end - start, combination, dag
+                    )
+
+    def _build_modality_dag(
+        self, modality: Modality, operator: Any
+    ) -> List[RepresentationDag]:
+        dags = []
+        builder = self.builders[modality.modality_id]
+        leaf_id = builder.create_leaf_node(modality.modality_id)
+
+        rep_node_id = builder.create_operation_node(
+            operator.__class__, [leaf_id], operator.parameters
+        )
+        current_node_id = rep_node_id
+        dags.append(builder.build(current_node_id))
+
+        if not operator.self_contained:
+            not_self_contained_reps = self._get_not_self_contained_reps(
+                modality.modality_type
+            )
+            not_self_contained_reps = [
+                rep for rep in not_self_contained_reps if rep != operator.__class__
+            ]
+
+            for combination in self._combination_operators:
+                current_node_id = rep_node_id
+                for other_rep in not_self_contained_reps:
+                    other_rep_id = builder.create_operation_node(
+                        other_rep, [leaf_id], other_rep().parameters
                     )
 
+                    combine_id = builder.create_operation_node(
+                        combination.__class__,
+                        [current_node_id, other_rep_id],
+                        combination.parameters,
+                    )
+                    dags.append(builder.build(combine_id))
+                    current_node_id = combine_id
+
+        context_operators = self._get_context_operators()
+
+        for context_op in context_operators:
+            if modality.modality_type != ModalityType.TEXT:
+                context_node_id = builder.create_operation_node(
+                    context_op,
+                    [leaf_id],
+                    context_op().parameters,
+                )
+                dags.append(builder.build(context_node_id))
+
+            context_node_id = builder.create_operation_node(
+                context_op,
+                [current_node_id],
+                context_op().parameters,
+            )
+            dags.append(builder.build(context_node_id))
+
+        return dags
+
 
 class UnimodalResults:
-    def __init__(self, modalities, tasks, debug=False):
+    def __init__(self, modalities, tasks, debug=False, run=None):
         self.modality_ids = [modality.modality_id for modality in modalities]
         self.task_names = [task.model.name for task in tasks]
         self.results = {}
@@ -250,48 +293,27 @@ class UnimodalResults:
         self.cache = {}
 
         for modality in self.modality_ids:
-            self.results[modality] = {}
-            self.cache[modality] = {}
-            for task_name in self.task_names:
-                self.cache[modality][task_name] = {}
-                self.results[modality][task_name] = []
-
-    def add_result(
-        self, scores, representations, modality, task_name, task_time, combination
-    ):
-        parameters = []
-        representation_names = []
-
-        for rep in representations:
-            representation_names.append(type(rep).__name__)
-            if isinstance(rep, AggregatedRepresentation):
-                parameters.append(rep.parameters)
-                continue
-
-            params = {}
-            for param in list(rep.parameters.keys()):
-                params[param] = getattr(rep, param)
-
-            if isinstance(rep, WindowAggregation):
-                params["aggregation_function"] = (
-                    rep.aggregation_function.aggregation_function_name
-                )
-
-            parameters.append(params)
+            self.results[modality] = {task_name: [] for task_name in self.task_names}
+            self.cache[modality] = {task_name: {} for task_name in self.task_names}
 
+    def add_result(self, scores, modality, task_name, task_time, combination, dag):
         entry = ResultEntry(
-            representations=representation_names,
-            params=parameters,
             train_score=scores[0],
             val_score=scores[1],
             representation_time=modality.transform_time,
             task_time=task_time,
             combination=combination.name if combination else "",
+            dag=dag,
         )
+
         self.results[modality.modality_id][task_name].append(entry)
-        self.cache[modality.modality_id][task_name][
-            (tuple(representation_names), scores[1], modality.transform_time)
-        ] = modality
+
+        cache_key = (
+            id(dag),
+            scores[1],
+            modality.transform_time,
+        )
+        self.cache[modality.modality_id][task_name][cache_key] = modality
 
         if self.debug:
             print(f"{modality.modality_id}_{task_name}: {entry}")
@@ -308,29 +330,29 @@ class UnimodalResults:
         :param modality: modality to get the best results for
         :param k: number of best results
         """
-        items = self.results[modality.modality_id][task.model.name]
-        sorted_indices = sorted(
-            range(len(items)), key=lambda x: items[x].val_score, reverse=True
-        )[:k]
+        task_results = self.results[modality.modality_id][task.model.name]
+
+        results = sorted(task_results, key=lambda x: x.val_score, reverse=True)[:k]
 
-        results = sorted(
-            self.results[modality.modality_id][task.model.name],
-            key=lambda x: x.val_score,
+        sorted_indices = sorted(
+            range(len(task_results)),
+            key=lambda x: task_results[x].val_score,
             reverse=True,
         )[:k]
 
-        items = list(self.cache[modality.modality_id][task.model.name].items())
-        reordered_cache = [items[i][1] for i in sorted_indices]
+        cache_items = list(self.cache[modality.modality_id][task.model.name].items())
+        reordered_cache = [
+            cache_items[i][1] for i in sorted_indices if i < len(cache_items)
+        ]
 
-        return results, list(reordered_cache)
+        return results, reordered_cache
 
 
 @dataclass(frozen=True)
 class ResultEntry:
     val_score: float
-    representations: list
-    params: list
     train_score: float
     representation_time: float
     task_time: float
     combination: str
+    dag: RepresentationDag
diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_representation_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_representation_optimizer.py
deleted file mode 100644
index e59ddbe9be..0000000000
--- a/src/main/python/systemds/scuro/drsearch/unimodal_representation_optimizer.py
+++ /dev/null
@@ -1,271 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
-import copy
-import os
-import pickle
-import time
-from typing import List
-
-from systemds.scuro.drsearch.operator_registry import Registry
-from systemds.scuro.drsearch.optimization_data import OptimizationResult
-from systemds.scuro.drsearch.representation_cache import RepresentationCache
-from systemds.scuro.drsearch.task import Task
-from systemds.scuro.modality.modality import Modality
-from systemds.scuro.representations.aggregate import Aggregation
-from systemds.scuro.representations.context import Context
-
-
-class UnimodalRepresentationOptimizer:
-    def __init__(
-        self,
-        modalities: List[Modality],
-        tasks: List[Task],
-        max_chain_depth=5,
-        debug=False,
-        folder_name=None,
-    ):
-        self.optimization_results = {}
-        self.modalities = modalities
-        self.tasks = tasks
-        self.operator_registry = Registry()
-        self.initialize_optimization_results()
-        self.max_chain_depth = max_chain_depth
-        self.debug = debug
-        self.cache = RepresentationCache(self.debug)
-        if self.debug:
-            self.folder_name = folder_name
-            os.makedirs(self.folder_name, exist_ok=True)
-
-    def initialize_optimization_results(self):
-        for modality in self.modalities:
-            self.optimization_results[modality.modality_id] = {}
-            for task in self.tasks:
-                self.optimization_results[modality.modality_id][task.model.name] = []
-
-    def optimize(self):
-        """
-        This method finds different unimodal representations for all given modalities
-        """
-
-        for modality in self.modalities:
-            self._optimize_modality(modality)
-
-            copy_results = copy.deepcopy(
-                self.optimization_results[modality.modality_id]
-            )
-            for model in copy_results:
-                for i, model_task in enumerate(copy_results[model]):
-                    ops = []
-                    for op in model_task.operator_chain:
-                        if not isinstance(op, str):
-                            ops.append(op.name)
-                    if len(ops) > 0:
-                        copy_results[model][i].operator_chain = ops
-                if self.debug:
-                    with open(
-                        f"{self.folder_name}/results_{model}_{modality.modality_type.name}.p",
-                        "wb",
-                    ) as fp:
-                        pickle.dump(
-                            copy_results[model], fp, protocol=pickle.HIGHEST_PROTOCOL
-                        )
-
-    def get_k_best_results(self, modality: Modality, k: int, task: Task):
-        """
-        Get the k best results for the given modality
-        :param modality: modality to get the best results for
-        :param k: number of best results
-        """
-        results = sorted(
-            self.optimization_results[modality.modality_id][task.model.name],
-            key=lambda x: x.test_accuracy,
-            reverse=True,
-        )[:k]
-
-        return results
-
-    def _optimize_modality(self, modality: Modality):
-        """
-        Optimize a single modality by leveraging modality specific heuristics and incorporating context and
-        stores the resulting operation chains as optimization results.
-        :param modality: modality to optimize
-        """
-
-        representations = self._get_compatible_operators(modality.modality_type, [])
-
-        for rep in representations:
-            self._build_operator_chain(modality, [rep()], 1)
-
-    def _get_compatible_operators(self, modality_type, used_operators):
-        next_operators = []
-        for operator in self.operator_registry.get_representations(modality_type):
-            if operator.__name__ not in used_operators:
-                next_operators.append(operator)
-
-        for context_operator in self.operator_registry.get_context_operators():
-            if (
-                len(used_operators) == 0
-                or context_operator.__name__ not in used_operators[-1]
-            ):
-                next_operators.append(context_operator)
-
-        return next_operators
-
-    def _build_operator_chain(self, modality, current_operator_chain, depth):
-
-        if depth > self.max_chain_depth:
-            return
-
-        self._apply_operator_chain(modality, current_operator_chain)
-
-        current_modality_type = modality.modality_type
-
-        for operator in current_operator_chain:
-            if hasattr(operator, "output_modality_type"):
-                current_modality_type = operator.output_modality_type
-
-        next_representations = self._get_compatible_operators(
-            current_modality_type, [type(op).__name__ for op in current_operator_chain]
-        )
-
-        for next_rep in next_representations:
-            rep_instance = next_rep()
-            new_chain = current_operator_chain + [rep_instance]
-            self._build_operator_chain(modality, new_chain, depth + 1)
-
-    def _evaluate_with_flattened_data(
-        self, modality, operator_chain, op_params, representation_time, task
-    ):
-        from systemds.scuro.representations.aggregated_representation import (
-            AggregatedRepresentation,
-        )
-
-        results = []
-        for aggregation in Aggregation().get_aggregation_functions():
-            start = time.time()
-            agg_operator = AggregatedRepresentation(Aggregation(aggregation, True))
-            agg_modality = agg_operator.transform(modality)
-            end = time.time()
-
-            agg_opperator_chain = operator_chain + [agg_operator]
-            agg_params = dict(op_params)
-            agg_params.update({agg_operator.name: agg_operator.parameters})
-
-            score = task.run(agg_modality.data)
-            result = OptimizationResult(
-                operator_chain=agg_opperator_chain,
-                parameters=agg_params,
-                train_accuracy=score[0],
-                test_accuracy=score[1],
-                # train_min_it_acc=score[2],
-                # test_min_it_acc=score[3],
-                training_runtime=task.training_time,
-                inference_runtime=task.inference_time,
-                representation_time=representation_time + end - start,
-                output_shape=(1, 1),  # TODO
-            )
-            results.append(result)
-
-            if self.debug:
-                op_name = ""
-                for operator in agg_opperator_chain:
-                    op_name += str(operator.__class__.__name__)
-                print(f"{task.name} {task.model.name} {op_name}: {score[1]}")
-
-        return results
-
-    def _evaluate_operator_chain(
-        self, modality, operator_chain, op_params, representation_time
-    ):
-        for task in self.tasks:
-            if isinstance(modality.data[0], str):
-                continue
-
-            if (
-                task.expected_dim == 1
-                and not isinstance(modality.data[0], list)
-                and modality.data[0].ndim > 1
-            ):
-                r = self._evaluate_with_flattened_data(
-                    modality, operator_chain, op_params, representation_time, task
-                )
-                self.optimization_results[modality.modality_id][task.model.name].extend(
-                    r
-                )
-            else:
-                score = task.run(modality.data)
-                result = OptimizationResult(
-                    operator_chain=operator_chain,
-                    parameters=op_params,
-                    train_accuracy=score[0],
-                    test_accuracy=score[1],
-                    # train_min_it_acc=score[2],
-                    # test_min_it_acc=score[3],
-                    training_runtime=task.training_time,
-                    inference_runtime=task.inference_time,
-                    representation_time=representation_time,
-                    output_shape=(1, 1),
-                )  # TODO
-                self.optimization_results[modality.modality_id][task.model.name].append(
-                    result
-                )
-                if self.debug:
-                    op_name = ""
-                    for operator in operator_chain:
-                        op_name += str(operator.__class__.__name__)
-                    print(f"{task.name} {task.model.name} - {op_name}: {score[1]}")
-
-    def _apply_operator_chain(self, current_modality, operator_chain):
-        op_params = {}
-        modified_modality = current_modality
-
-        representation_start = time.time()
-        try:
-            cached_representation, representation_ops, used_op_names = (
-                self.cache.load_from_cache(
-                    modified_modality, copy.deepcopy(operator_chain)
-                )
-            )
-            if cached_representation is not None:
-                modified_modality = cached_representation
-            store = False
-            for operator in representation_ops:
-                if isinstance(operator, Context):
-                    modified_modality = modified_modality.context(operator)
-                else:
-                    modified_modality = modified_modality.apply_representation(operator)
-                store = True
-                op_params[operator.name] = operator.get_current_parameters()
-            if store:
-                self.cache.save_to_cache(
-                    modified_modality, used_op_names, representation_ops
-                )
-            representation_end = time.time()
-
-            self._evaluate_operator_chain(
-                modified_modality,
-                operator_chain,
-                op_params,
-                representation_end - representation_start,
-            )
-        except Exception as e:
-            print(f"Failed to evaluate chain {operator_chain}: {str(e)}")
-            return
diff --git a/src/main/python/systemds/scuro/modality/joined_transformed.py b/src/main/python/systemds/scuro/modality/joined_transformed.py
index 3e0d8fb9df..078ce86f7a 100644
--- a/src/main/python/systemds/scuro/modality/joined_transformed.py
+++ b/src/main/python/systemds/scuro/modality/joined_transformed.py
@@ -72,7 +72,7 @@ class JoinedTransformedModality(Modality):
         return self
 
     def window_aggregation(self, window_size, aggregation):
-        w = WindowAggregation(window_size, aggregation)
+        w = WindowAggregation(aggregation, window_size)
         self.left_modality.data = w.execute(self.left_modality)
         self.right_modality.data = w.execute(self.right_modality)
         return self
diff --git a/src/main/python/systemds/scuro/modality/modality.py b/src/main/python/systemds/scuro/modality/modality.py
index f1b00fefcf..98dd631e12 100644
--- a/src/main/python/systemds/scuro/modality/modality.py
+++ b/src/main/python/systemds/scuro/modality/modality.py
@@ -132,6 +132,8 @@ class Modality:
         try:
             if max_len is None:
                 result = np.array(self.data)
+            elif isinstance(self.data, np.ndarray) and self.data.shape[1] == max_len:
+                result = self.data
             else:
                 raise "Needs padding to max_len"
         except:
@@ -144,8 +146,16 @@ class Modality:
             for i, seq in enumerate(self.data):
                 data = seq[:maxlen]
                 result[i, : len(data)] = data
-                # TODO: add padding to metadata as attention_masks
 
+                if self.has_metadata():
+                    attention_mask = np.zeros(result.shape[1], dtype=np.int8)
+                    attention_mask[: len(seq[:maxlen])] = 1
+                    md_key = list(self.metadata.keys())[i]
+                    if "attention_mask" in self.metadata[md_key]:
+                        self.metadata[md_key]["attention_mask"] = attention_mask
+                    else:
+                        self.metadata[md_key].update({"attention_mask": attention_mask})
+        # TODO: this might need to be a new modality (otherwise we loose the original data)
         self.data = result
 
     def get_data_layout(self):
diff --git a/src/main/python/systemds/scuro/modality/transformed.py b/src/main/python/systemds/scuro/modality/transformed.py
index 9481937e2c..7e8e54eff3 100644
--- a/src/main/python/systemds/scuro/modality/transformed.py
+++ b/src/main/python/systemds/scuro/modality/transformed.py
@@ -63,6 +63,7 @@ class TransformedModality(Modality):
     def add_transformation(self, transformation, modality):
         if (
             transformation.__class__.__bases__[0].__name__ == "Fusion"
+            and type(modality).__name__ == "TransformedModality"
             and modality.transformation[0].__class__.__bases__[0].__name__ != "Fusion"
         ):
             self.transformation = []
@@ -99,8 +100,8 @@ class TransformedModality(Modality):
 
         return joined_modality
 
-    def window_aggregation(self, windowSize, aggregation):
-        w = WindowAggregation(windowSize, aggregation)
+    def window_aggregation(self, window_size, aggregation):
+        w = WindowAggregation(aggregation, window_size)
         transformed_modality = TransformedModality(
             self, w, self_contained=self.self_contained
         )
@@ -135,11 +136,26 @@ class TransformedModality(Modality):
         fused_modality = TransformedModality(
             self, fusion_method, ModalityType.EMBEDDING
         )
+        fused_modality.data = fusion_method.transform(self.create_modality_list(other))
+
+        return fused_modality
+
+    def combine_with_training(
+        self, other: Union[Modality, List[Modality]], fusion_method, task
+    ):
+        fused_modality = TransformedModality(
+            self, fusion_method, ModalityType.EMBEDDING
+        )
+        modalities = self.create_modality_list(other)
+        fused_modality.data = fusion_method.transform_with_training(modalities, task)
+
+        return fused_modality
+
+    def create_modality_list(self, other: Union[Modality, List[Modality]]):
         modalities = [self]
         if isinstance(other, list):
             modalities.extend(other)
         else:
             modalities.append(other)
-        fused_modality.data = fusion_method.transform(modalities)
 
-        return fused_modality
+        return modalities
diff --git a/src/main/python/systemds/scuro/modality/type.py b/src/main/python/systemds/scuro/modality/type.py
index b2331d0fae..2853e8135d 100644
--- a/src/main/python/systemds/scuro/modality/type.py
+++ b/src/main/python/systemds/scuro/modality/type.py
@@ -230,6 +230,17 @@ class ModalityType(Flag):
         md["length"] = length
         return md
 
+    def create_ts_metadata(
+        self, signal_names, data, sampling_rate=None, is_single_instance=True
+    ):
+        md = deepcopy(self.get_schema())
+        md = ModalitySchemas.update_base_metadata(md, data, is_single_instance)
+        md["frequency"] = sampling_rate if sampling_rate is not None else 1
+        md["length"] = data.shape[0]
+        md["signal_names"] = signal_names
+        md["timestamp"] = create_timestamps(md["frequency"], md["length"])
+        return md
+
     def create_video_metadata(self, frequency, length, width, height, num_channels):
         md = deepcopy(self.get_schema())
         md["frequency"] = frequency
diff --git a/src/main/python/systemds/scuro/modality/unimodal_modality.py b/src/main/python/systemds/scuro/modality/unimodal_modality.py
index dd1674ea85..373921e95c 100644
--- a/src/main/python/systemds/scuro/modality/unimodal_modality.py
+++ b/src/main/python/systemds/scuro/modality/unimodal_modality.py
@@ -27,7 +27,7 @@ from systemds.scuro.dataloader.base_loader import BaseLoader
 from systemds.scuro.modality.modality import Modality
 from systemds.scuro.modality.joined import JoinedModality
 from systemds.scuro.modality.transformed import TransformedModality
-from systemds.scuro.modality.modality_identifier import ModalityIdentifier
+from systemds.scuro.utils.identifier import Identifier
 
 
 class UnimodalModality(Modality):
@@ -40,7 +40,7 @@ class UnimodalModality(Modality):
         """
         super().__init__(
             data_loader.modality_type,
-            ModalityIdentifier().new_id(),
+            Identifier().new_id(),
             {},
             data_loader.data_type,
         )
@@ -130,6 +130,11 @@ class UnimodalModality(Modality):
                 self.extract_raw_data()
             new_modality = representation.transform(self)
 
+            for i, d in enumerate(new_modality.data):
+                output = np.array(d)
+                if np.isnan(output).any():
+                    new_modality.data[i] = np.where(np.isnan(output), 0, output)
+
             if not all(
                 "attention_masks" in entry for entry in new_modality.metadata.values()
             ):
diff --git a/src/main/python/systemds/scuro/representations/aggregate.py b/src/main/python/systemds/scuro/representations/aggregate.py
index 2c046dc401..0a8438e684 100644
--- a/src/main/python/systemds/scuro/representations/aggregate.py
+++ b/src/main/python/systemds/scuro/representations/aggregate.py
@@ -20,6 +20,7 @@
 # -------------------------------------------------------------
 import numpy as np
 
+from systemds.scuro.modality.type import ModalityType
 from systemds.scuro.representations import utils
 
 
@@ -71,7 +72,13 @@ class Aggregation:
         for i, instance in enumerate(modality.data):
             data.append([])
             if isinstance(instance, np.ndarray):
-                aggregated_data = self._aggregation_func(instance)
+                if (
+                    modality.modality_type == ModalityType.IMAGE
+                    or modality.modality_type == ModalityType.VIDEO
+                ) and instance.ndim > 2:
+                    aggregated_data = instance.flatten()
+                else:
+                    aggregated_data = self._aggregation_func(instance)
             else:
                 aggregated_data = []
                 for entry in instance:
@@ -79,18 +86,17 @@ class Aggregation:
             max_len = max(max_len, len(aggregated_data))
             data[i] = aggregated_data
 
-        if self.pad_modality:
-            for i, instance in enumerate(data):
-                if isinstance(instance, np.ndarray):
-                    if len(instance) < max_len:
-                        padded_data = np.zeros(max_len, dtype=instance.dtype)
-                        padded_data[: len(instance)] = instance
-                        data[i] = padded_data
-                else:
-                    padded_data = []
-                    for entry in instance:
-                        padded_data.append(utils.pad_sequences(entry, max_len))
+        for i, instance in enumerate(data):
+            if isinstance(instance, np.ndarray):
+                if len(instance) < max_len:
+                    padded_data = np.zeros(max_len, dtype=instance.dtype)
+                    padded_data[: len(instance)] = instance
                     data[i] = padded_data
+            else:
+                padded_data = []
+                for entry in instance:
+                    padded_data.append(utils.pad_sequences(entry, max_len))
+                data[i] = padded_data
 
         return np.array(data)
 
diff --git a/src/main/python/systemds/scuro/representations/aggregated_representation.py b/src/main/python/systemds/scuro/representations/aggregated_representation.py
index 9119070a02..1e98d2f92a 100644
--- a/src/main/python/systemds/scuro/representations/aggregated_representation.py
+++ b/src/main/python/systemds/scuro/representations/aggregated_representation.py
@@ -20,12 +20,16 @@
 # -------------------------------------------------------------
 from systemds.scuro.modality.transformed import TransformedModality
 from systemds.scuro.representations.representation import Representation
+from systemds.scuro.representations.aggregate import Aggregation
 
 
 class AggregatedRepresentation(Representation):
-    def __init__(self, aggregation):
-        super().__init__("AggregatedRepresentation", aggregation.parameters)
-        self.aggregation = aggregation
+    def __init__(self, aggregation="mean"):
+        parameters = {
+            "aggregation": list(Aggregation().get_aggregation_functions()),
+        }
+        super().__init__("AggregatedRepresentation", parameters)
+        self.aggregation = Aggregation(aggregation)
         self.self_contained = True
 
     def transform(self, modality):
diff --git a/src/main/python/systemds/scuro/representations/bow.py b/src/main/python/systemds/scuro/representations/bow.py
index 7cfddbb506..2b338d30ee 100644
--- a/src/main/python/systemds/scuro/representations/bow.py
+++ b/src/main/python/systemds/scuro/representations/bow.py
@@ -34,8 +34,8 @@ class BoW(UnimodalRepresentation):
     def __init__(self, ngram_range=2, min_df=2, output_file=None):
         parameters = {"ngram_range": [ngram_range], "min_df": [min_df]}
         super().__init__("BoW", ModalityType.EMBEDDING, parameters)
-        self.ngram_range = ngram_range
-        self.min_df = min_df
+        self.ngram_range = int(ngram_range)
+        self.min_df = int(min_df)
         self.output_file = output_file
 
     def transform(self, modality):
diff --git a/src/main/python/systemds/scuro/representations/fusion.py b/src/main/python/systemds/scuro/representations/fusion.py
index ea614ac095..8cf67b1cb4 100644
--- a/src/main/python/systemds/scuro/representations/fusion.py
+++ b/src/main/python/systemds/scuro/representations/fusion.py
@@ -18,10 +18,14 @@
 # under the License.
 #
 # -------------------------------------------------------------
+import copy
 from typing import List
 
 import numpy as np
-from systemds.scuro import AggregatedRepresentation, Aggregation
+from systemds.scuro.representations.aggregated_representation import (
+    AggregatedRepresentation,
+)
+from systemds.scuro.modality.transformed import TransformedModality
 
 from systemds.scuro.modality.modality import Modality
 from systemds.scuro.representations.representation import Representation
@@ -52,7 +56,7 @@ class Fusion(Representation):
         for modality in modalities:
             agg_modality = None
             if get_shape(modality.metadata) > 1:
-                agg_operator = AggregatedRepresentation(Aggregation())
+                agg_operator = AggregatedRepresentation()
                 agg_modality = agg_operator.transform(modality)
             mods.append(agg_modality if agg_modality else modality)
 
@@ -63,26 +67,53 @@ class Fusion(Representation):
 
         return self.execute(mods)
 
-    def transform_with_training(
-        self, modalities: List[Modality], train_indices, labels
-    ):
-        # if self.needs_instance_alignment:
-        #     max_len = self.get_max_embedding_size(modalities)
-        #     for modality in modalities:
-        #         modality.pad(max_len=max_len)
-
-        self.execute(
-            [np.array(modality.data)[train_indices] for modality in modalities],
-            labels[train_indices],
+    def transform_with_training(self, modalities: List[Modality], task):
+        train_modalities = []
+        for modality in modalities:
+            train_data = [
+                d for i, d in enumerate(modality.data) if i in task.train_indices
+            ]
+            train_modality = TransformedModality(modality, self)
+            train_modality.data = copy.deepcopy(train_data)
+            train_modalities.append(train_modality)
+
+        transformed_train = self.execute(
+            train_modalities, task.labels[task.train_indices]
         )
+        transformed_val = self.transform_data(modalities, task.val_indices)
 
-    def transform_data(self, modalities: List[Modality], val_indices):
-        return self.apply_representation(
-            [np.array(modality.data)[val_indices] for modality in modalities]
+        transformed_data = np.zeros(
+            (len(modalities[0].data), transformed_train.shape[1])
         )
+        transformed_data[task.train_indices] = transformed_train
+        transformed_data[task.val_indices] = transformed_val
+
+        return transformed_data
+
+    def transform_data(self, modalities: List[Modality], indices=None):
+        val_modalities = []
+        for modality in modalities:
+            val_data = (
+                [d for i, d in enumerate(modality.data) if i in indices]
+                if indices
+                else modality.data
+            )
+            val_modality = type(modality)(modality, self)
+            val_modality.data = copy.deepcopy(val_data)
+            val_modalities.append(val_modality)
 
-    def execute(self, modalities: List[Modality]):
-        raise f"Not implemented for Fusion: {self.name}"
+        return self.apply_representation(val_modalities)
+
+    def execute(self, modalities: List[Modality], labels: np.ndarray = None):
+        raise NotImplementedError(f"Not implemented for Fusion: {self.name}")
+
+    def apply_representation(self, modalities: List[Modality]):
+        if self.needs_training:
+            raise NotImplementedError(
+                f"apply_representation not implemented for trainable fusion: {self.name}"
+            )
+        else:
+            return self.execute(modalities)
 
     def get_max_embedding_size(self, modalities: List[Modality]):
         """
@@ -90,6 +121,11 @@ class Fusion(Representation):
         :param modalities: List of modalities
         :return: maximum embedding size
         """
+        try:
+            modalities[0].data = np.array(modalities[0].data)
+        except:
+            pass
+
         if isinstance(modalities[0].data[0], list):
             max_size = modalities[0].data[0][0].shape[1]
         elif isinstance(modalities[0].data, np.ndarray):
diff --git a/src/main/python/systemds/scuro/representations/lstm.py b/src/main/python/systemds/scuro/representations/lstm.py
index 0cfafddefa..c8e9644881 100644
--- a/src/main/python/systemds/scuro/representations/lstm.py
+++ b/src/main/python/systemds/scuro/representations/lstm.py
@@ -22,10 +22,10 @@ import os
 import random
 
 import torch
-
 from torch import nn
-from typing import List
-
+from torch.utils.data import DataLoader, TensorDataset
+from typing import List, Dict, Any
+from systemds.scuro.utils.static_variables import get_device
 import numpy as np
 
 from systemds.scuro.modality.modality import Modality
@@ -34,21 +34,46 @@ from systemds.scuro.representations.fusion import Fusion
 from systemds.scuro.drsearch.operator_registry import register_fusion_operator
 
 
-# TODO: concatenate before embedding
-# Make this a hyperparameter
 @register_fusion_operator()
 class LSTM(Fusion):
-    def __init__(self, width=128, depth=1, dropout_rate=0.1):
-        """
-        Combines modalities using an LSTM
-        """
-        super().__init__("LSTM")
-        self.depth = depth
-        self.width = width
-        self.dropout_rate = dropout_rate
-        self.unimodal_embeddings = {}
-        seed = 42
-
+    def __init__(
+        self,
+        width=128,
+        depth=1,
+        dropout_rate=0.1,
+        learning_rate=0.001,
+        epochs=50,
+        batch_size=32,
+    ):
+        parameters = {
+            "width": [128, 256, 512],
+            "depth": [1, 2, 3],
+            "dropout_rate": [0.1, 0.2, 0.3, 0.4, 0.5],
+            "learning_rate": [0.001, 0.0001, 0.01, 0.1],
+            "epochs": [50, 100, 200],
+            "batch_size": [8, 16, 32, 64, 128],
+        }
+
+        super().__init__("LSTM", parameters)
+
+        self.width = int(width)
+        self.depth = int(depth)
+        self.dropout_rate = float(dropout_rate)
+        self.learning_rate = float(learning_rate)
+        self.epochs = int(epochs)
+        self.batch_size = int(batch_size)
+
+        self.needs_training = True
+        self.needs_alignment = True
+        self.model = None
+        self.input_dim = None
+        self.num_classes = None
+        self.is_trained = False
+        self.model_state = None
+
+        self._set_random_seeds()
+
+    def _set_random_seeds(self, seed=42):
         os.environ["PYTHONHASHSEED"] = str(seed)
         random.seed(seed)
         np.random.seed(seed)
@@ -57,40 +82,161 @@ class LSTM(Fusion):
         torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.benchmark = False
 
-    def transform(self, modalities: List[Modality]):
-        self.unimodal_embeddings = {}
-        size = len(modalities[0].data)
-
-        result = np.zeros((size, 0))
+    def _prepare_data(self, modalities: List[Modality]) -> np.ndarray:
+        processed_modalities = []
 
         for modality in modalities:
-            if modality.modality_type in list(self.unimodal_embeddings.keys()):
-                out = self.unimodal_embeddings.get(modality.modality_type)
+            try:
+                data = np.array(modality.data)
+            except:
+                max_len = -1
+                for md in modality.metadata.values():
+                    if max_len < md["data_layout"]["shape"][0]:
+                        max_len = md["data_layout"]["shape"][0]
+                data = np.zeros((len(modality.data), max_len))
+                for i, d in enumerate(modality.data):
+                    data[i, : len(d)] = d
+
+                modality.data = data
+
+            if data.ndim == 1:
+                data = data.reshape(-1, 1, 1)
+            elif data.ndim == 2:
+                data = data.reshape(data.shape[0], 1, data.shape[1])
+            elif data.ndim == 3:
+                pass
             else:
-                out = self.run_lstm(modality.data)
-                self.unimodal_embeddings[modality.modality_type] = out
+                raise ValueError(
+                    f"Unsupported data shape: {data.shape}. Expected 1D, 2D, or 3D arrays."
+                )
+
+            processed_modalities.append(data)
+
+        max_seq_len = max(mod.shape[1] for mod in processed_modalities)
+
+        aligned_modalities = []
+        for data in processed_modalities:
+            if data.shape[1] < max_seq_len:
+                pad_width = ((0, 0), (0, max_seq_len - data.shape[1]), (0, 0))
+                data = np.pad(data, pad_width, mode="constant", constant_values=0)
+            aligned_modalities.append(data)
+
+        concatenated_data = np.concatenate(aligned_modalities, axis=2)
+
+        return concatenated_data.astype(np.float32)
+
+    def _build_model(self, input_dim: int, num_classes: int) -> nn.Module:
+
+        class LSTMClassifier(nn.Module):
+            def __init__(
+                self, input_dim, hidden_dim, num_layers, num_classes, dropout_rate
+            ):
+                super(LSTMClassifier, self).__init__()
+                self.hidden_dim = hidden_dim
+                self.num_layers = num_layers
+
+                self.lstm = nn.LSTM(
+                    input_dim,
+                    hidden_dim,
+                    num_layers,
+                    batch_first=True,
+                    bidirectional=True,
+                    dropout=dropout_rate if num_layers > 1 else 0,
+                )
+
+                self.dropout = nn.Dropout(dropout_rate)
+                self.classifier = nn.Linear(hidden_dim * 2, num_classes)
+
+            def forward(self, x):
+                lstm_out, _ = self.lstm(x)
+                last_output = lstm_out[:, -1, :]
+                dropped = self.dropout(last_output)
+                output = self.classifier(dropped)
+
+                return last_output, output
+
+        return LSTMClassifier(
+            input_dim, self.width, self.depth, num_classes, self.dropout_rate
+        )
+
+    def execute(self, modalities: List[Modality], labels: np.ndarray = None):
+        if labels is None:
+            raise ValueError("LSTM fusion requires labels for training")
+
+        X = self._prepare_data(modalities)
+        y = np.array(labels)
+
+        self.input_dim = X.shape[2]
+        self.num_classes = len(np.unique(y))
+
+        self.model = self._build_model(self.input_dim, self.num_classes)
+        device = get_device()
+        self.model.to(device)
+
+        criterion = nn.CrossEntropyLoss()
+        optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)
+
+        X_tensor = torch.FloatTensor(X).to(device)
+        y_tensor = torch.LongTensor(y).to(device)
+
+        dataset = TensorDataset(X_tensor, y_tensor)
+        dataloader = DataLoader(dataset, batch_size=self.batch_size, shuffle=True)
+
+        self.model.train()
+        for epoch in range(self.epochs):
+            total_loss = 0
+            for batch_X, batch_y in dataloader:
+                optimizer.zero_grad()
+
+                features, predictions = self.model(batch_X)
+                loss = criterion(predictions, batch_y)
+
+                loss.backward()
+                optimizer.step()
+
+                total_loss += loss.item()
+
+        self.is_trained = True
+
+        self.model_state = {
+            "state_dict": self.model.state_dict(),
+            "input_dim": self.input_dim,
+            "num_classes": self.num_classes,
+            "width": self.width,
+            "depth": self.depth,
+            "dropout_rate": self.dropout_rate,
+        }
+
+        self.model.eval()
+        with torch.no_grad():
+            features, _ = self.model(X_tensor)
+        return features.cpu().numpy()
+
+    def apply_representation(self, modalities: List[Modality]) -> np.ndarray:
+        if not self.is_trained or self.model is None:
+            raise ValueError("Model must be trained before applying representation")
 
-            result = np.concatenate([result, out], axis=-1)
+        X = self._prepare_data(modalities)
 
-        return result
+        device = get_device()
+        self.model.to(device)
 
-    def run_lstm(self, data):
-        if isinstance(data, list):
-            data = np.array(data)
+        X_tensor = torch.FloatTensor(X).to(device)
 
-        d = data.astype(np.float32)
-        dim = d.shape[-1]
-        d = torch.from_numpy(d)
-        dropout_layer = torch.nn.Dropout(self.dropout_rate)
+        self.model.eval()
+        with torch.no_grad():
+            features, _ = self.model(X_tensor)
 
-        for x in range(0, self.depth):
-            lstm_x = nn.LSTM(dim, self.width, batch_first=True, bidirectional=True)
-            dim = 2 * self.width
-            d = lstm_x(d)[0]
+        return features.cpu().numpy()
 
-        out = dropout_layer(d)
+    def get_model_state(self) -> Dict[str, Any]:
+        return self.model_state
 
-        if d.ndim > 2:
-            out = torch.flatten(out, 1)
+    def set_model_state(self, state: Dict[str, Any]):
+        self.model_state = state
+        self.input_dim = state["input_dim"]
+        self.num_classes = state["num_classes"]
 
-        return out.detach().numpy()
+        self.model = self._build_model(self.input_dim, self.num_classes)
+        self.model.load_state_dict(state["state_dict"])
+        self.is_trained = True
diff --git a/src/main/python/systemds/scuro/representations/mel_spectrogram.py b/src/main/python/systemds/scuro/representations/mel_spectrogram.py
index dca1b0eec8..6ea9061901 100644
--- a/src/main/python/systemds/scuro/representations/mel_spectrogram.py
+++ b/src/main/python/systemds/scuro/representations/mel_spectrogram.py
@@ -37,9 +37,9 @@ class MelSpectrogram(UnimodalRepresentation):
             "n_fft": [1024, 2048, 4096],
         }
         super().__init__("MelSpectrogram", ModalityType.TIMESERIES, parameters, False)
-        self.n_mels = n_mels
-        self.hop_length = hop_length
-        self.n_fft = n_fft
+        self.n_mels = int(n_mels)
+        self.hop_length = int(hop_length)
+        self.n_fft = int(n_fft)
 
     def transform(self, modality):
         transformed_modality = TransformedModality(
diff --git a/src/main/python/systemds/scuro/representations/mfcc.py b/src/main/python/systemds/scuro/representations/mfcc.py
index c942f3076e..4d9989add9 100644
--- a/src/main/python/systemds/scuro/representations/mfcc.py
+++ b/src/main/python/systemds/scuro/representations/mfcc.py
@@ -38,10 +38,10 @@ class MFCC(UnimodalRepresentation):
             "n_mels": [20, 32, 64, 128],
         }  # TODO
         super().__init__("MFCC", ModalityType.TIMESERIES, parameters, False)
-        self.n_mfcc = n_mfcc
-        self.dct_type = dct_type
-        self.n_mels = n_mels
-        self.hop_length = hop_length
+        self.n_mfcc = int(n_mfcc)
+        self.dct_type = int(dct_type)
+        self.n_mels = int(n_mels)
+        self.hop_length = int(hop_length)
 
     def transform(self, modality):
         transformed_modality = TransformedModality(
diff --git a/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py b/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py
index 7928b9988b..d17451932e 100644
--- a/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py
+++ b/src/main/python/systemds/scuro/representations/multimodal_attention_fusion.py
@@ -19,10 +19,12 @@
 #
 # -------------------------------------------------------------
 
+import os
+import random
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Any
 import numpy as np
 from systemds.scuro.drsearch.operator_registry import register_fusion_operator
 from systemds.scuro.modality.modality import Modality
@@ -37,140 +39,237 @@ class AttentionFusion(Fusion):
         hidden_dim=256,
         num_heads=8,
         dropout=0.1,
-        fusion_strategy="attention",
         batch_size=32,
         num_epochs=50,
+        learning_rate=0.001,
     ):
-        self.encoder = None
         params = {
-            "hidden_dim": [128, 256, 512],
-            "num_heads": [1, 4, 8],
-            "dropout": [0.1, 0.2, 0.3],
-            "fusion_strategy": ["mean", "max", "attention", "cls"],
-            "batch_size": [32, 64, 128],
-            "num_epochs": [50, 70, 100, 150],
+            "hidden_dim": [32, 128, 256, 384, 512, 768],
+            "num_heads": [2, 4, 8, 12],
+            "dropout": [0.0, 0.1, 0.2, 0.3, 0.4],
+            "batch_size": [8, 16, 32, 64, 128],
+            "num_epochs": [50, 100, 150, 200],
+            "learning_rate": [1e-5, 1e-4, 1e-3, 1e-2],
         }
         super().__init__("AttentionFusion", params)
-        self.hidden_dim = hidden_dim
-        self.num_heads = num_heads
-        self.dropout = dropout
-        self.fusion_strategy = fusion_strategy
-        self.batch_size = batch_size
-        self.needs_training = True
-        self.needs_instance_alignment = True
-        self.num_epochs = num_epochs
 
-    def execute(
-        self,
-        data: List[np.ndarray],
-        labels: np.ndarray,
-    ):
-        input_dimension = {}
+        self.hidden_dim = int(hidden_dim)
+        self.num_heads = int(num_heads)
+        self.dropout = float(dropout)
+        self.batch_size = int(batch_size)
+        self.num_epochs = int(num_epochs)
+        self.learning_rate = float(learning_rate)
+
+        self.needs_training = True
+        self.needs_alignment = True
+        self.encoder = None
+        self.classification_head = None
+        self.input_dimensions = None
+        self.max_sequence_length = None
+        self.num_classes = None
+        self.is_trained = False
+        self.model_state = None
+
+        self._set_random_seeds()
+
+    def _set_random_seeds(self, seed=42):
+        os.environ["PYTHONHASHSEED"] = str(seed)
+        random.seed(seed)
+        np.random.seed(seed)
+        torch.manual_seed(seed)
+        torch.cuda.manual_seed(seed)
+        torch.backends.cudnn.deterministic = True
+        torch.backends.cudnn.benchmark = False
+
+    def _prepare_data(self, modalities: List[Modality]) -> Dict[str, torch.Tensor]:
         inputs = {}
+        input_dimensions = {}
         max_sequence_length = 0
-        masks = {}
-        for i, modality in enumerate(data):
-            modality_name = "modality_" + str(i)
-            shape = modality.shape
-            max_sequence_length = max(max_sequence_length, shape[1])
-            input_dimension[modality_name] = shape[2] if len(shape) > 2 else shape[1]
-            inputs[modality_name] = torch.from_numpy(np.stack(modality)).to(
-                get_device()
-            )
 
-            # attention_masks_list = [
-            #     entry["attention_masks"]
-            #     for entry in modality.metadata.values()
-            #     if "attention_masks" in entry
-            # ]
-            attention_masks_list = None
-            if attention_masks_list:
-                masks[modality_name] = (
-                    torch.tensor(np.array(attention_masks_list)).bool().to(get_device())
-                )
+        for i, modality in enumerate(modalities):
+            modality_name = f"modality_{i}"
+            data = np.array(modality.data)
+
+            if data.ndim == 1:
+                data = data.reshape(-1, 1, 1)
+            elif data.ndim == 2:
+                data = data.reshape(data.shape[0], 1, data.shape[1])
+            elif data.ndim == 3:
+                pass
             else:
-                masks[modality_name] = None
+                raise ValueError(
+                    f"Unsupported data shape: {data.shape}. Expected 1D, 2D, or 3D arrays."
+                )
+
+            input_dimensions[modality_name] = data.shape[2]  # Feature dimension
+            max_sequence_length = max(max_sequence_length, data.shape[1])
+
+            inputs[modality_name] = torch.from_numpy(data.astype(np.float32))
+
+        for modality_name, tensor in inputs.items():
+            if tensor.shape[1] < max_sequence_length:
+                pad_width = (0, 0, 0, max_sequence_length - tensor.shape[1], 0, 0)
+                inputs[modality_name] = F.pad(
+                    tensor, pad_width, mode="constant", value=0
+                )
+
+        return inputs, input_dimensions, max_sequence_length
+
+    def execute(self, modalities: List[Modality], labels: np.ndarray = None):
+        if labels is None:
+            raise ValueError("Attention fusion requires labels for training")
+
+        inputs, input_dimensions, max_sequence_length = self._prepare_data(modalities)
+        y = np.array(labels)
+
+        self.input_dimensions = input_dimensions
+        self.max_sequence_length = max_sequence_length
+        self.num_classes = len(np.unique(y))
 
         self.encoder = MultiModalAttentionFusion(
-            input_dimension,
+            self.input_dimensions,
             self.hidden_dim,
             self.num_heads,
             self.dropout,
-            max_sequence_length,
-            self.fusion_strategy,
+            self.max_sequence_length,
         )
 
-        head = FusedClassificationHead(
-            fused_dim=self.hidden_dim, num_classes=len(np.unique(labels))
+        self.classification_head = FusedClassificationHead(
+            fused_dim=self.hidden_dim, num_classes=self.num_classes
         )
+
+        device = get_device()
+        self.encoder.to(device)
+        self.classification_head.to(device)
+
         criterion = nn.CrossEntropyLoss()
         optimizer = torch.optim.Adam(
-            list(self.encoder.parameters()) + list(head.parameters()), lr=0.001
+            list(self.encoder.parameters())
+            + list(self.classification_head.parameters()),
+            lr=self.learning_rate,
         )
-        labels = torch.from_numpy(labels).to(get_device())
+
+        for modality_name in inputs:
+            inputs[modality_name] = inputs[modality_name].to(device)
+        labels_tensor = torch.from_numpy(y).long().to(device)
+
+        dataset_inputs = []
+        for i in range(len(y)):
+            sample_inputs = {name: tensor[i] for name, tensor in inputs.items()}
+            dataset_inputs.append((sample_inputs, labels_tensor[i]))
+
+        self.encoder.train()
+        self.classification_head.train()
 
         for epoch in range(self.num_epochs):
             total_loss = 0
-            total_accuracy = 0
-            for batch_idx in range(0, len(data), self.batch_size):
-                batched_input = {}
-                for modality, modality_data in inputs.items():
-                    batched_input[modality] = modality_data[
-                        batch_idx : batch_idx + self.batch_size
-                    ]
-                loss, predictions = self.train_encoder_step(
-                    head,
-                    inputs,
-                    labels[batch_idx : batch_idx + self.batch_size],
-                    criterion,
-                    optimizer,
-                )
-                total_loss += loss
-                total_accuracy += predictions
+            total_correct = 0
+            total_samples = 0
+
+            for batch_start in range(0, len(dataset_inputs), self.batch_size):
+                batch_end = min(batch_start + self.batch_size, len(dataset_inputs))
+                batch_data = dataset_inputs[batch_start:batch_end]
+
+                batch_inputs = {}
+                batch_labels = []
+
+                for sample_inputs, label in batch_data:
+                    batch_labels.append(label)
+                    for modality_name, tensor in sample_inputs.items():
+                        if modality_name not in batch_inputs:
+                            batch_inputs[modality_name] = []
+                        batch_inputs[modality_name].append(tensor)
+
+                for modality_name in batch_inputs:
+                    batch_inputs[modality_name] = torch.stack(
+                        batch_inputs[modality_name]
+                    )
+
+                batch_labels = torch.stack(batch_labels)
+
+                optimizer.zero_grad()
+
+                encoder_output = self.encoder(batch_inputs)
+                logits = self.classification_head(encoder_output["fused"])
+                loss = criterion(logits, batch_labels)
+
+                loss.backward()
+                optimizer.step()
+
+                total_loss += loss.item()
+                _, predicted = torch.max(logits.data, 1)
+                total_correct += (predicted == batch_labels).sum().item()
+                total_samples += batch_labels.size(0)
+
+        self.is_trained = True
+
+        self.model_state = {
+            "encoder_state_dict": self.encoder.state_dict(),
+            "classification_head_state_dict": self.classification_head.state_dict(),
+            "input_dimensions": self.input_dimensions,
+            "max_sequence_length": self.max_sequence_length,
+            "num_classes": self.num_classes,
+            "hidden_dim": self.hidden_dim,
+            "num_heads": self.num_heads,
+            "dropout": self.dropout,
+        }
 
-            if epoch % 50 == 0 or epoch == self.num_epochs - 1:
-                print(
-                    f"Epoch {epoch}, Loss: {total_loss:.4f}, accuracy: {total_accuracy/len(data):.4f}"
-                )
+    def apply_representation(self, modalities: List[Modality]) -> np.ndarray:
+        if not self.is_trained or self.encoder is None:
+            raise ValueError("Model must be trained before applying representation")
+
+        inputs, _, _ = self._prepare_data(modalities)
+
+        device = get_device()
+        self.encoder.to(device)
+
+        for modality_name in inputs:
+            inputs[modality_name] = inputs[modality_name].to(device)
 
-    # Training step (encoder + classification head)
-    def train_encoder_step(self, head, inputs, labels, criterion, optimizer):
-        self.encoder.train()
-        head.train()
-        optimizer.zero_grad()
-        output = self.encoder(inputs)
-        logits = head(output["fused"])
-        loss = criterion(logits, labels)
-        loss.backward()
-        optimizer.step()
-        _, predicted = torch.max(logits.data, 1)
-        return loss.item(), (predicted == labels).sum().item()
-
-    def apply_representation(self, modalities):
-        inputs = {}
-        for i, modality in enumerate(modalities):
-            modality_name = "modality_" + str(i)
-            inputs[modality_name] = torch.from_numpy(np.stack(modality)).to(
-                get_device()
-            )
         self.encoder.eval()
         with torch.no_grad():
-            output = self.encoder(inputs)
-        return output["fused"].cpu().numpy()
+            encoder_output = self.encoder(inputs)
+
+        return encoder_output["fused"].cpu().numpy()
+
+    def get_model_state(self) -> Dict[str, Any]:
+        return self.model_state
+
+    def set_model_state(self, state: Dict[str, Any]):
+        self.model_state = state
+        self.input_dimensions = state["input_dimensions"]
+        self.max_sequence_length = state["max_sequence_length"]
+        self.num_classes = state["num_classes"]
+
+        self.encoder = MultiModalAttentionFusion(
+            self.input_dimensions,
+            state["hidden_dim"],
+            state["num_heads"],
+            state["dropout"],
+            self.max_sequence_length,
+        )
+        self.encoder.load_state_dict(state["encoder_state_dict"])
+
+        self.classification_head = FusedClassificationHead(
+            fused_dim=state["hidden_dim"], num_classes=self.num_classes
+        )
+        self.classification_head.load_state_dict(
+            state["classification_head_state_dict"]
+        )
+
+        self.is_trained = True
 
 
 class FusedClassificationHead(nn.Module):
-    """
-    Simple classification head for supervision during training.
-    """
 
     def __init__(self, fused_dim, num_classes=2):
         super(FusedClassificationHead, self).__init__()
         self.head = nn.Sequential(
             nn.Linear(fused_dim, fused_dim // 2),
             nn.ReLU(),
+            nn.Dropout(0.1),
             nn.Linear(fused_dim // 2, num_classes),
-        ).to(get_device())
+        )
 
     def forward(self, fused):
         return self.head(fused)
@@ -184,7 +283,7 @@ class MultiModalAttentionFusion(nn.Module):
         num_heads: int,
         dropout: float,
         max_seq_len: int,
-        pooling_strategy: str,
+        pooling_strategy: str = "mean",
     ):
         super().__init__()
 
@@ -194,50 +293,42 @@ class MultiModalAttentionFusion(nn.Module):
         self.pooling_strategy = pooling_strategy
         self.max_seq_len = max_seq_len
 
-        # Project each modality to the same hidden dimension
         self.modality_projections = nn.ModuleDict(
             {
-                modality: nn.Linear(dim, hidden_dim).to(get_device())
+                modality: nn.Linear(dim, hidden_dim)
                 for modality, dim in modality_dims.items()
             }
         )
 
-        # Positional encoding for sequence modalities
         self.positional_encoding = nn.Parameter(
             torch.randn(max_seq_len, hidden_dim) * 0.1
-        ).to(get_device())
+        )
 
-        # Cross-modal attention
         self.cross_attention = nn.MultiheadAttention(
             embed_dim=hidden_dim, num_heads=num_heads, dropout=dropout, batch_first=True
-        ).to(get_device())
+        )
 
-        # Self-attention within each modality
         self.self_attention = nn.MultiheadAttention(
             embed_dim=hidden_dim, num_heads=num_heads, dropout=dropout, batch_first=True
-        ).to(get_device())
+        )
 
-        # Attention-based pooling for sequences
         if pooling_strategy == "attention":
             self.pooling_attention = nn.Sequential(
                 nn.Linear(hidden_dim, hidden_dim // 2),
                 nn.Tanh(),
                 nn.Linear(hidden_dim // 2, 1),
-            ).to(get_device())
+            )
 
-        # Modality-level attention for final fusion
         self.modality_attention = nn.Sequential(
             nn.Linear(hidden_dim, hidden_dim // 2),
             nn.ReLU(),
             nn.Linear(hidden_dim // 2, 1),
-        ).to(get_device())
+        )
 
-        # Layer normalization
-        self.layer_norm = nn.LayerNorm(hidden_dim).to(get_device())
-        self.dropout = nn.Dropout(dropout).to(get_device())
+        self.layer_norm = nn.LayerNorm(hidden_dim)
+        self.dropout_layer = nn.Dropout(dropout)
 
-        # Final projection
-        self.final_projection = nn.Linear(hidden_dim, hidden_dim).to(get_device())
+        self.final_projection = nn.Linear(hidden_dim, hidden_dim)
 
     def _handle_input_format(self, modality_tensor: torch.Tensor) -> torch.Tensor:
         if len(modality_tensor.shape) == 2:
@@ -269,29 +360,22 @@ class MultiModalAttentionFusion(nn.Module):
 
         elif self.pooling_strategy == "max":
             if mask is not None:
-                # Set masked positions to large negative value before max pooling
                 masked_seq = sequence.masked_fill(~mask.unsqueeze(-1), float("-inf"))
                 return masked_seq.max(dim=1)[0]
             else:
                 return sequence.max(dim=1)[0]
 
         elif self.pooling_strategy == "cls":
-            # Use the first token (assuming it's a CLS token)
             return sequence[:, 0, :]
 
         elif self.pooling_strategy == "attention":
-            # Attention-based pooling
-            attention_scores = self.pooling_attention(sequence).squeeze(
-                -1
-            )  # (batch, seq)
+            attention_scores = self.pooling_attention(sequence).squeeze(-1)
 
             if mask is not None:
                 attention_scores = attention_scores.masked_fill(~mask, float("-inf"))
 
-            attention_weights = F.softmax(attention_scores, dim=1)  # (batch, seq)
-            return (sequence * attention_weights.unsqueeze(-1)).sum(
-                dim=1
-            )  # (batch, hidden)
+            attention_weights = F.softmax(attention_scores, dim=1)
+            return (sequence * attention_weights.unsqueeze(-1)).sum(dim=1)
 
         else:
             raise ValueError(f"Unknown pooling strategy: {self.pooling_strategy}")
@@ -323,7 +407,7 @@ class MultiModalAttentionFusion(nn.Module):
                     key_padding_mask=~mask if mask is not None else None,
                 )
 
-                projected = self.layer_norm(projected + self.dropout(attended))
+                projected = self.layer_norm(projected + self.dropout_layer(attended))
 
                 pooled = self._pool_sequence(projected, mask)
             else:
@@ -339,7 +423,7 @@ class MultiModalAttentionFusion(nn.Module):
             )
 
             cross_attended = self.layer_norm(
-                modality_stack + self.dropout(cross_attended)
+                modality_stack + self.dropout_layer(cross_attended)
             )
 
             updated_embeddings = {
diff --git a/src/main/python/systemds/scuro/representations/representation.py b/src/main/python/systemds/scuro/representations/representation.py
index 144b88f34c..dac3bb2b98 100644
--- a/src/main/python/systemds/scuro/representations/representation.py
+++ b/src/main/python/systemds/scuro/representations/representation.py
@@ -18,7 +18,7 @@
 # under the License.
 #
 # -------------------------------------------------------------
-from abc import abstractmethod
+from systemds.scuro.utils.identifier import Identifier
 
 
 class Representation:
@@ -26,6 +26,7 @@ class Representation:
         self.name = name
         self._parameters = parameters
         self.self_contained = True
+        self.representation_id = Identifier().new_id()
 
     @property
     def parameters(self):
diff --git a/src/main/python/systemds/scuro/representations/resnet.py b/src/main/python/systemds/scuro/representations/resnet.py
index f961cb4588..7bb94d8bfd 100644
--- a/src/main/python/systemds/scuro/representations/resnet.py
+++ b/src/main/python/systemds/scuro/representations/resnet.py
@@ -32,17 +32,13 @@ from systemds.scuro.modality.type import ModalityType
 from systemds.scuro.utils.static_variables import get_device
 
 
-@register_representation(
-    [ModalityType.IMAGE, ModalityType.VIDEO, ModalityType.TIMESERIES]
-)
+@register_representation([ModalityType.IMAGE, ModalityType.VIDEO])
 class ResNet(UnimodalRepresentation):
-    def __init__(self, layer="avgpool", model_name="ResNet18", output_file=None):
+    def __init__(self, model_name="ResNet18", layer="avgpool", output_file=None):
         self.data_type = torch.bfloat16
         self.model_name = model_name
         parameters = self._get_parameters()
-        super().__init__(
-            "ResNet", ModalityType.TIMESERIES, parameters
-        )  # TODO: TIMESERIES only for videos - images would be handled as EMBEDDING
+        super().__init__("ResNet", ModalityType.EMBEDDING, parameters)
 
         self.output_file = output_file
         self.layer_name = layer
@@ -95,7 +91,6 @@ class ResNet(UnimodalRepresentation):
                 .to(get_device())
                 .to(self.data_type)
             )
-
         else:
             raise NotImplementedError
 
diff --git a/src/main/python/systemds/scuro/representations/spectrogram.py b/src/main/python/systemds/scuro/representations/spectrogram.py
index 51b69d7d87..662fb0d627 100644
--- a/src/main/python/systemds/scuro/representations/spectrogram.py
+++ b/src/main/python/systemds/scuro/representations/spectrogram.py
@@ -33,8 +33,8 @@ class Spectrogram(UnimodalRepresentation):
     def __init__(self, hop_length=512, n_fft=2048):
         parameters = {"hop_length": [256, 512, 1024, 2048], "n_fft": [1024, 2048, 4096]}
         super().__init__("Spectrogram", ModalityType.TIMESERIES, parameters, False)
-        self.hop_length = hop_length
-        self.n_fft = n_fft
+        self.hop_length = int(hop_length)
+        self.n_fft = int(n_fft)
 
     def transform(self, modality):
         transformed_modality = TransformedModality(
@@ -44,8 +44,8 @@ class Spectrogram(UnimodalRepresentation):
 
         for i, sample in enumerate(modality.data):
             spectrogram = librosa.stft(
-                y=np.array(sample), hop_length=self.hop_length, n_fft=self.n_fft
-            ).astype(modality.data_type)
+                y=np.array(np.abs(sample)), hop_length=self.hop_length, n_fft=self.n_fft
+            )
             S_dB = librosa.amplitude_to_db(np.abs(spectrogram))
 
             result.append(S_dB.T)
diff --git a/src/main/python/systemds/scuro/representations/tfidf.py b/src/main/python/systemds/scuro/representations/tfidf.py
index 3b8f069df8..c82961949f 100644
--- a/src/main/python/systemds/scuro/representations/tfidf.py
+++ b/src/main/python/systemds/scuro/representations/tfidf.py
@@ -34,7 +34,7 @@ class TfIdf(UnimodalRepresentation):
     def __init__(self, min_df=2, output_file=None):
         parameters = {"min_df": [min_df]}
         super().__init__("TF-IDF", ModalityType.EMBEDDING, parameters)
-        self.min_df = min_df
+        self.min_df = int(min_df)
         self.output_file = output_file
 
     def transform(self, modality):
diff --git a/src/main/python/systemds/scuro/representations/window_aggregation.py b/src/main/python/systemds/scuro/representations/window_aggregation.py
index b3ad9e1b93..c16f6d747f 100644
--- a/src/main/python/systemds/scuro/representations/window_aggregation.py
+++ b/src/main/python/systemds/scuro/representations/window_aggregation.py
@@ -49,10 +49,10 @@ class Window(Context):
 
 @register_context_operator()
 class WindowAggregation(Window):
-    def __init__(self, window_size=10, aggregation_function="mean", pad=False):
+    def __init__(self, aggregation_function="mean", window_size=10, pad=False):
         super().__init__("WindowAggregation", aggregation_function)
         self.parameters["window_size"] = [window_size]
-        self.window_size = window_size
+        self.window_size = int(window_size)
         self.pad = pad
 
     def execute(self, modality):
@@ -141,10 +141,10 @@ class WindowAggregation(Window):
 
 @register_context_operator()
 class StaticWindow(Window):
-    def __init__(self, num_windows=100, aggregation_function="mean"):
+    def __init__(self, aggregation_function="mean", num_windows=100):
         super().__init__("StaticWindow", aggregation_function)
         self.parameters["num_windows"] = [num_windows]
-        self.num_windows = num_windows
+        self.num_windows = int(num_windows)
 
     def execute(self, modality):
         windowed_data = []
@@ -172,10 +172,10 @@ class StaticWindow(Window):
 
 @register_context_operator()
 class DynamicWindow(Window):
-    def __init__(self, num_windows=100, aggregation_function="mean"):
+    def __init__(self, aggregation_function="mean", num_windows=100):
         super().__init__("DynamicWindow", aggregation_function)
         self.parameters["num_windows"] = [num_windows]
-        self.num_windows = num_windows
+        self.num_windows = int(num_windows)
 
     def execute(self, modality):
         windowed_data = []
@@ -198,6 +198,7 @@ class DynamicWindow(Window):
                 )
                 output.append(val)
                 start = end
+
             windowed_data.append(output)
 
         return np.array(windowed_data)
diff --git a/src/main/python/systemds/scuro/representations/word2vec.py b/src/main/python/systemds/scuro/representations/word2vec.py
index 06e082fb69..837811935c 100644
--- a/src/main/python/systemds/scuro/representations/word2vec.py
+++ b/src/main/python/systemds/scuro/representations/word2vec.py
@@ -41,16 +41,14 @@ def get_embedding(sentence, model):
 
 @register_representation(ModalityType.TEXT)
 class W2V(UnimodalRepresentation):
-    def __init__(self, vector_size=150, min_count=2, window=5, output_file=None):
+    def __init__(self, vector_size=150, min_count=1, output_file=None):
         parameters = {
             "vector_size": [vector_size],
             "min_count": [min_count],
-            "window": [window],
-        }  # TODO
+        }
         super().__init__("Word2Vec", ModalityType.EMBEDDING, parameters)
         self.vector_size = vector_size
         self.min_count = min_count
-        self.window = window
         self.output_file = output_file
 
     def transform(self, modality):
@@ -59,7 +57,6 @@ class W2V(UnimodalRepresentation):
         model = Word2Vec(
             sentences=t,
             vector_size=self.vector_size,
-            window=self.window,
             min_count=self.min_count,
         )
         embeddings = []
diff --git a/src/main/python/systemds/scuro/modality/modality_identifier.py b/src/main/python/systemds/scuro/utils/identifier.py
similarity index 64%
rename from src/main/python/systemds/scuro/modality/modality_identifier.py
rename to src/main/python/systemds/scuro/utils/identifier.py
index 5eeee7dc13..7b6802672c 100644
--- a/src/main/python/systemds/scuro/modality/modality_identifier.py
+++ b/src/main/python/systemds/scuro/utils/identifier.py
@@ -18,7 +18,10 @@
 # under the License.
 #
 # -------------------------------------------------------------
-class ModalityIdentifier:
+import threading
+
+
+class Identifier:
     """ """
 
     _instance = None
@@ -32,3 +35,27 @@ class ModalityIdentifier:
     def new_id(self):  # TODO: make threadsafe when parallelizing
         self.id += 1
         return self.id
+
+
+class IdGenerator:
+    _instance = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._ctr = 0
+                    cls._instance._ctr_lock = threading.Lock()
+        return cls._instance
+
+    def next(self) -> int:
+        with self._instance._ctr_lock:
+            self._instance._ctr += 1
+            n = self._instance._ctr
+        return n
+
+
+get_op_id = IdGenerator().next
+get_node_id = IdGenerator().next
diff --git a/src/main/python/tests/scuro/test_data_loaders.py b/src/main/python/tests/scuro/test_data_loaders.py
index 85da2919a0..fb07df7154 100644
--- a/src/main/python/tests/scuro/test_data_loaders.py
+++ b/src/main/python/tests/scuro/test_data_loaders.py
@@ -66,7 +66,6 @@ class TestDataLoaders(unittest.TestCase):
 
     @classmethod
     def tearDownClass(cls):
-        print("Cleaning up test data")
         shutil.rmtree(cls.test_file_path)
 
     def test_load_audio_data_from_file(self):
diff --git a/src/main/python/tests/scuro/test_dr_search.py b/src/main/python/tests/scuro/test_dr_search.py
deleted file mode 100644
index 3e0e702e6f..0000000000
--- a/src/main/python/tests/scuro/test_dr_search.py
+++ /dev/null
@@ -1,174 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
-import os
-import shutil
-import unittest
-
-import numpy as np
-from sklearn import svm
-from sklearn.metrics import classification_report
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import MinMaxScaler
-
-from systemds.scuro.modality.type import ModalityType
-from systemds.scuro.drsearch.dr_search import DRSearch
-from systemds.scuro.drsearch.task import Task
-from systemds.scuro.models.model import Model
-from systemds.scuro.representations.average import Average
-from systemds.scuro.representations.bert import Bert
-from systemds.scuro.representations.concatenation import Concatenation
-from systemds.scuro.representations.lstm import LSTM
-from systemds.scuro.representations.max import RowMax
-from systemds.scuro.representations.mel_spectrogram import MelSpectrogram
-from systemds.scuro.representations.hadamard import Hadamard
-from systemds.scuro.representations.resnet import ResNet
-from systemds.scuro.representations.sum import Sum
-from tests.scuro.data_generator import ModalityRandomDataGenerator
-
-
-import warnings
-
-warnings.filterwarnings("always")
-
-
-class TestSVM(Model):
-    def __init__(self):
-        super().__init__("Test")
-
-    def fit(self, X, y, X_test, y_test):
-        self.clf = svm.SVC(C=1, gamma="scale", kernel="rbf", verbose=False)
-        self.clf = self.clf.fit(X, np.array(y))
-        y_pred = self.clf.predict(X)
-
-        return classification_report(
-            y, y_pred, output_dict=True, digits=3, zero_division=1
-        )["accuracy"]
-
-    def test(self, test_X: np.ndarray, test_y: np.ndarray):
-        y_pred = self.clf.predict(np.array(test_X))  # noqa
-
-        return classification_report(
-            np.array(test_y), y_pred, output_dict=True, digits=3, zero_division=1
-        )["accuracy"]
-
-
-def scale_data(data, train_indizes):
-    data = np.array(data).reshape(len(data), -1)
-    scaler = MinMaxScaler(feature_range=(0, 1))
-    scaler.fit(data[train_indizes])
-    return scaler.transform(data)
-
-
-class TestDataLoaders(unittest.TestCase):
-    train_indizes = None
-    val_indizes = None
-    test_file_path = None
-    mods = None
-    text = None
-    audio = None
-    video = None
-    data_generator = None
-    num_instances = 0
-    representations = None
-
-    @classmethod
-    def setUpClass(cls):
-        cls.num_instances = 20
-        cls.data_generator = ModalityRandomDataGenerator()
-
-        cls.labels = ModalityRandomDataGenerator().create_balanced_labels(
-            num_instances=cls.num_instances
-        )
-        # TODO: adapt the representation so they return non aggregated values. Apply windowing operation instead
-
-        cls.video = cls.data_generator.create1DModality(
-            cls.num_instances, 100, ModalityType.VIDEO
-        )
-        cls.text = cls.data_generator.create1DModality(
-            cls.num_instances, 100, ModalityType.TEXT
-        )
-        cls.audio = cls.data_generator.create1DModality(
-            cls.num_instances, 100, ModalityType.AUDIO
-        )
-
-        cls.mods = [cls.video, cls.audio, cls.text]
-
-        split = train_test_split(
-            np.array(range(cls.num_instances)),
-            cls.labels,
-            test_size=0.2,
-            random_state=42,
-        )
-        cls.train_indizes, cls.val_indizes = [int(i) for i in split[0]], [
-            int(i) for i in split[1]
-        ]
-
-        for m in cls.mods:
-            m.data = scale_data(m.data, cls.train_indizes)
-
-        cls.representations = [
-            Concatenation(),
-            Average(),
-            RowMax(),
-            Hadamard(),
-            Sum(),
-            LSTM(width=256, depth=3),
-        ]
-
-    def test_enumerate_all(self):
-        task = Task(
-            "TestTask",
-            TestSVM(),
-            self.labels,
-            self.train_indizes,
-            self.val_indizes,
-        )
-        dr_search = DRSearch(self.mods, task, self.representations)
-        best_representation, best_score, best_modalities = dr_search.fit_enumerate_all()
-
-        for r in dr_search.scores.values():
-            for scores in r.values():
-                assert scores[1] <= best_score
-
-    def test_enumerate_all_vs_random(self):
-        task = Task(
-            "TestTask",
-            TestSVM(),
-            self.labels,
-            self.train_indizes,
-            self.val_indizes,
-        )
-        dr_search = DRSearch(self.mods, task, self.representations)
-        best_representation_enum, best_score_enum, best_modalities_enum = (
-            dr_search.fit_enumerate_all()
-        )
-
-        dr_search.reset_best_params()
-
-        best_representation_rand, best_score_rand, best_modalities_rand = (
-            dr_search.fit_random(seed=42)
-        )
-
-        assert best_score_rand <= best_score_enum
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_hp_tuner.py
similarity index 66%
copy from src/main/python/tests/scuro/test_unimodal_optimizer.py
copy to src/main/python/tests/scuro/test_hp_tuner.py
index a4952d29f9..73aab4493d 100644
--- a/src/main/python/tests/scuro/test_unimodal_optimizer.py
+++ b/src/main/python/tests/scuro/test_hp_tuner.py
@@ -19,7 +19,6 @@
 #
 # -------------------------------------------------------------
 
-
 import unittest
 
 import numpy as np
@@ -27,6 +26,10 @@ from sklearn import svm
 from sklearn.metrics import classification_report
 from sklearn.model_selection import train_test_split
 
+from systemds.scuro.drsearch.multimodal_optimizer import MultimodalOptimizer
+from systemds.scuro.representations.average import Average
+from systemds.scuro.representations.concatenation import Concatenation
+from systemds.scuro.representations.lstm import LSTM
 from systemds.scuro.drsearch.operator_registry import Registry
 from systemds.scuro.models.model import Model
 from systemds.scuro.drsearch.task import Task
@@ -39,11 +42,13 @@ from systemds.scuro.representations.covarep_audio_features import (
     Pitch,
 )
 from systemds.scuro.representations.word2vec import W2V
+from systemds.scuro.representations.bow import BoW
 from systemds.scuro.modality.unimodal_modality import UnimodalModality
 from systemds.scuro.representations.resnet import ResNet
 from tests.scuro.data_generator import ModalityRandomDataGenerator, TestDataLoader
 
 from systemds.scuro.modality.type import ModalityType
+from systemds.scuro.drsearch.hyperparameter_tuner import HyperparameterTuner
 
 
 class TestSVM(Model):
@@ -71,9 +76,9 @@ class TestSVM(Model):
         )["accuracy"]
 
 
-class TestCNN(Model):
+class TestSVM2(Model):
     def __init__(self):
-        super().__init__("TestCNN")
+        super().__init__("TestSVM2")
 
     def fit(self, X, y, X_test, y_test):
         if X.ndim > 2:
@@ -99,7 +104,7 @@ class TestCNN(Model):
 from unittest.mock import patch
 
 
-class TestUnimodalRepresentationOptimizer(unittest.TestCase):
+class TestHPTuner(unittest.TestCase):
     data_generator = None
     num_instances = 0
 
@@ -132,14 +137,14 @@ class TestUnimodalRepresentationOptimizer(unittest.TestCase):
             ),
             Task(
                 "UnimodalRepresentationTask2",
-                TestCNN(),
+                TestSVM2(),
                 cls.labels,
                 cls.train_indizes,
                 cls.val_indizes,
             ),
         ]
 
-    def test_unimodal_optimizer_for_audio_modality(self):
+    def test_hp_tuner_for_audio_modality(self):
         audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data(
             self.num_instances, 3000
         )
@@ -149,9 +154,35 @@ class TestUnimodalRepresentationOptimizer(unittest.TestCase):
             )
         )
 
-        self.optimize_unimodal_representation_for_modality(audio)
-
-    def test_unimodal_optimizer_for_text_modality(self):
+        self.run_hp_for_modality([audio])
+
+    # def test_multimodal_hp_tuning(self):
+    #     audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data(
+    #         self.num_instances, 3000
+    #     )
+    #     audio = UnimodalModality(
+    #         TestDataLoader(
+    #             self.indices, None, ModalityType.AUDIO, audio_data, np.float32, audio_md
+    #         )
+    #     )
+    #
+    #     text_data, text_md = ModalityRandomDataGenerator().create_text_data(
+    #         self.num_instances
+    #     )
+    #     text = UnimodalModality(
+    #         TestDataLoader(
+    #             self.indices, None, ModalityType.TEXT, text_data, str, text_md
+    #         )
+    #     )
+    #
+    #     self.run_hp_for_modality(
+    #         [audio, text], multimodal=True, tune_unimodal_representations=True
+    #     )
+    #     self.run_hp_for_modality(
+    #         [audio, text], multimodal=True, tune_unimodal_representations=False
+    #     )
+
+    def test_hp_tuner_for_text_modality(self):
         text_data, text_md = ModalityRandomDataGenerator().create_text_data(
             self.num_instances
         )
@@ -160,25 +191,16 @@ class TestUnimodalRepresentationOptimizer(unittest.TestCase):
                 self.indices, None, ModalityType.TEXT, text_data, str, text_md
             )
         )
-        self.optimize_unimodal_representation_for_modality(text)
-
-    def test_unimodal_optimizer_for_video_modality(self):
-        video_data, video_md = ModalityRandomDataGenerator().create_visual_modality(
-            self.num_instances, 60
-        )
-        video = UnimodalModality(
-            TestDataLoader(
-                self.indices, None, ModalityType.VIDEO, video_data, np.float32, video_md
-            )
-        )
-        self.optimize_unimodal_representation_for_modality(video)
+        self.run_hp_for_modality([text])
 
-    def optimize_unimodal_representation_for_modality(self, modality):
+    def run_hp_for_modality(
+        self, modalities, multimodal=False, tune_unimodal_representations=False
+    ):
         with patch.object(
             Registry,
             "_representations",
             {
-                ModalityType.TEXT: [W2V],
+                ModalityType.TEXT: [W2V, BoW],
                 ModalityType.AUDIO: [Spectrogram, ZeroCrossing, Spectral, Pitch],
                 ModalityType.TIMESERIES: [ResNet],
                 ModalityType.VIDEO: [ResNet],
@@ -186,20 +208,36 @@ class TestUnimodalRepresentationOptimizer(unittest.TestCase):
             },
         ):
             registry = Registry()
-
-            unimodal_optimizer = UnimodalOptimizer([modality], self.tasks, False)
+            registry._fusion_operators = [Average, Concatenation, LSTM]
+            unimodal_optimizer = UnimodalOptimizer(modalities, self.tasks, False)
             unimodal_optimizer.optimize()
 
-            assert (
-                unimodal_optimizer.operator_performance.modality_ids[0]
-                == modality.modality_id
+            hp = HyperparameterTuner(
+                modalities, self.tasks, unimodal_optimizer.operator_performance
             )
-            assert len(unimodal_optimizer.operator_performance.task_names) == 2
-            result, cached = unimodal_optimizer.operator_performance.get_k_best_results(
-                modality, 1, self.tasks[0]
-            )
-            assert len(result) == 1
-            assert len(cached) == 1
+
+            if multimodal:
+                m_o = MultimodalOptimizer(
+                    modalities,
+                    unimodal_optimizer.operator_performance,
+                    self.tasks,
+                    debug=False,
+                    min_modalities=2,
+                    max_modalities=3,
+                )
+                fusion_results = m_o.optimize()
+
+                hp.tune_multimodal_representations(
+                    fusion_results,
+                    k=1,
+                    optimize_unimodal=tune_unimodal_representations,
+                )
+
+            else:
+                hp.tune_unimodal_representations()
+
+            assert len(hp.results) == len(self.tasks)
+            assert len(hp.results[self.tasks[0].model.name]) == 2
 
 
 if __name__ == "__main__":
diff --git a/src/main/python/tests/scuro/test_multimodal_fusion.py b/src/main/python/tests/scuro/test_multimodal_fusion.py
index ae3ddedffb..0925e47cf2 100644
--- a/src/main/python/tests/scuro/test_multimodal_fusion.py
+++ b/src/main/python/tests/scuro/test_multimodal_fusion.py
@@ -19,10 +19,7 @@
 #
 # -------------------------------------------------------------
 
-
-import shutil
 import unittest
-from multiprocessing import freeze_support
 
 import numpy as np
 from sklearn import svm
@@ -32,30 +29,22 @@ from sklearn.model_selection import train_test_split
 from systemds.scuro.drsearch.multimodal_optimizer import MultimodalOptimizer
 from systemds.scuro.drsearch.unimodal_optimizer import UnimodalOptimizer
 from systemds.scuro.representations.concatenation import Concatenation
+from systemds.scuro.representations.lstm import LSTM
 from systemds.scuro.representations.average import Average
-from systemds.scuro.drsearch.fusion_optimizer import FusionOptimizer
 from systemds.scuro.drsearch.operator_registry import Registry
 from systemds.scuro.models.model import Model
 from systemds.scuro.drsearch.task import Task
-from systemds.scuro.drsearch.unimodal_representation_optimizer import (
-    UnimodalRepresentationOptimizer,
-)
 
 from systemds.scuro.representations.spectrogram import Spectrogram
 from systemds.scuro.representations.word2vec import W2V
 from systemds.scuro.modality.unimodal_modality import UnimodalModality
 from systemds.scuro.representations.resnet import ResNet
 from tests.scuro.data_generator import (
-    setup_data,
     TestDataLoader,
     ModalityRandomDataGenerator,
 )
 
-from systemds.scuro.dataloader.audio_loader import AudioLoader
-from systemds.scuro.dataloader.video_loader import VideoLoader
-from systemds.scuro.dataloader.text_loader import TextLoader
 from systemds.scuro.modality.type import ModalityType
-
 from unittest.mock import patch
 
 
@@ -144,24 +133,17 @@ class TestMultimodalRepresentationOptimizer(unittest.TestCase):
         )
 
         audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data(
-            self.num_instances, 100
+            self.num_instances, 1000
         )
         text_data, text_md = ModalityRandomDataGenerator().create_text_data(
             self.num_instances
         )
-        video_data, video_md = ModalityRandomDataGenerator().create_visual_modality(
-            self.num_instances, 60
-        )
+
         audio = UnimodalModality(
             TestDataLoader(
                 self.indices, None, ModalityType.AUDIO, audio_data, np.float32, audio_md
             )
         )
-        video = UnimodalModality(
-            TestDataLoader(
-                self.indices, None, ModalityType.VIDEO, video_data, np.float32, video_md
-            )
-        )
         text = UnimodalModality(
             TestDataLoader(
                 self.indices, None, ModalityType.TEXT, text_data, str, text_md
@@ -180,48 +162,26 @@ class TestMultimodalRepresentationOptimizer(unittest.TestCase):
             },
         ):
             registry = Registry()
-            registry._fusion_operators = [Average, Concatenation]
-            unimodal_optimizer = UnimodalOptimizer(
-                [audio, text, video], [task], debug=False
-            )
+            registry._fusion_operators = [Average, Concatenation, LSTM]
+            unimodal_optimizer = UnimodalOptimizer([audio, text], [task], debug=False)
             unimodal_optimizer.optimize()
             unimodal_optimizer.operator_performance.get_k_best_results(audio, 2, task)
-
-            multimodal_optimizer = MultimodalOptimizer(
-                [audio, text, video],
+            m_o = MultimodalOptimizer(
+                [audio, text],
                 unimodal_optimizer.operator_performance,
                 [task],
                 debug=False,
+                min_modalities=2,
+                max_modalities=3,
             )
+            fusion_results = m_o.optimize()
 
-            multimodal_optimizer.optimize()
+            best_results = sorted(
+                fusion_results[task.model.name], key=lambda x: x.val_score, reverse=True
+            )[:2]
 
-            assert (
-                len(multimodal_optimizer.optimization_results.results["TestSVM"].keys())
-                == 57
-            )
-            assert (
-                len(
-                    multimodal_optimizer.optimization_results.results["TestSVM"][
-                        "0_1_2_3_4_5"
-                    ]
-                )
-                == 62
-            )
-            assert (
-                len(
-                    multimodal_optimizer.optimization_results.results["TestSVM"][
-                        "3_4_5"
-                    ]
-                )
-                == 6
-            )
-            assert (
-                len(multimodal_optimizer.optimization_results.results["TestSVM"]["0_1"])
-                == 2
-            )
+            assert best_results[0].val_score >= best_results[1].val_score
 
 
 if __name__ == "__main__":
-    freeze_support()
     unittest.main()
diff --git a/src/main/python/tests/scuro/test_operator_registry.py b/src/main/python/tests/scuro/test_operator_registry.py
index 0d83d83bda..b5fa4b01b4 100644
--- a/src/main/python/tests/scuro/test_operator_registry.py
+++ b/src/main/python/tests/scuro/test_operator_registry.py
@@ -74,9 +74,9 @@ class TestOperatorRegistry(unittest.TestCase):
             # SwinVideoTransformer,
         ]
 
-    def test_timeseries_representations_in_registry(self):
-        registry = Registry()
-        assert registry.get_representations(ModalityType.TIMESERIES) == [ResNet]
+    # def test_timeseries_representations_in_registry(self):
+    #     registry = Registry()
+    #     assert registry.get_representations(ModalityType.TIMESERIES) == [ResNet]
 
     def test_text_representations_in_registry(self):
         registry = Registry()
diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_unimodal_optimizer.py
index a4952d29f9..e2f0378d58 100644
--- a/src/main/python/tests/scuro/test_unimodal_optimizer.py
+++ b/src/main/python/tests/scuro/test_unimodal_optimizer.py
@@ -39,6 +39,7 @@ from systemds.scuro.representations.covarep_audio_features import (
     Pitch,
 )
 from systemds.scuro.representations.word2vec import W2V
+from systemds.scuro.representations.bow import BoW
 from systemds.scuro.modality.unimodal_modality import UnimodalModality
 from systemds.scuro.representations.resnet import ResNet
 from tests.scuro.data_generator import ModalityRandomDataGenerator, TestDataLoader
@@ -164,7 +165,7 @@ class TestUnimodalRepresentationOptimizer(unittest.TestCase):
 
     def test_unimodal_optimizer_for_video_modality(self):
         video_data, video_md = ModalityRandomDataGenerator().create_visual_modality(
-            self.num_instances, 60
+            self.num_instances, 10, 10
         )
         video = UnimodalModality(
             TestDataLoader(
@@ -178,7 +179,7 @@ class TestUnimodalRepresentationOptimizer(unittest.TestCase):
             Registry,
             "_representations",
             {
-                ModalityType.TEXT: [W2V],
+                ModalityType.TEXT: [W2V, BoW],
                 ModalityType.AUDIO: [Spectrogram, ZeroCrossing, Spectral, Pitch],
                 ModalityType.TIMESERIES: [ResNet],
                 ModalityType.VIDEO: [ResNet],
@@ -201,6 +202,9 @@ class TestUnimodalRepresentationOptimizer(unittest.TestCase):
             assert len(result) == 1
             assert len(cached) == 1
 
+    # Todo: Add a test with all representations at once
+    # Todo: Add test with only one model
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/src/main/python/tests/scuro/test_unimodal_representations.py b/src/main/python/tests/scuro/test_unimodal_representations.py
index 52bca501ac..8c8e9baa2d 100644
--- a/src/main/python/tests/scuro/test_unimodal_representations.py
+++ b/src/main/python/tests/scuro/test_unimodal_representations.py
@@ -19,8 +19,6 @@
 #
 # -------------------------------------------------------------
 
-import os
-import shutil
 import unittest
 import copy
 import numpy as np
@@ -37,20 +35,14 @@ from systemds.scuro.representations.spectrogram import Spectrogram
 from systemds.scuro.representations.word2vec import W2V
 from systemds.scuro.representations.tfidf import TfIdf
 from systemds.scuro.modality.unimodal_modality import UnimodalModality
-from systemds.scuro.representations.bert import Bert
 from systemds.scuro.representations.mel_spectrogram import MelSpectrogram
 from systemds.scuro.representations.mfcc import MFCC
 from systemds.scuro.representations.resnet import ResNet
 from systemds.scuro.representations.swin_video_transformer import SwinVideoTransformer
-from tests.scuro.data_generator import setup_data
 from tests.scuro.data_generator import (
-    setup_data,
     TestDataLoader,
     ModalityRandomDataGenerator,
 )
-from systemds.scuro.dataloader.audio_loader import AudioLoader
-from systemds.scuro.dataloader.video_loader import VideoLoader
-from systemds.scuro.dataloader.text_loader import TextLoader
 from systemds.scuro.modality.type import ModalityType
 
 
@@ -78,7 +70,7 @@ class TestUnimodalRepresentations(unittest.TestCase):
             ZeroCrossing(),
             RMSE(),
             Pitch(),
-        ]  # TODO: add FFT, TFN, 1DCNN
+        ]
         audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data(
             self.num_instances, 1000
         )
@@ -120,7 +112,7 @@ class TestUnimodalRepresentations(unittest.TestCase):
             assert r.data[0].ndim == 2
 
     def test_text_representations(self):
-        test_representations = [BoW(2, 2), W2V(5, 2, 2), TfIdf(2), Bert()]
+        test_representations = [BoW(2, 2), TfIdf(), W2V()]
         text_data, text_md = ModalityRandomDataGenerator().create_text_data(
             self.num_instances
         )
diff --git a/src/main/python/tests/scuro/test_window_operations.py b/src/main/python/tests/scuro/test_window_operations.py
index 9aab25a814..d98e9ff4f3 100644
--- a/src/main/python/tests/scuro/test_window_operations.py
+++ b/src/main/python/tests/scuro/test_window_operations.py
@@ -53,7 +53,7 @@ class TestWindowOperations(unittest.TestCase):
                 md,
             )
         )
-        aggregated_window = modality.context(StaticWindow(num_windows))
+        aggregated_window = modality.context(StaticWindow(num_windows=num_windows))
 
         for i in range(0, self.num_instances):
             assert len(aggregated_window.data[i]) == num_windows
@@ -71,7 +71,7 @@ class TestWindowOperations(unittest.TestCase):
                 md,
             )
         )
-        aggregated_window = modality.context(DynamicWindow(num_windows))
+        aggregated_window = modality.context(DynamicWindow(num_windows=num_windows))
 
         for i in range(0, self.num_instances):
             assert len(aggregated_window.data[i]) == num_windows