From 45af76a1487dcc92155aef0f1efaf6de3e4c8ffe Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Sat, 29 Nov 2025 20:23:11 +0100 Subject: [PATCH 1/6] add metrics to optimizers --- .../scuro/drsearch/hyperparameter_tuner.py | 4 ++- .../scuro/drsearch/multimodal_optimizer.py | 4 ++- .../python/systemds/scuro/drsearch/ranking.py | 24 ++++++++-------- .../python/systemds/scuro/drsearch/task.py | 2 +- .../scuro/drsearch/unimodal_optimizer.py | 28 +++++++++++++------ 5 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py index 8c5e4c24e1e..a90485dfa2c 100644 --- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py +++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py @@ -103,7 +103,9 @@ def extract_k_best_modalities_per_task(self): representations[task.model.name] = {} for modality in self.modalities: k_best_results, cached_data = ( - self.optimization_results.get_k_best_results(modality, self.k, task) + self.optimization_results.get_k_best_results( + modality, self.k, task, self.scoring_metric + ) ) representations[task.model.name][modality.modality_id] = k_best_results self.k_best_representations[task.model.name].extend(k_best_results) diff --git a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py index 9d0088a976a..7266ad933d5 100644 --- a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py @@ -106,6 +106,7 @@ def __init__( debug: bool = True, min_modalities: int = 2, max_modalities: int = None, + metric: str = "accuracy", ): self.modalities = modalities self.tasks = tasks @@ -121,6 +122,7 @@ def __init__( unimodal_optimization_results ) self.optimization_results = [] + self.metric_name = metric def optimize_parallel( self, max_combinations: int = None, max_workers: int = 2, batch_size: int = 4 @@ -242,7 +244,7 @@ def _extract_k_best_representations( for modality in self.modalities: k_best_results, cached_data = ( unimodal_optimization_results.get_k_best_results( - modality, self.k, task + modality, self.k, task, self.metric_name ) ) diff --git a/src/main/python/systemds/scuro/drsearch/ranking.py b/src/main/python/systemds/scuro/drsearch/ranking.py index 831a059eb88..3a231de9133 100644 --- a/src/main/python/systemds/scuro/drsearch/ranking.py +++ b/src/main/python/systemds/scuro/drsearch/ranking.py @@ -19,8 +19,7 @@ # # ------------------------------------------------------------- -from dataclasses import replace -from typing import Callable, Iterable, List, Optional +from typing import Callable, Iterable, Optional, Union, Any def rank_by_tradeoff( @@ -31,7 +30,7 @@ def rank_by_tradeoff( runtime_accessor: Optional[Callable[[object], float]] = None, cache_scores: bool = True, score_attr: str = "tradeoff_score", -) -> List: +) -> Union[list[Any], tuple[list[Any], list[int]]]: entries = list(entries) if not entries: return [] @@ -77,14 +76,17 @@ def safe_normalize(values, vmin, vmax): if cache_scores: for entry, score in zip(entries, scores): if hasattr(entry, score_attr): - try: - new_entry = replace(entry, **{score_attr: score}) - entries[entries.index(entry)] = new_entry - except TypeError: - setattr(entry, score_attr, score) + setattr(entry, score_attr, score) else: setattr(entry, score_attr, score) - return sorted( - entries, key=lambda entry: getattr(entry, score_attr, 0.0), reverse=True - ) + sorted_entries = sorted(entries, key=lambda e: e.tradeoff_score, reverse=True) + + sorted_indices = [ + i + for i, _ in sorted( + enumerate(entries), key=lambda pair: pair[1].tradeoff_score, reverse=True + ) + ] + + return sorted_entries, sorted_indices diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py index bfd1f16ab37..425a91b87e2 100644 --- a/src/main/python/systemds/scuro/drsearch/task.py +++ b/src/main/python/systemds/scuro/drsearch/task.py @@ -69,7 +69,7 @@ def __init__( val_indices: List, kfold=5, measure_performance=True, - performance_measures="accuracy", + performance_measures=["accuracy"], ): """ Parent class for the prediction task that is performed on top of the aligned representation diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index 7735986c2e6..86c06930471 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -102,6 +102,17 @@ def store_results(self, file_name=None): with open(file_name, "wb") as f: pickle.dump(self.operator_performance.results, f) + def store_cache(self, file_name=None): + if file_name is None: + import time + + timestr = time.strftime("%Y%m%d-%H%M%S") + file_name = "unimodal_optimizer_cache" + timestr + ".pkl" + + file_name = f"{self.result_path}/{file_name}" + with open(file_name, "wb") as f: + pickle.dump(self.operator_performance.cache, f) + def load_results(self, file_name): with open(file_name, "rb") as f: self.operator_performance.results = pickle.load(f) @@ -414,22 +425,23 @@ def print_results(self): for entry in self.results[modality][task_name]: print(f"{modality}_{task_name}: {entry}") - def get_k_best_results(self, modality, k, task): + def get_k_best_results(self, modality, k, task, performance_metric_name): """ Get the k best results for the given modality :param modality: modality to get the best results for :param k: number of best results + :param task: task to get the best results for + :param performance_metric_name: name of the performance metric to use for ranking """ task_results = self.results[modality.modality_id][task.model.name] - results = rank_by_tradeoff(task_results)[:k] + results, sorted_indices = rank_by_tradeoff( + task_results, performance_metric_name=performance_metric_name + ) - sorted_indices = sorted( - range(len(task_results)), - key=lambda x: task_results[x].tradeoff_score, - reverse=True, - )[:k] + results = results[:k] + sorted_indices = sorted_indices[:k] task_cache = self.cache.get(modality.modality_id, {}).get(task.model.name, None) if not task_cache: @@ -446,7 +458,7 @@ def get_k_best_results(self, modality, k, task): return results, cache -@dataclass(frozen=True) +@dataclass class ResultEntry: val_score: PerformanceMeasure = None train_score: PerformanceMeasure = None From 66756ff8018a63210fed0e5ca66cc45b5305d641 Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Sat, 29 Nov 2025 20:39:51 +0100 Subject: [PATCH 2/6] adapt tests --- .../systemds/scuro/drsearch/multimodal_optimizer.py | 2 +- src/main/python/tests/scuro/test_multimodal_fusion.py | 8 ++++++-- src/main/python/tests/scuro/test_unimodal_optimizer.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py index 7266ad933d5..a1ad485547d 100644 --- a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py @@ -117,12 +117,12 @@ def __init__( self.operator_registry = Registry() self.fusion_operators = self.operator_registry.get_fusion_operators() + self.metric_name = metric self.k_best_representations = self._extract_k_best_representations( unimodal_optimization_results ) self.optimization_results = [] - self.metric_name = metric def optimize_parallel( self, max_combinations: int = None, max_workers: int = 2, batch_size: int = 4 diff --git a/src/main/python/tests/scuro/test_multimodal_fusion.py b/src/main/python/tests/scuro/test_multimodal_fusion.py index 395a9cd8623..72e9148edc1 100644 --- a/src/main/python/tests/scuro/test_multimodal_fusion.py +++ b/src/main/python/tests/scuro/test_multimodal_fusion.py @@ -174,7 +174,9 @@ def test_multimodal_fusion(self): registry._fusion_operators = [Average, Concatenation, LSTM] unimodal_optimizer = UnimodalOptimizer([audio, text], [task], debug=False) unimodal_optimizer.optimize() - unimodal_optimizer.operator_performance.get_k_best_results(audio, 2, task) + unimodal_optimizer.operator_performance.get_k_best_results( + audio, 2, task, "accuracy" + ) m_o = MultimodalOptimizer( [audio, text], unimodal_optimizer.operator_performance, @@ -238,7 +240,9 @@ def test_parallel_multimodal_fusion(self): registry._fusion_operators = [Average, Concatenation, LSTM] unimodal_optimizer = UnimodalOptimizer([audio, text], [task], debug=False) unimodal_optimizer.optimize() - unimodal_optimizer.operator_performance.get_k_best_results(audio, 2, task) + unimodal_optimizer.operator_performance.get_k_best_results( + audio, 2, task, "accuracy" + ) m_o = MultimodalOptimizer( [audio, text], unimodal_optimizer.operator_performance, diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_unimodal_optimizer.py index 252dfe997a7..0ca74212d82 100644 --- a/src/main/python/tests/scuro/test_unimodal_optimizer.py +++ b/src/main/python/tests/scuro/test_unimodal_optimizer.py @@ -218,7 +218,7 @@ def optimize_unimodal_representation_for_modality(self, modality): ) assert len(unimodal_optimizer.operator_performance.task_names) == 2 result, cached = unimodal_optimizer.operator_performance.get_k_best_results( - modality, 1, self.tasks[0] + modality, 1, self.tasks[0], "accuracy" ) assert len(result) == 1 assert len(cached) == 1 From 16f1c5c9cbd21ef7ef6de1469bb59faa549f4cec Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Mon, 1 Dec 2025 19:29:42 +0100 Subject: [PATCH 3/6] fix errors in ranking --- .../scuro/drsearch/multimodal_optimizer.py | 8 ++--- .../python/systemds/scuro/drsearch/task.py | 6 ++++ .../scuro/drsearch/unimodal_optimizer.py | 3 +- .../systemds/scuro/modality/modality.py | 2 ++ .../python/systemds/scuro/modality/type.py | 4 +-- .../systemds/scuro/representations/fusion.py | 35 ++++++------------- 6 files changed, 26 insertions(+), 32 deletions(-) diff --git a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py index a1ad485547d..5063ae6746c 100644 --- a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py @@ -83,8 +83,8 @@ def _evaluate_dag_worker(dag_pickle, task_pickle, modalities_pickle, debug=False return OptimizationResult( dag=dag_copy, - train_score=scores[0], - val_score=scores[1], + train_score=scores[0].average_scores, + val_score=scores[1].average_scores, runtime=total_time, task_name=task_copy.model.name, task_time=eval_time, @@ -390,8 +390,8 @@ def _evaluate_dag(self, dag: RepresentationDag, task: Task) -> "OptimizationResu return OptimizationResult( dag=dag_copy, - train_score=scores[0], - val_score=scores[1], + train_score=scores[0].average_scores, + val_score=scores[1].average_scores, runtime=total_time, representation_time=total_time - eval_time, task_name=task_copy.model.name, diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py index 425a91b87e2..78998964bb7 100644 --- a/src/main/python/systemds/scuro/drsearch/task.py +++ b/src/main/python/systemds/scuro/drsearch/task.py @@ -193,3 +193,9 @@ def create_representation_and_run( self.training_time = np.mean(self.training_time) return [np.mean(train_scores), np.mean(test_scores)] + + def evaluate_on_test_data(self, data): + X_train, y_train, X_test, y_test = self.get_train_test_split(data) + self.model.fit(X_train, y_train, X_val=None, y_val=None) + score = self.model.test(X_test, y_test) + return score diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index 86c06930471..c66f18e2783 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -54,7 +54,6 @@ def __init__( ): self.modalities = modalities self.tasks = tasks - self.run = None self.save_all_results = save_all_results self.result_path = result_path @@ -65,7 +64,7 @@ def __init__( self.debug = debug self.operator_registry = Registry() - self.operator_performance = UnimodalResults(modalities, tasks, debug, self.run) + self.operator_performance = UnimodalResults(modalities, tasks, debug, True) self._tasks_require_same_dims = True self.expected_dimensions = tasks[0].expected_dim diff --git a/src/main/python/systemds/scuro/modality/modality.py b/src/main/python/systemds/scuro/modality/modality.py index 98dd631e12c..07f80cbd9ff 100644 --- a/src/main/python/systemds/scuro/modality/modality.py +++ b/src/main/python/systemds/scuro/modality/modality.py @@ -93,6 +93,8 @@ def update_metadata(self): for i, (md_k, md_v) in enumerate(md_copy.items()): updated_md = self.modality_type.update_metadata(md_v, self.data[i]) self.metadata[md_k] = updated_md + if i == 0: + self.data_type = updated_md["data_layout"]["type"] def flatten(self, padding=False): """ diff --git a/src/main/python/systemds/scuro/modality/type.py b/src/main/python/systemds/scuro/modality/type.py index c6f713df240..c2fe38176f7 100644 --- a/src/main/python/systemds/scuro/modality/type.py +++ b/src/main/python/systemds/scuro/modality/type.py @@ -281,7 +281,7 @@ def create_video_metadata(self, frequency, length, width, height, num_channels): md["num_channels"] = num_channels md["timestamp"] = create_timestamps(frequency, length) md["data_layout"]["representation"] = DataLayout.NESTED_LEVEL - md["data_layout"]["type"] = float + md["data_layout"]["type"] = np.float32 md["data_layout"]["shape"] = (width, height, num_channels) return md @@ -291,7 +291,7 @@ def create_image_metadata(self, width, height, num_channels): md["height"] = height md["num_channels"] = num_channels md["data_layout"]["representation"] = DataLayout.SINGLE_LEVEL - md["data_layout"]["type"] = float + md["data_layout"]["type"] = np.float32 md["data_layout"]["shape"] = (width, height, num_channels) return md diff --git a/src/main/python/systemds/scuro/representations/fusion.py b/src/main/python/systemds/scuro/representations/fusion.py index 8cf67b1cb42..d491dcad6b9 100644 --- a/src/main/python/systemds/scuro/representations/fusion.py +++ b/src/main/python/systemds/scuro/representations/fusion.py @@ -121,29 +121,16 @@ def get_max_embedding_size(self, modalities: List[Modality]): :param modalities: List of modalities :return: maximum embedding size """ - try: - modalities[0].data = np.array(modalities[0].data) - except: - pass - - if isinstance(modalities[0].data[0], list): - max_size = modalities[0].data[0][0].shape[1] - elif isinstance(modalities[0].data, np.ndarray): - max_size = modalities[0].data.shape[1] - else: - max_size = modalities[0].data[0].shape[1] - for idx in range(1, len(modalities)): - if isinstance(modalities[idx].data[0], list): - curr_shape = modalities[idx].data[0][0].shape - elif isinstance(modalities[idx].data, np.ndarray): - curr_shape = modalities[idx].data.shape - else: - curr_shape = modalities[idx].data[0].shape - if len(modalities[idx - 1].data) != len(modalities[idx].data): - raise f"Modality sizes don't match!" - elif len(curr_shape) == 1: - continue - elif curr_shape[1] > max_size: - max_size = curr_shape[1] + max_size = 0 + for m in modalities: + data = m.data + if isinstance(data, memoryview): + data = np.array(data) + arr = np.asarray(data) + if arr.ndim < 2: + continue + emb_size = arr.shape[1] + if emb_size > max_size: + max_size = emb_size return max_size From f5287adf1e682253a21255f8c1c1382cff7075bd Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Mon, 1 Dec 2025 20:11:40 +0100 Subject: [PATCH 4/6] fix tests --- src/main/python/systemds/scuro/drsearch/ranking.py | 5 +++-- .../scuro/representations/concatenation.py | 2 +- .../representations/timeseries_representations.py | 2 +- src/main/python/tests/scuro/data_generator.py | 8 ++++---- .../python/tests/scuro/test_multimodal_fusion.py | 14 +++++++------- .../python/tests/scuro/test_unimodal_optimizer.py | 2 +- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/main/python/systemds/scuro/drsearch/ranking.py b/src/main/python/systemds/scuro/drsearch/ranking.py index 3a231de9133..b4b8a392ea1 100644 --- a/src/main/python/systemds/scuro/drsearch/ranking.py +++ b/src/main/python/systemds/scuro/drsearch/ranking.py @@ -19,7 +19,7 @@ # # ------------------------------------------------------------- -from typing import Callable, Iterable, Optional, Union, Any +from typing import Callable, Iterable, Optional def rank_by_tradeoff( @@ -30,7 +30,7 @@ def rank_by_tradeoff( runtime_accessor: Optional[Callable[[object], float]] = None, cache_scores: bool = True, score_attr: str = "tradeoff_score", -) -> Union[list[Any], tuple[list[Any], list[int]]]: +): entries = list(entries) if not entries: return [] @@ -38,6 +38,7 @@ def rank_by_tradeoff( performance_score_accessor = lambda entry: getattr(entry, "val_score")[ performance_metric_name ] + if runtime_accessor is None: def runtime_accessor(entry): diff --git a/src/main/python/systemds/scuro/representations/concatenation.py b/src/main/python/systemds/scuro/representations/concatenation.py index bf854a481fd..ea199d58274 100644 --- a/src/main/python/systemds/scuro/representations/concatenation.py +++ b/src/main/python/systemds/scuro/representations/concatenation.py @@ -51,7 +51,7 @@ def execute(self, modalities: List[Modality]): max_emb_size = self.get_max_embedding_size(modalities) size = len(modalities[0].data) - if modalities[0].data.ndim > 2: + if np.array(modalities[0].data).ndim > 2: data = np.zeros((size, max_emb_size, 0)) else: data = np.zeros((size, 0)) diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py index d1dee67f861..631294809ee 100644 --- a/src/main/python/systemds/scuro/representations/timeseries_representations.py +++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py @@ -46,7 +46,7 @@ def transform(self, modality): feature = self.compute_feature(signal) result.append(feature) - transformed_modality.data = np.vstack(result).astype( + transformed_modality.data = np.vstack(np.array(result)).astype( modality.metadata[list(modality.metadata.keys())[0]]["data_layout"]["type"] ) return transformed_modality diff --git a/src/main/python/tests/scuro/data_generator.py b/src/main/python/tests/scuro/data_generator.py index 3c43cabb3ee..5bec163fe76 100644 --- a/src/main/python/tests/scuro/data_generator.py +++ b/src/main/python/tests/scuro/data_generator.py @@ -198,14 +198,14 @@ def create_visual_modality( ): if max_num_frames > 1: data = [ - np.random.randint( - 0, - 256, + np.random.uniform( + 0.0, + 1.0, (np.random.randint(10, max_num_frames + 1), height, width, 3), - dtype=np.uint8, ) for _ in range(num_instances) ] + metadata = { i: ModalityType.VIDEO.create_metadata( 30, data[i].shape[0], width, height, 3 diff --git a/src/main/python/tests/scuro/test_multimodal_fusion.py b/src/main/python/tests/scuro/test_multimodal_fusion.py index 72e9148edc1..f98824a16a5 100644 --- a/src/main/python/tests/scuro/test_multimodal_fusion.py +++ b/src/main/python/tests/scuro/test_multimodal_fusion.py @@ -189,13 +189,13 @@ def test_multimodal_fusion(self): best_results = sorted( fusion_results[task.model.name], - key=lambda x: getattr(x, "val_score").average_scores["accuracy"], + key=lambda x: getattr(x, "val_score")["accuracy"], reverse=True, )[:2] assert ( - best_results[0].val_score.average_scores["accuracy"] - >= best_results[1].val_score.average_scores["accuracy"] + best_results[0].val_score["accuracy"] + >= best_results[1].val_score["accuracy"] ) def test_parallel_multimodal_fusion(self): @@ -256,21 +256,21 @@ def test_parallel_multimodal_fusion(self): best_results = sorted( fusion_results[task.model.name], - key=lambda x: getattr(x, "val_score").average_scores["accuracy"], + key=lambda x: getattr(x, "val_score")["accuracy"], reverse=True, ) best_results_parallel = sorted( parallel_fusion_results[task.model.name], - key=lambda x: getattr(x, "val_score").average_scores["accuracy"], + key=lambda x: getattr(x, "val_score")["accuracy"], reverse=True, ) assert len(best_results) == len(best_results_parallel) for i in range(len(best_results)): assert ( - best_results[i].val_score.average_scores["accuracy"] - == best_results_parallel[i].val_score.average_scores["accuracy"] + best_results[i].val_score["accuracy"] + == best_results_parallel[i].val_score["accuracy"] ) diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_unimodal_optimizer.py index 0ca74212d82..ca54ee64b1d 100644 --- a/src/main/python/tests/scuro/test_unimodal_optimizer.py +++ b/src/main/python/tests/scuro/test_unimodal_optimizer.py @@ -210,7 +210,7 @@ def optimize_unimodal_representation_for_modality(self, modality): registry = Registry() unimodal_optimizer = UnimodalOptimizer([modality], self.tasks, False) - unimodal_optimizer.optimize_parallel() + unimodal_optimizer.optimize() assert ( unimodal_optimizer.operator_performance.modality_ids[0] From 06b920ac3e8c22249d7f0879aa9f87e4663e0199 Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Tue, 2 Dec 2025 10:50:56 +0100 Subject: [PATCH 5/6] fix tests --- src/main/python/tests/scuro/test_unimodal_representations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/python/tests/scuro/test_unimodal_representations.py b/src/main/python/tests/scuro/test_unimodal_representations.py index 3bc28ee23c5..0313cd29f88 100644 --- a/src/main/python/tests/scuro/test_unimodal_representations.py +++ b/src/main/python/tests/scuro/test_unimodal_representations.py @@ -169,7 +169,6 @@ def test_image_representations(self): def test_video_representations(self): video_representations = [ CLIPVisual(), - ColorHistogram(), I3D(), X3D(), VGG19(), From c2f67b5de3c761d127d2ab5415e8e8edc8dda37c Mon Sep 17 00:00:00 2001 From: Christina Dionysio Date: Mon, 8 Dec 2025 09:43:33 +0100 Subject: [PATCH 6/6] access metrics correclty in optimizers --- .../scuro/drsearch/multimodal_optimizer.py | 6 + .../python/systemds/scuro/drsearch/task.py | 127 +++++++++--------- .../scuro/drsearch/unimodal_optimizer.py | 2 + 3 files changed, 75 insertions(+), 60 deletions(-) diff --git a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py index 5063ae6746c..76831f6aae8 100644 --- a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py @@ -19,6 +19,7 @@ # # ------------------------------------------------------------- import os +import torch import multiprocessing as mp import itertools import threading @@ -85,6 +86,7 @@ def _evaluate_dag_worker(dag_pickle, task_pickle, modalities_pickle, debug=False dag=dag_copy, train_score=scores[0].average_scores, val_score=scores[1].average_scores, + test_score=scores[2].average_scores, runtime=total_time, task_name=task_copy.model.name, task_time=eval_time, @@ -369,6 +371,8 @@ def _evaluate_dag(self, dag: RepresentationDag, task: Task) -> "OptimizationResu task_copy, ) + torch.cuda.empty_cache() + if fused_representation is None: return None @@ -392,6 +396,7 @@ def _evaluate_dag(self, dag: RepresentationDag, task: Task) -> "OptimizationResu dag=dag_copy, train_score=scores[0].average_scores, val_score=scores[1].average_scores, + test_score=scores[2].average_scores, runtime=total_time, representation_time=total_time - eval_time, task_name=task_copy.model.name, @@ -481,6 +486,7 @@ class OptimizationResult: dag: RepresentationDag train_score: PerformanceMeasure = None val_score: PerformanceMeasure = None + test_score: PerformanceMeasure = None runtime: float = 0.0 task_time: float = 0.0 representation_time: float = 0.0 diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py index 78998964bb7..fbe08bcc61e 100644 --- a/src/main/python/systemds/scuro/drsearch/task.py +++ b/src/main/python/systemds/scuro/drsearch/task.py @@ -20,12 +20,10 @@ # ------------------------------------------------------------- import copy import time -from typing import List, Union -from systemds.scuro.modality.modality import Modality -from systemds.scuro.representations.representation import Representation +from typing import List from systemds.scuro.models.model import Model import numpy as np -from sklearn.model_selection import KFold +from sklearn.model_selection import train_test_split class PerformanceMeasure: @@ -70,6 +68,7 @@ def __init__( kfold=5, measure_performance=True, performance_measures=["accuracy"], + fusion_train_split=0.8, ): """ Parent class for the prediction task that is performed on top of the aligned representation @@ -85,7 +84,7 @@ def __init__( self.model = model self.labels = labels self.train_indices = train_indices - self.val_indices = val_indices + self.test_indices = val_indices self.kfold = kfold self.measure_performance = measure_performance self.inference_time = [] @@ -94,6 +93,47 @@ def __init__( self.performance_measures = performance_measures self.train_scores = PerformanceMeasure("train", performance_measures) self.val_scores = PerformanceMeasure("val", performance_measures) + self.test_scores = PerformanceMeasure("test", performance_measures) + self.fusion_train_indices = None + self._create_cv_splits() + + def _create_cv_splits(self): + train_labels = [self.labels[i] for i in self.train_indices] + train_labels_array = np.array(train_labels) + + train_indices_array = np.array(self.train_indices) + + self.cv_train_indices = [] + self.cv_val_indices = [] + + for fold_idx in range(self.kfold): + fold_train_indices_array, fold_val_indices_array, _, _ = train_test_split( + train_indices_array, + train_labels_array, + test_size=0.2, + shuffle=True, + random_state=11 + fold_idx, + ) + + fold_train_indices = fold_train_indices_array.tolist() + fold_val_indices = fold_val_indices_array.tolist() + + self.cv_train_indices.append(fold_train_indices) + self.cv_val_indices.append(fold_val_indices) + + overlap = set(fold_train_indices) & set(fold_val_indices) + if overlap: + raise ValueError( + f"Fold {fold_idx}: Overlap detected between train and val indices: {overlap}" + ) + + all_val_indices = set() + for val_indices in self.cv_val_indices: + all_val_indices.update(val_indices) + + self.fusion_train_indices = [ + idx for idx in self.train_indices if idx not in all_val_indices + ] def create_model(self): """ @@ -107,12 +147,12 @@ def create_model(self): def get_train_test_split(self, data): X_train = [data[i] for i in self.train_indices] y_train = [self.labels[i] for i in self.train_indices] - if self.val_indices is None: + if self.test_indices is None: X_test = None y_test = None else: - X_test = [data[i] for i in self.val_indices] - y_test = [self.labels[i] for i in self.val_indices] + X_test = [data[i] for i in self.test_indices] + y_test = [self.labels[i] for i in self.test_indices] return X_train, y_train, X_test, y_test @@ -125,22 +165,25 @@ def run(self, data): """ self._reset_params() model = self.create_model() - skf = KFold(n_splits=self.kfold, shuffle=True, random_state=11) - fold = 0 - X, y, _, _ = self.get_train_test_split(data) + test_X = np.array([data[i] for i in self.test_indices]) + test_y = np.array([self.labels[i] for i in self.test_indices]) + + for fold_idx in range(self.kfold): + fold_train_indices = self.cv_train_indices[fold_idx] + fold_val_indices = self.cv_val_indices[fold_idx] - for train, test in skf.split(X, y): - train_X = np.array(X)[train] - train_y = np.array(y)[train] - test_X = np.array(X)[test] - test_y = np.array(y)[test] - self._run_fold(model, train_X, train_y, test_X, test_y) - fold += 1 + train_X = np.array([data[i] for i in fold_train_indices]) + train_y = np.array([self.labels[i] for i in fold_train_indices]) + val_X = np.array([data[i] for i in fold_val_indices]) + val_y = np.array([self.labels[i] for i in fold_val_indices]) + + self._run_fold(model, train_X, train_y, val_X, val_y, test_X, test_y) return [ self.train_scores.compute_averages(), self.val_scores.compute_averages(), + self.test_scores.compute_averages(), ] def _reset_params(self): @@ -148,54 +191,18 @@ def _reset_params(self): self.training_time = [] self.train_scores = PerformanceMeasure("train", self.performance_measures) self.val_scores = PerformanceMeasure("val", self.performance_measures) + self.test_scores = PerformanceMeasure("test", self.performance_measures) - def _run_fold(self, model, train_X, train_y, test_X, test_y): + def _run_fold(self, model, train_X, train_y, val_X, val_y, test_X, test_y): train_start = time.time() - train_score = model.fit(train_X, train_y, test_X, test_y) + train_score = model.fit(train_X, train_y, val_X, val_y) train_end = time.time() self.training_time.append(train_end - train_start) self.train_scores.add_scores(train_score[0]) + val_score = model.test(val_X, val_y) test_start = time.time() test_score = model.test(np.array(test_X), test_y) test_end = time.time() self.inference_time.append(test_end - test_start) - self.val_scores.add_scores(test_score[0]) - - def create_representation_and_run( - self, - representation: Representation, - modalities: Union[List[Modality], Modality], - ): - self._reset_params() - skf = KFold(n_splits=self.kfold, shuffle=True, random_state=11) - - fold = 0 - X, y, _, _ = self.get_train_test_split(data) - - for train, test in skf.split(X, y): - train_X = np.array(X)[train] - train_y = np.array(y)[train] - test_X = s.transform(np.array(X)[test]) - test_y = np.array(y)[test] - - if isinstance(modalities, Modality): - rep = modality.apply_representation(representation()) - else: - representation().transform( - train_X, train_y - ) # TODO: think about a way how to handle masks - - self._run_fold(train_X, train_y, test_X, test_y) - fold += 1 - - if self.measure_performance: - self.inference_time = np.mean(self.inference_time) - self.training_time = np.mean(self.training_time) - - return [np.mean(train_scores), np.mean(test_scores)] - - def evaluate_on_test_data(self, data): - X_train, y_train, X_test, y_test = self.get_train_test_split(data) - self.model.fit(X_train, y_train, X_val=None, y_val=None) - score = self.model.test(X_test, y_test) - return score + self.val_scores.add_scores(val_score[0]) + self.test_scores.add_scores(test_score[0]) diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index c66f18e2783..1a348a91df2 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -400,6 +400,7 @@ def add_result(self, scores, modality, task_name, task_time, combination, dag): entry = ResultEntry( train_score=scores[0].average_scores, val_score=scores[1].average_scores, + test_score=scores[2].average_scores, representation_time=modality.transform_time, task_time=task_time, combination=combination.name if combination else "", @@ -461,6 +462,7 @@ def get_k_best_results(self, modality, k, task, performance_metric_name): class ResultEntry: val_score: PerformanceMeasure = None train_score: PerformanceMeasure = None + test_score: PerformanceMeasure = None representation_time: float = 0.0 task_time: float = 0.0 combination: str = ""