From d871ea0ee042490e13c28958f270d0cc74d4e3bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bet=C3=BCl=20Tuba=20G=C3=BCm=C3=BC=C5=9F?= Date: Mon, 20 Jul 2026 13:46:02 +0300 Subject: [PATCH 01/21] checkpoint ve validation eklendi. --- services/ml/data/dataloader.py | 89 +++++++++++++++-- services/ml/training/train_with_validation.py | 97 +++++++++++++++++++ 2 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 services/ml/training/train_with_validation.py diff --git a/services/ml/data/dataloader.py b/services/ml/data/dataloader.py index 19bd366..2d80500 100644 --- a/services/ml/data/dataloader.py +++ b/services/ml/data/dataloader.py @@ -1,18 +1,89 @@ -from torch.utils.data import DataLoader +import torch +from torch.utils.data import DataLoader, random_split + from .dataset import BlackHoleDataset -def create_dataloader(root_dir, batch_size=16): +def _create_loader( + dataset, + batch_size=16, + shuffle=True, + num_workers=4, + pin_memory=True, + drop_last=True, +): + return DataLoader( + dataset, + batch_size=batch_size, + shuffle=shuffle, + num_workers=num_workers, + pin_memory=pin_memory, + drop_last=drop_last, + ) - dataset = BlackHoleDataset(root_dir) - loader = DataLoader( +def create_dataloader( + root_dir, + batch_size=16, + shuffle=True, + num_workers=4, + pin_memory=True, + drop_last=True, +): + dataset = BlackHoleDataset(root_dir) + return _create_loader( dataset, batch_size=batch_size, - shuffle=True, - num_workers=4, - pin_memory=True, - drop_last=True + shuffle=shuffle, + num_workers=num_workers, + pin_memory=pin_memory, + drop_last=drop_last, + ) + + +def create_train_val_loaders( + root_dir, + batch_size=16, + val_ratio=0.2, + seed=42, + shuffle=True, + num_workers=4, + pin_memory=True, + drop_last=True, +): + if not 0 < val_ratio < 1: + raise ValueError("val_ratio must be between 0 and 1") + + dataset = BlackHoleDataset(root_dir) + dataset_size = len(dataset) + + if dataset_size < 2: + raise ValueError("At least 2 samples are required to create train/val splits") + + val_size = int(dataset_size * val_ratio) + train_size = dataset_size - val_size + + if train_size == 0 or val_size == 0: + raise ValueError("The selected val_ratio results in an empty split") + + generator = torch.Generator().manual_seed(seed) + train_dataset, val_dataset = random_split(dataset, [train_size, val_size], generator=generator) + + train_loader = _create_loader( + train_dataset, + batch_size=batch_size, + shuffle=shuffle, + num_workers=num_workers, + pin_memory=pin_memory, + drop_last=drop_last, + ) + val_loader = _create_loader( + val_dataset, + batch_size=batch_size, + shuffle=False, + num_workers=num_workers, + pin_memory=pin_memory, + drop_last=False, ) - return loader + return train_loader, val_loader diff --git a/services/ml/training/train_with_validation.py b/services/ml/training/train_with_validation.py new file mode 100644 index 0000000..92b970f --- /dev/null +++ b/services/ml/training/train_with_validation.py @@ -0,0 +1,97 @@ +import os +from pathlib import Path + +import torch + +from services.ml.data.dataloader import create_train_val_loaders +from services.ml.models.unet import UNet + + +def train_model( + root_dir, + output_dir="./artifacts", + batch_size=16, + epochs=5, + learning_rate=1e-4, + val_ratio=0.2, + checkpoint_interval=1, +): + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader, val_loader = create_train_val_loaders( + root_dir, + batch_size=batch_size, + val_ratio=val_ratio, + ) + + model = UNet().to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) + criterion = torch.nn.L1Loss() + + best_val_loss = float("inf") + best_checkpoint_path = output_path / "best_model.pt" + + for epoch in range(epochs): + model.train() + running_loss = 0.0 + + for degraded, clean in train_loader: + degraded = degraded.to(device, dtype=torch.float32) + clean = clean.to(device, dtype=torch.float32) + + optimizer.zero_grad() + outputs = model(degraded) + loss = criterion(outputs, clean) + loss.backward() + optimizer.step() + running_loss += loss.item() + + train_loss = running_loss / max(1, len(train_loader)) + + model.eval() + val_loss = 0.0 + with torch.no_grad(): + for degraded, clean in val_loader: + degraded = degraded.to(device, dtype=torch.float32) + clean = clean.to(device, dtype=torch.float32) + outputs = model(degraded) + val_loss += criterion(outputs, clean).item() + + val_loss = val_loss / max(1, len(val_loader)) + + print(f"Epoch {epoch + 1}/{epochs} | train_loss={train_loss:.4f} | val_loss={val_loss:.4f}") + + if (epoch + 1) % checkpoint_interval == 0 or val_loss < best_val_loss: + checkpoint_path = output_path / f"checkpoint_epoch_{epoch + 1}.pt" + torch.save( + { + "epoch": epoch + 1, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "val_loss": val_loss, + }, + checkpoint_path, + ) + print(f"Saved checkpoint: {checkpoint_path}") + + if val_loss < best_val_loss: + best_val_loss = val_loss + torch.save( + { + "epoch": epoch + 1, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "val_loss": val_loss, + }, + best_checkpoint_path, + ) + print(f"Saved best checkpoint: {best_checkpoint_path}") + + return best_checkpoint_path + + +if __name__ == "__main__": + root_dir = os.environ.get("DEEPHORIZON_DATA_ROOT", "./data/raw/simulated") + train_model(root_dir=root_dir) From 3f3e4c2527469593684e5fbd935053451a2e9ea6 Mon Sep 17 00:00:00 2001 From: Eda Tosun Date: Mon, 20 Jul 2026 14:13:07 +0300 Subject: [PATCH 02/21] feat: add loss function and training loop --- services/ml/losses/loss.py | 6 ++++ services/ml/training/train.py | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 services/ml/losses/loss.py create mode 100644 services/ml/training/train.py diff --git a/services/ml/losses/loss.py b/services/ml/losses/loss.py new file mode 100644 index 0000000..bda5585 --- /dev/null +++ b/services/ml/losses/loss.py @@ -0,0 +1,6 @@ +import torch.nn as nn + + +def get_loss(): + + return nn.MSELoss() diff --git a/services/ml/training/train.py b/services/ml/training/train.py new file mode 100644 index 0000000..68ed308 --- /dev/null +++ b/services/ml/training/train.py @@ -0,0 +1,63 @@ +import torch + +from models.unet import UNet +from data.dataloader import create_dataloader +from losses.loss import get_loss + + +def train(): + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + model = UNet().to(device) + + dataloader = create_dataloader( + root_dir="data/train", + batch_size=16 + ) + + criterion = get_loss() + + optimizer = torch.optim.Adam( + model.parameters(), + lr=1e-4 + ) + + epochs = 10 + + for epoch in range(epochs): + + model.train() + + running_loss = 0.0 + + for degraded, clean in dataloader: + + degraded = degraded.to(device) + clean = clean.to(device) + + prediction = model(degraded) + + loss = criterion( + prediction, + clean + ) + + optimizer.zero_grad() + + loss.backward() + + optimizer.step() + + running_loss += loss.item() + + epoch_loss = running_loss / len(dataloader) + + print( + f"Epoch [{epoch + 1}/{epochs}] " + f"Loss: {epoch_loss:.6f}" + ) + + +if __name__ == "__main__": + train() From 2c4d92566cc7aec48c8954413884698bc6cff8df Mon Sep 17 00:00:00 2001 From: Eda Tosun Date: Tue, 21 Jul 2026 12:08:38 +0300 Subject: [PATCH 03/21] feat: add validation and checkpoint support to training --- services/ml/training/train.py | 134 +++++++++++++++++++++++++++++----- 1 file changed, 114 insertions(+), 20 deletions(-) diff --git a/services/ml/training/train.py b/services/ml/training/train.py index 68ed308..b35f0bf 100644 --- a/services/ml/training/train.py +++ b/services/ml/training/train.py @@ -1,46 +1,73 @@ -import torch +from pathlib import Path -from models.unet import UNet -from data.dataloader import create_dataloader -from losses.loss import get_loss +import torch +from services.ml.data.dataloader import create_train_val_loaders +from services.ml.losses.loss import get_loss +from services.ml.models.unet import UNet -def train(): - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +def train( + root_dir="data/train", + output_dir="checkpoints", + batch_size=16, + epochs=10, + learning_rate=1e-4, + val_ratio=0.2, +): - model = UNet().to(device) + device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu" + ) - dataloader = create_dataloader( - root_dir="data/train", - batch_size=16 + train_loader, val_loader = create_train_val_loaders( + root_dir=root_dir, + batch_size=batch_size, + val_ratio=val_ratio, ) + model = UNet().to(device) + criterion = get_loss() optimizer = torch.optim.Adam( model.parameters(), - lr=1e-4 + lr=learning_rate, + ) + + output_dir = Path(output_dir) + output_dir.mkdir( + parents=True, + exist_ok=True, ) - epochs = 10 + best_val_loss = float("inf") for epoch in range(epochs): + # TRAIN + model.train() - running_loss = 0.0 + running_train_loss = 0.0 - for degraded, clean in dataloader: + for degraded, clean in train_loader: - degraded = degraded.to(device) - clean = clean.to(device) + degraded = degraded.to( + device, + dtype=torch.float32, + ) + + clean = clean.to( + device, + dtype=torch.float32, + ) prediction = model(degraded) loss = criterion( prediction, - clean + clean, ) optimizer.zero_grad() @@ -49,15 +76,82 @@ def train(): optimizer.step() - running_loss += loss.item() + running_train_loss += loss.item() + + train_loss = running_train_loss / len(train_loader) + + # VALIDATION + + model.eval() + + running_val_loss = 0.0 + + with torch.no_grad(): + + for degraded, clean in val_loader: + + degraded = degraded.to( + device, + dtype=torch.float32, + ) - epoch_loss = running_loss / len(dataloader) + clean = clean.to( + device, + dtype=torch.float32, + ) + + prediction = model(degraded) + + loss = criterion( + prediction, + clean, + ) + + running_val_loss += loss.item() + + val_loss = running_val_loss / len(val_loader) + + # LOG print( f"Epoch [{epoch + 1}/{epochs}] " - f"Loss: {epoch_loss:.6f}" + f"Train Loss: {train_loss:.6f} " + f"Val Loss: {val_loss:.6f}" + ) + + # CHECKPOINT + + + checkpoint = { + "epoch": epoch + 1, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "train_loss": train_loss, + "val_loss": val_loss, + } + + torch.save( + checkpoint, + output_dir / f"epoch_{epoch + 1}.pt", ) + # BEST MODEL + + if val_loss < best_val_loss: + + best_val_loss = val_loss + + torch.save( + checkpoint, + output_dir / "best_model.pt", + ) + + print( + f"Best model updated! " + f"Validation Loss: {val_loss:.6f}" + ) + if __name__ == "__main__": + train() From d86a8f8a1179e2e042c2893fe5c7f2bdaf595547 Mon Sep 17 00:00:00 2001 From: Eda Tosun Date: Tue, 21 Jul 2026 12:32:59 +0300 Subject: [PATCH 04/21] Add checkpoint save/load integration to training pipeline --- services/ml/checkpoints/checkpoint.py | 50 +++++++++++++++++++++++++++ services/ml/training/train.py | 40 ++++++++------------- 2 files changed, 65 insertions(+), 25 deletions(-) create mode 100644 services/ml/checkpoints/checkpoint.py diff --git a/services/ml/checkpoints/checkpoint.py b/services/ml/checkpoints/checkpoint.py new file mode 100644 index 0000000..218eaa7 --- /dev/null +++ b/services/ml/checkpoints/checkpoint.py @@ -0,0 +1,50 @@ +from pathlib import Path +import torch + + +def save_checkpoint( + model, + optimizer, + epoch, + train_loss, + val_loss, + checkpoint_path, +): + + checkpoint_path = Path(checkpoint_path) + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + + torch.save( + { + "epoch": epoch, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "train_loss": train_loss, + "val_loss": val_loss, + }, + checkpoint_path, + ) + + +def load_checkpoint( + checkpoint_path, + model, + optimizer=None, + map_location="cpu", +): + + checkpoint = torch.load( + checkpoint_path, + map_location=map_location, + ) + + model.load_state_dict( + checkpoint["model_state_dict"] + ) + + if optimizer is not None: + optimizer.load_state_dict( + checkpoint["optimizer_state_dict"] + ) + + return checkpoint diff --git a/services/ml/training/train.py b/services/ml/training/train.py index b35f0bf..c9f1892 100644 --- a/services/ml/training/train.py +++ b/services/ml/training/train.py @@ -1,7 +1,6 @@ -from pathlib import Path - import torch +from services.ml.checkpoints.checkpoint import save_checkpoint from services.ml.data.dataloader import create_train_val_loaders from services.ml.losses.loss import get_loss from services.ml.models.unet import UNet @@ -35,12 +34,6 @@ def train( lr=learning_rate, ) - output_dir = Path(output_dir) - output_dir.mkdir( - parents=True, - exist_ok=True, - ) - best_val_loss = float("inf") for epoch in range(epochs): @@ -121,29 +114,26 @@ def train( # CHECKPOINT - - checkpoint = { - "epoch": epoch + 1, - "model_state_dict": model.state_dict(), - "optimizer_state_dict": optimizer.state_dict(), - "train_loss": train_loss, - "val_loss": val_loss, - } - - torch.save( - checkpoint, - output_dir / f"epoch_{epoch + 1}.pt", + save_checkpoint( + model=model, + optimizer=optimizer, + epoch=epoch + 1, + train_loss=train_loss, + val_loss=val_loss, + checkpoint_path=output_dir / f"epoch_{epoch + 1}.pt", ) - # BEST MODEL - if val_loss < best_val_loss: best_val_loss = val_loss - torch.save( - checkpoint, - output_dir / "best_model.pt", + save_checkpoint( + model=model, + optimizer=optimizer, + epoch=epoch + 1, + train_loss=train_loss, + val_loss=val_loss, + checkpoint_path=output_dir / "best_model.pt", ) print( From e74acb19c04d470bebb4abcf5e264ef42eb6947c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bet=C3=BCl=20Tuba=20G=C3=BCm=C3=BC=C5=9F?= Date: Tue, 21 Jul 2026 14:38:59 +0300 Subject: [PATCH 05/21] =?UTF-8?q?evulation=20ve=20metrics=20olu=C5=9Fturul?= =?UTF-8?q?du.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/ml/evaluation/__init__.py | 1 + services/ml/evaluation/benchmark.py | 136 ++++++++++++++++ services/ml/evaluation/metrics.py | 109 +++++++++++++ services/ml/training/train.py | 154 ++++++++---------- services/ml/training/train_with_validation.py | 97 ----------- 5 files changed, 311 insertions(+), 186 deletions(-) create mode 100644 services/ml/evaluation/__init__.py create mode 100644 services/ml/evaluation/benchmark.py create mode 100644 services/ml/evaluation/metrics.py delete mode 100644 services/ml/training/train_with_validation.py diff --git a/services/ml/evaluation/__init__.py b/services/ml/evaluation/__init__.py new file mode 100644 index 0000000..63bb28c --- /dev/null +++ b/services/ml/evaluation/__init__.py @@ -0,0 +1 @@ +"""Evaluation helpers for DeepHorizon ML workflows.""" diff --git a/services/ml/evaluation/benchmark.py b/services/ml/evaluation/benchmark.py new file mode 100644 index 0000000..a67f418 --- /dev/null +++ b/services/ml/evaluation/benchmark.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path + +import torch +from torchvision.utils import make_grid, save_image + +from .metrics import compute_metrics + + +@dataclass(frozen=True) +class ValidationSummary: + epoch: int + train_loss: float + val_loss: float + psnr: float + ssim: float + + +def _prepare_output_dir(output_dir: Path | str) -> Path: + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + return output_path + + +def save_validation_summary(summary: ValidationSummary, output_dir: Path | str) -> Path: + output_path = _prepare_output_dir(output_dir) + results_path = output_path / "validation_results.jsonl" + with results_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(asdict(summary), ensure_ascii=False) + "\n") + return results_path + + +def save_sample_outputs( + degraded: torch.Tensor, + prediction: torch.Tensor, + clean: torch.Tensor, + output_dir: Path | str, + epoch: int, + max_samples: int = 4, +) -> Path | None: + if degraded.numel() == 0: + return None + + output_path = _prepare_output_dir(output_dir) + sample_dir = output_path / "validation_samples" / f"epoch_{epoch:03d}" + sample_dir.mkdir(parents=True, exist_ok=True) + + sample_count = min(max_samples, degraded.shape[0]) + for index in range(sample_count): + triplet = torch.cat( + [degraded[index : index + 1], prediction[index : index + 1], clean[index : index + 1]], + dim=0, + ) + grid = make_grid(triplet, nrow=3, normalize=True, value_range=(0.0, 1.0)) + save_image(grid, sample_dir / f"sample_{index:02d}.png") + + return sample_dir + + +def evaluate_validation_loader( + model, + val_loader, + device: torch.device, + criterion, +) -> tuple[ValidationSummary, tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None]: + model.eval() + + running_val_loss = 0.0 + running_psnr = 0.0 + running_ssim = 0.0 + batch_count = 0 + sample_batch: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None + + with torch.no_grad(): + for degraded, clean in val_loader: + degraded = degraded.to(device, dtype=torch.float32) + clean = clean.to(device, dtype=torch.float32) + prediction = model(degraded) + + loss = criterion(prediction, clean) + metrics = compute_metrics(prediction, clean) + + running_val_loss += float(loss.item()) + running_psnr += metrics["psnr"] + running_ssim += metrics["ssim"] + batch_count += 1 + + if sample_batch is None: + sample_batch = ( + degraded.detach().cpu(), + prediction.detach().cpu(), + clean.detach().cpu(), + ) + + if batch_count == 0: + raise ValueError("val_loader produced no batches") + + summary = ValidationSummary( + epoch=0, + train_loss=0.0, + val_loss=running_val_loss / batch_count, + psnr=running_psnr / batch_count, + ssim=running_ssim / batch_count, + ) + return summary, sample_batch + + +def update_best_model( + summary: ValidationSummary, + model, + optimizer, + output_dir: Path | str, + best_val_loss: float, +) -> tuple[float, bool, Path]: + output_path = _prepare_output_dir(output_dir) + best_checkpoint_path = output_path / "best_model.pt" + + if summary.val_loss >= best_val_loss: + return best_val_loss, False, best_checkpoint_path + + torch.save( + { + "epoch": summary.epoch, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "train_loss": summary.train_loss, + "val_loss": summary.val_loss, + "psnr": summary.psnr, + "ssim": summary.ssim, + }, + best_checkpoint_path, + ) + return summary.val_loss, True, best_checkpoint_path diff --git a/services/ml/evaluation/metrics.py b/services/ml/evaluation/metrics.py new file mode 100644 index 0000000..b3e0415 --- /dev/null +++ b/services/ml/evaluation/metrics.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import math + +import torch +import torch.nn.functional as F + + +def _ensure_4d(tensor: torch.Tensor) -> torch.Tensor: + if tensor.ndim == 2: + return tensor.unsqueeze(0).unsqueeze(0) + if tensor.ndim == 3: + return tensor.unsqueeze(0) + if tensor.ndim != 4: + raise ValueError(f"Expected a 2D, 3D, or 4D tensor, got shape {tuple(tensor.shape)}") + return tensor + + +def _validate_pair(prediction: torch.Tensor, target: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + prediction = _ensure_4d(prediction).to(dtype=torch.float32) + target = _ensure_4d(target).to(dtype=torch.float32) + + if prediction.shape != target.shape: + raise ValueError( + "prediction and target must have the same shape, " + f"got {tuple(prediction.shape)} and {tuple(target.shape)}" + ) + + return prediction, target + + +def compute_psnr( + prediction: torch.Tensor, + target: torch.Tensor, + data_range: float = 1.0, +) -> float: + prediction, target = _validate_pair(prediction, target) + mse = F.mse_loss(prediction, target, reduction="mean") + + if mse <= 0: + return float("inf") + + return float(20.0 * math.log10(data_range) - 10.0 * math.log10(float(mse))) + + +def _gaussian_kernel( + window_size: int, + sigma: float, + channels: int, + device: torch.device, + dtype: torch.dtype, +) -> torch.Tensor: + coordinates = torch.arange(window_size, device=device, dtype=dtype) - window_size // 2 + gaussian = torch.exp(-(coordinates**2) / (2.0 * sigma**2)) + gaussian = gaussian / gaussian.sum() + window_2d = gaussian[:, None] * gaussian[None, :] + window_2d = window_2d / window_2d.sum() + return window_2d.expand(channels, 1, window_size, window_size).contiguous() + + +def compute_ssim( + prediction: torch.Tensor, + target: torch.Tensor, + data_range: float = 1.0, + window_size: int = 11, + sigma: float = 1.5, +) -> float: + prediction, target = _validate_pair(prediction, target) + + if window_size % 2 == 0: + raise ValueError("window_size must be odd") + + channels = prediction.shape[1] + kernel = _gaussian_kernel(window_size, sigma, channels, prediction.device, prediction.dtype) + padding = window_size // 2 + + mu_prediction = F.conv2d(prediction, kernel, padding=padding, groups=channels) + mu_target = F.conv2d(target, kernel, padding=padding, groups=channels) + + mu_prediction_sq = mu_prediction.pow(2) + mu_target_sq = mu_target.pow(2) + mu_prediction_target = mu_prediction * mu_target + + sigma_prediction_sq = F.conv2d(prediction * prediction, kernel, padding=padding, groups=channels) - mu_prediction_sq + sigma_target_sq = F.conv2d(target * target, kernel, padding=padding, groups=channels) - mu_target_sq + sigma_prediction_target = ( + F.conv2d(prediction * target, kernel, padding=padding, groups=channels) - mu_prediction_target + ) + + c1 = (0.01 * data_range) ** 2 + c2 = (0.03 * data_range) ** 2 + + numerator = (2.0 * mu_prediction_target + c1) * (2.0 * sigma_prediction_target + c2) + denominator = (mu_prediction_sq + mu_target_sq + c1) * (sigma_prediction_sq + sigma_target_sq + c2) + + ssim_map = numerator / denominator.clamp_min(1e-12) + return float(ssim_map.mean().item()) + + +def compute_metrics( + prediction: torch.Tensor, + target: torch.Tensor, + data_range: float = 1.0, +) -> dict[str, float]: + prediction, target = _validate_pair(prediction, target) + return { + "psnr": compute_psnr(prediction, target, data_range=data_range), + "ssim": compute_ssim(prediction, target, data_range=data_range), + } diff --git a/services/ml/training/train.py b/services/ml/training/train.py index c9f1892..4c0eb85 100644 --- a/services/ml/training/train.py +++ b/services/ml/training/train.py @@ -1,9 +1,17 @@ +from pathlib import Path + import torch -from services.ml.checkpoints.checkpoint import save_checkpoint +from services.ml.evaluation.benchmark import ( + ValidationSummary, + evaluate_validation_loader, + save_sample_outputs, + save_validation_summary, + update_best_model, +) from services.ml.data.dataloader import create_train_val_loaders -from services.ml.losses.loss import get_loss from services.ml.models.unet import UNet +from services.ml.losses.loss import get_loss def train( @@ -14,10 +22,10 @@ def train( learning_rate=1e-4, val_ratio=0.2, ): + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) - device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu" - ) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_loader, val_loader = create_train_val_loaders( root_dir=root_dir, @@ -26,122 +34,90 @@ def train( ) model = UNet().to(device) - criterion = get_loss() - - optimizer = torch.optim.Adam( - model.parameters(), - lr=learning_rate, - ) + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) best_val_loss = float("inf") + best_checkpoint_path = output_dir / "best_model.pt" for epoch in range(epochs): - - # TRAIN - model.train() - running_train_loss = 0.0 for degraded, clean in train_loader: - - degraded = degraded.to( - device, - dtype=torch.float32, - ) - - clean = clean.to( - device, - dtype=torch.float32, - ) + degraded = degraded.to(device, dtype=torch.float32) + clean = clean.to(device, dtype=torch.float32) prediction = model(degraded) - - loss = criterion( - prediction, - clean, - ) + loss = criterion(prediction, clean) optimizer.zero_grad() - loss.backward() - optimizer.step() - running_train_loss += loss.item() - - train_loss = running_train_loss / len(train_loader) - - # VALIDATION - - model.eval() - - running_val_loss = 0.0 + running_train_loss += float(loss.item()) - with torch.no_grad(): + train_loss = running_train_loss / max(1, len(train_loader)) - for degraded, clean in val_loader: - - degraded = degraded.to( - device, - dtype=torch.float32, - ) - - clean = clean.to( - device, - dtype=torch.float32, - ) - - prediction = model(degraded) - - loss = criterion( - prediction, - clean, - ) - - running_val_loss += loss.item() + validation_summary, sample_batch = evaluate_validation_loader( + model=model, + val_loader=val_loader, + device=device, + criterion=criterion, + ) + validation_summary = ValidationSummary( + epoch=epoch + 1, + train_loss=train_loss, + val_loss=validation_summary.val_loss, + psnr=validation_summary.psnr, + ssim=validation_summary.ssim, + ) - val_loss = running_val_loss / len(val_loader) + if sample_batch is not None: + save_sample_outputs( + degraded=sample_batch[0], + prediction=sample_batch[1], + clean=sample_batch[2], + output_dir=output_dir, + epoch=epoch + 1, + ) - # LOG + save_validation_summary(validation_summary, output_dir) print( f"Epoch [{epoch + 1}/{epochs}] " f"Train Loss: {train_loss:.6f} " - f"Val Loss: {val_loss:.6f}" + f"Val Loss: {validation_summary.val_loss:.6f} " + f"PSNR: {validation_summary.psnr:.4f} " + f"SSIM: {validation_summary.ssim:.4f}" ) - # CHECKPOINT + checkpoint_path = output_dir / f"epoch_{epoch + 1}.pt" + torch.save( + { + "epoch": epoch + 1, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "train_loss": train_loss, + "val_loss": validation_summary.val_loss, + "psnr": validation_summary.psnr, + "ssim": validation_summary.ssim, + }, + checkpoint_path, + ) - save_checkpoint( + best_val_loss, is_best, best_checkpoint_path = update_best_model( + summary=validation_summary, model=model, optimizer=optimizer, - epoch=epoch + 1, - train_loss=train_loss, - val_loss=val_loss, - checkpoint_path=output_dir / f"epoch_{epoch + 1}.pt", + output_dir=output_dir, + best_val_loss=best_val_loss, ) + if is_best: + print(f"Best model updated: {best_checkpoint_path}") - if val_loss < best_val_loss: - - best_val_loss = val_loss - - save_checkpoint( - model=model, - optimizer=optimizer, - epoch=epoch + 1, - train_loss=train_loss, - val_loss=val_loss, - checkpoint_path=output_dir / "best_model.pt", - ) - - print( - f"Best model updated! " - f"Validation Loss: {val_loss:.6f}" - ) + return best_checkpoint_path if __name__ == "__main__": - train() diff --git a/services/ml/training/train_with_validation.py b/services/ml/training/train_with_validation.py deleted file mode 100644 index 92b970f..0000000 --- a/services/ml/training/train_with_validation.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -from pathlib import Path - -import torch - -from services.ml.data.dataloader import create_train_val_loaders -from services.ml.models.unet import UNet - - -def train_model( - root_dir, - output_dir="./artifacts", - batch_size=16, - epochs=5, - learning_rate=1e-4, - val_ratio=0.2, - checkpoint_interval=1, -): - output_path = Path(output_dir) - output_path.mkdir(parents=True, exist_ok=True) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - train_loader, val_loader = create_train_val_loaders( - root_dir, - batch_size=batch_size, - val_ratio=val_ratio, - ) - - model = UNet().to(device) - optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) - criterion = torch.nn.L1Loss() - - best_val_loss = float("inf") - best_checkpoint_path = output_path / "best_model.pt" - - for epoch in range(epochs): - model.train() - running_loss = 0.0 - - for degraded, clean in train_loader: - degraded = degraded.to(device, dtype=torch.float32) - clean = clean.to(device, dtype=torch.float32) - - optimizer.zero_grad() - outputs = model(degraded) - loss = criterion(outputs, clean) - loss.backward() - optimizer.step() - running_loss += loss.item() - - train_loss = running_loss / max(1, len(train_loader)) - - model.eval() - val_loss = 0.0 - with torch.no_grad(): - for degraded, clean in val_loader: - degraded = degraded.to(device, dtype=torch.float32) - clean = clean.to(device, dtype=torch.float32) - outputs = model(degraded) - val_loss += criterion(outputs, clean).item() - - val_loss = val_loss / max(1, len(val_loader)) - - print(f"Epoch {epoch + 1}/{epochs} | train_loss={train_loss:.4f} | val_loss={val_loss:.4f}") - - if (epoch + 1) % checkpoint_interval == 0 or val_loss < best_val_loss: - checkpoint_path = output_path / f"checkpoint_epoch_{epoch + 1}.pt" - torch.save( - { - "epoch": epoch + 1, - "model_state_dict": model.state_dict(), - "optimizer_state_dict": optimizer.state_dict(), - "val_loss": val_loss, - }, - checkpoint_path, - ) - print(f"Saved checkpoint: {checkpoint_path}") - - if val_loss < best_val_loss: - best_val_loss = val_loss - torch.save( - { - "epoch": epoch + 1, - "model_state_dict": model.state_dict(), - "optimizer_state_dict": optimizer.state_dict(), - "val_loss": val_loss, - }, - best_checkpoint_path, - ) - print(f"Saved best checkpoint: {best_checkpoint_path}") - - return best_checkpoint_path - - -if __name__ == "__main__": - root_dir = os.environ.get("DEEPHORIZON_DATA_ROOT", "./data/raw/simulated") - train_model(root_dir=root_dir) From 2b5fffcce359801b161b6df3b33873bfe11df312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bet=C3=BCl=20Tuba=20G=C3=BCm=C3=BC=C5=9F?= Date: Wed, 22 Jul 2026 15:44:16 +0300 Subject: [PATCH 06/21] =?UTF-8?q?feature:=20sunucu=20verileri=20ba=C4=9Fla?= =?UTF-8?q?nd=C4=B1.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ services/ml/minio_loader.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 services/ml/minio_loader.py diff --git a/.gitignore b/.gitignore index e54f560..0af607c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ Thumbs.db *.ckpt mlruns/ wandb/ + +.env diff --git a/services/ml/minio_loader.py b/services/ml/minio_loader.py new file mode 100644 index 0000000..7812fc1 --- /dev/null +++ b/services/ml/minio_loader.py @@ -0,0 +1,33 @@ +import boto3 +import numpy as np +import io +import os +from dotenv import load_dotenv + +load_dotenv() + +def load_npy_from_minio(bucket_name, file_key): + endpoint = os.getenv("MINIO_ENDPOINT") + access_key = os.getenv("MINIO_ACCESS_KEY") + secret_key = os.getenv("MINIO_SECRET_KEY") + + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key + ) + + try: + response = s3_client.get_object(Bucket=bucket_name, Key=file_key) + + file_stream = response['Body'].read() + + image_array = np.load(io.BytesIO(file_stream)) + + print(f"Görüntü başarıyla yüklendi! Boyutları: {image_array.shape}") + return image_array + + except Exception as e: + print(f"Hata oluştu: {e}") + return None From 77d73f173fa0b59b3441e5f0f151d8d4e89ae601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bet=C3=BCl=20Tuba=20G=C3=BCm=C3=BC=C5=9F?= Date: Wed, 22 Jul 2026 16:20:10 +0300 Subject: [PATCH 07/21] =?UTF-8?q?fix:=20sunucu=20data=20ba=C4=9Flant=C4=B1?= =?UTF-8?q?s=C4=B1=20d=C3=BCzenlendi.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/ml/data/dataset.py | 45 ++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/services/ml/data/dataset.py b/services/ml/data/dataset.py index ec54c13..4e744ce 100644 --- a/services/ml/data/dataset.py +++ b/services/ml/data/dataset.py @@ -3,32 +3,41 @@ from torch.utils.data import Dataset from pathlib import Path +from services.ml.data.minio_loader import load_npy_from_minio class BlackHoleDataset(Dataset): - - def __init__(self, root_dir): + def __init__(self, root_dir, use_minio=False, bucket_name="karadelikler"): self.root_dir = Path(root_dir) - - self.clean_files = sorted( - (self.root_dir / "clean").glob("*.npy") - ) - - self.degraded_files = sorted( - (self.root_dir / "degraded").glob("*.npy") - ) - + self.use_minio = use_minio + self.bucket_name = bucket_name + + if not self.use_minio: + self.clean_files = sorted( + (self.root_dir / "clean").glob("*.npy") + ) + self.degraded_files = sorted( + (self.root_dir / "degraded").glob("*.npy") + ) + else: + self.clean_files = [] + self.degraded_files = [] def __len__(self): return len(self.clean_files) - def __getitem__(self, index): - - clean = np.load(self.clean_files[index]) - degraded = np.load(self.degraded_files[index]) - - clean = torch.from_numpy(clean) - degraded = torch.from_numpy(degraded) + if not self.use_minio: + clean_data = np.load(self.clean_files[index]) + degraded_data = np.load(self.degraded_files[index]) + else: + clean_file_key = f"clean/{self.clean_files[index]}" + degraded_file_key = f"degraded/{self.degraded_files[index]}" + + clean_data = load_npy_from_minio(self.bucket_name, clean_file_key) + degraded_data = load_npy_from_minio(self.bucket_name, degraded_file_key) + + clean = torch.from_numpy(clean_data) + degraded = torch.from_numpy(degraded_data) clean = clean.unsqueeze(0) degraded = degraded.unsqueeze(0) From a4abaf1a697223847adaf23c893d646907e9658f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bet=C3=BCl=20Tuba=20G=C3=BCm=C3=BC=C5=9F?= Date: Wed, 22 Jul 2026 17:57:51 +0300 Subject: [PATCH 08/21] =?UTF-8?q?data=20sunucu=20path=20g=C3=BCncellendi.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/data.txt | 3 +++ services/ml/data/dataset.py | 19 +++++++-------- services/ml/minio_loader.py | 46 +++++++++++++++++++++++++------------ 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/requirements/data.txt b/requirements/data.txt index 47c8a24..ca477e6 100644 --- a/requirements/data.txt +++ b/requirements/data.txt @@ -8,3 +8,6 @@ ehtim>=1.2.10 astropy>=7.0.0 dvc>=3.59.0 great-expectations>=1.4.0 + +boto3==1.34.0 +python-dotenv==1.0.1 diff --git a/services/ml/data/dataset.py b/services/ml/data/dataset.py index 4e744ce..c6b9a3e 100644 --- a/services/ml/data/dataset.py +++ b/services/ml/data/dataset.py @@ -3,13 +3,14 @@ from torch.utils.data import Dataset from pathlib import Path -from services.ml.data.minio_loader import load_npy_from_minio +from services.ml.data.minio_loader import load_npy_from_minio, list_files_in_minio class BlackHoleDataset(Dataset): - def __init__(self, root_dir, use_minio=False, bucket_name="karadelikler"): + def __init__(self, root_dir, use_minio=False, bucket_name="karadelikler", minio_prefix="datasets/training-512/v1"): self.root_dir = Path(root_dir) self.use_minio = use_minio self.bucket_name = bucket_name + self.minio_prefix = minio_prefix if not self.use_minio: self.clean_files = sorted( @@ -19,8 +20,11 @@ def __init__(self, root_dir, use_minio=False, bucket_name="karadelikler"): (self.root_dir / "degraded").glob("*.npy") ) else: - self.clean_files = [] - self.degraded_files = [] + clean_path = f"{self.minio_prefix}/clean/" + degraded_path = f"{self.minio_prefix}/degraded/" + + self.clean_files = list_files_in_minio(self.bucket_name, clean_path) + self.degraded_files = list_files_in_minio(self.bucket_name, degraded_path) def __len__(self): return len(self.clean_files) @@ -30,11 +34,8 @@ def __getitem__(self, index): clean_data = np.load(self.clean_files[index]) degraded_data = np.load(self.degraded_files[index]) else: - clean_file_key = f"clean/{self.clean_files[index]}" - degraded_file_key = f"degraded/{self.degraded_files[index]}" - - clean_data = load_npy_from_minio(self.bucket_name, clean_file_key) - degraded_data = load_npy_from_minio(self.bucket_name, degraded_file_key) + clean_data = load_npy_from_minio(self.bucket_name, self.clean_files[index]) + degraded_data = load_npy_from_minio(self.bucket_name, self.degraded_files[index]) clean = torch.from_numpy(clean_data) degraded = torch.from_numpy(degraded_data) diff --git a/services/ml/minio_loader.py b/services/ml/minio_loader.py index 7812fc1..56742a1 100644 --- a/services/ml/minio_loader.py +++ b/services/ml/minio_loader.py @@ -6,28 +6,44 @@ load_dotenv() -def load_npy_from_minio(bucket_name, file_key): - endpoint = os.getenv("MINIO_ENDPOINT") - access_key = os.getenv("MINIO_ACCESS_KEY") - secret_key = os.getenv("MINIO_SECRET_KEY") - - s3_client = boto3.client( +def get_s3_client(): + return boto3.client( 's3', - endpoint_url=endpoint, - aws_access_key_id=access_key, - aws_secret_access_key=secret_key + endpoint_url=os.getenv("MINIO_ENDPOINT"), + aws_access_key_id=os.getenv("MINIO_ACCESS_KEY"), + aws_secret_access_key=os.getenv("MINIO_SECRET_KEY") ) + +def list_files_in_minio(bucket_name, prefix_path): + """ + Sunucudaki belirli bir dosya yolu altındaki .npy dosyalarını bulur. + Örnek prefix_path: "data/training/clean/" + """ + s3_client = get_s3_client() + file_keys = [] try: - response = s3_client.get_object(Bucket=bucket_name, Key=file_key) + response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=prefix_path) + if 'Contents' in response: + for obj in response['Contents']: + # Sadece .npy uzantılı dosyaları listeye ekle + if obj['Key'].endswith('.npy'): + file_keys.append(obj['Key']) + + return sorted(file_keys) + except Exception as e: + print(f"Hata - Dosyalar listelenemedi: {e}") + return [] + +def load_npy_from_minio(bucket_name, file_key): + s3_client = get_s3_client() + try: + response = s3_client.get_object(Bucket=bucket_name, Key=file_key) file_stream = response['Body'].read() - image_array = np.load(io.BytesIO(file_stream)) - print(f"Görüntü başarıyla yüklendi! Boyutları: {image_array.shape}") return image_array - except Exception as e: - print(f"Hata oluştu: {e}") - return None + print(f"Hata oluştu ({file_key}): {e}") + return None \ No newline at end of file From 17ae30039a3a007d4e847dbbd89e0c1b4ef421af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bet=C3=BCl=20Tuba=20G=C3=BCm=C3=BC=C5=9F?= Date: Thu, 23 Jul 2026 14:42:02 +0300 Subject: [PATCH 09/21] feat: added normalizition test. --- scripts/test_normalization.py | 42 +++++++++++++++++++++++++++++++++++ services/ml/data/dataset.py | 4 ++-- services/ml/minio_loader.py | 4 ---- 3 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 scripts/test_normalization.py diff --git a/scripts/test_normalization.py b/scripts/test_normalization.py new file mode 100644 index 0000000..8996f53 --- /dev/null +++ b/scripts/test_normalization.py @@ -0,0 +1,42 @@ +import numpy as np +from services.ml.minio_loader import list_files_in_minio, load_npy_from_minio + +def test_normalization(bucket_name="blackhole", prefix_path="datasets/training-512/v1/clean/", num_samples=5): + print(f" '{bucket_name}' bucket'ındaki '{prefix_path}' dizini kontrol ediliyor...\n") + + files = list_files_in_minio(bucket_name, prefix_path) + + if not files: + print(" Dosya bulunamadı! Lütfen MinIO bağlantınızı, bucket adını ve prefix_path'i kontrol edin.") + return + + print(f" Toplam {len(files)} dosya bulundu. İlk {num_samples} dosya test ediliyor...\n") + + for i, file_key in enumerate(files[:num_samples]): + data = load_npy_from_minio(bucket_name, file_key) + + if data is not None: + min_val = np.min(data) + max_val = np.max(data) + + is_normalized = (min_val >= 0.0) and (max_val <= 1.0) + + print(f"📄 Dosya {i+1}: {file_key}") + print(f" -> Veri Boyutu: {data.shape}") + print(f" -> Min Değer: {min_val:.6f}, Max Değer: {max_val:.6f}") + + if is_normalized: + print(" SONUÇ: Başarılı! Dosya 0-1 aralığında normalize edilmiş.") + else: + print(" SONUÇ: Hatalı! Dosya 0-1 aralığında DEĞİL.") + else: + print(f" Dosya {i+1}: {file_key} yüklenirken hata oluştu.") + + print("-" * 60) + +if __name__ == "__main__": + test_normalization(bucket_name="blackhole", prefix_path="datasets/training-512/v1/clean/", num_samples=3) + + print("\n" + "="*60 + "\n") + + test_normalization(bucket_name="blackhole", prefix_path="datasets/training-512/v1/degraded/", num_samples=3) \ No newline at end of file diff --git a/services/ml/data/dataset.py b/services/ml/data/dataset.py index c6b9a3e..d7c3187 100644 --- a/services/ml/data/dataset.py +++ b/services/ml/data/dataset.py @@ -3,10 +3,10 @@ from torch.utils.data import Dataset from pathlib import Path -from services.ml.data.minio_loader import load_npy_from_minio, list_files_in_minio +from services.ml.minio_loader import load_npy_from_minio, list_files_in_minio class BlackHoleDataset(Dataset): - def __init__(self, root_dir, use_minio=False, bucket_name="karadelikler", minio_prefix="datasets/training-512/v1"): + def __init__(self, root_dir, use_minio=False, bucket_name="blackhole", minio_prefix="datasets/training-512/v1"): self.root_dir = Path(root_dir) self.use_minio = use_minio self.bucket_name = bucket_name diff --git a/services/ml/minio_loader.py b/services/ml/minio_loader.py index 56742a1..b58837b 100644 --- a/services/ml/minio_loader.py +++ b/services/ml/minio_loader.py @@ -15,10 +15,6 @@ def get_s3_client(): ) def list_files_in_minio(bucket_name, prefix_path): - """ - Sunucudaki belirli bir dosya yolu altındaki .npy dosyalarını bulur. - Örnek prefix_path: "data/training/clean/" - """ s3_client = get_s3_client() file_keys = [] From db4880f13d09b349fdd8e85666f2b20daa9f9e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bet=C3=BCl=20Tuba=20G=C3=BCm=C3=BC=C5=9F?= Date: Thu, 23 Jul 2026 16:51:29 +0300 Subject: [PATCH 10/21] Normalization control was achieved. --- scripts/test_normalization.py | 42 ------------------------------- scripts/test_normalization_run.py | 20 +++++++++++++++ 2 files changed, 20 insertions(+), 42 deletions(-) delete mode 100644 scripts/test_normalization.py create mode 100644 scripts/test_normalization_run.py diff --git a/scripts/test_normalization.py b/scripts/test_normalization.py deleted file mode 100644 index 8996f53..0000000 --- a/scripts/test_normalization.py +++ /dev/null @@ -1,42 +0,0 @@ -import numpy as np -from services.ml.minio_loader import list_files_in_minio, load_npy_from_minio - -def test_normalization(bucket_name="blackhole", prefix_path="datasets/training-512/v1/clean/", num_samples=5): - print(f" '{bucket_name}' bucket'ındaki '{prefix_path}' dizini kontrol ediliyor...\n") - - files = list_files_in_minio(bucket_name, prefix_path) - - if not files: - print(" Dosya bulunamadı! Lütfen MinIO bağlantınızı, bucket adını ve prefix_path'i kontrol edin.") - return - - print(f" Toplam {len(files)} dosya bulundu. İlk {num_samples} dosya test ediliyor...\n") - - for i, file_key in enumerate(files[:num_samples]): - data = load_npy_from_minio(bucket_name, file_key) - - if data is not None: - min_val = np.min(data) - max_val = np.max(data) - - is_normalized = (min_val >= 0.0) and (max_val <= 1.0) - - print(f"📄 Dosya {i+1}: {file_key}") - print(f" -> Veri Boyutu: {data.shape}") - print(f" -> Min Değer: {min_val:.6f}, Max Değer: {max_val:.6f}") - - if is_normalized: - print(" SONUÇ: Başarılı! Dosya 0-1 aralığında normalize edilmiş.") - else: - print(" SONUÇ: Hatalı! Dosya 0-1 aralığında DEĞİL.") - else: - print(f" Dosya {i+1}: {file_key} yüklenirken hata oluştu.") - - print("-" * 60) - -if __name__ == "__main__": - test_normalization(bucket_name="blackhole", prefix_path="datasets/training-512/v1/clean/", num_samples=3) - - print("\n" + "="*60 + "\n") - - test_normalization(bucket_name="blackhole", prefix_path="datasets/training-512/v1/degraded/", num_samples=3) \ No newline at end of file diff --git a/scripts/test_normalization_run.py b/scripts/test_normalization_run.py new file mode 100644 index 0000000..ef77ee6 --- /dev/null +++ b/scripts/test_normalization_run.py @@ -0,0 +1,20 @@ +from services.ml.minio_loader import list_files_in_minio, load_npy_from_minio +import numpy as np + +BUCKET = "datasets" +PREFIXES = ["training-512/v1/clean/", "training-512/v1/degraded/"] + +for prefix in PREFIXES: + print(f"\n=== {BUCKET}/{prefix} ===") + files = list_files_in_minio(BUCKET, prefix) + print(f"Toplam dosya: {len(files)}") + for i, key in enumerate(files[:3]): + data = load_npy_from_minio(BUCKET, key) + if data is None: + print(f" {key} -> YUKLENEMEDI") + continue + mn = float(np.min(data)) + mx = float(np.max(data)) + ok = (mn >= 0.0) and (mx <= 1.0) + status = "OK 0-1" if ok else "HATA" + print(f" {key} shape={data.shape} min={mn:.6f} max={mx:.6f} -> {status}") From 07b691a6e312df015fb6724ee3778c52ceaa860f Mon Sep 17 00:00:00 2001 From: Eda Tosun Date: Fri, 24 Jul 2026 12:54:04 +0300 Subject: [PATCH 11/21] fix: revised bucket_name as 'datasets' --- services/ml/data/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ml/data/dataset.py b/services/ml/data/dataset.py index d7c3187..558f123 100644 --- a/services/ml/data/dataset.py +++ b/services/ml/data/dataset.py @@ -6,7 +6,7 @@ from services.ml.minio_loader import load_npy_from_minio, list_files_in_minio class BlackHoleDataset(Dataset): - def __init__(self, root_dir, use_minio=False, bucket_name="blackhole", minio_prefix="datasets/training-512/v1"): + def __init__(self, root_dir, use_minio=False, bucket_name="datasets", minio_prefix="datasets/training-512/v1"): self.root_dir = Path(root_dir) self.use_minio = use_minio self.bucket_name = bucket_name From b902852454cf03302460d17d62376f990cb4fe4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bet=C3=BCl=20Tuba=20G=C3=BCm=C3=BC=C5=9F?= Date: Fri, 24 Jul 2026 16:20:09 +0300 Subject: [PATCH 12/21] The U-Net core training infrastructure has been completed. --- pyproject.toml | 2 + scripts/eval_baseline.py | 264 ++++++++++++++++++++++ services/ml/checkpoints/checkpoint.py | 33 +++ services/ml/conf/config.yaml | 34 +++ services/ml/conf/data/default.yaml | 19 ++ services/ml/conf/loss/default.yaml | 13 ++ services/ml/conf/model/unet.yaml | 14 ++ services/ml/conf/training/default.yaml | 32 +++ services/ml/data/dataloader.py | 66 +++++- services/ml/data/dataset.py | 77 ++++++- services/ml/evaluation/benchmark.py | 20 +- services/ml/evaluation/metrics.py | 299 ++++++++++++++++++++++++- services/ml/losses/loss.py | 35 ++- services/ml/tests/__init__.py | 0 services/ml/tests/conftest.py | 37 +++ services/ml/tests/test_checkpoint.py | 110 +++++++++ services/ml/tests/test_dataloader.py | 125 +++++++++++ services/ml/tests/test_dataset.py | 94 ++++++++ services/ml/tests/test_loss.py | 76 +++++++ services/ml/tests/test_metrics.py | 175 +++++++++++++++ services/ml/training/train.py | 267 +++++++++++++++------- 21 files changed, 1691 insertions(+), 101 deletions(-) create mode 100644 scripts/eval_baseline.py create mode 100644 services/ml/conf/config.yaml create mode 100644 services/ml/conf/data/default.yaml create mode 100644 services/ml/conf/loss/default.yaml create mode 100644 services/ml/conf/model/unet.yaml create mode 100644 services/ml/conf/training/default.yaml create mode 100644 services/ml/tests/__init__.py create mode 100644 services/ml/tests/conftest.py create mode 100644 services/ml/tests/test_checkpoint.py create mode 100644 services/ml/tests/test_dataloader.py create mode 100644 services/ml/tests/test_dataset.py create mode 100644 services/ml/tests/test_loss.py create mode 100644 services/ml/tests/test_metrics.py diff --git a/pyproject.toml b/pyproject.toml index 0319ce8..9a8102d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,8 @@ ml = [ "lpips>=0.1.4", "mlflow>=2.21.0", "optuna>=4.3.0", + "hydra-core>=1.3.2", + "omegaconf>=2.3.0", ] serving = [ "torch>=2.6.0", diff --git a/scripts/eval_baseline.py b/scripts/eval_baseline.py new file mode 100644 index 0000000..ee80154 --- /dev/null +++ b/scripts/eval_baseline.py @@ -0,0 +1,264 @@ +""" +DeepHorizon — Baseline Evaluation Script +========================================= +Bicubic upsample baseline + tüm metrikler (PSNR, SSIM, LPIPS, FID). + +README'deki baseline rakamlarını üretmek için kullanılır: +- PSNR ~18 dB, SSIM ~0.35, LPIPS ~0.55, FID ~180 (medium split, 2500 pairs) + +Kullanım: + python scripts/eval_baseline.py --split medium --num-samples 2500 \\ + --output-json baseline_medium.json + +DRY: Bu script `services.ml.evaluation.metrics` modülünü kullanır — metrik +hesaplama mantığı burada yeniden yazılmaz. +""" + +import argparse +import json +import sys +import time +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F + +# Proje kökünü path'e ekle (script doğrudan çalıştırılabilsin) +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from services.ml.data.dataset import BlackHoleDataset +from services.ml.evaluation.metrics import ( + compute_fid, + compute_lpips, + compute_metrics, + compute_psnr, + compute_ssim, +) + + +# README'deki baseline rakamları (karşılaştırma için) +EXPECTED_BASELINE = { + "psnr": 18.0, + "ssim": 0.35, + "lpips": 0.55, + "fid": 180.0, +} + +# Toleranslar (±) +TOLERANCE = { + "psnr": 0.5, + "ssim": 0.05, + "lpips": 0.05, + "fid": 20.0, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Bicubic upsample baseline + PSNR/SSIM/LPIPS/FID evaluation", + ) + parser.add_argument( + "--root-dir", + type=str, + default="data/training", + help="Yerel veri kök dizini (default: data/training)", + ) + parser.add_argument( + "--split", + type=str, + choices=["light", "medium", "heavy", "extreme"], + default="medium", + help="Degradation split (default: medium)", + ) + parser.add_argument( + "--output-json", + type=str, + default=None, + help="Sonuçları JSON olarak kaydet (örn: baseline_medium.json)", + ) + parser.add_argument( + "--num-samples", + type=int, + default=None, + help="Değerlendirilecek örnek sayısı (None = tüm split)", + ) + parser.add_argument( + "--batch-size", + type=int, + default=16, + help="Batch boyutu (default: 16)", + ) + parser.add_argument( + "--device", + type=str, + choices=["cpu", "cuda"], + default="cuda" if torch.cuda.is_available() else "cpu", + help="Cihaz (default: cuda varsa cuda)", + ) + parser.add_argument( + "--seed", + type=int, + default=42, + help="Random seed (default: 42)", + ) + return parser.parse_args() + + +def bicubic_upsample(degraded: torch.Tensor, scale_factor: int = 2) -> torch.Tensor: + """Bicubic upsample ile degraded görüntüyü hedef boyuta getir. + + Args: + degraded: (B, C, H, W) tensor. + scale_factor: Upsample çarpanı (default: 2). + + Returns: + (B, C, H*scale_factor, W*scale_factor) tensor. + """ + return F.interpolate(degraded, scale_factor=scale_factor, mode="bicubic", align_corners=False) + + +def evaluate_baseline(args: argparse.Namespace) -> dict: + """Ana değerlendirme döngüsü. + + Args: + args: argparse namespace. + + Returns: + Dict with keys: psnr, ssim, lpips, fid, num_samples, elapsed_sec. + """ + torch.manual_seed(args.seed) + np.random.seed(args.seed) + + device = torch.device(args.device) + print(f"[eval_baseline] Device: {device}") + print(f"[eval_baseline] Split: {args.split}") + print(f"[eval_baseline] Root dir: {args.root_dir}") + + # Dataset — augmentation KAPALI, sadece split + dataset = BlackHoleDataset( + root_dir=args.root_dir, + use_minio=False, + augment=False, + split=args.split, + ) + + if len(dataset) == 0: + raise FileNotFoundError( + f"Dataset bos: {args.root_dir}/{args.split}/clean/*.npy bulunamadi. " + f"Once `python scripts/generate_training_data.py` calistirin." + ) + + num_samples = min(args.num_samples, len(dataset)) if args.num_samples else len(dataset) + print(f"[eval_baseline] Toplam ornek: {len(dataset)}, degerlendirilecek: {num_samples}") + + # Accumulators + psnr_sum = 0.0 + ssim_sum = 0.0 + lpips_sum = 0.0 + count = 0 + + # FID için tüm görüntüleri topla (InceptionV3 pool_features) + real_features_list = [] + fake_features_list = [] + + start = time.time() + + for idx in range(num_samples): + degraded, clean = dataset[idx] + # (1, 1, H, W) — batch dimension ekle + degraded = degraded.unsqueeze(0).to(device) + clean = clean.unsqueeze(0).to(device) + + # Bicubic upsample + upsampled = bicubic_upsample(degraded, scale_factor=2) + + # Upsampled ile clean aynı boyutta olmalı + if upsampled.shape != clean.shape: + # Boyut uyumsuzsa clean'i upsampled boyutuna getir + clean = F.interpolate(clean, size=upsampled.shape[-2:], mode="bilinear", align_corners=False) + + # Metrikler + m = compute_metrics( + upsampled, clean, + data_range=1.0, + include_lpips=True, + include_physics=False, + ) + psnr_sum += m["psnr"] + ssim_sum += m["ssim"] + lpips_sum += m["lpips"] + + # FID için Inception features topla + real_features_list.append(clean) + fake_features_list.append(upsampled) + + count += 1 + if (idx + 1) % 100 == 0: + elapsed = time.time() - start + print(f" [{idx + 1}/{num_samples}] elapsed: {elapsed:.1f}s") + + # Ortalamalar + psnr_avg = psnr_sum / count + ssim_avg = ssim_sum / count + lpips_avg = lpips_sum / count + + # FID — tüm görüntüleri birleştir + print("[eval_baseline] FID hesaplaniyor...") + real_all = torch.cat(real_features_list, dim=0) + fake_all = torch.cat(fake_features_list, dim=0) + fid_value = compute_fid(real_all, fake_all) + + elapsed = time.time() - start + + results = { + "split": args.split, + "num_samples": count, + "psnr": psnr_avg, + "ssim": ssim_avg, + "lpips": lpips_avg, + "fid": fid_value, + "elapsed_sec": elapsed, + "device": str(device), + } + + return results + + +def print_table(results: dict) -> None: + """Konsol tablosu yazdir + README baseline ile karsilastir.""" + print("\n" + "=" * 70) + print(f"BICUBIC BASELINE — {results['split'].upper()} SPLIT ({results['num_samples']} samples)") + print("=" * 70) + print(f"{'Metric':<10} {'Measured':>12} {'README':>12} {'Delta':>10} {'Status':>10}") + print("-" * 70) + + for metric in ["psnr", "ssim", "lpips", "fid"]: + measured = results[metric] + expected = EXPECTED_BASELINE[metric] + tol = TOLERANCE[metric] + delta = abs(measured - expected) + status = "OK" if delta <= tol else "WARN" + print(f"{metric.upper():<10} {measured:>12.4f} {expected:>12.4f} {delta:>10.4f} {status:>10}") + + print("-" * 70) + print(f"Elapsed: {results['elapsed_sec']:.1f}s | Device: {results['device']}") + print("=" * 70) + + +def main() -> None: + args = parse_args() + results = evaluate_baseline(args) + print_table(results) + + if args.output_json: + output_path = Path(args.output_json) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(results, f, indent=2) + print(f"\n[eval_baseline] Sonuclar kaydedildi: {output_path}") + + +if __name__ == "__main__": + main() diff --git a/services/ml/checkpoints/checkpoint.py b/services/ml/checkpoints/checkpoint.py index 218eaa7..c98465e 100644 --- a/services/ml/checkpoints/checkpoint.py +++ b/services/ml/checkpoints/checkpoint.py @@ -8,9 +8,22 @@ def save_checkpoint( epoch, train_loss, val_loss, + psnr: float, + ssim: float, checkpoint_path, ): + """Save model + optimizer + metrics to a checkpoint file. + Args: + model: PyTorch model (state_dict saved). + optimizer: PyTorch optimizer (state_dict saved). + epoch: Current epoch number (1-indexed). + train_loss: Training loss for this epoch. + val_loss: Validation loss for this epoch. + psnr: Validation PSNR for this epoch. + ssim: Validation SSIM for this epoch. + checkpoint_path: Where to write the .pt file. + """ checkpoint_path = Path(checkpoint_path) checkpoint_path.parent.mkdir(parents=True, exist_ok=True) @@ -21,6 +34,8 @@ def save_checkpoint( "optimizer_state_dict": optimizer.state_dict(), "train_loss": train_loss, "val_loss": val_loss, + "psnr": psnr, + "ssim": ssim, }, checkpoint_path, ) @@ -32,7 +47,21 @@ def load_checkpoint( optimizer=None, map_location="cpu", ): + """Load model + optimizer + metrics from a checkpoint file. + Returns the full checkpoint dict. Missing keys (e.g. older checkpoints + saved before psnr/ssim were added) default to 0.0 for backward compat. + + Args: + checkpoint_path: Path to the .pt file. + model: PyTorch model (state_dict loaded into). + optimizer: PyTorch optimizer (state_dict loaded into, optional). + map_location: Device to map tensors to (default "cpu"). + + Returns: + dict with keys: epoch, model_state_dict, optimizer_state_dict, + train_loss, val_loss, psnr, ssim. + """ checkpoint = torch.load( checkpoint_path, map_location=map_location, @@ -47,4 +76,8 @@ def load_checkpoint( checkpoint["optimizer_state_dict"] ) + # Backward compat: eski checkpoint'lerde psnr/ssim yoksa 0.0 döndür + checkpoint.setdefault("psnr", 0.0) + checkpoint.setdefault("ssim", 0.0) + return checkpoint diff --git a/services/ml/conf/config.yaml b/services/ml/conf/config.yaml new file mode 100644 index 0000000..c194f9d --- /dev/null +++ b/services/ml/conf/config.yaml @@ -0,0 +1,34 @@ +# DeepHorizon — root Hydra config +# Loaded by `python -m services.ml.training.train` +# Override anything from CLI: `python -m services.ml.training.train training.epochs=2` + +defaults: + - model: unet + - training: default + - data: default + - loss: default + - _self_ + +# Paths +paths: + output_dir: checkpoints # where checkpoints, jsonl, sample PNGs land + root_dir: data/training # local fallback when data.use_minio=false + +# Reproducibility +seed: 42 + +# Device: "cuda" | "cpu" | "auto" (auto = cuda if available else cpu) +device: auto + +# MLflow tracking (Faz 1 Adım 2'de kullanılacak; şimdilik sadece config'de) +mlflow: + tracking_uri: "http://mlflow.deephorizon-ml.svc:5000" + experiment_name: "unet-baseline" + run_name: null # null → MLflow auto-generates + log_model: true # best_model.pt'yi artifact olarak logla + +# Hydra runtime config — CLI override'ların çalışması için struct modunu kapat +hydra: + run: + dir: outputs/${now:%Y-%m-%d_%H-%M-%S} + output_subdir: .hydra diff --git a/services/ml/conf/data/default.yaml b/services/ml/conf/data/default.yaml new file mode 100644 index 0000000..484ece6 --- /dev/null +++ b/services/ml/conf/data/default.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Data loading configuration +# MinIO mode (default) → 10K 512×512 pairs from docs/DATA.md bucket layout +# Local mode → root_dir/clean/*.npy + root_dir/degraded/*.npy + +data: + root_dir: data/training + use_minio: true + bucket_name: datasets + minio_prefix: datasets/training-512/v1 + + # DataLoader + num_workers: 4 + pin_memory: true + drop_last: true + + # Augmentation (Faz 1 Adım 8 — train set'e flip + 90° rot + random crop) + augment: true + crop_size: 256 # random crop from 512×512 (augment=true ise) diff --git a/services/ml/conf/loss/default.yaml b/services/ml/conf/loss/default.yaml new file mode 100644 index 0000000..487fc5f --- /dev/null +++ b/services/ml/conf/loss/default.yaml @@ -0,0 +1,13 @@ +# @package _global_ +# Loss function configuration +# Faz 1 Adım 6: mse | l1 | smooth_l1 +# Faz 2'de eklenecek: perceptual, gan, physics, combined + +loss: + name: l1 # mse | l1 | smooth_l1 + # Combined loss ağırlıkları (Faz 2'de kullanılacak) + weights: + pixel: 1.0 + perceptual: 0.0 + adversarial: 0.0 + physics: 0.0 diff --git a/services/ml/conf/model/unet.yaml b/services/ml/conf/model/unet.yaml new file mode 100644 index 0000000..39244ca --- /dev/null +++ b/services/ml/conf/model/unet.yaml @@ -0,0 +1,14 @@ +# @package _global_ +# U-Net model architecture +# Mirrors services/ml/models/unet.py defaults (1→64→128→256→512→1024) + +model: + name: unet + in_channels: 1 + out_channels: 1 + # Channel widths at each encoder level (length = depth) + features: + - 64 + - 128 + - 256 + - 512 diff --git a/services/ml/conf/training/default.yaml b/services/ml/conf/training/default.yaml new file mode 100644 index 0000000..6c5ab96 --- /dev/null +++ b/services/ml/conf/training/default.yaml @@ -0,0 +1,32 @@ +# @package _global_ +# Training loop hyperparameters + +training: + epochs: 10 + batch_size: 16 + learning_rate: 1.0e-4 + val_ratio: 0.2 + + # Optimizer (Adam) + optimizer: + name: adam # adam | adamw | sgd + betas: [0.9, 0.999] + weight_decay: 0.0 + + # LR scheduler (ReduceLROnPlateau) + scheduler: + name: reduce_on_plateau # reduce_on_plateau | cosine | none + factor: 0.5 + patience: 5 + min_lr: 1.0e-6 + + # Mixed precision (Faz 1 Adım 3'te kullanılacak) + amp: false # true → torch.amp.autocast + GradScaler + amp_dtype: bfloat16 # float16 | bfloat16 (L40S BF16 destekliyor) + + # Gradient accumulation (Faz 1 Adım 4'te kullanılacak) + grad_accum_steps: 1 # 1 = no accumulation, 4 = effective batch ×4 + + # Checkpointing + save_every_epoch: true + keep_last_n: 3 # sadece son N epoch checkpoint'ini tut diff --git a/services/ml/data/dataloader.py b/services/ml/data/dataloader.py index 2d80500..d5b6d2b 100644 --- a/services/ml/data/dataloader.py +++ b/services/ml/data/dataloader.py @@ -29,8 +29,22 @@ def create_dataloader( num_workers=4, pin_memory=True, drop_last=True, + use_minio=False, + bucket_name="datasets", + minio_prefix="datasets/training-512/v1", + augment=False, + crop_size=256, + split=None, ): - dataset = BlackHoleDataset(root_dir) + dataset = BlackHoleDataset( + root_dir, + use_minio=use_minio, + bucket_name=bucket_name, + minio_prefix=minio_prefix, + augment=augment, + crop_size=crop_size, + split=split, + ) return _create_loader( dataset, batch_size=batch_size, @@ -50,12 +64,38 @@ def create_train_val_loaders( num_workers=4, pin_memory=True, drop_last=True, + use_minio=False, + bucket_name="datasets", + minio_prefix="datasets/training-512/v1", + augment=False, + crop_size=256, + split=None, ): + """Train + validation DataLoader oluşturur. + + Args: + root_dir: Local kök dizin (use_minio=False ise). + use_minio: True ise MinIO/S3'ten okur. + bucket_name: MinIO bucket adı. + minio_prefix: MinIO prefix. + augment: True ise train set'e augmentation uygulanır (val set'e uygulanmaz). + crop_size: Random crop boyutu (augment=True ise). + split: Degradation split adı (light/medium/heavy/extreme). + """ if not 0 < val_ratio < 1: raise ValueError("val_ratio must be between 0 and 1") - dataset = BlackHoleDataset(root_dir) - dataset_size = len(dataset) + # Train dataset — augmentation açık + train_full = BlackHoleDataset( + root_dir, + use_minio=use_minio, + bucket_name=bucket_name, + minio_prefix=minio_prefix, + augment=augment, + crop_size=crop_size, + split=split, + ) + dataset_size = len(train_full) if dataset_size < 2: raise ValueError("At least 2 samples are required to create train/val splits") @@ -67,10 +107,26 @@ def create_train_val_loaders( raise ValueError("The selected val_ratio results in an empty split") generator = torch.Generator().manual_seed(seed) - train_dataset, val_dataset = random_split(dataset, [train_size, val_size], generator=generator) + train_subset, val_subset = random_split( + train_full, [train_size, val_size], generator=generator + ) + + # Validation dataset — augmentation KAPALI (deterministic ölçüm) + val_full = BlackHoleDataset( + root_dir, + use_minio=use_minio, + bucket_name=bucket_name, + minio_prefix=minio_prefix, + augment=False, + crop_size=crop_size, + split=split, + ) + # Aynı index'leri kullanmak için val_subset'in index'lerini val_full'e uygula + val_indices = val_subset.indices + val_dataset = torch.utils.data.Subset(val_full, val_indices) train_loader = _create_loader( - train_dataset, + train_subset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, diff --git a/services/ml/data/dataset.py b/services/ml/data/dataset.py index 558f123..e73d7be 100644 --- a/services/ml/data/dataset.py +++ b/services/ml/data/dataset.py @@ -5,23 +5,51 @@ from services.ml.minio_loader import load_npy_from_minio, list_files_in_minio + class BlackHoleDataset(Dataset): - def __init__(self, root_dir, use_minio=False, bucket_name="datasets", minio_prefix="datasets/training-512/v1"): + """Black hole image dataset (clean + degraded pairs). + + Args: + root_dir: Local kök dizin (use_minio=False ise kullanılır). + use_minio: True ise MinIO/S3'ten okur, False ise yerel dosya sisteminden. + bucket_name: MinIO bucket adı. + minio_prefix: MinIO prefix (clean/ ve degraded/ altında .npy dosyaları). + augment: True ise random flip + 90° rotation + random crop uygulanır. + crop_size: Random crop boyutu (augment=True ise kullanılır). + split: Degradation split adı (light/medium/heavy/extreme). None ise + root_dir/clean + root_dir/degraded kullanılır; belirtilirse + root_dir/{split}/clean + root_dir/{split}/degraded kullanılır. + """ + + def __init__( + self, + root_dir, + use_minio=False, + bucket_name="datasets", + minio_prefix="datasets/training-512/v1", + augment=False, + crop_size=256, + split=None, + ): self.root_dir = Path(root_dir) self.use_minio = use_minio self.bucket_name = bucket_name self.minio_prefix = minio_prefix + self.augment = augment + self.crop_size = crop_size + self.split = split if not self.use_minio: + base = self.root_dir / split if split else self.root_dir self.clean_files = sorted( - (self.root_dir / "clean").glob("*.npy") + (base / "clean").glob("*.npy") ) self.degraded_files = sorted( - (self.root_dir / "degraded").glob("*.npy") + (base / "degraded").glob("*.npy") ) else: - clean_path = f"{self.minio_prefix}/clean/" - degraded_path = f"{self.minio_prefix}/degraded/" + clean_path = f"{self.minio_prefix}/{split}/clean/" if split else f"{self.minio_prefix}/clean/" + degraded_path = f"{self.minio_prefix}/{split}/degraded/" if split else f"{self.minio_prefix}/degraded/" self.clean_files = list_files_in_minio(self.bucket_name, clean_path) self.degraded_files = list_files_in_minio(self.bucket_name, degraded_path) @@ -43,4 +71,43 @@ def __getitem__(self, index): clean = clean.unsqueeze(0) degraded = degraded.unsqueeze(0) + if self.augment: + clean, degraded = self._augment(clean, degraded) + + return degraded, clean + + def _augment(self, degraded, clean): + """Deterministic augmentation: random flip + 90° rotation + random crop. + + Her çağrıda yeni bir Generator oluşturulur — epoch başına farklı ama + aynı epoch içinde aynı index için aynı sonucu verir (DataLoader shuffle + ile birlikte her epoch'ta farklı augmentasyon görülür). + """ + gen = torch.Generator() + + # Random horizontal flip + if torch.rand(1, generator=gen).item() < 0.5: + degraded = torch.flip(degraded, dims=[-1]) + clean = torch.flip(clean, dims=[-1]) + + # Random vertical flip + if torch.rand(1, generator=gen).item() < 0.5: + degraded = torch.flip(degraded, dims=[-2]) + clean = torch.flip(clean, dims=[-2]) + + # Random 90° rotation (k ∈ {0, 1, 2, 3}) + k = int(torch.randint(0, 4, (1,), generator=gen).item()) + if k > 0: + degraded = torch.rot90(degraded, k=k, dims=[-2, -1]) + clean = torch.rot90(clean, k=k, dims=[-2, -1]) + + # Random crop (crop_size × crop_size) — padding ile sınır dışı korunur + _, _, h, w = degraded.shape + crop = self.crop_size + if h >= crop and w >= crop: + top = int(torch.randint(0, h - crop + 1, (1,), generator=gen).item()) + left = int(torch.randint(0, w - crop + 1, (1,), generator=gen).item()) + degraded = degraded[..., top:top + crop, left:left + crop] + clean = clean[..., top:top + crop, left:left + crop] + return degraded, clean \ No newline at end of file diff --git a/services/ml/evaluation/benchmark.py b/services/ml/evaluation/benchmark.py index a67f418..d7b5314 100644 --- a/services/ml/evaluation/benchmark.py +++ b/services/ml/evaluation/benchmark.py @@ -17,6 +17,10 @@ class ValidationSummary: val_loss: float psnr: float ssim: float + lpips: float = 0.0 + flux_error: float = 0.0 + ring_diameter_error: float = 0.0 + asymmetry_error: float = 0.0 def _prepare_output_dir(output_dir: Path | str) -> Path: @@ -71,6 +75,10 @@ def evaluate_validation_loader( running_val_loss = 0.0 running_psnr = 0.0 running_ssim = 0.0 + running_lpips = 0.0 + running_flux_error = 0.0 + running_ring_diameter_error = 0.0 + running_asymmetry_error = 0.0 batch_count = 0 sample_batch: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None @@ -81,11 +89,17 @@ def evaluate_validation_loader( prediction = model(degraded) loss = criterion(prediction, clean) - metrics = compute_metrics(prediction, clean) + metrics = compute_metrics( + prediction, clean, include_lpips=True, include_physics=True + ) running_val_loss += float(loss.item()) running_psnr += metrics["psnr"] running_ssim += metrics["ssim"] + running_lpips += metrics["lpips"] + running_flux_error += metrics["flux_error"] + running_ring_diameter_error += metrics["ring_diameter_error"] + running_asymmetry_error += metrics["asymmetry_error"] batch_count += 1 if sample_batch is None: @@ -104,6 +118,10 @@ def evaluate_validation_loader( val_loss=running_val_loss / batch_count, psnr=running_psnr / batch_count, ssim=running_ssim / batch_count, + lpips=running_lpips / batch_count, + flux_error=running_flux_error / batch_count, + ring_diameter_error=running_ring_diameter_error / batch_count, + asymmetry_error=running_asymmetry_error / batch_count, ) return summary, sample_batch diff --git a/services/ml/evaluation/metrics.py b/services/ml/evaluation/metrics.py index b3e0415..7ef3bea 100644 --- a/services/ml/evaluation/metrics.py +++ b/services/ml/evaluation/metrics.py @@ -1,9 +1,12 @@ from __future__ import annotations import math +from typing import Literal +import numpy as np import torch import torch.nn.functional as F +from scipy import linalg, signal def _ensure_4d(tensor: torch.Tensor) -> torch.Tensor: @@ -97,13 +100,307 @@ def compute_ssim( return float(ssim_map.mean().item()) +# --------------------------------------------------------------------------- +# LPIPS — Learned Perceptual Image Patch Similarity +# Lazy-loaded global cache; lpips paketi ml.txt'de zaten var. +# --------------------------------------------------------------------------- +_LPIPS_CACHE: dict[str, torch.nn.Module] = {} + + +def _get_lpips_model(net: str = "alex", device: torch.device | None = None) -> torch.nn.Module: + """Lazy-load and cache the LPIPS model.""" + import lpips # type: ignore[import-untyped] + + key = f"{net}:{device}" + if key not in _LPIPS_CACHE: + model = lpips.LPIPS(net=net, verbose=False) + if device is not None: + model = model.to(device) + model.eval() + _LPIPS_CACHE[key] = model + return _LPIPS_CACHE[key] + + +def compute_lpips( + prediction: torch.Tensor, + target: torch.Tensor, + net: Literal["alex", "vgg", "squeeze"] = "alex", +) -> float: + """Learned Perceptual Image Patch Similarity (lower = more similar). + + Args: + prediction: Predicted image tensor (any shape, will be 4D). + target: Target image tensor (same shape as prediction). + net: Backbone network — "alex" (fast), "vgg" (accurate), "squeeze". + + Returns: + LPIPS distance as a Python float. ~0.0 for identical images. + """ + prediction, target = _validate_pair(prediction, target) + device = prediction.device + model = _get_lpips_model(net=net, device=device) + + # LPIPS expects input in [-1, 1] range + prediction_norm = prediction * 2.0 - 1.0 + target_norm = target * 2.0 - 1.0 + + with torch.no_grad(): + distance = model(prediction_norm, target_norm) + + return float(distance.mean().item()) + + +# --------------------------------------------------------------------------- +# FID — Fréchet Inception Distance +# InceptionV3 pool_features (2048-dim) üzerinden iki dağılım arasındaki mesafe. +# --------------------------------------------------------------------------- +_INCEPTION_CACHE: dict[str, torch.nn.Module] = {} + + +def _get_inception_model(device: torch.device | None = None) -> torch.nn.Module: + """Lazy-load and cache the InceptionV3 feature extractor.""" + from torchvision.models import Inception_V3_Weights, inception_v3 # type: ignore[import-untyped] + + key = str(device) + if key not in _INCEPTION_CACHE: + model = inception_v3(weights=Inception_V3_Weights.DEFAULT, aux_logits=True) + # Remove final FC; use the 2048-dim feature before it + model.fc = torch.nn.Identity() + model.eval() + if device is not None: + model = model.to(device) + _INCEPTION_CACHE[key] = model + return _INCEPTION_CACHE[key] + + +def _inception_features( + images: torch.Tensor, + model: torch.nn.Module, + batch_size: int = 32, +) -> torch.Tensor: + """Extract 2048-dim features from InceptionV3 in batches.""" + features: list[torch.Tensor] = [] + with torch.no_grad(): + for start in range(0, images.shape[0], batch_size): + batch = images[start : start + batch_size] + # InceptionV3 expects 3-channel, 299x299, normalized + if batch.shape[1] == 1: + batch = batch.repeat(1, 3, 1, 1) + batch = F.interpolate(batch, size=(299, 299), mode="bilinear", align_corners=False) + # ImageNet normalization + mean = torch.tensor([0.485, 0.456, 0.406], device=batch.device).view(1, 3, 1, 1) + std = torch.tensor([0.229, 0.224, 0.225], device=batch.device).view(1, 3, 1, 1) + batch = (batch - mean) / std + feat = model(batch) + features.append(feat.cpu()) + return torch.cat(features, dim=0).numpy() + + +def compute_fid( + real_images: torch.Tensor, + fake_images: torch.Tensor, + batch_size: int = 32, +) -> float: + """Fréchet Inception Distance between two image sets (lower = more similar). + + Args: + real_images: Real images tensor (N, C, H, W) in [0, 1]. + fake_images: Generated images tensor (M, C, H, W) in [0, 1]. + batch_size: InceptionV3 batch size for feature extraction. + + Returns: + FID score as a Python float. + """ + real_images = _ensure_4d(real_images).to(dtype=torch.float32) + fake_images = _ensure_4d(fake_images).to(dtype=torch.float32) + device = real_images.device + model = _get_inception_model(device=device) + + real_features = _inception_features(real_images, model, batch_size=batch_size) + fake_features = _inception_features(fake_images, model, batch_size=batch_size) + + mu_real = real_features.mean(axis=0) + mu_fake = fake_features.mean(axis=0) + sigma_real = np.cov(real_features, rowvar=False) + sigma_fake = np.cov(fake_features, rowvar=False) + + diff = mu_real - mu_fake + # sqrt(sigma_real @ sigma_fake) — bazen kompleks sonuç verir, real part al + covmean = linalg.sqrtm(sigma_real @ sigma_fake) + if np.iscomplexobj(covmean): + covmean = covmean.real + + fid = float(diff @ diff + np.trace(sigma_real) + np.trace(sigma_fake) - 2.0 * np.trace(covmean)) + return fid + + +# --------------------------------------------------------------------------- +# Physics-informed metrics — README'deki formel tanıma uygun +# --------------------------------------------------------------------------- +def compute_flux_conservation( + prediction: torch.Tensor, + target: torch.Tensor, +) -> float: + """Flux conservation error: |sum(pred) - sum(target)| / sum(target). + + Toplam akı (flux) korunmalıdır; ideal değer 0.0. + """ + prediction, target = _validate_pair(prediction, target) + pred_sum = float(prediction.sum().item()) + target_sum = float(target.sum().item()) + + if abs(target_sum) < 1e-12: + return float("inf") if abs(pred_sum) > 1e-12 else 0.0 + + return abs(pred_sum - target_sum) / abs(target_sum) + + +def compute_ring_diameter( + image: torch.Tensor, + n_bins: int = 360, +) -> float: + """Ring diameter (pixels) via radial brightness profile peak detection. + + Görüntünün merkezinden dışa doğru radyal parlaklık profili çıkarılır, + en parlak halka pikselinin yarıçapı döndürülür. + + Args: + image: 2D, 3D veya 4D tensor (son iki boyut H, W). + n_bins: Açısal bin sayısı (varsayılan 360 = 1 derece). + + Returns: + Halka çapı (piksel cinsinden, çap = 2 * yarıçap). + """ + image = _ensure_4d(image).to(dtype=torch.float32) + # İlk batch ve kanalı al + img = image[0, 0] # (H, W) + + height, width = img.shape + center_y, center_x = height / 2.0, width / 2.0 + max_radius = min(center_y, center_x) + + # Radyal profil: her yarıçap için ortalama parlaklık + radii = torch.linspace(0, max_radius, n_bins + 1, device=img.device) + profile = torch.zeros(n_bins, device=img.device) + + y_coords = torch.arange(height, device=img.device).float() - center_y + x_coords = torch.arange(width, device=img.device).float() - center_x + yy, xx = torch.meshgrid(y_coords, x_coords, indexing="ij") + rr = torch.sqrt(yy * yy + xx * xx) + + for i in range(n_bins): + r_inner = radii[i].item() + r_outer = radii[i + 1].item() + mask = (rr >= r_inner) & (rr < r_outer) + if mask.any(): + profile[i] = img[mask].mean() + + profile_np = profile.cpu().numpy() + # En parlak piki bul (merkezden uzak, yarıçap > 5 piksel) + peaks, _ = signal.find_peaks(profile_np, distance=5) + if len(peaks) == 0: + return 0.0 + + # En yüksek piki seç + best_peak = peaks[profile_np[peaks].argmax()] + radius = float(radii[best_peak].item()) + return 2.0 * radius # çap = 2 * yarıçap + + +def compute_asymmetry_ratio( + image: torch.Tensor, + ring_radius_px: float, + n_bins: int = 360, +) -> float: + """Brightness asymmetry ratio along the ring: max / min. + + Halka boyunca parlaklık asimetrisi; ideal değer 1.0 (simetrik). + ring_radius_px verilmezse otomatik tespit edilir. + """ + image = _ensure_4d(image).to(dtype=torch.float32) + img = image[0, 0] # (H, W) + + height, width = img.shape + center_y, center_x = height / 2.0, width / 2.0 + + if ring_radius_px <= 0: + ring_radius_px = compute_ring_diameter(image, n_bins=n_bins) / 2.0 + + # Halka üzerindeki pikselleri topla + y_coords = torch.arange(height, device=img.device).float() - center_y + x_coords = torch.arange(width, device=img.device).float() - center_x + yy, xx = torch.meshgrid(y_coords, x_coords, indexing="ij") + rr = torch.sqrt(yy * yy + xx * xx) + + # Halka kalınlığı: ±2 piksel + ring_mask = (rr >= ring_radius_px - 2.0) & (rr <= ring_radius_px + 2.0) + if not ring_mask.any(): + return 1.0 + + ring_values = img[ring_mask] + min_val = float(ring_values.min().item()) + max_val = float(ring_values.max().item()) + + if min_val < 1e-12: + return float("inf") if max_val > 1e-12 else 1.0 + + return max_val / min_val + + +def compute_physics_metrics( + prediction: torch.Tensor, + target: torch.Tensor, +) -> dict[str, float]: + """Tüm physics-informed metrikleri tek seferde hesapla. + + Returns: + Dict with keys: flux_error, ring_diameter_error, asymmetry_error. + """ + prediction, target = _validate_pair(prediction, target) + + flux_error = compute_flux_conservation(prediction, target) + + pred_diameter = compute_ring_diameter(prediction) + target_diameter = compute_ring_diameter(target) + ring_diameter_error = abs(pred_diameter - target_diameter) + + pred_asym = compute_asymmetry_ratio(prediction, ring_radius_px=pred_diameter / 2.0) + target_asym = compute_asymmetry_ratio(target, ring_radius_px=target_diameter / 2.0) + asymmetry_error = abs(pred_asym - target_asym) + + return { + "flux_error": flux_error, + "ring_diameter_error": ring_diameter_error, + "asymmetry_error": asymmetry_error, + } + + def compute_metrics( prediction: torch.Tensor, target: torch.Tensor, data_range: float = 1.0, + include_lpips: bool = False, + include_physics: bool = False, ) -> dict[str, float]: + """Tüm metrikleri tek seferde hesapla. + + Args: + prediction: Predicted image tensor. + target: Target image tensor. + data_range: PSNR/SSIM için veri aralığı (varsayılan 1.0). + include_lpips: True ise "lpips" anahtarı eklenir (yavaş, lazy load). + include_physics: True ise flux/ring/asymmetry anahtarları eklenir. + + Returns: + Dict with keys: psnr, ssim, [lpips], [flux_error, ring_diameter_error, asymmetry_error]. + """ prediction, target = _validate_pair(prediction, target) - return { + metrics: dict[str, float] = { "psnr": compute_psnr(prediction, target, data_range=data_range), "ssim": compute_ssim(prediction, target, data_range=data_range), } + if include_lpips: + metrics["lpips"] = compute_lpips(prediction, target) + if include_physics: + metrics.update(compute_physics_metrics(prediction, target)) + return metrics diff --git a/services/ml/losses/loss.py b/services/ml/losses/loss.py index bda5585..d20b438 100644 --- a/services/ml/losses/loss.py +++ b/services/ml/losses/loss.py @@ -1,6 +1,37 @@ import torch.nn as nn +# Faz 1 Adım 6: mse | l1 | smooth_l1 +# Faz 2'de eklenecek: perceptual, gan, physics, combined +_SUPPORTED_LOSSES = frozenset({"mse", "l1", "smooth_l1"}) -def get_loss(): - return nn.MSELoss() +def get_loss(name: str = "mse", **kwargs) -> nn.Module: + """Factory for loss functions. + + Args: + name: Loss identifier. One of "mse", "l1", "smooth_l1". + **kwargs: Extra arguments forwarded to the loss constructor + (e.g. beta for SmoothL1Loss, reduction for any of them). + + Returns: + An instantiated `nn.Module` loss. + + Raises: + ValueError: If `name` is not a supported loss identifier. + + Note: + Faz 2'de eklenecek: "perceptual" (VGG), "gan" (adversarial), + "physics" (flux + ring + asymmetry), "combined" (weighted sum). + """ + if name == "mse": + return nn.MSELoss(**kwargs) + if name == "l1": + return nn.L1Loss(**kwargs) + if name == "smooth_l1": + return nn.SmoothL1Loss(**kwargs) + + supported = ", ".join(sorted(_SUPPORTED_LOSSES)) + raise ValueError( + f"Unknown loss: {name!r}. Supported losses: {supported}. " + f"physics/perceptual/gan will be added in Faz 2." + ) diff --git a/services/ml/tests/__init__.py b/services/ml/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/ml/tests/conftest.py b/services/ml/tests/conftest.py new file mode 100644 index 0000000..f5947eb --- /dev/null +++ b/services/ml/tests/conftest.py @@ -0,0 +1,37 @@ +"""Pytest fixtures for services.ml tests.""" +from __future__ import annotations + +from pathlib import Path + +import pytest +import torch + + +@pytest.fixture +def device() -> torch.device: + """Test cihazı — CUDA varsa cuda, yoksa cpu.""" + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +@pytest.fixture +def sample_batch() -> tuple[torch.Tensor, torch.Tensor]: + """Örnek batch — (degraded, clean) çifti, 64x64 float32 [0, 1].""" + torch.manual_seed(42) + degraded = torch.rand(2, 1, 64, 64, dtype=torch.float32) + clean = torch.rand(2, 1, 64, 64, dtype=torch.float32) + return degraded, clean + + +@pytest.fixture +def sample_image() -> torch.Tensor: + """Tek görüntü — (1, 1, 64, 64) float32 [0, 1].""" + torch.manual_seed(42) + return torch.rand(1, 1, 64, 64, dtype=torch.float32) + + +@pytest.fixture +def tmp_output_dir(tmp_path: Path) -> Path: + """Geçici çıktı dizini — checkpoint + artifact testleri için.""" + out = tmp_path / "outputs" + out.mkdir(parents=True, exist_ok=True) + return out diff --git a/services/ml/tests/test_checkpoint.py b/services/ml/tests/test_checkpoint.py new file mode 100644 index 0000000..890b7c8 --- /dev/null +++ b/services/ml/tests/test_checkpoint.py @@ -0,0 +1,110 @@ +"""Unit tests for services.ml.checkpoints.checkpoint.""" +from __future__ import annotations + +from pathlib import Path + +import torch + +from services.ml.checkpoints.checkpoint import load_checkpoint, save_checkpoint +from services.ml.models.unet import UNet + + +def test_checkpoint_roundtrip(tmp_path: Path) -> None: + """save → load → aynı değerler geri gelir.""" + model = UNet() + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + path = tmp_path / "ckpt.pt" + + save_checkpoint(model, optimizer, 1, 0.5, 0.4, 30.0, 0.9, path) + + # Yeni model + optimizer ile load + new_model = UNet() + new_optimizer = torch.optim.Adam(new_model.parameters(), lr=1e-3) + state = load_checkpoint(path, new_model, new_optimizer) + + assert state["epoch"] == 1 + assert state["train_loss"] == 0.5 + assert state["val_loss"] == 0.4 + assert state["psnr"] == 30.0 + assert state["ssim"] == 0.9 + + +def test_checkpoint_creates_parent_dir(tmp_path: Path) -> None: + """save_checkpoint üst dizini otomatik oluşturur.""" + model = UNet() + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + path = tmp_path / "nested" / "dir" / "ckpt.pt" + + save_checkpoint(model, optimizer, 1, 0.5, 0.4, 30.0, 0.9, path) + assert path.exists() + + +def test_checkpoint_load_without_optimizer(tmp_path: Path) -> None: + """optimizer=None → sadece model state_dict yüklenir.""" + model = UNet() + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + path = tmp_path / "ckpt.pt" + + save_checkpoint(model, optimizer, 5, 0.3, 0.2, 28.0, 0.85, path) + + new_model = UNet() + state = load_checkpoint(path, new_model, optimizer=None) + + assert state["epoch"] == 5 + assert state["psnr"] == 28.0 + + +def test_checkpoint_backward_compat_missing_psnr(tmp_path: Path) -> None: + """Eski checkpoint (psnr/ssim yok) → setdefault ile 0.0 döner.""" + model = UNet() + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + path = tmp_path / "old_ckpt.pt" + + # Eski format — sadece temel alanlar + torch.save( + { + "epoch": 3, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "train_loss": 0.5, + "val_loss": 0.4, + }, + path, + ) + + new_model = UNet() + new_optimizer = torch.optim.Adam(new_model.parameters(), lr=1e-3) + state = load_checkpoint(path, new_model, new_optimizer) + + assert state["epoch"] == 3 + assert state["psnr"] == 0.0 # backward compat + assert state["ssim"] == 0.0 # backward compat + + +def test_checkpoint_state_dict_matches(tmp_path: Path) -> None: + """Model state_dict load sonrası aynı ağırlıkları üretir.""" + model = UNet() + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + path = tmp_path / "ckpt.pt" + + # Modeli bir adım eğit (ağırlıklar değişsin) + x = torch.rand(1, 1, 64, 64) + y = model(x) + loss = y.sum() + loss.backward() + optimizer.step() + + save_checkpoint(model, optimizer, 1, 0.5, 0.4, 30.0, 0.9, path) + + # Yeni model + aynı input → aynı çıktı olmalı + new_model = UNet() + new_optimizer = torch.optim.Adam(new_model.parameters(), lr=1e-3) + load_checkpoint(path, new_model, new_optimizer) + + new_model.eval() + model.eval() + with torch.no_grad(): + out_original = model(x) + out_loaded = new_model(x) + + assert torch.allclose(out_original, out_loaded, atol=1e-6) diff --git a/services/ml/tests/test_dataloader.py b/services/ml/tests/test_dataloader.py new file mode 100644 index 0000000..ba8bda2 --- /dev/null +++ b/services/ml/tests/test_dataloader.py @@ -0,0 +1,125 @@ +"""Unit tests for services.ml.data.dataloader.""" +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +import torch + +from services.ml.data.dataloader import create_dataloader, create_train_val_loaders + + +@pytest.fixture +def dataset_10(tmp_path: Path) -> Path: + """10 çift .npy içeren geçici dataset.""" + clean_dir = tmp_path / "clean" + degraded_dir = tmp_path / "degraded" + clean_dir.mkdir(parents=True) + degraded_dir.mkdir(parents=True) + + for i in range(10): + np.save(clean_dir / f"img_{i:03d}.npy", np.random.rand(32, 32).astype(np.float32)) + np.save(degraded_dir / f"img_{i:03d}.npy", np.random.rand(32, 32).astype(np.float32)) + + return tmp_path + + +def test_create_dataloader_returns_dataloader(dataset_10: Path) -> None: + """create_dataloader → DataLoader instance.""" + loader = create_dataloader( + root_dir=dataset_10, + batch_size=2, + num_workers=0, + ) + assert isinstance(loader, torch.utils.data.DataLoader) + + +def test_create_dataloader_batch_shape(dataset_10: Path) -> None: + """Batch shape = (batch_size, 1, H, W).""" + loader = create_dataloader( + root_dir=dataset_10, + batch_size=2, + num_workers=0, + ) + batch = next(iter(loader)) + degraded, clean = batch + assert degraded.shape[0] == 2 + assert clean.shape[0] == 2 + assert degraded.shape[1] == 1 + assert clean.shape[1] == 1 + + +def test_create_train_val_loaders_split_ratio(dataset_10: Path) -> None: + """val_ratio=0.2 → 8 train + 2 val.""" + train_loader, val_loader = create_train_val_loaders( + root_dir=dataset_10, + batch_size=2, + val_ratio=0.2, + num_workers=0, + ) + train_size = len(train_loader.dataset) + val_size = len(val_loader.dataset) + assert train_size == 8 + assert val_size == 2 + + +def test_create_train_val_loaders_invalid_ratio(dataset_10: Path) -> None: + """val_ratio=0 veya 1 → ValueError.""" + with pytest.raises(ValueError, match="val_ratio"): + create_train_val_loaders( + root_dir=dataset_10, + batch_size=2, + val_ratio=0.0, + num_workers=0, + ) + + +def test_create_train_val_loaders_too_few_samples(tmp_path: Path) -> None: + """1 örnek → ValueError (en az 2 gerekli).""" + (tmp_path / "clean").mkdir() + (tmp_path / "degraded").mkdir() + np.save(tmp_path / "clean" / "img_000.npy", np.random.rand(8, 8).astype(np.float32)) + np.save(tmp_path / "degraded" / "img_000.npy", np.random.rand(8, 8).astype(np.float32)) + + with pytest.raises(ValueError, match="At least 2 samples"): + create_train_val_loaders( + root_dir=tmp_path, + batch_size=1, + val_ratio=0.5, + num_workers=0, + ) + + +def test_create_train_val_loaders_val_no_shuffle(dataset_10: Path) -> None: + """Validation loader shuffle=False (deterministic ölçüm).""" + _, val_loader = create_train_val_loaders( + root_dir=dataset_10, + batch_size=2, + val_ratio=0.2, + num_workers=0, + ) + # İlk batch'i iki kez al → aynı olmalı (shuffle=False) + batch1 = next(iter(val_loader)) + batch2 = next(iter(val_loader)) + assert torch.equal(batch1[0], batch2[0]) + + +def test_create_train_val_loaders_with_split(dataset_10: Path) -> None: + """split='medium' → root_dir/medium/clean + root_dir/medium/degraded.""" + medium_dir = dataset_10 / "medium" + (medium_dir / "clean").mkdir(parents=True) + (medium_dir / "degraded").mkdir(parents=True) + for i in range(6): + np.save(medium_dir / "clean" / f"img_{i:03d}.npy", np.random.rand(16, 16).astype(np.float32)) + np.save(medium_dir / "degraded" / f"img_{i:03d}.npy", np.random.rand(16, 16).astype(np.float32)) + + train_loader, val_loader = create_train_val_loaders( + root_dir=dataset_10, + batch_size=2, + val_ratio=0.5, + num_workers=0, + split="medium", + ) + assert len(train_loader.dataset) == 3 + assert len(val_loader.dataset) == 3 diff --git a/services/ml/tests/test_dataset.py b/services/ml/tests/test_dataset.py new file mode 100644 index 0000000..7caeace --- /dev/null +++ b/services/ml/tests/test_dataset.py @@ -0,0 +1,94 @@ +"""Unit tests for services.ml.data.dataset.BlackHoleDataset.""" +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +import torch + +from services.ml.data.dataset import BlackHoleDataset + + +@pytest.fixture +def local_dataset_dir(tmp_path: Path) -> Path: + """Geçici yerel dataset dizini — 4 çift clean/degraded .npy.""" + clean_dir = tmp_path / "clean" + degraded_dir = tmp_path / "degraded" + clean_dir.mkdir(parents=True) + degraded_dir.mkdir(parents=True) + + for i in range(4): + np.save(clean_dir / f"img_{i:03d}.npy", np.random.rand(64, 64).astype(np.float32)) + np.save(degraded_dir / f"img_{i:03d}.npy", np.random.rand(64, 64).astype(np.float32)) + + return tmp_path + + +def test_dataset_len(local_dataset_dir: Path) -> None: + """len(dataset) = dosya sayısı.""" + ds = BlackHoleDataset(root_dir=local_dataset_dir, use_minio=False) + assert len(ds) == 4 + + +def test_dataset_getitem_shape(local_dataset_dir: Path) -> None: + """__getitem__ → (degraded, clean) tuple, shape (1, 1, H, W).""" + ds = BlackHoleDataset(root_dir=local_dataset_dir, use_minio=False) + degraded, clean = ds[0] + assert isinstance(degraded, torch.Tensor) + assert isinstance(clean, torch.Tensor) + assert degraded.shape == (1, 1, 64, 64) + assert clean.shape == (1, 1, 64, 64) + + +def test_dataset_getitem_range(local_dataset_dir: Path) -> None: + """Pixel değerleri [0, 1] aralığında (float32).""" + ds = BlackHoleDataset(root_dir=local_dataset_dir, use_minio=False) + degraded, clean = ds[0] + assert degraded.dtype == torch.float32 + assert clean.dtype == torch.float32 + assert degraded.min() >= 0.0 + assert degraded.max() <= 1.0 + + +def test_dataset_split_subdir(local_dataset_dir: Path) -> None: + """split='medium' → root_dir/medium/clean + root_dir/medium/degraded.""" + medium_dir = local_dataset_dir / "medium" + (medium_dir / "clean").mkdir(parents=True) + (medium_dir / "degraded").mkdir(parents=True) + for i in range(2): + np.save(medium_dir / "clean" / f"img_{i:03d}.npy", np.random.rand(32, 32).astype(np.float32)) + np.save(medium_dir / "degraded" / f"img_{i:03d}.npy", np.random.rand(32, 32).astype(np.float32)) + + ds = BlackHoleDataset(root_dir=local_dataset_dir, use_minio=False, split="medium") + assert len(ds) == 2 + degraded, clean = ds[0] + assert degraded.shape == (1, 1, 32, 32) + + +def test_dataset_augment_changes_shape(local_dataset_dir: Path) -> None: + """augment=True + crop_size=32 → 64x64'den 32x32'ye crop.""" + ds = BlackHoleDataset( + root_dir=local_dataset_dir, + use_minio=False, + augment=True, + crop_size=32, + ) + degraded, clean = ds[0] + assert degraded.shape == (1, 1, 32, 32) + assert clean.shape == (1, 1, 32, 32) + + +def test_dataset_no_augment_keeps_shape(local_dataset_dir: Path) -> None: + """augment=False → orijinal shape korunur.""" + ds = BlackHoleDataset(root_dir=local_dataset_dir, use_minio=False, augment=False) + degraded, clean = ds[0] + assert degraded.shape == (1, 1, 64, 64) + + +def test_dataset_empty_dir(tmp_path: Path) -> None: + """Boş dizin → len = 0.""" + (tmp_path / "clean").mkdir() + (tmp_path / "degraded").mkdir() + ds = BlackHoleDataset(root_dir=tmp_path, use_minio=False) + assert len(ds) == 0 diff --git a/services/ml/tests/test_loss.py b/services/ml/tests/test_loss.py new file mode 100644 index 0000000..d56782b --- /dev/null +++ b/services/ml/tests/test_loss.py @@ -0,0 +1,76 @@ +"""Unit tests for services.ml.losses.loss factory.""" +from __future__ import annotations + +import pytest +import torch +import torch.nn as nn + +from services.ml.losses.loss import _SUPPORTED_LOSSES, get_loss + + +def test_get_loss_default_is_mse() -> None: + """Default loss = MSE.""" + loss = get_loss() + assert isinstance(loss, nn.MSELoss) + + +def test_get_loss_mse() -> None: + """name='mse' → MSELoss.""" + loss = get_loss("mse") + assert isinstance(loss, nn.MSELoss) + + +def test_get_loss_l1() -> None: + """name='l1' → L1Loss.""" + loss = get_loss("l1") + assert isinstance(loss, nn.L1Loss) + + +def test_get_loss_smooth_l1() -> None: + """name='smooth_l1' → SmoothL1Loss.""" + loss = get_loss("smooth_l1") + assert isinstance(loss, nn.SmoothL1Loss) + + +def test_get_loss_smooth_l1_with_beta() -> None: + """kwargs forward — beta parametresi SmoothL1Loss'a geçer.""" + loss = get_loss("smooth_l1", beta=0.5) + assert isinstance(loss, nn.SmoothL1Loss) + assert loss.beta == 0.5 + + +def test_get_loss_mse_with_reduction() -> None: + """kwargs forward — reduction parametresi MSELoss'a geçer.""" + loss = get_loss("mse", reduction="sum") + assert isinstance(loss, nn.MSELoss) + assert loss.reduction == "sum" + + +def test_get_loss_unknown_raises() -> None: + """Bilinmeyen loss → ValueError.""" + with pytest.raises(ValueError, match="Unknown loss"): + get_loss("unknown") + + +def test_get_loss_physics_not_yet_supported() -> None: + """Faz 2'de eklenecek physics loss şimdilik ValueError.""" + with pytest.raises(ValueError, match="Unknown loss"): + get_loss("physics") + + +def test_supported_losses_is_frozenset() -> None: + """_SUPPORTED_LOSSES immutable (frozenset).""" + assert isinstance(_SUPPORTED_LOSSES, frozenset) + assert "mse" in _SUPPORTED_LOSSES + assert "l1" in _SUPPORTED_LOSSES + assert "smooth_l1" in _SUPPORTED_LOSSES + + +def test_loss_forward_computes_scalar() -> None: + """Loss forward → scalar tensor.""" + loss_fn = get_loss("l1") + pred = torch.rand(2, 1, 8, 8) + target = torch.rand(2, 1, 8, 8) + value = loss_fn(pred, target) + assert value.dim() == 0 # scalar + assert value.item() >= 0.0 diff --git a/services/ml/tests/test_metrics.py b/services/ml/tests/test_metrics.py new file mode 100644 index 0000000..178d8e8 --- /dev/null +++ b/services/ml/tests/test_metrics.py @@ -0,0 +1,175 @@ +"""Unit tests for services.ml.evaluation.metrics.""" +from __future__ import annotations + +import pytest +import torch + +from services.ml.evaluation.metrics import ( + compute_asymmetry_ratio, + compute_fid, + compute_flux_conservation, + compute_lpips, + compute_metrics, + compute_physics_metrics, + compute_psnr, + compute_ring_diameter, + compute_ssim, +) + + +# ---------- PSNR ---------- + +def test_compute_psnr_identical_returns_inf(sample_image: torch.Tensor) -> None: + """Aynı görüntü → PSNR = inf.""" + assert compute_psnr(sample_image, sample_image) == float("inf") + + +def test_compute_psnr_different_returns_finite(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """Farklı görüntüler → PSNR sonlu ve negatif olmayan.""" + degraded, clean = sample_batch + psnr = compute_psnr(degraded, clean) + assert psnr != float("inf") + assert psnr > 0.0 + + +def test_compute_psnr_shape_mismatch_raises() -> None: + """Farklı shape → ValueError.""" + a = torch.rand(1, 1, 64, 64) + b = torch.rand(1, 1, 32, 32) + with pytest.raises(ValueError, match="same shape"): + compute_psnr(a, b) + + +# ---------- SSIM ---------- + +def test_compute_ssim_identical_high(sample_image: torch.Tensor) -> None: + """Aynı görüntü → SSIM > 0.99.""" + ssim = compute_ssim(sample_image, sample_image) + assert ssim > 0.99 + + +def test_compute_ssim_different_lower(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """Farklı görüntüler → SSIM < 1.0.""" + degraded, clean = sample_batch + ssim = compute_ssim(degraded, clean) + assert 0.0 <= ssim < 1.0 + + +# ---------- LPIPS ---------- + +def test_compute_lpips_identical_low(sample_image: torch.Tensor) -> None: + """Aynı görüntü → LPIPS ≈ 0.""" + lpips = compute_lpips(sample_image, sample_image) + assert lpips < 0.01 + + +def test_compute_lpips_different_higher(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """Farklı görüntüler → LPIPS > 0.""" + degraded, clean = sample_batch + lpips = compute_lpips(degraded, clean) + assert lpips > 0.0 + + +# ---------- FID ---------- + +def test_compute_fid_identical_low(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """Aynı görüntüler → FID ≈ 0.""" + _, clean = sample_batch + fid = compute_fid(clean, clean) + assert fid < 1.0 + + +def test_compute_fid_different_higher(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """Farklı görüntüler → FID > 0.""" + degraded, clean = sample_batch + fid = compute_fid(clean, degraded) + assert fid > 0.0 + + +# ---------- Physics: Flux Conservation ---------- + +def test_compute_flux_conservation_identical_zero(sample_image: torch.Tensor) -> None: + """Aynı görüntü → flux error = 0.""" + err = compute_flux_conservation(sample_image, sample_image) + assert err == 0.0 + + +def test_compute_flux_conservation_different_positive( + sample_batch: tuple[torch.Tensor, torch.Tensor], +) -> None: + """Farklı görüntüler → flux error > 0.""" + degraded, clean = sample_batch + err = compute_flux_conservation(degraded, clean) + assert err > 0.0 + + +# ---------- Physics: Ring Diameter ---------- + +def test_compute_ring_diameter_returns_float(sample_image: torch.Tensor) -> None: + """Ring diameter float döner.""" + diameter = compute_ring_diameter(sample_image) + assert isinstance(diameter, float) + + +def test_compute_ring_diameter_zero_image() -> None: + """Sıfır görüntü → diameter = 0 (peak yok).""" + img = torch.zeros(1, 1, 64, 64) + diameter = compute_ring_diameter(img) + assert diameter == 0.0 + + +# ---------- Physics: Asymmetry Ratio ---------- + +def test_compute_asymmetry_ratio_symmetric() -> None: + """Simetrik görüntü → asymmetry ≈ 0.""" + img = torch.zeros(1, 1, 64, 64) + img[:, :, 16:48, 16:48] = 1.0 # merkezi kare + ratio = compute_asymmetry_ratio(img) + assert ratio < 0.01 + + +def test_compute_asymmetry_ratio_asymmetric() -> None: + """Asimetrik görüntü → asymmetry > 0.""" + img = torch.zeros(1, 1, 64, 64) + img[:, :, 16:48, 32:48] = 1.0 # sağa yaslı kare + ratio = compute_asymmetry_ratio(img) + assert ratio > 0.0 + + +# ---------- Physics: Combined ---------- + +def test_compute_physics_metrics_keys(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """3 physics metriği döner.""" + degraded, clean = sample_batch + metrics = compute_physics_metrics(degraded, clean) + assert "flux_error" in metrics + assert "ring_diameter_error" in metrics + assert "asymmetry_error" in metrics + + +# ---------- compute_metrics (combined) ---------- + +def test_compute_metrics_basic(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """Default: sadece psnr + ssim.""" + degraded, clean = sample_batch + metrics = compute_metrics(degraded, clean) + assert "psnr" in metrics + assert "ssim" in metrics + assert "lpips" not in metrics + assert "flux_error" not in metrics + + +def test_compute_metrics_with_lpips(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """include_lpips=True → lpips eklenir.""" + degraded, clean = sample_batch + metrics = compute_metrics(degraded, clean, include_lpips=True) + assert "lpips" in metrics + + +def test_compute_metrics_with_physics(sample_batch: tuple[torch.Tensor, torch.Tensor]) -> None: + """include_physics=True → 3 physics metriği eklenir.""" + degraded, clean = sample_batch + metrics = compute_metrics(degraded, clean, include_physics=True) + assert "flux_error" in metrics + assert "ring_diameter_error" in metrics + assert "asymmetry_error" in metrics diff --git a/services/ml/training/train.py b/services/ml/training/train.py index 4c0eb85..9a65fda 100644 --- a/services/ml/training/train.py +++ b/services/ml/training/train.py @@ -1,6 +1,11 @@ from pathlib import Path +import hydra +import mlflow +import mlflow.pytorch import torch +from omegaconf import DictConfig, OmegaConf +from torch.amp import GradScaler, autocast from services.ml.evaluation.benchmark import ( ValidationSummary, @@ -12,109 +17,197 @@ from services.ml.data.dataloader import create_train_val_loaders from services.ml.models.unet import UNet from services.ml.losses.loss import get_loss +from services.ml.checkpoints.checkpoint import save_checkpoint -def train( - root_dir="data/train", - output_dir="checkpoints", - batch_size=16, - epochs=10, - learning_rate=1e-4, - val_ratio=0.2, -): - output_dir = Path(output_dir) +@hydra.main(config_path="../conf", config_name="config", version_base="1.3") +def train(cfg: DictConfig) -> Path: + """Train U-Net baseline on black hole image pairs. + + All hyperparameters come from Hydra config (services/ml/conf/*.yaml). + Override from CLI: `python -m services.ml.training.train training.epochs=2` + + Every run is logged to MLflow: params (full config), per-epoch metrics, + and artifacts (best_model.pt, sample PNGs, validation_results.jsonl). + """ + # Resolve device + if cfg.device == "auto": + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + else: + device = torch.device(cfg.device) + + # Output dir: Hydra already created outputs//