Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
d871ea0
checkpoint ve validation eklendi.
betultgumus Jul 20, 2026
3f3e4c2
feat: add loss function and training loop
Jul 20, 2026
2c4d925
feat: add validation and checkpoint support to training
Jul 21, 2026
d86a8f8
Add checkpoint save/load integration to training pipeline
Jul 21, 2026
e74acb1
evulation ve metrics oluşturuldu.
betultgumus Jul 21, 2026
2b5fffc
feature: sunucu verileri bağlandı.
betultgumus Jul 22, 2026
77d73f1
fix: sunucu data bağlantısı düzenlendi.
betultgumus Jul 22, 2026
a4abaf1
data sunucu path güncellendi.
betultgumus Jul 22, 2026
17ae300
feat: added normalizition test.
betultgumus Jul 23, 2026
db4880f
Normalization control was achieved.
betultgumus Jul 23, 2026
07b691a
fix: revised bucket_name as 'datasets'
Jul 24, 2026
b902852
The U-Net core training infrastructure has been completed.
betultgumus Jul 24, 2026
61b0fb6
add Pix2Pix GAN training pipeline and ONNX inference server
betultgumus Jul 28, 2026
81a635f
pix2pix tests and validations were performed.
betultgumus Jul 29, 2026
bfb7310
fix: resolve ML pipeline compatibility and test failures
Jul 30, 2026
fa6884b
Ready for training
betultgumus Aug 10, 2026
e0a5253
fix(ml): add /dev/shm volume to training jobs to fix DataLoader BusError
betultgumus Aug 11, 2026
10d3247
Implement ESRGAN training and service integration
Aug 12, 2026
0b15ee1
Merge remote-tracking branch 'origin/ml/feature' into ml/feature
Aug 20, 2026
f07e6bc
feat(ml): physics metrics, Optuna runner, Faz 3 evaluation
Aug 20, 2026
d705615
fix(ml): U-Net config uyumluluğu + ESRGAN test parametreleri
Aug 21, 2026
8e1ee93
fix(ml): U-Net config compatibility + ESRGAN test params
Aug 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added .coverage
Binary file not shown.
157 changes: 157 additions & 0 deletions infra/k8s/ml/inference.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
# DeepHorizon inference server — K8s Deployment + Service.
# Faz 3 Adım 28: gRPC inference + Prometheus metrics.
#
# Topology:
# - 1 replica (GPU pod, scale=0 ile başla, ihtiyaç olursa artır)
# - NodePort 30551 → gRPC (50051)
# - ClusterIP 8000 → Prometheus metrics
# - GPU: nvidia.com/gpu: 1 (L40S node selector)
# - PriorityClass: high-priority (training job'larından önce schedule)
#
# Deploy:
# kubectl apply -f infra/k8s/ml/inference.yaml
#
# Test:
# kubectl port-forward svc/inference 50051:50051 -n deephorizon-ml
# grpcurl -plaintext -d '{"image": {"data": "...", "mime_type": "image/png"}, "model_id": "pix2pix-v1"}' \
# localhost:50051 deephorizon.v1.InferenceService/Enhance

---
apiVersion: v1
kind: Service
metadata:
name: inference
namespace: deephorizon-ml
labels:
app: inference
component: ml-serving
spec:
type: NodePort
selector:
app: inference
ports:
- name: grpc
port: 50051
targetPort: 50051
nodePort: 30551
protocol: TCP
- name: metrics
port: 8000
targetPort: 8000
protocol: TCP

---
apiVersion: apps/v1
kind: Deployment
metadata:
name: inference
namespace: deephorizon-ml
labels:
app: inference
component: ml-serving
spec:
replicas: 1
strategy:
type: Recreate # GPU pod — rolling update sırasında 2 pod aynı anda çalışamaz
selector:
matchLabels:
app: inference
template:
metadata:
labels:
app: inference
component: ml-serving
spec:
restartPolicy: Always
automountServiceAccountToken: false
priorityClassName: high-priority
# L40S GPU node selector
nodeSelector:
nvidia.com/gpu.product: NVIDIA-L40S
tolerations:
- key: nvidia.com/gpu
operator: Exists
effect: NoSchedule
containers:
- name: inference-server
image: localhost:32000/deephorizon-inference:v1
imagePullPolicy: IfNotPresent
args:
- "--models-dir"
- "/app/exports"
- "--port"
- "50051"
- "--metrics-port"
- "8000"
- "--max-workers"
- "4"
- "--log-level"
- "INFO"
ports:
- name: grpc
containerPort: 50051
protocol: TCP
- name: metrics
containerPort: 8000
protocol: TCP
env:
- name: PYTHONUNBUFFERED
value: "1"
- name: PYTHONPATH
value: /app
resources:
requests:
cpu: "2"
memory: 4Gi
nvidia.com/gpu: 1
limits:
cpu: "4"
memory: 8Gi
nvidia.com/gpu: 1
volumeMounts:
- name: models
mountPath: /app/exports
readOnly: true
- name: dshm
mountPath: /dev/shm
livenessProbe:
tcpSocket:
port: 50051
initialDelaySeconds: 30
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
httpGet:
path: /metrics
port: 8000
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 3
volumes:
- name: models
persistentVolumeClaim:
claimName: inference-models-pvc
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 4Gi

---
# PVC for ONNX models (MinIO'dan indirilen veya build-time bake edilen)
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: inference-models-pvc
namespace: deephorizon-ml
labels:
app: inference
component: ml-serving
spec:
accessModes:
- ReadOnlyMany
resources:
Comment on lines +152 to +154
requests:
storage: 10Gi
storageClassName: microk8s-hostpath
23 changes: 23 additions & 0 deletions infra/k8s/ml/priority-class.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# DeepHorizon PriorityClass definitions.
# Faz 3 Adım 28: Inference pod'ları training job'larından önce schedule edilir.
#
# Deploy:
# kubectl apply -f infra/k8s/ml/priority-class.yaml

---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: high-priority
description: "Inference serving pods — GPU node'larda training'den önce schedule edilir."
globalDefault: false
value: 1000000

---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: training-priority
description: "Training job'ları — inference yokken GPU node'ları kullanır."
globalDefault: false
value: 100000
14 changes: 14 additions & 0 deletions infra/k8s/ml/pvcs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: training-outputs-pvc
namespace: deephorizon-ml
labels:
app: ml-training
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 50Gi
storageClassName: microk8s-hostpath
53 changes: 53 additions & 0 deletions infra/k8s/ml/training-image-build-job.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
apiVersion: batch/v1
kind: Job
metadata:
name: training-image-build-20260810-v2
namespace: deephorizon-ml
spec:
backoffLimit: 0
activeDeadlineSeconds: 1800
ttlSecondsAfterFinished: 3600
template:
spec:
restartPolicy: Never
automountServiceAccountToken: false
initContainers:
- name: unpack-context
image: localhost:32000/deephorizon-training:2026-08-07b
imagePullPolicy: IfNotPresent
command: ["/bin/bash", "-lc"]
args:
- mkdir -p /workspace && tar --touch --no-same-owner --no-same-permissions -xzf /context/context.tar.gz -C /workspace
volumeMounts:
- name: context-archive
mountPath: /context
readOnly: true
- name: workspace
mountPath: /workspace
containers:
- name: kaniko
image: gcr.io/kaniko-project/executor:v1.23.2-debug
args:
- --dockerfile=/workspace/Dockerfile
- --context=dir:///workspace
- --destination=registry.container-registry.svc.cluster.local:5000/deephorizon-training:20260810-unet100-v2
- --insecure-registry=registry.container-registry.svc.cluster.local:5000
- --skip-tls-verify-registry=registry.container-registry.svc.cluster.local:5000
- --snapshot-mode=redo
- --verbosity=info
resources:
requests:
cpu: "1"
memory: 1Gi
limits:
cpu: "4"
memory: 4Gi
volumeMounts:
- name: workspace
mountPath: /workspace
volumes:
- name: context-archive
configMap:
name: training-build-context-20260810-v2
- name: workspace
emptyDir: {}
Loading
Loading