From 3b51726551d67c58400df21e6134c1b5b156d310 Mon Sep 17 00:00:00 2001 From: hhhhsc <1710496817@qq.com> Date: Mon, 15 Jun 2026 11:55:40 +0800 Subject: [PATCH 1/3] Default Ray worker init containers to runtime image --- .../charts/ray-cluster/templates/_helpers.tpl | 21 +++++++++++ .../templates/raycluster-cluster.yaml | 4 +-- .../datamate/charts/ray-cluster/values.yaml | 11 ++++++ deployment/helm/datamate/values.yaml | 36 +++++++++++++++++++ scripts/images/runtime/Dockerfile | 2 +- 5 files changed, 71 insertions(+), 3 deletions(-) diff --git a/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl b/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl index 79fe3bb79..ce4799873 100644 --- a/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl +++ b/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl @@ -66,3 +66,24 @@ Name of image {{- $name }}:{{ $tag }} {{- end }} {{- end }} + +{{/* +Render worker init containers, defaulting missing image fields to the Ray runtime image. +*/}} +{{- define "ray-cluster.workerInitContainers" -}} +{{- $root := .root -}} +{{- $image := include "ray-cluster.image" $root -}} +{{- $imagePullPolicy := default $root.Values.image.pullPolicy $root.Values.global.image.pullPolicy -}} +{{- $containers := list -}} +{{- range $container := (.initContainers | default list) }} +{{- $containerWithDefaults := deepCopy $container -}} +{{- if not $containerWithDefaults.image }} +{{- $_ := set $containerWithDefaults "image" $image -}} +{{- end }} +{{- if not $containerWithDefaults.imagePullPolicy }} +{{- $_ := set $containerWithDefaults "imagePullPolicy" $imagePullPolicy -}} +{{- end }} +{{- $containers = append $containers $containerWithDefaults -}} +{{- end }} +{{- toYaml $containers -}} +{{- end }} diff --git a/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml b/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml index 2cb2ce887..5eaf3a8ab 100644 --- a/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml +++ b/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml @@ -190,7 +190,7 @@ spec: spec: {{- with .Values.worker.initContainers }} initContainers: - {{- toYaml . | nindent 8 }} + {{- include "ray-cluster.workerInitContainers" (dict "root" $ "initContainers" .) | nindent 8 }} {{- end }} containers: - name: ray-worker @@ -313,7 +313,7 @@ spec: spec: {{- with $values.initContainers }} initContainers: - {{- toYaml . | nindent 8 }} + {{- include "ray-cluster.workerInitContainers" (dict "root" $ "initContainers" .) | nindent 8 }} {{- end }} containers: - name: ray-worker diff --git a/deployment/helm/datamate/charts/ray-cluster/values.yaml b/deployment/helm/datamate/charts/ray-cluster/values.yaml index 58752d02e..2c71e2d50 100644 --- a/deployment/helm/datamate/charts/ray-cluster/values.yaml +++ b/deployment/helm/datamate/charts/ray-cluster/values.yaml @@ -131,7 +131,12 @@ worker: serviceAccountName: "" restartPolicy: "" rayStartParams: {} + # initContainers follows standard K8s init container schema. + # If image is omitted, it defaults to the Ray runtime image. initContainers: [] + # - name: init-runtime + # command: ["sh", "-c"] + # args: ["your custom command"] # containerEnv specifies environment variables for the Ray container, # Follows standard K8s container env schema. containerEnv: [] @@ -200,6 +205,12 @@ additionalWorkerGroups: serviceAccountName: "" restartPolicy: "" rayStartParams: {} + # initContainers follows standard K8s init container schema. + # If image is omitted, it defaults to the Ray runtime image. + initContainers: [] + # - name: init-runtime + # command: ["sh", "-c"] + # args: ["your custom command"] # containerEnv specifies environment variables for the Ray container, # Follows standard K8s container env schema. containerEnv: [] diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index d00d73ba9..ae57088b7 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -376,6 +376,24 @@ ray-cluster: worker: nodeSelector: {} tolerations: [] + initContainers: + - name: ray-init + securityContext: + runAsUser: 0 + capabilities: + add: + - "NET_ADMIN" + env: + - name: BLOCKED_IP + value: "" + command: + - bash + - -c + - | + IFS=',' read -ra bips <<< "${BLOCKED_IP}" + for bip in "${bips[@]}"; do + iptables -A OUTPUT -m owner --uid-owner 1100 -d ${bip} -j REJECT + done containerEnv: - name: RAY_DEDUP_LOGS value: "0" @@ -434,6 +452,24 @@ ray-cluster: maxReplicas: 8 rayStartParams: resources: '"{\"npu\": 1}"' + initContainers: + - name: ray-init + securityContext: + runAsUser: 0 + capabilities: + add: + - "NET_ADMIN" + env: + - name: BLOCKED_IP + value: "" + command: + - bash + - -c + - | + IFS=',' read -ra bips <<< "${BLOCKED_IP}" + for bip in "${bips[@]}"; do + iptables -A OUTPUT -m owner --uid-owner 1100 -d ${bip} -j REJECT + done containerEnv: - name: RAY_DEDUP_LOGS value: "0" diff --git a/scripts/images/runtime/Dockerfile b/scripts/images/runtime/Dockerfile index 5481ec391..e32f01315 100644 --- a/scripts/images/runtime/Dockerfile +++ b/scripts/images/runtime/Dockerfile @@ -3,7 +3,7 @@ FROM ghcr.io/astral-sh/uv:python3.11-bookworm RUN --mount=type=cache,target=/var/cache/apt \ --mount=type=cache,target=/var/lib/apt \ apt update \ - && apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix swig poppler-utils tesseract-ocr + && apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix swig poppler-utils tesseract-ocr iptables RUN mkdir -p /home/models \ && wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-LCNet_x1_0_doc_ori_infer.tar \ From ba1426c1393f89f542f35fa1fe21e7781fbd35ea Mon Sep 17 00:00:00 2001 From: hhhhsc <1710496817@qq.com> Date: Mon, 15 Jun 2026 14:23:18 +0800 Subject: [PATCH 2/3] Add Ray worker network policy for blocked CIDRs --- .../templates/worker-networkpolicy.yaml | 21 ++++++++++ .../datamate/charts/ray-cluster/values.yaml | 6 +++ deployment/helm/datamate/values.yaml | 40 ++----------------- 3 files changed, 31 insertions(+), 36 deletions(-) create mode 100644 deployment/helm/datamate/charts/ray-cluster/templates/worker-networkpolicy.yaml diff --git a/deployment/helm/datamate/charts/ray-cluster/templates/worker-networkpolicy.yaml b/deployment/helm/datamate/charts/ray-cluster/templates/worker-networkpolicy.yaml new file mode 100644 index 000000000..de9175ae9 --- /dev/null +++ b/deployment/helm/datamate/charts/ray-cluster/templates/worker-networkpolicy.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.workerNetworkPolicy.enabled .Values.workerNetworkPolicy.blockedCIDRs }} +apiVersion: crd.projectcalico.org/v1 +kind: NetworkPolicy +metadata: + name: {{ include "ray-cluster.fullname" . }}-worker-egress + namespace: {{ .Release.Namespace }} + labels: + {{- include "ray-cluster.labels" . | nindent 4 }} +spec: + selector: ray.io/node-type == "worker" + types: + - Egress + egress: + - action: Deny + destination: + nets: +{{- range .Values.workerNetworkPolicy.blockedCIDRs }} + - {{ . }} +{{- end }} + - action: Allow +{{- end }} diff --git a/deployment/helm/datamate/charts/ray-cluster/values.yaml b/deployment/helm/datamate/charts/ray-cluster/values.yaml index 2c71e2d50..6e004eb39 100644 --- a/deployment/helm/datamate/charts/ray-cluster/values.yaml +++ b/deployment/helm/datamate/charts/ray-cluster/values.yaml @@ -16,6 +16,12 @@ fullnameOverride: "datamate-raycluster" imagePullSecrets: [] # - name: an-existing-secret +workerNetworkPolicy: + enabled: false + # CIDRs to block from Ray worker pod egress. Use /32 for a single IPv4 address. + blockedCIDRs: [] + # - 192.0.2.10/32 + # common defined values shared between the head and worker common: # containerEnv specifies environment variables for the Ray head and worker containers. diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index ae57088b7..e75725a92 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -302,6 +302,10 @@ runtime: ray-cluster: enabled: true + workerNetworkPolicy: + enabled: true + # Use CIDR notation, for example 192.0.2.10/32 for a single IPv4 address. + blockedCIDRs: [] head: nodeSelector: {} tolerations: [] @@ -376,24 +380,6 @@ ray-cluster: worker: nodeSelector: {} tolerations: [] - initContainers: - - name: ray-init - securityContext: - runAsUser: 0 - capabilities: - add: - - "NET_ADMIN" - env: - - name: BLOCKED_IP - value: "" - command: - - bash - - -c - - | - IFS=',' read -ra bips <<< "${BLOCKED_IP}" - for bip in "${bips[@]}"; do - iptables -A OUTPUT -m owner --uid-owner 1100 -d ${bip} -j REJECT - done containerEnv: - name: RAY_DEDUP_LOGS value: "0" @@ -452,24 +438,6 @@ ray-cluster: maxReplicas: 8 rayStartParams: resources: '"{\"npu\": 1}"' - initContainers: - - name: ray-init - securityContext: - runAsUser: 0 - capabilities: - add: - - "NET_ADMIN" - env: - - name: BLOCKED_IP - value: "" - command: - - bash - - -c - - | - IFS=',' read -ra bips <<< "${BLOCKED_IP}" - for bip in "${bips[@]}"; do - iptables -A OUTPUT -m owner --uid-owner 1100 -d ${bip} -j REJECT - done containerEnv: - name: RAY_DEDUP_LOGS value: "0" From ec71859c85a04329c94f764016c84fd08b19efda Mon Sep 17 00:00:00 2001 From: hhhhsc <1710496817@qq.com> Date: Mon, 15 Jun 2026 14:40:01 +0800 Subject: [PATCH 3/3] clean code --- .../charts/ray-cluster/templates/_helpers.tpl | 21 ------------------- .../templates/raycluster-cluster.yaml | 4 ++-- .../datamate/charts/ray-cluster/values.yaml | 11 ---------- scripts/images/runtime/Dockerfile | 2 +- 4 files changed, 3 insertions(+), 35 deletions(-) diff --git a/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl b/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl index ce4799873..79fe3bb79 100644 --- a/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl +++ b/deployment/helm/datamate/charts/ray-cluster/templates/_helpers.tpl @@ -66,24 +66,3 @@ Name of image {{- $name }}:{{ $tag }} {{- end }} {{- end }} - -{{/* -Render worker init containers, defaulting missing image fields to the Ray runtime image. -*/}} -{{- define "ray-cluster.workerInitContainers" -}} -{{- $root := .root -}} -{{- $image := include "ray-cluster.image" $root -}} -{{- $imagePullPolicy := default $root.Values.image.pullPolicy $root.Values.global.image.pullPolicy -}} -{{- $containers := list -}} -{{- range $container := (.initContainers | default list) }} -{{- $containerWithDefaults := deepCopy $container -}} -{{- if not $containerWithDefaults.image }} -{{- $_ := set $containerWithDefaults "image" $image -}} -{{- end }} -{{- if not $containerWithDefaults.imagePullPolicy }} -{{- $_ := set $containerWithDefaults "imagePullPolicy" $imagePullPolicy -}} -{{- end }} -{{- $containers = append $containers $containerWithDefaults -}} -{{- end }} -{{- toYaml $containers -}} -{{- end }} diff --git a/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml b/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml index 5eaf3a8ab..2cb2ce887 100644 --- a/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml +++ b/deployment/helm/datamate/charts/ray-cluster/templates/raycluster-cluster.yaml @@ -190,7 +190,7 @@ spec: spec: {{- with .Values.worker.initContainers }} initContainers: - {{- include "ray-cluster.workerInitContainers" (dict "root" $ "initContainers" .) | nindent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} containers: - name: ray-worker @@ -313,7 +313,7 @@ spec: spec: {{- with $values.initContainers }} initContainers: - {{- include "ray-cluster.workerInitContainers" (dict "root" $ "initContainers" .) | nindent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} containers: - name: ray-worker diff --git a/deployment/helm/datamate/charts/ray-cluster/values.yaml b/deployment/helm/datamate/charts/ray-cluster/values.yaml index 6e004eb39..f519da661 100644 --- a/deployment/helm/datamate/charts/ray-cluster/values.yaml +++ b/deployment/helm/datamate/charts/ray-cluster/values.yaml @@ -137,12 +137,7 @@ worker: serviceAccountName: "" restartPolicy: "" rayStartParams: {} - # initContainers follows standard K8s init container schema. - # If image is omitted, it defaults to the Ray runtime image. initContainers: [] - # - name: init-runtime - # command: ["sh", "-c"] - # args: ["your custom command"] # containerEnv specifies environment variables for the Ray container, # Follows standard K8s container env schema. containerEnv: [] @@ -211,12 +206,6 @@ additionalWorkerGroups: serviceAccountName: "" restartPolicy: "" rayStartParams: {} - # initContainers follows standard K8s init container schema. - # If image is omitted, it defaults to the Ray runtime image. - initContainers: [] - # - name: init-runtime - # command: ["sh", "-c"] - # args: ["your custom command"] # containerEnv specifies environment variables for the Ray container, # Follows standard K8s container env schema. containerEnv: [] diff --git a/scripts/images/runtime/Dockerfile b/scripts/images/runtime/Dockerfile index e32f01315..5481ec391 100644 --- a/scripts/images/runtime/Dockerfile +++ b/scripts/images/runtime/Dockerfile @@ -3,7 +3,7 @@ FROM ghcr.io/astral-sh/uv:python3.11-bookworm RUN --mount=type=cache,target=/var/cache/apt \ --mount=type=cache,target=/var/lib/apt \ apt update \ - && apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix swig poppler-utils tesseract-ocr iptables + && apt install -y libgl1 libglib2.0-0 vim libmagic1 libreoffice dos2unix swig poppler-utils tesseract-ocr RUN mkdir -p /home/models \ && wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-LCNet_x1_0_doc_ori_infer.tar \