Add support for Ceph RGW

ansible/main.yml

@@ -30,5 +30,12 @@
     - lvms
     - common-osp
     - controlplane
+    - { role: ceph-rgw-prep, when: ceph_backend | default(false) | bool }
     - dataplane
     - post
+
+- name: Deploy Ceph RGW for object storage
+  hosts: ceph_admin
+  gather_facts: false
+  roles:
+    - { role: ceph-rgw, when: ceph_backend | default(false) | bool }

ansible/roles/ceph-rgw-prep/defaults/main.yml

@@ -0,0 +1,17 @@
+---
+openstack_namespace: openstack
+openstackclient_pod: openstackclient
+osp_secret_name: osp-secret
+osp_swift_password_key: SwiftPassword
+keystone_region: regionOne
+rgw_storage_vip: 172.20.0.24
+rgw_external_vip: 172.19.0.24
+rgw_frontend_port: 8080
+rgw_dns_zone: ceph.local
+rgw_internal_hostname: ceph-rgw-internal-vip.ceph.local
+rgw_external_hostname: ceph-rgw-external-vip.ceph.local
+rgw_dnsdata_name: ceph-storage
+rgw_certificate_name: cert-ceph-rgw
+rgw_certificate_secret: cert-ceph-rgw
+rgw_certificate_issuer: rootca-public
+rgw_certificate_duration: 43800h0m0s

ansible/roles/ceph-rgw-prep/tasks/includes/ensure_endpoint.yml

@@ -0,0 +1,30 @@
+---
+- name: "Get object-store {{ rgw_endpoint_interface }} endpoint"
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack endpoint list \
+        --service object-store \
+        --interface {{ rgw_endpoint_interface }} \
+        -f value -c ID -c URL
+  register: rgw_endpoint_list
+  changed_when: false
+  failed_when: false
+
+- name: "Create object-store {{ rgw_endpoint_interface }} endpoint"
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack endpoint create \
+        --region {{ keystone_region }} \
+        object-store {{ rgw_endpoint_interface }} \
+        '{{ rgw_endpoint_url }}'
+  when: rgw_endpoint_list.stdout_lines | length == 0
+
+- name: "Update object-store {{ rgw_endpoint_interface }} endpoint URL"
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack endpoint set \
+        --url '{{ rgw_endpoint_url }}' \
+        {{ rgw_endpoint_list.stdout_lines[0].split()[0] }}
+  when:
+    - rgw_endpoint_list.stdout_lines | length > 0
+    - (rgw_endpoint_list.stdout_lines[0].split(None, 1)[1] | default('')) != rgw_endpoint_url

ansible/roles/ceph-rgw-prep/tasks/includes/export_tls.yml

@@ -0,0 +1,17 @@
+---
+- name: Read RGW TLS cert and key from Certificate secret
+  no_log: true
+  shell: |
+    set -euo pipefail
+    crt=$(oc -n {{ openstack_namespace }} get secret {{ rgw_certificate_secret }} \
+      -o jsonpath='{.data.tls\.crt}' | base64 -d)
+    key=$(oc -n {{ openstack_namespace }} get secret {{ rgw_certificate_secret }} \
+      -o jsonpath='{.data.tls\.key}' | base64 -d)
+    printf '%s\n%s\n' "$crt" "$key"
+  register: rgw_tls_pem_raw
+  changed_when: false
+
+- name: Store concatenated TLS PEM on localhost for Stage 6
+  no_log: true
+  set_fact:
+    rgw_ssl_pem: "{{ rgw_tls_pem_raw.stdout }}"

ansible/roles/ceph-rgw-prep/tasks/includes/keystone_prep.yml

@@ -0,0 +1,115 @@
+---
+- name: Wait for openstackclient pod
+  shell: |
+    oc wait -n {{ openstack_namespace }} pod/{{ openstackclient_pod }} \
+      --for=condition=Ready --timeout=300s
+  changed_when: false
+
+- name: Fetch SwiftPassword from osp-secret
+  no_log: true
+  shell: |
+    oc get secret {{ osp_secret_name }} -n {{ openstack_namespace }} \
+      -o jsonpath='{.data.{{ osp_swift_password_key }}}' | base64 -d
+  register: osp_swift_password_raw
+  changed_when: false
+  failed_when: osp_swift_password_raw.rc != 0 or osp_swift_password_raw.stdout | length == 0
+
+- name: Set swift password fact
+  no_log: true
+  set_fact:
+    rgw_swift_password: "{{ osp_swift_password_raw.stdout }}"
+
+- name: Fetch Keystone internal endpoint URL
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack endpoint list --service identity --interface internal -f value -c URL
+  register: keystone_internal_urls
+  changed_when: false
+  failed_when: keystone_internal_urls.rc != 0 or keystone_internal_urls.stdout_lines | length == 0
+
+- name: Fetch keystone-internal LoadBalancer IP for Ceph reachability
+  shell: |
+    oc -n {{ openstack_namespace }} get svc keystone-internal \
+      -o jsonpath='{.status.loadBalancer.ingress[0].ip}'
+  register: keystone_lb_ip
+  changed_when: false
+  failed_when: false
+
+- name: Set Keystone URL and RGW endpoint URL facts
+  set_fact:
+    rgw_keystone_url: >-
+      {{
+        ('https://' ~ (keystone_lb_ip.stdout | trim) ~ ':5000')
+        if (keystone_lb_ip.stdout | default('') | trim | length > 0)
+        else (keystone_internal_urls.stdout_lines[0] | trim)
+      }}
+    rgw_endpoint_public_url: "https://{{ rgw_external_hostname }}:{{ rgw_frontend_port }}/swift/v1/AUTH_%(tenant_id)s"
+    rgw_endpoint_internal_url: "https://{{ rgw_internal_hostname }}:{{ rgw_frontend_port }}/swift/v1/AUTH_%(tenant_id)s"
+
+- name: Check object-store service
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack service show object-store
+  register: rgw_object_store_service
+  changed_when: false
+  failed_when: false
+
+- name: Create object-store service
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack service create \
+        --name swift \
+        --description "OpenStack Object Storage" \
+        object-store
+  when: rgw_object_store_service.rc != 0
+
+- name: Ensure swift user
+  no_log: true
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack user create --project service \
+        --password {{ rgw_swift_password | quote }} \
+        --or-show swift
+  changed_when: false
+
+- name: Ensure swiftoperator role
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack role create --or-show swiftoperator
+  changed_when: false
+
+- name: Ensure ResellerAdmin role
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack role create --or-show ResellerAdmin
+  changed_when: false
+
+- name: Ensure swift member role on service project
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack role add --user swift --project service member
+  changed_when: false
+
+- name: Ensure swift admin role on service project
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack role add --user swift --project service admin
+  changed_when: false
+
+- name: Ensure admin swiftoperator role on admin project
+  shell: |
+    oc rsh -n {{ openstack_namespace }} {{ openstackclient_pod }} \
+      openstack role add --project admin --user admin swiftoperator
+  changed_when: false
+
+- name: Ensure object-store public endpoint
+  include_tasks: ensure_endpoint.yml
+  vars:
+    rgw_endpoint_interface: public
+    rgw_endpoint_url: "{{ rgw_endpoint_public_url }}"
+
+- name: Ensure object-store internal endpoint
+  include_tasks: ensure_endpoint.yml
+  vars:
+    rgw_endpoint_interface: internal
+    rgw_endpoint_url: "{{ rgw_endpoint_internal_url }}"

ansible/roles/ceph-rgw-prep/tasks/includes/tls_prep.yml

@@ -0,0 +1,98 @@
+---
+- name: Render DNSData for RGW VIPs
+  template:
+    src: dnsdata_ceph_rgw.yaml.j2
+    dest: /tmp/jetbrew-ceph-rgw-dnsdata.yaml
+    mode: "0644"
+
+- name: Apply DNSData for RGW VIPs
+  shell: |
+    oc apply -f /tmp/jetbrew-ceph-rgw-dnsdata.yaml
+
+- name: Wait for DNSData Ready
+  shell: |
+    oc wait -n {{ openstack_namespace }} dnsdata/{{ rgw_dnsdata_name }} \
+      --for=jsonpath='{.status.conditions[?(@.type=="Ready")].status}'=True \
+      --timeout=300s
+  changed_when: false
+
+- name: Get dnsmasq-dns ClusterIP
+  shell: |
+    oc -n {{ openstack_namespace }} get svc dnsmasq-dns -o jsonpath='{.spec.clusterIP}'
+  register: rgw_dnsmasq_cluster_ip
+  changed_when: false
+  failed_when: rgw_dnsmasq_cluster_ip.stdout | length == 0
+
+- name: Ensure CoreDNS forwards ceph.local to dnsmasq-dns
+  shell: |
+    python3 - <<'PY'
+    import json
+    import subprocess
+    import sys
+
+    zone = "{{ rgw_dns_zone }}"
+    upstream = "{{ rgw_dnsmasq_cluster_ip.stdout | trim }}:53"
+    desired = {
+        "name": "ceph",
+        "zones": [zone],
+        "forwardPlugin": {"policy": "Random", "upstreams": [upstream]},
+    }
+
+    raw = subprocess.check_output(
+        ["oc", "-n", "openshift-dns", "get", "dns.operator/default", "-o", "json"],
+        text=True,
+    )
+    dns = json.loads(raw)
+    servers = list(dns.get("spec", {}).get("servers") or [])
+    changed = False
+    found = False
+    for i, server in enumerate(servers):
+        if zone in (server.get("zones") or []) or server.get("name") == "ceph":
+            found = True
+            if server != desired:
+                servers[i] = desired
+                changed = True
+            break
+    if not found:
+        servers.append(desired)
+        changed = True
+
+    if not changed:
+        print("unchanged")
+        sys.exit(0)
+
+    patch = {"spec": {"servers": servers}}
+    subprocess.run(
+        ["oc", "-n", "openshift-dns", "patch", "dns.operator/default", "--type=merge", "-p", json.dumps(patch)],
+        check=True,
+    )
+    print("updated")
+    PY
+  register: rgw_coredns_forward
+  changed_when: "'updated' in rgw_coredns_forward.stdout"
+
+- name: Render Certificate for RGW hostnames
+  template:
+    src: certificate_ceph_rgw.yaml.j2
+    dest: /tmp/jetbrew-ceph-rgw-certificate.yaml
+    mode: "0644"
+
+- name: Apply Certificate for RGW hostnames
+  shell: |
+    oc apply -f /tmp/jetbrew-ceph-rgw-certificate.yaml
+
+- name: Wait for RGW Certificate Ready
+  shell: |
+    oc wait -n {{ openstack_namespace }} certificate/{{ rgw_certificate_name }} \
+      --for=condition=Ready --timeout=300s
+  changed_when: false
+
+- name: Wait for RGW TLS secret
+  shell: |
+    oc -n {{ openstack_namespace }} get secret {{ rgw_certificate_secret }} \
+      -o jsonpath='{.data.tls\.crt}' | grep -q .
+  register: rgw_tls_secret
+  retries: 30
+  delay: 5
+  until: rgw_tls_secret.rc == 0
+  changed_when: false

ansible/roles/ceph-rgw-prep/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: Configure DNS and TLS for external RGW
+  include_tasks: includes/tls_prep.yml
+
+- name: Ensure external RGW VIP hostname in /etc/hosts
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    regexp: '.*\b{{ rgw_external_hostname | regex_escape }}\b.*'
+    line: "{{ rgw_external_vip }} {{ rgw_external_hostname }}"
+    state: present
+
+- name: Export TLS materials for Ceph RGW deploy
+  include_tasks: includes/export_tls.yml
+
+- name: Prepare Keystone for external RGW
+  include_tasks: includes/keystone_prep.yml

ansible/roles/ceph-rgw-prep/templates/certificate_ceph_rgw.yaml.j2

@@ -0,0 +1,15 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ rgw_certificate_name }}
+  namespace: {{ openstack_namespace }}
+spec:
+  duration: {{ rgw_certificate_duration }}
+  issuerRef:
+    group: cert-manager.io
+    kind: Issuer
+    name: {{ rgw_certificate_issuer }}
+  secretName: {{ rgw_certificate_secret }}
+  dnsNames:
+    - {{ rgw_internal_hostname }}
+    - {{ rgw_external_hostname }}

ansible/roles/ceph-rgw-prep/templates/dnsdata_ceph_rgw.yaml.j2

@@ -0,0 +1,17 @@
+apiVersion: network.openstack.org/v1beta1
+kind: DNSData
+metadata:
+  labels:
+    component: ceph-storage
+    service: ceph
+  name: {{ rgw_dnsdata_name }}
+  namespace: {{ openstack_namespace }}
+spec:
+  dnsDataLabelSelectorValue: dnsdata
+  hosts:
+    - hostnames:
+        - {{ rgw_internal_hostname }}
+      ip: {{ rgw_storage_vip }}
+    - hostnames:
+        - {{ rgw_external_hostname }}
+      ip: {{ rgw_external_vip }}

ansible/roles/ceph-rgw/defaults/main.yml

@@ -0,0 +1,32 @@
+---
+rgw_storage_vip: 172.20.0.24
+rgw_external_vip: 172.19.0.24
+rgw_frontend_port: 8080
+rgw_backend_port: 8082
+rgw_realm: default
+rgw_zone: default
+rgw_service_id: rgw
+rgw_ingress_service_id: rgw.default
+
+# Align with mon/mgr
+rgw_placement_count: 3
+
+storage_network: 172.20.0.0/16
+external_network: 172.19.0.0/16
+storage_network_prefix: "{{ storage_network.split('/')[1] }}"
+external_network_prefix: "{{ external_network.split('/')[1] }}"
+
+rgw_remote_workdir: /tmp/jetbrew-rgw
+rgw_spec_file: "{{ rgw_remote_workdir }}/rgw_spec.yaml"
+
+# Ingress needs one host slot per VIP
+rgw_ingress_monitor_port: 8999
+rgw_ingress_count: 2
+
+rgw_keystone_verify_ssl: false
+rgw_keystone_api_version: 3
+rgw_keystone_accepted_roles: "member, Member, admin"
+rgw_keystone_accepted_admin_roles: "ResellerAdmin, swiftoperator"
+rgw_keystone_admin_domain: default
+rgw_keystone_admin_project: service
+rgw_keystone_admin_user: swift