diff --git a/tests/templates/kuttl/ldap/30-test-opensearch.yaml b/tests/templates/kuttl/ldap/30-test-opensearch.yaml index a720d960..fc1798a3 100644 --- a/tests/templates/kuttl/ldap/30-test-opensearch.yaml +++ b/tests/templates/kuttl/ldap/30-test-opensearch.yaml @@ -16,7 +16,7 @@ spec: - -c args: - | - pip install opensearch-py==3.1.0 + pip install opensearch-py==3.2.0 python scripts/test.py env: # required for pip install diff --git a/tests/templates/kuttl/repository-azure-plugin/00-patch-ns.yaml b/tests/templates/kuttl/repository-azure-plugin/00-patch-ns.yaml new file mode 100644 index 00000000..d4f91fa5 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/00-patch-ns.yaml @@ -0,0 +1,15 @@ +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl patch namespace $NAMESPACE --patch=' + { + "metadata": { + "labels": { + "pod-security.kubernetes.io/enforce": "privileged" + } + } + }' + timeout: 120 diff --git a/tests/templates/kuttl/repository-azure-plugin/01-rbac.yaml b/tests/templates/kuttl/repository-azure-plugin/01-rbac.yaml new file mode 100644 index 00000000..64eced8c --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/01-rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-service-account +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - privileged + verbs: + - use +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role-binding +subjects: + - kind: ServiceAccount + name: test-service-account +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-role diff --git a/tests/templates/kuttl/repository-azure-plugin/02-assert.yaml.j2 b/tests/templates/kuttl/repository-azure-plugin/02-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/02-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/repository-azure-plugin/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/repository-azure-plugin/02-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/02-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/repository-azure-plugin/03-truststore.yaml b/tests/templates/kuttl/repository-azure-plugin/03-truststore.yaml new file mode 100644 index 00000000..2d55c6d4 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/03-truststore.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: TrustStore +metadata: + name: truststore-pem +spec: + secretClassName: tls + format: tls-pem + targetKind: ConfigMap diff --git a/tests/templates/kuttl/repository-azure-plugin/10-assert.yaml b/tests/templates/kuttl/repository-azure-plugin/10-assert.yaml new file mode 100644 index 00000000..6281c948 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/10-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: azurite +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/repository-azure-plugin/10-install-azurite.yaml b/tests/templates/kuttl/repository-azure-plugin/10-install-azurite.yaml new file mode 100644 index 00000000..4263e321 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/10-install-azurite.yaml @@ -0,0 +1,99 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: azurite + labels: + app.kubernetes.io/name: azurite +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: azurite + serviceName: azurite + template: + metadata: + labels: + app.kubernetes.io/name: azurite + spec: + containers: + - name: azurite + image: mcr.microsoft.com/azure-storage/azurite:3.37.0 + command: + - azurite-blob + - --location + - /data + - --disableTelemetry + # The account name is in the URL path, not in the FQDN hostname. + - --disableProductStyleUrl + - --blobHost + - 0.0.0.0 + - --cert + - /tls/tls.crt + - --key + - /tls/tls.key + ports: + - containerPort: 10000 + name: blobs + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + privileged: false + volumeMounts: + - name: storage + mountPath: /data + - name: tls + mountPath: /tls + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + fsGroup: 1000 + serviceAccountName: test-service-account + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=azurite + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: azurite +spec: + selector: + app.kubernetes.io/name: azurite + ports: + - port: 10000 + targetPort: 10000 +--- +apiVersion: v1 +kind: Secret +metadata: + name: azurite-credentials +stringData: + # The default storage account of Azurite, see https://github.com/Azure/Azurite#default-storage-account. + AZURITE_ACCOUNT_NAME: devstoreaccount1 + AZURITE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== diff --git a/tests/templates/kuttl/repository-azure-plugin/11-assert.yaml b/tests/templates/kuttl/repository-azure-plugin/11-assert.yaml new file mode 100644 index 00000000..65a42029 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/11-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-azure-storage-container +status: + succeeded: 1 diff --git a/tests/templates/kuttl/repository-azure-plugin/11-create-azure-storage-container.yaml b/tests/templates/kuttl/repository-azure-plugin/11-create-azure-storage-container.yaml new file mode 100644 index 00000000..c2d9ddd6 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/11-create-azure-storage-container.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-azure-storage-container +spec: + template: + spec: + containers: + - name: create-azure-storage-container + image: mcr.microsoft.com/azure-cli:2.91.0 + command: + - az + - storage + - container + - create + - --name + - opensearch-remote + envFrom: + - secretRef: + name: azurite-credentials + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: AZURE_STORAGE_CONNECTION_STRING + value: "\ + DefaultEndpointsProtocol=https;\ + AccountName=$(AZURITE_ACCOUNT_NAME);\ + AccountKey=$(AZURITE_ACCOUNT_KEY);\ + BlobEndpoint=https://azurite.$(NAMESPACE).svc.cluster.local:10000/$(AZURITE_ACCOUNT_NAME);\ + " + - name: REQUESTS_CA_BUNDLE + value: /tls/ca.crt + # The home directory of the user is not writable, so let the Azure + # CLI store its configuration in the writable /tmp directory. + - name: AZURE_CONFIG_DIR + value: /tmp/.azure + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + privileged: false + volumeMounts: + - name: tls + mountPath: /tls + # The Azure CLI requires a writable temporary directory, also for + # its configuration (see AZURE_CONFIG_DIR). + - name: tmp + mountPath: /tmp + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + fsGroup: 1000 + serviceAccountName: test-service-account + volumes: + - name: tls + configMap: + name: truststore-pem + - name: tmp + emptyDir: {} + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/repository-azure-plugin/20-assert.yaml b/tests/templates/kuttl/repository-azure-plugin/20-assert.yaml new file mode 100644 index 00000000..7e0d3b46 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/20-assert.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-nodes-cluster-manager +status: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-nodes-data +status: + readyReplicas: 2 + replicas: 2 diff --git a/tests/templates/kuttl/repository-azure-plugin/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/repository-azure-plugin/20-install-opensearch.yaml.j2 new file mode 100644 index 00000000..0bc25f6f --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/20-install-opensearch.yaml.j2 @@ -0,0 +1,248 @@ +--- +apiVersion: opensearch.stackable.tech/v1alpha1 +kind: OpenSearchCluster +metadata: + name: opensearch +spec: + image: +{% if test_scenario['values']['opensearch'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opensearch'].split(',')[1] }}" +{% endif %} + productVersion: "{{ test_scenario['values']['opensearch'].split(',')[0] }}" + pullPolicy: IfNotPresent + clusterConfig: + keystore: + - key: azure.client.azurite.account + secretKeyRef: + name: azurite-credentials + key: AZURITE_ACCOUNT_NAME + - key: azure.client.azurite.key + secretKeyRef: + name: azurite-credentials + key: AZURITE_ACCOUNT_KEY + security: + settings: + config: + managedBy: API + content: + value: + _meta: + type: config + config_version: 2 + config: + dynamic: + authc: + basic_internal_auth_domain: + description: Authenticate via HTTP Basic against internal users database + http_enabled: true + transport_enabled: true + order: 1 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + authz: {} + http: + anonymous_auth_enabled: true + internalUsers: + managedBy: API + content: + valueFrom: + secretKeyRef: + name: security-config-file-internal-users + key: internal_users.yml + roles: + managedBy: API + content: + valueFrom: + configMapKeyRef: + name: security-config + key: roles.yml + rolesMapping: + managedBy: API + content: + valueFrom: + configMapKeyRef: + name: security-config + key: roles_mapping.yml +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + discoveryServiceListenerClass: external-unstable + roleGroups: + cluster-manager: + config: + discoveryServiceExposed: true + nodeRoles: + - cluster_manager + resources: + cpu: + min: 500m + storage: + data: + capacity: 100Mi + replicas: 3 + data: + config: + discoveryServiceExposed: false + nodeRoles: + - ingest + - data + - remote_cluster_client + resources: + cpu: + min: 500m + storage: + data: + capacity: 200Mi + replicas: 2 + podOverrides: + spec: + initContainers: + - name: init-system-keystore +{% if test_scenario['values']['opensearch'].find(",") > 0 %} + image: "{{ test_scenario['values']['opensearch'].split(',')[1] }}" +{% else %} + image: oci.stackable.tech/sdp/opensearch:{{ test_scenario['values']['opensearch'].split(',')[0] }}-stackable{{ test_scenario['values']['release'] }} +{% endif %} + command: + - update-ca-trust + args: + - extract + - --output + - /stackable/ca-trust + volumeMounts: + - name: system-trust-store + mountPath: /stackable/ca-trust + readOnly: false + - name: azurite-ca + mountPath: /etc/pki/ca-trust/source/anchors/azurite-ca.crt + subPath: ca.crt + readOnly: true + containers: + - name: opensearch + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: AZURITE_ACCOUNT_NAME + valueFrom: + secretKeyRef: + key: AZURITE_ACCOUNT_NAME + name: azurite-credentials + volumeMounts: + - name: system-trust-store + mountPath: /etc/pki/java/cacerts + subPath: java/cacerts + readOnly: true + volumes: + - name: azurite-ca + configMap: + name: truststore-pem + - name: system-trust-store + emptyDir: + sizeLimit: 10Mi + configOverrides: + opensearch.yml: + jsonMergePatch: + azure: + client: + azurite: + # The plugin appends this value to the connection string of the Azure SDK as + # `;EndpointSuffix=`, see `AzureStorageSettings.buildConnectString`. As + # there is no setting for the endpoint URL of Azurite, `BlobEndpoint` is injected + # here. `ignored` is just a placeholder which terminates `EndpointSuffix=`. It is + # never used because an explicit `BlobEndpoint` takes precedence over an endpoint + # derived from the account name and the endpoint suffix. + endpoint_suffix: "\ + ignored;\ + BlobEndpoint=https://azurite.${NAMESPACE}.svc.cluster.local:10000/${AZURITE_ACCOUNT_NAME}\ + " + node: + attr: + remote_store: + segment: + repository: azurite + translog: + repository: azurite + state: + repository: azurite + repository: + azurite: + type: azure + settings: + client: azurite + container: opensearch-remote + base_path: remote-store + store: + # Disable memory mapping in this test; If memory mapping were activated, the kernel + # setting vm.max_map_count would have to be increased to 262144 on the node. + allow_mmap: false + cluster: + indices: + replication: + # Remote-backed storage requires segment replication. + strategy: SEGMENT + remote_store: + state: + # Publish the cluster metadata to the remote repository. + enabled: true + # Disable the disk allocation decider in this test; Otherwise the test depends on the + # disk usage of the node and if the relative watermark set in + # `cluster.routing.allocation.disk.watermark.high` is reached then the security index + # could not be created even if enough disk space would be available. + routing.allocation.disk.threshold_enabled: false +--- +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-credentials +stringData: + OPENSEARCH_USER: admin + OPENSEARCH_PASSWORD: AJVFsGJBbpT6mChn +--- +apiVersion: v1 +kind: Secret +metadata: + name: security-config-file-internal-users +stringData: + internal_users.yml: | + --- + _meta: + type: internalusers + config_version: 2 + + admin: + hash: $2y$10$xRtHZFJ9QhG9GcYhRpAGpufCZYsk//nxsuel5URh0GWEBgmiI4Q/e + reserved: true + backend_roles: + - admin + description: OpenSearch admin user +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: security-config +data: + roles.yml: | + --- + _meta: + type: roles + config_version: 2 + roles_mapping.yml: | + --- + _meta: + type: rolesmapping + config_version: 2 + + all_access: + reserved: false + backend_roles: + - admin diff --git a/tests/templates/kuttl/repository-azure-plugin/30-assert.yaml b/tests/templates/kuttl/repository-azure-plugin/30-assert.yaml new file mode 100644 index 00000000..bebbaa96 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/30-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-opensearch +status: + succeeded: 1 diff --git a/tests/templates/kuttl/repository-azure-plugin/30-test-opensearch.yaml b/tests/templates/kuttl/repository-azure-plugin/30-test-opensearch.yaml new file mode 100644 index 00000000..78d08b77 --- /dev/null +++ b/tests/templates/kuttl/repository-azure-plugin/30-test-opensearch.yaml @@ -0,0 +1,146 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-opensearch +spec: + template: + spec: + containers: + - name: test-opensearch + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + command: + - /bin/bash + - -euxo + - pipefail + - -c + args: + - | + pip install opensearch-py==3.2.0 + python scripts/test.py + env: + # required for pip install; the root filesystem is read-only, so + # point the home directory at the writable /tmp directory. + - name: HOME + value: /tmp + envFrom: + - configMapRef: + name: opensearch + - secretRef: + name: opensearch-credentials + volumeMounts: + - name: script + mountPath: /stackable/scripts + - name: tls + mountPath: /stackable/tls + # pip installs the packages into the home directory (see HOME) and + # Python requires a writable temporary directory. + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 128Mi + cpu: 400m + volumes: + - name: script + configMap: + name: test-opensearch + - name: tls + configMap: + name: truststore-pem + - name: tmp + emptyDir: {} + serviceAccountName: test-service-account + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + fsGroup: 1000 + restartPolicy: OnFailure + backoffLimit: 10 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-opensearch +data: + test.py: | + import os + from opensearchpy import OpenSearch + + INDEX_NAME = 'test-index' + + + class OpenSearchTestClient: + """Provides the OpenSearch operations required by this test.""" + + def __init__(self, index_name): + self.index_name = index_name + self.client = OpenSearch( + hosts=[{ + 'host': os.environ['OPENSEARCH_HOSTNAME'], + 'port': os.environ['OPENSEARCH_PORT'], + }], + http_auth=( + os.environ["OPENSEARCH_USER"], + os.environ["OPENSEARCH_PASSWORD"] + ), + http_compress=True, + use_ssl=os.environ['OPENSEARCH_PROTOCOL'] == 'https', + verify_certs=True, + ca_certs='/stackable/tls/ca.crt' + ) + + def create_index(self): + response = self.client.indices.create(index=self.index_name) + print(f'Index created; {response=}') + + def delete_index(self): + response = self.client.indices.delete(index=self.index_name, ignore_unavailable=True) + print(f'Index deleted; {response=}') + + def add_document(self): + response = self.client.index( + index=self.index_name, + body={ + 'name': 'Stackable' + }, + id=1, + # Segments are only uploaded to the remote store on a refresh and the remote store refresh + # listener uploads them synchronously, so the upload is finished when this call returns. + refresh=True + ) + print(f'Document added; {response=}') + + def bytes_uploaded_to_remote_store(self): + stats = self.client.indices.stats(index=self.index_name, metric='segments') + segments = stats['indices'][self.index_name]['total']['segments'] + return segments['remote_store']['upload']['total_upload_size']['succeeded_bytes'] + + + client = OpenSearchTestClient(INDEX_NAME) + + # The Job is retried on failure, so delete a left-over index from a previous attempt. + client.delete_index() + + client.create_index() + + uploaded_bytes_before = client.bytes_uploaded_to_remote_store() + client.add_document() + uploaded_bytes_after = client.bytes_uploaded_to_remote_store() + + assert uploaded_bytes_after > uploaded_bytes_before, \ + 'The segments of the added document were not uploaded to the remote store' + + client.delete_index() diff --git a/tests/templates/kuttl/security-config/20-test-initial-security-config.yaml b/tests/templates/kuttl/security-config/20-test-initial-security-config.yaml index c41667b3..e9f57bad 100644 --- a/tests/templates/kuttl/security-config/20-test-initial-security-config.yaml +++ b/tests/templates/kuttl/security-config/20-test-initial-security-config.yaml @@ -16,7 +16,7 @@ spec: - -c args: - | - pip install opensearch-py==3.1.0 + pip install opensearch-py==3.2.0 python scripts/test.py env: # required for pip install diff --git a/tests/templates/kuttl/security-config/22-test-updated-security-config.yaml b/tests/templates/kuttl/security-config/22-test-updated-security-config.yaml index de8e9fd5..c97c736f 100644 --- a/tests/templates/kuttl/security-config/22-test-updated-security-config.yaml +++ b/tests/templates/kuttl/security-config/22-test-updated-security-config.yaml @@ -16,7 +16,7 @@ spec: - -c args: - | - pip install opensearch-py==3.1.0 + pip install opensearch-py==3.2.0 python scripts/test.py env: # required for pip install diff --git a/tests/templates/kuttl/security-disabled/20-test-opensearch.yaml b/tests/templates/kuttl/security-disabled/20-test-opensearch.yaml index d878a179..af425904 100644 --- a/tests/templates/kuttl/security-disabled/20-test-opensearch.yaml +++ b/tests/templates/kuttl/security-disabled/20-test-opensearch.yaml @@ -16,7 +16,7 @@ spec: - -c args: - | - pip install opensearch-py==3.1.0 + pip install opensearch-py==3.2.0 python scripts/test.py env: # required for pip install diff --git a/tests/templates/kuttl/smoke/20-test-opensearch.yaml.j2 b/tests/templates/kuttl/smoke/20-test-opensearch.yaml.j2 index f76d04f0..d77cda68 100644 --- a/tests/templates/kuttl/smoke/20-test-opensearch.yaml.j2 +++ b/tests/templates/kuttl/smoke/20-test-opensearch.yaml.j2 @@ -16,7 +16,7 @@ spec: - -c args: - | - pip install opensearch-py==3.1.0 + pip install opensearch-py==3.2.0 python scripts/test.py env: # required for pip install diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index f178a936..3e6b97d1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -67,6 +67,12 @@ tests: dimensions: - opensearch - opensearch_home + # The test case "repository-azure-plugin" does not work with the original image because it + # requires the repository-azure plugin. + - name: repository-azure-plugin + dimensions: + - opensearch + - release suites: - name: nightly patch: