Migrate HPC Backend to svc_compose and /projects/CRBM

assets/dev/config/.dev_env_TEMPLATE

@@ -27,8 +27,8 @@ BATCH_SLURM_PARTITION=
 BATCH_SLURM_QOS=
 
 
-# The home compose_api dir for crbmapi
-INTERNAL_MOUNT_DIR=/mnt/crbmapi
+# The home compose_api dir for svc_compose
+INTERNAL_MOUNT_DIR=/mnt/svc_compose
 NAMESPACE=test
-SIMULATION_STORE_BASE_PATH=/home/crpbmapi
+SIMULATION_STORE_BASE_PATH=/projects/CRBM
 CONTAINER_SERVICE=APPTAINER

compose_api/common/hpc/models.py

@@ -52,17 +52,21 @@ def from_sacct_formatted_output(cls, line: str) -> "SlurmJob":
         job_state = fields[4]
         if "cancelled" in job_state.lower():  # Has 'cancelled by <User-ID>' which trips up mappings from string to Enum
             job_state = "CANCELLED"  # so just set it to 'canceled'
-        # Map fields to model attributes
+
+        # sacct emits "Unknown" / "N/A" / "" for fields that don't apply yet (e.g. start/end on PENDING jobs)
+        def _nullable(value: str) -> Optional[str]:
+            return None if value in ("", "Unknown", "N/A") else value
+
         return cls(
             job_id=int(fields[0]),
             name=fields[1],
             account=fields[2],
             user_name=fields[3],
             job_state=job_state,
-            start_time=fields[5],
-            end_time=fields[6],
-            elapsed=fields[7],
-            exit_code=fields[8],
+            start_time=_nullable(fields[5]),
+            end_time=_nullable(fields[6]),
+            elapsed=_nullable(fields[7]),
+            exit_code=_nullable(fields[8]),
         )
 
     @staticmethod

compose_api/config.py

@@ -79,8 +79,8 @@ class Settings(BaseSettings):
     assets_dir: str = f"{REPO_ROOT}/assets"
     marimo_api_server: str = ""
 
-    # Should mount to /home/FCAM/crbmapi/compose_api externally
-    internal_mount_dir: str = "/mnt/crpbmapi"
+    # Should mount to /projects/CRBM/compose_api externally
+    internal_mount_dir: str = "/mnt/projects/CRBM/compose_api"
     namespace: str = "test"
     containers_output_dir: str = "/experiment/output"
     container_service: str = "APPTAINER"

compose_api/simulation/hpc_utils.py

@@ -68,7 +68,7 @@ def get_correlation_id(random_string: str, job_type: JobType) -> str:
 
 
 def format_experiment_path(experiment_dirname: str, namespace: Namespace = Namespace.TEST) -> Path:
-    base_path = f"/home/FCAM/crbmapi/compose_api/{namespace}/sims"
+    base_path = f"/projects/CRBM/compose_api/{namespace}/sims"
     return Path(base_path) / experiment_dirname
 
 

compose_api/simulation/simulation_service.py

@@ -13,7 +13,6 @@
 from compose_api.common.hpc.slurm_service import SlurmService
 from compose_api.common.ssh.ssh_service import SSHService, get_ssh_service
 from compose_api.config import Settings, get_settings
-from compose_api.dependencies import get_required_database_service
 from compose_api.simulation.hpc_utils import (
     get_correlation_id,
     get_slurm_job_name,
@@ -195,6 +194,8 @@ async def build_container(self, simulator_version: SimulatorVersion, random_str:
                 remote_singularity_file=singularity_def_file,
             )
 
+            from compose_api.dependencies import get_required_database_service
+
             hpc_run = (
                 await get_required_database_service()
                 .get_hpc_db()
@@ -210,6 +211,8 @@ async def build_container(self, simulator_version: SimulatorVersion, random_str:
 
     @override
     async def download_container(self, remote_container_image: RemoteContainerImage) -> SimulatorVersion:
+        from compose_api.dependencies import get_required_database_service
+
         await get_ssh_service().download_container(remote_container_image=remote_container_image)
         return (
             await get_required_database_service()

kustomize/base/api.yaml

@@ -61,7 +61,7 @@ spec:
             - name: api-cache-pvc
               mountPath: /app/scratch
             - name: compose-api-home-pvc
-              mountPath: /home/FCAM/crbmapi
+              mountPath: /projects/CRBM
             - name: slurm-submit-key-file
               mountPath: /run/secrets/slurm_submit_key_file
             - name: ssh-known-hosts

kustomize/config/compose-api-local/shared.env

@@ -1,15 +1,15 @@
 #SLURM_SUBMIT_HOST=hamantis.cam.uchc.edu
 SLURM_SUBMIT_HOST=haproxy-ssh
-SLURM_SUBMIT_USER=crbmapi
+SLURM_SUBMIT_USER=svc_compose
 #SLURM_SUBMIT_KEY_PATH=/run/secrets/slurmsubmitkeyfile/ssh-privatekey
-SLURM_PARTITION=general
-SLURM_NODE_LIST=mantis-039
-SLURM_QOS=general
-SLURM_LOG_BASE_PATH=/home/FCAM/crbmapi/compose_api/dev/htclogs
+SLURM_PARTITION=vcell
+SLURM_NODE_LIST=
+SLURM_QOS=vcell-services
+SLURM_LOG_BASE_PATH=/projects/CRBM/compose_api/dev/htclogs
 
-HPC_IMAGE_BASE_PATH=/home/FCAM/crbmapi/compose_api/dev/images
-HPC_REPO_BASE_PATH=/home/FCAM/crbmapi/compose_api/dev/repos
-HPC_SIM_BASE_PATH=/home/FCAM/crbmapi/compose_api/dev/sims
+HPC_IMAGE_BASE_PATH=/projects/CRBM/compose_api/dev/images
+HPC_REPO_BASE_PATH=/projects/CRBM/compose_api/dev/repos
+HPC_SIM_BASE_PATH=/projects/CRBM/compose_api/dev/sims
 HPC_SIM_CONFIG_FILE=publish.json
 
 NATS_URL=nats://nats:4222

kustomize/config/compose-api-rke/api.env

@@ -1 +1 @@
-INTERNAL_MOUNT_DIR=/home/FCAM/crbmapi/compose_api
+INTERNAL_MOUNT_DIR=/projects/CRBM/compose_api

kustomize/config/compose-api-rke/shared.env

@@ -1,6 +1,6 @@
 #SLURM_SUBMIT_HOST=hamantis.cam.uchc.edu
 SLURM_SUBMIT_HOST=hamantis.cam.uchc.edu
-SLURM_SUBMIT_USER=crbmapi
+SLURM_SUBMIT_USER=svc_compose
 #SLURM_SUBMIT_KEY_PATH=/run/secrets/slurmsubmitkeyfile/ssh-privatekey
 SLURM_PARTITION=vcell
 SLURM_NODE_LIST=
@@ -9,7 +9,7 @@ SLURM_QOS=vcell-services
 BATCH_SLURM_PARTITION=vcell
 BATCH_SLURM_QOS=vcell-services
 
-SIMULATION_STORE_BASE_PATH=/home/FCAM/crbmapi/compose_api
+SIMULATION_STORE_BASE_PATH=/projects/CRBM/compose_api
 HPC_SIM_CONFIG_FILE=publish-uchc-hpc.json
 NAMESPACE=prod
 

kustomize/overlays/compose-api-local/compose-api-home-pv.yaml

@@ -10,7 +10,7 @@ spec:
   persistentVolumeReclaimPolicy: Retain
   storageClassName: nfs-csi
   nfs:
-    path: /home/FCAM
-    server: cfs09.cam.uchc.edu
+    path: /projects/CRBM
+    server: cfs15.cam.uchc.edu
   mountOptions:
     - nfsvers=3

kustomize/overlays/compose-api-local/compose-api-home-pvc.yaml

@@ -1,7 +1,6 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  # corresponds to the NFS mount of cfs07:/ifs/vcell to /share/apps/vcell7 on swarm nodes
   name: compose-api-home-pvc
 spec:
   accessModes:

kustomize/overlays/compose-api-rke-db-migration/secret-ghcr.yaml

@@ -7,7 +7,7 @@ metadata:
   namespace: compose-api-rke
 spec:
   encryptedData:
-    .dockerconfigjson: AgC46AppZujtPVRlR4pByBcIM8cJrGQ+YhM9WtCfTY4GoSVJUNaIE4AyjYhw3Z5fTi/tU+P5Fd3qLKK218OaTkohKdWwzRlSrHljAzETtdQA+Wo+fm9FGK+r0gfx2/G8+cVVF/z/tYMdG70ZnwMgnCp6Y41/TCVmOsO+zSFT1cnOtQKr/1pd/aRL0WS7zuAOnPnFihAhIPuVtFXM0+QuHV1QQUS8xlSu++1VKCB3wZfrwZE+PJqu0JV/LhVQOgwqxcK63Y/mvzlgWxaWcO1az9roc40I9BeFNmp+L0wSh3SlUMP4kSKSHn8IFKu7lu9k8Usu/HnZiAfgECHPPuYu1bn4qg/U/hBbpNwDEXVW3n1oKoueqHc6H7MblGJmItjoKLNbPlc3SkLqcczwxgo1yqyCbeeB068Y2pr44qWnOMi0DciM2YYeZCQcrUMbDt4/nIhudXpFISF4LYmEbcyHt24uVtWNBmnVx6tEbk7M+f36VS764DqBzbljbu8XOnst6St7N37lCW4rauuQ4b0XPH7n7L2YCDRmRGYwk4tW4D2puqQ+WzCcnW4Ibd2IDvG+DMHWucoKYiQgjWjjDEt9TIJenK4yWMPqOkftp0Q5fhL9D6myQJ5zVlzdcVBi9cULYt9KCei0SgmXUO5DQmSQ5tcjnJlQLRPxEzfiCj8+6jvZAxQol2CEP67Xn8yQPxNa+N/C1HlMWD2wxAFnepYApx/V08xd7PE5CxuRAWaf+2wX1ePVpiLpg81lcQx3+cy/vDEsb0IX6qoGq8skfts6tO2k27oTkqQEHUEOCCaO54Hys52DsYY9cGH5meOggM7IL74BlD1l2bsFXRh6Ik4Mhb/LltviNuwah7bYOv44RDbtBFW3cpm6K/+KJ/frgPJfE2YtOLdvpV4K3kIn+mCAbf0VClfXig0DAh0fHbat1+yvshiFgbJL3JmqvqPTmB3kuVaIw2Hl/Edz05q3L0qn9oNq
+    .dockerconfigjson: AgCjBaIbIJlBAQg2PFrqNFgL740v/DBf+9pbfztlCqJIhdPjKkRiSKiQQhlhzMfvaNW93CMLJdfv5ERz8cdfP/toGJGyETulsY8sAWt48GM5I6WsXJ7/SKkOy/StIJboh4gMK/wRQsvOhSou5W2dqMNxspNksnEbKsZ9KyTcwUeAW1I3esBCDolFcvf2/DRkC+oGVNTrN1lRXodewbKT/2Wn1wJL024nmxuzyak38fouDuN4FgnzNZw0X/boGak5dbJ9WH2CDmoD8Bl+jWKTb1yUdQzfLseYYUrxb2fGGD4SpfrxY/4McIxJ+2IgWH3aEEtmShrZYb5/uiaCnLrI3Ws8m3nNBFiI8oB+EfVGMI1lniUYvAxRdqeu7w45p6AKe8oV0NcdCPUQmAu/i+6YLDdH96fH8LiUNcan2QYrZHZHjjFPh43XML3gLfdGH6BBd/r2jxgcoOjrx/ymrEsVs/SGIf8s5y463gKHDjJNPy7n8MM2JBDkooxnQrzHDrjriM8KiL5h17mn9UyNJHRAq+/mV9LD20+4KZHf+BglsCRjxP5yR3QH8NpriW4fonYUBrneFvVtOe/O64IbA+fTA9Pg1HToCQUYJam10rgH0Y4ZcwQXF/zsq27EZmPHB7HyY0GmwzUtjL2IyKAqjgCVUsceEbkyHcUDwXzM13gz/ccbLyAc+bksxTz/NOqIV9kTGt+LgXXRhMZur+YBkwgbeAepOwyGBv4m3TZ3qC4mY48iN0elBPYGyMxsm2/LxvBqwJNpGCNcggAMtS3fNSxcacHojP8dkIGWPDLkQuSNnZMw+lJwxf6zIO0+Ufznn5cPkVJQVwa40poHStulWayDEoVl0YyNTgd+eD4QO1oaLZXKA8Qg4FEQzdjf7FMSMTtrGx/JBbMEi1bFzzlHL/oDTmfJQpFkNZdCmjPWJp0lEyI6SuFdmJVM7fvaT8ro9vjZmNMp/otka7FgJXZUt8RS3ZHS
   template:
     metadata:
       creationTimestamp: null

kustomize/overlays/compose-api-rke-db-migration/secret-shared.yaml

@@ -7,12 +7,12 @@ metadata:
   namespace: compose-api-rke
 spec:
   encryptedData:
-    postgres-database: AgBj9wdj0hMVwQ3HrFtDRmPFBT5Lmma9elpOLDk8A9sDFDfv3nbr5MuT9lh05mPndWAj+gJNJ/dzhzb3Qbr31QCbTMqe8U/mrPrmNiJAfeHp7B5dx0JpT/AE9FqPto6J+pb/Gz3mHwD1Ko+SvWTBGq6hfwn2TqmaeHkiQ/vV0tQjfqzIZDu+fzinFW1OtsYx1Umc07SKNi0wIaaQgjo1beEuOn3HnLaCivoWbTW4ppQEsVBjXEcducRLM56Ko7uV3NsnTeid7n0c1MhRsMKqjOiMYBvwUnN4fqXiwVntwq7nOWN5gfg+D4ndd9w7UgHJUQRVPreGgGy6t5E81OBIFR/RSgm7AR7bIUP1zCCBqPKSA6Ik+ZL2vyQH8o3+3tv6b7q9g9utGLqdQ+Cbn0HachfTB3qJWFxk99YVUBxsBYxK3rixR2L2Epv3ogS3aJAjaD9jK4qoTH90nySkIZw6MKXrN+3wSmdVjW4V2YEtTXd/08tpvEoczHlFet7qcaqPetcWZFS7zGB8lF2CwEFa6ilVWhCnbB9TPTp9zNEgXthVHQAyNbkuKOFokskzqPIe6y0DFwdRG8SSLEFSi2lEfEgZ2zOTotH6f0BX2zhaA5KeKdYhaFJGp6BEq3KGW/kyjzLsfN6m8zyQphviFLoewMdO0KLIJTxWHB/FUs1KkMcXRJdY/hbOhaT5E6Ns6J1fpRsMLl7n8HQd4YFu+Q==
-    postgres-host: AgByzOLirQ1SbSwIVx8WZcBhsuNx84Zu43w07mgtg55FB0KLA7wyDXxc3y7zP0K1n8Vx8hn35aP7OzyVFyWoKobhmsybFggOQO/8bBeczdvFRJ0pksLzvYw0XYOoxB99j0R6XimlEI8GxaN6Ay8oxrVYGn1VvBWXhSC4xGoFR+lg5yLBUilM5P+qq5U08ROVlcXd4q4ZxKo/Q56T/FcHhKc4pdc99zBVVcyGaOvPkRlE886POP+luIWdqa/2N9ZYchs4fbUt6imAL4PhWw1zIt8XAejGoiZwhtxfNMjgcytWcr8Sj0p9FA6+J82s6wAwAGHZ3y2rCoOYT9+DoQlzKu8jrk9WfsKdMb1RzNC6IIzFcQfKf679J9P86+UrXc+N3Z51paFiCfsqJaNNFY0EgDFOIDDWEkqC5CKTf7K7DyaW+gnWL5VTBNUdGs82TZaX2y2UCH+WsrO6RKoLxnUlYiwXS5/9RNhtU8uI3wQwro/zz4hs8V3LHjhgUAwTtqpvf7PL7x62+d0TtGaJ5Bft8lXQPvEl6TW4cha+hmzJAZw/ZvNDIivE1uRW3l721Y216FDAQ8/0yKsNlkCyPVf05EdCcYuI0Z33Jcn32WsoUrAe/fLyRBU2eURaw2hRhoDY26RcbsanghRfdtK2W7Dsh9n1fBVQo76JDm5BrWpts3WHpHgT4Zg7QlBkJGtzDzmOxloapxnplVgtGXM1bK1o/bBkHdY05l7iC3Fu/9QBsrFfvb/LelNkXuPcN7x1qvRdtb8pLHbwbv2yB9Ls0xxmqw==
-    postgres-password: AgA9VC0jx7+0h1fM68UsMGFNSPCOQpilkbcKWYOUwavguv91zwHhLEZ+eFZ4/o1VmiEHDM9jAdzuIjmvF5uM9fO4ruiecnANMrKle3CPYnf47RkwFZHKNtZNiDaAVlFAPpzUjbwbHMoI2leeKN652Jc23WWliiCSw0gkmCgcTAnJs6C669CaZZeCbOWcXg9QQ7ekl9/7pcfYibnbgxGiEkx2bh03zg1mTSM8Em7NiBADKV1p9XLWX9oY5Utqb9htEYIc+8O2BuYMlWM4fB+KWDU32CyUJ41kUZzCQ+ALpwW1WPVsXI/w2mTbK1gD9KxCPPWXfk5pQ4ylY26yg7loNOrJeNfh2qque72Dw1tu9VupvosdhjJNgoVnyarf27nOmYTn9GZmgjM0aAcON5cMJDJlambTLIldSLdL6ylTDIVMFAopZ6p3SFcvcDWXdhaX3JoIoMj0gUnHdAmafc4vh1vw/pnXFyaQYA3lht/2QX0DHq6HDg3DNE6aVY+likcMuEC8uXxOhxZzCFRA9bwXjcA+Gzit5CC4xivsr6rRkIHCAynjzaBCKmndUQf01FeZK559d2VJZz5CzOD8WXy7bjKwHESeyqDilijG6+isgl92wWHRCimTlALQg51bfbCBtPboL/vrKJegmeSiaL9kPeZJPj9iKqWAMtCO7hC97ALeKQ4wS9FFiFoSLcsNEXTcWz8B60Uazj3lttFbjdUOgNk5Dds2GDRYC2MYABrLAPga0+CQK2Yy6S39F2d9kJQJnrZp58JxXew3XCmQOkCxtCmG
-    postgres-port: AgABfMwP8FcCsIn0u1+jDIABpSQxJhgwfzrsHV5zo2slczU3PeqmLRy7UwLxpftZ+jRV0LlhR0gkfBcvM8P6ZHXmL557O9MsdJFF3dRxvv+cn2iPjfhPlLTANh7K2h7IFr4vd5gFZ+UQjmLo6gw9/38zfYOtA84wkupNe++KIkp7yhPb++pt9V8WFyplpGbx2nw+ExqE4jxCnd9xERNVAoHEhzvCzmFDGdxapwDpZKrU7tbANsE4znh7IDq+LloXvzECRfIHgrLmiTTi6J6esmYfPcGPdXRhVARD3kGzSaF25k24vACRuXSxtBG7zeKuH5PRz9bWWBkXpF5nRYOXuDWm94dqYdloVbAyPLnw0Vk/L0mcRqvtLUSiSH3Qh74CWxfE8N7SM54MU1EgFo4t6xHKGMkwpJ/tUF7+zlucVAB0t9EPoLCDtzyewM/Xp2c+Wj4yEmzBE3cTqoGgXNcS4bhWWgbCveLZw48nqCVnkUsKF/XZ0lildHzsPfqbBW+umvzzW2Mqye87XxtpA/Xu0/R2a1NQcZ2EXL+0+7z4tRKdrkzLAwnTB+G8R0ypdrbROpCePRMhrOwHGdK/zV0kOr9QX4gngCR9oERZQQlHY6F9S+ratOS68oNaG+NtlwxFvMqenvG+RTxJLt0o3T1t5TMQoAg9BvkJC3VoAFkajNdMZl0GU4Zr+xw19U070cztsEC/tS8A
-    postgres-uri: AgADYa6Ae5KSWHCCjxQeEmym/QZZPZm7UdqJmngZ7rAEcDwGf79Ovb6REovTgtg+EVgk+y19EE8/FFTGdVelOsYUb8igSBtQCnXL0p1Le/Zf5L0I+J11ngXoWY5ebMTQu2iXD+qidVD26xG70WjWksweRLanUl7SiqgkmXA9CizvNDixyBUHzt6hnCiLLxm8dwr0XI0pPDHbav3XlCt0V5TxG5g0iXDMmqAoVbMd4zAUL2bcvZ4pEFm5DWcQCSKZV50X5dopIoWv8wjmdrWrFOLQcEPoaKkZgotAvLj1xq1PofYG7EKLkTfmCEsWHkr8hC6froN1v1pN02PvR9uwyR06BwrxnDF6wRJPw41bakq44Gj1i5r+up2g7Lb3jyc8fOHmnW7YEOVLeeaBSCCLaE2Lq0xMPCRtgQDiwef97U4vrddOyf2Nuh/fl1Eir5jyEuNamiwkUzEHPuQFIV84KWCTcYu78RMo9VlDTbxo50L+2cz7BFzxpokPbb1xu9jfhoTLHO869IUX3rB87qgerZJWKbKhVGuk+kJ8Oa0nYI4BaA6oMdEStEaS9SNjnHoA5wEWAT13w35Y0yLjRJCZjGMaLakQH9A26qm7sp+hllac0tpVHHpQlHQsRDlUl/NkydKD47czZ2WHUZvoyggPNLYzKLuez5wS9IH8OptZ8vjFoC+LoQQriu+0WU2bbJEHcKH4SrUKRy/2PCgO0MjwnQXbs/ZBCa7ht5qCPRwxY7saMYhboImxMM8SGzkvKTXlNhxrID6GOXscmxXk/iZrxT6HC21PawECTsULnxJKWaAo9S7L6bnr2Tlk6UDUV0UYiE6EtCh8tlA661QoEBd7nKwYec8Y0TEa9wEaLUjg1Dy1Q962MMPxmBTyOrEiM1qXuTigrfuW361mq08kX300JpAMkgK4mhof9DIFwqm/7Ge1PMU=
-    postgres-user: AgAheN2FpKC39kpKVV/D9V7m988ejF6loz+52Y41H8cftDkS/YBFfPaGR86GUd+XXQbKtR4/ROiXmKJsI+Jxv8smSmLuqAWvSkCeGpOWGMNisNnjNurxVXXp6paOYXWAgfLVaiXqUwZDZNlhc1VyRFrFpGq9IlV3+DIqqBArEZ7ozztjEIs9dp2SmKbOjB183u6+/WrII53jJy7HwdPKiDaKoMBS+uKxQvk3pPKCdtjQTsvaKHV50QRwO4nxnI+SvbbaE9G2oD/X5viTZBFkMON3MCA6r/k1wJUKgeRkL9pgSIPL4bF45BnZOMb4m8Z7n+to7qyMDoZ6cdQyJLLXgW9aqv8wd8Mm8G7JSOEP2yyt0dqk+zCcrAFI9scxhIBE3UYmGPS1UW+gDcXvUrwdNso3XekT1nMYcJu7OpoqIPLdYk8pTAfdViMSM5KFAaW9tqQ9hpXihrinCmSLpNqJffFIlC9sI5rkZydiqnX63SIM+RgwXaeuz93imQ2BOAlFu1paP54IjnQKy3aA3BNyyGsvWLA2Osdl1F3aajkwcdeRGi8UfIlDiIWAHdK/NrNzQd23E7Yu+b/OsgDSZ37l6bWKtJ2Mrz3h486WGdon8b+FhOJUe8sJfwnbmZIPDquWFks+opuNWhP//3oMx1dyETKAglQhaoznxuq1OG+cvZQUKaf3XZuFY2FItj/CYlQKmfui/zJqS84uJ4O0Sg==
+    postgres-database: AgCtQWXDkFnVejlLewM0mbVUuhSZJC0LSNz8Lz5M33mxJHqFKcC6dkVZE2G/mL5dIt3LkQy2MrSWPgOIsOH1H9oRstm1Qbp/W5AGtFvs32GYVg/QLdddf7dCaurD0SF3JICYYVkhT/rZF7qr9jwXA9db5LgozCEfHO69JuBrVg0tB8+EEdsvjrP3w+Sp12RmChOZuciaovPhNtJK7O5WEdSQejafv0RVGL6RNcvdGSSYg2FywFdQNNg/+ud2ykG4t8kC3Sg+WJVt2emBJq5RNjAkYA2ShOC9xIx7z4iTw0glxHCymr1bg53V6GObDIIJAoinZx/EOJcMWkfJmbMxEghu//z/ZiswqwUNaG7PKtRuMNRvKk75V17JWVCO1SkxKDlTXPGvGckk98DdPc83QTAvA/8LsnvA2VQL+w76mxHNrm8OYUfr1WEDGp/MqlAVYkfW21NKoPoDjGHXimSaVGctoSzS34iDltbm1KXEnCqEY9Slxp8vnohLzhUGcpHmhgB6MDtUegSZBgVO66rJpYT4Kv+GfT/y5D4XLgma38qaDiy/nxt/p7TWQJy8YHCOKHeXbr+jwaJmft7nNKlmVDlAGOLdceSlN7o1f/5/4TwTIzrFDzqtkGlKW3nN1QgYGpj22rFhrFa+p4kuRge1jQ9vcolzC/iaPAkAaXRZe+/Gz4Rjkg8gs1FWiW63vKdxc6PZ9xal47NyxrOLeQ==
+    postgres-host: AgBMVNa9m4dXzp3rongt9TM7diYnJseGivo7vDJQeYc31MkHGdd4wEZ9cGmEgJjflYFASFBDJQ2jAaHBCbScybz2P+ZDsJ1p+W5kaSTDUxZCCSta6bTyhxnCtrkPmEKZg8m7mnjg10OjfNLSj/InBKfasXrORoKIx7ZKsGyRfZTvnMWgjKGbEDjlH1K12iink8t8xqSmtObkLSQZXa6ZA11Ul06IHIBpCmec5Q/Xf5F2H9xOU9n5nl/ddhjchd2DMsi4J0HX+nS4g47NMHlVpAcdLoqV7E8/0CgFv5BB/kOQq4WB9KstL+Aqf0IdKsI73ewB14XP+1EosVw9EwkXQYuTR5WKPKioQ7oXNL7w7PIRshZu0vTY/TjuicT1Uy4LfBOOqTDlwzCkjeNKf5n7t+gHT0dhak5o3kOuM3okc8COQSeLY7CHckNR4rkHvsLPNPz8Fr/4D3MCagtuqDfrk+/IcDnsM0XRmqia5oaWoM6tkr8gt6G3cZh34tLbuwuAZJ1srlnU9FZ/a4e8CcWi+z+aktZWktD3RL5fogRvoiKBTq3GrL6I2qNksHQCW3gZr2DrExwpOijH8QiZr9+JovHR3/uSm9fAcpS7CyVYZ623PW2LERvobpiVe40oB0HdfL+MsCEE5M2VN3UNxKB9PVXKTcIKoGreSJR4TSUpaLxXkswPRvzOAhmp1bKLYxg04rzaIeb5qIdfKYEZ6bS+5gEnFuuuGxi23/83jvkScaOwNELpo3S2XIYDS3v1un+s4K7NFzN/YiSz2e/gqXkf3Q==
+    postgres-password: AgCsL/Fyed3T7tsc78IzBU+dxi4XSp7poM1al6ojXtHVFBIw8WaViyBxGq6JSe5xx8Opb+X9ogocjNrJG2f1oYcf3IJzAWVJP2q2IMxxt/SBzg/KHgDQ3YXsoXm7egUorQb3+uovA58rfoi+VhcgRzpuOvZnfFpnIuiLYOn9jc/UF3A3KUGZ3Vc64y/4jvD4gz64HKPRl5jvJvwaLM/910rtfHsB+4sFxRIKbfBLWgs3TS3mTmFdGfA4rJ7+MiadtZHFnFYo2YdjbyPLS/emBx+hlJ5DtsQpVGAaEUWsiOob7wXu01fmnnUlWXfuDSodxUnpuxkKI2jrwSmEjliABlYLsPCX9AyEmTbE9ENcXtD26JoJCAyb0YzbPJBlrofvvFuOdTMrebu8rVp2CeYYQABOiGogjIGcrzANUNFBnhMoZR+BLrhd9hqwvtHEPwiXcCHK0WcTDkdT6CSfGJ+jgIIuAu1MfuGohadQZMRHTrjTlQdPV5ZPJX06yApiUMnJEHJZKY7issZZLKW4nUKzLcRWx3TY6Ob7xH5M2khAoCedp0h10Hh8tunyYojy+NCsfJPSYMwliy+xKF2M5ajbpTQbTlsv0vRnvJZxjj4fBOMrNHj1JmIdDLJm0Q9Wi6p6C14IqxxqqaaJVpjaQg9EdDaXHKJUnqTsUaAvwPKwe10eqAv1992FRd0CezZgkn41hMsl7RxOtUojk4gez5+EqN9Wmb9ZUNmC/bmgWFJPRfKF8iC8+jNM3nkoW9Q7WDPpJI3cHJPZ1KQM+L37m2rcEkrZ
+    postgres-port: AgBGP5ZHukGx4e+Sg3VQQHTjexIB3WivLeEY2ReRtOffQcpS1ZNd589SFJ9bEA6oOkK/rR3NZaJRM/n5tt+t3ZipzUIUwmirvrnOauTuJuulHwUOvkXruFLOis9bW2tvLP1sJLtcyQXJxNA5OnKwOcft6dpqO0pk8wFXjKJNtcdw9eSnQkWBINF4d3x0iwKCS9eZQc8q85DuoOnvAbXHQ0nPTmF7w/KrdVR8cOGhXXJ2N9gcuF3Wfa12DZM7M64qTK659uYC27jLTr+zUKokTxquIlJaQ3NtE3+4hAy85NZRTkrjKB1OUqia3zvo7XtxXfjINYC4mX1s1OPPUarzpi2yp33BxDAtYjhzxBTtPr+2yqGKU8wCGqmzzUvUbC/nE0Zch0XNWB+gwZTFNgXuSQMP7N2+bzsuLAkqb6TjqPMaCZHgkPH/wwJ9aWMaXW2AGi64Ajfm9kJzb3WXi11lEjhCMjF6JqsRdGFu0zixtS6cGFuNF8T3+Bz4ljPuiHlZ7InEz9EfPMdhOvcrNwEmVv4PWUJnEAn69KQamcAAG4f134fuus7RpMKFa4vuxX5xIDcGqFgFRTP4cnUGHBkAdxTqMWb9OIVoIhFSUaRiFPAUHZSNnl2dbt1JQzij8hz0CifQFZMmpDQpLQZ/f3SqZ+PRPne0OvH1ty3JnbCmQgeHaU/UUcZS7Tkn1xTv1T8++AeMqzVe
+    postgres-uri: AgAHbtNvsQR4qolSwTjWTCG8xfYX6ZtVuhfR5L/I5XsqfZeByYyWfJD9SQrxWGOdQ51Ar8V10OiDEYBzQ1JCOG8NAnt3DzgfwIeK/76DQ5meBusu7SuHWeovZKkDSUSaYZS7U4uzjFCqZgywVe30fVYMuTjOO/YspbyZKxT+o0AWHz7Nn6Iksv3jk0/65qaSrjsl4ymzAXUeHDqEjfjtxVNBPFx1coGlbjCCE1yXa1OrMVhIN0/q54i73yR9kvw+Y8Lg1iSqphNkDc6haZWS4wedi6nBxPgi8oQQc1kihHJoUIpf1jUJHEiF3ils+1s6i5xfqzdUlhSI+lQrvwrwQNG5qqt9uOB7PAQJsl39hprwRgLegOg8jfbQqXpVw0lINHU8FLfRKpiPYzabOmllM8ogQcZG7GpI1uSkNLZU+4doSsx8WN/0tLm/GsZguhD254lpCTYtYsHfxALOgww3dHm+OsHLLTdJukrVGb42YwktKNcWTDAc7gPTkv4DL1UrUA2+k4Zu6FmHA+s8A2Tf6BudXLMoAc/5w2JSU/SoO0vcAatvXq+ibfmsrcoohKkyra0VDxLkAcBKunrMkAushJ6On6Ybk4nDaypluFZG1MCXKJp1yFgpZDUz6fxt8gv76nwvxGKrN81GN7gm/AI+Z5F9a1EzPpfhO8ghaIoDe6IwLah6h4027R1HwfOcutLSDQatoxL6eFHWmU3N+JUX42iSkSPFcgBdeDUt193gkBng1czuqNyY9q2dTtEnTVQ5ZzOUjmE6y9RseivREgtvLBECjo5LsDsg7StZPBCoB0qVGG8GutxhVgrHtV8D94LXJPqmANsnZMGqh04xpTyb11qujyGg1O9jWlQYspUMjiePriifGeL68jodH4CNcrio4GYHPj2gSryKeoEG+eRO2/t0478/hZohuVYTW3/3vNrXFkc=
+    postgres-user: AgBGI5iNglBNqELOxNslr6hSuLqlsNAeIaZKu4c+RXYwEqsV0xlsGSz5cNcpOk3r9byW520ImHlpClpH/AdMYdKFJrYyGs0Jk6lvLeL8Lhkr5HvuCIYogxrREj0AoeiV5OXn2X4GHULq7urovCqYPjeUGE8oCS27Cg+/WJFwGV7sIiFdQ+zeWCBOPoo1WBPi5Bxjdyvymhgf1/SS1Uik6heYLfRDnktj3aABNgsqysmykECfT2OvTxjlF8AG/CudqBvOKg0Bc0ZzRa16KtiTO+WTkbxPBV73iZtD8uWRn8vCBdozUL6IfWpY1DbDcJzTsS7F2Xtu+upVo7zcabRwgAwen5XUEGo+ZAmG6paM7scS4Y+CXfuwjEduCroXMPpsi1RQ19+IokXZkCJu4lMTIWXIMG6hKYUHrhW2bGk2r2JQYj3ml1peDLRWw2XeGBL5/nIAHnKhREXAx/TksRfKRmZwuePry54rCaU5pUXtww/Wp81Jn5VRp/NW6qTEuwbzFIH6d16NCilSuKy0nuPn+2Pe0Vnv8K6V7f4eopRXHQjJYJuGZALJUiIicLbrS30CFc8P/theWYesrb5QjsulaOTeMc8KsZyW9Sr5sBw94/UEDwlzsnyfGrzqsGIX3dG27KJ/wR3EtKYFZFXJNJ+leXXU1qUueNI1vNIgHkVYVsqZaMheVvUGqGiJDYDnuskIav5fr+M/wqA0HEszFg==
   template:
     metadata:
       creationTimestamp: null

kustomize/overlays/compose-api-rke/compose-api-home-pv.yaml

@@ -10,7 +10,7 @@ spec:
   persistentVolumeReclaimPolicy: Retain
   storageClassName: nfs-csi
   nfs:
-    path: /home/FCAM/crbmapi
-    server: cfs09.cam.uchc.edu
+    path: /projects/CRBM
+    server: cfs15.cam.uchc.edu
   mountOptions:
     - nfsvers=3

kustomize/overlays/compose-api-rke/compose-api-home-pvc.yaml

@@ -1,7 +1,6 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  # corresponds to the NFS mount of cfs07:/ifs/vcell to /share/apps/vcell7 on swarm nodes
   name: compose-api-home-pvc
 spec:
   accessModes: