Expand app configuration through extension specifications

packages/app/src/cli/models/app/loader.test.ts

@@ -1,4 +1,5 @@
 import {
+  formatConfigurationError,
   getAppConfigurationShorthand,
   getAppConfigurationFileName,
   loadApp,
@@ -15,12 +16,16 @@ import {ExtensionInstance} from '../extensions/extension-instance.js'
 import {configurationFileNames, blocks} from '../../constants.js'
 import metadata from '../../metadata.js'
 import {loadLocalExtensionsSpecifications} from '../extensions/load-specifications.js'
-import {ExtensionSpecification} from '../extensions/specification.js'
+import {createConfigExtensionSpecification, ExtensionSpecification} from '../extensions/specification.js'
+import {BaseSchemaWithoutHandle} from '../extensions/schemas.js'
 import {getCachedAppInfo} from '../../services/local-storage.js'
 import use from '../../services/app/config/use.js'
+import {writeAppConfigurationFile} from '../../services/app/write-app-configuration-file.js'
+import {remoteAppConfigurationExtensionContent} from '../../services/app/select-app.js'
 import {WebhooksSchema} from '../extensions/specifications/app_config_webhook_schemas/webhooks_schema.js'
 import {WebhooksConfig} from '../extensions/specifications/types/app_config_webhook.js'
-import {Flag} from '../../utilities/developer-platform-client.js'
+import {AppModuleVersion, Flag} from '../../utilities/developer-platform-client.js'
+import {unifiedConfigurationParserFactory} from '../../utilities/json-schema.js'
 import {describe, expect, beforeEach, afterEach, beforeAll, test, vi} from 'vitest'
 import {installNodeModules, PackageJson} from '@shopify/cli-kit/node/node-package-manager'
 import {inTemporaryDirectory, moveFile, mkdir, mkTmpDir, rmdir, writeFile} from '@shopify/cli-kit/node/fs'
@@ -29,6 +34,7 @@ import {platformAndArch} from '@shopify/cli-kit/node/os'
 import {zod} from '@shopify/cli-kit/node/schema'
 import colors from '@shopify/cli-kit/node/colors'
 import {showMultipleCLIWarningIfNeeded} from '@shopify/cli-kit/node/multiple-installation-warning'
+import {getPathValue} from '@shopify/cli-kit/common/object'
 
 vi.mock('../../services/local-storage.js')
 vi.mock('@shopify/cli-kit/node/system')
@@ -2198,6 +2204,313 @@ describe('load', () => {
     ])
   })
 
+  describe('configuration expansion', () => {
+    test('creates and validates expanded modules without knowing the specification', async () => {
+      const specification = createConfigExtensionSpecification({
+        identifier: 'custom',
+        schema: BaseSchemaWithoutHandle.extend({entries: zod.array(zod.object({handle: zod.string()}))}),
+        expandConfig: (config) => config.entries.map((entry) => ({entries: [entry]})),
+        getIdentity: (config) => ({handle: config.entries[0]!.handle, uid: config.entries[0]!.handle}),
+        transformConfig: {
+          forward: (config) => ((config as {entries: {handle: string}[]}).entries[0]!.handle === 'omit' ? {} : config),
+        },
+      })
+      await writeConfig(`entries = [{handle = "keep"}, {handle = "omit"}]\n${buildAppConfiguration()}`)
+
+      const app = await loadApp({
+        directory: tmpDir,
+        userProvidedConfigName: undefined,
+        specifications: [...specifications, specification] as ExtensionSpecification[],
+      })
+
+      expect(app.errors.isEmpty()).toBe(true)
+      expect(app.allExtensions.filter((extension) => extension.specification.identifier === 'custom')).toMatchObject([
+        {handle: 'keep', uid: 'keep', configuration: {entries: [{handle: 'keep'}]}},
+      ])
+    })
+
+    test('claims configuration keys when expansion produces no modules', async () => {
+      const specification = createConfigExtensionSpecification({
+        identifier: 'custom',
+        schema: BaseSchemaWithoutHandle.extend({entries: zod.array(zod.string())}),
+        expandConfig: () => [],
+        transformConfig: {forward: (config) => config},
+      })
+      await writeConfig(`entries = []\n${buildAppConfiguration()}`)
+
+      const app = await loadApp({
+        directory: tmpDir,
+        userProvidedConfigName: undefined,
+        specifications: [...specifications, specification] as ExtensionSpecification[],
+      })
+
+      expect(app.errors.isEmpty()).toBe(true)
+      expect(app.allExtensions.some((extension) => extension.specification.identifier === 'custom')).toBe(false)
+    })
+
+    test('validates each expanded configuration against the specification schema', async () => {
+      const specification = createConfigExtensionSpecification({
+        identifier: 'custom',
+        schema: BaseSchemaWithoutHandle.extend({entries: zod.array(zod.string())}),
+        expandConfig: () => [{entries: [42]}],
+        transformConfig: {forward: (config) => config},
+      })
+      await writeConfig(`entries = ["valid"]\n${buildAppConfiguration()}`)
+
+      const app = await loadApp({
+        directory: tmpDir,
+        userProvidedConfigName: undefined,
+        specifications: [...specifications, specification] as ExtensionSpecification[],
+      })
+
+      expect(app.errors.getErrors()).toEqual([
+        expect.objectContaining({message: expect.stringContaining('Expected string, received number')}),
+      ])
+      expect(app.allExtensions.some((extension) => extension.specification.identifier === 'custom')).toBe(false)
+    })
+  })
+
+  describe('event subscription modules', () => {
+    const eventsConfiguration = `
+      [events]
+      api_version = "2024-01"
+
+      [[events.subscription]]
+      topic = "orders/create"
+      actions = ["create"]
+      handle = "order-notifier"
+      uri = "/events/orders"
+
+      [[events.subscription]]
+      topic = "products/update"
+      actions = ["update"]
+      handle = "product-sync"
+      api_version = "2025-01"
+      uri = "https://example.com/events/products"
+    `
+
+    async function specificationsWithEventsContract() {
+      const eventsSpec = specifications.find((specification) => specification.identifier === 'events')!
+      const subscriptionSchema = {
+        type: 'object',
+        properties: {
+          topic: {type: 'string'},
+          actions: {type: 'array', items: {type: 'string'}},
+          uri: {type: 'string'},
+          api_version: {type: 'string'},
+        },
+        required: ['topic', 'actions', 'uri'],
+      }
+      const parseConfigurationObject = await unifiedConfigurationParserFactory(
+        {...eventsSpec, loadedRemoteSpecs: true},
+        {
+          jsonSchema: JSON.stringify({
+            type: 'object',
+            properties: {
+              events: {
+                type: 'object',
+                properties: {
+                  api_version: {type: 'string'},
+                  subscription: {
+                    anyOf: [
+                      {...subscriptionSchema, not: {required: ['handle']}},
+                      {
+                        type: 'array',
+                        items: {
+                          ...subscriptionSchema,
+                          properties: {...subscriptionSchema.properties, handle: {type: 'string'}},
+                          required: [...subscriptionSchema.required, 'handle'],
+                        },
+                      },
+                    ],
+                  },
+                },
+              },
+            },
+            additionalProperties: false,
+          }),
+        },
+      )
+      return specifications.map((specification) =>
+        specification === eventsSpec ? {...specification, parseConfigurationObject} : specification,
+      )
+    }
+
+    test('loads and round-trips the remote contract without rollout flags', async () => {
+      const configuration = eventsConfiguration.replace(
+        'actions = ["create"]',
+        'actions = ["create"]\nfuture_field = "value"',
+      )
+      await writeConfig(buildAppConfiguration({handle: 'my_app', extra: configuration}))
+      const contractSpecifications = await specificationsWithEventsContract()
+      const loadOptions = {
+        directory: tmpDir,
+        userProvidedConfigName: undefined,
+        specifications: contractSpecifications,
+      }
+
+      const app = await loadApp(loadOptions)
+      const extensions = app.allExtensions.filter((extension) => extension.specification.identifier === 'events')
+      const expectedHandles = ['order-notifier', 'product-sync']
+      const payloads = await Promise.all(
+        extensions.map((extension) =>
+          extension.deployConfig({apiKey: 'test-client-id', appConfiguration: app.configuration}),
+        ),
+      )
+
+      expect(app.errors.getErrors()).toEqual([])
+      expect(app.configuration.handle).toBe('my_app')
+      expect(extensions.map((extension) => extension.handle)).toEqual(expectedHandles)
+      expect(extensions.map((extension) => extension.uid)).toEqual(expectedHandles)
+      expect(extensions.map((extension) => extension.configuration.handle)).toEqual(expectedHandles)
+      expect(extensions.map((extension) => extension.contextValue)).toEqual(['orders/create', 'products/update'])
+      expect(payloads.every((payload) => !Object.hasOwn(payload!, 'handle'))).toBe(true)
+      expect(payloads.map((payload) => Array.isArray(getPathValue(payload!, 'events.subscription')))).toEqual([
+        false,
+        false,
+      ])
+
+      const remoteModules = extensions.map(
+        (extension, index): AppModuleVersion => ({
+          registrationId: extension.uid,
+          registrationTitle: extension.handle,
+          type: 'events',
+          config: payloads[index],
+          specification: {
+            identifier: 'events',
+            name: 'Events',
+            experience: 'configuration',
+            options: {managementExperience: 'cli'},
+          },
+        }),
+      )
+      const linked = remoteAppConfigurationExtensionContent(remoteModules, contractSpecifications, [])
+      expect(linked).toEqual({
+        events: {
+          api_version: '2024-01',
+          subscription: [
+            {
+              handle: 'order-notifier',
+              topic: 'orders/create',
+              actions: ['create'],
+              uri: 'https://example.com/events/orders',
+              future_field: 'value',
+            },
+            {
+              handle: 'product-sync',
+              topic: 'products/update',
+              actions: ['update'],
+              api_version: '2025-01',
+              uri: 'https://example.com/events/products',
+            },
+          ],
+        },
+      })
+
+      await writeAppConfigurationFile({...app.configuration, ...linked}, app.configPath)
+      const linkedApp = await loadApp(loadOptions)
+      const linkedExtensions = linkedApp.allExtensions.filter(
+        (extension) => extension.specification.identifier === 'events',
+      )
+      expect(linkedApp.errors.getErrors()).toEqual([])
+      expect(linkedApp.configuration.handle).toBe('my_app')
+      expect(linkedExtensions.map((extension) => extension.uid)).toEqual(expectedHandles)
+      await expect(
+        Promise.all(
+          linkedExtensions.map((extension) =>
+            extension.deployConfig({apiKey: 'test-client-id', appConfiguration: linkedApp.configuration}),
+          ),
+        ),
+      ).resolves.toEqual(payloads)
+    })
+
+    test('does not create a module from the app handle without events configuration', async () => {
+      await writeConfig(buildAppConfiguration({handle: 'my-app'}))
+
+      const app = await loadApp({
+        directory: tmpDir,
+        userProvidedConfigName: undefined,
+        specifications: await specificationsWithEventsContract(),
+      })
+
+      expect(app.errors.getErrors()).toEqual([])
+      expect(app.allExtensions.some((extension) => extension.specification.identifier === 'events')).toBe(false)
+    })
+
+    test('expands subscriptions into individual modules by default', async () => {
+      await writeConfig(buildAppConfiguration({extra: eventsConfiguration}))
+
+      const app = await loadTestingApp()
+      const extensions = app.allExtensions.filter((extension) => extension.specification.identifier === 'events')
+
+      expect(app.errors.isEmpty()).toBe(true)
+      expect(extensions.map(({handle, uid, contextValue}) => ({handle, uid, contextValue}))).toEqual([
+        {handle: 'order-notifier', uid: 'order-notifier', contextValue: 'orders/create'},
+        {handle: 'product-sync', uid: 'product-sync', contextValue: 'products/update'},
+      ])
+      expect(app.configuration).toMatchObject({
+        events: {subscription: [{handle: 'order-notifier'}, {handle: 'product-sync'}]},
+      })
+      const deployed = await Promise.all(
+        extensions.map((extension) =>
+          extension.deployConfig({apiKey: 'test-client-id', appConfiguration: app.configuration}),
+        ),
+      )
+      expect(deployed).toEqual([
+        {
+          events: {
+            api_version: '2024-01',
+            subscription: {topic: 'orders/create', actions: ['create'], uri: 'https://example.com/events/orders'},
+          },
+        },
+        {
+          events: {
+            api_version: '2024-01',
+            subscription: {
+              topic: 'products/update',
+              actions: ['update'],
+              api_version: '2025-01',
+              uri: 'https://example.com/events/products',
+            },
+          },
+        },
+      ])
+    })
+
+    test('rejects duplicate subscription handles', async () => {
+      await writeConfig(buildAppConfiguration({extra: eventsConfiguration.replace('product-sync', 'order-notifier')}))
+
+      const app = await loadTestingApp()
+
+      expect(app.errors.getErrors()).toEqual([
+        expect.objectContaining({message: expect.stringContaining('Duplicated handle "order-notifier"')}),
+      ])
+    })
+
+    test('rejects missing subscription handles before identity resolution', async () => {
+      await writeConfig(
+        buildAppConfiguration({handle: 'my_app', extra: eventsConfiguration.replace('handle = "order-notifier"', '')}),
+      )
+
+      const app = await loadTestingApp()
+
+      expect(app.errors.isEmpty()).toBe(false)
+      expect(app.allExtensions.filter((extension) => extension.specification.identifier === 'events')).toMatchObject([
+        {handle: 'product-sync'},
+      ])
+      expect(app.errors.getErrors().map(formatConfigurationError).join('\n')).toContain('handle')
+    })
+
+    test('creates no events modules for an empty subscription list', async () => {
+      await writeConfig(buildAppConfiguration({extra: '[events]\napi_version = "2024-01"\nsubscription = []'}))
+
+      const app = await loadTestingApp()
+
+      expect(app.errors.isEmpty()).toBe(true)
+      expect(app.allExtensions.some((extension) => extension.specification.identifier === 'events')).toBe(false)
+    })
+  })
+
   test('loads the app with several functions that have valid configurations', async () => {
     // Given
     await writeConfig(appConfiguration)

packages/app/src/cli/models/app/loader.ts

@@ -787,25 +787,27 @@ class AppLoader<TConfig extends CurrentAppConfiguration, TModuleSpec extends Ext
           const specResult = parseConfigurationObjectAgainstSpecification(specification, configPath, appConfiguration)
           if (specResult.errors) {
             this.errors.addErrors(specResult.errors)
-            return [null, [] as string[]] as const
+            return [[], [] as string[]] as const
           }
           const specConfiguration = specResult.data
 
-          if (Object.keys(specConfiguration).length === 0) return [null, Object.keys(specConfiguration)] as const
+          if (Object.keys(specConfiguration).length === 0) return [[], Object.keys(specConfiguration)] as const
 
-          const instance = await this.createExtensionInstance(
-            specification.identifier,
+          const configurations = specification.expandConfig?.(specConfiguration, {flags: this.remoteFlags}) ?? [
             specConfiguration,
-            configPath,
-            directory,
-          ).then((extensionInstance) =>
-            this.validateConfigurationExtensionInstance(
-              appConfiguration.client_id,
-              appConfiguration,
-              extensionInstance,
-            ),
+          ]
+          const instances = await Promise.all(
+            configurations.map(async (configuration) => {
+              const instance = await this.createExtensionInstance(
+                specification.identifier,
+                configuration,
+                configPath,
+                directory,
+              )
+              return this.validateConfigurationExtensionInstance(appConfiguration.client_id, appConfiguration, instance)
+            }),
           )
-          return [instance, Object.keys(specConfiguration)] as const
+          return [instances, Object.keys(specConfiguration)] as const
         }),
     )
 
@@ -824,9 +826,7 @@ class AppLoader<TConfig extends CurrentAppConfiguration, TModuleSpec extends Ext
         message: `Unsupported section(s) in app configuration: ${unusedKeys.sort().join(', ')}`,
       })
     }
-    return extensionInstancesWithKeys
-      .filter(([instance]) => instance)
-      .map(([instance]) => instance as ExtensionInstance)
+    return getArrayRejectingUndefined(extensionInstancesWithKeys.flatMap(([instances]) => instances))
   }
 
   private async validateConfigurationExtensionInstance(