Implement Custom Stacks RFC: allow docker:// URIs as stacks

app/actions/build_create.rb

@@ -184,6 +184,7 @@ def staging_in_progress!
 
     def validate_stack_state!(lifecycle, app)
       return [] if lifecycle.type == Lifecycles::DOCKER
+      return [] if UriUtils.is_custom_stack_uri?(lifecycle.staging_stack)
 
       stack = Stack.find(name: lifecycle.staging_stack)
       return [] unless stack

app/controllers/v3/apps_controller.rb

@@ -98,6 +98,7 @@ def create
     FeatureFlag.raise_unless_enabled!(:diego_docker) if message.lifecycle_type == VCAP::CloudController::PackageModel::DOCKER_TYPE
     lifecycle = AppLifecycleProvider.provide_for_create(message)
     FeatureFlag.raise_unless_enabled!(:diego_cnb) if lifecycle.type == VCAP::CloudController::Lifecycles::CNB
+    raise_unless_custom_stack_enabled!(custom_stack_from_message(message))
     unprocessable!(lifecycle.errors.full_messages) unless lifecycle.valid?
 
     app_creator = AppCreate.new(user_audit_info)
@@ -128,6 +129,7 @@ def update
     require_writable_space!(space)
 
     lifecycle = AppLifecycleProvider.provide_for_update(message, app)
+    raise_unless_custom_stack_enabled!(custom_stack_from_message(message))
     unprocessable!(lifecycle.errors.full_messages) unless lifecycle.valid?
 
     app_updater = AppUpdate.new(user_audit_info)
@@ -188,6 +190,7 @@ def start
 
     FeatureFlag.raise_unless_enabled!(:diego_docker) if app.lifecycle_type == DockerLifecycleDataModel::LIFECYCLE_TYPE
     FeatureFlag.raise_unless_enabled!(:diego_cnb) if app.lifecycle_type == CNBLifecycleDataModel::LIFECYCLE_TYPE
+    raise_unless_custom_stack_enabled!(app.lifecycle_data.stack) if app.lifecycle_data.respond_to?(:stack)
 
     AppStart.start(app:, user_audit_info:)
     TelemetryLogger.v3_emit(
@@ -231,6 +234,7 @@ def restart
 
     FeatureFlag.raise_unless_enabled!(:diego_docker) if app.lifecycle_type == DockerLifecycleDataModel::LIFECYCLE_TYPE
     FeatureFlag.raise_unless_enabled!(:diego_cnb) if app.lifecycle_type == CNBLifecycleDataModel::LIFECYCLE_TYPE
+    raise_unless_custom_stack_enabled!(app.lifecycle_data.stack) if app.lifecycle_data.respond_to?(:stack)
 
     AppRestart.restart(app: app, config: Config.config, user_audit_info: user_audit_info)
     TelemetryLogger.v3_emit(
@@ -397,6 +401,14 @@ def translate_error(e)
 
   private
 
+  def raise_unless_custom_stack_enabled!(stack)
+    FeatureFlag.raise_unless_enabled!(:diego_custom_stacks) if stack && UriUtils.is_custom_stack_uri?(stack)
+  end
+
+  def custom_stack_from_message(message)
+    message.lifecycle_data&.dig(:stack)
+  end
+
   def async_recursive_delete_enabled?
     !!VCAP::CloudController::Config.config.get(:temporary_enable_async_recursive_delete, :apps)
   end

app/fetchers/buildpack_lifecycle_fetcher.rb

@@ -4,6 +4,8 @@ module VCAP::CloudController
   class BuildpackLifecycleFetcher
     class << self
       def fetch(buildpack_names, stack_name, lifecycle=Config.config.get(:default_app_lifecycle))
+        return { stack: nil, buildpack_infos: custom_stack_buildpacks(buildpack_names, lifecycle) } if UriUtils.is_custom_stack_uri?(stack_name)
+
         {
           stack: Stack.find(name: stack_name),
           buildpack_infos: ordered_buildpacks(buildpack_names, stack_name, lifecycle)
@@ -12,6 +14,15 @@ def fetch(buildpack_names, stack_name, lifecycle=Config.config.get(:default_app_
 
       private
 
+      def custom_stack_buildpacks(buildpack_names, lifecycle)
+        buildpacks_without_stacks = Buildpack.list_admin_buildpacks(nil, lifecycle).select { |buildpack| buildpack.stack.nil? }
+
+        Array(buildpack_names).map do |buildpack_name|
+          buildpack_record = buildpacks_without_stacks.find { |buildpack| buildpack.name == buildpack_name }
+          BuildpackInfo.new(buildpack_name, buildpack_record)
+        end
+      end
+
       def ordered_buildpacks(buildpack_names, stack_name, lifecycle)
         buildpacks_with_stacks, buildpacks_without_stacks = Buildpack.list_admin_buildpacks(stack_name, lifecycle).partition(&:stack)
 

app/messages/app_manifest_message.rb

@@ -75,6 +75,7 @@ def self.underscore_keys(hash)
     validate :validate_buildpack_and_buildpacks_combination!
     validate :validate_docker_enabled!
     validate :validate_cnb_enabled!
+    validate :validate_custom_stacks_enabled!
     validate :validate_docker_buildpacks_combination!
     validate :validate_credentials_combination!
     validate :validate_service_bindings_message!, if: ->(record) { record.requested?(:services) }
@@ -488,6 +489,12 @@ def validate_cnb_enabled!
       errors.add(:base, e.message)
     end
 
+    def validate_custom_stacks_enabled!
+      FeatureFlag.raise_unless_enabled!(:diego_custom_stacks) if requested?(:stack) && UriUtils.is_custom_stack_uri?(@stack)
+    rescue CloudController::Errors::ApiError => e
+      errors.add(:base, e.message)
+    end
+
     def validate_docker_buildpacks_combination!
       return unless requested?(:docker) && (requested?(:buildpack) || requested?(:buildpacks))
 

app/models/runtime/feature_flag.rb

@@ -14,6 +14,7 @@ class UndefinedFeatureFlagError < StandardError
       service_instance_creation: true,
       diego_docker: false,
       diego_cnb: false,
+      diego_custom_stacks: false,
       set_roles_by_username: true,
       unset_roles_by_username: true,
       task_creation: true,

docs/v3/source/includes/concepts/_lifecycles.md.erb

@@ -40,8 +40,8 @@ Name | Type | Description
 ---- | ---- | -----------
 **type** | _string_ | `buildpack`
 **data.buildpacks** | _list of strings_ | A list of the names of buildpacks, URLs from which they may be downloaded, or `null` to auto-detect a suitable buildpack during staging
-**data.stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`
-**data.credentials** | _object_ | Credentials used to download custom buildpacks from private git repositories, keyed by host. Each entry is a username/password pair. If any buildpack URL embeds credentials, **data.credentials** must be empty; provide credentials in one place only.
+**data.stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`. Can also be a container image URI with `docker://` scheme (e.g. `docker://docker.io/cloudfoundry/cflinuxfs4:1.268.0`) when the `diego_custom_stacks` feature flag is enabled. When using a custom stack, buildpacks must be custom (URL-based) or enabled admin buildpacks without a pinned stack. If no buildpacks are specified, at least one enabled unpinned admin buildpack must exist for auto-detection.
+**data.credentials** | _object_ | Credentials used to download custom buildpacks from private git repositories or pull a custom stack from a private registry, keyed by host. Each entry is a username/password pair. If any buildpack URL embeds credentials, **data.credentials** must be empty; provide credentials in one place only.
 
 For records whose historical lifecycle data is no longer available,
 **data.buildpacks** may be an empty list and **data.stack** may be `null`.
@@ -84,8 +84,8 @@ Name | Type | Description
 ---- | ---- | -----------
 **type** | _string_ | `cnb`
 **data.buildpacks** | _list of strings_ | A list of URLs with either `docker://` or `http(s)://` scheme, pointing to the Cloud Native Buildpack OCI image. <br/>When the scheme is `http(s)://`, an OCI tarball is expected to be present at the specified location.
-**data.credentials** | _object_ | Credentials used to download the configured buildpacks, keyed by registry host. Each entry contains either a username/password pair or a token. CNB buildpack URLs must not embed credentials; use **data.credentials** instead.
-**data.stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`
+**data.credentials** | _object_ | Credentials used to download the configured buildpacks or pull a custom stack, keyed by registry host. Each entry contains either a username/password pair or a token. Only a username/password pair matching a custom stack's registry is sent to Diego to pull the stack; token entries are used for CNB buildpacks. CNB buildpack URLs must not embed credentials; use **data.credentials** instead.
+**data.stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`. Can also be a container image URI with `docker://` scheme when the `diego_custom_stacks` feature flag is enabled. When using a custom stack, buildpacks must be custom (URL-based) or enabled admin buildpacks without a pinned stack. If no buildpacks are specified, at least one enabled unpinned CNB admin buildpack must exist for auto-detection.
 
 For records whose historical lifecycle data is no longer available,
 **data.buildpacks** may be an empty list and **data.stack** may be `null`.

docs/v3/source/includes/resources/feature_flags/_flags.md.erb

@@ -4,6 +4,7 @@
 ["app_scaling", "true", "When enabled, space developers can perform scaling operations (i.e. change memory, disk, log rate, or instances). When disabled, only admins can perform scaling operations."],
 ["diego_docker", "false", "When enabled, Docker applications are supported by Diego. When disabled, Docker applications will stop running. It will still be possible to stop and delete them and update their configurations."],
 ["diego_cnb (experimental)", "false", "When enabled, the staging process using Cloud Native Buildpacks is supported."],
+["diego_custom_stacks", "false", "When enabled, users can specify a container image URI (docker://...) as the stack for buildpack and CNB lifecycle apps. This allows users to bring their own stacks independent of the platform-provided stacks. When disabled, only platform-provided stacks registered in the stacks table can be used."],
 ["env_var_visibility", "true", "When enabled, all users can see their environment variables. When disabled, no users can see environment variables."],
 ["hide_marketplace_from_unauthenticated_users", "false", "When enabled, service offerings available in the marketplace will be hidden from unauthenticated users. When disabled, unauthenticated users will be able to see the service offerings available in the marketplace."],
 ["private_domain_creation", "true", "When enabled, an organization manager can create private domains for that organization. When disabled, only admin users can create private domains."],

docs/v3/source/includes/resources/manifests/_object.md.erb

@@ -105,7 +105,7 @@ Name | Type | Description
 **name** | _string_ | Name of the app
 **lifecycle** | _string_ | Type of the lifecycle; valid values are `buildpack`, `cnb`, `docker`. Defaults to `buildpack`, unless the **docker** property is present.
 **buildpacks** | _list of strings_ | a) An empty array, which will automatically select the appropriate default buildpack according to the coding language (incompatible with **lifecycle: cnb**) <br>b) An array of one or more URLs pointing to buildpacks <br>c) An array of one or more installed buildpack names <br>Replaces the legacy `buildpack` field
-**credentials** | _object_ | Credentials used to download custom buildpacks (buildpack lifecycle) or buildpack OCI images (cnb lifecycle) from private git repositories or registries, keyed by host. For the buildpack lifecycle each entry is a username/password pair; for the cnb lifecycle each entry is a username/password pair or a token. Set to `null` to clear previously stored credentials. If any buildpack URL embeds credentials, **credentials** must be empty; provide credentials in one place only.
+**credentials** | _object_ | Credentials used to download custom buildpacks (buildpack lifecycle), buildpack OCI images (cnb lifecycle), or pull a custom stack from a private registry, keyed by host. For the buildpack lifecycle each entry is a username/password pair; for the cnb lifecycle each entry is a username/password pair or a token. Only a username/password pair matching a custom stack's registry is sent to Diego to pull the stack. Set to `null` to clear previously stored credentials. If any buildpack URL embeds credentials, **credentials** must be empty; provide credentials in one place only.
 **cnb-credentials** | _object_ | Backward-compatible alias for **credentials**. Specifying both **credentials** and **cnb-credentials** is an error.
 **docker** | _object_ | If present, the created app will have Docker lifecycle type; the value of this key is ignored by the API but may be used by clients to source the registry address of the image and credentials, if needed; the [generate manifest endpoint](#generate-a-manifest-for-an-app) will return the registry address of the image and username provided with this key
 **env** | _object_ | A key-value mapping of environment variables to be used for the app when running
@@ -117,7 +117,7 @@ Name | Type | Description
 **routes** | _array of [route configurations](#space-manifest-route-level-configuration)_  | List declaring HTTP and TCP routes to be mapped to the app.
 **services** | _array of [service configurations](#space-manifest-service-level-configuration)_ | A list of service-instances to bind to the app
 **sidecars** | _array of [sidecar configurations](#space-manifest-sidecar-level-configuration)_ | A list of configurations for individual sidecars
-**stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`
+**stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`. When the `diego_custom_stacks` feature flag is enabled, this can also be a `docker://` container image URI. Custom stack URIs must not include credentials; use **credentials** instead. When using a custom stack with the buildpack lifecycle, buildpacks must be custom (URL-based) or admin buildpacks without a pinned stack.
 **metadata.labels** | _[label object](#labels)_ | Labels applied to the app
 **metadata.annotations** | _[annotation object](#annotations)_ | Annotations applied to the app
 **buildpack**  | _string_ | **DEPRECATED in favor of the `buildpacks` field above**

lib/cloud_controller/diego/app_recipe_builder.rb

@@ -4,6 +4,7 @@
 require 'cloud_controller/diego/buildpack/desired_lrp_builder'
 require 'cloud_controller/diego/docker/desired_lrp_builder'
 require 'cloud_controller/diego/cnb/desired_lrp_builder'
+require 'cloud_controller/diego/image_credential_resolver'
 require 'cloud_controller/diego/process_guid'
 require 'cloud_controller/diego/ssh_key'
 require 'cloud_controller/diego/service_binding_files_builder'
@@ -101,14 +102,25 @@ def app_lrp_arguments
           certificate_properties: ::Diego::Bbs::Models::CertificateProperties.new(
             organizational_unit: ["organization:#{process.organization.guid}", "space:#{process.space.guid}", "app:#{process.app_guid}"]
           ),
-          image_username: process.desired_droplet.docker_receipt_username,
-          image_password: process.desired_droplet.docker_receipt_password,
+          image_username: lrp_image_credentials[0],
+          image_password: lrp_image_credentials[1],
           volume_mounted_files: ServiceBindingFilesBuilder.build(process)
         }.compact
       rescue ServiceBindingFilesBuilder::IncompatibleBindings => e
         raise CloudController::Errors::ApiError.new_from_details('UnprocessableEntity', "Cannot build service binding files for app - #{e.message}")
       end
 
+      def lrp_image_credentials
+        @lrp_image_credentials ||= begin
+          droplet = process.desired_droplet
+          ImageCredentialResolver.resolve(
+            primary_username: droplet.docker_receipt_username,
+            primary_password: droplet.docker_receipt_password,
+            lifecycle_data: process.app.lifecycle_data
+          )
+        end
+      end
+
       def metric_tags(process)
         tags = {
           'source_id' => METRIC_TAG_VALUE.new(static: process.app_guid),

lib/cloud_controller/diego/buildpack/desired_lrp_builder.rb

@@ -1,8 +1,13 @@
+require 'cloud_controller/diego/custom_stack_uri_converter'
+require 'cloud_controller/diego/lifecycle_bundle_stack_resolver'
+require 'utils/uri_utils'
+
 module VCAP::CloudController
   module Diego
     module Buildpack
       class DesiredLrpBuilder
         include ::Diego::ActionBuilder
+        include LifecycleBundleStackResolver
 
         class InvalidStack < StandardError; end
 
@@ -25,20 +30,22 @@ def initialize(config, opts)
         def cached_dependencies
           return nil if @config.get(:diego, :enable_declarative_asset_downloads)
 
-          lifecycle_bundle_key = :"buildpack/#{@stack}"
+          lifecycle_bundle_key = resolved_lifecycle_bundle_key
           lifecycle_bundle = @config.get(:diego, :lifecycle_bundles)[lifecycle_bundle_key]
           raise InvalidStack.new("no compiler defined for requested stack '#{@stack}'") unless lifecycle_bundle
 
           [
             ::Diego::Bbs::Models::CachedDependency.new(
               from: LifecycleBundleUriGenerator.uri(lifecycle_bundle),
               to: '/tmp/lifecycle',
-              cache_key: "buildpack-#{@stack}-lifecycle"
+              cache_key: "buildpack-#{lifecycle_bundle_stack_name}-lifecycle"
             )
           ]
         end
 
         def root_fs
+          return CustomStackUriConverter.convert(@stack) if UriUtils.is_custom_stack_uri?(@stack)
+
           @stack_obj ||= Stack.find(name: @stack)
           raise CloudController::Errors::ApiError.new_from_details('StackNotFound', @stack) unless @stack_obj
 
@@ -64,16 +71,16 @@ def setup
         def image_layers
           return [] unless @config.get(:diego, :enable_declarative_asset_downloads)
 
-          lifecycle_bundle_key = :"buildpack/#{@stack}"
+          lifecycle_bundle_key = resolved_lifecycle_bundle_key
           lifecycle_bundle = @config.get(:diego, :lifecycle_bundles)[lifecycle_bundle_key]
           raise InvalidStack.new("no compiler defined for requested stack '#{@stack}'") unless lifecycle_bundle
 
-          destination = @config.get(:diego, :droplet_destinations)[@stack.to_sym]
+          destination = @config.get(:diego, :droplet_destinations)[lifecycle_bundle_stack_name.to_sym]
           raise InvalidStack.new("no droplet destination defined for requested stack '#{@stack}'") unless destination
 
           layers = [
             ::Diego::Bbs::Models::ImageLayer.new(
-              name: "buildpack-#{@stack}-lifecycle",
+              name: "buildpack-#{lifecycle_bundle_stack_name}-lifecycle",
               url: LifecycleBundleUriGenerator.uri(lifecycle_bundle),
               destination_path: '/tmp/lifecycle',
               layer_type: ::Diego::Bbs::Models::ImageLayer::Type::SHARED,
@@ -117,6 +124,12 @@ def port_environment_variables
         def privileged?
           @config.get(:diego, :use_privileged_containers_for_running)
         end
+
+        private
+
+        def resolved_lifecycle_bundle_key
+          :"buildpack/#{lifecycle_bundle_stack_name}"
+        end
       end
     end
   end