feat(file-service): add model upload limits and the models_enabled flag

common/config/src/main/resources/default.conf

@@ -64,6 +64,10 @@ gui {
     datasets_enabled = true
     datasets_enabled = ${?GUI_TABS_DATASETS_ENABLED}
 
+    # Hides the Models sidebar entry, not the route.
+    models_enabled = false
+    models_enabled = ${?GUI_TABS_MODELS_ENABLED}
+
     compute_enabled = true
     compute_enabled = ${?GUI_TABS_COMPUTE_ENABLED}
 
@@ -80,24 +84,41 @@ gui {
 }
 
 dataset {
-  single_file_upload_max_size_mib = 20
-  single_file_upload_max_size_mib = ${?DATASET_SINGLE_FILE_UPLOAD_MAX_SIZE_MIB}
+  dataset_single_file_upload_max_size_mib = 20
+  dataset_single_file_upload_max_size_mib = ${?DATASET_SINGLE_FILE_UPLOAD_MAX_SIZE_MIB}
+
+  dataset_max_number_of_concurrent_uploading_file = 3
+  dataset_max_number_of_concurrent_uploading_file = ${?DATASET_MAX_NUMBER_OF_CONCURRENT_UPLOADING_FILE}
+
+  # The maximum number of file chunks that can be held in the memory
+  dataset_max_number_of_concurrent_uploading_file_chunks = 10
+  dataset_max_number_of_concurrent_uploading_file_chunks = ${?DATASET_MAX_NUMBER_OF_CONCURRENT_UPLOADING_FILE_CHUNKS}
+
+  # the size of each chunk during the multipart upload of file
+  dataset_multipart_upload_chunk_size_mib = 50
+  dataset_multipart_upload_chunk_size_mib = ${?DATASET_MULTIPART_UPLOAD_CHUNK_SIZE_MIB}
+}
+
+model {
+  model_single_file_upload_max_size_mib = 2048
+  model_single_file_upload_max_size_mib = ${?MODEL_SINGLE_FILE_UPLOAD_MAX_SIZE_MIB}
 
-  max_number_of_concurrent_uploading_file = 3
-  max_number_of_concurrent_uploading_file = ${?MAX_NUMBER_OF_CONCURRENT_UPLOADING_FILE}
+  model_max_number_of_concurrent_uploading_file = 3
+  model_max_number_of_concurrent_uploading_file = ${?MODEL_MAX_NUMBER_OF_CONCURRENT_UPLOADING_FILE}
 
   # The maximum number of file chunks that can be held in the memory
-  max_number_of_concurrent_uploading_file_chunks = 10
-  max_number_of_concurrent_uploading_file_chunks = ${?DATASET_MAX_NUMBER_OF_CONCURRENT_UPLOADING_FILE_CHUNKS}
+  model_max_number_of_concurrent_uploading_file_chunks = 10
+  model_max_number_of_concurrent_uploading_file_chunks = ${?MODEL_MAX_NUMBER_OF_CONCURRENT_UPLOADING_FILE_CHUNKS}
 
   # the size of each chunk during the multipart upload of file
-  multipart_upload_chunk_size_mib = 50
-  multipart_upload_chunk_size_mib = ${?DATASET_MULTIPART_UPLOAD_CHUNK_SIZE_MIB}
+  model_multipart_upload_chunk_size_mib = 50
+  model_multipart_upload_chunk_size_mib = ${?MODEL_MULTIPART_UPLOAD_CHUNK_SIZE_MIB}
 }
 
 # Operator-level defaults. These are management-only site settings (edited from
-# the admin page, read by the engine) and are deliberately NOT under gui/dataset,
-# so they stay out of the anonymous /config/settings/public whitelist.
+# the admin page, read by the engine) and are deliberately NOT under
+# gui/dataset/model, so they stay out of the anonymous /config/settings/public
+# whitelist.
 operator {
   # Upper bound on the number of columns the CSV scan source will parse.
   csv_parser_max_columns = 512

common/config/src/test/scala/org/apache/texera/common/config/DefaultsConfigSpec.scala

@@ -43,9 +43,14 @@ class DefaultsConfigSpec extends AnyFlatSpec with Matchers {
     defaults should not be empty
     // scalar leaves are flattened to their last path segment
     ifUnset("DATASET_SINGLE_FILE_UPLOAD_MAX_SIZE_MIB")(
-      defaults.get("single_file_upload_max_size_mib") shouldBe Some("20")
+      defaults.get("dataset_single_file_upload_max_size_mib") shouldBe Some("20")
     )
     ifUnset("GUI_TABS_HUB_ENABLED")(defaults.get("hub_enabled") shouldBe Some("true"))
+    // the model block's leaves are `model_`-prefixed to keep short keys unique
+    ifUnset("MODEL_SINGLE_FILE_UPLOAD_MAX_SIZE_MIB")(
+      defaults.get("model_single_file_upload_max_size_mib") shouldBe Some("2048")
+    )
+    ifUnset("GUI_TABS_MODELS_ENABLED")(defaults.get("models_enabled") shouldBe Some("false"))
     // management-only keys are flattened too (used by reset + the startup seeder)
     ifUnset("OPERATOR_CSV_PARSER_MAX_COLUMNS")(
       defaults.get("csv_parser_max_columns") shouldBe Some("512")
@@ -54,31 +59,38 @@ class DefaultsConfigSpec extends AnyFlatSpec with Matchers {
     defaults.values.foreach(_ shouldBe a[String])
   }
 
-  it should "keep management-only keys out of the public gui/dataset whitelist" in {
+  it should "keep management-only keys out of the public gui/dataset/model whitelist" in {
     // csv_parser_max_columns is seeded and resettable (present in allDefaults)
     // but lives under `operator`, so it must never reach the anonymous
     // /config/settings/public payload.
     DefaultsConfig.allDefaults.keySet should contain("csv_parser_max_columns")
     DefaultsConfig.keysUnderSections(
-      Set("gui", "dataset")
+      Set("gui", "dataset", "model")
     ) should not contain "csv_parser_max_columns"
   }
 
   "DefaultsConfig.keysUnderSections" should "collect the short keys of the requested sections only" in {
     val guiKeys = DefaultsConfig.keysUnderSections(Set("gui"))
     guiKeys should contain allOf ("logo", "mini_logo", "favicon", "hub_enabled")
     // keys from other sections are excluded
-    guiKeys should not contain "single_file_upload_max_size_mib"
+    guiKeys should not contain "dataset_single_file_upload_max_size_mib"
     guiKeys should not contain "always-reset-configurations-to-default-values"
 
     val datasetKeys = DefaultsConfig.keysUnderSections(Set("dataset"))
-    datasetKeys should contain("single_file_upload_max_size_mib")
+    datasetKeys should contain("dataset_single_file_upload_max_size_mib")
     datasetKeys should not contain "logo"
+
+    // `model` is a sibling section, not a sub-section of `dataset`: disjoint sets
+    // are what let a model carry a different ceiling.
+    val modelKeys = DefaultsConfig.keysUnderSections(Set("model"))
+    modelKeys should contain("model_single_file_upload_max_size_mib")
+    modelKeys should not contain "dataset_single_file_upload_max_size_mib"
+    datasetKeys should not contain "model_single_file_upload_max_size_mib"
   }
 
   it should "union multiple sections and be empty for an unknown section" in {
-    val union = DefaultsConfig.keysUnderSections(Set("gui", "dataset"))
-    union should contain allOf ("logo", "single_file_upload_max_size_mib")
+    val union = DefaultsConfig.keysUnderSections(Set("gui", "dataset", "model"))
+    union should contain allOf ("logo", "dataset_single_file_upload_max_size_mib", "model_single_file_upload_max_size_mib")
     // every returned key exists in allDefaults under the same short name
     union.subsetOf(DefaultsConfig.allDefaults.keySet) shouldBe true
 

config-service/src/main/scala/org/apache/texera/service/resource/ConfigResource.scala

@@ -117,21 +117,21 @@ class ConfigResource {
     )
 
   // The site_settings keys that non-admin pages consume: dashboard branding,
-  // sidebar tab toggles, and dataset upload limits — exactly the gui.* and
-  // dataset.* sections of default.conf, which is also where the seeding
-  // pipeline gets them. Keys declared outside those sections (e.g.
+  // sidebar tab toggles, and the dataset/model upload limits — exactly the
+  // gui.*, dataset.* and model.* sections of default.conf, which is also where
+  // the seeding pipeline gets them. Keys declared outside those sections (e.g.
   // csv_parser_max_columns) are management-only. Deriving the set from the
   // file keeps "which section does this default live in" the single place
   // where visibility is decided.
   private val publicSettingKeys: Set[String] =
-    DefaultsConfig.keysUnderSections(Set("gui", "dataset"))
+    DefaultsConfig.keysUnderSections(Set("gui", "dataset", "model"))
 
   // SECURITY: every key returned here is served anonymously (see
   // /settings/public below), so `publicSettingKeys` is the anonymous-exposure
-  // surface. It is derived from the gui/dataset sections of default.conf and
-  // pinned by ConfigResourceSpec/DefaultsConfigSpec — adding a key under those
-  // sections (or moving one in) changes what unauthenticated callers can read
-  // and MUST be reviewed there. Never place a secret under gui/dataset.
+  // surface. It is derived from the gui/dataset/model sections of default.conf
+  // and pinned by ConfigResourceSpec/DefaultsConfigSpec — adding a key under
+  // those sections (or moving one in) changes what unauthenticated callers can
+  // read and MUST be reviewed there. Never place a secret under those sections.
 
   private def fetchSettings(condition: Condition): Map[String, String] =
     ctx

config-service/src/test/scala/org/apache/texera/service/resource/ConfigResourceSpec.scala

@@ -401,11 +401,11 @@ class ConfigResourceSpec
     publicSettings should not contain key("csv_parser_max_columns")
   }
 
-  // The public whitelist is derived from the gui/dataset sections of
+  // The public whitelist is derived from the gui/dataset/model sections of
   // default.conf. This pins the derived set, so moving a key between sections
   // (or adding one) forces the visibility decision into review here.
-  it should "expose exactly the gui and dataset section keys of default.conf" in {
-    DefaultsConfig.keysUnderSections(Set("gui", "dataset")) shouldBe Set(
+  it should "expose exactly the gui, dataset and model section keys of default.conf" in {
+    DefaultsConfig.keysUnderSections(Set("gui", "dataset", "model")) shouldBe Set(
       "logo",
       "mini_logo",
       "favicon",
@@ -417,14 +417,19 @@ class ConfigResourceSpec
       "projects_enabled",
       "workflows_enabled",
       "datasets_enabled",
+      "models_enabled",
       "compute_enabled",
       "quota_enabled",
       "forum_enabled",
       "about_enabled",
-      "single_file_upload_max_size_mib",
-      "multipart_upload_chunk_size_mib",
-      "max_number_of_concurrent_uploading_file",
-      "max_number_of_concurrent_uploading_file_chunks"
+      "dataset_single_file_upload_max_size_mib",
+      "dataset_multipart_upload_chunk_size_mib",
+      "dataset_max_number_of_concurrent_uploading_file",
+      "dataset_max_number_of_concurrent_uploading_file_chunks",
+      "model_single_file_upload_max_size_mib",
+      "model_multipart_upload_chunk_size_mib",
+      "model_max_number_of_concurrent_uploading_file",
+      "model_max_number_of_concurrent_uploading_file_chunks"
     )
   }
 

file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala

@@ -29,7 +29,6 @@ import org.apache.texera.common.util.EmailUtil
 import org.apache.texera.amber.core.storage.util.LakeFSStorageClient
 import org.apache.texera.amber.core.storage.ResourceType
 import org.apache.texera.auth.SessionUser
-import org.apache.texera.dao.SiteSettings
 import org.apache.texera.dao.SqlServer
 import org.apache.texera.dao.SqlServer.withTransaction
 import org.apache.texera.dao.jooq.generated.enums.{PrivilegeEnum, UserRoleEnum}
@@ -72,9 +71,6 @@ object DatasetResource {
       .getInstance()
       .createDSLContext()
 
-  private def singleFileUploadMaxBytes(defaultMiB: Long = 20L): Long =
-    SiteSettings.getLong("single_file_upload_max_size_mib", defaultMiB) * 1024L * 1024L
-
   /**
     * Helper function to get the dataset from DB using did
     */

file-service/src/main/scala/org/apache/texera/service/resource/ModelResource.scala

@@ -28,7 +28,7 @@ import org.apache.texera.amber.core.storage.ResourceType
 import org.apache.texera.amber.core.storage.util.LakeFSStorageClient
 import org.apache.texera.auth.SessionUser
 import org.apache.texera.common.config.StorageConfig
-import org.apache.texera.dao.{SiteSettings, SqlServer}
+import org.apache.texera.dao.SqlServer
 import org.apache.texera.dao.SqlServer.withTransaction
 import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum
 import org.apache.texera.dao.jooq.generated.tables.Model.MODEL
@@ -107,9 +107,6 @@ object ModelResource {
       .getInstance()
       .createDSLContext()
 
-  private def singleFileUploadMaxBytes(defaultMiB: Long = 20L): Long =
-    SiteSettings.getLong("single_file_upload_max_size_mib", defaultMiB) * 1024L * 1024L
-
   /**
     * Helper function to get the model from DB using mid
     */

file-service/src/main/scala/org/apache/texera/service/resource/ResourceUploadService.scala

@@ -26,7 +26,7 @@ import org.apache.texera.amber.core.storage.model.OnVersionedFileResource
 import org.apache.texera.amber.core.storage.util.LakeFSStorageClient
 import org.apache.texera.amber.core.storage.{DocumentFactory, FileResolver}
 import org.apache.texera.common.config.StorageConfig
-import org.apache.texera.dao.{SiteSettings, SqlServer}
+import org.apache.texera.dao.SqlServer
 import org.apache.texera.dao.SqlServer.withTransaction
 import org.apache.texera.dao.jooq.generated.tables.Dataset.DATASET
 import org.apache.texera.dao.jooq.generated.tables.Model.MODEL
@@ -74,11 +74,11 @@ import scala.util.Try
   * Describes where a resource's files live and how its in-progress uploads are tracked.
   *
   * [[ResourceTables]] names the columns that carry identity, ownership and grants; this adds
-  * the storage side — the LakeFS repository column plus the `*_upload_session` and
-  * `*_upload_session_part` tables that back resumable multipart uploads. Naming the columns
-  * keeps one implementation of the upload engine serving every resource type, so adding the
-  * next one costs a descriptor rather than another copy of the locking, part-size and
-  * resume rules.
+  * the storage side — the LakeFS repository column, the [[UploadLimits]] this type is held to,
+  * and the `*_upload_session` and `*_upload_session_part` tables that back resumable multipart
+  * uploads. Naming them keeps one implementation of the upload engine serving every resource
+  * type, so adding the next one costs a descriptor rather than another copy of the locking,
+  * part-size and resume rules.
   *
   * @tparam R record type of the resource table
   * @tparam A record type of the companion user-access table
@@ -88,6 +88,7 @@ import scala.util.Try
 case class ResourceStorage[R <: Record, A <: Record, S <: Record, P <: Record](
     resource: ResourceTables[R, A],
     resourceType: ResourceType.Value,
+    uploadLimits: UploadLimits,
     repositoryNameField: TableField[R, String],
     sessionResourceId: TableField[S, Integer],
     sessionUid: TableField[S, Integer],
@@ -117,6 +118,7 @@ object ResourceStorage {
     ResourceStorage(
       resource = ResourceTables.Dataset,
       resourceType = ResourceType.Dataset,
+      uploadLimits = UploadLimits.Dataset,
       repositoryNameField = DATASET.REPOSITORY_NAME,
       sessionResourceId = DATASET_UPLOAD_SESSION.DID,
       sessionUid = DATASET_UPLOAD_SESSION.UID,
@@ -141,6 +143,7 @@ object ResourceStorage {
     ResourceStorage(
       resource = ResourceTables.Model,
       resourceType = ResourceType.Model,
+      uploadLimits = UploadLimits.Model,
       repositoryNameField = MODEL.REPOSITORY_NAME,
       sessionResourceId = MODEL_UPLOAD_SESSION.MID,
       sessionUid = MODEL_UPLOAD_SESSION.UID,
@@ -172,9 +175,6 @@ object ResourceUploadService {
       .getInstance()
       .createDSLContext()
 
-  private def singleFileUploadMaxBytes(defaultMiB: Long = 20L): Long =
-    SiteSettings.getLong("single_file_upload_max_size_mib", defaultMiB) * 1024L * 1024L
-
   /**
     * Builds the file nodes of one committed version, plus the version's total size.
     *
@@ -714,7 +714,7 @@ object ResourceUploadService {
       if (fileSizeBytesValue <= 0L) throw new BadRequestException("fileSizeBytes must be > 0")
       if (partSizeBytesValue <= 0L) throw new BadRequestException("partSizeBytes must be > 0")
 
-      val totalMaxBytes: Long = singleFileUploadMaxBytes()
+      val totalMaxBytes: Long = s.uploadLimits.singleFileUploadMaxBytes
       if (totalMaxBytes <= 0L) {
         throw new WebApplicationException(
           "singleFileUploadMaxBytes must be > 0",
@@ -1105,7 +1105,7 @@ object ResourceUploadService {
         )
       }
 
-      val maxBytes = singleFileUploadMaxBytes()
+      val maxBytes = s.uploadLimits.singleFileUploadMaxBytes
       val tooLarge = actualSizeBytes > maxBytes
 
       if (tooLarge) {

file-service/src/main/scala/org/apache/texera/service/resource/UploadLimits.scala

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.service.resource
+
+import org.apache.texera.dao.SiteSettings
+
+/** An admin-editable limit: its `site_settings` key, and the fallback when the row is missing
+  * (must match the leaf's default in `default.conf`).
+  */
+case class UploadLimit(key: String, defaultValue: Long)
+
+/**
+  * The upload ceilings of one resource type.
+  *
+  * `site_settings` rows are keyed by a `default.conf` leaf's last path segment, so each type
+  * needs its own key names. Only the size ceiling is enforced server-side; the chunk-size and
+  * concurrency limits are client-side tuning served from `/config/settings/public`.
+  */
+case class UploadLimits(
+    singleFileMaxSizeMiB: UploadLimit,
+    multipartChunkSizeMiB: UploadLimit,
+    maxConcurrentFiles: UploadLimit,
+    maxConcurrentFileChunks: UploadLimit
+) {
+
+  /** The largest single file this resource type accepts, in bytes. */
+  def singleFileUploadMaxBytes: Long =
+    SiteSettings.getLong(
+      singleFileMaxSizeMiB.key,
+      singleFileMaxSizeMiB.defaultValue
+    ) * 1024L * 1024L
+
+  def all: Seq[UploadLimit] =
+    Seq(singleFileMaxSizeMiB, multipartChunkSizeMiB, maxConcurrentFiles, maxConcurrentFileChunks)
+}
+
+object UploadLimits {
+
+  val Dataset: UploadLimits =
+    UploadLimits(
+      singleFileMaxSizeMiB = UploadLimit("dataset_single_file_upload_max_size_mib", 20L),
+      multipartChunkSizeMiB = UploadLimit("dataset_multipart_upload_chunk_size_mib", 50L),
+      maxConcurrentFiles = UploadLimit("dataset_max_number_of_concurrent_uploading_file", 3L),
+      maxConcurrentFileChunks =
+        UploadLimit("dataset_max_number_of_concurrent_uploading_file_chunks", 10L)
+    )
+
+  // Model weights are far larger than the files datasets are sized for: 2 GiB, not 20 MiB.
+  val Model: UploadLimits =
+    UploadLimits(
+      singleFileMaxSizeMiB = UploadLimit("model_single_file_upload_max_size_mib", 2048L),
+      multipartChunkSizeMiB = UploadLimit("model_multipart_upload_chunk_size_mib", 50L),
+      maxConcurrentFiles = UploadLimit("model_max_number_of_concurrent_uploading_file", 3L),
+      maxConcurrentFileChunks =
+        UploadLimit("model_max_number_of_concurrent_uploading_file_chunks", 10L)
+    )
+}

file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala

@@ -1654,7 +1654,7 @@ class DatasetResourceSpec
     s"$prefix/${System.nanoTime()}-${Random.alphanumeric.take(8).mkString}.bin"
 
   // ---------- site_settings helpers (max upload size) ----------
-  private val MaxUploadKey = "single_file_upload_max_size_mib"
+  private val MaxUploadKey = "dataset_single_file_upload_max_size_mib"
 
   private def upsertSiteSetting(key: String, value: String): Unit = {
     val table = DSL.table(DSL.name("texera_db", "site_settings"))