diff --git a/doc/workloads/sglang.rst b/doc/workloads/sglang.rst index 6d62f6ee9..237537af4 100644 --- a/doc/workloads/sglang.rst +++ b/doc/workloads/sglang.rst @@ -70,6 +70,28 @@ supported under ``[[Tests]]`` in a test scenario. Define them in a test definiti ``test_name`` when custom benchmark or semantic-evaluation arguments are needed. +Local Models +------------ +Set ``cmd_args.model`` to an absolute, container-visible path to load an existing model directory instead of +downloading a model from Hugging Face. Use ``extra_container_mounts`` to mount a host or shared-filesystem directory. + +.. code-block:: toml + :caption: test.toml (local model) + + name = "sglang_local_model" + description = "SGLang with a local model" + test_template_name = "sglang" + extra_container_mounts = [ + "/lustre/models/custom:/models/custom:ro", + ] + + [cmd_args] + docker_image_url = "lmsysorg/sglang:dev-cu13" + model = "/models/custom" + +The mounted directory must contain a model, configuration, and tokenizer supported by SGLang. + + Semantic Validation ------------------- To run GSM8K semantic validation after the serving benchmark, add ``semantic_eval_cmd_args``. CloudAI reports diff --git a/doc/workloads/vllm.rst b/doc/workloads/vllm.rst index 009270b32..082faf70d 100644 --- a/doc/workloads/vllm.rst +++ b/doc/workloads/vllm.rst @@ -70,6 +70,28 @@ supported under ``[[Tests]]`` in a test scenario. Define them in a test definiti ``test_name`` when custom benchmark or semantic-evaluation arguments are needed. +Local Models +------------ +Set ``cmd_args.model`` to an absolute, container-visible path to load an existing model directory instead of +downloading a model from Hugging Face. Use ``extra_container_mounts`` to mount a host or shared-filesystem directory. + +.. code-block:: toml + :caption: test.toml (local model) + + name = "vllm_local_model" + description = "vLLM with a local model" + test_template_name = "vllm" + extra_container_mounts = [ + "/lustre/models/custom:/models/custom:ro", + ] + + [cmd_args] + docker_image_url = "vllm/vllm-openai:v0.14.0-cu130" + model = "/models/custom" + +The mounted directory must contain a model, configuration, and tokenizer supported by vLLM. + + Semantic Validation ------------------- To run GSM8K semantic validation after the serving benchmark, add ``semantic_eval_cmd_args``. CloudAI reports diff --git a/src/cloudai/workloads/common/llm_serving.py b/src/cloudai/workloads/common/llm_serving.py index 09e98270a..6b45ff854 100644 --- a/src/cloudai/workloads/common/llm_serving.py +++ b/src/cloudai/workloads/common/llm_serving.py @@ -190,7 +190,10 @@ def extra_installables(self) -> list[Installable]: @property def installables(self) -> list[Installable]: - return [*self.git_repos, self.docker_image, self.hf_model, *self.extra_installables] + installables: list[Installable] = [*self.git_repos, self.docker_image] + if not Path(self.cmd_args.model).is_absolute(): + installables.append(self.hf_model) + return [*installables, *self.extra_installables] def model_post_init(self, __context: Any) -> None: super().model_post_init(__context) diff --git a/tests/workloads/sglang/test_workload.py b/tests/workloads/sglang/test_workload.py index 23720c81e..ee921e4fd 100644 --- a/tests/workloads/sglang/test_workload.py +++ b/tests/workloads/sglang/test_workload.py @@ -14,8 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from cloudai.workloads.sglang import SglangArgs +from cloudai.workloads.sglang import SglangArgs, SglangCmdArgs, SglangTestDefinition def test_sglang_serve_args_exclude_internal_fields() -> None: assert SglangArgs(gpu_ids="0", disaggregation_transfer_backend="nccl").serve_args == [] + + +def test_local_model_is_not_installed_from_hugging_face() -> None: + tdef = SglangTestDefinition( + name="test", + description="test", + test_template_name="sglang", + cmd_args=SglangCmdArgs( + docker_image_url="test_url", + model="/models/custom-model", + ), + ) + + assert tdef.installables == [tdef.docker_image] diff --git a/tests/workloads/vllm/test_workload.py b/tests/workloads/vllm/test_workload.py index 6d123e56b..4d0be03df 100644 --- a/tests/workloads/vllm/test_workload.py +++ b/tests/workloads/vllm/test_workload.py @@ -54,6 +54,22 @@ def test_installables_include_proxy_script_repo() -> None: assert tdef.installables == [tdef.docker_image, tdef.hf_model, proxy_script_repo] +def test_local_model_is_not_installed_from_hugging_face() -> None: + proxy_script_repo = GitRepo(url="./proxy_script_repo", commit="commit") + tdef = VllmTestDefinition( + name="test", + description="test", + test_template_name="vllm", + cmd_args=VllmCmdArgs( + docker_image_url="test_url", + model="/models/custom-model", + ), + proxy_script_repo=proxy_script_repo, + ) + + assert tdef.installables == [tdef.docker_image, proxy_script_repo] + + def test_constraint_check_rejects_tp_pp_dp_above_available_gpus(tmp_path) -> None: tdef = VllmTestDefinition( name="test",