From 5a36d8ec3545b2bb3855cf5a729a7f0cdb961aef Mon Sep 17 00:00:00 2001 From: David del Real Sifuentes Date: Mon, 10 Aug 2026 22:39:57 +0000 Subject: [PATCH 1/4] Migrated samples to genai/model_tuning. Refactored tests to wrok with new changes. --- genai/model_tuning/evaluate_model_example.py | 66 +++++++++++++++++++ .../evaluate_model_example_test.py | 25 +++++++ genai/model_tuning/noxfile_config.py | 42 ++++++++++++ .../pretrained_codegen_example.py | 57 ++++++++++++++++ .../model_tuning/pretrained_examples_test.py | 52 +++++++++++++++ genai/model_tuning/requirements-test.txt | 3 + genai/model_tuning/requirements.txt | 2 + 7 files changed, 247 insertions(+) create mode 100644 genai/model_tuning/evaluate_model_example.py create mode 100644 genai/model_tuning/evaluate_model_example_test.py create mode 100644 genai/model_tuning/noxfile_config.py create mode 100644 genai/model_tuning/pretrained_codegen_example.py create mode 100644 genai/model_tuning/pretrained_examples_test.py create mode 100644 genai/model_tuning/requirements-test.txt create mode 100644 genai/model_tuning/requirements.txt diff --git a/genai/model_tuning/evaluate_model_example.py b/genai/model_tuning/evaluate_model_example.py new file mode 100644 index 00000000000..25850bf5d06 --- /dev/null +++ b/genai/model_tuning/evaluate_model_example.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_genai_evaluate_model] +import os + +import vertexai +from vertexai.evaluation import EvalResult, EvalTask + +# TODO (Developer) Set environment variables +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") +LOCATION_ID = os.getenv("LOCATION_ID", "us-central1") +MODEL_NAME = os.getenv("MODEL_NAME", "gemini-2.5-flash") + + +def evaluate_model() -> EvalResult: + """Evaluate the performance of a generative AI model.""" + + vertexai.init(project=PROJECT_ID, location=LOCATION_ID) + + # Dataset URI containing input prompts and ground truth labels + dataset_uri = "gs://cloud-samples-data/ai-platform/generative_ai/llm_classification_bp_input_prompts_with_ground_truth.jsonl" + + metric_column_mapping = {"reference": "ground_truth"} + + # Define evaluation task + eval_task = EvalTask( + dataset=dataset_uri, + metrics=["exact_match"], + experiment="gemini-classification-eval", + metric_column_mapping=metric_column_mapping, + ) + + # Define a prompt template so the generative Gemini model produces formatted labels + prompt_template = ( + "Classify the following text into exactly one category from " + "[nature, news, sports, health, startups]. Return only the category name:\n\n{prompt}" + ) + + # Evaluate using a modern Gemini model + eval_result = eval_task.evaluate( + model=MODEL_NAME, + prompt_template=prompt_template, + ) + + print("=== SUMMARY METRICS ===") + print(eval_result.summary_metrics) + + print("\n=== METRICS TABLE SAMPLE ===") + print(eval_result.metrics_table.head()) + + return eval_result + + +# [END aiplatform_genai_evaluate_model] diff --git a/genai/model_tuning/evaluate_model_example_test.py b/genai/model_tuning/evaluate_model_example_test.py new file mode 100644 index 00000000000..df9cc129c56 --- /dev/null +++ b/genai/model_tuning/evaluate_model_example_test.py @@ -0,0 +1,25 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import backoff + +from google.api_core.exceptions import ResourceExhausted + +import evaluate_model_example + + +@backoff.on_exception(backoff.expo, ResourceExhausted, max_time=10) +def test_evaluate_model() -> None: + eval_metrics = evaluate_model_example.evaluate_model() + assert hasattr(eval_metrics, "metrics_table") diff --git a/genai/model_tuning/noxfile_config.py b/genai/model_tuning/noxfile_config.py new file mode 100644 index 00000000000..0973c8621c7 --- /dev/null +++ b/genai/model_tuning/noxfile_config.py @@ -0,0 +1,42 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be imported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": True, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} diff --git a/genai/model_tuning/pretrained_codegen_example.py b/genai/model_tuning/pretrained_codegen_example.py new file mode 100644 index 00000000000..c0e9a926267 --- /dev/null +++ b/genai/model_tuning/pretrained_codegen_example.py @@ -0,0 +1,57 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_genai_tune_code_generation_model] +import os +from google import genai + +# TODO (Developer) Set environment variables +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") +LOCATION_ID = os.getenv("LOCATION_ID", "us-central1") + +# Resource format: 'publishers/google/models/{model_id}' +BASE_MODEL_RESOURCE = "publishers/google/models/gemini-2.5-flash" +TRAINING_DATASET = ( + "gs://cloud-samples-data/ai-platform/generative_ai/gemini/text/sft_train_data.jsonl" +) + + +def tune_code_generation_model() -> genai.types.TuningJob: + """Submits a supervised fine-tuning job for a Gemini model on code/text tasks.""" + + client = genai.Client( + enterprise=True, + project=PROJECT_ID, + location=LOCATION_ID, + ) + + tuning_job = client.tunings.tune( + base_model=BASE_MODEL_RESOURCE, + training_dataset=genai.types.TuningDataset( + gcs_uri=TRAINING_DATASET, + ), + config=genai.types.CreateTuningJobConfig( + tuned_model_display_name="tuned_gemini_code_model", + epoch_count=2, + learning_rate_multiplier=1.0, + ), + ) + + print(f"Tuning job submitted successfully: {tuning_job.name}") + print(f"Current State: {tuning_job.state}") + + return tuning_job + + +# [END aiplatform_genai_tune_code_generation_model] diff --git a/genai/model_tuning/pretrained_examples_test.py b/genai/model_tuning/pretrained_examples_test.py new file mode 100644 index 00000000000..d6359cfd824 --- /dev/null +++ b/genai/model_tuning/pretrained_examples_test.py @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google import genai + +import pretrained_codegen_example + +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") +LOCATION_ID = os.getenv("LOCATION_ID", "us-central1") + +JOB_STATES_CANCELLABLE = [ + genai.types.JobState.JOB_STATE_RUNNING, + genai.types.JobState.JOB_STATE_PENDING, +] +JOB_STATES_DELETABLE = [ + genai.types.JobState.JOB_STATE_SUCCEEDED, + genai.types.JobState.JOB_STATE_CANCELLED, + genai.types.JobState.JOB_STATE_FAILED, +] + + +def test_tuning_code_generation_model() -> None: + """Validate tuning creation, and cleans after execution.""" + + client = genai.Client(enterprise=True, project=PROJECT_ID, location=LOCATION_ID) + + tuned_model = pretrained_codegen_example.tune_code_generation_model() + + job_is_pending = tuned_model.state in JOB_STATES_CANCELLABLE + job_is_finished = tuned_model.state in JOB_STATES_DELETABLE + result = job_is_finished or job_is_pending + + assert result + + # cleanup + if job_is_pending: + client.tunings.cancel(name=tuned_model.name) + if job_is_finished: + client.models.delete(model=tuned_model.model) diff --git a/genai/model_tuning/requirements-test.txt b/genai/model_tuning/requirements-test.txt new file mode 100644 index 00000000000..28344f9b091 --- /dev/null +++ b/genai/model_tuning/requirements-test.txt @@ -0,0 +1,3 @@ +backoff==2.2.1 +google-api-core==2.33.0 +pytest==9.1.1 diff --git a/genai/model_tuning/requirements.txt b/genai/model_tuning/requirements.txt new file mode 100644 index 00000000000..d5ef3b41e1a --- /dev/null +++ b/genai/model_tuning/requirements.txt @@ -0,0 +1,2 @@ +google-genai==2.16.0 +google-cloud-aiplatform[pipelines, evaluation]==1.163.0 From b965c5b4b1b9430f4d91bbb4db9c8bec273aecf9 Mon Sep 17 00:00:00 2001 From: David del Real Sifuentes Date: Mon, 10 Aug 2026 22:47:40 +0000 Subject: [PATCH 2/4] ficing linting issues. --- genai/model_tuning/evaluate_model_example_test.py | 3 +-- genai/model_tuning/pretrained_codegen_example.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/genai/model_tuning/evaluate_model_example_test.py b/genai/model_tuning/evaluate_model_example_test.py index df9cc129c56..521651a7c46 100644 --- a/genai/model_tuning/evaluate_model_example_test.py +++ b/genai/model_tuning/evaluate_model_example_test.py @@ -14,9 +14,8 @@ import backoff -from google.api_core.exceptions import ResourceExhausted - import evaluate_model_example +from google.api_core.exceptions import ResourceExhausted @backoff.on_exception(backoff.expo, ResourceExhausted, max_time=10) diff --git a/genai/model_tuning/pretrained_codegen_example.py b/genai/model_tuning/pretrained_codegen_example.py index c0e9a926267..ed0b9a9d165 100644 --- a/genai/model_tuning/pretrained_codegen_example.py +++ b/genai/model_tuning/pretrained_codegen_example.py @@ -14,6 +14,7 @@ # [START aiplatform_genai_tune_code_generation_model] import os + from google import genai # TODO (Developer) Set environment variables From 6da633ac51bac8fbc3e1b3b91c4ab05f97cb1b3b Mon Sep 17 00:00:00 2001 From: David del Real Sifuentes Date: Mon, 10 Aug 2026 22:57:13 +0000 Subject: [PATCH 3/4] Fixed gemini comments. --- .../evaluate_model_example_test.py | 2 +- genai/model_tuning/noxfile_config.py | 2 +- .../pretrained_codegen_example.py | 2 +- .../model_tuning/pretrained_examples_test.py | 30 ++++++++++++------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/genai/model_tuning/evaluate_model_example_test.py b/genai/model_tuning/evaluate_model_example_test.py index 521651a7c46..a4f172f60df 100644 --- a/genai/model_tuning/evaluate_model_example_test.py +++ b/genai/model_tuning/evaluate_model_example_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/genai/model_tuning/noxfile_config.py b/genai/model_tuning/noxfile_config.py index 0973c8621c7..650b3c47840 100644 --- a/genai/model_tuning/noxfile_config.py +++ b/genai/model_tuning/noxfile_config.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/genai/model_tuning/pretrained_codegen_example.py b/genai/model_tuning/pretrained_codegen_example.py index ed0b9a9d165..f6bd7983590 100644 --- a/genai/model_tuning/pretrained_codegen_example.py +++ b/genai/model_tuning/pretrained_codegen_example.py @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/genai/model_tuning/pretrained_examples_test.py b/genai/model_tuning/pretrained_examples_test.py index d6359cfd824..dee81db1b99 100644 --- a/genai/model_tuning/pretrained_examples_test.py +++ b/genai/model_tuning/pretrained_examples_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,16 +37,24 @@ def test_tuning_code_generation_model() -> None: client = genai.Client(enterprise=True, project=PROJECT_ID, location=LOCATION_ID) - tuned_model = pretrained_codegen_example.tune_code_generation_model() + tuned_model = None + job_is_finished = False + job_is_pending = False - job_is_pending = tuned_model.state in JOB_STATES_CANCELLABLE - job_is_finished = tuned_model.state in JOB_STATES_DELETABLE - result = job_is_finished or job_is_pending + try: - assert result + tuned_model = pretrained_codegen_example.tune_code_generation_model() - # cleanup - if job_is_pending: - client.tunings.cancel(name=tuned_model.name) - if job_is_finished: - client.models.delete(model=tuned_model.model) + job_is_pending = tuned_model.state in JOB_STATES_CANCELLABLE + job_is_finished = tuned_model.state in JOB_STATES_DELETABLE + result = job_is_finished or job_is_pending + + assert result + + finally: + + # cleanup + if job_is_pending: + client.tunings.cancel(name=tuned_model.name) + if job_is_finished: + client.models.delete(model=tuned_model.model) From 465b1ad361ade7072cdeabb4147ccdfe45f5e77b Mon Sep 17 00:00:00 2001 From: David del Real Sifuentes Date: Tue, 11 Aug 2026 21:51:11 +0000 Subject: [PATCH 4/4] Changes region tag since the sample is automatic --- genai/model_tuning/pretrained_codegen_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genai/model_tuning/pretrained_codegen_example.py b/genai/model_tuning/pretrained_codegen_example.py index f6bd7983590..bd33bc6e080 100644 --- a/genai/model_tuning/pretrained_codegen_example.py +++ b/genai/model_tuning/pretrained_codegen_example.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START aiplatform_genai_tune_code_generation_model] +# [START googlegenaisdk_tune_code_generation_model] import os from google import genai @@ -55,4 +55,4 @@ def tune_code_generation_model() -> genai.types.TuningJob: return tuning_job -# [END aiplatform_genai_tune_code_generation_model] +# [END googlegenaisdk_tune_code_generation_model]