diff --git a/genai/model_tuning/evaluate_model_example.py b/genai/model_tuning/evaluate_model_example.py new file mode 100644 index 0000000000..25850bf5d0 --- /dev/null +++ b/genai/model_tuning/evaluate_model_example.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_genai_evaluate_model] +import os + +import vertexai +from vertexai.evaluation import EvalResult, EvalTask + +# TODO (Developer) Set environment variables +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") +LOCATION_ID = os.getenv("LOCATION_ID", "us-central1") +MODEL_NAME = os.getenv("MODEL_NAME", "gemini-2.5-flash") + + +def evaluate_model() -> EvalResult: + """Evaluate the performance of a generative AI model.""" + + vertexai.init(project=PROJECT_ID, location=LOCATION_ID) + + # Dataset URI containing input prompts and ground truth labels + dataset_uri = "gs://cloud-samples-data/ai-platform/generative_ai/llm_classification_bp_input_prompts_with_ground_truth.jsonl" + + metric_column_mapping = {"reference": "ground_truth"} + + # Define evaluation task + eval_task = EvalTask( + dataset=dataset_uri, + metrics=["exact_match"], + experiment="gemini-classification-eval", + metric_column_mapping=metric_column_mapping, + ) + + # Define a prompt template so the generative Gemini model produces formatted labels + prompt_template = ( + "Classify the following text into exactly one category from " + "[nature, news, sports, health, startups]. Return only the category name:\n\n{prompt}" + ) + + # Evaluate using a modern Gemini model + eval_result = eval_task.evaluate( + model=MODEL_NAME, + prompt_template=prompt_template, + ) + + print("=== SUMMARY METRICS ===") + print(eval_result.summary_metrics) + + print("\n=== METRICS TABLE SAMPLE ===") + print(eval_result.metrics_table.head()) + + return eval_result + + +# [END aiplatform_genai_evaluate_model] diff --git a/genai/model_tuning/evaluate_model_example_test.py b/genai/model_tuning/evaluate_model_example_test.py new file mode 100644 index 0000000000..a4f172f60d --- /dev/null +++ b/genai/model_tuning/evaluate_model_example_test.py @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import backoff + +import evaluate_model_example +from google.api_core.exceptions import ResourceExhausted + + +@backoff.on_exception(backoff.expo, ResourceExhausted, max_time=10) +def test_evaluate_model() -> None: + eval_metrics = evaluate_model_example.evaluate_model() + assert hasattr(eval_metrics, "metrics_table") diff --git a/genai/model_tuning/noxfile_config.py b/genai/model_tuning/noxfile_config.py new file mode 100644 index 0000000000..650b3c4784 --- /dev/null +++ b/genai/model_tuning/noxfile_config.py @@ -0,0 +1,42 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be imported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": True, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} diff --git a/genai/model_tuning/pretrained_codegen_example.py b/genai/model_tuning/pretrained_codegen_example.py new file mode 100644 index 0000000000..bd33bc6e08 --- /dev/null +++ b/genai/model_tuning/pretrained_codegen_example.py @@ -0,0 +1,58 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START googlegenaisdk_tune_code_generation_model] +import os + +from google import genai + +# TODO (Developer) Set environment variables +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") +LOCATION_ID = os.getenv("LOCATION_ID", "us-central1") + +# Resource format: 'publishers/google/models/{model_id}' +BASE_MODEL_RESOURCE = "publishers/google/models/gemini-2.5-flash" +TRAINING_DATASET = ( + "gs://cloud-samples-data/ai-platform/generative_ai/gemini/text/sft_train_data.jsonl" +) + + +def tune_code_generation_model() -> genai.types.TuningJob: + """Submits a supervised fine-tuning job for a Gemini model on code/text tasks.""" + + client = genai.Client( + enterprise=True, + project=PROJECT_ID, + location=LOCATION_ID, + ) + + tuning_job = client.tunings.tune( + base_model=BASE_MODEL_RESOURCE, + training_dataset=genai.types.TuningDataset( + gcs_uri=TRAINING_DATASET, + ), + config=genai.types.CreateTuningJobConfig( + tuned_model_display_name="tuned_gemini_code_model", + epoch_count=2, + learning_rate_multiplier=1.0, + ), + ) + + print(f"Tuning job submitted successfully: {tuning_job.name}") + print(f"Current State: {tuning_job.state}") + + return tuning_job + + +# [END googlegenaisdk_tune_code_generation_model] diff --git a/genai/model_tuning/pretrained_examples_test.py b/genai/model_tuning/pretrained_examples_test.py new file mode 100644 index 0000000000..dee81db1b9 --- /dev/null +++ b/genai/model_tuning/pretrained_examples_test.py @@ -0,0 +1,60 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google import genai + +import pretrained_codegen_example + +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") +LOCATION_ID = os.getenv("LOCATION_ID", "us-central1") + +JOB_STATES_CANCELLABLE = [ + genai.types.JobState.JOB_STATE_RUNNING, + genai.types.JobState.JOB_STATE_PENDING, +] +JOB_STATES_DELETABLE = [ + genai.types.JobState.JOB_STATE_SUCCEEDED, + genai.types.JobState.JOB_STATE_CANCELLED, + genai.types.JobState.JOB_STATE_FAILED, +] + + +def test_tuning_code_generation_model() -> None: + """Validate tuning creation, and cleans after execution.""" + + client = genai.Client(enterprise=True, project=PROJECT_ID, location=LOCATION_ID) + + tuned_model = None + job_is_finished = False + job_is_pending = False + + try: + + tuned_model = pretrained_codegen_example.tune_code_generation_model() + + job_is_pending = tuned_model.state in JOB_STATES_CANCELLABLE + job_is_finished = tuned_model.state in JOB_STATES_DELETABLE + result = job_is_finished or job_is_pending + + assert result + + finally: + + # cleanup + if job_is_pending: + client.tunings.cancel(name=tuned_model.name) + if job_is_finished: + client.models.delete(model=tuned_model.model) diff --git a/genai/model_tuning/requirements-test.txt b/genai/model_tuning/requirements-test.txt new file mode 100644 index 0000000000..28344f9b09 --- /dev/null +++ b/genai/model_tuning/requirements-test.txt @@ -0,0 +1,3 @@ +backoff==2.2.1 +google-api-core==2.33.0 +pytest==9.1.1 diff --git a/genai/model_tuning/requirements.txt b/genai/model_tuning/requirements.txt new file mode 100644 index 0000000000..d5ef3b41e1 --- /dev/null +++ b/genai/model_tuning/requirements.txt @@ -0,0 +1,2 @@ +google-genai==2.16.0 +google-cloud-aiplatform[pipelines, evaluation]==1.163.0