diff --git a/dataflow/conftest.py b/dataflow/conftest.py index 2bb6c6b3ec..d48ebc8f7f 100644 --- a/dataflow/conftest.py +++ b/dataflow/conftest.py @@ -334,6 +334,8 @@ def wait_until( @staticmethod def storage_bucket(name: str) -> str: if bucket_name := os.environ.get("GOOGLE_CLOUD_BUCKET"): + if bucket_name.startswith("gs://"): + bucket_name = bucket_name.replace("gs://", "", 1) logging.warning(f"Using bucket from GOOGLE_CLOUD_BUCKET: {bucket_name}") yield bucket_name return # don't delete diff --git a/dataflow/flex-templates/pipeline_with_dependencies/.gcloudignore b/dataflow/flex-templates/pipeline_with_dependencies/.gcloudignore new file mode 100644 index 0000000000..333da0ffcd --- /dev/null +++ b/dataflow/flex-templates/pipeline_with_dependencies/.gcloudignore @@ -0,0 +1,12 @@ +.gcloudignore +.git +.gitignore +.nox/ +.pytest_cache/ +__pycache__/ +**/*.egg-info/ +*sponge_log.xml +noxfile.py +noxfile_config.py +e2e_test.py +requirements-test.txt diff --git a/dataflow/flex-templates/pipeline_with_dependencies/Dockerfile b/dataflow/flex-templates/pipeline_with_dependencies/Dockerfile index 1ddf03253e..a6503dc935 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/Dockerfile +++ b/dataflow/flex-templates/pipeline_with_dependencies/Dockerfile @@ -24,14 +24,14 @@ # This Dockerfile illustrates how to use a custom base image when building # a custom contaier images for Dataflow. A 'slim' base image is smaller in size, # but does not include some preinstalled libraries, like google-cloud-debugger. -# To use a standard image, use apache/beam_python3.14_sdk:2.76.0 instead. +# To use a standard image, use apache/beam_python3.13_sdk:2.77.0 instead. # Use consistent versions of Python interpreter in the project. -FROM python:3.14-slim +FROM python:3.13-slim # Copy SDK entrypoint binary from Apache Beam image, which makes it possible to # use the image as SDK container image. If you explicitly depend on # apache-beam in setup.py, use the same version of Beam in both files. -COPY --from=apache/beam_python3.14_sdk:2.76.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.13_sdk:2.77.0 /opt/apache/beam /opt/apache/beam # Copy Flex Template launcher binary from the launcher image, which makes it # possible to use the image as a Flex Template base image. diff --git a/dataflow/flex-templates/pipeline_with_dependencies/noxfile_config.py b/dataflow/flex-templates/pipeline_with_dependencies/noxfile_config.py index c3badedaab..8d5df10b17 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/noxfile_config.py +++ b/dataflow/flex-templates/pipeline_with_dependencies/noxfile_config.py @@ -18,6 +18,9 @@ # You can opt out from the test for specific Python versions. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) - "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.13) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.14"], + "envs": { + "CLOUDSDK_BUILDS_USE_KANIKO": "True", + }, } diff --git a/dataflow/flex-templates/pipeline_with_dependencies/pyproject.toml b/dataflow/flex-templates/pipeline_with_dependencies/pyproject.toml index 03aeed04ed..a9bc14157e 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/pyproject.toml +++ b/dataflow/flex-templates/pipeline_with_dependencies/pyproject.toml @@ -20,6 +20,6 @@ build-backend = "setuptools.build_meta" name = "my_package" version = "0.1.0" dependencies = [ - "apache-beam[gcp]==2.76.0", # Must match the version in `Dockerfile``. + "apache-beam[gcp]==2.77.0", # Must match the version in `Dockerfile``. "pyfiglet", # This is the only non-Beam dependency of this pipeline. ] diff --git a/dataflow/flex-templates/pipeline_with_dependencies/requirements-test.txt b/dataflow/flex-templates/pipeline_with_dependencies/requirements-test.txt index f686d5a942..2231980787 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/requirements-test.txt +++ b/dataflow/flex-templates/pipeline_with_dependencies/requirements-test.txt @@ -14,8 +14,8 @@ # This is a test configuration file. It is not a part of the example. -google-api-python-client==2.87.0 -google-cloud-storage==2.9.0 -pytest-xdist==3.3.0 -pytest==9.0.3 -pyyaml==6.0 +google-api-python-client==2.201.0 +google-cloud-storage==3.17.1 +pytest-xdist==3.8.0 +pytest==9.1.1 +pyyaml==6.0.3 diff --git a/dataflow/flex-templates/pipeline_with_dependencies/requirements.txt b/dataflow/flex-templates/pipeline_with_dependencies/requirements.txt index 7d946a0d3b..ca9dbcb3ad 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/requirements.txt +++ b/dataflow/flex-templates/pipeline_with_dependencies/requirements.txt @@ -2,14 +2,11 @@ # This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --output-file=requirements.txt pyproject.toml +# pip-compile --no-emit-index-url --no-emit-trusted-host --output-file=requirements.txt # ---index-url http://airlock-proxy.uplink.goog:999/python/artifact-foundry-prod/ah-3p-staging-python/simple/ ---trusted-host airlock-proxy.uplink.goog - aiohappyeyeballs==2.7.1 # via aiohttp -aiohttp==3.14.3 +aiohttp==3.14.4 # via cloud-sql-python-connector aiosignal==1.4.0 # via aiohttp @@ -19,8 +16,8 @@ anyio==4.15.1 # via # google-genai # httpx -apache-beam[gcp]==2.76.0 - # via my_package (pyproject.toml) +apache-beam[gcp]==2.77.0 + # via my_package (setup.py) asn1crypto==1.5.1 # via scramp attrs==26.1.0 @@ -37,11 +34,11 @@ certifi==2026.7.22 # requests cffi==2.1.1 # via cryptography -charset-normalizer==3.5.1 +charset-normalizer==3.5.2 # via requests cloud-sql-python-connector==1.22.0 # via apache-beam -cryptography==50.0.1 +cryptography==50.0.2 # via # apache-beam # cloud-sql-python-connector @@ -55,7 +52,7 @@ dnspython==2.8.0 # pymongo docstring-parser==0.18.0 # via google-cloud-aiplatform -fastavro==1.12.2 +fastavro==1.13.0 # via apache-beam fasteners==0.20 # via @@ -65,7 +62,7 @@ frozenlist==1.8.0 # via # aiohttp # aiosignal -google-api-core[grpc]==2.36.0 +google-api-core[grpc]==2.42.0 # via # apache-beam # google-cloud-aiplatform @@ -90,7 +87,7 @@ google-api-core[grpc]==2.36.0 # google-cloud-vision google-apitools==0.5.35 # via apache-beam -google-auth[pyopenssl,requests]==2.57.1 +google-auth[pyopenssl,requests]==2.62.0 # via # apache-beam # cloud-sql-python-connector @@ -122,17 +119,17 @@ google-auth-httplib2==0.2.1 # via apache-beam google-cloud-aiplatform==1.165.1 # via apache-beam -google-cloud-bigquery==3.45.0 +google-cloud-bigquery==3.46.1 # via # apache-beam # google-cloud-aiplatform -google-cloud-bigquery-storage==2.41.0 +google-cloud-bigquery-storage==2.42.0 # via apache-beam -google-cloud-bigtable==2.44.0 +google-cloud-bigtable==2.50.0 # via apache-beam -google-cloud-build==3.39.0 +google-cloud-build==3.40.0 # via apache-beam -google-cloud-core==2.7.0 +google-cloud-core==2.8.0 # via # apache-beam # google-cloud-bigquery @@ -142,50 +139,50 @@ google-cloud-core==2.7.0 # google-cloud-storage google-cloud-dataflow-client==0.13.0 # via apache-beam -google-cloud-datastore==2.26.1 +google-cloud-datastore==2.27.0 # via apache-beam -google-cloud-dlp==3.39.0 +google-cloud-dlp==3.40.0 # via apache-beam -google-cloud-kms==3.16.0 +google-cloud-kms==3.18.0 # via apache-beam -google-cloud-language==2.21.0 +google-cloud-language==2.22.0 # via apache-beam -google-cloud-monitoring==2.31.0 +google-cloud-monitoring==2.32.0 # via # google-cloud-bigtable # google-cloud-spanner -google-cloud-pubsub==2.40.0 +google-cloud-pubsub==2.42.0 # via apache-beam google-cloud-recommendations-ai==0.10.18 # via apache-beam -google-cloud-resource-manager==1.18.0 +google-cloud-resource-manager==1.19.0 # via # apache-beam # google-cloud-aiplatform -google-cloud-secret-manager==2.30.0 +google-cloud-secret-manager==2.31.0 # via apache-beam google-cloud-spanner==3.71.0 # via apache-beam -google-cloud-storage==3.13.1 +google-cloud-storage==3.17.0 # via # apache-beam # google-cloud-aiplatform -google-cloud-videointelligence==2.20.0 +google-cloud-videointelligence==2.21.0 # via apache-beam -google-cloud-vision==3.15.0 +google-cloud-vision==3.16.0 # via apache-beam -google-crc32c==1.8.0 +google-crc32c==1.9.0 # via # google-cloud-bigtable # google-cloud-storage # google-resumable-media -google-genai==2.22.0 +google-genai==2.30.1 # via google-cloud-aiplatform -google-resumable-media==2.10.2 +google-resumable-media==2.11.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.75.3 +googleapis-common-protos[grpc]==1.75.5 # via # cloud-sql-python-connector # google-api-core @@ -202,7 +199,7 @@ grpc-google-iam-v1==0.14.5 # google-cloud-spanner grpc-interceptor==0.15.4 # via google-cloud-spanner -grpcio==1.83.1 +grpcio==1.84.0 # via # apache-beam # cloud-sql-python-connector @@ -226,7 +223,7 @@ grpcio==1.83.1 # grpc-google-iam-v1 # grpc-interceptor # grpcio-status -grpcio-status==1.83.1 +grpcio-status==1.84.0 # via # google-api-core # google-cloud-pubsub @@ -242,7 +239,7 @@ httplib2==0.32.0 # oauth2client httpx==0.28.1 # via google-genai -idna==3.19 +idna==3.20 # via # anyio # httpx @@ -258,19 +255,19 @@ jeepney==0.9.0 # via # keyring # secretstorage -jsonpickle==4.1.2 +jsonpickle==4.1.3 # via apache-beam keyring==25.7.0 # via keyrings-google-artifactregistry-auth keyrings-google-artifactregistry-auth==1.1.2 # via apache-beam -mmh3==5.3.0 +mmh3==5.3.1 # via google-cloud-spanner more-itertools==11.1.0 # via # jaraco-classes # jaraco-functools -multidict==6.7.1 +multidict==6.9.1 # via # aiohttp # yarl @@ -280,7 +277,7 @@ oauth2client==4.1.3 # via google-apitools objsize==0.7.1 # via apache-beam -opentelemetry-api==1.44.0 +opentelemetry-api==1.45.1 # via # google-api-core # google-cloud-bigtable @@ -291,17 +288,17 @@ opentelemetry-api==1.44.0 # opentelemetry-semantic-conventions opentelemetry-resourcedetector-gcp==1.14.0 # via google-cloud-spanner -opentelemetry-sdk==1.44.0 +opentelemetry-sdk==1.45.1 # via # google-cloud-bigtable # google-cloud-pubsub # google-cloud-spanner # opentelemetry-resourcedetector-gcp -opentelemetry-semantic-conventions==0.65b0 +opentelemetry-semantic-conventions==0.66b1 # via # google-cloud-spanner # opentelemetry-sdk -orjson==3.12.0 +orjson==3.13.0 # via apache-beam packaging==26.3 # via @@ -314,11 +311,11 @@ pillow==12.3.0 # via apache-beam pluggy==1.6.0 # via keyrings-google-artifactregistry-auth -propcache==0.5.2 +propcache==0.5.4 # via # aiohttp # yarl -proto-plus==1.28.4 +proto-plus==1.29.0 # via # apache-beam # google-api-core @@ -385,12 +382,12 @@ pydantic==2.13.5 pydantic-core==2.46.5 # via pydantic pyfiglet==1.0.4 - # via my_package (pyproject.toml) -pymongo==4.18.0 + # via my_package (setup.py) +pymongo==4.18.2 # via apache-beam -pymysql==1.2.0 +pymysql==1.2.3 # via apache-beam -pyparsing==3.3.2 +pyparsing==3.3.3 # via httplib2 python-dateutil==2.9.0.post0 # via @@ -399,11 +396,11 @@ python-dateutil==2.9.0.post0 # pg8000 python-tds==1.17.1 # via apache-beam -pytz==2026.3.post1 +pytz==2026.5 # via apache-beam pyyaml==6.0.3 # via apache-beam -regex==2026.9.3 +regex==2026.9.29 # via apache-beam requests==2.34.2 # via @@ -450,11 +447,11 @@ typing-extensions==4.16.0 # typing-inspection typing-inspection==0.4.4 # via pydantic -urllib3==2.7.0 +urllib3==2.8.0 # via requests websockets==16.1.1 # via google-genai -yarl==1.24.5 +yarl==1.25.1 # via aiohttp zstandard==0.25.0 # via apache-beam diff --git a/dataflow/flex-templates/pipeline_with_dependencies/setup.py b/dataflow/flex-templates/pipeline_with_dependencies/setup.py index 3316b9b522..22e708817c 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/setup.py +++ b/dataflow/flex-templates/pipeline_with_dependencies/setup.py @@ -20,4 +20,5 @@ """ import setuptools + setuptools.setup()