diff --git a/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/.gitignore b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/.gitignore new file mode 100644 index 000000000..2736243b8 --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/.gitignore @@ -0,0 +1,9 @@ +build/ +dist/ +.aws-sam/ +history-*/ +report-*.xml +report-*.json +__pycache__/ +*.pyc +output/ diff --git a/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/README.md b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/README.md new file mode 100644 index 000000000..c7c66ab11 --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/README.md @@ -0,0 +1,80 @@ +# Durable Execution Python SDK: Durable Execution Telemetry Conformance Tests + +This package answers one question. +Does [`aws-durable-execution-sdk-python-telemetry-exporter`](../aws-durable-execution-sdk-python-telemetry-exporter), with its `CloudWatchLogsExporter`, deliver a complete `DurableExecutionTelemetry` record to CloudWatch Logs for a real Lambda execution? + +Unit tests can assert that the plugin builds a record. +They cannot assert that the record reaches the configured log group on a real invocation. +They cannot assert that it carries the right identity fields for the deployed function. +Those are properties of a deployed function. +So the scenario here is a Lambda function. +The runner deploys it with the plugin configured and invokes it once. +It then fetches the record from CloudWatch Logs and checks it against the requirement. + +The requirement and the runner live in [`aws/aws-durable-execution-conformance-tests`](https://github.com/aws/aws-durable-execution-conformance-tests), package `aws-durable-execution-conformance-tests-telemetry`. +This package holds the SDK-specific part: the handler and the SAM template that maps it to the requirement ID. + +## Coverage + +| Suite | Requirements | What it pins down | +| ----------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `telemetry` | `telemetry-1` | One successful single-step execution emits one `SUCCEEDED` record with `input`, `output`, ARN-derived identity fields, and `operationsByName`. | + +## Layout + +``` +src/ + telemetry_1_success.py # single step "increment"; plugin + CloudWatchLogsExporter + Makefile # build-Telemetry1Success target used by `sam build` + requirements.txt # core SDK + telemetry exporter, pinned to ${PYTHON_SDK_REF} +template.yaml # declares TelemetryLogGroupName and creates the group +tests/ + test_telemetry_examples.py # guards template -> handler wiring and the parameter contract +``` + +The handler uses the real SDK API only. +It constructs no record and calls no exporter method. +A missing or wrong record is the plugin's defect and is the signal the suite measures. + +## Template contract + +The runner passes `TelemetryLogGroupName` from the extension. +Extensions have no pre-deploy hook. +So the template creates the group with an `AWS::Logs::LogGroup` resource, and the function depends on it. +The group name reaches the handler through `TELEMETRY_LOG_GROUP_NAME`. +The handler raises at import if that variable is unset. + +The `LambdaExecutionRoleArn` role must allow `logs:CreateLogStream` and `logs:PutLogEvents` on the group. +`AWSLambdaBasicDurableExecutionRolePolicy` grants both on `*`. + +## Building locally + +Prerequisites: Python 3.13 and the AWS SAM CLI. + +`src/requirements.txt` installs both SDK packages from git at `${PYTHON_SDK_REF}`. +The ref must be a pushed commit that contains the packages. + +```bash +export PYTHON_SDK_REF=$(git rev-parse HEAD) # must already be pushed +sam build --template packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/template.yaml +``` + +Run the wiring tests from the repository root: + +```bash +hatch run test:all packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/tests +``` + +## Running the suite + +Run from a checkout of the conformance repository with the telemetry package installed: + +```bash +durable-execution-conformance \ + --template /packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/template.yaml \ + --language python --region us-west-2 --suite telemetry \ + --report console json \ + --parameter-overrides LambdaExecutionRoleArn= +``` + +A caller GitHub workflow is a follow-on. diff --git a/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/pyproject.toml b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/pyproject.toml new file mode 100644 index 000000000..e7b683fbb --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["hatchling!=1.32.1,<1.33"] +build-backend = "hatchling.build" + +[project] +name = "aws-durable-execution-sdk-python-conformance-tests-telemetry" +version = "0.0.0" +description = "Durable Execution Telemetry conformance test handlers for the AWS Durable Execution SDK for Python, exercised by the aws-durable-execution-conformance-tests telemetry suite." +requires-python = ">=3.11" +dependencies = [ + "aws-durable-execution-sdk-python==3.0.0", + "aws-durable-execution-sdk-python-telemetry-exporter==0.0.1", +] + +[tool.hatch.build.targets.wheel] +# The handlers are flat Lambda modules deployed by SAM from src/, not an +# importable package, so there is nothing to ship in a wheel. +bypass-selection = true + +[tool.ruff] +line-length = 88 +target-version = "py311" + +[tool.ruff.lint] +preview = true +select = ["E4", "E7", "E9", "F", "TID252"] # pycodestyle (E4/E7/E9) + Pyflakes + absolute imports + +[tool.ruff.lint.isort] +known-first-party = [ + "aws_durable_execution_sdk_python", + "aws_durable_execution_sdk_python_telemetry_exporter", +] +force-single-line = false +lines-after-imports = 2 + +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "ARG001", + "ARG002", + "ARG005", + "E402", + "S101", + "PLR2004", + "SIM117", + "TRY301", +] diff --git a/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/Makefile b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/Makefile new file mode 100644 index 000000000..ea39cfdf4 --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/Makefile @@ -0,0 +1,11 @@ +.PHONY: build-Telemetry1Success + +build-Telemetry1Success: + python3 -m pip install --upgrade \ + --target "$(ARTIFACTS_DIR)" \ + --platform manylinux2014_x86_64 \ + --implementation cp \ + --python-version 3.13 \ + --only-binary=:all: \ + -r requirements.txt + cp ./*.py "$(ARTIFACTS_DIR)/" diff --git a/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/requirements.txt b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/requirements.txt new file mode 100644 index 000000000..103a92c37 --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/requirements.txt @@ -0,0 +1,2 @@ +aws-durable-execution-sdk-python @ git+https://github.com/aws/aws-durable-execution-sdk-python.git@${PYTHON_SDK_REF}#subdirectory=packages/aws-durable-execution-sdk-python +aws-durable-execution-sdk-python-telemetry-exporter @ git+https://github.com/aws/aws-durable-execution-sdk-python.git@${PYTHON_SDK_REF}#subdirectory=packages/aws-durable-execution-sdk-python-telemetry-exporter diff --git a/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/telemetry_1_success.py b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/telemetry_1_success.py new file mode 100644 index 000000000..d661146dd --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/src/telemetry_1_success.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +"""Successful single-step execution for requirement telemetry-1. + +The handler uses the real SDK API only. It builds no record and calls no +exporter method. A missing or wrong record is the plugin's defect, and that +defect is the signal the suite measures. +""" + +from __future__ import annotations + +import os +from typing import Any + +from aws_durable_execution_sdk_python import ( + DurableContext, + StepContext, + durable_execution, + durable_step, +) +from aws_durable_execution_sdk_python_telemetry_exporter import ( + CloudWatchLogsExporter, + ContentConfig, + DurableExecutionTelemetryConfig, + durable_execution_telemetry, +) + + +# The template sets this variable. Without it the exporter has no target. +# So the module fails at import instead of dropping records silently. +log_group_name = os.environ.get("TELEMETRY_LOG_GROUP_NAME") +if not log_group_name: + raise RuntimeError("TELEMETRY_LOG_GROUP_NAME must be set") + + +@durable_step +def increment(_step_context: StepContext, value: int) -> int: + return value + 1 + + +@durable_execution( + plugins=[ + durable_execution_telemetry( + DurableExecutionTelemetryConfig( + exporters=[CloudWatchLogsExporter(log_group_name=log_group_name)], + emit_mode="on-complete", + sampling_rate=1, + operation_detail="top-level", + content=ContentConfig(input=True, output=True), + ) + ) + ] +) +def handler(event: dict[str, Any], context: DurableContext) -> dict[str, int]: + result: int = context.step(increment(event["value"]), name="increment") + return {"value": result} diff --git a/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/template.yaml b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/template.yaml new file mode 100644 index 000000000..668e21643 --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/template.yaml @@ -0,0 +1,48 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: Python SDK examples for the Durable Execution Telemetry conformance suite + +Parameters: + LambdaExecutionRoleArn: + Type: String + Description: Pre-created execution role for the test functions + TelemetryLogGroupName: + Type: String + Description: CloudWatch Logs group that receives Durable Execution Telemetry records + +Globals: + Function: + Runtime: python3.13 + Architectures: + - x86_64 + Timeout: 60 + MemorySize: 256 + DurableConfig: + ExecutionTimeout: 300 + RetentionPeriodInDays: 1 + Environment: + Variables: + TELEMETRY_LOG_GROUP_NAME: !Ref TelemetryLogGroupName + +Resources: + # Extensions have no pre-deploy hook, so CloudFormation creates the group + # before the runner invokes any function. + TelemetryLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Ref TelemetryLogGroupName + RetentionInDays: 1 + + Telemetry1Success: + Type: AWS::Serverless::Function + DependsOn: TelemetryLogGroup + Metadata: + BuildMethod: makefile + TestingMetadata: + TestDescription: + - telemetry-1 + Properties: + CodeUri: src/ + Handler: telemetry_1_success.handler + FunctionName: !Sub "${AWS::StackName}-telemetry-1" + Role: !Ref LambdaExecutionRoleArn diff --git a/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/tests/test_telemetry_examples.py b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/tests/test_telemetry_examples.py new file mode 100644 index 000000000..64d3af368 --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/tests/test_telemetry_examples.py @@ -0,0 +1,188 @@ +# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +"""Contract tests for the Python Durable Execution Telemetry conformance handlers. + +The conformance runner reads requirement mappings from +``TestingMetadata.TestDescription``. SAM builds each function through +``src/Makefile``. A renamed handler or a missing build target fails only when a +cloud deployment runs. So these tests pin that wiring locally. +""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path +from typing import Any + +import pytest +import yaml + + +PKG_DIR = Path(__file__).resolve().parents[1] +SRC_DIR = PKG_DIR / "src" +TEMPLATE_PATH = PKG_DIR / "template.yaml" + +SDK_REPOSITORY_URL = "git+https://github.com/aws/aws-durable-execution-sdk-python.git" +EXPECTED_COVERAGE: list[str] = ["telemetry-1"] +EXPECTED_MODULES: frozenset[str] = frozenset({"telemetry_1_success"}) + + +class CfnLoader(yaml.SafeLoader): + """SafeLoader that keeps CloudFormation short-form tags as (tag, value).""" + + +def _construct_cfn_tag( + loader: CfnLoader, _suffix: str, node: yaml.Node +) -> tuple[str, Any]: + value: Any + if isinstance(node, yaml.ScalarNode): + value = loader.construct_scalar(node) + elif isinstance(node, yaml.SequenceNode): + value = loader.construct_sequence(node, deep=True) + else: + value = loader.construct_mapping(node, deep=True) + return (node.tag, value) + + +CfnLoader.add_multi_constructor("!", _construct_cfn_tag) + + +def load_template(path: Path) -> dict[str, Any]: + """Load a template like yaml.safe_load, keeping short-form tags.""" + with path.open(encoding="utf-8") as stream: + loader = CfnLoader(stream) + try: + template: dict[str, Any] = loader.get_single_data() + finally: + loader.dispose() + return template + + +def functions(template: dict[str, Any]) -> dict[str, dict[str, Any]]: + return { + logical_id: resource + for logical_id, resource in template["Resources"].items() + if resource["Type"] == "AWS::Serverless::Function" + } + + +@pytest.fixture(scope="module") +def template() -> dict[str, Any]: + return load_template(TEMPLATE_PATH) + + +@pytest.fixture(scope="module") +def makefile() -> str: + return (SRC_DIR / "Makefile").read_text(encoding="utf-8") + + +def test_coverage_is_exactly_telemetry_1(template: dict[str, Any]) -> None: + coverage: list[str] = [ + description + for resource in functions(template).values() + for description in resource["TestingMetadata"]["TestDescription"] + ] + + assert coverage == EXPECTED_COVERAGE + + +def test_every_function_points_at_an_existing_handler_module( + template: dict[str, Any], +) -> None: + for resource in functions(template).values(): + module, _, attribute = resource["Properties"]["Handler"].partition(".") + source = SRC_DIR / f"{module}.py" + + assert source.is_file() + assert attribute == "handler" + tree = ast.parse(source.read_text(encoding="utf-8"), filename=str(source)) + assert any( + isinstance(node, ast.FunctionDef) and node.name == "handler" + for node in tree.body + ) + + +def test_every_function_builds_from_src_with_the_makefile( + template: dict[str, Any], +) -> None: + for resource in functions(template).values(): + assert resource["Metadata"]["BuildMethod"] == "makefile" + assert resource["Properties"]["CodeUri"] == "src/" + + +def test_makefile_has_a_build_target_per_function( + template: dict[str, Any], makefile: str +) -> None: + for logical_id in functions(template): + assert re.search(rf"^build-{logical_id}:", makefile, re.MULTILINE) + assert re.search(rf"^\.PHONY:.*\bbuild-{logical_id}\b", makefile, re.MULTILINE) + + +def test_handler_modules_are_exactly_the_expected_set() -> None: + assert {path.stem for path in SRC_DIR.glob("*.py")} == EXPECTED_MODULES + + +def test_requirements_install_both_packages_from_one_commit() -> None: + requirements: str = (SRC_DIR / "requirements.txt").read_text(encoding="utf-8") + + for package in ( + "aws-durable-execution-sdk-python", + "aws-durable-execution-sdk-python-telemetry-exporter", + ): + assert ( + f"{package} @ {SDK_REPOSITORY_URL}@${{PYTHON_SDK_REF}}" + f"#subdirectory=packages/{package}" + ) in requirements + + +def test_log_group_parameter_is_declared(template: dict[str, Any]) -> None: + assert template["Parameters"]["TelemetryLogGroupName"]["Type"] == "String" + assert template["Parameters"]["LambdaExecutionRoleArn"]["Type"] == "String" + + +def test_log_group_is_created_from_the_parameter(template: dict[str, Any]) -> None: + group: dict[str, Any] = template["Resources"]["TelemetryLogGroup"] + + assert group["Type"] == "AWS::Logs::LogGroup" + assert group["Properties"]["LogGroupName"] == ("!Ref", "TelemetryLogGroupName") + assert group["Properties"]["RetentionInDays"] == 1 + + +def test_every_function_depends_on_the_log_group(template: dict[str, Any]) -> None: + for resource in functions(template).values(): + assert resource["DependsOn"] == "TelemetryLogGroup" + + +def test_log_group_name_reaches_the_handler_environment( + template: dict[str, Any], +) -> None: + variables: dict[str, Any] = template["Globals"]["Function"]["Environment"][ + "Variables" + ] + + assert variables["TELEMETRY_LOG_GROUP_NAME"] == ("!Ref", "TelemetryLogGroupName") + + +def test_function_settings_match_the_contract(template: dict[str, Any]) -> None: + function: dict[str, Any] = functions(template)["Telemetry1Success"] + + assert function["Properties"]["FunctionName"] == ( + "!Sub", + "${AWS::StackName}-telemetry-1", + ) + assert function["Properties"]["Role"] == ("!Ref", "LambdaExecutionRoleArn") + assert template["Globals"]["Function"]["Runtime"] == "python3.13" + assert template["Globals"]["Function"]["DurableConfig"] == { + "ExecutionTimeout": 300, + "RetentionPeriodInDays": 1, + } + + +def test_handler_reads_the_log_group_from_the_environment() -> None: + source: str = (SRC_DIR / "telemetry_1_success.py").read_text(encoding="utf-8") + + assert 'os.environ.get("TELEMETRY_LOG_GROUP_NAME")' in source + assert "raise RuntimeError" in source + assert "CloudWatchLogsExporter(log_group_name=log_group_name)" in source diff --git a/pyproject.toml b/pyproject.toml index 621244358..d86a1e8d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ testpaths = [ "packages/aws-durable-execution-sdk-python-examples/test", "packages/aws-durable-execution-sdk-python-conformance-tests/tests", "packages/aws-durable-execution-sdk-python-conformance-tests-otel/tests", + "packages/aws-durable-execution-sdk-python-conformance-tests-telemetry/tests", ] markers = [ # Used for test selection with -m example