Add OpenTelemetry support to AWS binary

cmd/tesseract/aws/README.md

@@ -16,6 +16,54 @@ For AWS-specific information on how to run this binary, see the documentation un
 TesseraCT expects the databases configured with the `db_name` and
 `antispam_db_name` flags to be located in the same Aurora DB cluster.
 
+## OpenTelemetry
+
+The AWS binary configures OpenTelemetry metrics, tracing, and Go runtime metrics
+using the same environment-driven exporters as the POSIX binary. The service name
+is the log's `--origin`, and the service namespace is `tesseract`.
+`OTEL_RESOURCE_ATTRIBUTES` can add or override resource attributes.
+
+For OTLP export to a collector, for example:
+
+```bash
+export OTEL_METRICS_EXPORTER=otlp
+export OTEL_TRACES_EXPORTER=otlp
+export OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+export OTEL_EXPORTER_OTLP_ENDPOINT=https://collector.example.com:4317
+```
+
+Set `--trace_fraction` to a value from `0` to `1` for the fraction of root traces
+to sample. The default is `0`; periodic Tessera root spans marked with
+`tessera.periodic` are still sampled. Child spans follow their parent's sampling
+decision. Set `OTEL_TRACES_EXPORTER=none` to disable trace export, or
+`OTEL_METRICS_EXPORTER=none` to disable metric export. Exporter selection defaults
+to OTLP when unset. This does not automatically configure CloudWatch or X-Ray.
+
+### Prometheus
+
+To expose a Prometheus scrape endpoint instead of pushing metrics over OTLP:
+
+```bash
+export OTEL_METRICS_EXPORTER=prometheus
+export OTEL_EXPORTER_PROMETHEUS_HOST=127.0.0.1
+export OTEL_EXPORTER_PROMETHEUS_PORT=9464
+export OTEL_TRACES_EXPORTER=none
+```
+
+Metrics are available at `http://127.0.0.1:9464/metrics` on a separate listener,
+not the CT HTTP endpoint. The default host is `localhost` and the default port is
+`9464`. Tracing is configured independently and can remain enabled if desired.
+
+Remote scrapers require a non-loopback bind address such as `0.0.0.0`, along with
+appropriate container port mappings and security group rules. Restrict access to
+trusted scrapers: the metrics listener does not provide authentication or TLS.
+Use a TLS/authentication proxy where required, validate deployment inputs, and
+inject exporter credentials securely rather than hardcoding secrets.
+
+Telemetry providers shut down after HTTP requests drain on normal termination.
+As in POSIX, startup failures that call `os.Exit` bypass deferred cleanup, and
+telemetry shutdown does not have a separate timeout.
+
 ## Vanilla S3+MySQL support
 
 Setting up S3 and MySQL infrastructure is out of scope for this document, but

cmd/tesseract/aws/main.go

@@ -111,6 +111,7 @@ var (
 	signerPublicKeyFile        = flag.String("signer_public_key_file", "", "Path to public key file for checkpoints and SCTs signer (alternative to secrets manager)")
 	signerPrivateKeyFile       = flag.String("signer_private_key_file", "", "Path to private key file for checkpoints and SCTs signer (alternative to secrets manager)")
 	usePathStyle               = flag.Bool("s3_use_path_style", false, "Whether to force the AWS S3 client to use path-style bucket references, probably only useful for on-prem deployments")
+	traceFraction              = flag.Float64("trace_fraction", 0, "Fraction of open-telemetry span traces to sample")
 	slogLevel                  = flag.Int("slog_level", 0, "The cut-off threshold for structured logging. See cmd/tesseract/README.md#Logging.")
 )
 
@@ -120,6 +121,9 @@ func main() {
 	ctx := context.Background()
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(*slogLevel)})))
 
+	shutdownOTel := initOTel(ctx, *traceFraction, *origin)
+	defer shutdownOTel(ctx)
+
 	var signer *ECDSAWithSHA256Signer
 	var err error
 

cmd/tesseract/aws/otel.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"os"
+
+	t_otel "github.com/transparency-dev/tesseract/internal/otel"
+	"go.opentelemetry.io/contrib/exporters/autoexport"
+	"go.opentelemetry.io/contrib/instrumentation/runtime"
+	"go.opentelemetry.io/otel"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+)
+
+func initOTel(ctx context.Context, traceFraction float64, origin string) func(context.Context) {
+	var shutdownFuncs []func(context.Context) error
+	shutdown := func(ctx context.Context) {
+		var err error
+		for _, shutdownFunc := range shutdownFuncs {
+			err = errors.Join(err, shutdownFunc(ctx))
+		}
+		shutdownFuncs = nil
+		if err != nil {
+			slog.ErrorContext(ctx, "OTel shutdown", slog.Any("error", err))
+		}
+	}
+
+	mr, err := autoexport.NewMetricReader(ctx)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to create the metric reader", slog.Any("error", err))
+		os.Exit(1)
+	}
+
+	resources, err := resource.New(ctx,
+		resource.WithTelemetrySDK(),
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(origin),
+			semconv.ServiceNamespaceKey.String("tesseract"),
+		),
+		resource.WithFromEnv(),
+	)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to detect resources", slog.Any("error", err))
+		os.Exit(1)
+	}
+
+	mp := sdkmetric.NewMeterProvider(
+		sdkmetric.WithReader(mr),
+		sdkmetric.WithResource(resources),
+	)
+	shutdownFuncs = append(shutdownFuncs, mp.Shutdown)
+	otel.SetMeterProvider(mp)
+
+	te, err := autoexport.NewSpanExporter(ctx)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to create the span exporter", slog.Any("error", err))
+		os.Exit(1)
+	}
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(sdktrace.ParentBased(t_otel.NewAttributeSampler([]string{"tessera.periodic"}, sdktrace.TraceIDRatioBased(traceFraction)))),
+		sdktrace.WithBatcher(te),
+		sdktrace.WithResource(resources),
+	)
+	shutdownFuncs = append(shutdownFuncs, tp.Shutdown)
+	otel.SetTracerProvider(tp)
+
+	if err := runtime.Start(runtime.WithMeterProvider(mp)); err != nil {
+		slog.ErrorContext(ctx, "Failed to start exporting Go runtime metrics", slog.Any("error", err))
+		os.Exit(1)
+	}
+	return shutdown
+}

cmd/tesseract/aws/otel_test.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func TestInitOTel(t *testing.T) {
+	for _, scenario := range []string{"prometheus", "none", "sample-all", "invalid-exporter", "invalid-port", "occupied-port"} {
+		t.Run(scenario, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+			defer cancel()
+			command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestOTelProcess$")
+			for _, entry := range os.Environ() {
+				if !strings.HasPrefix(entry, "OTEL_") && !strings.HasPrefix(entry, "TESSERACT_OTEL_TEST=") {
+					command.Env = append(command.Env, entry)
+				}
+			}
+			command.Env = append(command.Env,
+				"TESSERACT_OTEL_TEST="+scenario,
+				"OTEL_METRICS_EXPORTER=none",
+				"OTEL_TRACES_EXPORTER=none",
+			)
+			output, err := command.CombinedOutput()
+			if strings.HasPrefix(scenario, "invalid-") || scenario == "occupied-port" {
+				if err == nil || !strings.Contains(string(output), "Failed to create the metric reader") {
+					t.Fatalf("expected metric reader startup failure, got %v: %s", err, output)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("telemetry check failed: %v: %s", err, output)
+			}
+		})
+	}
+}
+
+func TestOTelProcess(t *testing.T) {
+	scenario := os.Getenv("TESSERACT_OTEL_TEST")
+	if scenario == "" {
+		t.Skip("subprocess helper")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	var address string
+	if scenario == "prometheus" || scenario == "occupied-port" {
+		listener, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatal(err)
+		}
+		address = listener.Addr().String()
+		_, port, err := net.SplitHostPort(address)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv("OTEL_METRICS_EXPORTER", "prometheus")
+		t.Setenv("OTEL_EXPORTER_PROMETHEUS_HOST", "127.0.0.1")
+		t.Setenv("OTEL_EXPORTER_PROMETHEUS_PORT", port)
+		if scenario == "occupied-port" {
+			defer func() {
+				if err := listener.Close(); err != nil {
+					t.Error(err)
+				}
+			}()
+		} else if err := listener.Close(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if scenario == "invalid-exporter" {
+		t.Setenv("OTEL_METRICS_EXPORTER", "invalid")
+	}
+	if scenario == "invalid-port" {
+		t.Setenv("OTEL_METRICS_EXPORTER", "prometheus")
+		t.Setenv("OTEL_EXPORTER_PROMETHEUS_PORT", "invalid-port")
+	}
+	t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=test")
+	traceFraction := 0.0
+	if scenario == "sample-all" {
+		traceFraction = 1
+	}
+	shutdown := initOTel(ctx, traceFraction, "example.com/testlog")
+	defer shutdown(ctx)
+
+	tracer := otel.Tracer("aws-otel-test")
+	_, ordinary := tracer.Start(ctx, "ordinary")
+	if ordinary.SpanContext().IsSampled() != (traceFraction == 1) {
+		t.Error("ordinary span sampling does not match trace_fraction")
+	}
+	ordinary.End()
+	_, periodic := tracer.Start(ctx, "periodic", trace.WithAttributes(attribute.Bool("tessera.periodic", true)))
+	if !periodic.SpanContext().IsSampled() {
+		t.Error("periodic root span was not sampled")
+	}
+	periodic.End()
+
+	if scenario != "prometheus" {
+		return
+	}
+	counter, err := otel.Meter("aws-otel-test").Int64Counter("aws_otel_test_requests")
+	if err != nil {
+		t.Fatal(err)
+	}
+	counter.Add(ctx, 1)
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+address+"/metrics", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	client := &http.Client{Timeout: 5 * time.Second}
+	response, err := client.Do(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := response.Body.Close(); err != nil {
+			t.Error(err)
+		}
+	}()
+	body, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("scrape status: %s", response.Status)
+	}
+	for _, expected := range []string{"aws_otel_test_requests_total", "go_goroutine", `service_name="example.com/testlog"`, `service_namespace="tesseract"`, `deployment_environment="test"`} {
+		if !strings.Contains(string(body), expected) {
+			t.Errorf("scrape missing %q", expected)
+		}
+	}
+	shutdown(ctx)
+	connection, err := net.DialTimeout("tcp", address, time.Second)
+	if err == nil {
+		if err := connection.Close(); err != nil {
+			t.Error(err)
+		}
+		t.Error("Prometheus listener still accepts connections after shutdown")
+	}
+}