diff --git a/cmd/tesseract/aws/README.md b/cmd/tesseract/aws/README.md index d10b8b1f..22a0c202 100644 --- a/cmd/tesseract/aws/README.md +++ b/cmd/tesseract/aws/README.md @@ -16,6 +16,54 @@ For AWS-specific information on how to run this binary, see the documentation un TesseraCT expects the databases configured with the `db_name` and `antispam_db_name` flags to be located in the same Aurora DB cluster. +## OpenTelemetry + +The AWS binary configures OpenTelemetry metrics, tracing, and Go runtime metrics +using the same environment-driven exporters as the POSIX binary. The service name +is the log's `--origin`, and the service namespace is `tesseract`. +`OTEL_RESOURCE_ATTRIBUTES` can add or override resource attributes. + +For OTLP export to a collector, for example: + +```bash +export OTEL_METRICS_EXPORTER=otlp +export OTEL_TRACES_EXPORTER=otlp +export OTEL_EXPORTER_OTLP_PROTOCOL=grpc +export OTEL_EXPORTER_OTLP_ENDPOINT=https://collector.example.com:4317 +``` + +Set `--trace_fraction` to a value from `0` to `1` for the fraction of root traces +to sample. The default is `0`; periodic Tessera root spans marked with +`tessera.periodic` are still sampled. Child spans follow their parent's sampling +decision. Set `OTEL_TRACES_EXPORTER=none` to disable trace export, or +`OTEL_METRICS_EXPORTER=none` to disable metric export. Exporter selection defaults +to OTLP when unset. This does not automatically configure CloudWatch or X-Ray. + +### Prometheus + +To expose a Prometheus scrape endpoint instead of pushing metrics over OTLP: + +```bash +export OTEL_METRICS_EXPORTER=prometheus +export OTEL_EXPORTER_PROMETHEUS_HOST=127.0.0.1 +export OTEL_EXPORTER_PROMETHEUS_PORT=9464 +export OTEL_TRACES_EXPORTER=none +``` + +Metrics are available at `http://127.0.0.1:9464/metrics` on a separate listener, +not the CT HTTP endpoint. The default host is `localhost` and the default port is +`9464`. Tracing is configured independently and can remain enabled if desired. + +Remote scrapers require a non-loopback bind address such as `0.0.0.0`, along with +appropriate container port mappings and security group rules. Restrict access to +trusted scrapers: the metrics listener does not provide authentication or TLS. +Use a TLS/authentication proxy where required, validate deployment inputs, and +inject exporter credentials securely rather than hardcoding secrets. + +Telemetry providers shut down after HTTP requests drain on normal termination. +As in POSIX, startup failures that call `os.Exit` bypass deferred cleanup, and +telemetry shutdown does not have a separate timeout. + ## Vanilla S3+MySQL support Setting up S3 and MySQL infrastructure is out of scope for this document, but diff --git a/cmd/tesseract/aws/main.go b/cmd/tesseract/aws/main.go index f8e45cb4..1fa0b856 100644 --- a/cmd/tesseract/aws/main.go +++ b/cmd/tesseract/aws/main.go @@ -111,6 +111,7 @@ var ( signerPublicKeyFile = flag.String("signer_public_key_file", "", "Path to public key file for checkpoints and SCTs signer (alternative to secrets manager)") signerPrivateKeyFile = flag.String("signer_private_key_file", "", "Path to private key file for checkpoints and SCTs signer (alternative to secrets manager)") usePathStyle = flag.Bool("s3_use_path_style", false, "Whether to force the AWS S3 client to use path-style bucket references, probably only useful for on-prem deployments") + traceFraction = flag.Float64("trace_fraction", 0, "Fraction of open-telemetry span traces to sample") slogLevel = flag.Int("slog_level", 0, "The cut-off threshold for structured logging. See cmd/tesseract/README.md#Logging.") ) @@ -120,6 +121,9 @@ func main() { ctx := context.Background() slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.Level(*slogLevel)}))) + shutdownOTel := initOTel(ctx, *traceFraction, *origin) + defer shutdownOTel(ctx) + var signer *ECDSAWithSHA256Signer var err error diff --git a/cmd/tesseract/aws/otel.go b/cmd/tesseract/aws/otel.go new file mode 100644 index 00000000..dbe920b9 --- /dev/null +++ b/cmd/tesseract/aws/otel.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "os" + + t_otel "github.com/transparency-dev/tesseract/internal/otel" + "go.opentelemetry.io/contrib/exporters/autoexport" + "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" +) + +func initOTel(ctx context.Context, traceFraction float64, origin string) func(context.Context) { + var shutdownFuncs []func(context.Context) error + shutdown := func(ctx context.Context) { + var err error + for _, shutdownFunc := range shutdownFuncs { + err = errors.Join(err, shutdownFunc(ctx)) + } + shutdownFuncs = nil + if err != nil { + slog.ErrorContext(ctx, "OTel shutdown", slog.Any("error", err)) + } + } + + mr, err := autoexport.NewMetricReader(ctx) + if err != nil { + slog.ErrorContext(ctx, "Failed to create the metric reader", slog.Any("error", err)) + os.Exit(1) + } + + resources, err := resource.New(ctx, + resource.WithTelemetrySDK(), + resource.WithAttributes( + semconv.ServiceNameKey.String(origin), + semconv.ServiceNamespaceKey.String("tesseract"), + ), + resource.WithFromEnv(), + ) + if err != nil { + slog.ErrorContext(ctx, "Failed to detect resources", slog.Any("error", err)) + os.Exit(1) + } + + mp := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(mr), + sdkmetric.WithResource(resources), + ) + shutdownFuncs = append(shutdownFuncs, mp.Shutdown) + otel.SetMeterProvider(mp) + + te, err := autoexport.NewSpanExporter(ctx) + if err != nil { + slog.ErrorContext(ctx, "Failed to create the span exporter", slog.Any("error", err)) + os.Exit(1) + } + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.ParentBased(t_otel.NewAttributeSampler([]string{"tessera.periodic"}, sdktrace.TraceIDRatioBased(traceFraction)))), + sdktrace.WithBatcher(te), + sdktrace.WithResource(resources), + ) + shutdownFuncs = append(shutdownFuncs, tp.Shutdown) + otel.SetTracerProvider(tp) + + if err := runtime.Start(runtime.WithMeterProvider(mp)); err != nil { + slog.ErrorContext(ctx, "Failed to start exporting Go runtime metrics", slog.Any("error", err)) + os.Exit(1) + } + return shutdown +} diff --git a/cmd/tesseract/aws/otel_test.go b/cmd/tesseract/aws/otel_test.go new file mode 100644 index 00000000..b9dd0a32 --- /dev/null +++ b/cmd/tesseract/aws/otel_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "context" + "io" + "net" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +func TestInitOTel(t *testing.T) { + for _, scenario := range []string{"prometheus", "none", "sample-all", "invalid-exporter", "invalid-port", "occupied-port"} { + t.Run(scenario, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestOTelProcess$") + for _, entry := range os.Environ() { + if !strings.HasPrefix(entry, "OTEL_") && !strings.HasPrefix(entry, "TESSERACT_OTEL_TEST=") { + command.Env = append(command.Env, entry) + } + } + command.Env = append(command.Env, + "TESSERACT_OTEL_TEST="+scenario, + "OTEL_METRICS_EXPORTER=none", + "OTEL_TRACES_EXPORTER=none", + ) + output, err := command.CombinedOutput() + if strings.HasPrefix(scenario, "invalid-") || scenario == "occupied-port" { + if err == nil || !strings.Contains(string(output), "Failed to create the metric reader") { + t.Fatalf("expected metric reader startup failure, got %v: %s", err, output) + } + return + } + if err != nil { + t.Fatalf("telemetry check failed: %v: %s", err, output) + } + }) + } +} + +func TestOTelProcess(t *testing.T) { + scenario := os.Getenv("TESSERACT_OTEL_TEST") + if scenario == "" { + t.Skip("subprocess helper") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + var address string + if scenario == "prometheus" || scenario == "occupied-port" { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + address = listener.Addr().String() + _, port, err := net.SplitHostPort(address) + if err != nil { + t.Fatal(err) + } + t.Setenv("OTEL_METRICS_EXPORTER", "prometheus") + t.Setenv("OTEL_EXPORTER_PROMETHEUS_HOST", "127.0.0.1") + t.Setenv("OTEL_EXPORTER_PROMETHEUS_PORT", port) + if scenario == "occupied-port" { + defer func() { + if err := listener.Close(); err != nil { + t.Error(err) + } + }() + } else if err := listener.Close(); err != nil { + t.Fatal(err) + } + } + if scenario == "invalid-exporter" { + t.Setenv("OTEL_METRICS_EXPORTER", "invalid") + } + if scenario == "invalid-port" { + t.Setenv("OTEL_METRICS_EXPORTER", "prometheus") + t.Setenv("OTEL_EXPORTER_PROMETHEUS_PORT", "invalid-port") + } + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=test") + traceFraction := 0.0 + if scenario == "sample-all" { + traceFraction = 1 + } + shutdown := initOTel(ctx, traceFraction, "example.com/testlog") + defer shutdown(ctx) + + tracer := otel.Tracer("aws-otel-test") + _, ordinary := tracer.Start(ctx, "ordinary") + if ordinary.SpanContext().IsSampled() != (traceFraction == 1) { + t.Error("ordinary span sampling does not match trace_fraction") + } + ordinary.End() + _, periodic := tracer.Start(ctx, "periodic", trace.WithAttributes(attribute.Bool("tessera.periodic", true))) + if !periodic.SpanContext().IsSampled() { + t.Error("periodic root span was not sampled") + } + periodic.End() + + if scenario != "prometheus" { + return + } + counter, err := otel.Meter("aws-otel-test").Int64Counter("aws_otel_test_requests") + if err != nil { + t.Fatal(err) + } + counter.Add(ctx, 1) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+address+"/metrics", nil) + if err != nil { + t.Fatal(err) + } + client := &http.Client{Timeout: 5 * time.Second} + response, err := client.Do(request) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := response.Body.Close(); err != nil { + t.Error(err) + } + }() + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("scrape status: %s", response.Status) + } + for _, expected := range []string{"aws_otel_test_requests_total", "go_goroutine", `service_name="example.com/testlog"`, `service_namespace="tesseract"`, `deployment_environment="test"`} { + if !strings.Contains(string(body), expected) { + t.Errorf("scrape missing %q", expected) + } + } + shutdown(ctx) + connection, err := net.DialTimeout("tcp", address, time.Second) + if err == nil { + if err := connection.Close(); err != nil { + t.Error(err) + } + t.Error("Prometheus listener still accepts connections after shutdown") + } +}