feat: add OTLP profile forwarding mode

README.md

@@ -74,6 +74,35 @@ The scrape configuration can be changed in the `parca.yaml` in the root of the r
 
 ### Configuration
 
+#### Forwarding profiles with OTLP
+
+Use `--mode=otlp-forwarder --store-address=<host:port>` to scrape pprof endpoints,
+convert the profiles with OpenTelemetry's pprof translator, and export them to an
+upstream that supports OTLP profiles over gRPC. No local profile store is started.
+The existing `--bearer-token`, `--bearer-token-file`, `--grpc-headers`, and TLS flags
+apply to this connection. Use `--insecure` for a plaintext upstream.
+
+All target labels retained after relabeling and any `--external-label` values
+become OTLP resource attributes, without renaming. Labels inside the pprof data
+remain attributes on the OTLP samples. Relabeling changes only target metadata,
+not labels within profiles. As with other scraping modes, discovery's internal
+`__meta_*` labels must be copied by relabeling to retain them.
+
+Example configs map target metadata to OpenTelemetry semantic-convention names:
+
+- [Static targets](deploy/otlp/static.yaml) sets `service.name`,
+  `service.namespace`, `service.instance.id`, and `deployment.environment.name`.
+- [Kubernetes pods](deploy/otlp/kubernetes.yaml) sets service attributes and
+  `k8s.*` attributes from pod discovery metadata.
+
+Run either example with `--config-path=deploy/otlp/<example>.yaml` and the mode
+and store address above. Parca's `job`, `instance`, and `__name__` labels are also
+forwarded as resource attributes for compatibility. The examples do not rename
+pprof sample types or units. The upstream must support the OTLP profiles
+`v1development` protocol used by this build. OTLP forwarding supports pprof
+`WriteRaw` requests and OTLP `Export`, but not Parca's Arrow or debuginfo APIs.
+The existing `scraper-only` and `forwarder` modes continue to use Parca's protocol.
+
 Flags:
 
 <!-- prettier-ignore-start -->
@@ -85,8 +114,11 @@ Flags:
   -h, --help                     Show context-sensitive help.
       --config-path="parca.yaml"
                                  Path to config file.
-      --mode="all"               Scraper only runs a scraper that sends to a
-                                 remote gRPC endpoint. All runs all components.
+      --mode="all"               All runs all components. Scraper-only and
+                                 forwarder send profiles to a remote Parca
+                                 gRPC endpoint. OTLP-forwarder converts pprof
+                                 profiles and sends them to an OTLP profiles
+                                 gRPC endpoint.
       --http-address=":7070"     Address to bind HTTP server to.
       --http-read-timeout=5s     Timeout duration for HTTP server to read
                                  request body.
@@ -164,8 +196,8 @@ Flags:
                                  TLS.
       --insecure-skip-verify     Skip TLS certificate verification.
       --external-label=KEY=VALUE;...
-                                 Label(s) to attach to all profiles in
-                                 scraper-only mode.
+                                 Label(s) to attach to all scraped profiles in
+                                 forwarding modes.
       --grpc-headers=KEY=VALUE;...
                                  Additional gRPC headers to send with each
                                  request to the remote store (key=value pairs).

deploy/otlp/kubernetes.yaml

@@ -0,0 +1,44 @@
+# Run in-cluster with --mode=otlp-forwarder and --store-address=<OTLP gRPC address>.
+# The service account needs permission to get, list, and watch pods.
+# Scrape containers that expose a port named "pprof" and whose pods have an
+# app.kubernetes.io/name label. Endpoints must serve Go's /debug/pprof paths.
+# Required by config validation, but unused in forwarding modes.
+object_storage:
+  bucket:
+    type: FILESYSTEM
+    config:
+      directory: ./data
+
+scrape_configs:
+  - job_name: kubernetes-pods
+    scrape_interval: 30s
+    kubernetes_sd_configs:
+      - role: pod
+    relabel_configs:
+      - action: keep
+        source_labels: [__meta_kubernetes_pod_container_port_name]
+        regex: pprof
+      - action: keep
+        source_labels: [__meta_kubernetes_pod_phase]
+        regex: Running
+      - action: keep
+        source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+        regex: .+
+      - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+        target_label: service.name
+      - source_labels: [__meta_kubernetes_namespace]
+        target_label: service.namespace
+      # A pod UID plus container name distinguishes replicas and containers.
+      - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
+        separator: /
+        target_label: service.instance.id
+      - source_labels: [__meta_kubernetes_namespace]
+        target_label: k8s.namespace.name
+      - source_labels: [__meta_kubernetes_pod_name]
+        target_label: k8s.pod.name
+      - source_labels: [__meta_kubernetes_pod_uid]
+        target_label: k8s.pod.uid
+      - source_labels: [__meta_kubernetes_pod_node_name]
+        target_label: k8s.node.name
+      - source_labels: [__meta_kubernetes_pod_container_name]
+        target_label: k8s.container.name

deploy/otlp/static.yaml

@@ -0,0 +1,30 @@
+# Run with --mode=otlp-forwarder --store-address=collector.example.com:4317.
+# Add --insecure only if the upstream uses plaintext gRPC.
+# Required by config validation, but unused in forwarding modes.
+object_storage:
+  bucket:
+    type: FILESYSTEM
+    config:
+      directory: ./data
+
+scrape_configs:
+  - job_name: checkout
+    scrape_interval: 30s
+    static_configs:
+      - targets: [checkout:6060]
+        labels:
+          namespace: shop
+          environment: development
+    relabel_configs:
+      - source_labels: [job]
+        target_label: service.name
+      # The default instance label is assigned AFTER relabeling, so use
+      # __address__ to identify the service instance here.
+      - source_labels: [__address__]
+        target_label: service.instance.id
+      - source_labels: [namespace]
+        target_label: service.namespace
+      - source_labels: [environment]
+        target_label: deployment.environment.name
+      - action: labeldrop
+        regex: namespace|environment

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/nanmu42/limitio v1.0.0
 	github.com/oklog/run v1.2.0
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/pprof v0.157.0
 	github.com/parquet-go/parquet-go v0.24.0
 	github.com/pierrec/lz4/v4 v4.1.25
 	github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25
@@ -45,6 +46,7 @@ require (
 	github.com/stretchr/testify v1.12.1
 	github.com/thanos-io/objstore v0.0.0-20241028145108-31c0873d5266
 	github.com/zeebo/xxh3 v1.1.0
+	go.opentelemetry.io/collector/pdata/pprofile v0.157.0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.67.0
 	go.opentelemetry.io/otel v1.46.0
@@ -196,6 +198,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-version v1.9.0 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/nomad/api v0.0.0-20260324203407-b27b0c2e019a // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
@@ -255,10 +258,13 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/collector/featuregate v1.63.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.63.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect
 	go.opentelemetry.io/otel/metric v1.46.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	golang.org/x/crypto v0.57.0 // indirect

go.sum

@@ -542,8 +542,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
-github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA=
+github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -745,6 +745,10 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
 github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/pprof v0.157.0 h1:JrYk+H65J18Gn6JXf5zjNQkCv4HoU7LRHm6nSz6qOVs=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/pprof v0.157.0/go.mod h1:Xb2YX/ehDKQHwhpwSkTCe0t6IBDv406KTA3eMpPY8Lw=
+github.com/open-telemetry/sig-profiling/profcheck v0.0.0-20260605055552-091960d5da90 h1:Pvu/hHr3XjXFlWQlNk/+dG69b0R1ScGWk8Bgm29lP2g=
+github.com/open-telemetry/sig-profiling/profcheck v0.0.0-20260605055552-091960d5da90/go.mod h1:WPdgk1BinVdvYdkbt1KIkgDw6qUiK8CnGq+ftaJ41Ns=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
@@ -964,6 +968,14 @@ go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/collector/featuregate v1.63.0 h1:6EWX1C5AtmIh8hFH97DwK6R7R8Jk3fTLxAUfZPXGutY=
+go.opentelemetry.io/collector/featuregate v1.63.0/go.mod h1:4ga1QBMPEejXXmpyJS8lmaRpknJ3Lb9Bvk6e420bUFU=
+go.opentelemetry.io/collector/internal/testutil v0.157.0 h1:plojUQwFC5l1ex9KUDaLmCFY/mTxEmf3zrlP7M23IEw=
+go.opentelemetry.io/collector/internal/testutil v0.157.0/go.mod h1:Jkjs6rkqs973LqgZ0Fe3zrokQRKULYXPIf4HuqStiEE=
+go.opentelemetry.io/collector/pdata v1.63.0 h1:fY2xSG2MnyoBwA4GUhzoogGZMuNS0qHpCoODqaKwiVQ=
+go.opentelemetry.io/collector/pdata v1.63.0/go.mod h1:jzozYYhQEkTQ/CCbCBNC+hYUeju9S2J8HIqIDHdxZWk=
+go.opentelemetry.io/collector/pdata/pprofile v0.157.0 h1:YRTPhwWzdG0pfJmb8p/qpQm1EdX+JfV20qzwG3ypDqI=
+go.opentelemetry.io/collector/pdata/pprofile v0.157.0/go.mod h1:kwy/ufNUBkw8PsFPQnAqCvD12OpGU8h9A1cz5S7xS6g=
 go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw=
 go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E=
@@ -1000,6 +1012,12 @@ go.opentelemetry.io/proto/otlp/collector/profiles/v1development v0.3.0 h1:AWBLtJ
 go.opentelemetry.io/proto/otlp/collector/profiles/v1development v0.3.0/go.mod h1:kCdbxyM9kfmv3v4ZQzZ2pACxoQ6lE0IsjP+UcZwQnr4=
 go.opentelemetry.io/proto/otlp/profiles/v1development v0.3.0 h1:ZQs05qo3Yh4KUHeVH6v89xErwmsvgA/cLX2/w5Ikp+k=
 go.opentelemetry.io/proto/otlp/profiles/v1development v0.3.0/go.mod h1:3iiRVKaCfVo0UI1ZaSMm5WbCBbINRqVlD9SUmvyBNrY=
+go.opentelemetry.io/proto/slim/otlp v1.10.0 h1:iR97Vs/ZDR+y9TfuP9b1XBtdPWeC+OMslIBmhcLU7jM=
+go.opentelemetry.io/proto/slim/otlp v1.10.0/go.mod h1:lV9250stpjYLPNA5viFabIgP2QlUGRT1GdTgAf8SIUk=
+go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:RUF5rO0hAlgiJt1fzQVzcVs3vZVNHIcMLgOgG4rWNcQ=
+go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U=
+go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0=
+go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -1011,6 +1029,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=

pkg/parca/parca.go

@@ -90,12 +90,13 @@ const (
 	symbolizationInterval = 10 * time.Second
 	flagModeScraperOnly   = "scraper-only"
 	flagModeForwarder     = "forwarder"
+	flagModeOTLPForwarder = "otlp-forwarder"
 	metaStoreBadger       = "badger"
 )
 
 type Flags struct {
 	ConfigPath       string        `default:"parca.yaml" help:"Path to config file."`
-	Mode             string        `default:"all" enum:"all,scraper-only,forwarder" help:"Scraper only runs a scraper that sends to a remote gRPC endpoint. All runs all components."`
+	Mode             string        `default:"all" enum:"all,scraper-only,forwarder,otlp-forwarder" help:"All runs all components. Scraper-only and forwarder send profiles to a remote Parca gRPC endpoint. OTLP-forwarder converts pprof profiles and sends them to an OTLP profiles gRPC endpoint."`
 	HTTPAddress      string        `default:":7070" help:"Address to bind HTTP server to."`
 	HTTPReadTimeout  time.Duration `default:"5s" help:"Timeout duration for HTTP server to read request body."`
 	HTTPWriteTimeout time.Duration `default:"1m" help:"Timeout duration for HTTP server to write response body."`
@@ -127,7 +128,7 @@ type Flags struct {
 	BearerTokenFile    string            `kong:"help='File to read bearer token from to authenticate with store.'"`
 	Insecure           bool              `kong:"help='Send gRPC requests via plaintext instead of TLS.'"`
 	InsecureSkipVerify bool              `kong:"help='Skip TLS certificate verification.'"`
-	ExternalLabel      map[string]string `kong:"help='Label(s) to attach to all profiles in scraper-only mode.'"`
+	ExternalLabel      map[string]string `kong:"help='Label(s) to attach to all scraped profiles in forwarding modes.'"`
 	GRPCHeaders        map[string]string `kong:"help='Additional gRPC headers to send with each request to the remote store (key=value pairs).'"`
 
 	Hidden FlagsHidden `embed:"" prefix:""`
@@ -243,13 +244,12 @@ func Run(ctx context.Context, logger log.Logger, reg *prometheus.Registry, flags
 		return fmt.Errorf("failed to initialize UI filesystem: %w", err)
 	}
 
-	if flags.StoreAddress != "" && flags.Mode != flagModeScraperOnly {
-		return fmt.Errorf("the mode should be set as `--mode=scraper-only`, if `StoreAddress` is set")
-	}
-
-	if flags.Mode == flagModeScraperOnly || flags.Mode == flagModeForwarder {
+	if flags.Mode == flagModeScraperOnly || flags.Mode == flagModeForwarder || flags.Mode == flagModeOTLPForwarder {
 		return runForwarder(ctx, logger, reg, tracerProvider, uiFS, flags, version, cfg)
 	}
+	if flags.StoreAddress != "" {
+		return fmt.Errorf("--store-address requires --mode=scraper-only, --mode=forwarder, or --mode=otlp-forwarder")
+	}
 
 	bucketCfg, err := yaml.Marshal(cfg.ObjectStorage.Bucket)
 	if err != nil {
@@ -856,13 +856,14 @@ func runForwarder(
 	if err != nil {
 		return fmt.Errorf("failed to create gRPC connection: %w", err)
 	}
+	defer conn.Close()
 
 	dbginfo := debuginfo.NewGRPCForwarder(debuginfopb.NewDebuginfoServiceClient(conn))
 	client := profilestore.NewClient(
 		profilestorepb.NewProfileStoreServiceClient(conn),
 		otelgrpcprofilingpb.NewProfilesServiceClient(conn),
 	)
-	store := profilestore.NewGRPCForwarder(client, logger)
+	store := profilestore.NewGRPCForwarder(client, logger, flags.Mode == flagModeOTLPForwarder)
 
 	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(reg)
 	if err != nil {
@@ -954,10 +955,13 @@ func runForwarder(
 					server.RegisterableFunc(func(ctx context.Context, srv *grpc.Server, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) error {
 						scrapepb.RegisterScrapeServiceServer(srv, m)
 						profilestorepb.RegisterProfileStoreServiceServer(srv, store)
-						debuginfopb.RegisterDebuginfoServiceServer(srv, dbginfo)
-
-						if err := debuginfopb.RegisterDebuginfoServiceHandlerFromEndpoint(ctx, mux, endpoint, opts); err != nil {
-							return err
+						if flags.Mode == flagModeOTLPForwarder {
+							otelgrpcprofilingpb.RegisterProfilesServiceServer(srv, store)
+						} else {
+							debuginfopb.RegisterDebuginfoServiceServer(srv, dbginfo)
+							if err := debuginfopb.RegisterDebuginfoServiceHandlerFromEndpoint(ctx, mux, endpoint, opts); err != nil {
+								return err
+							}
 						}
 
 						if err := scrapepb.RegisterScrapeServiceHandlerFromEndpoint(ctx, mux, endpoint, opts); err != nil {