diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f72d849..cc33cdf9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,6 +33,11 @@ jobs: go.sum go.mod + - name: Install libpcap + run: | + sudo apt-get update + sudo apt-get install -y libpcap-dev + - name: Tidy Go Modules run: go mod tidy diff --git a/.github/workflows/reflow.yml b/.github/workflows/reflow.yml new file mode 100644 index 00000000..80888e4d --- /dev/null +++ b/.github/workflows/reflow.yml @@ -0,0 +1,414 @@ +name: ReFlow + +on: + push: + branches: [ main ] + tags: + - 'v*' + pull_request: + branches: [ main ] + +env: + GHCR_REFLOW_IMAGE: ghcr.io/${{ github.repository_owner }}/reflow + DOCKERHUB_REFLOW_IMAGE: docker.io/${{ github.repository_owner }}/reflow + +jobs: + build: + name: Build ReFlow (${{ matrix.label }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - label: linux-amd64 + runner: ubuntu-latest + goos: linux + goarch: amd64 + cgo: "1" + libc: glibc + static: "0" + apt_packages: libpcap-dev + docker_package: "1" + docker_platform: linux/amd64 + package: tar + - label: linux-arm64 + runner: ubuntu-24.04-arm + goos: linux + goarch: arm64 + cgo: "1" + libc: glibc + static: "0" + apt_packages: libpcap-dev + docker_package: "1" + docker_platform: linux/arm64/v8 + package: tar + - label: linux-armv7 + runner: ubuntu-24.04-arm + goos: linux + goarch: arm + goarm: "7" + cgo: "1" + libc: musl + static: "1" + zig: "1" + zig_version: 0.15.2 + zig_archive_target: aarch64-linux + cc: zig cc -target arm-linux-musleabihf + host: arm-linux-musleabihf + apt_packages: make flex bison + libpcap_source: "1" + libpcap_version: 1.10.5 + cgo_cflags: -I/tmp/reflow-libpcap-armv7/include + cgo_ldflags: -static -L/tmp/reflow-libpcap-armv7/lib + package: tar + - label: darwin-amd64 + runner: macos-15-intel + goos: darwin + goarch: amd64 + cgo: "1" + package: tar + - label: darwin-arm64 + runner: macos-15 + goos: darwin + goarch: arm64 + cgo: "1" + package: tar + - label: windows-amd64 + runner: ubuntu-latest + goos: windows + goarch: amd64 + cgo: "1" + extension: .exe + package: zip + - label: windows-arm64 + runner: ubuntu-latest + goos: windows + goarch: arm64 + cgo: "1" + # gopacket/pcap v1.1.19 has generated Windows pcap definitions + # for 386/amd64 only, so keep this artifact buildable until the + # dependency supports windows/arm64 pcap. + build_tags: reflow_nopcap + extension: .exe + package: zip + # Future OpenWrt package architectures, intentionally inactive: + # x86_64, aarch64_generic, aarch64_cortex-a53, aarch64_cortex-a72, + # arm_cortex-a7_neon-vfpv4, arm_cortex-a9, arm_cortex-a15_neon-vfpv4, + # mips_24kc, mipsel_24kc, mipsel_74kc, mipsel_mips32, + # mips64_octeonplus, riscv64_riscv64. + # Future EdgeOS/UBNT candidates, intentionally inactive: + # mips64, mips64le, arm64. + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: ^1.24 + cache: true + cache-dependency-path: | + go.sum + go.mod + + - name: Set VERSION for tag builds + if: startsWith(github.ref, 'refs/tags/') + run: echo "VERSION=$(git describe --tags --abbrev=0 HEAD)" >> "$GITHUB_ENV" + + - name: Install Zig + if: matrix.zig == '1' + env: + ZIG_ARCHIVE_TARGET: ${{ matrix.zig_archive_target }} + ZIG_VERSION: ${{ matrix.zig_version }} + run: | + archive="zig-${ZIG_ARCHIVE_TARGET}-${ZIG_VERSION}" + curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/${archive}.tar.xz" -o /tmp/zig.tar.xz + tar -C /tmp -xf /tmp/zig.tar.xz + echo "/tmp/${archive}" >> "$GITHUB_PATH" + "/tmp/${archive}/zig" version + + - name: Install Linux build dependencies + if: matrix.goos == 'linux' + env: + APT_PACKAGES: ${{ matrix.apt_packages }} + run: | + sudo apt-get update + sudo apt-get install -y $APT_PACKAGES + + - name: Build libpcap for ARMv7 + if: matrix.libpcap_source == '1' + env: + CC: ${{ matrix.cc }} + HOST: ${{ matrix.host }} + LIBPCAP_VERSION: ${{ matrix.libpcap_version }} + run: | + prefix=/tmp/reflow-libpcap-armv7 + curl -fsSL "https://www.tcpdump.org/release/libpcap-${LIBPCAP_VERSION}.tar.gz" -o /tmp/libpcap.tar.gz + tar -C /tmp -xzf /tmp/libpcap.tar.gz + cd "/tmp/libpcap-${LIBPCAP_VERSION}" + ./configure \ + --host="$HOST" \ + --prefix="$prefix" \ + --with-pcap=linux \ + --disable-shared \ + --enable-static + make -j"$(nproc)" + make install + + - name: Patch gopacket pcap timestamp types for ARMv7 + if: matrix.libpcap_source == '1' + run: | + go mod download github.com/google/gopacket + module_dir="$(go env GOPATH)/pkg/mod/github.com/google/gopacket@v1.1.19" + chmod u+w "$module_dir" "$module_dir/pcap" "$module_dir/pcap/pcap_unix.go" + perl -0pi -e 's{// Windows, Macs, and Linux all use different time types\. Joy\.\n#ifdef __APPLE__\n#define gopacket_time_secs_t __darwin_time_t\n#define gopacket_time_usecs_t __darwin_suseconds_t\n#elif __ANDROID__\n#define gopacket_time_secs_t __kernel_time_t\n#define gopacket_time_usecs_t __kernel_suseconds_t\n#elif __GLIBC__\n#define gopacket_time_secs_t __time_t\n#define gopacket_time_usecs_t __suseconds_t\n#else // Some form of linux/bsd/etc\.\.\.\n#include \n#ifdef __OpenBSD__\n#define gopacket_time_secs_t u_int32_t\n#define gopacket_time_usecs_t u_int32_t\n#else\n#define gopacket_time_secs_t time_t\n#define gopacket_time_usecs_t suseconds_t\n#endif\n#endif}{// Match libpcap pcap_pkthdr timestamp fields across cross-libc builds.\n#define gopacket_time_secs_t __typeof__(((struct pcap_pkthdr *)0)->ts.tv_sec)\n#define gopacket_time_usecs_t __typeof__(((struct pcap_pkthdr *)0)->ts.tv_usec)}' "$module_dir/pcap/pcap_unix.go" + if grep -q 'gopacket_time_usecs_t __suseconds_t' "$module_dir/pcap/pcap_unix.go"; then + echo "failed to patch gopacket pcap time types" >&2 + exit 1 + fi + + - name: Compile pcap source + if: matrix.cgo == '1' + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + GOARM: ${{ matrix.goarm }} + CGO_ENABLED: ${{ matrix.cgo }} + CC: ${{ matrix.cc }} + CGO_CFLAGS: ${{ matrix.cgo_cflags }} + CGO_LDFLAGS: ${{ matrix.cgo_ldflags }} + REFLOW_BUILD_TAGS: ${{ matrix.build_tags }} + run: | + output="/tmp/reflow-pcap-${{ matrix.label }}.test" + if [[ -n "$REFLOW_BUILD_TAGS" ]]; then + go test -c -tags "$REFLOW_BUILD_TAGS" -o "$output" ./pkg/reflow/source/pcap + else + go test -c -o "$output" ./pkg/reflow/source/pcap + fi + + - name: Package ReFlow + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + GOARM: ${{ matrix.goarm }} + CGO_ENABLED: ${{ matrix.cgo }} + CC: ${{ matrix.cc }} + CGO_CFLAGS: ${{ matrix.cgo_cflags }} + CGO_LDFLAGS: ${{ matrix.cgo_ldflags }} + EXTENSION: ${{ matrix.extension }} + REFLOW_BUILD_TAGS: ${{ matrix.build_tags }} + run: | + if [[ "${{ matrix.package }}" == "zip" ]]; then + make package-reflow-zip + else + make package-reflow-tar + fi + + - name: Verify Linux ReFlow binary + if: matrix.goos == 'linux' + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + GOARM: ${{ matrix.goarm }} + EXTENSION: ${{ matrix.extension }} + STATIC: ${{ matrix.static }} + run: | + output="$(make print-reflow-output)" + file "$output" + if [[ "$STATIC" == "1" ]] && file "$output" | grep -q 'dynamically linked'; then + echo "$output should be statically linked" >&2 + exit 1 + fi + + - name: Set ReFlow Docker packaging vars + if: matrix.docker_package == '1' + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + GOARM: ${{ matrix.goarm }} + EXTENSION: ${{ matrix.extension }} + run: | + CGO_ENABLED=0 REFLOW_BUILD_TAGS=reflow_nopcap DIST_DIR=dist/docker/ make build-reflow + echo "REFLOW_BINARY_PATH=$(DIST_DIR=dist/docker/ make print-reflow-output)" >> "$GITHUB_ENV" + + - name: Set up QEMU + if: matrix.docker_package == '1' + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + if: matrix.docker_package == '1' + uses: docker/setup-buildx-action@v4 + + - name: Login to DockerHub + if: matrix.docker_package == '1' && github.event_name != 'pull_request' + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GHCR + if: matrix.docker_package == '1' && github.event_name != 'pull_request' + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + + - name: Docker package ReFlow + if: matrix.docker_package == '1' + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + DOCKER_PUSH_FLAG="--load" + else + DOCKER_PUSH_FLAG="--push" + fi + DOCKER_SUFFIX=-${{ matrix.goarch }} \ + DOCKER_IMAGE_PREFIXES="${{ env.DOCKERHUB_REFLOW_IMAGE }} ${{ env.GHCR_REFLOW_IMAGE }}" \ + DOCKER_CMD="buildx build $DOCKER_PUSH_FLAG --platform ${{ matrix.docker_platform }} -f Dockerfile.reflow --build-arg BIN_SOURCE=prebuilt --build-arg PREBUILT_BINARY=$REFLOW_BINARY_PATH" \ + NAME=reflow \ + DESCRIPTION="ReFlow: encode and decode flow and packet streams" \ + make docker + + - name: Upload ReFlow artifact + uses: actions/upload-artifact@v6 + with: + name: reflow-${{ matrix.label }} + path: | + dist/reflow-*.tar.gz + dist/reflow-*.tar.gz.sha256 + dist/reflow-*.zip + dist/reflow-*.zip.sha256 + if-no-files-found: error + retention-days: 14 + + manifest: + name: Publish ReFlow Docker Manifests + needs: build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + + - name: Set VERSION for tag builds + if: startsWith(github.ref, 'refs/tags/') + run: echo "VERSION=$(git describe --tags --abbrev=0 HEAD)" >> "$GITHUB_ENV" + + - name: Publish manifests (branch) + if: "!startsWith(github.ref, 'refs/tags/')" + run: | + images=("${{ env.DOCKERHUB_REFLOW_IMAGE }}" "${{ env.GHCR_REFLOW_IMAGE }}") + for image in "${images[@]}"; do + DOCKER_IMAGE="$image" make docker-manifest + DOCKER_IMAGE="$image" DOCKER_MANIFEST_TAG=latest make docker-manifest + done + + - name: Publish manifests (tag) + if: startsWith(github.ref, 'refs/tags/') + run: | + images=("${{ env.DOCKERHUB_REFLOW_IMAGE }}" "${{ env.GHCR_REFLOW_IMAGE }}") + for image in "${images[@]}"; do + DOCKER_IMAGE="$image" DOCKER_MANIFEST_TAG="${VERSION}" make docker-manifest + DOCKER_IMAGE="$image" DOCKER_MANIFEST_TAG=latest make docker-manifest + done + + release: + name: Publish ReFlow Release Assets + needs: build + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download ReFlow artifacts + uses: actions/download-artifact@v8 + with: + path: dist + pattern: reflow-* + merge-multiple: true + + - name: Create or find release + id: release + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const tag = process.env.GITHUB_REF_NAME; + async function getRelease() { + const { data } = await github.rest.repos.getReleaseByTag({ + owner: context.repo.owner, + repo: context.repo.repo, + tag, + }); + return data; + } + try { + const release = await getRelease(); + core.setOutput('upload_url', release.upload_url); + core.setOutput('release_id', release.id); + } catch (error) { + if (error.status !== 404) throw error; + try { + const { data } = await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + name: `Release ${tag}`, + draft: false, + prerelease: false, + }); + core.setOutput('upload_url', data.upload_url); + core.setOutput('release_id', data.id); + } catch (createError) { + if (createError.status !== 422) throw createError; + const release = await getRelease(); + core.setOutput('upload_url', release.upload_url); + core.setOutput('release_id', release.id); + } + } + + - name: Upload release assets + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs').promises; + const path = require('path'); + const uploadUrl = '${{ steps.release.outputs.upload_url }}'; + const releaseId = Number('${{ steps.release.outputs.release_id }}'); + const { data: assets } = await github.rest.repos.listReleaseAssets({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: releaseId, + per_page: 100, + }); + const existing = new Map(assets.map((asset) => [asset.name, asset.id])); + for (const file of await fs.readdir('./dist')) { + const fullPath = path.join('./dist', file); + if (!(await fs.stat(fullPath)).isFile()) continue; + if (existing.has(file)) { + await github.rest.repos.deleteReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + asset_id: existing.get(file), + }); + } + console.log('uploading', file); + await github.rest.repos.uploadReleaseAsset({ + url: uploadUrl, + name: file, + data: await fs.readFile(fullPath), + }); + } diff --git a/Dockerfile.reflow b/Dockerfile.reflow new file mode 100644 index 00000000..34cfcbc4 --- /dev/null +++ b/Dockerfile.reflow @@ -0,0 +1,23 @@ +FROM scratch +ARG PREBUILT_BINARY="reflow" +ARG VERSION="" +ARG CREATED="" +ARG DESCRIPTION="" +ARG NAME="" +ARG MAINTAINER="" +ARG URL="" +ARG LICENSE="" +ARG REV="" + +LABEL org.opencontainers.image.created="${CREATED}" +LABEL org.opencontainers.image.authors="${MAINTAINER}" +LABEL org.opencontainers.image.url="${URL}" +LABEL org.opencontainers.image.title="${NAME}" +LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.description="${DESCRIPTION}" +LABEL org.opencontainers.image.licenses="${LICENSE}" +LABEL org.opencontainers.image.revision="${REV}" + +COPY ${PREBUILT_BINARY} /reflow + +ENTRYPOINT ["/reflow"] diff --git a/Makefile b/Makefile index 28d53cd0..a6bbef3c 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,12 @@ EXTENSION ?= DIST_DIR ?= dist/ GOOS ?= linux GOARCH ?= $(shell go env GOARCH) +GOARM ?= +CGO_ENABLED ?= 0 BUILDINFOSDET ?= +REFLOW_BUILD_TAGS ?= -NAME := goflow2 +NAME ?= goflow2 DOCKER_IMAGE ?= netsampler/$(NAME) VERSION ?= $(shell git describe --abbrev --long HEAD) ABBREV ?= $(shell git rev-parse --short HEAD) @@ -14,7 +17,7 @@ TAG ?= $(shell git describe --tags --abbrev=0 HEAD) VERSION_PKG ?= $(shell echo $(VERSION) | sed 's/^v//g') LICENSE := BSD-3-Clause URL := https://github.com/netsampler/goflow2 -DESCRIPTION := GoFlow2: Open-Source and Scalable Network Sample Collector +DESCRIPTION ?= GoFlow2: Open-Source and Scalable Network Sample Collector DATE := $(shell date +%FT%T%z) BUILDINFOS ?= ($(DATE)$(BUILDINFOSDET)) LDFLAGS ?= '-X main.version=$(VERSION) -X main.buildinfos=$(BUILDINFOS)' @@ -28,6 +31,10 @@ DOCKER_TAG_ARGS := $(foreach tag,$(DOCKER_TAGS),-t $(tag)) DOCKER_MANIFEST_TAG ?= $(ABBREV) OUTPUT := $(DIST_DIR)goflow2-$(VERSION_PKG)-$(GOOS)-$(GOARCH)$(EXTENSION) +REFLOW_TARGET := $(GOOS)-$(GOARCH)$(if $(filter arm,$(GOARCH)),$(if $(GOARM),v$(GOARM),)) +REFLOW_OUTPUT := $(DIST_DIR)reflow-$(VERSION_PKG)-$(REFLOW_TARGET)$(EXTENSION) +REFLOW_PACKAGE := $(DIST_DIR)reflow-$(VERSION_PKG)-$(REFLOW_TARGET) +REFLOW_PACKAGE_DIR := $(DIST_DIR)reflow-package-$(REFLOW_TARGET) # fpm expects x86_64 for amd64, but use GOARCH otherwise. FPM_ARCH ?= $(GOARCH) @@ -85,10 +92,37 @@ clean: build: prepare CGO_ENABLED=0 go build -ldflags $(LDFLAGS) -o $(OUTPUT) cmd/goflow2/main.go +.PHONY: build-reflow +# Build the reflow binary. +build-reflow: prepare + CGO_ENABLED=$(CGO_ENABLED) go build $(if $(REFLOW_BUILD_TAGS),-tags '$(REFLOW_BUILD_TAGS)',) -ldflags $(LDFLAGS) -o $(REFLOW_OUTPUT) cmd/reflow/main.go + .PHONY: print-output print-output: @echo $(OUTPUT) +.PHONY: print-reflow-output +print-reflow-output: + @echo $(REFLOW_OUTPUT) + +.PHONY: package-reflow-tar +package-reflow-tar: build-reflow + rm -rf $(REFLOW_PACKAGE_DIR) + mkdir -p $(REFLOW_PACKAGE_DIR) + cp $(REFLOW_OUTPUT) $(REFLOW_PACKAGE_DIR)/reflow + cp README.md LICENSE $(REFLOW_PACKAGE_DIR)/ + tar -C $(DIST_DIR) -czf $(REFLOW_PACKAGE).tar.gz $(notdir $(REFLOW_PACKAGE_DIR)) + cd $(DIST_DIR) && shasum -a 256 $(notdir $(REFLOW_PACKAGE)).tar.gz > $(notdir $(REFLOW_PACKAGE)).tar.gz.sha256 + +.PHONY: package-reflow-zip +package-reflow-zip: build-reflow + rm -rf $(REFLOW_PACKAGE_DIR) + mkdir -p $(REFLOW_PACKAGE_DIR) + cp $(REFLOW_OUTPUT) $(REFLOW_PACKAGE_DIR)/reflow$(EXTENSION) + cp README.md LICENSE $(REFLOW_PACKAGE_DIR)/ + cd $(DIST_DIR) && zip -qr $(notdir $(REFLOW_PACKAGE)).zip $(notdir $(REFLOW_PACKAGE_DIR)) + cd $(DIST_DIR) && shasum -a 256 $(notdir $(REFLOW_PACKAGE)).zip > $(notdir $(REFLOW_PACKAGE)).zip.sha256 + .PHONY: docker # Build docker image for the current version. docker: diff --git a/README.md b/README.md index 82190d01..1387f419 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,22 @@ not provided. ![GoFlow2 System diagram](/graphics/diagram.png) +## ReFlow + +This repository now also includes an experimental `reflow` binary under `cmd/reflow`. +ReFlow keeps the collector roots of GoFlow2, but turns them into a small pipeline: +ingest from sockets or live packet capture, decode into internal events, optionally +process and aggregate them, then emit JSON, protobuf, sFlow, IPFIX, or NetFlow v9/v5. + +The goal is to make common traffic conversion and reshaping tasks easier to express in +one config file. A ReFlow config can start from UDP flow messages, JSON events, or live +packets, then write the result to stdout, a file, UDP, or a Unix datagram socket. + +ReFlow source framing is selected by `sources[].network`. `udp` and `unixgram` +are packet-oriented sources and produce one event per datagram. `stream` is a +byte-stream source; `address: "-"` means stdin, and JSON stream input is newline +delimited with one JSON object per line. + ## Origins This work is a fork of a previous [open-source GoFlow code](https://github.com/cloudflare/goflow) built and used at Cloudflare. diff --git a/cmd/reflow/main.go b/cmd/reflow/main.go new file mode 100644 index 00000000..ea421e30 --- /dev/null +++ b/cmd/reflow/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/netsampler/goflow2/v3/pkg/reflow/app" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/encode" + "gopkg.in/yaml.v3" +) + +var ( + version = "" + buildinfos = "" + appVersion = "ReFlow " + version + " " + buildinfos +) + +func main() { + cfg, versionFlag := config.BindFlags(flag.CommandLine) + if err := flag.CommandLine.Parse(config.NormalizeAggregateArgs(os.Args[1:])); err != nil { + log.Fatal(err) + } + + if *versionFlag { + fmt.Println(appVersion) + os.Exit(0) + } + if cfg.ListOptions { + fmt.Print(config.HelperOptionsText()) + return + } + + loadedCfg, generated, err := config.LoadFromFlags(cfg) + if err != nil { + log.Fatal(err) + } + loadedCfg.LogLevel = cfg.LogLevel + loadedCfg.LogFormat = cfg.LogFormat + + if cfg.GenConf { + if !generated { + log.Fatal("-genconf requires generated config mode; omit -config") + } + raw, err := yaml.Marshal(loadedCfg) + if err != nil { + log.Fatal(err) + } + if _, err := os.Stdout.Write(raw); err != nil { + log.Fatal(err) + } + return + } + if cfg.GenProto { + raw, err := encode.GenerateProtobufDefinition(loadedCfg.Encoder.Protobuf) + if err != nil { + log.Fatal(err) + } + fmt.Print(raw) + return + } + + // App.New wires the full runtime graph from the loaded config. + application, err := app.New(loadedCfg) + if err != nil { + log.Fatal(err) + } + + // ReFlow exits cleanly on Ctrl-C or SIGTERM so downstream flush paths run. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err := application.Run(ctx); err != nil { + slog.Error("application error", slog.String("error", err.Error())) + os.Exit(1) + } +} diff --git a/cmd/reflow/reflow-json-interface-options-to-ipfix.yaml b/cmd/reflow/reflow-json-interface-options-to-ipfix.yaml new file mode 100644 index 00000000..5a0b5a37 --- /dev/null +++ b/cmd/reflow/reflow-json-interface-options-to-ipfix.yaml @@ -0,0 +1,38 @@ +sources: + - network: udp + address: "127.0.0.1:18082" + type: json + json: + flavor: reflow + +processor: + type: builtin + builtin: + drop_message: true + +aggregators: + - stream: interface_options + match: + record_kind: interface_option + window: + idle_flush_after_ms: 500 + template_id: 1300 + fields: + - static:tflow_record_type:options + - key:observation_domain_id + - key:input_if + - current:interface_name + +encoder: + type: ipfix + batch: + enabled: true + max_records: 2 + flush_interval_ms: 500 + templated_flow: + observation_domain_id: 777 + options_refresh_ms: 30000 + +sink: + type: udp + address: "127.0.0.1:2055" diff --git a/decoders/netflow/encode.go b/decoders/netflow/encode.go index 6a91f2bf..bf05c8dc 100644 --- a/decoders/netflow/encode.go +++ b/decoders/netflow/encode.go @@ -103,16 +103,16 @@ func encodeIPFIXPacket(packet *IPFIXPacket) ([]byte, error) { return nil, err } - length := packet.Length - if length == 0 { - length = uint16(ipfixHeaderLen + len(flowSetPayload)) + length := ipfixHeaderLen + len(flowSetPayload) + if length > 0xffff { + return nil, fmt.Errorf("netflow: IPFIX message length %d exceeds 65535", length) } buf := bytes.NewBuffer(make([]byte, 0, int(length))) if err := utils.WriteU16(buf, version); err != nil { return nil, err } - if err := utils.WriteU16(buf, length); err != nil { + if err := utils.WriteU16(buf, uint16(length)); err != nil { return nil, err } if err := utils.WriteU32(buf, packet.ExportTime); err != nil { @@ -127,6 +127,9 @@ func encodeIPFIXPacket(packet *IPFIXPacket) ([]byte, error) { if _, err := buf.Write(flowSetPayload); err != nil { return nil, err } + if buf.Len() != length { + return nil, fmt.Errorf("netflow: IPFIX message length invariant failed header:%d emitted:%d", length, buf.Len()) + } return buf.Bytes(), nil } @@ -138,15 +141,23 @@ func encodeFlowSets(version uint16, flowSets []interface{}) ([]byte, error) { if err != nil { return nil, err } + setStart := buf.Len() if err := utils.WriteU16(buf, id); err != nil { return nil, err } - if err := utils.WriteU16(buf, uint16(flowSetHeaderLen+len(payload))); err != nil { + length := flowSetHeaderLen + len(payload) + if length > 0xffff { + return nil, fmt.Errorf("netflow: flow set %d length %d exceeds 65535", id, length) + } + if err := utils.WriteU16(buf, uint16(length)); err != nil { return nil, err } if _, err := buf.Write(payload); err != nil { return nil, err } + if buf.Len() != setStart+length { + return nil, fmt.Errorf("netflow: flow set %d length invariant failed header:%d emitted:%d", id, length, buf.Len()-setStart) + } } return buf.Bytes(), nil } @@ -477,7 +488,14 @@ func encodeDataFieldValue(buf *bytes.Buffer, field DataField, template *Field) e return fmt.Errorf("netflow: encode field %d: %w", field.Type, err) } - if template != nil && template.Length == 0xffff { + length := field.Length + if template != nil { + length = template.Length + } + if length == 0xffff { + if len(value) > 0xffff { + return fmt.Errorf("netflow: variable-length field %d length %d exceeds 65535", field.Type, len(value)) + } if len(value) < 255 { if err := utils.WriteU8(buf, uint8(len(value))); err != nil { return err @@ -490,8 +508,8 @@ func encodeDataFieldValue(buf *bytes.Buffer, field DataField, template *Field) e return err } } - } else if template != nil && template.Length != 0 && int(template.Length) != len(value) { - return fmt.Errorf("length mismatch header:%d value:%d", template.Length, len(value)) + } else if length != 0 && int(length) != len(value) { + return fmt.Errorf("length mismatch header:%d value:%d", length, len(value)) } _, err = buf.Write(value) diff --git a/decoders/netflow/encode_test.go b/decoders/netflow/encode_test.go index 57cd86a7..08b1dc8b 100644 --- a/decoders/netflow/encode_test.go +++ b/decoders/netflow/encode_test.go @@ -2,6 +2,7 @@ package netflow import ( "bytes" + "encoding/binary" "testing" "github.com/stretchr/testify/assert" @@ -217,3 +218,143 @@ func TestEncodeDecodeIPFIX(t *testing.T) { assert.Equal(t, []byte{0, 0, 3, 21}, optionsDataSet.Records[0].ScopesValues[0].Value) assert.Equal(t, []byte("abcde"), optionsDataSet.Records[0].OptionsValues[0].Value) } + +func TestEncodeIPFIXLengthsUseEmittedVariableLengthSection(t *testing.T) { + section := bytes.Repeat([]byte{0xab}, 88) + packet := IPFIXPacket{ + Version: 10, + Length: 856, + ExportTime: 123, + SequenceNumber: 456, + ObservationDomainId: 789, + FlowSets: []interface{}{ + TemplateFlowSet{ + FlowSetHeader: FlowSetHeader{Id: 2}, + Records: []TemplateRecord{ + { + TemplateId: 256, + Fields: []Field{ + {Type: IPFIX_FIELD_octetDeltaCount, Length: 8}, + {Type: IPFIX_FIELD_dataLinkFrameSize, Length: 2}, + {Type: IPFIX_FIELD_dataLinkFrameSection, Length: 0xffff}, + }, + }, + }, + }, + DataFlowSet{ + FlowSetHeader: FlowSetHeader{Id: 256}, + Records: []DataRecord{ + { + Values: []DataField{ + {Type: IPFIX_FIELD_octetDeltaCount, Value: []byte{0, 0, 0, 0, 0, 0, 0, 1}}, + {Type: IPFIX_FIELD_dataLinkFrameSize, Value: []byte{0, 0x60}}, + {Type: IPFIX_FIELD_dataLinkFrameSection, Value: section}, + }, + }, + { + Values: []DataField{ + {Type: IPFIX_FIELD_octetDeltaCount, Value: []byte{0, 0, 0, 0, 0, 0, 0, 2}}, + {Type: IPFIX_FIELD_dataLinkFrameSize, Value: []byte{0, 0x60}}, + {Type: IPFIX_FIELD_dataLinkFrameSection, Value: section}, + }, + }, + }, + }, + }, + } + + encoded, err := EncodeMessage(&packet) + assert.NoError(t, err) + assert.Equal(t, len(encoded), int(binary.BigEndian.Uint16(encoded[2:4]))) + + offset := ipfixHeaderLen + templateSetLength := int(binary.BigEndian.Uint16(encoded[offset+2 : offset+4])) + assert.Equal(t, offset+templateSetLength, nextSetOffset(t, encoded, offset)) + offset += templateSetLength + + dataSetLength := int(binary.BigEndian.Uint16(encoded[offset+2 : offset+4])) + recordLength := 8 + 2 + 1 + len(section) + padding := (4 - ((flowSetHeaderLen + recordLength*2) % 4)) % 4 + assert.Equal(t, flowSetHeaderLen+recordLength*2+padding, dataSetLength) + assert.Equal(t, len(encoded), offset+dataSetLength) + + recordOffset := offset + flowSetHeaderLen + assert.Equal(t, byte(0x58), encoded[recordOffset+8+2]) + assert.Equal(t, section, encoded[recordOffset+8+2+1:recordOffset+recordLength]) + + recordOffset += recordLength + assert.Equal(t, byte(0x58), encoded[recordOffset+8+2]) + assert.Equal(t, section, encoded[recordOffset+8+2+1:recordOffset+recordLength]) +} + +func TestEncodeIPFIXPayloadAggregateFrameSectionBoundary(t *testing.T) { + record := func(frameSize uint16, section []byte) DataRecord { + fixed := make([]byte, 99) + return DataRecord{ + Values: []DataField{ + {Type: IPFIX_FIELD_dataLinkFrameSize, Length: 2, Value: []byte{byte(frameSize >> 8), byte(frameSize)}}, + {Type: IPFIX_FIELD_octetTotalCount, Length: uint16(len(fixed)), Value: fixed}, + {Type: IPFIX_FIELD_dataLinkFrameSection, Length: 0xffff, Value: section}, + }, + } + } + packet := IPFIXPacket{ + Version: 10, + Length: 0x0408, + ExportTime: 123, + SequenceNumber: 456, + ObservationDomainId: 789, + FlowSets: []interface{}{ + DataFlowSet{ + FlowSetHeader: FlowSetHeader{Id: 256}, + Records: []DataRecord{ + record(74, bytes.Repeat([]byte{0xaa}, 74)), + record(96, bytes.Repeat([]byte{0xbb}, 96)), + record(96, bytes.Repeat([]byte{0xcc}, 88)), + }, + }, + DataFlowSet{ + FlowSetHeader: FlowSetHeader{Id: 257}, + Records: []DataRecord{ + { + Values: []DataField{ + {Type: IPFIX_FIELD_dataLinkFrameSection, Length: 436, Value: bytes.Repeat([]byte{0xdd}, 436)}, + }, + }, + }, + }, + }, + } + + encoded, err := EncodeMessage(&packet) + assert.NoError(t, err) + assert.Equal(t, 0x0400, int(binary.BigEndian.Uint16(encoded[2:4]))) + assert.Equal(t, 1024, len(encoded)) + + set256Offset := ipfixHeaderLen + assert.Equal(t, 256, int(binary.BigEndian.Uint16(encoded[set256Offset:set256Offset+2]))) + assert.Equal(t, 0x0238, int(binary.BigEndian.Uint16(encoded[set256Offset+2:set256Offset+4]))) + record3Offset := set256Offset + flowSetHeaderLen + (101 + 1 + 74) + (101 + 1 + 96) + assert.Equal(t, byte(0x58), encoded[record3Offset+101]) + + set257Offset := set256Offset + int(binary.BigEndian.Uint16(encoded[set256Offset+2:set256Offset+4])) + assert.Equal(t, 584, set257Offset) + assert.Equal(t, 257, int(binary.BigEndian.Uint16(encoded[set257Offset:set257Offset+2]))) + assert.Equal(t, 0x01b8, int(binary.BigEndian.Uint16(encoded[set257Offset+2:set257Offset+4]))) + assert.Equal(t, len(encoded), set257Offset+int(binary.BigEndian.Uint16(encoded[set257Offset+2:set257Offset+4]))) +} + +func nextSetOffset(t *testing.T, encoded []byte, offset int) int { + t.Helper() + if offset+flowSetHeaderLen > len(encoded) { + t.Fatalf("set header at offset %d exceeds message length %d", offset, len(encoded)) + } + length := int(binary.BigEndian.Uint16(encoded[offset+2 : offset+4])) + if length < flowSetHeaderLen { + t.Fatalf("set at offset %d has invalid length %d", offset, length) + } + if offset+length > len(encoded) { + t.Fatalf("set at offset %d length %d exceeds message length %d", offset, length, len(encoded)) + } + return offset + length +} diff --git a/decoders/netflow/packet.go b/decoders/netflow/packet.go index c6a8b60d..fd3c3246 100644 --- a/decoders/netflow/packet.go +++ b/decoders/netflow/packet.go @@ -101,6 +101,7 @@ type DataField struct { // A numeric value that represents the type of field. PenProvided bool `json:"pen-provided"` Type uint16 `json:"type"` + Length uint16 `json:"length,omitempty"` Pen uint32 `json:"pen"` // The value (in bytes) of the field. diff --git a/decoders/netflowlegacy/packet.go b/decoders/netflowlegacy/packet.go index 765db444..09b3d776 100644 --- a/decoders/netflowlegacy/packet.go +++ b/decoders/netflowlegacy/packet.go @@ -45,10 +45,14 @@ type RecordsNetFlowV5 struct { // IPAddress is a NetFlow legacy IPv4 address rendered as dotted decimal. type IPAddress uint32 +func (s IPAddress) String() string { + return fmt.Sprintf("%d.%d.%d.%d", s>>24, (s>>16)&0xFF, (s>>8)&0xFF, s&0xFF) +} + // MarshalJSON formats the IPv4 address as a JSON string. func IPAddressToUint32(addr IPAddress) uint32 { return uint32(addr) } func (s *IPAddress) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("\"%d.%d.%d.%d\"", *s>>24, (*s>>16)&0xFF, (*s>>8)&0xFF, *s&0xFF)), nil + return []byte(fmt.Sprintf("%q", s.String())), nil } diff --git a/decoders/sflow/datastructure.go b/decoders/sflow/datastructure.go index a4b3db8d..d93c0115 100644 --- a/decoders/sflow/datastructure.go +++ b/decoders/sflow/datastructure.go @@ -81,6 +81,47 @@ type ExtendedGateway struct { LocalPref uint32 `json:"local-pref"` } +// ExtendedMPLS carries MPLS next-hop and label stack metadata. +type ExtendedMPLS struct { + NextHopIPVersion uint32 `json:"next-hop-ip-version"` + NextHop utils.IPAddress `json:"next-hop"` + InLabelStack []uint32 `json:"in-label-stack"` + OutLabelStack []uint32 `json:"out-label-stack"` +} + +// ExtendedNAT carries translated source and destination addresses. +type ExtendedNAT struct { + SrcAddressIPVersion uint32 `json:"src-address-ip-version"` + SrcAddress utils.IPAddress `json:"src-address"` + DstAddressIPVersion uint32 `json:"dst-address-ip-version"` + DstAddress utils.IPAddress `json:"dst-address"` +} + +// ExtendedMPLSTunnel carries MPLS tunnel metadata. +type ExtendedMPLSTunnel struct { + TunnelLSPName string `json:"tunnel-lsp-name"` + TunnelID uint32 `json:"tunnel-id"` + TunnelCOS uint32 `json:"tunnel-cos"` +} + +// ExtendedMPLSVC carries MPLS virtual circuit metadata. +type ExtendedMPLSVC struct { + VCInstanceName string `json:"vc-instance-name"` + VLLVCID uint32 `json:"vll-vc-id"` + VCLabelCOS uint32 `json:"vc-label-cos"` +} + +// ExtendedMPLSFTN carries MPLS FTN metadata. +type ExtendedMPLSFTN struct { + MPLSFTNDescr string `json:"mpls-ftn-descr"` + MPLSFTNMask uint32 `json:"mpls-ftn-mask"` +} + +// ExtendedMPLSLDPFEC carries MPLS LDP FEC metadata. +type ExtendedMPLSLDPFEC struct { + MPLSFecAddrPrefixLength uint32 `json:"mpls-fec-addr-prefix-length"` +} + // EgressQueue reports a queue identifier for drop records. type EgressQueue struct { Queue uint32 `json:"queue"` @@ -142,3 +183,9 @@ type EthernetCounters struct { type RawRecord struct { Data []byte `json:"data"` } + +// RawSample stores unparsed sample bytes for unknown standard or enterprise samples. +type RawSample struct { + Header SampleHeader `json:"header"` + Data []byte `json:"data"` +} diff --git a/decoders/sflow/encode.go b/decoders/sflow/encode.go index e4e695a9..f2e4a7cf 100644 --- a/decoders/sflow/encode.go +++ b/decoders/sflow/encode.go @@ -149,6 +149,16 @@ func encodeSample(sample interface{}) ([]byte, uint32, error) { } payload, err := encodeDropSample(s, format) return payload, format, err + case RawSample: + if s.Header.Format == 0 { + return nil, 0, fmt.Errorf("sflow: raw sample format is not set") + } + return append([]byte(nil), s.Data...), s.Header.Format, nil + case *RawSample: + if s.Header.Format == 0 { + return nil, 0, fmt.Errorf("sflow: raw sample format is not set") + } + return append([]byte(nil), s.Data...), s.Header.Format, nil default: return nil, 0, fmt.Errorf("sflow: unsupported sample type %T", sample) } @@ -554,6 +564,90 @@ func encodeFlowRecord(buf *bytes.Buffer, record *FlowRecord) error { if err := encodeExtendedGateway(payload, data); err != nil { return err } + case ExtendedMPLS: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS + } + if err := encodeExtendedMPLS(payload, &data); err != nil { + return err + } + case *ExtendedMPLS: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS + } + if err := encodeExtendedMPLS(payload, data); err != nil { + return err + } + case ExtendedNAT: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_NAT + } + if err := encodeExtendedNAT(payload, &data); err != nil { + return err + } + case *ExtendedNAT: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_NAT + } + if err := encodeExtendedNAT(payload, data); err != nil { + return err + } + case ExtendedMPLSTunnel: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS_TUNNEL + } + if err := encodeExtendedMPLSTunnel(payload, &data); err != nil { + return err + } + case *ExtendedMPLSTunnel: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS_TUNNEL + } + if err := encodeExtendedMPLSTunnel(payload, data); err != nil { + return err + } + case ExtendedMPLSVC: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS_VC + } + if err := encodeExtendedMPLSVC(payload, &data); err != nil { + return err + } + case *ExtendedMPLSVC: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS_VC + } + if err := encodeExtendedMPLSVC(payload, data); err != nil { + return err + } + case ExtendedMPLSFTN: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS_FEC + } + if err := encodeExtendedMPLSFTN(payload, &data); err != nil { + return err + } + case *ExtendedMPLSFTN: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS_FEC + } + if err := encodeExtendedMPLSFTN(payload, data); err != nil { + return err + } + case ExtendedMPLSLDPFEC: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS_LVP_FEC + } + if err := utils.WriteU32(payload, data.MPLSFecAddrPrefixLength); err != nil { + return err + } + case *ExtendedMPLSLDPFEC: + if dataFormat == 0 { + dataFormat = FLOW_TYPE_EXT_MPLS_LVP_FEC + } + if err := utils.WriteU32(payload, data.MPLSFecAddrPrefixLength); err != nil { + return err + } case EgressQueue: if dataFormat == 0 { dataFormat = FLOW_TYPE_EGRESS_QUEUE @@ -968,8 +1062,67 @@ func encodeExtendedGateway(buf *bytes.Buffer, data *ExtendedGateway) error { return nil } +func encodeExtendedMPLS(buf *bytes.Buffer, data *ExtendedMPLS) error { + if err := encodeIP(buf, data.NextHopIPVersion, data.NextHop); err != nil { + return err + } + if err := writeU32Vector(buf, data.InLabelStack); err != nil { + return err + } + return writeU32Vector(buf, data.OutLabelStack) +} + +func encodeExtendedNAT(buf *bytes.Buffer, data *ExtendedNAT) error { + if err := encodeIP(buf, data.SrcAddressIPVersion, data.SrcAddress); err != nil { + return err + } + return encodeIP(buf, data.DstAddressIPVersion, data.DstAddress) +} + +func encodeExtendedMPLSTunnel(buf *bytes.Buffer, data *ExtendedMPLSTunnel) error { + if err := writeXDRString(buf, data.TunnelLSPName); err != nil { + return err + } + if err := utils.WriteU32(buf, data.TunnelID); err != nil { + return err + } + return utils.WriteU32(buf, data.TunnelCOS) +} + +func encodeExtendedMPLSVC(buf *bytes.Buffer, data *ExtendedMPLSVC) error { + if err := writeXDRString(buf, data.VCInstanceName); err != nil { + return err + } + if err := utils.WriteU32(buf, data.VLLVCID); err != nil { + return err + } + return utils.WriteU32(buf, data.VCLabelCOS) +} + +func encodeExtendedMPLSFTN(buf *bytes.Buffer, data *ExtendedMPLSFTN) error { + if err := writeXDRString(buf, data.MPLSFTNDescr); err != nil { + return err + } + return utils.WriteU32(buf, data.MPLSFTNMask) +} + +func writeU32Vector(buf *bytes.Buffer, values []uint32) error { + if err := utils.WriteU32(buf, uint32(len(values))); err != nil { + return err + } + for _, value := range values { + if err := utils.WriteU32(buf, value); err != nil { + return err + } + } + return nil +} + func encodeIP(buf *bytes.Buffer, ipVersion uint32, ip []byte) error { if ipVersion == 0 { + if len(ip) == 0 { + return utils.WriteU32(buf, 0) + } switch len(ip) { case 4: ipVersion = 1 diff --git a/decoders/sflow/encode_test.go b/decoders/sflow/encode_test.go index 0bfc582e..a17ef1b5 100644 --- a/decoders/sflow/encode_test.go +++ b/decoders/sflow/encode_test.go @@ -170,6 +170,106 @@ func TestEncodeDecodeSFlowExpandedFlowSample(t *testing.T) { assert.Equal(t, []uint32{100, 200}, gw.Communities) } +func TestEncodeDecodeSFlowExtendedNATMPLSAndMPLSFamily(t *testing.T) { + packet := Packet{ + Version: 5, + IPVersion: 1, + AgentIP: utils.IPAddress{192, 0, 2, 1}, + SubAgentId: 1, + SequenceNumber: 2, + Uptime: 3, + Samples: []interface{}{ + FlowSample{ + Header: SampleHeader{ + Format: SAMPLE_FORMAT_FLOW, + SampleSequenceNumber: 42, + SourceIdValue: 7, + }, + Records: []FlowRecord{ + {Data: ExtendedNAT{ + SrcAddress: utils.IPAddress{203, 0, 113, 10}, + DstAddress: utils.IPAddress{198, 51, 100, 20}, + }}, + {Data: ExtendedMPLS{ + NextHop: utils.IPAddress{192, 0, 2, 254}, + InLabelStack: []uint32{0x00011100}, + OutLabelStack: []uint32{0x00022200, 0x00033300}, + }}, + {Data: ExtendedMPLSTunnel{TunnelLSPName: "lsp-a", TunnelID: 11, TunnelCOS: 3}}, + {Data: ExtendedMPLSVC{VCInstanceName: "vc-a", VLLVCID: 12, VCLabelCOS: 4}}, + {Data: ExtendedMPLSFTN{MPLSFTNDescr: "ftn-a", MPLSFTNMask: 24}}, + {Data: ExtendedMPLSLDPFEC{MPLSFecAddrPrefixLength: 25}}, + }, + }, + }, + } + + encoded, err := EncodeMessage(&packet) + assert.NoError(t, err) + + var decoded Packet + assert.NoError(t, DecodeMessageVersion(bytes.NewBuffer(encoded), &decoded)) + sample := decoded.Samples[0].(FlowSample) + assert.Len(t, sample.Records, 6) + + nat := sample.Records[0].Data.(ExtendedNAT) + assert.Equal(t, utils.IPAddress{203, 0, 113, 10}, nat.SrcAddress) + assert.Equal(t, utils.IPAddress{198, 51, 100, 20}, nat.DstAddress) + mpls := sample.Records[1].Data.(ExtendedMPLS) + assert.Equal(t, utils.IPAddress{192, 0, 2, 254}, mpls.NextHop) + assert.Equal(t, []uint32{0x00011100}, mpls.InLabelStack) + assert.Equal(t, []uint32{0x00022200, 0x00033300}, mpls.OutLabelStack) + assert.Equal(t, "lsp-a", sample.Records[2].Data.(ExtendedMPLSTunnel).TunnelLSPName) + assert.Equal(t, uint32(12), sample.Records[3].Data.(ExtendedMPLSVC).VLLVCID) + assert.Equal(t, "ftn-a", sample.Records[4].Data.(ExtendedMPLSFTN).MPLSFTNDescr) + assert.Equal(t, uint32(25), sample.Records[5].Data.(ExtendedMPLSLDPFEC).MPLSFecAddrPrefixLength) +} + +func TestEncodeDecodeSFlowRawEnterpriseRecordAndSample(t *testing.T) { + enterpriseFormat := PackDataFormat(64512, 7) + packet := Packet{ + Version: 5, + IPVersion: 1, + AgentIP: utils.IPAddress{192, 0, 2, 1}, + SubAgentId: 1, + SequenceNumber: 2, + Uptime: 3, + Samples: []interface{}{ + FlowSample{ + Header: SampleHeader{Format: SAMPLE_FORMAT_FLOW, SampleSequenceNumber: 1}, + Records: []FlowRecord{ + {Header: RecordHeader{DataFormat: enterpriseFormat}, Data: RawRecord{Data: []byte{0xde, 0xad, 0xbe, 0xef}}}, + }, + }, + CounterSample{ + Header: SampleHeader{Format: SAMPLE_FORMAT_COUNTER, SampleSequenceNumber: 2}, + Records: []CounterRecord{ + {Header: RecordHeader{DataFormat: enterpriseFormat}, Data: RawRecord{Data: []byte{0xca, 0xfe, 0xba, 0xbe}}}, + }, + }, + RawSample{ + Header: SampleHeader{Format: PackDataFormat(64512, 8)}, + Data: []byte{0x01, 0x02, 0x03, 0x04}, + }, + }, + } + + encoded, err := EncodeMessage(&packet) + assert.NoError(t, err) + + var decoded Packet + assert.NoError(t, DecodeMessageVersion(bytes.NewBuffer(encoded), &decoded)) + flow := decoded.Samples[0].(FlowSample) + assert.Equal(t, enterpriseFormat, flow.Records[0].Header.DataFormat) + assert.Equal(t, []byte{0xde, 0xad, 0xbe, 0xef}, flow.Records[0].Data.(RawRecord).Data) + counter := decoded.Samples[1].(CounterSample) + assert.Equal(t, enterpriseFormat, counter.Records[0].Header.DataFormat) + assert.Equal(t, []byte{0xca, 0xfe, 0xba, 0xbe}, counter.Records[0].Data.(RawRecord).Data) + raw := decoded.Samples[2].(RawSample) + assert.Equal(t, PackDataFormat(64512, 8), raw.Header.Format) + assert.Equal(t, []byte{0x01, 0x02, 0x03, 0x04}, raw.Data) +} + func TestEncodeDecodeSFlowCounterSample(t *testing.T) { packet := Packet{ Version: 5, diff --git a/decoders/sflow/sflow.go b/decoders/sflow/sflow.go index 74335e7a..c0b0d02f 100644 --- a/decoders/sflow/sflow.go +++ b/decoders/sflow/sflow.go @@ -52,6 +52,23 @@ const ( COUNTER_TYPE_CPU = 1001 ) +const dataFormatFormatMask = 0x00000fff + +// PackDataFormat builds an sFlow data_format from enterprise and format parts. +func PackDataFormat(enterprise, format uint32) uint32 { + return (enterprise << 12) | (format & dataFormatFormatMask) +} + +// DataFormatEnterprise returns the enterprise number from a packed sFlow data_format. +func DataFormatEnterprise(dataFormat uint32) uint32 { + return dataFormat >> 12 +} + +// DataFormatFormat returns the format number from a packed sFlow data_format. +func DataFormatFormat(dataFormat uint32) uint32 { + return dataFormat & dataFormatFormatMask +} + // DecoderError wraps an sFlow decode error. type DecoderError struct { Err error @@ -126,7 +143,11 @@ func DecodeCounterRecord(header *RecordHeader, payload *bytes.Buffer) (CounterRe counterRecord := CounterRecord{ Header: *header, } - switch header.DataFormat { + if DataFormatEnterprise(header.DataFormat) != 0 { + counterRecord.Data = RawRecord{Data: append([]byte(nil), payload.Bytes()...)} + return counterRecord, nil + } + switch DataFormatFormat(header.DataFormat) { case COUNTER_TYPE_IF: var ifCounters IfCounters if err := utils.BinaryDecoder(payload, @@ -175,7 +196,7 @@ func DecodeCounterRecord(header *RecordHeader, payload *bytes.Buffer) (CounterRe counterRecord.Data = ethernetCounters default: var rawRecord RawRecord - rawRecord.Data = payload.Bytes() + rawRecord.Data = append([]byte(nil), payload.Bytes()...) counterRecord.Data = rawRecord } @@ -187,8 +208,13 @@ func DecodeFlowRecord(header *RecordHeader, payload *bytes.Buffer) (FlowRecord, flowRecord := FlowRecord{ Header: *header, } + if DataFormatEnterprise(header.DataFormat) != 0 { + flowRecord.Data = RawRecord{Data: append([]byte(nil), payload.Bytes()...)} + return flowRecord, nil + } + format := DataFormatFormat(header.DataFormat) var err error - switch header.DataFormat { + switch format { case FLOW_TYPE_RAW: sampledHeader := SampledHeader{} var headerLength uint32 @@ -200,9 +226,15 @@ func DecodeFlowRecord(header *RecordHeader, payload *bytes.Buffer) (FlowRecord, ); err != nil { return flowRecord, &RecordError{header.DataFormat, err} } - headerData, err := readXDROpaqueWithLength(payload, headerLength) - if err != nil { - return flowRecord, &RecordError{header.DataFormat, err} + var headerData []byte + if int(headerLength) > payload.Len() { + headerData = append([]byte(nil), payload.Bytes()...) + payload.Next(payload.Len()) + } else { + headerData, err = readXDROpaqueWithLength(payload, headerLength) + if err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } } sampledHeader.OriginalLength = headerLength sampledHeader.HeaderData = headerData @@ -361,6 +393,63 @@ func DecodeFlowRecord(header *RecordHeader, payload *bytes.Buffer) (FlowRecord, extendedGateway.Communities = communities flowRecord.Data = extendedGateway + case FLOW_TYPE_EXT_MPLS: + var mpls ExtendedMPLS + if mpls.NextHopIPVersion, mpls.NextHop, err = DecodeIP(payload); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + if mpls.InLabelStack, err = decodeU32Vector(payload, "mpls in label stack"); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + if mpls.OutLabelStack, err = decodeU32Vector(payload, "mpls out label stack"); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + flowRecord.Data = mpls + case FLOW_TYPE_EXT_NAT: + var nat ExtendedNAT + if nat.SrcAddressIPVersion, nat.SrcAddress, err = DecodeIP(payload); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + if nat.DstAddressIPVersion, nat.DstAddress, err = DecodeIP(payload); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + flowRecord.Data = nat + case FLOW_TYPE_EXT_MPLS_TUNNEL: + var tunnel ExtendedMPLSTunnel + tunnel.TunnelLSPName, err = readXDRString(payload) + if err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + if err := utils.BinaryDecoder(payload, &tunnel.TunnelID, &tunnel.TunnelCOS); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + flowRecord.Data = tunnel + case FLOW_TYPE_EXT_MPLS_VC: + var vc ExtendedMPLSVC + vc.VCInstanceName, err = readXDRString(payload) + if err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + if err := utils.BinaryDecoder(payload, &vc.VLLVCID, &vc.VCLabelCOS); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + flowRecord.Data = vc + case FLOW_TYPE_EXT_MPLS_FEC: + var ftn ExtendedMPLSFTN + ftn.MPLSFTNDescr, err = readXDRString(payload) + if err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + if err := utils.BinaryDecoder(payload, &ftn.MPLSFTNMask); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + flowRecord.Data = ftn + case FLOW_TYPE_EXT_MPLS_LVP_FEC: + var fec ExtendedMPLSLDPFEC + if err := utils.BinaryDecoder(payload, &fec.MPLSFecAddrPrefixLength); err != nil { + return flowRecord, &RecordError{header.DataFormat, err} + } + flowRecord.Data = fec case FLOW_TYPE_EGRESS_QUEUE: var queue EgressQueue if err := utils.BinaryDecoder(payload, &queue.Queue); err != nil { @@ -391,12 +480,32 @@ func DecodeFlowRecord(header *RecordHeader, payload *bytes.Buffer) (FlowRecord, flowRecord.Data = function default: var rawRecord RawRecord - rawRecord.Data = payload.Bytes() + rawRecord.Data = append([]byte(nil), payload.Bytes()...) flowRecord.Data = rawRecord } return flowRecord, nil } +func decodeU32Vector(payload *bytes.Buffer, name string) ([]uint32, error) { + var length uint32 + if err := utils.BinaryDecoder(payload, &length); err != nil { + return nil, err + } + if length > 1000 { + return nil, fmt.Errorf("%s length of %d seems quite large", name, length) + } + if int(length) > payload.Len()/4 { + return nil, fmt.Errorf("invalid %s length: %d", name, length) + } + values := make([]uint32, length) + if len(values) > 0 { + if err := utils.BinaryDecoder(payload, values); err != nil { + return nil, err + } + } + return values, nil +} + func DecodeSample(header *SampleHeader, payload *bytes.Buffer) (interface{}, error) { format := header.Format var sample interface{} @@ -593,16 +702,26 @@ func DecodeMessage(payload *bytes.Buffer, packetV5 *Packet) error { } packetV5.Samples = make([]interface{}, int(packetV5.SamplesCount)) // max size of 1000 for protection - for i := 0; i < int(packetV5.SamplesCount) && payload.Len() >= 8; i++ { + for i := 0; i < int(packetV5.SamplesCount); i++ { + if payload.Len() < 8 { + return &DecoderError{fmt.Errorf("sample %d header truncated: need 8 bytes, got %d", i, payload.Len())} + } header := SampleHeader{} if err := utils.BinaryDecoder(payload, &header.Format, &header.Length); err != nil { return &DecoderError{fmt.Errorf("header [%w]", err)} } if int(header.Length) > payload.Len() { - break + return &DecoderError{fmt.Errorf("sample %d length %d exceeds remaining payload %d", i, header.Length, payload.Len())} } sampleReader := bytes.NewBuffer(payload.Next(int(header.Length))) + if DataFormatEnterprise(header.Format) != 0 || !isKnownSampleFormat(DataFormatFormat(header.Format)) { + packetV5.Samples[i] = RawSample{ + Header: header, + Data: append([]byte(nil), sampleReader.Bytes()...), + } + continue + } sample, err := DecodeSample(&header, sampleReader) if err != nil { return &DecoderError{fmt.Errorf("sample [%w]", err)} @@ -613,3 +732,12 @@ func DecodeMessage(payload *bytes.Buffer, packetV5 *Packet) error { return nil } + +func isKnownSampleFormat(format uint32) bool { + switch format { + case SAMPLE_FORMAT_FLOW, SAMPLE_FORMAT_COUNTER, SAMPLE_FORMAT_EXPANDED_FLOW, SAMPLE_FORMAT_EXPANDED_COUNTER, SAMPLE_FORMAT_DROP: + return true + default: + return false + } +} diff --git a/decoders/sflow/sflow_test.go b/decoders/sflow/sflow_test.go index 4419ad38..ed26524a 100644 --- a/decoders/sflow/sflow_test.go +++ b/decoders/sflow/sflow_test.go @@ -208,6 +208,25 @@ func TestSFlowDecodeSampledEthernet(t *testing.T) { assert.Equal(t, 0, payload.Len()) } +func TestSFlowDecodeSampledHeaderUsesRemainingRecordBytesWhenHeaderLengthIsLarge(t *testing.T) { + payload := bytes.NewBuffer([]byte{ + 0x00, 0x00, 0x00, 0x01, // protocol + 0x00, 0x00, 0x00, 0x95, // frame length + 0x00, 0x00, 0x00, 0x04, // stripped + 0x00, 0x00, 0x00, 0x22, // declared header length + 0xde, 0xad, 0xbe, 0xef, // available header bytes + }) + + flowRecord, err := DecodeFlowRecord(&RecordHeader{DataFormat: FLOW_TYPE_RAW}, payload) + assert.NoError(t, err) + assert.Equal(t, 0, payload.Len()) + + sampledHeader, ok := flowRecord.Data.(SampledHeader) + assert.True(t, ok) + assert.Equal(t, uint32(0x22), sampledHeader.OriginalLength) + assert.Equal(t, []byte{0xde, 0xad, 0xbe, 0xef}, sampledHeader.HeaderData) +} + func TestSFlowDecodeDropExtendedFunction(t *testing.T) { data := []byte{ 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0xc0, 0xa8, 0x77, 0xb8, 0x00, 0x01, 0x86, 0xa0, diff --git a/docs/protocols.md b/docs/protocols.md index 1ea3ebd8..027fc322 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -31,7 +31,14 @@ The mapping to the protobuf format is listed in the table below. |dst_mac|Destination mac address| |Included|OUT_DST_MAC (57)|postDestinationMacAddress (57)| |src_vlan|Source VLAN ID| |From ExtendedSwitch|SRC_VLAN (58)|vlanId (58)| |dst_vlan|Destination VLAN ID| |From ExtendedSwitch|DST_VLAN (59)|postVlanId (59)| -|vlan_id|802.11q VLAN ID| |Included|SRC_VLAN (58)|vlanId (58)| +|vlan_id|VLAN ID compatibility alias| |Included|SRC_VLAN (58)|vlanId (58)| +|dot1q_vlan_id|Outer 802.1Q VLAN ID| |Included|SRC_VLAN (58)|dot1qVlanId (243)| +|dot1q_priority|Outer 802.1Q priority| |Included||dot1qPriority (244)| +|dot1q_dei|Outer 802.1Q drop eligible indicator| |Included||dot1qDEI (388)| +|dot1q_customer_vlan_id|Customer 802.1Q VLAN ID| |Included||dot1qCustomerVlanId (245)| +|dot1q_customer_priority|Customer 802.1Q priority| |Included||dot1qCustomerPriority (246)| +|dot1q_customer_dei|Customer 802.1Q drop eligible indicator| |Included||dot1qCustomerDEI (389)| +|layer2_segment_id|Overlay layer-2 segment identifier| |Included||layer2SegmentId (351)| |ip_tos|IP Type of Service|tos|Included|SRC_TOS (5)|ipClassOfService (5)| |forwarding_status|Forwarding status| | |FORWARDING_STATUS (89)|forwardingStatus (89)| |ip_ttl|IP Time to Live| |Included|IPTTL (52)|minimumTTL (52| diff --git a/go.mod b/go.mod index 31902c1c..b0a136d5 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,13 @@ toolchain go1.24.1 require ( github.com/Shopify/sarama v1.38.1 + github.com/google/gopacket v1.1.19 github.com/libp2p/go-reuseport v0.4.0 github.com/oschwald/geoip2-golang v1.13.0 github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.11.1 github.com/xdg-go/scram v1.2.0 + golang.org/x/sys v0.38.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) @@ -46,6 +48,5 @@ require ( go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect ) diff --git a/go.sum b/go.sum index 6027584e..822eeec3 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -94,11 +96,15 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -112,6 +118,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -130,8 +137,10 @@ golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/reflow/aggregate/aggregate.go b/pkg/reflow/aggregate/aggregate.go new file mode 100644 index 00000000..ae5dca08 --- /dev/null +++ b/pkg/reflow/aggregate/aggregate.go @@ -0,0 +1,1067 @@ +package aggregate + +import ( + "fmt" + "net/netip" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +// Aggregator receives processed events and optionally emits aggregated ones. +type Aggregator interface { + InitEvents() ([]*event.Event, error) + Process(evt *event.Event) ([]*event.Event, error) + Flush() ([]*event.Event, error) + Close() ([]*event.Event, error) + Interval() time.Duration +} + +// New builds the configured aggregation stage. Disabled aggregation keeps the input stream unchanged. +func New(cfg config.AggregatorConfig) (Aggregator, error) { + cfg.Fields = effectiveAggregatorFields(cfg) + if cfg.Passthrough { + return schemaPassthrough{cfg: cfg}, nil + } + return NewStateful(cfg), nil +} + +type schemaPassthrough struct { + cfg config.AggregatorConfig +} + +func (p schemaPassthrough) InitEvents() ([]*event.Event, error) { + return schemaInitEvents(p.cfg) +} + +func (p schemaPassthrough) Process(evt *event.Event) ([]*event.Event, error) { + if len(p.cfg.StaticFields) == 0 && p.cfg.EmitAs == "" && (evt == nil || evt.Stream == p.cfg.Stream || p.cfg.Stream == "") { + return []*event.Event{evt}, nil + } + if evt == nil { + return []*event.Event{evt}, nil + } + out := *evt + out.Fields = cloneFields(evt.Fields) + if p.cfg.Stream != "" { + out.Stream = p.cfg.Stream + } + if p.cfg.EmitAs == "data" { + out.Kind = "data" + out.Control = nil + } + for key, val := range p.cfg.StaticFields { + out.Fields[key] = val + } + return []*event.Event{&out}, nil +} + +func (schemaPassthrough) Flush() ([]*event.Event, error) { return nil, nil } +func (schemaPassthrough) Close() ([]*event.Event, error) { return nil, nil } +func (schemaPassthrough) Interval() time.Duration { return 0 } + +type aggregateRecord struct { + Fields map[string]any + FirstSeen time.Time + LastSeen time.Time + Dirty bool +} + +type missingAggregationKeyError struct { + Key string +} + +func (e *missingAggregationKeyError) Error() string { + return fmt.Sprintf("missing aggregation key field %q", e.Key) +} + +// Add merges a delta record into the accumulator while preserving first/last +// visibility windows and dirty-state for periodic export. +func (r *aggregateRecord) Add(delta aggregateRecord, existed bool) error { + if !existed { + delta.Dirty = true + *r = delta + return nil + } + if r.Fields == nil { + r.Fields = make(map[string]any) + } + mergeFields(r.Fields, delta.Fields) + if delta.FirstSeen.Before(r.FirstSeen) { + r.FirstSeen = delta.FirstSeen + } + if delta.LastSeen.After(r.LastSeen) { + r.LastSeen = delta.LastSeen + } + r.Dirty = true + return nil +} + +// Stateful keeps all aggregation policy in one place. +// +// The old aggregator relied on FlowStore TTL only, which was fine for +// "flush when idle" but not enough once ReFlow needed: +// - idle-based flush +// - max lifetime flush +// - periodic snapshot export +// - periodic export with optional bucket reset +// - silent stale bucket cleanup without exporting +// +// Keeping those timers together here makes the behavior explicit and easier to +// reason about than mixing TTL expiration with separate periodic logic. +type Stateful struct { + cfg config.AggregatorConfig + recordCapacity int + + mu sync.Mutex + state map[string]aggregateRecord + startedAt time.Time + lastPeriodicRun time.Time +} + +func NewStateful(cfg config.AggregatorConfig) *Stateful { + cfg.Fields = effectiveAggregatorFields(cfg) + return &Stateful{ + cfg: cfg, + recordCapacity: aggregateRecordCapacity(cfg), + state: make(map[string]aggregateRecord), + startedAt: time.Now(), + } +} + +// InitEvents emits one schema control event so downstream templated encoders can +// advertise aggregated streams before the first bucket flushes. +func (a *Stateful) InitEvents() ([]*event.Event, error) { + return schemaInitEvents(a.cfg) +} + +func schemaInitEvents(cfg config.AggregatorConfig) ([]*event.Event, error) { + fieldNames := orderedSchemaFields(cfg) + if len(fieldNames) == 0 { + return nil, nil + } + return []*event.Event{ + { + ReceivedAt: time.Now().UTC(), + Kind: "control", + Stream: cfg.Stream, + Source: event.SourceMetadata{ + Type: "aggregator", + }, + Control: &event.ControlMetadata{ + Type: "schema", + Stream: cfg.Stream, + }, + Payload: event.AggregationSchema{ + Stream: cfg.Stream, + FieldNames: fieldNames, + Fields: schemaFields(cfg), + KeyFields: append([]string(nil), cfg.KeyFields...), + SumFields: append([]string(nil), cfg.Sum...), + FirstFields: append([]string(nil), cfg.First...), + CurrentFields: append([]string(nil), cfg.Current...), + MinFields: append([]string(nil), cfg.Min...), + MaxFields: append([]string(nil), cfg.Max...), + AndFields: append([]string(nil), cfg.And...), + Match: cloneStringMap(cfg.Match), + StaticFields: cloneFields(cfg.StaticFields), + BaseTemplateID: cfg.TemplateID, + }, + }, + }, nil +} + +func cloneStringMap(in map[string]string) map[string]string { + if len(in) == 0 { + return nil + } + out := make(map[string]string, len(in)) + for key, val := range in { + out[key] = val + } + return out +} + +// Process adds one event into the keyed aggregation state. Missing key fields +// are treated as non-matches rather than hard pipeline failures. +func (a *Stateful) Process(evt *event.Event) ([]*event.Event, error) { + if evt != nil && evt.Kind == "control" { + return []*event.Event{evt}, nil + } + key, record, err := aggregateFromEvent(a.cfg, a.recordCapacity, evt) + if err != nil { + return nil, err + } + a.mu.Lock() + current, exists := a.state[key] + if err := current.Add(record, exists); err != nil { + a.mu.Unlock() + return nil, err + } + a.state[key] = current + a.mu.Unlock() + return nil, nil +} + +// orderedSchemaFields produces a stable field order for schema announcements and +// templated encoders that depend on deterministic field positions. +func orderedSchemaFields(cfg config.AggregatorConfig) []string { + if cfg.FieldsConfigured { + seen := make(map[string]struct{}) + out := make([]string, 0, len(cfg.Fields)) + for _, field := range cfg.Fields { + if field.Name == "" { + continue + } + if _, ok := seen[field.Name]; ok { + continue + } + seen[field.Name] = struct{}{} + out = append(out, field.Name) + } + return out + } + + seen := make(map[string]struct{}) + var out []string + appendField := func(field string) { + if field == "" { + return + } + if _, ok := seen[field]; ok { + return + } + seen[field] = struct{}{} + out = append(out, field) + } + for _, field := range cfg.KeyFields { + appendField(field) + } + for _, field := range cfg.Sum { + appendField(field) + } + for _, field := range cfg.First { + appendField(field) + } + for _, field := range cfg.Current { + appendField(field) + } + for _, field := range cfg.Min { + appendField(field) + } + for _, field := range cfg.Max { + appendField(field) + } + for _, field := range cfg.And { + appendField(field) + } + appendField("start_time_unix") + appendField("end_time_unix") + staticFields := make([]string, 0, len(cfg.StaticFields)) + for field := range cfg.StaticFields { + staticFields = append(staticFields, field) + } + sort.Strings(staticFields) + for _, field := range staticFields { + appendField(field) + } + return out +} + +func schemaFields(cfg config.AggregatorConfig) []event.SchemaField { + if cfg.FieldsConfigured { + out := make([]event.SchemaField, 0, len(cfg.Fields)) + seen := make(map[string]struct{}, len(cfg.Fields)) + for _, field := range cfg.Fields { + if field.Name == "" { + continue + } + if _, ok := seen[field.Name]; ok { + continue + } + seen[field.Name] = struct{}{} + out = append(out, event.SchemaField{ + Role: field.Role, + Name: field.Name, + Value: field.Value, + }) + } + return out + } + + names := orderedSchemaFields(cfg) + out := make([]event.SchemaField, 0, len(names)) + roleByName := make(map[string]string, len(names)) + for _, field := range cfg.KeyFields { + roleByName[field] = "key" + } + for _, field := range cfg.Sum { + roleByName[field] = "sum" + } + for _, field := range cfg.First { + roleByName[field] = "first" + } + for _, field := range cfg.Current { + roleByName[field] = "current" + } + for _, field := range cfg.Min { + roleByName[field] = "min" + } + for _, field := range cfg.Max { + roleByName[field] = "max" + } + for _, field := range cfg.And { + roleByName[field] = "and" + } + for _, name := range names { + role := roleByName[name] + if _, ok := cfg.StaticFields[name]; ok { + role = "static" + } + if role == "" { + role = "current" + } + out = append(out, event.SchemaField{ + Role: role, + Name: name, + Value: cfg.StaticFields[name], + }) + } + return out +} + +// Flush evaluates timer-based export policies against the current state map. +func (a *Stateful) Flush() ([]*event.Event, error) { + a.mu.Lock() + defer a.mu.Unlock() + return a.flushAt(time.Now(), false), nil +} + +// Close exports all remaining buckets regardless of timer state. +func (a *Stateful) Close() ([]*event.Event, error) { + a.mu.Lock() + defer a.mu.Unlock() + + // On shutdown ReFlow should not silently drop aggregate state. Close exports + // every remaining bucket one last time regardless of dirty state. + out := make([]*event.Event, 0, len(a.state)) + for key, record := range a.state { + out = append(out, buildAggregatedEvent(a.cfg.Stream, key, record)) + } + return out, nil +} + +// Interval reports the smallest configured timer so the runtime knows how often +// to wake this aggregator worker for flush evaluation. +func (a *Stateful) Interval() time.Duration { + var min time.Duration + add := func(ms int) { + if ms <= 0 { + return + } + d := time.Duration(ms) * time.Millisecond + if min == 0 || d < min { + min = d + } + } + add(a.cfg.Window.IdleFlushAfter) + add(a.cfg.Window.MaxFlushAfter) + add(a.cfg.Window.IdleEraseAfter) + add(a.cfg.Periodic.Every) + return min +} + +// flushAt evaluates every bucket against the configured timers. +// +// Evaluation order: +// 1. idle window flush +// 2. max lifetime window flush +// 3. periodic snapshot export +// 4. idle erase without export +// +// That order is intentional: +// - a bucket that qualifies for a real window flush should be exported, not only +// snapshotted and kept around +// - silent erase is last so it only applies when no export trigger fired +func (a *Stateful) flushAt(now time.Time, closing bool) []*event.Event { + out := make([]*event.Event, 0, len(a.state)) + periodicDue := a.periodicDue(now) + for key, record := range a.state { + if closing { + out = append(out, buildAggregatedEvent(a.cfg.Stream, key, record)) + continue + } + + if a.shouldFlushIdle(record, now) || a.shouldFlushMax(record, now) { + out = append(out, buildAggregatedEvent(a.cfg.Stream, key, record)) + delete(a.state, key) + continue + } + + if periodicDue && a.shouldEmitPeriodic(record) { + out = append(out, buildAggregatedEvent(a.cfg.Stream, key, record)) + if a.cfg.Periodic.ResetBuckets { + delete(a.state, key) + } else { + record.Dirty = false + a.state[key] = record + } + continue + } + + if a.shouldEraseIdle(record, now) { + delete(a.state, key) + } + } + if periodicDue { + a.lastPeriodicRun = now + } + return out +} + +// shouldFlushIdle exports a bucket after it has been inactive long enough. +func (a *Stateful) shouldFlushIdle(record aggregateRecord, now time.Time) bool { + if a.cfg.Window.IdleFlushAfter <= 0 { + return false + } + return now.Sub(record.LastSeen) >= time.Duration(a.cfg.Window.IdleFlushAfter)*time.Millisecond +} + +// shouldFlushMax enforces an absolute bucket lifetime even when updates keep arriving. +func (a *Stateful) shouldFlushMax(record aggregateRecord, now time.Time) bool { + if a.cfg.Window.MaxFlushAfter <= 0 { + return false + } + return now.Sub(record.FirstSeen) >= time.Duration(a.cfg.Window.MaxFlushAfter)*time.Millisecond +} + +// Periodic export is intentionally driven by the aggregate worker ticker. The +// dirty bit prevents the same untouched bucket from being emitted over and over +// when periodic snapshots are enabled. +func (a *Stateful) shouldEmitPeriodic(record aggregateRecord) bool { + return record.Dirty +} + +// shouldEraseIdle drops stale buckets that should disappear silently instead of exporting. +func (a *Stateful) shouldEraseIdle(record aggregateRecord, now time.Time) bool { + if a.cfg.Window.IdleEraseAfter <= 0 { + return false + } + return now.Sub(record.LastSeen) >= time.Duration(a.cfg.Window.IdleEraseAfter)*time.Millisecond +} + +// periodicDue ensures snapshot emission follows the aggregator's own wall clock +// rather than event timestamps. +func (a *Stateful) periodicDue(now time.Time) bool { + if a.cfg.Periodic.Every <= 0 { + return false + } + if a.lastPeriodicRun.IsZero() { + return now.Sub(a.startedAt) >= time.Duration(a.cfg.Periodic.Every)*time.Millisecond + } + return now.Sub(a.lastPeriodicRun) >= time.Duration(a.cfg.Periodic.Every)*time.Millisecond +} + +// aggregateFromEvent converts one processed event into the stored aggregation +// representation used inside a bucket. +func aggregateFromEvent(cfg config.AggregatorConfig, recordCapacity int, evt *event.Event) (string, aggregateRecord, error) { + fields := evt.Fields + if fields == nil { + return "", aggregateRecord{}, fmt.Errorf("event fields are empty") + } + + key, err := buildKey(evt, cfg.Fields, cfg.AggregateMissing) + if err != nil { + return "", aggregateRecord{}, err + } + + // Aggregation window timers are runtime policy, not event timestamps. + // + // Using evt.ReceivedAt here would make replayed traffic or synthetic test + // events look instantly ancient, which would trigger max/idle flushes + // incorrectly. The actual flow timestamps remain in start_time_unix and + // end_time_unix; the bucket lifecycle timers should follow wall-clock time. + now := time.Now() + + recordFields := make(map[string]any, recordCapacity) + for key, val := range cfg.StaticFields { + recordFields[key] = val + } + for _, field := range cfg.Fields { + switch field.Role { + case "key": + if val, ok := eventFieldValueForSpec(evt, field); ok { + recordFields[field.Name] = val + } + case "sum": + recordFields[field.Name] = sumValue{Value: int64EventFieldForSpec(evt, field)} + case "first": + if val, ok := eventFieldValueForSpec(evt, field); ok { + recordFields[field.Name] = firstValue{Value: val} + } + case "current": + if val, ok := eventFieldValueForSpec(evt, field); ok { + recordFields[field.Name] = currentValue{Value: val} + } + case "min": + if val, ok := eventFieldValueForSpec(evt, field); ok { + recordFields[field.Name] = minValue{Value: val} + } + case "max": + if val, ok := eventFieldValueForSpec(evt, field); ok { + recordFields[field.Name] = maxValue{Value: val} + } + case "and": + if val, ok := eventFieldValueForSpec(evt, field); ok { + recordFields[field.Name] = andValue{Value: val} + } + } + } + seedTimestamps(recordFields, fields, now) + + return key, aggregateRecord{ + Fields: recordFields, + FirstSeen: now, + LastSeen: now, + }, nil +} + +func aggregateRecordCapacity(cfg config.AggregatorConfig) int { + return len(cfg.StaticFields) + len(cfg.Fields) + 2 +} + +// seedTimestamps ensures aggregates always have start/end fields even when the +// input record omitted explicit flow timing. +func seedTimestamps(dst, src map[string]any, now time.Time) { + start := timestampFieldOrNow(src, "start_time_unix", now) + end := timestampFieldOrNow(src, "end_time_unix", now) + dst["start_time_unix"] = start + dst["end_time_unix"] = end +} + +// buildKey joins the configured key fields into one stable bucket identifier. +func buildKey(evt *event.Event, fields []config.AggregatorField, aggregateMissing bool) (string, error) { + keyFields := aggregatorFieldsByRole(fields, "key") + if len(keyFields) == 0 { + return "__global__", nil + } + var b strings.Builder + b.Grow(len(keyFields) * 16) + for i, field := range keyFields { + val, ok := eventFieldValueForSpec(evt, field) + if !ok { + if !aggregateMissing { + return "", &missingAggregationKeyError{Key: field.Name} + } + val = nil + } + if i > 0 { + b.WriteByte('|') + } + writeKeyValue(&b, val) + } + return b.String(), nil +} + +// EventHasRequiredKeyFields reports whether this event can enter the configured +// aggregate bucket without relying on aggregate_missing fallback semantics. +func EventHasRequiredKeyFields(cfg config.AggregatorConfig, evt *event.Event) bool { + return MissingRequiredKeyField(cfg, evt) == "" +} + +func MissingRequiredKeyField(cfg config.AggregatorConfig, evt *event.Event) string { + if cfg.AggregateMissing { + return "" + } + for _, field := range aggregatorFieldsByRole(effectiveAggregatorFields(cfg), "key") { + if _, ok := eventFieldValueForSpec(evt, field); !ok { + return field.Name + } + } + return "" +} + +func writeKeyValue(b *strings.Builder, val any) { + switch v := val.(type) { + case nil: + b.WriteString("nil") + case string: + b.WriteString(v) + case uint64: + b.WriteString(strconv.FormatUint(v, 10)) + case uint32: + b.WriteString(strconv.FormatUint(uint64(v), 10)) + case uint16: + b.WriteString(strconv.FormatUint(uint64(v), 10)) + case uint8: + b.WriteString(strconv.FormatUint(uint64(v), 10)) + case int64: + b.WriteString(strconv.FormatInt(v, 10)) + case int: + b.WriteString(strconv.Itoa(v)) + case float64: + b.WriteString(strconv.FormatFloat(v, 'g', -1, 64)) + case bool: + b.WriteString(strconv.FormatBool(v)) + default: + fmt.Fprint(b, v) + } +} + +// buildAggregatedEvent materializes one bucket into a regular runtime event. +func buildAggregatedEvent(stream, key string, record aggregateRecord) *event.Event { + fields := cloneFields(record.Fields) + + return &event.Event{ + ReceivedAt: time.Now(), + Stream: stream, + Source: event.SourceMetadata{ + Type: "aggregated_flow", + }, + Aggregation: &event.AggregationMetadata{ + Key: key, + FirstSeenUnix: record.FirstSeen.UnixMilli(), + LastSeenUnix: record.LastSeen.UnixMilli(), + }, + Fields: fields, + } +} + +// cloneFields unwraps internal accumulator marker types back into plain values +// suitable for encoding and logging. +func cloneFields(in map[string]any) map[string]any { + out := make(map[string]any, len(in)+3) + for key, val := range in { + switch typed := val.(type) { + case sumValue: + out[key] = typed.Value + case firstValue: + out[key] = typed.Value + case currentValue: + out[key] = typed.Value + case minValue: + out[key] = typed.Value + case maxValue: + out[key] = typed.Value + case andValue: + out[key] = typed.Value + default: + out[key] = val + } + } + return out +} + +type firstValue struct{ Value any } +type sumValue struct{ Value any } +type currentValue struct{ Value any } +type minValue struct{ Value any } +type maxValue struct{ Value any } +type andValue struct{ Value any } + +// mergeFields applies the configured aggregation semantics by field wrapper type: +// sum values add, first values stick, current values overwrite, min/max compare, +// and values bitwise-AND. +func mergeFields(dst, src map[string]any) { + for key, val := range src { + if key == "start_time_unix" { + dst[key] = minTimestamp(int64FromAny(dst[key]), int64FromAny(val)) + continue + } + if key == "end_time_unix" { + dst[key] = maxTimestamp(int64FromAny(dst[key]), int64FromAny(val)) + continue + } + switch incoming := val.(type) { + case sumValue: + dst[key] = sumValue{Value: sumValueOf(dst[key]) + int64FromAny(incoming.Value)} + continue + case firstValue: + if _, exists := dst[key]; !exists { + dst[key] = incoming + } + case currentValue: + dst[key] = incoming + case minValue: + dst[key] = minValue{Value: minAny(dst[key], incoming.Value)} + case maxValue: + dst[key] = maxValue{Value: maxAny(dst[key], incoming.Value)} + case andValue: + dst[key] = andValue{Value: andAny(dst[key], incoming.Value)} + default: + dst[key] = val + } + } +} + +// sumValueOf unwraps sum markers and normalizes plain integers into an additive value. +func sumValueOf(val any) int64 { + switch typed := val.(type) { + case sumValue: + return int64FromAny(typed.Value) + default: + return int64FromAny(val) + } +} + +func minAny(current, incoming any) any { + if wrapped, exists := current.(minValue); exists { + if int64FromAny(incoming) < int64FromAny(wrapped.Value) { + return incoming + } + return wrapped.Value + } + if int64FromAny(incoming) < int64FromAny(current) { + return incoming + } + return current +} + +func maxAny(current, incoming any) any { + if wrapped, exists := current.(maxValue); exists { + if int64FromAny(incoming) > int64FromAny(wrapped.Value) { + return incoming + } + return wrapped.Value + } + if int64FromAny(incoming) > int64FromAny(current) { + return incoming + } + return current +} + +func andAny(current, incoming any) any { + if wrapped, exists := current.(andValue); exists { + return uint32FromAny(wrapped.Value) & uint32FromAny(incoming) + } + return uint32FromAny(current) & uint32FromAny(incoming) +} + +// timestampFieldOrNow prefers an existing field timestamp and falls back to the +// current wall clock when the input record did not provide one. +func timestampFieldOrNow(fields map[string]any, key string, now time.Time) int64 { + if fields != nil { + if val, ok := fields[key]; ok { + ts := int64FromAny(val) + if ts != 0 { + return ts + } + } + } + return now.UnixMilli() +} + +// minTimestamp keeps the earlier non-zero timestamp. +func minTimestamp(a, b int64) int64 { + if a == 0 { + return b + } + if b == 0 || a <= b { + return a + } + return b +} + +// maxTimestamp keeps the later non-zero timestamp. +func maxTimestamp(a, b int64) int64 { + if a == 0 { + return b + } + if b == 0 || a >= b { + return a + } + return b +} + +// int64EventField reads one numeric field from the event field map or metadata. +func int64EventField(evt *event.Event, key string) int64 { + val, ok := eventFieldValue(evt, key) + if !ok { + return 0 + } + return int64FromAny(val) +} + +func int64EventFieldForSpec(evt *event.Event, field config.AggregatorField) int64 { + val, ok := eventFieldValueForSpec(evt, field) + if !ok { + return 0 + } + return int64FromAny(val) +} + +func eventFieldValueForSpec(evt *event.Event, field config.AggregatorField) (any, bool) { + val, ok := eventFieldValue(evt, field.Name) + if !ok { + return nil, false + } + return applyFieldFamily(val, aggregatorFieldFamily(field)) +} + +func applyFieldFamily(val any, family string) (any, bool) { + if family == "" { + return val, true + } + ip, ok := val.(string) + if !ok { + return nil, false + } + addr, err := netip.ParseAddr(ip) + if err != nil { + return nil, false + } + switch family { + case "ip4": + if !addr.Is4() { + return nil, false + } + return addr.String(), true + case "ip6": + if !addr.Is6() { + return nil, false + } + return addr.String(), true + case "ip4in6": + if addr.Is6() { + return addr.String(), true + } + if !addr.Is4() { + return nil, false + } + raw := addr.As4() + mapped := netip.AddrFrom16([16]byte{10: 0xff, 11: 0xff, 12: raw[0], 13: raw[1], 14: raw[2], 15: raw[3]}) + return mapped.String(), true + default: + return nil, false + } +} + +func aggregatorFieldFamily(field config.AggregatorField) string { + if field.Role == "static" { + return "" + } + family, _ := ipFamilyModifier(field.Value) + return family +} + +func ipFamilyModifier(value any) (string, bool) { + family, ok := value.(string) + if !ok { + return "", false + } + switch family { + case "ip4", "ip6", "ip4in6": + return family, true + default: + return "", false + } +} + +func aggregatorFieldsByRole(fields []config.AggregatorField, role string) []config.AggregatorField { + out := make([]config.AggregatorField, 0, len(fields)) + for _, field := range fields { + if field.Role == role { + out = append(out, field) + } + } + return out +} + +func effectiveAggregatorFields(cfg config.AggregatorConfig) []config.AggregatorField { + if len(cfg.Fields) > 0 { + return cfg.Fields + } + fields := make([]config.AggregatorField, 0, len(cfg.KeyFields)+len(cfg.Sum)+len(cfg.First)+len(cfg.Current)+len(cfg.Min)+len(cfg.Max)+len(cfg.And)+len(cfg.StaticFields)) + appendRole := func(role string, names []string) { + for _, name := range names { + fields = append(fields, config.AggregatorField{Role: role, Name: name}) + } + } + appendRole("key", cfg.KeyFields) + appendRole("sum", cfg.Sum) + appendRole("first", cfg.First) + appendRole("current", cfg.Current) + appendRole("min", cfg.Min) + appendRole("max", cfg.Max) + appendRole("and", cfg.And) + staticFields := make([]string, 0, len(cfg.StaticFields)) + for name := range cfg.StaticFields { + staticFields = append(staticFields, name) + } + sort.Strings(staticFields) + for _, name := range staticFields { + fields = append(fields, config.AggregatorField{Role: "static", Name: name, Value: cfg.StaticFields[name]}) + } + return fields +} + +func eventFieldValue(evt *event.Event, key string) (any, bool) { + if evt == nil { + return nil, false + } + switch key { + case "agent_ip": + if agentIP, ok := metadataAgentIP(evt); ok { + if agentIPv4, _ := agentIPByFamily(agentIP); agentIPv4 != "" { + return agentIPv4, true + } + return nil, false + } + case "agent_ipv6": + if agentIP, ok := metadataAgentIP(evt); ok { + if _, agentIPv6 := agentIPByFamily(agentIP); agentIPv6 != "" { + return agentIPv6, true + } + return nil, false + } + case "source_id": + if evt.SFlow != nil && evt.SFlow.SourceID != 0 { + return evt.SFlow.SourceID, true + } + if evt.Source.SourceIDSet || evt.Source.SourceID != 0 { + return evt.Source.SourceID, true + } + case "sampling_rate": + if evt.SFlow != nil && evt.SFlow.SamplingRate != 0 { + return evt.SFlow.SamplingRate, true + } + if evt.Source.Sampling != nil { + return evt.Source.Sampling.Rate, true + } + case "sample_pool": + if evt.SFlow != nil && evt.SFlow.SamplePool != 0 { + return evt.SFlow.SamplePool, true + } + if evt.Source.Sampling != nil { + return evt.Source.Sampling.SamplePool, true + } + case "drops": + if evt.SFlow != nil && evt.SFlow.Drops != 0 { + return evt.SFlow.Drops, true + } + if evt.Source.Sampling != nil { + return evt.Source.Sampling.Drops, true + } + } + return fieldValue(evt.Fields, key) +} + +func metadataAgentIP(evt *event.Event) (string, bool) { + if evt.SFlow != nil && evt.SFlow.AgentIP != "" { + return evt.SFlow.AgentIP, true + } + if evt.Source.AgentIP != "" { + return evt.Source.AgentIP, true + } + return "", false +} + +func agentIPByFamily(raw string) (string, string) { + if raw == "" { + return "", "" + } + addr, err := netip.ParseAddr(raw) + if err != nil { + return raw, "" + } + if addr.Is4() { + return raw, "" + } + return "", raw +} + +// fieldValue reads flat field names first, then dotted paths through nested +// maps and slices. This lets aggregate configs key on structured values without +// forcing the event field map itself to be flat. +func fieldValue(fields map[string]any, key string) (any, bool) { + if fields == nil { + return nil, false + } + if val, ok := fields[key]; ok { + return val, true + } + if !strings.Contains(key, ".") { + return nil, false + } + parts := strings.Split(key, ".") + var current any = fields + for _, part := range parts { + val, ok := nestedFieldValue(current, part) + if !ok { + return nil, false + } + current = val + } + return current, true +} + +func nestedFieldValue(current any, part string) (any, bool) { + switch typed := current.(type) { + case map[string]any: + val, ok := typed[part] + return val, ok + case []map[string]any: + index, ok := pathIndex(part, len(typed)) + if !ok { + return nil, false + } + return typed[index], true + case []any: + index, ok := pathIndex(part, len(typed)) + if !ok { + return nil, false + } + return typed[index], true + default: + return nil, false + } +} + +func pathIndex(part string, length int) (int, bool) { + index, err := strconv.Atoi(part) + if err != nil || index < 0 || index >= length { + return 0, false + } + return index, true +} + +// uint32FromAny normalizes the small set of number shapes aggregation expects. +func uint32FromAny(val any) uint32 { + switch v := val.(type) { + case uint32: + return v + case uint64: + return uint32(v) + case int: + return uint32(v) + case int64: + return uint32(v) + case float64: + return uint32(v) + default: + return 0 + } +} + +// int64FromAny normalizes the small set of number shapes aggregation expects. +func int64FromAny(val any) int64 { + switch v := val.(type) { + case int64: + return v + case uint64: + return int64(v) + case uint32: + return int64(v) + case int: + return int64(v) + case float64: + return int64(v) + default: + return 0 + } +} diff --git a/pkg/reflow/aggregate/aggregate_bench_test.go b/pkg/reflow/aggregate/aggregate_bench_test.go new file mode 100644 index 00000000..9d96ea45 --- /dev/null +++ b/pkg/reflow/aggregate/aggregate_bench_test.go @@ -0,0 +1,129 @@ +package aggregate + +import ( + "fmt" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func BenchmarkStatefulProcess(b *testing.B) { + benchmarks := []struct { + name string + new func(int) *event.Event + }{ + { + name: "same_bucket", + new: func(int) *event.Event { + return benchAggregateEvent("192.0.2.1", "198.51.100.2", 12345) + }, + }, + { + name: "many_buckets", + new: func(i int) *event.Event { + return benchAggregateEvent( + fmt.Sprintf("192.0.2.%d", 1+i%250), + fmt.Sprintf("198.51.100.%d", 1+(i/250)%250), + uint32(1024+i%50000), + ) + }, + }, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + agg, err := New(benchAggregateConfig()) + if err != nil { + b.Fatalf("New returned error: %v", err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + out, err := agg.Process(bm.new(i)) + if err != nil { + b.Fatalf("Process returned error: %v", err) + } + if len(out) != 0 { + b.Fatalf("expected no immediate output, got %d", len(out)) + } + } + b.StopTimer() + _, _ = agg.Close() + }) + } +} + +func BenchmarkStatefulClose(b *testing.B) { + benchmarks := []struct { + name string + buckets int + }{ + {name: "100_buckets", buckets: 100}, + {name: "10000_buckets", buckets: 10000}, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + b.ReportAllocs() + b.StopTimer() + for i := 0; i < b.N; i++ { + agg, err := New(benchAggregateConfig()) + if err != nil { + b.Fatalf("New returned error: %v", err) + } + for bucket := 0; bucket < bm.buckets; bucket++ { + _, err := agg.Process(benchAggregateEvent( + fmt.Sprintf("192.0.%d.%d", bucket/250, bucket%250), + fmt.Sprintf("198.51.%d.%d", bucket/250, bucket%250), + uint32(1024+bucket), + )) + if err != nil { + b.Fatalf("Process returned error: %v", err) + } + } + b.StartTimer() + out, err := agg.Close() + b.StopTimer() + if err != nil { + b.Fatalf("Close returned error: %v", err) + } + if len(out) != bm.buckets { + b.Fatalf("expected %d output events, got %d", bm.buckets, len(out)) + } + } + }) + } +} + +func benchAggregateConfig() config.AggregatorConfig { + return config.AggregatorConfig{ + KeyFields: []string{"src_addr", "dst_addr", "proto", "src_port", "dst_port"}, + Sum: []string{"bytes", "packets"}, + First: []string{"agent_ip", "input_if"}, + Current: []string{"output_if"}, + Min: []string{"start_time_unix"}, + Max: []string{"end_time_unix"}, + } +} + +func benchAggregateEvent(srcAddr, dstAddr string, srcPort uint32) *event.Event { + return &event.Event{ + ReceivedAt: time.Unix(1_700_000_001, 0).UTC(), + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "src_addr": srcAddr, + "dst_addr": dstAddr, + "proto": uint32(6), + "src_port": srcPort, + "dst_port": uint32(443), + "bytes": int64(321), + "packets": int64(7), + "input_if": uint32(9), + "output_if": uint32(10), + "start_time_unix": int64(1_700_000_000_100), + "end_time_unix": int64(1_700_000_000_900), + }, + } +} diff --git a/pkg/reflow/aggregate/aggregate_test.go b/pkg/reflow/aggregate/aggregate_test.go new file mode 100644 index 00000000..fb6b56fb --- /dev/null +++ b/pkg/reflow/aggregate/aggregate_test.go @@ -0,0 +1,1349 @@ +package aggregate + +import ( + "strings" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/processor" +) + +func TestStatefulFlushSumsPacketCounters(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"src_addr", "dst_addr", "proto", "src_port", "dst_port"}, + Sum: []string{"bytes", "packets"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + first := &event.Event{ + ReceivedAt: time.Unix(1, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(60), + "packets": int64(1), + }, + } + second := &event.Event{ + ReceivedAt: time.Unix(2, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(70), + "packets": int64(1), + }, + } + + if out, err := agg.Process(first); err != nil || len(out) != 0 { + t.Fatalf("first Process returned out=%d err=%v", len(out), err) + } + if out, err := agg.Process(second); err != nil || len(out) != 0 { + t.Fatalf("second Process returned out=%d err=%v", len(out), err) + } + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 aggregated event, got %d", len(out)) + } + if got := out[0].Fields["packets"]; got != int64(2) { + t.Fatalf("expected packets=2, got %#v", got) + } + if got := out[0].Fields["bytes"]; got != int64(130) { + t.Fatalf("expected bytes=130, got %#v", got) + } +} + +func TestStatefulReturnsErrorWithMissingKey(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Stream: "interface_options", + KeyFields: []string{"observation_domain_id", "input_if"}, + Current: []string{"interface_name"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + out, err := agg.Process(&event.Event{ + Fields: map[string]any{ + "observation_domain_id": uint32(777), + "interface_name": "eth0", + }, + }) + if err == nil || !strings.Contains(err.Error(), `missing aggregation key field "input_if"`) { + t.Fatalf("expected missing key error, got out=%d err=%v", len(out), err) + } +} + +func TestStatefulAggregatesMissingKeyWhenConfigured(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + AggregateMissing: true, + KeyFields: []string{"src_addr", "dst_addr"}, + Sum: []string{"bytes"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + out, err := agg.Process(&event.Event{ + Fields: map[string]any{ + "dst_addr": "198.51.100.2", + "bytes": int64(60), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(out) != 0 { + t.Fatalf("expected aggregation to buffer event, got %d events", len(out)) + } + + out, err = agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 aggregated event, got %d", len(out)) + } + if _, ok := out[0].Fields["src_addr"]; ok { + t.Fatalf("did not expect missing src_addr to be added to output fields: %#v", out[0].Fields) + } + if got := out[0].Fields["dst_addr"]; got != "198.51.100.2" { + t.Fatalf("expected dst_addr to be preserved, got %#v", got) + } + if got := out[0].Fields["bytes"]; got != int64(60) { + t.Fatalf("expected bytes=60, got %#v", got) + } + if got := out[0].Aggregation.Key; got != "nil|198.51.100.2" { + t.Fatalf("expected aggregate key to use missing marker, got %q", got) + } +} + +func TestStatefulAggregatesNestedPacketLayerFields(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + KeyFields: []string{ + "outer_src_addr", + "src_addr", + "dst_addr", + "proto", + "src_port", + "dst_port", + }, + Sum: []string{"bytes", "packets"}, + Current: []string{ + "outer_dst_addr", + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + fields := map[string]any{ + "outer_src_addr": "203.0.113.1", + "outer_dst_addr": "203.0.113.2", + "outer_proto": uint32(47), + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(60), + "packets": int64(1), + } + if _, err := agg.Process(&event.Event{Fields: fields}); err != nil { + t.Fatalf("Process returned error: %v", err) + } + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 aggregated event, got %d", len(out)) + } + if got := out[0].Fields["outer_src_addr"]; got != "203.0.113.1" { + t.Fatalf("expected outer source key to be preserved, got %#v", got) + } + if got := out[0].Fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected inner source key to be preserved, got %#v", got) + } + if got := out[0].Fields["outer_dst_addr"]; got != "203.0.113.2" { + t.Fatalf("expected current nested field to be preserved, got %#v", got) + } + if got := out[0].Fields["bytes"]; got != int64(60) { + t.Fatalf("expected bytes=60, got %#v", got) + } +} + +func TestStatefulAggregatesNestedReFlowJSON(t *testing.T) { + cfg := nestedReFlowJSONConfig() + proc, err := processor.New(cfg.Processor) + if err != nil { + t.Fatalf("processor.New returned error: %v", err) + } + + tests := []struct { + name string + raw string + wantBytes int64 + wantLayers []string + wantVLANID any + wantMPLS any + wantEther any + wantAggKey string + wantAggVal any + wantAggKey2 string + wantAggVal2 any + }{ + { + name: "nested ip layers", + raw: nestedIPLayersJSON, + wantBytes: 96, + wantAggKey: "outer_src_addr", + wantAggVal: "203.0.113.1", + wantAggKey2: "src_addr", + wantAggVal2: "192.0.2.1", + }, + { + name: "dot1q mpls gre", + raw: nestedDot1QMPLSGREJSON, + wantBytes: 128, + wantLayers: []string{"ethernet", "dot1q", "mpls", "ipv4", "gre", "ipv4", "tcp"}, + wantVLANID: uint32(100), + wantAggKey: "outer_dst_addr", + wantAggVal: "203.0.113.2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: cfg.Sources[0].Type, + JSON: event.JSONMetadata{ + Flavor: cfg.Sources[0].JSON.Flavor, + }, + }, + Message: []byte(tt.raw), + }) + if err != nil { + t.Fatalf("Process ReFlow JSON returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 processed event, got %d", len(events)) + } + if events[0].Packet == nil || len(events[0].Packet.Layers) == 0 { + t.Fatalf("expected sample JSON to populate packet.layers") + } + if len(tt.wantLayers) > 0 { + layers := packetLayerKinds(events[0].Packet) + if len(layers) != len(tt.wantLayers) { + t.Fatalf("expected packet layer count %d, got %#v", len(tt.wantLayers), layers) + } + for i, want := range tt.wantLayers { + if layers[i] != want { + t.Fatalf("expected packet.layers[%d].kind=%q, got %#v", i, want, layers[i]) + } + } + } + if tt.wantVLANID != nil { + if got := events[0].Fields["vlan_id"]; got != tt.wantVLANID { + t.Fatalf("expected vlan_id=%#v, got %#v", tt.wantVLANID, got) + } + } + if tt.wantMPLS != nil { + if got := events[0].Fields["mpls_label"]; got != tt.wantMPLS { + t.Fatalf("expected mpls_label=%#v, got %#v", tt.wantMPLS, got) + } + } + if tt.wantEther != nil { + if got := events[0].Fields["ether_type"]; got != tt.wantEther { + t.Fatalf("expected ether_type=%#v, got %#v", tt.wantEther, got) + } + } + + agg, err := New(cfg.Aggregators[0]) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + if _, err := agg.Process(events[0]); err != nil { + t.Fatalf("Aggregate sample event returned error: %v", err) + } + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 aggregate output, got %d", len(out)) + } + if out[0].Stream != "nested_flow_data" { + t.Fatalf("expected nested_flow_data stream, got %q", out[0].Stream) + } + if got := out[0].Fields[tt.wantAggKey]; got != tt.wantAggVal { + t.Fatalf("expected %s=%#v, got %#v", tt.wantAggKey, tt.wantAggVal, got) + } + if tt.wantAggKey2 != "" { + if got := out[0].Fields[tt.wantAggKey2]; got != tt.wantAggVal2 { + t.Fatalf("expected %s=%#v, got %#v", tt.wantAggKey2, tt.wantAggVal2, got) + } + } + if got := out[0].Fields["bytes"]; got != tt.wantBytes { + t.Fatalf("expected bytes=%d, got %#v", tt.wantBytes, got) + } + }) + } +} + +func nestedReFlowJSONConfig() *config.Config { + return &config.Config{ + Sources: []config.SourceConfig{ + { + Type: "json", + JSON: config.JSONConfig{Flavor: "reflow"}, + }, + }, + Processor: config.ProcessorConfig{ + Type: "builtin", + Workers: 1, + Builtin: config.BuiltinProcessorConfig{ + DropMessage: true, + DropPayload: true, + }, + }, + Aggregators: []config.AggregatorConfig{ + { + Stream: "nested_flow_data", + Window: config.AggregatorWindowConfig{ + IdleFlushAfter: 5000, + }, + Periodic: config.AggregatorPeriodicConfig{ + Every: 1000, + }, + KeyFields: []string{ + "outer_src_addr", + "outer_dst_addr", + "src_addr", + "dst_addr", + "proto", + "src_port", + "dst_port", + }, + TemplateID: 300, + StaticFields: map[string]any{ + "example": "nested-ip-layers", + }, + Sum: []string{ + "bytes", + "packets", + }, + First: []string{ + "agent_ip", + "start_time_unix", + }, + Current: []string{ + "agent_ip", + "input_if", + "output_if", + "sampling_rate", + "end_time_unix", + }, + }, + }, + } +} + +const nestedIPLayersJSON = `{ + "agent_ip": "192.0.2.10", + "source_id": 1, + "sampling_rate": 100, + "input_if": 10, + "output_if": 20, + "packet": { + "layers": [ + { + "kind": "ipv4", + "ipv4": { + "src_addr": "203.0.113.1", + "dst_addr": "203.0.113.2", + "protocol": 47, + "ttl": 64 + } + }, + { + "kind": "gre", + "gre": { + "protocol": 2048 + } + }, + { + "kind": "ipv4", + "ipv4": { + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "protocol": 6, + "ttl": 64 + } + }, + { + "kind": "tcp", + "tcp": { + "src_port": 12345, + "dst_port": 443, + "flags": 2, + "window": 65535 + } + } + ] + }, + "bytes": 96, + "packets": 1, + "start_time_unix": 1714483200000, + "end_time_unix": 1714483200100 +}` + +const nestedDot1QMPLSGREJSON = `{ + "agent_ip": "192.0.2.10", + "source_id": 1, + "sampling_rate": 100, + "input_if": 10, + "output_if": 20, + "packet": { + "layers": [ + { + "kind": "ethernet", + "ethernet": { + "src_mac": "66:77:88:99:aa:bb", + "dst_mac": "00:11:22:33:44:55" + } + }, + { + "kind": "dot1q", + "vlan": { + "id": 100, + "tpid": 33024 + } + }, + { + "kind": "mpls", + "mpls": { + "label": { + "label": 17, + "bos": true, + "ttl": 64 + } + } + }, + { + "kind": "ipv4", + "ipv4": { + "src_addr": "203.0.113.1", + "dst_addr": "203.0.113.2", + "protocol": 47, + "ttl": 64 + } + }, + { + "kind": "gre", + "gre": { + "protocol": 2048 + } + }, + { + "kind": "ipv4", + "ipv4": { + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "protocol": 6, + "ttl": 64 + } + }, + { + "kind": "tcp", + "tcp": { + "src_port": 12345, + "dst_port": 443, + "flags": 2, + "window": 65535 + } + } + ] + }, + "bytes": 128, + "packets": 1, + "start_time_unix": 1714483201000, + "end_time_unix": 1714483201100 +}` + +func packetLayerKinds(model *event.PacketModel) []string { + if model == nil { + return nil + } + layers := make([]string, 0, len(model.Layers)) + for _, layer := range model.Layers { + layers = append(layers, layer.Kind) + } + return layers +} + +func TestStatefulInitEventsCarryConfiguredStreamAndTemplateBaseID(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Stream: "agg_packets", + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + Match: map[string]string{"ip_family": "ipv4"}, + TemplateID: 512, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + events, err := agg.InitEvents() + if err != nil { + t.Fatalf("InitEvents returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 init event, got %d", len(events)) + } + if events[0].Stream != "agg_packets" { + t.Fatalf("expected event stream agg_packets, got %q", events[0].Stream) + } + if events[0].Control == nil || events[0].Control.Stream != "agg_packets" { + t.Fatalf("expected control stream agg_packets, got %#v", events[0].Control) + } + schema, ok := events[0].Payload.(event.AggregationSchema) + if !ok { + t.Fatalf("expected aggregation schema payload, got %T", events[0].Payload) + } + if schema.Stream != "agg_packets" { + t.Fatalf("expected schema stream agg_packets, got %q", schema.Stream) + } + if schema.BaseTemplateID != 512 { + t.Fatalf("expected base template id 512, got %d", schema.BaseTemplateID) + } + if schema.Match["ip_family"] != "ipv4" { + t.Fatalf("expected schema match ip_family=ipv4, got %#v", schema.Match) + } +} + +func TestSchemaPassthroughEmitsSchemaAndForwardsEvents(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Passthrough: true, + Stream: "flow_data", + TemplateID: 256, + FieldsConfigured: true, + Fields: []config.AggregatorField{ + {Role: "key", Name: "src_addr"}, + {Role: "current", Name: "bytes"}, + {Role: "static", Name: "exporter_name", Value: "edge-a"}, + }, + KeyFields: []string{"src_addr"}, + StaticFields: map[string]any{ + "exporter_name": "edge-a", + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + initEvents, err := agg.InitEvents() + if err != nil { + t.Fatalf("InitEvents returned error: %v", err) + } + if len(initEvents) != 1 { + t.Fatalf("expected 1 init event, got %d", len(initEvents)) + } + schema, ok := initEvents[0].Payload.(event.AggregationSchema) + if !ok { + t.Fatalf("expected aggregation schema payload, got %T", initEvents[0].Payload) + } + if len(schema.Fields) != 3 { + t.Fatalf("expected 3 schema fields, got %#v", schema.Fields) + } + if schema.Fields[0].Name != "src_addr" { + t.Fatalf("unexpected first schema field: %#v", schema.Fields[0]) + } + + evt := &event.Event{ + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "bytes": uint64(64), + }, + } + out, err := agg.Process(evt) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 forwarded event, got %d", len(out)) + } + if out[0].Fields["exporter_name"] != "edge-a" { + t.Fatalf("expected static field to be injected, got %#v", out[0].Fields) + } + if _, ok := evt.Fields["exporter_name"]; ok { + t.Fatalf("did not expect original event fields to be mutated") + } +} + +func TestSchemaInitEventsCarryFieldFamilyModifiers(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Passthrough: true, + Stream: "flow_data", + FieldsConfigured: true, + Fields: []config.AggregatorField{ + {Role: "key", Name: "src_addr", Value: "ip6"}, + {Role: "current", Name: "nat_src_addr", Value: "ip4in6"}, + {Role: "sum", Name: "bytes"}, + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + initEvents, err := agg.InitEvents() + if err != nil { + t.Fatalf("InitEvents returned error: %v", err) + } + schema := initEvents[0].Payload.(event.AggregationSchema) + if schema.Fields[0].Value != "ip6" || schema.Fields[1].Value != "ip4in6" { + t.Fatalf("expected schema values to preserve family modifiers, got %#v", schema.Fields) + } +} + +func TestStatefulAppliesFieldFamilyModifiers(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + FieldsConfigured: true, + Fields: []config.AggregatorField{ + {Role: "key", Name: "src_addr", Value: "ip4in6"}, + {Role: "current", Name: "dst_addr", Value: "ip6"}, + {Role: "sum", Name: "bytes"}, + }, + Periodic: config.AggregatorPeriodicConfig{Every: 1}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, err = agg.Process(&event.Event{Fields: map[string]any{ + "src_addr": "192.0.2.10", + "dst_addr": "2001:db8::20", + "bytes": uint64(64), + }}) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + fields := out[0].Fields + if fields["src_addr"] != "::ffff:192.0.2.10" { + t.Fatalf("expected IPv4 source to be mapped into IPv6, got %#v", fields["src_addr"]) + } + if fields["dst_addr"] != "2001:db8::20" { + t.Fatalf("expected IPv6 destination to be kept, got %#v", fields["dst_addr"]) + } +} + +func TestSchemaPassthroughCanEmitControlAsData(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Passthrough: true, + Stream: "options_data", + EmitAs: "data", + FieldsConfigured: true, + Fields: []config.AggregatorField{ + {Role: "static", Name: "tflow_record_type", Value: "options"}, + {Role: "key", Name: "source_id"}, + {Role: "current", Name: "sampling_rate"}, + }, + KeyFields: []string{"source_id"}, + StaticFields: map[string]any{ + "tflow_record_type": "options", + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + out, err := agg.Process(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init", + }, + Fields: map[string]any{ + "source_id": uint32(7), + "sampling_rate": uint32(100), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 forwarded event, got %d", len(out)) + } + if out[0].Kind != "data" || out[0].Control != nil { + t.Fatalf("expected control event to be emitted as data, got kind=%q control=%#v", out[0].Kind, out[0].Control) + } + if out[0].Stream != "options_data" { + t.Fatalf("expected options_data stream, got %q", out[0].Stream) + } + if out[0].Fields["tflow_record_type"] != "options" { + t.Fatalf("expected static options marker, got %#v", out[0].Fields) + } +} + +func TestSchemaFieldsDeduplicateConfiguredNames(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Passthrough: true, + Stream: "flow_data", + FieldsConfigured: true, + Fields: []config.AggregatorField{ + {Role: "first", Name: "agent_ip"}, + {Role: "current", Name: "agent_ip"}, + {Role: "sum", Name: "bytes"}, + }, + First: []string{"agent_ip"}, + Current: []string{"agent_ip"}, + Sum: []string{"bytes"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + initEvents, err := agg.InitEvents() + if err != nil { + t.Fatalf("InitEvents returned error: %v", err) + } + schema, ok := initEvents[0].Payload.(event.AggregationSchema) + if !ok { + t.Fatalf("expected aggregation schema payload, got %T", initEvents[0].Payload) + } + if len(schema.Fields) != 2 { + t.Fatalf("expected deduplicated schema fields, got %#v", schema.Fields) + } + if schema.Fields[0].Name != "agent_ip" || schema.Fields[1].Name != "bytes" { + t.Fatalf("unexpected schema field order: %#v", schema.Fields) + } +} + +func TestStatefulInitEventsSortStaticFieldsDeterministically(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Stream: "agg_packets", + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"src_addr"}, + Sum: []string{"bytes"}, + StaticFields: map[string]any{ + "z_field": "z", + "a_field": "a", + "m_field": "m", + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + events, err := agg.InitEvents() + if err != nil { + t.Fatalf("InitEvents returned error: %v", err) + } + schema, ok := events[0].Payload.(event.AggregationSchema) + if !ok { + t.Fatalf("expected aggregation schema payload, got %T", events[0].Payload) + } + want := []string{"src_addr", "bytes", "start_time_unix", "end_time_unix", "a_field", "m_field", "z_field"} + if len(schema.FieldNames) != len(want) { + t.Fatalf("expected %d field names, got %#v", len(want), schema.FieldNames) + } + for i, field := range want { + if schema.FieldNames[i] != field { + t.Fatalf("expected field_names[%d]=%q, got %#v", i, field, schema.FieldNames) + } + } +} + +func TestStatefulAggregatedEventsCarryConfiguredStream(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Stream: "agg_counters", + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + if _, err := agg.Process(&event.Event{ + Fields: map[string]any{ + "if_in_octets": int64(64), + }, + }); err != nil { + t.Fatalf("Process returned error: %v", err) + } + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 aggregated event, got %d", len(out)) + } + if out[0].Stream != "agg_counters" { + t.Fatalf("expected stream agg_counters, got %q", out[0].Stream) + } +} + +func TestStatefulStoresSyntheticMetadataInAggregationEnvelope(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"src_addr"}, + Sum: []string{"bytes"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + if _, err := agg.Process(&event.Event{ + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "bytes": int64(64), + }, + }); err != nil { + t.Fatalf("Process returned error: %v", err) + } + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 aggregated event, got %d", len(out)) + } + + metadata := out[0].Aggregation + if metadata == nil { + t.Fatalf("expected aggregation metadata") + } + if metadata.Key != "192.0.2.1" { + t.Fatalf("expected aggregation key 192.0.2.1, got %q", metadata.Key) + } + if metadata.FirstSeenUnix == 0 { + t.Fatalf("expected first_seen_unix to be set") + } + if metadata.LastSeenUnix == 0 { + t.Fatalf("expected last_seen_unix to be set") + } + for _, key := range []string{"_aggregation_key", "_first_seen_unix", "_last_seen_unix", "aggregation_key", "first_seen_unix", "last_seen_unix"} { + if _, ok := out[0].Fields[key]; ok { + t.Fatalf("did not expect synthetic metadata field %s", key) + } + } +} + +func TestStatefulTTLFlushSumsPacketCounters(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Window: config.AggregatorWindowConfig{ + IdleFlushAfter: 1, + }, + KeyFields: []string{"src_addr", "dst_addr", "proto", "src_port", "dst_port"}, + Sum: []string{"bytes", "packets"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + first := &event.Event{ + ReceivedAt: time.Unix(1, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(60), + "packets": int64(1), + }, + } + second := &event.Event{ + ReceivedAt: time.Unix(2, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(70), + "packets": int64(1), + }, + } + + if out, err := agg.Process(first); err != nil || len(out) != 0 { + t.Fatalf("first Process returned out=%d err=%v", len(out), err) + } + if out, err := agg.Process(second); err != nil || len(out) != 0 { + t.Fatalf("second Process returned out=%d err=%v", len(out), err) + } + + time.Sleep(10 * time.Millisecond) + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 aggregated event, got %d", len(out)) + } + if got := out[0].Fields["packets"]; got != int64(2) { + t.Fatalf("expected packets=2, got %#v", got) + } + if got := out[0].Fields["bytes"]; got != int64(130) { + t.Fatalf("expected bytes=130, got %#v", got) + } +} + +func TestStatefulTracksMinStartAndMaxEndTimestamps(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"src_addr", "dst_addr", "proto", "src_port", "dst_port"}, + Sum: []string{"bytes", "packets"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, _ = agg.Process(&event.Event{ + ReceivedAt: time.Unix(10, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(60), + "packets": int64(1), + "start_time_unix": int64(5_000), + "end_time_unix": int64(8_000), + }, + }) + _, _ = agg.Process(&event.Event{ + ReceivedAt: time.Unix(11, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(70), + "packets": int64(1), + "start_time_unix": int64(4_000), + "end_time_unix": int64(9_000), + }, + }) + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if got := out[0].Fields["start_time_unix"]; got != int64(4_000) { + t.Fatalf("expected start_time_unix=4000, got %#v", got) + } + if got := out[0].Fields["end_time_unix"]; got != int64(9_000) { + t.Fatalf("expected end_time_unix=9000, got %#v", got) + } +} + +func TestStatefulBitwiseAndsConfiguredFields(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"src_addr", "dst_addr", "proto", "src_port", "dst_port"}, + And: []string{"tcp_flags"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + initEvents, err := agg.InitEvents() + if err != nil { + t.Fatalf("InitEvents returned error: %v", err) + } + schema, ok := initEvents[0].Payload.(event.AggregationSchema) + if !ok { + t.Fatalf("expected aggregation schema payload, got %T", initEvents[0].Payload) + } + if len(schema.AndFields) != 1 || schema.AndFields[0] != "tcp_flags" { + t.Fatalf("expected schema and_fields tcp_flags, got %#v", schema.AndFields) + } + + for _, flags := range []uint32{0x13, 0x12, 0x16} { + _, err := agg.Process(&event.Event{ + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "tcp_flags": flags, + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + } + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 aggregated event, got %d", len(out)) + } + if got := out[0].Fields["tcp_flags"]; got != uint32(0x12) { + t.Fatalf("expected tcp_flags=0x12, got %#v", got) + } +} + +func TestStatefulOnlySumsConfiguredSumFields(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"src_addr", "dst_addr", "proto", "src_port", "dst_port"}, + Sum: []string{"bytes", "packets"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, _ = agg.Process(&event.Event{ + ReceivedAt: time.Unix(1, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(60), + "packets": int64(1), + }, + }) + _, _ = agg.Process(&event.Event{ + ReceivedAt: time.Unix(2, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(70), + "packets": int64(1), + }, + }) + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if got := out[0].Fields["proto"]; got != uint32(6) { + t.Fatalf("expected proto=6 to remain stable, got %#v", got) + } + if got := out[0].Fields["bytes"]; got != int64(130) { + t.Fatalf("expected bytes=130, got %#v", got) + } + if got := out[0].Fields["packets"]; got != int64(2) { + t.Fatalf("expected packets=2, got %#v", got) + } +} + +func TestStatefulReadsSourceMetadataFields(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"source_id"}, + Current: []string{"agent_ip", "sampling_rate", "sample_pool", "drops"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, err = agg.Process(&event.Event{ + Source: event.SourceMetadata{ + AgentIP: "198.51.100.99", + SourceID: 42, + Sampling: &event.SamplingMetadata{ + Rate: 250, + SamplePool: 54321, + Drops: 7, + }, + }, + Fields: map[string]any{ + "bytes": uint64(64), + "agent_ip": "192.0.2.1", + "source_id": uint32(9), + "sampling_rate": uint32(100), + "sample_pool": uint32(12345), + "drops": uint32(3), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected one aggregate, got %d", len(out)) + } + fields := out[0].Fields + if fields["agent_ip"] != "198.51.100.99" { + t.Fatalf("expected agent_ip from metadata, got %#v", fields["agent_ip"]) + } + if fields["source_id"] != uint32(42) { + t.Fatalf("expected source_id from metadata, got %#v", fields["source_id"]) + } + if fields["sampling_rate"] != uint32(250) || fields["sample_pool"] != uint32(54321) || fields["drops"] != uint32(7) { + t.Fatalf("expected sampling metadata fields, got %#v", fields) + } +} + +func TestStatefulReadsZeroSourceIDWhenSet(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"source_id"}, + Current: []string{"source_id"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, err = agg.Process(&event.Event{ + Source: event.SourceMetadata{ + SourceID: 0, + SourceIDSet: true, + }, + Fields: map[string]any{ + "bytes": uint64(64), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected one aggregate, got %d", len(out)) + } + if got := out[0].Fields["source_id"]; got != uint32(0) { + t.Fatalf("expected source_id=0 from metadata, got %#v", got) + } +} + +func TestStatefulSplitsAgentIPMetadataByFamily(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + Current: []string{"agent_ip", "agent_ipv6"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, err = agg.Process(&event.Event{ + Source: event.SourceMetadata{ + AgentIP: "2001:db8::99", + }, + Fields: map[string]any{ + "bytes": uint64(64), + "agent_ip": "192.0.2.1", + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + out, err := agg.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected one aggregate, got %d", len(out)) + } + fields := out[0].Fields + if _, ok := fields["agent_ip"]; ok { + t.Fatalf("expected IPv6 metadata not to populate agent_ip, got %#v", fields) + } + if fields["agent_ipv6"] != "2001:db8::99" { + t.Fatalf("expected agent_ipv6 from metadata, got %#v", fields["agent_ipv6"]) + } +} + +func TestStatefulPeriodicFlushOnlyEmitsDirtyBuckets(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + }, + KeyFields: []string{"src_addr", "dst_addr", "proto", "src_port", "dst_port"}, + Sum: []string{"bytes", "packets"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, _ = agg.Process(&event.Event{ + ReceivedAt: time.Unix(1, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(60), + "packets": int64(1), + }, + }) + + time.Sleep(10 * time.Millisecond) + + firstFlush, err := agg.Flush() + if err != nil { + t.Fatalf("first Flush returned error: %v", err) + } + if len(firstFlush) != 1 { + t.Fatalf("expected first flush to emit 1 event, got %d", len(firstFlush)) + } + + secondFlush, err := agg.Flush() + if err != nil { + t.Fatalf("second Flush returned error: %v", err) + } + if len(secondFlush) != 0 { + t.Fatalf("expected second flush to emit 0 events, got %d", len(secondFlush)) + } + + _, _ = agg.Process(&event.Event{ + ReceivedAt: time.Unix(2, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "src_port": uint32(12345), + "dst_port": uint32(443), + "bytes": int64(40), + "packets": int64(1), + }, + }) + + time.Sleep(10 * time.Millisecond) + + thirdFlush, err := agg.Flush() + if err != nil { + t.Fatalf("third Flush returned error: %v", err) + } + if len(thirdFlush) != 1 { + t.Fatalf("expected third flush to emit 1 event, got %d", len(thirdFlush)) + } + if got := thirdFlush[0].Fields["bytes"]; got != int64(100) { + t.Fatalf("expected bytes=100 after update, got %#v", got) + } +} + +func TestStatefulIdleEraseDropsUntouchedBucketWithoutEmit(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Window: config.AggregatorWindowConfig{ + IdleEraseAfter: 1, + MaxFlushAfter: 1000, + }, + KeyFields: []string{"src_addr"}, + Sum: []string{"bytes"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, _ = agg.Process(&event.Event{ + ReceivedAt: time.Unix(1, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "bytes": int64(60), + }, + }) + + time.Sleep(10 * time.Millisecond) + + out, err := agg.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(out) != 0 { + t.Fatalf("expected idle erase to drop bucket without emit, got %d events", len(out)) + } +} + +func TestStatefulPeriodicResetDeletesBucketsAfterEmit(t *testing.T) { + agg, err := New(config.AggregatorConfig{ + Periodic: config.AggregatorPeriodicConfig{ + Every: 1, + ResetBuckets: true, + }, + KeyFields: []string{"src_addr"}, + Sum: []string{"bytes"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + _, _ = agg.Process(&event.Event{ + ReceivedAt: time.Unix(1, 0), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "bytes": int64(60), + }, + }) + + time.Sleep(10 * time.Millisecond) + + firstFlush, err := agg.Flush() + if err != nil { + t.Fatalf("first Flush returned error: %v", err) + } + if len(firstFlush) != 1 { + t.Fatalf("expected first flush to emit 1 event, got %d", len(firstFlush)) + } + + secondFlush, err := agg.Flush() + if err != nil { + t.Fatalf("second Flush returned error: %v", err) + } + if len(secondFlush) != 0 { + t.Fatalf("expected second flush to emit 0 events after reset, got %d", len(secondFlush)) + } +} diff --git a/pkg/reflow/app/app.go b/pkg/reflow/app/app.go new file mode 100644 index 00000000..60fda904 --- /dev/null +++ b/pkg/reflow/app/app.go @@ -0,0 +1,710 @@ +package app + +import ( + "context" + "fmt" + "log/slog" + "net/netip" + "strings" + "sync" + "time" + + "github.com/netsampler/goflow2/v3/pkg/goflow2/logging" + "github.com/netsampler/goflow2/v3/pkg/reflow/aggregate" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/decode" + "github.com/netsampler/goflow2/v3/pkg/reflow/encode" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/processor" + "github.com/netsampler/goflow2/v3/pkg/reflow/sink" + "github.com/netsampler/goflow2/v3/pkg/reflow/source" +) + +type App struct { + logger *slog.Logger + sources []source.Source + sourceInitRefresh []time.Duration + decoder decode.Decoder + processor processor.Processor + processorWorkers int + aggregatorCfgs []config.AggregatorConfig + encoderCfg config.EncoderConfig + encoderWorkers int + sink sink.Sink +} + +// New wires the current ReFlow runtime from source to sink. +func New(cfg *config.Config) (*App, error) { + logger, err := logging.NewLogger(cfg.LogLevel, cfg.LogFormat) + if err != nil { + return nil, fmt.Errorf("init logger: %w", err) + } + slog.SetDefault(logger) + + sources := make([]source.Source, 0, len(cfg.Sources)) + sourceInitRefresh := make([]time.Duration, 0, len(cfg.Sources)) + for i, srcCfg := range cfg.Sources { + src, err := source.New(srcCfg) + if err != nil { + return nil, fmt.Errorf("init source %d: %w", i, err) + } + sources = append(sources, src) + sourceInitRefresh = append(sourceInitRefresh, sourceInitRefreshDuration(srcCfg)) + } + proc, err := processor.New(cfg.Processor) + if err != nil { + return nil, fmt.Errorf("init processor: %w", err) + } + out, err := sink.New(cfg.Sink) + if err != nil { + return nil, fmt.Errorf("init sink: %w", err) + } + + encoderCfg := cfg.Encoder + processorWorkers := config.ResolveProcessorWorkers(cfg.Processor.Workers) + encoderWorkers := config.ResolveEncoderWorkers(cfg.Encoder.Workers, encoderCfg.Type) + // Ordered exporters own protocol sequence numbers and template refresh state. + // The effective config is: + // + // encoder: + // type: sflow # also ipfix, netflowv9, or netflowv5 + // workers: 1 + // + if requiresOrderedExporter(encoderCfg.Type) && cfg.Encoder.Workers > 1 { + logger.Warn( + "forcing encoder workers to 1 for ordered exporter", + slog.String("encoder_type", encoderCfg.Type), + slog.Int("requested_workers", int(cfg.Encoder.Workers)), + ) + } + dec, err := decode.NewWithOptions(decode.Options{ + Catalog: cfg.TemplatedFields.Catalog, + EmbedTemplateFields: cfg.Processor.Builtin.TFlow.EmbedTemplateFields || cfg.TemplatedFields.EmbedTemplateFields, + CachePath: cfg.TFlow.Cache.Path, + CacheInterval: time.Duration(cfg.TFlow.Cache.FlushInterval) * time.Millisecond, + }) + if err != nil { + return nil, fmt.Errorf("init decoder: %w", err) + } + + return &App{ + logger: logger, + sources: sources, + sourceInitRefresh: sourceInitRefresh, + decoder: dec, + processor: proc, + processorWorkers: processorWorkers, + aggregatorCfgs: cfg.Aggregators, + encoderCfg: encoderCfg, + encoderWorkers: encoderWorkers, + sink: out, + }, nil +} + +// Run owns the source, processor, aggregation, encoding, and sink goroutines. +func (a *App) Run(ctx context.Context) error { + a.logger.Info("starting ReFlow") + defer func() { + if err := a.sink.Close(); err != nil { + a.logger.Error("sink close error", slog.String("error", err.Error())) + } + }() + + sourceCtx, stopSource := context.WithCancel(ctx) + defer stopSource() + + var decoderErrWG sync.WaitGroup + if errCh := a.decoder.Errors(); errCh != nil { + decoderErrWG.Add(1) + go func() { + defer decoderErrWG.Done() + for err := range errCh { + a.logger.Error("decoder cache error", slog.String("error", err.Error())) + } + }() + defer decoderErrWG.Wait() + } + defer a.decoder.Close() + + decodeJobs := make(chan *event.Event, a.processorWorkers*2) + processJobs := make(chan *event.Event, a.processorWorkers*2) + encodeJobs := make(chan *event.Event, a.encoderWorkers*2) + + var decodeWG sync.WaitGroup + decodeWG.Add(1) + go func() { + defer decodeWG.Done() + // Decode is centralized so protocol identification and shared state stay + // single-writer even when later stages fan out with workers. + for evt := range decodeJobs { + events, err := a.decoder.Decode(evt) + if err != nil { + a.logger.Error("decode error", slog.String("error", err.Error())) + continue + } + for _, item := range events { + processJobs <- item + } + } + }() + + var aggregateWG sync.WaitGroup + aggregateWorkers := make([]aggregateWorker, 0, len(a.aggregatorCfgs)) + for _, aggCfg := range a.aggregatorCfgs { + agg, err := aggregate.New(aggCfg) + if err != nil { + return fmt.Errorf("init aggregator %q: %w", aggCfg.Stream, err) + } + worker := aggregateWorker{ + cfg: aggCfg, + agg: agg, + jobs: make(chan *event.Event, a.processorWorkers*2), + } + aggregateWorkers = append(aggregateWorkers, worker) + aggregateWG.Add(1) + go func(worker aggregateWorker) { + defer aggregateWG.Done() + var ticker *time.Ticker + if interval := worker.agg.Interval(); interval > 0 { + ticker = time.NewTicker(interval) + defer ticker.Stop() + } + sendToEncoder := func(events []*event.Event) { + for _, evt := range events { + encodeJobs <- evt + } + } + // flush is shared by timer-driven and shutdown-driven paths so + // aggregate close semantics stay consistent. + flush := func(closeStore bool) { + var ( + events []*event.Event + err error + ) + if closeStore { + events, err = worker.agg.Close() + } else { + events, err = worker.agg.Flush() + } + if err != nil { + a.logger.Error("aggregate flush error", slog.String("stream", worker.cfg.Stream), slog.String("error", err.Error())) + return + } + // Aggregator Flush/Close returns bucket events that are ready + // to leave the aggregation stage, so pass them to the encoder. + sendToEncoder(events) + } + for { + select { + case <-tickerChannel(ticker): + flush(false) + case evt, ok := <-worker.jobs: + if !ok { + flush(true) + return + } + events, err := worker.agg.Process(evt) + if err != nil { + a.logger.Error("aggregate error", slog.String("stream", worker.cfg.Stream), slog.String("error", err.Error())) + continue + } + sendToEncoder(events) + } + } + }(worker) + } + hasAggregators := len(aggregateWorkers) > 0 + routeSourceInitControls := aggregateWorkersRouteSourceInit(aggregateWorkers) + var aggregateRouteJobs chan *event.Event + if hasAggregators { + aggregateRouteJobs = make(chan *event.Event, a.processorWorkers*2) + } + + var processWG sync.WaitGroup + for range a.processorWorkers { + processWG.Add(1) + go func() { + defer processWG.Done() + for evt := range processJobs { + events, err := a.processor.Process(evt) + if err != nil { + a.logger.Error("process error", slog.String("error", err.Error())) + continue + } + for _, item := range events { + if hasAggregators { + aggregateRouteJobs <- item + } else { + encodeJobs <- item + } + } + } + }() + } + + var aggregateRouteWG sync.WaitGroup + if hasAggregators { + aggregateRouteWG.Add(1) + go func() { + defer aggregateRouteWG.Done() + for evt := range aggregateRouteJobs { + if evt == nil { + continue + } + if evt.Kind == "control" && (!routeSourceInitControls || !controlEventCanRouteThroughAggregator(evt)) { + encodeJobs <- evt + continue + } + matchedWorkers, missingKeyStream, missingKey := matchingAggregateWorkers(aggregateWorkers, evt) + for _, worker := range matchedWorkers { + // One event may intentionally feed multiple aggregators. + worker.jobs <- evt + } + if len(matchedWorkers) == 0 { + if missingKey != "" { + a.logger.Warn( + "dropping aggregate event with missing key field", + slog.String("stream", missingKeyStream), + slog.String("key", missingKey), + ) + } + encodeJobs <- evt + } + } + for _, worker := range aggregateWorkers { + close(worker.jobs) + } + }() + } else { + for _, worker := range aggregateWorkers { + close(worker.jobs) + } + } + + var encodeWG sync.WaitGroup + encoders := make([]encode.Encoder, 0, a.encoderWorkers) + for i := range a.encoderWorkers { + enc, err := encode.New(a.encoderCfg) + if err != nil { + return fmt.Errorf("init encoder %d: %w", i, err) + } + encoders = append(encoders, enc) + } + for _, enc := range encoders { + encodeWG.Add(1) + go func(enc encode.Encoder) { + defer encodeWG.Done() + var ticker *time.Ticker + if interval := encoderTickInterval(a.encoderCfg); interval > 0 { + ticker = time.NewTicker(interval) + defer ticker.Stop() + } + flush := func() bool { + payloads, err := enc.Flush() + if err != nil { + a.logger.Error("encode flush error", slog.String("error", err.Error())) + return true + } + for _, payload := range payloads { + if err := a.sink.Send(payload); err != nil { + a.logger.Error("sink write error", slog.String("error", err.Error())) + } + } + return true + } + for { + select { + case <-tickerChannel(ticker): + if !flush() { + return + } + case evt, ok := <-encodeJobs: + if !ok { + flush() + return + } + payloads, err := enc.Encode(evt) + if err != nil { + a.logger.Error("encode error", slog.String("error", err.Error())) + continue + } + for _, payload := range payloads { + if err := a.sink.Send(payload); err != nil { + a.logger.Error("sink write error", slog.String("error", err.Error())) + } + } + } + } + }(enc) + } + + for _, worker := range aggregateWorkers { + initEvents, err := worker.agg.InitEvents() + if err != nil { + return fmt.Errorf("init aggregator events for %q: %w", worker.cfg.Stream, err) + } + for _, evt := range initEvents { + encodeJobs <- evt + } + } + emitSourceInitEvents := func(events []*event.Event) bool { + if len(events) == 0 { + return true + } + if hasAggregators && routeSourceInitControls { + for _, evt := range events { + select { + case aggregateRouteJobs <- evt: + case <-sourceCtx.Done(): + return false + } + } + return true + } + for _, evt := range sourceInitEventsForEncoder(a.encoderCfg.Type, events) { + select { + case encodeJobs <- evt: + case <-sourceCtx.Done(): + return false + } + } + return true + } + + var sourceInitEvents []*event.Event + for i, src := range a.sources { + initEvents, err := src.InitEvents() + if err != nil { + return fmt.Errorf("init source %d events: %w", i, err) + } + sourceInitEvents = append(sourceInitEvents, initEvents...) + } + if ok := emitSourceInitEvents(sourceInitEvents); !ok { + return nil + } + + var sourceInitWG sync.WaitGroup + for i, src := range a.sources { + interval := sourceInitRefreshInterval(a.sourceInitRefresh, i) + if interval <= 0 { + continue + } + sourceInitWG.Add(1) + go func(i int, src source.Source, interval time.Duration) { + defer sourceInitWG.Done() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + events, err := src.InitEvents() + if err != nil { + a.logger.Error("source init refresh error", slog.Int("source_index", i), slog.String("error", err.Error())) + continue + } + if ok := emitSourceInitEvents(events); !ok { + return + } + case <-sourceCtx.Done(): + return + } + } + }(i, src, interval) + } + + sourceDone := make(chan error, len(a.sources)) + var sourceWG sync.WaitGroup + for i, src := range a.sources { + sourceWG.Add(1) + go func(i int, src source.Source) { + defer sourceWG.Done() + err := src.Start(sourceCtx, func(evt *event.Event) error { + select { + case decodeJobs <- evt: + return nil + case <-sourceCtx.Done(): + return ctx.Err() + } + }) + sourceDone <- err + if err != nil && sourceCtx.Err() == nil { + a.logger.Error("source error", slog.Int("source_index", i), slog.String("error", err.Error())) + } + }(i, src) + } + + shutdown := func() { + // Stop stages from left to right so every downstream stage drains and gets + // a chance to flush buffered or stateful output. + close(decodeJobs) + decodeWG.Wait() + close(processJobs) + processWG.Wait() + if hasAggregators { + close(aggregateRouteJobs) + aggregateRouteWG.Wait() + aggregateWG.Wait() + } + close(encodeJobs) + encodeWG.Wait() + } + + select { + case err := <-sourceDone: + stopSource() + for _, src := range a.sources { + _ = src.Close() + } + sourceWG.Wait() + sourceInitWG.Wait() + shutdown() + if err != nil && sourceCtx.Err() == nil { + return fmt.Errorf("run source: %w", err) + } + return nil + case <-ctx.Done(): + stopSource() + for _, src := range a.sources { + _ = src.Close() + } + sourceWG.Wait() + sourceInitWG.Wait() + shutdown() + return nil + } +} + +func sourceInitRefreshDuration(cfg config.SourceConfig) time.Duration { + if cfg.SourceInitRefreshMS == nil { + return 30 * time.Second + } + if *cfg.SourceInitRefreshMS <= 0 { + return 0 + } + return time.Duration(*cfg.SourceInitRefreshMS) * time.Millisecond +} + +func sourceInitRefreshInterval(intervals []time.Duration, sourceIndex int) time.Duration { + if sourceIndex < 0 || sourceIndex >= len(intervals) { + return 0 + } + return intervals[sourceIndex] +} + +func sourceInitEventsForEncoder(encoderType string, events []*event.Event) []*event.Event { + if encoderType != "ipfix" || len(events) <= 1 { + return events + } + return []*event.Event{ + { + ReceivedAt: time.Now().UTC(), + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init_batch", + }, + Payload: events, + }, + } +} + +type aggregateWorker struct { + cfg config.AggregatorConfig + agg aggregate.Aggregator + jobs chan *event.Event +} + +func aggregateWorkersRouteSourceInit(workers []aggregateWorker) bool { + for _, worker := range workers { + if worker.cfg.EmitAs == "data" { + return true + } + if worker.cfg.Match["kind"] == "control" || worker.cfg.Match["control.type"] == "source_init" { + return true + } + } + return false +} + +func matchingAggregateWorkers(workers []aggregateWorker, evt *event.Event) ([]aggregateWorker, string, string) { + matched := make([]aggregateWorker, 0, len(workers)) + missingKey := "" + missingKeyStream := "" + for _, worker := range workers { + if !aggregatorFilterMatches(worker.cfg, evt) { + continue + } + if key := aggregate.MissingRequiredKeyField(worker.cfg, evt); key != "" { + if missingKey == "" { + missingKey = key + missingKeyStream = worker.cfg.Stream + } + continue + } + matched = append(matched, worker) + } + if len(matched) > 0 { + return matched, "", "" + } + return matched, missingKeyStream, missingKey +} + +// aggregatorMatches applies the optional aggregator.match filter to a processed event. +func aggregatorMatches(cfg config.AggregatorConfig, evt *event.Event) bool { + return aggregatorFilterMatches(cfg, evt) && aggregate.EventHasRequiredKeyFields(cfg, evt) +} + +func aggregatorFilterMatches(cfg config.AggregatorConfig, evt *event.Event) bool { + if len(cfg.Match) == 0 { + return true + } + for key, want := range cfg.Match { + if eventMatchValue(evt, key) != want { + return false + } + } + return true +} + +func controlEventCanRouteThroughAggregator(evt *event.Event) bool { + if evt == nil || evt.Control == nil { + return false + } + return evt.Control.Type == "source_init" +} + +// eventMatchValue exposes a small set of envelope metadata alongside event fields +// so match rules can route on either transport metadata or canonical fields. +func eventMatchValue(evt *event.Event, key string) string { + if evt == nil { + return "" + } + if field, ok := strings.CutSuffix(key, ".ip_version"); ok { + return eventIPVersion(evt, field) + } + if layer, ok := strings.CutPrefix(key, "packet.has_layer."); ok { + if packetHasLayer(evt.Packet, layer) { + return "true" + } + return "false" + } + switch key { + case "stream": + return evt.Stream + case "kind": + return evt.Kind + case "control.type": + if evt.Control != nil { + return evt.Control.Type + } + return "" + case "control.stream": + if evt.Control != nil { + return evt.Control.Stream + } + return "" + case "source.type": + return evt.Source.Type + case "source.network": + return evt.Source.Network + case "source.address": + return evt.Source.Address + case "source.agent_ip": + return evt.Source.AgentIP + case "source.source_id": + if evt.Source.SourceIDSet || evt.Source.SourceID != 0 { + return fmt.Sprint(evt.Source.SourceID) + } + return "" + case "source.sampling.rate": + if evt.Source.Sampling != nil { + return fmt.Sprint(evt.Source.Sampling.Rate) + } + return "" + case "source.sampling.sample_pool": + if evt.Source.Sampling != nil { + return fmt.Sprint(evt.Source.Sampling.SamplePool) + } + return "" + case "source.sampling.drops": + if evt.Source.Sampling != nil { + return fmt.Sprint(evt.Source.Sampling.Drops) + } + return "" + } + if evt.Fields == nil { + return "" + } + if val, ok := evt.Fields[key]; ok { + return fmt.Sprint(val) + } + return "" +} + +func eventIPVersion(evt *event.Event, key string) string { + val := eventMatchValue(evt, key) + if val == "" { + return "" + } + addr, err := netip.ParseAddr(val) + if err != nil { + return "" + } + if addr.Is4() { + return "ipv4" + } + if addr.Is6() { + return "ipv6" + } + return "" +} + +func packetHasLayer(packet *event.PacketModel, kind string) bool { + if packet == nil || kind == "" { + return false + } + for _, layer := range packet.Layers { + if layer.Kind == kind { + return true + } + } + return false +} + +// tickerChannel keeps select logic simple when a stage does not need timer-driven flushing. +func tickerChannel(t *time.Ticker) <-chan time.Time { + if t == nil { + return nil + } + return t.C +} + +// encoderTickInterval picks the smallest configured timer that should wake encoder workers. +func encoderTickInterval(cfg config.EncoderConfig) time.Duration { + var min time.Duration + add := func(ms int) { + if ms <= 0 { + return + } + d := time.Duration(ms) * time.Millisecond + if min == 0 || d < min { + min = d + } + } + if cfg.Batch.IsEnabled() { + add(cfg.Batch.FlushInterval) + } + add(cfg.TemplatedFlow.TemplateRefresh) + add(cfg.TemplatedFlow.OptionsRefresh) + return min +} + +// requiresOrderedExporter forces single-worker encoding for protocols whose +// packets, templates, or sequence numbers depend on ordering. +func requiresOrderedExporter(encoderType string) bool { + switch encoderType { + case "sflow", "ipfix", "netflowv9", "netflowv5", "pcap", "pcapng": + return true + default: + return false + } +} diff --git a/pkg/reflow/app/app_bench_test.go b/pkg/reflow/app/app_bench_test.go new file mode 100644 index 00000000..905ff9ba --- /dev/null +++ b/pkg/reflow/app/app_bench_test.go @@ -0,0 +1,132 @@ +package app + +import ( + "context" + "io" + "log/slog" + "sync/atomic" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/decode" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/processor" + "github.com/netsampler/goflow2/v3/pkg/reflow/sink" + "github.com/netsampler/goflow2/v3/pkg/reflow/source" +) + +func BenchmarkRunPipeline(b *testing.B) { + benchmarks := []struct { + name string + newEvent func() *event.Event + encoderCfg config.EncoderConfig + }{ + { + name: "bytes_to_json", + newEvent: newBenchPacketEvent, + encoderCfg: config.EncoderConfig{Type: "json"}, + }, + { + name: "bytes_to_protobuf", + newEvent: newBenchPacketEvent, + encoderCfg: config.EncoderConfig{Type: "protobuf"}, + }, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + src := &benchSource{ + count: b.N, + newEvent: bm.newEvent, + } + out := &benchSink{} + dec := decode.New() + defer dec.Close() + app := &App{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + sources: []source.Source{src}, + decoder: dec, + processor: processor.NewBuiltin(config.ProcessorConfig{}), + processorWorkers: 1, + encoderCfg: bm.encoderCfg, + encoderWorkers: 1, + sink: out, + } + + b.ReportAllocs() + b.ResetTimer() + if err := app.Run(context.Background()); err != nil { + b.Fatalf("Run returned error: %v", err) + } + b.StopTimer() + if got := out.count.Load(); got != int64(b.N) { + b.Fatalf("expected %d sink payloads, got %d", b.N, got) + } + }) + } +} + +type benchSource struct { + count int + newEvent func() *event.Event +} + +func (s *benchSource) InitEvents() ([]*event.Event, error) { + return nil, nil +} + +func (s *benchSource) Start(ctx context.Context, emit func(*event.Event) error) error { + for i := 0; i < s.count; i++ { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if err := emit(s.newEvent()); err != nil { + return err + } + } + return nil +} + +func (s *benchSource) Close() error { + return nil +} + +type benchSink struct { + count atomic.Int64 + bytes atomic.Int64 +} + +func (s *benchSink) Send(payload []byte) error { + s.count.Add(1) + s.bytes.Add(int64(len(payload))) + return nil +} + +func (s *benchSink) Close() error { + return nil +} + +func newBenchPacketEvent() *event.Event { + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x14, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + return &event.Event{ + ReceivedAt: time.Unix(1_700_000_001, 0).UTC(), + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + } +} + +var _ source.Source = (*benchSource)(nil) +var _ sink.Sink = (*benchSink)(nil) diff --git a/pkg/reflow/app/app_test.go b/pkg/reflow/app/app_test.go new file mode 100644 index 00000000..a1399a48 --- /dev/null +++ b/pkg/reflow/app/app_test.go @@ -0,0 +1,431 @@ +package app + +import ( + "context" + "io" + "log/slog" + "sync/atomic" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/decode" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/processor" + "github.com/netsampler/goflow2/v3/pkg/reflow/sink" + "github.com/netsampler/goflow2/v3/pkg/reflow/source" +) + +func TestAggregatorMatchesByFieldsAndMetadata(t *testing.T) { + evt := &event.Event{ + Kind: "data", + Stream: "agg_samples", + Source: event.SourceMetadata{ + Type: "flow", + Network: "udp", + Address: ":18081", + }, + Fields: map[string]any{ + "record_kind": "packet", + "source_id": uint32(7), + }, + Packet: &event.PacketModel{ + Layers: []event.LayerSpec{ + {Kind: "ethernet"}, + {Kind: "mpls"}, + {Kind: "ipv4"}, + }, + }, + } + + if !aggregatorMatches(config.AggregatorConfig{ + Match: map[string]string{ + "record_kind": "packet", + "stream": "agg_samples", + "source.type": "flow", + "source.network": "udp", + "source.address": ":18081", + "source_id": "7", + "packet.has_layer.mpls": "true", + }, + }, evt) { + t.Fatalf("expected match to succeed") + } + + if aggregatorMatches(config.AggregatorConfig{ + Match: map[string]string{ + "record_kind": "interface_counter", + }, + }, evt) { + t.Fatalf("expected match to fail") + } + + if aggregatorMatches(config.AggregatorConfig{ + Match: map[string]string{ + "packet.has_layer.vxlan": "true", + }, + }, evt) { + t.Fatalf("expected missing layer match to fail") + } + + sourceInit := &event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init", + Stream: "options_data", + }, + Source: event.SourceMetadata{ + SourceID: 7, + SourceIDSet: true, + Sampling: &event.SamplingMetadata{ + Rate: 100, + }, + }, + } + if !aggregatorMatches(config.AggregatorConfig{ + Match: map[string]string{ + "kind": "control", + "control.type": "source_init", + "control.stream": "options_data", + "source.source_id": "7", + "source.sampling.rate": "100", + }, + }, sourceInit) { + t.Fatalf("expected source_init control match to succeed") + } +} + +func TestAggregatorMatchesIPAddressVersion(t *testing.T) { + evt := &event.Event{ + Kind: "data", + Fields: map[string]any{ + "src_addr": "2001:db8::1", + "nat_src_addr": "192.0.2.10", + }, + } + + if !aggregatorMatches(config.AggregatorConfig{ + Match: map[string]string{ + "src_addr.ip_version": "ipv6", + "nat_src_addr.ip_version": "ipv4", + }, + }, evt) { + t.Fatalf("expected IP version match to succeed") + } + if aggregatorMatches(config.AggregatorConfig{ + Match: map[string]string{ + "src_addr.ip_version": "ipv4", + }, + }, evt) { + t.Fatalf("expected wrong IP version match to fail") + } + if aggregatorMatches(config.AggregatorConfig{ + Match: map[string]string{ + "missing_addr.ip_version": "ipv6", + }, + }, evt) { + t.Fatalf("expected missing IP version match to fail") + } +} + +func TestAggregatorMatchesMissingKeysOnlyWhenAggregateMissingEnabled(t *testing.T) { + evt := &event.Event{ + Kind: "data", + Fields: map[string]any{ + "dst_addr": "198.51.100.2", + "bytes": int64(60), + }, + } + + first := config.AggregatorConfig{ + KeyFields: []string{"src_addr", "dst_addr"}, + Sum: []string{"bytes"}, + } + if aggregatorMatches(first, evt) { + t.Fatalf("expected aggregator without aggregate_missing to reject missing src_addr") + } + + next := config.AggregatorConfig{ + AggregateMissing: true, + KeyFields: []string{"src_addr", "dst_addr"}, + Sum: []string{"bytes"}, + } + if !aggregatorMatches(next, evt) { + t.Fatalf("expected aggregator with aggregate_missing to accept missing src_addr") + } +} + +func TestMatchingAggregateWorkersWarnsOnlyWhenNoAggregatorAcceptsMissingKeys(t *testing.T) { + evt := &event.Event{ + Kind: "data", + Fields: map[string]any{ + "dst_addr": "198.51.100.2", + "bytes": int64(60), + }, + } + withoutAggregateMissing := aggregateWorker{ + cfg: config.AggregatorConfig{ + Stream: "strict_flow_data", + KeyFields: []string{"src_addr", "dst_addr"}, + Sum: []string{"bytes"}, + }, + } + withAggregateMissing := aggregateWorker{ + cfg: config.AggregatorConfig{ + Stream: "missing_flow_data", + AggregateMissing: true, + KeyFields: []string{"src_addr", "dst_addr"}, + Sum: []string{"bytes"}, + }, + } + + matched, stream, key := matchingAggregateWorkers([]aggregateWorker{ + withoutAggregateMissing, + withAggregateMissing, + }, evt) + if len(matched) != 1 || matched[0].cfg.Stream != "missing_flow_data" { + t.Fatalf("expected only aggregate_missing worker to match, got %#v", matched) + } + if stream != "" || key != "" { + t.Fatalf("did not expect warning details when an aggregator matched, got stream=%q key=%q", stream, key) + } + + matched, stream, key = matchingAggregateWorkers([]aggregateWorker{withoutAggregateMissing}, evt) + if len(matched) != 0 { + t.Fatalf("expected no workers to match, got %#v", matched) + } + if stream != "strict_flow_data" || key != "src_addr" { + t.Fatalf("expected warning details for unaggregated event, got stream=%q key=%q", stream, key) + } +} + +func TestRunClosesStdoutLikeSinkOnContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + src := &blockingSource{} + out := &closeTrackingSink{} + app := &App{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + sources: []source.Source{src}, + decoder: noopDecoder{}, + processor: passthroughProcessor{}, + processorWorkers: 1, + encoderCfg: config.EncoderConfig{Type: "json"}, + encoderWorkers: 1, + sink: out, + } + + done := make(chan error, 1) + go func() { + done <- app.Run(ctx) + }() + + cancel() + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(time.Second): + t.Fatalf("Run did not return after context cancellation") + } + if got := out.closeCount.Load(); got != 1 { + t.Fatalf("expected stdout-like sink Close to be called once, got %d", got) + } + if got := src.closeCount.Load(); got != 1 { + t.Fatalf("expected source Close to be called once, got %d", got) + } +} + +func TestRunRefreshesSourceInitEvents(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + src := &refreshingSource{} + out := &countingSink{ + after: 2, + cancel: cancel, + } + app := &App{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + sources: []source.Source{src}, + sourceInitRefresh: []time.Duration{10 * time.Millisecond}, + decoder: noopDecoder{}, + processor: passthroughProcessor{}, + processorWorkers: 1, + encoderCfg: config.EncoderConfig{Type: "json"}, + encoderWorkers: 1, + sink: out, + } + + done := make(chan error, 1) + go func() { + done <- app.Run(ctx) + }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(time.Second): + t.Fatalf("Run did not refresh source init events") + } + if got := src.initCount.Load(); got < 2 { + t.Fatalf("expected source InitEvents to be called at least twice, got %d", got) + } + if got := out.sendCount.Load(); got < 2 { + t.Fatalf("expected at least two encoded source init events, got %d", got) + } +} + +func TestRunRefreshesSourceInitEventsAfterEmptyInitialSet(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + src := &refreshingSource{emptyFirst: true} + out := &countingSink{ + after: 1, + cancel: cancel, + } + app := &App{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + sources: []source.Source{src}, + sourceInitRefresh: []time.Duration{10 * time.Millisecond}, + decoder: noopDecoder{}, + processor: passthroughProcessor{}, + processorWorkers: 1, + encoderCfg: config.EncoderConfig{Type: "json"}, + encoderWorkers: 1, + sink: out, + } + + done := make(chan error, 1) + go func() { + done <- app.Run(ctx) + }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(time.Second): + t.Fatalf("Run did not refresh source init events after an empty initial set") + } + if got := src.initCount.Load(); got < 2 { + t.Fatalf("expected source InitEvents to be retried, got %d calls", got) + } + if got := out.sendCount.Load(); got != 1 { + t.Fatalf("expected one encoded refreshed source init event, got %d", got) + } +} + +type blockingSource struct { + closeCount atomic.Int32 +} + +func (s *blockingSource) InitEvents() ([]*event.Event, error) { + return nil, nil +} + +func (s *blockingSource) Start(ctx context.Context, _ func(*event.Event) error) error { + <-ctx.Done() + return ctx.Err() +} + +func (s *blockingSource) Close() error { + s.closeCount.Add(1) + return nil +} + +type refreshingSource struct { + initCount atomic.Int32 + closeCount atomic.Int32 + emptyFirst bool +} + +func (s *refreshingSource) InitEvents() ([]*event.Event, error) { + count := s.initCount.Add(1) + if s.emptyFirst && count == 1 { + return nil, nil + } + return []*event.Event{ + { + ReceivedAt: time.Now().UTC(), + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init", + Stream: "options_data", + }, + Fields: map[string]any{ + "init_count": count, + }, + }, + }, nil +} + +func (s *refreshingSource) Start(ctx context.Context, _ func(*event.Event) error) error { + <-ctx.Done() + return ctx.Err() +} + +func (s *refreshingSource) Close() error { + s.closeCount.Add(1) + return nil +} + +type closeTrackingSink struct { + closeCount atomic.Int32 +} + +func (s *closeTrackingSink) Send(_ []byte) error { + return nil +} + +func (s *closeTrackingSink) Close() error { + s.closeCount.Add(1) + return nil +} + +type countingSink struct { + sendCount atomic.Int32 + after int32 + cancel context.CancelFunc +} + +func (s *countingSink) Send(_ []byte) error { + count := s.sendCount.Add(1) + if s.cancel != nil && s.after > 0 && count >= s.after { + s.cancel() + } + return nil +} + +func (s *countingSink) Close() error { + return nil +} + +type noopDecoder struct{} + +func (noopDecoder) Decode(evt *event.Event) ([]*event.Event, error) { + return []*event.Event{evt}, nil +} + +func (noopDecoder) Errors() <-chan error { return nil } + +func (noopDecoder) Close() {} + +var _ decode.Decoder = noopDecoder{} +var _ sink.Sink = (*closeTrackingSink)(nil) +var _ sink.Sink = (*countingSink)(nil) +var _ source.Source = (*blockingSource)(nil) +var _ source.Source = (*refreshingSource)(nil) + +type passthroughProcessor struct{} + +func (passthroughProcessor) Process(evt *event.Event) ([]*event.Event, error) { + return []*event.Event{evt}, nil +} + +var _ processor.Processor = passthroughProcessor{} diff --git a/pkg/reflow/app/ipfix_compare_bench_test.go b/pkg/reflow/app/ipfix_compare_bench_test.go new file mode 100644 index 00000000..2ab267e9 --- /dev/null +++ b/pkg/reflow/app/ipfix_compare_bench_test.go @@ -0,0 +1,156 @@ +package app + +import ( + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/decode" + reflowencode "github.com/netsampler/goflow2/v3/pkg/reflow/encode" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/processor" +) + +func BenchmarkReFlowRawIPFIXToEncode(b *testing.B) { + catalog := benchReFlowIPFIXCatalog() + templates, data := benchReFlowIPFIXPayloads(b, catalog) + + benchmarks := []struct { + name string + cfg config.EncoderConfig + }{ + {name: "protobuf", cfg: config.EncoderConfig{Type: "protobuf"}}, + {name: "json", cfg: config.EncoderConfig{Type: "json"}}, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + dec := decode.NewWithCatalog(catalog) + defer dec.Close() + for _, payload := range templates { + if _, err := dec.Decode(&event.Event{Source: event.SourceMetadata{Type: "flow"}, Payload: payload}); err != nil { + b.Fatalf("decode template payload: %v", err) + } + } + proc := processor.NewBuiltin(config.ProcessorConfig{}) + enc, err := reflowencode.New(bm.cfg) + if err != nil { + b.Fatalf("New encoder returned error: %v", err) + } + + var totalBytes int + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + decoded, err := dec.Decode(&event.Event{ + ReceivedAt: time.Unix(1_700_000_001, 0).UTC(), + Source: event.SourceMetadata{Type: "flow"}, + Payload: data, + }) + if err != nil { + b.Fatalf("Decode returned error: %v", err) + } + if len(decoded) != 1 { + b.Fatalf("expected 1 decoded event, got %d", len(decoded)) + } + processed, err := proc.Process(decoded[0]) + if err != nil { + b.Fatalf("Process returned error: %v", err) + } + for _, evt := range processed { + payloads, err := enc.Encode(evt) + if err != nil { + b.Fatalf("Encode returned error: %v", err) + } + for _, payload := range payloads { + totalBytes += len(payload) + } + } + } + if totalBytes == 0 { + b.Fatalf("expected encoded payload bytes") + } + }) + } +} + +func benchReFlowIPFIXPayloads(b *testing.B, catalog map[string]config.IPFIXFieldDefinition) ([][]byte, []byte) { + b.Helper() + enc := reflowencode.NewIPFIXEncoder(config.EncoderConfig{ + Type: "ipfix", + TemplatedFlow: config.TemplatedFlowConfig{ + TemplateBaseID: 256, + Data: config.TemplatedFlowDataConfig{ + Select: []string{"src_addr", "dst_addr", "src_port", "dst_port", "proto", "bytes", "packets", "input_if", "output_if", "start_time_unix", "end_time_unix"}, + Catalog: catalog, + }, + }, + }) + templates, err := enc.Encode(&event.Event{ + Kind: "control", + Stream: "flow_data", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{ + "src_addr", + "dst_addr", + "src_port", + "dst_port", + "proto", + "bytes", + "packets", + "input_if", + "output_if", + "start_time_unix", + "end_time_unix", + }, + }, + }) + if err != nil { + b.Fatalf("encode schema returned error: %v", err) + } + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(1_700_000_001, 0).UTC(), + Stream: "flow_data", + Fields: map[string]any{ + "src_addr": "192.0.2.10", + "dst_addr": "198.51.100.20", + "src_port": uint32(12345), + "dst_port": uint32(443), + "proto": uint32(6), + "bytes": int64(321), + "packets": int64(7), + "input_if": uint32(9), + "output_if": uint32(10), + "start_time_unix": int64(1_700_000_000_100), + "end_time_unix": int64(1_700_000_000_900), + }, + }) + if err != nil { + b.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + b.Fatalf("expected one data payload, got %d", len(payloads)) + } + return templates, payloads[0] +} + +func benchReFlowIPFIXCatalog() map[string]config.IPFIXFieldDefinition { + return map[string]config.IPFIXFieldDefinition{ + "src_addr": {ID: 8, Length: 4, Type: "ipv4Address"}, + "dst_addr": {ID: 12, Length: 4, Type: "ipv4Address"}, + "src_port": {ID: 7, Length: 2, Type: "unsigned16"}, + "dst_port": {ID: 11, Length: 2, Type: "unsigned16"}, + "proto": {ID: 4, Length: 1, Type: "unsigned8"}, + "bytes": {ID: 1, Length: 8, Type: "unsigned64"}, + "packets": {ID: 2, Length: 4, Type: "unsigned32"}, + "input_if": {ID: 10, Length: 4, Type: "unsigned32"}, + "output_if": {ID: 14, Length: 4, Type: "unsigned32"}, + "start_time_unix": {ID: 152, Length: 8, Type: "unsigned64"}, + "end_time_unix": {ID: 153, Length: 8, Type: "unsigned64"}, + } +} diff --git a/pkg/reflow/config/cli.go b/pkg/reflow/config/cli.go new file mode 100644 index 00000000..7c7ccb19 --- /dev/null +++ b/pkg/reflow/config/cli.go @@ -0,0 +1,1435 @@ +package config + +import ( + "fmt" + "net/netip" + "net/url" + "strconv" + "strings" +) + +const generatedConfigPath = "cmd/reflow/reflow.yaml" + +var ( + inputHelperOptions = []string{ + "udp::flow", + "udp::bytes", + "udp::json", + "unixgram::flow", + "unixgram::bytes", + "unixgram::json", + "socket::flow", + "socket::bytes", + "socket::json", + "stream::pcap", + "stream::pcapng", + "stream::json", + "ebpf::bytes[?snaplen=&sample_every=&sample_offset=&source_init_refresh_ms=&interface_filter=&direction=ingress|egress|both&skb_metadata=&conntrack=]", + "pcap_live::bytes[?snaplen=&sample_every=&sample_offset=&source_init_refresh_ms=&interface_filter=]", + } + outputEncoderOptions = []string{ + "json", + "protobuf", + "sflow", + "ipfix", + "netflowv9", + "netflowv5", + "pcap", + "pcapng", + } + outputSinkOptions = []string{ + "stdout", + "file:", + "udp:
[?resolve_interval_ms=]", + "unixgram:", + "socket:", + } + outputHelperOptions = []string{ + "protobuf:*[?export_all=]", + "sflow:*[?allow_truncate=&max_header_bytes=]", + "sflow|ipfix|netflowv9:*[?batch=&batch_max_records=&batch_max_bytes=&batch_flush_interval_ms=]", + } + inputHelperExamples = []string{ + "udp::6343:flow", + "udp:127.0.0.1:2055:flow", + "stream:-:json", + "'ebpf:eth0:bytes?direction=ingress'", + "'pcap_live:en0:bytes?snaplen=262144&sample_every=10'", + } + outputHelperExamples = []string{ + "json:stdout", + "ipfix:udp:127.0.0.1:4739", + "'ipfix:udp:collector.example.com:4739?resolve_interval_ms=0'", + "'protobuf:file:/tmp/reflow.pb?export_all=true'", + "pcap:stdout", + "'sflow:udp:127.0.0.1:6343?allow_truncate=true&max_header_bytes=128'", + "'ipfix:udp:127.0.0.1:4739?batch=true&batch_max_records=32&batch_max_bytes=4096&batch_flush_interval_ms=250'", + } + aggregateHelperOptions = []string{ + "-agg", + "-agg payload", + "-agg payload,limited", + "-agg nat", + "-agg encap", + "-agg mpls", + "-agg passthrough", + "-agg idle_flush_after_ms=,periodic_every_ms=", + "-agg max_flush_after_ms=,idle_erase_after_ms=,reset_buckets=,aggregate_missing=", + } + aggregateHelperExamples = []string{ + "-agg payload", + "-agg payload,limited", + "-agg payload,limited,nat", + "-agg payload,limited,encap", + "-agg nat", + "-agg encap", + "-agg mpls", + "-agg passthrough", + "-agg idle_flush_after_ms=5000,periodic_every_ms=30000", + "-agg periodic_every_ms=10000,reset_buckets=true", + "-agg aggregate_missing=false", + } + generatedTemplatedFlowFields = []string{ + "bytes", + "packets", + "proto", + "tcp_flags", + "src_port", + "src_addr", + "input_if", + "dst_port", + "dst_addr", + "output_if", + "src_mac", + "vlan_id", + "flow_direction", + "dst_mac", + "agent_ip", + "agent_ipv6", + "source_id", + "start_time_unix", + "end_time_unix", + "ether_type", + } + generatedTemplatedFlowMPLSFields = []string{ + "mpls_label_stack_section_1", + "mpls_label_stack_section_2", + "mpls_label_stack_section_3", + } + generatedTemplatedFlowNATFields = []string{ + "nat_src_addr", + "nat_dst_addr", + "nat_src_port", + "nat_dst_port", + } + generatedTemplatedFlowEncapFields = []string{ + "outer_proto", + "outer_proto_name", + "outer_src_port", + "outer_src_addr", + "outer_dst_port", + "outer_dst_addr", + "encap_depth", + } + generatedPacketParsedFlowFields = []string{ + "protocol", + "header_protocol", + "header_protocol_name", + "proto", + "proto_name", + "tcp_flags", + "src_port", + "src_addr", + "dst_port", + "dst_addr", + "ip_family", + "src_mac", + "vlan_id", + "dst_mac", + "ether_type", + "outer_proto", + "outer_proto_name", + "outer_src_port", + "outer_src_addr", + "outer_dst_port", + "outer_dst_addr", + "encap_depth", + } + generatedPacketParsedIPFields = map[string]struct{}{ + "src_addr": {}, + "dst_addr": {}, + "ip_family": {}, + } +) + +type inputFlags []string + +func (f *inputFlags) String() string { + if f == nil { + return "" + } + return strings.Join(*f, ",") +} + +func HelperOptionsText() string { + var b strings.Builder + b.WriteString("Input helper specs (-input network:target:type):\n") + for _, option := range inputHelperOptions { + fmt.Fprintf(&b, " %s\n", option) + } + b.WriteString("\nInput examples:\n") + for _, example := range inputHelperExamples { + fmt.Fprintf(&b, " -input %s\n", example) + } + b.WriteString("\nOutput helper specs (-output encoder:sink[:target]):\n") + b.WriteString(" encoders: ") + b.WriteString(strings.Join(outputEncoderOptions, ", ")) + b.WriteString("\n sinks:\n") + for _, option := range outputSinkOptions { + fmt.Fprintf(&b, " %s\n", option) + } + b.WriteString(" parameters:\n") + for _, option := range outputHelperOptions { + fmt.Fprintf(&b, " %s\n", option) + } + b.WriteString("\nOutput examples:\n") + for _, example := range outputHelperExamples { + fmt.Fprintf(&b, " -output %s\n", example) + } + b.WriteString("\nAggregation helper specs:\n") + for _, option := range aggregateHelperOptions { + fmt.Fprintf(&b, " %s\n", option) + } + b.WriteString("\nAggregation examples:\n") + for _, example := range aggregateHelperExamples { + fmt.Fprintf(&b, " %s\n", example) + } + return b.String() +} + +func (f *inputFlags) Set(value string) error { + *f = append(*f, value) + return nil +} + +type outputFlag struct { + cfg *FlagConfig +} + +func (f outputFlag) String() string { + if f.cfg == nil { + return "" + } + return f.cfg.Output +} + +func (f outputFlag) Set(value string) error { + if f.cfg.OutputSet { + return fmt.Errorf("only one -output/-o value is supported") + } + f.cfg.Output = value + f.cfg.OutputSet = true + return nil +} + +type aggregateFlag struct { + cfg *FlagConfig +} + +func (f aggregateFlag) String() string { + if f.cfg == nil || !f.cfg.Aggregate { + return "false" + } + var parts []string + if f.cfg.AggIdleFlushAfter != nil { + parts = append(parts, fmt.Sprintf("idle_flush_after_ms=%d", *f.cfg.AggIdleFlushAfter)) + } + if f.cfg.AggMaxFlushAfter != nil { + parts = append(parts, fmt.Sprintf("max_flush_after_ms=%d", *f.cfg.AggMaxFlushAfter)) + } + if f.cfg.AggIdleEraseAfter != nil { + parts = append(parts, fmt.Sprintf("idle_erase_after_ms=%d", *f.cfg.AggIdleEraseAfter)) + } + if f.cfg.AggPeriodicEvery != nil { + parts = append(parts, fmt.Sprintf("periodic_every_ms=%d", *f.cfg.AggPeriodicEvery)) + } + if f.cfg.AggResetBuckets != nil { + parts = append(parts, fmt.Sprintf("reset_buckets=%t", *f.cfg.AggResetBuckets)) + } + if f.cfg.AggAggregateMissing != nil { + parts = append(parts, fmt.Sprintf("aggregate_missing=%t", *f.cfg.AggAggregateMissing)) + } + parts = append(parts, f.cfg.AggPresets...) + if len(parts) == 0 { + return "true" + } + return strings.Join(parts, ",") +} + +func (f aggregateFlag) Set(value string) error { + if f.cfg == nil { + return fmt.Errorf("missing aggregate flag config") + } + value = strings.TrimSpace(value) + switch value { + case "", "true": + f.cfg.Aggregate = true + return nil + case "false": + f.cfg.Aggregate = false + return nil + } + f.cfg.Aggregate = true + return parseAggregateParams(f.cfg, value) +} + +func (aggregateFlag) IsBoolFlag() bool { return true } + +// LoadFromFlags resolves either explicit YAML mode or generated-helper mode. +func LoadFromFlags(flags *FlagConfig) (*Config, bool, error) { + if flags == nil { + flags = &FlagConfig{} + } + if flags.ConfigPath != "" { + if flags.usesGeneratedHelpersWithConfig() { + return nil, false, fmt.Errorf("-config cannot be combined with -input, -output/-o, or -genconf") + } + cfg, err := Load(flags.ConfigPath) + if err != nil { + return nil, false, err + } + if err := cfg.ApplyAggregationFlags(flags); err != nil { + return nil, false, err + } + return cfg, false, nil + } + + cfg, err := flags.generatedConfig() + if err != nil { + return nil, true, err + } + if err := cfg.setDefaults(generatedConfigPath); err != nil { + return nil, true, err + } + if flags.GenConf { + cfg.materializeGeneratedYAMLDefaults() + } + return cfg, true, nil +} + +func (c *FlagConfig) usesGeneratedHelpersWithConfig() bool { + return len(c.Inputs) > 0 || c.OutputSet || c.GenConf +} + +func (c *FlagConfig) generatedConfig() (*Config, error) { + inputs := c.Inputs + if len(inputs) == 0 { + inputs = []string{ + "udp::6343:flow", + "udp::2055:flow", + } + } + + sources := make([]SourceConfig, 0, len(inputs)) + for _, spec := range inputs { + source, err := parseInputSpec(spec) + if err != nil { + return nil, err + } + sources = append(sources, source) + } + + output := c.Output + if output == "" { + output = "json:stdout" + } + encoder, sink, err := parseOutputSpec(output) + if err != nil { + return nil, err + } + cfg := &Config{ + Sources: sources, + Processor: ProcessorConfig{ + Type: "builtin", + Workers: AutoWorkers, + Builtin: BuiltinProcessorConfig{ + DropMessage: true, + DropPayload: true, + }, + }, + Encoder: encoder, + Sink: sink, + } + if encoder.Type == "ipfix" || encoder.Type == "netflowv9" { + cfg.Encoder.TemplatedFlow.Data.Select = append([]string(nil), generatedTemplatedFlowFields...) + } + if c.Aggregate { + cfg.Aggregators = generatedAggregators(c) + if lastAggregatePreset(c) != "none" && aggregatePresetRequested(c, "payload") { + cfg.ensureTemplatedFlowFieldsSelected("frame_length", "header_data") + } + if lastAggregatePreset(c) != "none" && aggregatePresetRequested(c, "nat") { + cfg.ensureTemplatedFlowFieldsSelected(generatedTemplatedFlowNATFields...) + } + if lastAggregatePreset(c) != "none" && aggregatePresetRequested(c, "encap") { + cfg.enablePacketDecoderBeyondL4() + cfg.ensureTemplatedFlowFieldsSelected(generatedTemplatedFlowEncapFields...) + } + if lastAggregatePreset(c) != "none" && aggregatePresetRequested(c, "mpls") { + cfg.enableMPLSAggregationHelpers() + cfg.ensureTemplatedFlowFieldsSelected(generatedTemplatedFlowMPLSFields...) + } + if lastAggregatePreset(c) != "none" && aggregatePresetRequested(c, "limited") { + cfg.removeTemplatedFlowFieldsSelected(packetParsedFieldsToRemove( + aggregatePresetRequested(c, "nat"), + aggregatePresetRequested(c, "encap"), + )...) + } + } + if generatedSourceOptionsAggregatorEnabled(c, encoder.Type) { + cfg.ensureSourceOptionsAggregator() + } + return cfg, nil +} + +func (c *Config) materializeGeneratedYAMLDefaults() { + for i := range c.Sources { + if c.Sources[i].SnapLen == 0 { + c.Sources[i].SnapLen = 65535 + } + if c.Sources[i].SampleEvery == 0 { + c.Sources[i].SampleEvery = 1 + } + if c.Sources[i].Network == "ebpf" { + defaultTrue(&c.Sources[i].EBPF.SKBMetadata) + defaultTrue(&c.Sources[i].EBPF.Conntrack) + if c.Sources[i].EBPF.Direction == "" { + c.Sources[i].EBPF.Direction = "both" + } + } + } + defaultFalse(&c.Processor.Builtin.PacketDecoder.DecodeBeyondL4) + defaultFalse(&c.Processor.Builtin.PacketDecoder.Encapsulations.GRE.Enabled) + defaultFalse(&c.Processor.Builtin.PacketDecoder.Encapsulations.IPIP.Enabled) + defaultFalse(&c.Processor.Builtin.PacketDecoder.Encapsulations.VXLAN.Enabled) + defaultFalse(&c.Processor.Builtin.PacketDecoder.Encapsulations.Geneve.Enabled) + defaultFalse(&c.Processor.Builtin.PacketDecoder.Encapsulations.L2TP.Enabled) + defaultFalse(&c.Processor.Builtin.PacketDecoder.Encapsulations.GTPU.Enabled) + defaultFalse(&c.Processor.Builtin.PacketDecoder.Encapsulations.PPPoE.Enabled) + defaultFalse(&c.Encoder.Batch.Enabled) + if len(c.Processor.Builtin.PacketDecoder.Encapsulations.VXLAN.Ports) == 0 { + c.Processor.Builtin.PacketDecoder.Encapsulations.VXLAN.Ports = []uint32{4789} + } + if len(c.Processor.Builtin.PacketDecoder.Encapsulations.Geneve.Ports) == 0 { + c.Processor.Builtin.PacketDecoder.Encapsulations.Geneve.Ports = []uint32{6081} + } + if len(c.Processor.Builtin.PacketDecoder.Encapsulations.L2TP.Ports) == 0 { + c.Processor.Builtin.PacketDecoder.Encapsulations.L2TP.Ports = []uint32{1701} + } + if len(c.Processor.Builtin.PacketDecoder.Encapsulations.GTPU.Ports) == 0 { + c.Processor.Builtin.PacketDecoder.Encapsulations.GTPU.Ports = []uint32{2152} + } + if c.Encoder.TemplatedFlow.TemplateBaseID == 0 { + c.Encoder.TemplatedFlow.TemplateBaseID = 256 + } + if c.Encoder.TemplatedFlow.OptionsTemplateBaseID == 0 { + c.Encoder.TemplatedFlow.OptionsTemplateBaseID = 1024 + } + if c.Encoder.TemplatedFlow.TemplateRefresh == 0 { + c.Encoder.TemplatedFlow.TemplateRefresh = 60000 + } + if c.Encoder.TemplatedFlow.OptionsRefresh == 0 { + c.Encoder.TemplatedFlow.OptionsRefresh = 30000 + } +} + +func parseInputSpec(spec string) (SourceConfig, error) { + network, rest, ok := strings.Cut(spec, ":") + if !ok || network == "" || rest == "" { + return SourceConfig{}, fmt.Errorf("invalid -input %q: expected network:target:type", spec) + } + target, sourceType, ok := cutLast(rest, ":") + if !ok || sourceType == "" { + return SourceConfig{}, fmt.Errorf("invalid -input %q: expected network:target:type", spec) + } + network = normalizeSocketAlias(network) + sourceType, rawParams, ok := strings.Cut(sourceType, "?") + if ok && rawParams == "" { + return SourceConfig{}, fmt.Errorf("invalid -input %q: source parameters cannot be empty", spec) + } + + source := SourceConfig{ + Network: network, + Address: target, + Type: sourceType, + } + if source.Type == "json" { + source.JSON.Flavor = "reflow" + } + + switch network { + case "udp": + if err := validateSocketSourceType(sourceType); err != nil { + return SourceConfig{}, fmt.Errorf("invalid -input %q: %w", spec, err) + } + if err := validateUDPListenTarget(target); err != nil { + return SourceConfig{}, fmt.Errorf("invalid -input %q: %w", spec, err) + } + case "unixgram": + if err := validateSocketSourceType(sourceType); err != nil { + return SourceConfig{}, fmt.Errorf("invalid -input %q: %w", spec, err) + } + case "stream": + switch sourceType { + case "pcap", "pcapng", "json": + default: + return SourceConfig{}, fmt.Errorf("invalid -input %q: stream source type must be pcap, pcapng, or json", spec) + } + case "pcap_live", "ebpf": + if sourceType != "bytes" { + return SourceConfig{}, fmt.Errorf("invalid -input %q: %s source type must be bytes", spec, network) + } + source.Interface = target + source.Address = "" + default: + return SourceConfig{}, fmt.Errorf("invalid -input %q: unsupported network %q", spec, network) + } + if rawParams != "" { + if err := applyInputParams(&source, rawParams); err != nil { + return SourceConfig{}, fmt.Errorf("invalid -input %q: %w", spec, err) + } + } + return source, nil +} + +func applyInputParams(source *SourceConfig, rawParams string) error { + if source.Network != "pcap_live" && source.Network != "ebpf" { + return fmt.Errorf("source parameters are only supported for pcap_live and ebpf inputs") + } + params, err := url.ParseQuery(rawParams) + if err != nil { + return fmt.Errorf("parse source parameters: %w", err) + } + for key, values := range params { + if len(values) != 1 { + return fmt.Errorf("source parameter %q must be set once", key) + } + value := values[0] + switch key { + case "snaplen": + parsed, err := parseNonNegativeInputParam(key, value) + if err != nil { + return err + } + source.SnapLen = parsed + case "sample_every": + parsed, err := parseNonNegativeInputParam(key, value) + if err != nil { + return err + } + source.SampleEvery = parsed + case "sample_offset": + parsed, err := parseNonNegativeInputParam(key, value) + if err != nil { + return err + } + source.SampleOffset = parsed + case "source_init_refresh_ms": + parsed, err := parseNonNegativeInputParam(key, value) + if err != nil { + return err + } + source.SourceInitRefreshMS = &parsed + case "interface_filter": + if source.Network != "pcap_live" && source.Network != "ebpf" { + return fmt.Errorf("source parameter %q is only supported for pcap_live and ebpf inputs", key) + } + if value == "" { + return fmt.Errorf("source parameter %q cannot be empty", key) + } + source.InterfaceFilter = value + case "skb_metadata": + if source.Network != "ebpf" { + return fmt.Errorf("source parameter %q is only supported for ebpf inputs", key) + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("source parameter %q must be a boolean", key) + } + source.EBPF.SKBMetadata = &parsed + case "conntrack": + if source.Network != "ebpf" { + return fmt.Errorf("source parameter %q is only supported for ebpf inputs", key) + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("source parameter %q must be a boolean", key) + } + source.EBPF.Conntrack = &parsed + case "conntrack_path": + if source.Network != "ebpf" { + return fmt.Errorf("source parameter %q is only supported for ebpf inputs", key) + } + if value == "" { + return fmt.Errorf("source parameter %q cannot be empty", key) + } + source.EBPF.ConntrackPath = value + case "direction": + if source.Network != "ebpf" { + return fmt.Errorf("source parameter %q is only supported for ebpf inputs", key) + } + parsed, err := NormalizeEBPFDirection(value) + if err != nil { + return fmt.Errorf("source parameter %q: %w", key, err) + } + source.EBPF.Direction = parsed + default: + return fmt.Errorf("unsupported source parameter %q", key) + } + } + return nil +} + +func parseNonNegativeInputParam(name, value string) (int, error) { + parsed, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("source parameter %q must be an integer", name) + } + if parsed < 0 { + return 0, fmt.Errorf("source parameter %q must be >= 0", name) + } + return parsed, nil +} + +func parseAggregateParams(cfg *FlagConfig, rawParams string) error { + rawParams = strings.TrimPrefix(strings.TrimSpace(rawParams), "?") + if rawParams == "" { + return fmt.Errorf("parse aggregate parameters: parameters cannot be empty") + } + parts := strings.Split(strings.ReplaceAll(rawParams, "&", ","), ",") + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + return fmt.Errorf("parse aggregate parameters: parameters cannot be empty") + } + key, value, hasValue := strings.Cut(part, "=") + if !hasValue { + if err := parseAggregatePreset(cfg, key); err != nil { + return err + } + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch key { + case "idle_flush_after_ms": + parsed, err := parseNonNegativeAggregateParam(key, value) + if err != nil { + return err + } + cfg.AggIdleFlushAfter = &parsed + case "max_flush_after_ms": + parsed, err := parseNonNegativeAggregateParam(key, value) + if err != nil { + return err + } + cfg.AggMaxFlushAfter = &parsed + case "idle_erase_after_ms": + parsed, err := parseNonNegativeAggregateParam(key, value) + if err != nil { + return err + } + cfg.AggIdleEraseAfter = &parsed + case "periodic_every_ms", "every_ms": + parsed, err := parseNonNegativeAggregateParam(key, value) + if err != nil { + return err + } + cfg.AggPeriodicEvery = &parsed + case "reset_buckets": + parsed, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("aggregate parameter %q must be a boolean", key) + } + cfg.AggResetBuckets = &parsed + case "aggregate_missing": + parsed, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("aggregate parameter %q must be a boolean", key) + } + cfg.AggAggregateMissing = &parsed + default: + return fmt.Errorf("unsupported aggregate parameter %q", key) + } + } + return nil +} + +func parseAggregatePreset(cfg *FlagConfig, raw string) error { + preset := strings.ToLower(strings.TrimSpace(raw)) + switch preset { + case "payload", "header", "packet-header": + cfg.AggPresets = append(cfg.AggPresets, "payload") + case "nat": + cfg.AggPresets = append(cfg.AggPresets, "nat") + case "encap", "encapsulation", "outer": + cfg.AggPresets = append(cfg.AggPresets, "encap") + case "mpls": + cfg.AggPresets = append(cfg.AggPresets, "mpls") + case "limited", "no-packet-fields", "no-packet", "strip-packet-fields", "packetless": + cfg.AggPresets = append(cfg.AggPresets, "limited") + case "payload-only": + cfg.AggPresets = append(cfg.AggPresets, "payload", "limited") + case "passthrough": + cfg.AggPresets = append(cfg.AggPresets, "passthrough") + case "none", "off": + cfg.AggPresets = append(cfg.AggPresets, "none") + case "true": + // Useful when users write -agg=true,payload. + case "": + return fmt.Errorf("aggregate preset cannot be empty") + default: + return fmt.Errorf("unsupported aggregate preset %q", raw) + } + return nil +} + +// NormalizeAggregateArgs lets -agg accept either -agg=value or -agg value while +// keeping bare -agg as the existing boolean shorthand. +func NormalizeAggregateArgs(args []string) []string { + out := make([]string, 0, len(args)) + for i := 0; i < len(args); i++ { + arg := args[i] + if (arg == "-agg" || arg == "--agg") && i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") { + out = append(out, arg+"="+args[i+1]) + i++ + continue + } + out = append(out, arg) + } + return out +} + +func parseCommaQueryParams(rawParams string) (url.Values, error) { + rawParams = strings.TrimPrefix(strings.TrimSpace(rawParams), "?") + if rawParams == "" { + return nil, fmt.Errorf("parameters cannot be empty") + } + return url.ParseQuery(strings.ReplaceAll(rawParams, ",", "&")) +} + +func parseNonNegativeAggregateParam(name, value string) (int, error) { + parsed, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("aggregate parameter %q must be an integer", name) + } + if parsed < 0 { + return 0, fmt.Errorf("aggregate parameter %q must be >= 0", name) + } + return parsed, nil +} + +func parseOutputSpec(spec string) (EncoderConfig, SinkConfig, error) { + encoderType, rest, ok := strings.Cut(spec, ":") + if !ok || encoderType == "" || rest == "" { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: expected encoder:sink[:target]", spec) + } + if !supportedEncoderType(encoderType) { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: unsupported encoder %q", spec, encoderType) + } + + var params url.Values + if before, rawParams, ok := strings.Cut(rest, "?"); ok { + if rawParams == "" { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: output parameters cannot be empty", spec) + } + parsed, err := parseCommaQueryParams(rawParams) + if err != nil { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: %w", spec, err) + } + rest = before + params = parsed + } + if rest == "" { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: expected encoder:sink[:target]", spec) + } + + sinkType := rest + target := "" + if first, tail, ok := strings.Cut(rest, ":"); ok { + sinkType = first + target = tail + } + sinkType = normalizeSocketAlias(sinkType) + + sink := SinkConfig{Type: sinkType} + switch sinkType { + case "stdout": + if target != "" { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: stdout does not take a target", spec) + } + case "file": + if target == "" { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: file sink requires a path", spec) + } + sink.Path = target + case "udp": + if target == "" { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: udp sink requires an address", spec) + } + sink.Address = target + case "unixgram": + if target == "" { + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: unixgram sink requires a path", spec) + } + sink.Address = target + default: + return EncoderConfig{}, SinkConfig{}, fmt.Errorf("invalid -output %q: unsupported sink %q", spec, sinkType) + } + + encoder := EncoderConfig{Type: encoderType, Workers: AutoWorkers} + if encoderType == "sflow" { + encoder.AllowTruncate = boolPtr(true) + } + if encoderType == "sflow" || encoderType == "ipfix" || encoderType == "netflowv9" { + encoder.Batch.Enabled = boolPtr(true) + } + if err := applyOutputParams(spec, encoderType, params, &encoder, &sink); err != nil { + return EncoderConfig{}, SinkConfig{}, err + } + return encoder, sink, nil +} + +func applyOutputParams(spec, encoderType string, params url.Values, encoder *EncoderConfig, sink *SinkConfig) error { + for key, values := range params { + if len(values) == 0 { + return fmt.Errorf("invalid -output %q: output parameter %q cannot be empty", spec, key) + } + value := values[len(values)-1] + switch key { + case "allow_truncate": + if encoderType != "sflow" { + return fmt.Errorf("invalid -output %q: output parameter %q is only supported for sflow", spec, key) + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid -output %q: output parameter %q must be a boolean", spec, key) + } + encoder.AllowTruncate = boolPtr(parsed) + case "max_header_bytes": + if encoderType != "sflow" { + return fmt.Errorf("invalid -output %q: output parameter %q is only supported for sflow", spec, key) + } + parsed, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid -output %q: output parameter %q must be an integer", spec, key) + } + if parsed < 0 { + return fmt.Errorf("invalid -output %q: output parameter %q must be >= 0", spec, key) + } + encoder.SFlow.MaxHeaderBytes = parsed + case "batch": + if !supportsBatchOutputParams(encoderType) { + return fmt.Errorf("invalid -output %q: output parameter %q is only supported for sflow, ipfix, and netflowv9", spec, key) + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid -output %q: output parameter %q must be a boolean", spec, key) + } + encoder.Batch.Enabled = boolPtr(parsed) + case "batch_max_records": + if !supportsBatchOutputParams(encoderType) { + return fmt.Errorf("invalid -output %q: output parameter %q is only supported for sflow, ipfix, and netflowv9", spec, key) + } + parsed, err := parseNonNegativeOutputParam(spec, key, value) + if err != nil { + return err + } + encoder.Batch.MaxRecords = parsed + case "batch_max_bytes": + if !supportsBatchOutputParams(encoderType) { + return fmt.Errorf("invalid -output %q: output parameter %q is only supported for sflow, ipfix, and netflowv9", spec, key) + } + parsed, err := parseNonNegativeOutputParam(spec, key, value) + if err != nil { + return err + } + encoder.Batch.MaxBytes = parsed + case "batch_flush_interval_ms": + if !supportsBatchOutputParams(encoderType) { + return fmt.Errorf("invalid -output %q: output parameter %q is only supported for sflow, ipfix, and netflowv9", spec, key) + } + parsed, err := parseNonNegativeOutputParam(spec, key, value) + if err != nil { + return err + } + encoder.Batch.FlushInterval = parsed + case "export_all": + if encoderType != "protobuf" { + return fmt.Errorf("invalid -output %q: output parameter %q is only supported for protobuf", spec, key) + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid -output %q: output parameter %q must be a boolean", spec, key) + } + encoder.Protobuf.ExportAll = parsed + case "resolve_interval_ms": + if sink.Type != "udp" { + return fmt.Errorf("invalid -output %q: output parameter %q is only supported for udp sinks", spec, key) + } + parsed, err := parseNonNegativeOutputParam(spec, key, value) + if err != nil { + return err + } + sink.ResolveIntervalMS = &parsed + default: + return fmt.Errorf("invalid -output %q: unsupported output parameter %q", spec, key) + } + } + return nil +} + +func parseNonNegativeOutputParam(spec, key, value string) (int, error) { + parsed, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("invalid -output %q: output parameter %q must be an integer", spec, key) + } + if parsed < 0 { + return 0, fmt.Errorf("invalid -output %q: output parameter %q must be >= 0", spec, key) + } + return parsed, nil +} + +func supportsBatchOutputParams(encoderType string) bool { + switch encoderType { + case "sflow", "ipfix", "netflowv9": + return true + default: + return false + } +} + +func boolPtr(v bool) *bool { + return &v +} + +func uint32Ptr(v uint32) *uint32 { + return &v +} + +func validateSocketSourceType(sourceType string) error { + switch sourceType { + case "flow", "bytes", "json": + return nil + default: + return fmt.Errorf("socket source type must be flow, bytes, or json") + } +} + +func supportedEncoderType(encoderType string) bool { + switch encoderType { + case "json", "protobuf", "sflow", "ipfix", "netflowv9", "netflowv5", "pcap", "pcapng": + return true + default: + return false + } +} + +func validateUDPListenTarget(target string) error { + if _, err := netip.ParseAddrPort(target); err == nil { + return nil + } + if strings.HasPrefix(target, ":") { + return nil + } + if strings.Contains(target, ":") { + return nil + } + return fmt.Errorf("udp target must include a port") +} + +func normalizeSocketAlias(network string) string { + if network == "socket" { + return "unixgram" + } + return network +} + +func cutLast(s, sep string) (string, string, bool) { + idx := strings.LastIndex(s, sep) + if idx < 0 { + return "", "", false + } + return s[:idx], s[idx+len(sep):], true +} + +func generatedAggregators(flags *FlagConfig) []AggregatorConfig { + if lastAggregatePreset(flags) == "none" { + return nil + } + cfg := defaultGeneratedAggregator() + applyGeneratedAggregatorOverrides(&cfg, flags) + applyAggregationPresetsToAggregator(&cfg, aggregatePresets(flags)) + return []AggregatorConfig{cfg} +} + +func defaultGeneratedAggregator() AggregatorConfig { + return AggregatorConfig{ + Stream: "flow_data", + Match: map[string]string{ + "record_kind": "packet", + }, + AggregateMissing: true, + Window: AggregatorWindowConfig{ + IdleFlushAfter: 10000, + }, + Periodic: AggregatorPeriodicConfig{ + Every: 60000, + }, + Fields: []AggregatorField{ + {Role: "sum", Name: "bytes"}, + {Role: "sum", Name: "packets"}, + {Role: "key", Name: "proto"}, + {Role: "and", Name: "tcp_flags"}, + {Role: "key", Name: "src_port"}, + {Role: "key", Name: "src_addr"}, + {Role: "current", Name: "input_if"}, + {Role: "key", Name: "dst_port"}, + {Role: "key", Name: "dst_addr"}, + {Role: "current", Name: "output_if"}, + {Role: "current", Name: "src_mac"}, + {Role: "current", Name: "vlan_id"}, + {Role: "current", Name: "flow_direction"}, + {Role: "current", Name: "dst_mac"}, + {Role: "current", Name: "agent_ip"}, + {Role: "current", Name: "agent_ipv6"}, + {Role: "key", Name: "source_id"}, + {Role: "first", Name: "start_time_unix"}, + {Role: "current", Name: "end_time_unix"}, + {Role: "current", Name: "ether_type"}, + }, + TemplateID: 256, + } +} + +func defaultSourceOptionsAggregator() AggregatorConfig { + return AggregatorConfig{ + Stream: "options_data", + EmitAs: "data", + Match: map[string]string{ + "kind": "control", + "control.type": "source_init", + }, + Fields: []AggregatorField{ + {Role: "static", Name: "tflow_record_type", Value: "options"}, + {Role: "key", Name: "source_id"}, + {Role: "current", Name: "agent_ip"}, + {Role: "current", Name: "agent_ipv6"}, + {Role: "current", Name: "sampling_rate"}, + {Role: "current", Name: "sample_pool"}, + {Role: "current", Name: "drops"}, + {Role: "current", Name: "input_if"}, + {Role: "current", Name: "output_if"}, + }, + TemplateID: 1024, + } +} + +func generatedSourceOptionsAggregatorEnabled(flags *FlagConfig, encoderType string) bool { + switch encoderType { + case "ipfix", "netflowv9": + default: + return false + } + return lastAggregatePreset(flags) != "none" +} + +func (c *Config) ensureSourceOptionsAggregator() { + if c == nil { + return + } + for _, agg := range c.Aggregators { + if agg.Stream == "options_data" && agg.Match["control.type"] == "source_init" { + return + } + } + c.Aggregators = append(c.Aggregators, defaultSourceOptionsAggregator()) +} + +func applyGeneratedAggregatorOverrides(cfg *AggregatorConfig, flags *FlagConfig) { + if cfg == nil || flags == nil { + return + } + if flags.AggIdleFlushAfter != nil { + cfg.Window.IdleFlushAfter = *flags.AggIdleFlushAfter + } + if flags.AggMaxFlushAfter != nil { + cfg.Window.MaxFlushAfter = *flags.AggMaxFlushAfter + } + if flags.AggIdleEraseAfter != nil { + cfg.Window.IdleEraseAfter = *flags.AggIdleEraseAfter + } + if flags.AggPeriodicEvery != nil { + cfg.Periodic.Every = *flags.AggPeriodicEvery + } + if flags.AggResetBuckets != nil { + cfg.Periodic.ResetBuckets = *flags.AggResetBuckets + } + if flags.AggAggregateMissing != nil { + cfg.AggregateMissing = *flags.AggAggregateMissing + } +} + +// ApplyAggregationFlags applies -agg overlays to an explicit YAML config. +// Presets are useful with -config, while -input/-output remain generated-config +// helpers and are intentionally still rejected with -config. +func (c *Config) ApplyAggregationFlags(flags *FlagConfig) error { + if c == nil || flags == nil || !flags.Aggregate { + return nil + } + if len(c.Aggregators) == 0 && (len(flags.AggPresets) == 0 || hasAggregateOverrides(flags)) { + c.Aggregators = []AggregatorConfig{defaultGeneratedAggregator()} + } + if len(c.Aggregators) > 0 { + applyGeneratedAggregatorOverrides(&c.Aggregators[0], flags) + } + for _, preset := range aggregatePresets(flags) { + switch preset { + case "none": + c.Aggregators = nil + case "passthrough", "payload", "nat", "encap", "mpls", "limited": + agg := c.ensurePrimaryAggregator() + if preset == "limited" { + removePacketParsedFields(agg, + aggregatePresetRequested(flags, "nat"), + aggregatePresetRequested(flags, "encap"), + ) + } else { + applyAggregationPresetsToAggregator(agg, []string{preset}) + } + if preset == "payload" { + c.ensureTemplatedFlowFieldsSelected("frame_length", "header_data") + } + if preset == "nat" { + c.ensureTemplatedFlowFieldsSelected(generatedTemplatedFlowNATFields...) + } + if preset == "encap" { + c.enablePacketDecoderBeyondL4() + c.ensureTemplatedFlowFieldsSelected(generatedTemplatedFlowEncapFields...) + } + if preset == "mpls" { + c.enableMPLSAggregationHelpers() + c.ensureTemplatedFlowFieldsSelected(generatedTemplatedFlowMPLSFields...) + } + if preset == "limited" { + c.removeTemplatedFlowFieldsSelected(packetParsedFieldsToRemove( + aggregatePresetRequested(flags, "nat"), + aggregatePresetRequested(flags, "encap"), + )...) + } + } + } + for i := range c.Aggregators { + if c.Aggregators[i].Stream == "" { + c.Aggregators[i].Stream = "flow_data" + } + if err := validateAggregatorFields(c.Aggregators[i].Fields); err != nil { + return fmt.Errorf("aggregators[%d]: %w", i, err) + } + c.Aggregators[i].Passthrough = !aggregatorNeedsState(&c.Aggregators[i]) + if i == 0 { + applyAggregateMissingFlagDefault(&c.Aggregators[i], flags) + } + if err := validateAggregatorConfig(c.Aggregators[i]); err != nil { + return fmt.Errorf("aggregators[%d]: %w", i, err) + } + } + return nil +} + +func (c *Config) ensurePrimaryAggregator() *AggregatorConfig { + if len(c.Aggregators) == 0 { + c.Aggregators = []AggregatorConfig{defaultGeneratedAggregator()} + } + return &c.Aggregators[0] +} + +func applyAggregationPresetsToAggregator(cfg *AggregatorConfig, presets []string) { + keepIPFields := aggregatePresetSliceContains(presets, "nat") + keepEncapFields := aggregatePresetSliceContains(presets, "encap") + for _, preset := range presets { + switch preset { + case "none": + // Handled at Config level. + case "passthrough": + makeAggregatorPassthrough(cfg) + case "payload": + addPayloadFields(cfg) + case "nat": + addNATFields(cfg) + case "encap": + addEncapFields(cfg) + case "mpls": + addMPLSFields(cfg) + case "limited": + removePacketParsedFields(cfg, keepIPFields, keepEncapFields) + } + } +} + +func makeAggregatorPassthrough(cfg *AggregatorConfig) { + if cfg == nil { + return + } + cfg.Window = AggregatorWindowConfig{} + cfg.Periodic = AggregatorPeriodicConfig{} + cfg.ResetInterval = 0 + cfg.PeriodicInterval = 0 + cfg.AggregateMissing = false + cfg.Sum = nil + cfg.First = nil + cfg.Min = nil + cfg.Max = nil + cfg.And = nil + if len(cfg.Fields) > 0 { + out := cfg.Fields[:0] + for _, field := range cfg.Fields { + switch field.Role { + case "key", "current", "static": + out = append(out, field) + } + } + cfg.Fields = out + } +} + +func applyAggregateMissingFlagDefault(cfg *AggregatorConfig, flags *FlagConfig) { + if cfg == nil || flags == nil || cfg.Passthrough { + return + } + if flags.AggAggregateMissing != nil { + cfg.AggregateMissing = *flags.AggAggregateMissing + return + } + cfg.AggregateMissing = true +} + +func addPayloadFields(cfg *AggregatorConfig) { + if cfg == nil { + return + } + appendUniqueAggregatorField(cfg, AggregatorField{Role: "current", Name: "frame_length"}) + appendUniqueAggregatorField(cfg, AggregatorField{Role: "current", Name: "header_data"}) + appendUniqueString(&cfg.Current, "frame_length") + appendUniqueString(&cfg.Current, "header_data") +} + +func addNATFields(cfg *AggregatorConfig) { + if cfg == nil { + return + } + for _, name := range generatedTemplatedFlowNATFields { + appendUniqueAggregatorField(cfg, AggregatorField{Role: "current", Name: name}) + appendUniqueString(&cfg.Current, name) + } +} + +func addEncapFields(cfg *AggregatorConfig) { + if cfg == nil { + return + } + for _, name := range []string{"outer_proto", "outer_src_port", "outer_src_addr", "outer_dst_port", "outer_dst_addr"} { + appendUniqueAggregatorField(cfg, AggregatorField{Role: "key", Name: name}) + appendUniqueString(&cfg.KeyFields, name) + } + for _, name := range []string{"outer_proto_name", "encap_depth"} { + appendUniqueAggregatorField(cfg, AggregatorField{Role: "current", Name: name}) + appendUniqueString(&cfg.Current, name) + } +} + +func addMPLSFields(cfg *AggregatorConfig) { + if cfg == nil { + return + } + for _, name := range generatedTemplatedFlowMPLSFields { + appendUniqueAggregatorField(cfg, AggregatorField{Role: "current", Name: name}) + appendUniqueString(&cfg.Current, name) + } +} + +func removePacketParsedFields(cfg *AggregatorConfig, keepIPFields, keepEncapFields bool) { + if cfg == nil { + return + } + remove := packetParsedFieldSet(keepIPFields, keepEncapFields) + cfg.Fields = filterAggregatorFieldsByName(cfg.Fields, remove) + cfg.KeyFields = filterStringsBySet(cfg.KeyFields, remove) + cfg.Current = filterStringsBySet(cfg.Current, remove) + cfg.Sum = filterStringsBySet(cfg.Sum, remove) + cfg.First = filterStringsBySet(cfg.First, remove) + cfg.Min = filterStringsBySet(cfg.Min, remove) + cfg.Max = filterStringsBySet(cfg.Max, remove) + for name := range remove { + delete(cfg.StaticFields, name) + } +} + +func appendUniqueAggregatorField(cfg *AggregatorConfig, field AggregatorField) { + for _, existing := range cfg.Fields { + if existing.Role == field.Role && existing.Name == field.Name { + return + } + } + cfg.Fields = append(cfg.Fields, field) +} + +func appendUniqueString(values *[]string, value string) { + for _, existing := range *values { + if existing == value { + return + } + } + *values = append(*values, value) +} + +func (c *Config) enableMPLSAggregationHelpers() { + if c == nil { + return + } + if c.Processor.Builtin.AggregationHelpers.MPLSLabels < len(generatedTemplatedFlowMPLSFields) { + c.Processor.Builtin.AggregationHelpers.MPLSLabels = len(generatedTemplatedFlowMPLSFields) + } +} + +func (c *Config) enablePacketDecoderBeyondL4() { + if c == nil { + return + } + c.Processor.Builtin.PacketDecoder.DecodeBeyondL4 = boolPtr(true) +} + +func (c *Config) ensureTemplatedFlowFieldsSelected(names ...string) { + if c == nil || (c.Encoder.Type != "ipfix" && c.Encoder.Type != "netflowv9") || len(c.Encoder.TemplatedFlow.Data.Select) == 0 { + return + } + for _, name := range names { + appendUniqueString(&c.Encoder.TemplatedFlow.Data.Select, name) + } +} + +func (c *Config) removeTemplatedFlowFieldsSelected(names ...string) { + if c == nil || len(c.Encoder.TemplatedFlow.Data.Select) == 0 { + return + } + remove := make(map[string]struct{}, len(names)) + for _, name := range names { + remove[name] = struct{}{} + } + c.Encoder.TemplatedFlow.Data.Select = filterStringsBySet(c.Encoder.TemplatedFlow.Data.Select, remove) +} + +func aggregatePresets(flags *FlagConfig) []string { + if flags == nil { + return nil + } + return flags.AggPresets +} + +func hasAggregateOverrides(flags *FlagConfig) bool { + return flags != nil && (flags.AggIdleFlushAfter != nil || flags.AggMaxFlushAfter != nil || flags.AggIdleEraseAfter != nil || flags.AggPeriodicEvery != nil || flags.AggResetBuckets != nil || flags.AggAggregateMissing != nil) +} + +func aggregatePresetRequested(flags *FlagConfig, want string) bool { + return aggregatePresetSliceContains(aggregatePresets(flags), want) +} + +func lastAggregatePreset(flags *FlagConfig) string { + presets := aggregatePresets(flags) + if len(presets) == 0 { + return "" + } + return presets[len(presets)-1] +} + +func aggregatePresetSliceContains(presets []string, want string) bool { + for _, preset := range presets { + if preset == want { + return true + } + } + return false +} + +func packetParsedFieldsToRemove(keepIPFields, keepEncapFields bool) []string { + fields := make([]string, 0, len(generatedPacketParsedFlowFields)) + for _, name := range generatedPacketParsedFlowFields { + if keepIPFields { + if _, ok := generatedPacketParsedIPFields[name]; ok { + continue + } + } + if keepEncapFields { + if stringInSlice(generatedTemplatedFlowEncapFields, name) { + continue + } + } + fields = append(fields, name) + } + return fields +} + +func packetParsedFieldSet(keepIPFields, keepEncapFields bool) map[string]struct{} { + fields := packetParsedFieldsToRemove(keepIPFields, keepEncapFields) + out := make(map[string]struct{}, len(fields)) + for _, name := range fields { + out[name] = struct{}{} + } + return out +} + +func stringInSlice(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func filterAggregatorFieldsByName(fields []AggregatorField, remove map[string]struct{}) []AggregatorField { + if len(fields) == 0 || len(remove) == 0 { + return fields + } + out := fields[:0] + for _, field := range fields { + if _, ok := remove[field.Name]; ok { + continue + } + out = append(out, field) + } + return out +} + +func filterStringsBySet(values []string, remove map[string]struct{}) []string { + if len(values) == 0 || len(remove) == 0 { + return values + } + out := values[:0] + for _, value := range values { + if _, ok := remove[value]; ok { + continue + } + out = append(out, value) + } + return out +} + +func validateAggregatorFields(fields []AggregatorField) error { + for _, field := range fields { + if err := validateAggregatorField(field); err != nil { + return err + } + } + return nil +} diff --git a/pkg/reflow/config/config.go b/pkg/reflow/config/config.go new file mode 100644 index 00000000..7d9fce68 --- /dev/null +++ b/pkg/reflow/config/config.go @@ -0,0 +1,1452 @@ +package config + +import ( + "bytes" + _ "embed" + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +//go:embed default-fields.yaml +var defaultFlowFields []byte + +type FlagConfig struct { + ConfigPath string + LogLevel string + LogFormat string + Inputs []string + Output string + OutputSet bool + Aggregate bool + AggIdleFlushAfter *int + AggMaxFlushAfter *int + AggIdleEraseAfter *int + AggPeriodicEvery *int + AggResetBuckets *bool + AggAggregateMissing *bool + AggPresets []string + GenConf bool + GenProto bool + ListOptions bool +} + +type Config struct { + LogLevel string `yaml:"-"` + LogFormat string `yaml:"-"` + Sources []SourceConfig `yaml:"sources"` + TFlow TFlowConfig `yaml:"tflow,omitempty"` + Processor ProcessorConfig `yaml:"processor"` + Aggregators []AggregatorConfig `yaml:"aggregators"` + TemplatedFields TemplatedFieldsConfig `yaml:"templated_fields,omitempty"` + Encoder EncoderConfig `yaml:"encoder"` + Sink SinkConfig `yaml:"sink"` +} + +type CacheConfig struct { + Path string `yaml:"path"` + FlushInterval int `yaml:"flush_interval_ms"` +} + +func (c CacheConfig) IsZero() bool { + return c.Path == "" && c.FlushInterval == 0 +} + +type TFlowConfig struct { + Cache CacheConfig `yaml:"cache,omitempty"` +} + +func (c TFlowConfig) IsZero() bool { + return c.Cache.IsZero() +} + +type SourceConfig struct { + Network string `yaml:"network"` + Address string `yaml:"address"` + Interface string `yaml:"interface"` + SnapLen int `yaml:"snaplen"` + SampleEvery int `yaml:"sample_every"` + SampleOffset int `yaml:"sample_offset"` + InterfaceFilter string `yaml:"interface_filter,omitempty"` + SourceInitRefreshMS *int `yaml:"source_init_refresh_ms,omitempty"` + SourceID *uint32 `yaml:"source_id,omitempty"` + Type string `yaml:"type"` + JSON JSONConfig `yaml:"json"` + EBPF EBPFConfig `yaml:"ebpf,omitempty"` +} + +type EBPFConfig struct { + SKBMetadata *bool `yaml:"skb_metadata,omitempty"` + Conntrack *bool `yaml:"conntrack,omitempty"` + ConntrackPath string `yaml:"conntrack_path,omitempty"` + Direction string `yaml:"direction,omitempty"` +} + +func (c EBPFConfig) IsZero() bool { + return c.SKBMetadata == nil && c.Conntrack == nil && c.ConntrackPath == "" && c.Direction == "" +} + +func (c EBPFConfig) SKBMetadataEnabled() bool { + return c.SKBMetadata == nil || *c.SKBMetadata +} + +func (c EBPFConfig) ConntrackEnabled() bool { + return c.Conntrack == nil || *c.Conntrack +} + +func (c EBPFConfig) DirectionFilter() string { + if c.Direction == "" { + return "both" + } + return c.Direction +} + +type ProcessorConfig struct { + Type string `yaml:"type"` + Workers WorkerCount `yaml:"workers"` + Builtin BuiltinProcessorConfig `yaml:"builtin"` +} + +type WorkerCount int + +const AutoWorkers WorkerCount = -1 + +func (w *WorkerCount) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.ScalarNode { + return fmt.Errorf("workers must be an integer or \"auto\"") + } + if strings.EqualFold(value.Value, "auto") { + *w = AutoWorkers + return nil + } + var count int + if err := value.Decode(&count); err != nil { + return fmt.Errorf("workers must be an integer or \"auto\"") + } + *w = WorkerCount(count) + return nil +} + +func (w WorkerCount) MarshalYAML() (any, error) { + if w == AutoWorkers { + return "auto", nil + } + return int(w), nil +} + +func ResolveProcessorWorkers(workers WorkerCount) int { + if workers == AutoWorkers { + return autoWorkerCount() + } + if workers <= 0 { + return 1 + } + return int(workers) +} + +func ResolveEncoderWorkers(workers WorkerCount, encoderType string) int { + if workers == AutoWorkers { + if orderedEncoderType(encoderType) { + return 1 + } + return autoWorkerCount() + } + if workers <= 0 { + return 1 + } + if orderedEncoderType(encoderType) { + return 1 + } + return int(workers) +} + +func autoWorkerCount() int { + workers := runtime.NumCPU() + if workers < 1 { + return 1 + } + return workers +} + +func orderedEncoderType(encoderType string) bool { + switch encoderType { + case "sflow", "ipfix", "netflowv9", "netflowv5", "pcap", "pcapng": + return true + default: + return false + } +} + +type BuiltinProcessorConfig struct { + DropMessage bool `yaml:"drop_message"` + DropPayload bool `yaml:"drop_payload"` + DisablePacketMapping bool `yaml:"disable_packet_mapping"` + TruncatePacketBytes int `yaml:"truncate_packet_bytes"` + PacketDecoder PacketDecoderConfig `yaml:"packet_decoder"` + TFlow TFlowProcessorConfig `yaml:"tflow"` + AggregationHelpers AggregationHelperConfig `yaml:"aggregation_helpers"` + NAT NATProcessorConfig `yaml:"nat"` +} + +type TFlowProcessorConfig struct { + EmbedTemplateFields bool `yaml:"embed_template_fields"` +} + +type NATProcessorConfig struct { + SwapPrePost bool `yaml:"swap_pre_post"` +} + +type AggregationHelperConfig struct { + MPLSLabels int `yaml:"mpls_labels"` + IPLayers int `yaml:"ip_layers"` +} + +type PacketDecoderConfig struct { + DecodeBeyondL4 *bool `yaml:"decode_beyond_l4"` + Encapsulations PacketEncapsulationConfig `yaml:"encapsulations"` +} + +type PacketEncapsulationConfig struct { + GRE ToggleEncapsulationConfig `yaml:"gre"` + IPIP ToggleEncapsulationConfig `yaml:"ipip"` + VXLAN PortEncapsulationConfig `yaml:"vxlan"` + Geneve PortEncapsulationConfig `yaml:"geneve"` + L2TP PortEncapsulationConfig `yaml:"l2tp"` + GTPU PortEncapsulationConfig `yaml:"gtpu"` + PPPoE ToggleEncapsulationConfig `yaml:"pppoe"` +} + +type PortEncapsulationConfig struct { + Enabled *bool `yaml:"enabled"` + Ports []uint32 `yaml:"ports"` +} + +type ToggleEncapsulationConfig struct { + Enabled *bool `yaml:"enabled"` +} + +type AggregatorConfig struct { + Stream string `yaml:"stream"` + // Passthrough is derived from config. When no stateful rollup is required, + // matching events are forwarded immediately after schema registration. + Passthrough bool `yaml:"-"` + // EmitAs rewrites passthrough events into another pipeline kind. The primary + // use is converting source_init control records into schema-backed data. + EmitAs string `yaml:"emit_as,omitempty"` + // Window controls bucket closure based on activity and age. + Window AggregatorWindowConfig `yaml:"window"` + // Periodic controls snapshot-style exports of current bucket state. + Periodic AggregatorPeriodicConfig `yaml:"periodic"` + // Fields is the preferred ordered field/policy list. Compact entries use: + // role:name or static:name:value. IPFIX/NetFlow field mapping + // stays in encoder.tflow_data. + Fields []AggregatorField `yaml:"fields"` + FieldsConfigured bool `yaml:"-"` + KeyFields []string `yaml:"key_fields"` + // Legacy aggregation policy lists. They remain supported, but no longer + // receive hidden defaults. + Sum []string `yaml:"sum"` + First []string `yaml:"first"` + Current []string `yaml:"current"` + Min []string `yaml:"min"` + Max []string `yaml:"max"` + And []string `yaml:"and"` + Match map[string]string `yaml:"match"` + TemplateID uint16 `yaml:"template_id"` + StaticFields map[string]any `yaml:"static_fields"` + KeyDefaults map[string]any `yaml:"key_defaults"` + // AggregateMissing controls whether events with missing key fields still + // contribute to a bucket. Missing key fields are not added to output events. + AggregateMissing bool `yaml:"aggregate_missing"` + + // Deprecated compatibility knobs. They are still parsed so older configs keep + // loading, then mapped into the explicit window/periodic sections. + ResetInterval int `yaml:"reset_interval_ms"` + PeriodicInterval int `yaml:"periodic_interval_ms"` +} + +func (cfg AggregatorConfig) MarshalYAML() (any, error) { + if len(cfg.Fields) == 0 { + type rawAggregatorConfig AggregatorConfig + return rawAggregatorConfig(cfg), nil + } + return struct { + Stream string `yaml:"stream,omitempty"` + EmitAs string `yaml:"emit_as,omitempty"` + Window AggregatorWindowConfig `yaml:"window,omitempty"` + Periodic AggregatorPeriodicConfig `yaml:"periodic,omitempty"` + Fields []AggregatorField `yaml:"fields"` + Match map[string]string `yaml:"match,omitempty"` + TemplateID uint16 `yaml:"template_id,omitempty"` + KeyDefaults map[string]any `yaml:"key_defaults,omitempty"` + AggregateMissing bool `yaml:"aggregate_missing,omitempty"` + }{ + Stream: cfg.Stream, + EmitAs: cfg.EmitAs, + Window: cfg.Window, + Periodic: cfg.Periodic, + Fields: cfg.Fields, + Match: cfg.Match, + TemplateID: cfg.TemplateID, + KeyDefaults: cfg.KeyDefaults, + AggregateMissing: cfg.AggregateMissing, + }, nil +} + +type AggregatorField struct { + Role string `yaml:"role"` + Name string `yaml:"name"` + Value any `yaml:"value,omitempty"` + + // Path is accepted as a compatibility alias for mapping-style entries. + Path string `yaml:"path,omitempty"` +} + +func (f AggregatorField) MarshalYAML() (any, error) { + if f.Name == "" { + f.Name = f.Path + } + if err := validateAggregatorField(f); err != nil { + return nil, err + } + if f.Role == "static" { + if value, ok := f.Value.(string); ok { + return "static:" + f.Name + ":" + value, nil + } + return struct { + Role string `yaml:"role"` + Name string `yaml:"name"` + Value any `yaml:"value,omitempty"` + }{ + Role: f.Role, + Name: f.Name, + Value: f.Value, + }, nil + } + if family, ok := aggregatorFieldFamilyValue(f.Value); ok { + return f.Role + ":" + f.Name + ":" + family, nil + } + return f.Role + ":" + f.Name, nil +} + +func (f *AggregatorField) UnmarshalYAML(value *yaml.Node) error { + switch value.Kind { + case yaml.ScalarNode: + parsed, err := parseAggregatorField(value.Value) + if err != nil { + return err + } + *f = parsed + return nil + case yaml.MappingNode: + type rawAggregatorField AggregatorField + var raw rawAggregatorField + if err := value.Decode(&raw); err != nil { + return err + } + *f = AggregatorField(raw) + if f.Name == "" { + f.Name = f.Path + } + f.Path = "" + return validateAggregatorField(*f) + default: + return fmt.Errorf("aggregator field must be a string or mapping") + } +} + +type AggregatorWindowConfig struct { + // IdleFlushAfter exports and removes a bucket after this much time without updates. + IdleFlushAfter int `yaml:"idle_flush_after_ms"` + // MaxFlushAfter exports and removes a bucket once it reaches this lifetime. + MaxFlushAfter int `yaml:"max_flush_after_ms"` + // IdleEraseAfter removes stale buckets without exporting them. + IdleEraseAfter int `yaml:"idle_erase_after_ms"` +} + +type AggregatorPeriodicConfig struct { + // Every emits periodic snapshots of current bucket state. + Every int `yaml:"every_ms"` + // ResetBuckets turns periodic export into "emit and clear" instead of "emit snapshot and keep". + ResetBuckets bool `yaml:"reset_buckets"` +} + +type EncoderConfig struct { + Type string `yaml:"type"` + Workers WorkerCount `yaml:"workers"` + MaxDatagramBytes int `yaml:"max_datagram_bytes"` + AllowTruncate *bool `yaml:"allow_truncate"` + Batch BatchConfig `yaml:"batch"` + TemplatedFlow TemplatedFlowConfig `yaml:"templated_flow"` + JSON JSONConfig `yaml:"json"` + Protobuf ProtobufConfig `yaml:"protobuf"` + SFlow SFlowConfig `yaml:"sflow"` + Pcap PcapConfig `yaml:"pcap"` +} + +type TemplatedFlowConfig struct { + TemplateBaseID uint16 `yaml:"template_base_id"` + OptionsTemplateBaseID uint16 `yaml:"options_template_base_id"` + ObservationDomainID uint32 `yaml:"observation_domain_id"` + TemplateRefresh int `yaml:"template_refresh_ms"` + OptionsRefresh int `yaml:"options_refresh_ms"` + Data TemplatedFlowDataConfig `yaml:"data"` +} + +type JSONConfig struct { + Flavor string `yaml:"flavor"` + DropFields []string `yaml:"drop_fields"` +} + +type ProtobufConfig struct { + Flavor string `yaml:"flavor"` + LengthPrefixed bool `yaml:"length_prefixed"` + ExportAll bool `yaml:"export_all"` +} + +type BatchConfig struct { + Enabled *bool `yaml:"enabled"` + MaxRecords int `yaml:"max_records"` + MaxBytes int `yaml:"max_bytes"` + FlushInterval int `yaml:"flush_interval_ms"` +} + +func (c BatchConfig) IsEnabled() bool { + return c.Enabled != nil && *c.Enabled +} + +type SFlowConfig struct { + AgentIP string `yaml:"agent_ip"` + CounterFormat string `yaml:"counter_format"` + EmitExtendedRecords *bool `yaml:"emit_extended_records"` + UseMetadataSequenceNumber bool `yaml:"use_metadata_sequence_number"` + MaxHeaderBytes int `yaml:"max_header_bytes"` + BatchOver SFlowBatchOverConfig `yaml:"batch_over"` +} + +type SFlowBatchOverConfig struct { + AgentIP *bool `yaml:"agent_ip"` + SubAgentID *bool `yaml:"sub_agent_id"` + SequenceNumber *bool `yaml:"sequence_number"` + Uptime *bool `yaml:"uptime"` +} + +type PcapConfig struct { + PacketSource string `yaml:"packet_source"` + LinkType string `yaml:"link_type"` + SnapLen int `yaml:"snaplen"` +} + +type TemplatedFlowDataConfig struct { + Select []string `yaml:"fields"` + Matches []TemplatedFlowFieldMatch `yaml:"matches,omitempty"` + FieldsPath string `yaml:"fields_path"` + Catalog map[string]IPFIXFieldDefinition `yaml:"-"` + Overrides map[string]IPFIXFieldDefinition `yaml:"overrides"` + fieldsPathSet bool +} + +type TemplatedFlowFieldMatch struct { + Field string `yaml:"field"` + From string `yaml:"from"` +} + +type TemplatedFieldsConfig struct { + FieldsPath string `yaml:"fields_path"` + EmbedTemplateFields bool `yaml:"embed_template_fields"` + Catalog map[string]IPFIXFieldDefinition `yaml:"-"` + Overrides map[string]IPFIXFieldDefinition `yaml:"overrides"` + fieldsPathSet bool +} + +func (c TemplatedFieldsConfig) IsZero() bool { + return !c.fieldsPathSet && c.FieldsPath == "" && !c.EmbedTemplateFields && len(c.Overrides) == 0 +} + +func (c *TemplatedFlowDataConfig) UnmarshalYAML(value *yaml.Node) error { + type rawConfig TemplatedFlowDataConfig + var raw rawConfig + if err := value.Decode(&raw); err != nil { + return err + } + *c = TemplatedFlowDataConfig(raw) + c.fieldsPathSet = yamlMappingHasKey(value, "fields_path") + return nil +} + +func (c *TemplatedFieldsConfig) UnmarshalYAML(value *yaml.Node) error { + type rawConfig TemplatedFieldsConfig + var raw rawConfig + if err := value.Decode(&raw); err != nil { + return err + } + *c = TemplatedFieldsConfig(raw) + c.fieldsPathSet = yamlMappingHasKey(value, "fields_path") + return nil +} + +func yamlMappingHasKey(value *yaml.Node, key string) bool { + if value == nil || value.Kind != yaml.MappingNode { + return false + } + for i := 0; i+1 < len(value.Content); i += 2 { + if value.Content[i].Value == key { + return true + } + } + return false +} + +type IPFIXFieldDefinition struct { + Name string `yaml:"name"` + ID uint16 `yaml:"id"` + PEN uint32 `yaml:"pen"` + Length uint16 `yaml:"length"` + Type string `yaml:"type"` + Format string `yaml:"format"` + EnterpriseScoped bool `yaml:"enterprise_scoped"` +} + +func (d *IPFIXFieldDefinition) UnmarshalYAML(value *yaml.Node) error { + type rawDefinition IPFIXFieldDefinition + if value.Kind != yaml.ScalarNode { + var raw rawDefinition + if err := value.Decode(&raw); err != nil { + return err + } + *d = IPFIXFieldDefinition(raw) + return nil + } + + var compact string + if err := value.Decode(&compact); err != nil { + return err + } + def, err := parseCompactFieldDefinition(compact) + if err != nil { + return err + } + *d = def + return nil +} + +func parseCompactFieldDefinition(raw string) (IPFIXFieldDefinition, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return IPFIXFieldDefinition{}, fmt.Errorf("empty field definition") + } + + var penInfo string + if open := strings.LastIndex(raw, "["); open >= 0 { + if !strings.HasSuffix(raw, "]") { + return IPFIXFieldDefinition{}, fmt.Errorf("invalid field definition %q: missing closing bracket", raw) + } + penInfo = strings.TrimSpace(raw[open+1 : len(raw)-1]) + raw = strings.TrimSpace(raw[:open]) + } + + parts := strings.Split(raw, ":") + if len(parts) < 3 || len(parts) > 4 { + return IPFIXFieldDefinition{}, fmt.Errorf("invalid field definition %q: expected id:length:type[:format]", raw) + } + id, err := parseUint16Part(parts[0], "id") + if err != nil { + return IPFIXFieldDefinition{}, err + } + length, err := parseUint16Part(parts[1], "length") + if err != nil { + return IPFIXFieldDefinition{}, err + } + def := IPFIXFieldDefinition{ + ID: id, + Length: length, + Type: expandFieldTypeAlias(parts[2]), + } + if len(parts) == 4 { + def.Format = strings.TrimSpace(parts[3]) + } + if penInfo != "" { + pen, enterpriseScoped, err := parsePENInfo(penInfo) + if err != nil { + return IPFIXFieldDefinition{}, err + } + def.PEN = pen + def.EnterpriseScoped = enterpriseScoped + } + return def, nil +} + +func parseUint16Part(raw, name string) (uint16, error) { + v, err := strconv.ParseUint(strings.TrimSpace(raw), 0, 16) + if err != nil { + return 0, fmt.Errorf("invalid %s %q: %w", name, raw, err) + } + return uint16(v), nil +} + +func expandFieldTypeAlias(raw string) string { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "u8": + return "unsigned8" + case "u16": + return "unsigned16" + case "u32": + return "unsigned32" + case "u64": + return "unsigned64" + case "s8": + return "signed8" + case "s16": + return "signed16" + case "s32": + return "signed32" + case "s64": + return "signed64" + case "ip4", "ipv4": + return "ipv4Address" + case "ip6", "ipv6": + return "ipv6Address" + case "mac": + return "macAddress" + case "str": + return "string" + default: + return strings.TrimSpace(raw) + } +} + +func parsePENInfo(raw string) (uint32, bool, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return 0, false, nil + } + enterpriseScoped := true + if key, val, ok := strings.Cut(raw, "="); ok { + switch strings.ToLower(strings.TrimSpace(key)) { + case "pen": + raw = val + case "enterprise", "enterprise_scoped": + enterpriseScoped = strings.TrimSpace(val) != "false" + return 0, enterpriseScoped, nil + default: + return 0, false, fmt.Errorf("invalid PEN info key %q", key) + } + } + pen, err := strconv.ParseUint(strings.TrimSpace(raw), 0, 32) + if err != nil { + return 0, false, fmt.Errorf("invalid PEN info %q: %w", raw, err) + } + return uint32(pen), enterpriseScoped, nil +} + +type SinkConfig struct { + Type string `yaml:"type"` + Path string `yaml:"path"` + Address string `yaml:"address"` + ResolveIntervalMS *int `yaml:"resolve_interval_ms,omitempty"` + Framing string `yaml:"framing"` + Mode string `yaml:"mode"` +} + +// BindFlags defines CLI bootstrap flags. When -config is omitted, helper flags +// generate an in-memory config and pass it through the same defaults/validation. +func BindFlags(fs *flag.FlagSet) (*FlagConfig, *bool) { + cfg := &FlagConfig{} + version := fs.Bool("v", false, "Print version") + fs.StringVar(&cfg.ConfigPath, "config", "", "Path to ReFlow YAML config") + fs.StringVar(&cfg.LogLevel, "loglevel", "info", "Log level") + fs.StringVar(&cfg.LogFormat, "logfmt", "text", "Log format (text or json)") + fs.Var((*inputFlags)(&cfg.Inputs), "input", "Input helper spec network:target:type (repeatable)") + out := outputFlag{cfg: cfg} + fs.Var(out, "output", "Output helper spec encoder:sink[:target]") + fs.Var(out, "o", "Output helper spec encoder:sink[:target]") + fs.Var(aggregateFlag{cfg: cfg}, "agg", "Generate packet aggregation config or apply aggregation presets") + fs.BoolVar(&cfg.GenConf, "genconf", false, "Print generated config and exit") + fs.BoolVar(&cfg.GenProto, "genproto", false, "Print ReFlow protobuf definition and exit") + fs.BoolVar(&cfg.ListOptions, "list-options", false, "List helper -input/-output/-agg options and exit") + return cfg, version +} + +// Load reads, decodes, defaults, and validates the runtime config in one step. +func Load(path string) (*Config, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("load config %s: %w", path, err) + } + return LoadBytes(path, raw) +} + +// LoadBytes decodes, defaults, and validates a runtime config from in-memory YAML. +func LoadBytes(name string, raw []byte) (*Config, error) { + cfg := &Config{} + if err := yaml.Unmarshal(raw, cfg); err != nil { + return nil, fmt.Errorf("decode config %s: %w", name, err) + } + if err := cfg.setDefaults(name); err != nil { + return nil, err + } + return cfg, nil +} + +// setDefaults applies runtime defaults after YAML decoding so the rest of the +// code can treat the config as normalized. +func (c *Config) setDefaults(configPath string) error { + if len(c.Sources) == 0 { + return fmt.Errorf("sources must contain at least one source") + } + assignDefaultSourceIDs(c.Sources) + for i := range c.Sources { + if err := applySourceDefaults(&c.Sources[i]); err != nil { + return fmt.Errorf("sources[%d]: %w", i, err) + } + } + if c.Processor.Type == "" { + c.Processor.Type = "builtin" + } + if c.Processor.Workers == 0 { + c.Processor.Workers = 1 + } else if c.Processor.Workers < 0 && c.Processor.Workers != AutoWorkers { + return fmt.Errorf("processor.workers must be >= 1 or \"auto\"") + } + if c.Processor.Builtin.TruncatePacketBytes < 0 { + return fmt.Errorf("processor.builtin.truncate_packet_bytes must be >= 0") + } + if c.Processor.Builtin.AggregationHelpers.MPLSLabels < 0 { + return fmt.Errorf("processor.builtin.aggregation_helpers.mpls_labels must be >= 0") + } + if c.Processor.Builtin.AggregationHelpers.IPLayers < 0 { + return fmt.Errorf("processor.builtin.aggregation_helpers.ip_layers must be >= 0") + } + if err := validatePacketDecoderConfig(c.Processor.Builtin.PacketDecoder); err != nil { + return fmt.Errorf("processor.builtin.packet_decoder: %w", err) + } + if len(c.Aggregators) > 0 { + for i := range c.Aggregators { + applyAggregatorCompatibility(&c.Aggregators[i]) + if err := normalizeAggregatorConfig(&c.Aggregators[i]); err != nil { + return fmt.Errorf("aggregators[%d]: %w", i, err) + } + if err := validateAggregatorConfig(c.Aggregators[i]); err != nil { + return fmt.Errorf("aggregators[%d]: %w", i, err) + } + } + } + if err := c.applyCacheDefaults(configPath); err != nil { + return err + } + if err := c.loadFlowDataCatalog(configPath); err != nil { + return err + } + if err := validateTemplatedFlowDataMatches(c.Encoder.TemplatedFlow.Data); err != nil { + return err + } + if c.Encoder.Type == "" { + c.Encoder.Type = "json" + } + if c.Encoder.Workers == 0 { + c.Encoder.Workers = 1 + } else if c.Encoder.Workers < 0 && c.Encoder.Workers != AutoWorkers { + return fmt.Errorf("encoder.workers must be >= 1 or \"auto\"") + } + if c.Encoder.AllowTruncate == nil { + v := c.Encoder.Type == "sflow" + c.Encoder.AllowTruncate = &v + } + if c.Encoder.Batch.IsEnabled() { + if c.Encoder.Batch.MaxRecords == 0 { + c.Encoder.Batch.MaxRecords = 32 + } + if c.Encoder.Batch.MaxBytes == 0 { + c.Encoder.Batch.MaxBytes = 1200 + } + if c.Encoder.Batch.FlushInterval == 0 { + c.Encoder.Batch.FlushInterval = 1000 + } + } + if c.Encoder.MaxDatagramBytes <= 0 { + c.Encoder.MaxDatagramBytes = 1400 + if c.Encoder.Batch.IsEnabled() && c.Encoder.Batch.MaxBytes > 0 { + c.Encoder.MaxDatagramBytes = c.Encoder.Batch.MaxBytes + } + } + if (c.Encoder.Type == "ipfix" || c.Encoder.Type == "netflowv9") && c.Encoder.TemplatedFlow.TemplateBaseID == 0 { + c.Encoder.TemplatedFlow.TemplateBaseID = 256 + } + if (c.Encoder.Type == "ipfix" || c.Encoder.Type == "netflowv9") && c.Encoder.TemplatedFlow.OptionsTemplateBaseID == 0 { + c.Encoder.TemplatedFlow.OptionsTemplateBaseID = 1024 + } + if c.Encoder.TemplatedFlow.TemplateRefresh < 0 { + return fmt.Errorf("encoder.templated_flow.template_refresh_ms must be >= 0") + } + if c.Encoder.TemplatedFlow.OptionsRefresh < 0 { + return fmt.Errorf("encoder.templated_flow.options_refresh_ms must be >= 0") + } + if (c.Encoder.Type == "ipfix" || c.Encoder.Type == "netflowv9") && c.Encoder.TemplatedFlow.TemplateRefresh == 0 { + c.Encoder.TemplatedFlow.TemplateRefresh = 60000 + } + if (c.Encoder.Type == "ipfix" || c.Encoder.Type == "netflowv9") && c.Encoder.TemplatedFlow.OptionsRefresh == 0 { + c.Encoder.TemplatedFlow.OptionsRefresh = 30000 + } + defaultTrue(&c.Encoder.SFlow.BatchOver.AgentIP) + defaultTrue(&c.Encoder.SFlow.BatchOver.SubAgentID) + defaultTrue(&c.Encoder.SFlow.BatchOver.SequenceNumber) + defaultTrue(&c.Encoder.SFlow.BatchOver.Uptime) + defaultTrue(&c.Encoder.SFlow.EmitExtendedRecords) + if c.Encoder.SFlow.MaxHeaderBytes < 0 { + return fmt.Errorf("encoder.sflow.max_header_bytes must be >= 0") + } + if c.Encoder.Type == "sflow" && c.Encoder.SFlow.MaxHeaderBytes == 0 { + c.Encoder.SFlow.MaxHeaderBytes = 128 + } + switch c.Encoder.SFlow.CounterFormat { + case "", "standard": + c.Encoder.SFlow.CounterFormat = "standard" + case "expanded": + default: + return fmt.Errorf("unsupported encoder.sflow.counter_format %q", c.Encoder.SFlow.CounterFormat) + } + switch c.Encoder.Protobuf.Flavor { + case "", "canonical": + c.Encoder.Protobuf.Flavor = "canonical" + case "goflow2v2": + default: + return fmt.Errorf("unsupported encoder.protobuf.flavor %q", c.Encoder.Protobuf.Flavor) + } + if c.Encoder.Batch.MaxRecords < 0 { + return fmt.Errorf("encoder.batch.max_records must be >= 0") + } + if c.Encoder.Batch.MaxBytes < 0 { + return fmt.Errorf("encoder.batch.max_bytes must be >= 0") + } + if c.Encoder.Batch.FlushInterval < 0 { + return fmt.Errorf("encoder.batch.flush_interval_ms must be >= 0") + } + switch c.Encoder.Pcap.PacketSource { + case "", "auto": + c.Encoder.Pcap.PacketSource = "auto" + case "header_data", "payload", "pseudo": + default: + return fmt.Errorf("unsupported encoder.pcap.packet_source %q", c.Encoder.Pcap.PacketSource) + } + switch c.Encoder.Pcap.LinkType { + case "", "ethernet": + c.Encoder.Pcap.LinkType = "ethernet" + case "raw", "ipv4", "ipv6": + default: + return fmt.Errorf("unsupported encoder.pcap.link_type %q", c.Encoder.Pcap.LinkType) + } + if c.Encoder.Pcap.SnapLen < 0 { + return fmt.Errorf("encoder.pcap.snaplen must be >= 0") + } + if c.Encoder.Pcap.SnapLen == 0 { + c.Encoder.Pcap.SnapLen = 65535 + } + switch c.Encoder.Type { + case "json", "protobuf", "sflow", "ipfix", "netflowv9", "netflowv5", "pcap", "pcapng": + default: + return fmt.Errorf("unsupported encoder.type %q", c.Encoder.Type) + } + if c.Sink.Type == "" { + c.Sink.Type = "stdout" + } + if (c.Encoder.Type == "pcap" || c.Encoder.Type == "pcapng") && (c.Sink.Type == "udp" || c.Sink.Type == "unixgram") { + return fmt.Errorf("encoder.type=%s requires a stream sink, got sink.type=%s", c.Encoder.Type, c.Sink.Type) + } + switch c.Sink.Type { + case "stdout", "file", "udp", "unixgram": + default: + return fmt.Errorf("unsupported sink.type %q", c.Sink.Type) + } + if c.Sink.Framing == "" { + if c.Encoder.Type == "pcap" || c.Encoder.Type == "pcapng" { + c.Sink.Framing = "none" + } else { + c.Sink.Framing = "line" + } + } + switch c.Sink.Framing { + case "line", "none": + default: + return fmt.Errorf("unsupported sink.framing %q", c.Sink.Framing) + } + if c.Sink.Mode == "" { + if (c.Encoder.Type == "pcap" || c.Encoder.Type == "pcapng") && c.Sink.Type == "file" { + c.Sink.Mode = "truncate" + } else { + c.Sink.Mode = "append" + } + } + switch c.Sink.Mode { + case "append", "truncate": + default: + return fmt.Errorf("unsupported sink.mode %q", c.Sink.Mode) + } + if c.Sink.Type == "file" && c.Sink.Path == "" { + return fmt.Errorf("sink.path is required when sink.type=file") + } + if (c.Sink.Type == "udp" || c.Sink.Type == "unixgram") && c.Sink.Address == "" { + return fmt.Errorf("sink.address is required when sink.type=%s", c.Sink.Type) + } + if c.Sink.Type == "udp" && c.Sink.ResolveIntervalMS == nil { + v := 60000 + c.Sink.ResolveIntervalMS = &v + } + if c.Sink.ResolveIntervalMS != nil { + if *c.Sink.ResolveIntervalMS < 0 { + return fmt.Errorf("sink.resolve_interval_ms must be >= 0") + } + if c.Sink.Type != "udp" { + return fmt.Errorf("sink.resolve_interval_ms is only supported when sink.type=udp") + } + } + return nil +} + +func (c *Config) applyCacheDefaults(configPath string) error { + cache := &c.TFlow.Cache + if cache.FlushInterval < 0 { + return fmt.Errorf("tflow.cache.flush_interval_ms must be >= 0") + } + if cache.Path == "" { + return nil + } + if cache.FlushInterval == 0 { + cache.FlushInterval = 10000 + } + if !filepath.IsAbs(cache.Path) { + cache.Path = filepath.Join(filepath.Dir(configPath), cache.Path) + } + return nil +} + +func assignDefaultSourceIDs(sources []SourceConfig) { + var next uint32 + for i := range sources { + if sources[i].SourceID != nil { + next = *sources[i].SourceID + 1 + continue + } + if !sourceUsesConfiguredObservationPoint(sources[i]) { + continue + } + sources[i].SourceID = uint32Ptr(next) + next++ + } +} + +func sourceUsesConfiguredObservationPoint(src SourceConfig) bool { + switch src.Network { + case "pcap_live", "ebpf": + return true + case "stream": + return src.Type == "pcap" || src.Type == "pcapng" + default: + return false + } +} + +func validatePacketDecoderConfig(cfg PacketDecoderConfig) error { + if err := validateUDPPorts("encapsulations.vxlan.ports", cfg.Encapsulations.VXLAN.Ports); err != nil { + return err + } + if err := validateUDPPorts("encapsulations.geneve.ports", cfg.Encapsulations.Geneve.Ports); err != nil { + return err + } + if err := validateUDPPorts("encapsulations.l2tp.ports", cfg.Encapsulations.L2TP.Ports); err != nil { + return err + } + if err := validateUDPPorts("encapsulations.gtpu.ports", cfg.Encapsulations.GTPU.Ports); err != nil { + return err + } + return nil +} + +func validateUDPPorts(name string, ports []uint32) error { + for _, port := range ports { + if port > 65535 { + return fmt.Errorf("%s contains invalid UDP port %d", name, port) + } + } + return nil +} + +// applySourceDefaults normalizes per-source defaults that depend on source.network. +func applySourceDefaults(src *SourceConfig) error { + if src.SourceInitRefreshMS == nil { + v := 30000 + src.SourceInitRefreshMS = &v + } + if *src.SourceInitRefreshMS < 0 { + return fmt.Errorf("source.source_init_refresh_ms must be >= 0") + } + if src.Network == "" { + src.Network = "udp" + } + if src.Network == "stream" { + if src.Address == "" { + src.Address = "-" + } + switch src.Type { + case "pcap", "pcapng", "json", "bytes", "flow": + case "": + return fmt.Errorf("source.type is required when source.network=stream") + default: + return fmt.Errorf("unsupported source.type %q for source.network=stream", src.Type) + } + return nil + } + if src.Network != "pcap_live" && src.Network != "ebpf" && src.Address == "" { + src.Address = ":18080" + } + if src.Network == "pcap_live" || src.Network == "ebpf" { + if src.Interface == "" { + return fmt.Errorf("source.interface is required when source.network=%s", src.Network) + } + if src.SnapLen <= 0 { + src.SnapLen = 65535 + } + if src.SampleEvery <= 0 { + src.SampleEvery = 1 + } + if src.SampleOffset < 0 || src.SampleOffset >= src.SampleEvery { + return fmt.Errorf("source.sample_offset must be >= 0 and < source.sample_every") + } + if src.InterfaceFilter != "" { + if _, err := regexp.Compile(src.InterfaceFilter); err != nil { + return fmt.Errorf("source.interface_filter is invalid: %w", err) + } + } + if src.Address == "" { + src.Address = src.Interface + } + if src.Type == "" { + src.Type = "bytes" + } + if src.Network == "ebpf" && src.Type != "bytes" { + return fmt.Errorf("source.type must be bytes when source.network=ebpf") + } + if src.Network == "ebpf" { + direction, err := NormalizeEBPFDirection(src.EBPF.Direction) + if err != nil { + return err + } + src.EBPF.Direction = direction + } + } + return nil +} + +func NormalizeEBPFDirection(raw string) (string, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "", "both", "all": + return "both", nil + case "ingress", "in", "input": + return "ingress", nil + case "egress", "out", "output": + return "egress", nil + default: + return "", fmt.Errorf("unsupported source.ebpf.direction %q", raw) + } +} + +// defaultTrue fills optional boolean pointers with true so config can distinguish +// "unset" from an explicit false. +func defaultTrue(dst **bool) { + if *dst != nil { + return + } + v := true + *dst = &v +} + +func defaultFalse(dst **bool) { + if *dst != nil { + return + } + v := false + *dst = &v +} + +// loadFlowDataCatalog resolves the shared templated flow field catalog. Empty +// fields_path uses the embedded default catalog; explicit paths are resolved +// relative to config. Legacy encoder.templated_flow.data catalog settings keep +// their previous behavior when the top-level shared catalog is not configured. +func (c *Config) loadFlowDataCatalog(configPath string) error { + sharedConfigured := c.TemplatedFields.fieldsPathSet || c.TemplatedFields.FieldsPath != "" || len(c.TemplatedFields.Overrides) > 0 || len(c.TemplatedFields.Catalog) > 0 + if sharedConfigured { + fields, err := loadIPFIXFieldCatalog(configPath, &c.TemplatedFields.FieldsPath, "templated_fields", !c.TemplatedFields.fieldsPathSet) + if err != nil { + return err + } + shared := mergeIPFIXFields(fields, c.TemplatedFields.Catalog, c.TemplatedFields.Overrides) + if err := validateTemplatedDecodeCatalog(shared); err != nil { + return fmt.Errorf("templated_fields: %w", err) + } + var exportFields map[string]IPFIXFieldDefinition + if c.Encoder.TemplatedFlow.Data.fieldsPathSet { + exportFields, err = loadIPFIXFieldCatalog(configPath, &c.Encoder.TemplatedFlow.Data.FieldsPath, "encoder.templated_flow.data", false) + if err != nil { + return err + } + } + c.TemplatedFields.Catalog = shared + c.Encoder.TemplatedFlow.Data.Catalog = mergeIPFIXFields(shared, exportFields, c.Encoder.TemplatedFlow.Data.Catalog, c.Encoder.TemplatedFlow.Data.Overrides) + return nil + } + + fields, err := loadIPFIXFieldCatalog(configPath, &c.Encoder.TemplatedFlow.Data.FieldsPath, "encoder.templated_flow.data", true) + if err != nil { + return err + } + legacy := mergeIPFIXFields(fields, c.Encoder.TemplatedFlow.Data.Catalog, c.Encoder.TemplatedFlow.Data.Overrides) + if err := validateTemplatedDecodeCatalog(legacy); err != nil { + return fmt.Errorf("encoder.templated_flow.data: %w", err) + } + c.TemplatedFields.Catalog = legacy + c.Encoder.TemplatedFlow.Data.Catalog = legacy + return nil +} + +func loadIPFIXFieldCatalog(configPath string, fieldsPath *string, label string, emptyPathDefault bool) (map[string]IPFIXFieldDefinition, error) { + type ipfixCatalog struct { + Fields map[string]IPFIXFieldDefinition `yaml:"fields"` + } + + raw := defaultFlowFields + source := "embedded default flow fields" + if (fieldsPath == nil || *fieldsPath == "") && !emptyPathDefault { + return map[string]IPFIXFieldDefinition{}, nil + } + if fieldsPath != nil && *fieldsPath != "" { + if !filepath.IsAbs(*fieldsPath) { + *fieldsPath = filepath.Join(filepath.Dir(configPath), *fieldsPath) + } + source = *fieldsPath + var err error + raw, err = os.ReadFile(*fieldsPath) + if err != nil { + return nil, fmt.Errorf("load %s fields %s: %w", label, source, err) + } + if len(bytes.TrimSpace(raw)) == 0 { + raw = defaultFlowFields + source = "embedded default flow fields" + } + } + + catalog := ipfixCatalog{} + if err := yaml.Unmarshal(raw, &catalog); err != nil { + return nil, fmt.Errorf("decode %s fields %s: %w", label, source, err) + } + return catalog.Fields, nil +} + +// mergeIPFIXFields applies later catalogs over earlier ones. +func mergeIPFIXFields(sources ...map[string]IPFIXFieldDefinition) map[string]IPFIXFieldDefinition { + merged := make(map[string]IPFIXFieldDefinition) + for _, source := range sources { + for key, def := range source { + merged[key] = def + } + } + return merged +} + +func validateTemplatedFlowDataMatches(cfg TemplatedFlowDataConfig) error { + for i, match := range cfg.Matches { + if match.Field == "" { + return fmt.Errorf("encoder.templated_flow.data.matches[%d].field is required", i) + } + if match.From == "" { + return fmt.Errorf("encoder.templated_flow.data.matches[%d].from is required", i) + } + if _, ok := cfg.Catalog[match.Field]; !ok { + return fmt.Errorf("encoder.templated_flow.data.matches[%d].field %q is not defined in templated field catalog", i, match.Field) + } + } + return nil +} + +type decodeIPFIXKey struct { + id uint16 + pen uint32 +} + +func validateTemplatedDecodeCatalog(catalog map[string]IPFIXFieldDefinition) error { + ipfix := make(map[decodeIPFIXKey]string) + netflowV9 := make(map[uint16]string) + for name, def := range catalog { + for _, key := range ipfixDecodeKeys(name, def) { + if existing, ok := ipfix[key]; ok && existing != name { + return fmt.Errorf("duplicate IPFIX decode mapping id=%d pen=%d for %q and %q", key.id, key.pen, existing, name) + } + ipfix[key] = name + } + for _, id := range netflowV9DecodeIDs(name, def) { + if existing, ok := netflowV9[id]; ok && existing != name { + return fmt.Errorf("duplicate NetFlow v9 decode mapping id=%d for %q and %q", id, existing, name) + } + netflowV9[id] = name + } + } + return nil +} + +func ipfixDecodeKeys(name string, def IPFIXFieldDefinition) []decodeIPFIXKey { + if def.ID == 0 { + return nil + } + pen := uint32(0) + if def.EnterpriseScoped || def.PEN != 0 { + pen = def.PEN + } + keys := []decodeIPFIXKey{{id: def.ID, pen: pen}} + switch name { + case "src_addr": + keys = append(keys, decodeIPFIXKey{id: 27}) + case "dst_addr": + keys = append(keys, decodeIPFIXKey{id: 28}) + } + return keys +} + +func netflowV9DecodeIDs(name string, def IPFIXFieldDefinition) []uint16 { + ids := make([]uint16, 0, 2) + if def.ID != 0 && def.PEN == 0 && !def.EnterpriseScoped { + ids = append(ids, def.ID) + } + switch name { + case "src_addr": + ids = append(ids, 27) + case "dst_addr": + ids = append(ids, 28) + case "start_time_unix": + ids = append(ids, 21) + case "end_time_unix": + ids = append(ids, 22) + } + return ids +} + +// normalizeAggregatorConfig applies defaults that do not change aggregation +// semantics and translates legacy field lists into the preferred field DSL. +func normalizeAggregatorConfig(cfg *AggregatorConfig) error { + if cfg.Stream == "" { + cfg.Stream = "flow_data" + } + + cfg.FieldsConfigured = len(cfg.Fields) > 0 + if !cfg.FieldsConfigured { + for _, field := range cfg.KeyFields { + cfg.Fields = append(cfg.Fields, AggregatorField{Role: "key", Name: field}) + } + for _, field := range cfg.Sum { + cfg.Fields = append(cfg.Fields, AggregatorField{Role: "sum", Name: field}) + } + for _, field := range cfg.First { + cfg.Fields = append(cfg.Fields, AggregatorField{Role: "first", Name: field}) + } + for _, field := range cfg.Current { + cfg.Fields = append(cfg.Fields, AggregatorField{Role: "current", Name: field}) + } + for _, field := range cfg.Min { + cfg.Fields = append(cfg.Fields, AggregatorField{Role: "min", Name: field}) + } + for _, field := range cfg.Max { + cfg.Fields = append(cfg.Fields, AggregatorField{Role: "max", Name: field}) + } + for _, field := range cfg.And { + cfg.Fields = append(cfg.Fields, AggregatorField{Role: "and", Name: field}) + } + staticFields := make([]string, 0, len(cfg.StaticFields)) + for field := range cfg.StaticFields { + staticFields = append(staticFields, field) + } + sort.Strings(staticFields) + for _, field := range staticFields { + cfg.Fields = append(cfg.Fields, AggregatorField{Role: "static", Name: field, Value: cfg.StaticFields[field]}) + } + } else { + cfg.KeyFields = nil + cfg.Sum = nil + cfg.First = nil + cfg.Current = nil + cfg.Min = nil + cfg.Max = nil + cfg.And = nil + cfg.StaticFields = nil + for _, field := range cfg.Fields { + if err := validateAggregatorField(field); err != nil { + return err + } + switch field.Role { + case "key": + cfg.KeyFields = append(cfg.KeyFields, field.Name) + case "sum": + cfg.Sum = append(cfg.Sum, field.Name) + case "first": + cfg.First = append(cfg.First, field.Name) + case "current": + cfg.Current = append(cfg.Current, field.Name) + case "min": + cfg.Min = append(cfg.Min, field.Name) + case "max": + cfg.Max = append(cfg.Max, field.Name) + case "and": + cfg.And = append(cfg.And, field.Name) + case "static": + if cfg.StaticFields == nil { + cfg.StaticFields = make(map[string]any) + } + cfg.StaticFields[field.Name] = field.Value + } + } + } + + cfg.Passthrough = !aggregatorNeedsState(cfg) + return nil +} + +// applyAggregatorCompatibility maps legacy knobs into the explicit window and +// periodic sections used by the current runtime. +func applyAggregatorCompatibility(cfg *AggregatorConfig) { + if cfg.Window.IdleFlushAfter == 0 && cfg.ResetInterval > 0 { + cfg.Window.IdleFlushAfter = cfg.ResetInterval + } + if cfg.Periodic.Every == 0 && cfg.PeriodicInterval > 0 { + cfg.Periodic.Every = cfg.PeriodicInterval + } +} + +// validateAggregatorConfig rejects timer combinations that would never emit data +// or would make periodic reset semantics ambiguous. +func validateAggregatorConfig(cfg AggregatorConfig) error { + if cfg.Window.IdleFlushAfter < 0 { + return fmt.Errorf("aggregator.window.idle_flush_after_ms must be >= 0") + } + if cfg.Window.MaxFlushAfter < 0 { + return fmt.Errorf("aggregator.window.max_flush_after_ms must be >= 0") + } + if cfg.Window.IdleEraseAfter < 0 { + return fmt.Errorf("aggregator.window.idle_erase_after_ms must be >= 0") + } + if cfg.Periodic.Every < 0 { + return fmt.Errorf("aggregator.periodic.every_ms must be >= 0") + } + if len(cfg.KeyDefaults) > 0 { + keyFields := make(map[string]struct{}, len(cfg.KeyFields)) + for _, field := range cfg.KeyFields { + keyFields[field] = struct{}{} + } + for field := range cfg.KeyDefaults { + if _, ok := keyFields[field]; !ok { + return fmt.Errorf("aggregator.key_defaults.%s requires matching key field", field) + } + } + } + if cfg.Passthrough { + if cfg.EmitAs == "" || cfg.EmitAs == "data" { + return nil + } + return fmt.Errorf("aggregator.emit_as must be \"data\" when set") + } + if cfg.EmitAs != "" { + return fmt.Errorf("aggregator.emit_as is only supported for passthrough aggregators") + } + if cfg.Periodic.ResetBuckets && cfg.Periodic.Every == 0 { + return fmt.Errorf("aggregator.periodic.reset_buckets requires aggregator.periodic.every_ms > 0") + } + if cfg.Window.IdleFlushAfter == 0 && cfg.Window.MaxFlushAfter == 0 && cfg.Periodic.Every == 0 { + return fmt.Errorf("aggregator requires at least one export trigger: window.idle_flush_after_ms, window.max_flush_after_ms, or periodic.every_ms") + } + return nil +} + +func parseAggregatorField(raw string) (AggregatorField, error) { + if raw == "" { + return AggregatorField{}, fmt.Errorf("aggregator field entry cannot be empty") + } + if strings.HasPrefix(raw, "static:") { + parts := strings.SplitN(raw, ":", 3) + if len(parts) != 3 || parts[1] == "" { + return AggregatorField{}, fmt.Errorf("invalid static field entry %q", raw) + } + field := AggregatorField{Role: "static", Name: parts[1], Value: parts[2]} + return field, validateAggregatorField(field) + } + + parts := strings.Split(raw, ":") + if len(parts) != 2 && len(parts) != 3 { + return AggregatorField{}, fmt.Errorf("invalid aggregator field entry %q", raw) + } + field := AggregatorField{ + Role: parts[0], + Name: parts[1], + } + if len(parts) == 3 { + field.Value = parts[2] + } + return field, validateAggregatorField(field) +} + +func validateAggregatorField(field AggregatorField) error { + switch field.Role { + case "key", "sum", "first", "current", "min", "max", "and", "static": + default: + return fmt.Errorf("unsupported aggregator field role %q", field.Role) + } + if field.Name == "" { + return fmt.Errorf("aggregator field name is required for role %q", field.Role) + } + if field.Role == "static" { + return nil + } + if field.Value == nil { + return nil + } + value, ok := field.Value.(string) + if !ok { + return fmt.Errorf("aggregator field value is only supported for static fields or IP family modifiers") + } + if _, ok := aggregatorFieldFamilyValue(value); !ok { + return fmt.Errorf("unsupported aggregator field family %q", value) + } + return nil +} + +func aggregatorFieldFamilyValue(value any) (string, bool) { + family, ok := value.(string) + if !ok { + return "", false + } + switch family { + case "ip4", "ip6", "ip4in6": + return family, true + default: + return "", false + } +} + +func aggregatorNeedsState(cfg *AggregatorConfig) bool { + hasCurrent := false + for _, field := range cfg.Fields { + switch field.Role { + case "sum", "first", "min", "max", "and": + return true + case "current": + hasCurrent = true + } + } + return hasCurrent && aggregatorHasExportTrigger(*cfg) +} + +func aggregatorHasExportTrigger(cfg AggregatorConfig) bool { + return cfg.Window.IdleFlushAfter > 0 || cfg.Window.MaxFlushAfter > 0 || cfg.Periodic.Every > 0 +} diff --git a/pkg/reflow/config/config_bench_test.go b/pkg/reflow/config/config_bench_test.go new file mode 100644 index 00000000..a0d90a93 --- /dev/null +++ b/pkg/reflow/config/config_bench_test.go @@ -0,0 +1,110 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func BenchmarkLoadConfig(b *testing.B) { + path := filepath.Join(b.TempDir(), "reflow.yaml") + if err := os.WriteFile(path, []byte(benchmarkConfigYAML), 0o600); err != nil { + b.Fatalf("write benchmark config: %v", err) + } + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := Load(path); err != nil { + b.Fatalf("Load returned error: %v", err) + } + } +} + +const benchmarkConfigYAML = ` +sources: + - network: stream + address: "-" + type: bytes + +processor: + type: builtin + workers: 2 + builtin: + drop_message: true + drop_payload: false + disable_packet_mapping: false + aggregation_helpers: + mpls_labels: 3 + ip_layers: 2 + +aggregators: + - stream: flow_data + window: + idle_flush_after_ms: 10000 + periodic: + every_ms: 60000 + key_fields: + - src_addr + - dst_addr + - proto + - src_port + - dst_port + template_id: 256 + sum: + - bytes + - packets + current: + - input_if + - output_if + - end_time_unix + +encoder: + type: json + workers: 1 + json: + flavor: canonical + +sink: + type: stdout +` + +func BenchmarkLoadFromFlags(b *testing.B) { + benchmarks := []struct { + name string + flags *FlagConfig + }{ + { + name: "stream_pcap_to_json", + flags: &FlagConfig{ + Inputs: []string{"stream:fixtures/input.pcap:pcap"}, + Output: "json:stdout", + }, + }, + { + name: "stream_pcap_aggregate_to_ipfix", + flags: &FlagConfig{ + Inputs: []string{"stream:fixtures/input.pcap:pcap"}, + Output: "ipfix:udp:127.0.0.1:4739", + Aggregate: true, + }, + }, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + cfg, generated, err := LoadFromFlags(bm.flags) + if err != nil { + b.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + b.Fatalf("expected generated config") + } + if len(cfg.Sources) == 0 { + b.Fatalf("expected at least one source") + } + } + }) + } +} diff --git a/pkg/reflow/config/config_test.go b/pkg/reflow/config/config_test.go new file mode 100644 index 00000000..c34d1865 --- /dev/null +++ b/pkg/reflow/config/config_test.go @@ -0,0 +1,2842 @@ +package config + +import ( + "flag" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestLoadSetsAggregatorDefaultsAndLoadsFlowDataFields(t *testing.T) { + dir := t.TempDir() + + ipfixPath := filepath.Join(dir, "fields.yaml") + if err := os.WriteFile(ipfixPath, []byte(` +fields: + bytes: 1:8:u64:delta + custom_counter: + name: customCounter + id: 1000 + pen: 32473 + enterprise_scoped: true + length: 8 + type: unsigned64 + compact_enterprise: 4000:4:u32[pen=64513] +`), 0o644); err != nil { + t.Fatalf("write ipfix fields: %v", err) + } + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + json: + flavor: reflow + +processor: + type: builtin + builtin: + packet_decoder: + decode_beyond_l4: false + encapsulations: + gre: + enabled: true + ipip: + enabled: true + vxlan: + enabled: false + ports: [4789, 4790] + geneve: + enabled: true + ports: [6081] + l2tp: + enabled: true + ports: [1701] + gtpu: + enabled: true + ports: [2152] + pppoe: + enabled: false + aggregation_helpers: + mpls_labels: 3 + ip_layers: 2 + nat: + swap_pre_post: true + +aggregators: + - window: + idle_flush_after_ms: 5000 + template_id: 256 + static_fields: + exporter_name: reflow-test + +encoder: + type: json + templated_flow: + data: + fields_path: fields.yaml + overrides: + custom_counter: + name: customCounterOverride + id: 2000 + pen: 64512 + enterprise_scoped: true + length: 8 + type: unsigned64 + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + decoder := cfg.Processor.Builtin.PacketDecoder + if decoder.DecodeBeyondL4 == nil || *decoder.DecodeBeyondL4 { + t.Fatalf("expected processor packet decoder decode_beyond_l4=false, got %#v", decoder.DecodeBeyondL4) + } + if decoder.Encapsulations.VXLAN.Enabled == nil || *decoder.Encapsulations.VXLAN.Enabled { + t.Fatalf("expected VXLAN encapsulation enabled=false, got %#v", decoder.Encapsulations.VXLAN.Enabled) + } + if len(decoder.Encapsulations.VXLAN.Ports) != 2 || decoder.Encapsulations.VXLAN.Ports[1] != 4790 { + t.Fatalf("expected VXLAN ports [4789 4790], got %#v", decoder.Encapsulations.VXLAN.Ports) + } + if cfg.Processor.Builtin.AggregationHelpers.MPLSLabels != 3 { + t.Fatalf("expected aggregation helper mpls_labels=3, got %d", cfg.Processor.Builtin.AggregationHelpers.MPLSLabels) + } + if !cfg.Processor.Builtin.NAT.SwapPrePost { + t.Fatalf("expected processor.builtin.nat.swap_pre_post=true") + } + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected 1 aggregator, got %d", len(cfg.Aggregators)) + } + if cfg.Aggregators[0].Window.IdleFlushAfter != 5000 { + t.Fatalf("expected aggregators[0].window.idle_flush_after_ms=5000, got %d", cfg.Aggregators[0].Window.IdleFlushAfter) + } + if !cfg.Aggregators[0].Passthrough { + t.Fatalf("expected aggregators[0] without aggregation fields to use pass-through schema mode") + } + custom := cfg.Encoder.TemplatedFlow.Data.Catalog["custom_counter"] + if custom.ID != 2000 || custom.PEN != 64512 { + t.Fatalf("expected override for custom_counter to win, got %#v", custom) + } + compact := cfg.Encoder.TemplatedFlow.Data.Catalog["compact_enterprise"] + if compact.ID != 4000 || compact.Length != 4 || compact.Type != "unsigned32" || compact.PEN != 64513 || !compact.EnterpriseScoped { + t.Fatalf("expected compact enterprise field to load, got %#v", compact) + } + if cfg.Aggregators[0].TemplateID != 256 { + t.Fatalf("expected aggregators[0].template_id=256, got %d", cfg.Aggregators[0].TemplateID) + } + if cfg.Aggregators[0].StaticFields["exporter_name"] != "reflow-test" { + t.Fatalf("expected static field exporter_name to be loaded, got %#v", cfg.Aggregators[0].StaticFields["exporter_name"]) + } +} + +func TestLoadSupportsTFlowCacheConfig(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +tflow: + cache: + path: reflow-cache.json + flush_interval_ms: 250 + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.TFlow.Cache.Path != filepath.Join(dir, "reflow-cache.json") { + t.Fatalf("expected cache path relative to config dir, got %q", cfg.TFlow.Cache.Path) + } + if cfg.TFlow.Cache.FlushInterval != 250 { + t.Fatalf("expected cache flush interval 250, got %d", cfg.TFlow.Cache.FlushInterval) + } +} + +func TestLoadDefaultsCacheFlushIntervalWhenPathSet(t *testing.T) { + cfg, err := LoadBytes("reflow.yaml", []byte(` +sources: + - network: udp + type: flow + +tflow: + cache: + path: reflow-cache.json + +encoder: + type: json + +sink: + type: stdout +`)) + if err != nil { + t.Fatalf("LoadBytes returned error: %v", err) + } + if cfg.TFlow.Cache.FlushInterval != 10000 { + t.Fatalf("expected default cache flush interval 10000, got %d", cfg.TFlow.Cache.FlushInterval) + } +} + +func TestLoadBytesMatchesLoad(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + raw := []byte(` +sources: + - network: stream + type: json + json: + flavor: reflow + +processor: + type: builtin + +encoder: + type: json + +sink: + type: stdout +`) + if err := os.WriteFile(cfgPath, raw, 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + fromFile, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + fromBytes, err := LoadBytes(cfgPath, raw) + if err != nil { + t.Fatalf("LoadBytes returned error: %v", err) + } + if fromBytes.Sources[0].Network != fromFile.Sources[0].Network || + fromBytes.Sources[0].Type != fromFile.Sources[0].Type || + fromBytes.Encoder.Type != fromFile.Encoder.Type || + fromBytes.Sink.Type != fromFile.Sink.Type { + t.Fatalf("LoadBytes did not match Load: %#v vs %#v", fromBytes, fromFile) + } +} + +func TestLoadSupportsStreamBytesAndFlowSources(t *testing.T) { + for _, sourceType := range []string{"bytes", "flow"} { + t.Run(sourceType, func(t *testing.T) { + _, err := LoadBytes("browser.yaml", []byte(` +sources: + - network: stream + type: `+sourceType+` + +processor: + type: builtin + +encoder: + type: json + +sink: + type: stdout +`)) + if err != nil { + t.Fatalf("LoadBytes returned error: %v", err) + } + }) + } +} + +func TestLoadUsesEmbeddedFlowDataCatalogWhenFieldsPathEmpty(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +encoder: + type: ipfix + templated_flow: + data: + matches: + - field: source_id + from: tid + overrides: + custom_counter: + name: customCounter + id: 2000 + length: 8 + type: unsigned64 + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Encoder.TemplatedFlow.Data.FieldsPath != "" { + t.Fatalf("expected empty fields_path to keep using embedded catalog, got %q", cfg.Encoder.TemplatedFlow.Data.FieldsPath) + } + if cfg.Encoder.TemplatedFlow.Data.Catalog["bytes"].ID != 1 { + t.Fatalf("expected bytes field from embedded catalog, got %#v", cfg.Encoder.TemplatedFlow.Data.Catalog["bytes"]) + } + mpls := cfg.Encoder.TemplatedFlow.Data.Catalog["mpls_label_stack_section_3"] + if mpls.ID != 72 || mpls.Length != 3 || mpls.Type != "bytes" { + t.Fatalf("expected mpls_label_stack_section_3 bytes/3 field from embedded catalog, got %#v", mpls) + } + headerData := cfg.Encoder.TemplatedFlow.Data.Catalog["header_data"] + if headerData.ID != 315 || headerData.Length != 65535 || headerData.Type != "bytes" { + t.Fatalf("expected header_data variable dataLinkFrameSection field from embedded catalog, got %#v", headerData) + } + interfaceName := cfg.Encoder.TemplatedFlow.Data.Catalog["interface_name"] + if interfaceName.ID != 82 || interfaceName.Type != "string" { + t.Fatalf("expected embedded interface_name interfaceName field, got %#v", interfaceName) + } + if cfg.Encoder.TemplatedFlow.Data.Catalog["custom_counter"].ID != 2000 { + t.Fatalf("expected override to merge over embedded catalog, got %#v", cfg.Encoder.TemplatedFlow.Data.Catalog["custom_counter"]) + } + if len(cfg.Encoder.TemplatedFlow.Data.Matches) != 1 || + cfg.Encoder.TemplatedFlow.Data.Matches[0].Field != "source_id" || + cfg.Encoder.TemplatedFlow.Data.Matches[0].From != "tid" { + t.Fatalf("expected source_id match from tid, got %#v", cfg.Encoder.TemplatedFlow.Data.Matches) + } + + emptyCatalog := filepath.Join(dir, "empty-fields.yaml") + if err := os.WriteFile(emptyCatalog, nil, 0o644); err != nil { + t.Fatalf("write empty catalog: %v", err) + } + emptyPathCfg := filepath.Join(dir, "empty-file.yaml") + if err := os.WriteFile(emptyPathCfg, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +encoder: + type: ipfix + templated_flow: + data: + fields_path: empty-fields.yaml + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write empty-file config: %v", err) + } + emptyFile, err := Load(emptyPathCfg) + if err != nil { + t.Fatalf("Load empty file returned error: %v", err) + } + if emptyFile.Encoder.TemplatedFlow.Data.Catalog["bytes"].ID != 1 { + t.Fatalf("expected empty catalog file to use embedded bytes field, got %#v", emptyFile.Encoder.TemplatedFlow.Data.Catalog["bytes"]) + } + if got := emptyFile.Encoder.TemplatedFlow.Data.Catalog["agent_ipv6"]; got.ID != 131 || got.Length != 16 || got.Type != "ipv6Address" { + t.Fatalf("expected embedded agent_ipv6 exporter IPv6 field, got %#v", got) + } +} + +func TestLoadRejectsUnknownTemplatedFlowDataMatchField(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "match.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +encoder: + type: ipfix + templated_flow: + data: + matches: + - field: not_in_catalog + from: tid + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load(cfgPath) + if err == nil || !strings.Contains(err.Error(), `field "not_in_catalog" is not defined`) { + t.Fatalf("expected unknown match field error, got %v", err) + } +} + +func TestLoadSupportsSharedTemplatedFieldsCatalog(t *testing.T) { + dir := t.TempDir() + fieldsPath := filepath.Join(dir, "shared-fields.yaml") + if err := os.WriteFile(fieldsPath, []byte(` +fields: + bytes: 1:8:u64:delta + custom_counter: + name: customCounter + id: 1000 + pen: 32473 + enterprise_scoped: true + length: 8 + type: unsigned64 + if_index: + id: 10 + length: 4 + type: unsigned32 +`), 0o644); err != nil { + t.Fatalf("write fields: %v", err) + } + exportFieldsPath := filepath.Join(dir, "export-fields.yaml") + if err := os.WriteFile(exportFieldsPath, []byte(` +fields: + file_export_only: + id: 3000 + length: 4 + type: unsigned32 +`), 0o644); err != nil { + t.Fatalf("write export fields: %v", err) + } + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +templated_fields: + fields_path: shared-fields.yaml + overrides: + custom_counter: + id: 1001 + pen: 32473 + enterprise_scoped: true + length: 8 + type: unsigned64 + +encoder: + type: ipfix + templated_flow: + data: + fields_path: export-fields.yaml + fields: + - bytes + overrides: + export_only: + id: 2000 + length: 4 + type: unsigned32 + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + shared := cfg.TemplatedFields.Catalog["custom_counter"] + if shared.ID != 1001 || shared.PEN != 32473 { + t.Fatalf("expected top-level override to apply to shared catalog, got %#v", shared) + } + if sharedScope := cfg.TemplatedFields.Catalog["if_index"]; sharedScope.ID != 10 { + t.Fatalf("expected if_index to load in shared catalog, got %#v", sharedScope) + } + if _, ok := cfg.TemplatedFields.Catalog["export_only"]; ok { + t.Fatalf("expected encoder-only override not to change shared decode catalog") + } + if _, ok := cfg.TemplatedFields.Catalog["file_export_only"]; ok { + t.Fatalf("expected encoder-only fields_path not to change shared decode catalog") + } + if cfg.Encoder.TemplatedFlow.Data.Catalog["file_export_only"].ID != 3000 { + t.Fatalf("expected encoder-only fields_path in export catalog, got %#v", cfg.Encoder.TemplatedFlow.Data.Catalog["file_export_only"]) + } + if len(cfg.Encoder.TemplatedFlow.Data.Select) != 1 || cfg.Encoder.TemplatedFlow.Data.Select[0] != "bytes" { + t.Fatalf("expected export field selection to remain export-only, got %#v", cfg.Encoder.TemplatedFlow.Data.Select) + } +} + +func TestLoadSupportsEmbedTemplateFields(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + builtin: + tflow: + embed_template_fields: true + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if !cfg.Processor.Builtin.TFlow.EmbedTemplateFields { + t.Fatalf("expected processor.builtin.tflow.embed_template_fields=true") + } +} + +func TestLoadTreatsExplicitEmptySharedTemplatedFieldsPathAsEmptyCatalog(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +templated_fields: + fields_path: "" + overrides: + custom_counter: + id: 1000 + pen: 32473 + enterprise_scoped: true + length: 8 + type: unsigned64 + +encoder: + type: ipfix + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if _, ok := cfg.TemplatedFields.Catalog["bytes"]; ok { + t.Fatalf("expected explicit empty templated_fields.fields_path not to load embedded defaults") + } + if cfg.TemplatedFields.Catalog["custom_counter"].ID != 1000 { + t.Fatalf("expected override to remain in shared catalog, got %#v", cfg.TemplatedFields.Catalog["custom_counter"]) + } +} + +func TestLoadRejectsDuplicateTemplatedDecodeFields(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +templated_fields: + overrides: + custom_bytes: + id: 1 + length: 8 + type: unsigned64 + +encoder: + type: ipfix + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := Load(cfgPath); err == nil || !strings.Contains(err.Error(), "duplicate IPFIX decode mapping") { + t.Fatalf("expected duplicate IPFIX decode mapping error, got %v", err) + } +} + +func TestLoadSupportsAccumulativeAggregatorWithExplicitFields(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + json: + flavor: reflow + +processor: + type: builtin + +aggregators: + - periodic: + every_ms: 60000 + fields: + - key:agent_ip + - current:sampling_rate + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected 1 aggregator, got %d", len(cfg.Aggregators)) + } + if cfg.Aggregators[0].Periodic.Every != 60000 { + t.Fatalf("expected periodic.every_ms=60000, got %d", cfg.Aggregators[0].Periodic.Every) + } + if cfg.Aggregators[0].Passthrough { + t.Fatalf("expected current field with export trigger to force aggregate mode") + } + if len(cfg.Aggregators[0].KeyFields) != 1 || cfg.Aggregators[0].KeyFields[0] != "agent_ip" { + t.Fatalf("expected key field agent_ip, got %#v", cfg.Aggregators[0].KeyFields) + } + if len(cfg.Aggregators[0].Current) != 1 || cfg.Aggregators[0].Current[0] != "sampling_rate" { + t.Fatalf("expected current field sampling_rate, got %#v", cfg.Aggregators[0].Current) + } + if cfg.Aggregators[0].Stream != "flow_data" { + t.Fatalf("expected default stream=flow_data, got %q", cfg.Aggregators[0].Stream) + } +} + +func TestLoadParsesAggregatorFieldDSL(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - fields: + - key:src_addr + - key:dst_addr + - current:tenant_id + - static:exporter_name:edge-a + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + agg := cfg.Aggregators[0] + if !agg.Passthrough { + t.Fatalf("expected key/current/static fields without export triggers to use pass-through schema mode") + } + if len(agg.Fields) != 4 { + t.Fatalf("expected 4 parsed fields, got %#v", agg.Fields) + } + if got := agg.Fields[0]; got.Role != "key" || got.Name != "src_addr" { + t.Fatalf("unexpected first field: %#v", got) + } + if got := agg.Fields[2]; got.Role != "current" || got.Name != "tenant_id" { + t.Fatalf("unexpected tenant field: %#v", got) + } + if agg.StaticFields["exporter_name"] != "edge-a" { + t.Fatalf("expected static exporter_name edge-a, got %#v", agg.StaticFields["exporter_name"]) + } +} + +func TestLoadParsesAggregatorAndField(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - periodic: + every_ms: 1000 + fields: + - key:src_addr + - and:tcp_flags + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + agg := cfg.Aggregators[0] + if agg.Passthrough { + t.Fatalf("expected and field to force aggregate mode") + } + if !aggregatorHasField(agg, "and", "tcp_flags") { + t.Fatalf("expected and:tcp_flags, got %#v", agg.Fields) + } + if len(agg.And) != 1 || agg.And[0] != "tcp_flags" { + t.Fatalf("expected And list to contain tcp_flags, got %#v", agg.And) + } +} + +func TestLoadParsesAggregatorAggregateMissing(t *testing.T) { + cfg, err := LoadBytes("reflow.yaml", []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - periodic: + every_ms: 1000 + fields: + - key:src_addr + - key:dst_addr + - sum:bytes + aggregate_missing: true + +encoder: + type: json + +sink: + type: stdout +`)) + if err != nil { + t.Fatalf("LoadBytes returned error: %v", err) + } + if !cfg.Aggregators[0].AggregateMissing { + t.Fatalf("expected aggregate_missing=true") + } +} + +func TestLoadDefaultsAggregatorAggregateMissingFalse(t *testing.T) { + cfg, err := LoadBytes("reflow.yaml", []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - periodic: + every_ms: 1000 + fields: + - key:dst_addr + - sum:bytes + +encoder: + type: json + +sink: + type: stdout +`)) + if err != nil { + t.Fatalf("LoadBytes returned error: %v", err) + } + if cfg.Aggregators[0].AggregateMissing { + t.Fatalf("expected aggregate_missing default false") + } +} + +func TestLoadRejectsPlainAggregatorFieldRole(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - fields: + - key:src_addr + - field:bytes + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := Load(cfgPath); err == nil { + t.Fatalf("expected Load to reject plain field role") + } +} + +func TestLoadAcceptsAggregatorFieldFamilyModifier(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - fields: + - key:src_addr:ip4 + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if got := cfg.Aggregators[0].Fields[0].Value; got != "ip4" { + t.Fatalf("expected field value ip4, got %q", got) + } +} + +func TestLoadRejectsInvalidAggregatorFieldFamilyModifier(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - fields: + - key:src_addr:ipv4 + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := Load(cfgPath); err == nil { + t.Fatalf("expected Load to reject invalid field family modifier") + } +} + +func TestLoadSupportsStreamPcapAndPcapSinkDefaults(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: stream + address: "-" + type: pcap + +processor: + type: builtin + +encoder: + type: pcap + +sink: + type: file + path: out.pcap +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Sources[0].Address != "-" { + t.Fatalf("expected stream stdin address, got %q", cfg.Sources[0].Address) + } + if cfg.Encoder.Pcap.PacketSource != "auto" { + t.Fatalf("expected default packet_source auto, got %q", cfg.Encoder.Pcap.PacketSource) + } + if cfg.Encoder.Pcap.LinkType != "ethernet" { + t.Fatalf("expected default pcap link_type ethernet, got %q", cfg.Encoder.Pcap.LinkType) + } + if cfg.Encoder.Pcap.SnapLen != 65535 { + t.Fatalf("expected default pcap snaplen 65535, got %d", cfg.Encoder.Pcap.SnapLen) + } + if cfg.Sink.Framing != "none" { + t.Fatalf("expected pcap sink framing none, got %q", cfg.Sink.Framing) + } + if cfg.Sink.Mode != "truncate" { + t.Fatalf("expected pcap file sink mode truncate, got %q", cfg.Sink.Mode) + } +} + +func TestLoadAssignsSourceIDsByLocalObservationPointOrder(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: stream + address: first.pcap + type: pcap + - network: stream + address: events.ndjson + type: json + - network: stream + address: second.pcapng + type: pcapng + - network: pcap_live + interface: lo0 + type: bytes + source_id: 9 + - network: ebpf + interface: eth0 + type: bytes +processor: + type: builtin +encoder: + type: ipfix +sink: + type: file + path: out.ipfix +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Sources[0].SourceID == nil || *cfg.Sources[0].SourceID != 0 { + t.Fatalf("expected first stream pcap source_id 0, got %#v", cfg.Sources[0].SourceID) + } + if cfg.Sources[1].SourceID != nil { + t.Fatalf("expected stream json not to receive source_id, got %#v", cfg.Sources[1].SourceID) + } + if cfg.Sources[2].SourceID == nil || *cfg.Sources[2].SourceID != 1 { + t.Fatalf("expected stream pcapng source_id 1, got %#v", cfg.Sources[2].SourceID) + } + if cfg.Sources[3].SourceID == nil || *cfg.Sources[3].SourceID != 9 { + t.Fatalf("expected explicit pcap_live source_id 9, got %#v", cfg.Sources[3].SourceID) + } + if cfg.Sources[4].SourceID == nil || *cfg.Sources[4].SourceID != 10 { + t.Fatalf("expected ebpf source_id 10 after explicit id, got %#v", cfg.Sources[4].SourceID) + } +} + +func TestLoadRejectsPcapEncoderWithDatagramSink(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: stream + address: "-" + type: pcap + +processor: + type: builtin + +encoder: + type: pcap + +sink: + type: udp + address: "127.0.0.1:9000" +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := Load(cfgPath); err == nil { + t.Fatalf("expected pcap encoder with UDP sink to be rejected") + } +} + +func TestLoadRejectsPcapNGEncoderWithDatagramSink(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: stream + address: "-" + type: pcapng + +processor: + type: builtin + +encoder: + type: pcapng + +sink: + type: udp + address: "127.0.0.1:9000" +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := Load(cfgPath); err == nil { + t.Fatalf("expected pcapng encoder with UDP sink to be rejected") + } +} + +func TestLoadSupportsPcapNGEncoderDefaults(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: stream + address: "-" + type: pcapng + +processor: + type: builtin + +encoder: + type: pcapng + +sink: + type: file + path: out.pcapng +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Encoder.Pcap.LinkType != "ethernet" { + t.Fatalf("expected default pcapng link_type ethernet, got %q", cfg.Encoder.Pcap.LinkType) + } + if cfg.Sink.Framing != "none" { + t.Fatalf("expected pcapng sink framing none, got %q", cfg.Sink.Framing) + } + if cfg.Sink.Mode != "truncate" { + t.Fatalf("expected pcapng file sink mode truncate, got %q", cfg.Sink.Mode) + } +} + +func TestLoadAllowsPassthroughAggregatorWithoutExportTrigger(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - fields: + - key:src_addr + - current:bytes + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if !cfg.Aggregators[0].Passthrough { + t.Fatalf("expected aggregator without stateful rollup to use pass-through schema mode") + } +} + +func TestLoadSupportsAggregatorList(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - stream: agg_samples + periodic: + every_ms: 1000 + match: + record_kind: packet + - stream: agg_counters + periodic: + every_ms: 1000 + match: + record_kind: interface_counter + current: + - if_in_octets + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if len(cfg.Aggregators) != 2 { + t.Fatalf("expected 2 aggregators, got %d", len(cfg.Aggregators)) + } + if cfg.Aggregators[0].Stream != "agg_samples" { + t.Fatalf("expected first stream agg_samples, got %q", cfg.Aggregators[0].Stream) + } + if cfg.Aggregators[0].Match["record_kind"] != "packet" { + t.Fatalf("expected first match record_kind=packet, got %#v", cfg.Aggregators[0].Match) + } + if cfg.Aggregators[1].Stream != "agg_counters" { + t.Fatalf("expected second stream agg_counters, got %q", cfg.Aggregators[1].Stream) + } + if cfg.Aggregators[1].Match["record_kind"] != "interface_counter" { + t.Fatalf("expected second match record_kind=interface_counter, got %#v", cfg.Aggregators[1].Match) + } +} + +func TestGeneratedAggregateConfigUsesFieldDSL(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Aggregate: true, + GenConf: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected one generated aggregator, got %d", len(cfg.Aggregators)) + } + agg := cfg.Aggregators[0] + if agg.Match["record_kind"] != "packet" { + t.Fatalf("expected generated aggregator to match packets, got %#v", agg.Match) + } + if agg.TemplateID != 256 { + t.Fatalf("expected generated template_id 256, got %d", agg.TemplateID) + } + if !agg.AggregateMissing { + t.Fatalf("expected generated -agg config to default aggregate_missing=true") + } + for _, field := range []struct { + role string + name string + }{ + {"key", "src_addr"}, + {"sum", "bytes"}, + {"and", "tcp_flags"}, + {"current", "src_mac"}, + {"current", "flow_direction"}, + {"current", "agent_ipv6"}, + } { + if !aggregatorHasField(agg, field.role, field.name) { + t.Fatalf("expected generated aggregate field %s:%s, got %#v", field.role, field.name, agg.Fields) + } + } + raw, err := yaml.Marshal(cfg) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + out := string(raw) + for _, unwanted := range []string{"key_fields:", "static_fields:", "reset_interval_ms:", "periodic_interval_ms:"} { + if strings.Contains(out, unwanted) { + t.Fatalf("expected generated config not to contain %q:\n%s", unwanted, out) + } + } +} + +func TestGeneratedAggregateConfigSupportsCLIParams(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{ + "-agg=idle_flush_after_ms=0,max_flush_after_ms=45000,periodic_every_ms=15000,reset_buckets=true,aggregate_missing=false", + "-genconf", + }); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected generated aggregators, got %d", len(cfg.Aggregators)) + } + agg := cfg.Aggregators[0] + if agg.Match["record_kind"] != "packet" { + t.Fatalf("match = %#v, want record_kind=packet", agg.Match) + } + if agg.Window.IdleFlushAfter != 0 { + t.Fatalf("idle flush = %d", agg.Window.IdleFlushAfter) + } + if agg.Window.MaxFlushAfter != 45000 { + t.Fatalf("max flush = %d", agg.Window.MaxFlushAfter) + } + if agg.Periodic.Every != 15000 { + t.Fatalf("periodic every = %d", agg.Periodic.Every) + } + if !agg.Periodic.ResetBuckets { + t.Fatalf("expected reset_buckets=true") + } + if agg.AggregateMissing { + t.Fatalf("expected aggregate_missing=false") + } +} + +func TestNormalizeAggregateArgsSupportsSpaceSeparatedValue(t *testing.T) { + tests := []struct { + name string + args []string + want []string + }{ + { + name: "space separated aggregate preset", + args: []string{"-agg", "passthrough,payload", "-genconf"}, + want: []string{"-agg=passthrough,payload", "-genconf"}, + }, + { + name: "bare aggregate remains boolean shorthand", + args: []string{"-agg", "-genconf"}, + want: []string{"-agg", "-genconf"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NormalizeAggregateArgs(tt.args) + if !slices.Equal(got, tt.want) { + t.Fatalf("NormalizeAggregateArgs() = %#v, want %#v", got, tt.want) + } + }) + } +} + +func TestGeneratedAggregateConfigSupportsSpaceSeparatedPresets(t *testing.T) { + tests := []struct { + name string + preset string + checkFunc func(t *testing.T, cfg *Config) + }{ + { + name: "passthrough payload", + preset: "passthrough,payload", + checkFunc: func(t *testing.T, cfg *Config) { + t.Helper() + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected generated aggregator, got %d", len(cfg.Aggregators)) + } + agg := cfg.Aggregators[0] + if !agg.Passthrough { + t.Fatalf("expected passthrough preset to use schema passthrough") + } + if !aggregatorHasField(agg, "current", "frame_length") || !aggregatorHasField(agg, "current", "header_data") { + t.Fatalf("expected payload fields, got %#v", agg.Fields) + } + }, + }, + { + name: "none", + preset: "none", + checkFunc: func(t *testing.T, cfg *Config) { + t.Helper() + if len(cfg.Aggregators) != 0 { + t.Fatalf("expected no generated aggregators, got %d", len(cfg.Aggregators)) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse(NormalizeAggregateArgs([]string{"-agg", tt.preset, "-genconf"})); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + tt.checkFunc(t, cfg) + }) + } +} + +func TestGeneratedIPFIXConfigExcludesDataLinkFrameByDefault(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=ipfix:udp:127.0.0.1:4739", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + fields := cfg.Encoder.TemplatedFlow.Data.Select + if len(fields) == 0 { + t.Fatalf("expected generated ipfix config to select explicit fields") + } + for _, name := range []string{"header_data", "nat_src_addr", "mpls_label_stack_section_1"} { + if slices.Contains(fields, name) { + t.Fatalf("expected generated ipfix config not to export %s by default: %#v", name, fields) + } + } + for _, name := range []string{"src_addr", "bytes", "source_id", "flow_direction"} { + if !slices.Contains(fields, name) { + t.Fatalf("expected generated ipfix config to export %s: %#v", name, fields) + } + } +} + +func TestGeneratedTemplatedOutputAddsSourceOptionsAggregator(t *testing.T) { + for _, output := range []string{ + "ipfix:udp:127.0.0.1:4739", + "netflowv9:udp:127.0.0.1:2055", + } { + t.Run(output, func(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=" + output, "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + agg := sourceOptionsAggregator(t, cfg) + if !agg.Passthrough { + t.Fatalf("expected source options aggregator to be passthrough") + } + if agg.EmitAs != "data" { + t.Fatalf("expected emit_as=data, got %q", agg.EmitAs) + } + if agg.TemplateID != 1024 { + t.Fatalf("expected source options template 1024, got %d", agg.TemplateID) + } + if agg.Match["kind"] != "control" || agg.Match["control.type"] != "source_init" { + t.Fatalf("unexpected source options match: %#v", agg.Match) + } + if !aggregatorHasField(agg, "key", "source_id") || !aggregatorHasField(agg, "current", "sampling_rate") { + t.Fatalf("expected source options sampling fields, got %#v", agg.Fields) + } + }) + } +} + +func TestGeneratedTemplatedOutputAggNoneDisablesSourceOptionsAggregator(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=ipfix:udp:127.0.0.1:4739", "-agg=none", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if len(cfg.Aggregators) != 0 { + t.Fatalf("expected -agg=none to disable generated aggregators, got %#v", cfg.Aggregators) + } +} + +func TestGeneratedTemplatedFlowPickerOrdersAgentFields(t *testing.T) { + agentIP := slices.Index(generatedTemplatedFlowFields, "agent_ip") + agentIPv6 := slices.Index(generatedTemplatedFlowFields, "agent_ipv6") + if agentIP < 0 || agentIPv6 < 0 { + t.Fatalf("expected picker fields to include agent_ip and agent_ipv6: %#v", generatedTemplatedFlowFields) + } + if agentIP > agentIPv6 { + t.Fatalf("expected picker order agent_ip, agent_ipv6; got %#v", generatedTemplatedFlowFields) + } +} + +func TestGeneratedAggregateConfigPayloadPresetEnablesDataLinkFrame(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=ipfix:udp:127.0.0.1:4739", "-agg=payload", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + agg := primaryPacketAggregator(t, cfg) + if agg.Passthrough { + t.Fatalf("expected payload preset alone to keep stateful aggregation") + } + if !aggregatorHasField(agg, "current", "frame_length") || !aggregatorHasField(agg, "current", "header_data") { + t.Fatalf("expected payload preset to enable data-link fields, got %#v", agg.Fields) + } + fields := cfg.Encoder.TemplatedFlow.Data.Select + if !slices.Contains(fields, "frame_length") || !slices.Contains(fields, "header_data") { + t.Fatalf("expected payload preset to select data-link fields for IPFIX, got %#v", fields) + } + if aggregatorHasField(agg, "current", "nat_src_addr") || slices.Contains(fields, "nat_src_addr") { + t.Fatalf("expected payload preset not to select NAT fields, agg=%#v fields=%#v", agg.Fields, fields) + } +} + +func TestGeneratedAggregateConfigLimitedPresetRemovesParsedPacketFields(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=ipfix:udp:127.0.0.1:4739", "-agg=payload,limited", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + agg := primaryPacketAggregator(t, cfg) + fields := cfg.Encoder.TemplatedFlow.Data.Select + if !aggregatorHasField(agg, "current", "frame_length") || !aggregatorHasField(agg, "current", "header_data") { + t.Fatalf("expected payload fields to remain, got %#v", agg.Fields) + } + for _, name := range []string{"src_addr", "dst_addr", "src_port", "dst_port", "src_mac", "dst_mac", "proto", "tcp_flags", "ether_type"} { + if aggregatorHasFieldName(agg, name) || slices.Contains(fields, name) { + t.Fatalf("expected limited preset to remove %s, agg=%#v fields=%#v", name, agg.Fields, fields) + } + } + for _, name := range []string{"bytes", "packets", "input_if", "output_if", "flow_direction", "agent_ip", "agent_ipv6", "source_id"} { + if !aggregatorHasFieldName(agg, name) || !slices.Contains(fields, name) { + t.Fatalf("expected limited preset to preserve %s, agg=%#v fields=%#v", name, agg.Fields, fields) + } + } +} + +func TestGeneratedAggregateConfigLimitedKeepsIPFieldsWithNATPreset(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=ipfix:udp:127.0.0.1:4739", "-agg=payload,limited,nat", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + agg := primaryPacketAggregator(t, cfg) + fields := cfg.Encoder.TemplatedFlow.Data.Select + for _, name := range []string{"src_addr", "dst_addr", "nat_src_addr", "nat_dst_addr", "nat_src_port", "nat_dst_port"} { + if !aggregatorHasFieldName(agg, name) || !slices.Contains(fields, name) { + t.Fatalf("expected NAT limited preset to preserve %s, agg=%#v fields=%#v", name, agg.Fields, fields) + } + } + for _, name := range []string{"src_port", "dst_port", "src_mac", "dst_mac", "proto", "tcp_flags", "ether_type"} { + if aggregatorHasFieldName(agg, name) || slices.Contains(fields, name) { + t.Fatalf("expected NAT limited preset to remove %s, agg=%#v fields=%#v", name, agg.Fields, fields) + } + } +} + +func TestGeneratedAggregateConfigEncapPresetEnablesOuterFields(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=ipfix:udp:127.0.0.1:4739", "-agg=payload,limited,encap", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if cfg.Processor.Builtin.PacketDecoder.DecodeBeyondL4 == nil || !*cfg.Processor.Builtin.PacketDecoder.DecodeBeyondL4 { + t.Fatalf("expected encap preset to enable decode_beyond_l4, got %#v", cfg.Processor.Builtin.PacketDecoder.DecodeBeyondL4) + } + agg := primaryPacketAggregator(t, cfg) + fields := cfg.Encoder.TemplatedFlow.Data.Select + for _, name := range []string{"outer_proto", "outer_src_port", "outer_src_addr", "outer_dst_port", "outer_dst_addr"} { + if !aggregatorHasField(agg, "key", name) || !slices.Contains(fields, name) { + t.Fatalf("expected encap preset to add key %s, agg=%#v fields=%#v", name, agg.Fields, fields) + } + } + for _, name := range []string{"outer_proto_name", "encap_depth"} { + if !aggregatorHasField(agg, "current", name) || !slices.Contains(fields, name) { + t.Fatalf("expected encap preset to add current %s, agg=%#v fields=%#v", name, agg.Fields, fields) + } + } + for _, name := range []string{"src_addr", "dst_addr", "src_port", "dst_port"} { + if aggregatorHasFieldName(agg, name) || slices.Contains(fields, name) { + t.Fatalf("expected limited encap preset to keep inner field %s removed, agg=%#v fields=%#v", name, agg.Fields, fields) + } + } +} + +func TestGeneratedAggregateConfigNATPresetEnablesNATFields(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=ipfix:udp:127.0.0.1:4739", "-agg=nat", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + agg := primaryPacketAggregator(t, cfg) + fields := cfg.Encoder.TemplatedFlow.Data.Select + if !aggregatorHasField(agg, "current", "nat_src_addr") || !slices.Contains(fields, "nat_src_addr") { + t.Fatalf("expected nat preset to add and select NAT fields, agg=%#v fields=%#v", agg.Fields, fields) + } + if aggregatorHasField(agg, "current", "frame_length") || aggregatorHasField(agg, "current", "header_data") { + t.Fatalf("expected nat preset not to enable payload fields, got %#v", agg.Fields) + } +} + +func TestGeneratedAggregateConfigMPLSPresetEnablesMPLSFields(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-output=ipfix:udp:127.0.0.1:4739", "-agg=mpls", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if cfg.Processor.Builtin.AggregationHelpers.MPLSLabels != len(generatedTemplatedFlowMPLSFields) { + t.Fatalf("expected mpls aggregation helpers to be enabled, got %d", cfg.Processor.Builtin.AggregationHelpers.MPLSLabels) + } + agg := primaryPacketAggregator(t, cfg) + fields := cfg.Encoder.TemplatedFlow.Data.Select + if !aggregatorHasField(agg, "current", "mpls_label_stack_section_1") || !slices.Contains(fields, "mpls_label_stack_section_1") { + t.Fatalf("expected mpls preset to add and select MPLS fields, agg=%#v fields=%#v", agg.Fields, fields) + } +} + +func TestGeneratedAggregateConfigSupportsPassthroughPreset(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-agg=passthrough", "-genconf"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected generated aggregator, got %d", len(cfg.Aggregators)) + } + agg := cfg.Aggregators[0] + if !agg.Passthrough { + t.Fatalf("expected passthrough preset to use schema passthrough") + } + if len(agg.Sum) != 0 || aggregatorHasRole(agg, "sum") { + t.Fatalf("expected passthrough preset to remove sum fields, got sum=%#v fields=%#v", agg.Sum, agg.Fields) + } + if agg.Window.IdleFlushAfter != 0 || agg.Periodic.Every != 0 { + t.Fatalf("expected passthrough preset to remove export timers, got window=%#v periodic=%#v", agg.Window, agg.Periodic) + } + if agg.AggregateMissing { + t.Fatalf("expected passthrough preset to disable aggregate_missing") + } +} + +func TestConfigModeAllowsAggregationPresets(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +encoder: + type: ipfix + +sink: + type: udp + address: "127.0.0.1:4739" +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-config", cfgPath, "-agg=passthrough,payload"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if generated { + t.Fatalf("expected explicit config mode") + } + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected one overlay aggregator, got %d", len(cfg.Aggregators)) + } + agg := cfg.Aggregators[0] + if !agg.Passthrough { + t.Fatalf("expected passthrough preset to use schema passthrough") + } + if !aggregatorHasField(agg, "current", "frame_length") || !aggregatorHasField(agg, "current", "header_data") { + t.Fatalf("expected payload fields, got %#v", agg.Fields) + } +} + +func TestConfigModeAggDefaultsAggregateMissingUnlessDisabled(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + for _, tt := range []struct { + args string + want bool + }{ + {args: "payload", want: true}, + {args: "payload,aggregate_missing=false", want: false}, + } { + t.Run(tt.args, func(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-config", cfgPath, "-agg=" + tt.args}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if generated { + t.Fatalf("expected explicit config mode") + } + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected one overlay aggregator, got %d", len(cfg.Aggregators)) + } + if cfg.Aggregators[0].AggregateMissing != tt.want { + t.Fatalf("aggregate_missing=%t, want %t", cfg.Aggregators[0].AggregateMissing, tt.want) + } + }) + } +} + +func TestConfigModeMPLSPresetAddsMPLSFields(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +aggregators: + - fields: + - key:src_addr + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + fs := flag.NewFlagSet("test", flag.ContinueOnError) + flags, _ := BindFlags(fs) + if err := fs.Parse([]string{"-config", cfgPath, "-agg=mpls"}); err != nil { + t.Fatalf("Parse returned error: %v", err) + } + cfg, generated, err := LoadFromFlags(flags) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if generated { + t.Fatalf("expected explicit config mode") + } + if cfg.Processor.Builtin.AggregationHelpers.MPLSLabels != len(generatedTemplatedFlowMPLSFields) { + t.Fatalf("expected mpls aggregation helpers to be enabled, got %d", cfg.Processor.Builtin.AggregationHelpers.MPLSLabels) + } + if len(cfg.Aggregators) != 1 { + t.Fatalf("expected one overlay aggregator, got %d", len(cfg.Aggregators)) + } + if !aggregatorHasField(cfg.Aggregators[0], "current", "mpls_label_stack_section_1") { + t.Fatalf("expected mpls preset to add aggregate MPLS fields, got %#v", cfg.Aggregators[0].Fields) + } +} + +func TestLoadSupportsJSONDropFields(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +encoder: + type: json + json: + flavor: canonical + drop_fields: + - header_data + - payload + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if len(cfg.Encoder.JSON.DropFields) != 2 { + t.Fatalf("expected 2 json.drop_fields entries, got %#v", cfg.Encoder.JSON.DropFields) + } + if cfg.Encoder.JSON.DropFields[0] != "header_data" || cfg.Encoder.JSON.DropFields[1] != "payload" { + t.Fatalf("unexpected json.drop_fields contents: %#v", cfg.Encoder.JSON.DropFields) + } +} + +func TestLoadSupportsSFlowCounterFormat(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +encoder: + type: sflow + sflow: + counter_format: expanded + emit_extended_records: false + use_metadata_sequence_number: true + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if cfg.Encoder.SFlow.CounterFormat != "expanded" { + t.Fatalf("expected sflow.counter_format=expanded, got %#v", cfg.Encoder.SFlow.CounterFormat) + } + if !cfg.Encoder.SFlow.UseMetadataSequenceNumber { + t.Fatalf("expected sflow.use_metadata_sequence_number=true") + } + if cfg.Encoder.SFlow.EmitExtendedRecords == nil || *cfg.Encoder.SFlow.EmitExtendedRecords { + t.Fatalf("expected sflow.emit_extended_records=false") + } + if cfg.Encoder.AllowTruncate == nil || !*cfg.Encoder.AllowTruncate { + t.Fatalf("expected sflow allow_truncate to default true") + } + if cfg.Encoder.SFlow.MaxHeaderBytes != 128 { + t.Fatalf("expected sflow.max_header_bytes default 128, got %d", cfg.Encoder.SFlow.MaxHeaderBytes) + } + if cfg.Encoder.Batch.IsEnabled() { + t.Fatalf("expected YAML sflow config not to enable batching by default") + } +} + +func TestLoadSupportsAutoWorkers(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + workers: auto + +encoder: + type: ipfix + workers: auto + +sink: + type: udp + address: "127.0.0.1:4739" +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Processor.Workers != AutoWorkers { + t.Fatalf("expected processor workers auto, got %d", cfg.Processor.Workers) + } + if cfg.Encoder.Workers != AutoWorkers { + t.Fatalf("expected encoder workers auto, got %d", cfg.Encoder.Workers) + } + if got := ResolveProcessorWorkers(cfg.Processor.Workers); got != runtime.NumCPU() { + t.Fatalf("expected auto processor workers to resolve to CPU count, got %d", got) + } + if got := ResolveEncoderWorkers(cfg.Encoder.Workers, cfg.Encoder.Type); got != 1 { + t.Fatalf("expected ordered auto encoder workers to resolve to 1, got %d", got) + } + if got := ResolveEncoderWorkers(2, cfg.Encoder.Type); got != 1 { + t.Fatalf("expected ordered explicit encoder workers to resolve to 1, got %d", got) + } + if got := ResolveEncoderWorkers(AutoWorkers, "pcap"); got != 1 { + t.Fatalf("expected pcap auto encoder workers to resolve to 1, got %d", got) + } + if got := ResolveEncoderWorkers(AutoWorkers, "json"); got != runtime.NumCPU() { + t.Fatalf("expected unordered auto encoder workers to resolve to CPU count, got %d", got) + } +} + +func TestLoadSupportsSFlowTruncationOverrides(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +encoder: + type: sflow + allow_truncate: false + sflow: + max_header_bytes: 256 + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Encoder.AllowTruncate == nil || *cfg.Encoder.AllowTruncate { + t.Fatalf("expected explicit allow_truncate=false") + } + if cfg.Encoder.SFlow.MaxHeaderBytes != 256 { + t.Fatalf("expected sflow.max_header_bytes=256, got %d", cfg.Encoder.SFlow.MaxHeaderBytes) + } +} + +func TestLoadSupportsProtobufFlavor(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +encoder: + type: protobuf + protobuf: + flavor: goflow2v2 + length_prefixed: true + export_all: true + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if cfg.Encoder.Protobuf.Flavor != "goflow2v2" { + t.Fatalf("expected protobuf.flavor=goflow2v2, got %#v", cfg.Encoder.Protobuf.Flavor) + } + if !cfg.Encoder.Protobuf.LengthPrefixed { + t.Fatalf("expected protobuf.length_prefixed=true") + } + if !cfg.Encoder.Protobuf.ExportAll { + t.Fatalf("expected protobuf.export_all=true") + } +} + +func TestLoadSupportsTemplatedFlowEncoderSubsection(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +encoder: + type: ipfix + templated_flow: + template_base_id: 300 + options_template_base_id: 1300 + observation_domain_id: 42 + template_refresh_ms: 70000 + options_refresh_ms: 40000 + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Encoder.TemplatedFlow.TemplateBaseID != 300 { + t.Fatalf("expected templated_flow.template_base_id=300, got %d", cfg.Encoder.TemplatedFlow.TemplateBaseID) + } + if cfg.Encoder.TemplatedFlow.OptionsTemplateBaseID != 1300 { + t.Fatalf("expected templated_flow.options_template_base_id=1300, got %d", cfg.Encoder.TemplatedFlow.OptionsTemplateBaseID) + } + if cfg.Encoder.TemplatedFlow.ObservationDomainID != 42 { + t.Fatalf("expected templated_flow.observation_domain_id=42, got %d", cfg.Encoder.TemplatedFlow.ObservationDomainID) + } + if cfg.Encoder.TemplatedFlow.TemplateRefresh != 70000 { + t.Fatalf("expected templated_flow.template_refresh_ms=70000, got %d", cfg.Encoder.TemplatedFlow.TemplateRefresh) + } + if cfg.Encoder.TemplatedFlow.OptionsRefresh != 40000 { + t.Fatalf("expected templated_flow.options_refresh_ms=40000, got %d", cfg.Encoder.TemplatedFlow.OptionsRefresh) + } +} + +func TestLoadDefaultsSourceInitRefresh(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Sources[0].SourceInitRefreshMS == nil || *cfg.Sources[0].SourceInitRefreshMS != 30000 { + t.Fatalf("expected source_init_refresh_ms default 30000, got %#v", cfg.Sources[0].SourceInitRefreshMS) + } +} + +func TestLoadAllowsDisablingSourceInitRefresh(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + source_init_refresh_ms: 0 + +processor: + type: builtin + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Sources[0].SourceInitRefreshMS == nil || *cfg.Sources[0].SourceInitRefreshMS != 0 { + t.Fatalf("expected source_init_refresh_ms=0, got %#v", cfg.Sources[0].SourceInitRefreshMS) + } +} + +func TestLoadDefaultsUDPSinkResolveInterval(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +encoder: + type: json + +sink: + type: udp + address: "collector.example.com:4739" +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Sink.ResolveIntervalMS == nil || *cfg.Sink.ResolveIntervalMS != 60000 { + t.Fatalf("expected sink.resolve_interval_ms default 60000, got %#v", cfg.Sink.ResolveIntervalMS) + } +} + +func TestLoadAllowsDisablingUDPSinkResolveInterval(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +encoder: + type: json + +sink: + type: udp + address: "collector.example.com:4739" + resolve_interval_ms: 0 +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Sink.ResolveIntervalMS == nil || *cfg.Sink.ResolveIntervalMS != 0 { + t.Fatalf("expected sink.resolve_interval_ms=0, got %#v", cfg.Sink.ResolveIntervalMS) + } +} + +func TestLoadRejectsInvalidUDPSinkResolveInterval(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + +processor: + type: builtin + +encoder: + type: json + +sink: + type: udp + address: "collector.example.com:4739" + resolve_interval_ms: -1 +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := Load(cfgPath); err == nil || !strings.Contains(err.Error(), "sink.resolve_interval_ms must be >= 0") { + t.Fatalf("expected sink.resolve_interval_ms validation error, got %v", err) + } +} + +func TestLoadRejectsNegativeSourceInitRefresh(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: flow + source_init_refresh_ms: -1 + +processor: + type: builtin + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := Load(cfgPath); err == nil || !strings.Contains(err.Error(), "source.source_init_refresh_ms must be >= 0") { + t.Fatalf("expected source_init_refresh_ms validation error, got %v", err) + } +} + +func TestLoadRejectsMissingSources(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +processor: + type: builtin + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := Load(cfgPath); err == nil { + t.Fatalf("expected Load to reject missing sources") + } +} + +func TestHelperOptionsTextListsInputOutputAndAggregationExamples(t *testing.T) { + text := HelperOptionsText() + for _, want := range []string{ + "Input helper specs", + "Output helper specs", + "Aggregation helper specs", + "-input udp::6343:flow", + "-output json:stdout", + "-agg passthrough", + "-agg mpls", + "-agg idle_flush_after_ms=,periodic_every_ms=", + "encoders: json, protobuf, sflow, ipfix, netflowv9, netflowv5, pcap, pcapng", + "protobuf:*[?export_all=]", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected helper options to contain %q, got:\n%s", want, text) + } + } +} + +func TestGeneratedSFlowOutputAllowsTruncate(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Inputs: []string{"ebpf:br-lan:bytes?snaplen=65535&sample_every=1"}, + Output: "sflow:udp:127.0.0.1:6343", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if cfg.Encoder.Type != "sflow" { + t.Fatalf("expected sflow encoder, got %q", cfg.Encoder.Type) + } + if cfg.Encoder.AllowTruncate == nil || !*cfg.Encoder.AllowTruncate { + t.Fatalf("expected helper sflow output to enable allow_truncate") + } + if !cfg.Encoder.Batch.IsEnabled() { + t.Fatalf("expected helper sflow output to enable batching by default") + } + if cfg.Encoder.Workers != AutoWorkers { + t.Fatalf("expected helper sflow output to keep encoder workers auto, got %d", cfg.Encoder.Workers) + } +} + +func TestGeneratedIPFIXOutputEnablesBatching(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "ipfix:udp:127.0.0.1:4739", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if !cfg.Encoder.Batch.IsEnabled() { + t.Fatalf("expected helper ipfix output to enable batching by default") + } + if cfg.Encoder.Batch.MaxRecords != 32 { + t.Fatalf("expected helper ipfix output to default batch max records to 32, got %d", cfg.Encoder.Batch.MaxRecords) + } + if cfg.Encoder.Workers != AutoWorkers { + t.Fatalf("expected helper ipfix output to keep encoder workers auto, got %d", cfg.Encoder.Workers) + } +} + +func TestGeneratedNetFlowV9OutputEnablesBatching(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "netflowv9:udp:127.0.0.1:2055", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if !cfg.Encoder.Batch.IsEnabled() { + t.Fatalf("expected helper netflowv9 output to enable batching by default") + } + if cfg.Encoder.Batch.MaxRecords != 32 { + t.Fatalf("expected helper netflowv9 output to default batch max records to 32, got %d", cfg.Encoder.Batch.MaxRecords) + } + if cfg.Encoder.Workers != AutoWorkers { + t.Fatalf("expected helper netflowv9 output to keep encoder workers auto, got %d", cfg.Encoder.Workers) + } +} + +func TestGeneratedOutputParsesBatchParams(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "ipfix:udp:127.0.0.1:4739?batch=true&batch_max_records=32&batch_max_bytes=4096&batch_flush_interval_ms=250", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if !cfg.Encoder.Batch.IsEnabled() { + t.Fatalf("expected explicit batch=true") + } + if cfg.Encoder.Batch.MaxRecords != 32 || cfg.Encoder.Batch.MaxBytes != 4096 || cfg.Encoder.Batch.FlushInterval != 250 { + t.Fatalf("unexpected batch params: %#v", cfg.Encoder.Batch) + } + + sflowCfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "sflow:udp:127.0.0.1:6343?batch=true&batch_max_records=32&batch_max_bytes=4096&batch_flush_interval_ms=250", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags sflow returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated sflow config") + } + if sflowCfg.Encoder.Batch.MaxBytes != 4096 || sflowCfg.Encoder.MaxDatagramBytes != 4096 { + t.Fatalf("expected sflow batch/max datagram bytes 4096, got batch=%d max_datagram=%d", sflowCfg.Encoder.Batch.MaxBytes, sflowCfg.Encoder.MaxDatagramBytes) + } + + netflowV9Cfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "netflowv9:udp:127.0.0.1:2055?batch=true&batch_max_records=32&batch_max_bytes=4096&batch_flush_interval_ms=250", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags netflowv9 returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated netflowv9 config") + } + if netflowV9Cfg.Encoder.Batch.MaxRecords != 32 || netflowV9Cfg.Encoder.Batch.MaxBytes != 4096 || netflowV9Cfg.Encoder.Batch.FlushInterval != 250 { + t.Fatalf("unexpected netflowv9 batch params: %#v", netflowV9Cfg.Encoder.Batch) + } + if netflowV9Cfg.Encoder.MaxDatagramBytes != 4096 { + t.Fatalf("expected netflowv9 max datagram bytes 4096, got %d", netflowV9Cfg.Encoder.MaxDatagramBytes) + } +} + +func TestGeneratedOutputParsesUDPSinkResolveInterval(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "ipfix:udp:collector.example.com:4739?resolve_interval_ms=0", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if cfg.Sink.ResolveIntervalMS == nil || *cfg.Sink.ResolveIntervalMS != 0 { + t.Fatalf("expected resolve_interval_ms=0, got %#v", cfg.Sink.ResolveIntervalMS) + } + + defaultCfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "ipfix:udp:collector.example.com:4739", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags default returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if defaultCfg.Sink.ResolveIntervalMS == nil || *defaultCfg.Sink.ResolveIntervalMS != 60000 { + t.Fatalf("expected default resolve_interval_ms=60000, got %#v", defaultCfg.Sink.ResolveIntervalMS) + } +} + +func TestGeneratedOutputParsesBatchDisable(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "sflow:udp:127.0.0.1:6343?batch=false", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if cfg.Encoder.Batch.IsEnabled() { + t.Fatalf("expected explicit batch=false") + } +} + +func TestGeneratedProtobufOutputParsesExportAllParam(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "protobuf:file:/tmp/reflow.pb?export_all=true", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if cfg.Encoder.Type != "protobuf" { + t.Fatalf("expected protobuf encoder, got %q", cfg.Encoder.Type) + } + if !cfg.Encoder.Protobuf.ExportAll { + t.Fatalf("expected protobuf export_all=true") + } +} + +func TestLoadDoesNotEnableIPFIXBatchingByDefault(t *testing.T) { + dir := t.TempDir() + + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: udp + address: ":18081" + type: json + +processor: + type: builtin + +encoder: + type: ipfix + +sink: + type: udp + address: "127.0.0.1:4739" +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Encoder.Batch.IsEnabled() { + t.Fatalf("expected YAML ipfix config not to enable batching by default") + } +} + +func TestGeneratedSFlowOutputParsesAllowTruncateParam(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Output: "sflow:udp:127.0.0.1:6343?allow_truncate=false&max_header_bytes=256", + OutputSet: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config") + } + if cfg.Encoder.AllowTruncate == nil || *cfg.Encoder.AllowTruncate { + t.Fatalf("expected explicit allow_truncate=false to override helper default") + } + if cfg.Encoder.SFlow.MaxHeaderBytes != 256 { + t.Fatalf("expected explicit max_header_bytes=256, got %d", cfg.Encoder.SFlow.MaxHeaderBytes) + } +} + +func TestGeneratedOutputRejectsUnsupportedParams(t *testing.T) { + _, _, err := LoadFromFlags(&FlagConfig{ + Output: "json:stdout?allow_truncate=true", + OutputSet: true, + }) + if err == nil { + t.Fatalf("expected json output to reject allow_truncate") + } + if !strings.Contains(err.Error(), "only supported for sflow") { + t.Fatalf("expected sflow-only error, got %v", err) + } +} + +func TestGeneratedOutputRejectsUnsupportedBatchParams(t *testing.T) { + _, _, err := LoadFromFlags(&FlagConfig{ + Output: "json:stdout?batch=true", + OutputSet: true, + }) + if err == nil { + t.Fatalf("expected json output to reject batch") + } + if !strings.Contains(err.Error(), "only supported for sflow, ipfix, and netflowv9") { + t.Fatalf("expected sflow/ipfix/netflowv9-only error, got %v", err) + } +} + +func TestGeneratedOutputRejectsUnsupportedProtobufParams(t *testing.T) { + _, _, err := LoadFromFlags(&FlagConfig{ + Output: "json:stdout?export_all=true", + OutputSet: true, + }) + if err == nil { + t.Fatalf("expected json output to reject export_all") + } + if !strings.Contains(err.Error(), "only supported for protobuf") { + t.Fatalf("expected protobuf-only error, got %v", err) + } + + _, _, err = LoadFromFlags(&FlagConfig{ + Output: "protobuf:stdout?export_all=maybe", + OutputSet: true, + }) + if err == nil { + t.Fatalf("expected protobuf output to reject invalid export_all") + } + if !strings.Contains(err.Error(), "must be a boolean") { + t.Fatalf("expected boolean parse error, got %v", err) + } +} + +func TestGeneratedConfigYAMLUsesFalseForPacketDecoderBooleans(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{GenConf: true}) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config mode") + } + if cfg.Sources[0].SnapLen != 65535 || cfg.Sources[0].SampleEvery != 1 { + t.Fatalf("expected generated source defaults, got %#v", cfg.Sources[0]) + } + if cfg.Processor.Workers != AutoWorkers { + t.Fatalf("expected generated processor workers auto, got %d", cfg.Processor.Workers) + } + if cfg.Processor.Builtin.PacketDecoder.DecodeBeyondL4 == nil || *cfg.Processor.Builtin.PacketDecoder.DecodeBeyondL4 { + t.Fatalf("expected generated packet decoder decode_beyond_l4=false") + } + if cfg.Processor.Builtin.PacketDecoder.Encapsulations.VXLAN.Enabled == nil || *cfg.Processor.Builtin.PacketDecoder.Encapsulations.VXLAN.Enabled { + t.Fatalf("expected generated vxlan enabled=false") + } + if !slices.Contains(cfg.Processor.Builtin.PacketDecoder.Encapsulations.VXLAN.Ports, 4789) { + t.Fatalf("expected generated vxlan default port 4789, got %#v", cfg.Processor.Builtin.PacketDecoder.Encapsulations.VXLAN.Ports) + } + if cfg.Encoder.TemplatedFlow.TemplateBaseID != 256 || cfg.Encoder.TemplatedFlow.OptionsTemplateBaseID != 1024 { + t.Fatalf("expected generated templated flow base IDs, got %#v", cfg.Encoder.TemplatedFlow) + } + if cfg.Encoder.TemplatedFlow.TemplateRefresh != 60000 || cfg.Encoder.TemplatedFlow.OptionsRefresh != 30000 { + t.Fatalf("expected generated templated flow refresh defaults, got %#v", cfg.Encoder.TemplatedFlow) + } + raw, err := yaml.Marshal(cfg) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + text := string(raw) + if strings.Contains(text, ": null") { + t.Fatalf("expected generated config to avoid null booleans, got:\n%s", text) + } + if strings.Contains(text, "ports: []") { + t.Fatalf("expected generated config to materialize encapsulation ports, got:\n%s", text) + } +} + +func TestGeneratedEBPFConfigMaterializesFeatureDefaults(t *testing.T) { + cfg, generated, err := LoadFromFlags(&FlagConfig{ + Inputs: []string{"ebpf:br-lan:bytes"}, + GenConf: true, + }) + if err != nil { + t.Fatalf("LoadFromFlags returned error: %v", err) + } + if !generated { + t.Fatalf("expected generated config mode") + } + if len(cfg.Sources) != 1 { + t.Fatalf("expected one generated source, got %d", len(cfg.Sources)) + } + src := cfg.Sources[0] + if src.Network != "ebpf" || src.Interface != "br-lan" || src.Type != "bytes" { + t.Fatalf("unexpected generated ebpf source: %#v", src) + } + if src.EBPF.SKBMetadata == nil || !*src.EBPF.SKBMetadata { + t.Fatalf("expected generated ebpf skb_metadata=true, got %#v", src.EBPF.SKBMetadata) + } + if src.EBPF.Conntrack == nil || !*src.EBPF.Conntrack { + t.Fatalf("expected generated ebpf conntrack=true, got %#v", src.EBPF.Conntrack) + } + if src.EBPF.Direction != "both" { + t.Fatalf("expected generated ebpf direction both, got %q", src.EBPF.Direction) + } +} + +func TestLoadSupportsEBPFSourceDefaults(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "reflow.yaml") + if err := os.WriteFile(cfgPath, []byte(` +sources: + - network: ebpf + interface: eth0 + +processor: + type: builtin + +encoder: + type: json + +sink: + type: stdout +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Sources[0].Type != "bytes" { + t.Fatalf("expected ebpf source type bytes, got %q", cfg.Sources[0].Type) + } + if cfg.Sources[0].Address != "eth0" { + t.Fatalf("expected ebpf address eth0, got %q", cfg.Sources[0].Address) + } + if cfg.Sources[0].SnapLen != 65535 { + t.Fatalf("expected ebpf snaplen 65535, got %d", cfg.Sources[0].SnapLen) + } + if cfg.Sources[0].SampleEvery != 1 { + t.Fatalf("expected ebpf sample_every 1, got %d", cfg.Sources[0].SampleEvery) + } + if cfg.Sources[0].EBPF.Direction != "both" { + t.Fatalf("expected ebpf direction both, got %q", cfg.Sources[0].EBPF.Direction) + } +} + +func TestParseInputSpecSupportsEBPF(t *testing.T) { + src, err := parseInputSpec("ebpf:eth0:bytes") + if err != nil { + t.Fatalf("parseInputSpec returned error: %v", err) + } + if src.Network != "ebpf" || src.Interface != "eth0" || src.Type != "bytes" || src.Address != "" { + t.Fatalf("unexpected ebpf source config: %#v", src) + } +} + +func TestParseInputSpecSupportsCaptureParams(t *testing.T) { + src, err := parseInputSpec("pcap_live:en0:bytes?snaplen=262144&sample_every=10&sample_offset=3&source_init_refresh_ms=0") + if err != nil { + t.Fatalf("parseInputSpec returned error: %v", err) + } + if src.Network != "pcap_live" || src.Interface != "en0" || src.Type != "bytes" || src.Address != "" { + t.Fatalf("unexpected pcap_live source config: %#v", src) + } + if src.SnapLen != 262144 || src.SampleEvery != 10 || src.SampleOffset != 3 { + t.Fatalf("unexpected capture params: snaplen=%d sample_every=%d sample_offset=%d", src.SnapLen, src.SampleEvery, src.SampleOffset) + } + if src.SourceInitRefreshMS == nil || *src.SourceInitRefreshMS != 0 { + t.Fatalf("expected source_init_refresh_ms=0, got %#v", src.SourceInitRefreshMS) + } +} + +func TestParseInputSpecSupportsPcapLiveInterfaceFilter(t *testing.T) { + src, err := parseInputSpec(`pcap_live:any:bytes?interface_filter=^(eth|wan)`) + if err != nil { + t.Fatalf("parseInputSpec returned error: %v", err) + } + if src.Network != "pcap_live" || src.Interface != "any" || src.InterfaceFilter != "^(eth|wan)" { + t.Fatalf("unexpected pcap_live source config: %#v", src) + } +} + +func TestParseInputSpecSupportsEBPFInterfaceFilter(t *testing.T) { + src, err := parseInputSpec(`ebpf:any:bytes?interface_filter=^(eth|wan)`) + if err != nil { + t.Fatalf("parseInputSpec returned error: %v", err) + } + if src.Network != "ebpf" || src.Interface != "any" || src.InterfaceFilter != "^(eth|wan)" { + t.Fatalf("unexpected ebpf source config: %#v", src) + } +} + +func TestParseInputSpecSupportsEBPFFeatureParams(t *testing.T) { + src, err := parseInputSpec("ebpf:br-lan:bytes?skb_metadata=false&conntrack=false&conntrack_path=/tmp/nf_conntrack&direction=ingress") + if err != nil { + t.Fatalf("parseInputSpec returned error: %v", err) + } + if src.EBPF.SKBMetadata == nil || *src.EBPF.SKBMetadata { + t.Fatalf("expected skb_metadata=false, got %#v", src.EBPF.SKBMetadata) + } + if src.EBPF.Conntrack == nil || *src.EBPF.Conntrack { + t.Fatalf("expected conntrack=false, got %#v", src.EBPF.Conntrack) + } + if src.EBPF.ConntrackPath != "/tmp/nf_conntrack" { + t.Fatalf("expected conntrack path, got %q", src.EBPF.ConntrackPath) + } + if src.EBPF.Direction != "ingress" { + t.Fatalf("expected direction ingress, got %q", src.EBPF.Direction) + } +} + +func TestParseInputSpecRejectsInvalidEBPFDirection(t *testing.T) { + if _, err := parseInputSpec("ebpf:br-lan:bytes?direction=sideways"); err == nil { + t.Fatalf("expected invalid ebpf direction to fail") + } +} + +func TestParseInputSpecRejectsCaptureParamsOnSocketInputs(t *testing.T) { + if _, err := parseInputSpec("udp::6343:flow?snaplen=128"); err == nil { + t.Fatalf("expected capture params on udp input to fail") + } +} + +func aggregatorHasField(agg AggregatorConfig, role, name string) bool { + for _, field := range agg.Fields { + if field.Role == role && field.Name == name { + return true + } + } + return false +} + +func aggregatorHasFieldName(agg AggregatorConfig, name string) bool { + for _, field := range agg.Fields { + if field.Name == name { + return true + } + } + return false +} + +func primaryPacketAggregator(t *testing.T, cfg *Config) AggregatorConfig { + t.Helper() + for _, agg := range cfg.Aggregators { + if agg.Match["record_kind"] == "packet" { + return agg + } + } + t.Fatalf("expected generated packet aggregator, got %#v", cfg.Aggregators) + return AggregatorConfig{} +} + +func sourceOptionsAggregator(t *testing.T, cfg *Config) AggregatorConfig { + t.Helper() + for _, agg := range cfg.Aggregators { + if agg.Stream == "options_data" && agg.Match["control.type"] == "source_init" { + return agg + } + } + t.Fatalf("expected generated source options aggregator, got %#v", cfg.Aggregators) + return AggregatorConfig{} +} + +func aggregatorHasRole(agg AggregatorConfig, role string) bool { + for _, field := range agg.Fields { + if field.Role == role { + return true + } + } + return false +} diff --git a/pkg/reflow/config/default-fields.yaml b/pkg/reflow/config/default-fields.yaml new file mode 100644 index 00000000..3c45ee7a --- /dev/null +++ b/pkg/reflow/config/default-fields.yaml @@ -0,0 +1,58 @@ +# Embedded default field catalog for templated IPFIX/NetFlow v9 exports. +# Format: id:length:type[:format][pen=] +# Type aliases: u8/u16/u32/u64, s8/s16/s32/s64, ip4, ip6, mac, str. +# Enterprise-specific example, commented out: +# custom_metric: 40000:4:u32[pen=42424242] # example enterprise field + +fields: + bytes: 1:8:u64:delta # octetDeltaCount + packets: 2:8:u64:delta # packetDeltaCount + proto: 4:1:u8 # protocolIdentifier + tcp_flags: 6:1:u8 # tcpControlBits + src_port: 7:2:u16 # sourceTransportPort + src_addr: 8:4:ip4 # sourceIPv4Address + input_if: 10:4:u32 # ingressInterface + dst_port: 11:2:u16 # destinationTransportPort + dst_addr: 12:4:ip4 # destinationIPv4Address + output_if: 14:4:u32 # egressInterface + sampling_rate: 34:4:u32 # samplingInterval + src_mac: 56:6:mac # sourceMacAddress + vlan_id: 58:2:u16 # vlanId + flow_direction: 61:1:u8 # flowDirection + mpls_label_stack_section_1: 70:3:bytes # mplsTopLabelStackSection + mpls_label_stack_section_2: 71:3:bytes # mplsLabelStackSection2 + mpls_label_stack_section_3: 72:3:bytes # mplsLabelStackSection3 + dst_mac: 80:6:mac # destinationMacAddress + interface_name: 82:0:str # interfaceName + interface_description: 83:0:str # interfaceDescription + sampler_name: 84:0:str # samplerName + agent_ip: 130:4:ip4 # exporterIPv4Address + agent_ipv6: 131:16:ip6 # exporterIPv6Address + drops: 133:8:u64:delta # droppedPacketDeltaCount + source_id: 138:8:u64 # observationPointId + sub_agent_id: 144:4:u32 # exportingProcessId + template_id: 145:2:u16 # templateId + observation_domain_id: 149:4:u32 # observationDomainId + start_time_unix: 152:8:u64 # flowStartMilliseconds + end_time_unix: 153:8:u64 # flowEndMilliseconds + nat_src_addr: 225:4:ip4 # postNATSourceIPv4Address + nat_dst_addr: 226:4:ip4 # postNATDestinationIPv4Address + nat_src_port: 227:2:u16 # postNAPTSourceTransportPort + nat_dst_port: 228:2:u16 # postNAPTDestinationTransportPort + ether_type: 256:2:u16 # ethernetType + sample_pool: 310:4:u32 # samplingPopulation + frame_length: 312:2:u16 # dataLinkFrameSize + header_data: 315:65535:bytes # dataLinkFrameSection + # Linux __sk_buff fields need a real implementation PEN before enabling. + # 32473 is reserved for documentation examples, so it is intentionally not used here. + # skb_len: 40000:4:u32[pen=] # Linux __sk_buff len + # skb_packet_type: 40001:4:u32[pen=] # Linux __sk_buff pkt_type + # skb_mark: 40002:4:u32[pen=] # Linux __sk_buff mark + # skb_queue_mapping: 40003:4:u32[pen=] # Linux __sk_buff queue_mapping + # skb_protocol: 40004:4:u32[pen=] # Linux __sk_buff protocol + # skb_priority: 40005:4:u32[pen=] # Linux __sk_buff priority + # skb_ingress_ifindex: 40006:4:u32[pen=] # Linux __sk_buff ingress_ifindex + # skb_ifindex: 40007:4:u32[pen=] # Linux __sk_buff ifindex + # skb_tc_index: 40008:4:u32[pen=] # Linux __sk_buff tc_index + # skb_hash: 40009:4:u32[pen=] # Linux __sk_buff hash + # skb_tc_classid: 40010:4:u32[pen=] # Linux __sk_buff tc_classid diff --git a/pkg/reflow/decode/catalog.go b/pkg/reflow/decode/catalog.go new file mode 100644 index 00000000..e4545966 --- /dev/null +++ b/pkg/reflow/decode/catalog.go @@ -0,0 +1,273 @@ +package decode + +import ( + "encoding/base64" + "fmt" + "math" + "net" + "net/netip" + + "github.com/netsampler/goflow2/v3/decoders/netflow" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" +) + +type decodeCatalog struct { + ipfix map[decodeCatalogIPFIXKey]decodeCatalogField + netflowV9 map[uint16]decodeCatalogField + netflowV9Scopes map[uint16]decodeCatalogField +} + +type decodeCatalogIPFIXKey struct { + id uint16 + pen uint32 +} + +type decodeCatalogField struct { + key string + def config.IPFIXFieldDefinition +} + +func newDecodeCatalog(catalog map[string]config.IPFIXFieldDefinition) decodeCatalog { + out := decodeCatalog{ + ipfix: make(map[decodeCatalogIPFIXKey]decodeCatalogField), + netflowV9: make(map[uint16]decodeCatalogField), + netflowV9Scopes: make(map[uint16]decodeCatalogField), + } + for key, def := range catalog { + field := decodeCatalogField{key: key, def: def} + for _, ipfixKey := range decodeCatalogIPFIXKeys(key, def) { + out.ipfix[ipfixKey] = field + } + for _, id := range decodeCatalogNetFlowV9IDs(key, def) { + out.netflowV9[id] = field + } + for _, id := range decodeCatalogNetFlowV9ScopeIDs(key, def) { + out.netflowV9Scopes[id] = field + } + } + return out +} + +func (c decodeCatalog) empty() bool { + return len(c.ipfix) == 0 && len(c.netflowV9) == 0 && len(c.netflowV9Scopes) == 0 +} + +func (c decodeCatalog) lookup(field netflow.DataField, netflowV9 bool) (decodeCatalogField, bool) { + if netflowV9 { + out, ok := c.netflowV9[field.Type] + return out, ok + } + pen := uint32(0) + if field.PenProvided { + pen = field.Pen + } + out, ok := c.ipfix[decodeCatalogIPFIXKey{id: field.Type, pen: pen}] + return out, ok +} + +func (c decodeCatalog) lookupOptions(field netflow.DataField, netflowV9 bool, scope bool) (decodeCatalogField, bool) { + if netflowV9 && scope { + out, ok := c.netflowV9Scopes[field.Type] + return out, ok + } + return c.lookup(field, netflowV9) +} + +func (c decodeCatalog) lookupTemplateField(field netflow.Field, netflowV9 bool, scope bool) (decodeCatalogField, bool) { + dataField := netflow.DataField{ + PenProvided: field.PenProvided, + Type: field.Type, + Length: field.Length, + Pen: field.Pen, + } + return c.lookupOptions(dataField, netflowV9, scope) +} + +func decodeCatalogIPFIXKeys(name string, def config.IPFIXFieldDefinition) []decodeCatalogIPFIXKey { + if def.ID == 0 { + return nil + } + pen := uint32(0) + if def.EnterpriseScoped || def.PEN != 0 { + pen = def.PEN + } + keys := []decodeCatalogIPFIXKey{{id: def.ID, pen: pen}} + switch name { + case "src_addr": + keys = append(keys, decodeCatalogIPFIXKey{id: 27}) + case "dst_addr": + keys = append(keys, decodeCatalogIPFIXKey{id: 28}) + } + return keys +} + +func decodeCatalogNetFlowV9IDs(name string, def config.IPFIXFieldDefinition) []uint16 { + ids := make([]uint16, 0, 2) + if def.ID != 0 && def.PEN == 0 && !def.EnterpriseScoped { + ids = append(ids, def.ID) + } + switch name { + case "src_addr": + ids = append(ids, 27) + case "dst_addr": + ids = append(ids, 28) + case "start_time_unix": + ids = append(ids, netflow.NFV9_FIELD_FIRST_SWITCHED) + case "end_time_unix": + ids = append(ids, netflow.NFV9_FIELD_LAST_SWITCHED) + } + return ids +} + +func decodeCatalogNetFlowV9ScopeIDs(name string, def config.IPFIXFieldDefinition) []uint16 { + ids := make([]uint16, 0, 1) + if id := defaultNetFlowV9ScopeID(name); id != 0 { + ids = append(ids, id) + } + return ids +} + +func defaultNetFlowV9ScopeID(name string) uint16 { + switch name { + case "observation_domain_id", "source_id": + return 1 + case "if_index", "input_if": + return 2 + default: + return 0 + } +} + +func applyCatalogDataField(fields map[string]any, field netflow.DataField, catalogField decodeCatalogField, sysUptime, unixSeconds uint32, netflowV9 bool) { + key := catalogField.key + switch key { + case "proto": + proto := decodeUint32(field.Value) + fields["proto"] = proto + fields["proto_name"] = ipProtocolName(proto) + case "bytes", "packets": + fields[key] = int64(decodeUint64(field.Value)) + case "start_time_unix": + if netflowV9 && field.Type == netflow.NFV9_FIELD_FIRST_SWITCHED { + fields[key] = flowTimeFromV9(sysUptime, unixSeconds, decodeUint32(field.Value)) + } else { + fields[key] = int64(decodeUint64(field.Value)) + } + case "end_time_unix": + if netflowV9 && field.Type == netflow.NFV9_FIELD_LAST_SWITCHED { + fields[key] = flowTimeFromV9(sysUptime, unixSeconds, decodeUint32(field.Value)) + } else { + fields[key] = int64(decodeUint64(field.Value)) + } + case "src_addr", "dst_addr", "nat_src_addr", "nat_dst_addr": + setDecodedIPField(fields, key, field.Value) + default: + fields[key] = decodeCatalogValue(catalogField.def, field.Value) + } +} + +func setDecodedIPField(fields map[string]any, key string, val any) { + ip := decodeIPString(val) + if ip == "" { + return + } + addr, err := netip.ParseAddr(ip) + if err == nil && addr.IsUnspecified() { + if existing, ok := fields[key].(string); ok && existing != "" { + return + } + } + fields[key] = ip +} + +func decodeCatalogValue(def config.IPFIXFieldDefinition, val any) any { + switch def.Type { + case "ipv4Address", "ipv6Address": + return decodeIPString(val) + case "string": + switch v := val.(type) { + case []byte: + return string(v) + case string: + return v + default: + return fmt.Sprint(v) + } + case "unsigned8", "unsigned16", "unsigned32": + return decodeUint32(val) + case "unsigned64": + return decodeUint64(val) + case "signed8", "signed16", "signed32": + return int32(decodeSigned64(val, def.Length)) + case "signed64": + return decodeSigned64(val, def.Length) + case "bytes": + return cloneRawBytes(val) + case "macAddress": + return decodeMACString(val) + case "boolean": + return decodeUint32(val) != 0 + default: + return base64RawValue(val) + } +} + +func decodeMACString(val any) any { + raw, ok := val.([]byte) + if !ok { + return val + } + if len(raw) == 0 { + return "" + } + return net.HardwareAddr(raw).String() +} + +func decodeSigned64(val any, length uint16) int64 { + switch v := val.(type) { + case []byte: + u := decodeUint64(v) + bits := uint(length) * 8 + if bits == 0 { + bits = uint(len(v)) * 8 + } + if bits == 0 || bits >= 64 { + return int64(u) + } + sign := uint64(1) << (bits - 1) + if u&sign == 0 { + return int64(u) + } + mask := uint64(math.MaxUint64) << bits + return int64(u | mask) + case int64: + return v + case int32: + return int64(v) + case uint64: + return int64(v) + case uint32: + return int64(v) + default: + return 0 + } +} + +func cloneRawBytes(val any) any { + raw, ok := val.([]byte) + if !ok { + return val + } + return append([]byte(nil), raw...) +} + +func base64RawValue(val any) string { + switch v := val.(type) { + case []byte: + return base64.StdEncoding.EncodeToString(v) + case string: + return base64.StdEncoding.EncodeToString([]byte(v)) + default: + return base64.StdEncoding.EncodeToString([]byte(fmt.Sprint(v))) + } +} diff --git a/pkg/reflow/decode/decode.go b/pkg/reflow/decode/decode.go new file mode 100644 index 00000000..7bf219c2 --- /dev/null +++ b/pkg/reflow/decode/decode.go @@ -0,0 +1,144 @@ +package decode + +import ( + "fmt" + "time" + + "github.com/netsampler/goflow2/v3/decoders/netflow" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/utils/store/persistence" + "github.com/netsampler/goflow2/v3/utils/store/samplingrate" + "github.com/netsampler/goflow2/v3/utils/store/templates" +) + +// Decoder identifies and decodes source payloads into the next runtime event shape. +type Decoder interface { + Decode(evt *event.Event) ([]*event.Event, error) + Errors() <-chan error + Close() +} + +// Options configures the built-in decoder. +type Options struct { + Catalog map[string]config.IPFIXFieldDefinition + EmbedTemplateFields bool + CachePath string + CacheInterval time.Duration +} + +// New returns the built-in decoder used by the current runtime. +func New() Decoder { + return NewWithCatalog(nil) +} + +// NewWithCatalog returns a decoder that uses the shared templated field catalog +// for IPFIX and NetFlow v9 data record expansion. +func NewWithCatalog(catalog map[string]config.IPFIXFieldDefinition) Decoder { + decoder, err := NewWithOptions(Options{Catalog: catalog}) + if err != nil { + panic(err) + } + return decoder +} + +// NewWithOptions returns a decoder with optional template and sampling-rate cache persistence. +func NewWithOptions(opts Options) (Decoder, error) { + var ( + manager *persistence.Manager + store netflow.ManagedTemplateStore + sampling samplingrate.Store + err error + ) + if opts.CachePath != "" { + manager = persistence.New(persistence.Config{ + Path: opts.CachePath, + Interval: opts.CacheInterval, + }) + sampling, err = manager.NewSamplingRateStore() + if err != nil { + manager.Close() + return nil, fmt.Errorf("init sampling cache: %w", err) + } + store, err = manager.NewTemplateStore() + if err != nil { + manager.Close() + return nil, fmt.Errorf("init template cache: %w", err) + } + } else { + store = templates.NewTemplateFlowStore() + sampling = samplingrate.NewSamplingRateFlowStore() + } + store.Start() + sampling.Start() + if manager != nil { + manager.Start() + } + return &builtIn{ + templates: store, + sampling: sampling, + catalog: newDecodeCatalog(opts.Catalog), + embedTemplateFields: opts.EmbedTemplateFields, + persistence: manager, + }, nil +} + +type builtIn struct { + templates netflow.ManagedTemplateStore + sampling samplingrate.Store + catalog decodeCatalog + embedTemplateFields bool + persistence *persistence.Manager +} + +// Close stops decoder-owned stores and their background sweepers. +func (d *builtIn) Close() { + if d == nil { + return + } + if d.templates != nil { + d.templates.Close() + } + if d.sampling != nil { + d.sampling.Close() + } + if d.persistence != nil { + d.persistence.Close() + } +} + +// Errors exposes asynchronous decoder cache persistence errors. +func (d *builtIn) Errors() <-chan error { + if d == nil || d.persistence == nil { + return nil + } + return d.persistence.Errors() +} + +// Decode handles protocol identification for raw flow payloads and passes through other event types. +func (d *builtIn) Decode(evt *event.Event) ([]*event.Event, error) { + if evt != nil && evt.Kind == "control" { + return []*event.Event{evt}, nil + } + switch evt.Source.Type { + case "flow": + return d.decodeFlow(evt) + case "bytes": + return decodeBytes(evt) + default: + return []*event.Event{evt}, nil + } +} + +// decodeBytes marks raw packet inputs as bytes payloads and leaves the actual +// tuple extraction to the processor stage. +func decodeBytes(evt *event.Event) ([]*event.Event, error) { + payload, ok := evt.Payload.([]byte) + if !ok || len(payload) == 0 { + return nil, fmt.Errorf("decode bytes: missing payload bytes") + } + if evt.Fields == nil { + evt.Fields = make(map[string]any, 1) + } + return []*event.Event{evt}, nil +} diff --git a/pkg/reflow/decode/decode_bench_test.go b/pkg/reflow/decode/decode_bench_test.go new file mode 100644 index 00000000..863f7f6d --- /dev/null +++ b/pkg/reflow/decode/decode_bench_test.go @@ -0,0 +1,134 @@ +package decode + +import ( + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/encode" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func BenchmarkDecode(b *testing.B) { + b.Run("bytes_passthrough", func(b *testing.B) { + dec := New() + defer dec.Close() + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x14, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + b.ReportAllocs() + for i := 0; i < b.N; i++ { + events, err := dec.Decode(&event.Event{ + ReceivedAt: time.Unix(1_700_000_001, 0).UTC(), + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + b.Fatalf("Decode returned error: %v", err) + } + if len(events) != 1 { + b.Fatalf("expected 1 event, got %d", len(events)) + } + } + }) + + b.Run("ipfix_data", func(b *testing.B) { + catalog := benchDecodeCatalog() + templates, data := benchIPFIXPayloads(b, catalog) + dec := NewWithCatalog(catalog) + defer dec.Close() + for _, payload := range templates { + if _, err := dec.Decode(&event.Event{Source: event.SourceMetadata{Type: "flow"}, Payload: payload}); err != nil { + b.Fatalf("decode template payload: %v", err) + } + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + events, err := dec.Decode(&event.Event{ + ReceivedAt: time.Unix(1_700_000_001, 0).UTC(), + Source: event.SourceMetadata{Type: "flow"}, + Payload: data, + }) + if err != nil { + b.Fatalf("Decode returned error: %v", err) + } + if len(events) != 1 { + b.Fatalf("expected 1 data event, got %d", len(events)) + } + } + }) +} + +func benchIPFIXPayloads(b *testing.B, catalog map[string]config.IPFIXFieldDefinition) ([][]byte, []byte) { + b.Helper() + enc := encode.NewIPFIXEncoder(config.EncoderConfig{ + Type: "ipfix", + TemplatedFlow: config.TemplatedFlowConfig{ + TemplateBaseID: 256, + Data: config.TemplatedFlowDataConfig{ + Select: []string{"src_addr", "dst_addr", "src_port", "dst_port", "proto", "bytes", "packets"}, + Catalog: catalog, + }, + }, + }) + templates, err := enc.Encode(&event.Event{ + Kind: "control", + Stream: "flow_data", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"src_addr", "dst_addr", "src_port", "dst_port", "proto", "bytes", "packets"}, + }, + }) + if err != nil { + b.Fatalf("encode schema returned error: %v", err) + } + if len(templates) == 0 { + b.Fatalf("expected template payloads") + } + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(1_700_000_001, 0).UTC(), + Stream: "flow_data", + Fields: map[string]any{ + "src_addr": "192.0.2.10", + "dst_addr": "198.51.100.20", + "src_port": uint32(12345), + "dst_port": uint32(443), + "proto": uint32(6), + "bytes": int64(321), + "packets": int64(7), + }, + }) + if err != nil { + b.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + b.Fatalf("expected one data payload, got %d", len(payloads)) + } + return templates, payloads[0] +} + +func benchDecodeCatalog() map[string]config.IPFIXFieldDefinition { + return map[string]config.IPFIXFieldDefinition{ + "src_addr": {ID: 8, Length: 4, Type: "ipv4Address"}, + "dst_addr": {ID: 12, Length: 4, Type: "ipv4Address"}, + "src_port": {ID: 7, Length: 2, Type: "unsigned16"}, + "dst_port": {ID: 11, Length: 2, Type: "unsigned16"}, + "proto": {ID: 4, Length: 1, Type: "unsigned8"}, + "bytes": {ID: 1, Length: 8, Type: "unsigned64"}, + "packets": {ID: 2, Length: 8, Type: "unsigned64"}, + } +} diff --git a/pkg/reflow/decode/flow.go b/pkg/reflow/decode/flow.go new file mode 100644 index 00000000..9af0959d --- /dev/null +++ b/pkg/reflow/decode/flow.go @@ -0,0 +1,52 @@ +package decode + +import ( + "encoding/binary" + "fmt" + + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func (d *builtIn) decodeFlow(evt *event.Event) ([]*event.Event, error) { + payload, ok := evt.Payload.([]byte) + if !ok || len(payload) == 0 { + return nil, fmt.Errorf("decode flow: missing payload bytes") + } + flowType, flowVersion, err := identifyFlow(payload) + if err != nil { + return nil, err + } + switch flowType { + case "sflow": + return d.decodeSFlow(evt, payload, flowVersion) + case "netflowv5": + return d.decodeNetFlowV5(evt, payload) + case "netflowv9": + return d.decodeNetFlowV9(evt, payload) + case "ipfix": + return d.decodeIPFIX(evt, payload) + default: + return nil, fmt.Errorf("decode flow: unsupported flow type %q", flowType) + } +} + +// identifyFlow distinguishes sFlow from NetFlow/IPFIX before full decoding. +// sFlow uses a 32-bit leading version field while NetFlow/IPFIX use 16-bit. +func identifyFlow(payload []byte) (string, uint32, error) { + if len(payload) < 4 { + return "", 0, fmt.Errorf("identify flow: payload too short") + } + if binary.BigEndian.Uint32(payload[:4]) == 5 { + return "sflow", 5, nil + } + switch version := binary.BigEndian.Uint16(payload[:2]); version { + case 5: + return "netflowv5", uint32(version), nil + case 9: + return "netflowv9", uint32(version), nil + case 10: + return "ipfix", uint32(version), nil + default: + return "", 0, fmt.Errorf("identify flow: unsupported version %d", version) + } +} diff --git a/pkg/reflow/decode/helpers.go b/pkg/reflow/decode/helpers.go new file mode 100644 index 00000000..a551f259 --- /dev/null +++ b/pkg/reflow/decode/helpers.go @@ -0,0 +1,146 @@ +package decode + +import ( + "encoding/binary" + "fmt" + "net/netip" + + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func routerKey(evt *event.Event) string { + if evt.Source.Remote != "" { + return evt.Source.Remote + } + return evt.Source.Address +} + +// cloneEvent preserves source metadata and shared envelope fields while giving +// downstream decode paths an isolated Fields map to mutate. +func cloneEvent(evt *event.Event) *event.Event { + item := &event.Event{ + ReceivedAt: evt.ReceivedAt, + Source: evt.Source, + Message: evt.Message, + Payload: evt.Payload, + Packet: evt.Packet, + } + if evt.SFlow != nil { + copy := *evt.SFlow + item.SFlow = © + } + if evt.Aggregation != nil { + copy := *evt.Aggregation + item.Aggregation = © + } + if evt.Fields != nil { + item.Fields = make(map[string]any, len(evt.Fields)) + for k, v := range evt.Fields { + item.Fields[k] = v + } + } + return item +} + +// ensureFields lazily allocates the mutable field map for an event. +func ensureFields(evt *event.Event, capacity int) map[string]any { + if evt.Fields == nil { + evt.Fields = make(map[string]any, capacity) + } + return evt.Fields +} + +// decodeUint32 normalizes decoder field payloads that may arrive either as raw +// bytes or already-decoded integer types. +func decodeUint32(val any) uint32 { + switch v := val.(type) { + case []byte: + return uint32(decodeUint64(v)) + case uint32: + return v + case uint64: + return uint32(v) + default: + return 0 + } +} + +// fieldUint32 reads one field from a generic field map and normalizes it to u32. +func fieldUint32(fields map[string]any, key string) uint32 { + if fields == nil { + return 0 + } + val, ok := fields[key] + if !ok { + return 0 + } + return decodeUint32(val) +} + +// decodeUint64 handles big-endian raw field encodings with varying lengths. +func decodeUint64(val any) uint64 { + switch v := val.(type) { + case []byte: + switch len(v) { + case 1: + return uint64(v[0]) + case 2: + return uint64(binary.BigEndian.Uint16(v)) + case 4: + return uint64(binary.BigEndian.Uint32(v)) + case 8: + return binary.BigEndian.Uint64(v) + default: + var out uint64 + for _, b := range v { + out = (out << 8) | uint64(b) + } + return out + } + case uint64: + return v + case uint32: + return uint64(v) + default: + return 0 + } +} + +// decodeIPString converts raw IP bytes into Go's canonical string form. +func decodeIPString(val any) string { + raw, ok := val.([]byte) + if !ok { + return fmt.Sprint(val) + } + addr, ok := netip.AddrFromSlice(raw) + if !ok { + return "" + } + return addr.String() +} + +// ipSliceString is the byte-slice-only variant used by decoder paths that +// already know the source type. +func ipSliceString(raw []byte) string { + addr, ok := netip.AddrFromSlice(raw) + if !ok { + return "" + } + return addr.String() +} + +// flowTimeFromV5 reconstructs an absolute Unix millisecond timestamp from the +// exporter's uptime-relative NetFlow v5 timestamps. +func flowTimeFromV5(unixSecs, unixNSecs, sysUptime, switched uint32) int64 { + exportMs := int64(unixSecs)*1000 + int64(unixNSecs)/1_000_000 + uptimeMs := int64(sysUptime) + return exportMs - (uptimeMs - int64(switched)) +} + +// flowTimeFromV9 reconstructs an absolute Unix millisecond timestamp from the +// exporter's uptime-relative NetFlow v9 timestamps. +func flowTimeFromV9(sysUptime, unixSeconds, switched uint32) int64 { + exportMs := int64(unixSeconds) * 1000 + uptimeMs := int64(sysUptime) + return exportMs - (uptimeMs - int64(switched)) +} diff --git a/pkg/reflow/decode/netflow.go b/pkg/reflow/decode/netflow.go new file mode 100644 index 00000000..c26f66f8 --- /dev/null +++ b/pkg/reflow/decode/netflow.go @@ -0,0 +1,459 @@ +package decode + +import ( + "bytes" + "fmt" + + "github.com/netsampler/goflow2/v3/decoders/netflow" + "github.com/netsampler/goflow2/v3/decoders/netflowlegacy" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func (d *builtIn) decodeNetFlowV5(evt *event.Event, payload []byte) ([]*event.Event, error) { + packet := &netflowlegacy.PacketNetFlowV5{} + if err := netflowlegacy.DecodeMessageVersion(bytes.NewBuffer(payload), packet); err != nil { + return nil, fmt.Errorf("decode netflow v5: %w", err) + } + + out := make([]*event.Event, 0, len(packet.Records)) + for _, record := range packet.Records { + item := cloneEvent(evt) + fields := ensureFields(item, 32) + fields["flow_type"] = "netflowv5" + fields["flow_version"] = packet.Version + fields["record_kind"] = "packet" + fields["flow_sequence"] = packet.FlowSequence + fields["engine_type"] = uint32(packet.EngineType) + fields["engine_id"] = uint32(packet.EngineId) + fields["sampling_rate"] = uint32(packet.SamplingInterval) + fields["src_addr"] = fmt.Sprint(record.SrcAddr) + fields["dst_addr"] = fmt.Sprint(record.DstAddr) + fields["next_hop"] = fmt.Sprint(record.NextHop) + fields["src_port"] = uint32(record.SrcPort) + fields["dst_port"] = uint32(record.DstPort) + fields["proto"] = uint32(record.Proto) + fields["proto_name"] = ipProtocolName(uint32(record.Proto)) + fields["tcp_flags"] = uint32(record.TCPFlags) + fields["tos"] = uint32(record.Tos) + fields["bytes"] = int64(record.DOctets) + fields["packets"] = int64(record.DPkts) + fields["input_if"] = uint32(record.Input) + fields["output_if"] = uint32(record.Output) + fields["src_as"] = uint32(record.SrcAS) + fields["dst_as"] = uint32(record.DstAS) + fields["src_mask"] = uint32(record.SrcMask) + fields["dst_mask"] = uint32(record.DstMask) + fields["start_time_unix"] = flowTimeFromV5(packet.UnixSecs, packet.UnixNSecs, packet.SysUptime, record.First) + fields["end_time_unix"] = flowTimeFromV5(packet.UnixSecs, packet.UnixNSecs, packet.SysUptime, record.Last) + out = append(out, item) + } + return out, nil +} + +// decodeNetFlowV9 expands a packet into template, options, and data events and +// keeps a sampling-rate cache keyed by exporter identity. +func (d *builtIn) decodeNetFlowV9(evt *event.Event, payload []byte) ([]*event.Event, error) { + packet := &netflow.NFv9Packet{} + ctx := netflow.FlowContext{RouterKey: routerKey(evt)} + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), d.templates, ctx, packet, nil); err != nil { + return nil, fmt.Errorf("decode netflow v9: %w", err) + } + + var out []*event.Event + templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet := splitNetFlowV9Sets(packet) + samplingRate, found, err := searchNetFlowOptionDataSets(optionsDataFlowSet) + if err != nil { + return nil, fmt.Errorf("netflow v9 options data sets: %w", err) + } + if d.sampling != nil { + if found { + _ = d.sampling.Set(ctx, packet.Version, packet.SourceId, samplingRate) + } else if stored, ok, _ := d.sampling.Get(ctx, packet.Version, packet.SourceId); ok { + samplingRate = stored + } + } + out = append(out, d.templateEventsFromV9(evt, packet, templatesFlowSet, optionsTemplatesFlowSet)...) + out = append(out, d.optionsEventsFromV9(evt, packet, optionsDataFlowSet)...) + for _, flowSet := range packet.FlowSets { + dataSet, ok := flowSet.(netflow.DataFlowSet) + if !ok { + continue + } + for _, record := range dataSet.Records { + item := cloneEvent(evt) + fields := ensureFields(item, 16) + fields["flow_type"] = "netflowv9" + fields["flow_version"] = packet.Version + fields["record_kind"] = "packet" + d.mapDataFields(fields, record.Values, packet.SystemUptime, packet.UnixSeconds, true) + if fieldUint32(fields, "sampling_rate") == 0 && samplingRate != 0 { + fields["sampling_rate"] = samplingRate + } + out = append(out, item) + } + } + return out, nil +} + +// decodeIPFIX mirrors the NetFlow v9 path but uses the IPFIX observation-domain +// identity and template semantics. +func (d *builtIn) decodeIPFIX(evt *event.Event, payload []byte) ([]*event.Event, error) { + packet := &netflow.IPFIXPacket{} + ctx := netflow.FlowContext{RouterKey: routerKey(evt)} + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), d.templates, ctx, nil, packet); err != nil { + return nil, fmt.Errorf("decode ipfix: %w", err) + } + + var out []*event.Event + templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet := splitIPFIXSets(packet) + samplingRate, found, err := searchNetFlowOptionDataSets(optionsDataFlowSet) + if err != nil { + return nil, fmt.Errorf("ipfix options data sets: %w", err) + } + if d.sampling != nil { + if found { + _ = d.sampling.Set(ctx, packet.Version, packet.ObservationDomainId, samplingRate) + } else if stored, ok, _ := d.sampling.Get(ctx, packet.Version, packet.ObservationDomainId); ok { + samplingRate = stored + } + } + out = append(out, d.templateEventsFromIPFIX(evt, packet, templatesFlowSet, optionsTemplatesFlowSet)...) + out = append(out, d.optionsEventsFromIPFIX(evt, packet, optionsDataFlowSet)...) + for _, flowSet := range packet.FlowSets { + dataSet, ok := flowSet.(netflow.DataFlowSet) + if !ok { + continue + } + for _, record := range dataSet.Records { + item := cloneEvent(evt) + fields := ensureFields(item, 16) + fields["flow_type"] = "ipfix" + fields["flow_version"] = packet.Version + fields["record_kind"] = "packet" + d.mapDataFields(fields, record.Values, 0, 0, false) + if fieldUint32(fields, "sampling_rate") == 0 && samplingRate != 0 { + fields["sampling_rate"] = samplingRate + } + out = append(out, item) + } + } + return out, nil +} + +// splitNetFlowV9Sets separates template-bearing sets from data sets so they can +// produce different control and data events downstream. +func splitNetFlowV9Sets(packet *netflow.NFv9Packet) ([]netflow.TemplateFlowSet, []netflow.NFv9OptionsTemplateFlowSet, []netflow.OptionsDataFlowSet) { + var templatesFlowSet []netflow.TemplateFlowSet + var optionsTemplatesFlowSet []netflow.NFv9OptionsTemplateFlowSet + var optionsDataFlowSet []netflow.OptionsDataFlowSet + for _, flowSet := range packet.FlowSets { + switch tFlowSet := flowSet.(type) { + case netflow.TemplateFlowSet: + templatesFlowSet = append(templatesFlowSet, tFlowSet) + case netflow.NFv9OptionsTemplateFlowSet: + optionsTemplatesFlowSet = append(optionsTemplatesFlowSet, tFlowSet) + case netflow.OptionsDataFlowSet: + optionsDataFlowSet = append(optionsDataFlowSet, tFlowSet) + } + } + return templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet +} + +// splitIPFIXSets is the IPFIX equivalent of splitNetFlowV9Sets. +func splitIPFIXSets(packet *netflow.IPFIXPacket) ([]netflow.TemplateFlowSet, []netflow.IPFIXOptionsTemplateFlowSet, []netflow.OptionsDataFlowSet) { + var templatesFlowSet []netflow.TemplateFlowSet + var optionsTemplatesFlowSet []netflow.IPFIXOptionsTemplateFlowSet + var optionsDataFlowSet []netflow.OptionsDataFlowSet + for _, flowSet := range packet.FlowSets { + switch tFlowSet := flowSet.(type) { + case netflow.TemplateFlowSet: + templatesFlowSet = append(templatesFlowSet, tFlowSet) + case netflow.IPFIXOptionsTemplateFlowSet: + optionsTemplatesFlowSet = append(optionsTemplatesFlowSet, tFlowSet) + case netflow.OptionsDataFlowSet: + optionsDataFlowSet = append(optionsDataFlowSet, tFlowSet) + } + } + return templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet +} + +// searchNetFlowOptionDataSets scans common sampling-rate field IDs across +// NetFlow/IPFIX options data records. +func searchNetFlowOptionDataSets(dataFlowSet []netflow.OptionsDataFlowSet) (uint32, bool, error) { + var samplingRate uint32 + for _, dataFlowSetItem := range dataFlowSet { + for _, record := range dataFlowSetItem.Records { + if found, err := netFlowPopulate(record.OptionsValues, 305, &samplingRate); err != nil || found { + return samplingRate, found, err + } + if found, err := netFlowPopulate(record.OptionsValues, 50, &samplingRate); err != nil || found { + return samplingRate, found, err + } + if found, err := netFlowPopulate(record.OptionsValues, 34, &samplingRate); err != nil || found { + return samplingRate, found, err + } + } + } + return samplingRate, false, nil +} + +// netFlowPopulate copies one matching options field into the provided output. +func netFlowPopulate(dataFields []netflow.DataField, typeID uint16, out *uint32) (bool, error) { + for _, dataField := range dataFields { + if dataField.Type != typeID { + continue + } + *out = decodeUint32(dataField.Value) + return true, nil + } + return false, nil +} + +// templateEventsFromV9 turns v9 templates into explicit runtime events so +// encoders can forward or translate template state. +func (d *builtIn) templateEventsFromV9(base *event.Event, packet *netflow.NFv9Packet, templatesSets []netflow.TemplateFlowSet, optionsSets []netflow.NFv9OptionsTemplateFlowSet) []*event.Event { + var out []*event.Event + for _, set := range templatesSets { + for _, record := range set.Records { + item := cloneEvent(base) + fields := ensureFields(item, 8) + fields["flow_type"] = "netflowv9_template" + fields["flow_version"] = packet.Version + fields["record_kind"] = "template" + fields["template_id"] = uint32(record.TemplateId) + fields["field_count"] = uint32(record.FieldCount) + if d.embedTemplateFields { + fields["tflow.fields"] = d.templateFieldSpecs(record.Fields, true, false) + } + item.Payload = record + out = append(out, item) + } + } + for _, set := range optionsSets { + for _, record := range set.Records { + item := cloneEvent(base) + fields := ensureFields(item, 8) + fields["flow_type"] = "netflowv9_options_template" + fields["flow_version"] = packet.Version + fields["record_kind"] = "options_template" + fields["tflow_record_type"] = "options" + fields["template_id"] = uint32(record.TemplateId) + fields["scope_field_count"] = uint32(len(record.Scopes)) + fields["option_field_count"] = uint32(len(record.Options)) + if d.embedTemplateFields { + fields["tflow.scopes"] = d.templateFieldSpecs(record.Scopes, true, true) + fields["tflow.options"] = d.templateFieldSpecs(record.Options, true, false) + } + item.Payload = record + out = append(out, item) + } + } + return out +} + +// templateEventsFromIPFIX turns IPFIX templates into explicit runtime events. +func (d *builtIn) templateEventsFromIPFIX(base *event.Event, packet *netflow.IPFIXPacket, templatesSets []netflow.TemplateFlowSet, optionsSets []netflow.IPFIXOptionsTemplateFlowSet) []*event.Event { + var out []*event.Event + for _, set := range templatesSets { + for _, record := range set.Records { + item := cloneEvent(base) + fields := ensureFields(item, 8) + fields["flow_type"] = "ipfix_template" + fields["flow_version"] = packet.Version + fields["record_kind"] = "template" + fields["template_id"] = uint32(record.TemplateId) + fields["field_count"] = uint32(record.FieldCount) + if d.embedTemplateFields { + fields["tflow.fields"] = d.templateFieldSpecs(record.Fields, false, false) + } + item.Payload = record + out = append(out, item) + } + } + for _, set := range optionsSets { + for _, record := range set.Records { + item := cloneEvent(base) + fields := ensureFields(item, 8) + fields["flow_type"] = "ipfix_options_template" + fields["flow_version"] = packet.Version + fields["record_kind"] = "options_template" + fields["tflow_record_type"] = "options" + fields["template_id"] = uint32(record.TemplateId) + fields["scope_field_count"] = uint32(record.ScopeFieldCount) + fields["option_field_count"] = uint32(int(record.FieldCount) - int(record.ScopeFieldCount)) + if d.embedTemplateFields { + fields["tflow.scopes"] = d.templateFieldSpecs(record.Scopes, false, true) + fields["tflow.options"] = d.templateFieldSpecs(record.Options, false, false) + } + item.Payload = record + out = append(out, item) + } + } + return out +} + +func (d *builtIn) templateFieldSpecs(templateFields []netflow.Field, netflowV9 bool, scope bool) []map[string]any { + specs := make([]map[string]any, 0, len(templateFields)) + for _, field := range templateFields { + spec := map[string]any{ + "id": uint32(field.Type), + "length": uint32(field.Length), + "pen": field.Pen, + "pen_provided": field.PenProvided, + } + if catalogField, ok := d.catalog.lookupTemplateField(field, netflowV9, scope); ok { + spec["name"] = catalogFieldName(catalogField) + if catalogField.key != "" { + spec["key"] = catalogField.key + } + if catalogField.def.Type != "" { + spec["data_type"] = catalogField.def.Type + } + if catalogField.def.Format != "" { + spec["format"] = catalogField.def.Format + } + } + specs = append(specs, spec) + } + return specs +} + +func catalogFieldName(field decodeCatalogField) string { + if field.def.Name != "" { + return field.def.Name + } + return field.key +} + +// optionsEventsFromV9 wraps NetFlow v9 options records in runtime events. +func (d *builtIn) optionsEventsFromV9(base *event.Event, packet *netflow.NFv9Packet, optionsSets []netflow.OptionsDataFlowSet) []*event.Event { + return d.optionsEvents(base, "netflowv9_options_data", packet.Version, optionsSets, true) +} + +// optionsEventsFromIPFIX wraps IPFIX options records in runtime events. +func (d *builtIn) optionsEventsFromIPFIX(base *event.Event, packet *netflow.IPFIXPacket, optionsSets []netflow.OptionsDataFlowSet) []*event.Event { + return d.optionsEvents(base, "ipfix_options_data", packet.Version, optionsSets, false) +} + +// optionsEvents shares the event-building logic for NetFlow/IPFIX options data. +func (d *builtIn) optionsEvents(base *event.Event, flowType string, version uint16, optionsSets []netflow.OptionsDataFlowSet, netflowV9 bool) []*event.Event { + var out []*event.Event + for _, set := range optionsSets { + for _, record := range set.Records { + item := cloneEvent(base) + fields := ensureFields(item, 8) + fields["flow_type"] = flowType + fields["flow_version"] = version + fields["record_kind"] = "options_data" + fields["tflow_record_type"] = "options" + fields["template_id"] = uint32(set.Id) + scopeKeys := d.mapOptionsFields(fields, record.ScopesValues, netflowV9, true, "scope") + if len(scopeKeys) > 0 { + fields["tflow.scope"] = scopeKeys + } + for _, dataField := range record.OptionsValues { + if catalogField, ok := d.catalog.lookupOptions(dataField, netflowV9, false); ok { + applyCatalogDataField(fields, dataField, catalogField, 0, 0, netflowV9) + continue + } + switch dataField.Type { + case 34, 50, 305: + fields["sampling_rate"] = decodeUint32(dataField.Value) + default: + fields[fmt.Sprintf("option_%d", dataField.Type)] = decodeOptionFallback(dataField.Value) + } + } + item.Payload = record + out = append(out, item) + } + } + return out +} + +func (d *builtIn) mapOptionsFields(fields map[string]any, values []netflow.DataField, netflowV9 bool, scope bool, prefix string) []string { + keys := make([]string, 0, len(values)) + for _, dataField := range values { + if catalogField, ok := d.catalog.lookupOptions(dataField, netflowV9, scope); ok { + applyCatalogDataField(fields, dataField, catalogField, 0, 0, netflowV9) + keys = append(keys, catalogField.key) + continue + } + key := fmt.Sprintf("%s_%d", prefix, dataField.Type) + fields[key] = decodeOptionFallback(dataField.Value) + keys = append(keys, key) + } + return keys +} + +func decodeOptionFallback(val any) any { + switch v := val.(type) { + case []byte: + return cloneRawBytes(v) + default: + return v + } +} + +// mapDataFields keeps the first cut of canonical field mapping close to the +// decode boundary so later stages can remain protocol-agnostic. +func (d *builtIn) mapDataFields(fields map[string]any, values []netflow.DataField, sysUptime, unixSeconds uint32, netflowV9 bool) []string { + keys := make([]string, 0, len(values)) + for _, field := range values { + if d != nil && !d.catalog.empty() { + if catalogField, ok := d.catalog.lookup(field, netflowV9); ok { + applyCatalogDataField(fields, field, catalogField, sysUptime, unixSeconds, netflowV9) + keys = append(keys, catalogField.key) + } + continue + } + switch field.Type { + case 4: + fields["proto"] = decodeUint32(field.Value) + fields["proto_name"] = ipProtocolName(decodeUint32(field.Value)) + keys = append(keys, "proto") + case netflow.NFV9_FIELD_TCP_FLAGS: + fields["tcp_flags"] = decodeUint32(field.Value) + keys = append(keys, "tcp_flags") + case 7: + fields["src_port"] = decodeUint32(field.Value) + keys = append(keys, "src_port") + case 11: + fields["dst_port"] = decodeUint32(field.Value) + keys = append(keys, "dst_port") + case 8, 27: + setDecodedIPField(fields, "src_addr", field.Value) + keys = append(keys, "src_addr") + case 12, 28: + setDecodedIPField(fields, "dst_addr", field.Value) + keys = append(keys, "dst_addr") + case 1: + fields["bytes"] = int64(decodeUint64(field.Value)) + keys = append(keys, "bytes") + case 2: + fields["packets"] = int64(decodeUint64(field.Value)) + keys = append(keys, "packets") + case 10: + fields["input_if"] = decodeUint32(field.Value) + keys = append(keys, "input_if") + case 14: + fields["output_if"] = decodeUint32(field.Value) + keys = append(keys, "output_if") + case 34: + fields["sampling_rate"] = decodeUint32(field.Value) + keys = append(keys, "sampling_rate") + case netflow.NFV9_FIELD_FIRST_SWITCHED: + fields["start_time_unix"] = flowTimeFromV9(sysUptime, unixSeconds, decodeUint32(field.Value)) + keys = append(keys, "start_time_unix") + case netflow.NFV9_FIELD_LAST_SWITCHED: + fields["end_time_unix"] = flowTimeFromV9(sysUptime, unixSeconds, decodeUint32(field.Value)) + keys = append(keys, "end_time_unix") + case netflow.IPFIX_FIELD_flowStartMilliseconds: + fields["start_time_unix"] = int64(decodeUint64(field.Value)) + keys = append(keys, "start_time_unix") + case netflow.IPFIX_FIELD_flowEndMilliseconds: + fields["end_time_unix"] = int64(decodeUint64(field.Value)) + keys = append(keys, "end_time_unix") + } + } + return keys +} diff --git a/pkg/reflow/decode/netflow_test.go b/pkg/reflow/decode/netflow_test.go new file mode 100644 index 00000000..23aef18e --- /dev/null +++ b/pkg/reflow/decode/netflow_test.go @@ -0,0 +1,661 @@ +package decode + +import ( + "encoding/binary" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/decoders/netflow" + flowpb "github.com/netsampler/goflow2/v3/pb" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/encode" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "google.golang.org/protobuf/proto" +) + +func TestSearchNetFlowOptionDataSetsFindsSamplingRate(t *testing.T) { + rate, found, err := searchNetFlowOptionDataSets([]netflow.OptionsDataFlowSet{ + { + Records: []netflow.OptionsDataRecord{ + { + OptionsValues: []netflow.DataField{ + {Type: 34, Value: []byte{0x00, 0x00, 0x03, 0xe8}}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("searchNetFlowOptionDataSets returned error: %v", err) + } + if !found { + t.Fatalf("expected sampling rate to be found") + } + if rate != 1000 { + t.Fatalf("expected sampling rate 1000, got %d", rate) + } +} + +func TestTemplateAndOptionsEventsAreEmitted(t *testing.T) { + d := &builtIn{} + base := &event.Event{ + ReceivedAt: time.Unix(1, 0), + Source: event.SourceMetadata{Type: "flow"}, + } + packet := &netflow.IPFIXPacket{Version: 10} + + templateEvents := d.templateEventsFromIPFIX(base, packet, + []netflow.TemplateFlowSet{ + { + Records: []netflow.TemplateRecord{ + {TemplateId: 256, FieldCount: 3, Fields: []netflow.Field{{Type: 1, Length: 8}}}, + }, + }, + }, + []netflow.IPFIXOptionsTemplateFlowSet{ + { + Records: []netflow.IPFIXOptionsTemplateRecord{ + {TemplateId: 300, FieldCount: 2, ScopeFieldCount: 1}, + }, + }, + }, + ) + if len(templateEvents) != 2 { + t.Fatalf("expected 2 template events, got %d", len(templateEvents)) + } + if templateEvents[0].Fields["flow_type"] != "ipfix_template" { + t.Fatalf("expected first template event flow_type=ipfix_template, got %#v", templateEvents[0].Fields["flow_type"]) + } + if templateEvents[1].Fields["flow_type"] != "ipfix_options_template" { + t.Fatalf("expected second template event flow_type=ipfix_options_template, got %#v", templateEvents[1].Fields["flow_type"]) + } + if got := templateEvents[1].Fields["tflow_record_type"]; got != "options" { + t.Fatalf("expected options template tflow_record_type=options, got %#v", got) + } + if _, ok := templateEvents[0].Fields["tflow.fields"]; ok { + t.Fatalf("expected compact template event by default, got %#v", templateEvents[0].Fields["tflow.fields"]) + } + + optionsEvents := d.optionsEventsFromIPFIX(base, packet, []netflow.OptionsDataFlowSet{ + { + Records: []netflow.OptionsDataRecord{ + { + OptionsValues: []netflow.DataField{ + {Type: 34, Value: []byte{0x00, 0x00, 0x00, 0x64}}, + }, + }, + }, + }, + }) + if len(optionsEvents) != 1 { + t.Fatalf("expected 1 options event, got %d", len(optionsEvents)) + } + if optionsEvents[0].Fields["record_kind"] != "options_data" { + t.Fatalf("expected record_kind=options_data, got %#v", optionsEvents[0].Fields["record_kind"]) + } + if got := optionsEvents[0].Fields["tflow_record_type"]; got != "options" { + t.Fatalf("expected options data tflow_record_type=options, got %#v", got) + } + if got := optionsEvents[0].Fields["sampling_rate"]; got != uint32(100) { + t.Fatalf("expected sampling_rate=100, got %#v", got) + } +} + +func TestTemplateEventsEmbedFieldsWhenConfigured(t *testing.T) { + d := &builtIn{ + embedTemplateFields: true, + catalog: newDecodeCatalog(map[string]config.IPFIXFieldDefinition{ + "bytes": { + ID: 1, + Length: 8, + Type: "unsigned64", + Format: "delta", + }, + "custom_counter": { + Name: "customCounter", + ID: 4000, + PEN: 32473, + Length: 4, + Type: "unsigned32", + EnterpriseScoped: true, + }, + "input_if": { + ID: 10, + Length: 4, + Type: "unsigned32", + }, + "sampling_rate": { + ID: 34, + Length: 4, + Type: "unsigned32", + }, + }), + } + base := &event.Event{ + ReceivedAt: time.Unix(1, 0), + Source: event.SourceMetadata{Type: "flow"}, + } + packet := &netflow.IPFIXPacket{Version: 10} + + templateEvents := d.templateEventsFromIPFIX(base, packet, + []netflow.TemplateFlowSet{ + { + Records: []netflow.TemplateRecord{ + { + TemplateId: 256, + FieldCount: 3, + Fields: []netflow.Field{ + {Type: 1, Length: 8}, + {PenProvided: true, Type: 4000, Length: 4, Pen: 32473}, + {Type: 999, Length: 2}, + }, + }, + }, + }, + }, + []netflow.IPFIXOptionsTemplateFlowSet{ + { + Records: []netflow.IPFIXOptionsTemplateRecord{ + { + TemplateId: 300, + FieldCount: 2, + ScopeFieldCount: 1, + Scopes: []netflow.Field{{Type: 10, Length: 4}}, + Options: []netflow.Field{{Type: 34, Length: 4}}, + }, + }, + }, + }, + ) + if len(templateEvents) != 2 { + t.Fatalf("expected 2 template events, got %d", len(templateEvents)) + } + + fields, ok := templateEvents[0].Fields["tflow.fields"].([]map[string]any) + if !ok { + t.Fatalf("expected tflow.fields, got %#v", templateEvents[0].Fields["tflow.fields"]) + } + if len(fields) != 3 { + t.Fatalf("expected 3 embedded fields, got %#v", fields) + } + if fields[0]["name"] != "bytes" || fields[0]["data_type"] != "unsigned64" || fields[0]["format"] != "delta" { + t.Fatalf("expected catalog-enriched bytes field, got %#v", fields[0]) + } + if fields[1]["name"] != "customCounter" || fields[1]["key"] != "custom_counter" || fields[1]["pen"] != uint32(32473) { + t.Fatalf("expected enterprise catalog field, got %#v", fields[1]) + } + if _, ok := fields[2]["name"]; ok { + t.Fatalf("expected unknown field to stay raw, got %#v", fields[2]) + } + + scopes, ok := templateEvents[1].Fields["tflow.scopes"].([]map[string]any) + if !ok || len(scopes) != 1 || scopes[0]["name"] != "input_if" { + t.Fatalf("expected embedded scope field, got %#v", templateEvents[1].Fields["tflow.scopes"]) + } + options, ok := templateEvents[1].Fields["tflow.options"].([]map[string]any) + if !ok || len(options) != 1 || options[0]["name"] != "sampling_rate" { + t.Fatalf("expected embedded option field, got %#v", templateEvents[1].Fields["tflow.options"]) + } +} + +func TestDecodeNetFlowV5EmitsPacketMetadataAndCanonicalFields(t *testing.T) { + nfv5 := encode.NewNFv5Encoder(config.EncoderConfig{Type: "netflowv5"}) + payloads, err := nfv5.Encode(&event.Event{ + ReceivedAt: time.Unix(1700000000, 0).UTC(), + Fields: map[string]any{ + "src_addr": "192.0.2.10", + "dst_addr": "198.51.100.20", + "next_hop": "203.0.113.1", + "src_port": uint32(12345), + "dst_port": uint32(443), + "proto": uint32(6), + "tcp_flags": uint32(0x12), + "tos": uint32(184), + "bytes": int64(321), + "packets": int64(7), + "input_if": uint32(10), + "output_if": uint32(20), + "src_as": uint32(64512), + "dst_as": uint32(64513), + "src_mask": uint32(24), + "dst_mask": uint32(25), + "engine_type": uint32(1), + "engine_id": uint32(2), + "sampling_rate": uint32(100), + "start_time_unix": int64(1699999999000), + "end_time_unix": int64(1700000000000), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + dec := New() + defer dec.Close() + events, err := dec.Decode(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: payloads[0], + }) + if err != nil { + t.Fatalf("Decode returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected one NetFlow v5 event, got %d", len(events)) + } + fields := events[0].Fields + if fields["flow_type"] != "netflowv5" || fields["flow_version"] != uint16(5) || fields["record_kind"] != "packet" { + t.Fatalf("expected NetFlow v5 packet metadata, got %#v", fields) + } + if fields["src_addr"] != "192.0.2.10" || fields["dst_addr"] != "198.51.100.20" || fields["next_hop"] != "203.0.113.1" { + t.Fatalf("expected dotted IPv4 fields, got src=%#v dst=%#v next_hop=%#v", fields["src_addr"], fields["dst_addr"], fields["next_hop"]) + } + for key, want := range map[string]any{ + "sampling_rate": uint32(100), + "engine_type": uint32(1), + "engine_id": uint32(2), + "tos": uint32(184), + "src_as": uint32(64512), + "dst_as": uint32(64513), + "src_mask": uint32(24), + "dst_mask": uint32(25), + } { + if fields[key] != want { + t.Fatalf("expected %s=%#v, got %#v", key, want, fields[key]) + } + } + + protobufEnc, err := encode.New(config.EncoderConfig{Type: "protobuf"}) + if err != nil { + t.Fatalf("New protobuf encoder returned error: %v", err) + } + protobufPayloads, err := protobufEnc.Encode(events[0]) + if err != nil { + t.Fatalf("protobuf Encode returned error: %v", err) + } + var msg flowpb.FlowMessage + if err := proto.Unmarshal(protobufPayloads[0], &msg); err != nil { + t.Fatalf("unmarshal protobuf flow message: %v", err) + } + if msg.Type != flowpb.FlowMessage_NETFLOW_V5 { + t.Fatalf("expected protobuf type NETFLOW_V5, got %v", msg.Type) + } + if msg.Bytes != 321 || msg.Packets != 7 { + t.Fatalf("expected protobuf counters bytes=321 packets=7, got bytes=%d packets=%d", msg.Bytes, msg.Packets) + } +} + +func TestMapDataFieldsUsesSharedCatalog(t *testing.T) { + d := &builtIn{catalog: newDecodeCatalog(map[string]config.IPFIXFieldDefinition{ + "src_addr": {ID: 8, Length: 4, Type: "ipv4Address"}, + "dst_addr": {ID: 12, Length: 4, Type: "ipv4Address"}, + "proto": {ID: 4, Length: 1, Type: "unsigned8"}, + "src_port": {ID: 7, Length: 2, Type: "unsigned16"}, + "dst_port": {ID: 11, Length: 2, Type: "unsigned16"}, + "bytes": {ID: 1, Length: 8, Type: "unsigned64"}, + "packets": {ID: 2, Length: 8, Type: "unsigned64"}, + "start_time_unix": {ID: 152, Length: 8, Type: "unsigned64"}, + "tenant_id": { + ID: 4000, + PEN: 32473, + EnterpriseScoped: true, + Length: 8, + Type: "unsigned64", + }, + })} + fields := map[string]any{} + d.mapDataFields(fields, []netflow.DataField{ + {Type: 27, Value: []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}}, + {Type: 12, Value: []byte{192, 0, 2, 10}}, + {Type: 4, Value: []byte{6}}, + {Type: 7, Value: []byte{0x1f, 0x90}}, + {Type: 11, Value: []byte{0x01, 0xbb}}, + {Type: 1, Value: testUint64(1234, 8)}, + {Type: 2, Value: testUint64(12, 8)}, + {Type: 152, Value: testUint64(1714860000000, 8)}, + {Type: 4000, PenProvided: true, Pen: 32473, Value: testUint64(99, 8)}, + }, 0, 0, false) + + if fields["src_addr"] != "2001:db8::1" { + t.Fatalf("expected IPv6 src_addr alias to decode, got %#v", fields["src_addr"]) + } + if fields["dst_addr"] != "192.0.2.10" { + t.Fatalf("expected dst_addr to decode, got %#v", fields["dst_addr"]) + } + if fields["proto"] != uint32(6) || fields["proto_name"] != "tcp" { + t.Fatalf("expected proto and proto_name, got proto=%#v proto_name=%#v", fields["proto"], fields["proto_name"]) + } + if fields["src_port"] != uint32(8080) || fields["dst_port"] != uint32(443) { + t.Fatalf("expected ports from catalog, got src=%#v dst=%#v", fields["src_port"], fields["dst_port"]) + } + if fields["bytes"] != int64(1234) || fields["packets"] != int64(12) { + t.Fatalf("expected bytes/packets int64 values, got bytes=%#v packets=%#v", fields["bytes"], fields["packets"]) + } + if fields["start_time_unix"] != int64(1714860000000) { + t.Fatalf("expected start_time_unix int64 milliseconds, got %#v", fields["start_time_unix"]) + } + if fields["tenant_id"] != uint64(99) { + t.Fatalf("expected enterprise custom field to decode, got %#v", fields["tenant_id"]) + } +} + +func TestMapDataFieldsUsesNetFlowV9FieldIDs(t *testing.T) { + d := &builtIn{catalog: newDecodeCatalog(map[string]config.IPFIXFieldDefinition{ + "custom_counter": {ID: 77, Length: 4, Type: "unsigned32"}, + "start_time_unix": { + ID: 152, + Length: 8, + Type: "unsigned64", + }, + })} + fields := map[string]any{} + d.mapDataFields(fields, []netflow.DataField{ + {Type: 77, Value: []byte{0, 0, 0, 42}}, + {Type: netflow.NFV9_FIELD_FIRST_SWITCHED, Value: []byte{0, 0, 0x03, 0xe8}}, + }, 2000, 100, true) + + if fields["custom_counter"] != uint32(42) { + t.Fatalf("expected custom_counter from NetFlow v9 field id, got %#v", fields["custom_counter"]) + } + if fields["start_time_unix"] != int64(99000) { + t.Fatalf("expected uptime-relative FIRST_SWITCHED conversion, got %#v", fields["start_time_unix"]) + } +} + +func TestOptionsEventsUseCatalogForScopesAndOptions(t *testing.T) { + d := &builtIn{catalog: newDecodeCatalog(map[string]config.IPFIXFieldDefinition{ + "packets": {ID: 2, Length: 8, Type: "unsigned64"}, + "if_index": {ID: 10, Length: 4, Type: "unsigned32"}, + "interface_name": {ID: 82, Length: 0xffff, Type: "string"}, + })} + base := &event.Event{ + ReceivedAt: time.Unix(1, 0), + Source: event.SourceMetadata{Type: "flow"}, + } + + ipfixEvents := d.optionsEventsFromIPFIX(base, &netflow.IPFIXPacket{Version: 10}, []netflow.OptionsDataFlowSet{ + { + FlowSetHeader: netflow.FlowSetHeader{Id: 1300}, + Records: []netflow.OptionsDataRecord{ + { + ScopesValues: []netflow.DataField{{Type: 10, Value: []byte{0, 0, 0, 3}}}, + OptionsValues: []netflow.DataField{{Type: 82, Value: []byte("eth0")}}, + }, + }, + }, + }) + if len(ipfixEvents) != 1 { + t.Fatalf("expected one IPFIX options event, got %d", len(ipfixEvents)) + } + if ipfixEvents[0].Fields["if_index"] != uint32(3) || ipfixEvents[0].Fields["interface_name"] != "eth0" { + t.Fatalf("expected IPFIX options catalog fields, got %#v", ipfixEvents[0].Fields) + } + if got := ipfixEvents[0].Fields["tflow_record_type"]; got != "options" { + t.Fatalf("expected IPFIX options tflow_record_type=options, got %#v", got) + } + + v9Events := d.optionsEventsFromV9(base, &netflow.NFv9Packet{Version: 9}, []netflow.OptionsDataFlowSet{ + { + FlowSetHeader: netflow.FlowSetHeader{Id: 1300}, + Records: []netflow.OptionsDataRecord{ + { + ScopesValues: []netflow.DataField{{Type: 2, Value: []byte{0, 0, 0, 4}}}, + OptionsValues: []netflow.DataField{{Type: 82, Value: []byte("eth1")}}, + }, + }, + }, + }) + if len(v9Events) != 1 { + t.Fatalf("expected one NetFlow v9 options event, got %d", len(v9Events)) + } + if v9Events[0].Fields["if_index"] != uint32(4) || v9Events[0].Fields["interface_name"] != "eth1" { + t.Fatalf("expected NetFlow v9 options catalog fields, got %#v", v9Events[0].Fields) + } + if got := v9Events[0].Fields["tflow_record_type"]; got != "options" { + t.Fatalf("expected NetFlow v9 options tflow_record_type=options, got %#v", got) + } + if _, ok := v9Events[0].Fields["packets"]; ok { + t.Fatalf("expected NetFlow v9 scope id 2 not to decode as packets, got %#v", v9Events[0].Fields) + } +} + +func TestMapDataFieldsPopulatesTCPFlagsWithoutCatalog(t *testing.T) { + d := &builtIn{} + fields := map[string]any{} + d.mapDataFields(fields, []netflow.DataField{ + {Type: netflow.NFV9_FIELD_TCP_FLAGS, Value: []byte{0x12}}, + }, 0, 0, true) + + if fields["tcp_flags"] != uint32(0x12) { + t.Fatalf("expected tcp_flags to decode from field 6, got %#v", fields["tcp_flags"]) + } +} + +func TestMapDataFieldsUsesCatalogAsSourceOfTruth(t *testing.T) { + d := &builtIn{catalog: newDecodeCatalog(map[string]config.IPFIXFieldDefinition{ + "interface_name": {ID: 82, Type: "string"}, + })} + fields := map[string]any{} + d.mapDataFields(fields, []netflow.DataField{ + {Type: netflow.NFV9_FIELD_TCP_FLAGS, Value: []byte{0x12}}, + }, 0, 0, true) + + if _, ok := fields["tcp_flags"]; ok { + t.Fatalf("expected unmapped tcp_flags to be omitted when catalog is present, got %#v", fields["tcp_flags"]) + } +} + +func TestMapDataFieldsDecodesCatalogStringsAndUnknownTypes(t *testing.T) { + d := &builtIn{catalog: newDecodeCatalog(map[string]config.IPFIXFieldDefinition{ + "interface_name": {ID: netflow.IPFIX_FIELD_interfaceName, Type: "string"}, + "mystery": {ID: 5000, Type: "frobnitz"}, + })} + fields := map[string]any{} + keys := d.mapDataFields(fields, []netflow.DataField{ + {Type: netflow.IPFIX_FIELD_interfaceName, Value: []byte("eth0")}, + {Type: 5000, Value: []byte{1, 2, 3}}, + }, 0, 0, false) + + if fields["interface_name"] != "eth0" { + t.Fatalf("expected interface_name string decode, got %#v", fields["interface_name"]) + } + if fields["mystery"] != "AQID" { + t.Fatalf("expected unknown catalog type as base64, got %#v", fields["mystery"]) + } + if !reflect.DeepEqual(keys, []string{"interface_name", "mystery"}) { + t.Fatalf("expected decoded keys, got %#v", keys) + } +} + +func TestOptionsEventsDecodeScopeAndOptionFields(t *testing.T) { + d := &builtIn{catalog: newDecodeCatalog(map[string]config.IPFIXFieldDefinition{ + "observation_domain_id": {ID: netflow.IPFIX_FIELD_observationDomainId, Type: "unsigned32"}, + "interface_name": {ID: netflow.IPFIX_FIELD_interfaceName, Type: "string"}, + "sampling_rate": {ID: netflow.IPFIX_FIELD_samplingInterval, Type: "unsigned32"}, + })} + base := &event.Event{ + ReceivedAt: time.Unix(1, 0), + Source: event.SourceMetadata{Type: "flow"}, + } + events := d.optionsEventsFromIPFIX(base, &netflow.IPFIXPacket{Version: 10}, []netflow.OptionsDataFlowSet{ + { + FlowSetHeader: netflow.FlowSetHeader{Id: 1024}, + Records: []netflow.OptionsDataRecord{ + { + ScopesValues: []netflow.DataField{ + {Type: netflow.IPFIX_FIELD_observationDomainId, Value: []byte{0, 0, 0, 42}}, + }, + OptionsValues: []netflow.DataField{ + {Type: netflow.IPFIX_FIELD_interfaceName, Value: []byte("eth0")}, + {Type: netflow.IPFIX_FIELD_samplingInterval, Value: []byte{0, 0, 0, 100}}, + }, + }, + }, + }, + }) + + if len(events) != 1 { + t.Fatalf("expected one options event, got %d", len(events)) + } + fields := events[0].Fields + if fields["observation_domain_id"] != uint32(42) { + t.Fatalf("expected decoded scope observation_domain_id, got %#v", fields["observation_domain_id"]) + } + if fields["interface_name"] != "eth0" { + t.Fatalf("expected decoded option interface_name, got %#v", fields["interface_name"]) + } + if fields["sampling_rate"] != uint32(100) { + t.Fatalf("expected decoded sampling_rate, got %#v", fields["sampling_rate"]) + } + if !reflect.DeepEqual(fields["tflow.scope"], []string{"observation_domain_id"}) { + t.Fatalf("expected tflow.scope to list scope keys, got %#v", fields["tflow.scope"]) + } +} + +func TestIPFIXCustomCatalogFieldRoundTrip(t *testing.T) { + catalog := map[string]config.IPFIXFieldDefinition{ + "bytes": {ID: 1, Length: 8, Type: "unsigned64"}, + "tenant_id": { + ID: 4000, + PEN: 32473, + EnterpriseScoped: true, + Length: 8, + Type: "unsigned64", + }, + } + enc := encode.NewIPFIXEncoder(config.EncoderConfig{ + Type: "ipfix", + TemplatedFlow: config.TemplatedFlowConfig{ + TemplateBaseID: 256, + Data: config.TemplatedFlowDataConfig{ + Select: []string{"bytes", "tenant_id"}, + Catalog: catalog, + }, + }, + }) + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(1, 0), + Fields: map[string]any{ + "bytes": int64(1234), + "tenant_id": uint64(99), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + dec := NewWithCatalog(catalog) + defer dec.Close() + var flowFields map[string]any + for _, payload := range payloads { + events, err := dec.Decode(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: payload, + }) + if err != nil { + t.Fatalf("Decode returned error: %v", err) + } + for _, item := range events { + if item.Fields["flow_type"] == "ipfix" { + flowFields = item.Fields + } + } + } + if flowFields == nil { + t.Fatalf("expected decoded IPFIX data event") + } + if flowFields["bytes"] != int64(1234) { + t.Fatalf("expected bytes to survive round trip, got %#v", flowFields["bytes"]) + } + if flowFields["tenant_id"] != uint64(99) { + t.Fatalf("expected tenant_id to survive round trip, got %#v", flowFields["tenant_id"]) + } +} + +func TestDecoderCachePersistsTemplatesAcrossRestart(t *testing.T) { + catalog := map[string]config.IPFIXFieldDefinition{ + "bytes": {ID: 1, Length: 8, Type: "unsigned64"}, + } + enc := encode.NewIPFIXEncoder(config.EncoderConfig{ + Type: "ipfix", + TemplatedFlow: config.TemplatedFlowConfig{ + TemplateBaseID: 256, + TemplateRefresh: 60000, + Data: config.TemplatedFlowDataConfig{ + Select: []string{"bytes"}, + Catalog: catalog, + }, + }, + }) + + firstPayloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(1, 0), + Fields: map[string]any{ + "bytes": int64(1234), + }, + }) + if err != nil { + t.Fatalf("first Encode returned error: %v", err) + } + secondPayloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(2, 0), + Fields: map[string]any{ + "bytes": int64(5678), + }, + }) + if err != nil { + t.Fatalf("second Encode returned error: %v", err) + } + + cachePath := filepath.Join(t.TempDir(), "reflow-cache.json") + dec, err := NewWithOptions(Options{ + Catalog: catalog, + CachePath: cachePath, + CacheInterval: time.Hour, + }) + if err != nil { + t.Fatalf("NewWithOptions returned error: %v", err) + } + for _, payload := range firstPayloads { + if _, err := dec.Decode(&event.Event{Source: event.SourceMetadata{Type: "flow"}, Payload: payload}); err != nil { + t.Fatalf("Decode first payload returned error: %v", err) + } + } + dec.Close() + + restarted, err := NewWithOptions(Options{ + Catalog: catalog, + CachePath: cachePath, + CacheInterval: time.Hour, + }) + if err != nil { + t.Fatalf("NewWithOptions after restart returned error: %v", err) + } + defer restarted.Close() + + var flowFields map[string]any + for _, payload := range secondPayloads { + events, err := restarted.Decode(&event.Event{Source: event.SourceMetadata{Type: "flow"}, Payload: payload}) + if err != nil { + t.Fatalf("Decode cached payload returned error: %v", err) + } + for _, item := range events { + if item.Fields["flow_type"] == "ipfix" { + flowFields = item.Fields + } + } + } + if flowFields == nil { + t.Fatalf("expected cached decoder to decode IPFIX data") + } + if flowFields["bytes"] != int64(5678) { + t.Fatalf("expected cached bytes 5678, got %#v", flowFields["bytes"]) + } +} + +func testUint64(v uint64, size int) []byte { + raw := make([]byte, 8) + binary.BigEndian.PutUint64(raw, v) + return raw[8-size:] +} diff --git a/pkg/reflow/decode/protocol.go b/pkg/reflow/decode/protocol.go new file mode 100644 index 00000000..00c09752 --- /dev/null +++ b/pkg/reflow/decode/protocol.go @@ -0,0 +1,28 @@ +package decode + +func ipProtocolName(proto uint32) string { + switch proto { + case 1: + return "icmp" + case 2: + return "igmp" + case 6: + return "tcp" + case 17: + return "udp" + case 41: + return "ipv6" + case 47: + return "gre" + case 50: + return "esp" + case 51: + return "ah" + case 58: + return "icmpv6" + case 132: + return "sctp" + default: + return "" + } +} diff --git a/pkg/reflow/decode/sflow.go b/pkg/reflow/decode/sflow.go new file mode 100644 index 00000000..ee7e6d93 --- /dev/null +++ b/pkg/reflow/decode/sflow.go @@ -0,0 +1,387 @@ +package decode + +import ( + "bytes" + "fmt" + "net/netip" + + "github.com/netsampler/goflow2/v3/decoders/sflow" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +const sflowFlowRecordsInternalKey = "sflow_flow_records" +const sflowSampleInternalKey = "sflow_sample" +const sflowCounterRecordsInternalKey = "sflow_counter_records" + +func (d *builtIn) decodeSFlow(evt *event.Event, payload []byte, version uint32) ([]*event.Event, error) { + packet := &sflow.Packet{} + if err := sflow.DecodeMessageVersion(bytes.NewBuffer(payload), packet); err != nil { + return nil, fmt.Errorf("decode sflow: %w", err) + } + + out := make([]*event.Event, 0, len(packet.Samples)) + for _, sample := range packet.Samples { + switch s := sample.(type) { + case sflow.FlowSample: + out = append(out, d.eventFromSFlowSample(evt, packet, s)) + case *sflow.FlowSample: + out = append(out, d.eventFromSFlowSample(evt, packet, *s)) + case sflow.ExpandedFlowSample: + out = append(out, d.eventFromExpandedSFlowSample(evt, packet, s)) + case *sflow.ExpandedFlowSample: + out = append(out, d.eventFromExpandedSFlowSample(evt, packet, *s)) + case sflow.CounterSample: + out = append(out, d.eventsFromSFlowCounterSample(evt, packet, s)...) + case *sflow.CounterSample: + out = append(out, d.eventsFromSFlowCounterSample(evt, packet, *s)...) + case sflow.RawSample: + out = append(out, d.eventFromRawSFlowSample(evt, packet, s)) + case *sflow.RawSample: + out = append(out, d.eventFromRawSFlowSample(evt, packet, *s)) + } + } + + if len(out) == 0 { + base := cloneEvent(evt) + base.Fields = ensureFields(base, 3) + base.Fields["flow_type"] = "sflow" + base.Fields["flow_version"] = version + return []*event.Event{base}, nil + } + for _, item := range out { + ensureSFlowDecodedFields(item, packet.Version) + } + return out, nil +} + +func ensureSFlowDecodedFields(evt *event.Event, version uint32) { + fields := ensureFields(evt, 2) + if _, ok := fields["flow_type"]; !ok { + fields["flow_type"] = "sflow" + } + if _, ok := fields["flow_version"]; !ok { + fields["flow_version"] = version + } +} + +// eventFromSFlowSample converts one sFlow flow sample into the runtime's +// canonical field map while retaining original sFlow metadata. +func (d *builtIn) eventFromSFlowSample(base *event.Event, packet *sflow.Packet, sample sflow.FlowSample) *event.Event { + evt := cloneEvent(base) + fields := ensureFields(evt, 16) + agentIP := ipSliceString(packet.AgentIP) + evt.SFlow = &event.SFlowMetadata{ + AgentIP: agentIP, + SubAgentID: packet.SubAgentId, + SequenceNumber: packet.SequenceNumber, + Uptime: packet.Uptime, + SourceID: sample.Header.SourceIdValue, + SamplingRate: sample.SamplingRate, + SamplePool: sample.SamplePool, + Drops: sample.Drops, + } + fields["flow_type"] = "sflow" + fields["flow_version"] = packet.Version + fields["agent_ip"] = agentIP + fields["sub_agent_id"] = packet.SubAgentId + fields["source_id"] = sample.Header.SourceIdValue + fields["sampling_rate"] = sample.SamplingRate + fields["sample_pool"] = sample.SamplePool + fields["drops"] = sample.Drops + fields["input_if"] = sample.Input + fields["output_if"] = sample.Output + fields["packets"] = int64(1) + + for _, record := range sample.Records { + if shouldPreserveSFlowFlowRecord(record) { + preserveSFlowFlowRecord(evt, record) + } + switch data := record.Data.(type) { + case sflow.SampledHeader: + fields["protocol"] = data.Protocol + fields["frame_length"] = data.FrameLength + fields["stripped"] = data.Stripped + fields["original_length"] = data.OriginalLength + fields["record_kind"] = "packet" + // Keep the raw sampled header as bytes so downstream packet handling can + // treat sFlow packet samples and source.type=bytes events consistently. + fields["header_data"] = append([]byte(nil), data.HeaderData...) + fields["bytes"] = int64(data.FrameLength) + case sflow.SampledIPv4: + fields["src_addr"] = string(data.SrcIP) + fields["dst_addr"] = string(data.DstIP) + fields["src_port"] = data.SrcPort + fields["dst_port"] = data.DstPort + fields["proto"] = data.Protocol + fields["proto_name"] = ipProtocolName(uint32(data.Protocol)) + fields["bytes"] = int64(data.Length) + case sflow.SampledIPv6: + fields["src_addr"] = string(data.SrcIP) + fields["dst_addr"] = string(data.DstIP) + fields["src_port"] = data.SrcPort + fields["dst_port"] = data.DstPort + fields["proto"] = data.Protocol + fields["proto_name"] = ipProtocolName(uint32(data.Protocol)) + fields["bytes"] = int64(data.Length) + case sflow.ExtendedNAT: + applySFlowExtendedNATFields(fields, data) + case sflow.ExtendedMPLS: + applySFlowExtendedMPLSFields(fields, data) + case sflow.ExtendedMPLSTunnel: + fields["mpls_tunnel_lsp_name"] = data.TunnelLSPName + fields["mpls_tunnel_id"] = data.TunnelID + fields["mpls_tunnel_cos"] = data.TunnelCOS + case sflow.ExtendedMPLSVC: + fields["mpls_vc_instance_name"] = data.VCInstanceName + fields["mpls_vll_vc_id"] = data.VLLVCID + fields["mpls_vc_label_cos"] = data.VCLabelCOS + case sflow.ExtendedMPLSFTN: + fields["mpls_ftn_descr"] = data.MPLSFTNDescr + fields["mpls_ftn_mask"] = data.MPLSFTNMask + case sflow.ExtendedMPLSLDPFEC: + fields["mpls_fec_addr_prefix_length"] = data.MPLSFecAddrPrefixLength + } + } + + return evt +} + +// eventFromExpandedSFlowSample handles the expanded sFlow format, which carries +// the same logical data with explicit interface encodings. +func (d *builtIn) eventFromExpandedSFlowSample(base *event.Event, packet *sflow.Packet, sample sflow.ExpandedFlowSample) *event.Event { + evt := cloneEvent(base) + fields := ensureFields(evt, 16) + agentIP := ipSliceString(packet.AgentIP) + evt.SFlow = &event.SFlowMetadata{ + AgentIP: agentIP, + SubAgentID: packet.SubAgentId, + SequenceNumber: packet.SequenceNumber, + Uptime: packet.Uptime, + SourceID: sample.Header.SourceIdValue, + SamplingRate: sample.SamplingRate, + SamplePool: sample.SamplePool, + Drops: sample.Drops, + } + fields["flow_type"] = "sflow" + fields["flow_version"] = packet.Version + fields["agent_ip"] = agentIP + fields["sub_agent_id"] = packet.SubAgentId + fields["source_id"] = sample.Header.SourceIdValue + fields["sampling_rate"] = sample.SamplingRate + fields["sample_pool"] = sample.SamplePool + fields["drops"] = sample.Drops + fields["input_if"] = sample.InputIfValue + fields["output_if"] = sample.OutputIfValue + fields["packets"] = int64(1) + + for _, record := range sample.Records { + if shouldPreserveSFlowFlowRecord(record) { + preserveSFlowFlowRecord(evt, record) + } + if data, ok := record.Data.(sflow.SampledHeader); ok { + fields["protocol"] = data.Protocol + fields["frame_length"] = data.FrameLength + fields["stripped"] = data.Stripped + fields["original_length"] = data.OriginalLength + fields["record_kind"] = "packet" + fields["header_data"] = append([]byte(nil), data.HeaderData...) + fields["bytes"] = int64(data.FrameLength) + continue + } + applySFlowExtendedRecordFields(fields, record.Data) + } + + return evt +} + +func (d *builtIn) eventFromRawSFlowSample(base *event.Event, packet *sflow.Packet, sample sflow.RawSample) *event.Event { + evt := cloneEvent(base) + fields := ensureFields(evt, 8) + agentIP := ipSliceString(packet.AgentIP) + evt.SFlow = &event.SFlowMetadata{ + AgentIP: agentIP, + SubAgentID: packet.SubAgentId, + SequenceNumber: packet.SequenceNumber, + Uptime: packet.Uptime, + } + fields["flow_type"] = "sflow" + fields["flow_version"] = packet.Version + fields["agent_ip"] = agentIP + fields["sub_agent_id"] = packet.SubAgentId + fields["record_kind"] = "sflow_raw_sample" + if evt.Internal == nil { + evt.Internal = make(map[string]any, 1) + } + evt.Internal[sflowSampleInternalKey] = sample + return evt +} + +// eventsFromSFlowCounterSample emits one event per interface counter record so +// later stages can aggregate or re-encode them independently. +func (d *builtIn) eventsFromSFlowCounterSample(base *event.Event, packet *sflow.Packet, sample sflow.CounterSample) []*event.Event { + agentIP := ipSliceString(packet.AgentIP) + out := make([]*event.Event, 0, len(sample.Records)) + for _, record := range sample.Records { + evt := cloneEvent(base) + fields := ensureFields(evt, 24) + evt.SFlow = &event.SFlowMetadata{ + AgentIP: agentIP, + SubAgentID: packet.SubAgentId, + SequenceNumber: packet.SequenceNumber, + Uptime: packet.Uptime, + SourceID: sample.Header.SourceIdValue, + } + fields["flow_type"] = "sflow" + fields["flow_version"] = packet.Version + fields["counter_type"] = "sflow" + fields["sflow_version"] = packet.Version + fields["agent_ip"] = agentIP + fields["sub_agent_id"] = packet.SubAgentId + fields["source_id_type"] = sample.Header.SourceIdType + fields["source_id"] = sample.Header.SourceIdValue + fields["counter_record_format"] = record.Header.DataFormat + if sample.Header.Format == sflow.SAMPLE_FORMAT_EXPANDED_COUNTER { + fields["counter_format"] = "expanded" + } + applySFlowCounterRecordFields(evt, fields, record) + out = append(out, evt) + } + return out +} + +func preserveSFlowCounterRecord(evt *event.Event, record sflow.CounterRecord) { + if evt.Internal == nil { + evt.Internal = make(map[string]any, 1) + } + records, _ := evt.Internal[sflowCounterRecordsInternalKey].([]sflow.CounterRecord) + evt.Internal[sflowCounterRecordsInternalKey] = append(records, record) +} + +func applySFlowCounterRecordFields(evt *event.Event, fields map[string]any, record sflow.CounterRecord) { + switch data := record.Data.(type) { + case sflow.IfCounters: + fields["record_kind"] = "interface_counter" + applySFlowIfCounterFields(fields, data) + case sflow.EthernetCounters: + fields["record_kind"] = "ethernet_counter" + applySFlowEthernetCounterFields(fields, data) + default: + fields["record_kind"] = "counter_record" + preserveSFlowCounterRecord(evt, record) + } +} + +func applySFlowIfCounterFields(fields map[string]any, data sflow.IfCounters) { + fields["if_index"] = data.IfIndex + fields["if_type"] = data.IfType + fields["if_speed"] = data.IfSpeed + fields["if_direction"] = data.IfDirection + fields["if_status"] = data.IfStatus + fields["if_in_octets"] = data.IfInOctets + fields["if_in_ucast_pkts"] = data.IfInUcastPkts + fields["if_in_multicast_pkts"] = data.IfInMulticastPkts + fields["if_in_broadcast_pkts"] = data.IfInBroadcastPkts + fields["if_in_discards"] = data.IfInDiscards + fields["if_in_errors"] = data.IfInErrors + fields["if_in_unknown_protos"] = data.IfInUnknownProtos + fields["if_out_octets"] = data.IfOutOctets + fields["if_out_ucast_pkts"] = data.IfOutUcastPkts + fields["if_out_multicast_pkts"] = data.IfOutMulticastPkts + fields["if_out_broadcast_pkts"] = data.IfOutBroadcastPkts + fields["if_out_discards"] = data.IfOutDiscards + fields["if_out_errors"] = data.IfOutErrors + fields["if_promiscuous_mode"] = data.IfPromiscuousMode +} + +func applySFlowEthernetCounterFields(fields map[string]any, data sflow.EthernetCounters) { + fields["dot3_stats_alignment_errors"] = data.Dot3StatsAlignmentErrors + fields["dot3_stats_fcs_errors"] = data.Dot3StatsFCSErrors + fields["dot3_stats_single_collision_frames"] = data.Dot3StatsSingleCollisionFrames + fields["dot3_stats_multiple_collision_frames"] = data.Dot3StatsMultipleCollisionFrames + fields["dot3_stats_sqe_test_errors"] = data.Dot3StatsSQETestErrors + fields["dot3_stats_deferred_transmissions"] = data.Dot3StatsDeferredTransmissions + fields["dot3_stats_late_collisions"] = data.Dot3StatsLateCollisions + fields["dot3_stats_excessive_collisions"] = data.Dot3StatsExcessiveCollisions + fields["dot3_stats_internal_mac_transmit_errors"] = data.Dot3StatsInternalMacTransmitErrors + fields["dot3_stats_carrier_sense_errors"] = data.Dot3StatsCarrierSenseErrors + fields["dot3_stats_frame_too_longs"] = data.Dot3StatsFrameTooLongs + fields["dot3_stats_internal_mac_receive_errors"] = data.Dot3StatsInternalMacReceiveErrors + fields["dot3_stats_symbol_errors"] = data.Dot3StatsSymbolErrors +} + +func shouldPreserveSFlowFlowRecord(record sflow.FlowRecord) bool { + switch record.Data.(type) { + case sflow.SampledHeader, sflow.SampledEthernet, sflow.SampledIPv4, sflow.SampledIPv6: + return false + default: + return true + } +} + +func preserveSFlowFlowRecord(evt *event.Event, record sflow.FlowRecord) { + if evt.Internal == nil { + evt.Internal = make(map[string]any, 1) + } + records, _ := evt.Internal[sflowFlowRecordsInternalKey].([]sflow.FlowRecord) + evt.Internal[sflowFlowRecordsInternalKey] = append(records, record) +} + +func applySFlowExtendedRecordFields(fields map[string]any, data any) { + switch data := data.(type) { + case sflow.ExtendedNAT: + applySFlowExtendedNATFields(fields, data) + case sflow.ExtendedMPLS: + applySFlowExtendedMPLSFields(fields, data) + case sflow.ExtendedMPLSTunnel: + fields["mpls_tunnel_lsp_name"] = data.TunnelLSPName + fields["mpls_tunnel_id"] = data.TunnelID + fields["mpls_tunnel_cos"] = data.TunnelCOS + case sflow.ExtendedMPLSVC: + fields["mpls_vc_instance_name"] = data.VCInstanceName + fields["mpls_vll_vc_id"] = data.VLLVCID + fields["mpls_vc_label_cos"] = data.VCLabelCOS + case sflow.ExtendedMPLSFTN: + fields["mpls_ftn_descr"] = data.MPLSFTNDescr + fields["mpls_ftn_mask"] = data.MPLSFTNMask + case sflow.ExtendedMPLSLDPFEC: + fields["mpls_fec_addr_prefix_length"] = data.MPLSFecAddrPrefixLength + } +} + +func applySFlowExtendedNATFields(fields map[string]any, data sflow.ExtendedNAT) { + if src := sflowIPString(data.SrcAddress); src != "" { + fields["nat_src_addr"] = src + } + if dst := sflowIPString(data.DstAddress); dst != "" { + fields["nat_dst_addr"] = dst + } +} + +func applySFlowExtendedMPLSFields(fields map[string]any, data sflow.ExtendedMPLS) { + if nextHop := sflowIPString(data.NextHop); nextHop != "" { + fields["mpls_next_hop_addr"] = nextHop + } + fields["mpls_in_label_stack"] = append([]uint32(nil), data.InLabelStack...) + fields["mpls_out_label_stack"] = append([]uint32(nil), data.OutLabelStack...) + for i, entry := range data.InLabelStack { + index := i + 1 + fields[fmt.Sprintf("mpls_label_%d", index)] = mplsLabelValueFromStackEntry(entry) + fields[fmt.Sprintf("mpls_label_stack_section_%d", index)] = mplsLabelStackSectionFromEntry(entry) + } +} + +func sflowIPString(ip []byte) string { + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return "" + } + return addr.String() +} + +func mplsLabelValueFromStackEntry(entry uint32) uint32 { + return (entry >> 12) & 0xfffff +} + +func mplsLabelStackSectionFromEntry(entry uint32) []byte { + return []byte{byte(entry >> 24), byte(entry >> 16), byte(entry >> 8)} +} diff --git a/pkg/reflow/decode/sflow_test.go b/pkg/reflow/decode/sflow_test.go new file mode 100644 index 00000000..2e7177b2 --- /dev/null +++ b/pkg/reflow/decode/sflow_test.go @@ -0,0 +1,374 @@ +package decode + +import ( + "bytes" + "encoding/hex" + "strings" + "testing" + + "github.com/netsampler/goflow2/v3/decoders/sflow" + "github.com/netsampler/goflow2/v3/decoders/utils" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func TestDecodeSFlowCounterSampleEmitsInterfaceCounterEvent(t *testing.T) { + packet := &sflow.Packet{ + Version: 5, + AgentIP: utils.IPAddress{198, 51, 100, 1}, + SubAgentId: 7, + SequenceNumber: 8, + Uptime: 9, + Samples: []interface{}{ + sflow.CounterSample{ + Header: sflow.SampleHeader{ + Format: sflow.SAMPLE_FORMAT_COUNTER, + SampleSequenceNumber: 10, + SourceIdType: 0, + SourceIdValue: 11, + }, + CounterRecordsCount: 1, + Records: []sflow.CounterRecord{ + { + Data: sflow.IfCounters{ + IfIndex: 12, + IfSpeed: 1000, + IfInOctets: 123, + IfOutOctets: 456, + IfOutErrors: 3, + IfStatus: 5, + IfDirection: 1, + IfInDiscards: 2, + IfOutDiscards: 4, + }, + }, + }, + }, + }, + } + + encoded, err := sflow.EncodeMessage(packet) + if err != nil { + t.Fatalf("EncodeMessage returned error: %v", err) + } + + decoded, err := (&builtIn{}).decodeSFlow(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: append([]byte(nil), encoded...), + }, encoded, 5) + if err != nil { + t.Fatalf("decodeSFlow returned error: %v", err) + } + if len(decoded) != 1 { + t.Fatalf("expected 1 decoded event, got %d", len(decoded)) + } + + fields := decoded[0].Fields + if got := fields["flow_type"]; got != "sflow" { + t.Fatalf("expected flow_type=sflow, got %#v", got) + } + if got := fields["flow_version"]; got != uint32(5) { + t.Fatalf("expected flow_version=5, got %#v", got) + } + if got := fields["record_kind"]; got != "interface_counter" { + t.Fatalf("expected record_kind=interface_counter, got %#v", got) + } + if got := fields["agent_ip"]; got != "198.51.100.1" { + t.Fatalf("expected agent_ip=198.51.100.1, got %#v", got) + } + if got := fields["if_index"]; got != uint32(12) { + t.Fatalf("expected if_index=12, got %#v", got) + } + if got := fields["if_in_octets"]; got != uint64(123) { + t.Fatalf("expected if_in_octets=123, got %#v", got) + } + if got := fields["if_out_octets"]; got != uint64(456) { + t.Fatalf("expected if_out_octets=456, got %#v", got) + } + + if decoded[0].SFlow == nil || decoded[0].SFlow.SourceID != 11 { + t.Fatalf("expected sflow metadata source_id=11, got %#v", decoded[0].SFlow) + } + + roundTrip := &sflow.Packet{} + if err := sflow.DecodeMessageVersion(bytes.NewBuffer(encoded), roundTrip); err != nil { + t.Fatalf("DecodeMessageVersion sanity check failed: %v", err) + } +} + +func TestDecodeSFlowExpandedCounterSampleCarriesFlowAndCounterFormat(t *testing.T) { + packet := &sflow.Packet{ + Version: 5, + AgentIP: utils.IPAddress{198, 51, 100, 1}, + SubAgentId: 7, + SequenceNumber: 8, + Uptime: 9, + Samples: []interface{}{ + sflow.CounterSample{ + Header: sflow.SampleHeader{ + Format: sflow.SAMPLE_FORMAT_EXPANDED_COUNTER, + SampleSequenceNumber: 10, + SourceIdType: 2, + SourceIdValue: 11, + }, + CounterRecordsCount: 1, + Records: []sflow.CounterRecord{ + { + Data: sflow.IfCounters{ + IfIndex: 12, + IfInOctets: 123, + IfOutOctets: 456, + }, + }, + }, + }, + }, + } + + encoded, err := sflow.EncodeMessage(packet) + if err != nil { + t.Fatalf("EncodeMessage returned error: %v", err) + } + + decoded, err := (&builtIn{}).decodeSFlow(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: append([]byte(nil), encoded...), + }, encoded, 5) + if err != nil { + t.Fatalf("decodeSFlow returned error: %v", err) + } + if len(decoded) != 1 { + t.Fatalf("expected 1 decoded event, got %d", len(decoded)) + } + + fields := decoded[0].Fields + if got := fields["flow_type"]; got != "sflow" { + t.Fatalf("expected flow_type=sflow, got %#v", got) + } + if got := fields["counter_format"]; got != "expanded" { + t.Fatalf("expected counter_format=expanded, got %#v", got) + } + if got := fields["source_id_type"]; got != uint32(2) { + t.Fatalf("expected source_id_type=2, got %#v", got) + } +} + +func TestDecodeSFlowEthernetCounterSampleEmitsCounterEvent(t *testing.T) { + packet := &sflow.Packet{ + Version: 5, + AgentIP: utils.IPAddress{198, 51, 100, 1}, + SubAgentId: 7, + SequenceNumber: 8, + Uptime: 9, + Samples: []interface{}{ + sflow.CounterSample{ + Header: sflow.SampleHeader{ + Format: sflow.SAMPLE_FORMAT_COUNTER, + SampleSequenceNumber: 10, + SourceIdValue: 11, + }, + CounterRecordsCount: 1, + Records: []sflow.CounterRecord{ + { + Data: sflow.EthernetCounters{ + Dot3StatsFCSErrors: 2, + Dot3StatsSymbolErrors: 13, + Dot3StatsFrameTooLongs: 11, + }, + }, + }, + }, + }, + } + + encoded, err := sflow.EncodeMessage(packet) + if err != nil { + t.Fatalf("EncodeMessage returned error: %v", err) + } + + decoded, err := (&builtIn{}).decodeSFlow(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: append([]byte(nil), encoded...), + }, encoded, 5) + if err != nil { + t.Fatalf("decodeSFlow returned error: %v", err) + } + if len(decoded) != 1 { + t.Fatalf("expected 1 decoded event, got %d", len(decoded)) + } + + fields := decoded[0].Fields + if got := fields["flow_type"]; got != "sflow" { + t.Fatalf("expected flow_type=sflow, got %#v", got) + } + if got := fields["record_kind"]; got != "ethernet_counter" { + t.Fatalf("expected record_kind=ethernet_counter, got %#v", got) + } + if got := fields["dot3_stats_fcs_errors"]; got != uint32(2) { + t.Fatalf("expected dot3_stats_fcs_errors=2, got %#v", got) + } + if got := fields["dot3_stats_symbol_errors"]; got != uint32(13) { + t.Fatalf("expected dot3_stats_symbol_errors=13, got %#v", got) + } +} + +func TestDecodeSFlowCounterPacketExampleRejectsMalformedSampleLengths(t *testing.T) { + payload, err := hex.DecodeString("0000000500000001c0a800ac00000000000000290010b171000000040000000400000070000000a10000000000000000000000010000000100000058000000110000000600000000000000000000000000000003000000000ae66a130002336f00000000000000000000000000000000000000000000000d786230000bee300000000000000000000000000000000000000020000000400000070000000a200000000000000000000000100000001000000580000000f0000000600000000000000000000000000000003000000083726775302207a1a000000000000000000000000000000000000000002293935c100e65a01000000000000000000000000000000000000000200000004000000700000000a3000000000000000000000001000000010000005800000010000000060000000000000000000000000000000300000009ab59d28c020ae0cb00000000000000000000000000000000000000000105914cc500b29fff00000000000000000000000000000000000000020000000400000070000000a4000000000000000000000001000000010000005800000012000000060000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002") + if err != nil { + t.Fatalf("DecodeString returned error: %v", err) + } + + _, err = (&builtIn{}).decodeSFlow(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: append([]byte(nil), payload...), + }, payload, 5) + if err == nil { + t.Fatalf("expected malformed packet to return a decode error") + } + if got := err.Error(); !strings.Contains(got, "sample 1 length") { + t.Fatalf("expected sample length error, got %v", err) + } +} + +func TestDecodeSFlowExpandedCounterPacketExample(t *testing.T) { + payload, err := hex.DecodeString("0000000500000001c0a800ac0000000000000074001c03e0000000040000000400000070000001c90000000000000000000000010000000100000058000000110000000600000000000000000000000000000003000000000ae6e33e000233de00000000000000000000000000000000000000000000000000d8085d0000bf6100000000000000000000000000000000000000020000000400000070000001ca00000000000000000000000100000001000000580000000f0000000600000000000000000000000000000003000000083728c33e02207f49000000000000000000000000000000000000000000000002293a906600e65d6e00000000000000000000000000000000000000020000000400000070000001cb000000000000000000000001000000010000005800000010000000060000000000000000000000000000000300000009ad4bd9b5020bddac00000000000000000000000000000000000000000000000106ee9ec200b39a9700000000000000000000000000000000000000010000000400000070000001cc000000000000000000000001000000010000005800000012000000060000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002") + if err != nil { + t.Fatalf("DecodeString returned error: %v", err) + } + + decoded, err := (&builtIn{}).decodeSFlow(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: append([]byte(nil), payload...), + }, payload, 5) + if err != nil { + t.Fatalf("decodeSFlow returned error: %v", err) + } + if len(decoded) != 4 { + t.Fatalf("expected 4 decoded events, got %d", len(decoded)) + } + for i, evt := range decoded { + fields := evt.Fields + if got := fields["flow_type"]; got != "sflow" { + t.Fatalf("event %d expected flow_type=sflow, got %#v fields=%#v", i, got, fields) + } + if got := fields["record_kind"]; got != "interface_counter" { + t.Fatalf("event %d expected record_kind=interface_counter, got %#v fields=%#v", i, got, fields) + } + if got := fields["counter_format"]; got != "expanded" { + t.Fatalf("event %d expected counter_format=expanded, got %#v fields=%#v", i, got, fields) + } + } +} + +func TestDecodeSFlowSampledHeaderBytesUseFrameLength(t *testing.T) { + packet := &sflow.Packet{ + Version: 5, + AgentIP: utils.IPAddress{198, 51, 100, 1}, + SubAgentId: 7, + SequenceNumber: 8, + Uptime: 9, + Samples: []interface{}{ + sflow.FlowSample{ + Header: sflow.SampleHeader{ + Format: sflow.SAMPLE_FORMAT_FLOW, + SampleSequenceNumber: 10, + SourceIdType: 0, + SourceIdValue: 11, + }, + SamplingRate: 100, + SamplePool: 1000, + FlowRecordsCount: 1, + Records: []sflow.FlowRecord{ + { + Data: sflow.SampledHeader{ + Protocol: 1, + FrameLength: 74, + OriginalLength: 54, + HeaderData: bytes.Repeat([]byte{0xaa}, 54), + }, + }, + }, + }, + }, + } + + encoded, err := sflow.EncodeMessage(packet) + if err != nil { + t.Fatalf("EncodeMessage returned error: %v", err) + } + + decoded, err := (&builtIn{}).decodeSFlow(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: append([]byte(nil), encoded...), + }, encoded, 5) + if err != nil { + t.Fatalf("decodeSFlow returned error: %v", err) + } + if len(decoded) != 1 { + t.Fatalf("expected 1 decoded event, got %d", len(decoded)) + } + if got := decoded[0].Fields["bytes"]; got != int64(74) { + t.Fatalf("expected bytes to use frame_length=74, got %#v", got) + } +} + +func TestDecodeSFlowExtendedNATMPLSFieldsAndPreservesRecords(t *testing.T) { + packet := &sflow.Packet{ + Version: 5, + AgentIP: utils.IPAddress{198, 51, 100, 1}, + SubAgentId: 7, + SequenceNumber: 8, + Uptime: 9, + Samples: []interface{}{ + sflow.FlowSample{ + Header: sflow.SampleHeader{ + Format: sflow.SAMPLE_FORMAT_FLOW, + SampleSequenceNumber: 10, + SourceIdValue: 11, + }, + Records: []sflow.FlowRecord{ + {Data: sflow.SampledHeader{Protocol: 1, FrameLength: 64, OriginalLength: 4, HeaderData: []byte{1, 2, 3, 4}}}, + {Data: sflow.ExtendedNAT{SrcAddress: utils.IPAddress{203, 0, 113, 10}, DstAddress: utils.IPAddress{192, 0, 2, 20}}}, + {Data: sflow.ExtendedMPLS{NextHop: utils.IPAddress{192, 0, 2, 254}, InLabelStack: []uint32{0x00011100}, OutLabelStack: []uint32{0x00022200}}}, + {Data: sflow.ExtendedMPLSTunnel{TunnelLSPName: "lsp-a", TunnelID: 12, TunnelCOS: 3}}, + }, + }, + }, + } + + encoded, err := sflow.EncodeMessage(packet) + if err != nil { + t.Fatalf("EncodeMessage returned error: %v", err) + } + + decoded, err := (&builtIn{}).decodeSFlow(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: append([]byte(nil), encoded...), + }, encoded, 5) + if err != nil { + t.Fatalf("decodeSFlow returned error: %v", err) + } + if len(decoded) != 1 { + t.Fatalf("expected 1 decoded event, got %d", len(decoded)) + } + + fields := decoded[0].Fields + if got := fields["nat_src_addr"]; got != "203.0.113.10" { + t.Fatalf("expected nat_src_addr, got %#v", got) + } + if got := fields["nat_dst_addr"]; got != "192.0.2.20" { + t.Fatalf("expected nat_dst_addr, got %#v", got) + } + if got := fields["mpls_next_hop_addr"]; got != "192.0.2.254" { + t.Fatalf("expected mpls_next_hop_addr, got %#v", got) + } + if got := fields["mpls_label_1"]; got != uint32(17) { + t.Fatalf("expected decoded MPLS label 17, got %#v", got) + } + if got := fields["mpls_tunnel_lsp_name"]; got != "lsp-a" { + t.Fatalf("expected mpls_tunnel_lsp_name, got %#v", got) + } + records, ok := decoded[0].Internal[sflowFlowRecordsInternalKey].([]sflow.FlowRecord) + if !ok || len(records) != 3 { + t.Fatalf("expected 3 preserved extended records, got %#v", decoded[0].Internal[sflowFlowRecordsInternalKey]) + } +} diff --git a/pkg/reflow/encode/encode.go b/pkg/reflow/encode/encode.go new file mode 100644 index 00000000..8f8303fe --- /dev/null +++ b/pkg/reflow/encode/encode.go @@ -0,0 +1,37 @@ +package encode + +import ( + "fmt" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type Encoder interface { + Encode(evt *event.Event) ([][]byte, error) + Flush() ([][]byte, error) +} + +// New builds the configured encoder. Each encoder worker gets its own instance. +func New(cfg config.EncoderConfig) (Encoder, error) { + switch cfg.Type { + case "", "json": + return NewJSONEncoder(cfg), nil + case "protobuf": + return NewProtobufEncoder(cfg) + case "sflow": + return NewSFlowEncoder(cfg), nil + case "ipfix": + return NewIPFIXEncoder(cfg), nil + case "netflowv9": + return NewNFv9Encoder(cfg), nil + case "netflowv5": + return NewNFv5Encoder(cfg), nil + case "pcap": + return NewPcapEncoder(cfg) + case "pcapng": + return NewPcapNGEncoder(cfg) + default: + return nil, fmt.Errorf("unsupported encoder.type %q", cfg.Type) + } +} diff --git a/pkg/reflow/encode/encode_bench_test.go b/pkg/reflow/encode/encode_bench_test.go new file mode 100644 index 00000000..c77c387e --- /dev/null +++ b/pkg/reflow/encode/encode_bench_test.go @@ -0,0 +1,187 @@ +package encode + +import ( + "bytes" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func BenchmarkEncode(b *testing.B) { + benchmarks := []struct { + name string + cfg config.EncoderConfig + evt *event.Event + }{ + { + name: "json_canonical", + cfg: config.EncoderConfig{Type: "json"}, + evt: benchFlowEvent(), + }, + { + name: "json_goflow2v2", + cfg: config.EncoderConfig{ + Type: "json", + JSON: config.JSONConfig{Flavor: "goflow2v2"}, + }, + evt: benchFlowEvent(), + }, + { + name: "protobuf_canonical", + cfg: config.EncoderConfig{ + Type: "protobuf", + Protobuf: config.ProtobufConfig{Flavor: "canonical"}, + }, + evt: benchFlowEvent(), + }, + { + name: "sflow_sampled_header", + cfg: config.EncoderConfig{Type: "sflow"}, + evt: benchSFlowPacketEvent(), + }, + { + name: "ipfix_templated", + cfg: benchTemplatedFlowConfig("ipfix"), + evt: benchFlowEvent(), + }, + { + name: "netflowv9_templated", + cfg: benchTemplatedFlowConfig("netflowv9"), + evt: benchFlowEvent(), + }, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + enc, err := New(bm.cfg) + if err != nil { + b.Fatalf("New returned error: %v", err) + } + if _, err := enc.Encode(bm.evt); err != nil { + b.Fatalf("warm Encode returned error: %v", err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + payloads, err := enc.Encode(bm.evt) + if err != nil { + b.Fatalf("Encode returned error: %v", err) + } + if len(payloads) == 0 { + b.Fatalf("expected at least one payload") + } + } + }) + } +} + +func BenchmarkEncodeSFlowBatch(b *testing.B) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + Batch: config.BatchConfig{ + Enabled: boolPtr(true), + MaxRecords: 32, + }, + }) + evt := benchSFlowPacketEvent() + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := enc.Encode(evt); err != nil { + b.Fatalf("Encode returned error: %v", err) + } + } + if _, err := enc.Flush(); err != nil { + b.Fatalf("Flush returned error: %v", err) + } +} + +func BenchmarkEncodeJSONDropFields(b *testing.B) { + evt := benchFlowEvent() + evt.Fields["header_data"] = bytes.Repeat([]byte{0xaa}, 128) + enc := NewJSONEncoder(config.EncoderConfig{ + Type: "json", + JSON: config.JSONConfig{ + DropFields: []string{"header_data"}, + }, + }) + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + payloads, err := enc.Encode(evt) + if err != nil { + b.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + b.Fatalf("expected 1 payload, got %d", len(payloads)) + } + } +} + +func benchFlowEvent() *event.Event { + return &event.Event{ + ReceivedAt: time.Unix(1_700_000_001, 0).UTC(), + Source: event.SourceMetadata{ + AgentIP: "192.0.2.1", + Sampling: &event.SamplingMetadata{ + Rate: 100, + }, + }, + Fields: map[string]any{ + "flow_type": "sflow", + "agent_ip": "192.0.2.1", + "source_id": uint32(42), + "src_addr": "192.0.2.10", + "dst_addr": "198.51.100.20", + "src_port": uint32(12345), + "dst_port": uint32(443), + "proto": uint32(6), + "bytes": int64(321), + "packets": int64(7), + "input_if": uint32(9), + "output_if": uint32(10), + "start_time_unix": int64(1_700_000_000_100), + "end_time_unix": int64(1_700_000_000_900), + "time_flow_start_ns": int64(1_700_000_000_100_123_456), + "time_flow_end_ns": int64(1_700_000_000_900_123_456), + }, + } +} + +func benchSFlowPacketEvent() *event.Event { + evt := benchFlowEvent() + evt.Fields["protocol"] = uint32(1) + evt.Fields["frame_length"] = uint32(60) + evt.Fields["original_length"] = uint32(60) + evt.Fields["header_data"] = []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x14, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + evt.SFlow = &event.SFlowMetadata{ + AgentIP: "192.0.2.1", + SourceID: 42, + SamplingRate: 100, + } + return evt +} + +func benchTemplatedFlowConfig(typ string) config.EncoderConfig { + cfg := testTFlowEncoderConfig(typ) + cfg.TemplatedFlow.TemplateBaseID = 256 + cfg.TemplatedFlow.TemplateRefresh = 0 + cfg.TemplatedFlow.OptionsRefresh = 0 + return cfg +} + +func boolPtr(v bool) *bool { + return &v +} diff --git a/pkg/reflow/encode/encode_test.go b/pkg/reflow/encode/encode_test.go new file mode 100644 index 00000000..a3c1801a --- /dev/null +++ b/pkg/reflow/encode/encode_test.go @@ -0,0 +1,4622 @@ +package encode + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "net/netip" + "strings" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/decoders/netflow" + "github.com/netsampler/goflow2/v3/decoders/netflowlegacy" + "github.com/netsampler/goflow2/v3/decoders/sflow" + flowpb "github.com/netsampler/goflow2/v3/pb" + "github.com/netsampler/goflow2/v3/pkg/reflow/aggregate" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/processor" + "github.com/netsampler/goflow2/v3/utils/store/templates" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/proto" +) + +func TestJSONEncoderDropsConfiguredFieldsFromCanonicalOutput(t *testing.T) { + enc := NewJSONEncoder(config.EncoderConfig{ + Type: "json", + JSON: config.JSONConfig{ + Flavor: "canonical", + DropFields: []string{"header_data"}, + }, + }) + + evt := &event.Event{ + Fields: map[string]any{ + "header_data": []byte{0, 1, 2, 3}, + "src_addr": "192.0.2.10", + }, + } + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + var decoded map[string]any + if err := json.Unmarshal(payloads[0], &decoded); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + fields, ok := decoded["fields"].(map[string]any) + if !ok { + t.Fatalf("expected fields object in payload, got %#v", decoded["fields"]) + } + if _, exists := fields["header_data"]; exists { + t.Fatalf("expected header_data to be dropped, got %#v", fields) + } + if fields["src_addr"] != "192.0.2.10" { + t.Fatalf("expected src_addr to be preserved, got %#v", fields["src_addr"]) + } + + if _, exists := evt.Fields["header_data"]; !exists { + t.Fatalf("expected original event fields to remain unchanged") + } +} + +func TestJSONEncoderGoFlow2V2PrefersNanosecondTimeFields(t *testing.T) { + enc := NewJSONEncoder(config.EncoderConfig{ + Type: "json", + JSON: config.JSONConfig{ + Flavor: "goflow2v2", + }, + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "flow_type": "sflow", + "start_time_unix": int64(1_700_000_000_100), + "end_time_unix": int64(1_700_000_000_900), + "time_flow_start_ns": int64(1_700_000_000_100_123_456), + "time_flow_end_ns": int64(1_700_000_000_900_123_456), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + dec := json.NewDecoder(bytes.NewReader(payloads[0])) + dec.UseNumber() + var decoded map[string]any + if err := dec.Decode(&decoded); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + numberField := func(key string) int64 { + t.Helper() + number, ok := decoded[key].(json.Number) + if !ok { + t.Fatalf("expected %s to be json.Number, got %T", key, decoded[key]) + } + value, err := number.Int64() + if err != nil { + t.Fatalf("parse %s: %v", key, err) + } + return value + } + + startNS := numberField("time_flow_start_ns") + if startNS != 1_700_000_000_100_123_456 { + t.Fatalf("expected nanosecond start time, got %d", startNS) + } + endNS := numberField("time_flow_end_ns") + if endNS != 1_700_000_000_900_123_456 { + t.Fatalf("expected nanosecond end time, got %d", endNS) + } +} + +func TestJSONEncoderGoFlow2V2UsesSourceSamplingMetadata(t *testing.T) { + enc := NewJSONEncoder(config.EncoderConfig{ + Type: "json", + JSON: config.JSONConfig{ + Flavor: "goflow2v2", + }, + }) + + payloads, err := enc.Encode(&event.Event{ + Source: event.SourceMetadata{ + AgentIP: "198.51.100.99", + Sampling: &event.SamplingMetadata{ + Rate: 250, + }, + }, + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "sampling_rate": uint32(100), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + dec := json.NewDecoder(bytes.NewReader(payloads[0])) + dec.UseNumber() + var decoded map[string]any + if err := dec.Decode(&decoded); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + if decoded["sampler_address"] != encodeIPBytes("198.51.100.99") { + t.Fatalf("expected source sampler_address, got %#v", decoded["sampler_address"]) + } + rate, ok := decoded["sampling_rate"].(json.Number) + if !ok { + t.Fatalf("expected sampling_rate json.Number, got %T", decoded["sampling_rate"]) + } + if got, _ := rate.Int64(); got != 250 { + t.Fatalf("expected source sampling_rate 250, got %d", got) + } +} + +func TestProtobufEncoderEncodesCanonicalFlowMessage(t *testing.T) { + enc, err := New(config.EncoderConfig{ + Type: "protobuf", + Protobuf: config.ProtobufConfig{ + Flavor: "canonical", + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(1, 200).UTC(), + Source: event.SourceMetadata{ + AgentIP: "192.0.2.1", + Sampling: &event.SamplingMetadata{ + Rate: 100, + }, + }, + Fields: map[string]any{ + "flow_type": "sflow", + "start_time_unix": int64(1700000000100), + "end_time_unix": int64(1700000000900), + "time_flow_start_ns": int64(1_700_000_000_100_123_456), + "time_flow_end_ns": int64(1_700_000_000_900_123_456), + "bytes": int64(321), + "packets": int64(7), + "src_addr": "192.0.2.10", + "dst_addr": "192.0.2.20", + "proto": uint32(17), + "src_port": uint32(1234), + "dst_port": uint32(4321), + "input_if": uint32(9), + "output_if": uint32(10), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + msg := decodeFlowMessage(t, payloads[0]) + if msg.Type != flowpb.FlowMessage_SFLOW_5 { + t.Fatalf("expected type SFLOW_5, got %v", msg.Type) + } + if msg.SamplingRate != 100 { + t.Fatalf("expected sampling_rate 100, got %d", msg.SamplingRate) + } + if !bytes.Equal(msg.SamplerAddress, []byte{192, 0, 2, 1}) { + t.Fatalf("expected sampler_address 192.0.2.1, got %v", msg.SamplerAddress) + } + if msg.Bytes != 321 || msg.Packets != 7 { + t.Fatalf("expected bytes=321 packets=7, got bytes=%d packets=%d", msg.Bytes, msg.Packets) + } + if msg.TimeFlowStartNs != 1_700_000_000_100_123_456 || msg.TimeFlowEndNs != 1_700_000_000_900_123_456 { + t.Fatalf("expected nanosecond flow window, got start=%d end=%d", msg.TimeFlowStartNs, msg.TimeFlowEndNs) + } + if msg.SrcPort != 1234 || msg.DstPort != 4321 { + t.Fatalf("expected ports 1234/4321, got %d/%d", msg.SrcPort, msg.DstPort) + } +} + +func TestProtobufEncoderUsesSourceSamplingMetadata(t *testing.T) { + enc, err := New(config.EncoderConfig{ + Type: "protobuf", + Protobuf: config.ProtobufConfig{ + Flavor: "canonical", + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + Source: event.SourceMetadata{ + AgentIP: "198.51.100.99", + Sampling: &event.SamplingMetadata{ + Rate: 250, + }, + }, + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "sampling_rate": uint32(100), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + msg := decodeFlowMessage(t, payloads[0]) + if msg.SamplingRate != 250 { + t.Fatalf("expected source sampling_rate 250, got %d", msg.SamplingRate) + } + if !bytes.Equal(msg.SamplerAddress, []byte{198, 51, 100, 99}) { + t.Fatalf("expected source sampler_address 198.51.100.99, got %v", msg.SamplerAddress) + } +} + +func TestProtobufEncoderSupportsGoFlow2V2Flavor(t *testing.T) { + enc, err := New(config.EncoderConfig{ + Type: "protobuf", + Protobuf: config.ProtobufConfig{ + Flavor: "goflow2v2", + LengthPrefixed: true, + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "flow_type": "netflowv9", + "src_addr": "192.0.2.10", + "dst_addr": "192.0.2.20", + "bytes": int64(321), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + msg := decodeDelimitedFlowMessage(t, payloads[0]) + if msg.Type != flowpb.FlowMessage_NETFLOW_V9 { + t.Fatalf("expected type NETFLOW_V9, got %v", msg.Type) + } + if msg.Bytes != 321 { + t.Fatalf("expected bytes=321, got %d", msg.Bytes) + } +} + +func TestGenerateProtobufDefinitionDescribesReFlowFields(t *testing.T) { + definition, err := GenerateProtobufDefinition(config.ProtobufConfig{Flavor: "canonical"}) + if err != nil { + t.Fatalf("GenerateProtobufDefinition returned error: %v", err) + } + + for _, want := range []string{ + "syntax = \"proto3\";", + "enum FlowType {", + "FlowType type = 1;", + "uint64 time_received_ns = 110;", + "bytes src_addr = 6;", + "bytes dst_addr = 7;", + "uint32 proto = 20;", + "uint32 observation_domain_id = 70;", + } { + if !strings.Contains(definition, want) { + t.Fatalf("expected generated protobuf definition to contain %q:\n%s", want, definition) + } + } +} + +func TestGenerateProtobufDefinitionRejectsUnsupportedFlavor(t *testing.T) { + if _, err := GenerateProtobufDefinition(config.ProtobufConfig{Flavor: "custom"}); err == nil { + t.Fatalf("expected unsupported flavor error") + } +} + +func TestProtobufEncoderExportAllFieldsAsExtraKVs(t *testing.T) { + enc, err := New(config.EncoderConfig{ + Type: "protobuf", + Protobuf: config.ProtobufConfig{ + Flavor: "canonical", + ExportAll: true, + }, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "src_addr": "192.0.2.10", + "custom_name": "edge-a", + "custom_count": uint64(42), + "custom_delta": int64(-7), + "custom_ok": true, + "custom_raw": []byte{0xde, 0xad}, + "custom_load": float64(1.5), + "custom_tags": []string{"wan", "sampled"}, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + extras := decodeProtobufExtraFields(t, payloads[0]) + + if got := extras["custom_name"]; got != "edge-a" { + t.Fatalf("expected custom_name=edge-a, got %#v", got) + } + if got := extras["custom_count"]; got != uint64(42) { + t.Fatalf("expected custom_count=42, got %#v", got) + } + if got := extras["custom_delta"]; got != int64(-7) { + t.Fatalf("expected custom_delta=-7, got %#v", got) + } + if got := extras["custom_ok"]; got != true { + t.Fatalf("expected custom_ok=true, got %#v", got) + } + if got := extras["custom_raw"]; !bytes.Equal(got.([]byte), []byte{0xde, 0xad}) { + t.Fatalf("expected custom_raw bytes, got %#v", got) + } + if got := extras["custom_load"]; got != float64(1.5) { + t.Fatalf("expected custom_load=1.5, got %#v", got) + } + if got := extras["custom_tags"]; got != `["wan","sampled"]` { + t.Fatalf("expected custom_tags JSON, got %#v", got) + } + if got := extras["src_addr"]; got != "192.0.2.10" { + t.Fatalf("expected export_all to include src_addr extra, got %#v", got) + } +} + +func TestGenerateProtobufDefinitionIncludesExtraKVSchema(t *testing.T) { + definition, err := GenerateProtobufDefinition(config.ProtobufConfig{ + Flavor: "canonical", + ExportAll: true, + }) + if err != nil { + t.Fatalf("GenerateProtobufDefinition returned error: %v", err) + } + + for _, want := range []string{ + "repeated ExtraField extra = 1000;", + "message ExtraField {", + "oneof value {", + "string string_value = 2;", + "string json_value = 8;", + } { + if !strings.Contains(definition, want) { + t.Fatalf("expected generated protobuf definition to contain %q:\n%s", want, definition) + } + } +} + +func TestSFlowEncoderUsesEventAgentIPBeforeConfiguredFallback(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + SFlow: config.SFlowConfig{ + AgentIP: "203.0.113.10", + }, + }) + + payloads, err := enc.Encode(testSFlowEvent("198.51.100.10")) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + packet := decodeSFlowPacket(t, payloads[0]) + got, ok := netip.AddrFromSlice(packet.AgentIP) + if !ok || got.String() != "198.51.100.10" { + t.Fatalf("expected event agent_ip 198.51.100.10, got %s", got.String()) + } +} + +func TestSFlowEncoderUsesConfiguredAgentIPFallback(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + SFlow: config.SFlowConfig{ + AgentIP: "203.0.113.10", + }, + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "protocol": uint32(1), + "frame_length": uint32(60), + "original_length": uint32(60), + "header_data": []byte{0, 1, 2, 3}, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + packet := decodeSFlowPacket(t, payloads[0]) + got, ok := netip.AddrFromSlice(packet.AgentIP) + if !ok || got.String() != "203.0.113.10" { + t.Fatalf("expected configured agent_ip fallback 203.0.113.10, got %s", got.String()) + } +} + +func TestSFlowEncoderUsesAgentIPv6Field(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ipv6": "2001:db8::10", + "agent_ip": "192.0.2.10", + "protocol": uint32(1), + "frame_length": uint32(60), + "original_length": uint32(60), + "header_data": []byte{0, 1, 2, 3}, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + packet := decodeSFlowPacket(t, payloads[0]) + got, ok := netip.AddrFromSlice(packet.AgentIP) + if !ok || got.String() != "2001:db8::10" { + t.Fatalf("expected agent_ipv6 field 2001:db8::10, got %s", got.String()) + } +} + +func TestSFlowEncoderIgnoresControlEvents(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + payloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init", + Stream: "options_data", + }, + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "source_id": uint32(4), + "sampling_rate": uint32(1), + "sample_pool": uint32(0), + "drops": uint32(0), + "input_if": uint32(4), + "output_if": uint32(4), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 0 { + t.Fatalf("expected control event to be ignored, got %d payloads", len(payloads)) + } + + payloads, err = enc.Encode(testSFlowEvent("198.51.100.10")) + if err != nil { + t.Fatalf("Encode(data) returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected data event to still encode, got %d payloads", len(payloads)) + } +} + +func TestSFlowEncoderFallsBackToLoopbackAgentIP(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "protocol": uint32(1), + "frame_length": uint32(60), + "original_length": uint32(60), + "header_data": []byte{0, 1, 2, 3}, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + packet := decodeSFlowPacket(t, payloads[0]) + got, ok := netip.AddrFromSlice(packet.AgentIP) + if !ok || got.String() != "127.0.0.1" { + t.Fatalf("expected loopback fallback agent_ip 127.0.0.1, got %s", got.String()) + } +} + +func TestSFlowEncoderSplitsBatchByAgentIPWhenConfigured(t *testing.T) { + batchOverAgent := false + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + Batch: config.BatchConfig{ + Enabled: testBoolPtr(true), + }, + SFlow: config.SFlowConfig{ + BatchOver: config.SFlowBatchOverConfig{ + AgentIP: &batchOverAgent, + }, + }, + }) + + payloads, err := enc.Encode(testSFlowEvent("198.51.100.10")) + if err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } + if len(payloads) != 0 { + t.Fatalf("expected no payload on first buffered encode, got %d", len(payloads)) + } + + payloads, err = enc.Encode(testSFlowEvent("198.51.100.20")) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 flushed payload on incompatible second encode, got %d", len(payloads)) + } + firstPacket := decodeSFlowPacket(t, payloads[0]) + firstAgent, ok := netip.AddrFromSlice(firstPacket.AgentIP) + if !ok || firstAgent.String() != "198.51.100.10" { + t.Fatalf("expected first flushed packet agent_ip 198.51.100.10, got %v", firstPacket.AgentIP) + } + + payloads, err = enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 remaining payload on final flush, got %d", len(payloads)) + } + secondPacket := decodeSFlowPacket(t, payloads[0]) + secondAgent, ok := netip.AddrFromSlice(secondPacket.AgentIP) + if !ok || secondAgent.String() != "198.51.100.20" { + t.Fatalf("expected second flushed packet agent_ip 198.51.100.20, got %v", secondPacket.AgentIP) + } +} + +func TestSFlowEncoderDropsOversizedSampleWithoutError(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + MaxDatagramBytes: 64, + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "protocol": uint32(1), + "frame_length": uint32(200), + "original_length": uint32(200), + "header_data": bytes.Repeat([]byte{0xaa}, 200), + }, + }) + if err != nil { + t.Fatalf("Encode returned error for oversized sample: %v", err) + } + if len(payloads) != 0 { + t.Fatalf("expected oversized sample to be dropped without payload, got %d payloads", len(payloads)) + } +} + +func TestSFlowEncoderTruncatesOversizedSampleWhenEnabled(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + MaxDatagramBytes: 96, + AllowTruncate: testBoolPtr(true), + }) + + originalHeader := bytes.Repeat([]byte{0xaa}, 200) + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "protocol": uint32(1), + "frame_length": uint32(200), + "original_length": uint32(200), + "header_data": originalHeader, + }, + }) + if err != nil { + t.Fatalf("Encode returned error for truncatable sample: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one truncated payload, got %d", len(payloads)) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + header := sample.Records[0].Data.(sflow.SampledHeader) + if len(header.HeaderData) >= len(originalHeader) { + t.Fatalf("expected truncated header_data length < %d, got %d", len(originalHeader), len(header.HeaderData)) + } + if int(header.OriginalLength) != len(header.HeaderData) { + t.Fatalf("expected OriginalLength=%d to match truncated header_data length, got %d", len(header.HeaderData), header.OriginalLength) + } +} + +func TestSFlowEncoderCapsHeadersBeforeBatching(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + MaxDatagramBytes: 400, + AllowTruncate: testBoolPtr(true), + Batch: config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + }, + SFlow: config.SFlowConfig{ + MaxHeaderBytes: 64, + }, + }) + + first, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "protocol": uint32(1), + "frame_length": uint32(200), + "original_length": uint32(200), + "header_data": bytes.Repeat([]byte{0xaa}, 200), + }, + }) + if err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } + if len(first) != 0 { + t.Fatalf("expected first batched event to stay buffered, got %d payloads", len(first)) + } + second, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "protocol": uint32(1), + "frame_length": uint32(200), + "original_length": uint32(200), + "header_data": bytes.Repeat([]byte{0xbb}, 200), + }, + }) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(second) != 1 { + t.Fatalf("expected one batched payload, got %d", len(second)) + } + + packet := decodeSFlowPacket(t, second[0]) + if len(packet.Samples) != 2 { + t.Fatalf("expected two samples in one datagram, got %d", len(packet.Samples)) + } + for i, raw := range packet.Samples { + sample := raw.(sflow.FlowSample) + header := sample.Records[0].Data.(sflow.SampledHeader) + if len(header.HeaderData) != 64 { + t.Fatalf("sample %d expected capped header_data length 64, got %d", i, len(header.HeaderData)) + } + if header.OriginalLength != 64 { + t.Fatalf("sample %d expected original_length/header_size 64, got %d", i, header.OriginalLength) + } + if header.FrameLength != 200 { + t.Fatalf("sample %d expected frame_length 200, got %d", i, header.FrameLength) + } + } +} + +func TestSFlowEncoderBatchMaxBytesDoesNotUseFieldEstimate(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + MaxDatagramBytes: 4096, + AllowTruncate: testBoolPtr(true), + Batch: config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 32, + MaxBytes: 4096, + }, + SFlow: config.SFlowConfig{ + MaxHeaderBytes: 128, + }, + }) + + for i := 0; i < 3; i++ { + evt := testSFlowEvent("198.51.100.10") + for j := 0; j < 128; j++ { + evt.Fields[fmt.Sprintf("unused_%03d", j)] = "this field is not encoded into sflow" + } + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode(%d) returned error: %v", i, err) + } + if len(payloads) != 0 { + t.Fatalf("expected event %d to stay buffered, got %d payloads", i, len(payloads)) + } + } + + payloads, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one flushed sflow packet, got %d", len(payloads)) + } + packet := decodeSFlowPacket(t, payloads[0]) + if len(packet.Samples) != 3 { + t.Fatalf("expected three samples after timer-style flush, got %d", len(packet.Samples)) + } +} + +func TestSFlowEncoderPacketSequenceAdvancesPerDatagram(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + Batch: config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + }, + }) + + firstPayloads, err := enc.Encode(testSFlowEvent("198.51.100.10")) + if err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } + if len(firstPayloads) != 0 { + t.Fatalf("expected first event to stay buffered, got %d payloads", len(firstPayloads)) + } + + secondPayloads, err := enc.Encode(testSFlowEvent("198.51.100.10")) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(secondPayloads) != 1 { + t.Fatalf("expected one flushed payload after second event, got %d", len(secondPayloads)) + } + + thirdPayloads, err := enc.Encode(testSFlowEvent("198.51.100.10")) + if err != nil { + t.Fatalf("Encode(third) returned error: %v", err) + } + if len(thirdPayloads) != 0 { + t.Fatalf("expected third event to stay buffered, got %d payloads", len(thirdPayloads)) + } + + fourthPayloads, err := enc.Encode(testSFlowEvent("198.51.100.10")) + if err != nil { + t.Fatalf("Encode(fourth) returned error: %v", err) + } + if len(fourthPayloads) != 1 { + t.Fatalf("expected one flushed payload after fourth event, got %d", len(fourthPayloads)) + } + + firstPacket := decodeSFlowPacket(t, secondPayloads[0]) + secondPacket := decodeSFlowPacket(t, fourthPayloads[0]) + if firstPacket.SequenceNumber != 1 { + t.Fatalf("expected first packet sequence 1, got %d", firstPacket.SequenceNumber) + } + if secondPacket.SequenceNumber != 2 { + t.Fatalf("expected second packet sequence 2, got %d", secondPacket.SequenceNumber) + } +} + +func TestSFlowEncoderDefaultsToOwnedPacketSequence(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + first := testSFlowEvent("198.51.100.10") + first.SFlow.SequenceNumber = 900 + second := testSFlowEvent("198.51.100.10") + second.SFlow.SequenceNumber = 900 + + firstPayloads, err := enc.Encode(first) + if err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } + secondPayloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + + firstPacket := decodeSFlowPacket(t, firstPayloads[0]) + secondPacket := decodeSFlowPacket(t, secondPayloads[0]) + if firstPacket.SequenceNumber != 1 { + t.Fatalf("expected first packet sequence 1, got %d", firstPacket.SequenceNumber) + } + if secondPacket.SequenceNumber != 2 { + t.Fatalf("expected second packet sequence 2, got %d", secondPacket.SequenceNumber) + } +} + +func TestSFlowEncoderCanUseMetadataPacketSequence(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + SFlow: config.SFlowConfig{ + UseMetadataSequenceNumber: true, + }, + }) + + evt := testSFlowEvent("198.51.100.10") + evt.SFlow.SequenceNumber = 900 + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + if packet.SequenceNumber != 900 { + t.Fatalf("expected packet sequence from metadata 900, got %d", packet.SequenceNumber) + } +} + +func TestSFlowEncoderUsesEventSamplingRate(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + }) + + evt := testSFlowEvent("198.51.100.10") + evt.SFlow.SamplingRate = 100 + evt.SFlow.SamplePool = 12345 + evt.Fields["sampling_rate"] = uint32(100) + evt.Fields["sample_pool"] = uint32(12345) + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + if sample.SamplingRate != 100 { + t.Fatalf("expected sampling_rate 100, got %d", sample.SamplingRate) + } + if sample.SamplePool != 12345 { + t.Fatalf("expected sample_pool 12345, got %d", sample.SamplePool) + } +} + +func TestSFlowEncoderUsesSourceSamplingMetadata(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + }) + + evt := testSFlowEvent("198.51.100.10") + evt.SFlow = nil + evt.Source.AgentIP = "198.51.100.99" + evt.Source.SourceID = 42 + evt.Source.Sampling = &event.SamplingMetadata{ + Rate: 250, + SamplePool: 54321, + Drops: 7, + } + evt.Fields["agent_ip"] = "192.0.2.1" + evt.Fields["source_id"] = uint32(9) + evt.Fields["sampling_rate"] = uint32(100) + evt.Fields["sample_pool"] = uint32(12345) + evt.Fields["drops"] = uint32(3) + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + gotAgent, ok := netip.AddrFromSlice(packet.AgentIP) + if !ok || gotAgent.String() != "198.51.100.99" { + t.Fatalf("expected source agent_ip 198.51.100.99, got %s", gotAgent.String()) + } + sample := packet.Samples[0].(sflow.FlowSample) + if sample.Header.SourceIdValue != 42 { + t.Fatalf("expected source_id 42, got %d", sample.Header.SourceIdValue) + } + if sample.SamplingRate != 250 { + t.Fatalf("expected source sampling_rate 250, got %d", sample.SamplingRate) + } + if sample.SamplePool != 54321 { + t.Fatalf("expected source sample_pool 54321, got %d", sample.SamplePool) + } + if sample.Drops != 7 { + t.Fatalf("expected source drops 7, got %d", sample.Drops) + } +} + +func TestSFlowEncoderSynthesizesExtendedNATAndMPLSRecords(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.10", + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "nat_src_addr": "203.0.113.10", + "nat_dst_addr": "203.0.113.20", + "mpls_next_hop_addr": "192.0.2.254", + "mpls_label_stack_section_1": []byte{0x00, 0x01, 0x11}, + "mpls_out_label_stack": []uint32{0x00022200}, + "mpls_tunnel_lsp_name": "lsp-a", + "mpls_tunnel_id": uint32(12), + "mpls_vc_instance_name": "vc-a", + "mpls_vll_vc_id": uint32(13), + "mpls_ftn_descr": "ftn-a", + "mpls_ftn_mask": uint32(24), + "mpls_fec_addr_prefix_length": uint32(25), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + if len(sample.Records) != 7 { + t.Fatalf("expected sampled header plus 6 extended records, got %d", len(sample.Records)) + } + nat, ok := sample.Records[1].Data.(sflow.ExtendedNAT) + if !ok { + t.Fatalf("expected record 1 ExtendedNAT, got %T", sample.Records[1].Data) + } + if !bytes.Equal(nat.SrcAddress, []byte{203, 0, 113, 10}) || !bytes.Equal(nat.DstAddress, []byte{203, 0, 113, 20}) { + t.Fatalf("unexpected NAT addresses: %#v", nat) + } + mpls, ok := sample.Records[2].Data.(sflow.ExtendedMPLS) + if !ok { + t.Fatalf("expected record 2 ExtendedMPLS, got %T", sample.Records[2].Data) + } + if !bytes.Equal(mpls.NextHop, []byte{192, 0, 2, 254}) || len(mpls.InLabelStack) != 1 || len(mpls.OutLabelStack) != 1 { + t.Fatalf("unexpected MPLS record: %#v", mpls) + } + if got := sample.Records[3].Data.(sflow.ExtendedMPLSTunnel).TunnelLSPName; got != "lsp-a" { + t.Fatalf("expected tunnel lsp name, got %q", got) + } + if got := sample.Records[4].Data.(sflow.ExtendedMPLSVC).VCInstanceName; got != "vc-a" { + t.Fatalf("expected vc name, got %q", got) + } + if got := sample.Records[5].Data.(sflow.ExtendedMPLSFTN).MPLSFTNDescr; got != "ftn-a" { + t.Fatalf("expected ftn descr, got %q", got) + } + if got := sample.Records[6].Data.(sflow.ExtendedMPLSLDPFEC).MPLSFecAddrPrefixLength; got != 25 { + t.Fatalf("expected fec prefix length 25, got %d", got) + } +} + +func TestSFlowEncoderCanDisableSyntheticExtendedRecords(t *testing.T) { + emitExtended := false + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + SFlow: config.SFlowConfig{ + EmitExtendedRecords: &emitExtended, + }, + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.10", + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(6), + "nat_src_addr": "203.0.113.10", + "mpls_label_1": uint32(17), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + if len(sample.Records) != 1 { + t.Fatalf("expected only sampled header record, got %d", len(sample.Records)) + } +} + +func TestSFlowEncoderForwardsPreservedEnterpriseRecord(t *testing.T) { + enterpriseFormat := sflow.PackDataFormat(64512, 7) + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.10", + "protocol": uint32(1), + "frame_length": uint32(60), + "original_length": uint32(4), + "header_data": []byte{1, 2, 3, 4}, + "nat_src_addr": "203.0.113.10", + "nat_dst_addr": "203.0.113.20", + }, + Internal: map[string]any{ + sflowFlowRecordsInternalKey: []sflow.FlowRecord{ + {Header: sflow.RecordHeader{DataFormat: enterpriseFormat}, Data: sflow.RawRecord{Data: []byte{0xde, 0xad, 0xbe, 0xef}}}, + {Data: sflow.ExtendedNAT{SrcAddress: []byte{198, 51, 100, 10}, DstAddress: []byte{198, 51, 100, 20}}}, + }, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + if len(sample.Records) != 3 { + t.Fatalf("expected sampled header plus preserved enterprise/NAT records, got %d", len(sample.Records)) + } + raw := sample.Records[1].Data.(sflow.RawRecord) + if sample.Records[1].Header.DataFormat != enterpriseFormat || !bytes.Equal(raw.Data, []byte{0xde, 0xad, 0xbe, 0xef}) { + t.Fatalf("expected enterprise raw record to be forwarded, got %#v", sample.Records[1]) + } + natCount := 0 + for _, record := range sample.Records { + if _, ok := record.Data.(sflow.ExtendedNAT); ok { + natCount++ + } + } + if natCount != 1 { + t.Fatalf("expected preserved NAT to suppress synthetic duplicate, got %d NAT records", natCount) + } +} + +func TestSFlowEncoderEncodesPublicRawFlowRecordEnvelope(t *testing.T) { + enterpriseFormat := sflow.PackDataFormat(64512, 7) + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.10", + "protocol": uint32(1), + "frame_length": uint32(60), + "original_length": uint32(4), + "header_data": []byte{1, 2, 3, 4}, + }, + SFlow: &event.SFlowMetadata{ + Samples: []event.SFlowSampleMetadata{ + { + RawFlowRecords: []event.SFlowRawRecord{ + {Enterprise: 64512, Format: 7, Data: []byte{0xde, 0xad, 0xbe, 0xef}}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + if len(sample.Records) != 2 { + t.Fatalf("expected sampled header plus raw enterprise record, got %d", len(sample.Records)) + } + raw := sample.Records[1].Data.(sflow.RawRecord) + if sample.Records[1].Header.DataFormat != enterpriseFormat || !bytes.Equal(raw.Data, []byte{0xde, 0xad, 0xbe, 0xef}) { + t.Fatalf("expected public enterprise raw record to encode, got %#v", sample.Records[1]) + } +} + +func TestSFlowEncoderEncodesPublicRawCounterRecordEnvelope(t *testing.T) { + enterpriseFormat := sflow.PackDataFormat(64512, 9) + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + payloads, err := enc.Encode(&event.Event{ + SFlow: &event.SFlowMetadata{ + AgentIP: "192.0.2.10", + SourceID: 7, + Samples: []event.SFlowSampleMetadata{ + { + RawCounterRecords: []event.SFlowRawRecord{ + {Enterprise: 64512, Format: 9, Data: []byte{0xca, 0xfe, 0xba, 0xbe}}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.CounterSample) + if sample.Header.SourceIdValue != 7 { + t.Fatalf("expected source_id 7, got %d", sample.Header.SourceIdValue) + } + if len(sample.Records) != 1 { + t.Fatalf("expected one raw counter record, got %d", len(sample.Records)) + } + raw := sample.Records[0].Data.(sflow.RawRecord) + if sample.Records[0].Header.DataFormat != enterpriseFormat || !bytes.Equal(raw.Data, []byte{0xca, 0xfe, 0xba, 0xbe}) { + t.Fatalf("expected public enterprise counter record to encode, got %#v", sample.Records[0]) + } +} + +func TestSFlowEncoderEncodesPublicRawSampleEnvelope(t *testing.T) { + enterpriseFormat := sflow.PackDataFormat(64512, 8) + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + payloads, err := enc.Encode(&event.Event{ + SFlow: &event.SFlowMetadata{ + AgentIP: "192.0.2.10", + Samples: []event.SFlowSampleMetadata{ + {Enterprise: 64512, Format: 8, Data: []byte{1, 2, 3, 4}}, + }, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.RawSample) + if sample.Header.Format != enterpriseFormat || !bytes.Equal(sample.Data, []byte{1, 2, 3, 4}) { + t.Fatalf("expected public raw sample to encode, got %#v", sample) + } +} + +func TestSFlowEncoderFallsBackWhenStandardRawSampleEnvelopeIsPartial(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + headerData := mustDecodeHex(t, "00112233445566778899aabb8847000641404500004a00030000402f027ecb007101cb00710200006558aabbccddeeff020000000001810000640800450000200002000040018e93c000020ac633640a0800600d12340001cafebabe") + + // This is the malformed shape reported from reflow-squash: it is tagged as a + // standard flow sample, but the bytes are only raw flow-record payload bytes. + partialStandardSampleData := mustDecodeHex(t, "000000010000005c000000000000005c"+ + "00112233445566778899aabb8847000641404500004a00030000402f027ecb007101cb00710200006558aabbccddeeff020000000001810000640800450000200002000040018e93c000020ac633640a0800600d12340001cafebabe"+ + "000003ee0000001000000000000000010006410000000000") + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "agent_ip": "127.0.0.1", + "protocol": uint32(1), + "frame_length": uint32(len(headerData)), + "original_length": uint32(len(headerData)), + "header_data": headerData, + }, + SFlow: &event.SFlowMetadata{ + Samples: []event.SFlowSampleMetadata{ + {Format: sflow.SAMPLE_FORMAT_FLOW, Data: partialStandardSampleData}, + }, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one payload, got %d", len(payloads)) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample, ok := packet.Samples[0].(sflow.FlowSample) + if !ok { + t.Fatalf("expected valid flow sample, got %T", packet.Samples[0]) + } + if len(sample.Records) != 1 { + t.Fatalf("expected one sampled-header record, got %d", len(sample.Records)) + } + header := sample.Records[0].Data.(sflow.SampledHeader) + if !bytes.Equal(header.HeaderData, headerData) { + t.Fatalf("expected sampled header bytes to round-trip") + } +} + +func TestSFlowEncoderBuildsPseudoHeaderFromTuple(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + evt := &event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.10", + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "src_port": uint32(12345), + "dst_port": uint32(443), + "proto": uint32(6), + "input_if": uint32(10), + "output_if": uint32(20), + }, + } + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + header := sample.Records[0].Data.(sflow.SampledHeader) + if len(header.HeaderData) == 0 { + t.Fatalf("expected pseudo header data") + } + if header.Protocol != 11 { + t.Fatalf("expected sampled header protocol=11, got %d", header.Protocol) + } + if header.FrameLength != uint32(len(header.HeaderData)) { + t.Fatalf("expected frame length %d, got %d", len(header.HeaderData), header.FrameLength) + } + if header.HeaderData[0]>>4 != 4 { + t.Fatalf("expected direct IPv4 pseudo header, got first byte 0x%02x", header.HeaderData[0]) + } + if evt.Packet == nil || len(evt.Packet.Layers) == 0 || evt.Packet.Layers[0].Kind != "ipv4" { + t.Fatalf("expected pseudo packet model to be attached") + } +} + +func TestSFlowEncoderBuildsDirectIPEncapsulatedPseudoHeader(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + evt := &event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.10", + }, + Packet: directGRETCPPacketModel(), + } + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + header := sample.Records[0].Data.(sflow.SampledHeader) + if header.Protocol != 11 { + t.Fatalf("expected sampled header protocol=11, got %d", header.Protocol) + } + if header.HeaderData[0]>>4 != 4 { + t.Fatalf("expected direct IPv4 pseudo header, got first byte 0x%02x", header.HeaderData[0]) + } + if header.HeaderData[9] != 47 { + t.Fatalf("expected outer IPv4 protocol GRE, got %d", header.HeaderData[9]) + } + if header.HeaderData[22] != 0x08 || header.HeaderData[23] != 0x00 { + t.Fatalf("expected GRE inner protocol 0x0800, got %02x%02x", header.HeaderData[22], header.HeaderData[23]) + } + if evt.Packet == nil || len(evt.Packet.Layers) < 4 || evt.Packet.Layers[0].Kind != "ipv4" { + t.Fatalf("expected direct nested packet model, got %#v", evt.Packet) + } +} + +func TestSFlowEncoderBuildsDirectIPEncapsulatedPseudoHeaderFromReFlowJSON(t *testing.T) { + proc := processor.NewBuiltin(config.ProcessorConfig{}) + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "reflow"}, + }, + Message: []byte(nestedIPLayersJSON), + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + payloads, err := enc.Encode(events[0]) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + header := sample.Records[0].Data.(sflow.SampledHeader) + if header.Protocol != 11 { + t.Fatalf("expected sampled header protocol=11, got %d", header.Protocol) + } + if header.HeaderData[9] != 47 { + t.Fatalf("expected outer IPv4 protocol GRE, got %d", header.HeaderData[9]) + } + if header.HeaderData[22] != 0x08 || header.HeaderData[23] != 0x00 { + t.Fatalf("expected GRE inner protocol 0x0800, got %02x%02x", header.HeaderData[22], header.HeaderData[23]) + } +} + +const nestedIPLayersJSON = `{ + "agent_ip": "192.0.2.10", + "source_id": 1, + "sampling_rate": 100, + "input_if": 10, + "output_if": 20, + "packet": { + "layers": [ + { + "kind": "ipv4", + "ipv4": { + "src_addr": "203.0.113.1", + "dst_addr": "203.0.113.2", + "protocol": 47, + "ttl": 64 + } + }, + { + "kind": "gre", + "gre": { + "protocol": 2048 + } + }, + { + "kind": "ipv4", + "ipv4": { + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "protocol": 6, + "ttl": 64 + } + }, + { + "kind": "tcp", + "tcp": { + "src_port": 12345, + "dst_port": 443, + "flags": 2, + "window": 65535 + } + } + ] + }, + "bytes": 96, + "packets": 1, + "start_time_unix": 1714483200000, + "end_time_unix": 1714483200100 +}` + +func TestSFlowEncoderBuildsEncapsulatedPseudoHeader(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + evt := &event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.10", + "src_mac": "66:77:88:99:aa:bb", + "dst_mac": "00:11:22:33:44:55", + "vlan_id": uint32(100), + "mpls_label": uint32(17), + "original_length": uint32(86), + }, + Packet: encapsulatedGRETCPPacketModel(), + } + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + header := sample.Records[0].Data.(sflow.SampledHeader) + if len(header.HeaderData) == 0 { + t.Fatalf("expected encapsulated pseudo header data") + } + if header.Protocol != 1 { + t.Fatalf("expected sampled header protocol=1, got %d", header.Protocol) + } + if header.OriginalLength != 86 { + t.Fatalf("expected original_length=86, got %d", header.OriginalLength) + } + if len(header.HeaderData) < 46 { + t.Fatalf("expected enough header data for ethernet/vlan/mpls/gre, got %d", len(header.HeaderData)) + } + if header.HeaderData[12] != 0x81 || header.HeaderData[13] != 0x00 { + t.Fatalf("expected ethernet type 0x8100 for dot1q, got %02x%02x", header.HeaderData[12], header.HeaderData[13]) + } + if header.HeaderData[16] != 0x88 || header.HeaderData[17] != 0x47 { + t.Fatalf("expected vlan inner type 0x8847 for mpls, got %02x%02x", header.HeaderData[16], header.HeaderData[17]) + } + if header.HeaderData[31] != 47 { + t.Fatalf("expected outer IPv4 protocol GRE, got %d", header.HeaderData[31]) + } + if header.HeaderData[44] != 0x08 || header.HeaderData[45] != 0x00 { + t.Fatalf("expected GRE inner protocol 0x0800, got %02x%02x", header.HeaderData[44], header.HeaderData[45]) + } + if evt.Packet == nil || len(evt.Packet.Layers) < 4 { + t.Fatalf("expected nested packet model, got %#v", evt.Packet) + } +} + +func TestSFlowEncoderBuildsVXLANPseudoHeaderFromPorts(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{Type: "sflow"}) + + evt := &event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.10", + }, + Packet: vxlanTCPPacketModel(), + } + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.FlowSample) + header := sample.Records[0].Data.(sflow.SampledHeader) + if len(header.HeaderData) == 0 { + t.Fatalf("expected vxlan pseudo header data") + } + if header.Protocol != 11 { + t.Fatalf("expected sampled header protocol=11, got %d", header.Protocol) + } + if evt.Packet == nil || len(evt.Packet.Layers) < 6 { + t.Fatalf("expected vxlan pseudo packet model, got %#v", evt.Packet) + } + if evt.Packet.Layers[1].Kind != "udp" || evt.Packet.Layers[2].Kind != "vxlan" { + t.Fatalf("expected outer UDP and VXLAN layers, got %#v", evt.Packet.Layers) + } +} + +func directGRETCPPacketModel() *event.PacketModel { + return &event.PacketModel{ + Layers: []event.LayerSpec{ + ipv4Layer("203.0.113.1", "203.0.113.2", 47), + {Kind: "gre", GRE: &event.GRELayer{Protocol: 0x0800}}, + ipv4Layer("192.0.2.1", "198.51.100.2", 6), + tcpLayer(12345, 443), + }, + } +} + +func encapsulatedGRETCPPacketModel() *event.PacketModel { + return &event.PacketModel{ + Layers: []event.LayerSpec{ + {Kind: "ethernet", Ethernet: &event.EthernetLayer{SrcMAC: "66:77:88:99:aa:bb", DstMAC: "00:11:22:33:44:55"}}, + {Kind: "dot1q", VLAN: &event.VLANLayer{ID: 100, TPID: 0x8100}}, + {Kind: "mpls", MPLS: &event.MPLSLayer{Label: event.MPLSLabel{Label: 17, BOS: true, TTL: 64}}}, + ipv4Layer("203.0.113.1", "203.0.113.2", 47), + {Kind: "gre", GRE: &event.GRELayer{Protocol: 0x0800}}, + ipv4Layer("192.0.2.1", "198.51.100.2", 6), + tcpLayer(12345, 443), + }, + } +} + +func vxlanTCPPacketModel() *event.PacketModel { + return &event.PacketModel{ + Layers: []event.LayerSpec{ + ipv4Layer("203.0.113.1", "203.0.113.2", 17), + {Kind: "udp", UDP: &event.UDPLayer{SrcPort: 49152, DstPort: 4789}}, + {Kind: "vxlan", VXLAN: &event.VXLANLayer{}}, + {Kind: "ethernet", Ethernet: &event.EthernetLayer{}}, + ipv4Layer("192.0.2.1", "198.51.100.2", 6), + tcpLayer(12345, 443), + }, + } +} + +func ipv4Layer(src, dst string, proto uint8) event.LayerSpec { + return event.LayerSpec{ + Kind: "ipv4", + IPv4: &event.IPv4Layer{ + SrcAddr: netip.MustParseAddr(src), + DstAddr: netip.MustParseAddr(dst), + Protocol: proto, + TTL: 64, + }, + } +} + +func tcpLayer(srcPort, dstPort uint16) event.LayerSpec { + return event.LayerSpec{ + Kind: "tcp", + TCP: &event.TCPLayer{ + SrcPort: srcPort, + DstPort: dstPort, + Flags: 0x02, + Window: 65535, + }, + } +} + +func TestSFlowEncoderEmitsInterfaceCounterSample(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "record_kind": "interface_counter", + "agent_ip": "192.0.2.1", + "sub_agent_id": uint32(7), + "source_id": uint32(8), + "if_index": uint32(9), + "if_type": uint32(6), + "if_speed": uint64(1000), + "if_direction": uint32(1), + "if_status": uint32(3), + "if_in_octets": uint64(100), + "if_out_octets": uint64(200), + "if_in_errors": uint32(2), + "if_out_errors": uint32(4), + "if_promiscuous_mode": uint32(1), + }, + SFlow: &event.SFlowMetadata{ + AgentIP: "192.0.2.1", + SubAgentID: 7, + SourceID: 8, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample, ok := packet.Samples[0].(sflow.CounterSample) + if !ok { + t.Fatalf("expected counter sample, got %T", packet.Samples[0]) + } + if sample.Header.SourceIdValue != 8 { + t.Fatalf("expected source id 8, got %d", sample.Header.SourceIdValue) + } + if sample.CounterRecordsCount != 1 { + t.Fatalf("expected 1 counter record, got %d", sample.CounterRecordsCount) + } + record, ok := sample.Records[0].Data.(sflow.IfCounters) + if !ok { + t.Fatalf("expected IfCounters record, got %T", sample.Records[0].Data) + } + if record.IfIndex != 9 { + t.Fatalf("expected if_index 9, got %d", record.IfIndex) + } + if record.IfOutOctets != 200 { + t.Fatalf("expected if_out_octets 200, got %d", record.IfOutOctets) + } +} + +func TestSFlowEncoderUsesConfiguredExpandedCounterFormat(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + SFlow: config.SFlowConfig{ + CounterFormat: "expanded", + }, + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "record_kind": "interface_counter", + "agent_ip": "192.0.2.1", + "source_id": uint32(8), + "source_id_type": uint32(2), + "if_index": uint32(9), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.CounterSample) + if sample.Header.Format != sflow.SAMPLE_FORMAT_EXPANDED_COUNTER { + t.Fatalf("expected expanded counter sample format, got %d", sample.Header.Format) + } + if sample.Header.SourceIdType != 2 { + t.Fatalf("expected source_id_type 2, got %d", sample.Header.SourceIdType) + } +} + +func TestSFlowCounterEventOverridesConfiguredFormat(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + SFlow: config.SFlowConfig{ + CounterFormat: "standard", + }, + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "record_kind": "interface_counter", + "counter_format": "expanded", + "agent_ip": "192.0.2.1", + "source_id": uint32(8), + "source_id_type": uint32(3), + "if_index": uint32(9), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample := packet.Samples[0].(sflow.CounterSample) + if sample.Header.Format != sflow.SAMPLE_FORMAT_EXPANDED_COUNTER { + t.Fatalf("expected event override to force expanded format, got %d", sample.Header.Format) + } + if sample.Header.SourceIdType != 3 { + t.Fatalf("expected source_id_type 3, got %d", sample.Header.SourceIdType) + } +} + +func TestSFlowEncoderEmitsEthernetCounterSample(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + }) + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "record_kind": "ethernet_counter", + "agent_ip": "192.0.2.1", + "source_id": uint32(8), + "dot3_stats_fcs_errors": uint32(2), + "dot3_stats_frame_too_longs": uint32(11), + "dot3_stats_internal_mac_receive_errors": uint32(12), + "dot3_stats_symbol_errors": uint32(13), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + packet := decodeSFlowPacket(t, payloads[0]) + sample, ok := packet.Samples[0].(sflow.CounterSample) + if !ok { + t.Fatalf("expected counter sample, got %T", packet.Samples[0]) + } + record, ok := sample.Records[0].Data.(sflow.EthernetCounters) + if !ok { + t.Fatalf("expected EthernetCounters record, got %T", sample.Records[0].Data) + } + if record.Dot3StatsFCSErrors != 2 { + t.Fatalf("expected FCS errors 2, got %d", record.Dot3StatsFCSErrors) + } + if record.Dot3StatsSymbolErrors != 13 { + t.Fatalf("expected symbol errors 13, got %d", record.Dot3StatsSymbolErrors) + } +} + +func TestSFlowEncoderBatchesInterfaceCounterSamples(t *testing.T) { + enc := NewSFlowEncoder(config.EncoderConfig{ + Type: "sflow", + Batch: config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 4, + }, + }) + + for i := 0; i < 5; i++ { + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "record_kind": "interface_counter", + "agent_ip": "192.168.0.172", + "source_id": uint32(0), + "if_index": uint32(15 + i), + "if_type": uint32(6), + "if_status": uint32(3), + "if_in_octets": uint64(100 + i), + "if_out_octets": uint64(200 + i), + }, + }) + if err != nil { + t.Fatalf("Encode(%d) returned error: %v", i, err) + } + if i < 3 && len(payloads) != 0 { + t.Fatalf("expected event %d to stay buffered, got %d payloads", i, len(payloads)) + } + if i == 3 { + if len(payloads) != 1 { + t.Fatalf("expected fourth event to flush one payload, got %d", len(payloads)) + } + packet := decodeSFlowPacket(t, payloads[0]) + if len(packet.Samples) != 4 { + t.Fatalf("expected four counter samples, got %d", len(packet.Samples)) + } + for j, sample := range packet.Samples { + counter, ok := sample.(sflow.CounterSample) + if !ok { + t.Fatalf("sample %d expected CounterSample, got %T", j, sample) + } + if len(counter.Records) != 1 { + t.Fatalf("sample %d expected one record, got %d", j, len(counter.Records)) + } + if _, ok := counter.Records[0].Data.(sflow.IfCounters); !ok { + t.Fatalf("sample %d expected IfCounters, got %T", j, counter.Records[0].Data) + } + } + } + } +} + +func TestIPFIXEncoderEmitsTemplateAndDataRecord(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 42 + enc := NewIPFIXEncoder(cfg) + evt := testTemplatedFlowEvent() + evt.Fields["observation_domain_id"] = uint32(42) + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + + if decoded.ObservationDomainId != 42 { + t.Fatalf("expected observation domain 42, got %d", decoded.ObservationDomainId) + } + if decoded.SequenceNumber != 0 { + t.Fatalf("expected sequence 0, got %d", decoded.SequenceNumber) + } + if len(decoded.FlowSets) != 2 { + t.Fatalf("expected 2 flow sets, got %d", len(decoded.FlowSets)) + } + + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + if got := dataSet.Records[0].Values[0].Value.([]byte); !bytes.Equal(got, []byte{192, 0, 2, 10}) { + t.Fatalf("expected src_addr bytes 192.0.2.10, got %v", got) + } +} + +func TestTemplatedFlowEncoderExportsFlowDirection(t *testing.T) { + tests := []struct { + name string + typ string + fieldType uint16 + }{ + {name: "ipfix", typ: "ipfix", fieldType: netflow.IPFIX_FIELD_flowDirection}, + {name: "netflowv9", typ: "netflowv9", fieldType: netflow.NFV9_FIELD_DIRECTION}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := testTFlowEncoderConfig(tt.typ) + cfg.TemplatedFlow.Data.Select = []string{"flow_direction"} + cfg.TemplatedFlow.Data.Catalog["flow_direction"] = config.IPFIXFieldDefinition{ + ID: netflow.IPFIX_FIELD_flowDirection, + Length: 1, + Type: "unsigned8", + } + evt := testTemplatedFlowEvent() + evt.Fields["flow_direction"] = uint32(1) + + var payloads [][]byte + var err error + store := templates.NewTemplateFlowStore() + store.Start() + switch tt.typ { + case "ipfix": + payloads, err = NewIPFIXEncoder(cfg).Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + assertFlowDirectionTemplateAndValue(t, decoded.FlowSets, tt.fieldType) + case "netflowv9": + payloads, err = NewNFv9Encoder(cfg).Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + assertFlowDirectionTemplateAndValue(t, decoded.FlowSets, tt.fieldType) + } + }) + } +} + +func TestIPFIXEncoderBatchesCompatibleDataRecords(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewIPFIXEncoder(cfg) + + firstPayloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } + if len(firstPayloads) != 0 { + t.Fatalf("expected first IPFIX record to stay buffered, got %d payloads", len(firstPayloads)) + } + second := testTemplatedFlowEvent() + second.Fields["bytes"] = int64(654) + secondPayloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(secondPayloads) != 1 { + t.Fatalf("expected batched IPFIX payload, got %d", len(secondPayloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(secondPayloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + if decoded.SequenceNumber != 0 { + t.Fatalf("expected first batched packet sequence 0, got %d", decoded.SequenceNumber) + } + if len(decoded.FlowSets) != 2 { + t.Fatalf("expected template and data flow sets, got %d", len(decoded.FlowSets)) + } + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + if len(dataSet.Records) != 2 { + t.Fatalf("expected two IPFIX data records, got %d", len(dataSet.Records)) + } + + thirdPayloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode(third) returned error: %v", err) + } + if len(thirdPayloads) != 0 { + t.Fatalf("expected third IPFIX record to stay buffered, got %d payloads", len(thirdPayloads)) + } + flushed, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(flushed) != 1 { + t.Fatalf("expected one flushed IPFIX payload, got %d", len(flushed)) + } + var flushedDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(flushed[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &flushedDecoded); err != nil { + t.Fatalf("decode flushed ipfix payload: %v", err) + } + if flushedDecoded.SequenceNumber != 2 { + t.Fatalf("expected flushed packet sequence 2, got %d", flushedDecoded.SequenceNumber) + } +} + +func TestIPFIXEncoderBatchMaxBytesUsesRenderedDataFields(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Select = []string{"src_port", "dst_port", "proto", "bytes", "packets"} + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 32, + MaxBytes: 100, + } + enc := NewIPFIXEncoder(cfg) + + for i := 0; i < 3; i++ { + evt := testTemplatedFlowEvent() + evt.Fields["bytes"] = int64(i) + for j := 0; j < 128; j++ { + evt.Fields[fmt.Sprintf("unused_%03d", j)] = "this field is not encoded into ipfix" + } + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode(%d) returned error: %v", i, err) + } + if len(payloads) != 0 { + t.Fatalf("expected event %d to stay buffered, got %d payloads", i, len(payloads)) + } + } + if enc.estimatedBytes != 63 { + t.Fatalf("expected rendered field estimate 63, got %d", enc.estimatedBytes) + } + + payloads, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one flushed IPFIX packet, got %d", len(payloads)) + } + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + if len(dataSet.Records) != 3 { + t.Fatalf("expected three IPFIX data records, got %d", len(dataSet.Records)) + } +} + +func TestIPFIXEncoderCapacityFlushKeepsSmallTailBuffered(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Select = []string{"src_port", "dst_port", "proto", "bytes", "packets"} + cfg.MaxDatagramBytes = 150 + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 4, + MaxBytes: 70, + } + enc := NewIPFIXEncoder(cfg) + + var payloads [][]byte + for i := 0; i < 5; i++ { + evt := testTemplatedFlowEvent() + evt.Fields["bytes"] = int64(i) + got, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode(%d) returned error: %v", i, err) + } + payloads = append(payloads, got...) + } + if len(payloads) != 1 { + t.Fatalf("expected capacity flush to emit one packet and keep the tail buffered, got %d payloads", len(payloads)) + } + if len(enc.events) == 0 { + t.Fatalf("expected small tail to remain buffered") + } + + flushed, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(flushed) != 1 { + t.Fatalf("expected timer/shutdown flush to emit buffered tail, got %d payloads", len(flushed)) + } +} + +func TestIPFIXEncoderPayloadBatchFlowSetLengthsAreConsistent(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Catalog["frame_length"] = config.IPFIXFieldDefinition{ID: 312, Length: 2, Type: "unsigned16"} + cfg.TemplatedFlow.Data.Catalog["header_data"] = config.IPFIXFieldDefinition{ID: 315, Length: 0xffff, Type: "bytes"} + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + MaxBytes: 1200, + } + enc := NewIPFIXEncoder(cfg) + + if _, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + Fields: []event.SchemaField{ + {Role: "key", Name: "src_addr", Value: "ip4"}, + {Role: "key", Name: "dst_addr", Value: "ip4"}, + {Role: "current", Name: "frame_length"}, + {Role: "current", Name: "header_data"}, + }, + BaseTemplateID: 256, + }, + }); err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + first := testTemplatedFlowEvent() + firstHeader := append(bytes.Repeat([]byte{0xff}, 6), bytes.Repeat([]byte{0xab}, 90)...) + first.Fields["frame_length"] = uint32(0xffff) + first.Fields["header_data"] = firstHeader + if payloads, err := enc.Encode(first); err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } else if len(payloads) != 0 { + t.Fatalf("expected first payload record to stay buffered, got %d payloads", len(payloads)) + } + + second := testTemplatedFlowEvent() + second.Fields["src_addr"] = "2001:db8::10" + second.Fields["dst_addr"] = "2001:db8::20" + second.Fields["frame_length"] = uint32(300) + second.Fields["header_data"] = bytes.Repeat([]byte{0xcd}, 300) + payloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one batched payload, got %d", len(payloads)) + } + + firstHeaderOffset := 16 + 4 + 4 + 4 + 2 + if payloads[0][firstHeaderOffset] != byte(len(firstHeader)) { + t.Fatalf("expected first header_data short length prefix %d at offset %d, got %#x", len(firstHeader), firstHeaderOffset, payloads[0][firstHeaderOffset]) + } + if got := payloads[0][firstHeaderOffset+1 : firstHeaderOffset+7]; !bytes.Equal(got, bytes.Repeat([]byte{0xff}, 6)) { + t.Fatalf("expected broadcast header_data after length prefix, got %x", got) + } + packetLength := int(binary.BigEndian.Uint16(payloads[0][2:4])) + if packetLength != len(payloads[0]) { + t.Fatalf("expected IPFIX packet length %d to equal payload bytes %d", packetLength, len(payloads[0])) + } + offset := 16 + for flowSet := 0; offset < len(payloads[0]); flowSet++ { + if offset+4 > len(payloads[0]) { + t.Fatalf("flowset %d header at offset %d exceeds payload length %d", flowSet, offset, len(payloads[0])) + } + length := int(binary.BigEndian.Uint16(payloads[0][offset+2 : offset+4])) + if length < 4 { + t.Fatalf("flowset %d at offset %d has invalid length %d", flowSet, offset, length) + } + if offset+length > len(payloads[0]) { + t.Fatalf("flowset %d at offset %d length %d exceeds payload length %d", flowSet, offset, length, len(payloads[0])) + } + offset += length + } + if offset != len(payloads[0]) { + t.Fatalf("expected flowset lengths to end at %d, got %d", len(payloads[0]), offset) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + templatePayloads, err := enc.encodeSchemaTemplates(enc.dataSchemas["flow_data"]) + if err != nil { + t.Fatalf("encodeSchemaTemplates returned error: %v", err) + } + for _, payload := range templatePayloads { + var templatePacket netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), store, ctx, nil, &templatePacket); err != nil { + t.Fatalf("decode schema template payload: %v", err) + } + } + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, ctx, nil, &decoded); err != nil { + t.Fatalf("decode batched payload: %v", err) + } + if len(decoded.FlowSets) != 1 { + t.Fatalf("expected one data set, got %d", len(decoded.FlowSets)) + } + firstSet := decoded.FlowSets[0].(netflow.DataFlowSet) + if len(firstSet.Records) != 2 { + t.Fatalf("expected two records in one data set, got %d", len(firstSet.Records)) + } + values := firstSet.Records[0].Values + if got := values[len(values)-1].Value.([]byte); !bytes.Equal(got, firstHeader) { + t.Fatalf("expected decoded broadcast header_data length %d, got %d", len(firstHeader), len(got)) + } +} + +func TestIPFIXEncoderFallbackUsesConfiguredSelectWidth(t *testing.T) { + enc := NewIPFIXEncoder(testTFlowEncoderConfig("ipfix")) + evt := testTemplatedFlowEvent() + delete(evt.Fields, "bytes") + delete(evt.Fields, "packets") + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one IPFIX payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + if len(decoded.FlowSets) != 2 { + t.Fatalf("expected template and data flow sets, got %d", len(decoded.FlowSets)) + } + templateSet, ok := decoded.FlowSets[0].(netflow.TemplateFlowSet) + if !ok { + t.Fatalf("expected first flow set to be TemplateFlowSet, got %T", decoded.FlowSets[0]) + } + if templateSet.Records[0].FieldCount != 9 { + t.Fatalf("expected configured 9-field dual-stack template, got %d", templateSet.Records[0].FieldCount) + } + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + values := dataSet.Records[0].Values + if len(values) != 9 { + t.Fatalf("expected configured 9-value dual-stack data record, got %d", len(values)) + } + if got := values[7].Value.([]byte); !bytes.Equal(got, make([]byte, 8)) { + t.Fatalf("expected missing bytes field to default to zero, got %v", got) + } + if got := values[8].Value.([]byte); !bytes.Equal(got, make([]byte, 8)) { + t.Fatalf("expected missing packets field to default to zero, got %v", got) + } +} + +func TestIPFIXEncoderFallbackUsesConfiguredFieldMatch(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Select = []string{"source_id"} + cfg.TemplatedFlow.Data.Matches = []config.TemplatedFlowFieldMatch{ + {Field: "source_id", From: "tid"}, + } + cfg.TemplatedFlow.Data.Catalog["source_id"] = config.IPFIXFieldDefinition{ + ID: netflow.IPFIX_FIELD_observationPointId, + Length: 8, + Type: "unsigned64", + } + enc := NewIPFIXEncoder(cfg) + evt := testTemplatedFlowEvent() + delete(evt.Fields, "source_id") + evt.Fields["tid"] = uint32(99) + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one IPFIX payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + values := dataSet.Records[0].Values + if len(values) != 1 { + t.Fatalf("expected one matched value, got %#v", values) + } + if got := values[0].Value.([]byte); !bytes.Equal(got, encodeU64(99)) { + t.Fatalf("expected source_id value remapped from tid=99, got %v", got) + } +} + +func TestIPFIXEncoderSchemaUsesConfiguredFieldMatch(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Matches = []config.TemplatedFlowFieldMatch{ + {Field: "source_id", From: "tid"}, + } + cfg.TemplatedFlow.Data.Catalog["source_id"] = config.IPFIXFieldDefinition{ + ID: netflow.IPFIX_FIELD_observationPointId, + Length: 8, + Type: "unsigned64", + } + enc := NewIPFIXEncoder(cfg) + + templatePayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Stream: "flow_data", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + BaseTemplateID: 256, + Fields: []event.SchemaField{ + {Role: "current", Name: "source_id"}, + }, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + evt := testTemplatedFlowEvent() + evt.Stream = "flow_data" + delete(evt.Fields, "source_id") + evt.Fields["tid"] = uint32(99) + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one IPFIX payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + for _, payload := range templatePayloads { + var templateDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), store, ctx, nil, &templateDecoded); err != nil { + t.Fatalf("decode schema payload: %v", err) + } + } + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, ctx, nil, &decoded); err != nil { + t.Fatalf("decode data payload: %v", err) + } + dataSet, ok := decoded.FlowSets[0].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected data flow set, got %T", decoded.FlowSets[0]) + } + values := dataSet.Records[0].Values + if len(values) != 1 { + t.Fatalf("expected one matched value, got %#v", values) + } + if got := values[0].Value.([]byte); !bytes.Equal(got, encodeU64(99)) { + t.Fatalf("expected source_id value remapped from tid=99, got %v", got) + } +} + +func TestIPFIXEncoderBatchesMultipleDataSetsInOnePacket(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewIPFIXEncoder(cfg) + + firstPayloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } + if len(firstPayloads) != 0 { + t.Fatalf("expected first IPFIX record to stay buffered, got %d payloads", len(firstPayloads)) + } + + second := testTemplatedFlowEvent() + second.Fields["template_id"] = uint32(257) + secondPayloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(secondPayloads) != 1 { + t.Fatalf("expected one batched IPFIX payload, got %d", len(secondPayloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(secondPayloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + if decoded.SequenceNumber != 0 { + t.Fatalf("expected first batched packet sequence 0, got %d", decoded.SequenceNumber) + } + if len(decoded.FlowSets) != 3 { + t.Fatalf("expected template set and two data sets, got %d", len(decoded.FlowSets)) + } + templateSet, ok := decoded.FlowSets[0].(netflow.TemplateFlowSet) + if !ok { + t.Fatalf("expected first flow set to be TemplateFlowSet, got %T", decoded.FlowSets[0]) + } + if len(templateSet.Records) != 2 { + t.Fatalf("expected two template records, got %d", len(templateSet.Records)) + } + firstSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + secondSet, ok := decoded.FlowSets[2].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected third flow set to be DataFlowSet, got %T", decoded.FlowSets[2]) + } + if firstSet.Id != 256 || secondSet.Id != 257 { + t.Fatalf("expected data set ids 256 and 257, got %d and %d", firstSet.Id, secondSet.Id) + } + if len(firstSet.Records) != 1 || len(secondSet.Records) != 1 { + t.Fatalf("expected one record per data set, got %d and %d", len(firstSet.Records), len(secondSet.Records)) + } + + if payloads, err := enc.Encode(testTemplatedFlowEvent()); err != nil { + t.Fatalf("Encode(third) returned error: %v", err) + } else if len(payloads) != 0 { + t.Fatalf("expected third IPFIX record to stay buffered, got %d payloads", len(payloads)) + } + flushed, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(flushed) != 1 { + t.Fatalf("expected one flushed IPFIX payload, got %d", len(flushed)) + } + var flushedDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(flushed[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &flushedDecoded); err != nil { + t.Fatalf("decode flushed ipfix payload: %v", err) + } + if flushedDecoded.SequenceNumber != 2 { + t.Fatalf("expected flushed packet sequence 2 after two data records, got %d", flushedDecoded.SequenceNumber) + } +} + +func TestIPFIXEncoderBatchesSchemaDataSetsInOnePacket(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewIPFIXEncoder(cfg) + + templatePayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"src_addr", "dst_addr", "src_port", "dst_port", "proto", "bytes", "packets"}, + BaseTemplateID: 256, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + for _, payload := range templatePayloads { + var templatePacket netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), store, ctx, nil, &templatePacket); err != nil { + t.Fatalf("decode schema template payload: %v", err) + } + } + + if payloads, err := enc.Encode(testTemplatedFlowEvent()); err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } else if len(payloads) != 0 { + t.Fatalf("expected first schema record to stay buffered, got %d payloads", len(payloads)) + } + + second := testTemplatedFlowEvent() + second.Fields["src_addr"] = "2001:db8::10" + second.Fields["dst_addr"] = "2001:db8::20" + payloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one schema batched IPFIX payload, got %d", len(payloads)) + } + + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, ctx, nil, &decoded); err != nil { + t.Fatalf("decode schema data payload: %v", err) + } + if len(decoded.FlowSets) != 1 { + t.Fatalf("expected one dual-stack schema data set, got %d", len(decoded.FlowSets)) + } + dataSet, ok := decoded.FlowSets[0].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected first flow set to be DataFlowSet, got %T", decoded.FlowSets[0]) + } + if dataSet.Id != 256 || len(dataSet.Records) != 2 { + t.Fatalf("expected schema data set 256 with two records, got id=%d records=%d", dataSet.Id, len(dataSet.Records)) + } +} + +func TestIPFIXEncoderBatchesFallbackAddressFamiliesWithOneDualStackTemplate(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewIPFIXEncoder(cfg) + + if payloads, err := enc.Encode(testTemplatedFlowEvent()); err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } else if len(payloads) != 0 { + t.Fatalf("expected first IPFIX record to stay buffered, got %d payloads", len(payloads)) + } + + second := testTemplatedFlowEvent() + second.Fields["src_addr"] = "2001:db8::10" + second.Fields["dst_addr"] = "2001:db8::20" + payloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one mixed IPv4/IPv6 payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, ctx, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + if len(decoded.FlowSets) != 2 { + t.Fatalf("expected template set plus one dual-stack data set, got %d", len(decoded.FlowSets)) + } + templateSet, ok := decoded.FlowSets[0].(netflow.TemplateFlowSet) + if !ok { + t.Fatalf("expected first flow set to be TemplateFlowSet, got %T", decoded.FlowSets[0]) + } + if len(templateSet.Records) != 1 { + t.Fatalf("expected one dual-stack address-family template, got %d", len(templateSet.Records)) + } + if templateSet.Records[0].Fields[0].Type != netflow.IPFIX_FIELD_sourceIPv4Address { + t.Fatalf("expected first template field sourceIPv4Address, got %d", templateSet.Records[0].Fields[0].Type) + } + if templateSet.Records[0].Fields[1].Type != netflow.IPFIX_FIELD_sourceIPv6Address { + t.Fatalf("expected second template field sourceIPv6Address, got %d", templateSet.Records[0].Fields[1].Type) + } + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + if dataSet.Id != 256 || len(dataSet.Records) != 2 { + t.Fatalf("expected dual-stack data set 256 with two records, got id=%d records=%d", dataSet.Id, len(dataSet.Records)) + } +} + +func TestIPFIXEncoderPreservesBufferedEventsOnBatchError(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewIPFIXEncoder(cfg) + first := testTemplatedFlowEvent() + bad := &event.Event{ + ReceivedAt: time.Unix(1, 0).UTC(), + Fields: map[string]any{ + "unmapped_field": uint32(1), + }, + } + + payloads, err := enc.Encode(first) + if err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } + if len(payloads) != 0 { + t.Fatalf("expected first record to stay buffered, got %d payloads", len(payloads)) + } + payloads, err = enc.Encode(bad) + if err == nil { + t.Fatalf("expected bad batch event to return error") + } + if len(payloads) != 0 { + t.Fatalf("expected no payloads from failed batch, got %d", len(payloads)) + } + if len(enc.events) != 2 { + t.Fatalf("expected failed batch to preserve 2 buffered events, got %d", len(enc.events)) + } + if enc.events[0] != first || enc.events[1] != bad { + t.Fatalf("expected buffered events to remain in original order") + } + if enc.estimatedBytes == 0 { + t.Fatalf("expected estimated byte count to be preserved") + } +} + +func TestIPFIXEncoderIgnoresEventObservationDomainID(t *testing.T) { + enc := NewIPFIXEncoder(testTFlowEncoderConfig("ipfix")) + evt := testTemplatedFlowEvent() + evt.Fields["observation_domain_id"] = uint32(777) + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + if decoded.ObservationDomainId != 0 { + t.Fatalf("expected exporter observation domain 0, got %d", decoded.ObservationDomainId) + } +} + +func TestIPFIXEncoderConfigObservationDomainIDOverridesEvent(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 888 + enc := NewIPFIXEncoder(cfg) + evt := testTemplatedFlowEvent() + evt.Fields["observation_domain_id"] = uint32(777) + evt.Fields["source_id"] = uint32(42) + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + if decoded.ObservationDomainId != 888 { + t.Fatalf("expected observation domain 888, got %d", decoded.ObservationDomainId) + } +} + +func TestIPFIXDataRecordUsesSourceIDAsObservationPointID(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Select = []string{"source_id", "bytes"} + cfg.TemplatedFlow.Data.Catalog["source_id"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_observationPointId, Length: 8, Type: "unsigned64"} + enc := NewIPFIXEncoder(cfg) + + evt := testTemplatedFlowEvent() + delete(evt.Fields, "source_id") + evt.Source.SourceID = 0 + evt.Source.SourceIDSet = true + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + templateSet, ok := decoded.FlowSets[0].(netflow.TemplateFlowSet) + if !ok { + t.Fatalf("expected template flow set, got %T", decoded.FlowSets[0]) + } + if templateSet.Records[0].Fields[0].Type != netflow.IPFIX_FIELD_observationPointId { + t.Fatalf("expected observationPointId template field, got %d", templateSet.Records[0].Fields[0].Type) + } + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected data flow set, got %T", decoded.FlowSets[1]) + } + if got := dataSet.Records[0].Values[0].Value.([]byte); !bytes.Equal(got, encodeU64(0)) { + t.Fatalf("expected observation point value 0, got %v", got) + } +} + +func TestIPFIXSourceOptionsUseObservationPointScope(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 888 + enc := NewIPFIXEncoder(cfg) + + payloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init", + }, + Source: event.SourceMetadata{ + AgentIP: "198.51.100.99", + SourceID: 42, + Sampling: &event.SamplingMetadata{ + Rate: 250, + }, + }, + Fields: map[string]any{ + "observation_domain_id": uint32(777), + }, + Payload: event.SourceInit{ + ObservationDomainID: 777, + SourceID: 42, + SamplingRate: 100, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one options payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix source options payload: %v", err) + } + if decoded.ObservationDomainId != 888 { + t.Fatalf("expected packet observation domain 888, got %d", decoded.ObservationDomainId) + } + optionsTemplate, ok := decoded.FlowSets[0].(netflow.IPFIXOptionsTemplateFlowSet) + if !ok { + t.Fatalf("expected options template flow set, got %T", decoded.FlowSets[0]) + } + scopeField := optionsTemplate.Records[0].Scopes[0] + if scopeField.Type != netflow.IPFIX_FIELD_observationPointId { + t.Fatalf("expected observationPointId scope field, got %d", scopeField.Type) + } + if scopeField.Length != 8 { + t.Fatalf("expected observationPointId scope length 8, got %d", scopeField.Length) + } + optionsData, ok := decoded.FlowSets[1].(netflow.OptionsDataFlowSet) + if !ok { + t.Fatalf("expected options data flow set, got %T", decoded.FlowSets[1]) + } + scope := optionsData.Records[0].ScopesValues[0] + if scope.Type != netflow.IPFIX_FIELD_observationPointId { + t.Fatalf("expected observationPointId scope, got %d", scope.Type) + } + if got := scope.Value.([]byte); !bytes.Equal(got, encodeU64(42)) { + t.Fatalf("expected options observation point scope 42, got %v", got) + } + option := optionsData.Records[0].OptionsValues[0] + if got := option.Value.([]byte); !bytes.Equal(got, encodeU32(250)) { + t.Fatalf("expected source metadata sampling rate 250, got %v", got) + } + + dataPayloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode data returned error: %v", err) + } + if len(dataPayloads) != 1 { + t.Fatalf("expected one data payload, got %d", len(dataPayloads)) + } + var dataDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(dataPayloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &dataDecoded); err != nil { + t.Fatalf("decode ipfix data payload: %v", err) + } + if dataDecoded.SequenceNumber != 1 { + t.Fatalf("expected first data packet after options data record sequence 1, got %d", dataDecoded.SequenceNumber) + } +} + +func TestIPFIXAggregationSchemaOptionsDataUsesKeysAsScopes(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 888 + cfg.TemplatedFlow.Data.Catalog["observation_domain_id"] = config.IPFIXFieldDefinition{ID: 149, Length: 4, Type: "unsigned32"} + cfg.TemplatedFlow.Data.Catalog["if_index"] = config.IPFIXFieldDefinition{ID: 10, Length: 4, Type: "unsigned32"} + cfg.TemplatedFlow.Data.Catalog["interface_name"] = config.IPFIXFieldDefinition{ID: 82, Length: 0xffff, Type: "string"} + enc := NewIPFIXEncoder(cfg) + + schemaPayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "interface_options", + }, + Payload: event.AggregationSchema{ + Stream: "interface_options", + Fields: []event.SchemaField{ + {Role: "static", Name: "tflow_record_type", Value: "options"}, + {Role: "key", Name: "observation_domain_id"}, + {Role: "key", Name: "if_index"}, + {Role: "current", Name: "interface_name"}, + }, + BaseTemplateID: 1300, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + if len(schemaPayloads) != 1 { + t.Fatalf("expected one options schema payload, got %d", len(schemaPayloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + var schemaDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(schemaPayloads[0]), store, ctx, nil, &schemaDecoded); err != nil { + t.Fatalf("decode options schema payload: %v", err) + } + optionsTemplate, ok := schemaDecoded.FlowSets[0].(netflow.IPFIXOptionsTemplateFlowSet) + if !ok { + t.Fatalf("expected options template flow set, got %T", schemaDecoded.FlowSets[0]) + } + record := optionsTemplate.Records[0] + if record.TemplateId != 1300 || record.FieldCount != 3 || record.ScopeFieldCount != 2 { + t.Fatalf("unexpected options template counts: %#v", record) + } + if record.Scopes[0].Type != 149 || record.Scopes[1].Type != 10 || record.Options[0].Type != 82 { + t.Fatalf("unexpected options template fields: scopes=%#v options=%#v", record.Scopes, record.Options) + } + + dataPayloads, err := enc.Encode(&event.Event{ + ReceivedAt: testEventTime(), + Stream: "interface_options", + Fields: map[string]any{ + "observation_domain_id": uint32(777), + "if_index": uint32(2), + "interface_name": "eth0", + }, + }) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + if len(dataPayloads) != 1 { + t.Fatalf("expected one options data payload, got %d", len(dataPayloads)) + } + var dataDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(dataPayloads[0]), store, ctx, nil, &dataDecoded); err != nil { + t.Fatalf("decode options data payload: %v", err) + } + optionsData, ok := dataDecoded.FlowSets[0].(netflow.OptionsDataFlowSet) + if !ok { + t.Fatalf("expected options data flow set, got %T", dataDecoded.FlowSets[0]) + } + values := optionsData.Records[0] + if !bytes.Equal(values.ScopesValues[0].Value.([]byte), encodeU32(777)) || !bytes.Equal(values.ScopesValues[1].Value.([]byte), encodeU32(2)) { + t.Fatalf("unexpected options scope values: %#v", values.ScopesValues) + } + if !bytes.Equal(values.OptionsValues[0].Value.([]byte), []byte("eth0")) { + t.Fatalf("expected interface name option eth0, got %#v", values.OptionsValues[0].Value) + } +} + +func TestIPFIXAggregationOptionsBatchCombinesRecordsInOneSet(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 888 + cfg.TemplatedFlow.Data.Catalog["observation_domain_id"] = config.IPFIXFieldDefinition{ID: 149, Length: 4, Type: "unsigned32"} + cfg.TemplatedFlow.Data.Catalog["if_index"] = config.IPFIXFieldDefinition{ID: 10, Length: 4, Type: "unsigned32"} + cfg.TemplatedFlow.Data.Catalog["interface_name"] = config.IPFIXFieldDefinition{ID: 82, Length: 0xffff, Type: "string"} + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewIPFIXEncoder(cfg) + + schemaPayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "interface_options", + }, + Payload: event.AggregationSchema{ + Stream: "interface_options", + Fields: []event.SchemaField{ + {Role: "static", Name: "tflow_record_type", Value: "options"}, + {Role: "key", Name: "observation_domain_id"}, + {Role: "key", Name: "if_index"}, + {Role: "current", Name: "interface_name"}, + }, + BaseTemplateID: 1300, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + firstPayloads, err := enc.Encode(&event.Event{ + ReceivedAt: testEventTime(), + Stream: "interface_options", + Fields: map[string]any{ + "observation_domain_id": uint32(777), + "if_index": uint32(2), + "interface_name": "eth0", + }, + }) + if err != nil { + t.Fatalf("first Encode returned error: %v", err) + } + if len(firstPayloads) != 0 { + t.Fatalf("expected first options record to stay buffered, got %d payloads", len(firstPayloads)) + } + + secondPayloads, err := enc.Encode(&event.Event{ + ReceivedAt: testEventTime(), + Stream: "interface_options", + Fields: map[string]any{ + "observation_domain_id": uint32(777), + "if_index": uint32(3), + "interface_name": "eth1", + }, + }) + if err != nil { + t.Fatalf("second Encode returned error: %v", err) + } + if len(secondPayloads) != 1 { + t.Fatalf("expected one batched options payload, got %d", len(secondPayloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + for _, payload := range schemaPayloads { + var schemaDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &schemaDecoded); err != nil { + t.Fatalf("decode options schema payload: %v", err) + } + } + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(secondPayloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode batched options payload: %v", err) + } + if len(decoded.FlowSets) != 1 { + t.Fatalf("expected one options data set, got %d flow sets", len(decoded.FlowSets)) + } + optionsData, ok := decoded.FlowSets[0].(netflow.OptionsDataFlowSet) + if !ok { + t.Fatalf("expected flow set to be options data, got %T", decoded.FlowSets[0]) + } + if optionsData.Id != 1300 { + t.Fatalf("expected options data set id 1300, got %d", optionsData.Id) + } + if len(optionsData.Records) != 2 { + t.Fatalf("expected two options records in one set, got %d", len(optionsData.Records)) + } + if !bytes.Equal(optionsData.Records[0].OptionsValues[0].Value.([]byte), []byte("eth0")) || !bytes.Equal(optionsData.Records[1].OptionsValues[0].Value.([]byte), []byte("eth1")) { + t.Fatalf("expected eth0/eth1 options records, got %#v", optionsData.Records) + } +} + +func TestIPFIXSourceOptionsRefreshMergesObservationPointRecords(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 888 + cfg.TemplatedFlow.OptionsRefresh = 1 + enc := NewIPFIXEncoder(cfg) + + for _, sourceID := range []uint32{42, 43} { + if _, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init", + }, + Source: event.SourceMetadata{ + SourceID: sourceID, + Sampling: &event.SamplingMetadata{ + Rate: sourceID + 100, + }, + }, + }); err != nil { + t.Fatalf("Encode source_init %d returned error: %v", sourceID, err) + } + } + if len(enc.sourceOptions) != 2 { + t.Fatalf("expected two source options states, got %d", len(enc.sourceOptions)) + } + + enc.lastOptionsRun = time.Time{} + payloads, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one merged source options payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode merged source options payload: %v", err) + } + optionsData, ok := decoded.FlowSets[1].(netflow.OptionsDataFlowSet) + if !ok { + t.Fatalf("expected options data flow set, got %T", decoded.FlowSets[1]) + } + if len(optionsData.Records) != 2 { + t.Fatalf("expected two merged options data records, got %d", len(optionsData.Records)) + } + for i, wantSourceID := range []uint64{42, 43} { + if got := optionsData.Records[i].ScopesValues[0].Value.([]byte); !bytes.Equal(got, encodeU64(wantSourceID)) { + t.Fatalf("expected options record %d observation point %d, got %v", i, wantSourceID, got) + } + } +} + +func TestIPFIXSourceInitBatchSendsOneTemplateAndMultipleDataRecords(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 888 + enc := NewIPFIXEncoder(cfg) + + payloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init_batch", + }, + Payload: []*event.Event{ + { + Source: event.SourceMetadata{ + SourceID: 0, + SourceIDSet: true, + Sampling: &event.SamplingMetadata{ + Rate: 100, + }, + }, + }, + { + Source: event.SourceMetadata{ + SourceID: 1, + SourceIDSet: true, + Sampling: &event.SamplingMetadata{ + Rate: 200, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one source-init batch payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode source-init batch payload: %v", err) + } + optionsTemplate, ok := decoded.FlowSets[0].(netflow.IPFIXOptionsTemplateFlowSet) + if !ok { + t.Fatalf("expected options template flow set, got %T", decoded.FlowSets[0]) + } + if len(optionsTemplate.Records) != 1 || optionsTemplate.Records[0].TemplateId != 1024 { + t.Fatalf("expected one options template 1024, got %#v", optionsTemplate.Records) + } + optionsData, ok := decoded.FlowSets[1].(netflow.OptionsDataFlowSet) + if !ok { + t.Fatalf("expected options data flow set, got %T", decoded.FlowSets[1]) + } + if len(optionsData.Records) != 2 { + t.Fatalf("expected two options data records, got %d", len(optionsData.Records)) + } + for i, wantSourceID := range []uint64{0, 1} { + if got := optionsData.Records[i].ScopesValues[0].Value.([]byte); !bytes.Equal(got, encodeU64(wantSourceID)) { + t.Fatalf("expected options data record %d observation point %d, got %v", i, wantSourceID, got) + } + } +} + +func TestNFv9EncoderEmitsTemplateAndDataRecord(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.ObservationDomainID = 42 + enc := NewNFv9Encoder(cfg) + + payloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + + if decoded.SourceId != 42 { + t.Fatalf("expected source id 42, got %d", decoded.SourceId) + } + if decoded.SequenceNumber != 0 { + t.Fatalf("expected sequence 0, got %d", decoded.SequenceNumber) + } + if len(decoded.FlowSets) != 2 { + t.Fatalf("expected 2 flow sets, got %d", len(decoded.FlowSets)) + } + + templateSet, ok := decoded.FlowSets[0].(netflow.TemplateFlowSet) + if !ok { + t.Fatalf("expected first flow set to be TemplateFlowSet, got %T", decoded.FlowSets[0]) + } + if templateSet.Records[0].Fields[0].Type != 8 { + t.Fatalf("expected first field type 8 for src_addr, got %d", templateSet.Records[0].Fields[0].Type) + } +} + +func TestNFv9EncoderSequenceAdvancesPerExportPacket(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.ObservationDomainID = 42 + enc := NewNFv9Encoder(cfg) + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + decodePacket := func(payload []byte) netflow.NFv9Packet { + t.Helper() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), store, ctx, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + return decoded + } + + templatePayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"bytes"}, + BaseTemplateID: 300, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + if len(templatePayloads) != 1 { + t.Fatalf("expected one template payload, got %d", len(templatePayloads)) + } + templateDecoded := decodePacket(templatePayloads[0]) + if templateDecoded.SequenceNumber != 0 { + t.Fatalf("expected template packet sequence 0, got %d", templateDecoded.SequenceNumber) + } + + optionsPayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "source_init", + }, + Source: event.SourceMetadata{ + SourceID: 7, + SourceIDSet: true, + Sampling: &event.SamplingMetadata{ + Rate: 100, + }, + }, + }) + if err != nil { + t.Fatalf("source_init Encode returned error: %v", err) + } + if len(optionsPayloads) != 1 { + t.Fatalf("expected one options payload, got %d", len(optionsPayloads)) + } + optionsDecoded := decodePacket(optionsPayloads[0]) + if optionsDecoded.SequenceNumber != 1 { + t.Fatalf("expected options packet sequence 1, got %d", optionsDecoded.SequenceNumber) + } + + dataPayloads, err := enc.Encode(&event.Event{ + ReceivedAt: testEventTime(), + Stream: "flow_data", + Fields: map[string]any{ + "bytes": uint64(64), + }, + }) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + if len(dataPayloads) != 1 { + t.Fatalf("expected one data payload, got %d", len(dataPayloads)) + } + dataDecoded := decodePacket(dataPayloads[0]) + if dataDecoded.SequenceNumber != 2 { + t.Fatalf("expected data packet sequence 2, got %d", dataDecoded.SequenceNumber) + } +} + +func TestNFv9EncoderBatchesCompatibleDataRecords(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewNFv9Encoder(cfg) + + firstPayloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } + if len(firstPayloads) != 0 { + t.Fatalf("expected first NetFlow v9 record to stay buffered, got %d payloads", len(firstPayloads)) + } + second := testTemplatedFlowEvent() + second.Fields["bytes"] = int64(654) + secondPayloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(secondPayloads) != 1 { + t.Fatalf("expected batched NetFlow v9 payload, got %d", len(secondPayloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(secondPayloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + if decoded.SequenceNumber != 0 { + t.Fatalf("expected first batched packet sequence 0, got %d", decoded.SequenceNumber) + } + if decoded.Count != 2 { + t.Fatalf("expected template and data flow set count 2, got %d", decoded.Count) + } + if len(decoded.FlowSets) != 2 { + t.Fatalf("expected template and data flow sets, got %d", len(decoded.FlowSets)) + } + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + if len(dataSet.Records) != 2 { + t.Fatalf("expected two NetFlow v9 data records, got %d", len(dataSet.Records)) + } + + thirdPayloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode(third) returned error: %v", err) + } + if len(thirdPayloads) != 0 { + t.Fatalf("expected third NetFlow v9 record to stay buffered, got %d payloads", len(thirdPayloads)) + } + flushed, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(flushed) != 1 { + t.Fatalf("expected one flushed NetFlow v9 payload, got %d", len(flushed)) + } + var flushedDecoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(flushed[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &flushedDecoded, nil); err != nil { + t.Fatalf("decode flushed netflow v9 payload: %v", err) + } + if flushedDecoded.SequenceNumber != 1 { + t.Fatalf("expected flushed packet sequence 1, got %d", flushedDecoded.SequenceNumber) + } +} + +func TestNFv9EncoderBatchMaxBytesUsesRenderedDataFields(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.Data.Select = []string{"src_port", "dst_port", "proto", "bytes", "packets"} + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 32, + MaxBytes: 100, + } + enc := NewNFv9Encoder(cfg) + + for i := 0; i < 3; i++ { + evt := testTemplatedFlowEvent() + evt.Fields["bytes"] = int64(i) + for j := 0; j < 128; j++ { + evt.Fields[fmt.Sprintf("unused_%03d", j)] = "this field is not encoded into netflow v9" + } + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode(%d) returned error: %v", i, err) + } + if len(payloads) != 0 { + t.Fatalf("expected event %d to stay buffered, got %d payloads", i, len(payloads)) + } + } + if enc.estimatedBytes != 63 { + t.Fatalf("expected rendered field estimate 63, got %d", enc.estimatedBytes) + } + + payloads, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one flushed NetFlow v9 packet, got %d", len(payloads)) + } + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + dataSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + if len(dataSet.Records) != 3 { + t.Fatalf("expected three NetFlow v9 data records, got %d", len(dataSet.Records)) + } +} + +func TestNFv9EncoderCapacityFlushKeepsSmallTailBuffered(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.Data.Select = []string{"src_port", "dst_port", "proto", "bytes", "packets"} + cfg.MaxDatagramBytes = 150 + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 4, + MaxBytes: 70, + } + enc := NewNFv9Encoder(cfg) + + var payloads [][]byte + for i := 0; i < 5; i++ { + evt := testTemplatedFlowEvent() + evt.Fields["bytes"] = int64(i) + got, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode(%d) returned error: %v", i, err) + } + payloads = append(payloads, got...) + } + if len(payloads) != 1 { + t.Fatalf("expected capacity flush to emit one packet and keep the tail buffered, got %d payloads", len(payloads)) + } + if len(enc.events) == 0 { + t.Fatalf("expected small tail to remain buffered") + } + + flushed, err := enc.Flush() + if err != nil { + t.Fatalf("Flush returned error: %v", err) + } + if len(flushed) != 1 { + t.Fatalf("expected timer/shutdown flush to emit buffered tail, got %d payloads", len(flushed)) + } +} + +func TestNFv9EncoderBatchesMultipleDataSetsInOnePacket(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewNFv9Encoder(cfg) + + if payloads, err := enc.Encode(testTemplatedFlowEvent()); err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } else if len(payloads) != 0 { + t.Fatalf("expected first NetFlow v9 record to stay buffered, got %d payloads", len(payloads)) + } + + second := testTemplatedFlowEvent() + second.Fields["template_id"] = uint32(257) + payloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one batched NetFlow v9 payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + if len(decoded.FlowSets) != 3 { + t.Fatalf("expected template set and two data sets, got %d", len(decoded.FlowSets)) + } + templateSet, ok := decoded.FlowSets[0].(netflow.TemplateFlowSet) + if !ok { + t.Fatalf("expected first flow set to be TemplateFlowSet, got %T", decoded.FlowSets[0]) + } + if len(templateSet.Records) != 2 { + t.Fatalf("expected two template records, got %d", len(templateSet.Records)) + } + firstSet, ok := decoded.FlowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", decoded.FlowSets[1]) + } + secondSet, ok := decoded.FlowSets[2].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected third flow set to be DataFlowSet, got %T", decoded.FlowSets[2]) + } + if firstSet.Id != 256 || secondSet.Id != 257 { + t.Fatalf("expected data set ids 256 and 257, got %d and %d", firstSet.Id, secondSet.Id) + } +} + +func TestNFv9EncoderBatchedSwitchedTimesUseSharedPacketBase(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.Data.Select = append(cfg.TemplatedFlow.Data.Select, "start_time_unix", "end_time_unix") + cfg.Batch = config.BatchConfig{ + Enabled: testBoolPtr(true), + MaxRecords: 2, + } + enc := NewNFv9Encoder(cfg) + + if payloads, err := enc.Encode(testTemplatedFlowEvent()); err != nil { + t.Fatalf("Encode(first) returned error: %v", err) + } else if len(payloads) != 0 { + t.Fatalf("expected first NetFlow v9 record to stay buffered, got %d payloads", len(payloads)) + } + + second := testTemplatedFlowEvent() + second.Fields["start_time_unix"] = int64(1_699_999_999_900) + second.Fields["end_time_unix"] = int64(1_700_000_000_200) + payloads, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode(second) returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one batched NetFlow v9 payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + dataSet := decoded.FlowSets[1].(netflow.DataFlowSet) + if len(dataSet.Records) != 2 { + t.Fatalf("expected two records, got %d", len(dataSet.Records)) + } + decodeV9Time := func(switched uint32) int64 { + return int64(decoded.UnixSeconds)*1000 - (int64(decoded.SystemUptime) - int64(switched)) + } + for i, want := range []struct { + start int64 + end int64 + }{ + {start: 1_700_000_000_100, end: 1_700_000_000_900}, + {start: 1_699_999_999_900, end: 1_700_000_000_200}, + } { + values := dataSet.Records[i].Values + startRaw := values[len(values)-2].Value.([]byte) + endRaw := values[len(values)-1].Value.([]byte) + if got := decodeV9Time(binary.BigEndian.Uint32(startRaw)); got != want.start { + t.Fatalf("record %d start time got %d, want %d", i, got, want.start) + } + if got := decodeV9Time(binary.BigEndian.Uint32(endRaw)); got != want.end { + t.Fatalf("record %d end time got %d, want %d", i, got, want.end) + } + } +} + +func TestNFv9EncoderUsesSwitchedTimeFields(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.Data.Select = append(cfg.TemplatedFlow.Data.Select, "start_time_unix", "end_time_unix") + enc := NewNFv9Encoder(cfg) + + payloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + if decoded.UnixSeconds != 1_700_000_001 { + t.Fatalf("expected rounded packet timestamp 1700000001, got %d", decoded.UnixSeconds) + } + if decoded.SystemUptime != 900 { + t.Fatalf("expected system uptime to include packet timestamp rounding remainder, got %d", decoded.SystemUptime) + } + + templateSet := decoded.FlowSets[0].(netflow.TemplateFlowSet) + fields := templateSet.Records[0].Fields + startField := fields[len(fields)-2] + endField := fields[len(fields)-1] + if startField.Type != netflow.NFV9_FIELD_FIRST_SWITCHED || startField.Length != 4 { + t.Fatalf("expected start_time_unix to use FIRST_SWITCHED/4, got type=%d length=%d", startField.Type, startField.Length) + } + if endField.Type != netflow.NFV9_FIELD_LAST_SWITCHED || endField.Length != 4 { + t.Fatalf("expected end_time_unix to use LAST_SWITCHED/4, got type=%d length=%d", endField.Type, endField.Length) + } + + dataSet := decoded.FlowSets[1].(netflow.DataFlowSet) + values := dataSet.Records[0].Values + startValue := values[len(values)-2] + endValue := values[len(values)-1] + startRaw, ok := startValue.Value.([]byte) + if !ok { + t.Fatalf("expected FIRST_SWITCHED value bytes, got %T", startValue.Value) + } + endRaw, ok := endValue.Value.([]byte) + if !ok { + t.Fatalf("expected LAST_SWITCHED value bytes, got %T", endValue.Value) + } + if got := binary.BigEndian.Uint32(startRaw); got != 0 { + t.Fatalf("expected FIRST_SWITCHED value 0, got %d", got) + } + if got := binary.BigEndian.Uint32(endRaw); got != 800 { + t.Fatalf("expected LAST_SWITCHED value 800, got %d", got) + } + decodeV9Time := func(switched uint32) int64 { + return int64(decoded.UnixSeconds)*1000 - (int64(decoded.SystemUptime) - int64(switched)) + } + if got := decodeV9Time(binary.BigEndian.Uint32(startRaw)); got != int64(1_700_000_000_100) { + t.Fatalf("expected reconstructed start_time_unix=1700000000100, got %d", got) + } + if got := decodeV9Time(binary.BigEndian.Uint32(endRaw)); got != int64(1_700_000_000_900) { + t.Fatalf("expected reconstructed end_time_unix=1700000000900, got %d", got) + } +} + +func TestNFv9EncoderSkipsIPFIXOnlyFallbackFields(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.Data.Select = append(cfg.TemplatedFlow.Data.Select, "source_id", "header_data", "start_time_unix") + cfg.TemplatedFlow.Data.Catalog["source_id"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_observationPointId, Length: 8, Type: "unsigned64"} + cfg.TemplatedFlow.Data.Catalog["header_data"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_dataLinkFrameSection, Length: 0xffff, Type: "bytes"} + enc := NewNFv9Encoder(cfg) + + evt := testTemplatedFlowEvent() + evt.Fields["header_data"] = []byte{0xde, 0xad, 0xbe, 0xef} + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + + templateSet := decoded.FlowSets[0].(netflow.TemplateFlowSet) + dataSet := decoded.FlowSets[1].(netflow.DataFlowSet) + fields := templateSet.Records[0].Fields + values := dataSet.Records[0].Values + if len(fields) != len(values) { + t.Fatalf("expected template/data widths to match, got fields=%d values=%d", len(fields), len(values)) + } + for _, field := range fields { + if field.Type == netflow.IPFIX_FIELD_observationPointId || field.Type == netflow.IPFIX_FIELD_dataLinkFrameSection { + t.Fatalf("expected IPFIX-only field to be skipped, got template field %#v", field) + } + if !isNetFlowV9FieldID(field.Type) { + t.Fatalf("expected only NetFlow v9 field IDs, got %#v", field) + } + } + if got := fields[len(fields)-1]; got.Type != netflow.NFV9_FIELD_FIRST_SWITCHED || got.Length != 4 { + t.Fatalf("expected supported remapped start_time_unix to remain, got %#v", got) + } +} + +func TestNFv9EncoderSkipsIPFIXOnlySchemaFields(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.Data.Catalog["source_id"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_observationPointId, Length: 8, Type: "unsigned64"} + cfg.TemplatedFlow.Data.Catalog["header_data"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_dataLinkFrameSection, Length: 0xffff, Type: "bytes"} + enc := NewNFv9Encoder(cfg) + + templatePayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"source_id", "header_data", "bytes"}, + BaseTemplateID: 300, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + if len(templatePayloads) != 1 { + t.Fatalf("expected one schema template payload, got %d", len(templatePayloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + var templateDecoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(templatePayloads[0]), store, ctx, &templateDecoded, nil); err != nil { + t.Fatalf("decode netflow v9 schema payload: %v", err) + } + templateSet := templateDecoded.FlowSets[0].(netflow.TemplateFlowSet) + if got := templateSet.Records[0].Fields; len(got) != 1 || got[0].Type != netflow.NFV9_FIELD_IN_BYTES { + t.Fatalf("expected only supported bytes template field, got %#v", got) + } + + evt := testTemplatedFlowEvent() + evt.Stream = "flow_data" + evt.Fields["header_data"] = []byte{0xde, 0xad, 0xbe, 0xef} + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, ctx, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 data payload: %v", err) + } + dataSet := decoded.FlowSets[0].(netflow.DataFlowSet) + if got := dataSet.Records[0].Values; len(got) != 1 || got[0].Type != netflow.NFV9_FIELD_IN_BYTES { + t.Fatalf("expected only supported bytes data value, got %#v", got) + } +} + +func TestTemplatedEncoderEncodesMacAddressFields(t *testing.T) { + cfg := config.TemplatedFlowDataConfig{ + Select: []string{"src_mac"}, + Catalog: map[string]config.IPFIXFieldDefinition{ + "src_mac": {ID: 56, Length: 6, Type: "macAddress"}, + }, + } + template, record, err := buildTemplatedDataRecord(cfg, map[string]any{ + "src_mac": "66:77:88:99:aa:bb", + }, 256, false) + if err != nil { + t.Fatalf("buildTemplatedDataRecord returned error: %v", err) + } + if template.Fields[0].Type != netflow.IPFIX_FIELD_sourceMacAddress || template.Fields[0].Length != 6 { + t.Fatalf("expected sourceMacAddress/6 template field, got %#v", template.Fields[0]) + } + got, ok := record.Values[0].Value.([]byte) + if !ok { + t.Fatalf("expected encoded MAC bytes, got %T", record.Values[0].Value) + } + want := []byte{0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb} + if !bytes.Equal(got, want) { + t.Fatalf("expected MAC %x, got %x", want, got) + } +} + +func TestNFv5EncoderEmitsRecord(t *testing.T) { + enc := NewNFv5Encoder(config.EncoderConfig{Type: "netflowv5"}) + + payloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected 1 payload, got %d", len(payloads)) + } + + var decoded netflowlegacy.PacketNetFlowV5 + if err := netflowlegacy.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), &decoded); err != nil { + t.Fatalf("decode netflow v5 payload: %v", err) + } + + if decoded.FlowSequence != 1 { + t.Fatalf("expected flow sequence 1, got %d", decoded.FlowSequence) + } + if len(decoded.Records) != 1 { + t.Fatalf("expected 1 record, got %d", len(decoded.Records)) + } + if decoded.Records[0].SrcPort != 1234 { + t.Fatalf("expected src port 1234, got %d", decoded.Records[0].SrcPort) + } + if decoded.Records[0].DOctets != 321 { + t.Fatalf("expected bytes 321, got %d", decoded.Records[0].DOctets) + } +} + +func TestNFv5EncoderIgnoresControlEvents(t *testing.T) { + enc := NewNFv5Encoder(config.EncoderConfig{Type: "netflowv5"}) + + payloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 0 { + t.Fatalf("expected no payloads for control event, got %d", len(payloads)) + } +} + +func TestSourceOptionsUseSourceSamplingMetadata(t *testing.T) { + state := sourceOptionsFromEvent(&event.Event{ + Source: event.SourceMetadata{ + AgentIP: "198.51.100.99", + SourceID: 42, + Sampling: &event.SamplingMetadata{ + Rate: 250, + SamplePool: 54321, + Drops: 7, + }, + }, + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "source_id": uint32(9), + "sampling_rate": uint32(100), + "sample_pool": uint32(12345), + "drops": uint32(3), + }, + Payload: event.SourceInit{ + AgentIP: "192.0.2.2", + SourceID: 10, + SamplingRate: 101, + SamplePool: 12346, + Drops: 4, + }, + }) + + if state.agentIP != "198.51.100.99" { + t.Fatalf("expected source agent_ip 198.51.100.99, got %q", state.agentIP) + } + if state.sourceID != 42 { + t.Fatalf("expected source_id 42, got %d", state.sourceID) + } + if state.samplingRate != 250 { + t.Fatalf("expected source sampling_rate 250, got %d", state.samplingRate) + } + if state.samplePool != 54321 { + t.Fatalf("expected source sample_pool 54321, got %d", state.samplePool) + } + if state.drops != 7 { + t.Fatalf("expected source drops 7, got %d", state.drops) + } +} + +func TestIPFIXSchemaDataUsesSourceSamplingMetadata(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Catalog["sampling_rate"] = config.IPFIXFieldDefinition{ID: 34, Length: 4, Type: "unsigned32"} + cfg.TemplatedFlow.Data.Catalog["sample_pool"] = config.IPFIXFieldDefinition{ID: 310, Length: 4, Type: "unsigned32"} + cfg.TemplatedFlow.Data.Catalog["drops"] = config.IPFIXFieldDefinition{ID: 133, Length: 8, Type: "unsigned64"} + cfg.TemplatedFlow.Data.Catalog["source_id"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_observationPointId, Length: 8, Type: "unsigned64"} + enc := NewIPFIXEncoder(cfg) + + if _, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"source_id", "sampling_rate", "sample_pool", "drops"}, + BaseTemplateID: 300, + }, + }); err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + evt := &event.Event{ + ReceivedAt: testEventTime(), + Stream: "flow_data", + Source: event.SourceMetadata{ + SourceID: 42, + Sampling: &event.SamplingMetadata{ + Rate: 250, + SamplePool: 54321, + Drops: 7, + }, + }, + Fields: map[string]any{ + "bytes": uint64(64), + "sampling_rate": uint32(100), + "sample_pool": uint32(12345), + "drops": uint32(3), + }, + } + fields := eventFieldsWithMetadataForSchema(evt, enc.dataSchemas["flow_data"].fields) + if fields["source_id"] != uint32(42) || fields["sampling_rate"] != uint32(250) || fields["sample_pool"] != uint32(54321) || fields["drops"] != uint32(7) { + t.Fatalf("expected source sampling metadata to materialize for schema, got %#v", fields) + } + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one data payload, got %d", len(payloads)) + } + record := enc.dataSchemas["flow_data"].templateForFields(nil) + if record.Fields[0].Type != netflow.IPFIX_FIELD_observationPointId || record.Fields[1].Type != 34 || record.Fields[2].Type != 310 || record.Fields[3].Type != 133 { + t.Fatalf("unexpected metadata template fields: %#v", record.Fields) + } +} + +func TestIPFIXSchemaDataUsesAgentIPv6Metadata(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Catalog["agent_ip"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_exporterIPv4Address, Length: 4, Type: "ipv4Address"} + cfg.TemplatedFlow.Data.Catalog["agent_ipv6"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_exporterIPv6Address, Length: 16, Type: "ipv6Address"} + enc := NewIPFIXEncoder(cfg) + + if _, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"agent_ip", "agent_ipv6"}, + BaseTemplateID: 300, + }, + }); err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + evt := &event.Event{ + ReceivedAt: testEventTime(), + Stream: "flow_data", + Source: event.SourceMetadata{ + AgentIP: "2001:db8::99", + }, + Fields: map[string]any{ + "bytes": uint64(64), + }, + } + fields := eventFieldsWithMetadataForSchema(evt, enc.dataSchemas["flow_data"].fields) + if _, ok := fields["agent_ip"]; ok { + t.Fatalf("expected IPv6 metadata not to materialize as agent_ip, got %#v", fields) + } + if fields["agent_ipv6"] != "2001:db8::99" { + t.Fatalf("expected agent_ipv6 metadata, got %#v", fields) + } + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one data payload, got %d", len(payloads)) + } + record := enc.dataSchemas["flow_data"].templateForFields(nil) + if record.Fields[0].Type != netflow.IPFIX_FIELD_exporterIPv4Address || record.Fields[1].Type != netflow.IPFIX_FIELD_exporterIPv6Address { + t.Fatalf("unexpected agent address template fields: %#v", record.Fields) + } +} + +func TestNFv9EncoderPassesThroughOptionsTemplate(t *testing.T) { + enc := NewNFv9Encoder(config.EncoderConfig{Type: "netflowv9"}) + evt := &event.Event{ + Fields: map[string]any{ + "source_id": uint32(9), + }, + Payload: netflow.NFv9OptionsTemplateRecord{ + TemplateId: 300, + ScopeLength: 4, + OptionLength: 4, + Scopes: []netflow.Field{{Type: 1, Length: 4}}, + Options: []netflow.Field{{Type: 34, Length: 4}}, + }, + } + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, &decoded, nil); err != nil { + t.Fatalf("decode netflow v9 payload: %v", err) + } + + if _, ok := decoded.FlowSets[0].(netflow.NFv9OptionsTemplateFlowSet); !ok { + t.Fatalf("expected options template flow set, got %T", decoded.FlowSets[0]) + } +} + +func TestNFv9AggregationSchemaOptionsDataUsesScopeIDs(t *testing.T) { + cfg := testTFlowEncoderConfig("netflowv9") + cfg.TemplatedFlow.ObservationDomainID = 888 + cfg.TemplatedFlow.Data.Catalog["observation_domain_id"] = config.IPFIXFieldDefinition{ID: 149, Length: 4, Type: "unsigned32"} + cfg.TemplatedFlow.Data.Catalog["if_index"] = config.IPFIXFieldDefinition{ID: 10, Length: 4, Type: "unsigned32"} + cfg.TemplatedFlow.Data.Catalog["interface_name"] = config.IPFIXFieldDefinition{ID: 82, Length: 0xffff, Type: "string"} + enc := NewNFv9Encoder(cfg) + + schemaPayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "interface_options", + }, + Payload: event.AggregationSchema{ + Stream: "interface_options", + Fields: []event.SchemaField{ + {Role: "static", Name: "tflow_record_type", Value: "options"}, + {Role: "key", Name: "observation_domain_id"}, + {Role: "key", Name: "if_index"}, + {Role: "current", Name: "interface_name"}, + }, + BaseTemplateID: 1300, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + if len(schemaPayloads) != 1 { + t.Fatalf("expected one options schema payload, got %d", len(schemaPayloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + var schemaDecoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(schemaPayloads[0]), store, ctx, &schemaDecoded, nil); err != nil { + t.Fatalf("decode options schema payload: %v", err) + } + optionsTemplate, ok := schemaDecoded.FlowSets[0].(netflow.NFv9OptionsTemplateFlowSet) + if !ok { + t.Fatalf("expected options template flow set, got %T", schemaDecoded.FlowSets[0]) + } + record := optionsTemplate.Records[0] + if record.TemplateId != 1300 || record.ScopeLength != 8 || record.OptionLength != 4 { + t.Fatalf("unexpected options template lengths: %#v", record) + } + if record.Scopes[0].Type != 1 || record.Scopes[1].Type != 2 || record.Options[0].Type != 82 { + t.Fatalf("unexpected options template fields: scopes=%#v options=%#v", record.Scopes, record.Options) + } + + dataPayloads, err := enc.Encode(&event.Event{ + ReceivedAt: testEventTime(), + Stream: "interface_options", + Fields: map[string]any{ + "observation_domain_id": uint32(777), + "if_index": uint32(2), + "interface_name": "eth0", + }, + }) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + if len(dataPayloads) != 1 { + t.Fatalf("expected one options data payload, got %d", len(dataPayloads)) + } + var dataDecoded netflow.NFv9Packet + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(dataPayloads[0]), store, ctx, &dataDecoded, nil); err != nil { + t.Fatalf("decode options data payload: %v", err) + } + optionsData, ok := dataDecoded.FlowSets[0].(netflow.OptionsDataFlowSet) + if !ok { + t.Fatalf("expected options data flow set, got %T", dataDecoded.FlowSets[0]) + } + values := optionsData.Records[0] + if !bytes.Equal(values.ScopesValues[0].Value.([]byte), encodeU32(777)) || !bytes.Equal(values.ScopesValues[1].Value.([]byte), encodeU32(2)) { + t.Fatalf("unexpected options scope values: %#v", values.ScopesValues) + } + if !bytes.Equal(values.OptionsValues[0].Value.([]byte), []byte("eth0")) { + t.Fatalf("expected interface name option eth0, got %#v", values.OptionsValues[0].Value) + } +} + +func TestIPFIXTemplatePacketDoesNotAdvanceSequence(t *testing.T) { + enc := NewIPFIXEncoder(testTFlowEncoderConfig("ipfix")) + + templatePayloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "source_id": uint32(42), + }, + Payload: netflow.TemplateRecord{ + TemplateId: 256, + FieldCount: 1, + Fields: []netflow.Field{{Type: netflow.IPFIX_FIELD_octetDeltaCount, Length: 8}}, + }, + }) + if err != nil { + t.Fatalf("template Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var templateDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(templatePayloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &templateDecoded); err != nil { + t.Fatalf("decode ipfix template payload: %v", err) + } + if templateDecoded.SequenceNumber != 0 { + t.Fatalf("expected template sequence 0, got %d", templateDecoded.SequenceNumber) + } + + dataPayloads, err := enc.Encode(testTemplatedFlowEvent()) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + + var dataDecoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(dataPayloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &dataDecoded); err != nil { + t.Fatalf("decode ipfix data payload: %v", err) + } + if dataDecoded.SequenceNumber != 0 { + t.Fatalf("expected first data packet sequence 0 after template set, got %d", dataDecoded.SequenceNumber) + } +} + +func TestIPFIXSchemaDrivenDataRecordKeepsTemplateWidth(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 42 + enc := NewIPFIXEncoder(cfg) + + _, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"src_addr", "dst_addr", "src_port", "dst_port", "proto", "bytes", "packets", "start_time_unix", "end_time_unix"}, + BaseTemplateID: 256, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + evt := testTemplatedFlowEvent() + delete(evt.Fields, "end_time_unix") + evt.Fields["observation_domain_id"] = uint32(42) + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + + // Prime the decode-side template store with the same schema packet. + templatePayloads, err := enc.encodeSchemaTemplates(enc.dataSchemas["flow_data"]) + if err != nil { + t.Fatalf("encodeSchemaTemplates returned error: %v", err) + } + for _, payload := range templatePayloads { + var templatePacket netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), store, ctx, nil, &templatePacket); err != nil { + t.Fatalf("decode schema template payload: %v", err) + } + } + + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, ctx, nil, &decoded); err != nil { + t.Fatalf("decode ipfix data payload: %v", err) + } + dataSet := decoded.FlowSets[0].(netflow.DataFlowSet) + if len(dataSet.Records[0].Values) != 11 { + t.Fatalf("expected 11 values in dual-stack data record, got %d", len(dataSet.Records[0].Values)) + } +} + +func TestIPFIXSchemaDrivenVariableLengthDataIsLengthPrefixed(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Catalog["header_data"] = config.IPFIXFieldDefinition{ID: 315, Length: 0xffff, Type: "bytes"} + enc := NewIPFIXEncoder(cfg) + + _, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"header_data", "bytes"}, + BaseTemplateID: 256, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + headerData := append([]byte{0x01}, bytes.Repeat([]byte{0xab}, 299)...) + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: testEventTime(), + Stream: "flow_data", + Fields: map[string]any{ + "header_data": headerData, + "bytes": uint64(300), + }, + }) + if err != nil { + t.Fatalf("data Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one data payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + ctx := netflow.FlowContext{RouterKey: "test-router"} + templatePayloads, err := enc.encodeSchemaTemplates(enc.dataSchemas["flow_data"]) + if err != nil { + t.Fatalf("encodeSchemaTemplates returned error: %v", err) + } + for _, payload := range templatePayloads { + var templatePacket netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payload), store, ctx, nil, &templatePacket); err != nil { + t.Fatalf("decode schema template payload: %v", err) + } + } + + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, ctx, nil, &decoded); err != nil { + t.Fatalf("decode ipfix data payload: %v", err) + } + dataSet := decoded.FlowSets[0].(netflow.DataFlowSet) + if len(dataSet.Records) != 1 { + t.Fatalf("expected one data record, got %d", len(dataSet.Records)) + } + values := dataSet.Records[0].Values + if len(values) != 2 { + t.Fatalf("expected two values, got %d", len(values)) + } + if got := values[0].Value.([]byte); !bytes.Equal(got, headerData) { + t.Fatalf("expected decoded header_data length %d, got %d", len(headerData), len(got)) + } + if got := values[1].Value.([]byte); !bytes.Equal(got, []byte{0, 0, 0, 0, 0, 0, 0x01, 0x2c}) { + t.Fatalf("expected decoded bytes value 300, got %v", got) + } +} + +func TestIPFIXSchemaDrivenLayerAddressFieldsDefaultToDualStackFields(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Catalog["ip_1_src_addr"] = config.IPFIXFieldDefinition{ID: 8, Length: 4, Type: "ipv4Address"} + cfg.TemplatedFlow.Data.Catalog["ip_1_dst_addr"] = config.IPFIXFieldDefinition{ID: 12, Length: 4, Type: "ipv4Address"} + cfg.TemplatedFlow.Data.Catalog["ip_2_src_addr"] = config.IPFIXFieldDefinition{ID: 8, Length: 4, Type: "ipv4Address"} + cfg.TemplatedFlow.Data.Catalog["ip_2_dst_addr"] = config.IPFIXFieldDefinition{ID: 12, Length: 4, Type: "ipv4Address"} + enc := NewIPFIXEncoder(cfg) + + templatePayloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"ip_1_src_addr", "ip_1_dst_addr", "ip_2_src_addr", "ip_2_dst_addr"}, + BaseTemplateID: 300, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + if len(templatePayloads) != 1 { + t.Fatalf("expected one dual-stack template, got %d", len(templatePayloads)) + } + + evt := &event.Event{ + ReceivedAt: testEventTime(), + Fields: map[string]any{ + "ip_1_src_addr": "203.0.113.1", + "ip_1_dst_addr": "203.0.113.2", + "ip_2_src_addr": "2001:db8::1", + "ip_2_dst_addr": "2001:db8::2", + }, + } + template := enc.dataSchemas["flow_data"].templateForFields(evt.Fields) + if template.TemplateId != 300 { + t.Fatalf("expected single template id 300, got %d", template.TemplateId) + } + if len(template.Fields) != 8 { + t.Fatalf("expected v4/v6 field pair per logical address, got %#v", template.Fields) + } + if template.Fields[0].Type != netflow.IPFIX_FIELD_sourceIPv4Address || template.Fields[1].Type != netflow.IPFIX_FIELD_sourceIPv6Address { + t.Fatalf("expected first source layer to use IPv4/IPv6 fields, got %#v", template.Fields[:2]) + } + if template.Fields[4].Type != netflow.IPFIX_FIELD_sourceIPv4Address || template.Fields[5].Type != netflow.IPFIX_FIELD_sourceIPv6Address { + t.Fatalf("expected second source layer to use IPv4/IPv6 fields, got %#v", template.Fields[4:6]) + } + if _, err := enc.Encode(evt); err != nil { + t.Fatalf("data Encode returned error: %v", err) + } +} + +func TestIPFIXSchemaDrivenFixedFamilyEmitsSingleTemplate(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + enc := NewIPFIXEncoder(cfg) + + ipv4Templates, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data_ipv4", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data_ipv4", + Fields: []event.SchemaField{ + {Role: "key", Name: "src_addr", Value: "ip4"}, + {Role: "key", Name: "dst_addr", Value: "ip4"}, + {Role: "sum", Name: "bytes"}, + }, + FieldNames: []string{"src_addr", "dst_addr", "bytes"}, + BaseTemplateID: 256, + }, + }) + if err != nil { + t.Fatalf("ipv4 schema Encode returned error: %v", err) + } + if len(ipv4Templates) != 1 { + t.Fatalf("expected one ipv4 template, got %d", len(ipv4Templates)) + } + + ipv6Templates, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data_ipv6", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data_ipv6", + Fields: []event.SchemaField{ + {Role: "key", Name: "src_addr", Value: "ip6"}, + {Role: "key", Name: "dst_addr", Value: "ip6"}, + {Role: "sum", Name: "bytes"}, + }, + FieldNames: []string{"src_addr", "dst_addr", "bytes"}, + BaseTemplateID: 258, + }, + }) + if err != nil { + t.Fatalf("ipv6 schema Encode returned error: %v", err) + } + if len(ipv6Templates) != 1 { + t.Fatalf("expected one ipv6 template, got %d", len(ipv6Templates)) + } + + if template := enc.dataSchemas["flow_data_ipv4"].templates()[0]; template.TemplateId != 256 { + t.Fatalf("expected ipv4 template id 256, got %d", template.TemplateId) + } else if template.Fields[0].Type != netflow.IPFIX_FIELD_sourceIPv4Address || template.Fields[1].Type != netflow.IPFIX_FIELD_destinationIPv4Address { + t.Fatalf("expected ipv4 template to use IPv4 source/destination fields, got %#v", template.Fields[:2]) + } + if template := enc.dataSchemas["flow_data_ipv6"].templates()[0]; template.TemplateId != 258 { + t.Fatalf("expected ipv6 template id 258, got %d", template.TemplateId) + } else if template.Fields[0].Type != netflow.IPFIX_FIELD_sourceIPv6Address || template.Fields[1].Type != netflow.IPFIX_FIELD_destinationIPv6Address { + t.Fatalf("expected ipv6 template to use IPv6 source/destination fields, got %#v", template.Fields[:2]) + } +} + +func TestIPFIXSchemaFieldFamiliesDriveWireDefinitions(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Catalog["nat_src_addr"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_postNATSourceIPv4Address, Length: 4, Type: "ipv4Address"} + enc := NewIPFIXEncoder(cfg) + + if _, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + Fields: []event.SchemaField{ + {Role: "key", Name: "src_addr", Value: "ip4in6"}, + {Role: "current", Name: "nat_src_addr", Value: "ip4in6"}, + {Role: "sum", Name: "bytes"}, + }, + FieldNames: []string{"src_addr", "nat_src_addr", "bytes"}, + BaseTemplateID: 300, + }, + }); err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + + state := enc.dataSchemas["flow_data"] + template := state.templates()[0] + if template.Fields[0].Type != netflow.IPFIX_FIELD_sourceIPv6Address { + t.Fatalf("expected src_addr:ip4in6 to use sourceIPv6Address, got %d", template.Fields[0].Type) + } + if template.Fields[1].Type != netflow.IPFIX_FIELD_postNATSourceIPv6Address { + t.Fatalf("expected nat_src_addr:ip4in6 to use postNATSourceIPv6Address, got %d", template.Fields[1].Type) + } + + record, err := buildTemplatedValuesFromSchemaFields(cfg.TemplatedFlow.Data, map[string]any{ + "src_addr": "192.0.2.10", + "nat_src_addr": "198.51.100.20", + "bytes": uint64(64), + }, state.fields, templatedEncodingContext{}) + if err != nil { + t.Fatalf("buildTemplatedValuesFromSchemaFields returned error: %v", err) + } + if got := record.Values[0].Value.([]byte); !bytes.Equal(got, netip.MustParseAddr("::ffff:192.0.2.10").AsSlice()) { + t.Fatalf("expected src_addr to be IPv4-mapped IPv6, got %x", got) + } + if got := record.Values[1].Value.([]byte); !bytes.Equal(got, netip.MustParseAddr("::ffff:198.51.100.20").AsSlice()) { + t.Fatalf("expected nat_src_addr to be IPv4-mapped IPv6, got %x", got) + } +} + +func TestIPFIXSchemaRejectsFamilyModifierOnNonAddressField(t *testing.T) { + enc := NewIPFIXEncoder(testTFlowEncoderConfig("ipfix")) + _, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + Fields: []event.SchemaField{ + {Role: "sum", Name: "bytes", Value: "ip6"}, + }, + FieldNames: []string{"bytes"}, + BaseTemplateID: 300, + }, + }) + if err == nil { + t.Fatalf("expected schema with non-address family modifier to fail") + } +} + +func TestIPFIXSchemaDrivenNATFieldsDefaultToDualStackFields(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Catalog["nat_src_addr"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_postNATSourceIPv4Address, Length: 4, Type: "ipv4Address"} + cfg.TemplatedFlow.Data.Catalog["nat_dst_addr"] = config.IPFIXFieldDefinition{ID: netflow.IPFIX_FIELD_postNATDestinationIPv4Address, Length: 4, Type: "ipv4Address"} + enc := NewIPFIXEncoder(cfg) + + payloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + FieldNames: []string{"src_addr", "dst_addr", "nat_src_addr", "nat_dst_addr"}, + BaseTemplateID: 256, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one NAT dual-stack template, got %d", len(payloads)) + } + + templates := enc.dataSchemas["flow_data"].templates() + if len(templates) != 1 { + t.Fatalf("expected one template, got %d", len(templates)) + } + template := templates[0] + if template.TemplateId != 256 || len(template.Fields) != 8 { + t.Fatalf("expected template 256 with 8 address fields, got id=%d fields=%#v", template.TemplateId, template.Fields) + } + if template.Fields[4].Type != netflow.IPFIX_FIELD_postNATSourceIPv4Address || template.Fields[5].Type != netflow.IPFIX_FIELD_postNATSourceIPv6Address { + t.Fatalf("expected NAT source to use post-NAT IPv4/IPv6 fields, got %#v", template.Fields[4:6]) + } + if template.Fields[6].Type != netflow.IPFIX_FIELD_postNATDestinationIPv4Address || template.Fields[7].Type != netflow.IPFIX_FIELD_postNATDestinationIPv6Address { + t.Fatalf("expected NAT destination to use post-NAT IPv4/IPv6 fields, got %#v", template.Fields[6:8]) + } +} + +func TestIPFIXDefaultMPLSStackSectionUsesThreeByteField(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.Data.Catalog["mpls_label_stack_section_1"] = config.IPFIXFieldDefinition{ID: 70, Length: 3, Type: "bytes"} + def := cfg.TemplatedFlow.Data.Catalog["mpls_label_stack_section_1"] + if def.ID != 70 || def.Length != 3 || def.Type != "bytes" { + t.Fatalf("expected MPLS stack section IE 70 length 3 bytes, got %#v", def) + } + + template, record, err := buildTemplatedDataRecordWithNames( + cfg.TemplatedFlow.Data, + map[string]any{"mpls_label_stack_section_1": []byte{0x00, 0x01, 0x31}}, + []string{"mpls_label_stack_section_1"}, + 256, + templatedEncodingContext{}, + ) + if err != nil { + t.Fatalf("buildTemplatedDataRecordWithNames returned error: %v", err) + } + if template.Fields[0].Type != 70 || template.Fields[0].Length != 3 { + t.Fatalf("expected template field IE 70 length 3, got %#v", template.Fields[0]) + } + if got, ok := record.Values[0].Value.([]byte); !ok || !bytes.Equal(got, []byte{0x00, 0x01, 0x31}) { + t.Fatalf("expected three MPLS stack-section bytes, got %#v", got) + } +} + +func TestIPFIXSchemaFieldsUseEncoderCatalogForEnterpriseMapping(t *testing.T) { + cfg := testTFlowEncoderConfig("ipfix") + cfg.TemplatedFlow.ObservationDomainID = 42 + cfg.TemplatedFlow.Data.Catalog["tenant_id"] = config.IPFIXFieldDefinition{ + Name: "tenantId", + ID: 12345, + PEN: 32473, + Length: 4, + Type: "unsigned32", + EnterpriseScoped: true, + } + enc := NewIPFIXEncoder(cfg) + + payloads, err := enc.Encode(&event.Event{ + Kind: "control", + Control: &event.ControlMetadata{ + Type: "schema", + Stream: "flow_data", + }, + Payload: event.AggregationSchema{ + Stream: "flow_data", + Fields: []event.SchemaField{ + {Role: "current", Name: "tenant_id"}, + }, + FieldNames: []string{"tenant_id"}, + BaseTemplateID: 300, + }, + }) + if err != nil { + t.Fatalf("schema Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one schema payload, got %d", len(payloads)) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode schema payload: %v", err) + } + templateSet := decoded.FlowSets[0].(netflow.TemplateFlowSet) + field := templateSet.Records[0].Fields[0] + if !field.PenProvided || field.Pen != 32473 || field.Type != 12345 || field.Length != 4 { + t.Fatalf("expected enterprise field 12345/32473 length 4, got %#v", field) + } +} + +func TestIPFIXEncoderUsesDualStackInformationElementsForUnmodifiedAddresses(t *testing.T) { + enc := NewIPFIXEncoder(testTFlowEncoderConfig("ipfix")) + evt := testTemplatedFlowEvent() + evt.Fields["src_addr"] = "2001:db8::10" + evt.Fields["dst_addr"] = "2001:db8::20" + + payloads, err := enc.Encode(evt) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + store := templates.NewTemplateFlowStore() + store.Start() + var decoded netflow.IPFIXPacket + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(payloads[0]), store, netflow.FlowContext{RouterKey: "test-router"}, nil, &decoded); err != nil { + t.Fatalf("decode ipfix payload: %v", err) + } + + templateSet := decoded.FlowSets[0].(netflow.TemplateFlowSet) + if templateSet.Records[0].Fields[0].Type != netflow.IPFIX_FIELD_sourceIPv4Address { + t.Fatalf("expected IPv4 src IE, got %d", templateSet.Records[0].Fields[0].Type) + } + if templateSet.Records[0].Fields[1].Type != netflow.IPFIX_FIELD_sourceIPv6Address { + t.Fatalf("expected IPv6 src IE, got %d", templateSet.Records[0].Fields[1].Type) + } + dataSet := decoded.FlowSets[1].(netflow.DataFlowSet) + values := dataSet.Records[0].Values + if got := values[0].Value.([]byte); !bytes.Equal(got, make([]byte, 4)) { + t.Fatalf("expected IPv4 source slot to be zero-filled, got %x", got) + } + if got := values[1].Value.([]byte); !bytes.Equal(got, netip.MustParseAddr("2001:db8::10").AsSlice()) { + t.Fatalf("expected IPv6 source slot to contain address, got %x", got) + } +} + +func TestEncodeIPFIXBooleanValue(t *testing.T) { + def := config.IPFIXFieldDefinition{ID: 388, Length: 1, Type: "boolean"} + encoded, err := encodeIPFIXValue(def, true) + if err != nil { + t.Fatalf("encodeIPFIXValue returned error: %v", err) + } + if !bytes.Equal(encoded, []byte{1}) { + t.Fatalf("expected true boolean to encode as 1, got %x", encoded) + } + + encoded, err = encodeIPFIXValue(def, false) + if err != nil { + t.Fatalf("encodeIPFIXValue returned error: %v", err) + } + if !bytes.Equal(encoded, []byte{0}) { + t.Fatalf("expected false boolean to encode as 0, got %x", encoded) + } +} + +func TestAggregatorReturnsErrorForPacketsMissingConfiguredKeys(t *testing.T) { + agg, err := aggregate.New(config.AggregatorConfig{ + KeyFields: []string{"src_addr", "dst_addr"}, + }) + if err != nil { + t.Fatalf("New aggregator returned error: %v", err) + } + + events, err := agg.Process(&event.Event{ + Fields: map[string]any{ + "bytes": int64(64), + }, + }) + if err == nil || !strings.Contains(err.Error(), `missing aggregation key field "src_addr"`) { + t.Fatalf("expected missing key error, got events=%d err=%v", len(events), err) + } +} + +func testSFlowEvent(agentIP string) *event.Event { + return &event.Event{ + Fields: map[string]any{ + "agent_ip": "192.0.2.1", + "protocol": uint32(1), + "frame_length": uint32(60), + "original_length": uint32(60), + "header_data": []byte{0, 1, 2, 3}, + }, + SFlow: &event.SFlowMetadata{ + AgentIP: agentIP, + }, + } +} + +func decodeSFlowPacket(t *testing.T, payload []byte) *sflow.Packet { + t.Helper() + packet := &sflow.Packet{} + if err := sflow.DecodeMessageVersion(bytes.NewBuffer(payload), packet); err != nil { + t.Fatalf("decode sflow payload: %v", err) + } + return packet +} + +func mustDecodeHex(t *testing.T, value string) []byte { + t.Helper() + data, err := hex.DecodeString(value) + if err != nil { + t.Fatalf("decode hex: %v", err) + } + return data +} + +func decodeDelimitedFlowMessage(t *testing.T, payload []byte) *flowpb.FlowMessage { + t.Helper() + size, n := protowire.ConsumeVarint(payload) + if n < 0 { + t.Fatalf("failed to decode protobuf length prefix") + } + msg := &flowpb.FlowMessage{} + if err := proto.Unmarshal(payload[n:n+int(size)], msg); err != nil { + t.Fatalf("unmarshal flow message: %v", err) + } + return msg +} + +func decodeFlowMessage(t *testing.T, payload []byte) *flowpb.FlowMessage { + t.Helper() + msg := &flowpb.FlowMessage{} + if err := proto.Unmarshal(payload, msg); err != nil { + t.Fatalf("unmarshal flow message: %v", err) + } + return msg +} + +func decodeProtobufExtraFields(t *testing.T, payload []byte) map[string]any { + t.Helper() + extras := map[string]any{} + for len(payload) > 0 { + num, typ, n := protowire.ConsumeTag(payload) + if n < 0 { + t.Fatalf("consume top-level tag: %v", protowire.ParseError(n)) + } + payload = payload[n:] + if num != 1000 { + n = protowire.ConsumeFieldValue(num, typ, payload) + if n < 0 { + t.Fatalf("consume top-level field %d: %v", num, protowire.ParseError(n)) + } + payload = payload[n:] + continue + } + if typ != protowire.BytesType { + t.Fatalf("expected extra field bytes type, got %v", typ) + } + raw, n := protowire.ConsumeBytes(payload) + if n < 0 { + t.Fatalf("consume extra field message: %v", protowire.ParseError(n)) + } + payload = payload[n:] + key, value := decodeProtobufExtraField(t, raw) + extras[key] = value + } + return extras +} + +func decodeProtobufExtraField(t *testing.T, payload []byte) (string, any) { + t.Helper() + var key string + var value any + for len(payload) > 0 { + num, typ, n := protowire.ConsumeTag(payload) + if n < 0 { + t.Fatalf("consume extra tag: %v", protowire.ParseError(n)) + } + payload = payload[n:] + switch num { + case 1: + if typ != protowire.BytesType { + t.Fatalf("expected extra key bytes type, got %v", typ) + } + key, n = protowire.ConsumeString(payload) + case 2: + if typ != protowire.BytesType { + t.Fatalf("expected extra string bytes type, got %v", typ) + } + value, n = protowire.ConsumeString(payload) + case 3: + if typ != protowire.VarintType { + t.Fatalf("expected extra uint64 varint type, got %v", typ) + } + value, n = protowire.ConsumeVarint(payload) + case 4: + if typ != protowire.VarintType { + t.Fatalf("expected extra int64 varint type, got %v", typ) + } + var raw uint64 + raw, n = protowire.ConsumeVarint(payload) + value = protowire.DecodeZigZag(raw) + case 5: + if typ != protowire.VarintType { + t.Fatalf("expected extra bool varint type, got %v", typ) + } + var raw uint64 + raw, n = protowire.ConsumeVarint(payload) + value = raw != 0 + case 6: + if typ != protowire.BytesType { + t.Fatalf("expected extra bytes type, got %v", typ) + } + value, n = protowire.ConsumeBytes(payload) + case 7: + if typ != protowire.Fixed64Type { + t.Fatalf("expected extra double fixed64 type, got %v", typ) + } + var raw uint64 + raw, n = protowire.ConsumeFixed64(payload) + value = math.Float64frombits(raw) + case 8: + if typ != protowire.BytesType { + t.Fatalf("expected extra JSON bytes type, got %v", typ) + } + value, n = protowire.ConsumeString(payload) + default: + n = protowire.ConsumeFieldValue(num, typ, payload) + } + if n < 0 { + t.Fatalf("consume extra field %d: %v", num, protowire.ParseError(n)) + } + payload = payload[n:] + } + if key == "" { + t.Fatalf("decoded extra field without key") + } + return key, value +} + +func testBoolPtr(v bool) *bool { + return &v +} + +func assertFlowDirectionTemplateAndValue(t *testing.T, flowSets []interface{}, fieldType uint16) { + t.Helper() + if len(flowSets) != 2 { + t.Fatalf("expected template and data flow sets, got %d", len(flowSets)) + } + templateSet, ok := flowSets[0].(netflow.TemplateFlowSet) + if !ok { + t.Fatalf("expected first flow set to be TemplateFlowSet, got %T", flowSets[0]) + } + if len(templateSet.Records) != 1 || len(templateSet.Records[0].Fields) != 1 { + t.Fatalf("expected one-field template, got %#v", templateSet.Records) + } + field := templateSet.Records[0].Fields[0] + if field.Type != fieldType || field.Length != 1 { + t.Fatalf("expected flow direction field type=%d length=1, got type=%d length=%d", fieldType, field.Type, field.Length) + } + dataSet, ok := flowSets[1].(netflow.DataFlowSet) + if !ok { + t.Fatalf("expected second flow set to be DataFlowSet, got %T", flowSets[1]) + } + if len(dataSet.Records) < 1 || len(dataSet.Records[0].Values) != 1 { + t.Fatalf("expected first flow direction value, got %#v", dataSet.Records) + } + if got := dataSet.Records[0].Values[0].Value.([]byte); !bytes.Equal(got, []byte{1}) { + t.Fatalf("expected flow direction value 1, got %v", got) + } +} + +func testTFlowEncoderConfig(typ string) config.EncoderConfig { + return config.EncoderConfig{ + Type: typ, + TemplatedFlow: config.TemplatedFlowConfig{ + OptionsTemplateBaseID: 1024, + Data: config.TemplatedFlowDataConfig{ + Select: []string{"src_addr", "dst_addr", "src_port", "dst_port", "proto", "bytes", "packets"}, + Catalog: map[string]config.IPFIXFieldDefinition{ + "src_addr": {ID: 8, Length: 4, Type: "ipv4Address"}, + "dst_addr": {ID: 12, Length: 4, Type: "ipv4Address"}, + "src_port": {ID: 7, Length: 2, Type: "unsigned16"}, + "dst_port": {ID: 11, Length: 2, Type: "unsigned16"}, + "proto": {ID: 4, Length: 1, Type: "unsigned8"}, + "bytes": {ID: 1, Length: 8, Type: "unsigned64"}, + "packets": {ID: 2, Length: 8, Type: "unsigned64"}, + "start_time_unix": {ID: 152, Length: 8, Type: "unsigned64"}, + "end_time_unix": {ID: 153, Length: 8, Type: "unsigned64"}, + }, + }, + }, + } +} + +func testTemplatedFlowEvent() *event.Event { + return &event.Event{ + ReceivedAt: testEventTime(), + Fields: map[string]any{ + "source_id": uint32(42), + "src_addr": "192.0.2.10", + "dst_addr": "192.0.2.20", + "src_port": uint32(1234), + "dst_port": uint32(4321), + "proto": uint32(17), + "bytes": int64(321), + "packets": int64(7), + "start_time_unix": int64(1_700_000_000_100), + "end_time_unix": int64(1_700_000_000_900), + }, + } +} + +func testEventTime() time.Time { + return time.Unix(1_700_000_001, 0).UTC() +} diff --git a/pkg/reflow/encode/fields.go b/pkg/reflow/encode/fields.go new file mode 100644 index 00000000..6011d577 --- /dev/null +++ b/pkg/reflow/encode/fields.go @@ -0,0 +1,262 @@ +package encode + +import ( + "encoding/base64" + "encoding/json" + "net/netip" + "time" + + "github.com/netsampler/goflow2/v3/decoders/netflowlegacy" +) + +// stringFieldOrZero reads an optional string field and returns an empty string when absent. +func stringFieldOrZero(fields map[string]any, key string) string { + if fields == nil { + return "" + } + val, ok := fields[key] + if !ok { + return "" + } + s, _ := val.(string) + return s +} + +// uint32Field normalizes common integer representations from the generic field map. +func uint32Field(fields map[string]any, key string) uint32 { + if fields == nil { + return 0 + } + val, ok := fields[key] + if !ok { + return 0 + } + switch v := val.(type) { + case uint32: + return v + case uint64: + return uint32(v) + case int: + if v < 0 { + return 0 + } + return uint32(v) + case int64: + if v < 0 { + return 0 + } + return uint32(v) + case float64: + if v < 0 { + return 0 + } + return uint32(v) + case json.Number: + n, _ := v.Int64() + if n < 0 { + return 0 + } + return uint32(n) + default: + return 0 + } +} + +// uint64Field normalizes common integer representations from the generic field map. +func uint64Field(fields map[string]any, key string) uint64 { + if fields == nil { + return 0 + } + val, ok := fields[key] + if !ok { + return 0 + } + return uint64FromAny(val) +} + +// int64Field normalizes common integer representations from the generic field map. +func int64Field(fields map[string]any, key string) int64 { + if fields == nil { + return 0 + } + val, ok := fields[key] + if !ok { + return 0 + } + switch v := val.(type) { + case int64: + return v + case uint64: + return int64(v) + case int: + return int64(v) + case uint32: + return int64(v) + case float64: + return int64(v) + case json.Number: + n, _ := v.Int64() + return n + default: + return 0 + } +} + +// bytesField returns a byte-oriented field in either raw []byte or string form. +func bytesField(fields map[string]any, key string) []byte { + if fields == nil { + return nil + } + val, ok := fields[key] + if !ok { + return nil + } + switch v := val.(type) { + case []byte: + return v + case string: + return []byte(v) + default: + return nil + } +} + +// encodeIPBytes converts string IPs into the byte-oriented base64 shape used by +// the legacy goflow2v2 JSON payload. +func encodeIPBytes(ip string) string { + if ip == "" { + return "" + } + addr, err := netip.ParseAddr(ip) + if err != nil { + return "" + } + return base64.StdEncoding.EncodeToString(addr.AsSlice()) +} + +// flowTypeField maps string flow-type labels to the integer enum values expected +// by the legacy goflow2v2 JSON layout. +func flowTypeField(fields map[string]any) any { + val := stringFieldOrZero(fields, "flow_type") + switch val { + case "sflow": + return 1 + case "netflowv5": + return 2 + case "netflowv9": + return 3 + case "ipfix": + return 4 + case "": + return 0 + default: + return val + } +} + +// timeFlowNS prefers the nanosecond timestamp aliases and falls back to the +// millisecond fields that older ReFlow paths still use for aggregation/export. +func timeFlowNS(fields map[string]any, nsKey, msKey string) int64 { + if ns := int64Field(fields, nsKey); ns > 0 { + return ns + } + if ms := int64Field(fields, msKey); ms > 0 { + return ms * int64(time.Millisecond) + } + return 0 +} + +// uint16Field is a convenience wrapper around the generic uint32 field reader. +func uint16Field(fields map[string]any, key string) uint16 { + return uint16(uint32Field(fields, key)) +} + +// exportUnixMilliseconds picks the best available export timestamp for encoders +// that need an absolute export time even when flow timings are absent. +func exportUnixMilliseconds(receivedAt time.Time, fields map[string]any) int64 { + endMS := int64Field(fields, "end_time_unix") + if endMS > 0 { + return endMS + } + startMS := int64Field(fields, "start_time_unix") + if startMS > 0 { + return startMS + } + if !receivedAt.IsZero() { + return receivedAt.UnixMilli() + } + return time.Now().UnixMilli() +} + +// uptimeWindow derives the relative uptime values expected by NetFlow v5 from +// absolute millisecond timestamps. +func uptimeWindow(exportMS, startMS, endMS int64) (sysUptime, first, last uint32) { + if startMS <= 0 { + startMS = exportMS + } + if endMS <= 0 { + endMS = exportMS + } + baseMS := exportMS + if startMS < baseMS { + baseMS = startMS + } + if endMS < baseMS { + baseMS = endMS + } + if exportMS < baseMS { + baseMS = exportMS + } + return uint32(exportMS - baseMS), uint32(startMS - baseMS), uint32(endMS - baseMS) +} + +// mustIPv4Address parses an IPv4 string field into the legacy NetFlow v5 integer form. +func mustIPv4Address(fields map[string]any, key string) netflowlegacy.IPAddress { + ip := stringFieldOrZero(fields, key) + if ip == "" { + return 0 + } + addr, err := netip.ParseAddr(ip) + if err != nil || !addr.Is4() { + return 0 + } + raw := addr.As4() + return netflowlegacy.IPAddress(uint32(raw[0])<<24 | uint32(raw[1])<<16 | uint32(raw[2])<<8 | uint32(raw[3])) +} + +// uint64FromAny normalizes several integer representations into uint64 for encoders. +func uint64FromAny(val any) uint64 { + switch v := val.(type) { + case uint64: + return v + case uint32: + return uint64(v) + case uint16: + return uint64(v) + case uint8: + return uint64(v) + case int64: + if v < 0 { + return 0 + } + return uint64(v) + case int: + if v < 0 { + return 0 + } + return uint64(v) + case float64: + if v < 0 { + return 0 + } + return uint64(v) + case json.Number: + n, _ := v.Int64() + if n < 0 { + return 0 + } + return uint64(n) + default: + return 0 + } +} diff --git a/pkg/reflow/encode/json.go b/pkg/reflow/encode/json.go new file mode 100644 index 00000000..ada04cd3 --- /dev/null +++ b/pkg/reflow/encode/json.go @@ -0,0 +1,120 @@ +package encode + +import ( + "encoding/json" + "fmt" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type JSONEncoder struct { + flavor string + dropFields map[string]struct{} +} + +// NewJSONEncoder creates the stateless JSON event encoder. +func NewJSONEncoder(cfg config.EncoderConfig) *JSONEncoder { + dropFields := make(map[string]struct{}, len(cfg.JSON.DropFields)) + for _, field := range cfg.JSON.DropFields { + dropFields[field] = struct{}{} + } + return &JSONEncoder{ + flavor: cfg.JSON.Flavor, + dropFields: dropFields, + } +} + +// Encode serializes one event as a JSON payload in the configured output flavor. +func (e JSONEncoder) Encode(evt *event.Event) ([][]byte, error) { + payload := e.formatEvent(evt) + data, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("marshal event: %w", err) + } + return [][]byte{data}, nil +} + +// Flush is a no-op for JSON because it does not keep internal batching state. +func (JSONEncoder) Flush() ([][]byte, error) { + return nil, nil +} + +// formatEvent selects the output view used for one JSON-encoded event. +func (e JSONEncoder) formatEvent(evt *event.Event) any { + switch e.flavor { + case "", "canonical": + return e.filterEvent(evt) + case "vendor": + return e.filterMap(map[string]any{ + "src_addr": stringFieldOrZero(evt.Fields, "src_addr"), + "dst_addr": stringFieldOrZero(evt.Fields, "dst_addr"), + "src_port": uint32Field(evt.Fields, "src_port"), + "dst_port": uint32Field(evt.Fields, "dst_port"), + "proto": uint32Field(evt.Fields, "proto"), + "packets": int64Field(evt.Fields, "packets"), + "bytes": int64Field(evt.Fields, "bytes"), + "start_time_unix": int64Field(evt.Fields, "start_time_unix"), + "end_time_unix": int64Field(evt.Fields, "end_time_unix"), + "flow_direction": stringFieldOrZero(evt.Fields, "flow_direction"), + "traffic_decision": stringFieldOrZero(evt.Fields, "traffic_decision"), + "action": stringFieldOrZero(evt.Fields, "action"), + "log_status": stringFieldOrZero(evt.Fields, "log_status"), + "reporter": stringFieldOrZero(evt.Fields, "reporter"), + "disposition": stringFieldOrZero(evt.Fields, "disposition"), + }) + case "goflow2v2": + out := map[string]any{ + "sampler_address": encodeIPBytes(eventAgentIP(evt)), + "src_addr": encodeIPBytes(stringFieldOrZero(evt.Fields, "src_addr")), + "dst_addr": encodeIPBytes(stringFieldOrZero(evt.Fields, "dst_addr")), + "src_port": uint32Field(evt.Fields, "src_port"), + "dst_port": uint32Field(evt.Fields, "dst_port"), + "proto": uint32Field(evt.Fields, "proto"), + "bytes": int64Field(evt.Fields, "bytes"), + "packets": int64Field(evt.Fields, "packets"), + "time_flow_start_ns": timeFlowNS(evt.Fields, "time_flow_start_ns", "start_time_unix"), + "time_flow_end_ns": timeFlowNS(evt.Fields, "time_flow_end_ns", "end_time_unix"), + "sampling_rate": eventSamplingRate(evt), + "in_if": uint32Field(evt.Fields, "input_if"), + "out_if": uint32Field(evt.Fields, "output_if"), + "type": flowTypeField(evt.Fields), + } + return e.filterMap(out) + default: + return e.filterEvent(evt) + } +} + +// filterEvent preserves the original event when no fields were dropped to avoid +// allocating a shallow copy for the common case. +func (e JSONEncoder) filterEvent(evt *event.Event) any { + if len(e.dropFields) == 0 || len(evt.Fields) == 0 { + return evt + } + + filteredFields := e.filterMap(evt.Fields) + if len(filteredFields) == len(evt.Fields) { + return evt + } + + filtered := *evt + filtered.Fields = filteredFields + return &filtered +} + +// filterMap applies the configured drop-fields policy to a map payload. +func (e JSONEncoder) filterMap(fields map[string]any) map[string]any { + if len(fields) == 0 || len(e.dropFields) == 0 { + return fields + } + + filtered := make(map[string]any, len(fields)) + for key, value := range fields { + if _, drop := e.dropFields[key]; drop { + continue + } + filtered[key] = value + } + return filtered +} diff --git a/pkg/reflow/encode/metadata.go b/pkg/reflow/encode/metadata.go new file mode 100644 index 00000000..c5c5ab1f --- /dev/null +++ b/pkg/reflow/encode/metadata.go @@ -0,0 +1,168 @@ +package encode + +import ( + "net/netip" + + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func eventAgentIP(evt *event.Event) string { + if evt == nil { + return "" + } + if evt.SFlow != nil && evt.SFlow.AgentIP != "" { + return evt.SFlow.AgentIP + } + if evt.Source.AgentIP != "" { + return evt.Source.AgentIP + } + if agentIPv6 := stringFieldOrZero(evt.Fields, "agent_ipv6"); agentIPv6 != "" { + return agentIPv6 + } + if agentIP := stringFieldOrZero(evt.Fields, "agent_ip"); agentIP != "" { + return agentIP + } + return "" +} + +func eventSamplingRate(evt *event.Event) uint32 { + if evt == nil { + return 0 + } + if evt.SFlow != nil && evt.SFlow.SamplingRate != 0 { + return evt.SFlow.SamplingRate + } + if evt.Source.Sampling != nil && evt.Source.Sampling.Rate != 0 { + return evt.Source.Sampling.Rate + } + return 0 +} + +func eventSourceID(evt *event.Event) uint32 { + sourceID, _ := eventSourceIDValue(evt) + return sourceID +} + +func eventSourceIDValue(evt *event.Event) (uint32, bool) { + if evt == nil { + return 0, false + } + if evt.SFlow != nil && evt.SFlow.SourceID != 0 { + return evt.SFlow.SourceID, true + } + if evt.Source.SourceIDSet || evt.Source.SourceID != 0 { + return evt.Source.SourceID, true + } + return 0, false +} + +func eventSamplePool(evt *event.Event) uint32 { + if evt == nil { + return 0 + } + if evt.SFlow != nil && evt.SFlow.SamplePool != 0 { + return evt.SFlow.SamplePool + } + if evt.Source.Sampling != nil && evt.Source.Sampling.SamplePool != 0 { + return evt.Source.Sampling.SamplePool + } + return 0 +} + +func eventDrops(evt *event.Event) uint32 { + if evt == nil { + return 0 + } + if evt.SFlow != nil && evt.SFlow.Drops != 0 { + return evt.SFlow.Drops + } + if evt.Source.Sampling != nil && evt.Source.Sampling.Drops != 0 { + return evt.Source.Sampling.Drops + } + return 0 +} + +func eventFieldsWithMetadata(evt *event.Event) map[string]any { + if evt == nil { + return nil + } + fields := cloneFieldMap(evt.Fields) + addEventMetadataFields(fields, evt, nil) + return fields +} + +func eventFieldsWithMetadataForSchema(evt *event.Event, schemaFields []event.SchemaField) map[string]any { + if evt == nil { + return nil + } + fields := cloneFieldMap(evt.Fields) + names := make(map[string]struct{}, len(schemaFields)) + for _, field := range schemaFields { + names[field.Name] = struct{}{} + } + addEventMetadataFields(fields, evt, names) + return fields +} + +func cloneFieldMap(in map[string]any) map[string]any { + if len(in) == 0 { + return make(map[string]any) + } + out := make(map[string]any, len(in)+6) + for key, val := range in { + out[key] = val + } + return out +} + +func addEventMetadataFields(fields map[string]any, evt *event.Event, names map[string]struct{}) { + add := func(name string, val any, present bool) { + if !present { + return + } + if names != nil { + if _, ok := names[name]; !ok { + return + } + } + fields[name] = val + } + agentIP := eventAgentIP(evt) + agentIPv4, agentIPv6 := agentIPByFamily(agentIP) + sourceID, hasSourceID := eventSourceIDValue(evt) + samplingRate := eventSamplingRate(evt) + samplePool := eventSamplePool(evt) + drops := eventDrops(evt) + add("agent_ip", agentIPv4, agentIPv4 != "") + add("agent_ipv6", agentIPv6, agentIPv6 != "") + add("source_id", sourceID, hasSourceID) + add("sampling_rate", samplingRate, hasSamplingMetadata(evt)) + add("sample_pool", samplePool, hasSamplePoolMetadata(evt)) + add("drops", drops, hasDropsMetadata(evt)) +} + +func agentIPByFamily(raw string) (string, string) { + if raw == "" { + return "", "" + } + addr, err := netip.ParseAddr(raw) + if err != nil { + return raw, "" + } + if addr.Is4() { + return raw, "" + } + return "", raw +} + +func hasSamplingMetadata(evt *event.Event) bool { + return evt != nil && ((evt.SFlow != nil && evt.SFlow.SamplingRate != 0) || evt.Source.Sampling != nil) +} + +func hasSamplePoolMetadata(evt *event.Event) bool { + return evt != nil && ((evt.SFlow != nil && evt.SFlow.SamplePool != 0) || evt.Source.Sampling != nil) +} + +func hasDropsMetadata(evt *event.Event) bool { + return evt != nil && ((evt.SFlow != nil && evt.SFlow.Drops != 0) || evt.Source.Sampling != nil) +} diff --git a/pkg/reflow/encode/netflowv5.go b/pkg/reflow/encode/netflowv5.go new file mode 100644 index 00000000..84ef5b32 --- /dev/null +++ b/pkg/reflow/encode/netflowv5.go @@ -0,0 +1,89 @@ +package encode + +import ( + "fmt" + "sync/atomic" + "time" + + "github.com/netsampler/goflow2/v3/decoders/netflowlegacy" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type NFv5Encoder struct { + seq atomic.Uint32 +} + +func NewNFv5Encoder(cfg config.EncoderConfig) *NFv5Encoder { + return &NFv5Encoder{} +} + +// Encode turns one canonical event into one NetFlow v5 datagram. +func (e *NFv5Encoder) Encode(evt *event.Event) ([][]byte, error) { + if evt == nil || evt.Kind == "control" { + return nil, nil + } + + packet, err := e.buildPacket(evt) + if err != nil { + return nil, err + } + data, err := netflowlegacy.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode netflow v5 packet: %w", err) + } + return [][]byte{data}, nil +} + +// Flush is a no-op because the NetFlow v5 encoder is stateless. +func (e *NFv5Encoder) Flush() ([][]byte, error) { + return nil, nil +} + +// buildPacket converts one canonical event into a NetFlow v5 export packet. +func (e *NFv5Encoder) buildPacket(evt *event.Event) (*netflowlegacy.PacketNetFlowV5, error) { + if evt == nil { + return nil, fmt.Errorf("nil event") + } + if evt.Fields == nil { + return nil, fmt.Errorf("event fields are empty") + } + + exportMS := exportUnixMilliseconds(evt.ReceivedAt, evt.Fields) + sysUptime, first, last := uptimeWindow(exportMS, int64Field(evt.Fields, "start_time_unix"), int64Field(evt.Fields, "end_time_unix")) + exportTime := time.UnixMilli(exportMS).UTC() + + record := netflowlegacy.RecordsNetFlowV5{ + SrcAddr: mustIPv4Address(evt.Fields, "src_addr"), + DstAddr: mustIPv4Address(evt.Fields, "dst_addr"), + NextHop: mustIPv4Address(evt.Fields, "next_hop"), + Input: uint16Field(evt.Fields, "input_if"), + Output: uint16Field(evt.Fields, "output_if"), + DPkts: uint32(uint64FromAny(evt.Fields["packets"])), + DOctets: uint32(uint64FromAny(evt.Fields["bytes"])), + First: first, + Last: last, + SrcPort: uint16Field(evt.Fields, "src_port"), + DstPort: uint16Field(evt.Fields, "dst_port"), + TCPFlags: uint8(uint32Field(evt.Fields, "tcp_flags")), + Proto: uint8(uint32Field(evt.Fields, "proto")), + Tos: uint8(uint32Field(evt.Fields, "tos")), + SrcAS: uint16Field(evt.Fields, "src_as"), + DstAS: uint16Field(evt.Fields, "dst_as"), + SrcMask: uint8(uint32Field(evt.Fields, "src_mask")), + DstMask: uint8(uint32Field(evt.Fields, "dst_mask")), + } + + return &netflowlegacy.PacketNetFlowV5{ + Version: 5, + Count: 1, + SysUptime: sysUptime, + UnixSecs: uint32(exportTime.Unix()), + UnixNSecs: uint32(exportTime.Nanosecond()), + FlowSequence: e.seq.Add(1), + EngineType: uint8(uint32Field(evt.Fields, "engine_type")), + EngineId: uint8(uint32Field(evt.Fields, "engine_id")), + SamplingInterval: uint16Field(evt.Fields, "sampling_rate"), + Records: []netflowlegacy.RecordsNetFlowV5{record}, + }, nil +} diff --git a/pkg/reflow/encode/pcap.go b/pkg/reflow/encode/pcap.go new file mode 100644 index 00000000..336b4a92 --- /dev/null +++ b/pkg/reflow/encode/pcap.go @@ -0,0 +1,286 @@ +package encode + +import ( + "bytes" + "encoding/hex" + "fmt" + "strings" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/packet" +) + +type PcapEncoder struct { + cfg config.PcapConfig + linkType layers.LinkType + buf bytes.Buffer + writer *pcapgo.Writer + headerWritten bool +} + +func NewPcapEncoder(cfg config.EncoderConfig) (*PcapEncoder, error) { + linkType, err := pcapLinkType(cfg.Pcap.LinkType) + if err != nil { + return nil, err + } + return &PcapEncoder{ + cfg: cfg.Pcap, + linkType: linkType, + }, nil +} + +func (e *PcapEncoder) Encode(evt *event.Event) ([][]byte, error) { + if evt == nil || evt.Kind == "control" { + return nil, nil + } + + data, source, err := e.packetBytes(evt) + if err != nil { + return nil, err + } + if len(data) == 0 { + return nil, fmt.Errorf("encode pcap: empty packet bytes") + } + data = e.adaptPacketBytes(evt, data, source) + if err := e.validateLinkType(evt, data, source); err != nil { + return nil, err + } + + if !e.headerWritten { + e.writer = pcapgo.NewWriterNanos(&e.buf) + if err := e.writer.WriteFileHeader(uint32(e.cfg.SnapLen), e.linkType); err != nil { + return nil, fmt.Errorf("write pcap header: %w", err) + } + e.headerWritten = true + } + + ci := gopacket.CaptureInfo{ + Timestamp: pcapTimestamp(evt), + CaptureLength: len(data), + Length: pcapWireLength(evt.Fields, len(data)), + } + if err := e.writer.WritePacket(ci, data); err != nil { + return nil, fmt.Errorf("write pcap packet: %w", err) + } + + out := append([]byte(nil), e.buf.Bytes()...) + e.buf.Reset() + return [][]byte{out}, nil +} + +func (e *PcapEncoder) Flush() ([][]byte, error) { + return nil, nil +} + +func (e *PcapEncoder) packetBytes(evt *event.Event) ([]byte, string, error) { + switch e.cfg.PacketSource { + case "header_data": + if data := packetHeaderBytes(evt.Fields); len(data) > 0 { + return data, "header_data", nil + } + if data, ok := packet.BuildPseudoHeader(evt, evt.Fields); ok && len(data) > 0 { + return data, "pseudo", nil + } + return nil, "", fmt.Errorf("encode pcap: missing header_data and cannot build pseudo packet") + case "payload": + data, ok := evt.Payload.([]byte) + if !ok || len(data) == 0 { + return nil, "", fmt.Errorf("encode pcap: missing payload bytes") + } + return append([]byte(nil), data...), "payload", nil + case "pseudo": + data, ok := packet.BuildPseudoHeader(evt, evt.Fields) + if !ok || len(data) == 0 { + return nil, "", fmt.Errorf("encode pcap: cannot build pseudo packet") + } + return data, "pseudo", nil + case "auto": + if data := packetHeaderBytes(evt.Fields); len(data) > 0 { + return data, "header_data", nil + } + if data, ok := evt.Payload.([]byte); ok && len(data) > 0 { + return append([]byte(nil), data...), "payload", nil + } + if data, ok := packet.BuildPseudoHeader(evt, evt.Fields); ok && len(data) > 0 { + return data, "pseudo", nil + } + return nil, "", fmt.Errorf("encode pcap: missing packet bytes") + default: + return nil, "", fmt.Errorf("unsupported pcap packet_source %q", e.cfg.PacketSource) + } +} + +func packetHeaderBytes(fields map[string]any) []byte { + if data := bytesField(fields, "header_data"); len(data) > 0 { + return append([]byte(nil), data...) + } + raw := stringFieldOrZero(fields, "header_hex") + if raw == "" { + return nil + } + raw = strings.ReplaceAll(raw, " ", "") + raw = strings.ReplaceAll(raw, ":", "") + data, err := hex.DecodeString(raw) + if err != nil || len(data) == 0 { + return nil + } + return data +} + +func (e *PcapEncoder) adaptPacketBytes(evt *event.Event, data []byte, source string) []byte { + if source != "pseudo" || len(data) == 0 { + return data + } + startLayer := pseudoPacketStartLayer(evt, data) + if e.linkType == layers.LinkTypeEthernet && startLayer == "ip" { + switch data[0] >> 4 { + case 4: + return prependSyntheticEthernet(data, 0x0800) + case 6: + return prependSyntheticEthernet(data, 0x86dd) + } + } + switch e.linkType { + case layers.LinkTypeRaw, layers.LinkTypeIPv4, layers.LinkTypeIPv6: + if startLayer == "ethernet" && len(data) >= 14 { + etherType := uint16(data[12])<<8 | uint16(data[13]) + switch etherType { + case 0x0800, 0x86dd: + return append([]byte(nil), data[14:]...) + } + } + } + return data +} + +func (e *PcapEncoder) validateLinkType(evt *event.Event, data []byte, source string) error { + protocol := uint32Field(evt.Fields, "header_protocol") + if protocol == 0 { + protocol = uint32Field(evt.Fields, "protocol") + } + if source == "pseudo" { + protocol = sampledProtocolFromBytes(data) + } + switch e.linkType { + case layers.LinkTypeEthernet: + if protocol != 0 && protocol != 1 { + return fmt.Errorf("encode pcap: packet protocol %d is incompatible with link_type ethernet", protocol) + } + case layers.LinkTypeIPv4: + if protocol != 0 && protocol != 11 { + return fmt.Errorf("encode pcap: packet protocol %d is incompatible with link_type ipv4", protocol) + } + case layers.LinkTypeIPv6: + if protocol != 0 && protocol != 12 { + return fmt.Errorf("encode pcap: packet protocol %d is incompatible with link_type ipv6", protocol) + } + case layers.LinkTypeRaw: + if protocol != 0 && protocol != 11 && protocol != 12 { + return fmt.Errorf("encode pcap: packet protocol %d is incompatible with link_type raw", protocol) + } + } + return nil +} + +func pcapTimestamp(evt *event.Event) time.Time { + if evt != nil && !evt.ReceivedAt.IsZero() { + return evt.ReceivedAt.UTC() + } + return time.Now().UTC() +} + +func pcapWireLength(fields map[string]any, captured int) int { + for _, key := range []string{"original_length", "frame_length", "wire_length"} { + if value := uint32Field(fields, key); value > 0 { + return int(value) + } + } + if avg := averagePacketLength(fields); avg > captured { + return avg + } + return captured +} + +func averagePacketLength(fields map[string]any) int { + bytes := uint64Field(fields, "bytes") + packets := uint64Field(fields, "packets") + if bytes == 0 || packets == 0 { + return 0 + } + avg := (bytes + packets - 1) / packets + maxInt := uint64(^uint(0) >> 1) + if avg > maxInt { + return int(maxInt) + } + return int(avg) +} + +func pcapLinkType(name string) (layers.LinkType, error) { + switch name { + case "", "ethernet": + return layers.LinkTypeEthernet, nil + case "raw": + return layers.LinkTypeRaw, nil + case "ipv4": + return layers.LinkTypeIPv4, nil + case "ipv6": + return layers.LinkTypeIPv6, nil + default: + return 0, fmt.Errorf("unsupported pcap link_type %q", name) + } +} + +func sampledProtocolFromBytes(data []byte) uint32 { + if len(data) >= 14 { + etherType := uint16(data[12])<<8 | uint16(data[13]) + switch etherType { + case 0x0800, 0x86dd: + return 1 + } + } + if len(data) > 0 { + switch data[0] >> 4 { + case 4: + return 11 + case 6: + return 12 + } + } + return 0 +} + +func pseudoPacketStartLayer(evt *event.Event, data []byte) string { + if evt != nil && evt.Packet != nil && len(evt.Packet.Layers) > 0 { + switch evt.Packet.Layers[0].Kind { + case "ethernet", "dot1q", "mpls", "pppoe": + return "ethernet" + case "ipv4", "ipv6": + return "ip" + } + } + switch sampledProtocolFromBytes(data) { + case 1: + return "ethernet" + case 11, 12: + return "ip" + default: + return "" + } +} + +func prependSyntheticEthernet(payload []byte, etherType uint16) []byte { + // When pcap is configured as Ethernet but the pseudo packet starts at IP, + // wrap it with a minimal Ethernet header. The zero-value bytes are unknown + // destination/source MACs; only EtherType is meaningful. + out := make([]byte, 14+len(payload)) + out[12] = byte(etherType >> 8) + out[13] = byte(etherType) + copy(out[14:], payload) + return out +} diff --git a/pkg/reflow/encode/pcap_test.go b/pkg/reflow/encode/pcap_test.go new file mode 100644 index 00000000..c417cd67 --- /dev/null +++ b/pkg/reflow/encode/pcap_test.go @@ -0,0 +1,353 @@ +package encode + +import ( + "bytes" + "encoding/hex" + "testing" + "time" + + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func TestPcapEncoderWritesHeaderOnceAndPackets(t *testing.T) { + enc, err := NewPcapEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "auto", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapEncoder returned error: %v", err) + } + + first := &event.Event{ + ReceivedAt: time.Unix(10, 123).UTC(), + Fields: map[string]any{ + "header_data": ethernetFrame(0x0800), + "protocol": uint32(1), + "original_length": uint32(128), + }, + } + second := &event.Event{ + ReceivedAt: time.Unix(11, 456).UTC(), + Fields: map[string]any{ + "header_data": ethernetFrame(0x86dd), + "protocol": uint32(1), + "frame_length": uint32(96), + }, + } + + payloads, err := enc.Encode(first) + if err != nil { + t.Fatalf("Encode first returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one payload, got %d", len(payloads)) + } + payloads2, err := enc.Encode(second) + if err != nil { + t.Fatalf("Encode second returned error: %v", err) + } + combined := append(append([]byte(nil), payloads[0]...), payloads2[0]...) + + reader, err := pcapgo.NewReader(bytes.NewReader(combined)) + if err != nil { + t.Fatalf("NewReader returned error: %v", err) + } + if reader.LinkType() != layers.LinkTypeEthernet { + t.Fatalf("expected ethernet link type, got %v", reader.LinkType()) + } + data, ci, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData first returned error: %v", err) + } + if len(data) != len(first.Fields["header_data"].([]byte)) { + t.Fatalf("expected first caplen %d, got %d", len(first.Fields["header_data"].([]byte)), len(data)) + } + if ci.Length != 128 { + t.Fatalf("expected first wire length 128, got %d", ci.Length) + } + _, ci, err = reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData second returned error: %v", err) + } + if ci.Length != 96 { + t.Fatalf("expected second wire length 96, got %d", ci.Length) + } + if bytes.Count(combined, combined[:24]) != 1 { + t.Fatalf("expected pcap global header to appear once") + } +} + +func TestPcapEncoderUsesPseudoPacket(t *testing.T) { + enc, err := NewPcapEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "pseudo", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapEncoder returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(20, 0).UTC(), + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(17), + "src_port": uint32(12345), + "dst_port": uint32(53), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + reader, err := pcapgo.NewReader(bytes.NewReader(payloads[0])) + if err != nil { + t.Fatalf("NewReader returned error: %v", err) + } + data, _, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData returned error: %v", err) + } + if len(data) < 14 || data[12] != 0x08 || data[13] != 0x00 { + t.Fatalf("expected synthetic ethernet IPv4 frame, got %x", data[:min(len(data), 14)]) + } +} + +func TestPcapEncoderUsesAverageAggregateLengthForPseudoPacket(t *testing.T) { + enc, err := NewPcapEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "auto", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapEncoder returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(17), + "src_port": uint32(12345), + "dst_port": uint32(53), + "bytes": uint64(3000), + "packets": uint64(2), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + reader, err := pcapgo.NewReader(bytes.NewReader(payloads[0])) + if err != nil { + t.Fatalf("NewReader returned error: %v", err) + } + data, ci, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData returned error: %v", err) + } + if len(data) >= ci.Length { + t.Fatalf("expected pseudo packet to be captured/truncated, caplen=%d wirelen=%d", len(data), ci.Length) + } + if ci.Length != 1500 { + t.Fatalf("expected average aggregate wire length 1500, got %d", ci.Length) + } +} + +func TestPcapEncoderUsesHeaderHexWhenHeaderDataMissing(t *testing.T) { + enc, err := NewPcapEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "auto", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapEncoder returned error: %v", err) + } + frame := ethernetFrame(0x0800) + + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(20, 0).UTC(), + Fields: map[string]any{ + "header_hex": " " + hex.EncodeToString(frame[:10]) + ":" + hex.EncodeToString(frame[10:]), + "protocol": uint32(1), + "original_length": uint32(len(frame)), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + reader, err := pcapgo.NewReader(bytes.NewReader(payloads[0])) + if err != nil { + t.Fatalf("NewReader returned error: %v", err) + } + data, _, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData returned error: %v", err) + } + if !bytes.Equal(data, frame) { + t.Fatalf("expected decoded header_hex frame, got %x", data) + } +} + +func TestPcapEncoderKeepsPseudoEthernetStartLayer(t *testing.T) { + enc, err := NewPcapEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "pseudo", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapEncoder returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "src_mac": "00:11:22:33:44:55", + "dst_mac": "66:77:88:99:aa:bb", + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(17), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + reader, err := pcapgo.NewReader(bytes.NewReader(payloads[0])) + if err != nil { + t.Fatalf("NewReader returned error: %v", err) + } + data, _, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData returned error: %v", err) + } + if len(data) < 14 || data[0] != 0x66 || data[6] != 0x00 || data[12] != 0x08 || data[13] != 0x00 { + t.Fatalf("expected original pseudo ethernet frame, got %x", data[:min(len(data), 14)]) + } +} + +func TestPcapEncoderPseudoStartLayerIPStripsEthernet(t *testing.T) { + enc, err := NewPcapEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "pseudo", + LinkType: "raw", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapEncoder returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "src_mac": "00:11:22:33:44:55", + "dst_mac": "66:77:88:99:aa:bb", + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(17), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + reader, err := pcapgo.NewReader(bytes.NewReader(payloads[0])) + if err != nil { + t.Fatalf("NewReader returned error: %v", err) + } + data, _, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData returned error: %v", err) + } + if len(data) == 0 || data[0]>>4 != 4 { + t.Fatalf("expected raw IPv4 packet, got %x", data[:min(len(data), 14)]) + } +} + +func TestPcapEncoderHeaderDataFallsBackToPseudoPacket(t *testing.T) { + enc, err := NewPcapEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "header_data", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapEncoder returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(21, 0).UTC(), + Fields: map[string]any{ + "record_kind": "packet", + "src_addr": "192.0.2.10", + "dst_addr": "198.51.100.20", + "proto": uint32(6), + "src_port": uint32(54321), + "dst_port": uint32(443), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + reader, err := pcapgo.NewReader(bytes.NewReader(payloads[0])) + if err != nil { + t.Fatalf("NewReader returned error: %v", err) + } + data, _, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData returned error: %v", err) + } + if len(data) < 14 || data[12] != 0x08 || data[13] != 0x00 { + t.Fatalf("expected pseudo ethernet IPv4 frame, got %x", data[:min(len(data), 14)]) + } +} + +func TestPcapEncoderRejectsIncompatibleLinkType(t *testing.T) { + enc, err := NewPcapEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "header_data", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapEncoder returned error: %v", err) + } + _, err = enc.Encode(&event.Event{ + Fields: map[string]any{ + "header_data": []byte{0x45, 0, 0, 20}, + "protocol": uint32(11), + }, + }) + if err == nil { + t.Fatalf("expected incompatible link type error") + } +} + +func ethernetFrame(etherType uint16) []byte { + data := make([]byte, 34) + data[12] = byte(etherType >> 8) + data[13] = byte(etherType) + if etherType == 0x0800 { + data[14] = 0x45 + data[23] = 17 + } + if etherType == 0x86dd { + data[14] = 0x60 + data[20] = 17 + } + return data +} diff --git a/pkg/reflow/encode/pcapng.go b/pkg/reflow/encode/pcapng.go new file mode 100644 index 00000000..16a0da74 --- /dev/null +++ b/pkg/reflow/encode/pcapng.go @@ -0,0 +1,165 @@ +package encode + +import ( + "bytes" + "fmt" + "path/filepath" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type PcapNGEncoder struct { + packet *PcapEncoder + buf bytes.Buffer + writer *pcapgo.NgWriter + headerWritten bool + iface pcapNGInterfaceMetadata +} + +func NewPcapNGEncoder(cfg config.EncoderConfig) (*PcapNGEncoder, error) { + packet, err := NewPcapEncoder(cfg) + if err != nil { + return nil, err + } + return &PcapNGEncoder{packet: packet}, nil +} + +func (e *PcapNGEncoder) Encode(evt *event.Event) ([][]byte, error) { + if evt == nil { + return nil, nil + } + if evt.Kind == "control" { + e.handleControl(evt) + return nil, nil + } + + data, source, err := e.packet.packetBytes(evt) + if err != nil { + return nil, err + } + if len(data) == 0 { + return nil, fmt.Errorf("encode pcapng: empty packet bytes") + } + data = e.packet.adaptPacketBytes(evt, data, source) + if err := e.packet.validateLinkType(evt, data, source); err != nil { + return nil, err + } + + if !e.headerWritten { + iface := e.interfaceBlock() + writer, err := pcapgo.NewNgWriterInterface(&e.buf, pcapgo.NgInterface{ + Name: iface.Name, + Description: iface.Description, + Comment: iface.Comment, + LinkType: e.packet.linkType, + SnapLength: uint32(e.packet.cfg.SnapLen), + }, pcapgo.NgWriterOptions{ + SectionInfo: pcapgo.NgSectionInfo{ + Application: "reflow", + }, + }) + if err != nil { + return nil, fmt.Errorf("write pcapng header: %w", err) + } + e.writer = writer + e.headerWritten = true + } + + ci := gopacket.CaptureInfo{ + Timestamp: pcapTimestamp(evt), + CaptureLength: len(data), + Length: pcapWireLength(evt.Fields, len(data)), + InterfaceIndex: 0, + } + if err := e.writer.WritePacket(ci, data); err != nil { + return nil, fmt.Errorf("write pcapng packet: %w", err) + } + if err := e.writer.Flush(); err != nil { + return nil, fmt.Errorf("flush pcapng packet: %w", err) + } + + out := append([]byte(nil), e.buf.Bytes()...) + e.buf.Reset() + return [][]byte{out}, nil +} + +func (e *PcapNGEncoder) handleControl(evt *event.Event) { + if evt == nil || evt.Control == nil || evt.Control.Type != "source_init" || e.headerWritten { + return + } + if name := pcapNGInterfaceNameFromSource(evt.Source); name != "" { + e.iface.Name = name + } + if e.iface.Description == "" { + e.iface.Description = pcapNGInterfaceDescription(evt.Source) + } +} + +func (e *PcapNGEncoder) Flush() ([][]byte, error) { + if e.writer == nil { + return nil, nil + } + if err := e.writer.Flush(); err != nil { + return nil, fmt.Errorf("flush pcapng encoder: %w", err) + } + if e.buf.Len() == 0 { + return nil, nil + } + out := append([]byte(nil), e.buf.Bytes()...) + e.buf.Reset() + return [][]byte{out}, nil +} + +func pcapNGInterfaceName(linkType layers.LinkType) string { + switch linkType { + case layers.LinkTypeEthernet: + return "reflow-ethernet" + case layers.LinkTypeRaw, layers.LinkTypeIPv4, layers.LinkTypeIPv6: + return "reflow-raw" + default: + return "reflow" + } +} + +type pcapNGInterfaceMetadata struct { + Name string + Description string + Comment string +} + +func (e *PcapNGEncoder) interfaceBlock() pcapNGInterfaceMetadata { + iface := e.iface + if iface.Name == "" { + iface.Name = pcapNGInterfaceName(e.packet.linkType) + } + return iface +} + +func pcapNGInterfaceNameFromSource(src event.SourceMetadata) string { + if src.CaptureInterface != "" { + return src.CaptureInterface + } + if src.Address == "" { + return "" + } + if src.Address == "-" { + return "stdin" + } + base := filepath.Base(src.Address) + if base == "." || base == string(filepath.Separator) { + return src.Address + } + return base +} + +func pcapNGInterfaceDescription(src event.SourceMetadata) string { + if src.Network == "" && src.Type == "" && src.Address == "" { + return "" + } + return fmt.Sprintf("network=%s type=%s address=%s", src.Network, src.Type, src.Address) +} diff --git a/pkg/reflow/encode/pcapng_test.go b/pkg/reflow/encode/pcapng_test.go new file mode 100644 index 00000000..5778a38d --- /dev/null +++ b/pkg/reflow/encode/pcapng_test.go @@ -0,0 +1,149 @@ +package encode + +import ( + "bytes" + "testing" + "time" + + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func TestPcapNGEncoderWritesReadablePacket(t *testing.T) { + enc, err := NewPcapNGEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "auto", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapNGEncoder returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + ReceivedAt: time.Unix(30, 123).UTC(), + Fields: map[string]any{ + "header_data": ethernetFrame(0x0800), + "protocol": uint32(1), + "original_length": uint32(128), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + if len(payloads) != 1 { + t.Fatalf("expected one payload, got %d", len(payloads)) + } + + reader, err := pcapgo.NewNgReader(bytes.NewReader(payloads[0]), pcapgo.NgReaderOptions{}) + if err != nil { + t.Fatalf("NewNgReader returned error: %v", err) + } + if reader.LinkType() != layers.LinkTypeEthernet { + t.Fatalf("expected ethernet link type, got %v", reader.LinkType()) + } + data, ci, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData returned error: %v", err) + } + if len(data) != len(ethernetFrame(0x0800)) { + t.Fatalf("expected packet bytes, got caplen %d", len(data)) + } + if ci.Length != 128 { + t.Fatalf("expected wire length 128, got %d", ci.Length) + } +} + +func TestPcapNGEncoderUsesSourceInitInterfaceName(t *testing.T) { + enc, err := NewPcapNGEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "auto", + LinkType: "ethernet", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapNGEncoder returned error: %v", err) + } + + if payloads, err := enc.Encode(&event.Event{ + Kind: "control", + Source: event.SourceMetadata{ + Network: "pcap_live", + Address: "en7", + Type: "bytes", + CaptureInterface: "en7", + }, + Control: &event.ControlMetadata{ + Type: "source_init", + }, + }); err != nil || len(payloads) != 0 { + t.Fatalf("expected source_init to be consumed without payloads, payloads=%d err=%v", len(payloads), err) + } + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "header_data": ethernetFrame(0x0800), + "protocol": uint32(1), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + reader, err := pcapgo.NewNgReader(bytes.NewReader(payloads[0]), pcapgo.NgReaderOptions{}) + if err != nil { + t.Fatalf("NewNgReader returned error: %v", err) + } + iface, err := reader.Interface(0) + if err != nil { + t.Fatalf("Interface returned error: %v", err) + } + if iface.Name != "en7" { + t.Fatalf("expected interface name en7, got %q", iface.Name) + } +} + +func TestPcapNGEncoderUsesPseudoRawPacket(t *testing.T) { + enc, err := NewPcapNGEncoder(config.EncoderConfig{ + Pcap: config.PcapConfig{ + PacketSource: "pseudo", + LinkType: "raw", + SnapLen: 65535, + }, + }) + if err != nil { + t.Fatalf("NewPcapNGEncoder returned error: %v", err) + } + + payloads, err := enc.Encode(&event.Event{ + Fields: map[string]any{ + "src_addr": "192.0.2.1", + "dst_addr": "198.51.100.2", + "proto": uint32(17), + "src_port": uint32(12345), + "dst_port": uint32(53), + }, + }) + if err != nil { + t.Fatalf("Encode returned error: %v", err) + } + + reader, err := pcapgo.NewNgReader(bytes.NewReader(payloads[0]), pcapgo.NgReaderOptions{}) + if err != nil { + t.Fatalf("NewNgReader returned error: %v", err) + } + if reader.LinkType() != layers.LinkTypeRaw { + t.Fatalf("expected raw link type, got %v", reader.LinkType()) + } + data, _, err := reader.ReadPacketData() + if err != nil { + t.Fatalf("ReadPacketData returned error: %v", err) + } + if len(data) == 0 || data[0]>>4 != 4 { + t.Fatalf("expected raw IPv4 packet, got %x", data[:min(len(data), 14)]) + } +} diff --git a/pkg/reflow/encode/protobuf.go b/pkg/reflow/encode/protobuf.go new file mode 100644 index 00000000..59335b05 --- /dev/null +++ b/pkg/reflow/encode/protobuf.go @@ -0,0 +1,392 @@ +package encode + +import ( + "encoding/json" + "fmt" + "math" + "net/netip" + "sort" + "strings" + + flowpb "github.com/netsampler/goflow2/v3/pb" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "google.golang.org/protobuf/encoding/protowire" +) + +type protobufFieldSpec struct { + name string + number protowire.Number + protoType string + append func([]byte, *event.Event) ([]byte, bool, error) +} + +type ProtobufEncoder struct { + flavor string + lengthPrefixed bool + exportAll bool + fields []protobufFieldSpec +} + +func NewProtobufEncoder(cfg config.EncoderConfig) (*ProtobufEncoder, error) { + fields, err := compileProtobufFieldPlan(cfg.Protobuf.Flavor) + if err != nil { + return nil, err + } + return &ProtobufEncoder{ + flavor: cfg.Protobuf.Flavor, + lengthPrefixed: cfg.Protobuf.LengthPrefixed, + exportAll: cfg.Protobuf.ExportAll, + fields: fields, + }, nil +} + +func (e *ProtobufEncoder) Encode(evt *event.Event) ([][]byte, error) { + buf := make([]byte, 0, 256) + for _, field := range e.fields { + next, ok, err := field.append(buf, evt) + if err != nil { + return nil, fmt.Errorf("encode protobuf field %s: %w", field.name, err) + } + if !ok { + continue + } + buf = next + } + if e.exportAll { + var err error + buf, err = appendProtobufExtraFields(buf, evt) + if err != nil { + return nil, err + } + } + + if e.lengthPrefixed { + framed := protowire.AppendVarint(make([]byte, 0, len(buf)+10), uint64(len(buf))) + framed = append(framed, buf...) + return [][]byte{framed}, nil + } + return [][]byte{buf}, nil +} + +func (e *ProtobufEncoder) Flush() ([][]byte, error) { + return nil, nil +} + +func GenerateProtobufDefinition(cfg config.ProtobufConfig) (string, error) { + fields, err := compileProtobufFieldPlan(cfg.Flavor) + if err != nil { + return "", err + } + + var b strings.Builder + b.WriteString("syntax = \"proto3\";\n") + b.WriteString("package flowpb;\n") + b.WriteString("option go_package = \"github.com/netsampler/goflow2/pb;flowpb\";\n\n") + b.WriteString("message FlowMessage {\n\n") + b.WriteString(" enum FlowType {\n") + b.WriteString(" FLOWUNKNOWN = 0;\n") + b.WriteString(" SFLOW_5 = 1;\n") + b.WriteString(" NETFLOW_V5 = 2;\n") + b.WriteString(" NETFLOW_V9 = 3;\n") + b.WriteString(" IPFIX = 4;\n") + b.WriteString(" }\n\n") + for _, field := range fields { + fmt.Fprintf(&b, " %s %s = %d;\n", field.protoType, field.name, field.number) + } + if cfg.ExportAll { + b.WriteString(" repeated ExtraField extra = 1000;\n\n") + b.WriteString(" message ExtraField {\n") + b.WriteString(" string key = 1;\n") + b.WriteString(" oneof value {\n") + b.WriteString(" string string_value = 2;\n") + b.WriteString(" uint64 uint64_value = 3;\n") + b.WriteString(" sint64 int64_value = 4;\n") + b.WriteString(" bool bool_value = 5;\n") + b.WriteString(" bytes bytes_value = 6;\n") + b.WriteString(" double double_value = 7;\n") + b.WriteString(" string json_value = 8;\n") + b.WriteString(" }\n") + b.WriteString(" }\n") + } + b.WriteString("\n}\n") + return b.String(), nil +} + +func compileProtobufFieldPlan(flavor string) ([]protobufFieldSpec, error) { + switch flavor { + case "", "canonical", "goflow2v2": + return []protobufFieldSpec{ + protobufEnumField("type", 1, protobufFlowTypeValue), + protobufUint64Field("time_received_ns", 110, func(evt *event.Event) uint64 { + if evt == nil || evt.ReceivedAt.IsZero() { + return 0 + } + return uint64(evt.ReceivedAt.UnixNano()) + }), + protobufUint32Field("sequence_num", 4, func(evt *event.Event) uint32 { + if evt != nil && evt.SFlow != nil && evt.SFlow.SequenceNumber != 0 { + return evt.SFlow.SequenceNumber + } + return uint32Field(evt.Fields, "sequence_num") + }), + protobufUint64Field("sampling_rate", 3, func(evt *event.Event) uint64 { + return uint64(eventSamplingRate(evt)) + }), + protobufIPField("sampler_address", 11, func(evt *event.Event) string { + return eventAgentIP(evt) + }), + protobufUint64Field("time_flow_start_ns", 111, func(evt *event.Event) uint64 { + startNS := timeFlowNS(evt.Fields, "time_flow_start_ns", "start_time_unix") + if startNS == 0 { + return 0 + } + return uint64(startNS) + }), + protobufUint64Field("time_flow_end_ns", 112, func(evt *event.Event) uint64 { + endNS := timeFlowNS(evt.Fields, "time_flow_end_ns", "end_time_unix") + if endNS == 0 { + return 0 + } + return uint64(endNS) + }), + protobufUint64Field("bytes", 9, func(evt *event.Event) uint64 { + return uint64Field(evt.Fields, "bytes") + }), + protobufUint64Field("packets", 10, func(evt *event.Event) uint64 { + return uint64Field(evt.Fields, "packets") + }), + protobufIPField("src_addr", 6, func(evt *event.Event) string { + return stringFieldOrZero(evt.Fields, "src_addr") + }), + protobufIPField("dst_addr", 7, func(evt *event.Event) string { + return stringFieldOrZero(evt.Fields, "dst_addr") + }), + protobufUint32Field("etype", 30, func(evt *event.Event) uint32 { + return uint32Field(evt.Fields, "etype") + }), + protobufUint32Field("proto", 20, func(evt *event.Event) uint32 { + return uint32Field(evt.Fields, "proto") + }), + protobufUint32Field("src_port", 21, func(evt *event.Event) uint32 { + return uint32Field(evt.Fields, "src_port") + }), + protobufUint32Field("dst_port", 22, func(evt *event.Event) uint32 { + return uint32Field(evt.Fields, "dst_port") + }), + protobufUint32Field("in_if", 18, func(evt *event.Event) uint32 { + return uint32Field(evt.Fields, "input_if") + }), + protobufUint32Field("out_if", 19, func(evt *event.Event) uint32 { + return uint32Field(evt.Fields, "output_if") + }), + protobufUint32Field("observation_domain_id", 70, func(evt *event.Event) uint32 { + return uint32Field(evt.Fields, "observation_domain_id") + }), + protobufUint32Field("observation_point_id", 71, func(evt *event.Event) uint32 { + return uint32Field(evt.Fields, "observation_point_id") + }), + }, nil + default: + return nil, fmt.Errorf("unsupported protobuf flavor %q", flavor) + } +} + +func protobufUint32Field(name string, num protowire.Number, get func(*event.Event) uint32) protobufFieldSpec { + return protobufFieldSpec{ + name: name, + number: num, + protoType: "uint32", + append: func(dst []byte, evt *event.Event) ([]byte, bool, error) { + val := get(evt) + if val == 0 { + return dst, false, nil + } + dst = protowire.AppendTag(dst, num, protowire.VarintType) + dst = protowire.AppendVarint(dst, uint64(val)) + return dst, true, nil + }, + } +} + +func protobufUint64Field(name string, num protowire.Number, get func(*event.Event) uint64) protobufFieldSpec { + return protobufFieldSpec{ + name: name, + number: num, + protoType: "uint64", + append: func(dst []byte, evt *event.Event) ([]byte, bool, error) { + val := get(evt) + if val == 0 { + return dst, false, nil + } + dst = protowire.AppendTag(dst, num, protowire.VarintType) + dst = protowire.AppendVarint(dst, val) + return dst, true, nil + }, + } +} + +func protobufIPField(name string, num protowire.Number, get func(*event.Event) string) protobufFieldSpec { + return protobufFieldSpec{ + name: name, + number: num, + protoType: "bytes", + append: func(dst []byte, evt *event.Event) ([]byte, bool, error) { + ip := get(evt) + if ip == "" { + return dst, false, nil + } + addr, err := netip.ParseAddr(ip) + if err != nil { + return dst, false, nil + } + dst = protowire.AppendTag(dst, num, protowire.BytesType) + dst = protowire.AppendBytes(dst, addr.AsSlice()) + return dst, true, nil + }, + } +} + +func protobufEnumField(name string, num protowire.Number, get func(*event.Event) uint64) protobufFieldSpec { + return protobufFieldSpec{ + name: name, + number: num, + protoType: "FlowType", + append: func(dst []byte, evt *event.Event) ([]byte, bool, error) { + val := get(evt) + if val == 0 { + return dst, false, nil + } + dst = protowire.AppendTag(dst, num, protowire.VarintType) + dst = protowire.AppendVarint(dst, val) + return dst, true, nil + }, + } +} + +func appendProtobufExtraFields(dst []byte, evt *event.Event) ([]byte, error) { + if evt == nil || len(evt.Fields) == 0 { + return dst, nil + } + keys := make([]string, 0, len(evt.Fields)) + for key := range evt.Fields { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + msg, ok, err := protobufExtraFieldMessage(key, evt.Fields[key]) + if err != nil { + return nil, fmt.Errorf("encode protobuf extra field %s: %w", key, err) + } + if !ok { + continue + } + dst = protowire.AppendTag(dst, 1000, protowire.BytesType) + dst = protowire.AppendBytes(dst, msg) + } + return dst, nil +} + +func protobufExtraFieldMessage(key string, val any) ([]byte, bool, error) { + if key == "" { + return nil, false, nil + } + msg := protowire.AppendTag(nil, 1, protowire.BytesType) + msg = protowire.AppendString(msg, key) + return appendProtobufExtraValue(msg, val) +} + +func appendProtobufExtraValue(dst []byte, val any) ([]byte, bool, error) { + switch v := val.(type) { + case nil: + return appendProtobufExtraJSONValue(dst, nil) + case string: + dst = protowire.AppendTag(dst, 2, protowire.BytesType) + dst = protowire.AppendString(dst, v) + case []byte: + dst = protowire.AppendTag(dst, 6, protowire.BytesType) + dst = protowire.AppendBytes(dst, v) + case bool: + dst = protowire.AppendTag(dst, 5, protowire.VarintType) + dst = protowire.AppendVarint(dst, protowire.EncodeBool(v)) + case int: + dst = appendProtobufExtraInt64Value(dst, int64(v)) + case int8: + dst = appendProtobufExtraInt64Value(dst, int64(v)) + case int16: + dst = appendProtobufExtraInt64Value(dst, int64(v)) + case int32: + dst = appendProtobufExtraInt64Value(dst, int64(v)) + case int64: + dst = appendProtobufExtraInt64Value(dst, v) + case uint: + dst = appendProtobufExtraUint64Value(dst, uint64(v)) + case uint8: + dst = appendProtobufExtraUint64Value(dst, uint64(v)) + case uint16: + dst = appendProtobufExtraUint64Value(dst, uint64(v)) + case uint32: + dst = appendProtobufExtraUint64Value(dst, uint64(v)) + case uint64: + dst = appendProtobufExtraUint64Value(dst, v) + case float32: + dst = appendProtobufExtraDoubleValue(dst, float64(v)) + case float64: + dst = appendProtobufExtraDoubleValue(dst, v) + default: + return appendProtobufExtraJSONValue(dst, v) + } + return dst, true, nil +} + +func appendProtobufExtraUint64Value(dst []byte, val uint64) []byte { + dst = protowire.AppendTag(dst, 3, protowire.VarintType) + return protowire.AppendVarint(dst, val) +} + +func appendProtobufExtraInt64Value(dst []byte, val int64) []byte { + dst = protowire.AppendTag(dst, 4, protowire.VarintType) + return protowire.AppendVarint(dst, protowire.EncodeZigZag(val)) +} + +func appendProtobufExtraDoubleValue(dst []byte, val float64) []byte { + dst = protowire.AppendTag(dst, 7, protowire.Fixed64Type) + return protowire.AppendFixed64(dst, math.Float64bits(val)) +} + +func appendProtobufExtraJSONValue(dst []byte, val any) ([]byte, bool, error) { + raw, err := json.Marshal(val) + if err != nil { + raw, err = json.Marshal(fmt.Sprint(val)) + if err != nil { + return nil, false, err + } + } + dst = protowire.AppendTag(dst, 8, protowire.BytesType) + dst = protowire.AppendBytes(dst, raw) + return dst, true, nil +} + +func protobufFlowTypeValue(evt *event.Event) uint64 { + if evt == nil { + return 0 + } + if evt.Fields != nil { + switch stringFieldOrZero(evt.Fields, "counter_type") { + case "sflow": + return uint64(flowpb.FlowMessage_SFLOW_5) + } + switch val := flowTypeField(evt.Fields).(type) { + case int: + return uint64(val) + case uint32: + return uint64(val) + case uint64: + return val + } + } + if evt.SFlow != nil { + return uint64(flowpb.FlowMessage_SFLOW_5) + } + return 0 +} diff --git a/pkg/reflow/encode/sflow.go b/pkg/reflow/encode/sflow.go new file mode 100644 index 00000000..c1a0a03b --- /dev/null +++ b/pkg/reflow/encode/sflow.go @@ -0,0 +1,1019 @@ +package encode + +import ( + "bytes" + "errors" + "fmt" + "log/slog" + "net/netip" + "sync/atomic" + "time" + + "github.com/netsampler/goflow2/v3/decoders/sflow" + "github.com/netsampler/goflow2/v3/decoders/utils" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/packet" +) + +var ErrSFlowSampleTooLarge = errors.New("sflow sample exceeds max_datagram_bytes") + +const sflowFlowRecordsInternalKey = "sflow_flow_records" +const sflowSampleInternalKey = "sflow_sample" +const sflowCounterRecordsInternalKey = "sflow_counter_records" + +type sflowSampleTooLargeError struct { + MaxDatagramBytes int + CurrentSize int +} + +// Error reports both the configured datagram limit and the offending packet size. +func (e *sflowSampleTooLargeError) Error() string { + return fmt.Sprintf("%s: current_size=%d max_datagram_bytes=%d", ErrSFlowSampleTooLarge, e.CurrentSize, e.MaxDatagramBytes) +} + +// Unwrap lets callers treat the concrete size error as ErrSFlowSampleTooLarge. +func (e *sflowSampleTooLargeError) Unwrap() error { + return ErrSFlowSampleTooLarge +} + +type SFlowEncoder struct { + packetSeq atomic.Uint32 + sampleSeq atomic.Uint32 + started time.Time + maxDatagramBytes int + allowTruncate bool + maxHeaderBytes int + batch config.BatchConfig + cfg config.SFlowConfig + events []*event.Event +} + +// NewSFlowEncoder builds one encoder instance per runtime worker. Each instance +// keeps its own batch buffer and sequence counters. +func NewSFlowEncoder(cfg config.EncoderConfig) *SFlowEncoder { + return &SFlowEncoder{ + started: time.Now(), + maxDatagramBytes: cfg.MaxDatagramBytes, + allowTruncate: cfg.AllowTruncate != nil && *cfg.AllowTruncate, + maxHeaderBytes: cfg.SFlow.MaxHeaderBytes, + batch: cfg.Batch, + cfg: cfg.SFlow, + } +} + +// Encode appends an event to the encoder-local batch or encodes it immediately. +func (e *SFlowEncoder) Encode(evt *event.Event) ([][]byte, error) { + if evt == nil || evt.Kind == "control" { + return nil, nil + } + + if !e.batch.IsEnabled() { + packet, err := e.buildPacket([]*event.Event{evt}) + if err != nil { + if errors.Is(err, ErrSFlowSampleTooLarge) { + logOversizedSample(err) + return nil, nil + } + return nil, err + } + return e.encodePacket(packet) + } + + if len(e.events) > 0 && !e.compatibleTopLevel(e.events[0], evt) { + payloads, err := e.Flush() + if err != nil { + return nil, err + } + e.appendEvent(evt) + if e.shouldFlush() { + flushed, err := e.Flush() + if err != nil { + return nil, err + } + return append(payloads, flushed...), nil + } + return payloads, nil + } + + e.appendEvent(evt) + if e.shouldFlush() { + return e.Flush() + } + return nil, nil +} + +// Flush emits all buffered events, splitting them into multiple sFlow datagrams if needed. +func (e *SFlowEncoder) Flush() ([][]byte, error) { + if len(e.events) == 0 { + return nil, nil + } + + var payloads [][]byte + pending := e.events + e.events = nil + + for len(pending) > 0 { + packet, accepted, err := e.buildPacketWithLimit(pending) + if err != nil { + if errors.Is(err, ErrSFlowSampleTooLarge) { + logOversizedSample(err) + pending = pending[1:] + continue + } + return nil, err + } + encoded, err := e.encodePacket(packet) + if err != nil { + return nil, err + } + payloads = append(payloads, encoded...) + pending = pending[accepted:] + } + + return payloads, nil +} + +// shouldFlush checks the configured batch thresholds before the timer fires. +func (e *SFlowEncoder) shouldFlush() bool { + if len(e.events) == 0 { + return false + } + if e.batch.MaxRecords > 0 && len(e.events) >= e.batch.MaxRecords { + return true + } + return false +} + +func (e *SFlowEncoder) appendEvent(evt *event.Event) { + e.events = append(e.events, evt) +} + +func estimatedEventSize(evt *event.Event) int { + total := len(evt.Message) + 128 + for key, val := range evt.Fields { + total += len(key) + switch v := val.(type) { + case string: + total += len(v) + case []byte: + total += len(v) + default: + total += 16 + } + } + return total +} + +// encodePacket turns one populated sFlow packet into the UDP payload sent by the sink. +func (e *SFlowEncoder) encodePacket(packet *sflow.Packet) ([][]byte, error) { + data, err := sflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode sflow packet: %w", err) + } + return [][]byte{data}, nil +} + +// buildPacket requires all input events to fit in a single datagram. +func (e *SFlowEncoder) buildPacket(events []*event.Event) (*sflow.Packet, error) { + packet, accepted, err := e.buildPacketWithLimit(events) + if err != nil { + return nil, err + } + if accepted != len(events) { + return nil, fmt.Errorf("sflow packet exceeds max_datagram_bytes=%d", e.maxDatagramBytes) + } + return packet, nil +} + +// buildPacketWithLimit packs as many events as possible into one sFlow datagram. +func (e *SFlowEncoder) buildPacketWithLimit(events []*event.Event) (*sflow.Packet, int, error) { + if len(events) == 0 { + return nil, 0, fmt.Errorf("empty sflow packet batch") + } + + first := events[0] + top, err := e.packetTopLevel(first) + if err != nil { + return nil, 0, err + } + addr, err := netip.ParseAddr(top.AgentIP) + if err != nil { + return nil, 0, fmt.Errorf("parse agent_ip %q: %w", top.AgentIP, err) + } + + packetSeq := top.SequenceNumber + if packetSeq == 0 { + packetSeq = e.packetSeq.Add(1) + } + uptime := top.Uptime + if uptime == 0 { + uptime = uint32(time.Since(e.started).Milliseconds()) + } + packet := &sflow.Packet{ + Version: 5, + AgentIP: utils.IPAddress(addr.AsSlice()), + SubAgentId: top.SubAgentID, + SequenceNumber: packetSeq, + Uptime: uptime, + Samples: make([]interface{}, 0, len(events)), + } + + accepted := 0 + for _, evt := range events { + if accepted > 0 && !e.compatibleTopLevel(first, evt) { + break + } + sample, err := e.buildSample(evt) + if err != nil { + return nil, accepted, err + } + packet.Samples = append(packet.Samples, sample) + lastSize := 0 + if limit := e.batchDatagramLimit(); limit > 0 { + data, err := sflow.EncodeMessage(packet) + if err != nil { + return nil, accepted, fmt.Errorf("encode sflow packet: %w", err) + } + lastSize = len(data) + if len(data) > limit { + if e.allowTruncate { + truncated, ok, err := e.truncateLastSampleToFit(packet) + if err != nil { + return nil, accepted, err + } + if ok { + packet.Samples[len(packet.Samples)-1] = truncated + accepted++ + continue + } + } + packet.Samples = packet.Samples[:len(packet.Samples)-1] + if accepted == 0 { + return nil, 0, &sflowSampleTooLargeError{ + MaxDatagramBytes: limit, + CurrentSize: lastSize, + } + } + break + } + } + accepted++ + } + + if accepted == 0 { + return nil, 0, &sflowSampleTooLargeError{ + MaxDatagramBytes: e.batchDatagramLimit(), + CurrentSize: 0, + } + } + + packet.SamplesCount = uint32(len(packet.Samples)) + return packet, accepted, nil +} + +func (e *SFlowEncoder) batchDatagramLimit() int { + limit := e.maxDatagramBytes + if e.batch.IsEnabled() && e.batch.MaxBytes > 0 && (limit <= 0 || e.batch.MaxBytes < limit) { + limit = e.batch.MaxBytes + } + return limit +} + +// buildSample dispatches between flow-sample and counter-sample output. +func (e *SFlowEncoder) buildSample(evt *event.Event) (interface{}, error) { + if sample, ok := evt.Internal[sflowSampleInternalKey].(sflow.RawSample); ok { + return sample, nil + } + if sample, ok := evt.Internal[sflowSampleInternalKey].(*sflow.RawSample); ok { + return sample, nil + } + if sample, ok := publicSFlowRawSample(evt); ok { + return sample, nil + } + if isSFlowCounterEvent(evt) { + return e.buildCounterSample(evt) + } + return e.buildFlowSample(evt) +} + +// buildFlowSample maps the canonical event fields into one sFlow raw-header flow sample. +func (e *SFlowEncoder) buildFlowSample(evt *event.Event) (sflow.FlowSample, error) { + fields := evt.Fields + if fields == nil { + return sflow.FlowSample{}, fmt.Errorf("event fields are empty") + } + headerData, protocol, frameLength, originalLength := e.sampledHeaderFields(evt, fields) + + records := []sflow.FlowRecord{ + { + Data: sflow.SampledHeader{ + Protocol: protocol, + FrameLength: frameLength, + Stripped: uint32Field(fields, "stripped"), + OriginalLength: originalLength, + HeaderData: headerData, + }, + }, + } + records = append(records, preservedSFlowFlowRecords(evt)...) + records = append(records, publicSFlowFlowRecords(evt)...) + if e.emitExtendedRecords() { + records = append(records, e.syntheticExtendedRecords(fields, records)...) + } + + return sflow.FlowSample{ + Header: sflow.SampleHeader{ + Format: sflow.SAMPLE_FORMAT_FLOW, + SampleSequenceNumber: e.sampleSequence(evt), + SourceIdType: 0, + SourceIdValue: sflowSourceID(evt), + }, + SamplingRate: sflowSamplingRate(evt), + SamplePool: sflowSamplePool(evt), + Drops: sflowDrops(evt), + Input: uint32Field(fields, "input_if"), + Output: uint32Field(fields, "output_if"), + Records: records, + }, nil +} + +func (e *SFlowEncoder) emitExtendedRecords() bool { + return e.cfg.EmitExtendedRecords == nil || *e.cfg.EmitExtendedRecords +} + +func preservedSFlowFlowRecords(evt *event.Event) []sflow.FlowRecord { + if evt == nil || evt.Internal == nil { + return nil + } + records, _ := evt.Internal[sflowFlowRecordsInternalKey].([]sflow.FlowRecord) + if len(records) == 0 { + return nil + } + out := make([]sflow.FlowRecord, len(records)) + copy(out, records) + return out +} + +func publicSFlowRawSample(evt *event.Event) (sflow.RawSample, bool) { + if evt == nil || evt.SFlow == nil || len(evt.SFlow.Samples) == 0 { + return sflow.RawSample{}, false + } + sample := evt.SFlow.Samples[0] + if len(sample.Data) == 0 { + return sflow.RawSample{}, false + } + dataFormat, ok := publicSFlowDataFormat(sample.Enterprise, sample.Format, sample.DataFormat) + if !ok { + return sflow.RawSample{}, false + } + if !validPublicRawSamplePayload(dataFormat, sample.Data) { + return sflow.RawSample{}, false + } + return sflow.RawSample{ + Header: sflow.SampleHeader{Format: dataFormat}, + Data: append([]byte(nil), sample.Data...), + }, true +} + +func validPublicRawSamplePayload(dataFormat uint32, data []byte) bool { + if sflow.DataFormatEnterprise(dataFormat) != 0 { + return true + } + switch sflow.DataFormatFormat(dataFormat) { + case sflow.SAMPLE_FORMAT_FLOW, sflow.SAMPLE_FORMAT_COUNTER, sflow.SAMPLE_FORMAT_EXPANDED_FLOW, sflow.SAMPLE_FORMAT_EXPANDED_COUNTER, sflow.SAMPLE_FORMAT_DROP: + header := sflow.SampleHeader{Format: dataFormat, Length: uint32(len(data))} + payload := bytes.NewBuffer(data) + if _, err := sflow.DecodeSample(&header, payload); err != nil { + return false + } + return payload.Len() == 0 + default: + return true + } +} + +func publicSFlowFlowRecords(evt *event.Event) []sflow.FlowRecord { + if evt == nil || evt.SFlow == nil || len(evt.SFlow.Samples) == 0 { + return nil + } + rawRecords := evt.SFlow.Samples[0].RawFlowRecords + records := make([]sflow.FlowRecord, 0, len(rawRecords)) + for _, raw := range rawRecords { + dataFormat, ok := publicSFlowDataFormat(raw.Enterprise, raw.Format, raw.DataFormat) + if !ok { + continue + } + records = append(records, sflow.FlowRecord{ + Header: sflow.RecordHeader{DataFormat: dataFormat}, + Data: sflow.RawRecord{Data: append([]byte(nil), raw.Data...)}, + }) + } + return records +} + +func publicSFlowDataFormat(enterprise, format, dataFormat uint32) (uint32, bool) { + if dataFormat != 0 { + return dataFormat, true + } + if enterprise == 0 && format == 0 { + return 0, false + } + return sflow.PackDataFormat(enterprise, format), true +} + +func (e *SFlowEncoder) syntheticExtendedRecords(fields map[string]any, existing []sflow.FlowRecord) []sflow.FlowRecord { + var records []sflow.FlowRecord + if !hasStandardSFlowRecord(existing, sflow.FLOW_TYPE_EXT_NAT) { + if record, ok := syntheticNATRecord(fields); ok { + records = append(records, record) + } + } + if !hasStandardSFlowRecord(existing, sflow.FLOW_TYPE_EXT_MPLS) { + if record, ok := syntheticMPLSRecord(fields); ok { + records = append(records, record) + } + } + if !hasStandardSFlowRecord(existing, sflow.FLOW_TYPE_EXT_MPLS_TUNNEL) { + if record, ok := syntheticMPLSTunnelRecord(fields); ok { + records = append(records, record) + } + } + if !hasStandardSFlowRecord(existing, sflow.FLOW_TYPE_EXT_MPLS_VC) { + if record, ok := syntheticMPLSVCRecord(fields); ok { + records = append(records, record) + } + } + if !hasStandardSFlowRecord(existing, sflow.FLOW_TYPE_EXT_MPLS_FEC) { + if record, ok := syntheticMPLSFTNRecord(fields); ok { + records = append(records, record) + } + } + if !hasStandardSFlowRecord(existing, sflow.FLOW_TYPE_EXT_MPLS_LVP_FEC) { + if record, ok := syntheticMPLSLDPFECRecord(fields); ok { + records = append(records, record) + } + } + return records +} + +func hasStandardSFlowRecord(records []sflow.FlowRecord, format uint32) bool { + for _, record := range records { + dataFormat := flowRecordDataFormat(record) + if sflow.DataFormatEnterprise(dataFormat) == 0 && sflow.DataFormatFormat(dataFormat) == format { + return true + } + } + return false +} + +func flowRecordDataFormat(record sflow.FlowRecord) uint32 { + if record.Header.DataFormat != 0 { + return record.Header.DataFormat + } + switch record.Data.(type) { + case sflow.ExtendedNAT, *sflow.ExtendedNAT: + return sflow.FLOW_TYPE_EXT_NAT + case sflow.ExtendedMPLS, *sflow.ExtendedMPLS: + return sflow.FLOW_TYPE_EXT_MPLS + case sflow.ExtendedMPLSTunnel, *sflow.ExtendedMPLSTunnel: + return sflow.FLOW_TYPE_EXT_MPLS_TUNNEL + case sflow.ExtendedMPLSVC, *sflow.ExtendedMPLSVC: + return sflow.FLOW_TYPE_EXT_MPLS_VC + case sflow.ExtendedMPLSFTN, *sflow.ExtendedMPLSFTN: + return sflow.FLOW_TYPE_EXT_MPLS_FEC + case sflow.ExtendedMPLSLDPFEC, *sflow.ExtendedMPLSLDPFEC: + return sflow.FLOW_TYPE_EXT_MPLS_LVP_FEC + default: + return 0 + } +} + +func syntheticNATRecord(fields map[string]any) (sflow.FlowRecord, bool) { + src := stringFieldOrZero(fields, "nat_src_addr") + dst := stringFieldOrZero(fields, "nat_dst_addr") + if src == "" && dst == "" { + return sflow.FlowRecord{}, false + } + if src == "" { + src = stringFieldOrZero(fields, "src_addr") + } + if dst == "" { + dst = stringFieldOrZero(fields, "dst_addr") + } + srcAddr, srcOK := parseSFlowIP(src) + dstAddr, dstOK := parseSFlowIP(dst) + if !srcOK || !dstOK { + return sflow.FlowRecord{}, false + } + return sflow.FlowRecord{Data: sflow.ExtendedNAT{ + SrcAddress: srcAddr, + DstAddress: dstAddr, + }}, true +} + +func syntheticMPLSRecord(fields map[string]any) (sflow.FlowRecord, bool) { + inStack := uint32SliceField(fields, "mpls_in_label_stack") + outStack := uint32SliceField(fields, "mpls_out_label_stack") + if len(inStack) == 0 { + inStack = mplsStackFromHelperFields(fields) + } + if len(inStack) == 0 && len(outStack) == 0 { + return sflow.FlowRecord{}, false + } + nextHop, _ := parseSFlowIP(stringFieldOrZero(fields, "mpls_next_hop_addr")) + return sflow.FlowRecord{Data: sflow.ExtendedMPLS{ + NextHop: nextHop, + InLabelStack: inStack, + OutLabelStack: outStack, + }}, true +} + +func syntheticMPLSTunnelRecord(fields map[string]any) (sflow.FlowRecord, bool) { + name := stringFieldOrZero(fields, "mpls_tunnel_lsp_name") + id := uint32Field(fields, "mpls_tunnel_id") + cos := uint32Field(fields, "mpls_tunnel_cos") + if name == "" && id == 0 && cos == 0 { + return sflow.FlowRecord{}, false + } + return sflow.FlowRecord{Data: sflow.ExtendedMPLSTunnel{ + TunnelLSPName: name, + TunnelID: id, + TunnelCOS: cos, + }}, true +} + +func syntheticMPLSVCRecord(fields map[string]any) (sflow.FlowRecord, bool) { + name := stringFieldOrZero(fields, "mpls_vc_instance_name") + id := uint32Field(fields, "mpls_vll_vc_id") + cos := uint32Field(fields, "mpls_vc_label_cos") + if name == "" && id == 0 && cos == 0 { + return sflow.FlowRecord{}, false + } + return sflow.FlowRecord{Data: sflow.ExtendedMPLSVC{ + VCInstanceName: name, + VLLVCID: id, + VCLabelCOS: cos, + }}, true +} + +func syntheticMPLSFTNRecord(fields map[string]any) (sflow.FlowRecord, bool) { + descr := stringFieldOrZero(fields, "mpls_ftn_descr") + mask := uint32Field(fields, "mpls_ftn_mask") + if descr == "" && mask == 0 { + return sflow.FlowRecord{}, false + } + return sflow.FlowRecord{Data: sflow.ExtendedMPLSFTN{ + MPLSFTNDescr: descr, + MPLSFTNMask: mask, + }}, true +} + +func syntheticMPLSLDPFECRecord(fields map[string]any) (sflow.FlowRecord, bool) { + prefixLength := uint32Field(fields, "mpls_fec_addr_prefix_length") + if prefixLength == 0 { + return sflow.FlowRecord{}, false + } + return sflow.FlowRecord{Data: sflow.ExtendedMPLSLDPFEC{ + MPLSFecAddrPrefixLength: prefixLength, + }}, true +} + +func parseSFlowIP(value string) ([]byte, bool) { + if value == "" { + return nil, true + } + addr, err := netip.ParseAddr(value) + if err != nil { + return nil, false + } + return append([]byte(nil), addr.AsSlice()...), true +} + +func uint32SliceField(fields map[string]any, key string) []uint32 { + if fields == nil { + return nil + } + val, ok := fields[key] + if !ok { + return nil + } + switch v := val.(type) { + case []uint32: + return append([]uint32(nil), v...) + case []any: + out := make([]uint32, 0, len(v)) + for _, item := range v { + out = append(out, uint32(uint64FromAny(item))) + } + return out + default: + return nil + } +} + +func mplsStackFromHelperFields(fields map[string]any) []uint32 { + var entries []uint32 + for i := 1; i <= 16; i++ { + section := bytesField(fields, fmt.Sprintf("mpls_label_stack_section_%d", i)) + if len(section) >= 3 { + entries = append(entries, (uint32(section[0])<<24)|(uint32(section[1])<<16)|(uint32(section[2])<<8)) + continue + } + label := uint32Field(fields, fmt.Sprintf("mpls_label_%d", i)) + if label == 0 { + label = uint32Field(fields, fmt.Sprintf("mpls_label%d", i)) + } + if label == 0 { + continue + } + entries = append(entries, (label&0xfffff)<<12) + } + if len(entries) > 0 { + entries[len(entries)-1] |= 1 << 8 + } + return entries +} + +func (e *SFlowEncoder) sampledHeaderFields(evt *event.Event, fields map[string]any) ([]byte, uint32, uint32, uint32) { + headerData := bytesField(fields, "header_data") + protocol := uint32Field(fields, "protocol") + frameLength := uint32Field(fields, "frame_length") + originalLength := uint32Field(fields, "original_length") + if len(headerData) == 0 { + if pseudoHeader, ok := packet.BuildPseudoHeader(evt, fields); ok { + headerData = pseudoHeader + if protocol == 0 { + protocol = sampledHeaderProtocolForPacket(evt, headerData) + } + if frameLength == 0 { + frameLength = uint32(len(headerData)) + } + if originalLength == 0 { + originalLength = uint32(len(headerData)) + } + } + } + if e.allowTruncate && e.maxHeaderBytes > 0 && len(headerData) > e.maxHeaderBytes { + headerData = append([]byte(nil), headerData[:e.maxHeaderBytes]...) + originalLength = uint32(len(headerData)) + } + if uint32(len(headerData)) != originalLength { + originalLength = uint32(len(headerData)) + } + return headerData, protocol, frameLength, originalLength +} + +func sampledHeaderProtocolForPacket(evt *event.Event, headerData []byte) uint32 { + if evt != nil && evt.Packet != nil && len(evt.Packet.Layers) > 0 { + switch evt.Packet.Layers[0].Kind { + case "ipv4": + return 11 + case "ipv6": + return 12 + default: + return 1 + } + } + if len(headerData) > 0 { + switch headerData[0] >> 4 { + case 4: + return 11 + case 6: + return 12 + } + } + return 1 +} + +func (e *SFlowEncoder) buildCounterSample(evt *event.Event) (sflow.CounterSample, error) { + fields := evt.Fields + if fields == nil { + if len(publicSFlowCounterRecords(evt)) == 0 && len(preservedSFlowCounterRecords(evt)) == 0 { + return sflow.CounterSample{}, fmt.Errorf("event fields are empty") + } + fields = map[string]any{} + } + format, sourceIDType := e.counterSampleFormat(fields) + records, err := e.counterRecords(evt, fields) + if err != nil { + return sflow.CounterSample{}, err + } + + return sflow.CounterSample{ + Header: sflow.SampleHeader{ + Format: format, + SampleSequenceNumber: e.sampleSequence(evt), + SourceIdType: sourceIDType, + SourceIdValue: sflowSourceID(evt), + }, + CounterRecordsCount: uint32(len(records)), + Records: records, + }, nil +} + +func (e *SFlowEncoder) counterRecords(evt *event.Event, fields map[string]any) ([]sflow.CounterRecord, error) { + if records := preservedSFlowCounterRecords(evt); len(records) > 0 { + return records, nil + } + publicRecords := publicSFlowCounterRecords(evt) + if len(publicRecords) > 0 && stringFieldOrZero(fields, "record_kind") == "" { + return publicRecords, nil + } + + switch stringFieldOrZero(fields, "record_kind") { + case "interface_counter": + records := []sflow.CounterRecord{{Data: sflow.IfCounters{ + IfIndex: uint32Field(fields, "if_index"), + IfType: uint32Field(fields, "if_type"), + IfSpeed: uint64Field(fields, "if_speed"), + IfDirection: uint32Field(fields, "if_direction"), + IfStatus: uint32Field(fields, "if_status"), + IfInOctets: uint64Field(fields, "if_in_octets"), + IfInUcastPkts: uint32Field(fields, "if_in_ucast_pkts"), + IfInMulticastPkts: uint32Field(fields, "if_in_multicast_pkts"), + IfInBroadcastPkts: uint32Field(fields, "if_in_broadcast_pkts"), + IfInDiscards: uint32Field(fields, "if_in_discards"), + IfInErrors: uint32Field(fields, "if_in_errors"), + IfInUnknownProtos: uint32Field(fields, "if_in_unknown_protos"), + IfOutOctets: uint64Field(fields, "if_out_octets"), + IfOutUcastPkts: uint32Field(fields, "if_out_ucast_pkts"), + IfOutMulticastPkts: uint32Field(fields, "if_out_multicast_pkts"), + IfOutBroadcastPkts: uint32Field(fields, "if_out_broadcast_pkts"), + IfOutDiscards: uint32Field(fields, "if_out_discards"), + IfOutErrors: uint32Field(fields, "if_out_errors"), + IfPromiscuousMode: uint32Field(fields, "if_promiscuous_mode"), + }}} + return append(records, publicRecords...), nil + case "ethernet_counter": + records := []sflow.CounterRecord{{Data: sflow.EthernetCounters{ + Dot3StatsAlignmentErrors: uint32Field(fields, "dot3_stats_alignment_errors"), + Dot3StatsFCSErrors: uint32Field(fields, "dot3_stats_fcs_errors"), + Dot3StatsSingleCollisionFrames: uint32Field(fields, "dot3_stats_single_collision_frames"), + Dot3StatsMultipleCollisionFrames: uint32Field(fields, "dot3_stats_multiple_collision_frames"), + Dot3StatsSQETestErrors: uint32Field(fields, "dot3_stats_sqe_test_errors"), + Dot3StatsDeferredTransmissions: uint32Field(fields, "dot3_stats_deferred_transmissions"), + Dot3StatsLateCollisions: uint32Field(fields, "dot3_stats_late_collisions"), + Dot3StatsExcessiveCollisions: uint32Field(fields, "dot3_stats_excessive_collisions"), + Dot3StatsInternalMacTransmitErrors: uint32Field(fields, "dot3_stats_internal_mac_transmit_errors"), + Dot3StatsCarrierSenseErrors: uint32Field(fields, "dot3_stats_carrier_sense_errors"), + Dot3StatsFrameTooLongs: uint32Field(fields, "dot3_stats_frame_too_longs"), + Dot3StatsInternalMacReceiveErrors: uint32Field(fields, "dot3_stats_internal_mac_receive_errors"), + Dot3StatsSymbolErrors: uint32Field(fields, "dot3_stats_symbol_errors"), + }}} + return append(records, publicRecords...), nil + default: + return nil, fmt.Errorf("unsupported sflow counter record_kind %q", stringFieldOrZero(fields, "record_kind")) + } +} + +func preservedSFlowCounterRecords(evt *event.Event) []sflow.CounterRecord { + if evt == nil || evt.Internal == nil { + return nil + } + records, _ := evt.Internal[sflowCounterRecordsInternalKey].([]sflow.CounterRecord) + if len(records) == 0 { + return nil + } + out := make([]sflow.CounterRecord, len(records)) + copy(out, records) + return out +} + +func publicSFlowCounterRecords(evt *event.Event) []sflow.CounterRecord { + if evt == nil || evt.SFlow == nil || len(evt.SFlow.Samples) == 0 { + return nil + } + rawRecords := evt.SFlow.Samples[0].RawCounterRecords + records := make([]sflow.CounterRecord, 0, len(rawRecords)) + for _, raw := range rawRecords { + dataFormat, ok := publicSFlowDataFormat(raw.Enterprise, raw.Format, raw.DataFormat) + if !ok { + continue + } + records = append(records, sflow.CounterRecord{ + Header: sflow.RecordHeader{DataFormat: dataFormat}, + Data: sflow.RawRecord{Data: append([]byte(nil), raw.Data...)}, + }) + } + return records +} + +// counterSampleFormat chooses the sFlow counter record format and source index +// from the canonical event fields. +func (e *SFlowEncoder) counterSampleFormat(fields map[string]any) (uint32, uint32) { + switch stringFieldOrZero(fields, "counter_format") { + case "expanded": + return sflow.SAMPLE_FORMAT_EXPANDED_COUNTER, uint32Field(fields, "source_id_type") + case "standard": + return sflow.SAMPLE_FORMAT_COUNTER, 0 + } + + switch e.cfg.CounterFormat { + case "expanded": + return sflow.SAMPLE_FORMAT_EXPANDED_COUNTER, uint32Field(fields, "source_id_type") + default: + return sflow.SAMPLE_FORMAT_COUNTER, 0 + } +} + +type sflowPacketTopLevel struct { + AgentIP string + SubAgentID uint32 + SequenceNumber uint32 + Uptime uint32 +} + +// packetTopLevel extracts the per-datagram sFlow attributes that must match +// across every sample batched into one packet. +func (e *SFlowEncoder) packetTopLevel(evt *event.Event) (sflowPacketTopLevel, error) { + top := sflowPacketTopLevel{ + AgentIP: e.sflowAgentIP(evt), + SubAgentID: sflowSubAgentID(evt.SFlow, evt.Fields), + Uptime: sflowUptime(evt.SFlow), + } + if e.cfg.UseMetadataSequenceNumber { + top.SequenceNumber = sflowSequenceNumber(evt.SFlow) + } + if top.AgentIP == "" { + return sflowPacketTopLevel{}, fmt.Errorf("missing field \"agent_ip\"") + } + return top, nil +} + +// sflowAgentIP resolves the emitted agent IP from event metadata first, then +// the encoder config fallback. +func (e *SFlowEncoder) sflowAgentIP(evt *event.Event) string { + if agentIP := eventAgentIP(evt); agentIP != "" { + return agentIP + } + if e.cfg.AgentIP != "" { + return e.cfg.AgentIP + } + return "127.0.0.1" +} + +// compatibleTopLevel ensures two events can coexist in the same sFlow datagram. +func (e *SFlowEncoder) compatibleTopLevel(left, right *event.Event) bool { + leftTop, err := e.packetTopLevel(left) + if err != nil { + return false + } + rightTop, err := e.packetTopLevel(right) + if err != nil { + return false + } + if !batchOverEnabled(e.cfg.BatchOver.AgentIP) && leftTop.AgentIP != rightTop.AgentIP { + return false + } + if !batchOverEnabled(e.cfg.BatchOver.SubAgentID) && leftTop.SubAgentID != rightTop.SubAgentID { + return false + } + if !batchOverEnabled(e.cfg.BatchOver.SequenceNumber) && leftTop.SequenceNumber != rightTop.SequenceNumber { + return false + } + if !batchOverEnabled(e.cfg.BatchOver.Uptime) && leftTop.Uptime != rightTop.Uptime { + return false + } + return true +} + +// sampleSequence uses an encoder-local counter for outgoing sample ordering. +func (e *SFlowEncoder) sampleSequence(evt *event.Event) uint32 { + return e.sampleSeq.Add(1) +} + +// isSFlowCounterEvent identifies events that should become counter samples +// instead of raw-header flow samples. +func isSFlowCounterEvent(evt *event.Event) bool { + if evt == nil { + return false + } + if len(preservedSFlowCounterRecords(evt)) > 0 { + return true + } + if len(publicSFlowCounterRecords(evt)) > 0 { + return true + } + switch stringFieldOrZero(evt.Fields, "record_kind") { + case "interface_counter", "ethernet_counter": + return true + default: + return false + } +} + +// truncateLastSampleToFit rewrites only the newest sample when that is enough to +// keep the current packet under the datagram limit. +func (e *SFlowEncoder) truncateLastSampleToFit(packet *sflow.Packet) (sflow.FlowSample, bool, error) { + lastIdx := len(packet.Samples) - 1 + if lastIdx < 0 { + return sflow.FlowSample{}, false, nil + } + sample, ok := packet.Samples[lastIdx].(sflow.FlowSample) + if !ok || len(sample.Records) == 0 { + return sflow.FlowSample{}, false, nil + } + header, ok := sample.Records[0].Data.(sflow.SampledHeader) + if !ok || len(header.HeaderData) == 0 { + return sflow.FlowSample{}, false, nil + } + + original := append([]byte(nil), header.HeaderData...) + best := sample + fit := false + low, high := 0, len(original) + for low <= high { + mid := (low + high) / 2 + candidate := sample + candidateHeader := header + candidateHeader.HeaderData = append([]byte(nil), original[:mid]...) + candidateHeader.OriginalLength = uint32(len(candidateHeader.HeaderData)) + candidate.Records = append([]sflow.FlowRecord(nil), sample.Records...) + candidate.Records[0] = sflow.FlowRecord{ + Header: sample.Records[0].Header, + Data: candidateHeader, + } + packet.Samples[lastIdx] = candidate + data, err := sflow.EncodeMessage(packet) + if err != nil { + packet.Samples[lastIdx] = sample + return sflow.FlowSample{}, false, fmt.Errorf("encode truncated sflow packet: %w", err) + } + if len(data) <= e.maxDatagramBytes { + best = candidate + fit = true + low = mid + 1 + continue + } + high = mid - 1 + } + packet.Samples[lastIdx] = sample + if !fit { + return sflow.FlowSample{}, false, nil + } + return best, true, nil +} + +// sflowSubAgentID prefers explicit event metadata over generic fields. +func sflowSubAgentID(sf *event.SFlowMetadata, fields map[string]any) uint32 { + if sf != nil && sf.SubAgentID != 0 { + return sf.SubAgentID + } + return uint32Field(fields, "sub_agent_id") +} + +// sflowSequenceNumber prefers explicit event metadata over encoder-local sequencing. +func sflowSequenceNumber(sf *event.SFlowMetadata) uint32 { + if sf == nil { + return 0 + } + return sf.SequenceNumber +} + +// sflowUptime prefers explicit event metadata over encoder-derived uptime. +func sflowUptime(sf *event.SFlowMetadata) uint32 { + if sf == nil { + return 0 + } + return sf.Uptime +} + +// sflowSourceID comes from decoded sFlow metadata or source metadata. +func sflowSourceID(evt *event.Event) uint32 { + return eventSourceID(evt) +} + +// sflowSamplingRate comes from decoded sFlow metadata or source metadata. +func sflowSamplingRate(evt *event.Event) uint32 { + return eventSamplingRate(evt) +} + +// sflowSamplePool comes from decoded sFlow metadata or source metadata. +func sflowSamplePool(evt *event.Event) uint32 { + return eventSamplePool(evt) +} + +// sflowDrops comes from decoded sFlow metadata or source metadata. +func sflowDrops(evt *event.Event) uint32 { + return eventDrops(evt) +} + +// batchOverEnabled defaults unset batch-over toggles to true. +func batchOverEnabled(v *bool) bool { + return v == nil || *v +} + +// logOversizedSample keeps oversize-drop behavior visible without failing the entire pipeline. +func logOversizedSample(err error) { + var sizeErr *sflowSampleTooLargeError + if errors.As(err, &sizeErr) { + slog.Warn( + "dropping oversized sflow sample", + slog.Int("max_datagram_bytes", sizeErr.MaxDatagramBytes), + slog.Int("current_size", sizeErr.CurrentSize), + ) + return + } + slog.Warn("dropping oversized sflow sample", slog.String("error", err.Error())) +} diff --git a/pkg/reflow/encode/tflow.go b/pkg/reflow/encode/tflow.go new file mode 100644 index 00000000..790379ea --- /dev/null +++ b/pkg/reflow/encode/tflow.go @@ -0,0 +1,2998 @@ +package encode + +import ( + "encoding/json" + "fmt" + "net" + "net/netip" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/netsampler/goflow2/v3/decoders/netflow" + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type IPFIXEncoder struct { + seq atomic.Uint32 + cfg config.EncoderConfig + dataSchemas map[string]templatedSchemaState + optionsSchemas map[string]optionsSchemaState + fallbackPlans map[string]fallbackTemplatePlan + sourceOptions map[string]sourceOptionsState + lastTemplateRun time.Time + lastOptionsRun time.Time + events []*event.Event + estimatedBytes int +} + +type NFv9Encoder struct { + seq atomic.Uint32 + cfg config.EncoderConfig + dataSchemas map[string]templatedSchemaState + optionsSchemas map[string]optionsSchemaState + fallbackPlans map[string]fallbackTemplatePlan + sourceOptions map[string]sourceOptionsState + lastTemplateRun time.Time + lastOptionsRun time.Time + events []*event.Event + estimatedBytes int +} + +type templatedSchemaState struct { + stream string + fieldNames []string + fields []event.SchemaField + baseTemplateID uint16 + template netflow.TemplateRecord +} + +type optionsSchemaState struct { + stream string + templateID uint16 + scopes []event.SchemaField + options []event.SchemaField + ipfixTemplate netflow.IPFIXOptionsTemplateRecord + nfv9Template netflow.NFv9OptionsTemplateRecord +} + +type sourceOptionsState struct { + stream string + agentIP string + sourceID uint32 + observationDomainID uint32 + samplingRate uint32 + samplePool uint32 + drops uint32 + inputIf uint32 + outputIf uint32 + templateID uint16 +} + +type fallbackTemplatePlan struct { + template netflow.TemplateRecord + names []string + defs []config.IPFIXFieldDefinition + families []string +} + +type ipfixBatchRecord struct { + key string + recordKind tflowBatchRecordKind + exportTime uint32 + observationDomainID uint32 + templateID uint16 + template *netflow.TemplateRecord + data netflow.DataRecord + ipfixOptionsTemplate *netflow.IPFIXOptionsTemplateRecord + nfv9OptionsTemplate *netflow.NFv9OptionsTemplateRecord + options netflow.OptionsDataRecord +} + +type ipfixBatchSet struct { + key string + recordKind tflowBatchRecordKind + templateID uint16 + template *netflow.TemplateRecord + ipfixOptionsTemplate *netflow.IPFIXOptionsTemplateRecord + nfv9OptionsTemplate *netflow.NFv9OptionsTemplateRecord + records []netflow.DataRecord + optionsRecords []netflow.OptionsDataRecord +} + +type nfv9BatchRecord struct { + key string + recordKind tflowBatchRecordKind + unixSeconds uint32 + systemUptime uint32 + sourceID uint32 + templateID uint16 + template *netflow.TemplateRecord + data netflow.DataRecord + optionsTemplate *netflow.NFv9OptionsTemplateRecord + options netflow.OptionsDataRecord +} + +type nfv9BatchTiming struct { + packetExportMS int64 + baseMS int64 + unixSeconds uint32 + systemUptime uint32 +} + +type templatedEncodingContext struct { + netflowV9 bool + firstSwitched uint32 + lastSwitched uint32 +} + +type tflowBatchRecordKind int + +const ( + tflowBatchData tflowBatchRecordKind = iota + tflowBatchOptions +) + +const tflowRecordTypeField = "tflow_record_type" + +func NewIPFIXEncoder(cfg config.EncoderConfig) *IPFIXEncoder { + return &IPFIXEncoder{ + cfg: cfg, + dataSchemas: make(map[string]templatedSchemaState), + optionsSchemas: make(map[string]optionsSchemaState), + fallbackPlans: make(map[string]fallbackTemplatePlan), + sourceOptions: make(map[string]sourceOptionsState), + } +} + +func (e *IPFIXEncoder) Encode(evt *event.Event) ([][]byte, error) { + if evt != nil && evt.Kind == "control" { + payloads, err := e.flushDataPackets("control") + if err != nil { + return nil, err + } + controlPayloads, err := e.handleControl(evt) + if err != nil { + return nil, err + } + return append(payloads, controlPayloads...), nil + } + if !e.cfg.Batch.IsEnabled() { + packet, err := e.buildPacket(evt) + if err != nil { + return nil, err + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode ipfix packet: %w", err) + } + return [][]byte{data}, nil + } + e.appendEvent(evt) + if trigger := e.batchFlushTrigger(); trigger != "" { + return e.flushDataPackets(trigger) + } + return nil, nil +} + +func (e *IPFIXEncoder) Flush() ([][]byte, error) { + payloads, err := e.flushDataPackets("flush") + if err != nil { + return nil, err + } + controlPayloads, err := e.flushControlPackets(time.Now().UTC()) + if err != nil { + return nil, err + } + return append(payloads, controlPayloads...), nil +} + +func (e *IPFIXEncoder) appendEvent(evt *event.Event) { + e.events = append(e.events, evt) + e.estimatedBytes += e.estimatedEventRenderedSize(evt) +} + +func (e *IPFIXEncoder) batchFlushTrigger() string { + if len(e.events) == 0 { + return "" + } + if e.cfg.Batch.MaxRecords > 0 && len(e.events) >= e.cfg.Batch.MaxRecords { + return "max_records" + } + if e.cfg.Batch.MaxBytes > 0 && e.estimatedBytes >= e.cfg.Batch.MaxBytes { + return "max_bytes" + } + return "" +} + +func (e *IPFIXEncoder) flushDataPackets(trigger string) ([][]byte, error) { + if len(e.events) == 0 { + return nil, nil + } + + pending := e.events + + var payloads [][]byte + for len(pending) > 0 { + packet, accepted, err := e.buildBatchedPacket(pending) + if err != nil { + e.events = pending + e.estimatedBytes = e.estimatedEventsRenderedSize(pending) + return payloads, err + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + e.events = pending + e.estimatedBytes = e.estimatedEventsRenderedSize(pending) + return payloads, fmt.Errorf("encode ipfix packet: %w", err) + } + acceptedBytes := renderedFlowSetsDataSize(packet.FlowSets) + payloads = append(payloads, data) + pending = pending[accepted:] + e.events = pending + e.estimatedBytes -= acceptedBytes + if e.estimatedBytes < 0 { + e.estimatedBytes = e.estimatedEventsRenderedSize(pending) + } + if !e.shouldContinueDataFlush(trigger) { + return payloads, nil + } + } + e.events = nil + e.estimatedBytes = 0 + return payloads, nil +} + +func (e *IPFIXEncoder) shouldContinueDataFlush(trigger string) bool { + if len(e.events) == 0 { + return false + } + switch trigger { + case "max_records", "max_bytes": + return e.batchFlushTrigger() != "" + default: + return true + } +} + +func (e *IPFIXEncoder) estimatedEventsRenderedSize(events []*event.Event) int { + var total int + for _, evt := range events { + total += e.estimatedEventRenderedSize(evt) + } + return total +} + +func (e *IPFIXEncoder) estimatedEventRenderedSize(evt *event.Event) int { + record, err := e.ipfixBatchRecord(evt) + if err != nil { + return estimatedEventSize(evt) + } + if record.recordKind == tflowBatchOptions { + return renderedOptionsDataRecordsSize([]netflow.OptionsDataRecord{record.options}) + } + return renderedDataRecordSize(record.data) +} + +func (e *NFv9Encoder) estimatedEventsRenderedSize(events []*event.Event) int { + var total int + for _, evt := range events { + total += e.estimatedEventRenderedSize(evt) + } + return total +} + +func (e *NFv9Encoder) estimatedEventRenderedSize(evt *event.Event) int { + timing := nfv9BatchTimingFor([]*event.Event{evt}) + record, err := e.nfv9BatchRecord(evt, timing) + if err != nil { + return estimatedEventSize(evt) + } + if record.recordKind == tflowBatchOptions { + return renderedOptionsDataRecordsSize([]netflow.OptionsDataRecord{record.options}) + } + return renderedDataRecordSize(record.data) +} + +func renderedDataRecordSize(record netflow.DataRecord) int { + var total int + for _, field := range record.Values { + total += renderedDataFieldSize(field) + } + return total +} + +func renderedDataFieldSize(field netflow.DataField) int { + value, ok := field.Value.([]byte) + if !ok { + return 0 + } + if field.Length == 0xffff { + if len(value) < 255 { + return 1 + len(value) + } + return 3 + len(value) + } + return len(value) +} + +func renderedFlowSetsDataSize(flowSets []interface{}) int { + var total int + for _, flowSet := range flowSets { + switch fs := flowSet.(type) { + case netflow.DataFlowSet: + total += renderedDataRecordsSize(fs.Records) + case *netflow.DataFlowSet: + if fs != nil { + total += renderedDataRecordsSize(fs.Records) + } + case netflow.OptionsDataFlowSet: + total += renderedOptionsDataRecordsSize(fs.Records) + case *netflow.OptionsDataFlowSet: + if fs != nil { + total += renderedOptionsDataRecordsSize(fs.Records) + } + } + } + return total +} + +func renderedDataRecordsSize(records []netflow.DataRecord) int { + var total int + for _, record := range records { + total += renderedDataRecordSize(record) + } + return total +} + +func renderedOptionsDataRecordsSize(records []netflow.OptionsDataRecord) int { + var total int + for _, record := range records { + for _, field := range record.ScopesValues { + total += renderedDataFieldSize(field) + } + for _, field := range record.OptionsValues { + total += renderedDataFieldSize(field) + } + } + return total +} + +func (e *IPFIXEncoder) buildBatchedPacket(events []*event.Event) (*netflow.IPFIXPacket, int, error) { + if len(events) == 0 { + return nil, 0, fmt.Errorf("empty ipfix packet batch") + } + first, err := e.ipfixBatchRecord(events[0]) + if err != nil { + return nil, 0, err + } + sets := []ipfixBatchSet{{ + key: first.key, + recordKind: first.recordKind, + templateID: first.templateID, + template: first.template, + ipfixOptionsTemplate: first.ipfixOptionsTemplate, + nfv9OptionsTemplate: first.nfv9OptionsTemplate, + records: dataRecordsForBatchRecord(first), + optionsRecords: optionsRecordsForBatchRecord(first), + }} + packet := e.ipfixBatchPacket(first, sets) + accepted := 1 + + for _, evt := range events[1:] { + next, err := e.ipfixBatchRecord(evt) + if err != nil { + return nil, accepted, err + } + if next.observationDomainID != first.observationDomainID { + break + } + nextSets, ok := appendIPFIXBatchRecord(sets, next) + if !ok { + break + } + packet = e.ipfixBatchPacket(first, nextSets) + if e.cfg.MaxDatagramBytes > 0 { + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, accepted, fmt.Errorf("encode ipfix packet: %w", err) + } + if len(data) > e.cfg.MaxDatagramBytes { + packet = e.ipfixBatchPacket(first, sets) + break + } + } + sets = nextSets + accepted++ + } + + e.advanceIPFIXSequence(packet) + return packet, accepted, nil +} + +func appendIPFIXBatchRecord(sets []ipfixBatchSet, record ipfixBatchRecord) ([]ipfixBatchSet, bool) { + next := make([]ipfixBatchSet, len(sets)) + copy(next, sets) + for i := range next { + if next[i].templateID != record.templateID || next[i].recordKind != record.recordKind { + continue + } + if next[i].key != record.key { + return nil, false + } + if record.recordKind == tflowBatchOptions { + next[i].optionsRecords = append(append([]netflow.OptionsDataRecord(nil), next[i].optionsRecords...), record.options) + } else { + next[i].records = append(append([]netflow.DataRecord(nil), next[i].records...), record.data) + } + return next, true + } + next = append(next, ipfixBatchSet{ + key: record.key, + recordKind: record.recordKind, + templateID: record.templateID, + template: record.template, + ipfixOptionsTemplate: record.ipfixOptionsTemplate, + nfv9OptionsTemplate: record.nfv9OptionsTemplate, + records: dataRecordsForBatchRecord(record), + optionsRecords: optionsRecordsForBatchRecord(record), + }) + return next, true +} + +func dataRecordsForBatchRecord(record ipfixBatchRecord) []netflow.DataRecord { + if record.recordKind == tflowBatchOptions { + return nil + } + return []netflow.DataRecord{record.data} +} + +func optionsRecordsForBatchRecord(record ipfixBatchRecord) []netflow.OptionsDataRecord { + if record.recordKind != tflowBatchOptions { + return nil + } + return []netflow.OptionsDataRecord{record.options} +} + +func dataRecordsForNFv9BatchRecord(record nfv9BatchRecord) []netflow.DataRecord { + if record.recordKind == tflowBatchOptions { + return nil + } + return []netflow.DataRecord{record.data} +} + +func optionsRecordsForNFv9BatchRecord(record nfv9BatchRecord) []netflow.OptionsDataRecord { + if record.recordKind != tflowBatchOptions { + return nil + } + return []netflow.OptionsDataRecord{record.options} +} + +func (e *IPFIXEncoder) ipfixBatchPacket(first ipfixBatchRecord, sets []ipfixBatchSet) *netflow.IPFIXPacket { + flowSets := make([]interface{}, 0, len(sets)+2) + templates := make([]netflow.TemplateRecord, 0, len(sets)) + optionsTemplates := make([]netflow.IPFIXOptionsTemplateRecord, 0, len(sets)) + seenTemplates := make(map[uint16]struct{}, len(sets)) + seenOptionsTemplates := make(map[uint16]struct{}, len(sets)) + for _, set := range sets { + if set.recordKind == tflowBatchOptions { + if set.ipfixOptionsTemplate == nil { + continue + } + if _, ok := seenOptionsTemplates[set.templateID]; ok { + continue + } + seenOptionsTemplates[set.templateID] = struct{}{} + optionsTemplates = append(optionsTemplates, *set.ipfixOptionsTemplate) + continue + } + if set.template != nil { + if _, ok := seenTemplates[set.templateID]; ok { + continue + } + seenTemplates[set.templateID] = struct{}{} + templates = append(templates, *set.template) + } + } + if len(templates) > 0 { + flowSets = append(flowSets, netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 2}, + Records: templates, + }) + } + if len(optionsTemplates) > 0 { + flowSets = append(flowSets, netflow.IPFIXOptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 3}, + Records: optionsTemplates, + }) + } + for _, set := range sets { + if set.recordKind == tflowBatchOptions { + flowSets = append(flowSets, netflow.OptionsDataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: set.templateID}, + Records: set.optionsRecords, + }) + } else { + flowSets = append(flowSets, netflow.DataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: set.templateID}, + Records: set.records, + }) + } + } + return &netflow.IPFIXPacket{ + Version: 10, + ExportTime: first.exportTime, + SequenceNumber: e.seq.Load(), + ObservationDomainId: first.observationDomainID, + FlowSets: flowSets, + } +} + +func (e *IPFIXEncoder) advanceIPFIXSequence(packet *netflow.IPFIXPacket) { + if packet == nil { + return + } + e.seq.Add(ipfixSequenceIncrement(packet.FlowSets)) +} + +func ipfixSequenceIncrement(flowSets []interface{}) uint32 { + var count uint32 + for _, flowSet := range flowSets { + switch fs := flowSet.(type) { + case netflow.DataFlowSet: + count += uint32(len(fs.Records)) + case *netflow.DataFlowSet: + if fs != nil { + count += uint32(len(fs.Records)) + } + case netflow.OptionsDataFlowSet: + count += uint32(len(fs.Records)) + case *netflow.OptionsDataFlowSet: + if fs != nil { + count += uint32(len(fs.Records)) + } + } + } + return count +} + +func (e *IPFIXEncoder) ipfixBatchRecord(evt *event.Event) (ipfixBatchRecord, error) { + if evt == nil { + return ipfixBatchRecord{}, fmt.Errorf("nil event") + } + if evt.Fields == nil { + return ipfixBatchRecord{}, fmt.Errorf("event fields are empty") + } + fieldMap := eventFieldsWithMetadata(evt) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + templateID := uint16Field(fieldMap, "template_id") + if templateID == 0 { + templateID = 256 + } + exportTime := uint32(evt.ReceivedAt.Unix()) + if evt.ReceivedAt.IsZero() { + exportTime = uint32(time.Now().Unix()) + } + obsDomainID := e.observationDomainID() + + stream := eventStream(evt, "flow_data") + if schema, ok := e.optionsSchemas[stream]; ok { + fieldMap = eventFieldsWithMetadataForSchema(evt, optionsSchemaFields(schema)) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + optionsRecord, err := buildOptionsDataRecordFromSchemaFields(e.cfg.TemplatedFlow.Data, fieldMap, schema.scopes, schema.options, templatedEncodingContext{}) + if err != nil { + return ipfixBatchRecord{}, err + } + return ipfixBatchRecord{ + key: fmt.Sprintf("options-schema:%s:%d", stream, schema.templateID), + recordKind: tflowBatchOptions, + exportTime: exportTime, + observationDomainID: obsDomainID, + templateID: schema.templateID, + options: optionsRecord, + }, nil + } + if schema, ok := e.dataSchemas[stream]; ok { + fieldMap = eventFieldsWithMetadataForSchema(evt, schema.fields) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + templateRecord := schema.template + dataRecord, err := buildTemplatedValuesFromSchemaFields(e.cfg.TemplatedFlow.Data, fieldMap, schema.fields, templatedEncodingContext{}) + if err != nil { + return ipfixBatchRecord{}, err + } + return ipfixBatchRecord{ + key: fmt.Sprintf("schema:%s:%d", stream, templateRecord.TemplateId), + exportTime: exportTime, + observationDomainID: obsDomainID, + templateID: templateRecord.TemplateId, + data: dataRecord, + }, nil + } + + plan, dataRecord, err := e.fallbackDataRecord(fieldMap, templateID) + if err != nil { + return ipfixBatchRecord{}, err + } + key, err := fallbackPlanKey(e.cfg.TemplatedFlow.Data, fieldMap, templateID, false) + if err != nil { + return ipfixBatchRecord{}, err + } + return ipfixBatchRecord{ + key: key, + exportTime: exportTime, + observationDomainID: obsDomainID, + templateID: plan.template.TemplateId, + template: &plan.template, + data: dataRecord, + }, nil +} + +func nfv9BatchTimingFor(events []*event.Event) nfv9BatchTiming { + exportMS := time.Now().UnixMilli() + if len(events) > 0 && events[0] != nil { + exportMS = exportUnixMilliseconds(events[0].ReceivedAt, events[0].Fields) + } + unixSeconds := uint32((exportMS + 999) / 1000) + packetExportMS := int64(unixSeconds) * 1000 + baseMS := packetExportMS + for _, evt := range events { + if evt == nil { + continue + } + for _, key := range []string{"start_time_unix", "end_time_unix"} { + ms := int64Field(evt.Fields, key) + if ms > 0 && ms < baseMS { + baseMS = ms + } + } + } + return nfv9BatchTiming{ + packetExportMS: packetExportMS, + baseMS: baseMS, + unixSeconds: unixSeconds, + systemUptime: uint32(packetExportMS - baseMS), + } +} + +func (e *NFv9Encoder) nfv9BatchRecord(evt *event.Event, timing nfv9BatchTiming) (nfv9BatchRecord, error) { + if evt == nil { + return nfv9BatchRecord{}, fmt.Errorf("nil event") + } + if evt.Fields == nil { + return nfv9BatchRecord{}, fmt.Errorf("event fields are empty") + } + fieldMap := eventFieldsWithMetadata(evt) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + templateID := uint16Field(fieldMap, "template_id") + if templateID == 0 { + templateID = 256 + } + startMS := int64Field(evt.Fields, "start_time_unix") + if startMS <= 0 { + startMS = timing.packetExportMS + } + endMS := int64Field(evt.Fields, "end_time_unix") + if endMS <= 0 { + endMS = timing.packetExportMS + } + encodingCtx := templatedEncodingContext{ + netflowV9: true, + firstSwitched: uint32(startMS - timing.baseMS), + lastSwitched: uint32(endMS - timing.baseMS), + } + sourceID := e.sourceID() + + stream := eventStream(evt, "flow_data") + if schema, ok := e.optionsSchemas[stream]; ok { + fieldMap = eventFieldsWithMetadataForSchema(evt, optionsSchemaFields(schema)) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + optionsRecord, err := buildOptionsDataRecordFromSchemaFields(e.cfg.TemplatedFlow.Data, fieldMap, schema.scopes, schema.options, encodingCtx) + if err != nil { + return nfv9BatchRecord{}, err + } + return nfv9BatchRecord{ + key: fmt.Sprintf("options-schema:%s:%d", stream, schema.templateID), + recordKind: tflowBatchOptions, + unixSeconds: timing.unixSeconds, + systemUptime: timing.systemUptime, + sourceID: sourceID, + templateID: schema.templateID, + options: optionsRecord, + }, nil + } + if schema, ok := e.dataSchemas[stream]; ok { + fieldMap = eventFieldsWithMetadataForSchema(evt, schema.fields) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + templateRecord := schema.template + dataRecord, err := buildTemplatedValuesFromSchemaFields(e.cfg.TemplatedFlow.Data, fieldMap, schema.fields, encodingCtx) + if err != nil { + return nfv9BatchRecord{}, err + } + return nfv9BatchRecord{ + key: fmt.Sprintf("schema:%s:%d", stream, templateRecord.TemplateId), + unixSeconds: timing.unixSeconds, + systemUptime: timing.systemUptime, + sourceID: sourceID, + templateID: templateRecord.TemplateId, + data: dataRecord, + }, nil + } + + plan, dataRecord, err := e.fallbackDataRecord(fieldMap, templateID, encodingCtx) + if err != nil { + return nfv9BatchRecord{}, err + } + key, err := fallbackPlanKey(e.cfg.TemplatedFlow.Data, fieldMap, templateID, true) + if err != nil { + return nfv9BatchRecord{}, err + } + return nfv9BatchRecord{ + key: key, + unixSeconds: timing.unixSeconds, + systemUptime: timing.systemUptime, + sourceID: sourceID, + templateID: plan.template.TemplateId, + template: &plan.template, + data: dataRecord, + }, nil +} + +func (e *NFv9Encoder) nfv9BatchPacket(first nfv9BatchRecord, sets []ipfixBatchSet) *netflow.NFv9Packet { + flowSets := make([]interface{}, 0, len(sets)+2) + templates := make([]netflow.TemplateRecord, 0, len(sets)) + optionsTemplates := make([]netflow.NFv9OptionsTemplateRecord, 0, len(sets)) + seenTemplates := make(map[uint16]struct{}, len(sets)) + seenOptionsTemplates := make(map[uint16]struct{}, len(sets)) + for _, set := range sets { + if set.recordKind == tflowBatchOptions { + if set.nfv9OptionsTemplate == nil { + continue + } + if _, ok := seenOptionsTemplates[set.templateID]; ok { + continue + } + seenOptionsTemplates[set.templateID] = struct{}{} + optionsTemplates = append(optionsTemplates, *set.nfv9OptionsTemplate) + continue + } + if set.template != nil { + if _, ok := seenTemplates[set.templateID]; ok { + continue + } + seenTemplates[set.templateID] = struct{}{} + templates = append(templates, *set.template) + } + } + if len(templates) > 0 { + flowSets = append(flowSets, netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 0}, + Records: templates, + }) + } + if len(optionsTemplates) > 0 { + flowSets = append(flowSets, netflow.NFv9OptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 1}, + Records: optionsTemplates, + }) + } + for _, set := range sets { + if set.recordKind == tflowBatchOptions { + flowSets = append(flowSets, netflow.OptionsDataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: set.templateID}, + Records: set.optionsRecords, + }) + } else { + flowSets = append(flowSets, netflow.DataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: set.templateID}, + Records: set.records, + }) + } + } + return &netflow.NFv9Packet{ + Version: 9, + Count: uint16(len(flowSets)), + SystemUptime: first.systemUptime, + UnixSeconds: first.unixSeconds, + SequenceNumber: e.seq.Load(), + SourceId: first.sourceID, + FlowSets: flowSets, + } +} + +func NewNFv9Encoder(cfg config.EncoderConfig) *NFv9Encoder { + return &NFv9Encoder{ + cfg: cfg, + dataSchemas: make(map[string]templatedSchemaState), + optionsSchemas: make(map[string]optionsSchemaState), + fallbackPlans: make(map[string]fallbackTemplatePlan), + sourceOptions: make(map[string]sourceOptionsState), + } +} + +func (e *NFv9Encoder) Encode(evt *event.Event) ([][]byte, error) { + if evt != nil && evt.Kind == "control" { + payloads, err := e.flushDataPackets("control") + if err != nil { + return nil, err + } + controlPayloads, err := e.handleControl(evt) + if err != nil { + return nil, err + } + return append(payloads, controlPayloads...), nil + } + if !e.cfg.Batch.IsEnabled() { + packet, err := e.buildPacket(evt) + if err != nil { + return nil, err + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode netflow v9 packet: %w", err) + } + e.advanceSequence(packet) + return [][]byte{data}, nil + } + e.appendEvent(evt) + if trigger := e.batchFlushTrigger(); trigger != "" { + return e.flushDataPackets(trigger) + } + return nil, nil +} + +func (e *NFv9Encoder) Flush() ([][]byte, error) { + payloads, err := e.flushDataPackets("flush") + if err != nil { + return nil, err + } + controlPayloads, err := e.flushControlPackets(time.Now().UTC()) + if err != nil { + return nil, err + } + return append(payloads, controlPayloads...), nil +} + +func (e *NFv9Encoder) appendEvent(evt *event.Event) { + e.events = append(e.events, evt) + e.estimatedBytes += e.estimatedEventRenderedSize(evt) +} + +func (e *NFv9Encoder) batchFlushTrigger() string { + if len(e.events) == 0 { + return "" + } + if e.cfg.Batch.MaxRecords > 0 && len(e.events) >= e.cfg.Batch.MaxRecords { + return "max_records" + } + if e.cfg.Batch.MaxBytes > 0 && e.estimatedBytes >= e.cfg.Batch.MaxBytes { + return "max_bytes" + } + return "" +} + +func (e *NFv9Encoder) flushDataPackets(trigger string) ([][]byte, error) { + if len(e.events) == 0 { + return nil, nil + } + + pending := e.events + + var payloads [][]byte + for len(pending) > 0 { + packet, accepted, err := e.buildBatchedPacket(pending) + if err != nil { + e.events = pending + e.estimatedBytes = e.estimatedEventsRenderedSize(pending) + return payloads, err + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + e.events = pending + e.estimatedBytes = e.estimatedEventsRenderedSize(pending) + return payloads, fmt.Errorf("encode netflow v9 packet: %w", err) + } + acceptedBytes := renderedFlowSetsDataSize(packet.FlowSets) + payloads = append(payloads, data) + pending = pending[accepted:] + e.events = pending + e.estimatedBytes -= acceptedBytes + if e.estimatedBytes < 0 { + e.estimatedBytes = e.estimatedEventsRenderedSize(pending) + } + if !e.shouldContinueDataFlush(trigger) { + return payloads, nil + } + } + e.events = nil + e.estimatedBytes = 0 + return payloads, nil +} + +func (e *NFv9Encoder) shouldContinueDataFlush(trigger string) bool { + if len(e.events) == 0 { + return false + } + switch trigger { + case "max_records", "max_bytes": + return e.batchFlushTrigger() != "" + default: + return true + } +} + +func (e *NFv9Encoder) buildBatchedPacket(events []*event.Event) (*netflow.NFv9Packet, int, error) { + if len(events) == 0 { + return nil, 0, fmt.Errorf("empty netflow v9 packet batch") + } + candidate := []*event.Event{events[0]} + packet, ok, err := e.nfv9BatchPacketForEvents(candidate) + if err != nil { + return nil, 0, err + } + if !ok { + return nil, 0, fmt.Errorf("empty netflow v9 packet batch") + } + accepted := 1 + + for _, evt := range events[1:] { + nextCandidate := append(append([]*event.Event(nil), candidate...), evt) + nextPacket, ok, err := e.nfv9BatchPacketForEvents(nextCandidate) + if err != nil { + return nil, accepted, err + } + if !ok { + break + } + if e.cfg.MaxDatagramBytes > 0 { + data, err := netflow.EncodeMessage(nextPacket) + if err != nil { + return nil, accepted, fmt.Errorf("encode netflow v9 packet: %w", err) + } + if len(data) > e.cfg.MaxDatagramBytes { + break + } + } + candidate = nextCandidate + packet = nextPacket + accepted++ + } + + e.advanceSequence(packet) + return packet, accepted, nil +} + +func (e *NFv9Encoder) nfv9BatchPacketForEvents(events []*event.Event) (*netflow.NFv9Packet, bool, error) { + if len(events) == 0 { + return nil, false, nil + } + timing := nfv9BatchTimingFor(events) + first, err := e.nfv9BatchRecord(events[0], timing) + if err != nil { + return nil, false, err + } + sets := []ipfixBatchSet{{ + key: first.key, + recordKind: first.recordKind, + templateID: first.templateID, + template: first.template, + nfv9OptionsTemplate: first.optionsTemplate, + records: dataRecordsForNFv9BatchRecord(first), + optionsRecords: optionsRecordsForNFv9BatchRecord(first), + }} + for _, evt := range events[1:] { + next, err := e.nfv9BatchRecord(evt, timing) + if err != nil { + return nil, false, err + } + if next.sourceID != first.sourceID { + return nil, false, nil + } + nextSets, ok := appendIPFIXBatchRecord(sets, ipfixBatchRecord{ + key: next.key, + recordKind: next.recordKind, + templateID: next.templateID, + template: next.template, + nfv9OptionsTemplate: next.optionsTemplate, + data: next.data, + options: next.options, + }) + if !ok { + return nil, false, nil + } + sets = nextSets + } + return e.nfv9BatchPacket(first, sets), true, nil +} + +// advanceSequence tracks NetFlow v9's export-packet sequence counter. IPFIX +// uses a data-record counter instead. +func (e *NFv9Encoder) advanceSequence(packet *netflow.NFv9Packet) { + if packet == nil { + return + } + e.seq.Add(1) +} + +// buildPacket translates one runtime event into the appropriate IPFIX packet +// flavor: control/template output or a normal data set. +func (e *IPFIXEncoder) buildPacket(evt *event.Event) (*netflow.IPFIXPacket, error) { + if evt == nil { + return nil, fmt.Errorf("nil event") + } + if evt.Fields == nil { + return nil, fmt.Errorf("event fields are empty") + } + + fieldMap := eventFieldsWithMetadata(evt) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + templateID := uint16Field(fieldMap, "template_id") + if templateID == 0 { + templateID = 256 + } + obsDomainID := e.observationDomainID() + exportTime := uint32(evt.ReceivedAt.Unix()) + if evt.ReceivedAt.IsZero() { + exportTime = uint32(time.Now().Unix()) + } + + switch evt.Payload.(type) { + case netflow.TemplateRecord: + record := evt.Payload.(netflow.TemplateRecord) + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: exportTime, + SequenceNumber: e.seq.Load(), + ObservationDomainId: obsDomainID, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 2}, + Records: []netflow.TemplateRecord{record}, + }, + }, + } + e.advanceIPFIXSequence(packet) + return packet, nil + case *netflow.TemplateRecord: + record := *evt.Payload.(*netflow.TemplateRecord) + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: exportTime, + SequenceNumber: e.seq.Load(), + ObservationDomainId: obsDomainID, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 2}, + Records: []netflow.TemplateRecord{record}, + }, + }, + } + e.advanceIPFIXSequence(packet) + return packet, nil + case netflow.IPFIXOptionsTemplateRecord: + record := evt.Payload.(netflow.IPFIXOptionsTemplateRecord) + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: exportTime, + SequenceNumber: e.seq.Load(), + ObservationDomainId: obsDomainID, + FlowSets: []interface{}{ + netflow.IPFIXOptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 3}, + Records: []netflow.IPFIXOptionsTemplateRecord{record}, + }, + }, + } + e.advanceIPFIXSequence(packet) + return packet, nil + case *netflow.IPFIXOptionsTemplateRecord: + record := *evt.Payload.(*netflow.IPFIXOptionsTemplateRecord) + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: exportTime, + SequenceNumber: e.seq.Load(), + ObservationDomainId: obsDomainID, + FlowSets: []interface{}{ + netflow.IPFIXOptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 3}, + Records: []netflow.IPFIXOptionsTemplateRecord{record}, + }, + }, + } + e.advanceIPFIXSequence(packet) + return packet, nil + } + + stream := eventStream(evt, "flow_data") + if schema, ok := e.optionsSchemas[stream]; ok { + fieldMap = eventFieldsWithMetadataForSchema(evt, optionsSchemaFields(schema)) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + optionsRecord, err := buildOptionsDataRecordFromSchemaFields(e.cfg.TemplatedFlow.Data, fieldMap, schema.scopes, schema.options, templatedEncodingContext{}) + if err != nil { + return nil, err + } + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: exportTime, + SequenceNumber: e.seq.Load(), + ObservationDomainId: obsDomainID, + FlowSets: []interface{}{ + netflow.OptionsDataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: schema.templateID}, + Records: []netflow.OptionsDataRecord{optionsRecord}, + }, + }, + } + e.advanceIPFIXSequence(packet) + return packet, nil + } + if schema, ok := e.dataSchemas[stream]; ok { + fieldMap = eventFieldsWithMetadataForSchema(evt, schema.fields) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + templateRecord := schema.template + dataRecord, err := buildTemplatedValuesFromSchemaFields(e.cfg.TemplatedFlow.Data, fieldMap, schema.fields, templatedEncodingContext{}) + if err != nil { + return nil, err + } + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: exportTime, + SequenceNumber: e.seq.Load(), + ObservationDomainId: obsDomainID, + FlowSets: []interface{}{ + netflow.DataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: templateRecord.TemplateId}, + Records: []netflow.DataRecord{dataRecord}, + }, + }, + } + e.advanceIPFIXSequence(packet) + return packet, nil + } + + plan, dataRecord, err := e.fallbackDataRecord(fieldMap, templateID) + if err != nil { + return nil, err + } + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: exportTime, + SequenceNumber: e.seq.Load(), + ObservationDomainId: obsDomainID, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 2}, + Records: []netflow.TemplateRecord{plan.template}, + }, + netflow.DataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: plan.template.TemplateId}, + Records: []netflow.DataRecord{dataRecord}, + }, + }, + } + e.advanceIPFIXSequence(packet) + return packet, nil +} + +func (e *NFv9Encoder) buildPacket(evt *event.Event) (*netflow.NFv9Packet, error) { + if evt == nil { + return nil, fmt.Errorf("nil event") + } + if evt.Fields == nil { + return nil, fmt.Errorf("event fields are empty") + } + + fieldMap := eventFieldsWithMetadata(evt) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + templateID := uint16Field(fieldMap, "template_id") + if templateID == 0 { + templateID = 256 + } + sourceID := e.sourceID() + exportMS := exportUnixMilliseconds(evt.ReceivedAt, evt.Fields) + unixSeconds := uint32((exportMS + 999) / 1000) + packetExportMS := int64(unixSeconds) * 1000 + sysUptime, firstSwitched, lastSwitched := uptimeWindow(packetExportMS, int64Field(evt.Fields, "start_time_unix"), int64Field(evt.Fields, "end_time_unix")) + encodingCtx := templatedEncodingContext{ + netflowV9: true, + firstSwitched: firstSwitched, + lastSwitched: lastSwitched, + } + + switch payload := evt.Payload.(type) { + case netflow.TemplateRecord: + return &netflow.NFv9Packet{ + Version: 9, + Count: 1, + SystemUptime: sysUptime, + UnixSeconds: unixSeconds, + SequenceNumber: e.seq.Load(), + SourceId: sourceID, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 0}, + Records: []netflow.TemplateRecord{payload}, + }, + }, + }, nil + case *netflow.TemplateRecord: + return &netflow.NFv9Packet{ + Version: 9, + Count: 1, + SystemUptime: sysUptime, + UnixSeconds: unixSeconds, + SequenceNumber: e.seq.Load(), + SourceId: sourceID, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 0}, + Records: []netflow.TemplateRecord{*payload}, + }, + }, + }, nil + case netflow.NFv9OptionsTemplateRecord: + return &netflow.NFv9Packet{ + Version: 9, + Count: 1, + SystemUptime: sysUptime, + UnixSeconds: unixSeconds, + SequenceNumber: e.seq.Load(), + SourceId: sourceID, + FlowSets: []interface{}{ + netflow.NFv9OptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 1}, + Records: []netflow.NFv9OptionsTemplateRecord{payload}, + }, + }, + }, nil + case *netflow.NFv9OptionsTemplateRecord: + return &netflow.NFv9Packet{ + Version: 9, + Count: 1, + SystemUptime: sysUptime, + UnixSeconds: unixSeconds, + SequenceNumber: e.seq.Load(), + SourceId: sourceID, + FlowSets: []interface{}{ + netflow.NFv9OptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 1}, + Records: []netflow.NFv9OptionsTemplateRecord{*payload}, + }, + }, + }, nil + } + + stream := eventStream(evt, "flow_data") + if schema, ok := e.optionsSchemas[stream]; ok { + fieldMap = eventFieldsWithMetadataForSchema(evt, optionsSchemaFields(schema)) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + optionsRecord, err := buildOptionsDataRecordFromSchemaFields(e.cfg.TemplatedFlow.Data, fieldMap, schema.scopes, schema.options, encodingCtx) + if err != nil { + return nil, err + } + packet := &netflow.NFv9Packet{ + Version: 9, + Count: 1, + SystemUptime: sysUptime, + UnixSeconds: unixSeconds, + SequenceNumber: e.seq.Load(), + SourceId: sourceID, + FlowSets: []interface{}{ + netflow.OptionsDataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: schema.templateID}, + Records: []netflow.OptionsDataRecord{optionsRecord}, + }, + }, + } + return packet, nil + } + if schema, ok := e.dataSchemas[stream]; ok { + fieldMap = eventFieldsWithMetadataForSchema(evt, schema.fields) + applyTemplatedFlowDataMatches(e.cfg.TemplatedFlow.Data, fieldMap) + templateRecord := schema.template + dataRecord, err := buildTemplatedValuesFromSchemaFields(e.cfg.TemplatedFlow.Data, fieldMap, schema.fields, encodingCtx) + if err != nil { + return nil, err + } + packet := &netflow.NFv9Packet{ + Version: 9, + Count: 1, + SystemUptime: sysUptime, + UnixSeconds: unixSeconds, + SequenceNumber: e.seq.Load(), + SourceId: sourceID, + FlowSets: []interface{}{ + netflow.DataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: templateRecord.TemplateId}, + Records: []netflow.DataRecord{dataRecord}, + }, + }, + } + return packet, nil + } + + plan, dataRecord, err := e.fallbackDataRecord(fieldMap, templateID, encodingCtx) + if err != nil { + return nil, err + } + packet := &netflow.NFv9Packet{ + Version: 9, + Count: 2, + SystemUptime: sysUptime, + UnixSeconds: unixSeconds, + SequenceNumber: e.seq.Load(), + SourceId: sourceID, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 0}, + Records: []netflow.TemplateRecord{plan.template}, + }, + netflow.DataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: plan.template.TemplateId}, + Records: []netflow.DataRecord{dataRecord}, + }, + }, + } + return packet, nil +} + +// observationDomainID is exporter-scoped. Per-interface IDs stay in data fields +// such as source_id/observationPointId, not in the IPFIX packet header. +func (e *IPFIXEncoder) observationDomainID() uint32 { + return e.cfg.TemplatedFlow.ObservationDomainID +} + +// sourceID is the NetFlow v9 exporter source ID, equivalent to the configured +// exporter observation domain. +func (e *NFv9Encoder) sourceID() uint32 { + return e.cfg.TemplatedFlow.ObservationDomainID +} + +// handleControl routes control events into encoder-specific schema/source registration. +func (e *IPFIXEncoder) handleControl(evt *event.Event) ([][]byte, error) { + switch controlType(evt) { + case "schema": + return e.registerSchema(evt) + case "source_init": + return e.registerSourceInit(evt) + case "source_init_batch": + return e.registerSourceInitBatch(evt) + default: + return nil, nil + } +} + +// handleControl routes control events into encoder-specific schema/source registration. +func (e *NFv9Encoder) handleControl(evt *event.Event) ([][]byte, error) { + switch controlType(evt) { + case "schema": + return e.registerSchema(evt) + case "source_init": + return e.registerSourceInit(evt) + default: + return nil, nil + } +} + +// registerSchema stores aggregation schema state and emits any required template packets. +func (e *IPFIXEncoder) registerSchema(evt *event.Event) ([][]byte, error) { + schema, ok := evt.Payload.(event.AggregationSchema) + if !ok { + if ptr, ok := evt.Payload.(*event.AggregationSchema); ok && ptr != nil { + schema = *ptr + } else { + return nil, nil + } + } + if schemaIsOptions(schema) { + state, err := buildOptionsSchemaState(e.cfg.TemplatedFlow.Data, schema, false, e.cfg.TemplatedFlow.OptionsTemplateBaseID) + if err != nil { + return nil, err + } + e.optionsSchemas[eventStream(evt, schema.Stream)] = state + payloads, err := e.encodeOptionsSchemaTemplate(state) + if err != nil { + return nil, err + } + e.lastOptionsRun = time.Now().UTC() + return payloads, nil + } + state, err := buildSchemaState(e.cfg.TemplatedFlow.Data, schema, false) + if err != nil { + return nil, err + } + if state.baseTemplateID == 0 { + state.baseTemplateID = e.cfg.TemplatedFlow.TemplateBaseID + } + if state.template.TemplateId == 0 || state.baseTemplateID != state.template.TemplateId { + state, err = buildSchemaStateWithBase(e.cfg.TemplatedFlow.Data, schema, false, state.baseTemplateID) + if err != nil { + return nil, err + } + } + e.dataSchemas[eventStream(evt, schema.Stream)] = state + payloads, err := e.encodeSchemaTemplates(state) + if err != nil { + return nil, err + } + e.lastTemplateRun = time.Now().UTC() + return payloads, nil +} + +// registerSchema stores aggregation schema state and emits any required template packets. +func (e *NFv9Encoder) registerSchema(evt *event.Event) ([][]byte, error) { + schema, ok := evt.Payload.(event.AggregationSchema) + if !ok { + if ptr, ok := evt.Payload.(*event.AggregationSchema); ok && ptr != nil { + schema = *ptr + } else { + return nil, nil + } + } + if schemaIsOptions(schema) { + state, err := buildOptionsSchemaState(e.cfg.TemplatedFlow.Data, schema, true, e.cfg.TemplatedFlow.OptionsTemplateBaseID) + if err != nil { + return nil, err + } + e.optionsSchemas[eventStream(evt, schema.Stream)] = state + payloads, err := e.encodeOptionsSchemaTemplate(state) + if err != nil { + return nil, err + } + e.lastOptionsRun = time.Now().UTC() + return payloads, nil + } + state, err := buildSchemaState(e.cfg.TemplatedFlow.Data, schema, true) + if err != nil { + return nil, err + } + if state.baseTemplateID == 0 { + state.baseTemplateID = e.cfg.TemplatedFlow.TemplateBaseID + } + if state.template.TemplateId == 0 || state.baseTemplateID != state.template.TemplateId { + state, err = buildSchemaStateWithBase(e.cfg.TemplatedFlow.Data, schema, true, state.baseTemplateID) + if err != nil { + return nil, err + } + } + e.dataSchemas[eventStream(evt, schema.Stream)] = state + payloads, err := e.encodeSchemaTemplates(state) + if err != nil { + return nil, err + } + e.lastTemplateRun = time.Now().UTC() + return payloads, nil +} + +// registerSourceInit stores source-scoped exporter metadata and may emit options templates/data. +func (e *IPFIXEncoder) registerSourceInit(evt *event.Event) ([][]byte, error) { + state := sourceOptionsFromEvent(evt) + if state.stream == "" { + state.stream = eventStream(evt, "options_data") + } + if state.templateID == 0 { + state.templateID = e.cfg.TemplatedFlow.OptionsTemplateBaseID + } + state.observationDomainID = e.observationDomainID() + e.sourceOptions[sourceOptionsKey(state)] = state + payloads, err := e.encodeSourceOptions(state) + if err != nil { + return nil, err + } + e.lastOptionsRun = time.Now().UTC() + return payloads, nil +} + +func (e *IPFIXEncoder) registerSourceInitBatch(evt *event.Event) ([][]byte, error) { + events, ok := evt.Payload.([]*event.Event) + if !ok { + return nil, nil + } + states := make([]sourceOptionsState, 0, len(events)) + for _, sourceEvt := range events { + state := sourceOptionsFromEvent(sourceEvt) + if state.stream == "" { + state.stream = eventStream(sourceEvt, "options_data") + } + if state.templateID == 0 { + state.templateID = e.cfg.TemplatedFlow.OptionsTemplateBaseID + } + state.observationDomainID = e.observationDomainID() + e.sourceOptions[sourceOptionsKey(state)] = state + states = append(states, state) + } + payloads, err := e.encodeSourceOptionsBatch(states) + if err != nil { + return nil, err + } + e.lastOptionsRun = time.Now().UTC() + return payloads, nil +} + +// registerSourceInit stores source-scoped exporter metadata and may emit options templates/data. +func (e *NFv9Encoder) registerSourceInit(evt *event.Event) ([][]byte, error) { + state := sourceOptionsFromEvent(evt) + if state.stream == "" { + state.stream = eventStream(evt, "options_data") + } + if state.templateID == 0 { + state.templateID = e.cfg.TemplatedFlow.OptionsTemplateBaseID + } + state.observationDomainID = e.sourceID() + e.sourceOptions[sourceOptionsKey(state)] = state + payloads, err := e.encodeSourceOptions(state) + if err != nil { + return nil, err + } + e.lastOptionsRun = time.Now().UTC() + return payloads, nil +} + +// flushControlPackets emits periodic template/options refresh packets when due. +func (e *IPFIXEncoder) flushControlPackets(now time.Time) ([][]byte, error) { + var payloads [][]byte + if e.cfg.TemplatedFlow.TemplateRefresh > 0 && (e.lastTemplateRun.IsZero() || now.Sub(e.lastTemplateRun) >= time.Duration(e.cfg.TemplatedFlow.TemplateRefresh)*time.Millisecond) { + for _, schema := range e.dataSchemas { + encoded, err := e.encodeSchemaTemplates(schema) + if err != nil { + return nil, err + } + payloads = append(payloads, encoded...) + } + e.lastTemplateRun = now + } + if e.cfg.TemplatedFlow.OptionsRefresh > 0 && (e.lastOptionsRun.IsZero() || now.Sub(e.lastOptionsRun) >= time.Duration(e.cfg.TemplatedFlow.OptionsRefresh)*time.Millisecond) { + encoded, err := e.encodeSourceOptionsBatch(sortedSourceOptions(e.sourceOptions)) + if err != nil { + return nil, err + } + payloads = append(payloads, encoded...) + for _, schema := range e.optionsSchemas { + encoded, err := e.encodeOptionsSchemaTemplate(schema) + if err != nil { + return nil, err + } + payloads = append(payloads, encoded...) + } + e.lastOptionsRun = now + } + return payloads, nil +} + +// flushControlPackets emits periodic template/options refresh packets when due. +func (e *NFv9Encoder) flushControlPackets(now time.Time) ([][]byte, error) { + var payloads [][]byte + if e.cfg.TemplatedFlow.TemplateRefresh > 0 && (e.lastTemplateRun.IsZero() || now.Sub(e.lastTemplateRun) >= time.Duration(e.cfg.TemplatedFlow.TemplateRefresh)*time.Millisecond) { + for _, schema := range e.dataSchemas { + encoded, err := e.encodeSchemaTemplates(schema) + if err != nil { + return nil, err + } + payloads = append(payloads, encoded...) + } + e.lastTemplateRun = now + } + if e.cfg.TemplatedFlow.OptionsRefresh > 0 && (e.lastOptionsRun.IsZero() || now.Sub(e.lastOptionsRun) >= time.Duration(e.cfg.TemplatedFlow.OptionsRefresh)*time.Millisecond) { + for _, state := range e.sourceOptions { + encoded, err := e.encodeSourceOptions(state) + if err != nil { + return nil, err + } + payloads = append(payloads, encoded...) + } + for _, schema := range e.optionsSchemas { + encoded, err := e.encodeOptionsSchemaTemplate(schema) + if err != nil { + return nil, err + } + payloads = append(payloads, encoded...) + } + e.lastOptionsRun = now + } + return payloads, nil +} + +// encodeSchemaTemplates serializes the current stream schema into one or more IPFIX template sets. +func (e *IPFIXEncoder) encodeSchemaTemplates(state templatedSchemaState) ([][]byte, error) { + now := uint32(time.Now().Unix()) + var out [][]byte + for _, templateRecord := range state.templates() { + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: now, + SequenceNumber: e.seq.Load(), + ObservationDomainId: e.cfg.TemplatedFlow.ObservationDomainID, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 2}, + Records: []netflow.TemplateRecord{templateRecord}, + }, + }, + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode ipfix schema template: %w", err) + } + out = append(out, data) + e.advanceIPFIXSequence(packet) + } + return out, nil +} + +// encodeSchemaTemplates serializes the current stream schema into one or more NetFlow v9 template sets. +func (e *NFv9Encoder) encodeSchemaTemplates(state templatedSchemaState) ([][]byte, error) { + nowMS := time.Now().UnixMilli() + nowSec := uint32((nowMS + 999) / 1000) + var out [][]byte + for _, templateRecord := range state.templates() { + packet := &netflow.NFv9Packet{ + Version: 9, + Count: 1, + SystemUptime: 0, + UnixSeconds: nowSec, + SequenceNumber: e.seq.Load(), + SourceId: e.cfg.TemplatedFlow.ObservationDomainID, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 0}, + Records: []netflow.TemplateRecord{templateRecord}, + }, + }, + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode netflow v9 schema template: %w", err) + } + out = append(out, data) + e.advanceSequence(packet) + } + return out, nil +} + +// encodeOptionsSchemaTemplate serializes a schema-derived IPFIX options template. +func (e *IPFIXEncoder) encodeOptionsSchemaTemplate(state optionsSchemaState) ([][]byte, error) { + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: uint32(time.Now().Unix()), + SequenceNumber: e.seq.Load(), + ObservationDomainId: e.cfg.TemplatedFlow.ObservationDomainID, + FlowSets: []interface{}{ + netflow.IPFIXOptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 3}, + Records: []netflow.IPFIXOptionsTemplateRecord{state.ipfixTemplate}, + }, + }, + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode ipfix options schema template: %w", err) + } + e.advanceIPFIXSequence(packet) + return [][]byte{data}, nil +} + +// encodeOptionsSchemaTemplate serializes a schema-derived NetFlow v9 options template. +func (e *NFv9Encoder) encodeOptionsSchemaTemplate(state optionsSchemaState) ([][]byte, error) { + packet := &netflow.NFv9Packet{ + Version: 9, + Count: 1, + SystemUptime: 0, + UnixSeconds: uint32(time.Now().Unix()), + SequenceNumber: e.seq.Load(), + SourceId: e.cfg.TemplatedFlow.ObservationDomainID, + FlowSets: []interface{}{ + netflow.NFv9OptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 1}, + Records: []netflow.NFv9OptionsTemplateRecord{state.nfv9Template}, + }, + }, + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode netflow v9 options schema template: %w", err) + } + e.advanceSequence(packet) + return [][]byte{data}, nil +} + +// encodeSourceOptions serializes source-level exporter metadata as IPFIX options records. +func (e *IPFIXEncoder) encodeSourceOptions(state sourceOptionsState) ([][]byte, error) { + return e.encodeSourceOptionsBatch([]sourceOptionsState{state}) +} + +func (e *IPFIXEncoder) encodeSourceOptionsBatch(states []sourceOptionsState) ([][]byte, error) { + groups := groupedIPFIXSourceOptions(states) + out := make([][]byte, 0, len(groups)) + for _, states := range groups { + data, err := e.encodeSourceOptionsGroup(states) + if err != nil { + return nil, err + } + out = append(out, data) + } + return out, nil +} + +func (e *IPFIXEncoder) encodeSourceOptionsGroup(states []sourceOptionsState) ([]byte, error) { + if len(states) == 0 { + return nil, nil + } + first := states[0] + records := make([]netflow.OptionsDataRecord, 0, len(states)) + for _, state := range states { + records = append(records, netflow.OptionsDataRecord{ + ScopesValues: []netflow.DataField{ + {Type: netflow.IPFIX_FIELD_observationPointId, Value: encodeU64(uint64(state.sourceID))}, + }, + OptionsValues: []netflow.DataField{ + {Type: netflow.IPFIX_FIELD_samplingInterval, Value: encodeU32(state.samplingRate)}, + }, + }) + } + packet := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: uint32(time.Now().Unix()), + SequenceNumber: e.seq.Load(), + ObservationDomainId: first.observationDomainID, + FlowSets: []interface{}{ + netflow.IPFIXOptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 3}, + Records: []netflow.IPFIXOptionsTemplateRecord{ + { + TemplateId: first.templateID, + FieldCount: 2, + ScopeFieldCount: 1, + Scopes: []netflow.Field{ + {Type: netflow.IPFIX_FIELD_observationPointId, Length: 8}, + }, + Options: []netflow.Field{ + {Type: netflow.IPFIX_FIELD_samplingInterval, Length: 4}, + }, + }, + }, + }, + netflow.OptionsDataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: first.templateID}, + Records: records, + }, + }, + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode ipfix source options: %w", err) + } + e.advanceIPFIXSequence(packet) + return data, nil +} + +// encodeSourceOptions serializes source-level exporter metadata as NetFlow v9 options records. +func (e *NFv9Encoder) encodeSourceOptions(state sourceOptionsState) ([][]byte, error) { + packet := &netflow.NFv9Packet{ + Version: 9, + Count: 2, + SystemUptime: 0, + UnixSeconds: uint32(time.Now().Unix()), + SequenceNumber: e.seq.Load(), + SourceId: state.sourceID, + FlowSets: []interface{}{ + netflow.NFv9OptionsTemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 1}, + Records: []netflow.NFv9OptionsTemplateRecord{ + { + TemplateId: state.templateID, + ScopeLength: 4, + OptionLength: 4, + Scopes: []netflow.Field{ + {Type: 1, Length: 4}, + }, + Options: []netflow.Field{ + {Type: netflow.NFV9_FIELD_SAMPLING_INTERVAL, Length: 4}, + }, + }, + }, + }, + netflow.OptionsDataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: state.templateID}, + Records: []netflow.OptionsDataRecord{ + { + ScopesValues: []netflow.DataField{ + {Type: 1, Value: encodeU32(state.sourceID)}, + }, + OptionsValues: []netflow.DataField{ + {Type: netflow.NFV9_FIELD_SAMPLING_INTERVAL, Value: encodeU32(state.samplingRate)}, + }, + }, + }, + }, + }, + } + data, err := netflow.EncodeMessage(packet) + if err != nil { + return nil, fmt.Errorf("encode netflow v9 source options: %w", err) + } + e.advanceSequence(packet) + return [][]byte{data}, nil +} + +// buildSchemaState prepares the template state for an aggregated stream, defaulting +// the base template ID when the schema did not set one explicitly. +func buildSchemaState(cfg config.TemplatedFlowDataConfig, schema event.AggregationSchema, netflowV9 bool) (templatedSchemaState, error) { + baseTemplateID := schema.BaseTemplateID + if baseTemplateID == 0 { + baseTemplateID = 256 + } + return buildSchemaStateWithBase(cfg, schema, netflowV9, baseTemplateID) +} + +func schemaIsOptions(schema event.AggregationSchema) bool { + for _, field := range schemaFieldsOrNames(schema) { + if field.Role == "static" && field.Name == tflowRecordTypeField && fmt.Sprint(field.Value) == "options" { + return true + } + } + if val, ok := schema.StaticFields[tflowRecordTypeField]; ok && fmt.Sprint(val) == "options" { + return true + } + return false +} + +func buildOptionsSchemaState(cfg config.TemplatedFlowDataConfig, schema event.AggregationSchema, netflowV9 bool, defaultTemplateID uint16) (optionsSchemaState, error) { + templateID := schema.BaseTemplateID + if templateID == 0 { + templateID = defaultTemplateID + } + if templateID == 0 { + templateID = 1024 + } + + state := optionsSchemaState{ + stream: schema.Stream, + templateID: templateID, + } + if state.stream == "" { + state.stream = "options_data" + } + for _, field := range schemaFieldsOrNames(schema) { + if isTFlowRecordTypeMarker(field) { + continue + } + if _, ok := cfg.Catalog[field.Name]; !ok { + continue + } + if field.Role == "key" { + state.scopes = append(state.scopes, field) + continue + } + state.options = append(state.options, field) + } + if len(state.scopes) == 0 { + return optionsSchemaState{}, fmt.Errorf("options schema %q has no key scope fields", state.stream) + } + if len(state.options) == 0 { + return optionsSchemaState{}, fmt.Errorf("options schema %q has no option fields", state.stream) + } + + if netflowV9 { + record, err := buildNFv9OptionsTemplateRecordFromSchemaFields(cfg, state.scopes, state.options, templateID) + if err != nil { + return optionsSchemaState{}, err + } + state.nfv9Template = record + } else { + record, err := buildIPFIXOptionsTemplateRecordFromSchemaFields(cfg, state.scopes, state.options, templateID) + if err != nil { + return optionsSchemaState{}, err + } + state.ipfixTemplate = record + } + return state, nil +} + +func isTFlowRecordTypeMarker(field event.SchemaField) bool { + return field.Role == "static" && field.Name == tflowRecordTypeField +} + +func optionsSchemaFields(schema optionsSchemaState) []event.SchemaField { + fields := make([]event.SchemaField, 0, len(schema.scopes)+len(schema.options)) + fields = append(fields, schema.scopes...) + fields = append(fields, schema.options...) + return fields +} + +// buildSchemaStateWithBase precomputes the single template for one aggregated +// stream schema. Address-family choices are carried by SchemaField.Value; an +// unmodified address field expands to both IPv4 and IPv6 wire fields. +func buildSchemaStateWithBase(cfg config.TemplatedFlowDataConfig, schema event.AggregationSchema, netflowV9 bool, baseTemplateID uint16) (templatedSchemaState, error) { + stream := schema.Stream + if stream == "" { + stream = "flow_data" + } + if baseTemplateID == 0 { + baseTemplateID = 256 + } + state := templatedSchemaState{ + stream: stream, + fieldNames: append([]string(nil), schema.FieldNames...), + fields: schemaFieldsOrNames(schema), + baseTemplateID: baseTemplateID, + } + template, err := buildTemplateRecordFromSchemaFields(cfg, state.fields, baseTemplateID, netflowV9) + if err != nil { + return templatedSchemaState{}, err + } + state.template = template + return state, nil +} + +func schemaFieldsOrNames(schema event.AggregationSchema) []event.SchemaField { + if len(schema.Fields) > 0 { + return append([]event.SchemaField(nil), schema.Fields...) + } + fields := make([]event.SchemaField, 0, len(schema.FieldNames)) + for _, name := range schema.FieldNames { + fields = append(fields, event.SchemaField{Name: name, Role: "current"}) + } + return fields +} + +// templateForFields returns the schema template for compatibility with tests and +// older call sites that inspect schema state. +func (s templatedSchemaState) templateForFields(fields map[string]any) netflow.TemplateRecord { + return s.template +} + +// templates returns every template record that must be announced for this schema. +func (s templatedSchemaState) templates() []netflow.TemplateRecord { + if s.template.FieldCount == 0 { + return nil + } + return []netflow.TemplateRecord{s.template} +} + +func sourceOptionsKey(state sourceOptionsState) string { + return fmt.Sprintf("%s|%d|%d|%d", state.stream, state.observationDomainID, state.sourceID, state.templateID) +} + +func sortedSourceOptions(options map[string]sourceOptionsState) []sourceOptionsState { + if len(options) == 0 { + return nil + } + keys := make([]string, 0, len(options)) + for key := range options { + keys = append(keys, key) + } + sort.Strings(keys) + states := make([]sourceOptionsState, 0, len(keys)) + for _, key := range keys { + states = append(states, options[key]) + } + return states +} + +func groupedIPFIXSourceOptions(states []sourceOptionsState) [][]sourceOptionsState { + if len(states) == 0 { + return nil + } + groups := make([][]sourceOptionsState, 0, len(states)) + groupIndexes := make(map[string]int, len(states)) + for _, state := range states { + key := fmt.Sprintf("%d|%d", state.observationDomainID, state.templateID) + if index, ok := groupIndexes[key]; ok { + groups[index] = append(groups[index], state) + continue + } + groupIndexes[key] = len(groups) + groups = append(groups, []sourceOptionsState{state}) + } + return groups +} + +// sourceOptionsFromEvent extracts source-level exporter metadata from event +// metadata, with payload/field values used only for exporter control knobs that +// are not represented in SourceMetadata. +func sourceOptionsFromEvent(evt *event.Event) sourceOptionsState { + state := sourceOptionsState{ + stream: eventStream(evt, "options_data"), + agentIP: eventAgentIP(evt), + sourceID: eventSourceID(evt), + samplingRate: eventSamplingRate(evt), + samplePool: eventSamplePool(evt), + drops: eventDrops(evt), + inputIf: uint32Field(evt.Fields, "input_if"), + outputIf: uint32Field(evt.Fields, "output_if"), + } + if payload, ok := evt.Payload.(event.SourceInit); ok { + if payload.Stream != "" { + state.stream = payload.Stream + } + if payload.InputIf != 0 { + state.inputIf = payload.InputIf + } + if payload.OutputIf != 0 { + state.outputIf = payload.OutputIf + } + } + return state +} + +func (e *IPFIXEncoder) fallbackDataRecord(fieldMap map[string]any, templateID uint16) (fallbackTemplatePlan, netflow.DataRecord, error) { + key, err := fallbackPlanKey(e.cfg.TemplatedFlow.Data, fieldMap, templateID, false) + if err != nil { + return fallbackTemplatePlan{}, netflow.DataRecord{}, err + } + plan, ok := e.fallbackPlans[key] + if !ok { + plan, err = buildFallbackPlan(e.cfg.TemplatedFlow.Data, fieldMap, templateID, false) + if err != nil { + return fallbackTemplatePlan{}, netflow.DataRecord{}, err + } + e.fallbackPlans[key] = plan + } + record, err := buildFallbackValues(plan, fieldMap, templatedEncodingContext{}) + if err != nil { + return fallbackTemplatePlan{}, netflow.DataRecord{}, err + } + return plan, record, nil +} + +func (e *NFv9Encoder) fallbackDataRecord(fieldMap map[string]any, templateID uint16, encodingCtx templatedEncodingContext) (fallbackTemplatePlan, netflow.DataRecord, error) { + key, err := fallbackPlanKey(e.cfg.TemplatedFlow.Data, fieldMap, templateID, true) + if err != nil { + return fallbackTemplatePlan{}, netflow.DataRecord{}, err + } + plan, ok := e.fallbackPlans[key] + if !ok { + plan, err = buildFallbackPlan(e.cfg.TemplatedFlow.Data, fieldMap, templateID, true) + if err != nil { + return fallbackTemplatePlan{}, netflow.DataRecord{}, err + } + e.fallbackPlans[key] = plan + } + record, err := buildFallbackValues(plan, fieldMap, encodingCtx) + if err != nil { + return fallbackTemplatePlan{}, netflow.DataRecord{}, err + } + return plan, record, nil +} + +func fallbackPlanKey(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any, templateID uint16, netflowV9 bool) (string, error) { + names := selectPresentFlowFields(cfg, fieldMap) + var b strings.Builder + fmt.Fprintf(&b, "%t|%d", netflowV9, templateID) + for _, name := range names { + defs, err := fieldDefinitionsForFamily(name, cfg.Catalog[name], "") + if err != nil { + return "", err + } + for _, def := range defs { + def = wireFieldDefinition(name, def, netflowV9) + if !fieldAllowedForEncoding(def, netflowV9) { + continue + } + length := def.Length + if length == 0 || length == 0xffff { + encoded, err := encodeFallbackValue(name, def, fieldMap, templatedEncodingContext{netflowV9: netflowV9}) + if err == nil { + length = ipfixFieldLength(def, encoded) + } + } + fmt.Fprintf(&b, "|%s:%d:%d:%d:%d:%t", name, def.ID, length, def.Length, def.PEN, def.EnterpriseScoped) + } + } + return b.String(), nil +} + +func buildFallbackPlan(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any, templateID uint16, netflowV9 bool) (fallbackTemplatePlan, error) { + names := selectPresentFlowFields(cfg, fieldMap) + fields := make([]netflow.Field, 0, len(names)) + defs := make([]config.IPFIXFieldDefinition, 0, len(names)) + keptNames := make([]string, 0, len(names)) + families := make([]string, 0, len(names)) + for _, name := range names { + fieldDefs, err := fieldDefinitionsForFamily(name, cfg.Catalog[name], "") + if err != nil { + return fallbackTemplatePlan{}, err + } + for _, def := range fieldDefs { + def = wireFieldDefinition(name, def, netflowV9) + if !fieldAllowedForEncoding(def, netflowV9) { + continue + } + encoded, err := encodeFallbackValue(name, def, fieldMap, templatedEncodingContext{netflowV9: netflowV9}) + if err != nil { + return fallbackTemplatePlan{}, fmt.Errorf("encode field %q: %w", name, err) + } + fields = append(fields, netflow.Field{ + PenProvided: def.EnterpriseScoped || def.PEN != 0, + Type: def.ID, + Length: ipfixFieldLength(def, encoded), + Pen: def.PEN, + }) + defs = append(defs, def) + keptNames = append(keptNames, name) + families = append(families, "") + } + } + if len(fields) == 0 { + return fallbackTemplatePlan{}, fmt.Errorf("no encodable fields found for ipfix packet") + } + return fallbackTemplatePlan{ + template: netflow.TemplateRecord{ + TemplateId: templateID, + FieldCount: uint16(len(fields)), + Fields: fields, + }, + names: keptNames, + defs: defs, + families: families, + }, nil +} + +func buildFallbackValues(plan fallbackTemplatePlan, fieldMap map[string]any, encodingCtx templatedEncodingContext) (netflow.DataRecord, error) { + values := make([]netflow.DataField, 0, len(plan.names)) + present := 0 + for i, name := range plan.names { + def := plan.defs[i] + if _, ok := fieldMap[name]; ok { + present++ + } + encoded, err := encodeFallbackValueWithFamily(name, def, fieldMap, plan.families[i], encodingCtx) + if err != nil { + return netflow.DataRecord{}, fmt.Errorf("encode field %q: %w", name, err) + } + values = append(values, netflow.DataField{ + PenProvided: def.EnterpriseScoped || def.PEN != 0, + Type: plan.template.Fields[i].Type, + Length: plan.template.Fields[i].Length, + Pen: def.PEN, + Value: encoded, + }) + } + if len(values) == 0 || present == 0 { + return netflow.DataRecord{}, fmt.Errorf("no encodable values found for templated packet") + } + return netflow.DataRecord{Values: values}, nil +} + +func selectPresentFlowFields(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any) []string { + if len(cfg.Select) > 0 { + names := make([]string, 0, len(cfg.Select)) + for _, name := range cfg.Select { + if _, ok := cfg.Catalog[name]; !ok { + continue + } + names = append(names, name) + } + return names + } + names := make([]string, 0, len(fieldMap)) + for name := range fieldMap { + if _, ok := cfg.Catalog[name]; ok { + names = append(names, name) + } + } + sort.Strings(names) + return names +} + +func applyTemplatedFlowDataMatches(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any) { + if len(cfg.Matches) == 0 || len(fieldMap) == 0 { + return + } + for _, match := range cfg.Matches { + if match.Field == "" || match.From == "" { + continue + } + if _, exists := fieldMap[match.Field]; exists { + continue + } + if val, ok := fieldMap[match.From]; ok { + fieldMap[match.Field] = val + } + } +} + +func encodeFallbackValue(name string, def config.IPFIXFieldDefinition, fieldMap map[string]any, encodingCtx templatedEncodingContext) ([]byte, error) { + return encodeFallbackValueWithFamily(name, def, fieldMap, "", encodingCtx) +} + +func encodeFallbackValueWithFamily(name string, def config.IPFIXFieldDefinition, fieldMap map[string]any, family string, encodingCtx templatedEncodingContext) ([]byte, error) { + if val, ok := fieldMap[name]; ok { + return encodeTemplatedValueForFamily(name, def, val, family, encodingCtx) + } + return defaultEncodedValue(def) +} + +// buildTemplatedDataRecord picks fields from a runtime event and builds both the +// template and one matching data record. +func buildTemplatedDataRecord(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any, templateID uint16, netflowV9 bool) (netflow.TemplateRecord, netflow.DataRecord, error) { + names := selectFlowFields(cfg, fieldMap) + return buildTemplatedDataRecordWithNames(cfg, fieldMap, names, templateID, templatedEncodingContext{netflowV9: netflowV9}) +} + +// buildTemplatedDataRecordWithNames uses an explicit field order, which matters +// when schema events already fixed the record layout. +func buildTemplatedDataRecordWithNames(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any, names []string, templateID uint16, encodingCtx templatedEncodingContext) (netflow.TemplateRecord, netflow.DataRecord, error) { + templateFields := make([]netflow.Field, 0, len(names)) + values := make([]netflow.DataField, 0, len(names)) + for _, name := range names { + def, ok := cfg.Catalog[name] + if !ok { + continue + } + val, ok := fieldMap[name] + if !ok { + continue + } + def = resolvedFieldDefinition(name, def, val) + def = wireFieldDefinition(name, def, encodingCtx.netflowV9) + if !fieldAllowedForEncoding(def, encodingCtx.netflowV9) { + continue + } + encoded, err := encodeTemplatedValue(name, def, val, encodingCtx) + if err != nil { + return netflow.TemplateRecord{}, netflow.DataRecord{}, fmt.Errorf("encode field %q: %w", name, err) + } + templateFields = append(templateFields, netflow.Field{ + PenProvided: def.EnterpriseScoped || def.PEN != 0, + Type: def.ID, + Length: ipfixFieldLength(def, encoded), + Pen: def.PEN, + }) + values = append(values, netflow.DataField{ + PenProvided: def.EnterpriseScoped || def.PEN != 0, + Type: def.ID, + Length: ipfixFieldLength(def, encoded), + Pen: def.PEN, + Value: encoded, + }) + } + if len(templateFields) == 0 { + return netflow.TemplateRecord{}, netflow.DataRecord{}, fmt.Errorf("no encodable fields found for ipfix packet") + } + return netflow.TemplateRecord{ + TemplateId: templateID, + FieldCount: uint16(len(templateFields)), + Fields: templateFields, + }, netflow.DataRecord{ + Values: values, + }, nil +} + +func schemaFieldDefinitions(cfg config.TemplatedFlowDataConfig, field event.SchemaField) ([]config.IPFIXFieldDefinition, bool, error) { + def, ok := cfg.Catalog[field.Name] + if !ok { + return nil, false, nil + } + + if def.Name == "" { + def.Name = field.Name + } + defs, err := fieldDefinitionsForFamily(field.Name, def, schemaFieldFamily(field)) + return defs, true, err +} + +func buildTemplatedValuesFromSchemaFields(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any, fields []event.SchemaField, encodingCtx templatedEncodingContext) (netflow.DataRecord, error) { + values := make([]netflow.DataField, 0, len(fields)) + for _, field := range fields { + defs, ok, err := schemaFieldDefinitions(cfg, field) + if err != nil { + return netflow.DataRecord{}, err + } + if !ok { + continue + } + val, ok := fieldMap[field.Name] + if !ok && field.Role == "static" && field.Value != nil { + val = field.Value + ok = true + } + for _, def := range defs { + def = wireFieldDefinition(field.Name, def, encodingCtx.netflowV9) + if !fieldAllowedForEncoding(def, encodingCtx.netflowV9) { + continue + } + var encoded []byte + if ok { + encoded, err = encodeTemplatedValueForFamily(field.Name, def, val, schemaFieldFamily(field), encodingCtx) + if err != nil { + return netflow.DataRecord{}, fmt.Errorf("encode field %q: %w", field.Name, err) + } + } else { + encoded, err = defaultEncodedValue(def) + if err != nil { + return netflow.DataRecord{}, fmt.Errorf("default field %q: %w", field.Name, err) + } + } + values = append(values, netflow.DataField{ + PenProvided: def.EnterpriseScoped || def.PEN != 0, + Type: def.ID, + Length: def.Length, + Pen: def.PEN, + Value: encoded, + }) + } + } + if len(values) == 0 { + return netflow.DataRecord{}, fmt.Errorf("no encodable values found for templated packet") + } + return netflow.DataRecord{Values: values}, nil +} + +func buildOptionsDataRecordFromSchemaFields(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any, scopes, options []event.SchemaField, encodingCtx templatedEncodingContext) (netflow.OptionsDataRecord, error) { + scopeValues, err := buildOptionsDataFieldsFromSchemaFields(cfg, fieldMap, scopes, encodingCtx, true) + if err != nil { + return netflow.OptionsDataRecord{}, err + } + optionValues, err := buildOptionsDataFieldsFromSchemaFields(cfg, fieldMap, options, encodingCtx, false) + if err != nil { + return netflow.OptionsDataRecord{}, err + } + if len(scopeValues) == 0 { + return netflow.OptionsDataRecord{}, fmt.Errorf("no encodable scope values found for options packet") + } + if len(optionValues) == 0 { + return netflow.OptionsDataRecord{}, fmt.Errorf("no encodable option values found for options packet") + } + return netflow.OptionsDataRecord{ + ScopesValues: scopeValues, + OptionsValues: optionValues, + }, nil +} + +func buildOptionsDataFieldsFromSchemaFields(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any, fields []event.SchemaField, encodingCtx templatedEncodingContext, scope bool) ([]netflow.DataField, error) { + values := make([]netflow.DataField, 0, len(fields)) + for _, field := range fields { + def, ok := cfg.Catalog[field.Name] + if !ok { + continue + } + def = optionWireFieldDefinition(field.Name, def, encodingCtx.netflowV9, scope) + if !fieldAllowedForEncoding(def, encodingCtx.netflowV9) { + continue + } + val, ok := fieldMap[field.Name] + if !ok && field.Role == "static" && field.Value != nil { + val = field.Value + ok = true + } + var encoded []byte + var err error + if ok { + encoded, err = encodeTemplatedValue(field.Name, def, val, encodingCtx) + if err != nil { + return nil, fmt.Errorf("encode field %q: %w", field.Name, err) + } + } else { + encoded, err = defaultEncodedValue(def) + if err != nil { + return nil, fmt.Errorf("default field %q: %w", field.Name, err) + } + } + values = append(values, netflow.DataField{ + PenProvided: def.EnterpriseScoped || def.PEN != 0, + Type: def.ID, + Length: def.Length, + Pen: def.PEN, + Value: encoded, + }) + } + return values, nil +} + +func buildTemplateRecordFromSchemaFields(cfg config.TemplatedFlowDataConfig, schemaFields []event.SchemaField, templateID uint16, netflowV9 bool) (netflow.TemplateRecord, error) { + fields := make([]netflow.Field, 0, len(schemaFields)) + for _, field := range schemaFields { + defs, ok, err := schemaFieldDefinitions(cfg, field) + if err != nil { + return netflow.TemplateRecord{}, err + } + if !ok { + continue + } + for _, def := range defs { + def = wireFieldDefinition(field.Name, def, netflowV9) + if !fieldAllowedForEncoding(def, netflowV9) { + continue + } + fields = append(fields, netflow.Field{ + PenProvided: def.EnterpriseScoped || def.PEN != 0, + Type: def.ID, + Length: def.Length, + Pen: def.PEN, + }) + } + } + if len(fields) == 0 { + return netflow.TemplateRecord{}, fmt.Errorf("no encodable fields found for schema template") + } + return netflow.TemplateRecord{ + TemplateId: templateID, + FieldCount: uint16(len(fields)), + Fields: fields, + }, nil +} + +func buildIPFIXOptionsTemplateRecordFromSchemaFields(cfg config.TemplatedFlowDataConfig, scopes, options []event.SchemaField, templateID uint16) (netflow.IPFIXOptionsTemplateRecord, error) { + scopeFields, err := buildOptionsTemplateFields(cfg, scopes, false, true) + if err != nil { + return netflow.IPFIXOptionsTemplateRecord{}, err + } + optionFields, err := buildOptionsTemplateFields(cfg, options, false, false) + if err != nil { + return netflow.IPFIXOptionsTemplateRecord{}, err + } + if len(scopeFields) == 0 || len(optionFields) == 0 { + return netflow.IPFIXOptionsTemplateRecord{}, fmt.Errorf("options template %d requires at least one scope and option field", templateID) + } + return netflow.IPFIXOptionsTemplateRecord{ + TemplateId: templateID, + FieldCount: uint16(len(scopeFields) + len(optionFields)), + ScopeFieldCount: uint16(len(scopeFields)), + Scopes: scopeFields, + Options: optionFields, + }, nil +} + +func buildNFv9OptionsTemplateRecordFromSchemaFields(cfg config.TemplatedFlowDataConfig, scopes, options []event.SchemaField, templateID uint16) (netflow.NFv9OptionsTemplateRecord, error) { + scopeFields, err := buildOptionsTemplateFields(cfg, scopes, true, true) + if err != nil { + return netflow.NFv9OptionsTemplateRecord{}, err + } + optionFields, err := buildOptionsTemplateFields(cfg, options, true, false) + if err != nil { + return netflow.NFv9OptionsTemplateRecord{}, err + } + if len(scopeFields) == 0 || len(optionFields) == 0 { + return netflow.NFv9OptionsTemplateRecord{}, fmt.Errorf("options template %d requires at least one scope and option field", templateID) + } + return netflow.NFv9OptionsTemplateRecord{ + TemplateId: templateID, + ScopeLength: uint16(len(scopeFields) * 4), + OptionLength: uint16(len(optionFields) * 4), + Scopes: scopeFields, + Options: optionFields, + }, nil +} + +func buildOptionsTemplateFields(cfg config.TemplatedFlowDataConfig, schemaFields []event.SchemaField, netflowV9 bool, scope bool) ([]netflow.Field, error) { + fields := make([]netflow.Field, 0, len(schemaFields)) + for _, field := range schemaFields { + def, ok := cfg.Catalog[field.Name] + if !ok { + continue + } + def = optionWireFieldDefinition(field.Name, def, netflowV9, scope) + if !fieldAllowedForEncoding(def, netflowV9) { + continue + } + fields = append(fields, netflow.Field{ + PenProvided: def.EnterpriseScoped || def.PEN != 0, + Type: def.ID, + Length: def.Length, + Pen: def.PEN, + }) + } + return fields, nil +} + +// selectFlowFields uses the configured field whitelist when present, otherwise it +// exports all available fields in sorted order for determinism. +func selectFlowFields(cfg config.TemplatedFlowDataConfig, fieldMap map[string]any) []string { + if len(cfg.Select) > 0 { + return append([]string(nil), cfg.Select...) + } + names := make([]string, 0, len(fieldMap)) + for name := range fieldMap { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// encodeIPFIXValue encodes one canonical field into the wire representation +// expected by IPFIX or NetFlow v9. +func encodeIPFIXValue(def config.IPFIXFieldDefinition, val any) ([]byte, error) { + switch def.Type { + case "ipv4Address", "ipv6Address": + s, ok := val.(string) + if !ok { + return nil, fmt.Errorf("expected string IP, got %T", val) + } + addr, err := netip.ParseAddr(s) + if err != nil { + return nil, err + } + if def.Type == "ipv4Address" { + if !addr.Is4() { + return nil, fmt.Errorf("expected IPv4 address, got %s", s) + } + return append([]byte(nil), addr.AsSlice()...), nil + } + if !addr.Is6() { + return nil, fmt.Errorf("expected IPv6 address, got %s", s) + } + return append([]byte(nil), addr.AsSlice()...), nil + case "macAddress": + switch v := val.(type) { + case net.HardwareAddr: + if len(v) != 6 { + return nil, fmt.Errorf("expected 6-byte MAC address, got %d bytes", len(v)) + } + return append([]byte(nil), v...), nil + case []byte: + if len(v) != 6 { + return nil, fmt.Errorf("expected 6-byte MAC address, got %d bytes", len(v)) + } + return append([]byte(nil), v...), nil + case string: + hw, err := net.ParseMAC(v) + if err != nil { + return nil, err + } + if len(hw) != 6 { + return nil, fmt.Errorf("expected 6-byte MAC address, got %d bytes", len(hw)) + } + return append([]byte(nil), hw...), nil + default: + return nil, fmt.Errorf("expected MAC string/[]byte, got %T", val) + } + case "unsigned8", "unsigned16", "unsigned32", "unsigned64": + return encodeUnsigned(def.Type, val) + case "signed8", "signed16", "signed32", "signed64": + return encodeSigned(def.Type, val) + case "boolean": + if boolField(val) { + return []byte{1}, nil + } + return []byte{0}, nil + case "string": + switch v := val.(type) { + case string: + return []byte(v), nil + case []byte: + return append([]byte(nil), v...), nil + default: + return nil, fmt.Errorf("expected string/[]byte, got %T", val) + } + default: + switch v := val.(type) { + case []byte: + return append([]byte(nil), v...), nil + case string: + return []byte(v), nil + default: + return encodeUnsigned("unsigned64", val) + } + } +} + +func wireFieldDefinition(name string, def config.IPFIXFieldDefinition, netflowV9 bool) config.IPFIXFieldDefinition { + if !netflowV9 { + return def + } + switch name { + case "start_time_unix": + def.Name = "FIRST_SWITCHED" + def.ID = netflow.NFV9_FIELD_FIRST_SWITCHED + def.Length = 4 + def.Type = "unsigned32" + def.Format = "" + case "end_time_unix": + def.Name = "LAST_SWITCHED" + def.ID = netflow.NFV9_FIELD_LAST_SWITCHED + def.Length = 4 + def.Type = "unsigned32" + def.Format = "" + } + return def +} + +func optionWireFieldDefinition(name string, def config.IPFIXFieldDefinition, netflowV9 bool, scope bool) config.IPFIXFieldDefinition { + if !netflowV9 { + return def + } + if scope { + if id := defaultNetFlowV9ScopeID(name); id != 0 { + def.ID = id + def.Length = 4 + def.Type = "unsigned32" + def.PEN = 0 + def.EnterpriseScoped = false + return def + } + } + return wireFieldDefinition(name, def, netflowV9) +} + +func defaultNetFlowV9ScopeID(name string) uint16 { + switch name { + case "observation_domain_id", "source_id": + return 1 + case "if_index", "input_if": + return 2 + default: + return 0 + } +} + +func fieldAllowedForEncoding(def config.IPFIXFieldDefinition, netflowV9 bool) bool { + if !netflowV9 { + return true + } + return isNetFlowV9FieldID(def.ID) +} + +func isNetFlowV9FieldID(id uint16) bool { + name := netflow.NFv9TypeToString(id) + return name != "" && name != "Unassigned" +} + +func encodeTemplatedValue(name string, def config.IPFIXFieldDefinition, val any, encodingCtx templatedEncodingContext) ([]byte, error) { + if encodingCtx.netflowV9 { + switch name { + case "start_time_unix": + return encodeU32(encodingCtx.firstSwitched), nil + case "end_time_unix": + return encodeU32(encodingCtx.lastSwitched), nil + } + } + return encodeIPFIXValue(def, val) +} + +func encodeTemplatedValueForFamily(name string, def config.IPFIXFieldDefinition, val any, family string, encodingCtx templatedEncodingContext) ([]byte, error) { + if def.Type != "ipv4Address" && def.Type != "ipv6Address" { + if family != "" { + return nil, fmt.Errorf("family modifier %q requires IP address field", family) + } + return encodeTemplatedValue(name, def, val, encodingCtx) + } + ip, ok := val.(string) + if !ok { + return nil, fmt.Errorf("expected string IP, got %T", val) + } + addr, err := netip.ParseAddr(ip) + if err != nil { + return nil, err + } + if family == "ip4in6" && addr.Is4() { + ip = mappedIPv4String(addr) + addr, _ = netip.ParseAddr(ip) + } + switch def.Type { + case "ipv4Address": + if !addr.Is4() { + return defaultEncodedValue(def) + } + case "ipv6Address": + if !addr.Is6() { + return defaultEncodedValue(def) + } + } + return encodeTemplatedValue(name, def, ip, encodingCtx) +} + +// defaultEncodedValue provides a zero representation for fields omitted from a +// templated event but still required by the selected template. +func defaultEncodedValue(def config.IPFIXFieldDefinition) ([]byte, error) { + switch def.Type { + case "ipv4Address": + return make([]byte, 4), nil + case "ipv6Address": + return make([]byte, 16), nil + case "macAddress": + return make([]byte, 6), nil + case "unsigned8", "signed8", "boolean": + return make([]byte, 1), nil + case "unsigned16", "signed16": + return make([]byte, 2), nil + case "unsigned32", "signed32": + return make([]byte, 4), nil + case "unsigned64", "signed64": + return make([]byte, 8), nil + case "string": + if def.Length == 0xffff || def.Length == 0 { + return []byte{}, nil + } + return make([]byte, def.Length), nil + default: + if def.Length == 0xffff || def.Length == 0 { + return []byte{}, nil + } + return make([]byte, def.Length), nil + } +} + +// resolvedFieldDefinition upgrades address fields to their IPv6 definitions +// when the concrete runtime value contains an IPv6 address. +func resolvedFieldDefinition(name string, def config.IPFIXFieldDefinition, val any) config.IPFIXFieldDefinition { + ipStr, ok := val.(string) + if !ok { + return def + } + addr, err := netip.ParseAddr(ipStr) + if err != nil { + return def + } + switch { + case name == "agent_ip" && addr.Is6(): + def.Name = "exporterIPv6Address" + def.ID = netflow.IPFIX_FIELD_exporterIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + case name == "nat_src_addr" && isPostNATSourceIPv4Definition(def): + if addr.Is6() { + def.Name = "postNATSourceIPv6Address" + def.ID = netflow.IPFIX_FIELD_postNATSourceIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + } + case name == "nat_dst_addr" && isPostNATDestinationIPv4Definition(def): + if addr.Is6() { + def.Name = "postNATDestinationIPv6Address" + def.ID = netflow.IPFIX_FIELD_postNATDestinationIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + } + case isSourceAddressField(name) && isStandardSourceIPv4Definition(def): + if addr.Is6() { + def.Name = "sourceIPv6Address" + def.ID = netflow.IPFIX_FIELD_sourceIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + } + case isDestinationAddressField(name) && isStandardDestinationIPv4Definition(def): + if addr.Is6() { + def.Name = "destinationIPv6Address" + def.ID = netflow.IPFIX_FIELD_destinationIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + } + } + return def +} + +func fieldDefinitionsForFamily(name string, def config.IPFIXFieldDefinition, family string) ([]config.IPFIXFieldDefinition, error) { + if def.Name == "" { + def.Name = name + } + switch family { + case "": + if isDualStackAddressDefinition(name, def) { + return []config.IPFIXFieldDefinition{ + def, + resolvedFieldDefinitionForFamily(name, def, true), + }, nil + } + return []config.IPFIXFieldDefinition{def}, nil + case "ip4": + if !isIPAddressDefinition(def) { + return nil, fmt.Errorf("field %q family %q requires IP address field", name, family) + } + if def.Type != "ipv4Address" { + return nil, fmt.Errorf("field %q cannot be exported as IPv4", name) + } + return []config.IPFIXFieldDefinition{def}, nil + case "ip6", "ip4in6": + if !isIPAddressDefinition(def) { + return nil, fmt.Errorf("field %q family %q requires IP address field", name, family) + } + if def.Type == "ipv6Address" { + return []config.IPFIXFieldDefinition{def}, nil + } + promoted := resolvedFieldDefinitionForFamily(name, def, true) + if promoted.Type != "ipv6Address" || promoted.ID == def.ID { + return nil, fmt.Errorf("field %q cannot be exported as IPv6", name) + } + return []config.IPFIXFieldDefinition{promoted}, nil + default: + return nil, fmt.Errorf("unsupported field family %q", family) + } +} + +func schemaFieldFamily(field event.SchemaField) string { + if field.Role == "static" { + return "" + } + family, _ := ipFamilyModifier(field.Value) + return family +} + +func ipFamilyModifier(value any) (string, bool) { + family, ok := value.(string) + if !ok { + return "", false + } + switch family { + case "ip4", "ip6", "ip4in6": + return family, true + default: + return "", false + } +} + +func isIPAddressDefinition(def config.IPFIXFieldDefinition) bool { + return def.Type == "ipv4Address" || def.Type == "ipv6Address" +} + +func isDualStackAddressDefinition(name string, def config.IPFIXFieldDefinition) bool { + return isStandardSourceIPv4Definition(def) && isSourceAddressField(name) || + isStandardDestinationIPv4Definition(def) && isDestinationAddressField(name) || + isPostNATSourceIPv4Definition(def) && name == "nat_src_addr" || + isPostNATDestinationIPv4Definition(def) && name == "nat_dst_addr" +} + +func mappedIPv4String(addr netip.Addr) string { + raw := addr.As4() + mapped := netip.AddrFrom16([16]byte{10: 0xff, 11: 0xff, 12: raw[0], 13: raw[1], 14: raw[2], 15: raw[3]}) + return mapped.String() +} + +// resolvedFieldDefinitionForFamily performs the same promotion as +// resolvedFieldDefinition, but from a preselected IP family. +func resolvedFieldDefinitionForFamily(name string, def config.IPFIXFieldDefinition, ipv6 bool) config.IPFIXFieldDefinition { + if !ipv6 { + return def + } + switch { + case name == "agent_ip": + def.Name = "exporterIPv6Address" + def.ID = netflow.IPFIX_FIELD_exporterIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + case name == "nat_src_addr" && isPostNATSourceIPv4Definition(def): + def.Name = "postNATSourceIPv6Address" + def.ID = netflow.IPFIX_FIELD_postNATSourceIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + case name == "nat_dst_addr" && isPostNATDestinationIPv4Definition(def): + def.Name = "postNATDestinationIPv6Address" + def.ID = netflow.IPFIX_FIELD_postNATDestinationIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + case isSourceAddressField(name) && isStandardSourceIPv4Definition(def): + def.Name = "sourceIPv6Address" + def.ID = netflow.IPFIX_FIELD_sourceIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + case isDestinationAddressField(name) && isStandardDestinationIPv4Definition(def): + def.Name = "destinationIPv6Address" + def.ID = netflow.IPFIX_FIELD_destinationIPv6Address + def.Length = 16 + def.Type = "ipv6Address" + } + return def +} + +func isSourceAddressField(name string) bool { + return name == "src_addr" || strings.HasSuffix(name, "_src_addr") +} + +func isDestinationAddressField(name string) bool { + return name == "dst_addr" || strings.HasSuffix(name, "_dst_addr") +} + +func isStandardSourceIPv4Definition(def config.IPFIXFieldDefinition) bool { + return def.Type == "ipv4Address" && (def.ID == netflow.IPFIX_FIELD_sourceIPv4Address || def.Name == "sourceIPv4Address") +} + +func isStandardDestinationIPv4Definition(def config.IPFIXFieldDefinition) bool { + return def.Type == "ipv4Address" && (def.ID == netflow.IPFIX_FIELD_destinationIPv4Address || def.Name == "destinationIPv4Address") +} + +func isPostNATSourceIPv4Definition(def config.IPFIXFieldDefinition) bool { + return def.Type == "ipv4Address" && (def.ID == netflow.IPFIX_FIELD_postNATSourceIPv4Address || def.Name == "postNATSourceIPv4Address") +} + +func isPostNATDestinationIPv4Definition(def config.IPFIXFieldDefinition) bool { + return def.Type == "ipv4Address" && (def.ID == netflow.IPFIX_FIELD_postNATDestinationIPv4Address || def.Name == "postNATDestinationIPv4Address") +} + +func boolField(val any) bool { + switch v := val.(type) { + case bool: + return v + case uint64: + return v != 0 + case uint32: + return v != 0 + case uint16: + return v != 0 + case uint8: + return v != 0 + case int64: + return v != 0 + case int: + return v != 0 + case float64: + return v != 0 + case json.Number: + n, _ := v.Int64() + return n != 0 + case string: + return v == "true" || v == "1" + default: + return false + } +} + +// eventStream prefers the explicit event stream, then control stream, then a caller fallback. +func eventStream(evt *event.Event, fallback string) string { + if evt != nil && evt.Stream != "" { + return evt.Stream + } + if evt != nil && evt.Control != nil && evt.Control.Stream != "" { + return evt.Control.Stream + } + return fallback +} + +// controlType safely returns the event control type when present. +func controlType(evt *event.Event) string { + if evt == nil || evt.Control == nil { + return "" + } + return evt.Control.Type +} + +// encodeU32 writes one uint32 in big-endian order. +func encodeU32(v uint32) []byte { + return []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} +} + +// encodeU64 writes one uint64 in big-endian order. +func encodeU64(v uint64) []byte { + return []byte{ + byte(v >> 56), byte(v >> 48), byte(v >> 40), byte(v >> 32), + byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v), + } +} + +// ipfixFieldLength honors explicit field lengths and falls back to encoded size +// for variable-length definitions. +func ipfixFieldLength(def config.IPFIXFieldDefinition, encoded []byte) uint16 { + if def.Length != 0 { + return def.Length + } + if len(encoded) > 65535 { + return 0xffff + } + return uint16(len(encoded)) +} + +// encodeUnsigned serializes unsigned integer field kinds in big-endian order. +func encodeUnsigned(kind string, val any) ([]byte, error) { + v := uint64FromAny(val) + switch kind { + case "unsigned8": + return []byte{byte(v)}, nil + case "unsigned16": + return []byte{byte(v >> 8), byte(v)}, nil + case "unsigned32": + return []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + default: + return []byte{ + byte(v >> 56), byte(v >> 48), byte(v >> 40), byte(v >> 32), + byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v), + }, nil + } +} + +// encodeSigned serializes signed integer field kinds in big-endian order. +func encodeSigned(kind string, val any) ([]byte, error) { + v := int64Field(map[string]any{"v": val}, "v") + switch kind { + case "signed8": + return []byte{byte(v)}, nil + case "signed16": + return []byte{byte(v >> 8), byte(v)}, nil + case "signed32": + return []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + default: + return []byte{ + byte(v >> 56), byte(v >> 48), byte(v >> 40), byte(v >> 32), + byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v), + }, nil + } +} diff --git a/pkg/reflow/event/event.go b/pkg/reflow/event/event.go new file mode 100644 index 00000000..c5e1fe59 --- /dev/null +++ b/pkg/reflow/event/event.go @@ -0,0 +1,120 @@ +package event + +import ( + "encoding/json" + "time" +) + +type Event struct { + ReceivedAt time.Time `json:"received_at"` + Kind string `json:"kind,omitempty"` + Stream string `json:"stream,omitempty"` + Source SourceMetadata `json:"source"` + Control *ControlMetadata `json:"control,omitempty"` + Message json.RawMessage `json:"message,omitempty"` + Fields map[string]any `json:"fields,omitempty"` + Packet *PacketModel `json:"packet,omitempty"` + SFlow *SFlowMetadata `json:"sflow,omitempty"` + Aggregation *AggregationMetadata `json:"aggregation,omitempty"` + Payload any `json:"-"` + Internal map[string]any `json:"-"` +} + +type ControlMetadata struct { + Type string `json:"type,omitempty"` + Stream string `json:"stream,omitempty"` +} + +type AggregationSchema struct { + Stream string `json:"stream,omitempty"` + FieldNames []string `json:"field_names,omitempty"` + Fields []SchemaField `json:"fields,omitempty"` + KeyFields []string `json:"key_fields,omitempty"` + SumFields []string `json:"sum_fields,omitempty"` + FirstFields []string `json:"first_fields,omitempty"` + CurrentFields []string `json:"current_fields,omitempty"` + MinFields []string `json:"min_fields,omitempty"` + MaxFields []string `json:"max_fields,omitempty"` + AndFields []string `json:"and_fields,omitempty"` + Match map[string]string `json:"match,omitempty"` + StaticFields map[string]any `json:"static_fields,omitempty"` + BaseTemplateID uint16 `json:"base_template_id,omitempty"` +} + +type SchemaField struct { + Role string `json:"role,omitempty"` + Name string `json:"name,omitempty"` + Value any `json:"value,omitempty"` +} + +type SourceInit struct { + Stream string `json:"stream,omitempty"` + AgentIP string `json:"agent_ip,omitempty"` + SourceID uint32 `json:"source_id,omitempty"` + ObservationDomainID uint32 `json:"observation_domain_id,omitempty"` + SamplingRate uint32 `json:"sampling_rate,omitempty"` + SamplePool uint32 `json:"sample_pool,omitempty"` + Drops uint32 `json:"drops,omitempty"` + InputIf uint32 `json:"input_if,omitempty"` + OutputIf uint32 `json:"output_if,omitempty"` +} + +type SourceMetadata struct { + Network string `json:"network"` + Address string `json:"address"` + Remote string `json:"remote,omitempty"` + Type string `json:"type,omitempty"` + CaptureInterface string `json:"capture_interface,omitempty"` + CaptureInterfaceIndex int `json:"capture_interface_index,omitempty"` + CaptureDirection string `json:"capture_direction,omitempty"` + CapturePacketType string `json:"capture_packet_type,omitempty"` + AgentIP string `json:"agent_ip,omitempty"` + SourceID uint32 `json:"source_id,omitempty"` + SourceIDSet bool `json:"-"` + Sampling *SamplingMetadata `json:"sampling,omitempty"` + JSON JSONMetadata `json:"json,omitempty"` +} + +type JSONMetadata struct { + Flavor string `json:"flavor,omitempty"` +} + +type SamplingMetadata struct { + Rate uint32 `json:"rate,omitempty"` + SamplePool uint32 `json:"sample_pool,omitempty"` + Drops uint32 `json:"drops,omitempty"` +} + +type AggregationMetadata struct { + Key string `json:"key,omitempty"` + FirstSeenUnix int64 `json:"first_seen_unix,omitempty"` + LastSeenUnix int64 `json:"last_seen_unix,omitempty"` +} + +type SFlowMetadata struct { + AgentIP string `json:"agent_ip,omitempty"` + SubAgentID uint32 `json:"sub_agent_id,omitempty"` + SequenceNumber uint32 `json:"sequence_number,omitempty"` + Uptime uint32 `json:"uptime,omitempty"` + SourceID uint32 `json:"source_id,omitempty"` + SamplingRate uint32 `json:"sampling_rate,omitempty"` + SamplePool uint32 `json:"sample_pool,omitempty"` + Drops uint32 `json:"drops,omitempty"` + Samples []SFlowSampleMetadata `json:"samples,omitempty"` +} + +type SFlowSampleMetadata struct { + Enterprise uint32 `json:"enterprise,omitempty"` + Format uint32 `json:"format,omitempty"` + DataFormat uint32 `json:"data_format,omitempty"` + Data []byte `json:"data,omitempty"` + RawFlowRecords []SFlowRawRecord `json:"raw_flow_records,omitempty"` + RawCounterRecords []SFlowRawRecord `json:"raw_counter_records,omitempty"` +} + +type SFlowRawRecord struct { + Enterprise uint32 `json:"enterprise,omitempty"` + Format uint32 `json:"format,omitempty"` + DataFormat uint32 `json:"data_format,omitempty"` + Data []byte `json:"data,omitempty"` +} diff --git a/pkg/reflow/event/packet_model.go b/pkg/reflow/event/packet_model.go new file mode 100644 index 00000000..e4349d08 --- /dev/null +++ b/pkg/reflow/event/packet_model.go @@ -0,0 +1,165 @@ +package event + +import "net/netip" + +type PacketModel struct { + Layers []LayerSpec `json:"layers,omitempty"` + Features map[string]FeatureValue `json:"features,omitempty"` +} + +type LayerSpec struct { + Kind string `json:"kind"` + Name string `json:"name,omitempty"` + Length uint32 `json:"length,omitempty"` + Ethernet *EthernetLayer `json:"ethernet,omitempty"` + VLAN *VLANLayer `json:"vlan,omitempty"` + MPLS *MPLSLayer `json:"mpls,omitempty"` + IPv4 *IPv4Layer `json:"ipv4,omitempty"` + IPv6 *IPv6Layer `json:"ipv6,omitempty"` + GRE *GRELayer `json:"gre,omitempty"` + VXLAN *VXLANLayer `json:"vxlan,omitempty"` + Geneve *GeneveLayer `json:"geneve,omitempty"` + PPPoE *PPPoELayer `json:"pppoe,omitempty"` + TCP *TCPLayer `json:"tcp,omitempty"` + UDP *UDPLayer `json:"udp,omitempty"` + ICMP *ICMPLayer `json:"icmp,omitempty"` + Payload *PayloadLayer `json:"payload,omitempty"` + Features map[string]FeatureValue `json:"features,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +type EthernetLayer struct { + SrcMAC string `json:"src_mac,omitempty"` + DstMAC string `json:"dst_mac,omitempty"` + EtherType uint32 `json:"ether_type,omitempty"` +} + +type VLANLayer struct { + ID uint16 `json:"id,omitempty"` + PCP uint8 `json:"pcp,omitempty"` + DEI bool `json:"dei,omitempty"` + TPID uint16 `json:"tpid,omitempty"` +} + +type MPLSLabel struct { + Label uint32 `json:"label,omitempty"` + TC uint8 `json:"tc,omitempty"` + BOS bool `json:"bos,omitempty"` + TTL uint8 `json:"ttl,omitempty"` +} + +type MPLSLayer struct { + Label MPLSLabel `json:"label"` +} + +type IPv4Layer struct { + SrcAddr netip.Addr `json:"src_addr,omitempty"` + DstAddr netip.Addr `json:"dst_addr,omitempty"` + Protocol uint8 `json:"protocol,omitempty"` + TTL uint8 `json:"ttl,omitempty"` + DSCP uint8 `json:"dscp,omitempty"` + ECN uint8 `json:"ecn,omitempty"` + Identification uint16 `json:"identification,omitempty"` + Flags uint8 `json:"flags,omitempty"` + FragmentOffset uint16 `json:"fragment_offset,omitempty"` +} + +type IPv6Layer struct { + SrcAddr netip.Addr `json:"src_addr,omitempty"` + DstAddr netip.Addr `json:"dst_addr,omitempty"` + NextHeader uint8 `json:"next_header,omitempty"` + HopLimit uint8 `json:"hop_limit,omitempty"` + TrafficClass uint8 `json:"traffic_class,omitempty"` + FlowLabel uint32 `json:"flow_label,omitempty"` +} + +type GRELayer struct { + Protocol uint16 `json:"protocol,omitempty"` + Checksum bool `json:"checksum,omitempty"` + Key bool `json:"key,omitempty"` + Sequence bool `json:"sequence,omitempty"` +} + +type VXLANLayer struct { + VNI uint32 `json:"vni,omitempty"` +} + +type GeneveLayer struct { + VNI uint32 `json:"vni,omitempty"` + Protocol uint16 `json:"protocol,omitempty"` + Options map[string]FeatureValue `json:"options,omitempty"` +} + +type PPPoELayer struct { + SessionID uint16 `json:"session_id,omitempty"` + Protocol uint16 `json:"protocol,omitempty"` +} + +type TCPLayer struct { + SrcPort uint16 `json:"src_port,omitempty"` + DstPort uint16 `json:"dst_port,omitempty"` + Flags uint8 `json:"flags,omitempty"` + Seq uint32 `json:"seq,omitempty"` + Ack uint32 `json:"ack,omitempty"` + Window uint16 `json:"window,omitempty"` +} + +type UDPLayer struct { + SrcPort uint16 `json:"src_port,omitempty"` + DstPort uint16 `json:"dst_port,omitempty"` +} + +type ICMPLayer struct { + Type uint8 `json:"type,omitempty"` + Code uint8 `json:"code,omitempty"` +} + +type PayloadLayer struct { + Length uint32 `json:"length,omitempty"` + Pattern string `json:"pattern,omitempty"` +} + +type FeatureValue struct { + String *string `json:"string,omitempty"` + Uint64 *uint64 `json:"uint64,omitempty"` + Int64 *int64 `json:"int64,omitempty"` + Bool *bool `json:"bool,omitempty"` + Bytes []byte `json:"bytes,omitempty"` + List []FeatureValue `json:"list,omitempty"` + Map map[string]FeatureValue `json:"map,omitempty"` +} + +func FeatureString(v string) FeatureValue { + return FeatureValue{String: &v} +} + +func FeatureUint64(v uint64) FeatureValue { + return FeatureValue{Uint64: &v} +} + +func FeatureInt64(v int64) FeatureValue { + return FeatureValue{Int64: &v} +} + +func FeatureBool(v bool) FeatureValue { + return FeatureValue{Bool: &v} +} + +func FeatureBytes(v []byte) FeatureValue { + return FeatureValue{Bytes: append([]byte(nil), v...)} +} + +func FeatureList(v ...FeatureValue) FeatureValue { + return FeatureValue{List: append([]FeatureValue(nil), v...)} +} + +func FeatureMap(v map[string]FeatureValue) FeatureValue { + if len(v) == 0 { + return FeatureValue{} + } + out := make(map[string]FeatureValue, len(v)) + for k, item := range v { + out[k] = item + } + return FeatureValue{Map: out} +} diff --git a/pkg/reflow/packet/normalize.go b/pkg/reflow/packet/normalize.go new file mode 100644 index 00000000..e1362967 --- /dev/null +++ b/pkg/reflow/packet/normalize.go @@ -0,0 +1,2039 @@ +package packet + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "net/netip" + "strings" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type NormalizeOptions struct { + DisablePacketMapping bool + TruncatePacketBytes int + UsePayloadAsPacket bool + TruncatePayload bool + HeaderProtocol uint32 + Decode DecodeOptions + AggregationHelpers AggregationHelperOptions + Extractors []FeatureExtractor +} + +type AggregationHelperOptions struct { + MPLSLabels int + IPLayers int +} + +type DecodeOptions struct { + Configured bool + DecodeBeyondL4 bool + DecodeGRE bool + DecodeIPIP bool + DecodeVXLAN bool + VXLANPorts []uint32 + DecodeGeneve bool + GenevePorts []uint32 + DecodeL2TP bool + L2TPPorts []uint32 + DecodeGTPU bool + GTPUPorts []uint32 + DecodePPPoE bool +} + +var defaultDecodeOptions = DecodeOptions{ + Configured: true, + DecodeBeyondL4: true, + DecodeGRE: true, + DecodeIPIP: true, + DecodeVXLAN: true, + DecodeGeneve: true, + DecodeL2TP: true, + DecodeGTPU: true, + DecodePPPoE: true, +} + +func DefaultDecodeOptions() DecodeOptions { + opts := defaultDecodeOptions + opts.VXLANPorts = []uint32{4789} + opts.GenevePorts = []uint32{6081} + opts.L2TPPorts = []uint32{1701} + opts.GTPUPorts = []uint32{2152} + return opts +} + +func (opts DecodeOptions) withDefaults() DecodeOptions { + if !opts.Configured { + return defaultDecodeOptions + } + return opts +} + +type FeatureExtractor interface { + Name() string + Extract(*event.PacketModel, []byte) error +} + +// NormalizeEvent applies the shared packet-field normalization used by raw +// packet sources and packet-carrying decoded flow records such as sFlow sampled +// headers. +func NormalizeEvent(evt *event.Event, opts NormalizeOptions) error { + if evt == nil { + return fmt.Errorf("normalize packet event: nil event") + } + + fields := ensureFields(evt, 16) + if opts.UsePayloadAsPacket { + payload, ok := evt.Payload.([]byte) + if !ok || len(payload) == 0 { + return fmt.Errorf("normalize packet event: missing payload bytes") + } + fields["header_data"] = append([]byte(nil), payload...) + } + + headerData := bytesField(fields, "header_data") + if len(headerData) == 0 { + return nil + } + + if evt.ReceivedAt.IsZero() { + evt.ReceivedAt = time.Now().UTC() + } + if _, ok := fields["record_kind"]; !ok { + fields["record_kind"] = "packet" + } + if fieldUint32(fields, "frame_length") == 0 { + fields["frame_length"] = uint32(len(headerData)) + } + if fieldUint32(fields, "original_length") == 0 { + fields["original_length"] = uint32(len(headerData)) + } + if _, ok := fields["bytes"]; !ok { + fields["bytes"] = int64(fieldUint32(fields, "frame_length")) + } + if _, ok := fields["packets"]; !ok { + fields["packets"] = int64(1) + } + if _, ok := fields["start_time_unix"]; !ok { + fields["start_time_unix"] = evt.ReceivedAt.UnixMilli() + } + if _, ok := fields["end_time_unix"]; !ok { + fields["end_time_unix"] = evt.ReceivedAt.UnixMilli() + } + if opts.HeaderProtocol != 0 && fieldUint32(fields, "protocol") == 0 { + fields["protocol"] = opts.HeaderProtocol + } + if opts.HeaderProtocol != 0 { + if _, ok := fields["header_protocol_name"]; !ok { + fields["header_protocol_name"] = sampledHeaderProtocolName(opts.HeaderProtocol) + } + } + setDefaultInterfaces(evt, fields) + + if view, err := parsePacketViewWithOptions(headerData, opts.HeaderProtocol, opts.Decode.withDefaults()); err == nil { + evt.Packet = view.Model + for _, extractor := range opts.Extractors { + if extractor == nil || evt.Packet == nil { + continue + } + if err := extractor.Extract(evt.Packet, headerData); err != nil { + return fmt.Errorf("extract packet feature %q: %w", extractor.Name(), err) + } + } + if !opts.DisablePacketMapping { + applyPacketViewFields(fields, view, opts.AggregationHelpers) + } + } + truncatePacketData(evt, fields, opts.TruncatePacketBytes, opts.TruncatePayload) + return nil +} + +func setDefaultInterfaces(evt *event.Event, fields map[string]any) { + if fieldUint32(fields, "input_if") != 0 || fieldUint32(fields, "output_if") != 0 { + return + } + if evt.Source.CaptureInterfaceIndex <= 0 { + return + } + ifIndex := uint32(evt.Source.CaptureInterfaceIndex) + fields["input_if"] = ifIndex + fields["output_if"] = ifIndex +} + +func truncatePacketData(evt *event.Event, fields map[string]any, maxBytes int, truncatePayload bool) { + if maxBytes <= 0 { + return + } + headerData := bytesField(fields, "header_data") + if len(headerData) > maxBytes { + fields["header_data"] = append([]byte(nil), headerData[:maxBytes]...) + } + if !truncatePayload { + return + } + if payload, ok := evt.Payload.([]byte); ok && len(payload) > maxBytes { + evt.Payload = append([]byte(nil), payload[:maxBytes]...) + } +} + +func bytesField(fields map[string]any, key string) []byte { + if fields == nil { + return nil + } + val, ok := fields[key] + if !ok { + return nil + } + switch v := val.(type) { + case []byte: + return v + case string: + return []byte(v) + default: + return nil + } +} + +// BuildPseudoHeader synthesizes minimal packet bytes from a packet model or +// canonical tuple fields. It is intended for encoders that require packet bytes +// even when the input event only carries flow fields. +func BuildPseudoHeader(evt *event.Event, fields map[string]any) ([]byte, bool) { + var model *event.PacketModel + if evt != nil { + model = evt.Packet + } + if model == nil || len(model.Layers) == 0 { + model = pseudoPacketModelFromFields(fields) + } + if model == nil || len(model.Layers) == 0 { + return nil, false + } + ensurePacketModelLayerLengths(model) + data, err := encodePacketModel(model) + if err != nil || len(data) == 0 { + return nil, false + } + if evt != nil && evt.Packet == nil { + evt.Packet = model + } + return data, true +} + +func pseudoPacketModelFromFields(fields map[string]any) *event.PacketModel { + inner := pseudoIPLayerFromFields(fields, "src_addr", "dst_addr", "proto", "src_port", "dst_port") + outer := pseudoIPLayerFromFields(fields, "outer_src_addr", "outer_dst_addr", "outer_proto", "outer_src_port", "outer_dst_port") + if !inner.Valid() { + return nil + } + + model := &event.PacketModel{ + Features: make(map[string]event.FeatureValue), + } + + if pseudoPacketHasLinkLayer(fields) { + dstMAC := fieldStringOrZero(fields, "dst_mac") + if dstMAC == "" { + // Pseudo packets may need an L2 envelope without source metadata. + // Use zero MACs so the frame remains structurally valid. + dstMAC = "00:00:00:00:00:00" + } + srcMAC := fieldStringOrZero(fields, "src_mac") + if srcMAC == "" { + srcMAC = "00:00:00:00:00:00" + } + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "ethernet", + Ethernet: &event.EthernetLayer{ + SrcMAC: srcMAC, + DstMAC: dstMAC, + }, + }) + + appendVLANLayers(model, fields) + appendMPLSLayers(model, fields) + appendPPPoELayer(model, fields) + } + + if outer.Valid() { + appendPseudoIPLayer(model, outer.SrcAddr, outer.DstAddr, outer.Proto) + appendPseudoTunnelLayers(model, outer, inner) + } + + appendPseudoIPLayer(model, inner.SrcAddr, inner.DstAddr, inner.Proto) + appendPseudoTransportLayer(model, inner.Proto, inner.SrcPort, inner.DstPort) + + if frameLen := fieldUint32(fields, "original_length"); frameLen != 0 { + model.Features["target_wire_length"] = event.FeatureUint64(uint64(frameLen)) + } + return model +} + +func pseudoPacketHasLinkLayer(fields map[string]any) bool { + if fieldStringOrZero(fields, "src_mac") != "" || fieldStringOrZero(fields, "dst_mac") != "" { + return true + } + if fieldUint32(fields, "ether_type") != 0 || + fieldUint32(fields, "vlan_id") != 0 || + fieldUint32(fields, "mpls_label") != 0 || + fieldUint32(fields, "pppoe_session_id") != 0 { + return true + } + if vals, ok := fields["vlan_ids"].([]uint32); ok && len(vals) > 0 { + return true + } + return false +} + +type pseudoIPLayer struct { + SrcAddr netip.Addr + DstAddr netip.Addr + Proto uint32 + SrcPort uint32 + DstPort uint32 +} + +func (l pseudoIPLayer) Valid() bool { + return l.SrcAddr.IsValid() && l.DstAddr.IsValid() +} + +func pseudoIPLayerFromFields(fields map[string]any, srcKey, dstKey, protoKey, srcPortKey, dstPortKey string) pseudoIPLayer { + srcAddr, err := netip.ParseAddr(fieldStringOrZero(fields, srcKey)) + if err != nil { + return pseudoIPLayer{} + } + dstAddr, err := netip.ParseAddr(fieldStringOrZero(fields, dstKey)) + if err != nil { + return pseudoIPLayer{} + } + return pseudoIPLayer{ + SrcAddr: srcAddr, + DstAddr: dstAddr, + Proto: fieldUint32(fields, protoKey), + SrcPort: fieldUint32(fields, srcPortKey), + DstPort: fieldUint32(fields, dstPortKey), + } +} + +func appendVLANLayers(model *event.PacketModel, fields map[string]any) { + if model == nil { + return + } + if vals, ok := fields["vlan_ids"].([]uint32); ok && len(vals) > 0 { + for _, id := range vals { + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "dot1q", + VLAN: &event.VLANLayer{ID: uint16(id), TPID: 0x8100}, + }) + } + return + } + if id := fieldUint32(fields, "vlan_id"); id != 0 { + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "dot1q", + VLAN: &event.VLANLayer{ID: uint16(id), TPID: 0x8100}, + }) + } +} + +func appendMPLSLayers(model *event.PacketModel, fields map[string]any) { + if model == nil { + return + } + for i := 1; i <= 3; i++ { + if label := fieldUint32(fields, fmt.Sprintf("mpls_label%d", i)); label != 0 { + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "mpls", + MPLS: &event.MPLSLayer{ + Label: event.MPLSLabel{ + Label: label, + BOS: true, + TTL: 64, + }, + }, + }) + } + } + if len(model.Layers) > 0 && model.Layers[len(model.Layers)-1].Kind == "mpls" { + return + } + if label := fieldUint32(fields, "mpls_label"); label != 0 { + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "mpls", + MPLS: &event.MPLSLayer{ + Label: event.MPLSLabel{ + Label: label, + BOS: true, + TTL: 64, + }, + }, + }) + } +} + +func appendPPPoELayer(model *event.PacketModel, fields map[string]any) { + if model == nil { + return + } + sessionID := fieldUint32(fields, "pppoe_session_id") + if sessionID == 0 { + return + } + proto := uint16(0x0021) + if strings.Contains(fieldStringOrZero(fields, "src_addr"), ":") { + proto = 0x0057 + } + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "pppoe", + PPPoE: &event.PPPoELayer{ + SessionID: uint16(sessionID), + Protocol: proto, + }, + }) +} + +func appendPseudoTunnelLayers(model *event.PacketModel, outer, inner pseudoIPLayer) { + if model == nil { + return + } + switch pseudoTunnelKind(outer) { + case "gre": + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "gre", + GRE: &event.GRELayer{Protocol: pseudoInnerEtherType(inner.SrcAddr, inner.DstAddr)}, + }) + case "vxlan": + model.Layers = append(model.Layers, + event.LayerSpec{Kind: "udp", UDP: &event.UDPLayer{SrcPort: uint16(outer.SrcPort), DstPort: uint16(outer.DstPort)}}, + event.LayerSpec{Kind: "vxlan", VXLAN: &event.VXLANLayer{}}, + event.LayerSpec{Kind: "ethernet", Ethernet: &event.EthernetLayer{SrcMAC: "00:00:00:00:00:00", DstMAC: "00:00:00:00:00:00"}}, + ) + case "geneve": + model.Layers = append(model.Layers, + event.LayerSpec{Kind: "udp", UDP: &event.UDPLayer{SrcPort: uint16(outer.SrcPort), DstPort: uint16(outer.DstPort)}}, + event.LayerSpec{Kind: "geneve", Geneve: &event.GeneveLayer{Protocol: pseudoInnerEtherType(inner.SrcAddr, inner.DstAddr)}}, + event.LayerSpec{Kind: "ethernet", Ethernet: &event.EthernetLayer{SrcMAC: "00:00:00:00:00:00", DstMAC: "00:00:00:00:00:00"}}, + ) + } +} + +func pseudoTunnelKind(outer pseudoIPLayer) string { + switch { + case outer.Proto == 47: + return "gre" + case outer.Proto == 17 && (outer.SrcPort == 4789 || outer.DstPort == 4789): + return "vxlan" + case outer.Proto == 17 && (outer.SrcPort == 6081 || outer.DstPort == 6081): + return "geneve" + default: + return "" + } +} + +func appendPseudoIPLayer(model *event.PacketModel, srcAddr, dstAddr netip.Addr, proto uint32) { + if srcAddr.Is4() && dstAddr.Is4() { + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "ipv4", + IPv4: &event.IPv4Layer{ + SrcAddr: srcAddr, + DstAddr: dstAddr, + Protocol: uint8(proto), + TTL: 64, + }, + }) + return + } + if srcAddr.Is6() && dstAddr.Is6() { + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "ipv6", + IPv6: &event.IPv6Layer{ + SrcAddr: srcAddr, + DstAddr: dstAddr, + NextHeader: uint8(proto), + HopLimit: 64, + }, + }) + } +} + +func appendPseudoTransportLayer(model *event.PacketModel, proto, srcPort, dstPort uint32) { + switch proto { + case 6: + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "tcp", + TCP: &event.TCPLayer{ + SrcPort: uint16(srcPort), + DstPort: uint16(dstPort), + Flags: 0x02, + Window: 65535, + }, + }) + case 17: + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "udp", + UDP: &event.UDPLayer{ + SrcPort: uint16(srcPort), + DstPort: uint16(dstPort), + }, + }) + case 1, 58: + model.Layers = append(model.Layers, event.LayerSpec{ + Kind: "icmp", + ICMP: &event.ICMPLayer{}, + }) + } +} + +func pseudoInnerEtherType(srcAddr, dstAddr netip.Addr) uint16 { + if srcAddr.Is6() && dstAddr.Is6() { + return 0x86dd + } + return 0x0800 +} + +func encodePacketModel(model *event.PacketModel) ([]byte, error) { + if model == nil || len(model.Layers) == 0 { + return nil, fmt.Errorf("empty packet model") + } + payload := []byte(nil) + for i := len(model.Layers) - 1; i >= 0; i-- { + layer := model.Layers[i] + var next event.LayerSpec + if i+1 < len(model.Layers) { + next = model.Layers[i+1] + } + var err error + payload, err = prependLayer(layer, payload, next) + if err != nil { + return nil, err + } + } + return payload, nil +} + +func prependLayer(layer event.LayerSpec, payload []byte, next event.LayerSpec) ([]byte, error) { + switch layer.Kind { + case "ethernet": + return prependEthernet(layer.Ethernet, payload, next) + case "dot1q": + return prependVLAN(layer.VLAN, payload, next) + case "mpls": + return prependMPLS(layer.MPLS, payload) + case "pppoe": + return prependPPPoE(layer.PPPoE, payload) + case "ipv4": + return prependIPv4(layer.IPv4, payload) + case "ipv6": + return prependIPv6(layer.IPv6, payload) + case "gre": + return prependGRE(layer.GRE, payload) + case "udp": + return prependUDP(layer.UDP, payload) + case "tcp": + return prependTCP(layer.TCP, payload) + case "vxlan": + return prependVXLAN(layer.VXLAN, payload) + case "geneve": + return prependGeneve(layer.Geneve, payload) + case "icmp", "icmpv6": + return prependICMP(layer.ICMP, payload), nil + case "payload": + return prependPayload(layer.Payload, payload), nil + default: + return payload, nil + } +} + +func prependEthernet(layer *event.EthernetLayer, payload []byte, next event.LayerSpec) ([]byte, error) { + if layer == nil { + layer = &event.EthernetLayer{} + } + out := make([]byte, 14+len(payload)) + copy(out[0:6], parseMACOrZero(layer.DstMAC)) + copy(out[6:12], parseMACOrZero(layer.SrcMAC)) + etherType := inferLayerEtherType(layer.EtherType, next, payload) + binary.BigEndian.PutUint16(out[12:14], uint16(etherType)) + copy(out[14:], payload) + return out, nil +} + +func prependVLAN(layer *event.VLANLayer, payload []byte, next event.LayerSpec) ([]byte, error) { + if layer == nil { + layer = &event.VLANLayer{TPID: 0x8100} + } + out := make([]byte, 4+len(payload)) + tci := uint16(layer.ID&0x0fff) | uint16(layer.PCP&0x7)<<13 + if layer.DEI { + tci |= 1 << 12 + } + binary.BigEndian.PutUint16(out[0:2], tci) + binary.BigEndian.PutUint16(out[2:4], uint16(inferLayerEtherType(0, next, payload))) + copy(out[4:], payload) + return out, nil +} + +func prependMPLS(layer *event.MPLSLayer, payload []byte) ([]byte, error) { + if layer == nil { + layer = &event.MPLSLayer{Label: event.MPLSLabel{BOS: true, TTL: 64}} + } + out := make([]byte, 4+len(payload)) + label := (layer.Label.Label & 0xfffff) << 12 + label |= uint32(layer.Label.TC&0x7) << 9 + if layer.Label.BOS { + label |= 1 << 8 + } + label |= uint32(layer.Label.TTL) + binary.BigEndian.PutUint32(out[0:4], label) + copy(out[4:], payload) + return out, nil +} + +func prependPPPoE(layer *event.PPPoELayer, payload []byte) ([]byte, error) { + if layer == nil { + layer = &event.PPPoELayer{Protocol: 0x0021} + } + out := make([]byte, 8+len(payload)) + out[0] = 0x11 + out[1] = 0x00 + binary.BigEndian.PutUint16(out[2:4], layer.SessionID) + binary.BigEndian.PutUint16(out[4:6], uint16(len(payload)+2)) + binary.BigEndian.PutUint16(out[6:8], layer.Protocol) + copy(out[8:], payload) + return out, nil +} + +func prependIPv4(layer *event.IPv4Layer, payload []byte) ([]byte, error) { + if layer == nil { + return nil, fmt.Errorf("missing ipv4 layer") + } + out := make([]byte, 20+len(payload)) + out[0] = 0x45 + out[1] = byte((layer.DSCP << 2) | (layer.ECN & 0x3)) + binary.BigEndian.PutUint16(out[2:4], uint16(len(out))) + binary.BigEndian.PutUint16(out[4:6], layer.Identification) + flagsFrag := uint16(layer.Flags&0x7)<<13 | (layer.FragmentOffset & 0x1fff) + binary.BigEndian.PutUint16(out[6:8], flagsFrag) + ttl := layer.TTL + if ttl == 0 { + ttl = 64 + } + out[8] = ttl + out[9] = layer.Protocol + copy(out[12:16], layer.SrcAddr.AsSlice()) + copy(out[16:20], layer.DstAddr.AsSlice()) + copy(out[20:], payload) + return out, nil +} + +func prependIPv6(layer *event.IPv6Layer, payload []byte) ([]byte, error) { + if layer == nil { + return nil, fmt.Errorf("missing ipv6 layer") + } + out := make([]byte, 40+len(payload)) + out[0] = 0x60 | (layer.TrafficClass >> 4) + out[1] = (layer.TrafficClass << 4) | byte((layer.FlowLabel>>16)&0x0f) + out[2] = byte(layer.FlowLabel >> 8) + out[3] = byte(layer.FlowLabel) + binary.BigEndian.PutUint16(out[4:6], uint16(len(payload))) + out[6] = layer.NextHeader + hopLimit := layer.HopLimit + if hopLimit == 0 { + hopLimit = 64 + } + out[7] = hopLimit + copy(out[8:24], layer.SrcAddr.AsSlice()) + copy(out[24:40], layer.DstAddr.AsSlice()) + copy(out[40:], payload) + return out, nil +} + +func prependGRE(layer *event.GRELayer, payload []byte) ([]byte, error) { + if layer == nil { + layer = &event.GRELayer{Protocol: uint16(inferEtherType(0, payload))} + } + offset := 4 + flags := uint16(0) + if layer.Checksum { + flags |= 0x8000 + offset += 4 + } + if layer.Key { + flags |= 0x2000 + offset += 4 + } + if layer.Sequence { + flags |= 0x1000 + offset += 4 + } + out := make([]byte, offset+len(payload)) + binary.BigEndian.PutUint16(out[0:2], flags) + binary.BigEndian.PutUint16(out[2:4], layer.Protocol) + copy(out[offset:], payload) + return out, nil +} + +func prependUDP(layer *event.UDPLayer, payload []byte) ([]byte, error) { + if layer == nil { + layer = &event.UDPLayer{} + } + out := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint16(out[0:2], layer.SrcPort) + binary.BigEndian.PutUint16(out[2:4], layer.DstPort) + binary.BigEndian.PutUint16(out[4:6], uint16(len(out))) + copy(out[8:], payload) + return out, nil +} + +func prependTCP(layer *event.TCPLayer, payload []byte) ([]byte, error) { + if layer == nil { + layer = &event.TCPLayer{Flags: 0x02, Window: 65535} + } + out := make([]byte, 20+len(payload)) + binary.BigEndian.PutUint16(out[0:2], layer.SrcPort) + binary.BigEndian.PutUint16(out[2:4], layer.DstPort) + binary.BigEndian.PutUint32(out[4:8], layer.Seq) + binary.BigEndian.PutUint32(out[8:12], layer.Ack) + out[12] = 0x50 + out[13] = layer.Flags + window := layer.Window + if window == 0 { + window = 65535 + } + binary.BigEndian.PutUint16(out[14:16], window) + copy(out[20:], payload) + return out, nil +} + +func prependVXLAN(layer *event.VXLANLayer, payload []byte) ([]byte, error) { + if layer == nil { + layer = &event.VXLANLayer{} + } + out := make([]byte, 8+len(payload)) + out[0] = 0x08 + out[4] = byte(layer.VNI >> 16) + out[5] = byte(layer.VNI >> 8) + out[6] = byte(layer.VNI) + copy(out[8:], payload) + return out, nil +} + +func prependGeneve(layer *event.GeneveLayer, payload []byte) ([]byte, error) { + if layer == nil { + layer = &event.GeneveLayer{Protocol: uint16(inferEtherType(0, payload))} + } + out := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint16(out[2:4], layer.Protocol) + out[4] = byte(layer.VNI >> 16) + out[5] = byte(layer.VNI >> 8) + out[6] = byte(layer.VNI) + copy(out[8:], payload) + return out, nil +} + +func prependICMP(layer *event.ICMPLayer, payload []byte) []byte { + if layer == nil { + layer = &event.ICMPLayer{} + } + out := make([]byte, 8+len(payload)) + out[0] = layer.Type + out[1] = layer.Code + copy(out[8:], payload) + return out +} + +func prependPayload(layer *event.PayloadLayer, payload []byte) []byte { + if layer == nil || layer.Length == 0 { + return payload + } + if len(payload) >= int(layer.Length) { + return payload + } + fill := make([]byte, int(layer.Length)-len(payload)) + switch layer.Pattern { + case "repeat": + for i := range fill { + fill[i] = 0xaa + } + } + return append(fill, payload...) +} + +func inferEtherType(explicit uint32, payload []byte) uint32 { + if explicit != 0 { + return explicit + } + if len(payload) == 0 { + return 0x0800 + } + switch payload[0] >> 4 { + case 4: + return 0x0800 + case 6: + return 0x86dd + default: + return 0x6558 + } +} + +func inferLayerEtherType(explicit uint32, next event.LayerSpec, payload []byte) uint32 { + if explicit != 0 { + return explicit + } + switch next.Kind { + case "dot1q": + if next.VLAN != nil && next.VLAN.TPID != 0 { + return uint32(next.VLAN.TPID) + } + return 0x8100 + case "mpls": + return 0x8847 + case "pppoe": + return 0x8864 + case "ipv4": + return 0x0800 + case "ipv6": + return 0x86dd + case "arp": + return 0x0806 + default: + return inferEtherType(0, payload) + } +} + +func parseMACOrZero(raw string) []byte { + out := make([]byte, 6) + if raw == "" { + return out + } + var vals [6]byte + if _, err := fmt.Sscanf(raw, "%02x:%02x:%02x:%02x:%02x:%02x", &vals[0], &vals[1], &vals[2], &vals[3], &vals[4], &vals[5]); err != nil { + return out + } + copy(out, vals[:]) + return out +} + +type packetTuple struct { + SrcAddr netip.Addr + DstAddr netip.Addr + Proto uint32 + SrcPort uint32 + DstPort uint32 + TCPFlags uint32 +} + +type packetView struct { + Layers []string + SrcMAC string + DstMAC string + EtherType uint32 + VLANIDs []uint32 + MPLSLabels []uint32 + Tuples []packetTuple + Model *event.PacketModel +} + +func (v *packetView) appendLayer(layer event.LayerSpec) { + ensureLayerLength(&layer) + v.Layers = append(v.Layers, layer.Kind) + if v.Model == nil { + v.Model = &event.PacketModel{} + } + v.Model.Layers = append(v.Model.Layers, layer) +} + +func ensurePacketModelLayerLengths(model *event.PacketModel) { + if model == nil { + return + } + for i := range model.Layers { + ensureLayerLength(&model.Layers[i]) + } +} + +func ensureLayerLength(layer *event.LayerSpec) { + if layer == nil || layer.Length != 0 { + return + } + layer.Length = defaultLayerLength(*layer) +} + +func defaultLayerLength(layer event.LayerSpec) uint32 { + switch layer.Kind { + case "ethernet": + return 14 + case "dot1q", "mpls": + return 4 + case "ipv4": + return 20 + case "ipv6": + return 40 + case "gre": + length := uint32(4) + if layer.GRE != nil { + if layer.GRE.Checksum { + length += 4 + } + if layer.GRE.Key { + length += 4 + } + if layer.GRE.Sequence { + length += 4 + } + } + return length + case "vxlan", "geneve", "pppoe", "udp", "icmp", "icmpv6": + return 8 + case "tcp": + return 20 + case "sctp": + return 12 + case "payload": + if layer.Payload != nil { + return layer.Payload.Length + } + } + return 0 +} + +func applyPacketViewFields(fields map[string]any, view packetView, helpers AggregationHelperOptions) { + if view.SrcMAC != "" { + fields["src_mac"] = view.SrcMAC + } + if view.DstMAC != "" { + fields["dst_mac"] = view.DstMAC + } + if view.EtherType != 0 { + fields["ether_type"] = view.EtherType + } + if view.Model != nil { + fields["packet_ip_depth"] = uint32(packetIPDepth(view.Model)) + applyDot1QFields(fields, view.Model) + applyLayer2SegmentFields(fields, view.Model) + } + applyAggregationHelperFields(fields, view, helpers) + if len(view.VLANIDs) > 0 { + fields["vlan_ids"] = append([]uint32(nil), view.VLANIDs...) + fields["vlan_id"] = view.VLANIDs[0] + } + if len(view.MPLSLabels) > 0 { + fields["mpls_label"] = view.MPLSLabels[0] + for i, label := range view.MPLSLabels { + if i >= 3 { + break + } + fields[fmt.Sprintf("mpls_label%d", i+1)] = label + } + } + if len(view.Tuples) > 0 { + tuple := view.Tuples[len(view.Tuples)-1] + fields["src_addr"] = tuple.SrcAddr.String() + fields["dst_addr"] = tuple.DstAddr.String() + if tuple.SrcAddr.Is6() || tuple.DstAddr.Is6() { + fields["ip_family"] = "ipv6" + } else { + fields["ip_family"] = "ipv4" + } + fields["proto"] = tuple.Proto + fields["proto_name"] = ipProtocolName(tuple.Proto) + fields["src_port"] = tuple.SrcPort + fields["dst_port"] = tuple.DstPort + if tuple.Proto == 6 { + fields["tcp_flags"] = tuple.TCPFlags + } + } + if len(view.Tuples) > 1 { + outer := view.Tuples[0] + fields["outer_src_addr"] = outer.SrcAddr.String() + fields["outer_dst_addr"] = outer.DstAddr.String() + fields["outer_proto"] = outer.Proto + fields["outer_proto_name"] = ipProtocolName(outer.Proto) + fields["outer_src_port"] = outer.SrcPort + fields["outer_dst_port"] = outer.DstPort + if outer.Proto == 6 { + fields["outer_tcp_flags"] = outer.TCPFlags + } + fields["encap_depth"] = uint32(len(view.Tuples) - 1) + } +} + +func applyAggregationHelperFields(fields map[string]any, view packetView, helpers AggregationHelperOptions) { + if view.Model == nil { + return + } + if helpers.MPLSLabels > 0 { + applyMPLSLabelHelperFields(fields, view.Model, helpers.MPLSLabels) + } + if helpers.IPLayers > 0 { + applyIPLayerHelperFields(fields, view.Tuples, helpers.IPLayers) + } +} + +func applyDot1QFields(fields map[string]any, model *event.PacketModel) { + index := 0 + for _, layer := range model.Layers { + if layer.Kind != "dot1q" || layer.VLAN == nil { + continue + } + index++ + id := uint32(layer.VLAN.ID) + priority := uint32(layer.VLAN.PCP) + dei := layer.VLAN.DEI + fields[fmt.Sprintf("dot1q_%d_vlan_id", index)] = id + fields[fmt.Sprintf("dot1q_%d_priority", index)] = priority + fields[fmt.Sprintf("dot1q_%d_dei", index)] = dei + switch index { + case 1: + fields["dot1q_vlan_id"] = id + fields["dot1q_priority"] = priority + fields["dot1q_dei"] = dei + case 2: + fields["dot1q_customer_vlan_id"] = id + fields["dot1q_customer_priority"] = priority + fields["dot1q_customer_dei"] = dei + } + } +} + +func applyLayer2SegmentFields(fields map[string]any, model *event.PacketModel) { + for _, layer := range model.Layers { + if layer.Kind != "vxlan" || layer.VXLAN == nil { + continue + } + vni := layer.VXLAN.VNI & 0x00ffffff + fields["vxlan_vni"] = vni + fields["layer2_segment_id"] = (uint64(0x01) << 56) | uint64(vni) + return + } +} + +func applyMPLSLabelHelperFields(fields map[string]any, model *event.PacketModel, limit int) { + if limit <= 0 { + return + } + index := 0 + for _, layer := range model.Layers { + if layer.Kind != "mpls" || layer.MPLS == nil { + continue + } + index++ + if index > limit { + return + } + label := layer.MPLS.Label + fields[fmt.Sprintf("mpls_label_%d", index)] = label.Label + fields[fmt.Sprintf("mpls_label_stack_section_%d", index)] = mplsLabelStackSection(label) + } +} + +func applyIPLayerHelperFields(fields map[string]any, tuples []packetTuple, limit int) { + if limit <= 0 { + return + } + for i, tuple := range tuples { + if i >= limit { + return + } + prefix := fmt.Sprintf("ip_%d_", i+1) + fields[prefix+"src_addr"] = tuple.SrcAddr.String() + fields[prefix+"dst_addr"] = tuple.DstAddr.String() + fields[prefix+"proto"] = tuple.Proto + fields[prefix+"proto_name"] = ipProtocolName(tuple.Proto) + fields[prefix+"src_port"] = tuple.SrcPort + fields[prefix+"dst_port"] = tuple.DstPort + } +} + +func mplsLabelStackSection(label event.MPLSLabel) []byte { + raw := (label.Label & 0xfffff) << 4 + raw |= uint32(label.TC&0x7) << 1 + if label.BOS { + raw |= 1 + } + return []byte{byte(raw >> 16), byte(raw >> 8), byte(raw)} +} + +// ApplyModelFields projects a packet model into the canonical tuple aliases +// used by aggregation and encoders. The packet model remains the authoritative +// full layer structure. +func ApplyModelFields(fields map[string]any, model *event.PacketModel) { + ApplyModelFieldsWithHelpers(fields, model, AggregationHelperOptions{}) +} + +// ApplyModelFieldsWithHelpers projects a packet model into the canonical tuple +// aliases and any configured aggregation helper aliases. +func ApplyModelFieldsWithHelpers(fields map[string]any, model *event.PacketModel, helpers AggregationHelperOptions) { + if fields == nil || model == nil { + return + } + applyPacketViewFields(fields, packetViewFromModel(model), helpers) +} + +func packetViewFromModel(model *event.PacketModel) packetView { + view := packetView{ + Layers: make([]string, 0, len(model.Layers)), + Tuples: make([]packetTuple, 0, packetIPDepth(model)), + Model: model, + } + currentTuple := -1 + for _, layer := range model.Layers { + if layer.Kind != "" { + view.Layers = append(view.Layers, layer.Kind) + } + switch layer.Kind { + case "ethernet": + if layer.Ethernet == nil { + continue + } + view.SrcMAC = layer.Ethernet.SrcMAC + view.DstMAC = layer.Ethernet.DstMAC + view.EtherType = layer.Ethernet.EtherType + case "dot1q": + if layer.VLAN != nil { + view.VLANIDs = append(view.VLANIDs, uint32(layer.VLAN.ID)) + } + case "mpls": + if layer.MPLS != nil { + view.MPLSLabels = append(view.MPLSLabels, layer.MPLS.Label.Label) + } + case "ipv4": + if layer.IPv4 == nil { + continue + } + view.Tuples = append(view.Tuples, packetTuple{ + SrcAddr: layer.IPv4.SrcAddr, + DstAddr: layer.IPv4.DstAddr, + Proto: uint32(layer.IPv4.Protocol), + }) + currentTuple = len(view.Tuples) - 1 + case "ipv6": + if layer.IPv6 == nil { + continue + } + view.Tuples = append(view.Tuples, packetTuple{ + SrcAddr: layer.IPv6.SrcAddr, + DstAddr: layer.IPv6.DstAddr, + Proto: uint32(layer.IPv6.NextHeader), + }) + currentTuple = len(view.Tuples) - 1 + case "tcp": + if layer.TCP == nil || currentTuple < 0 { + continue + } + if view.Tuples[currentTuple].Proto == 0 { + view.Tuples[currentTuple].Proto = 6 + } + view.Tuples[currentTuple].SrcPort = uint32(layer.TCP.SrcPort) + view.Tuples[currentTuple].DstPort = uint32(layer.TCP.DstPort) + view.Tuples[currentTuple].TCPFlags = uint32(layer.TCP.Flags) + case "udp": + if layer.UDP == nil || currentTuple < 0 { + continue + } + if view.Tuples[currentTuple].Proto == 0 { + view.Tuples[currentTuple].Proto = 17 + } + view.Tuples[currentTuple].SrcPort = uint32(layer.UDP.SrcPort) + view.Tuples[currentTuple].DstPort = uint32(layer.UDP.DstPort) + } + } + return view +} + +func packetIPDepth(model *event.PacketModel) int { + if model == nil { + return 0 + } + count := 0 + for _, layer := range model.Layers { + if layer.IPv4 != nil || layer.IPv6 != nil { + count++ + } + } + return count +} + +func finalizePacketView(view packetView) packetView { + if view.Model == nil { + return view + } + ensurePacketModelLayerLengths(view.Model) + if view.Model.Features == nil { + view.Model.Features = make(map[string]event.FeatureValue) + } + layerFeatures := make([]event.FeatureValue, len(view.Layers)) + for i, layer := range view.Layers { + layerFeatures[i] = event.FeatureString(layer) + } + view.Model.Features["layer_kinds"] = event.FeatureValue{List: layerFeatures} + view.Model.Features["ip_depth"] = event.FeatureUint64(uint64(packetIPDepth(view.Model))) + if len(view.Tuples) > 1 { + view.Model.Features["encap_depth"] = event.FeatureUint64(uint64(len(view.Tuples) - 1)) + } + return view +} + +func parsePacketViewWithOptions(data []byte, protocol uint32, opts DecodeOptions) (packetView, error) { + if len(data) == 0 { + return packetView{}, fmt.Errorf("empty packet header") + } + view := packetView{ + Layers: make([]string, 0, 6), + Tuples: make([]packetTuple, 0, 2), + Model: &event.PacketModel{ + Layers: make([]event.LayerSpec, 0, 8), + Features: make(map[string]event.FeatureValue, 2), + }, + } + offset := 0 + etherType := uint16(0) + directIP := false + switch protocol { + case 1: + var err error + offset, etherType, err = parseEthernet(data, &view, true) + if err != nil { + return packetView{}, err + } + case 11: + etherType = 0x0800 + directIP = true + case 12: + etherType = 0x86dd + directIP = true + default: + if len(data) >= 14 { + var err error + offset, etherType, err = parseEthernet(data, &view, true) + if err != nil { + return packetView{}, err + } + break + } + switch data[0] >> 4 { + case 4, 6: + directIP = true + default: + return packetView{}, fmt.Errorf("truncated packet header") + } + } + if len(data) <= offset { + return packetView{}, fmt.Errorf("truncated packet header") + } + for { + switch { + case etherType == 0x0800 || (directIP && data[offset]>>4 == 4): + directIP = false + nextOffset, tuple, nextProto, err := parseIPv4Tuple(data[offset:]) + if err != nil { + return packetView{}, err + } + view.appendLayer(event.LayerSpec{ + Kind: "ipv4", + Length: uint32(nextOffset), + IPv4: &event.IPv4Layer{ + SrcAddr: tuple.SrcAddr, + DstAddr: tuple.DstAddr, + Protocol: uint8(tuple.Proto), + TTL: data[offset+8], + DSCP: data[offset+1] >> 2, + ECN: data[offset+1] & 0x03, + Identification: binary.BigEndian.Uint16(data[offset+4 : offset+6]), + Flags: uint8(binary.BigEndian.Uint16(data[offset+6:offset+8]) >> 13), + FragmentOffset: binary.BigEndian.Uint16(data[offset+6:offset+8]) & 0x1fff, + }, + }) + view.Tuples = append(view.Tuples, tuple) + if innerEtherType, ok := ipInIPInnerEtherType(nextProto, opts); ok { + offset += nextOffset + etherType = innerEtherType + continue + } + if opts.DecodeGRE && nextProto == 47 { + innerOffset, innerProto, err := parseGRE(data[offset+nextOffset:], &view) + if err != nil { + return view, nil + } + if !opts.DecodeBeyondL4 { + return finalizePacketView(view), nil + } + offset += nextOffset + innerOffset + etherType = innerProto + if shouldParseEthernetPayload(etherType) { + nextOffset, nextEtherType, err := parseEthernet(data[offset:], &view, false) + if err != nil { + return view, nil + } + offset += nextOffset + etherType = nextEtherType + continue + } + if canContinueEtherType(etherType) { + continue + } + } + if opts.DecodeBeyondL4 && tuple.Proto == 17 { + if innerOffset, innerEtherType, err := parseUDPTunnel(data[offset+nextOffset:], tuple, &view, opts); err == nil { + offset += nextOffset + innerOffset + if shouldParseEthernetPayload(innerEtherType) { + nextOffset, nextEtherType, err := parseEthernet(data[offset:], &view, false) + if err != nil { + return view, nil + } + offset += nextOffset + etherType = nextEtherType + continue + } + etherType = innerEtherType + if canContinueEtherType(etherType) { + continue + } + } + } + appendTransportLayer(&view, tuple.Proto, data[offset+nextOffset:]) + return finalizePacketView(view), nil + case etherType == 0x86dd || (directIP && data[offset]>>4 == 6): + directIP = false + nextOffset, tuple, nextProto, extensionLayers, err := parseIPv6Tuple(data[offset:]) + if err != nil { + return packetView{}, err + } + view.appendLayer(event.LayerSpec{ + Kind: "ipv6", + Length: 40, + IPv6: &event.IPv6Layer{ + SrcAddr: tuple.SrcAddr, + DstAddr: tuple.DstAddr, + NextHeader: uint8(tuple.Proto), + HopLimit: data[offset+7], + TrafficClass: ((data[offset]&0x0f)<<4 | (data[offset+1] >> 4)), + FlowLabel: uint32(data[offset+1]&0x0f)<<16 | uint32(data[offset+2])<<8 | uint32(data[offset+3]), + }, + }) + for _, layer := range extensionLayers { + view.appendLayer(layer) + } + view.Tuples = append(view.Tuples, tuple) + if innerEtherType, ok := ipInIPInnerEtherType(nextProto, opts); ok { + offset += nextOffset + etherType = innerEtherType + continue + } + if opts.DecodeGRE && nextProto == 47 { + innerOffset, innerProto, err := parseGRE(data[offset+nextOffset:], &view) + if err != nil { + return view, nil + } + if !opts.DecodeBeyondL4 { + return finalizePacketView(view), nil + } + offset += nextOffset + innerOffset + etherType = innerProto + if shouldParseEthernetPayload(etherType) { + nextOffset, nextEtherType, err := parseEthernet(data[offset:], &view, false) + if err != nil { + return view, nil + } + offset += nextOffset + etherType = nextEtherType + continue + } + if canContinueEtherType(etherType) { + continue + } + } + if opts.DecodeBeyondL4 && tuple.Proto == 17 { + if innerOffset, innerEtherType, err := parseUDPTunnel(data[offset+nextOffset:], tuple, &view, opts); err == nil { + offset += nextOffset + innerOffset + if shouldParseEthernetPayload(innerEtherType) { + nextOffset, nextEtherType, err := parseEthernet(data[offset:], &view, false) + if err != nil { + return view, nil + } + offset += nextOffset + etherType = nextEtherType + continue + } + etherType = innerEtherType + if canContinueEtherType(etherType) { + continue + } + } + } + appendTransportLayer(&view, tuple.Proto, data[offset+nextOffset:]) + return finalizePacketView(view), nil + case etherType == 0x8847 || etherType == 0x8848: + innerOffset, innerProto, err := parseMPLS(data[offset:], &view) + if err != nil { + return view, nil + } + if !opts.DecodeBeyondL4 { + return finalizePacketView(view), nil + } + offset += innerOffset + etherType = innerProto + continue + case etherType == 0x8864: + if !opts.DecodePPPoE { + return finalizePacketView(view), nil + } + innerOffset, innerProto, err := parsePPPoE(data[offset:], &view) + if err != nil { + return view, nil + } + if !opts.DecodeBeyondL4 { + return finalizePacketView(view), nil + } + offset += innerOffset + etherType = innerProto + continue + default: + return packetView{}, fmt.Errorf("unsupported ip version") + } + } +} + +func parseEthernet(data []byte, view *packetView, captureMAC bool) (int, uint16, error) { + if len(data) < 14 { + return 0, 0, fmt.Errorf("truncated ethernet header") + } + layer := event.LayerSpec{ + Kind: "ethernet", + Ethernet: &event.EthernetLayer{}, + } + if captureMAC { + view.DstMAC = formatMAC(data[0:6]) + view.SrcMAC = formatMAC(data[6:12]) + layer.Ethernet.DstMAC = view.DstMAC + layer.Ethernet.SrcMAC = view.SrcMAC + } + etherType := binary.BigEndian.Uint16(data[12:14]) + layer.Ethernet.EtherType = uint32(etherType) + view.appendLayer(layer) + offset := 14 + for etherType == 0x8100 || etherType == 0x88a8 { + if len(data) < offset+4 { + return 0, 0, fmt.Errorf("truncated vlan header") + } + tci := binary.BigEndian.Uint16(data[offset : offset+2]) + view.VLANIDs = append(view.VLANIDs, uint32(tci&0x0fff)) + view.appendLayer(event.LayerSpec{ + Kind: "dot1q", + VLAN: &event.VLANLayer{ + ID: tci & 0x0fff, + PCP: uint8((tci >> 13) & 0x7), + DEI: ((tci >> 12) & 0x1) == 1, + TPID: etherType, + }, + }) + etherType = binary.BigEndian.Uint16(data[offset+2 : offset+4]) + offset += 4 + } + if view.EtherType == 0 { + view.EtherType = uint32(etherType) + } + return offset, etherType, nil +} + +func shouldParseEthernetPayload(etherType uint16) bool { + return etherType == 0x6558 +} + +func canContinueEtherType(etherType uint16) bool { + switch etherType { + case 0x0800, 0x86dd, 0x8847, 0x8848, 0x8864: + return true + default: + return false + } +} + +func ipInIPInnerEtherType(protocol uint32, opts DecodeOptions) (uint16, bool) { + if !opts.DecodeBeyondL4 || !opts.DecodeIPIP { + return 0, false + } + switch protocol { + case 4: + return 0x0800, true + case 41: + return 0x86dd, true + default: + return 0, false + } +} + +func parseIPv4Tuple(data []byte) (int, packetTuple, uint32, error) { + if len(data) < 20 { + return 0, packetTuple{}, 0, fmt.Errorf("truncated ipv4 header") + } + ihl := int(data[0]&0x0f) * 4 + if ihl < 20 || len(data) < ihl { + return 0, packetTuple{}, 0, fmt.Errorf("invalid ipv4 header length") + } + src, ok := netip.AddrFromSlice(data[12:16]) + if !ok { + return 0, packetTuple{}, 0, fmt.Errorf("invalid ipv4 source address") + } + dst, ok := netip.AddrFromSlice(data[16:20]) + if !ok { + return 0, packetTuple{}, 0, fmt.Errorf("invalid ipv4 destination address") + } + tuple := packetTuple{ + SrcAddr: src, + DstAddr: dst, + Proto: uint32(data[9]), + } + if len(data) >= ihl+4 && (tuple.Proto == 6 || tuple.Proto == 17) { + tuple.SrcPort = uint32(uint16(data[ihl])<<8 | uint16(data[ihl+1])) + tuple.DstPort = uint32(uint16(data[ihl+2])<<8 | uint16(data[ihl+3])) + } + if len(data) >= ihl+14 && tuple.Proto == 6 { + tuple.TCPFlags = uint32(data[ihl+13]) + } + return ihl, tuple, tuple.Proto, nil +} + +func parseIPv6Tuple(data []byte) (int, packetTuple, uint32, []event.LayerSpec, error) { + if len(data) < 40 { + return 0, packetTuple{}, 0, nil, fmt.Errorf("truncated ipv6 header") + } + src, ok := netip.AddrFromSlice(data[8:24]) + if !ok { + return 0, packetTuple{}, 0, nil, fmt.Errorf("invalid ipv6 source address") + } + dst, ok := netip.AddrFromSlice(data[24:40]) + if !ok { + return 0, packetTuple{}, 0, nil, fmt.Errorf("invalid ipv6 destination address") + } + nextHeader := data[6] + offset := 40 + var extensionLayers []event.LayerSpec + for isIPv6ExtensionHeader(nextHeader) { + if len(data) < offset+2 { + return 0, packetTuple{}, 0, nil, fmt.Errorf("truncated ipv6 extension header") + } + switch nextHeader { + case 44: + if len(data) < offset+8 { + return 0, packetTuple{}, 0, nil, fmt.Errorf("truncated ipv6 fragment header") + } + extensionLayers = append(extensionLayers, event.LayerSpec{ + Kind: "ipv6_fragment", + Length: 8, + Features: map[string]event.FeatureValue{ + "next_header": event.FeatureUint64(uint64(data[offset])), + }, + }) + nextHeader = data[offset] + offset += 8 + case 51: + hdrLen := (int(data[offset+1]) + 2) * 4 + if len(data) < offset+hdrLen { + return 0, packetTuple{}, 0, nil, fmt.Errorf("truncated ipv6 authentication header") + } + extensionLayers = append(extensionLayers, event.LayerSpec{ + Kind: "ipv6_authentication", + Length: uint32(hdrLen), + Features: map[string]event.FeatureValue{ + "next_header": event.FeatureUint64(uint64(data[offset])), + }, + }) + nextHeader = data[offset] + offset += hdrLen + default: + hdrLen := (int(data[offset+1]) + 1) * 8 + if len(data) < offset+hdrLen { + return 0, packetTuple{}, 0, nil, fmt.Errorf("truncated ipv6 extension header") + } + if nextHeader == 43 { + extensionLayers = append(extensionLayers, event.LayerSpec{ + Kind: "ipv6_routing", + Length: uint32(hdrLen), + Features: map[string]event.FeatureValue{ + "next_header": event.FeatureUint64(uint64(data[offset])), + "routing_type": event.FeatureUint64(uint64(data[offset+2])), + }, + }) + } else { + extensionLayers = append(extensionLayers, event.LayerSpec{ + Kind: "ipv6_extension", + Length: uint32(hdrLen), + Features: map[string]event.FeatureValue{ + "header": event.FeatureUint64(uint64(nextHeader)), + "next_header": event.FeatureUint64(uint64(data[offset])), + }, + }) + } + nextHeader = data[offset] + offset += hdrLen + } + } + tuple := packetTuple{ + SrcAddr: src, + DstAddr: dst, + Proto: uint32(nextHeader), + } + if len(data) >= offset+4 && (tuple.Proto == 6 || tuple.Proto == 17) { + tuple.SrcPort = uint32(uint16(data[offset])<<8 | uint16(data[offset+1])) + tuple.DstPort = uint32(uint16(data[offset+2])<<8 | uint16(data[offset+3])) + } + if len(data) >= offset+14 && tuple.Proto == 6 { + tuple.TCPFlags = uint32(data[offset+13]) + } + return offset, tuple, tuple.Proto, extensionLayers, nil +} + +func parseGRE(data []byte, view *packetView) (int, uint16, error) { + if len(data) < 4 { + return 0, 0, fmt.Errorf("truncated gre header") + } + flags := binary.BigEndian.Uint16(data[0:2]) + proto := binary.BigEndian.Uint16(data[2:4]) + offset := 4 + if flags&0x8000 != 0 { + offset += 4 + } + if flags&0x2000 != 0 { + offset += 4 + } + if flags&0x1000 != 0 { + offset += 4 + } + if len(data) < offset { + return 0, 0, fmt.Errorf("truncated gre optional fields") + } + view.appendLayer(event.LayerSpec{ + Kind: "gre", + Length: uint32(offset), + GRE: &event.GRELayer{ + Protocol: proto, + Checksum: flags&0x8000 != 0, + Key: flags&0x2000 != 0, + Sequence: flags&0x1000 != 0, + }, + }) + return offset, proto, nil +} + +func parseUDPTunnel(data []byte, tuple packetTuple, view *packetView, opts DecodeOptions) (int, uint16, error) { + if len(data) < 8 { + return 0, 0, fmt.Errorf("truncated udp header") + } + switch { + case opts.DecodeVXLAN && portMatches(tuple, opts.VXLANPorts, 4789): + if len(data) < 16 { + return 0, 0, fmt.Errorf("truncated vxlan header") + } + view.appendLayer(event.LayerSpec{ + Kind: "vxlan", + Length: 8, + VXLAN: &event.VXLANLayer{ + VNI: uint32(data[12])<<16 | uint32(data[13])<<8 | uint32(data[14]), + }, + }) + return 16, 0x6558, nil + case opts.DecodeGeneve && portMatches(tuple, opts.GenevePorts, 6081): + if len(data) < 16 { + return 0, 0, fmt.Errorf("truncated geneve header") + } + optLen := int((data[8] & 0x3f) * 4) + proto := binary.BigEndian.Uint16(data[10:12]) + offset := 8 + 8 + optLen + if len(data) < offset { + return 0, 0, fmt.Errorf("truncated geneve options") + } + view.appendLayer(event.LayerSpec{ + Kind: "geneve", + Length: uint32(8 + optLen), + Geneve: &event.GeneveLayer{ + VNI: uint32(data[12])<<16 | uint32(data[13])<<8 | uint32(data[14]), + Protocol: proto, + }, + }) + return offset, proto, nil + case opts.DecodeL2TP && portMatches(tuple, opts.L2TPPorts, 1701): + return parseL2TP(data, view) + case opts.DecodeGTPU && portMatches(tuple, opts.GTPUPorts, 2152): + return parseGTPU(data, view) + default: + return 0, 0, fmt.Errorf("not a supported udp tunnel") + } +} + +func parseL2TP(data []byte, view *packetView) (int, uint16, error) { + if len(data) < 14 { + return 0, 0, fmt.Errorf("truncated l2tp header") + } + flags := binary.BigEndian.Uint16(data[8:10]) + version := flags & 0x000f + if version != 2 { + return 0, 0, fmt.Errorf("unsupported l2tp version") + } + offset := 10 + if flags&0x4000 != 0 { + if len(data) < offset+2 { + return 0, 0, fmt.Errorf("truncated l2tp length") + } + offset += 2 + } + if len(data) < offset+4 { + return 0, 0, fmt.Errorf("truncated l2tp session") + } + tunnelID := binary.BigEndian.Uint16(data[offset : offset+2]) + sessionID := binary.BigEndian.Uint16(data[offset+2 : offset+4]) + offset += 4 + if flags&0x0800 != 0 { + if len(data) < offset+4 { + return 0, 0, fmt.Errorf("truncated l2tp sequence") + } + offset += 4 + } + if flags&0x0200 != 0 { + if len(data) < offset+2 { + return 0, 0, fmt.Errorf("truncated l2tp offset size") + } + offsetSize := int(binary.BigEndian.Uint16(data[offset : offset+2])) + offset += 2 + offsetSize + } + if len(data) < offset+2 { + return 0, 0, fmt.Errorf("truncated l2tp payload") + } + if len(data) >= offset+4 && data[offset] == 0xff && data[offset+1] == 0x03 { + offset += 2 + } + proto := binary.BigEndian.Uint16(data[offset : offset+2]) + offset += 2 + view.appendLayer(event.LayerSpec{ + Kind: "l2tp", + Length: uint32(offset - 8), + Features: map[string]event.FeatureValue{ + "tunnel_id": event.FeatureUint64(uint64(tunnelID)), + "session_id": event.FeatureUint64(uint64(sessionID)), + "version": event.FeatureUint64(uint64(version)), + }, + }) + switch proto { + case 0x0021: + return offset, 0x0800, nil + case 0x0057: + return offset, 0x86dd, nil + default: + return 0, 0, fmt.Errorf("unsupported l2tp ppp payload") + } +} + +func parseGTPU(data []byte, view *packetView) (int, uint16, error) { + if len(data) < 16 { + return 0, 0, fmt.Errorf("truncated gtpu header") + } + flags := data[8] + messageType := data[9] + if flags>>5 != 1 || messageType != 0xff { + return 0, 0, fmt.Errorf("unsupported gtpu message") + } + teid := binary.BigEndian.Uint32(data[12:16]) + offset := 16 + if flags&0x07 != 0 { + if len(data) < offset+4 { + return 0, 0, fmt.Errorf("truncated gtpu optional fields") + } + nextExt := data[offset+3] + offset += 4 + for nextExt != 0 { + if len(data) < offset+2 { + return 0, 0, fmt.Errorf("truncated gtpu extension") + } + extLen := int(data[offset+1]) * 4 + if extLen == 0 || len(data) < offset+2+extLen { + return 0, 0, fmt.Errorf("truncated gtpu extension payload") + } + nextExt = data[offset+1+extLen] + offset += 2 + extLen + } + } + if len(data) <= offset { + return 0, 0, fmt.Errorf("missing gtpu payload") + } + view.appendLayer(event.LayerSpec{ + Kind: "gtpu", + Length: uint32(offset - 8), + Features: map[string]event.FeatureValue{ + "teid": event.FeatureUint64(uint64(teid)), + "message_type": event.FeatureUint64(uint64(messageType)), + }, + }) + switch data[offset] >> 4 { + case 4: + return offset, 0x0800, nil + case 6: + return offset, 0x86dd, nil + default: + return 0, 0, fmt.Errorf("unsupported gtpu payload") + } +} + +func portMatches(tuple packetTuple, ports []uint32, defaultPort uint32) bool { + if len(ports) == 0 { + return tuple.DstPort == defaultPort || tuple.SrcPort == defaultPort + } + for _, port := range ports { + if port != 0 && (tuple.DstPort == port || tuple.SrcPort == port) { + return true + } + } + return false +} + +func parseMPLS(data []byte, view *packetView) (int, uint16, error) { + if len(data) < 4 { + return 0, 0, fmt.Errorf("truncated mpls header") + } + offset := 0 + for { + if len(data) < offset+4 { + return 0, 0, fmt.Errorf("truncated mpls label stack") + } + label := binary.BigEndian.Uint32(data[offset : offset+4]) + labelValue := label >> 12 + view.MPLSLabels = append(view.MPLSLabels, labelValue) + view.appendLayer(event.LayerSpec{ + Kind: "mpls", + MPLS: &event.MPLSLayer{ + Label: event.MPLSLabel{ + Label: labelValue, + TC: uint8((label >> 9) & 0x7), + BOS: ((label >> 8) & 0x1) == 1, + TTL: uint8(label & 0xff), + }, + }, + }) + offset += 4 + if label&0x100 != 0 { + break + } + } + if len(data) <= offset { + return 0, 0, fmt.Errorf("missing mpls payload") + } + switch data[offset] >> 4 { + case 4: + return offset, 0x0800, nil + case 6: + return offset, 0x86dd, nil + default: + return 0, 0, fmt.Errorf("unsupported mpls payload") + } +} + +func parsePPPoE(data []byte, view *packetView) (int, uint16, error) { + if len(data) < 8 { + return 0, 0, fmt.Errorf("truncated pppoe session header") + } + proto := binary.BigEndian.Uint16(data[6:8]) + view.appendLayer(event.LayerSpec{ + Kind: "pppoe", + PPPoE: &event.PPPoELayer{ + SessionID: binary.BigEndian.Uint16(data[2:4]), + Protocol: proto, + }, + }) + switch proto { + case 0x0021: + return 8, 0x0800, nil + case 0x0057: + return 8, 0x86dd, nil + case 0x0281, 0x0283: + return 8, 0x8847, nil + default: + return 0, 0, fmt.Errorf("unsupported ppp payload") + } +} + +func appendTransportLayer(view *packetView, proto uint32, data []byte) { + switch proto { + case 6: + layer := event.LayerSpec{Kind: "tcp", Length: tcpHeaderLength(data)} + if len(view.Tuples) > 0 { + tuple := view.Tuples[len(view.Tuples)-1] + layer.TCP = &event.TCPLayer{ + SrcPort: uint16(tuple.SrcPort), + DstPort: uint16(tuple.DstPort), + Flags: uint8(tuple.TCPFlags), + } + } + view.appendLayer(layer) + case 17: + layer := event.LayerSpec{Kind: "udp", Length: 8} + if len(view.Tuples) > 0 { + tuple := view.Tuples[len(view.Tuples)-1] + layer.UDP = &event.UDPLayer{ + SrcPort: uint16(tuple.SrcPort), + DstPort: uint16(tuple.DstPort), + } + } + view.appendLayer(layer) + case 132: + view.appendLayer(event.LayerSpec{ + Kind: "sctp", + Length: 12, + Features: map[string]event.FeatureValue{ + "transport": event.FeatureString("sctp"), + }, + }) + case 1: + view.appendLayer(event.LayerSpec{ + Kind: "icmp", + Length: 8, + ICMP: &event.ICMPLayer{}, + }) + case 58: + view.appendLayer(event.LayerSpec{ + Kind: "icmpv6", + Length: 8, + ICMP: &event.ICMPLayer{}, + }) + } +} + +func tcpHeaderLength(data []byte) uint32 { + if len(data) < 13 { + return 20 + } + length := uint32(data[12]>>4) * 4 + if length < 20 { + return 20 + } + return length +} + +func formatMAC(data []byte) string { + if len(data) != 6 { + return "" + } + const hex = "0123456789abcdef" + var out [17]byte + j := 0 + for i := 0; i < 6; i++ { + if i > 0 { + out[j] = ':' + j++ + } + out[j] = hex[data[i]>>4] + out[j+1] = hex[data[i]&0x0f] + j += 2 + } + return string(out[:]) +} + +func isIPv6ExtensionHeader(nextHeader byte) bool { + switch nextHeader { + case 0, 43, 44, 50, 51, 60, 135, 139, 140: + return true + default: + return false + } +} + +func ensureFields(evt *event.Event, capacity int) map[string]any { + if evt.Fields == nil { + evt.Fields = make(map[string]any, capacity) + } + return evt.Fields +} + +func fieldStringOrZero(fields map[string]any, key string) string { + if fields == nil { + return "" + } + val, ok := fields[key] + if !ok { + return "" + } + switch v := val.(type) { + case string: + return v + default: + return fmt.Sprint(v) + } +} + +func fieldUint32(fields map[string]any, key string) uint32 { + if fields == nil { + return 0 + } + val, ok := fields[key] + if !ok { + return 0 + } + return uint32FromAny(val) +} + +func uint32FromAny(val any) uint32 { + switch v := val.(type) { + case uint32: + return v + case uint64: + return uint32(v) + case int: + return uint32(v) + case int64: + return uint32(v) + case float64: + return uint32(v) + case json.Number: + n, err := v.Int64() + if err != nil || n < 0 { + return 0 + } + return uint32(n) + case string: + var n uint64 + if _, err := fmt.Sscan(v, &n); err != nil { + return 0 + } + return uint32(n) + default: + return 0 + } +} + +func ipProtocolName(proto uint32) string { + switch proto { + case 1: + return "icmp" + case 2: + return "igmp" + case 6: + return "tcp" + case 17: + return "udp" + case 41: + return "ipv6" + case 47: + return "gre" + case 50: + return "esp" + case 51: + return "ah" + case 58: + return "icmpv6" + case 132: + return "sctp" + default: + return "" + } +} + +func sampledHeaderProtocolName(proto uint32) string { + switch proto { + case 1: + return "ethernet" + case 11: + return "ipv4" + case 12: + return "ipv6" + default: + return "" + } +} diff --git a/pkg/reflow/packet/normalize_test.go b/pkg/reflow/packet/normalize_test.go new file mode 100644 index 00000000..12993854 --- /dev/null +++ b/pkg/reflow/packet/normalize_test.go @@ -0,0 +1,145 @@ +package packet + +import ( + "encoding/hex" + "testing" + + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func TestNormalizeEventParsesDirectIPv4SampledHeader(t *testing.T) { + header := mustDecodeHex(t, "450000281234400040060000c0000201c6336401303901bb00000001000000005002200000000000") + evt := &event.Event{ + Fields: map[string]any{ + "protocol": uint32(253), + "frame_length": uint32(74), + "original_length": uint32(len(header)), + "header_data": header, + }, + } + + if err := NormalizeEvent(evt, NormalizeOptions{HeaderProtocol: 11}); err != nil { + t.Fatalf("NormalizeEvent returned error: %v", err) + } + + if got := evt.Fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected src_addr=192.0.2.1, got %#v", got) + } + if got := evt.Fields["dst_addr"]; got != "198.51.100.1" { + t.Fatalf("expected dst_addr=198.51.100.1, got %#v", got) + } + if got := evt.Fields["src_port"]; got != uint32(12345) { + t.Fatalf("expected src_port=12345, got %#v", got) + } + if got := evt.Fields["dst_port"]; got != uint32(443) { + t.Fatalf("expected dst_port=443, got %#v", got) + } + if got := evt.Fields["tcp_flags"]; got != uint32(2) { + t.Fatalf("expected tcp_flags=2, got %#v", got) + } + if got := evt.Fields["bytes"]; got != int64(74) { + t.Fatalf("expected bytes to use frame_length=74, got %#v", got) + } + if evt.Packet == nil || len(evt.Packet.Layers) != 2 { + t.Fatalf("expected ipv4/tcp packet model, got %#v", evt.Packet) + } + if got := evt.Packet.Layers[0].Length; got != 20 { + t.Fatalf("expected ipv4 layer length 20, got %d", got) + } + if got := evt.Packet.Layers[1].Length; got != 20 { + t.Fatalf("expected tcp layer length 20, got %d", got) + } + if evt.Packet.Layers[1].TCP == nil || evt.Packet.Layers[1].TCP.Flags != 2 { + t.Fatalf("expected tcp layer flags 2, got %#v", evt.Packet.Layers[1].TCP) + } +} + +func TestNormalizeEventParsesDirectIPv6SampledHeader(t *testing.T) { + header := []byte{ + 0x60, 0x00, 0x00, 0x00, 0x00, 0x08, 0x11, 0x40, + 0x20, 0x01, 0x0d, 0xb8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x20, 0x01, 0x0d, 0xb8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x30, 0x39, 0x01, 0xbb, 0x00, 0x08, 0x00, 0x00, + } + evt := &event.Event{ + Fields: map[string]any{ + "protocol": uint32(253), + "frame_length": uint32(88), + "original_length": uint32(len(header)), + "header_data": header, + }, + } + + if err := NormalizeEvent(evt, NormalizeOptions{HeaderProtocol: 12}); err != nil { + t.Fatalf("NormalizeEvent returned error: %v", err) + } + + if got := evt.Fields["src_addr"]; got != "2001:db8::1" { + t.Fatalf("expected src_addr=2001:db8::1, got %#v", got) + } + if got := evt.Fields["dst_addr"]; got != "2001:db8::2" { + t.Fatalf("expected dst_addr=2001:db8::2, got %#v", got) + } + if got := evt.Fields["src_port"]; got != uint32(12345) { + t.Fatalf("expected src_port=12345, got %#v", got) + } + if got := evt.Fields["dst_port"]; got != uint32(443) { + t.Fatalf("expected dst_port=443, got %#v", got) + } +} + +func TestNormalizeEventDoesNotTreatProtocolFieldAsHeaderProtocol(t *testing.T) { + header := mustDecodeHex(t, "450000281234400040060000c0000201c6336401303901bb00000001000000005002200000000000") + evt := &event.Event{ + Fields: map[string]any{ + "protocol": uint32(11), + "frame_length": uint32(74), + "original_length": uint32(len(header)), + "header_data": header, + }, + } + + if err := NormalizeEvent(evt, NormalizeOptions{}); err != nil { + t.Fatalf("NormalizeEvent returned error: %v", err) + } + if got := evt.Fields["src_addr"]; got != nil { + t.Fatalf("did not expect generic protocol field to drive packet parsing, got src_addr=%#v", got) + } +} + +func TestNormalizeEventDoesNotParseUnsupportedEtherTypePayloadAsIP(t *testing.T) { + ipHeader := mustDecodeHex(t, "450000281234400040060000c0000201c6336401303901bb00000001000000005002200000000000") + header := append([]byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x88, 0xb5, + }, ipHeader...) + evt := &event.Event{ + Fields: map[string]any{ + "frame_length": uint32(len(header)), + "original_length": uint32(len(header)), + "header_data": header, + }, + } + + if err := NormalizeEvent(evt, NormalizeOptions{}); err != nil { + t.Fatalf("NormalizeEvent returned error: %v", err) + } + if got := evt.Fields["src_addr"]; got != nil { + t.Fatalf("did not expect unsupported EtherType payload to drive packet parsing, got src_addr=%#v", got) + } + if evt.Packet != nil { + t.Fatalf("did not expect packet model for unsupported EtherType, got %#v", evt.Packet) + } +} + +func mustDecodeHex(t *testing.T, raw string) []byte { + t.Helper() + out, err := hex.DecodeString(raw) + if err != nil { + t.Fatalf("decode hex: %v", err) + } + return out +} diff --git a/pkg/reflow/processor/processor.go b/pkg/reflow/processor/processor.go new file mode 100644 index 00000000..af0a70d8 --- /dev/null +++ b/pkg/reflow/processor/processor.go @@ -0,0 +1,991 @@ +package processor + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net/netip" + "strconv" + "strings" + "time" + + flowmessage "github.com/netsampler/goflow2/v3/pb" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/packet" +) + +// Processor converts source-specific payloads into ReFlow's in-memory event shape. +type Processor interface { + Process(evt *event.Event) ([]*event.Event, error) +} + +// New returns the configured processor implementation. +func New(cfg config.ProcessorConfig) (Processor, error) { + switch cfg.Type { + case "", "builtin": + return NewBuiltin(cfg), nil + default: + return nil, fmt.Errorf("unsupported processor.type %q", cfg.Type) + } +} + +type Builtin struct { + cfg config.BuiltinProcessorConfig + decode packet.DecodeOptions +} + +// NewBuiltin constructs the default in-code processor used when WASM is disabled. +func NewBuiltin(cfg config.ProcessorConfig) *Builtin { + return &Builtin{ + cfg: cfg.Builtin, + decode: packetDecodeOptions(cfg.Builtin.PacketDecoder), + } +} + +func (p *Builtin) aggregationHelpers() packet.AggregationHelperOptions { + return packet.AggregationHelperOptions{ + MPLSLabels: p.cfg.AggregationHelpers.MPLSLabels, + IPLayers: p.cfg.AggregationHelpers.IPLayers, + } +} + +// Process dispatches to the built-in mapper for the incoming source message type. +func (p *Builtin) Process(evt *event.Event) ([]*event.Event, error) { + if evt != nil && evt.Kind == "control" { + return []*event.Event{evt}, nil + } + switch evt.Source.Type { + case "bytes": + return p.processBytes(evt) + case "json": + return p.processJSONFlavor(evt) + case "flow": + return p.processFlow(evt) + default: + return []*event.Event{evt}, nil + } +} + +// processBytes treats the payload as raw packet bytes and extracts the canonical +// L3/L4 tuple while preserving the raw bytes for later encoding if needed. +func (p *Builtin) processBytes(evt *event.Event) ([]*event.Event, error) { + payload, ok := evt.Payload.([]byte) + if !ok || len(payload) == 0 { + return nil, fmt.Errorf("decode bytes packet: missing payload bytes") + } + + fields := ensureFields(evt, 16) + frameLength := uint32(len(payload)) + if wireLength := fieldUint32(fields, "wire_length"); wireLength != 0 { + frameLength = wireLength + } + fields["record_kind"] = "packet" + fields["frame_length"] = frameLength + fields["original_length"] = uint32(len(payload)) + fields["bytes"] = int64(frameLength) + fields["packets"] = int64(1) + if evt.ReceivedAt.IsZero() { + evt.ReceivedAt = time.Now().UTC() + } + startNS, endNS := packetTimeWindowNS(evt.ReceivedAt, frameLength, fieldUint64(fields, "if_speed")) + fields["time_flow_start_ns"] = startNS + fields["time_flow_end_ns"] = endNS + // Aggregators use the existing millisecond fields for bucket windows and + // min/max merging; nanosecond aliases above preserve packet timing precision. + fields["start_time_unix"] = startNS / int64(time.Millisecond) + fields["end_time_unix"] = endNS / int64(time.Millisecond) + + // sFlow raw packet headers expect protocol metadata in addition to the bytes. + // Live pcap defaults to Ethernet, while stream sources can provide a header_protocol. + if _, ok := fields["protocol"]; !ok { + fields["protocol"] = uint32(1) + } + if err := packet.NormalizeEvent(evt, packet.NormalizeOptions{ + DisablePacketMapping: p.cfg.DisablePacketMapping, + TruncatePacketBytes: p.cfg.TruncatePacketBytes, + UsePayloadAsPacket: true, + TruncatePayload: true, + HeaderProtocol: fieldUint32(fields, "header_protocol"), + Decode: p.decode, + AggregationHelpers: p.aggregationHelpers(), + }); err != nil { + return nil, err + } + p.applyDerivedFieldMappings(evt) + + if p.cfg.DropMessage { + evt.Message = nil + } + if p.cfg.DropPayload { + evt.Payload = nil + } + return []*event.Event{evt}, nil +} + +// bytesField reads either []byte or string data from the generic field map. +func bytesField(fields map[string]any, key string) []byte { + if fields == nil { + return nil + } + val, ok := fields[key] + if !ok { + return nil + } + switch v := val.(type) { + case []byte: + return v + case string: + return []byte(v) + default: + return nil + } +} + +// processFlow treats the built-in processor as the post-decode normalization boundary. +// The decode stage already expands exporter packets into canonical flow events, so the +// processor mainly validates the shape and optionally drops the raw datagram payload. +func (p *Builtin) processFlow(evt *event.Event) ([]*event.Event, error) { + fields := ensureFields(evt, 2) + if _, ok := fields["flow_type"]; !ok { + return nil, fmt.Errorf("decoded flow event is missing flow_type") + } + if len(bytesField(fields, "header_data")) > 0 || fieldStringOrZero(fields, "record_kind") == "packet" { + if err := packet.NormalizeEvent(evt, packet.NormalizeOptions{ + DisablePacketMapping: p.cfg.DisablePacketMapping, + TruncatePacketBytes: p.cfg.TruncatePacketBytes, + HeaderProtocol: packetHeaderProtocol(fields), + Decode: p.decode, + AggregationHelpers: p.aggregationHelpers(), + }); err != nil { + return nil, err + } + } + setIPFamilyFromFields(fields) + p.applyDerivedFieldMappings(evt) + if p.cfg.DropMessage { + evt.Message = nil + } + if p.cfg.DropPayload { + evt.Payload = nil + } + return []*event.Event{evt}, nil +} + +func packetHeaderProtocol(fields map[string]any) uint32 { + if fieldStringOrZero(fields, "flow_type") != "sflow" || fieldStringOrZero(fields, "record_kind") != "packet" { + return 0 + } + return fieldUint32(fields, "protocol") +} + +type rawPacketHeaderInput struct { + AgentIP string `json:"agent_ip"` + SubAgentID uint32 `json:"sub_agent_id"` + SourceID uint32 `json:"source_id"` + SamplingRate uint32 `json:"sampling_rate"` + SamplePool uint32 `json:"sample_pool"` + Drops uint32 `json:"drops"` + InputIf uint32 `json:"input_if"` + OutputIf uint32 `json:"output_if"` + Protocol uint32 `json:"protocol"` + FrameLength uint32 `json:"frame_length"` + Stripped uint32 `json:"stripped"` + OriginalLength uint32 `json:"original_length"` + HeaderHex string `json:"header_hex"` +} + +// processJSONRawPacketHeader accepts a user-friendly JSON representation of an +// sFlow sampled-header event and normalizes it into canonical packet fields. +func (p *Builtin) processJSONRawPacketHeader(evt *event.Event) ([]*event.Event, error) { + var in rawPacketHeaderInput + if err := json.Unmarshal(evt.Message, &in); err != nil { + return nil, fmt.Errorf("decode json_raw_packet_header: %w", err) + } + + headerData, err := hex.DecodeString(in.HeaderHex) + if err != nil { + return nil, fmt.Errorf("decode header_hex: %w", err) + } + + if in.SamplingRate == 0 { + in.SamplingRate = 1 + } + if in.OriginalLength == 0 { + in.OriginalLength = uint32(len(headerData)) + } + if in.FrameLength == 0 { + in.FrameLength = in.OriginalLength + } + + fields := ensureFields(evt, 16) + fields["agent_ip"] = in.AgentIP + fields["sub_agent_id"] = in.SubAgentID + fields["source_id"] = in.SourceID + fields["sampling_rate"] = in.SamplingRate + fields["sample_pool"] = in.SamplePool + fields["drops"] = in.Drops + fields["input_if"] = in.InputIf + fields["output_if"] = in.OutputIf + fields["protocol"] = in.Protocol + fields["header_protocol_name"] = sampledHeaderProtocolName(in.Protocol) + fields["frame_length"] = in.FrameLength + fields["stripped"] = in.Stripped + fields["original_length"] = in.OriginalLength + fields["header_data"] = headerData + fields["bytes"] = int64(in.FrameLength) + fields["packets"] = int64(1) + if err := packet.NormalizeEvent(evt, packet.NormalizeOptions{ + DisablePacketMapping: p.cfg.DisablePacketMapping, + TruncatePacketBytes: p.cfg.TruncatePacketBytes, + HeaderProtocol: in.Protocol, + Decode: p.decode, + AggregationHelpers: p.aggregationHelpers(), + }); err != nil { + return nil, err + } + p.applyDerivedFieldMappings(evt) + + if p.cfg.DropMessage { + evt.Message = nil + } + + return []*event.Event{evt}, nil +} + +// processJSONFlavor maps generic JSON messages into canonical fields based on source metadata. +func (p *Builtin) processJSONFlavor(evt *event.Event) ([]*event.Event, error) { + var payload any + decoder := json.NewDecoder(bytes.NewReader(evt.Message)) + decoder.UseNumber() + if err := decoder.Decode(&payload); err != nil { + return nil, fmt.Errorf("decode json flavor %q: %w", evt.Source.JSON.Flavor, err) + } + + switch evt.Source.JSON.Flavor { + case "reflow": + return p.processReFlowJSON(evt, payload) + case "raw_packet_header": + record, ok := payload.(map[string]any) + if !ok { + return nil, fmt.Errorf("raw_packet_header expects a JSON object") + } + data, err := json.Marshal(record) + if err != nil { + return nil, fmt.Errorf("encode raw_packet_header payload: %w", err) + } + evt.Message = data + return p.processJSONRawPacketHeader(evt) + case "vendor": + return p.processVendor(evt, payload) + case "goflow2v2": + return p.processGoFlow2V2(evt, payload) + default: + return nil, fmt.Errorf("unsupported source.json.flavor %q", evt.Source.JSON.Flavor) + } +} + +// processVendor keeps opaque vendor JSON intact while tagging it so routing and +// encoding can treat it as a distinct message class. +func (p *Builtin) processVendor(evt *event.Event, payload any) ([]*event.Event, error) { + _, ok := payload.(map[string]any) + if !ok { + return nil, fmt.Errorf("vendor expects a JSON object") + } + + fields := ensureFields(evt, 2) + fields["json_flavor"] = evt.Source.JSON.Flavor + if p.cfg.DropMessage { + evt.Message = nil + } + return []*event.Event{evt}, nil +} + +// processReFlowJSON accepts the native ReFlow JSON shape and forwards field +// extraction to processReFlowFields. +func (p *Builtin) processReFlowJSON(evt *event.Event, payload any) ([]*event.Event, error) { + record, ok := payload.(map[string]any) + if !ok { + return nil, fmt.Errorf("reflow expects a JSON object") + } + if sflowValue, ok := record["sflow"]; ok && reFlowSFlowEnvelopeHasSamples(sflowValue) { + if err := applyReFlowSFlowEnvelope(evt, sflowValue); err != nil { + return nil, err + } + } + if fields, ok := canonicalEventFields(record); ok { + return p.processReFlowFields(evt, fields) + } + return p.processReFlowFields(evt, record) +} + +// processReFlowFields copies a native ReFlow JSON object directly into the event field map. +func (p *Builtin) processReFlowFields(evt *event.Event, record map[string]any) ([]*event.Event, error) { + fields := ensureFields(evt, len(record)) + for key, value := range record { + if key == "sflow" { + if reFlowSFlowEnvelopeHasSamples(value) { + if err := applyReFlowSFlowEnvelope(evt, value); err != nil { + return nil, err + } + } + continue + } + if key == "packet" { + model, err := reFlowPacketModelFromValue(value) + if err != nil { + return nil, err + } + evt.Packet = model + packet.ApplyModelFieldsWithHelpers(fields, model, p.aggregationHelpers()) + continue + } + if key == "aggregation" { + meta, err := reFlowAggregationMetadataFromValue(value) + if err != nil { + return nil, err + } + evt.Aggregation = meta + continue + } + fields[key] = normalizeReFlowJSONValue(key, value) + } + setIPFamilyFromFields(fields) + p.applyDerivedFieldMappings(evt) + if p.cfg.DropMessage { + evt.Message = nil + } + return []*event.Event{evt}, nil +} + +func reFlowSFlowEnvelopeHasSamples(value any) bool { + record, ok := value.(map[string]any) + if !ok { + return false + } + samples, ok := record["samples"].([]any) + return ok && len(samples) > 0 +} + +func applyReFlowSFlowEnvelope(evt *event.Event, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("encode reflow sflow envelope: %w", err) + } + var meta event.SFlowMetadata + if err := json.Unmarshal(raw, &meta); err != nil { + return fmt.Errorf("decode reflow sflow envelope: %w", err) + } + evt.SFlow = &meta + return nil +} + +func (p *Builtin) applyDerivedFieldMappings(evt *event.Event) { + if evt == nil { + return + } + deriveFlowDirectionFromCapture(evt.Fields) + deriveNATFieldsFromConntrack(evt.Fields, evt.Internal, p.cfg.NAT) +} + +func deriveFlowDirectionFromCapture(fields map[string]any) { + if fields == nil { + return + } + if _, ok := fields["flow_direction"]; ok { + return + } + switch strings.ToLower(fieldStringOrZero(fields, "capture_direction")) { + case "in", "ingress", "input": + fields["flow_direction"] = uint32(0) + case "out", "egress", "output": + fields["flow_direction"] = uint32(1) + } +} + +func deriveNATFieldsFromConntrack(fields, internal map[string]any, cfg config.NATProcessorConfig) { + if fields == nil { + return + } + originalSrc := stringAlias(fields, "conntrack_original_src_addr") + originalDst := stringAlias(fields, "conntrack_original_dst_addr") + replySrc := stringAlias(fields, "conntrack_reply_src_addr") + if replySrc == "" { + replySrc = stringAlias(internal, "conntrack_reply_src_addr") + } + replyDst := stringAlias(fields, "conntrack_reply_dst_addr") + if replyDst == "" { + replyDst = stringAlias(internal, "conntrack_reply_dst_addr") + } + if originalSrc == "" || originalDst == "" || replySrc == "" || replyDst == "" { + return + } + + originalSrcPort := uint32FromAny(fields["conntrack_original_src_port"]) + originalDstPort := uint32FromAny(fields["conntrack_original_dst_port"]) + replySrcPort := uint32FromAny(firstValue(fields, internal, "conntrack_reply_src_port")) + replyDstPort := uint32FromAny(firstValue(fields, internal, "conntrack_reply_dst_port")) + + hasSNAT := originalSrc != replyDst || portsDiffer(originalSrcPort, replyDstPort) + hasDNAT := originalDst != replySrc || portsDiffer(originalDstPort, replySrcPort) + if hasSNAT { + fields["nat_src_addr"] = replyDst + fields["nat_src_port"] = replyDstPort + } + if hasDNAT { + fields["nat_dst_addr"] = replySrc + fields["nat_dst_port"] = replySrcPort + } + if cfg.SwapPrePost && isEgressCapture(fields) && (hasSNAT || hasDNAT) { + fields["src_addr"] = originalSrc + fields["src_port"] = originalSrcPort + fields["dst_addr"] = originalDst + fields["dst_port"] = originalDstPort + refreshIPFamilyFromFields(fields) + } +} + +func portsDiffer(a, b uint32) bool { + return a != 0 && b != 0 && a != b +} + +func isEgressCapture(fields map[string]any) bool { + switch strings.ToLower(fieldStringOrZero(fields, "capture_direction")) { + case "out", "egress", "output": + return true + default: + return false + } +} + +func firstValue(primary, fallback map[string]any, key string) any { + if primary != nil { + if val, ok := primary[key]; ok { + return val + } + } + if fallback != nil { + return fallback[key] + } + return nil +} + +func refreshIPFamilyFromFields(fields map[string]any) { + if fields == nil { + return + } + delete(fields, "ip_family") + setIPFamilyFromFields(fields) +} + +func reFlowPacketModelFromValue(value any) (*event.PacketModel, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("encode reflow packet model: %w", err) + } + var model event.PacketModel + if err := json.Unmarshal(data, &model); err != nil { + return nil, fmt.Errorf("decode reflow packet model: %w", err) + } + return &model, nil +} + +func reFlowAggregationMetadataFromValue(value any) (*event.AggregationMetadata, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("encode reflow aggregation metadata: %w", err) + } + var meta event.AggregationMetadata + if err := json.Unmarshal(data, &meta); err != nil { + return nil, fmt.Errorf("decode reflow aggregation metadata: %w", err) + } + return &meta, nil +} + +func canonicalEventFields(record map[string]any) (map[string]any, bool) { + fields, ok := record["fields"].(map[string]any) + if !ok { + return nil, false + } + for _, key := range []string{"received_at", "kind", "stream", "source", "control", "message", "packet", "sflow", "aggregation"} { + if _, ok := record[key]; ok { + return fields, true + } + } + return nil, false +} + +func normalizeReFlowJSONValue(key string, value any) any { + switch typed := value.(type) { + case json.Number: + if n, err := typed.Int64(); err == nil { + return n + } + f, err := typed.Float64() + if err != nil { + return value + } + return f + case []any: + out := make([]any, len(typed)) + for i, item := range typed { + out[i] = normalizeReFlowJSONValue("", item) + } + return out + case map[string]any: + out := make(map[string]any, len(typed)) + for k, item := range typed { + out[k] = normalizeReFlowJSONValue(k, item) + } + return out + default: + return value + } +} + +func packetDecodeOptions(cfg config.PacketDecoderConfig) packet.DecodeOptions { + opts := packet.DecodeOptions{ + Configured: true, + DecodeBeyondL4: true, + DecodeGRE: true, + DecodeIPIP: true, + DecodeVXLAN: true, + DecodeGeneve: true, + DecodeL2TP: true, + DecodeGTPU: true, + DecodePPPoE: true, + } + if cfg.DecodeBeyondL4 != nil { + opts.DecodeBeyondL4 = *cfg.DecodeBeyondL4 + } + encaps := cfg.Encapsulations + if encaps.GRE.Enabled != nil { + opts.DecodeGRE = *encaps.GRE.Enabled + } + if encaps.IPIP.Enabled != nil { + opts.DecodeIPIP = *encaps.IPIP.Enabled + } + if encaps.VXLAN.Enabled != nil { + opts.DecodeVXLAN = *encaps.VXLAN.Enabled + } + if len(encaps.VXLAN.Ports) > 0 { + opts.VXLANPorts = append([]uint32(nil), encaps.VXLAN.Ports...) + } + if encaps.Geneve.Enabled != nil { + opts.DecodeGeneve = *encaps.Geneve.Enabled + } + if len(encaps.Geneve.Ports) > 0 { + opts.GenevePorts = append([]uint32(nil), encaps.Geneve.Ports...) + } + if encaps.L2TP.Enabled != nil { + opts.DecodeL2TP = *encaps.L2TP.Enabled + } + if len(encaps.L2TP.Ports) > 0 { + opts.L2TPPorts = append([]uint32(nil), encaps.L2TP.Ports...) + } + if encaps.GTPU.Enabled != nil { + opts.DecodeGTPU = *encaps.GTPU.Enabled + } + if len(encaps.GTPU.Ports) > 0 { + opts.GTPUPorts = append([]uint32(nil), encaps.GTPU.Ports...) + } + if encaps.PPPoE.Enabled != nil { + opts.DecodePPPoE = *encaps.PPPoE.Enabled + } + return opts +} + +// processGoFlow2V2 translates the existing goflow2 JSON schema into ReFlow's +// canonical field set so users can bridge between the old and new binaries. +func (p *Builtin) processGoFlow2V2(evt *event.Event, payload any) ([]*event.Event, error) { + record, ok := payload.(map[string]any) + if !ok { + return nil, fmt.Errorf("goflow2v2 expects a JSON object") + } + + fields := ensureFields(evt, 24) + if ip := decodeMaybeBase64IP(stringAlias(record, "sampler_address")); ip != "" { + fields["agent_ip"] = ip + } + if ip := decodeMaybeBase64IP(stringAlias(record, "src_addr")); ip != "" { + fields["src_addr"] = ip + } + if ip := decodeMaybeBase64IP(stringAlias(record, "dst_addr")); ip != "" { + fields["dst_addr"] = ip + } + setUint32Alias(fields, record, "src_port", "src_port") + setUint32Alias(fields, record, "dst_port", "dst_port") + setUint32Alias(fields, record, "proto", "proto") + setUint32Alias(fields, record, "tcp_flags", "tcp_flags") + if proto := fieldUint32(fields, "proto"); proto != 0 { + fields["proto_name"] = ipProtocolName(proto) + } + setUint32Alias(fields, record, "sampling_rate", "sampling_rate") + setUint32Alias(fields, record, "input_if", "in_if") + setUint32Alias(fields, record, "output_if", "out_if") + setInt64Alias(fields, record, "bytes", "bytes") + setInt64Alias(fields, record, "packets", "packets") + setTimeNSAlias(fields, record, "start_time_unix", "time_flow_start_ns") + setTimeNSAlias(fields, record, "end_time_unix", "time_flow_end_ns") + fields["json_flavor"] = evt.Source.JSON.Flavor + + if typeVal, ok := record["type"]; ok { + switch flowType := uint32FromAny(typeVal); flowType { + case uint32(flowmessage.FlowMessage_SFLOW_5): + fields["flow_type"] = "sflow" + case uint32(flowmessage.FlowMessage_NETFLOW_V5): + fields["flow_type"] = "netflowv5" + case uint32(flowmessage.FlowMessage_NETFLOW_V9): + fields["flow_type"] = "netflowv9" + case uint32(flowmessage.FlowMessage_IPFIX): + fields["flow_type"] = "ipfix" + default: + fields["flow_type"] = flowType + } + } + if p.cfg.DropMessage { + evt.Message = nil + } + return []*event.Event{evt}, nil +} + +// ensureFields lazily allocates the processor field map. +func ensureFields(evt *event.Event, capacity int) map[string]any { + if evt.Fields == nil { + evt.Fields = make(map[string]any, capacity) + } + return evt.Fields +} + +// fieldStringOrZero normalizes string-like field values from the generic map. +func fieldStringOrZero(fields map[string]any, key string) string { + if fields == nil { + return "" + } + val, ok := fields[key] + if !ok { + return "" + } + switch v := val.(type) { + case string: + return v + default: + return fmt.Sprint(v) + } +} + +func setIPFamilyFromFields(fields map[string]any) { + if fields == nil || fieldStringOrZero(fields, "ip_family") != "" { + return + } + for _, raw := range []string{fieldStringOrZero(fields, "src_addr"), fieldStringOrZero(fields, "dst_addr")} { + if raw == "" { + continue + } + addr, err := netip.ParseAddr(raw) + if err != nil { + continue + } + if addr.Is6() { + fields["ip_family"] = "ipv6" + } else { + fields["ip_family"] = "ipv4" + } + return + } +} + +// fieldUint32 reads one field from the generic field map and normalizes it to u32. +func fieldUint32(fields map[string]any, key string) uint32 { + if fields == nil { + return 0 + } + val, ok := fields[key] + if !ok { + return 0 + } + return uint32FromAny(val) +} + +// fieldUint64 reads one field from the generic field map and normalizes it to u64. +func fieldUint64(fields map[string]any, key string) uint64 { + if fields == nil { + return 0 + } + val, ok := fields[key] + if !ok { + return 0 + } + return uint64FromAny(val) +} + +// packetTimeWindowNS preserves nanosecond timing for raw packet events. When an +// interface speed is available, the end time includes the estimated wire time; +// otherwise a sampled packet is treated as an instant. +func packetTimeWindowNS(start time.Time, frameLength uint32, ifSpeed uint64) (int64, int64) { + startNS := start.UnixNano() + durationNS := packetTransmissionDurationNS(frameLength, ifSpeed) + const maxInt64 = int64(1<<63 - 1) + if durationNS > 0 && startNS > maxInt64-durationNS { + return startNS, maxInt64 + } + return startNS, startNS + durationNS +} + +func packetTransmissionDurationNS(frameLength uint32, ifSpeed uint64) int64 { + if frameLength == 0 || ifSpeed == 0 { + return 0 + } + const maxInt64 = uint64(1<<63 - 1) + bits := uint64(frameLength) * 8 + if bits > maxInt64/uint64(time.Second) { + return int64(maxInt64) + } + bitNanos := bits * uint64(time.Second) + durationNS := bitNanos / ifSpeed + if bitNanos%ifSpeed != 0 { + durationNS++ + } + if durationNS > maxInt64 { + return int64(maxInt64) + } + return int64(durationNS) +} + +// stringAlias returns the first present source key as a string. +func stringAlias(m map[string]any, keys ...string) string { + for _, key := range keys { + if val, ok := m[key]; ok { + switch v := val.(type) { + case string: + return v + case fmt.Stringer: + return v.String() + default: + return fmt.Sprint(v) + } + } + } + return "" +} + +// setUint32Alias copies the first matching source key into the destination under dstKey. +func setUint32Alias(dst map[string]any, src map[string]any, dstKey string, srcKeys ...string) { + if src == nil { + return + } + for _, key := range srcKeys { + if val, ok := src[key]; ok { + dst[dstKey] = uint32FromAny(val) + return + } + } +} + +// setInt64Alias copies one numeric value from any of the alias keys. +func setInt64Alias(dst map[string]any, src map[string]any, dstKey string, srcKeys ...string) { + if src == nil { + return + } + for _, key := range srcKeys { + if val, ok := src[key]; ok { + dst[dstKey] = int64FromAny(val) + return + } + } +} + +// setTimeNSAlias converts nanosecond timestamps into the millisecond unit used by ReFlow fields. +func setTimeNSAlias(dst map[string]any, src map[string]any, dstKey string, srcKeys ...string) { + if src == nil { + return + } + for _, key := range srcKeys { + if val, ok := src[key]; ok { + ns := int64FromAny(val) + if ns == 0 { + return + } + dst[dstKey] = ns / int64(time.Millisecond) + switch dstKey { + case "start_time_unix": + dst["time_flow_start_ns"] = ns + case "end_time_unix": + dst["time_flow_end_ns"] = ns + } + return + } + } +} + +// uint32FromAny normalizes the common JSON and generic-map number representations. +func uint32FromAny(val any) uint32 { + switch v := val.(type) { + case uint32: + return v + case uint64: + return uint32(v) + case int: + if v < 0 { + return 0 + } + return uint32(v) + case int64: + if v < 0 { + return 0 + } + return uint32(v) + case float64: + if v < 0 { + return 0 + } + return uint32(v) + case json.Number: + n, _ := v.Int64() + if n < 0 { + return 0 + } + return uint32(n) + case string: + n := int64FromString(v) + if n < 0 { + return 0 + } + return uint32(n) + default: + return 0 + } +} + +// uint64FromAny normalizes the common JSON and generic-map number representations. +func uint64FromAny(val any) uint64 { + switch v := val.(type) { + case uint64: + return v + case uint32: + return uint64(v) + case int: + if v < 0 { + return 0 + } + return uint64(v) + case int64: + if v < 0 { + return 0 + } + return uint64(v) + case float64: + if v < 0 { + return 0 + } + return uint64(v) + case json.Number: + n, _ := v.Int64() + if n < 0 { + return 0 + } + return uint64(n) + case string: + n := int64FromString(v) + if n < 0 { + return 0 + } + return uint64(n) + default: + return 0 + } +} + +// int64FromAny normalizes the common JSON and generic-map number representations. +func int64FromAny(val any) int64 { + switch v := val.(type) { + case int64: + return v + case uint64: + return int64(v) + case uint32: + return int64(v) + case int: + return int64(v) + case float64: + return int64(v) + case json.Number: + n, _ := v.Int64() + return n + case string: + return int64FromString(v) + default: + return 0 + } +} + +// int64FromString parses a decimal string and returns zero on failure. +func int64FromString(s string) int64 { + n, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + return n +} + +// decodeMaybeBase64IP accepts either a normal IP string or the base64-encoded +// byte form used by legacy goflow2 JSON. +func decodeMaybeBase64IP(val string) string { + if val == "" { + return "" + } + if addr, err := netip.ParseAddr(val); err == nil { + return addr.String() + } + raw, err := base64.StdEncoding.DecodeString(val) + if err != nil { + return "" + } + addr, ok := netip.AddrFromSlice(raw) + if !ok { + return "" + } + return addr.String() +} + +// ipProtocolName maps common IANA protocol numbers to readable names. +func ipProtocolName(proto uint32) string { + switch proto { + case 1: + return "icmp" + case 2: + return "igmp" + case 6: + return "tcp" + case 17: + return "udp" + case 41: + return "ipv6" + case 47: + return "gre" + case 50: + return "esp" + case 51: + return "ah" + case 58: + return "icmpv6" + case 132: + return "sctp" + default: + return "" + } +} + +// sampledHeaderProtocolName maps sFlow sampled-header protocol IDs to names. +func sampledHeaderProtocolName(proto uint32) string { + switch proto { + case 1: + return "ethernet" + case 11: + return "ipv4" + case 12: + return "ipv6" + default: + return "" + } +} diff --git a/pkg/reflow/processor/processor_bench_test.go b/pkg/reflow/processor/processor_bench_test.go new file mode 100644 index 00000000..bafa99f7 --- /dev/null +++ b/pkg/reflow/processor/processor_bench_test.go @@ -0,0 +1,150 @@ +package processor + +import ( + "encoding/json" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func BenchmarkBuiltinProcess(b *testing.B) { + benchmarks := []struct { + name string + proc *Builtin + new func() *event.Event + }{ + { + name: "bytes_ipv4_tcp", + proc: NewBuiltin(config.ProcessorConfig{}), + new: func() *event.Event { + packet := benchIPv4TCPPacket() + return &event.Event{ + ReceivedAt: benchEventTime(), + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + } + }, + }, + { + name: "bytes_vxlan", + proc: NewBuiltin(config.ProcessorConfig{}), + new: func() *event.Event { + return &event.Event{ + ReceivedAt: benchEventTime(), + Source: event.SourceMetadata{Type: "bytes"}, + Payload: vxlanTestPacket(4789), + } + }, + }, + { + name: "goflow2v2_json", + proc: NewBuiltin(config.ProcessorConfig{}), + new: func() *event.Event { + return &event.Event{ + ReceivedAt: benchEventTime(), + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "goflow2v2"}, + }, + Message: append([]byte(nil), benchGoFlow2V2JSON...), + } + }, + }, + { + name: "reflow_json", + proc: NewBuiltin(config.ProcessorConfig{}), + new: func() *event.Event { + return &event.Event{ + ReceivedAt: benchEventTime(), + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "reflow"}, + }, + Message: append([]byte(nil), benchReFlowJSON...), + } + }, + }, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + events, err := bm.proc.Process(bm.new()) + if err != nil { + b.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + b.Fatalf("expected 1 event, got %d", len(events)) + } + } + }) + } +} + +func benchIPv4TCPPacket() []byte { + return ethernetPayload( + 0x0800, + ipv4Packet(6, [4]byte{192, 0, 2, 1}, [4]byte{198, 51, 100, 2}, tcpHeader(12345, 443)), + ) +} + +func benchEventTime() time.Time { + return time.Unix(1_700_000_001, 123_000_000).UTC() +} + +var benchGoFlow2V2JSON = []byte(`{ + "type": 1, + "time_received_ns": 1700000001123000000, + "time_flow_start_ns": 1700000000100123456, + "time_flow_end_ns": 1700000000900123456, + "sampler_address": "wAACAQ==", + "src_addr": "wAACCg==", + "dst_addr": "xjNkFA==", + "src_port": 12345, + "dst_port": 443, + "proto": 6, + "bytes": 321, + "packets": 7, + "in_if": 9, + "out_if": 10, + "sampling_rate": 100 +}`) + +var benchReFlowJSON = mustMarshalBenchReFlowJSON() + +func mustMarshalBenchReFlowJSON() []byte { + raw := map[string]any{ + "received_at": benchEventTime().Format(time.RFC3339Nano), + "source": map[string]any{ + "type": "flow", + "network": "stream", + "address": "-", + }, + "fields": map[string]any{ + "flow_type": "sflow", + "record_kind": "packet", + "agent_ip": "192.0.2.1", + "src_addr": "192.0.2.10", + "dst_addr": "198.51.100.20", + "src_port": 12345, + "dst_port": 443, + "proto": 6, + "bytes": 321, + "packets": 7, + "time_flow_start_ns": int64(1_700_000_000_100_123_456), + "time_flow_end_ns": int64(1_700_000_000_900_123_456), + "protocol": 1, + "frame_length": len(benchIPv4TCPPacket()), + "original_length": len(benchIPv4TCPPacket()), + "header_data": benchIPv4TCPPacket(), + }, + } + out, err := json.Marshal(raw) + if err != nil { + panic(err) + } + return out +} diff --git a/pkg/reflow/processor/processor_test.go b/pkg/reflow/processor/processor_test.go new file mode 100644 index 00000000..2c5646ad --- /dev/null +++ b/pkg/reflow/processor/processor_test.go @@ -0,0 +1,1730 @@ +package processor + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func TestBuiltinProcessFlowDropsPayload(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + DropMessage: true, + DropPayload: true, + }, + }) + + evt := &event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Message: []byte(`{"ignored":true}`), + Payload: []byte{1, 2, 3}, + Fields: map[string]any{ + "flow_type": "sflow", + }, + } + + events, err := proc.Process(evt) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + if events[0].Message != nil { + t.Fatalf("expected message to be dropped") + } + if events[0].Payload != nil { + t.Fatalf("expected payload to be dropped") + } +} + +func TestBuiltinProcessFlowRequiresFlowType(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{}, + }) + + _, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + }) + if err == nil { + t.Fatalf("expected error for missing flow_type") + } +} + +func TestBuiltinProcessFlowNormalizesPacketRecordWithoutTruncatingDatagramPayload(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + TruncatePacketBytes: 32, + }, + }) + + header := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x02, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + datagram := make([]byte, 128) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Payload: datagram, + Fields: map[string]any{ + "flow_type": "sflow", + "record_kind": "packet", + "protocol": uint32(1), + "original_length": uint32(len(header)), + "frame_length": uint32(len(header)), + "header_data": append([]byte(nil), header...), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + fields := events[0].Fields + if got := fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected src_addr=192.0.2.1, got %#v", got) + } + if got := fields["dst_addr"]; got != "198.51.100.2" { + t.Fatalf("expected dst_addr=198.51.100.2, got %#v", got) + } + if got := fields["src_port"]; got != uint32(12345) { + t.Fatalf("expected src_port=12345, got %#v", got) + } + if got := fields["dst_port"]; got != uint32(443) { + t.Fatalf("expected dst_port=443, got %#v", got) + } + headerData, ok := fields["header_data"].([]byte) + if !ok { + t.Fatalf("expected header_data to be []byte, got %T", fields["header_data"]) + } + if len(headerData) != 32 { + t.Fatalf("expected truncated header_data length=32, got %d", len(headerData)) + } + payload, ok := events[0].Payload.([]byte) + if !ok { + t.Fatalf("expected payload to remain []byte, got %T", events[0].Payload) + } + if len(payload) != len(datagram) { + t.Fatalf("expected exporter datagram payload length=%d, got %d", len(datagram), len(payload)) + } +} + +func TestBuiltinProcessBytesDecodesPacketTuple(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + DropPayload: false, + }, + }) + + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x02, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + + evt := &event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + } + + events, err := proc.Process(evt) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + fields := events[0].Fields + if got := fields["record_kind"]; got != "packet" { + t.Fatalf("expected record_kind=packet, got %#v", got) + } + if got := fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected src_addr=192.0.2.1, got %#v", got) + } + if got := fields["dst_addr"]; got != "198.51.100.2" { + t.Fatalf("expected dst_addr=198.51.100.2, got %#v", got) + } + if got := fields["src_port"]; got != uint32(12345) { + t.Fatalf("expected src_port=12345, got %#v", got) + } + if got := fields["dst_port"]; got != uint32(443) { + t.Fatalf("expected dst_port=443, got %#v", got) + } + if got := fields["proto"]; got != uint32(6) { + t.Fatalf("expected proto=6, got %#v", got) + } + if got := fields["bytes"]; got != int64(len(packet)) { + t.Fatalf("expected bytes=%d, got %#v", len(packet), got) + } + headerData, ok := fields["header_data"].([]byte) + if !ok { + t.Fatalf("expected header_data to be []byte, got %T", fields["header_data"]) + } + if len(headerData) != len(packet) { + t.Fatalf("expected header_data length=%d, got %d", len(packet), len(headerData)) + } + if events[0].Packet == nil { + t.Fatalf("expected packet model to be attached") + } + if len(events[0].Packet.Layers) != 3 { + t.Fatalf("expected 3 packet layers, got %d", len(events[0].Packet.Layers)) + } +} + +func TestBuiltinProcessBytesSetsNanosecondPacketWindowFromInterfaceSpeed(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + receivedAt := time.Unix(1_700_000_000, 123_000_000).UTC() + packet := ethernetPayload( + 0x0800, + ipv4Packet(6, [4]byte{192, 0, 2, 1}, [4]byte{198, 51, 100, 2}, tcpHeader(12345, 443)), + ) + + events, err := proc.Process(&event.Event{ + ReceivedAt: receivedAt, + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + Fields: map[string]any{ + "wire_length": uint32(250_000), + "if_speed": uint64(1_000_000_000), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + fields := events[0].Fields + startNS := receivedAt.UnixNano() + endNS := startNS + int64(2*time.Millisecond) + if got := fields["time_flow_start_ns"]; got != startNS { + t.Fatalf("expected time_flow_start_ns=%d, got %#v", startNS, got) + } + if got := fields["time_flow_end_ns"]; got != endNS { + t.Fatalf("expected time_flow_end_ns=%d, got %#v", endNS, got) + } + if got := fields["start_time_unix"]; got != receivedAt.UnixMilli() { + t.Fatalf("expected start_time_unix=%d, got %#v", receivedAt.UnixMilli(), got) + } + if got := fields["end_time_unix"]; got != receivedAt.Add(2*time.Millisecond).UnixMilli() { + t.Fatalf("expected end_time_unix=%d, got %#v", receivedAt.Add(2*time.Millisecond).UnixMilli(), got) + } +} + +func TestBuiltinProcessGoFlow2V2PreservesNanosecondTimeAliases(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "goflow2v2"}, + }, + Message: []byte(`{ + "type": 1, + "tcp_flags": 18, + "time_flow_start_ns": 1700000000100123456, + "time_flow_end_ns": 1700000000900123456 + }`), + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + fields := events[0].Fields + if got := fields["time_flow_start_ns"]; got != int64(1_700_000_000_100_123_456) { + t.Fatalf("expected time_flow_start_ns to preserve nanoseconds, got %#v", got) + } + if got := fields["time_flow_end_ns"]; got != int64(1_700_000_000_900_123_456) { + t.Fatalf("expected time_flow_end_ns to preserve nanoseconds, got %#v", got) + } + if got := fields["start_time_unix"]; got != int64(1_700_000_000_100) { + t.Fatalf("expected start_time_unix milliseconds, got %#v", got) + } + if got := fields["end_time_unix"]; got != int64(1_700_000_000_900) { + t.Fatalf("expected end_time_unix milliseconds, got %#v", got) + } + if got := fields["tcp_flags"]; got != uint32(18) { + t.Fatalf("expected tcp_flags to be copied, got %#v", got) + } + if events[0].SFlow != nil { + t.Fatalf("expected sflow metadata to remain unset, got %#v", events[0].SFlow) + } +} + +func TestBuiltinProcessBytesDerivesNATFieldsFromConntrack(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + packet := ethernetPayload( + 0x0800, + ipv4Packet(6, [4]byte{192, 168, 1, 10}, [4]byte{198, 51, 100, 20}, tcpHeader(12345, 443)), + ) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + Fields: map[string]any{ + "conntrack_original_src_addr": "192.168.1.10", + "conntrack_original_src_port": uint32(12345), + "conntrack_original_dst_addr": "198.51.100.20", + "conntrack_original_dst_port": uint32(443), + "conntrack_reply_src_addr": "198.51.100.20", + "conntrack_reply_src_port": uint32(443), + "conntrack_reply_dst_addr": "203.0.113.9", + "conntrack_reply_dst_port": uint32(54321), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + fields := events[0].Fields + if got := fields["nat_src_addr"]; got != "203.0.113.9" { + t.Fatalf("expected nat_src_addr=203.0.113.9, got %#v", got) + } + if got := fields["nat_src_port"]; got != uint32(54321) { + t.Fatalf("expected nat_src_port=54321, got %#v", got) + } + if _, ok := fields["nat_dst_addr"]; ok { + t.Fatalf("expected no nat_dst_addr for SNAT-only tuple, got %#v", fields["nat_dst_addr"]) + } +} + +func TestBuiltinProcessBytesDerivesNATFieldsFromInternalConntrackReply(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + packet := ethernetPayload( + 0x0800, + ipv4Packet(6, [4]byte{192, 168, 1, 10}, [4]byte{198, 51, 100, 20}, tcpHeader(12345, 443)), + ) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + Fields: map[string]any{ + "conntrack_original_src_addr": "192.168.1.10", + "conntrack_original_src_port": uint32(12345), + "conntrack_original_dst_addr": "198.51.100.20", + "conntrack_original_dst_port": uint32(443), + }, + Internal: map[string]any{ + "conntrack_reply_src_addr": "198.51.100.20", + "conntrack_reply_src_port": uint32(443), + "conntrack_reply_dst_addr": "203.0.113.9", + "conntrack_reply_dst_port": uint32(54321), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + fields := events[0].Fields + if got := fields["nat_src_addr"]; got != "203.0.113.9" { + t.Fatalf("expected nat_src_addr=203.0.113.9, got %#v", got) + } + if got := fields["nat_src_port"]; got != uint32(54321) { + t.Fatalf("expected nat_src_port=54321, got %#v", got) + } + if _, ok := fields["conntrack_reply_dst_addr"]; ok { + t.Fatalf("expected conntrack_reply_dst_addr not to be exported, got %#v", fields["conntrack_reply_dst_addr"]) + } +} + +func TestBuiltinProcessBytesCanSwapPreNATTupleIntoCanonicalFields(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + NAT: config.NATProcessorConfig{SwapPrePost: true}, + }, + }) + packet := ethernetPayload( + 0x0800, + ipv4Packet(6, [4]byte{203, 0, 113, 9}, [4]byte{198, 51, 100, 20}, tcpHeader(54321, 443)), + ) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + Fields: map[string]any{ + "conntrack_original_src_addr": "192.168.1.10", + "conntrack_original_src_port": uint32(12345), + "conntrack_original_dst_addr": "198.51.100.20", + "conntrack_original_dst_port": uint32(443), + "capture_direction": "out", + "ip_family": "ipv6", + }, + Internal: map[string]any{ + "conntrack_reply_src_addr": "198.51.100.20", + "conntrack_reply_src_port": uint32(443), + "conntrack_reply_dst_addr": "203.0.113.9", + "conntrack_reply_dst_port": uint32(54321), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + fields := events[0].Fields + if got := fields["src_addr"]; got != "192.168.1.10" { + t.Fatalf("expected src_addr to be pre-NAT source, got %#v", got) + } + if got := fields["src_port"]; got != uint32(12345) { + t.Fatalf("expected src_port to be pre-NAT source port, got %#v", got) + } + if got := fields["dst_addr"]; got != "198.51.100.20" { + t.Fatalf("expected dst_addr to be pre-NAT destination, got %#v", got) + } + if got := fields["dst_port"]; got != uint32(443) { + t.Fatalf("expected dst_port to be pre-NAT destination port, got %#v", got) + } + if got := fields["nat_src_addr"]; got != "203.0.113.9" { + t.Fatalf("expected nat_src_addr to remain translated source, got %#v", got) + } + if got := fields["nat_src_port"]; got != uint32(54321) { + t.Fatalf("expected nat_src_port to remain translated source port, got %#v", got) + } + if got := fields["ip_family"]; got != "ipv4" { + t.Fatalf("expected ip_family to refresh from swapped tuple, got %#v", got) + } +} + +func TestBuiltinProcessBytesDerivesFlowDirectionFromCaptureDirection(t *testing.T) { + tests := []struct { + name string + direction string + want uint32 + }{ + {name: "ingress", direction: "in", want: 0}, + {name: "egress", direction: "out", want: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: ethernetPayload( + 0x0800, + ipv4Packet(6, [4]byte{192, 0, 2, 10}, [4]byte{198, 51, 100, 20}, tcpHeader(12345, 443)), + ), + Fields: map[string]any{ + "capture_direction": tt.direction, + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if got := events[0].Fields["flow_direction"]; got != tt.want { + t.Fatalf("expected flow_direction=%d, got %#v", tt.want, got) + } + }) + } +} + +func TestBuiltinProcessBytesDoesNotSwapPreNATTupleForIngressCapture(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + NAT: config.NATProcessorConfig{SwapPrePost: true}, + }, + }) + packet := ethernetPayload( + 0x0800, + ipv4Packet(6, [4]byte{203, 0, 113, 9}, [4]byte{198, 51, 100, 20}, tcpHeader(54321, 443)), + ) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + Fields: map[string]any{ + "conntrack_original_src_addr": "192.168.1.10", + "conntrack_original_src_port": uint32(12345), + "conntrack_original_dst_addr": "198.51.100.20", + "conntrack_original_dst_port": uint32(443), + "capture_direction": "in", + }, + Internal: map[string]any{ + "conntrack_reply_src_addr": "198.51.100.20", + "conntrack_reply_src_port": uint32(443), + "conntrack_reply_dst_addr": "203.0.113.9", + "conntrack_reply_dst_port": uint32(54321), + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + fields := events[0].Fields + if got := fields["src_addr"]; got != "203.0.113.9" { + t.Fatalf("expected ingress src_addr to stay captured/post-NAT source, got %#v", got) + } + if got := fields["src_port"]; got != uint32(54321) { + t.Fatalf("expected ingress src_port to stay captured/post-NAT source port, got %#v", got) + } + if got := fields["nat_src_addr"]; got != "203.0.113.9" { + t.Fatalf("expected nat_src_addr to still be derived, got %#v", got) + } + if got := fields["nat_src_port"]; got != uint32(54321) { + t.Fatalf("expected nat_src_port to still be derived, got %#v", got) + } +} + +func TestBuiltinProcessReFlowJSONDerivesNATFieldsFromConntrack(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "json", JSON: event.JSONMetadata{Flavor: "reflow"}}, + Message: []byte(`{ + "conntrack_original_src_addr": "2001:db8::10", + "conntrack_original_src_port": 12345, + "conntrack_original_dst_addr": "2001:db8::20", + "conntrack_original_dst_port": 443, + "conntrack_reply_src_addr": "192.0.2.20", + "conntrack_reply_src_port": 8443, + "conntrack_reply_dst_addr": "2001:db8::10", + "conntrack_reply_dst_port": 12345 + }`), + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + fields := events[0].Fields + if got := fields["nat_dst_addr"]; got != "192.0.2.20" { + t.Fatalf("expected nat_dst_addr=192.0.2.20, got %#v", got) + } + if got := fields["nat_dst_port"]; got != uint32(8443) { + t.Fatalf("expected nat_dst_port=8443, got %#v", got) + } + if _, ok := fields["nat_src_addr"]; ok { + t.Fatalf("expected no nat_src_addr for DNAT-only tuple, got %#v", fields["nat_src_addr"]) + } +} + +func TestDerivedNATFieldsKeepUnchangedEndpointParts(t *testing.T) { + fields := map[string]any{ + "conntrack_original_src_addr": "192.168.1.10", + "conntrack_original_src_port": uint32(12345), + "conntrack_original_dst_addr": "198.51.100.20", + "conntrack_original_dst_port": uint32(443), + "conntrack_reply_src_addr": "198.51.100.20", + "conntrack_reply_src_port": uint32(443), + "conntrack_reply_dst_addr": "203.0.113.9", + "conntrack_reply_dst_port": uint32(12345), + } + + NewBuiltin(config.ProcessorConfig{}).applyDerivedFieldMappings(&event.Event{Fields: fields}) + + if got := fields["nat_src_addr"]; got != "203.0.113.9" { + t.Fatalf("expected changed NAT source address, got %#v", got) + } + if got := fields["nat_src_port"]; got != uint32(12345) { + t.Fatalf("expected unchanged NAT source port to be kept, got %#v", got) + } + + fields = map[string]any{ + "conntrack_original_src_addr": "192.168.1.10", + "conntrack_original_src_port": uint32(12345), + "conntrack_original_dst_addr": "198.51.100.20", + "conntrack_original_dst_port": uint32(443), + "conntrack_reply_src_addr": "198.51.100.20", + "conntrack_reply_src_port": uint32(443), + "conntrack_reply_dst_addr": "192.168.1.10", + "conntrack_reply_dst_port": uint32(54321), + } + + NewBuiltin(config.ProcessorConfig{}).applyDerivedFieldMappings(&event.Event{Fields: fields}) + + if got := fields["nat_src_addr"]; got != "192.168.1.10" { + t.Fatalf("expected unchanged NAT source address to be kept, got %#v", got) + } + if got := fields["nat_src_port"]; got != uint32(54321) { + t.Fatalf("expected changed NAT source port, got %#v", got) + } +} + +func TestBuiltinProcessBytesCanDisablePacketMapping(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + DisablePacketMapping: true, + }, + }) + + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x02, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + if _, ok := fields["src_addr"]; ok { + t.Fatalf("expected src_addr to be absent when packet mapping is disabled") + } + if _, ok := fields["dst_addr"]; ok { + t.Fatalf("expected dst_addr to be absent when packet mapping is disabled") + } +} + +func TestBuiltinProcessBytesTruncatesRetainedPacketData(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + TruncatePacketBytes: 32, + }, + }) + + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x02, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + headerData, ok := fields["header_data"].([]byte) + if !ok { + t.Fatalf("expected header_data to be []byte, got %T", fields["header_data"]) + } + if len(headerData) != 32 { + t.Fatalf("expected truncated header_data length=32, got %d", len(headerData)) + } + payload, ok := events[0].Payload.([]byte) + if !ok { + t.Fatalf("expected payload to remain []byte, got %T", events[0].Payload) + } + if len(payload) != 32 { + t.Fatalf("expected truncated payload length=32, got %d", len(payload)) + } + if got := fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected src_addr to be mapped before truncation, got %#v", got) + } +} + +func TestBuiltinProcessBytesFallsBackToCaptureInterfaceIndex(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x02, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "bytes", + CaptureInterfaceIndex: 15, + }, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + if got := fields["input_if"]; got != uint32(15) { + t.Fatalf("expected input_if=15, got %#v", got) + } + if got := fields["output_if"]; got != uint32(15) { + t.Fatalf("expected output_if=15, got %#v", got) + } +} + +func TestBuiltinProcessBytesExtractsEncapsulatedLayers(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x81, 0x00, + 0x00, 0x64, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x34, 0x00, 0x01, 0x00, 0x00, 0x40, 0x2f, 0x00, 0x00, + 0xcb, 0x00, 0x71, 0x01, + 0xcb, 0x00, 0x71, 0x02, + 0x00, 0x00, 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x02, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x02, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + + if got := fields["src_mac"]; got != "66:77:88:99:aa:bb" { + t.Fatalf("expected src_mac=66:77:88:99:aa:bb, got %#v", got) + } + if got := fields["dst_mac"]; got != "00:11:22:33:44:55" { + t.Fatalf("expected dst_mac=00:11:22:33:44:55, got %#v", got) + } + if got := fields["vlan_id"]; got != uint32(100) { + t.Fatalf("expected vlan_id=100, got %#v", got) + } + if got := fields["dot1q_vlan_id"]; got != uint32(100) { + t.Fatalf("expected dot1q_vlan_id=100, got %#v", got) + } + if got := fields["dot1q_priority"]; got != uint32(0) { + t.Fatalf("expected dot1q_priority=0, got %#v", got) + } + if got := fields["dot1q_dei"]; got != false { + t.Fatalf("expected dot1q_dei=false, got %#v", got) + } + vlanIDs, ok := fields["vlan_ids"].([]uint32) + if !ok { + t.Fatalf("expected vlan_ids to be []uint32, got %T", fields["vlan_ids"]) + } + if len(vlanIDs) != 1 || vlanIDs[0] != 100 { + t.Fatalf("expected vlan_ids=[100], got %#v", vlanIDs) + } + expectedLayers := []string{"ethernet", "dot1q", "ipv4", "gre", "ipv4", "tcp"} + expectPacketLayers(t, events[0], expectedLayers) + if got := fields["outer_src_addr"]; got != "203.0.113.1" { + t.Fatalf("expected outer_src_addr=203.0.113.1, got %#v", got) + } + if got := fields["outer_dst_addr"]; got != "203.0.113.2" { + t.Fatalf("expected outer_dst_addr=203.0.113.2, got %#v", got) + } + if got := fields["outer_proto"]; got != uint32(47) { + t.Fatalf("expected outer_proto=47, got %#v", got) + } + expectIPLayer(t, fields, 0, "outer", "203.0.113.1", "203.0.113.2", 47, 0, 0) + expectIPLayer(t, fields, 1, "inner", "192.0.2.1", "198.51.100.2", 6, 12345, 443) + if got := fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected inner src_addr=192.0.2.1, got %#v", got) + } + if got := fields["dst_addr"]; got != "198.51.100.2" { + t.Fatalf("expected inner dst_addr=198.51.100.2, got %#v", got) + } + if got := fields["src_port"]; got != uint32(12345) { + t.Fatalf("expected inner src_port=12345, got %#v", got) + } + if got := fields["dst_port"]; got != uint32(443) { + t.Fatalf("expected inner dst_port=443, got %#v", got) + } +} + +func TestBuiltinProcessBytesExtractsVXLANInnerPacket(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + packet := vxlanTestPacket(4789) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + + expectedLayers := []string{"ethernet", "ipv4", "vxlan", "ethernet", "ipv4", "tcp"} + expectPacketLayers(t, events[0], expectedLayers) + if got := fields["vxlan_vni"]; got != uint32(100) { + t.Fatalf("expected vxlan_vni=100, got %#v", got) + } + if got := fields["layer2_segment_id"]; got != uint64(0x0100000000000064) { + t.Fatalf("expected VXLAN layer2_segment_id, got %#v", got) + } + if got := fields["outer_src_addr"]; got != "203.0.113.1" { + t.Fatalf("expected outer_src_addr=203.0.113.1, got %#v", got) + } + if got := fields["outer_dst_addr"]; got != "203.0.113.2" { + t.Fatalf("expected outer_dst_addr=203.0.113.2, got %#v", got) + } + expectIPLayer(t, fields, 0, "outer", "203.0.113.1", "203.0.113.2", 17, 4660, 4789) + expectIPLayer(t, fields, 1, "inner", "192.0.2.1", "198.51.100.2", 6, 12345, 443) + if got := fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected inner src_addr=192.0.2.1, got %#v", got) + } + if got := fields["dst_addr"]; got != "198.51.100.2" { + t.Fatalf("expected inner dst_addr=198.51.100.2, got %#v", got) + } +} + +func TestBuiltinProcessBytesExtractsDot1QCustomerFields(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + inner := ipv4Packet(6, [4]byte{192, 0, 2, 1}, [4]byte{198, 51, 100, 2}, tcpHeader(12345, 443)) + packet := ethernetPayload( + 0x88a8, + dot1qTCIPayload((5<<13)|(1<<12)|100, 0x8100, dot1qTCIPayload((3<<13)|200, 0x0800, inner)), + ) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + if got := fields["dot1q_vlan_id"]; got != uint32(100) { + t.Fatalf("expected dot1q_vlan_id=100, got %#v", got) + } + if got := fields["dot1q_priority"]; got != uint32(5) { + t.Fatalf("expected dot1q_priority=5, got %#v", got) + } + if got := fields["dot1q_dei"]; got != true { + t.Fatalf("expected dot1q_dei=true, got %#v", got) + } + if got := fields["dot1q_customer_vlan_id"]; got != uint32(200) { + t.Fatalf("expected dot1q_customer_vlan_id=200, got %#v", got) + } + if got := fields["dot1q_customer_priority"]; got != uint32(3) { + t.Fatalf("expected dot1q_customer_priority=3, got %#v", got) + } + if got := fields["dot1q_customer_dei"]; got != false { + t.Fatalf("expected dot1q_customer_dei=false, got %#v", got) + } +} + +func TestBuiltinProcessBytesCanDisableUDPTunnelDecoding(t *testing.T) { + disabled := false + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + PacketDecoder: config.PacketDecoderConfig{ + DecodeBeyondL4: &disabled, + }, + }, + }) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: vxlanTestPacket(4789), + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + expectedLayers := []string{"ethernet", "ipv4", "udp"} + expectPacketLayers(t, events[0], expectedLayers) + if got := fields["src_addr"]; got != "203.0.113.1" { + t.Fatalf("expected outer src_addr=203.0.113.1, got %#v", got) + } + if _, ok := fields["outer_src_addr"]; ok { + t.Fatalf("did not expect outer_src_addr when UDP tunnel decoding is disabled") + } +} + +func TestBuiltinProcessBytesCanDisableVXLANEncapsulation(t *testing.T) { + disabled := false + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + PacketDecoder: config.PacketDecoderConfig{ + Encapsulations: config.PacketEncapsulationConfig{ + VXLAN: config.PortEncapsulationConfig{ + Enabled: &disabled, + }, + }, + }, + }, + }) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: vxlanTestPacket(4789), + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + expectedLayers := []string{"ethernet", "ipv4", "udp"} + expectPacketLayers(t, events[0], expectedLayers) +} + +func TestBuiltinProcessBytesDecodesCustomVXLANPort(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + PacketDecoder: config.PacketDecoderConfig{ + Encapsulations: config.PacketEncapsulationConfig{ + VXLAN: config.PortEncapsulationConfig{ + Ports: []uint32{4790}, + }, + }, + }, + }, + }) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: vxlanTestPacket(4790), + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + expectIPLayer(t, fields, 0, "outer", "203.0.113.1", "203.0.113.2", 17, 4660, 4790) + expectIPLayer(t, fields, 1, "inner", "192.0.2.1", "198.51.100.2", 6, 12345, 443) + if got := fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected inner src_addr=192.0.2.1, got %#v", got) + } +} + +func TestBuiltinProcessBytesExtractsIPInIPFamilies(t *testing.T) { + inner4 := ipv4Packet(6, [4]byte{192, 0, 2, 1}, [4]byte{198, 51, 100, 2}, tcpHeader(12345, 443)) + inner6 := ipv6Packet(6, + [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}, + tcpHeader(12345, 443)) + + tests := []struct { + name string + payload []byte + outerLayer string + outerSrc string + outerDst string + outerProto uint32 + innerLayer string + innerSrc string + innerDst string + expectedSrc string + }{ + { + name: "ipv4 in ipv4", + payload: ethernetPayload(0x0800, ipv4Packet(4, [4]byte{203, 0, 113, 1}, [4]byte{203, 0, 113, 2}, inner4)), + outerLayer: "ipv4", + outerSrc: "203.0.113.1", + outerDst: "203.0.113.2", + outerProto: 4, + innerLayer: "ipv4", + innerSrc: "192.0.2.1", + innerDst: "198.51.100.2", + expectedSrc: "192.0.2.1", + }, + { + name: "ipv6 in ipv4", + payload: ethernetPayload(0x0800, ipv4Packet(41, [4]byte{203, 0, 113, 1}, [4]byte{203, 0, 113, 2}, inner6)), + outerLayer: "ipv4", + outerSrc: "203.0.113.1", + outerDst: "203.0.113.2", + outerProto: 41, + innerLayer: "ipv6", + innerSrc: "2001:db8::1", + innerDst: "2001:db8::2", + expectedSrc: "2001:db8::1", + }, + { + name: "ipv4 in ipv6", + payload: ethernetPayload(0x86dd, ipv6Packet(4, + [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}, + inner4)), + outerLayer: "ipv6", + outerSrc: "2001:db8:1::1", + outerDst: "2001:db8:1::2", + outerProto: 4, + innerLayer: "ipv4", + innerSrc: "192.0.2.1", + innerDst: "198.51.100.2", + expectedSrc: "192.0.2.1", + }, + { + name: "ipv6 in ipv6", + payload: ethernetPayload(0x86dd, ipv6Packet(41, + [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}, + inner6)), + outerLayer: "ipv6", + outerSrc: "2001:db8:1::1", + outerDst: "2001:db8:1::2", + outerProto: 41, + innerLayer: "ipv6", + innerSrc: "2001:db8::1", + innerDst: "2001:db8::2", + expectedSrc: "2001:db8::1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: tt.payload, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + expectPacketLayers(t, events[0], []string{"ethernet", tt.outerLayer, tt.innerLayer, "tcp"}) + expectIPLayer(t, fields, 0, "outer", tt.outerSrc, tt.outerDst, tt.outerProto, 0, 0) + expectIPLayer(t, fields, 1, "inner", tt.innerSrc, tt.innerDst, 6, 12345, 443) + if got := fields["src_addr"]; got != tt.expectedSrc { + t.Fatalf("expected inner src_addr=%s, got %#v", tt.expectedSrc, got) + } + }) + } +} + +func TestBuiltinProcessBytesExtractsIPv6InIPWithRoutingHeader(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + inner := ipv6RoutingPacket(tcpHeader(12345, 443)) + outer := ipv4Packet(41, [4]byte{203, 0, 113, 1}, [4]byte{203, 0, 113, 2}, inner) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: ethernetPayload(0x0800, outer), + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + expectedLayers := []string{"ethernet", "ipv4", "ipv6", "ipv6_routing", "tcp"} + expectPacketLayers(t, events[0], expectedLayers) + expectIPLayer(t, fields, 0, "outer", "203.0.113.1", "203.0.113.2", 41, 0, 0) + expectIPLayer(t, fields, 1, "inner", "2001:db8::1", "2001:db8::2", 6, 12345, 443) +} + +func TestBuiltinProcessBytesExtractsL2TPAndGTPUInnerPackets(t *testing.T) { + tests := []struct { + name string + payload []byte + wantLayer string + wantOuter packetLayerExpectation + wantInner packetLayerExpectation + wantLayers []string + }{ + { + name: "l2tp", + payload: ethernetPayload(0x0800, ipv4Packet(17, [4]byte{203, 0, 113, 1}, [4]byte{203, 0, 113, 2}, udpPacket(49152, 1701, l2tpPayload(ipv4Packet(6, [4]byte{192, 0, 2, 1}, [4]byte{198, 51, 100, 2}, tcpHeader(12345, 443)))))), + wantLayer: "l2tp", + wantOuter: packetLayerExpectation{"outer", "203.0.113.1", "203.0.113.2", 17, 49152, 1701}, + wantInner: packetLayerExpectation{"inner", "192.0.2.1", "198.51.100.2", 6, 12345, 443}, + }, + { + name: "gtpu", + payload: ethernetPayload(0x0800, ipv4Packet(17, [4]byte{203, 0, 113, 1}, [4]byte{203, 0, 113, 2}, udpPacket(49152, 2152, gtpuPayload(ipv4Packet(6, [4]byte{192, 0, 2, 1}, [4]byte{198, 51, 100, 2}, tcpHeader(12345, 443)))))), + wantLayer: "gtpu", + wantOuter: packetLayerExpectation{"outer", "203.0.113.1", "203.0.113.2", 17, 49152, 2152}, + wantInner: packetLayerExpectation{"inner", "192.0.2.1", "198.51.100.2", 6, 12345, 443}, + }, + } + + proc := NewBuiltin(config.ProcessorConfig{}) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: tt.payload, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + if !packetContainsLayer(events[0], tt.wantLayer) { + t.Fatalf("expected packet.layers to contain %q, got %#v", tt.wantLayer, events[0].Packet) + } + expectIPLayer(t, fields, 0, tt.wantOuter.role, tt.wantOuter.srcAddr, tt.wantOuter.dstAddr, tt.wantOuter.proto, tt.wantOuter.srcPort, tt.wantOuter.dstPort) + expectIPLayer(t, fields, 1, tt.wantInner.role, tt.wantInner.srcAddr, tt.wantInner.dstAddr, tt.wantInner.proto, tt.wantInner.srcPort, tt.wantInner.dstPort) + }) + } +} + +func TestBuiltinProcessBytesExtractsStackedEncapsulations(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + inner := ipv4Packet(6, [4]byte{192, 0, 2, 1}, [4]byte{198, 51, 100, 2}, tcpHeader(12345, 443)) + gre := grePayload(0x0800, inner) + outer := ipv4Packet(47, [4]byte{203, 0, 113, 1}, [4]byte{203, 0, 113, 2}, gre) + mpls := mplsPayload(17, true, outer) + pppoe := pppoePayload(0x0281, mpls) + packet := ethernetPayload(0x8100, dot1qPayload(100, 0x8864, pppoe)) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + expectedLayers := []string{"ethernet", "dot1q", "pppoe", "mpls", "ipv4", "gre", "ipv4", "tcp"} + expectPacketLayers(t, events[0], expectedLayers) + if got := fields["vlan_id"]; got != uint32(100) { + t.Fatalf("expected vlan_id=100, got %#v", got) + } + expectIPLayer(t, fields, 0, "outer", "203.0.113.1", "203.0.113.2", 47, 0, 0) + expectIPLayer(t, fields, 1, "inner", "192.0.2.1", "198.51.100.2", 6, 12345, 443) +} + +func TestBuiltinProcessBytesExtractsMPLSInnerPacket(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x88, 0x47, + 0x00, 0x01, 0x11, 0x40, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x02, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x02, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + + expectedLayers := []string{"ethernet", "mpls", "ipv4", "tcp"} + expectPacketLayers(t, events[0], expectedLayers) + expectIPLayer(t, fields, 0, "single", "192.0.2.1", "198.51.100.2", 6, 12345, 443) + if got := fields["src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected src_addr=192.0.2.1, got %#v", got) + } + if got := fields["dst_addr"]; got != "198.51.100.2" { + t.Fatalf("expected dst_addr=198.51.100.2, got %#v", got) + } +} + +func TestBuiltinProcessBytesAddsAggregationHelperFields(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + AggregationHelpers: config.AggregationHelperConfig{ + MPLSLabels: 3, + IPLayers: 2, + }, + }, + }) + inner := ipv4Packet(6, [4]byte{192, 0, 2, 1}, [4]byte{198, 51, 100, 2}, tcpHeader(12345, 443)) + outer := ipv4Packet(47, [4]byte{203, 0, 113, 1}, [4]byte{203, 0, 113, 2}, grePayload(0x0800, inner)) + packet := ethernetPayload(0x8847, mplsPayload(17, false, mplsPayload(18, false, mplsPayload(19, true, outer)))) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "bytes"}, + Payload: packet, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + if got := fields["mpls_label_1"]; got != uint32(17) { + t.Fatalf("expected mpls_label_1=17, got %#v", got) + } + if got := fields["mpls_label_2"]; got != uint32(18) { + t.Fatalf("expected mpls_label_2=18, got %#v", got) + } + if got := fields["mpls_label_3"]; got != uint32(19) { + t.Fatalf("expected mpls_label_3=19, got %#v", got) + } + if got := fields["mpls_label_stack_section_3"]; !bytes.Equal(got.([]byte), []byte{0x00, 0x01, 0x31}) { + t.Fatalf("expected MPLS label stack section bytes for label 3, got %#v", got) + } + if got := fields["ip_1_src_addr"]; got != "203.0.113.1" { + t.Fatalf("expected ip_1_src_addr outer address, got %#v", got) + } + if got := fields["ip_2_src_addr"]; got != "192.0.2.1" { + t.Fatalf("expected ip_2_src_addr inner address, got %#v", got) + } + if got := fields["ip_2_dst_port"]; got != uint32(443) { + t.Fatalf("expected ip_2_dst_port=443, got %#v", got) + } + if _, ok := fields["ip_3_src_addr"]; ok { + t.Fatalf("did not expect ip_3_src_addr beyond configured helper depth") + } +} + +func TestBuiltinProcessReFlowJSONAddsAggregationHelperFieldsFromPacketModel(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{ + Builtin: config.BuiltinProcessorConfig{ + AggregationHelpers: config.AggregationHelperConfig{ + MPLSLabels: 1, + IPLayers: 2, + }, + }, + }) + msg := []byte(`{ + "packet": { + "layers": [ + {"kind": "mpls", "mpls": {"label": {"label": 17, "bos": true}}}, + {"kind": "ipv4", "ipv4": {"src_addr": "203.0.113.1", "dst_addr": "203.0.113.2", "protocol": 47}}, + {"kind": "gre", "gre": {"protocol": 34525}}, + {"kind": "ipv6", "ipv6": {"src_addr": "2001:db8::1", "dst_addr": "2001:db8::2", "next_header": 6}}, + {"kind": "tcp", "tcp": {"src_port": 12345, "dst_port": 443}} + ] + } + }`) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "reflow"}, + }, + Message: msg, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + fields := events[0].Fields + if got := fields["mpls_label_1"]; got != uint32(17) { + t.Fatalf("expected mpls_label_1=17, got %#v", got) + } + if got := fields["ip_1_src_addr"]; got != "203.0.113.1" { + t.Fatalf("expected ip_1_src_addr outer address, got %#v", got) + } + if got := fields["ip_2_src_addr"]; got != "2001:db8::1" { + t.Fatalf("expected ip_2_src_addr inner address, got %#v", got) + } + if got := fields["ip_2_dst_port"]; got != uint32(443) { + t.Fatalf("expected ip_2_dst_port=443, got %#v", got) + } +} + +func TestBuiltinProcessReFlowJSONPreservesCounterFieldsWithoutSFlowMetadata(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + msg, err := json.Marshal(map[string]any{ + "record_kind": "interface_counter", + "agent_ip": "192.0.2.10", + "sub_agent_id": 3, + "source_id": 4, + "if_index": 5, + "if_in_octets": 1234, + "if_out_octets": 5678, + "if_out_errors": 9, + "if_status": 3, + "if_direction": 1, + "if_promiscuous_mode": 0, + }) + if err != nil { + t.Fatalf("Marshal returned error: %v", err) + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "reflow"}, + }, + Message: msg, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + fields := events[0].Fields + if got := fields["record_kind"]; got != "interface_counter" { + t.Fatalf("expected record_kind=interface_counter, got %#v", got) + } + if got := fields["if_index"]; got != int64(5) { + t.Fatalf("expected if_index=5, got %#v", got) + } + if _, ok := fields["input_if"]; ok { + t.Fatalf("did not expect interface_counter if_index to alias input_if, got %#v", fields["input_if"]) + } + if events[0].SFlow != nil { + t.Fatalf("expected sflow metadata to remain unset, got %#v", events[0].SFlow) + } +} + +func TestBuiltinProcessReFlowJSONParsesSFlowSampleEnvelope(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + msg := []byte(`{ + "fields": { + "agent_ip": "192.0.2.10", + "record_kind": "packet", + "protocol": 1, + "frame_length": 4, + "original_length": 4, + "header_data": "AQIDBA==" + }, + "sflow": { + "agent_ip": "198.51.100.1", + "source_id": 7, + "samples": [ + { + "raw_flow_records": [ + {"enterprise": 64512, "format": 7, "data": "3q2+7w=="} + ], + "raw_counter_records": [ + {"data_format": 264241161, "data": "yv66vg=="} + ] + } + ] + } + }`) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "reflow"}, + }, + Message: msg, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + sf := events[0].SFlow + if sf == nil || sf.AgentIP != "198.51.100.1" || sf.SourceID != 7 { + t.Fatalf("expected parsed sflow metadata, got %#v", sf) + } + if len(sf.Samples) != 1 || len(sf.Samples[0].RawFlowRecords) != 1 || len(sf.Samples[0].RawCounterRecords) != 1 { + t.Fatalf("expected parsed raw records, got %#v", sf.Samples) + } + if got := sf.Samples[0].RawFlowRecords[0].Data; !bytes.Equal(got, []byte{0xde, 0xad, 0xbe, 0xef}) { + t.Fatalf("expected decoded raw flow record bytes, got %#v", got) + } + if got := sf.Samples[0].RawCounterRecords[0].Data; !bytes.Equal(got, []byte{0xca, 0xfe, 0xba, 0xbe}) { + t.Fatalf("expected decoded raw counter record bytes, got %#v", got) + } +} + +func TestBuiltinProcessReFlowJSONUsesCanonicalEventFieldsWithoutEnvelopeMetadata(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + msg, err := json.Marshal(map[string]any{ + "received_at": "2026-04-28T00:00:00Z", + "source": map[string]any{ + "network": "udp", + "address": ":18081", + "type": "json", + }, + "fields": map[string]any{ + "agent_ip": "192.0.2.10", + "bytes": 1234, + }, + "packet": map[string]any{ + "layers": []string{"ipv4"}, + }, + "sflow": map[string]any{ + "agent_ip": "192.0.2.10", + }, + }) + if err != nil { + t.Fatalf("Marshal returned error: %v", err) + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "reflow"}, + }, + Message: msg, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + processed := events[0] + if processed.SFlow != nil { + t.Fatalf("expected sflow metadata to remain unset, got %#v", processed.SFlow) + } + if processed.Packet != nil { + t.Fatalf("expected packet metadata to remain unset, got %#v", processed.Packet) + } + if _, ok := processed.Fields["sflow"]; ok { + t.Fatalf("did not expect sflow envelope to be copied into fields") + } + if _, ok := processed.Fields["packet"]; ok { + t.Fatalf("did not expect packet envelope to be copied into fields") + } + if _, ok := processed.Fields["source"]; ok { + t.Fatalf("did not expect source envelope to be copied into fields") + } + if got := processed.Fields["agent_ip"]; got != "192.0.2.10" { + t.Fatalf("expected agent_ip field to be preserved, got %#v", got) + } + if got := processed.Fields["bytes"]; got != int64(1234) { + t.Fatalf("expected bytes field to be preserved, got %#v", got) + } +} + +func TestBuiltinProcessReFlowJSONPreservesLargeIntegerCounters(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + const largeCounter int64 = 9_007_199_254_740_993 + msg := []byte(`{ + "bytes": 9007199254740993, + "packets": 9007199254740993, + "sample_pool": 9007199254740993, + "if_in_octets": 9007199254740993, + "end_time_unix": 9007199254740993 + }`) + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "reflow"}, + }, + Message: msg, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + + for _, key := range []string{"bytes", "packets", "sample_pool", "if_in_octets", "end_time_unix"} { + if got := events[0].Fields[key]; got != largeCounter { + t.Fatalf("expected %s=%d, got %#v", key, largeCounter, got) + } + } +} + +func TestBuiltinProcessRawPacketHeaderRequiresExplicitFlavor(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + + msg, err := json.Marshal(map[string]any{ + "agent_ip": "127.0.0.1", + "sub_agent_id": 1, + "source_id": 1, + "sampling_rate": 100, + "sample_pool": 1000, + "drops": 0, + "input_if": 10, + "output_if": 20, + "protocol": 1, + "frame_length": 74, + "stripped": 0, + "original_length": 54, + "header_hex": "00112233445566778899aabb0800450000281234400040060000c0000201c6336401303901bb00000001000000005002200000000000", + }) + if err != nil { + t.Fatalf("Marshal returned error: %v", err) + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "reflow"}, + }, + Message: msg, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + + fields := events[0].Fields + if got := fields["header_hex"]; got == nil { + t.Fatalf("expected header_hex to remain a canonical field when flavor=reflow") + } + if _, ok := fields["header_data"]; ok { + t.Fatalf("did not expect raw packet header decoding for flavor=reflow") + } + + events, err = proc.Process(&event.Event{ + Source: event.SourceMetadata{ + Type: "json", + JSON: event.JSONMetadata{Flavor: "raw_packet_header"}, + }, + Message: msg, + }) + if err != nil { + t.Fatalf("Process returned error for raw_packet_header flavor: %v", err) + } + if _, ok := events[0].Fields["header_data"]; !ok { + t.Fatalf("expected raw_packet_header flavor to decode header_data") + } + if got := events[0].Fields["bytes"]; got != int64(74) { + t.Fatalf("expected bytes to use frame_length=74, got %#v", got) + } +} + +func TestBuiltinProcessFlowDoesNotTreatGenericProtocolAsSFlowHeaderProtocol(t *testing.T) { + proc := NewBuiltin(config.ProcessorConfig{}) + header := []byte{ + 0x45, 0x00, 0x00, 0x28, 0x12, 0x34, 0x40, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x01, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, + 0x50, 0x02, 0x20, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + + events, err := proc.Process(&event.Event{ + Source: event.SourceMetadata{Type: "flow"}, + Fields: map[string]any{ + "flow_type": "ipfix", + "record_kind": "packet", + "protocol": uint32(11), + "frame_length": uint32(74), + "original_length": uint32(len(header)), + "header_data": header, + }, + }) + if err != nil { + t.Fatalf("Process returned error: %v", err) + } + if got := events[0].Fields["src_addr"]; got != nil { + t.Fatalf("did not expect generic protocol field to drive packet parsing, got src_addr=%#v", got) + } +} + +func expectIPLayer(t *testing.T, fields map[string]any, index int, role, srcAddr, dstAddr string, proto, srcPort, dstPort uint32) { + t.Helper() + prefix := "" + if role == "outer" && index == 0 { + prefix = "outer_" + } + if got := fields[prefix+"src_addr"]; got != srcAddr { + t.Fatalf("expected %ssrc_addr=%q, got %#v", prefix, srcAddr, got) + } + if got := fields[prefix+"dst_addr"]; got != dstAddr { + t.Fatalf("expected %sdst_addr=%q, got %#v", prefix, dstAddr, got) + } + if got := fields[prefix+"proto"]; got != proto { + t.Fatalf("expected %sproto=%d, got %#v", prefix, proto, got) + } + if got := fields[prefix+"src_port"]; got != srcPort { + t.Fatalf("expected %ssrc_port=%d, got %#v", prefix, srcPort, got) + } + if got := fields[prefix+"dst_port"]; got != dstPort { + t.Fatalf("expected %sdst_port=%d, got %#v", prefix, dstPort, got) + } +} + +func expectPacketLayers(t *testing.T, evt *event.Event, expected []string) { + t.Helper() + if evt.Packet == nil { + t.Fatalf("expected packet model") + } + if len(evt.Packet.Layers) != len(expected) { + t.Fatalf("expected %d packet layers, got %#v", len(expected), evt.Packet.Layers) + } + for i, want := range expected { + if evt.Packet.Layers[i].Kind != want { + t.Fatalf("expected packet.layers[%d].kind=%q, got %#v", i, want, evt.Packet.Layers[i].Kind) + } + } +} + +func packetContainsLayer(evt *event.Event, want string) bool { + if evt.Packet == nil { + return false + } + for _, layer := range evt.Packet.Layers { + if layer.Kind == want { + return true + } + } + return false +} + +type packetLayerExpectation struct { + role string + srcAddr string + dstAddr string + proto uint32 + srcPort uint32 + dstPort uint32 +} + +func vxlanTestPacket(dstPort uint16) []byte { + packet := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, + 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x5c, 0x00, 0x01, 0x00, 0x00, 0x40, 0x11, 0x00, 0x00, + 0xcb, 0x00, 0x71, 0x01, + 0xcb, 0x00, 0x71, 0x02, + 0x12, 0x34, 0x12, 0xb5, 0x00, 0x48, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x01, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x02, + 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x02, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, + 0xc0, 0x00, 0x02, 0x01, + 0xc6, 0x33, 0x64, 0x02, + 0x30, 0x39, 0x01, 0xbb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + } + packet[36] = byte(dstPort >> 8) + packet[37] = byte(dstPort) + return packet +} + +func ethernetPayload(etherType uint16, payload []byte) []byte { + out := make([]byte, 14+len(payload)) + copy(out[0:6], []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}) + copy(out[6:12], []byte{0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb}) + binary.BigEndian.PutUint16(out[12:14], etherType) + copy(out[14:], payload) + return out +} + +func dot1qPayload(vlanID uint16, etherType uint16, payload []byte) []byte { + return dot1qTCIPayload(vlanID&0x0fff, etherType, payload) +} + +func dot1qTCIPayload(tci uint16, etherType uint16, payload []byte) []byte { + out := make([]byte, 4+len(payload)) + binary.BigEndian.PutUint16(out[0:2], tci) + binary.BigEndian.PutUint16(out[2:4], etherType) + copy(out[4:], payload) + return out +} + +func ipv4Packet(proto byte, src, dst [4]byte, payload []byte) []byte { + out := make([]byte, 20+len(payload)) + out[0] = 0x45 + binary.BigEndian.PutUint16(out[2:4], uint16(len(out))) + out[8] = 64 + out[9] = proto + copy(out[12:16], src[:]) + copy(out[16:20], dst[:]) + copy(out[20:], payload) + return out +} + +func ipv6Packet(nextHeader byte, src, dst [16]byte, payload []byte) []byte { + out := make([]byte, 40+len(payload)) + out[0] = 0x60 + binary.BigEndian.PutUint16(out[4:6], uint16(len(payload))) + out[6] = nextHeader + out[7] = 64 + copy(out[8:24], src[:]) + copy(out[24:40], dst[:]) + copy(out[40:], payload) + return out +} + +func ipv6RoutingPacket(payload []byte) []byte { + out := make([]byte, 48+len(payload)) + out[0] = 0x60 + binary.BigEndian.PutUint16(out[4:6], uint16(8+len(payload))) + out[6] = 43 + out[7] = 64 + copy(out[8:24], []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + copy(out[24:40], []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + out[40] = 6 + out[41] = 0 + copy(out[48:], payload) + return out +} + +func tcpHeader(srcPort, dstPort uint16) []byte { + out := make([]byte, 20) + binary.BigEndian.PutUint16(out[0:2], srcPort) + binary.BigEndian.PutUint16(out[2:4], dstPort) + out[12] = 0x50 + out[13] = 0x02 + return out +} + +func udpPacket(srcPort, dstPort uint16, payload []byte) []byte { + out := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint16(out[0:2], srcPort) + binary.BigEndian.PutUint16(out[2:4], dstPort) + binary.BigEndian.PutUint16(out[4:6], uint16(len(out))) + copy(out[8:], payload) + return out +} + +func l2tpPayload(inner []byte) []byte { + out := make([]byte, 10+len(inner)) + binary.BigEndian.PutUint16(out[0:2], 0x0002) + binary.BigEndian.PutUint16(out[2:4], 1) + binary.BigEndian.PutUint16(out[4:6], 2) + out[6] = 0xff + out[7] = 0x03 + binary.BigEndian.PutUint16(out[8:10], 0x0021) + copy(out[10:], inner) + return out +} + +func gtpuPayload(inner []byte) []byte { + out := make([]byte, 8+len(inner)) + out[0] = 0x30 + out[1] = 0xff + binary.BigEndian.PutUint16(out[2:4], uint16(len(inner))) + binary.BigEndian.PutUint32(out[4:8], 0x01020304) + copy(out[8:], inner) + return out +} + +func grePayload(proto uint16, inner []byte) []byte { + out := make([]byte, 4+len(inner)) + binary.BigEndian.PutUint16(out[2:4], proto) + copy(out[4:], inner) + return out +} + +func mplsPayload(label uint32, bos bool, inner []byte) []byte { + out := make([]byte, 4+len(inner)) + raw := (label & 0xfffff) << 12 + if bos { + raw |= 1 << 8 + } + raw |= 64 + binary.BigEndian.PutUint32(out[0:4], raw) + copy(out[4:], inner) + return out +} + +func pppoePayload(proto uint16, inner []byte) []byte { + out := make([]byte, 8+len(inner)) + out[0] = 0x11 + binary.BigEndian.PutUint16(out[2:4], 1) + binary.BigEndian.PutUint16(out[4:6], uint16(len(inner)+2)) + binary.BigEndian.PutUint16(out[6:8], proto) + copy(out[8:], inner) + return out +} diff --git a/pkg/reflow/sink/sink.go b/pkg/reflow/sink/sink.go new file mode 100644 index 00000000..9165b900 --- /dev/null +++ b/pkg/reflow/sink/sink.go @@ -0,0 +1,167 @@ +package sink + +import ( + "fmt" + "io" + "net" + "os" + "sync" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" +) + +type Sink interface { + Send(payload []byte) error + Close() error +} + +// New selects the concrete output sink from the normalized sink config. +func New(cfg config.SinkConfig) (Sink, error) { + sep := []byte("\n") + if cfg.Framing == "none" { + sep = nil + } + switch cfg.Type { + case "", "stdout": + return &writerSink{w: os.Stdout, sep: sep}, nil + case "file": + flags := os.O_APPEND | os.O_CREATE | os.O_WRONLY + if cfg.Mode == "truncate" { + flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY + } + f, err := os.OpenFile(cfg.Path, flags, 0644) + if err != nil { + return nil, fmt.Errorf("open sink file %s: %w", cfg.Path, err) + } + return &writerSink{w: f, f: f, sep: sep}, nil + case "udp", "unixgram": + resolveInterval := time.Duration(0) + if cfg.ResolveIntervalMS != nil { + resolveInterval = time.Duration(*cfg.ResolveIntervalMS) * time.Millisecond + } + sink, err := newPacketSink(cfg.Type, cfg.Address, resolveInterval, net.Dial, time.Now) + if err != nil { + return nil, fmt.Errorf("dial sink %s %s: %w", cfg.Type, cfg.Address, err) + } + return sink, nil + default: + return nil, fmt.Errorf("unsupported sink.type %q", cfg.Type) + } +} + +type writerSink struct { + mu sync.Mutex + w io.Writer + f *os.File + sep []byte +} + +// Send writes one payload and appends the configured record separator. +func (s *writerSink) Send(payload []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, err := s.w.Write(payload); err != nil { + return fmt.Errorf("write payload: %w", err) + } + if len(s.sep) > 0 { + if _, err := s.w.Write(s.sep); err != nil { + return fmt.Errorf("write separator: %w", err) + } + } + return nil +} + +// Close only needs to close file-backed sinks; stdout remains owned by the process. +func (s *writerSink) Close() error { + if s.f != nil { + return s.f.Close() + } + return nil +} + +type packetSink struct { + mu sync.Mutex + network string + address string + resolveInterval time.Duration + lastResolve time.Time + conn net.Conn + dial func(network, address string) (net.Conn, error) + now func() time.Time +} + +func newPacketSink(network, address string, resolveInterval time.Duration, dial func(string, string) (net.Conn, error), now func() time.Time) (*packetSink, error) { + if dial == nil { + dial = net.Dial + } + if now == nil { + now = time.Now + } + conn, err := dial(network, address) + if err != nil { + return nil, err + } + return &packetSink{ + network: network, + address: address, + resolveInterval: resolveInterval, + lastResolve: now(), + conn: conn, + dial: dial, + now: now, + }, nil +} + +// Send writes one already-encoded packet to the connected datagram sink. +func (s *packetSink) Send(payload []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.shouldRefreshLocked() { + if err := s.refreshLocked(); err != nil { + // Keep exporting through the previous socket if DNS is temporarily unavailable. + s.lastResolve = s.now() + } + } + if _, err := s.conn.Write(payload); err != nil { + if s.resolveInterval > 0 { + if refreshErr := s.refreshLocked(); refreshErr != nil { + return fmt.Errorf("write packet: %w; refresh packet sink: %v", err, refreshErr) + } + if _, retryErr := s.conn.Write(payload); retryErr != nil { + return fmt.Errorf("write packet after refresh: %w", retryErr) + } + return nil + } + return fmt.Errorf("write packet: %w", err) + } + return nil +} + +func (s *packetSink) shouldRefreshLocked() bool { + if s.resolveInterval <= 0 { + return false + } + return !s.now().Before(s.lastResolve.Add(s.resolveInterval)) +} + +func (s *packetSink) refreshLocked() error { + conn, err := s.dial(s.network, s.address) + if err != nil { + return err + } + old := s.conn + s.conn = conn + s.lastResolve = s.now() + if old != nil { + _ = old.Close() + } + return nil +} + +// Close releases the connected socket. +func (s *packetSink) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.conn.Close() +} diff --git a/pkg/reflow/sink/sink_test.go b/pkg/reflow/sink/sink_test.go new file mode 100644 index 00000000..2ea14cb7 --- /dev/null +++ b/pkg/reflow/sink/sink_test.go @@ -0,0 +1,233 @@ +package sink + +import ( + "errors" + "io" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" +) + +func TestFileSinkFramingNoneAndTruncate(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "out.bin") + if err := os.WriteFile(path, []byte("old data"), 0o644); err != nil { + t.Fatalf("seed file: %v", err) + } + + s, err := New(config.SinkConfig{ + Type: "file", + Path: path, + Framing: "none", + Mode: "truncate", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + if err := s.Send([]byte{0, 1, 2}); err != nil { + t.Fatalf("Send returned error: %v", err) + } + if err := s.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read output: %v", err) + } + if string(got) != string([]byte{0, 1, 2}) { + t.Fatalf("expected exact binary payload, got %v", got) + } +} + +func TestPacketSinkRefreshesConnectionAfterInterval(t *testing.T) { + now := time.Unix(1000, 0) + var conns []*packetTestConn + s, err := newPacketSink("udp", "collector.example.com:4739", time.Second, func(network, address string) (net.Conn, error) { + if network != "udp" || address != "collector.example.com:4739" { + t.Fatalf("unexpected dial target %s %s", network, address) + } + conn := &packetTestConn{} + conns = append(conns, conn) + return conn, nil + }, func() time.Time { + return now + }) + if err != nil { + t.Fatalf("newPacketSink returned error: %v", err) + } + + if err := s.Send([]byte("first")); err != nil { + t.Fatalf("first Send returned error: %v", err) + } + now = now.Add(time.Second) + if err := s.Send([]byte("second")); err != nil { + t.Fatalf("second Send returned error: %v", err) + } + + if len(conns) != 2 { + t.Fatalf("expected two dials, got %d", len(conns)) + } + if !conns[0].closed { + t.Fatalf("expected old connection to be closed after refresh") + } + if string(conns[0].writes[0]) != "first" || string(conns[1].writes[0]) != "second" { + t.Fatalf("unexpected writes: first=%q second=%q", conns[0].writes, conns[1].writes) + } +} + +func TestPacketSinkResolveIntervalZeroDisablesRefresh(t *testing.T) { + now := time.Unix(1000, 0) + var conns []*packetTestConn + s, err := newPacketSink("udp", "collector.example.com:4739", 0, func(string, string) (net.Conn, error) { + conn := &packetTestConn{} + conns = append(conns, conn) + return conn, nil + }, func() time.Time { + return now + }) + if err != nil { + t.Fatalf("newPacketSink returned error: %v", err) + } + + if err := s.Send([]byte("first")); err != nil { + t.Fatalf("first Send returned error: %v", err) + } + now = now.Add(24 * time.Hour) + if err := s.Send([]byte("second")); err != nil { + t.Fatalf("second Send returned error: %v", err) + } + + if len(conns) != 1 { + t.Fatalf("expected one dial when refresh is disabled, got %d", len(conns)) + } + if len(conns[0].writes) != 2 { + t.Fatalf("expected both writes on initial connection, got %d", len(conns[0].writes)) + } +} + +func TestPacketSinkUsesExistingConnectionWhenScheduledRefreshFails(t *testing.T) { + resolveErr := errors.New("resolve failed") + now := time.Unix(1000, 0) + var dialAttempts int + var conns []*packetTestConn + s, err := newPacketSink("udp", "collector.example.com:4739", time.Second, func(string, string) (net.Conn, error) { + dialAttempts++ + if dialAttempts > 1 { + return nil, resolveErr + } + conn := &packetTestConn{} + conns = append(conns, conn) + return conn, nil + }, func() time.Time { + return now + }) + if err != nil { + t.Fatalf("newPacketSink returned error: %v", err) + } + + if err := s.Send([]byte("first")); err != nil { + t.Fatalf("first Send returned error: %v", err) + } + now = now.Add(time.Second) + if err := s.Send([]byte("second")); err != nil { + t.Fatalf("second Send returned error: %v", err) + } + + if dialAttempts != 2 { + t.Fatalf("expected scheduled refresh attempt, got %d dials", dialAttempts) + } + if len(conns) != 1 || conns[0].closed { + t.Fatalf("expected original connection to remain open, conns=%d closed=%v", len(conns), conns[0].closed) + } + if len(conns[0].writes) != 2 || string(conns[0].writes[1]) != "second" { + t.Fatalf("expected payload to use original connection after refresh failure, got %#v", conns[0].writes) + } +} + +func TestPacketSinkRefreshesAndRetriesAfterWriteError(t *testing.T) { + writeErr := errors.New("write failed") + var conns []*packetTestConn + s, err := newPacketSink("udp", "collector.example.com:4739", time.Minute, func(string, string) (net.Conn, error) { + conn := &packetTestConn{} + if len(conns) == 0 { + conn.writeErr = writeErr + } + conns = append(conns, conn) + return conn, nil + }, time.Now) + if err != nil { + t.Fatalf("newPacketSink returned error: %v", err) + } + + if err := s.Send([]byte("retry")); err != nil { + t.Fatalf("Send returned error: %v", err) + } + if len(conns) != 2 { + t.Fatalf("expected write error to trigger a refresh, got %d dials", len(conns)) + } + if !conns[0].closed { + t.Fatalf("expected failed connection to be closed") + } + if len(conns[1].writes) != 1 || string(conns[1].writes[0]) != "retry" { + t.Fatalf("expected retry payload on refreshed connection, got %#v", conns[1].writes) + } +} + +type packetTestConn struct { + writes [][]byte + writeErr error + closed bool +} + +func (c *packetTestConn) Read([]byte) (int, error) { + return 0, io.ErrClosedPipe +} + +func (c *packetTestConn) Write(payload []byte) (int, error) { + if c.writeErr != nil { + return 0, c.writeErr + } + copied := append([]byte(nil), payload...) + c.writes = append(c.writes, copied) + return len(payload), nil +} + +func (c *packetTestConn) Close() error { + c.closed = true + return nil +} + +func (c *packetTestConn) LocalAddr() net.Addr { + return packetTestAddr("local") +} + +func (c *packetTestConn) RemoteAddr() net.Addr { + return packetTestAddr("remote") +} + +func (c *packetTestConn) SetDeadline(time.Time) error { + return nil +} + +func (c *packetTestConn) SetReadDeadline(time.Time) error { + return nil +} + +func (c *packetTestConn) SetWriteDeadline(time.Time) error { + return nil +} + +type packetTestAddr string + +func (a packetTestAddr) Network() string { + return string(a) +} + +func (a packetTestAddr) String() string { + return string(a) +} diff --git a/pkg/reflow/source/ebpf/conntrack_linux.go b/pkg/reflow/source/ebpf/conntrack_linux.go new file mode 100644 index 00000000..52a3eec5 --- /dev/null +++ b/pkg/reflow/source/ebpf/conntrack_linux.go @@ -0,0 +1,403 @@ +//go:build linux && !reflow_noebpf + +package ebpf + +import ( + "bufio" + "encoding/binary" + "net/netip" + "os" + "strconv" + "strings" + "time" +) + +const conntrackRefreshInterval = 2 * time.Second + +var defaultConntrackPaths = []string{ + "/proc/net/nf_conntrack", + "/proc/net/ip_conntrack", +} + +type conntrackTracker struct { + path string + lastLoad time.Time + entries map[conntrackTuple]conntrackMetadata +} + +type conntrackTuple struct { + family string + proto string + src netip.Addr + dst netip.Addr + srcPort uint16 + dstPort uint16 +} + +type conntrackMetadata struct { + direction string + family string + proto string + state string + status string + original conntrackTuple + reply conntrackTuple + hasSNAT bool + natSrc netip.Addr + natSPort uint16 + hasDNAT bool + natDst netip.Addr + natDPort uint16 +} + +func newConntrackTracker(path string) *conntrackTracker { + return &conntrackTracker{ + path: path, + entries: make(map[conntrackTuple]conntrackMetadata), + } +} + +func (t *conntrackTracker) Lookup(frame []byte) (conntrackMetadata, bool) { + if t == nil { + return conntrackMetadata{}, false + } + tuple, ok := packetConntrackTuple(frame) + if !ok { + return conntrackMetadata{}, false + } + if time.Since(t.lastLoad) >= conntrackRefreshInterval { + t.refresh() + } + meta, ok := t.entries[tuple] + return meta, ok +} + +func (t *conntrackTracker) refresh() { + t.lastLoad = time.Now() + entries, ok := loadConntrackEntries(t.path) + if ok { + t.entries = entries + } +} + +func loadConntrackEntries(path string) (map[conntrackTuple]conntrackMetadata, bool) { + paths := []string{path} + if path == "" { + paths = defaultConntrackPaths + } + for _, candidate := range paths { + if candidate == "" { + continue + } + entries, err := loadConntrackPath(candidate) + if err == nil { + return entries, true + } + } + return nil, false +} + +func loadConntrackPath(path string) (map[conntrackTuple]conntrackMetadata, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + entries := make(map[conntrackTuple]conntrackMetadata) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + registerConntrackLine(entries, scanner.Text()) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return entries, nil +} + +func registerConntrackLine(entries map[conntrackTuple]conntrackMetadata, line string) { + meta, ok := parseConntrackLine(line) + if !ok { + return + } + orig := meta + orig.direction = "original" + reply := meta + reply.direction = "reply" + entries[meta.original] = orig + entries[meta.reply] = reply +} + +func parseConntrackLine(line string) (conntrackMetadata, bool) { + tokens := strings.Fields(line) + if len(tokens) < 4 { + return conntrackMetadata{}, false + } + family := tokens[0] + if family != "ipv4" && family != "ipv6" { + return conntrackMetadata{}, false + } + protoIndex := -1 + proto := "" + for i, token := range tokens { + switch token { + case "tcp", "udp": + protoIndex = i + proto = token + } + if protoIndex >= 0 { + break + } + } + if protoIndex < 0 { + return conntrackMetadata{}, false + } + + tuples := parseConntrackTuples(tokens[protoIndex+1:], family, proto) + if len(tuples) < 2 { + return conntrackMetadata{}, false + } + meta := conntrackMetadata{ + family: family, + proto: proto, + state: parseConntrackState(tokens, protoIndex), + status: parseConntrackStatus(tokens), + original: tuples[0], + reply: tuples[1], + } + if tuples[0].src != tuples[1].dst || tuples[0].srcPort != tuples[1].dstPort { + meta.hasSNAT = true + meta.natSrc = tuples[1].dst + meta.natSPort = tuples[1].dstPort + } + if tuples[0].dst != tuples[1].src || tuples[0].dstPort != tuples[1].srcPort { + meta.hasDNAT = true + meta.natDst = tuples[1].src + meta.natDPort = tuples[1].srcPort + } + return meta, true +} + +func parseConntrackTuples(tokens []string, family, proto string) []conntrackTuple { + var tuples []conntrackTuple + var cur conntrackTuple + cur.family = family + cur.proto = proto + haveSrc, haveDst, haveSPort, haveDPort := false, false, false, false + reset := func() { + cur = conntrackTuple{family: family, proto: proto} + haveSrc, haveDst, haveSPort, haveDPort = false, false, false, false + } + for _, token := range tokens { + key, val, ok := strings.Cut(token, "=") + if !ok { + continue + } + switch key { + case "src": + addr, err := netip.ParseAddr(val) + if err == nil { + cur.src = addr + haveSrc = true + } + case "dst": + addr, err := netip.ParseAddr(val) + if err == nil { + cur.dst = addr + haveDst = true + } + case "sport": + if port, ok := parseUint16(val); ok { + cur.srcPort = port + haveSPort = true + } + case "dport": + if port, ok := parseUint16(val); ok { + cur.dstPort = port + haveDPort = true + } + } + if haveSrc && haveDst && haveSPort && haveDPort { + tuples = append(tuples, cur) + reset() + } + } + return tuples +} + +func parseConntrackState(tokens []string, protoIndex int) string { + for i := protoIndex + 1; i < len(tokens); i++ { + token := tokens[i] + if strings.Contains(token, "=") || strings.HasPrefix(token, "[") { + continue + } + if _, err := strconv.Atoi(token); err == nil { + continue + } + if strings.ToUpper(token) == token { + return strings.ToLower(token) + } + } + return "" +} + +func parseConntrackStatus(tokens []string) string { + var status []string + for _, token := range tokens { + if strings.HasPrefix(token, "[") && strings.HasSuffix(token, "]") { + status = append(status, strings.ToLower(strings.Trim(token, "[]"))) + } + } + return strings.Join(status, ",") +} + +func parseUint16(raw string) (uint16, bool) { + val, err := strconv.ParseUint(raw, 10, 16) + if err != nil { + return 0, false + } + return uint16(val), true +} + +func packetConntrackTuple(frame []byte) (conntrackTuple, bool) { + if len(frame) < 14 { + return conntrackTuple{}, false + } + offset := 14 + etherType := binary.BigEndian.Uint16(frame[12:14]) + for etherType == 0x8100 || etherType == 0x88a8 || etherType == 0x9100 { + if len(frame) < offset+4 { + return conntrackTuple{}, false + } + etherType = binary.BigEndian.Uint16(frame[offset+2 : offset+4]) + offset += 4 + } + switch etherType { + case 0x0800: + return ipv4ConntrackTuple(frame[offset:]) + case 0x86dd: + return ipv6ConntrackTuple(frame[offset:]) + default: + return conntrackTuple{}, false + } +} + +func ipv4ConntrackTuple(packet []byte) (conntrackTuple, bool) { + if len(packet) < 20 || packet[0]>>4 != 4 { + return conntrackTuple{}, false + } + ihl := int(packet[0]&0x0f) * 4 + if ihl < 20 || len(packet) < ihl+4 { + return conntrackTuple{}, false + } + proto := packet[9] + src := netip.AddrFrom4([4]byte{packet[12], packet[13], packet[14], packet[15]}) + dst := netip.AddrFrom4([4]byte{packet[16], packet[17], packet[18], packet[19]}) + srcPort, dstPort, protoName, ok := transportTuple(packet[ihl:], proto) + if !ok { + return conntrackTuple{}, false + } + return conntrackTuple{ + family: "ipv4", + proto: protoName, + src: src, + dst: dst, + srcPort: srcPort, + dstPort: dstPort, + }, true +} + +func ipv6ConntrackTuple(packet []byte) (conntrackTuple, bool) { + if len(packet) < 40 || packet[0]>>4 != 6 { + return conntrackTuple{}, false + } + nextHeader := packet[6] + offset := 40 + for { + switch nextHeader { + case 0, 43, 60: + if len(packet) < offset+2 { + return conntrackTuple{}, false + } + nextHeader = packet[offset] + offset += (int(packet[offset+1]) + 1) * 8 + case 44: + if len(packet) < offset+8 { + return conntrackTuple{}, false + } + fragmentOffset := binary.BigEndian.Uint16(packet[offset+2:offset+4]) & 0xfff8 + if fragmentOffset != 0 { + return conntrackTuple{}, false + } + nextHeader = packet[offset] + offset += 8 + default: + goto transport + } + if len(packet) < offset+4 { + return conntrackTuple{}, false + } + } + +transport: + if len(packet) < offset+4 { + return conntrackTuple{}, false + } + src, ok := netip.AddrFromSlice(packet[8:24]) + if !ok { + return conntrackTuple{}, false + } + dst, ok := netip.AddrFromSlice(packet[24:40]) + if !ok { + return conntrackTuple{}, false + } + srcPort, dstPort, protoName, ok := transportTuple(packet[offset:], nextHeader) + if !ok { + return conntrackTuple{}, false + } + return conntrackTuple{ + family: "ipv6", + proto: protoName, + src: src, + dst: dst, + srcPort: srcPort, + dstPort: dstPort, + }, true +} + +func transportTuple(packet []byte, proto uint8) (uint16, uint16, string, bool) { + if len(packet) < 4 { + return 0, 0, "", false + } + switch proto { + case 6: + return binary.BigEndian.Uint16(packet[0:2]), binary.BigEndian.Uint16(packet[2:4]), "tcp", true + case 17: + return binary.BigEndian.Uint16(packet[0:2]), binary.BigEndian.Uint16(packet[2:4]), "udp", true + default: + return 0, 0, "", false + } +} + +func applyConntrackFields(fields map[string]any, meta conntrackMetadata) { + fields["conntrack_direction"] = meta.direction + fields["conntrack_family"] = meta.family + fields["conntrack_proto"] = meta.proto + if meta.state != "" { + fields["conntrack_state"] = meta.state + } + if meta.status != "" { + fields["conntrack_status"] = meta.status + } + addTupleFields(fields, "conntrack_original", meta.original) + // Keep reply tuple details out of exported fields; packetEvent attaches them + // to Event.Internal so the processor can derive the NAT endpoint aliases. + // addTupleFields(fields, "conntrack_reply", meta.reply) +} + +func addTupleFields(fields map[string]any, prefix string, tuple conntrackTuple) { + fields[prefix+"_src_addr"] = tuple.src.String() + fields[prefix+"_dst_addr"] = tuple.dst.String() + fields[prefix+"_src_port"] = uint32(tuple.srcPort) + fields[prefix+"_dst_port"] = uint32(tuple.dstPort) +} diff --git a/pkg/reflow/source/ebpf/metadata.go b/pkg/reflow/source/ebpf/metadata.go new file mode 100644 index 00000000..1dbfb191 --- /dev/null +++ b/pkg/reflow/source/ebpf/metadata.go @@ -0,0 +1,132 @@ +package ebpf + +import ( + "fmt" + "net" + "sort" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +// InitEvents emits a source_init control event so template-based encoders can +// learn source-scoped metadata before the first captured packet arrives. +func (s *Source) InitEvents() ([]*event.Event, error) { + if !s.captureAny { + iface := net.Interface{ + Index: s.captureInterfaceIndex, + Name: s.cfg.Interface, + } + s.markInterfaceInitialized(uint32(iface.Index), iface.Name) + return []*event.Event{s.sourceInitEvent(iface, s.agentIP, 0)}, nil + } + + ifaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("list capture interfaces: %w", err) + } + sort.Slice(ifaces, func(i, j int) bool { + return ifaces[i].Index < ifaces[j].Index + }) + events := make([]*event.Event, 0, len(ifaces)) + for _, iface := range ifaces { + if !s.interfaceAllowed(iface.Name) { + continue + } + agentIP := firstInterfaceIP(&iface) + s.markInterfaceInitialized(uint32(iface.Index), iface.Name) + events = append(events, s.sourceInitEvent(iface, agentIP, 0)) + } + return events, nil +} + +func (s *Source) sourceInitEvent(iface net.Interface, agentIP string, drops uint32) *event.Event { + ifIndex := uint32(iface.Index) + sourceID := s.sourceIDForInterface(ifIndex) + return &event.Event{ + ReceivedAt: time.Now().UTC(), + Kind: "control", + Source: event.SourceMetadata{ + Network: s.cfg.Network, + Address: s.cfg.Interface, + Type: s.cfg.Type, + CaptureInterface: iface.Name, + CaptureInterfaceIndex: iface.Index, + AgentIP: agentIP, + SourceID: sourceID, + SourceIDSet: true, + Sampling: &event.SamplingMetadata{ + Rate: uint32(s.cfg.SampleEvery), + SamplePool: 0, + Drops: drops, + }, + }, + Control: &event.ControlMetadata{ + Type: "source_init", + Stream: "options_data", + }, + Fields: map[string]any{ + "input_if": ifIndex, + "output_if": ifIndex, + }, + Payload: event.SourceInit{ + Stream: "options_data", + AgentIP: agentIP, + SourceID: sourceID, + SamplingRate: uint32(s.cfg.SampleEvery), + SamplePool: 0, + Drops: drops, + InputIf: ifIndex, + OutputIf: ifIndex, + }, + } +} + +func (s *Source) sourceID() uint32 { + return s.sourceIDForInterface(0) +} + +func (s *Source) sourceIDForInterface(ifIndex uint32) uint32 { + if s.cfg.SourceID != nil { + return *s.cfg.SourceID + } + if s.captureAny && ifIndex != 0 { + return ifIndex + } + return uint32(s.captureInterfaceIndex) +} + +func (s *Source) markInterfaceInitialized(ifIndex uint32, name string) { + if ifIndex == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.interfaceNames == nil { + s.interfaceNames = make(map[uint32]string) + } + if name != "" { + s.interfaceNames[ifIndex] = name + } + if s.initializedInterfaces == nil { + s.initializedInterfaces = make(map[uint32]struct{}) + } + s.initializedInterfaces[ifIndex] = struct{}{} +} + +func (s *Source) interfaceInitialized(ifIndex uint32) bool { + if ifIndex == 0 { + return true + } + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.initializedInterfaces[ifIndex] + return ok +} + +func (s *Source) interfaceAllowed(name string) bool { + if s.interfaceFilter == nil { + return true + } + return name != "" && s.interfaceFilter.MatchString(name) +} diff --git a/pkg/reflow/source/ebpf/metadata_linux.go b/pkg/reflow/source/ebpf/metadata_linux.go new file mode 100644 index 00000000..49d59264 --- /dev/null +++ b/pkg/reflow/source/ebpf/metadata_linux.go @@ -0,0 +1,85 @@ +//go:build linux && !reflow_noebpf + +package ebpf + +import ( + "net" + "regexp" + "sort" + "sync" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" +) + +type Source struct { + cfg config.SourceConfig + agentIP string + captureInterfaceIndex int + captureAny bool + interfaceFilter *regexp.Regexp + interfaceNames map[uint32]string + initializedInterfaces map[uint32]struct{} + seenCount uint64 + mu sync.Mutex + fd int + progFD int + eventMapFD int + perfEvents []*perfEventReader + conntrack *conntrackTracker +} + +func (s *Source) shouldEmitCurrentPacket() bool { + if s.cfg.SampleEvery <= 1 { + return true + } + index := int((s.seenCount - 1) % uint64(s.cfg.SampleEvery)) + return index == s.cfg.SampleOffset +} + +// firstInterfaceIP picks a stable agent IP for exported metadata, preferring +// non-loopback IPv4, then any non-loopback address, then localhost. +func firstInterfaceIP(iface *net.Interface) string { + addrs, err := iface.Addrs() + if err != nil { + return "127.0.0.1" + } + var fallback string + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + if ip == nil || ip.IsLoopback() { + continue + } + if ipv4 := ip.To4(); ipv4 != nil { + return ipv4.String() + } + if fallback == "" { + fallback = ip.String() + } + } + if fallback != "" { + return fallback + } + return "127.0.0.1" +} + +func firstSystemIP() string { + ifaces, err := net.Interfaces() + if err != nil { + return "127.0.0.1" + } + sort.Slice(ifaces, func(i, j int) bool { + return ifaces[i].Index < ifaces[j].Index + }) + for i := range ifaces { + if ip := firstInterfaceIP(&ifaces[i]); ip != "127.0.0.1" { + return ip + } + } + return "127.0.0.1" +} diff --git a/pkg/reflow/source/ebpf/metadata_unsupported.go b/pkg/reflow/source/ebpf/metadata_unsupported.go new file mode 100644 index 00000000..b503db9f --- /dev/null +++ b/pkg/reflow/source/ebpf/metadata_unsupported.go @@ -0,0 +1,30 @@ +//go:build !linux || reflow_noebpf + +package ebpf + +import ( + "net" + "regexp" + "sync" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" +) + +type Source struct { + cfg config.SourceConfig + agentIP string + captureInterfaceIndex int + captureAny bool + interfaceFilter *regexp.Regexp + interfaceNames map[uint32]string + initializedInterfaces map[uint32]struct{} + mu sync.Mutex +} + +func firstInterfaceIP(*net.Interface) string { + return "127.0.0.1" +} + +func firstSystemIP() string { + return "127.0.0.1" +} diff --git a/pkg/reflow/source/ebpf/source_linux.go b/pkg/reflow/source/ebpf/source_linux.go new file mode 100644 index 00000000..db655f9b --- /dev/null +++ b/pkg/reflow/source/ebpf/source_linux.go @@ -0,0 +1,1228 @@ +//go:build linux && !reflow_noebpf + +package ebpf + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "net" + "os" + "regexp" + "runtime" + "strconv" + "strings" + "sync/atomic" + "time" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +const ( + anyInterfaceName = "any" + + ethPAll = 0x0003 + + packetHost = 0 + packetBroadcast = 1 + packetMulticast = 2 + packetOtherHost = 3 + packetOutgoing = 4 + packetLoopback = 5 + packetUser = 6 + packetKernel = 7 + + bpfProgLoad = 5 + bpfMapCreate = 0 + bpfMapUpdateElem = 2 + bpfMapTypePerfEvent = 4 + bpfProgTypeSocket = 1 + bpfPseudoMapFD = 1 + bpfLD = 0x00 + bpfLDX = 0x01 + bpfST = 0x02 + bpfSTX = 0x03 + bpfALU64 = 0x07 + bpfMov = 0xb0 + bpfAdd = 0x00 + bpfOr = 0x40 + bpfLSh = 0x60 + bpfK = 0x00 + bpfX = 0x08 + bpfDW = 0x18 + bpfW = 0x00 + bpfMem = 0x60 + bpfImm = 0x00 + bpfExit = 0x90 + bpfCall = 0x80 + bpfJEQ = 0x10 + bpfJLE = 0xb0 + bpfJMP = 0x05 + bpfReg0 = 0 + bpfReg1 = 1 + bpfReg2 = 2 + bpfReg3 = 3 + bpfReg4 = 4 + bpfReg5 = 5 + bpfReg6 = 6 + bpfReg7 = 7 + bpfReg10 = 10 + bpfFuncPerfOutput = 25 + bpfLogBufferSize = 16 * 1024 + defaultPollInterval = 500 * time.Millisecond + defaultPerfDataPages = 8 + + perfRecordSample = 9 + + skbLenOffset = 0 + skbPacketTypeOffset = 4 + skbMarkOffset = 8 + skbQueueMappingOffset = 12 + skbProtocolOffset = 16 + skbPriorityOffset = 32 + skbIngressIfindexOffset = 36 + skbIfindexOffset = 40 + skbTCIndexOffset = 44 + skbHashOffset = 68 + skbTCClassIDOffset = 72 +) + +type bpfInsn struct { + Code uint8 + DstSrc uint8 + Off int16 + Imm int32 +} + +type packetMetadata struct { + packetType string + direction string + inputIf uint32 + outputIf uint32 + inputInterface string + outputInterface string + skb skbMetadata + hasSKBMetadata bool + conntrack conntrackMetadata + hasConntrack bool +} + +type skbMetadata struct { + Len uint32 + PacketType uint32 + Mark uint32 + QueueMapping uint32 + Protocol uint32 + Priority uint32 + IngressIfindex uint32 + Ifindex uint32 + TCIndex uint32 + Hash uint32 + TCClassID uint32 +} + +type bpfMapCreateAttr struct { + MapType uint32 + KeySize uint32 + ValueSize uint32 + MaxEntries uint32 + MapFlags uint32 + InnerMapFD uint32 + NumaNode uint32 + MapName [16]byte + MapIfindex uint32 + BTFFD uint32 + BTFKeyType uint32 + BTFValType uint32 + BTFVMLinux uint32 +} + +type bpfMapElemAttr struct { + MapFD uint32 + Pad uint32 + Key uint64 + Value uint64 + Flags uint64 +} + +type socketFilterHandles struct { + progFD int +} + +type bpfProgLoadAttr struct { + ProgType uint32 + InsnCnt uint32 + Insns uint64 + License uint64 + LogLevel uint32 + LogSize uint32 + LogBuf uint64 + KernVersion uint32 + ProgFlags uint32 + ProgName [16]byte + ProgIfindex uint32 + ExpectedAttachType uint32 + ProgBTFD uint32 + FuncInfoRecSize uint32 + FuncInfo uint64 + FuncInfoCnt uint32 + LineInfoRecSize uint32 + LineInfo uint64 + LineInfoCnt uint32 + AttachBTFID uint32 + AttachProgFD uint32 + FDArray uint64 +} + +// New validates an eBPF live-capture source and precomputes interface metadata +// used in emitted events and source_init control records. +func New(cfg config.SourceConfig) (*Source, error) { + if cfg.Network != "ebpf" { + return nil, fmt.Errorf("unsupported source.network %q", cfg.Network) + } + if cfg.Interface == "" { + return nil, fmt.Errorf("source.interface is required when source.network=ebpf") + } + if cfg.Type == "" { + cfg.Type = "bytes" + } + if cfg.Type != "bytes" { + return nil, fmt.Errorf("source.type must be bytes when source.network=ebpf") + } + if cfg.SnapLen <= 0 { + cfg.SnapLen = 65535 + } + if cfg.SampleEvery <= 0 { + cfg.SampleEvery = 1 + } + var interfaceFilter *regexp.Regexp + if cfg.InterfaceFilter != "" { + compiled, err := regexp.Compile(cfg.InterfaceFilter) + if err != nil { + return nil, fmt.Errorf("compile source.interface_filter: %w", err) + } + interfaceFilter = compiled + } + direction, err := config.NormalizeEBPFDirection(cfg.EBPF.Direction) + if err != nil { + return nil, err + } + cfg.EBPF.Direction = direction + var conntrack *conntrackTracker + if cfg.EBPF.ConntrackEnabled() { + conntrack = newConntrackTracker(cfg.EBPF.ConntrackPath) + } + if cfg.Interface == anyInterfaceName { + return &Source{ + cfg: cfg, + agentIP: firstSystemIP(), + captureAny: true, + interfaceFilter: interfaceFilter, + interfaceNames: make(map[uint32]string), + initializedInterfaces: make(map[uint32]struct{}), + fd: -1, + progFD: -1, + eventMapFD: -1, + conntrack: conntrack, + }, nil + } + iface, err := net.InterfaceByName(cfg.Interface) + if err != nil { + return nil, fmt.Errorf("lookup capture interface %s: %w", cfg.Interface, err) + } + return &Source{ + cfg: cfg, + agentIP: firstInterfaceIP(iface), + captureInterfaceIndex: iface.Index, + interfaceFilter: interfaceFilter, + interfaceNames: map[uint32]string{ + uint32(iface.Index): iface.Name, + }, + initializedInterfaces: make(map[uint32]struct{}), + fd: -1, + progFD: -1, + eventMapFD: -1, + conntrack: conntrack, + }, nil +} + +// Start captures raw Ethernet frames from an AF_PACKET socket with a small eBPF +// socket filter attached. The filter emits one perf-buffer record containing +// both SKB metadata and packet bytes, so user space never has to pair a packet +// read with a separate metadata map lookup. +func (s *Source) Start(ctx context.Context, emit func(*event.Event) error) error { + fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW|unix.SOCK_CLOEXEC, int(htons(ethPAll))) + if err != nil { + return fmt.Errorf("open ebpf packet socket on %s: %w", s.cfg.Interface, err) + } + s.mu.Lock() + s.fd = fd + s.mu.Unlock() + defer s.Close() + + if err := unix.Bind(fd, &unix.SockaddrLinklayer{ + Protocol: htons(ethPAll), + Ifindex: s.captureInterfaceIndex, + }); err != nil { + return fmt.Errorf("bind ebpf packet socket to %s: %w", s.cfg.Interface, err) + } + tv := unix.NsecToTimeval(defaultPollInterval.Nanoseconds()) + if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv); err != nil { + return fmt.Errorf("set ebpf packet socket timeout: %w", err) + } + + eventMapFD, perfEvents, err := createPerfEventOutput(defaultPerfDataPages) + if err != nil { + return fmt.Errorf("create ebpf perf output: %w", err) + } + s.mu.Lock() + s.eventMapFD = eventMapFD + s.perfEvents = perfEvents + s.mu.Unlock() + + filter, err := attachSocketFilter(fd, s.cfg.SnapLen, eventMapFD) + if err != nil { + return fmt.Errorf("attach packet socket filter: %w", err) + } + if filter.progFD >= 0 { + s.mu.Lock() + s.progFD = filter.progFD + s.mu.Unlock() + } + + go func() { + <-ctx.Done() + _ = s.Close() + }() + + return s.pollPerfEvents(ctx, perfEvents, emit) +} + +func (s *Source) packetEvent(data []byte, meta packetMetadata) *event.Event { + now := time.Now().UTC() + samplePool := uint32(s.seenCount) + payload := append([]byte(nil), data...) + wireLength := len(payload) + if meta.hasSKBMetadata && meta.skb.Len > uint32(wireLength) { + wireLength = int(meta.skb.Len) + } + evt := &event.Event{ + ReceivedAt: now, + Source: event.SourceMetadata{ + Network: s.cfg.Network, + Address: s.cfg.Interface, + Type: s.cfg.Type, + CaptureInterface: s.eventCaptureInterface(meta), + CaptureInterfaceIndex: s.eventCaptureInterfaceIndex(meta), + CaptureDirection: meta.direction, + CapturePacketType: meta.packetType, + AgentIP: s.agentIP, + SourceID: s.eventSourceID(meta), + SourceIDSet: true, + Sampling: &event.SamplingMetadata{ + Rate: uint32(s.cfg.SampleEvery), + SamplePool: samplePool, + Drops: 0, + }, + JSON: event.JSONMetadata{ + Flavor: s.cfg.JSON.Flavor, + }, + }, + Payload: payload, + Fields: map[string]any{ + "capture_length": len(payload), + "wire_length": wireLength, + }, + } + applyPacketMetadataFields(evt.Fields, meta) + if meta.hasConntrack { + evt.Internal = make(map[string]any, 4) + addTupleFields(evt.Internal, "conntrack_reply", meta.conntrack.reply) + } + return evt +} + +func (s *Source) pollPerfEvents(ctx context.Context, readers []*perfEventReader, emit func(*event.Event) error) error { + pollFDs := make([]unix.PollFd, 0, len(readers)) + for _, reader := range readers { + pollFDs = append(pollFDs, unix.PollFd{Fd: int32(reader.fd), Events: unix.POLLIN}) + } + timeout := int(defaultPollInterval / time.Millisecond) + for { + if ctx.Err() != nil { + return nil + } + _, err := unix.Poll(pollFDs, timeout) + if err != nil { + if err == unix.EINTR { + continue + } + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("poll ebpf perf events: %w", err) + } + for i := range readers { + if pollFDs[i].Revents&(unix.POLLERR|unix.POLLHUP|unix.POLLNVAL) != 0 { + return fmt.Errorf("poll ebpf perf event cpu %d: revents=%#x", readers[i].cpu, pollFDs[i].Revents) + } + if err := readers[i].readSamples(func(sample []byte) error { + return s.emitPerfSample(sample, emit) + }); err != nil { + return err + } + } + } +} + +func (s *Source) emitPerfSample(sample []byte, emit func(*event.Event) error) error { + skb, packet, err := parsePerfPacketSample(sample) + if err != nil { + return err + } + meta := s.packetMetadataFromSKB(skb) + if !allowDirection(s.cfg.EBPF.DirectionFilter(), meta.direction) { + return nil + } + if !s.packetInterfaceAllowed(meta) { + return nil + } + if s.captureAny { + if err := s.emitFirstSeenInterfaceInits(meta, emit); err != nil { + return err + } + } + s.seenCount++ + if !s.shouldEmitCurrentPacket() { + return nil + } + if s.conntrack != nil { + if ct, ok := s.conntrack.Lookup(packet); ok { + meta.conntrack = ct + meta.hasConntrack = true + } + } + return emit(s.packetEvent(packet, meta)) +} + +func (s *Source) emitFirstSeenInterfaceInits(meta packetMetadata, emit func(*event.Event) error) error { + for _, item := range []struct { + ifIndex uint32 + name string + }{ + {ifIndex: meta.inputIf, name: meta.inputInterface}, + {ifIndex: meta.outputIf, name: meta.outputInterface}, + } { + if item.ifIndex == 0 || s.interfaceInitialized(item.ifIndex) { + continue + } + name := item.name + if name == "" { + name = s.interfaceName(item.ifIndex) + } + if name == "" { + name = fmt.Sprintf("ifindex-%d", item.ifIndex) + } + if !s.interfaceAllowed(name) { + continue + } + s.markInterfaceInitialized(item.ifIndex, name) + evt := s.sourceInitEvent(net.Interface{Index: int(item.ifIndex), Name: name}, s.agentIP, 0) + if err := emit(evt); err != nil { + return err + } + } + return nil +} + +func (s *Source) packetInterfaceAllowed(meta packetMetadata) bool { + if s.interfaceFilter == nil { + return true + } + for _, item := range []struct { + ifIndex uint32 + name string + }{ + {ifIndex: meta.inputIf, name: meta.inputInterface}, + {ifIndex: meta.outputIf, name: meta.outputInterface}, + } { + name := item.name + if name == "" && item.ifIndex != 0 { + name = s.interfaceName(item.ifIndex) + } + if s.interfaceAllowed(name) { + return true + } + } + return false +} + +func (s *Source) eventCaptureInterface(meta packetMetadata) string { + switch meta.direction { + case "out": + if meta.outputInterface != "" { + return meta.outputInterface + } + if meta.outputIf != 0 { + if name := s.interfaceName(meta.outputIf); name != "" { + return name + } + } + default: + if meta.inputInterface != "" { + return meta.inputInterface + } + if meta.inputIf != 0 { + if name := s.interfaceName(meta.inputIf); name != "" { + return name + } + } + if meta.outputInterface != "" { + return meta.outputInterface + } + if meta.outputIf != 0 { + if name := s.interfaceName(meta.outputIf); name != "" { + return name + } + } + } + return s.cfg.Interface +} + +func (s *Source) eventCaptureInterfaceIndex(meta packetMetadata) int { + switch meta.direction { + case "out": + if meta.outputIf != 0 { + return int(meta.outputIf) + } + default: + if meta.inputIf != 0 { + return int(meta.inputIf) + } + if meta.outputIf != 0 { + return int(meta.outputIf) + } + } + return s.captureInterfaceIndex +} + +func (s *Source) eventSourceID(meta packetMetadata) uint32 { + switch meta.direction { + case "out": + if meta.outputIf != 0 { + return s.sourceIDForInterface(meta.outputIf) + } + default: + if meta.inputIf != 0 { + return s.sourceIDForInterface(meta.inputIf) + } + if meta.outputIf != 0 { + return s.sourceIDForInterface(meta.outputIf) + } + } + return s.sourceID() +} + +func parsePerfPacketSample(sample []byte) (skbMetadata, []byte, error) { + metadataSize := int(unsafe.Sizeof(skbMetadata{})) + if len(sample) < metadataSize { + return skbMetadata{}, nil, fmt.Errorf("short ebpf packet event: got %d bytes, need at least %d", len(sample), metadataSize) + } + meta := skbMetadata{ + Len: binary.LittleEndian.Uint32(sample[0:4]), + PacketType: binary.LittleEndian.Uint32(sample[4:8]), + Mark: binary.LittleEndian.Uint32(sample[8:12]), + QueueMapping: binary.LittleEndian.Uint32(sample[12:16]), + Protocol: binary.LittleEndian.Uint32(sample[16:20]), + Priority: binary.LittleEndian.Uint32(sample[20:24]), + IngressIfindex: binary.LittleEndian.Uint32(sample[24:28]), + Ifindex: binary.LittleEndian.Uint32(sample[28:32]), + TCIndex: binary.LittleEndian.Uint32(sample[32:36]), + Hash: binary.LittleEndian.Uint32(sample[36:40]), + TCClassID: binary.LittleEndian.Uint32(sample[40:44]), + } + packet := append([]byte(nil), sample[metadataSize:]...) + return meta, packet, nil +} + +func (s *Source) packetMetadataFromSKB(skb skbMetadata) packetMetadata { + packetType := uint8(skb.PacketType) + meta := packetMetadata{ + packetType: packetTypeName(packetType), + direction: "unknown", + } + switch packetType { + case packetOutgoing: + meta.direction = "out" + case packetHost, packetBroadcast, packetMulticast, packetOtherHost: + meta.direction = "in" + case packetLoopback: + meta.direction = "loopback" + } + if s.cfg.EBPF.SKBMetadataEnabled() { + meta = s.mergeSKBMetadata(meta, skb) + } + switch meta.direction { + case "in": + if meta.inputIf == 0 { + meta.inputIf = firstNonZero(skb.IngressIfindex, skb.Ifindex, uint32(s.captureInterfaceIndex)) + meta.inputInterface = s.interfaceName(meta.inputIf) + } + case "out": + if meta.outputIf == 0 { + meta.outputIf = firstNonZero(skb.Ifindex, uint32(s.captureInterfaceIndex)) + meta.outputInterface = s.interfaceName(meta.outputIf) + } + case "loopback": + if meta.inputIf == 0 { + meta.inputIf = firstNonZero(skb.IngressIfindex, skb.Ifindex, uint32(s.captureInterfaceIndex)) + meta.inputInterface = s.interfaceName(meta.inputIf) + } + if meta.outputIf == 0 { + meta.outputIf = firstNonZero(skb.Ifindex, uint32(s.captureInterfaceIndex)) + meta.outputInterface = s.interfaceName(meta.outputIf) + } + } + return meta +} + +func firstNonZero(values ...uint32) uint32 { + for _, value := range values { + if value != 0 { + return value + } + } + return 0 +} + +func (s *Source) packetMetadata(from unix.Sockaddr) packetMetadata { + meta := packetMetadata{ + packetType: "unknown", + direction: "unknown", + } + link, ok := from.(*unix.SockaddrLinklayer) + if !ok || link == nil { + return meta + } + + ifIndex := uint32(s.captureInterfaceIndex) + if link.Ifindex > 0 { + ifIndex = uint32(link.Ifindex) + } + meta.packetType = packetTypeName(link.Pkttype) + switch link.Pkttype { + case packetOutgoing: + meta.direction = "out" + meta.outputIf = ifIndex + meta.outputInterface = s.cfg.Interface + case packetHost, packetBroadcast, packetMulticast, packetOtherHost: + meta.direction = "in" + meta.inputIf = ifIndex + meta.inputInterface = s.cfg.Interface + case packetLoopback: + meta.direction = "loopback" + meta.inputIf = ifIndex + meta.outputIf = ifIndex + meta.inputInterface = s.cfg.Interface + meta.outputInterface = s.cfg.Interface + } + return meta +} + +func allowDirection(filter, direction string) bool { + switch filter { + case "", "both": + return true + case "ingress": + return direction == "in" + case "egress": + return direction == "out" + default: + return true + } +} + +func (s *Source) mergeSKBMetadata(meta packetMetadata, skb skbMetadata) packetMetadata { + meta.skb = skb + meta.hasSKBMetadata = true + if skb.PacketType != 0 { + meta.packetType = packetTypeName(uint8(skb.PacketType)) + } + if skb.IngressIfindex != 0 && meta.inputIf == 0 { + meta.inputIf = skb.IngressIfindex + meta.inputInterface = s.interfaceName(skb.IngressIfindex) + } + if skb.Ifindex != 0 { + switch meta.direction { + case "out": + meta.outputIf = skb.Ifindex + meta.outputInterface = s.interfaceName(skb.Ifindex) + case "in", "loopback": + if meta.inputIf == 0 { + meta.inputIf = skb.Ifindex + meta.inputInterface = s.interfaceName(skb.Ifindex) + } + default: + meta.outputIf = skb.Ifindex + meta.outputInterface = s.interfaceName(skb.Ifindex) + } + } + if meta.inputInterface == "" && meta.inputIf != 0 { + meta.inputInterface = s.interfaceName(meta.inputIf) + } + if meta.outputInterface == "" && meta.outputIf != 0 { + meta.outputInterface = s.interfaceName(meta.outputIf) + } + return meta +} + +func (s *Source) interfaceName(ifindex uint32) string { + if ifindex == 0 { + return "" + } + s.mu.Lock() + if name := s.interfaceNames[ifindex]; name != "" { + s.mu.Unlock() + return name + } + s.mu.Unlock() + iface, err := net.InterfaceByIndex(int(ifindex)) + if err != nil || iface == nil { + return "" + } + s.mu.Lock() + defer s.mu.Unlock() + if s.interfaceNames == nil { + s.interfaceNames = make(map[uint32]string) + } + s.interfaceNames[ifindex] = iface.Name + return iface.Name +} + +func packetTypeName(packetType uint8) string { + switch packetType { + case packetHost: + return "host" + case packetBroadcast: + return "broadcast" + case packetMulticast: + return "multicast" + case packetOtherHost: + return "otherhost" + case packetOutgoing: + return "outgoing" + case packetLoopback: + return "loopback" + case packetUser: + return "user" + case packetKernel: + return "kernel" + default: + return "unknown" + } +} + +func applyPacketMetadataFields(fields map[string]any, meta packetMetadata) { + if meta.packetType != "" { + fields["capture_packet_type"] = meta.packetType + } + if meta.direction != "" { + fields["capture_direction"] = meta.direction + } + if meta.inputIf != 0 { + fields["input_if"] = meta.inputIf + } + if meta.outputIf != 0 { + fields["output_if"] = meta.outputIf + } + if meta.inputInterface != "" { + fields["input_interface"] = meta.inputInterface + } + if meta.outputInterface != "" { + fields["output_interface"] = meta.outputInterface + } + if meta.hasSKBMetadata { + fields["skb_len"] = meta.skb.Len + fields["skb_packet_type"] = meta.skb.PacketType + fields["skb_mark"] = meta.skb.Mark + fields["skb_queue_mapping"] = meta.skb.QueueMapping + fields["skb_protocol"] = meta.skb.Protocol + fields["skb_priority"] = meta.skb.Priority + fields["skb_ingress_ifindex"] = meta.skb.IngressIfindex + fields["skb_ifindex"] = meta.skb.Ifindex + fields["skb_tc_index"] = meta.skb.TCIndex + fields["skb_hash"] = meta.skb.Hash + fields["skb_tc_classid"] = meta.skb.TCClassID + } + if meta.hasConntrack { + applyConntrackFields(fields, meta.conntrack) + } +} + +func (s *Source) Close() error { + var firstErr error + s.mu.Lock() + defer s.mu.Unlock() + if s.fd >= 0 { + if err := unix.Close(s.fd); err != nil && err != unix.EBADF { + firstErr = err + } + s.fd = -1 + } + if s.progFD >= 0 { + if err := unix.Close(s.progFD); err != nil && err != unix.EBADF && firstErr == nil { + firstErr = err + } + s.progFD = -1 + } + if s.eventMapFD >= 0 { + if err := unix.Close(s.eventMapFD); err != nil && err != unix.EBADF && firstErr == nil { + firstErr = err + } + s.eventMapFD = -1 + } + for _, reader := range s.perfEvents { + if err := reader.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + s.perfEvents = nil + return firstErr +} + +func loadSocketFilterProgram(snapLen, eventMapFD int, fullMetadata bool) (int, error) { + if snapLen <= 0 { + return -1, fmt.Errorf("snaplen must be > 0") + } + insns := socketFilterInstructions(snapLen, eventMapFD, fullMetadata) + license := []byte("GPL\x00") + attr := bpfProgLoadAttr{ + ProgType: bpfProgTypeSocket, + InsnCnt: uint32(len(insns)), + Insns: uint64(uintptr(unsafe.Pointer(&insns[0]))), + License: uint64(uintptr(unsafe.Pointer(&license[0]))), + } + copy(attr.ProgName[:], "reflow_capture") + + fd, errno := bpfProgLoadProgram(attr) + if errno == 0 { + return fd, nil + } + + logBuf := make([]byte, bpfLogBufferSize) + attr.LogLevel = 1 + attr.LogSize = uint32(len(logBuf)) + attr.LogBuf = uint64(uintptr(unsafe.Pointer(&logBuf[0]))) + fd, loggedErrno := bpfProgLoadProgram(attr) + if loggedErrno == 0 { + return fd, nil + } + if msg := strings.TrimRight(string(logBuf), "\x00"); msg != "" { + return -1, fmt.Errorf("%w: %s", loggedErrno, msg) + } + return -1, errno +} + +func socketFilterInstructions(snapLen, eventMapFD int, fullMetadata bool) []bpfInsn { + if eventMapFD < 0 { + return []bpfInsn{ + mov64Imm(bpfReg0, int32(snapLen)), + exitInsn(), + } + } + metadataSize := int16(unsafe.Sizeof(skbMetadata{})) + stackSize := alignInt16(metadataSize, 8) + stackBase := -stackSize + insns := []bpfInsn{ + mov64Reg(bpfReg6, bpfReg1), + } + fields := []struct { + skbOffset int16 + eventOffset int16 + }{ + {skbLenOffset, 0}, + {skbPacketTypeOffset, 4}, + {skbMarkOffset, 8}, + {skbQueueMappingOffset, 12}, + {skbProtocolOffset, 16}, + {skbPriorityOffset, 20}, + {skbIngressIfindexOffset, 24}, + {skbIfindexOffset, 28}, + {skbHashOffset, 36}, + } + if fullMetadata { + fields = append(fields, + struct { + skbOffset int16 + eventOffset int16 + }{skbTCIndexOffset, 32}, + struct { + skbOffset int16 + eventOffset int16 + }{skbTCClassIDOffset, 40}, + ) + } + for offset := int16(0); offset < stackSize; offset += 4 { + insns = append(insns, storeImm(bpfReg10, stackBase+offset, 0)) + } + for _, field := range fields { + insns = append(insns, + loadMem(bpfReg1, bpfReg6, field.skbOffset), + storeReg(bpfReg10, bpfReg1, stackBase+field.eventOffset), + ) + } + insns = append(insns, + loadMem(bpfReg3, bpfReg6, skbLenOffset), + jumpImm(bpfJLE, bpfReg3, int32(snapLen), 1), + mov64Imm(bpfReg3, int32(snapLen)), + lsh64Imm(bpfReg3, 32), + ) + insns = append(insns, loadImm64(bpfReg7, uint64(unix.BPF_F_CURRENT_CPU))...) + insns = append(insns, or64Reg(bpfReg3, bpfReg7)) + insns = append(insns, loadMapFD(bpfReg2, eventMapFD)...) + insns = append(insns, + mov64Reg(bpfReg1, bpfReg6), + mov64Reg(bpfReg4, bpfReg10), + add64Imm(bpfReg4, int32(stackBase)), + mov64Imm(bpfReg5, int32(metadataSize)), + callInsn(bpfFuncPerfOutput), + mov64Imm(bpfReg0, 0), + exitInsn(), + ) + return insns +} + +func reg(dst, src uint8) uint8 { + return dst | src<<4 +} + +func alignInt16(v, alignment int16) int16 { + if alignment <= 0 || v%alignment == 0 { + return v + } + return v + alignment - v%alignment +} + +func mov64Imm(dst uint8, imm int32) bpfInsn { + return bpfInsn{Code: bpfALU64 | bpfMov | bpfK, DstSrc: reg(dst, 0), Imm: imm} +} + +func mov64Reg(dst, src uint8) bpfInsn { + return bpfInsn{Code: bpfALU64 | bpfMov | bpfX, DstSrc: reg(dst, src)} +} + +func add64Imm(dst uint8, imm int32) bpfInsn { + return bpfInsn{Code: bpfALU64 | bpfAdd | bpfK, DstSrc: reg(dst, 0), Imm: imm} +} + +func or64Reg(dst, src uint8) bpfInsn { + return bpfInsn{Code: bpfALU64 | bpfOr | bpfX, DstSrc: reg(dst, src)} +} + +func lsh64Imm(dst uint8, imm int32) bpfInsn { + return bpfInsn{Code: bpfALU64 | bpfLSh | bpfK, DstSrc: reg(dst, 0), Imm: imm} +} + +func loadMem(dst, src uint8, off int16) bpfInsn { + return bpfInsn{Code: bpfLDX | bpfMem | bpfW, DstSrc: reg(dst, src), Off: off} +} + +func storeImm(dst uint8, off int16, imm int32) bpfInsn { + return bpfInsn{Code: bpfST | bpfMem | bpfW, DstSrc: reg(dst, 0), Off: off, Imm: imm} +} + +func storeReg(dst, src uint8, off int16) bpfInsn { + return bpfInsn{Code: bpfSTX | bpfMem | bpfW, DstSrc: reg(dst, src), Off: off} +} + +func loadMapFD(dst uint8, mapFD int) []bpfInsn { + return []bpfInsn{ + {Code: bpfLD | bpfDW | bpfImm, DstSrc: reg(dst, bpfPseudoMapFD), Imm: int32(mapFD)}, + {}, + } +} + +func loadImm64(dst uint8, imm uint64) []bpfInsn { + return []bpfInsn{ + {Code: bpfLD | bpfDW | bpfImm, DstSrc: reg(dst, 0), Imm: int32(imm)}, + {Imm: int32(imm >> 32)}, + } +} + +func callInsn(helper int32) bpfInsn { + return bpfInsn{Code: bpfJMP | bpfCall, Imm: helper} +} + +func jumpImm(op uint8, dst uint8, imm int32, off int16) bpfInsn { + return bpfInsn{Code: bpfJMP | op | bpfK, DstSrc: reg(dst, 0), Off: off, Imm: imm} +} + +func exitInsn() bpfInsn { + return bpfInsn{Code: bpfJMP | bpfExit} +} + +func bpfProgLoadProgram(attr bpfProgLoadAttr) (int, unix.Errno) { + fd, _, errno := unix.Syscall(unix.SYS_BPF, uintptr(bpfProgLoad), uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr)) + if errno != 0 { + return -1, errno + } + return int(fd), 0 +} + +func attachSocketFilter(fd, snapLen, eventMapFD int) (socketFilterHandles, error) { + progFD, err := loadSocketFilterProgram(snapLen, eventMapFD, true) + if err != nil { + progFD, err = loadSocketFilterProgram(snapLen, eventMapFD, false) + } + if err == nil { + if attachErr := unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_ATTACH_BPF, progFD); attachErr != nil { + _ = unix.Close(progFD) + return socketFilterHandles{}, fmt.Errorf("attach ebpf socket filter: %w", attachErr) + } + return socketFilterHandles{progFD: progFD}, nil + } + return socketFilterHandles{}, fmt.Errorf("load ebpf socket perf filter: %w", err) +} + +func createPerfEventArrayMap(maxEntries int) (int, error) { + attr := bpfMapCreateAttr{ + MapType: bpfMapTypePerfEvent, + KeySize: 4, + ValueSize: 4, + MaxEntries: uint32(maxEntries), + } + copy(attr.MapName[:], "reflow_events") + fd, _, errno := unix.Syscall(unix.SYS_BPF, uintptr(bpfMapCreate), uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr)) + if errno != 0 { + return -1, errno + } + return int(fd), nil +} + +func updatePerfEventArrayMap(mapFD int, cpu int, perfFD int) error { + key := uint32(cpu) + value := uint32(perfFD) + attr := bpfMapElemAttr{ + MapFD: uint32(mapFD), + Key: uint64(uintptr(unsafe.Pointer(&key))), + Value: uint64(uintptr(unsafe.Pointer(&value))), + Flags: unix.BPF_ANY, + } + _, _, errno := unix.Syscall(unix.SYS_BPF, uintptr(bpfMapUpdateElem), uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr)) + if errno != 0 { + return errno + } + return nil +} + +type perfEventReader struct { + cpu int + fd int + pageSize int + data []byte +} + +func createPerfEventOutput(dataPages int) (int, []*perfEventReader, error) { + cpus := onlineCPUs() + if len(cpus) == 0 { + return -1, nil, fmt.Errorf("no online CPUs found") + } + maxCPU := cpus[0] + for _, cpu := range cpus[1:] { + if cpu > maxCPU { + maxCPU = cpu + } + } + mapFD, err := createPerfEventArrayMap(maxCPU + 1) + if err != nil { + return -1, nil, err + } + readers := make([]*perfEventReader, 0, len(cpus)) + for _, cpu := range cpus { + reader, err := openPerfEventReader(cpu, dataPages) + if err != nil { + closePerfEventReaders(readers) + _ = unix.Close(mapFD) + return -1, nil, fmt.Errorf("open perf event for cpu %d: %w", cpu, err) + } + if err := updatePerfEventArrayMap(mapFD, cpu, reader.fd); err != nil { + _ = reader.Close() + closePerfEventReaders(readers) + _ = unix.Close(mapFD) + return -1, nil, fmt.Errorf("populate perf event map for cpu %d: %w", cpu, err) + } + readers = append(readers, reader) + } + return mapFD, readers, nil +} + +func openPerfEventReader(cpu, dataPages int) (*perfEventReader, error) { + if dataPages <= 0 { + dataPages = defaultPerfDataPages + } + attr := unix.PerfEventAttr{ + Type: unix.PERF_TYPE_SOFTWARE, + Size: uint32(unsafe.Sizeof(unix.PerfEventAttr{})), + Config: unix.PERF_COUNT_SW_BPF_OUTPUT, + Sample_type: unix.PERF_SAMPLE_RAW, + Wakeup: 1, + } + fd, err := unix.PerfEventOpen(&attr, -1, cpu, -1, unix.PERF_FLAG_FD_CLOEXEC) + if err != nil { + return nil, err + } + pageSize := os.Getpagesize() + data, err := unix.Mmap(fd, 0, pageSize*(dataPages+1), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) + if err != nil { + _ = unix.Close(fd) + return nil, err + } + if err := unix.IoctlSetInt(fd, unix.PERF_EVENT_IOC_ENABLE, 0); err != nil { + _ = unix.Munmap(data) + _ = unix.Close(fd) + return nil, err + } + return &perfEventReader{cpu: cpu, fd: fd, pageSize: pageSize, data: data}, nil +} + +func closePerfEventReaders(readers []*perfEventReader) { + for _, reader := range readers { + _ = reader.Close() + } +} + +func (r *perfEventReader) Close() error { + var firstErr error + if r.fd >= 0 { + if err := unix.IoctlSetInt(r.fd, unix.PERF_EVENT_IOC_DISABLE, 0); err != nil && !errors.Is(err, unix.EBADF) { + firstErr = err + } + } + if len(r.data) > 0 { + if err := unix.Munmap(r.data); err != nil && firstErr == nil { + firstErr = err + } + r.data = nil + } + if r.fd >= 0 { + if err := unix.Close(r.fd); err != nil && err != unix.EBADF && firstErr == nil { + firstErr = err + } + r.fd = -1 + } + return firstErr +} + +func (r *perfEventReader) readSamples(emit func([]byte) error) error { + if len(r.data) < r.pageSize { + return fmt.Errorf("perf event cpu %d is not mapped", r.cpu) + } + page := (*unix.PerfEventMmapPage)(unsafe.Pointer(&r.data[0])) + head := atomic.LoadUint64(&page.Data_head) + tail := atomic.LoadUint64(&page.Data_tail) + dataOffset := int(page.Data_offset) + dataSize := int(page.Data_size) + if dataOffset == 0 { + dataOffset = r.pageSize + } + if dataSize == 0 { + dataSize = len(r.data) - dataOffset + } + if dataOffset < 0 || dataSize <= 0 || dataOffset+dataSize > len(r.data) { + return fmt.Errorf("invalid perf ring layout for cpu %d", r.cpu) + } + ring := r.data[dataOffset : dataOffset+dataSize] + for tail < head { + record, size, err := readPerfRecord(ring, tail) + if err != nil { + return fmt.Errorf("read ebpf perf record cpu %d: %w", r.cpu, err) + } + tail += uint64(size) + if len(record) < 12 || binary.LittleEndian.Uint32(record[0:4]) != perfRecordSample { + continue + } + rawSize := int(binary.LittleEndian.Uint32(record[8:12])) + if rawSize < 0 || 12+rawSize > len(record) { + return fmt.Errorf("invalid ebpf perf sample size %d in record size %d", rawSize, len(record)) + } + if err := emit(record[12 : 12+rawSize]); err != nil { + return err + } + } + atomic.StoreUint64(&page.Data_tail, tail) + return nil +} + +func readPerfRecord(ring []byte, tail uint64) ([]byte, uint16, error) { + if len(ring) < 8 { + return nil, 0, fmt.Errorf("perf ring is too small") + } + headerBytes := readPerfRingBytes(ring, tail, 8) + size := binary.LittleEndian.Uint16(headerBytes[6:8]) + if size < 8 { + return nil, 0, fmt.Errorf("invalid perf record size %d", size) + } + record := readPerfRingBytes(ring, tail, int(size)) + return record, size, nil +} + +func readPerfRingBytes(ring []byte, offset uint64, size int) []byte { + start := int(offset % uint64(len(ring))) + if start+size <= len(ring) { + return append([]byte(nil), ring[start:start+size]...) + } + out := make([]byte, size) + n := copy(out, ring[start:]) + copy(out[n:], ring[:size-n]) + return out +} + +func onlineCPUs() []int { + data, err := os.ReadFile("/sys/devices/system/cpu/online") + if err == nil { + if cpus := parseCPUList(strings.TrimSpace(string(data))); len(cpus) > 0 { + return cpus + } + } + n := runtime.NumCPU() + cpus := make([]int, 0, n) + for i := 0; i < n; i++ { + cpus = append(cpus, i) + } + return cpus +} + +func parseCPUList(text string) []int { + var cpus []int + for _, part := range strings.Split(text, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + startText, endText, ranged := strings.Cut(part, "-") + start, err := strconv.Atoi(startText) + if err != nil || start < 0 { + return nil + } + end := start + if ranged { + end, err = strconv.Atoi(endText) + if err != nil || end < start { + return nil + } + } + for cpu := start; cpu <= end; cpu++ { + cpus = append(cpus, cpu) + } + } + return cpus +} + +func htons(v uint16) uint16 { + return (v << 8) | (v >> 8) +} diff --git a/pkg/reflow/source/ebpf/source_linux_test.go b/pkg/reflow/source/ebpf/source_linux_test.go new file mode 100644 index 00000000..137c4c2e --- /dev/null +++ b/pkg/reflow/source/ebpf/source_linux_test.go @@ -0,0 +1,513 @@ +//go:build linux && !reflow_noebpf + +package ebpf + +import ( + "encoding/binary" + "net/netip" + "regexp" + "testing" + + "golang.org/x/sys/unix" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func TestNewSupportsAnyInterface(t *testing.T) { + source, err := New(config.SourceConfig{ + Network: "ebpf", + Interface: "any", + Type: "bytes", + SampleEvery: 1, + InterfaceFilter: "^wan", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + if !source.captureAny { + t.Fatalf("expected captureAny to be enabled") + } + if source.captureInterfaceIndex != 0 { + t.Fatalf("expected any capture interface index 0, got %d", source.captureInterfaceIndex) + } + if source.interfaceFilter == nil || !source.interfaceFilter.MatchString("wan0") { + t.Fatalf("expected compiled interface filter, got %#v", source.interfaceFilter) + } +} + +func TestPacketMetadataMarksOutgoingCapture(t *testing.T) { + source := &Source{ + cfg: config.SourceConfig{Network: "ebpf", Interface: "br-lan", Type: "bytes", SampleEvery: 1}, + agentIP: "192.0.2.10", + captureInterfaceIndex: 7, + seenCount: 1, + } + + meta := source.packetMetadata(&unix.SockaddrLinklayer{ + Ifindex: 7, + Pkttype: packetOutgoing, + }) + evt := source.packetEvent([]byte{0, 1, 2, 3}, meta) + + if evt.Source.CaptureDirection != "out" { + t.Fatalf("expected source capture direction out, got %q", evt.Source.CaptureDirection) + } + if evt.Source.CapturePacketType != "outgoing" { + t.Fatalf("expected packet type outgoing, got %q", evt.Source.CapturePacketType) + } + if got := evt.Fields["output_if"]; got != uint32(7) { + t.Fatalf("expected output_if=7, got %#v", got) + } + if _, ok := evt.Fields["input_if"]; ok { + t.Fatalf("expected input_if to be unset for outgoing packet, got %#v", evt.Fields["input_if"]) + } + if got := evt.Fields["output_interface"]; got != "br-lan" { + t.Fatalf("expected output_interface=br-lan, got %#v", got) + } + if _, ok := evt.Fields["dst_interface"]; ok { + t.Fatalf("expected dst_interface alias to be unset, got %#v", evt.Fields["dst_interface"]) + } + for _, key := range []string{"agent_ip", "sampling_rate", "sample_pool", "drops"} { + if _, ok := evt.Fields[key]; ok { + t.Fatalf("expected %s to stay in source metadata, got field value %#v", key, evt.Fields[key]) + } + } +} + +func TestParsePerfPacketSampleCombinesMetadataAndPacket(t *testing.T) { + packet := []byte{0xde, 0xad, 0xbe, 0xef} + sample := make([]byte, testSKBMetadataSize()+len(packet)) + binary.LittleEndian.PutUint32(sample[0:4], 1514) + binary.LittleEndian.PutUint32(sample[4:8], packetOutgoing) + binary.LittleEndian.PutUint32(sample[8:12], 42) + binary.LittleEndian.PutUint32(sample[12:16], 3) + binary.LittleEndian.PutUint32(sample[16:20], 0x0800) + binary.LittleEndian.PutUint32(sample[20:24], 5) + binary.LittleEndian.PutUint32(sample[24:28], 7) + binary.LittleEndian.PutUint32(sample[28:32], 9) + binary.LittleEndian.PutUint32(sample[32:36], 11) + binary.LittleEndian.PutUint32(sample[36:40], 0x12345678) + binary.LittleEndian.PutUint32(sample[40:44], 0x10002) + copy(sample[testSKBMetadataSize():], packet) + + meta, gotPacket, err := parsePerfPacketSample(sample) + if err != nil { + t.Fatalf("parse perf packet sample: %v", err) + } + if meta.Len != 1514 || meta.PacketType != packetOutgoing || meta.Mark != 42 || meta.Hash != 0x12345678 { + t.Fatalf("unexpected metadata: %#v", meta) + } + if string(gotPacket) != string(packet) { + t.Fatalf("expected packet %x, got %x", packet, gotPacket) + } + gotPacket[0] = 0 + if packet[0] == 0 { + t.Fatalf("expected parsed packet to be copied away from the sample buffer") + } +} + +func TestPacketMetadataFromSKBEventPreservesBranchAliases(t *testing.T) { + source := &Source{ + cfg: config.SourceConfig{Network: "ebpf", Interface: "br-lan", Type: "bytes", SampleEvery: 1}, + agentIP: "192.0.2.10", + captureInterfaceIndex: 7, + interfaceNames: map[uint32]string{ + 7: "br-lan", + 9: "wan", + }, + } + meta := source.packetMetadataFromSKB(skbMetadata{ + Len: 1514, + PacketType: packetOutgoing, + Mark: 42, + IngressIfindex: 7, + Ifindex: 9, + }) + if meta.direction != "out" || meta.packetType != "outgoing" { + t.Fatalf("unexpected packet direction/type: %#v", meta) + } + if meta.outputIf != 9 || meta.outputInterface != "wan" { + t.Fatalf("expected output interface from skb ifindex, got %#v", meta) + } + if !meta.hasSKBMetadata { + t.Fatalf("expected skb metadata to be marked present") + } + evt := source.packetEvent([]byte{0, 1, 2, 3}, meta) + if _, ok := evt.Fields["firewall_mark"]; ok { + t.Fatalf("expected firewall_mark alias to be unset, got %#v", evt.Fields["firewall_mark"]) + } + if _, ok := evt.Fields["dst_interface"]; ok { + t.Fatalf("expected dst_interface alias to be unset, got %#v", evt.Fields["dst_interface"]) + } +} + +func TestEmitPerfSampleAppliesDirectionBeforeSampling(t *testing.T) { + source := &Source{ + cfg: config.SourceConfig{ + Network: "ebpf", + Interface: "br-lan", + Type: "bytes", + SampleEvery: 1, + EBPF: config.EBPFConfig{Direction: "egress"}, + }, + agentIP: "192.0.2.10", + captureInterfaceIndex: 7, + interfaceNames: map[uint32]string{ + 7: "br-lan", + }, + } + sample := make([]byte, testSKBMetadataSize()+4) + binary.LittleEndian.PutUint32(sample[4:8], packetHost) + binary.LittleEndian.PutUint32(sample[24:28], 7) + if err := source.emitPerfSample(sample, func(*event.Event) error { + t.Fatalf("did not expect ingress packet to be emitted") + return nil + }); err != nil { + t.Fatalf("emit perf sample: %v", err) + } + if source.seenCount != 0 { + t.Fatalf("expected filtered packet not to advance sampling pool, got %d", source.seenCount) + } +} + +func TestEmitPerfSampleOnAnyEmitsFirstSeenInterfaceInit(t *testing.T) { + source := &Source{ + cfg: config.SourceConfig{ + Network: "ebpf", + Interface: "any", + Type: "bytes", + SampleEvery: 1, + EBPF: config.EBPFConfig{Direction: "both"}, + }, + agentIP: "192.0.2.10", + captureAny: true, + interfaceFilter: regexp.MustCompile(`^wan`), + initializedInterfaces: make(map[uint32]struct{}), + interfaceNames: map[uint32]string{ + 9: "wan0", + }, + } + sample := make([]byte, testSKBMetadataSize()+4) + binary.LittleEndian.PutUint32(sample[4:8], packetOutgoing) + binary.LittleEndian.PutUint32(sample[28:32], 9) + var events []*event.Event + if err := source.emitPerfSample(sample, func(evt *event.Event) error { + events = append(events, evt) + return nil + }); err != nil { + t.Fatalf("emit perf sample: %v", err) + } + if len(events) != 2 { + t.Fatalf("expected source_init and packet events, got %d", len(events)) + } + if events[0].Control == nil || events[0].Control.Type != "source_init" { + t.Fatalf("expected first event to be source_init, got %#v", events[0]) + } + if events[0].Source.CaptureInterface != "wan0" || events[0].Source.CaptureInterfaceIndex != 9 { + t.Fatalf("unexpected source_init metadata: %#v", events[0].Source) + } + if events[0].Source.SourceID != 9 || !events[0].Source.SourceIDSet { + t.Fatalf("expected source_init source_id from interface index, got %#v", events[0].Source) + } + if events[1].Source.CaptureInterface != "wan0" || events[1].Source.CaptureInterfaceIndex != 9 { + t.Fatalf("unexpected packet source metadata: %#v", events[1].Source) + } + if events[1].Source.SourceID != 9 || !events[1].Source.SourceIDSet { + t.Fatalf("expected packet source_id from interface index, got %#v", events[1].Source) + } + if !source.interfaceInitialized(9) { + t.Fatalf("expected interface 9 to be marked initialized") + } +} + +func TestEmitPerfSampleOnAnyAppliesInterfaceFilterBeforeSampling(t *testing.T) { + source := &Source{ + cfg: config.SourceConfig{ + Network: "ebpf", + Interface: "any", + Type: "bytes", + SampleEvery: 1, + EBPF: config.EBPFConfig{Direction: "both"}, + }, + agentIP: "192.0.2.10", + captureAny: true, + interfaceFilter: regexp.MustCompile(`^wan`), + interfaceNames: map[uint32]string{ + 9: "lan0", + }, + } + sample := make([]byte, testSKBMetadataSize()+4) + binary.LittleEndian.PutUint32(sample[4:8], packetOutgoing) + binary.LittleEndian.PutUint32(sample[28:32], 9) + if err := source.emitPerfSample(sample, func(*event.Event) error { + t.Fatalf("did not expect filtered interface packet to be emitted") + return nil + }); err != nil { + t.Fatalf("emit perf sample: %v", err) + } + if source.seenCount != 0 { + t.Fatalf("expected filtered packet not to advance sampling pool, got %d", source.seenCount) + } +} + +func TestParseCPUList(t *testing.T) { + got := parseCPUList("0-2,4,6-7") + want := []int{0, 1, 2, 4, 6, 7} + if len(got) != len(want) { + t.Fatalf("expected %v, got %v", want, got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("expected %v, got %v", want, got) + } + } + if got := parseCPUList("2-1"); got != nil { + t.Fatalf("expected invalid range to return nil, got %v", got) + } +} + +func testSKBMetadataSize() int { + return 44 +} + +func TestConntrackLineExtractsNATMetadata(t *testing.T) { + line := "ipv4 2 tcp 6 431999 ESTABLISHED src=192.168.1.10 dst=198.51.100.20 sport=12345 dport=443 packets=10 bytes=1000 src=198.51.100.20 dst=203.0.113.9 sport=443 dport=54321 packets=8 bytes=900 [ASSURED] mark=0 use=1" + meta, ok := parseConntrackLine(line) + if !ok { + t.Fatalf("expected conntrack line to parse") + } + if !meta.hasSNAT { + t.Fatalf("expected SNAT metadata: %#v", meta) + } + if meta.natSrc.String() != "203.0.113.9" || meta.natSPort != 54321 { + t.Fatalf("unexpected SNAT endpoint %s:%d", meta.natSrc, meta.natSPort) + } + if meta.hasDNAT { + t.Fatalf("did not expect DNAT metadata: %#v", meta) + } + if meta.state != "established" || meta.status != "assured" { + t.Fatalf("unexpected state/status: %q/%q", meta.state, meta.status) + } +} + +func TestPacketConntrackTupleParsesIPv4TCP(t *testing.T) { + frame := testIPv4TCPFrame("192.168.1.10", "198.51.100.20", 12345, 443) + tuple, ok := packetConntrackTuple(frame) + if !ok { + t.Fatalf("expected packet tuple") + } + if tuple.family != "ipv4" || tuple.proto != "tcp" { + t.Fatalf("unexpected tuple family/proto: %#v", tuple) + } + if tuple.src.String() != "192.168.1.10" || tuple.dst.String() != "198.51.100.20" { + t.Fatalf("unexpected tuple addresses: %#v", tuple) + } + if tuple.srcPort != 12345 || tuple.dstPort != 443 { + t.Fatalf("unexpected tuple ports: %#v", tuple) + } +} + +func TestPacketEventAddsConntrackFields(t *testing.T) { + source := &Source{ + cfg: config.SourceConfig{Network: "ebpf", Interface: "br-lan", Type: "bytes", SampleEvery: 1}, + agentIP: "192.0.2.10", + captureInterfaceIndex: 7, + seenCount: 1, + } + meta := packetMetadata{ + packetType: "host", + direction: "in", + hasConntrack: true, + conntrack: conntrackMetadata{ + direction: "original", + family: "ipv4", + proto: "tcp", + state: "established", + status: "assured", + original: conntrackTuple{ + family: "ipv4", + proto: "tcp", + src: netip.MustParseAddr("192.168.1.10"), + dst: netip.MustParseAddr("198.51.100.20"), + srcPort: 12345, + dstPort: 443, + }, + reply: conntrackTuple{ + family: "ipv4", + proto: "tcp", + src: netip.MustParseAddr("198.51.100.20"), + dst: netip.MustParseAddr("203.0.113.9"), + srcPort: 443, + dstPort: 54321, + }, + hasSNAT: true, + natSrc: netip.MustParseAddr("203.0.113.9"), + natSPort: 54321, + }, + } + + evt := source.packetEvent([]byte{0, 1, 2, 3}, meta) + if got := evt.Fields["conntrack_state"]; got != "established" { + t.Fatalf("expected conntrack_state=established, got %#v", got) + } + if got := evt.Fields["conntrack_status"]; got != "assured" { + t.Fatalf("expected conntrack_status=assured, got %#v", got) + } + if _, ok := evt.Fields["nat_src_addr"]; ok { + t.Fatalf("expected source event not to derive nat_src_addr, got %#v", evt.Fields["nat_src_addr"]) + } + if _, ok := evt.Fields["nat_src_port"]; ok { + t.Fatalf("expected source event not to derive nat_src_port, got %#v", evt.Fields["nat_src_port"]) + } + for _, key := range []string{"conntrack_reply_src_addr", "conntrack_reply_dst_addr", "conntrack_reply_src_port", "conntrack_reply_dst_port"} { + if _, ok := evt.Fields[key]; ok { + t.Fatalf("expected source event not to export %s, got %#v", key, evt.Fields[key]) + } + } + if got := evt.Internal["conntrack_reply_dst_addr"]; got != "203.0.113.9" { + t.Fatalf("expected internal conntrack_reply_dst_addr, got %#v", got) + } + if got := evt.Internal["conntrack_reply_dst_port"]; got != uint32(54321) { + t.Fatalf("expected internal conntrack_reply_dst_port=54321, got %#v", got) + } +} + +func testIPv4TCPFrame(src, dst string, sport, dport uint16) []byte { + frame := make([]byte, 14+20+20) + frame[12] = 0x08 + frame[13] = 0x00 + ip := frame[14:] + ip[0] = 0x45 + ip[9] = 6 + srcAddr := netip.MustParseAddr(src).As4() + dstAddr := netip.MustParseAddr(dst).As4() + copy(ip[12:16], srcAddr[:]) + copy(ip[16:20], dstAddr[:]) + tcp := ip[20:] + tcp[0] = byte(sport >> 8) + tcp[1] = byte(sport) + tcp[2] = byte(dport >> 8) + tcp[3] = byte(dport) + return frame +} + +func TestPacketMetadataMarksIncomingCapture(t *testing.T) { + source := &Source{ + cfg: config.SourceConfig{Network: "ebpf", Interface: "br-lan", Type: "bytes", SampleEvery: 1}, + agentIP: "192.0.2.10", + captureInterfaceIndex: 7, + seenCount: 1, + } + + meta := source.packetMetadata(&unix.SockaddrLinklayer{ + Ifindex: 7, + Pkttype: packetHost, + }) + evt := source.packetEvent([]byte{0, 1, 2, 3}, meta) + + if evt.Source.CaptureDirection != "in" { + t.Fatalf("expected source capture direction in, got %q", evt.Source.CaptureDirection) + } + if evt.Source.CapturePacketType != "host" { + t.Fatalf("expected packet type host, got %q", evt.Source.CapturePacketType) + } + if got := evt.Fields["input_if"]; got != uint32(7) { + t.Fatalf("expected input_if=7, got %#v", got) + } + if _, ok := evt.Fields["output_if"]; ok { + t.Fatalf("expected output_if to be unset for incoming packet, got %#v", evt.Fields["output_if"]) + } + if got := evt.Fields["input_interface"]; got != "br-lan" { + t.Fatalf("expected input_interface=br-lan, got %#v", got) + } + if _, ok := evt.Fields["src_interface"]; ok { + t.Fatalf("expected src_interface alias to be unset, got %#v", evt.Fields["src_interface"]) + } +} + +func TestAllowDirectionFiltersIngressAndEgress(t *testing.T) { + tests := []struct { + filter string + direction string + want bool + }{ + {filter: "both", direction: "in", want: true}, + {filter: "both", direction: "out", want: true}, + {filter: "both", direction: "loopback", want: true}, + {filter: "ingress", direction: "in", want: true}, + {filter: "ingress", direction: "out", want: false}, + {filter: "egress", direction: "out", want: true}, + {filter: "egress", direction: "in", want: false}, + {filter: "egress", direction: "loopback", want: false}, + } + for _, tt := range tests { + if got := allowDirection(tt.filter, tt.direction); got != tt.want { + t.Fatalf("allowDirection(%q, %q)=%t, want %t", tt.filter, tt.direction, got, tt.want) + } + } +} + +func TestPacketMetadataAddsSKBFields(t *testing.T) { + source := &Source{ + cfg: config.SourceConfig{Network: "ebpf", Interface: "br-lan", Type: "bytes", SampleEvery: 1}, + agentIP: "192.0.2.10", + captureInterfaceIndex: 7, + interfaceNames: map[uint32]string{ + 7: "br-lan", + 9: "wan", + }, + seenCount: 1, + } + + meta := source.packetMetadata(&unix.SockaddrLinklayer{ + Ifindex: 7, + Pkttype: packetOutgoing, + }) + meta = source.mergeSKBMetadata(meta, skbMetadata{ + Len: 1514, + Mark: 42, + QueueMapping: 3, + Protocol: 0x0800, + Priority: 5, + IngressIfindex: 7, + Ifindex: 9, + TCIndex: 11, + Hash: 0x12345678, + TCClassID: 0x10002, + }) + evt := source.packetEvent([]byte{0, 1, 2, 3}, meta) + + if got := evt.Fields["skb_mark"]; got != uint32(42) { + t.Fatalf("expected skb_mark=42, got %#v", got) + } + if _, ok := evt.Fields["firewall_mark"]; ok { + t.Fatalf("expected firewall_mark alias to be unset, got %#v", evt.Fields["firewall_mark"]) + } + if got := evt.Fields["skb_len"]; got != uint32(1514) { + t.Fatalf("expected skb_len=1514, got %#v", got) + } + if got := evt.Fields["capture_length"]; got != 4 { + t.Fatalf("expected capture_length=4, got %#v", got) + } + if got := evt.Fields["wire_length"]; got != 1514 { + t.Fatalf("expected wire_length=1514, got %#v", got) + } + if got := evt.Fields["skb_hash"]; got != uint32(0x12345678) { + t.Fatalf("expected skb_hash=0x12345678, got %#v", got) + } + if got := evt.Fields["skb_ingress_ifindex"]; got != uint32(7) { + t.Fatalf("expected skb_ingress_ifindex=7, got %#v", got) + } + if got := evt.Fields["skb_ifindex"]; got != uint32(9) { + t.Fatalf("expected skb_ifindex=9, got %#v", got) + } + for _, key := range []string{"route_input_if", "route_output_if", "dst_interface"} { + if _, ok := evt.Fields[key]; ok { + t.Fatalf("expected %s alias to be unset, got %#v", key, evt.Fields[key]) + } + } + if got := evt.Fields["output_interface"]; got != "wan" { + t.Fatalf("expected output_interface=wan, got %#v", got) + } +} diff --git a/pkg/reflow/source/ebpf/source_unsupported.go b/pkg/reflow/source/ebpf/source_unsupported.go new file mode 100644 index 00000000..d2e038bb --- /dev/null +++ b/pkg/reflow/source/ebpf/source_unsupported.go @@ -0,0 +1,30 @@ +//go:build !linux || reflow_noebpf + +package ebpf + +import ( + "context" + "fmt" + "runtime" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func New(cfg config.SourceConfig) (*Source, error) { + if runtime.GOOS == "linux" { + return nil, fmt.Errorf("source.network=%s is disabled in this build; rebuild without -tags reflow_noebpf", cfg.Network) + } + return nil, fmt.Errorf("source.network=%s requires linux, got %s", cfg.Network, runtime.GOOS) +} + +func (s *Source) Start(context.Context, func(*event.Event) error) error { + if runtime.GOOS == "linux" { + return fmt.Errorf("source.network=ebpf is disabled in this build; rebuild without -tags reflow_noebpf") + } + return fmt.Errorf("source.network=ebpf requires linux, got %s", runtime.GOOS) +} + +func (s *Source) Close() error { + return nil +} diff --git a/pkg/reflow/source/pcap/pcap.go b/pkg/reflow/source/pcap/pcap.go new file mode 100644 index 00000000..c69d8288 --- /dev/null +++ b/pkg/reflow/source/pcap/pcap.go @@ -0,0 +1,608 @@ +//go:build !reflow_nopcap && cgo + +package pcap + +import ( + "context" + "encoding/binary" + "fmt" + "net" + "regexp" + "sort" + "sync" + "time" + + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +const ( + anyInterfaceName = "any" + linkTypeLinuxSLL2 = 276 + linkTypeLinuxSLL2Truncated = 20 +) + +type Source struct { + cfg config.SourceConfig + agentIP string + handle *pcap.Handle + captureInterfaceIndex int + captureAny bool + linkType uint32 + interfaceFilter *regexp.Regexp + interfaceMu sync.Mutex + interfaceNames map[uint32]string + initializedInterfaces map[uint32]struct{} + wg sync.WaitGroup + seenCount uint64 +} + +type cookedMetadata struct { + linkType uint32 + linkTypeName string + headerLength int + packetType string + protocol uint16 + ifIndex uint32 + ifName string + inputIf uint32 + outputIf uint32 + inputName string + outputName string +} + +// New validates a live-capture source and precomputes interface metadata used +// in emitted events and source_init control records. +func New(cfg config.SourceConfig) (*Source, error) { + if cfg.Network != "pcap_live" { + return nil, fmt.Errorf("unsupported source.network %q", cfg.Network) + } + if cfg.Interface == "" { + return nil, fmt.Errorf("source.interface is required when source.network=pcap_live") + } + if cfg.Type == "" { + cfg.Type = "bytes" + } + if cfg.SnapLen <= 0 { + cfg.SnapLen = 65535 + } + if cfg.SampleEvery <= 0 { + cfg.SampleEvery = 1 + } + var interfaceFilter *regexp.Regexp + if cfg.InterfaceFilter != "" { + compiled, err := regexp.Compile(cfg.InterfaceFilter) + if err != nil { + return nil, fmt.Errorf("compile source.interface_filter: %w", err) + } + interfaceFilter = compiled + } + if cfg.Interface == anyInterfaceName { + agentIP := firstSystemIP() + return &Source{ + cfg: cfg, + agentIP: agentIP, + captureAny: true, + interfaceFilter: interfaceFilter, + interfaceNames: make(map[uint32]string), + initializedInterfaces: make(map[uint32]struct{}), + }, nil + } + iface, err := net.InterfaceByName(cfg.Interface) + if err != nil { + return nil, fmt.Errorf("lookup capture interface %s: %w", cfg.Interface, err) + } + agentIP := firstInterfaceIP(iface) + return &Source{ + cfg: cfg, + agentIP: agentIP, + captureInterfaceIndex: iface.Index, + interfaceFilter: interfaceFilter, + interfaceNames: map[uint32]string{ + uint32(iface.Index): iface.Name, + }, + initializedInterfaces: make(map[uint32]struct{}), + }, nil +} + +// InitEvents emits a source_init control event so template-based encoders can +// learn source-scoped metadata before the first captured packet arrives. +func (s *Source) InitEvents() ([]*event.Event, error) { + if !s.captureAny { + iface := net.Interface{ + Index: s.captureInterfaceIndex, + Name: s.cfg.Interface, + } + s.markInterfaceInitialized(uint32(iface.Index), iface.Name) + return []*event.Event{s.sourceInitEvent(iface, s.agentIP, 0)}, nil + } + + ifaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("list capture interfaces: %w", err) + } + sort.Slice(ifaces, func(i, j int) bool { + return ifaces[i].Index < ifaces[j].Index + }) + events := make([]*event.Event, 0, len(ifaces)) + for _, iface := range ifaces { + if !s.interfaceAllowed(iface.Name) { + continue + } + agentIP := firstInterfaceIP(&iface) + s.markInterfaceInitialized(uint32(iface.Index), iface.Name) + events = append(events, s.sourceInitEvent(iface, agentIP, 0)) + } + return events, nil +} + +// Start reads packets from libpcap, applies packet-level sampling, and emits +// one raw bytes event per accepted capture. +func (s *Source) Start(ctx context.Context, emit func(*event.Event) error) error { + handle, err := pcap.OpenLive(s.cfg.Interface, int32(s.cfg.SnapLen), true, 500*time.Millisecond) + if err != nil { + return fmt.Errorf("open pcap device %s: %w", s.cfg.Interface, err) + } + s.handle = handle + s.linkType = s.normalizeHandleLinkType(handle.LinkType()) + + s.wg.Add(1) + go func() { + defer s.wg.Done() + <-ctx.Done() + if s.handle != nil { + s.handle.Close() + } + }() + + for { + data, ci, err := s.handle.ReadPacketData() + if err != nil { + switch err { + case pcap.NextErrorTimeoutExpired: + select { + case <-ctx.Done(): + return nil + default: + continue + } + case pcap.NextErrorNotActivated: + return fmt.Errorf("read pcap packet: %w", err) + default: + select { + case <-ctx.Done(): + return nil + default: + return fmt.Errorf("read pcap packet: %w", err) + } + } + } + payload, cooked := s.translatePacket(data) + if !s.cookedInterfaceAllowed(cooked) { + continue + } + // Drop counters come from the capture engine, not from packet contents. + drops := s.currentDropCount() + if s.captureAny && cooked.ifIndex != 0 && !s.interfaceInitialized(cooked.ifIndex) { + if evt := s.dynamicSourceInitEvent(cooked, drops); evt != nil { + if err := emit(evt); err != nil { + return err + } + } + } + s.seenCount++ + if !s.shouldEmitCurrentPacket() { + continue + } + sourceID := s.sourceIDForInterface(cooked.ifIndex) + + evt := &event.Event{ + ReceivedAt: time.Now().UTC(), + Source: event.SourceMetadata{ + Network: s.cfg.Network, + Address: s.cfg.Interface, + Type: s.cfg.Type, + CaptureInterface: s.eventCaptureInterface(cooked), + CaptureInterfaceIndex: s.eventCaptureInterfaceIndex(cooked), + CapturePacketType: cooked.packetType, + AgentIP: s.agentIP, + SourceID: sourceID, + SourceIDSet: true, + Sampling: &event.SamplingMetadata{ + Rate: uint32(s.cfg.SampleEvery), + SamplePool: uint32(s.seenCount), + Drops: drops, + }, + JSON: event.JSONMetadata{ + Flavor: s.cfg.JSON.Flavor, + }, + }, + Payload: payload, + Fields: map[string]any{ + "capture_length": ci.CaptureLength, + "wire_length": ci.Length, + }, + } + s.applyPcapMetadata(evt.Fields, cooked) + + if err := emit(evt); err != nil { + return err + } + } +} + +// currentDropCount asks libpcap for the current cumulative drop counters and +// folds both capture and interface drops into one exported value. +func (s *Source) currentDropCount() uint32 { + if s.handle == nil { + return 0 + } + stats, err := s.handle.Stats() + if err != nil || stats == nil { + return 0 + } + return uint32(stats.PacketsDropped + stats.PacketsIfDropped) +} + +// shouldEmitCurrentPacket applies deterministic packet sampling based on +// sample_every/sample_offset so replay and testing stay reproducible. +func (s *Source) shouldEmitCurrentPacket() bool { + if s.cfg.SampleEvery <= 1 { + return true + } + index := int((s.seenCount - 1) % uint64(s.cfg.SampleEvery)) + return index == s.cfg.SampleOffset +} + +// Close terminates the capture handle and waits for the shutdown goroutine. +func (s *Source) Close() error { + if s.handle != nil { + s.handle.Close() + } + s.wg.Wait() + return nil +} + +func (s *Source) sourceID() uint32 { + return s.sourceIDForInterface(0) +} + +func (s *Source) sourceIDForInterface(ifIndex uint32) uint32 { + if s.cfg.SourceID != nil { + return *s.cfg.SourceID + } + if s.captureAny && ifIndex != 0 { + return ifIndex + } + return uint32(s.captureInterfaceIndex) +} + +func (s *Source) sourceInitEvent(iface net.Interface, agentIP string, drops uint32) *event.Event { + ifIndex := uint32(iface.Index) + sourceID := s.sourceIDForInterface(ifIndex) + return &event.Event{ + ReceivedAt: time.Now().UTC(), + Kind: "control", + Source: event.SourceMetadata{ + Network: s.cfg.Network, + Address: s.cfg.Interface, + Type: s.cfg.Type, + CaptureInterface: iface.Name, + CaptureInterfaceIndex: iface.Index, + AgentIP: agentIP, + SourceID: sourceID, + SourceIDSet: true, + Sampling: &event.SamplingMetadata{ + Rate: uint32(s.cfg.SampleEvery), + SamplePool: 0, + Drops: drops, + }, + }, + Control: &event.ControlMetadata{ + Type: "source_init", + Stream: "options_data", + }, + Fields: map[string]any{ + "input_if": ifIndex, + "output_if": ifIndex, + }, + Payload: event.SourceInit{ + Stream: "options_data", + AgentIP: agentIP, + SourceID: sourceID, + SamplingRate: uint32(s.cfg.SampleEvery), + SamplePool: 0, + Drops: drops, + InputIf: ifIndex, + OutputIf: ifIndex, + }, + } +} + +func (s *Source) dynamicSourceInitEvent(cooked cookedMetadata, drops uint32) *event.Event { + name := cooked.ifName + if name == "" { + name = s.interfaceName(cooked.ifIndex) + } + if name == "" { + name = fmt.Sprintf("ifindex-%d", cooked.ifIndex) + } + if !s.interfaceAllowed(name) { + return nil + } + s.markInterfaceInitialized(cooked.ifIndex, name) + return s.sourceInitEvent(net.Interface{Index: int(cooked.ifIndex), Name: name}, s.agentIP, drops) +} + +func (s *Source) markInterfaceInitialized(ifIndex uint32, name string) { + if ifIndex == 0 { + return + } + s.interfaceMu.Lock() + defer s.interfaceMu.Unlock() + if s.interfaceNames == nil { + s.interfaceNames = make(map[uint32]string) + } + if name != "" { + s.interfaceNames[ifIndex] = name + } + if s.initializedInterfaces == nil { + s.initializedInterfaces = make(map[uint32]struct{}) + } + s.initializedInterfaces[ifIndex] = struct{}{} +} + +func (s *Source) interfaceInitialized(ifIndex uint32) bool { + if ifIndex == 0 { + return true + } + s.interfaceMu.Lock() + defer s.interfaceMu.Unlock() + _, ok := s.initializedInterfaces[ifIndex] + return ok +} + +func (s *Source) interfaceName(ifIndex uint32) string { + if ifIndex == 0 { + return "" + } + s.interfaceMu.Lock() + defer s.interfaceMu.Unlock() + if name := s.interfaceNames[ifIndex]; name != "" { + return name + } + iface, err := net.InterfaceByIndex(int(ifIndex)) + if err != nil || iface == nil { + return "" + } + if s.interfaceNames == nil { + s.interfaceNames = make(map[uint32]string) + } + s.interfaceNames[ifIndex] = iface.Name + return iface.Name +} + +func (s *Source) interfaceAllowed(name string) bool { + if s.interfaceFilter == nil { + return true + } + return name != "" && s.interfaceFilter.MatchString(name) +} + +func (s *Source) cookedInterfaceAllowed(cooked cookedMetadata) bool { + if s.interfaceFilter == nil || cooked.ifIndex == 0 { + return true + } + name := cooked.ifName + if name == "" { + name = s.interfaceName(cooked.ifIndex) + } + return s.interfaceAllowed(name) +} + +func (s *Source) eventCaptureInterface(cooked cookedMetadata) string { + if cooked.ifName != "" { + return cooked.ifName + } + if cooked.ifIndex != 0 { + if name := s.interfaceName(cooked.ifIndex); name != "" { + return name + } + } + return s.cfg.Interface +} + +func (s *Source) eventCaptureInterfaceIndex(cooked cookedMetadata) int { + if cooked.ifIndex != 0 { + return int(cooked.ifIndex) + } + return s.captureInterfaceIndex +} + +func (s *Source) normalizeHandleLinkType(linkType layers.LinkType) uint32 { + value := uint32(linkType) + if s.captureAny && value == linkTypeLinuxSLL2Truncated && pcap.DatalinkNameToVal("LINUX_SLL2") == linkTypeLinuxSLL2 { + return linkTypeLinuxSLL2 + } + return value +} + +func (s *Source) translatePacket(data []byte) ([]byte, cookedMetadata) { + cooked := cookedMetadata{ + linkType: s.linkType, + linkTypeName: pcapLinkTypeName(s.linkType), + } + switch s.linkType { + case uint32(layers.LinkTypeLinuxSLL): + return translateLinuxSLL(data, cooked) + case linkTypeLinuxSLL2: + return s.translateLinuxSLL2(data, cooked) + default: + return append([]byte(nil), data...), cooked + } +} + +func translateLinuxSLL(data []byte, cooked cookedMetadata) ([]byte, cookedMetadata) { + if len(data) < 16 { + return append([]byte(nil), data...), cooked + } + cooked.headerLength = 16 + cooked.packetType = linuxSLLPacketTypeName(uint8(binary.BigEndian.Uint16(data[0:2]))) + cooked.protocol = binary.BigEndian.Uint16(data[14:16]) + return synthesizeEthernetFromCooked(data[16:], cooked.protocol, data[6:14]), cooked +} + +func (s *Source) translateLinuxSLL2(data []byte, cooked cookedMetadata) ([]byte, cookedMetadata) { + if len(data) < 20 { + return append([]byte(nil), data...), cooked + } + cooked.headerLength = 20 + cooked.protocol = binary.BigEndian.Uint16(data[0:2]) + cooked.ifIndex = binary.BigEndian.Uint32(data[4:8]) + cooked.packetType = linuxSLLPacketTypeName(data[10]) + cooked.ifName = s.interfaceName(cooked.ifIndex) + switch cooked.packetType { + case "outgoing": + cooked.outputIf = cooked.ifIndex + cooked.outputName = cooked.ifName + case "loopback": + cooked.inputIf = cooked.ifIndex + cooked.outputIf = cooked.ifIndex + cooked.inputName = cooked.ifName + cooked.outputName = cooked.ifName + default: + cooked.inputIf = cooked.ifIndex + cooked.inputName = cooked.ifName + } + return synthesizeEthernetFromCooked(data[20:], cooked.protocol, data[12:20]), cooked +} + +func synthesizeEthernetFromCooked(payload []byte, protocol uint16, addr []byte) []byte { + out := make([]byte, 14+len(payload)) + copy(out[6:12], cookedHardwareAddr(addr)) + binary.BigEndian.PutUint16(out[12:14], protocol) + copy(out[14:], payload) + return out +} + +func cookedHardwareAddr(addr []byte) []byte { + hw := make([]byte, 6) + copy(hw, addr) + return hw +} + +func (s *Source) applyPcapMetadata(fields map[string]any, cooked cookedMetadata) { + if cooked.linkType != 0 { + fields["pcap_link_type"] = cooked.linkType + fields["pcap_link_type_name"] = cooked.linkTypeName + } + if cooked.headerLength != 0 { + fields["pcap_cooked_header_length"] = cooked.headerLength + fields["linux_sll_protocol"] = uint32(cooked.protocol) + fields["header_protocol"] = uint32(1) + fields["protocol"] = uint32(1) + } + if cooked.packetType != "" { + fields["capture_packet_type"] = cooked.packetType + } + if cooked.ifIndex != 0 { + fields["linux_sll_ifindex"] = cooked.ifIndex + } + if cooked.inputIf != 0 { + fields["input_if"] = cooked.inputIf + } + if cooked.outputIf != 0 { + fields["output_if"] = cooked.outputIf + } + if cooked.inputName != "" { + fields["input_interface"] = cooked.inputName + } + if cooked.outputName != "" { + fields["output_interface"] = cooked.outputName + } +} + +func pcapLinkTypeName(linkType uint32) string { + if linkType == linkTypeLinuxSLL2 { + return "Linux SLL2" + } + if linkType <= 255 { + return layers.LinkType(uint8(linkType)).String() + } + if name := pcap.DatalinkValToName(int(linkType)); name != "" { + return name + } + return fmt.Sprintf("LinkType(%d)", linkType) +} + +func linuxSLLPacketTypeName(packetType uint8) string { + switch packetType { + case 0: + return "host" + case 1: + return "broadcast" + case 2: + return "multicast" + case 3: + return "otherhost" + case 4: + return "outgoing" + case 5: + return "loopback" + case 6: + return "fastroute" + default: + return fmt.Sprintf("unknown(%d)", packetType) + } +} + +func firstSystemIP() string { + ifaces, err := net.Interfaces() + if err != nil { + return "127.0.0.1" + } + sort.Slice(ifaces, func(i, j int) bool { + return ifaces[i].Index < ifaces[j].Index + }) + for i := range ifaces { + if ip := firstInterfaceIP(&ifaces[i]); ip != "127.0.0.1" { + return ip + } + } + return "127.0.0.1" +} + +// firstInterfaceIP picks a stable agent IP for exported metadata, preferring +// non-loopback IPv4, then any non-loopback address, then localhost. +func firstInterfaceIP(iface *net.Interface) string { + addrs, err := iface.Addrs() + if err != nil { + return "127.0.0.1" + } + var fallback string + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + if ip == nil || ip.IsLoopback() { + continue + } + if ipv4 := ip.To4(); ipv4 != nil { + return ipv4.String() + } + if fallback == "" { + fallback = ip.String() + } + } + if fallback != "" { + return fallback + } + return "127.0.0.1" +} diff --git a/pkg/reflow/source/pcap/pcap_disabled.go b/pkg/reflow/source/pcap/pcap_disabled.go new file mode 100644 index 00000000..8bfabfd8 --- /dev/null +++ b/pkg/reflow/source/pcap/pcap_disabled.go @@ -0,0 +1,29 @@ +//go:build reflow_nopcap || !cgo + +package pcap + +import ( + "context" + "fmt" + "runtime" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type Source struct{} + +func New(cfg config.SourceConfig) (*Source, error) { + if cfg.Network != "pcap_live" { + return nil, fmt.Errorf("unsupported source.network %q", cfg.Network) + } + return nil, fmt.Errorf("source.network=pcap_live is disabled in this build; rebuild with CGO_ENABLED=1 and libpcap installed on %s", runtime.GOOS) +} + +func (s *Source) InitEvents() ([]*event.Event, error) { return nil, nil } + +func (s *Source) Start(context.Context, func(*event.Event) error) error { + return fmt.Errorf("source.network=pcap_live is disabled in this build") +} + +func (s *Source) Close() error { return nil } diff --git a/pkg/reflow/source/pcap/pcap_test.go b/pkg/reflow/source/pcap/pcap_test.go new file mode 100644 index 00000000..654961f3 --- /dev/null +++ b/pkg/reflow/source/pcap/pcap_test.go @@ -0,0 +1,151 @@ +//go:build !reflow_nopcap && cgo + +package pcap + +import ( + "encoding/binary" + "net" + "regexp" + "testing" + + "github.com/google/gopacket/layers" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func TestTranslateLinuxSLLSynthesizesEthernetPacket(t *testing.T) { + src := &Source{linkType: uint32(layers.LinkTypeLinuxSLL)} + inner := []byte{0x45, 0, 0, 20} + packet := make([]byte, 16+len(inner)) + binary.BigEndian.PutUint16(packet[0:2], 4) + binary.BigEndian.PutUint16(packet[14:16], 0x0800) + copy(packet[6:14], []byte{1, 2, 3, 4, 5, 6, 7, 8}) + copy(packet[16:], inner) + + payload, meta := src.translatePacket(packet) + if meta.headerLength != 16 { + t.Fatalf("expected SLL header length 16, got %d", meta.headerLength) + } + if meta.packetType != "outgoing" { + t.Fatalf("expected packet type outgoing, got %q", meta.packetType) + } + if meta.protocol != 0x0800 { + t.Fatalf("expected protocol 0x0800, got %#x", meta.protocol) + } + if len(payload) != 14+len(inner) { + t.Fatalf("expected synthesized ethernet length %d, got %d", 14+len(inner), len(payload)) + } + if got := payload[6:12]; string(got) != string([]byte{1, 2, 3, 4, 5, 6}) { + t.Fatalf("unexpected synthesized source MAC %v", got) + } + if got := binary.BigEndian.Uint16(payload[12:14]); got != 0x0800 { + t.Fatalf("expected synthesized ethertype 0x0800, got %#x", got) + } +} + +func TestTranslateLinuxSLL2ResolvesInterfaceMetadata(t *testing.T) { + src := &Source{ + linkType: linkTypeLinuxSLL2, + interfaceNames: map[uint32]string{ + 77: "wan0", + }, + } + inner := []byte{0x45, 0, 0, 20} + packet := make([]byte, 20+len(inner)) + binary.BigEndian.PutUint16(packet[0:2], 0x0800) + binary.BigEndian.PutUint32(packet[4:8], 77) + packet[10] = 0 + copy(packet[12:20], []byte{6, 5, 4, 3, 2, 1, 0, 0}) + copy(packet[20:], inner) + + payload, meta := src.translatePacket(packet) + if meta.headerLength != 20 { + t.Fatalf("expected SLL2 header length 20, got %d", meta.headerLength) + } + if meta.ifIndex != 77 || meta.inputIf != 77 || meta.inputName != "wan0" { + t.Fatalf("expected input interface metadata for wan0/77, got %#v", meta) + } + if meta.outputIf != 0 || meta.outputName != "" { + t.Fatalf("did not expect output interface metadata, got %#v", meta) + } + if got := binary.BigEndian.Uint16(payload[12:14]); got != 0x0800 { + t.Fatalf("expected synthesized ethertype 0x0800, got %#x", got) + } +} + +func TestDynamicSourceInitEventMarksFirstSeenInterface(t *testing.T) { + src := &Source{ + cfg: config.SourceConfig{ + Network: "pcap_live", + Interface: "any", + Type: "bytes", + SampleEvery: 1, + }, + agentIP: "192.0.2.1", + captureAny: true, + interfaceFilter: regexp.MustCompile(`^wan`), + initializedInterfaces: make(map[uint32]struct{}), + } + + evt := src.dynamicSourceInitEvent(cookedMetadata{ifIndex: 77, ifName: "wan0"}, 3) + if evt == nil { + t.Fatalf("expected source_init event") + } + if evt.Source.CaptureInterface != "wan0" || evt.Source.CaptureInterfaceIndex != 77 { + t.Fatalf("unexpected source metadata: %#v", evt.Source) + } + if evt.Fields["input_if"] != uint32(77) || evt.Fields["output_if"] != uint32(77) { + t.Fatalf("unexpected source fields: %#v", evt.Fields) + } + if !src.interfaceInitialized(77) { + t.Fatalf("expected interface 77 to be marked initialized") + } +} + +func TestAnySourceInitEventUsesInterfaceIndexAsDefaultSourceID(t *testing.T) { + src := &Source{ + cfg: config.SourceConfig{ + Network: "pcap_live", + Interface: "any", + Type: "bytes", + SampleEvery: 1, + }, + captureAny: true, + } + + evt := src.sourceInitEvent(net.Interface{Index: 77, Name: "wan0"}, "192.0.2.1", 0) + if evt.Source.SourceID != 77 || !evt.Source.SourceIDSet { + t.Fatalf("expected source_id from concrete interface index, got %#v", evt.Source) + } + payload, ok := evt.Payload.(event.SourceInit) + if !ok { + t.Fatalf("expected SourceInit payload, got %T", evt.Payload) + } + if payload.SourceID != 77 { + t.Fatalf("expected payload source_id=77, got %#v", payload) + } +} + +func TestAnySourceInitEventKeepsExplicitSourceID(t *testing.T) { + sourceID := uint32(1234) + src := &Source{ + cfg: config.SourceConfig{ + Network: "pcap_live", + Interface: "any", + Type: "bytes", + SampleEvery: 1, + SourceID: &sourceID, + }, + captureAny: true, + } + + evt := src.sourceInitEvent(net.Interface{Index: 77, Name: "wan0"}, "192.0.2.1", 0) + if evt.Source.SourceID != sourceID { + t.Fatalf("expected explicit source_id=%d, got %#v", sourceID, evt.Source) + } + payload := evt.Payload.(event.SourceInit) + if payload.SourceID != sourceID { + t.Fatalf("expected payload source_id=%d, got %#v", sourceID, payload) + } +} diff --git a/pkg/reflow/source/socket/socket.go b/pkg/reflow/source/socket/socket.go new file mode 100644 index 00000000..e5a5fe13 --- /dev/null +++ b/pkg/reflow/source/socket/socket.go @@ -0,0 +1,115 @@ +package socket + +import ( + "context" + "encoding/json" + "fmt" + "net" + "sync" + "time" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type Source struct { + cfg config.SourceConfig + conn net.PacketConn + wg sync.WaitGroup +} + +// New validates the socket-oriented source configuration before runtime start. +func New(cfg config.SourceConfig) (*Source, error) { + switch cfg.Network { + case "udp", "unixgram": + default: + return nil, fmt.Errorf("unsupported source.network %q", cfg.Network) + } + return &Source{cfg: cfg}, nil +} + +// InitEvents returns no control events because socket sources do not have +// source-side metadata that needs to be announced before the first packet. +func (s *Source) InitEvents() ([]*event.Event, error) { + return nil, nil +} + +// Start listens for packet-oriented input and turns each datagram into one raw +// source event for the decode stage. +func (s *Source) Start(ctx context.Context, emit func(*event.Event) error) error { + conn, err := net.ListenPacket(s.cfg.Network, s.cfg.Address) + if err != nil { + return fmt.Errorf("listen on %s %s: %w", s.cfg.Network, s.cfg.Address, err) + } + s.conn = conn + + s.wg.Add(1) + go func() { + defer s.wg.Done() + <-ctx.Done() + _ = s.conn.Close() + }() + + buf := make([]byte, 64*1024) + for { + n, remote, err := s.conn.ReadFrom(buf) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + return fmt.Errorf("read datagram: %w", err) + } + } + payload := append([]byte(nil), buf[:n]...) + evt := &event.Event{ + ReceivedAt: time.Now().UTC(), + Source: event.SourceMetadata{ + Network: s.cfg.Network, + Address: s.cfg.Address, + Type: s.cfg.Type, + JSON: event.JSONMetadata{ + Flavor: s.cfg.JSON.Flavor, + }, + }, + } + if remote != nil { + evt.Source.Remote = remote.String() + } + switch s.cfg.Type { + case "flow", "bytes": + // Binary-oriented source types keep the datagram in Payload so the + // decode stage can treat it as raw bytes. + evt.Payload = payload + default: + // Message-oriented inputs preserve the raw JSON body when valid and + // fall back to a quoted string so the event stays representable. + raw := json.RawMessage(payload) + if !json.Valid(raw) { + raw = json.RawMessage([]byte(strconvQuoteBytes(payload))) + } + evt.Message = raw + } + if err := emit(evt); err != nil { + return err + } + } +} + +// Close stops the listener and waits for the background context closer to exit. +func (s *Source) Close() error { + if s.conn != nil { + err := s.conn.Close() + s.wg.Wait() + return err + } + s.wg.Wait() + return nil +} + +// strconvQuoteBytes mirrors strconv.Quote semantics for arbitrary bytes by +// round-tripping through JSON string encoding. +func strconvQuoteBytes(b []byte) string { + quoted, _ := json.Marshal(string(b)) + return string(quoted) +} diff --git a/pkg/reflow/source/source.go b/pkg/reflow/source/source.go new file mode 100644 index 00000000..f76c1e05 --- /dev/null +++ b/pkg/reflow/source/source.go @@ -0,0 +1,35 @@ +package source + +import ( + "context" + "fmt" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" + "github.com/netsampler/goflow2/v3/pkg/reflow/source/ebpf" + "github.com/netsampler/goflow2/v3/pkg/reflow/source/pcap" + "github.com/netsampler/goflow2/v3/pkg/reflow/source/socket" + "github.com/netsampler/goflow2/v3/pkg/reflow/source/stream" +) + +type Source interface { + InitEvents() ([]*event.Event, error) + Start(context.Context, func(*event.Event) error) error + Close() error +} + +// New selects the source implementation from the source.network setting. +func New(cfg config.SourceConfig) (Source, error) { + switch cfg.Network { + case "udp", "unixgram": + return socket.New(cfg) + case "ebpf": + return ebpf.New(cfg) + case "pcap_live": + return pcap.New(cfg) + case "stream": + return stream.New(cfg) + default: + return nil, fmt.Errorf("unsupported source.network %q", cfg.Network) + } +} diff --git a/pkg/reflow/source/stream/stream.go b/pkg/reflow/source/stream/stream.go new file mode 100644 index 00000000..6cf3a9d5 --- /dev/null +++ b/pkg/reflow/source/stream/stream.go @@ -0,0 +1,278 @@ +package stream + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +type packetReader interface { + ReadPacketData() ([]byte, gopacket.CaptureInfo, error) +} + +type Source struct { + cfg config.SourceConfig + closer io.Closer +} + +func New(cfg config.SourceConfig) (*Source, error) { + if cfg.Network != "stream" { + return nil, fmt.Errorf("unsupported source.network %q", cfg.Network) + } + switch cfg.Type { + case "pcap", "pcapng", "json": + default: + return nil, fmt.Errorf("unsupported stream source type %q", cfg.Type) + } + return &Source{cfg: cfg}, nil +} + +func (s *Source) InitEvents() ([]*event.Event, error) { + now := time.Now().UTC() + name := streamInterfaceName(s.cfg.Address) + sourceMeta := event.SourceMetadata{ + Network: s.cfg.Network, + Address: s.cfg.Address, + Type: s.cfg.Type, + CaptureInterface: name, + } + payload := event.SourceInit{ + Stream: s.cfg.Type, + } + if s.cfg.SourceID != nil { + sourceMeta.SourceID = *s.cfg.SourceID + sourceMeta.SourceIDSet = true + payload.SourceID = *s.cfg.SourceID + } + return []*event.Event{ + { + ReceivedAt: now, + Kind: "control", + Source: sourceMeta, + Control: &event.ControlMetadata{ + Type: "source_init", + Stream: s.cfg.Type, + }, + Fields: map[string]any{ + "stream_type": s.cfg.Type, + }, + Payload: payload, + }, + }, nil +} + +func (s *Source) Start(ctx context.Context, emit func(*event.Event) error) error { + r, closer, err := s.open() + if err != nil { + return err + } + s.closer = closer + + switch s.cfg.Type { + case "pcap": + reader, err := pcapgo.NewReader(r) + if err != nil { + return fmt.Errorf("open pcap stream: %w", err) + } + return s.readPackets(ctx, emit, reader, reader.LinkType()) + case "pcapng": + reader, err := pcapgo.NewNgReader(r, pcapgo.NgReaderOptions{ + WantMixedLinkType: true, + }) + if err != nil { + return fmt.Errorf("open pcapng stream: %w", err) + } + return s.readPackets(ctx, emit, reader, 0) + case "json": + return s.readNDJSON(ctx, emit, r) + default: + return fmt.Errorf("unsupported stream source type %q", s.cfg.Type) + } +} + +func (s *Source) open() (io.Reader, io.Closer, error) { + if s.cfg.Address == "-" { + return os.Stdin, os.Stdin, nil + } + f, err := os.Open(s.cfg.Address) + if err != nil { + return nil, nil, fmt.Errorf("open stream source %s: %w", s.cfg.Address, err) + } + return f, f, nil +} + +func (s *Source) readPackets(ctx context.Context, emit func(*event.Event) error, reader packetReader, defaultLink layers.LinkType) error { + for { + data, ci, err := reader.ReadPacketData() + if err != nil { + if err == io.EOF || ctx.Err() != nil { + return nil + } + return fmt.Errorf("read %s packet: %w", s.cfg.Type, err) + } + + linkType := defaultLink + if len(ci.AncillaryData) > 0 { + if typed, ok := ci.AncillaryData[0].(layers.LinkType); ok { + linkType = typed + } + } + receivedAt := ci.Timestamp.UTC() + if receivedAt.IsZero() { + receivedAt = time.Now().UTC() + } + + fields := map[string]any{ + "capture_length": ci.CaptureLength, + "wire_length": ci.Length, + "pcap_link_type": uint32(linkType), + "pcap_link_type_name": linkType.String(), + "stream_type": s.cfg.Type, + } + if protocol := sampledHeaderProtocol(linkType, data); protocol != 0 { + fields["header_protocol"] = protocol + fields["protocol"] = protocol + } + + evt := &event.Event{ + ReceivedAt: receivedAt, + Source: s.sourceMetadata("bytes"), + Payload: append([]byte(nil), data...), + Fields: fields, + } + if err := emit(evt); err != nil { + return err + } + } +} + +func (s *Source) sourceMetadata(sourceType string) event.SourceMetadata { + meta := event.SourceMetadata{ + Network: s.cfg.Network, + Address: s.cfg.Address, + Type: sourceType, + } + if s.cfg.SourceID != nil { + meta.SourceID = *s.cfg.SourceID + meta.SourceIDSet = true + } + return meta +} + +func (s *Source) readNDJSON(ctx context.Context, emit func(*event.Event) error, r io.Reader) error { + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 64*1024), 16*1024*1024) + type scanResult struct { + line []byte + err error + } + lines := make(chan scanResult) + go func() { + defer close(lines) + for scanner.Scan() { + line := append([]byte(nil), scanner.Bytes()...) + select { + case lines <- scanResult{line: line}: + case <-ctx.Done(): + return + } + } + if err := scanner.Err(); err != nil { + select { + case lines <- scanResult{err: err}: + case <-ctx.Done(): + } + } + }() + + for { + var line []byte + select { + case <-ctx.Done(): + return nil + case result, ok := <-lines: + if !ok { + return nil + } + if result.err != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("read ndjson stream: %w", result.err) + } + line = result.line + } + if len(line) == 0 { + continue + } + raw := json.RawMessage(line) + if !json.Valid(raw) { + return fmt.Errorf("decode ndjson stream: invalid JSON") + } + evt := &event.Event{ + ReceivedAt: time.Now().UTC(), + Source: func() event.SourceMetadata { + meta := s.sourceMetadata("json") + meta.JSON = event.JSONMetadata{ + Flavor: s.cfg.JSON.Flavor, + } + return meta + }(), + Message: raw, + } + if err := emit(evt); err != nil { + return err + } + } +} +func (s *Source) Close() error { + if s.closer != nil { + return s.closer.Close() + } + return nil +} + +func sampledHeaderProtocol(linkType layers.LinkType, data []byte) uint32 { + switch linkType { + case layers.LinkTypeEthernet: + return 1 + case layers.LinkTypeIPv4: + return 11 + case layers.LinkTypeIPv6: + return 12 + case layers.LinkTypeRaw: + if len(data) == 0 { + return 0 + } + switch data[0] >> 4 { + case 4: + return 11 + case 6: + return 12 + } + } + return 0 +} + +func streamInterfaceName(address string) string { + if address == "" || address == "-" { + return "stdin" + } + base := filepath.Base(address) + if base == "." || base == string(filepath.Separator) { + return address + } + return base +} diff --git a/pkg/reflow/source/stream/stream_test.go b/pkg/reflow/source/stream/stream_test.go new file mode 100644 index 00000000..116a8762 --- /dev/null +++ b/pkg/reflow/source/stream/stream_test.go @@ -0,0 +1,229 @@ +package stream + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + + "github.com/netsampler/goflow2/v3/pkg/reflow/config" + "github.com/netsampler/goflow2/v3/pkg/reflow/event" +) + +func TestSourceReadsPcapStream(t *testing.T) { + path := writePcapFile(t, false) + + src, err := New(config.SourceConfig{ + Network: "stream", + Address: path, + Type: "pcap", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + events := collectEvents(t, src) + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + evt := events[0] + if evt.Source.Type != "bytes" { + t.Fatalf("expected source type bytes, got %q", evt.Source.Type) + } + if evt.Fields["stream_type"] != "pcap" { + t.Fatalf("expected stream_type pcap, got %#v", evt.Fields["stream_type"]) + } + if evt.Fields["protocol"] != uint32(1) { + t.Fatalf("expected ethernet protocol metadata, got %#v", evt.Fields["protocol"]) + } + if evt.Fields["wire_length"] != 60 { + t.Fatalf("expected wire_length 60, got %#v", evt.Fields["wire_length"]) + } + if payload, ok := evt.Payload.([]byte); !ok || len(payload) != 34 { + t.Fatalf("expected 34 payload bytes, got %T len=%d", evt.Payload, len(payload)) + } + if !evt.ReceivedAt.Equal(time.Unix(42, 123).UTC()) { + t.Fatalf("expected pcap timestamp, got %s", evt.ReceivedAt) + } +} + +func TestSourceInitEventsExposeStreamInterfaceName(t *testing.T) { + src, err := New(config.SourceConfig{ + Network: "stream", + Address: "/tmp/capture.pipe", + Type: "pcap", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + events, err := src.InitEvents() + if err != nil { + t.Fatalf("InitEvents returned error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected one init event, got %d", len(events)) + } + if events[0].Control == nil || events[0].Control.Type != "source_init" { + t.Fatalf("expected source_init control event, got %#v", events[0].Control) + } + if events[0].Source.CaptureInterface != "capture.pipe" { + t.Fatalf("expected capture interface capture.pipe, got %q", events[0].Source.CaptureInterface) + } +} + +func TestSourceReadsPcapngStream(t *testing.T) { + path := writePcapFile(t, true) + + src, err := New(config.SourceConfig{ + Network: "stream", + Address: path, + Type: "pcapng", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + events := collectEvents(t, src) + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + if events[0].Fields["stream_type"] != "pcapng" { + t.Fatalf("expected stream_type pcapng, got %#v", events[0].Fields["stream_type"]) + } + if events[0].Fields["pcap_link_type"] != uint32(layers.LinkTypeEthernet) { + t.Fatalf("expected ethernet link type, got %#v", events[0].Fields["pcap_link_type"]) + } +} + +func TestSourceReadsNDJSON(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "events.ndjson") + if err := os.WriteFile(path, []byte("{\"src_addr\":\"192.0.2.1\"}\n{\"dst_addr\":\"198.51.100.2\"}\n"), 0o644); err != nil { + t.Fatalf("write ndjson: %v", err) + } + + src, err := New(config.SourceConfig{ + Network: "stream", + Address: path, + Type: "json", + JSON: config.JSONConfig{Flavor: "reflow"}, + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + events := collectEvents(t, src) + if len(events) != 2 { + t.Fatalf("expected 2 events, got %d", len(events)) + } + if events[0].Source.Type != "json" || events[0].Source.JSON.Flavor != "reflow" { + t.Fatalf("expected reflow JSON source metadata, got %#v", events[0].Source) + } + if !bytes.Contains(events[0].Message, []byte("src_addr")) { + t.Fatalf("expected first message to contain src_addr, got %s", events[0].Message) + } +} + +func TestSourceReadNDJSONReturnsOnContextCancelWhenReaderBlocks(t *testing.T) { + src := &Source{} + reader, writer := io.Pipe() + defer func() { + if err := reader.Close(); err != nil { + t.Errorf("reader close: %v", err) + } + }() + defer func() { + if err := writer.Close(); err != nil { + t.Errorf("writer close: %v", err) + } + }() + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + go func() { + done <- src.readNDJSON(ctx, func(*event.Event) error { + return nil + }, reader) + }() + + cancel() + + select { + case err := <-done: + if err != nil { + t.Fatalf("readNDJSON returned error: %v", err) + } + case <-time.After(time.Second): + t.Fatalf("readNDJSON did not return after context cancellation") + } +} + +func collectEvents(t *testing.T, src *Source) []*event.Event { + t.Helper() + var events []*event.Event + if err := src.Start(context.Background(), func(evt *event.Event) error { + events = append(events, evt) + return nil + }); err != nil { + t.Fatalf("Start returned error: %v", err) + } + if err := src.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + return events +} + +func writePcapFile(t *testing.T, pcapng bool) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "capture") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create capture: %v", err) + } + defer func() { + if err := f.Close(); err != nil { + t.Errorf("close capture: %v", err) + } + }() + + ci := gopacket.CaptureInfo{ + Timestamp: time.Unix(42, 123).UTC(), + CaptureLength: 34, + Length: 60, + } + data := make([]byte, 34) + data[12] = 0x08 + data[13] = 0x00 + data[14] = 0x45 + + if pcapng { + w, err := pcapgo.NewNgWriter(f, layers.LinkTypeEthernet) + if err != nil { + t.Fatalf("NewNgWriter: %v", err) + } + if err := w.WritePacket(ci, data); err != nil { + t.Fatalf("WritePacket pcapng: %v", err) + } + if err := w.Flush(); err != nil { + t.Fatalf("Flush pcapng: %v", err) + } + return path + } + + w := pcapgo.NewWriterNanos(f) + if err := w.WriteFileHeader(65535, layers.LinkTypeEthernet); err != nil { + t.Fatalf("WriteFileHeader: %v", err) + } + if err := w.WritePacket(ci, data); err != nil { + t.Fatalf("WritePacket pcap: %v", err) + } + return path +} diff --git a/producer/proto/ipfix_compare_bench_test.go b/producer/proto/ipfix_compare_bench_test.go new file mode 100644 index 00000000..e7c2ef8e --- /dev/null +++ b/producer/proto/ipfix_compare_bench_test.go @@ -0,0 +1,254 @@ +package protoproducer + +import ( + "bytes" + "net/netip" + "testing" + "time" + + "github.com/netsampler/goflow2/v3/decoders/netflow" + "github.com/netsampler/goflow2/v3/format" + _ "github.com/netsampler/goflow2/v3/format/binary" + _ "github.com/netsampler/goflow2/v3/format/json" + "github.com/netsampler/goflow2/v3/producer" + "github.com/netsampler/goflow2/v3/utils/store/templates" +) + +func BenchmarkOriginalIPFIXToFormat(b *testing.B) { + cfg, err := (&ProducerConfig{}).Compile() + if err != nil { + b.Fatalf("Compile returned error: %v", err) + } + prod, err := CreateProtoProducer(cfg, nil) + if err != nil { + b.Fatalf("CreateProtoProducer returned error: %v", err) + } + defer prod.Close() + + args := &producer.ProduceArgs{ + Src: netip.MustParseAddrPort("192.0.2.1:4739"), + Dst: netip.MustParseAddrPort("192.0.2.2:4739"), + SamplerAddress: netip.MustParseAddr("192.0.2.1"), + TimeReceived: time.Unix(1_700_000_001, 0).UTC(), + FlowContext: &netflow.FlowContext{ + RouterKey: "192.0.2.1", + }, + } + packet := benchOriginalIPFIXPacket() + + benchmarks := []struct { + name string + format string + }{ + {name: "protobuf_payload_for_kafka", format: "bin"}, + {name: "json", format: "json"}, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + formatter, err := format.FindFormat(bm.format) + if err != nil { + b.Fatalf("FindFormat returned error: %v", err) + } + var totalBytes int + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + messages, err := prod.Produce(packet, args) + if err != nil { + b.Fatalf("Produce returned error: %v", err) + } + if len(messages) != 1 { + b.Fatalf("expected 1 message, got %d", len(messages)) + } + _, payload, err := formatter.Format(messages[0]) + if err != nil { + b.Fatalf("Format returned error: %v", err) + } + totalBytes += len(payload) + prod.Commit(messages) + } + if totalBytes == 0 { + b.Fatalf("expected formatted payload bytes") + } + }) + } +} + +func BenchmarkOriginalIPFIXProduceOnly(b *testing.B) { + cfg, err := (&ProducerConfig{}).Compile() + if err != nil { + b.Fatalf("Compile returned error: %v", err) + } + prod, err := CreateProtoProducer(cfg, nil) + if err != nil { + b.Fatalf("CreateProtoProducer returned error: %v", err) + } + defer prod.Close() + + args := &producer.ProduceArgs{ + Src: netip.MustParseAddrPort("192.0.2.1:4739"), + SamplerAddress: netip.MustParseAddr("192.0.2.1"), + TimeReceived: time.Unix(1_700_000_001, 0).UTC(), + FlowContext: &netflow.FlowContext{RouterKey: "192.0.2.1"}, + } + packet := benchOriginalIPFIXPacket() + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + messages, err := prod.Produce(packet, args) + if err != nil { + b.Fatalf("Produce returned error: %v", err) + } + if len(messages) != 1 { + b.Fatalf("expected 1 message, got %d", len(messages)) + } + prod.Commit(messages) + } +} + +func BenchmarkOriginalRawIPFIXToFormat(b *testing.B) { + cfg, err := (&ProducerConfig{}).Compile() + if err != nil { + b.Fatalf("Compile returned error: %v", err) + } + prod, err := CreateProtoProducer(cfg, nil) + if err != nil { + b.Fatalf("CreateProtoProducer returned error: %v", err) + } + defer prod.Close() + + ctx := netflow.FlowContext{RouterKey: "192.0.2.1"} + store := templates.NewTemplateFlowStore() + store.Start() + defer store.Close() + templatePayload, dataPayload := benchOriginalIPFIXPayloads(b) + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(templatePayload), store, ctx, nil, &netflow.IPFIXPacket{}); err != nil { + b.Fatalf("decode template payload: %v", err) + } + + args := &producer.ProduceArgs{ + Src: netip.MustParseAddrPort("192.0.2.1:4739"), + SamplerAddress: netip.MustParseAddr("192.0.2.1"), + TimeReceived: time.Unix(1_700_000_001, 0).UTC(), + FlowContext: &ctx, + } + + benchmarks := []struct { + name string + format string + }{ + {name: "protobuf_payload_for_kafka", format: "bin"}, + {name: "json", format: "json"}, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + formatter, err := format.FindFormat(bm.format) + if err != nil { + b.Fatalf("FindFormat returned error: %v", err) + } + var totalBytes int + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + packet := &netflow.IPFIXPacket{} + if err := netflow.DecodeMessageVersion(bytes.NewBuffer(dataPayload), store, ctx, nil, packet); err != nil { + b.Fatalf("DecodeMessageVersion returned error: %v", err) + } + messages, err := prod.Produce(packet, args) + if err != nil { + b.Fatalf("Produce returned error: %v", err) + } + if len(messages) != 1 { + b.Fatalf("expected 1 message, got %d", len(messages)) + } + _, payload, err := formatter.Format(messages[0]) + if err != nil { + b.Fatalf("Format returned error: %v", err) + } + totalBytes += len(payload) + prod.Commit(messages) + } + if totalBytes == 0 { + b.Fatalf("expected formatted payload bytes") + } + }) + } +} + +func benchOriginalIPFIXPacket() *netflow.IPFIXPacket { + return &netflow.IPFIXPacket{ + Version: 10, + ExportTime: uint32(time.Unix(1_700_000_001, 0).Unix()), + SequenceNumber: 42, + ObservationDomainId: 7, + FlowSets: []interface{}{ + netflow.DataFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 256}, + Records: []netflow.DataRecord{ + { + Values: []netflow.DataField{ + {Type: netflow.IPFIX_FIELD_sourceIPv4Address, Value: []byte{192, 0, 2, 10}}, + {Type: netflow.IPFIX_FIELD_destinationIPv4Address, Value: []byte{198, 51, 100, 20}}, + {Type: netflow.IPFIX_FIELD_sourceTransportPort, Value: []byte{0x30, 0x39}}, + {Type: netflow.IPFIX_FIELD_destinationTransportPort, Value: []byte{0x01, 0xbb}}, + {Type: netflow.IPFIX_FIELD_protocolIdentifier, Value: []byte{6}}, + {Type: netflow.IPFIX_FIELD_octetDeltaCount, Value: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x41}}, + {Type: netflow.IPFIX_FIELD_packetDeltaCount, Value: []byte{0x00, 0x00, 0x00, 0x07}}, + {Type: netflow.IPFIX_FIELD_ingressInterface, Value: []byte{0x00, 0x00, 0x00, 0x09}}, + {Type: netflow.IPFIX_FIELD_egressInterface, Value: []byte{0x00, 0x00, 0x00, 0x0a}}, + {Type: netflow.IPFIX_FIELD_flowStartMilliseconds, Value: []byte{0x00, 0x00, 0x01, 0x8b, 0xcf, 0xe5, 0x6b, 0x64}}, + {Type: netflow.IPFIX_FIELD_flowEndMilliseconds, Value: []byte{0x00, 0x00, 0x01, 0x8b, 0xcf, 0xe5, 0x6e, 0x84}}, + }, + }, + }, + }, + }, + } +} + +func benchOriginalIPFIXPayloads(b *testing.B) ([]byte, []byte) { + b.Helper() + templatePacket := &netflow.IPFIXPacket{ + Version: 10, + ExportTime: uint32(time.Unix(1_700_000_001, 0).Unix()), + SequenceNumber: 41, + ObservationDomainId: 7, + FlowSets: []interface{}{ + netflow.TemplateFlowSet{ + FlowSetHeader: netflow.FlowSetHeader{Id: 2}, + Records: []netflow.TemplateRecord{ + { + TemplateId: 256, + FieldCount: 11, + Fields: []netflow.Field{ + {Type: netflow.IPFIX_FIELD_sourceIPv4Address, Length: 4}, + {Type: netflow.IPFIX_FIELD_destinationIPv4Address, Length: 4}, + {Type: netflow.IPFIX_FIELD_sourceTransportPort, Length: 2}, + {Type: netflow.IPFIX_FIELD_destinationTransportPort, Length: 2}, + {Type: netflow.IPFIX_FIELD_protocolIdentifier, Length: 1}, + {Type: netflow.IPFIX_FIELD_octetDeltaCount, Length: 8}, + {Type: netflow.IPFIX_FIELD_packetDeltaCount, Length: 4}, + {Type: netflow.IPFIX_FIELD_ingressInterface, Length: 4}, + {Type: netflow.IPFIX_FIELD_egressInterface, Length: 4}, + {Type: netflow.IPFIX_FIELD_flowStartMilliseconds, Length: 8}, + {Type: netflow.IPFIX_FIELD_flowEndMilliseconds, Length: 8}, + }, + }, + }, + }, + }, + } + dataPacket := benchOriginalIPFIXPacket() + + templatePayload, err := netflow.EncodeMessage(templatePacket) + if err != nil { + b.Fatalf("encode template packet: %v", err) + } + dataPayload, err := netflow.EncodeMessage(dataPacket) + if err != nil { + b.Fatalf("encode data packet: %v", err) + } + return templatePayload, dataPayload +} diff --git a/scripts/reflow-profile.sh b/scripts/reflow-profile.sh new file mode 100755 index 00000000..94d3ddd8 --- /dev/null +++ b/scripts/reflow-profile.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env sh +set -eu + +usage() { + cat <<'EOF' +Usage: scripts/reflow-profile.sh [preset] [out-dir] [benchtime] + +Presets: + processor-json Profile JSON-heavy processor benchmarks. + encode Profile ReFlow encoder benchmarks. + pipeline Profile in-memory app pipeline benchmarks. + aggregate Profile stateful aggregation benchmarks. + decode Profile decode benchmarks. + all Run all presets. + +Defaults: + preset processor-json + out-dir /tmp/reflow-profiles + benchtime 5s + +The script always writes standard Go CPU and memory profiles. If go tool pprof is +available, it also writes top reports. On toolchains that only expose +go tool preprofile, it writes the converted preprofile text instead. +EOF +} + +preset="${1:-processor-json}" +case "$preset" in + -h|--help) + usage + exit 0 + ;; +esac + +out_root="${2:-/tmp/reflow-profiles}" +benchtime="${3:-5s}" +mkdir -p "$out_root" + +run_profile() { + name="$1" + pkg="$2" + bench="$3" + out_dir="$out_root/$name" + mkdir -p "$out_dir" + + cpu_profile="$out_dir/cpu.pprof" + mem_profile="$out_dir/mem.pprof" + bench_log="$out_dir/bench.txt" + + echo "==> profiling $name" + go test "$pkg" \ + -run '^$' \ + -bench "$bench" \ + -benchmem \ + -benchtime "$benchtime" \ + -count=1 \ + -cpuprofile "$cpu_profile" \ + -memprofile "$mem_profile" | tee "$bench_log" + + if go tool pprof -h >/dev/null 2>&1; then + go tool pprof -top -nodecount=30 "$cpu_profile" > "$out_dir/cpu-top.txt" + go tool pprof -top -alloc_space -nodecount=30 "$mem_profile" > "$out_dir/mem-alloc-top.txt" + elif go tool preprofile -V >/dev/null 2>&1; then + if ! go tool preprofile -i "$cpu_profile" -o "$out_dir/cpu.preprofile.txt"; then + echo " preprofile could not convert CPU profile; kept $cpu_profile" >&2 + fi + if ! go tool preprofile -i "$mem_profile" -o "$out_dir/mem.preprofile.txt"; then + echo " preprofile could not convert memory profile; kept $mem_profile" >&2 + fi + fi + + echo " wrote $out_dir" +} + +case "$preset" in + processor-json) + run_profile processor-json ./pkg/reflow/processor 'BenchmarkBuiltinProcess/(goflow2v2_json|reflow_json)' + ;; + encode) + run_profile encode ./pkg/reflow/encode 'BenchmarkEncode' + ;; + pipeline) + run_profile pipeline ./pkg/reflow/app 'BenchmarkRunPipeline' + ;; + aggregate) + run_profile aggregate ./pkg/reflow/aggregate 'BenchmarkStateful(Process|Close)' + ;; + decode) + run_profile decode ./pkg/reflow/decode 'BenchmarkDecode' + ;; + all) + run_profile processor-json ./pkg/reflow/processor 'BenchmarkBuiltinProcess/(goflow2v2_json|reflow_json)' + run_profile encode ./pkg/reflow/encode 'BenchmarkEncode' + run_profile pipeline ./pkg/reflow/app 'BenchmarkRunPipeline' + run_profile aggregate ./pkg/reflow/aggregate 'BenchmarkStateful(Process|Close)' + run_profile decode ./pkg/reflow/decode 'BenchmarkDecode' + ;; + *) + echo "unknown preset: $preset" >&2 + usage >&2 + exit 2 + ;; +esac diff --git a/utils/udp_test.go b/utils/udp_test.go index 5094d6ca..f2b5c1dd 100644 --- a/utils/udp_test.go +++ b/utils/udp_test.go @@ -4,8 +4,10 @@ import ( "errors" "fmt" "net" + "os" "strconv" "sync/atomic" + "syscall" "testing" "time" @@ -16,13 +18,16 @@ import ( func TestUDPReceiver(t *testing.T) { addr := "::1" port, err := getFreeUDPPort() + skipOnSocketPermissionError(t, err) require.NoError(t, err) t.Logf("starting UDP receiver on %s:%d\n", addr, port) r, err := NewUDPReceiver(nil) require.NoError(t, err) - require.NoError(t, r.Start(addr, port, nil)) + err = r.Start(addr, port, nil) + skipOnSocketPermissionError(t, err) + require.NoError(t, err) sendMessage := func(msg string) error { conn, err := net.Dial("udp", net.JoinHostPort(addr, strconv.Itoa(port))) if err != nil { @@ -48,12 +53,15 @@ func TestUDPReceiver(t *testing.T) { func TestUDPClose(t *testing.T) { addr := "::1" port, err := getFreeUDPPort() + skipOnSocketPermissionError(t, err) require.NoError(t, err) t.Logf("starting UDP receiver on %s:%d\n", addr, port) r, err := NewUDPReceiver(nil) require.NoError(t, err) - require.NoError(t, r.Start(addr, port, nil)) + err = r.Start(addr, port, nil) + skipOnSocketPermissionError(t, err) + require.NoError(t, err) require.NoError(t, r.Stop()) require.NoError(t, r.Start(addr, port, nil)) require.Error(t, r.Start(addr, port, nil)) @@ -97,6 +105,7 @@ func TestUDPReceiverDrainOnStop(t *testing.T) { func TestUDPReceiverDecodeError(t *testing.T) { addr := "::1" port, err := getFreeUDPPort() + skipOnSocketPermissionError(t, err) require.NoError(t, err) r, err := NewUDPReceiver(nil) @@ -116,7 +125,9 @@ func TestUDPReceiverDecodeError(t *testing.T) { return wantErr } - require.NoError(t, r.Start(addr, port, decodeFunc)) + err = r.Start(addr, port, decodeFunc) + skipOnSocketPermissionError(t, err) + require.NoError(t, err) defer func() { require.NoError(t, r.Stop()) }() @@ -150,3 +161,13 @@ func getFreeUDPPort() (int, error) { } return port, nil } + +func skipOnSocketPermissionError(t *testing.T, err error) { + t.Helper() + if err == nil { + return + } + if errors.Is(err, os.ErrPermission) || errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) { + t.Skipf("skipping UDP socket test: %v", err) + } +}