diff --git a/.github/workflows/pipeline-daily.yml b/.github/workflows/pipeline-daily.yml new file mode 100644 index 0000000..6ffcb6f --- /dev/null +++ b/.github/workflows/pipeline-daily.yml @@ -0,0 +1,77 @@ +name: pipeline-daily + +# Runs the onchain analytics pipeline daily at 01:00 UTC (1h after midnight +# to ensure yesterday's data is fully finalized on all chains), then refreshes +# the dbt models so dashboards show the latest data. +# +# Required secrets: +# ENVIO_API_TOKEN -- HyperSync API token +# GCP_SA_KEY -- GCP service account JSON key (BigQuery access) +# SLACK_WEBHOOK_URL -- Slack incoming webhook for alerts (optional) +# +# Manual trigger: use workflow_dispatch to run on demand. + +on: + schedule: + - cron: '0 1 * * *' # 01:00 UTC daily + workflow_dispatch: # Manual trigger from GitHub UI + +concurrency: + group: pipeline-daily + cancel-in-progress: false # Let running pipeline finish; don't start another + +defaults: + run: + working-directory: projects/onchain-analytics/pipeline-v5 + +jobs: + ingest: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: projects/onchain-analytics/pipeline-v5/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Authenticate to GCP + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Run pipeline (daily) + env: + ENVIO_API_TOKEN: ${{ secrets.ENVIO_API_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: npx tsx src/index.ts daily + + # dbt refresh runs after successful ingestion + # Uncomment when dbt is ready for automated runs: + # + # transform: + # needs: ingest + # runs-on: ubuntu-latest + # timeout-minutes: 15 + # defaults: + # run: + # working-directory: projects/onchain-analytics/gd_dbt + # steps: + # - uses: actions/checkout@v4 + # - uses: actions/setup-python@v5 + # with: + # python-version: '3.11' + # - name: Install dbt + # run: pip install dbt-bigquery + # - name: Authenticate to GCP + # uses: google-github-actions/auth@v2 + # with: + # credentials_json: ${{ secrets.GCP_SA_KEY }} + # - name: dbt run + # run: dbt run --select marts diff --git a/.gitignore b/.gitignore index be644e1..9932fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -42,7 +42,10 @@ projects/dashboard-scripts/handoff.md docs/data-dictionary/ docs/datasources/ docs/other/ - +# Internal strategy / planning docs â€" local only, never published +projects/onchain-analytics/docs/antseed-tracking-plan.md +projects/onchain-analytics/docs/tracking-plan-research.md +docs/utm-tracking-architecture.md # Agent context & process files — local only, never published # This is a public repo: instructions files of any kind do not belong here, # only working code and reader-facing documentation. Keep this list ahead of @@ -61,3 +64,51 @@ CLAUDE.md **/plan.md **/audit.md **/specs/ + +# Private working files - local only, never committed +_internal/ + +# Investigation working files - local only +investigations/jul15-spike/ +investigations/sybil-farm/amplitude-prompts.md +investigations/sybil-farm/cross_reference_results.json +queries/dune/reserve-governance/cross_reference.py +queries/dune/sybil-monitor/BUILD-GUIDE.md + +investigations/sybil-farm/startup-prompt.md + +# --------------------------------------------------------------------------- +# THE UNDERSCORE CONVENTION (added 2026-07-30) +# A leading "_" means LOCAL-ONLY working file: raw dumps, exports, scratch +# analysis, and anything candid about people. One rule to remember instead of +# listing every future filename. Prefix a file with "_" and it can never be +# committed to this public repo by accident. +# NOTE: *.yml is deliberately NOT matched — dbt uses _sources.yml / _schema.yml +# as real tracked code. +_*.md +_*.txt +_*.json +_*.csv +_*.tsv +**/_*.md +**/_*.txt +**/_*.json +**/_*.csv +**/_*.tsv + +# Agent kickoff / seed prompts (same family as the handoff + prompt rules above) +**/*kickoff*.md +**/*startup-prompt*.md +**/*-primer.md + +# Internal meeting material + working digests — never reader-facing +**/meeting-prep*.md +**/meeting-notes*.md +**/project-checklist-meeting.md +docs/monday-digest-*.md +docs/monday-digest-pull-guide.md +docs/monday-digest-template.md + +# Candid docs about colleagues / stakeholders — NEVER in a repo others can read +**/*-alignment-analysis.md +**/*-profile.md diff --git a/projects/goodwidget-components/spec.md b/projects/goodwidget-components/spec.md new file mode 100644 index 0000000..8ec2a22 --- /dev/null +++ b/projects/goodwidget-components/spec.md @@ -0,0 +1,749 @@ +# GoodWidget Analytics Chart Components -- Spec (All 4 Remaining) + +*Spec for: Pie/Donut Chart, Bar Chart, Line/Area Chart, Data Table* +*Target repo: GoodDollar/GoodWidget* +*Target branch: feat/analytics-components (rename from feat/analytics-component-scorecard-plan)* +*PR: #142 (same PR as Scorecard)* +*Task naming: analytics-chart-pie-donut, analytics-chart-bar, analytics-chart-line-area, analytics-chart-table* + +--- + +## Shared Preamble (applies to ALL 4 components) + +### Why these exist + +Standardized chart components in GoodWidget packages/ui -- cross-platform building blocks that any widget can compose. These 4 components, combined with the already-built Scorecard, form the complete analytics visualization layer. They replace one-off HTML+Chart.js dashboards with reusable, consistent, theme-aware components. + +### Technical constraints (non-negotiable) + +- Must use **react-native-svg** for all graphical elements (Svg, Path, Rect, Circle, Line, G, Polygon, Polyline, Text) +- Must use **Tamagui primitives** from @goodwidget/ui (Stack, YStack, XStack, Text, Heading) for non-SVG layout +- Must use **useTheme()** for all colors -- zero hardcoded hex values +- Must use **createComponent()** for all internal styled sub-pieces (enables theme targeting) +- Must support all 3 GoodWidget pipelines: React web, React Native, Web Components +- Must use **formatMetricValue** from `packages/ui/src/utils/formatMetricValue.ts` for number formatting +- Must follow **Scorecard.tsx** component structure, spacing system, and type scale (use the same SCORECARD_BASE_SIZE_PX and GOLDEN_RATIO constants) +- Push changes to PR #142 branch (renamed to `feat/analytics-components`) +- **No animation.** Do NOT add react-native-reanimated or any animation library. No entrance animations, no transitions. Static rendering only. +- **No new dependencies.** react-native-svg is already a peerDependency. Do not add any new packages. + +### Shared design system + +**Type scale and spacing:** Follow Scorecard.tsx's established system exactly. Use its `SCORECARD_BASE_SIZE_PX` and `GOLDEN_RATIO` constants for all typography sizing and gap computation. Do NOT invent a separate spacing system -- derive from the same ratio so all analytics components breathe identically. + +**Color palette for multi-category charts:** + +```typescript +const CHART_COLOR_KEYS = ['primary', 'success', 'warning', 'colorDim', 'error'] as const +``` + +Resolved via `resolveThemeColor(theme, key)` at render time (same pattern as FundingDistributionChart). Always show direct labels (text) on every segment/bar/line -- never rely on color alone to convey information. + +**Visual hierarchy rules:** +- Main data (values, primary metrics) visually dominates labels and chrome +- Labels/units use lighter weight and secondary color ($placeholderColor or $colorDim) +- SVG data elements (bars, line strokes, pie segments) are the heaviest visual elements +- Grid lines are the lightest (0.5-1px, $borderColor at reduced opacity) + +### Design quality target + +**Visual quality benchmark:** Nivo charts (https://nivo.rocks/). These are the aesthetic standard. Our components should look this polished in both light and dark themes. + +**Specific aesthetic rules (measurable, not subjective):** + +1. **Grid lines: nearly invisible.** strokeOpacity between 0.08 and 0.15. They orient the reader without competing with data. If you squint and the grid lines are the first thing you see, they're too strong. + +2. **Axis labels: muted and small.** Use $placeholderColor (not $color). Font size at the smallest tier of the type scale. They exist to orient, not to inform. + +3. **Data elements: saturated and bold.** Bars, line strokes, and pie segments use full-strength theme colors ($primary, $success, etc.) at 100% opacity. They are the hero. + +4. **Area fills: vertical gradient, not flat.** When showArea=true, the fill should gradient from ~30% opacity at the line to ~5% opacity at the baseline. Use react-native-svg LinearGradient (Defs + LinearGradient + Stop elements). This is what makes area charts look premium vs flat/cheap. + +5. **No borders on data shapes.** Bars and pie segments have fill only, no strokeWidth on the shape itself. (Pie uses strokeDasharray on a Circle for the arc technique -- that's different from a border.) + +6. **Generous internal padding.** Chart content should breathe. The padding defaults ({ top: 16, right: 16, bottom: 40, left: 48 }) give axes room without cramping the data area. + +7. **Title and metadata OUTSIDE the SVG.** Title, legend, description, trend badges -- all rendered as Tamagui Text/Heading ABOVE or BELOW the SVG element, not crammed inside it. + +8. **Legend dots, not squares.** Legend color indicators are small circles (borderRadius full, 8x8 or 10x10), not squares. Consistent with FundingDistributionChart's existing 11x11 circles. + +9. **Line charts: thin and precise.** Default strokeWidth=2. Not thick marker-style lines. Dots small (r=3) when shown. The line itself carries the information, not fat markers. + +10. **Data glow on dark backgrounds (optional).** When the active theme is dark, primary-colored elements can have a very subtle shadow/glow effect (shadowColor matching the element color at 0.2-0.3 opacity, shadowRadius 4-8). On light themes, skip the glow -- it looks out of place. Match the glow pattern already used by GlowCard and ClaimActionGlow in the codebase. + +**Card variant behavior:** +- `variant="bare"` (default): No wrapper, for embedding in existing layouts +- `variant="card"`: Wraps in existing Card primitive with elevation. DO NOT modify Card.ts. + +**Accessibility baseline (every component):** +- Root SVG: `accessibilityRole="image"` + `accessibilityLabel` prop +- Decorative SVG elements (grid, axes, backgrounds): `accessible={false}` +- Cross-platform testID: both `testID={testID}` AND `data-testid={testID}` on root +- Minimum touch target for interactive elements: 44x44pt + +**Integer formatting rule:** +``` +IF value === Math.floor(value) THEN format without decimals +ELSE format with 1 decimal place +``` + +**NaN/Infinity guard:** +``` +IF !Number.isFinite(value) THEN treat as 0, do not render that data point +``` + +### Scope boundary (HARD RULE) + +For each component, ONLY create/modify: +1. `packages/ui/src/components/[ComponentName].tsx` +2. `packages/ui/src/index.ts` -- add export under `// Analytics` section +3. `examples/storybook/src/stories/design-system/[ComponentName].stories.tsx` +4. `tests/design-system/smoke.spec.ts` -- add test cases + +**DO NOT touch:** Card.ts, Text.ts, Icon.tsx, theme.ts, presets.ts, config.ts, any file in packages/governance-widget/, package.json. + +### Testing expectations + +**Storybook story:** Default (bare), card variant, empty state, single data point, stress test (extreme data volume). + +**Playwright smoke test:** Navigate to story, assert testID visible, screenshot, assert key text present. + +--- + +## Component 1: Pie/Donut Chart + +### Goal + +A pie/donut chart component that displays categorical data as proportional arc segments. It generalizes the governance-widget's FundingDistributionChart into a reusable building block. An inner radius > 0 makes it a donut (with center metric); inner radius = 0 makes it a classic pie. + +### When to use + +**Best for:** +- Showing that parts sum to a meaningful whole (budget allocation, market share, chain distribution) +- 2-5 categories where the "100% total" message matters more than precise between-category comparison +- When one dominant category vs several smaller ones is the story (e.g., "Celo handles 70% of claims") +- Situations where the reader's question is "what fraction?" not "how much more?" + +**Not appropriate for:** +- Comparing exact values between categories -- bar chart is far more accurate for this +- More than 7 categories -- aggregate or use a bar chart +- Showing change over time -- use stacked area or grouped bar +- Comparing two pie charts side by side -- nearly impossible to compare angles across separate charts +- Very similar-sized segments (e.g., three categories at ~33% each) -- bar chart shows differences much more clearly + +### Visual reference + +- **Aesthetic target:** Nivo Pie interactive demo (center metric, arc labels, legends, theming): https://nivo.rocks/pie/ +- Existing codebase pattern: FundingDistributionChart.tsx in governance-widget (SVG arc technique) + +### Props / API surface (MVP) + +| Prop | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| data | Array<{ label: string; value: number; color?: string }> | Yes | - | Category data (absolute values, not percentages) | +| title | string | No | undefined | Chart heading | +| innerRadius | number | No | 0.6 | Inner radius as fraction of outer (0 = pie, 0.6 = donut) | +| centerLabel | string | No | undefined | Top text inside donut hole | +| centerValue | string | number | No | undefined | Main metric in center | +| centerValueFormatter | (value: number) => string | No | formatMetricValue | Center value formatting | +| centerSubLabel | string | No | undefined | Bottom text in center | +| maxSlices | number | No | 7 | Segments beyond this aggregate into "Other" | +| otherLabel | string | No | "Other" | Aggregated remainder label | +| sort | 'descending' | 'ascending' | 'none' | No | 'descending' | Segment sort order | +| showLegend | boolean | No | true | Display legend below chart | +| showPercentages | boolean | No | true | Show percentage in legend items | +| onSegmentPress | (item, index) => void | No | undefined | Segment tap callback | +| variant | 'bare' | 'card' | No | 'bare' | Visual variant | +| testID | string | No | undefined | Testing identifier | +| accessibilityLabel | string | No | auto-generated | Screen reader description | +| width | number | No | 188 | SVG width | +| height | number | No | 188 | SVG height | + +### Behavioral rules + +1. **Segment rendering:** Circle elements with strokeDasharray/strokeDashoffset (same technique as FundingDistributionChart). strokeWidth = 20. + +2. **Sort and start position:** Sorted descending by default. First segment starts at -90 degrees (12-o'clock) via G rotation="-90". + +3. **Aggregation:** If data.length > maxSlices, smallest items merge into one "Other" segment using the last palette color. + +4. **Percentage display:** percentage = (item.value / sum(all values)) * 100. Integers show no decimal (25%), non-integers show one (33.3%). + +5. **Color assignment:** From CHART_COLOR_KEYS via useTheme(). Custom item.color overrides. Cycle if data exceeds palette length. + +6. **Center metric:** Formatted by centerValueFormatter (default: formatMetricValue). Constrained to inner radius width to prevent overflow. + +7. **Empty state:** Empty data OR all values 0: grey ring ($borderColor, 0.18 opacity), centerLabel or "No data", no legend. + +8. **Legend:** Vertical stack below chart. Each row: color swatch + label + percentage text. + +9. **NaN/null/negative filtering:** Silently excluded from rendering and total calculation. + +### Mock data for testing/screenshots + +```typescript +// 1. Standard (GoodDollar funding) +const funding = [ + { label: 'Education Hubs', value: 157500 }, + { label: 'Merchant Onboard', value: 112500 }, + { label: 'Dev Grants', value: 90000 }, + { label: 'Creator Fund', value: 90000 }, +] + +// 2. Single item +const single = [{ label: 'UBI Distribution', value: 1000000 }] + +// 3. Two near-equal +const nearEqual = [{ label: 'Celo', value: 51 }, { label: 'Fuse', value: 49 }] + +// 4. Empty +const empty: [] = [] + +// 5. STRESS TEST -- 100+ items (triggers maxSlices aggregation heavily) +const stress = Array.from({ length: 120 }, (_, i) => ({ + label: `Category ${i + 1}`, + value: Math.floor(Math.random() * 10000) + 100, +})) +// Expected: with maxSlices=7, this produces 7 segments (top 6 + one massive "Other") +// Tests: legend overflow, color cycling, aggregation math, percentage rounding at tiny values +``` + +### Acceptance criteria + +- [ ] Renders proportional arcs starting at 12-o'clock, sorted descending +- [ ] Center label, value, sublabel display (donut mode, innerRadius > 0) +- [ ] Pure pie renders when innerRadius=0 (no center content) +- [ ] Legend shows items with correct swatches and percentages +- [ ] maxSlices=7 with 120 items produces exactly 7 segments +- [ ] Integers: 25% (not 25.0%). Non-integers: 33.3% (not 33%) +- [ ] Empty state: grey ring + "No data" +- [ ] variant="card" wraps correctly +- [ ] testID + data-testid both present +- [ ] accessibilityRole="image" on Svg +- [ ] No hardcoded colors +- [ ] onSegmentPress fires correctly +- [ ] Stress test (120 items): renders without crash, legend does not overflow container + +### Out of scope (future) + +- Tooltip on hover (web) +- Labels on segments (outside with leader lines) +- Custom arc styling per segment +- Interactive legend (click to hide segment) +- Gradient fills + +### DO NOT + +- DO NOT copy FundingDistributionChart wholesale -- extract only the SVG arc technique +- DO NOT use Canvas or web-only APIs +- DO NOT use Icon.tsx (web-only DOM SVG) + +--- + +## Component 2: Bar Chart + +### Goal + +A bar chart component for discrete categorical comparison. This is the highest-accuracy chart type for comparing values across categories (position on a common scale). Supports vertical and horizontal layouts. + +### When to use + +**Best for:** +- Comparing magnitudes across discrete categories (claims per chain, funding per house, monthly totals) +- Showing ranking or ordering (sorted bars make rank instantly visible) +- When the reader's question is "which is bigger?" or "by how much?" +- Discrete time periods where each period is a complete unit (monthly totals, quarterly results) +- Long category labels (horizontal mode) + +**Not appropriate for:** +- Continuous time series with many points -- use line chart (bars become too narrow and lose readability) +- Part-to-whole relationships -- use pie/donut or 100% stacked bar +- Showing trends/velocity -- line chart communicates rate of change better via slope +- More than ~20 categories without scrolling or filtering + +**Horizontal vs vertical decision:** +- Use vertical (default) for short category labels and up to ~12 categories +- Switch to horizontal when labels exceed ~10 characters (avoids rotated text) +- Switch to horizontal when comparing many categories (>8) -- horizontal bars scale better vertically + +### Visual reference + +- **Aesthetic target:** Nivo Bar interactive demo (clean fills, muted axes, generous padding): https://nivo.rocks/bar/ + +### Props / API surface (MVP) + +| Prop | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| data | Array<{ category: string; value: number }> | Yes | - | Bar data | +| title | string | No | undefined | Chart heading | +| layout | 'vertical' | 'horizontal' | No | 'vertical' | Bar orientation | +| showGrid | boolean | No | true | Show grid lines | +| showValueLabels | boolean | No | false | Display values on bars | +| valueFormatter | (value: number) => string | No | formatMetricValue | Value/axis formatting | +| xAxisLabel | string | No | undefined | X-axis title | +| yAxisLabel | string | No | undefined | Y-axis title | +| barCornerRadius | number | No | 0 | Rounded top corners | +| onBarPress | (item, index) => void | No | undefined | Bar tap callback | +| variant | 'bare' | 'card' | No | 'bare' | Visual variant | +| testID | string | No | undefined | Testing identifier | +| accessibilityLabel | string | No | auto-generated | Screen reader description | +| width | number | string | No | '100%' | Chart width | +| height | number | No | 200 | Chart height | +| padding | { top, right, bottom, left } | No | { 16, 16, 40, 48 } | Internal padding for axes | + +### Behavioral rules + +1. **Zero baseline:** Y-axis always INCLUDES zero. If all values are positive, axis starts at 0. If values include negatives, axis extends below 0 (diverging bars from zero line). + +2. **Axis calculation:** Max = ceil(maxValue * 1.1) rounded to a "nice" number (multiples of 1/2/5/10/20/50/100/1K/etc.). ~5 tick marks at nice intervals. + +3. **Bar sizing:** barWidth = (availableWidth / categoryCount) * 0.7. Gap = barWidth * 0.3. + +4. **Color:** All bars use $primary (single series MVP). + +5. **Value labels:** When enabled, display above bars (vertical) or right of bars (horizontal). Hide if bar height < 20px. + +6. **Horizontal layout:** Axes swap. Categories on y-axis (left), values on x-axis (bottom). + +7. **Grid lines:** Horizontal grid lines at each y-axis tick. 0.5px, $borderColor, dashed "3 3". + +8. **Axis labels:** X-axis centered below bars. If label overflows bar width, truncate with ellipsis. Y-axis right-aligned, formatted with formatMetricValue. + +9. **Empty state:** Show axes with zero line + "No data" centered. + +10. **NaN/null filtering:** Items excluded silently. + +### Mock data for testing/screenshots + +```typescript +// 1. Standard (claims by chain) +const chains = [ + { category: 'Celo', value: 45200 }, + { category: 'Fuse', value: 32100 }, + { category: 'Ethereum', value: 8500 }, +] + +// 2. Horizontal with long labels +const houses = [ + { category: 'House of Alignment', value: 450000 }, + { category: 'House of Innovation', value: 320000 }, + { category: 'House of Community', value: 180000 }, +] + +// 3. Single bar +const single = [{ category: 'Total Claims', value: 85800 }] + +// 4. Empty +const empty: [] = [] + +// 5. STRESS TEST -- 100+ categories +const stress = Array.from({ length: 150 }, (_, i) => ({ + category: `Wallet ${String(i + 1).padStart(3, '0')}`, + value: Math.floor(Math.random() * 100000), +})) +// Expected: bars become extremely narrow (< 1px each), labels overlap/disappear +// Tests: layout doesn't crash, axis still renders, bars clip rather than overflow +// This reveals: need to handle overflow gracefully (either scroll or show only first N) +``` + +### Acceptance criteria + +- [ ] Renders vertical bars with correct proportional heights +- [ ] Y-axis includes zero (positive data: starts at 0; mixed: extends below) +- [ ] Y-axis uses nice-number ticks +- [ ] X-axis labels centered under bars, truncated if overflow +- [ ] Grid lines render as subtle dashed lines +- [ ] Horizontal layout works with axes swapped +- [ ] Value labels appear when enabled, hide when bar < 20px +- [ ] Empty state renders correctly +- [ ] 150-item stress test: renders without crash +- [ ] variant="card" wraps correctly +- [ ] testID + data-testid present +- [ ] No hardcoded colors +- [ ] formatMetricValue used for axis/value labels +- [ ] onBarPress fires correctly + +### Out of scope (future) + +- Grouped bars (multiple series side-by-side) +- Stacked bars +- Reference/target lines +- Interactive sort +- Scrollable overflow for many categories +- 100%-stacked bars + +### DO NOT + +- DO NOT implement grouped or stacked bars -- single series only +- DO NOT add a secondary y-axis +- DO NOT add horizontal scroll (truncate/clip instead) +- DO NOT use Icon.tsx or web-only APIs + +--- + +## Component 3: Line/Area Chart + +### Goal + +A line/area chart for time-series and continuous data visualization. The most complex of the 5 components. It communicates trends, velocity, and cumulative patterns. Area mode (shaded fill below line) emphasizes volume/accumulation. + +### When to use + +**Best for:** +- Showing trends over time (daily claims, member growth, reserve balance history) +- Communicating rate of change (slope = velocity -- "is it accelerating or slowing?") +- Comparing multiple time series on the same scale (up to 5 overlaid lines) +- Showing cumulative totals (area fill emphasizes "total so far") +- Identifying anomalies and pattern breaks in temporal data + +**Not appropriate for:** +- Comparing exact values at a specific point -- bar chart or table is more accurate +- Categorical data with no natural ordering -- lines imply continuity between points +- More than 5-7 overlaid series -- becomes unreadable "spaghetti" +- When individual data points matter more than the connection between them -- use scatter plot + +**Line vs area decision:** +- Use line when the trend/shape is the message +- Use area when cumulative volume or "total magnitude" is the message +- Use area when you want to emphasize "how much" over time, line when you want to emphasize "what direction" + +### Visual reference + +- **Aesthetic target:** Nivo Line interactive demo (multi-series, area fills, dots, clean grid): https://nivo.rocks/line/ +- Nivo Area stacked example: https://nivo.rocks/stacked-area/ + +### Props / API surface (MVP) + +| Prop | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| data | Array<{ x: string | number; y: number; series?: string }> | Yes | - | Data points | +| title | string | No | undefined | Chart heading | +| series | Array<{ key: string; label: string; color?: string; strokeDasharray?: string }> | No | auto-detect | Series definitions | +| type | 'linear' | 'monotone' | 'step' | No | 'linear' | Interpolation curve | +| showArea | boolean | No | false | Fill area below line | +| areaOpacity | number | No | 0.15 | Area fill transparency (0-1) | +| showDots | boolean | 'auto' | No | 'auto' | Show point markers (auto = show if <20 points) | +| showGrid | boolean | No | true | Show grid lines | +| connectNulls | boolean | No | false | Bridge gaps in data (false = show visible gap) | +| strokeWidth | number | No | 2 | Line thickness | +| xAxisLabel | string | No | undefined | X-axis title | +| yAxisLabel | string | No | undefined | Y-axis title | +| xAxisFormatter | (value) => string | No | identity | X-axis label formatting | +| yAxisFormatter | (value) => string | No | formatMetricValue | Y-axis label formatting | +| yAxisDomain | [number | 'auto', number | 'auto'] | No | ['auto', 'auto'] | Y-axis range | +| secondaryYAxis | { key: string; label?: string; formatter?: (v) => string } | No | undefined | Secondary y-axis for a specific series | +| referenceLines | Array<{ value: number; label?: string; color?: string }> | No | [] | Horizontal marker lines | +| onPointPress | (point, seriesKey) => void | No | undefined | Point tap callback | +| variant | 'bare' | 'card' | No | 'bare' | Visual variant | +| testID | string | No | undefined | Testing identifier | +| accessibilityLabel | string | No | auto-generated | Screen reader description | +| width | number | string | No | '100%' | Chart width | +| height | number | No | 200 | Chart height | +| padding | { top, right, bottom, left } | No | { 16, 16, 40, 48 } | Internal padding | + +### Behavioral rules + +1. **Coordinate mapping:** X-values mapped to evenly-spaced positions. Y-values mapped linearly from domain to height (inverted: higher value = lower SVG y). + +2. **Path generation:** 'linear' = straight segments (M, L). 'monotone' = monotone cubic Hermite spline (smooth, no overshoot). 'step' = hold value until next point (H then V). + +3. **Area fill:** Duplicate line path, extend to bottom, close path. Fill with a **vertical LinearGradient** (react-native-svg Defs + LinearGradient + Stop): top Stop at series color with opacity 0.3, bottom Stop at series color with opacity 0.05. This gradient treatment is what distinguishes premium area charts from flat/cheap-looking fills. + +4. **Multi-series:** Each series = separate Path with its own color. Layered in array order. + +5. **Missing data (null y-values):** connectNulls=false (default): break path, visible gap. connectNulls=true: skip null, connect adjacent. + +6. **Dots:** auto = show if fewer than 20 data points. Circle r=3, fill=series color. + +7. **Axis:** Nice-number algorithm (same as bar). Y extends 10% beyond data range. X labels at regular intervals; skip every Nth if they would overlap. + +8. **Grid:** Horizontal only, subtle dashed (same as bar chart). + +9. **Reference lines:** Horizontal line at y-value + optional right-aligned label. 1px solid, custom color or $colorDim. + +10. **Secondary y-axis:** When secondaryYAxis is provided, the specified series maps to a second y-axis on the RIGHT side of the chart with its own scale/domain/formatter. All other series use the left axis. Render the secondary axis labels on the right edge. Use the series color for the axis labels to associate them. NOTE: dual axes can mislead readers -- the consumer is responsible for appropriate use. + +11. **Empty state:** Axes only + "No data" centered. + +12. **Single data point:** Render only a dot. + +### Mock data for testing/screenshots + +```typescript +// 1. Single series (daily claims, 14 days) +const daily = [ + { x: 'Jul 24', y: 18200 }, { x: 'Jul 25', y: 19400 }, + { x: 'Jul 26', y: 17800 }, { x: 'Jul 27', y: 21000 }, + { x: 'Jul 28', y: 22500 }, { x: 'Jul 29', y: 20100 }, + { x: 'Jul 30', y: 23800 }, { x: 'Jul 31', y: 25200 }, + { x: 'Aug 1', y: 24100 }, { x: 'Aug 2', y: 26800 }, + { x: 'Aug 3', y: 28400 }, { x: 'Aug 4', y: 27200 }, + { x: 'Aug 5', y: 30100 }, { x: 'Aug 6', y: 31500 }, +] +// title="Daily UBI Claims", showArea=true, referenceLines=[{value:25000, label:"Target"}] + +// 2. Multi-series with secondary y-axis +const multiAxis = [ + { x: 'Jan', y: 12000, series: 'claims' }, + { x: 'Jan', y: 0.012, series: 'price' }, + { x: 'Feb', y: 14500, series: 'claims' }, + { x: 'Feb', y: 0.011, series: 'price' }, + // ... (7 months) +] +// secondaryYAxis={ key: 'price', label: 'G$ Price', formatter: (v) => `$${v}` } + +// 3. With missing data (gap) +const withGap = [ + { x: 'Day 1', y: 100 }, { x: 'Day 2', y: 120 }, + { x: 'Day 3', y: null }, { x: 'Day 4', y: null }, + { x: 'Day 5', y: 150 }, { x: 'Day 6', y: 160 }, +] + +// 4. Empty +const empty: [] = [] + +// 5. STRESS TEST -- 1000+ data points (daily data for 3 years) +const stress = Array.from({ length: 1095 }, (_, i) => { + const date = new Date(2024, 0, 1) + date.setDate(date.getDate() + i) + return { + x: date.toISOString().slice(0, 10), + y: 10000 + Math.floor(Math.random() * 5000) + i * 10, + } +}) +// Expected: line becomes very dense, individual points invisible +// Tests: SVG path doesn't crash, axis label thinning kicks in, performance is acceptable +// Reveals: might need downsampling strategy for paths with 1000+ points +``` + +### Acceptance criteria + +- [ ] Renders line connecting data points proportionally +- [ ] Y-axis nice-number ticks with auto domain +- [ ] X-axis labels display without overlapping (adaptive thinning) +- [ ] Grid lines render correctly +- [ ] Area fill below line at correct opacity +- [ ] Multi-series with distinct colors and legend +- [ ] Null y-values create visible gap (connectNulls=false) +- [ ] Null y-values bridge (connectNulls=true) +- [ ] Dots show at auto threshold (< 20 points) +- [ ] Reference line at correct position with label +- [ ] Step interpolation produces staircase +- [ ] Monotone interpolation produces smooth curve +- [ ] Secondary y-axis renders on right with correct scale +- [ ] Empty state: axes + "No data" +- [ ] Single point: dot only +- [ ] 1000-point stress test: renders without crash or hang +- [ ] variant="card" works +- [ ] Accessibility attributes present +- [ ] No hardcoded colors + +### Out of scope (future) + +- Tooltip/crosshair on hover +- Brush/zoom for time range selection +- Stacked area +- Custom dot shapes per series +- Data downsampling algorithm (MVP renders all points as-is) +- Pan/scroll for long series +- Sparkline variant (no axes) + +### DO NOT + +- DO NOT use 'natural' or 'basis' interpolation (overshoot, implies non-existent values) +- DO NOT add entrance animations on path drawing +- DO NOT use web-only APIs (no requestAnimationFrame, no CSS transitions) + +--- + +## Component 4: Data Table + +### Goal + +A data table for exact value display with typed columns, formatting, and sorting. The complement to visual charts -- providing precise value lookup (the task charts are worst at). Uses only Tamagui layout (no SVG). + +### When to use + +**Best for:** +- Exact value lookup ("what was Celo's claim count on Tuesday?") +- Multi-attribute comparison across entities (address + volume + tx count + date) +- When the reader needs to find a specific number, not perceive a pattern +- As a companion to any chart ("see the chart for the trend, switch to table for exact numbers") +- Sorted leaderboards, ranked lists, detailed breakdowns + +**Not appropriate for:** +- Pattern/trend detection -- charts are far faster for seeing shapes +- Very few data points (< 3 rows) -- just write a sentence +- Very many rows (> 100 visible) without pagination/filter -- becomes overwhelming + +### Visual reference + +- Aesthetic reference: StatCell pattern in ai-credits-widget's CreditsManagementCard.tsx +- Layout reference: CreditsManagementCard stat grid (backgroundColor=$backgroundHover, borderRadius, padding) + +### Props / API surface (MVP) + +| Prop | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| data | Array> | Yes | - | Row data | +| columns | Array | Yes | - | Column configuration | +| title | string | No | undefined | Table heading | +| striped | boolean | No | true | Alternating row backgrounds | +| compact | boolean | No | false | Reduced row padding | +| stickyHeader | boolean | No | true | Fixed header on scroll | +| maxHeight | number | No | undefined | Max height before scroll | +| defaultSort | { key: string; direction: 'asc' | 'desc' } | No | undefined | Initial sort | +| onSort | (key, direction) => void | No | undefined | Sort callback | +| emptyMessage | string | No | "No data" | Empty state text | +| onRowPress | (row, index) => void | No | undefined | Row tap callback | +| variant | 'bare' | 'card' | No | 'bare' | Visual variant | +| testID | string | No | undefined | Testing identifier | +| accessibilityLabel | string | No | auto-generated | Screen reader description | + +**ColumnDef:** + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| key | string | Yes | - | Data field accessor | +| label | string | Yes | - | Header text | +| type | 'text' | 'number' | 'date' | 'currency' | No | 'text' | Formatting hint | +| align | 'left' | 'center' | 'right' | No | 'center' | Cell alignment | +| width | number | string | No | 'auto' | Column width | +| minWidth | number | No | 60 | Minimum width | +| formatter | (value, row) => string | No | type-based | Custom formatting | +| sortable | boolean | No | false | Enable column sorting | +| truncate | boolean | No | true | Truncate with ellipsis | + +### Behavioral rules + +1. **Column alignment:** Default is 'center' for all columns. Explicit `align` prop overrides. + +2. **Default formatters by type:** + - text: String(value), truncate if needed + - number: formatMetricValue. Integers: no decimal. + - date: Display as-is (consumer pre-formats) + - currency: formatMetricValue with consumer-provided prefix + +3. **Sorting:** Client-side when sortable=true. Tap header: ascending -> descending -> clear. Unicode arrow indicator. Default sort on first render. + +4. **Striped rows:** Even rows: $backgroundHover. Odd: transparent. Header: fontWeight 700. + +5. **Sticky header:** Remains fixed on vertical scroll. + +6. **Horizontal overflow:** If columns exceed container width, enable horizontal ScrollView. + +7. **Empty state:** Header row + single merged cell with emptyMessage. + +8. **Null/undefined values:** Display "--" in the cell. + +9. **Row press:** Tappable when onRowPress provided. Press feedback. + +10. **Compact mode:** Reduced padding, smaller font. + +### Mock data for testing/screenshots + +```typescript +// 1. Multi-column (top wallets) +const wallets = [ + { address: '0x1a2b...', volume: 1234567, txCount: 847, lastActive: 'Aug 5' }, + { address: '0x3c4d...', volume: 892000, txCount: 623, lastActive: 'Aug 4' }, + { address: '0x5e6f...', volume: 445000, txCount: 312, lastActive: 'Aug 3' }, + { address: '0x7g8h...', volume: 128000, txCount: 95, lastActive: 'Aug 1' }, + { address: '0x9i0j...', volume: 45200, txCount: 42, lastActive: 'Jul 28' }, +] + +// 2. Compact metrics summary +const metrics = [ + { metric: 'Daily Claims', value: 31500, change: '+8.2%' }, + { metric: 'Active Wallets', value: 12400, change: '+3.1%' }, + { metric: 'Reserve Balance', value: 4500000, change: '-1.2%' }, +] + +// 3. Empty +const empty: [] = [] + +// 4. Single column +const names = [{ name: 'Education Hubs' }, { name: 'Merchant Onboard' }, { name: 'Dev Grants' }] + +// 5. STRESS TEST -- 100+ rows with scroll +const stress = Array.from({ length: 150 }, (_, i) => ({ + rank: i + 1, + address: `0x${i.toString(16).padStart(8, '0')}`, + amount: Math.floor(Math.random() * 1000000), + txCount: Math.floor(Math.random() * 500), +})) +// maxHeight=300. Tests: vertical scroll, sticky header, sort performance with many rows +``` + +### Acceptance criteria + +- [ ] Renders header + data rows matching column definitions +- [ ] Default center alignment on all columns +- [ ] formatMetricValue applied to number/currency columns +- [ ] Integers: no decimal (312 not 312.0) +- [ ] Striped rows alternate backgrounds +- [ ] Sticky header visible during scroll +- [ ] Sort toggles asc/desc/none on header tap with arrow indicator +- [ ] Empty state: header + emptyMessage +- [ ] Null cells show "--" +- [ ] Horizontal scroll activates when needed +- [ ] Compact mode reduces padding/font +- [ ] Truncation: ellipsis on overflow +- [ ] 150-row stress test: renders, scrolls smoothly, sort works +- [ ] variant="card" works +- [ ] testID + data-testid present +- [ ] onRowPress fires correctly + +### Out of scope (future) + +- Column resizing +- Multi-column sort +- Search/filter +- Export CSV +- Frozen first column +- Pagination +- Virtualized rows for 1000+ +- Expandable rows +- Responsive card layout at small breakpoints + +### DO NOT + +- DO NOT use SVG (pure Tamagui layout: YStack, XStack, Text, ScrollView) +- DO NOT add FlatList/VirtualizedList +- DO NOT use Icon.tsx for sort arrows -- use unicode characters +- DO NOT add pagination + +--- + +## Appendix + +### Branch rename + +Rename PR #142 branch: `feat/analytics-component-scorecard-plan` -> `feat/analytics-components` + +### Build order + +Pie/Donut -> Bar -> Line/Area -> Table (simplest to most complex). + +### File structure (final state) + +``` +packages/ui/src/components/PieDonutChart.tsx (NEW) +packages/ui/src/components/BarChart.tsx (NEW) +packages/ui/src/components/LineAreaChart.tsx (NEW) +packages/ui/src/components/DataTable.tsx (NEW) +packages/ui/src/index.ts (MODIFIED - add exports) +examples/storybook/src/stories/design-system/PieDonutChart.stories.tsx (NEW) +examples/storybook/src/stories/design-system/BarChart.stories.tsx (NEW) +examples/storybook/src/stories/design-system/LineAreaChart.stories.tsx (NEW) +examples/storybook/src/stories/design-system/DataTable.stories.tsx (NEW) +tests/design-system/smoke.spec.ts (MODIFIED - add cases) +``` + +### resolveThemeColor utility + +FundingDistributionChart has `resolveThemeColor` in governance-widget/src/shared.tsx. Duplicate into each chart component or extract to packages/ui/src/utils/. Must have fallback to $color token. + +### Screenshots required + +After implementation, provide screenshots of bare and card variants for each component in dark theme. diff --git a/projects/onchain-analytics/pipeline-v5/.env.example b/projects/onchain-analytics/pipeline-v5/.env.example new file mode 100644 index 0000000..08599ce --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/.env.example @@ -0,0 +1,13 @@ +# Required +ENVIO_API_TOKEN=your-envio-hypersync-api-token + +# Optional (with defaults) +# GCP_PROJECT_ID=gooddollar +# DATASET_ID=BlockchainEvents +# SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +# LOG_FILE=pipeline.log +# CHUNK_SIZE_TARGET=50000 +# BATCH_DELAY_MS=200 +# BQ_RETRIES=5 +# HYPERSYNC_RETRIES=5 +# FRESHNESS_THRESHOLD_HOURS=36 diff --git a/projects/onchain-analytics/pipeline-v5/package-lock.json b/projects/onchain-analytics/pipeline-v5/package-lock.json new file mode 100644 index 0000000..805e733 --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/package-lock.json @@ -0,0 +1,1767 @@ +{ + "name": "gooddollar-pipeline-v5", + "version": "5.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "gooddollar-pipeline-v5", + "version": "5.0.0", + "dependencies": { + "@envio-dev/hypersync-client": "^0.6.3", + "@google-cloud/bigquery": "^7.9.1", + "dotenv": "^16.4.7", + "viem": "^2.21.0" + }, + "devDependencies": { + "tsx": "^4.19.2", + "typescript": "^5.7.2" + } + }, + "node_modules/@adraffy/ens-normalize": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.11.1.tgz", + "integrity": "sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==", + "license": "MIT" + }, + "node_modules/@envio-dev/hypersync-client": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/@envio-dev/hypersync-client/-/hypersync-client-0.6.7.tgz", + "integrity": "sha512-KONNp/inLNWC80hw4FZckkXI7pigxnugxnfNBWOjAF0DZJDpFn4wgdGRlSb/dtsUc5Gk8JCI5oXWGSXmSOqW/A==", + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@envio-dev/hypersync-client-darwin-arm64": "0.6.7", + "@envio-dev/hypersync-client-darwin-x64": "0.6.7", + "@envio-dev/hypersync-client-linux-arm64-gnu": "0.6.7", + "@envio-dev/hypersync-client-linux-x64-gnu": "0.6.7", + "@envio-dev/hypersync-client-linux-x64-musl": "0.6.7", + "@envio-dev/hypersync-client-win32-x64-msvc": "0.6.7" + } + }, + "node_modules/@envio-dev/hypersync-client-darwin-arm64": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/@envio-dev/hypersync-client-darwin-arm64/-/hypersync-client-darwin-arm64-0.6.7.tgz", + "integrity": "sha512-cEMcdzorkuYn/c+bopKAdnrxAcRAwSYwkLfYg+V9/vEyNvMR86vCMnV5D2zxpMg/Urv9z80V/Pfg6buT/iPDkA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@envio-dev/hypersync-client-darwin-x64": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/@envio-dev/hypersync-client-darwin-x64/-/hypersync-client-darwin-x64-0.6.7.tgz", + "integrity": "sha512-aw0gpWIMGgSTWorPnt9Jk4S24h1mH0srSIGlMphs70It1raiSE9aFowLFxNXDR/mepfnzQrZMccnEGlLpgTC7g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@envio-dev/hypersync-client-linux-arm64-gnu": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/@envio-dev/hypersync-client-linux-arm64-gnu/-/hypersync-client-linux-arm64-gnu-0.6.7.tgz", + "integrity": "sha512-Kt2r9qrlab9JEK44GR70VaLOTLVojGY7wt7DdnzvgniYt+nrS42bmgq3GDhas/Sn/mM3Tl7T6AcpFLzdZxrAXQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@envio-dev/hypersync-client-linux-x64-gnu": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/@envio-dev/hypersync-client-linux-x64-gnu/-/hypersync-client-linux-x64-gnu-0.6.7.tgz", + "integrity": "sha512-TDfJZq3Aag+c4xROZOwNvPNKZdbaFq52yQhAj6ZvG13bMQzEM9Lj1UAiNcDRxCDqSDMANhKFWTlFd0JW7jG3hA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@envio-dev/hypersync-client-linux-x64-musl": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/@envio-dev/hypersync-client-linux-x64-musl/-/hypersync-client-linux-x64-musl-0.6.7.tgz", + "integrity": "sha512-OX3+OGSwWzjHwSnt/M31CdJXNjAAH5IthyffvtDnsKUGbg4y2FGdR1sVJir2L3CD1Kyv/VJmpZFygBDmQW1EZg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@envio-dev/hypersync-client-win32-x64-msvc": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/@envio-dev/hypersync-client-win32-x64-msvc/-/hypersync-client-win32-x64-msvc-0.6.7.tgz", + "integrity": "sha512-3eTaz/yEOx5h8oBFgp4fVNyb/ptaH4Iav7chhsYRBmrlx8BwoI669Xd5u+Qw5O/mr3jCDMKPiIG4cGH9MJsRNw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/bigquery": { + "version": "7.9.4", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.9.4.tgz", + "integrity": "sha512-C7jeI+9lnCDYK3cRDujcBsPgiwshWKn/f0BiaJmClplfyosCLfWE83iGQ0eKH113UZzjR9c9q7aZQg0nU388sw==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/common": "^5.0.0", + "@google-cloud/paginator": "^5.0.2", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/promisify": "4.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/common": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", + "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "extend": "^3.0.2", + "google-auth-library": "^9.0.0", + "html-entities": "^2.5.2", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", + "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/precise-date": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-4.0.0.tgz", + "integrity": "sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@noble/ciphers": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", + "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/curves": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.1.tgz", + "integrity": "sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/base": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.6.tgz", + "integrity": "sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.7.0.tgz", + "integrity": "sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==", + "license": "MIT", + "dependencies": { + "@noble/curves": "~1.9.0", + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.6.0.tgz", + "integrity": "sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@tootallnate/once": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.1.tgz", + "integrity": "sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "26.1.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.2.tgz", + "integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@types/request": { + "version": "2.48.13", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", + "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", + "license": "MIT", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.5" + } + }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "license": "MIT" + }, + "node_modules/abitype": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.2.3.tgz", + "integrity": "sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/wevm" + }, + "peerDependencies": { + "typescript": ">=5.0.4", + "zod": "^3.22.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/big.js": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", + "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "license": "MIT" + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/form-data": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.6.tgz", + "integrity": "sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", + "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.1.1", + "google-logging-utils": "^0.0.2", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/google-auth-library": { + "version": "9.15.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", + "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-logging-utils": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", + "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-entities": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", + "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/is": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.2.tgz", + "integrity": "sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isows": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/isows/-/isows-1.0.7.tgz", + "integrity": "sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "peerDependencies": { + "ws": "*" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/ox": { + "version": "0.14.33", + "resolved": "https://registry.npmjs.org/ox/-/ox-0.14.33.tgz", + "integrity": "sha512-rooA/4o7bBof4Ge2VH/eovfNPb/AEEYyrNj03wggc55g5HZD8Pjs/OeWhttgjic3dDcqn0r29bDuvQEdTiUemQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@adraffy/ens-normalize": "^1.11.0", + "@noble/ciphers": "^1.3.0", + "@noble/curves": "1.9.1", + "@noble/hashes": "^1.8.0", + "@scure/bip32": "^1.7.0", + "@scure/bip39": "^1.6.0", + "abitype": "^1.2.3", + "eventemitter3": "5.0.1" + }, + "peerDependencies": { + "typescript": ">=5.4.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "license": "MIT", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "license": "MIT" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "license": "MIT" + }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/teeny-request/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/tsx": { + "version": "4.23.1", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.1.tgz", + "integrity": "sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "license": "MIT" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/viem": { + "version": "2.55.10", + "resolved": "https://registry.npmjs.org/viem/-/viem-2.55.10.tgz", + "integrity": "sha512-Q9Ba+/ma81U2M5o5P2AQ7Ux8rTIwmCZvUcr8rKdQ22bV0IBFHllM2m5gWDP8hFaUN2nH2oW3QG44amRazflYNQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@noble/curves": "1.9.1", + "@noble/hashes": "1.8.0", + "@scure/bip32": "1.7.0", + "@scure/bip39": "1.6.0", + "abitype": "1.2.3", + "isows": "1.0.7", + "ox": "0.14.33", + "ws": "8.21.0" + }, + "peerDependencies": { + "typescript": ">=5.0.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/projects/onchain-analytics/pipeline-v5/package.json b/projects/onchain-analytics/pipeline-v5/package.json new file mode 100644 index 0000000..a315e26 --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/package.json @@ -0,0 +1,21 @@ +{ + "name": "gooddollar-pipeline-v5", + "version": "5.0.0", + "private": true, + "type": "module", + "scripts": { + "daily": "tsx src/index.ts daily", + "backfill": "tsx src/index.ts backfill", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@envio-dev/hypersync-client": "^0.6.3", + "@google-cloud/bigquery": "^7.9.1", + "dotenv": "^16.4.7", + "viem": "^2.21.0" + }, + "devDependencies": { + "tsx": "^4.19.2", + "typescript": "^5.7.2" + } +} diff --git a/projects/onchain-analytics/pipeline-v5/src/bq.ts b/projects/onchain-analytics/pipeline-v5/src/bq.ts new file mode 100644 index 0000000..92fbc71 --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/src/bq.ts @@ -0,0 +1,240 @@ +/** + * bq.ts -- BigQuery operations. Staging + MERGE write path, state queries. + */ + +import { BigQuery } from "@google-cloud/bigquery"; +import { writeFileSync, unlinkSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { randomUUID } from "crypto"; +import { CONFIG, fullTableName, stagingTableId } from "./config.js"; +import { log, RUN_ID } from "./log.js"; +import type { SchemaField, IngestionRecord, PipelineRunRecord } from "./types.js"; + +export const bigquery = new BigQuery({ projectId: CONFIG.GCP_PROJECT_ID }); +const dataset = bigquery.dataset(CONFIG.DATASET_ID, { projectId: CONFIG.GCP_PROJECT_ID }); + +// -- Retry helpers -- + +function isRetriable(e: any): boolean { + const msg = String(e?.message ?? "").toLowerCase(); + return ( + msg.includes("timeout") || msg.includes("rate limit") || + msg.includes("backend error") || msg.includes("internal error") || + msg.includes("unavailable") || msg.includes("econnreset") || + msg.includes("socket hang up") + ); +} + +function backoffMs(attempt: number): number { + const base = 1000 * Math.pow(2, attempt - 1); + return base + Math.random() * base * 0.3; +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +// -- Query with retry -- + +export async function bqQuery(sql: string, params?: Record): Promise { + let lastErr: any; + for (let attempt = 1; attempt <= CONFIG.BQ_RETRIES; attempt++) { + try { + const [rows] = await bigquery.query({ query: sql, params, projectId: CONFIG.GCP_PROJECT_ID }); + return rows; + } catch (e: any) { + lastErr = e; + if (!isRetriable(e) || attempt === CONFIG.BQ_RETRIES) throw e; + const delay = backoffMs(attempt); + log.warn(`BQ query retry ${attempt}/${CONFIG.BQ_RETRIES} in ${Math.round(delay)}ms`, { error: e.message }); + await sleep(delay); + } + } + throw lastErr; +} + +// -- State queries (fail-fast) -- + +export async function getLastBlock(tableId: string, network: string): Promise { + const rows = await bqQuery( + `SELECT MAX(block_number) AS last_block FROM ${fullTableName(tableId)} WHERE network = @network`, + { network } + ); + const last = rows[0]?.last_block; + if (last === null || last === undefined) return 0; + return Number(last); +} + +export async function countRows(tableId: string, network: string, fromBlock: number, toBlock: number): Promise { + const rows = await bqQuery( + `SELECT COUNT(*) AS cnt FROM ${fullTableName(tableId)} WHERE network = @network AND block_number >= @from AND block_number <= @to`, + { network, from: fromBlock, to: toBlock } + ); + return Number(rows[0]?.cnt ?? 0); +} + +export async function getMaxBlockTimestamp(tableId: string, network: string): Promise { + const rows = await bqQuery( + `SELECT MAX(block_timestamp) AS max_ts FROM ${fullTableName(tableId)} WHERE network = @network`, + { network } + ); + const ts = rows[0]?.max_ts; + if (!ts) return null; + return new Date(ts.value ?? ts); +} + +// -- Infrastructure tables -- + +export async function ensureInfraTables(): Promise { + await bqQuery(` + CREATE TABLE IF NOT EXISTS ${fullTableName("IngestionStatus")} ( + network STRING, + table_id STRING, + ingestion_date DATE, + status STRING, + last_block INT64, + row_count INT64, + started_at TIMESTAMP, + completed_at TIMESTAMP, + error_message STRING, + run_id STRING + ) + PARTITION BY ingestion_date + CLUSTER BY network, table_id + `); + + await bqQuery(` + CREATE TABLE IF NOT EXISTS ${fullTableName("PipelineRuns")} ( + run_id STRING, + mode STRING, + started_at TIMESTAMP, + completed_at TIMESTAMP, + exit_code INT64, + total_rows_merged INT64, + contracts_processed INT64, + contracts_failed INT64, + host STRING, + error_message STRING + ) + PARTITION BY DATE(started_at) + `); +} + +// -- Record keeping -- + +export async function recordIngestionStatus(record: IngestionRecord): Promise { + await bqQuery( + `INSERT INTO ${fullTableName("IngestionStatus")} + (network, table_id, ingestion_date, status, last_block, row_count, started_at, completed_at, error_message, run_id) + VALUES (@network, @tableId, @ingestionDate, @status, @lastBlock, @rowCount, TIMESTAMP(@startedAt), TIMESTAMP(@completedAt), @errorMessage, @runId)`, + { + network: record.network, + tableId: record.tableId, + ingestionDate: record.ingestionDate, + status: record.status, + lastBlock: record.lastBlock, + rowCount: record.rowCount, + startedAt: record.startedAt, + completedAt: record.completedAt, + errorMessage: record.errorMessage || "", + runId: record.runId, + } + ); +} + +export async function recordPipelineRun(record: PipelineRunRecord): Promise { + await bqQuery( + `INSERT INTO ${fullTableName("PipelineRuns")} + (run_id, mode, started_at, completed_at, exit_code, total_rows_merged, contracts_processed, contracts_failed, host, error_message) + VALUES (@runId, @mode, TIMESTAMP(@startedAt), TIMESTAMP(@completedAt), @exitCode, @totalRowsMerged, @contractsProcessed, @contractsFailed, @host, @errorMessage)`, + { + runId: record.runId, + mode: record.mode, + startedAt: record.startedAt, + completedAt: record.completedAt, + exitCode: record.exitCode, + totalRowsMerged: record.totalRowsMerged, + contractsProcessed: record.contractsProcessed, + contractsFailed: record.contractsFailed, + host: record.host, + errorMessage: record.errorMessage || "", + } + ); +} + +// -- Staging + MERGE (the one true write path) -- + +export async function stageAndMerge( + tableId: string, + rows: Record[], + schema: SchemaField[], + runId: string +): Promise<{ rowsMerged: number }> { + if (rows.length === 0) return { rowsMerged: 0 }; + + const staging = stagingTableId(tableId, runId); + const stagingRef = fullTableName(staging); + const productionRef = fullTableName(tableId); + + try { + // Load rows as NDJSON via temp file (BQ load requires file path or GCS URI) + const ndjson = rows.map((r) => JSON.stringify(r)).join("\n"); + const tmpFile = join(tmpdir(), `bq_staging_${runId}_${randomUUID().slice(0, 8)}.ndjson`); + writeFileSync(tmpFile, ndjson); + + const tbl = dataset.table(staging); + const metadata = { + sourceFormat: "NEWLINE_DELIMITED_JSON" as const, + writeDisposition: "WRITE_TRUNCATE" as const, + schema: { fields: schema }, + }; + + // Retry load job + let loadErr: any; + for (let attempt = 1; attempt <= CONFIG.BQ_RETRIES; attempt++) { + try { + await tbl.load(tmpFile, metadata); + loadErr = null; + break; + } catch (e: any) { + loadErr = e; + if (!isRetriable(e) || attempt === CONFIG.BQ_RETRIES) break; + const delay = backoffMs(attempt); + log.warn(`BQ load retry ${attempt}/${CONFIG.BQ_RETRIES} in ${Math.round(delay)}ms`, { error: e.message }); + await sleep(delay); + } + } + + // Clean up temp file + try { unlinkSync(tmpFile); } catch { /* ignore */ } + + if (loadErr) throw loadErr; + + // Build MERGE UPDATE clause (all non-key columns) + const keyCols = new Set(["network", "tx_hash", "log_index"]); + const updateCols = schema.filter((f) => !keyCols.has(f.name)); + const updateClause = updateCols.map((f) => `T.${f.name} = S.${f.name}`).join(", "); + + // MERGE + await bqQuery(` + MERGE ${productionRef} AS T + USING ${stagingRef} AS S + ON T.network = S.network + AND T.tx_hash = S.tx_hash + AND T.log_index = S.log_index + WHEN MATCHED THEN UPDATE SET ${updateClause} + WHEN NOT MATCHED THEN INSERT ROW + `); + + log.info(`MERGE complete: ${rows.length} rows`, { tableId }); + return { rowsMerged: rows.length }; + } finally { + // Always clean up staging table + try { + await bqQuery(`DROP TABLE IF EXISTS ${stagingRef}`); + } catch (e: any) { + log.warn(`Failed to drop staging table ${staging}`, { error: e.message }); + } + } +} diff --git a/projects/onchain-analytics/pipeline-v5/src/config.ts b/projects/onchain-analytics/pipeline-v5/src/config.ts new file mode 100644 index 0000000..1f84e7e --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/src/config.ts @@ -0,0 +1,224 @@ +/** + * config.ts -- Centralized configuration. Fails fast on missing required env vars. + */ + +import { config as loadDotenv } from "dotenv"; +import type { NetworkConfig, ContractConfig, SchemaField } from "./types.js"; + +loadDotenv(); + +// -- Env helpers -- + +function requireEnv(key: string): string { + const v = process.env[key]; + if (!v || v.trim() === "") { + console.error(`[FATAL] Required environment variable ${key} is missing or empty.`); + process.exit(2); + } + return v; +} + +function env(key: string, fallback: string): string { + const v = process.env[key]; + return v && v.trim() !== "" ? v : fallback; +} + +function envInt(key: string, fallback: number): number { + const v = process.env[key]; + if (v === undefined || v === "") return fallback; + const n = parseInt(v, 10); + if (Number.isNaN(n)) { + console.error(`[FATAL] Env ${key} is not a valid integer: "${v}"`); + process.exit(2); + } + return n; +} + +// -- Core config -- + +export const CONFIG = { + GCP_PROJECT_ID: env("GCP_PROJECT_ID", "gooddollar"), + DATASET_ID: env("DATASET_ID", "BlockchainEvents"), + ENVIO_API_TOKEN: requireEnv("ENVIO_API_TOKEN"), + SLACK_WEBHOOK_URL: env("SLACK_WEBHOOK_URL", ""), + LOG_FILE: env("LOG_FILE", "pipeline.log"), + CHUNK_SIZE_TARGET: envInt("CHUNK_SIZE_TARGET", 50_000), + BATCH_DELAY_MS: envInt("BATCH_DELAY_MS", 200), + BQ_RETRIES: envInt("BQ_RETRIES", 5), + HYPERSYNC_RETRIES: envInt("HYPERSYNC_RETRIES", 5), + FRESHNESS_THRESHOLD_HOURS: envInt("FRESHNESS_THRESHOLD_HOURS", 36), +}; + +// -- Networks -- + +export const NETWORKS: Record = { + XDC: { + url: "https://xdc.hypersync.xyz", + name: "XDC", + chainId: 50, + finalityBlocks: 15, + blocksPerDay: 8_640, + }, + CELO: { + url: "https://celo.hypersync.xyz", + name: "CELO", + chainId: 42220, + finalityBlocks: 64, + blocksPerDay: 17_500, + }, +}; + +// -- Common schema columns (every L1 table) -- + +const COMMON_SCHEMA: SchemaField[] = [ + { name: "network", type: "STRING" }, + { name: "chain_id", type: "INTEGER" }, + { name: "block_number", type: "INTEGER" }, + { name: "block_hash", type: "STRING" }, + { name: "block_timestamp", type: "TIMESTAMP" }, + { name: "tx_hash", type: "STRING" }, + { name: "tx_index", type: "INTEGER" }, + { name: "tx_from", type: "STRING" }, + { name: "tx_to", type: "STRING" }, + { name: "tx_value", type: "STRING" }, + { name: "tx_status", type: "INTEGER" }, + { name: "tx_nonce", type: "INTEGER" }, + { name: "log_index", type: "INTEGER" }, + { name: "contract_address", type: "STRING" }, + { name: "event_name", type: "STRING" }, + { name: "ingested_at", type: "TIMESTAMP" }, +]; + +// -- Contract configs -- + +const CLAIM_CONFIG: ContractConfig = { + tableId: "ClaimContractEvents", + schema: [ + ...COMMON_SCHEMA, + { name: "claimer", type: "STRING" }, + { name: "amount", type: "STRING" }, + ], + abi: [ + { + anonymous: false, + inputs: [ + { indexed: true, name: "claimer", type: "address" }, + { indexed: false, name: "amount", type: "uint256" }, + ], + name: "UBIClaimed", + type: "event", + }, + ] as const, + networkBindings: [ + { + network: NETWORKS.XDC, + firstBlock: 95_249_624, + contracts: ["0x22867567E2D80f2049200E25C6F31CB6Ec2F0faf"], + }, + // Celo: re-enable post-MVP + // { network: NETWORKS.CELO, firstBlock: 18_006_679, contracts: ["0x43d72Ff17701B2DA814620735C39C620Ce0ea4A1"] }, + ], + decodeToRow: (eventName, args, logCtx, networkName) => ({ + network: networkName, + chain_id: logCtx.blockNumber > 0 ? 50 : 42220, // derived from networkName in practice + block_number: logCtx.blockNumber, + block_hash: logCtx.blockHash || null, + block_timestamp: logCtx.blockTimestamp > 0 + ? new Date(logCtx.blockTimestamp * 1000).toISOString() + : null, + tx_hash: logCtx.txHash, + tx_index: logCtx.txIndex, + tx_from: null, // Not available in v5 log-only fetch; add if tx fields needed + tx_to: null, + tx_value: "0", + tx_status: 1, + tx_nonce: 0, + log_index: logCtx.logIndex, + contract_address: logCtx.contractAddress, + event_name: eventName, + ingested_at: new Date().toISOString(), + claimer: args.claimer ?? null, + amount: args.amount?.toString() ?? null, + }), +}; + +const INVITE_CONFIG: ContractConfig = { + tableId: "InviteContractEvents", + schema: [ + ...COMMON_SCHEMA, + { name: "inviter", type: "STRING" }, + { name: "invitee", type: "STRING" }, + { name: "bounty_paid", type: "STRING" }, + { name: "inviter_level", type: "STRING" }, + { name: "earned_level", type: "BOOLEAN" }, + ], + abi: [ + { + anonymous: false, + inputs: [ + { indexed: true, name: "inviter", type: "address" }, + { indexed: true, name: "invitee", type: "address" }, + ], + name: "InviteeJoined", + type: "event", + }, + { + anonymous: false, + inputs: [ + { indexed: true, name: "inviter", type: "address" }, + { indexed: true, name: "invitee", type: "address" }, + { indexed: false, name: "bountyPaid", type: "uint256" }, + { indexed: false, name: "inviterLevel", type: "uint256" }, + { indexed: false, name: "earnedLevel", type: "bool" }, + ], + name: "InviterBounty", + type: "event", + }, + ] as const, + networkBindings: [ + { + network: NETWORKS.XDC, + firstBlock: 95_144_756, + contracts: ["0x6bd698566632bf2e81e2278f1656CB24aAF06D2e"], + }, + // Celo: re-enable post-MVP + // { network: NETWORKS.CELO, firstBlock: 18_483_200, contracts: ["0x36829D1Cda92FFF5782d5d48991620664FC857d3"] }, + ], + decodeToRow: (eventName, args, logCtx, networkName) => ({ + network: networkName, + chain_id: 50, + block_number: logCtx.blockNumber, + block_hash: logCtx.blockHash || null, + block_timestamp: logCtx.blockTimestamp > 0 + ? new Date(logCtx.blockTimestamp * 1000).toISOString() + : null, + tx_hash: logCtx.txHash, + tx_index: logCtx.txIndex, + tx_from: null, + tx_to: null, + tx_value: "0", + tx_status: 1, + tx_nonce: 0, + log_index: logCtx.logIndex, + contract_address: logCtx.contractAddress, + event_name: eventName, + ingested_at: new Date().toISOString(), + inviter: args.inviter ?? null, + invitee: args.invitee ?? null, + bounty_paid: args.bountyPaid?.toString() ?? null, + inviter_level: args.inviterLevel?.toString() ?? null, + earned_level: args.earnedLevel ?? null, + }), +}; + +export const CONTRACTS: ContractConfig[] = [CLAIM_CONFIG, INVITE_CONFIG]; + +// -- Helpers -- + +export function fullTableName(tableId: string): string { + return `\`${CONFIG.GCP_PROJECT_ID}.${CONFIG.DATASET_ID}.${tableId}\``; +} + +export function stagingTableId(tableId: string, runId: string): string { + return `_staging_${tableId}_${runId.replace(/-/g, "")}`; +} diff --git a/projects/onchain-analytics/pipeline-v5/src/hypersync.ts b/projects/onchain-analytics/pipeline-v5/src/hypersync.ts new file mode 100644 index 0000000..1acd091 --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/src/hypersync.ts @@ -0,0 +1,200 @@ +/** + * hypersync.ts -- HyperSync client wrapper. + * Streaming, chain tip resolution, day boundary binary search. + */ + +// @ts-ignore -- NAPI-RS generated package has a known TS export bug +import { HypersyncClient } from "@envio-dev/hypersync-client"; +import { decodeEventLog } from "viem"; +import { CONFIG } from "./config.js"; +import { log } from "./log.js"; +import type { NetworkConfig, ContractConfig, DecodedRow, LogContext } from "./types.js"; + +// -- Client cache (one per URL) -- + +const clientCache = new Map(); + +function getClient(network: NetworkConfig): any { + const existing = clientCache.get(network.url); + if (existing) return existing; + const client = (HypersyncClient as any).new({ + url: network.url, + bearerToken: CONFIG.ENVIO_API_TOKEN, + }); + clientCache.set(network.url, client); + return client; +} + +// -- Backoff helper -- + +function backoffMs(attempt: number): number { + const base = 1000 * Math.pow(2, attempt - 1); + return base + Math.random() * base * 0.3; +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +// -- Chain tip -- + +export async function getChainTip(network: NetworkConfig): Promise { + try { + const client = getClient(network); + const height: number = await client.getHeight(); + return height; // No finality subtraction -- daily batch data is always hours/days past finality + } catch (e: any) { + log.warn(`getChainTip failed for ${network.name}: ${e.message}. Will stream to latest available.`); + return undefined; + } +} + +// -- Day boundary resolution -- + +export async function resolveBlockBeforeTimestamp( + network: NetworkConfig, + targetTimestamp: number +): Promise { + try { + const client = getClient(network); + // Binary search: find the last block with timestamp < target + let lo = 0; + let hi: number = await client.getHeight(); + let result = lo; + + while (lo <= hi) { + const mid = Math.floor((lo + hi) / 2); + const query = { + fromBlock: mid, + toBlock: mid + 1, + logs: [], + fieldSelection: { block: ["Number", "Timestamp"] }, + }; + const response = await client.sendReq(query); + const block = response?.data?.blocks?.[0]; + if (!block || block.timestamp === undefined) { + hi = mid - 1; + continue; + } + if (Number(block.timestamp) < targetTimestamp) { + result = mid; + lo = mid + 1; + } else { + hi = mid - 1; + } + } + return result; + } catch (e: any) { + log.warn(`resolveBlockBeforeTimestamp failed for ${network.name}: ${e.message}`); + return undefined; + } +} + +/** + * Resolve the toBlock for daily mode. + * Uses chain tip with finality margin (sufficient for daily batch -- + * data is always 24h+ past finality). + */ +export async function resolveDailyToBlock(network: NetworkConfig): Promise { + return getChainTip(network); +} + +// -- Event streaming -- + +export async function* streamEvents( + network: NetworkConfig, + contracts: string[], + abi: readonly any[], + fromBlock: number, + toBlock?: number +): AsyncGenerator { + const client = getClient(network); + + const query = { + fromBlock, + toBlock, + logs: [{ address: contracts }], + fieldSelection: { + log: [ + "BlockNumber", "BlockHash", "TransactionHash", "TransactionIndex", + "LogIndex", "Address", "Data", "Topic0", "Topic1", "Topic2", "Topic3", + ], + transaction: ["Hash", "From", "To", "Value", "Status", "Nonce"], + block: ["Number", "Timestamp"], + }, + }; + + const stream = await client.stream(query, {}); + let recvAttempts = 0; + + while (true) { + let res: any; + try { + res = await stream.recv(); + recvAttempts = 0; // reset on success + } catch (e: any) { + recvAttempts++; + if (recvAttempts >= CONFIG.HYPERSYNC_RETRIES) { + log.error(`HyperSync stream failed after ${recvAttempts} attempts`, { network: network.name, error: e.message }); + throw e; + } + const delay = backoffMs(recvAttempts); + log.warn(`HyperSync recv error (attempt ${recvAttempts}/${CONFIG.HYPERSYNC_RETRIES}), retrying in ${Math.round(delay)}ms`, { error: e.message }); + await sleep(delay); + continue; + } + + if (res === null) break; + + // Build lookup maps + const txByHash = new Map(); + for (const tx of res.data?.transactions ?? []) { + const h = (tx.hash as string)?.toLowerCase(); + if (h) txByHash.set(h, tx); + } + const blockByNumber = new Map(); + for (const block of res.data?.blocks ?? []) { + const n = block.number !== null && block.number !== undefined ? Number(block.number) : -1; + if (n >= 0) blockByNumber.set(n, block); + } + + const batch: DecodedRow[] = []; + + for (const logEntry of res.data?.logs ?? []) { + const topics = (logEntry.topics || []).filter( + (t: any): t is string => typeof t === "string" + ) as [`0x${string}`, ...`0x${string}`[]]; + if (topics.length === 0) continue; + + try { + const decoded = decodeEventLog({ + abi, + data: ((logEntry.data as string) ?? "0x") as `0x${string}`, + topics, + }); + + const txHash = (logEntry.transactionHash as string) ?? ""; + const block = blockByNumber.get(Number(logEntry.blockNumber)); + + const logCtx: LogContext = { + blockNumber: Number(logEntry.blockNumber), + blockHash: (logEntry.blockHash as string) ?? (block?.hash as string) ?? "", + blockTimestamp: block?.timestamp ? Number(block.timestamp) : 0, + txHash, + txIndex: logEntry.transactionIndex !== undefined ? Number(logEntry.transactionIndex) : 0, + logIndex: Number(logEntry.logIndex), + contractAddress: logEntry.address as string, + }; + + batch.push({ ...logCtx, _eventName: decoded.eventName, _args: decoded.args }); + } catch { + // Event not in ABI -- skip + } + } + + if (batch.length > 0) yield batch; + + // Inter-batch delay to prevent 429 rate-limiting + if (CONFIG.BATCH_DELAY_MS > 0) await sleep(CONFIG.BATCH_DELAY_MS); + } +} diff --git a/projects/onchain-analytics/pipeline-v5/src/index.ts b/projects/onchain-analytics/pipeline-v5/src/index.ts new file mode 100644 index 0000000..68dc04b --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/src/index.ts @@ -0,0 +1,107 @@ +/** + * index.ts -- CLI entry point. Arg parsing, exit codes, PipelineRun recording, alerting. + */ + +import { hostname } from "os"; +import { CONFIG } from "./config.js"; +import { log, flushLogs, RUN_ID } from "./log.js"; +import { runPipeline } from "./pipeline.js"; +import { recordPipelineRun } from "./bq.js"; +import { alertFailure } from "./slack.js"; +import type { PipelineOpts } from "./types.js"; + +const VALID_MODES = ["daily", "backfill"] as const; +type Mode = (typeof VALID_MODES)[number]; + +function parseArgs(): PipelineOpts { + const modeArg = (process.argv[2] || "daily").toLowerCase(); + if (!VALID_MODES.includes(modeArg as Mode)) { + console.error(`Unknown mode: "${modeArg}". Valid: ${VALID_MODES.join(", ")}`); + process.exit(2); + } + + let contracts: string[] | undefined; + let fromBlock: number | undefined; + let toBlock: number | undefined; + + for (const arg of process.argv.slice(3)) { + if (arg.startsWith("--contracts=")) { + contracts = arg.slice("--contracts=".length).split(",").map((s) => s.trim()).filter(Boolean); + } else if (arg.startsWith("--from=")) { + fromBlock = parseInt(arg.slice("--from=".length), 10); + } else if (arg.startsWith("--to=")) { + toBlock = parseInt(arg.slice("--to=".length), 10); + } + } + + return { mode: modeArg as Mode, contracts, fromBlock, toBlock }; +} + +async function main(): Promise { + const opts = parseArgs(); + const startedAt = new Date().toISOString(); + + log.info("Pipeline starting", { + mode: opts.mode, + runId: RUN_ID, + project: CONFIG.GCP_PROJECT_ID, + dataset: CONFIG.DATASET_ID, + contracts: opts.contracts ?? "all", + }); + + let exitCode = 0; + let errorMessage: string | undefined; + let totalRows = 0; + let succeeded = 0; + let failed = 0; + + try { + const result = await runPipeline(opts); + succeeded = result.succeeded; + failed = result.failed; + totalRows = result.totalRows; + + if (failed === 0) { + exitCode = 0; + } else if (succeeded > 0) { + exitCode = 1; // partial + } else { + exitCode = 2; // complete failure + } + } catch (e: any) { + exitCode = 2; + errorMessage = e.message; + log.error(`Pipeline crashed: ${e.message}`, { error: e.message }); + } + + const completedAt = new Date().toISOString(); + + // Record PipelineRun + try { + await recordPipelineRun({ + runId: RUN_ID, + mode: opts.mode, + startedAt, + completedAt, + exitCode, + totalRowsMerged: totalRows, + contractsProcessed: succeeded, + contractsFailed: failed, + host: hostname(), + errorMessage, + }); + } catch (e: any) { + log.warn(`Failed to record PipelineRun: ${e.message}`); + } + + // Alert on failure + if (exitCode !== 0) { + await alertFailure(RUN_ID, opts.mode, exitCode, errorMessage ?? `${failed} contracts failed`, totalRows); + } + + log.info(`Pipeline complete: exit ${exitCode}`, { succeeded, failed, totalRows, runId: RUN_ID }); + await flushLogs(); + process.exit(exitCode); +} + +main(); diff --git a/projects/onchain-analytics/pipeline-v5/src/log.ts b/projects/onchain-analytics/pipeline-v5/src/log.ts new file mode 100644 index 0000000..900d949 --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/src/log.ts @@ -0,0 +1,60 @@ +/** + * log.ts -- Structured logger. JSON to file, human-readable to stdout. + * BigInt-safe. Every line carries run_id. Async file writes. + */ + +import { createWriteStream, WriteStream } from "fs"; +import { randomUUID } from "crypto"; +import { CONFIG } from "./config.js"; + +export const RUN_ID = randomUUID().slice(0, 8); + +type LogLevel = "INFO" | "WARN" | "ERROR"; + +const logStream: WriteStream = createWriteStream(CONFIG.LOG_FILE, { flags: "a" }); +logStream.on("error", (err) => { + process.stderr.write(`[log.ts] Write stream error: ${err.message}\n`); +}); + +function safeReplacer() { + const seen = new WeakSet(); + return (_key: string, value: unknown) => { + if (typeof value === "bigint") return value.toString(); + if (typeof value === "object" && value !== null) { + if (seen.has(value as object)) return "[Circular]"; + seen.add(value as object); + } + return value; + }; +} + +function emit(level: LogLevel, msg: string, meta: Record = {}): void { + const ts = new Date().toISOString(); + const entry = { ts, level, run_id: RUN_ID, msg, ...meta }; + + // JSON to file + try { + logStream.write(JSON.stringify(entry, safeReplacer()) + "\n"); + } catch { + // Never let logging crash the pipeline + } + + // Human-readable to stdout + const metaStr = Object.keys(meta).length > 0 + ? " " + JSON.stringify(meta, safeReplacer()) + : ""; + const prefix = level === "INFO" ? "" : `[${level}] `; + console.log(`${ts} ${prefix}${msg}${metaStr}`); +} + +export const log = { + info: (msg: string, meta?: Record) => emit("INFO", msg, meta), + warn: (msg: string, meta?: Record) => emit("WARN", msg, meta), + error: (msg: string, meta?: Record) => emit("ERROR", msg, meta), +}; + +export function flushLogs(): Promise { + return new Promise((resolve) => { + logStream.end(resolve); + }); +} diff --git a/projects/onchain-analytics/pipeline-v5/src/pipeline.ts b/projects/onchain-analytics/pipeline-v5/src/pipeline.ts new file mode 100644 index 0000000..db0522e --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/src/pipeline.ts @@ -0,0 +1,224 @@ +/** + * pipeline.ts -- Orchestration. Daily and backfill flows. + * Sequential processing, block-aligned chunking, freshness monitoring. + */ + +import { CONFIG, CONTRACTS } from "./config.js"; +import { log, RUN_ID } from "./log.js"; +import { getChainTip, resolveDailyToBlock, streamEvents } from "./hypersync.js"; +import { + getLastBlock, + countRows, + getMaxBlockTimestamp, + ensureInfraTables, + stageAndMerge, + recordIngestionStatus, + recordPipelineRun, +} from "./bq.js"; +import { alertStaleness } from "./slack.js"; +import type { PipelineOpts, PipelineResult, ContractConfig, DecodedRow, LogContext } from "./types.js"; + +/** + * Process one (contract, network) pair. Returns rows merged. + */ +async function processBinding( + cfg: ContractConfig, + bindingIdx: number, + opts: PipelineOpts +): Promise { + const binding = cfg.networkBindings[bindingIdx]; + const { network, firstBlock, contracts } = binding; + const startedAt = new Date().toISOString(); + + // Determine block range + let fromBlock: number; + if (opts.fromBlock !== undefined) { + fromBlock = opts.fromBlock; + } else if (opts.mode === "backfill") { + fromBlock = firstBlock; + } else { + const lastBlock = await getLastBlock(cfg.tableId, network.name); + // +1 is safe because block-aligned chunking guarantees the last block was fully written + fromBlock = lastBlock > 0 ? lastBlock + 1 : firstBlock; + } + + let toBlock: number | undefined; + if (opts.toBlock !== undefined) { + toBlock = opts.toBlock; + } else if (opts.mode === "daily") { + toBlock = await resolveDailyToBlock(network) ?? undefined; + } else { + toBlock = await getChainTip(network) ?? undefined; + } + + log.info(`Processing ${cfg.tableId}/${network.name}: blocks ${fromBlock}..${toBlock ?? "latest"}`); + + // Detect stuck indexer: if toBlock is defined and barely ahead of fromBlock, warn + if (toBlock !== undefined && toBlock <= fromBlock) { + log.warn(`Nothing to fetch: toBlock (${toBlock}) <= fromBlock (${fromBlock}). Indexer may be stuck.`, { + tableId: cfg.tableId, network: network.name, fromBlock, toBlock, + }); + // Record as success with 0 rows (not an error -- the pipeline worked, the source is stale) + const completedAt = new Date().toISOString(); + const today = new Date().toISOString().slice(0, 10); + await recordIngestionStatus({ + network: network.name, + tableId: cfg.tableId, + ingestionDate: today, + status: "success", + lastBlock: fromBlock, + rowCount: 0, + startedAt, + completedAt, + runId: RUN_ID, + }); + return 0; + } + + if (toBlock !== undefined && toBlock - fromBlock < 100) { + log.warn(`Very small range (${toBlock - fromBlock} blocks). Data source may not be advancing.`, { + tableId: cfg.tableId, network: network.name, + }); + } + // Stream and buffer into block-aligned chunks + let buffer: Record[] = []; + let lastBlockInBuffer = -1; + let totalMerged = 0; + + for await (const batch of streamEvents(network, contracts, cfg.abi, fromBlock, toBlock)) { + for (const raw of batch) { + const logCtx: LogContext = { + blockNumber: raw.blockNumber, + blockHash: raw.blockHash, + blockTimestamp: raw.blockTimestamp, + txHash: raw.txHash, + txIndex: raw.txIndex, + logIndex: raw.logIndex, + contractAddress: raw.contractAddress, + }; + const row = cfg.decodeToRow(raw._eventName, raw._args, logCtx, network.name); + if (row === null) continue; + + // Block-aligned flush: flush when buffer >= target AND we've crossed a block boundary + if (buffer.length >= CONFIG.CHUNK_SIZE_TARGET && row.block_number > lastBlockInBuffer) { + const { rowsMerged } = await stageAndMerge(cfg.tableId, buffer, cfg.schema, RUN_ID); + totalMerged += rowsMerged; + log.info(`Chunk merged: ${rowsMerged} rows (total: ${totalMerged})`, { tableId: cfg.tableId, network: network.name }); + buffer = []; + } + + buffer.push(row); + lastBlockInBuffer = row.block_number; + } + } + + // Flush remaining + if (buffer.length > 0) { + const { rowsMerged } = await stageAndMerge(cfg.tableId, buffer, cfg.schema, RUN_ID); + totalMerged += rowsMerged; + log.info(`Final chunk merged: ${rowsMerged} rows (total: ${totalMerged})`, { tableId: cfg.tableId, network: network.name }); + } + + // Record ingestion status + const completedAt = new Date().toISOString(); + const today = new Date().toISOString().slice(0, 10); + await recordIngestionStatus({ + network: network.name, + tableId: cfg.tableId, + ingestionDate: today, + status: "success", + lastBlock: lastBlockInBuffer, + rowCount: totalMerged, + startedAt, + completedAt, + runId: RUN_ID, + }); + + log.info(`Done: ${cfg.tableId}/${network.name} -- ${totalMerged} rows merged`); + return totalMerged; +} + +/** + * Check freshness for all tables. Alerts if data is stale. + */ +async function checkFreshness(): Promise { + const threshold = CONFIG.FRESHNESS_THRESHOLD_HOURS; + const now = Date.now(); + + for (const cfg of CONTRACTS) { + for (const binding of cfg.networkBindings) { + try { + const maxTs = await getMaxBlockTimestamp(cfg.tableId, binding.network.name); + if (!maxTs) continue; + + const hoursStale = (now - maxTs.getTime()) / (1000 * 60 * 60); + if (hoursStale > threshold) { + log.warn(`Data stale: ${cfg.tableId}/${binding.network.name} is ${Math.round(hoursStale)}h behind`); + await alertStaleness(cfg.tableId, binding.network.name, maxTs, hoursStale); + } + } catch (e: any) { + log.warn(`Freshness check failed for ${cfg.tableId}/${binding.network.name}: ${e.message}`); + } + } + } +} + +/** + * Main pipeline entry. Processes all enabled contracts sequentially. + */ +export async function runPipeline(opts: PipelineOpts): Promise { + await ensureInfraTables(); + + // Filter contracts if specified + const contracts = opts.contracts + ? CONTRACTS.filter((c) => opts.contracts!.includes(c.tableId)) + : CONTRACTS; + + if (contracts.length === 0) { + log.error(`No contracts matched filter: ${opts.contracts?.join(", ")}`); + return { succeeded: 0, failed: 0, totalRows: 0 }; + } + + let succeeded = 0; + let failed = 0; + let totalRows = 0; + + for (const cfg of contracts) { + for (let i = 0; i < cfg.networkBindings.length; i++) { + const binding = cfg.networkBindings[i]; + try { + const rows = await processBinding(cfg, i, opts); + totalRows += rows; + succeeded++; + } catch (e: any) { + log.error(`Failed: ${cfg.tableId}/${binding.network.name}: ${e.message}`, { error: e.message }); + // Record failed ingestion status + const today = new Date().toISOString().slice(0, 10); + try { + await recordIngestionStatus({ + network: binding.network.name, + tableId: cfg.tableId, + ingestionDate: today, + status: "failed", + lastBlock: 0, + rowCount: 0, + startedAt: new Date().toISOString(), + completedAt: new Date().toISOString(), + errorMessage: e.message?.slice(0, 500), + runId: RUN_ID, + }); + } catch { + // Don't let recording failure mask the original error + } + failed++; + } + } + } + + // Freshness check (daily mode only) + if (opts.mode === "daily") { + await checkFreshness(); + } + + return { succeeded, failed, totalRows }; +} diff --git a/projects/onchain-analytics/pipeline-v5/src/slack.ts b/projects/onchain-analytics/pipeline-v5/src/slack.ts new file mode 100644 index 0000000..eccd440 --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/src/slack.ts @@ -0,0 +1,74 @@ +/** + * slack.ts -- Webhook alerting. No-op if SLACK_WEBHOOK_URL is empty. + */ + +import { CONFIG } from "./config.js"; +import { log } from "./log.js"; + +async function postToSlack(payload: Record): Promise { + if (!CONFIG.SLACK_WEBHOOK_URL) return; + + try { + const response = await fetch(CONFIG.SLACK_WEBHOOK_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + if (!response.ok) { + log.warn(`Slack webhook returned ${response.status}`, { status: response.status }); + } + } catch (e: any) { + log.warn(`Slack webhook failed: ${e.message}`); + } +} + +export async function alertFailure( + runId: string, + mode: string, + exitCode: number, + error: string, + rowsProcessed: number +): Promise { + await postToSlack({ + text: `:rotating_light: *Pipeline failed* (exit ${exitCode})`, + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: [ + `:rotating_light: *Pipeline failed*`, + `*Mode:* ${mode} | *Exit:* ${exitCode} | *Run:* ${runId}`, + `*Rows processed:* ${rowsProcessed}`, + `*Error:* \`${error.slice(0, 200)}\``, + ].join("\n"), + }, + }, + ], + }); +} + +export async function alertStaleness( + tableId: string, + network: string, + lastTimestamp: Date, + hoursStale: number +): Promise { + await postToSlack({ + text: `:warning: Data stale: ${tableId}/${network} is ${Math.round(hoursStale)}h behind`, + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: [ + `:warning: *Data freshness alert*`, + `*Table:* ${tableId} | *Network:* ${network}`, + `*Last data:* ${lastTimestamp.toISOString()} (${Math.round(hoursStale)}h ago)`, + `*Threshold:* ${CONFIG.FRESHNESS_THRESHOLD_HOURS}h`, + ].join("\n"), + }, + }, + ], + }); +} diff --git a/projects/onchain-analytics/pipeline-v5/src/types.ts b/projects/onchain-analytics/pipeline-v5/src/types.ts new file mode 100644 index 0000000..22029a7 --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/src/types.ts @@ -0,0 +1,88 @@ +/** + * types.ts -- Shared interfaces for the v5 pipeline. + */ + +export interface NetworkConfig { + url: string; + name: string; + chainId: number; + finalityBlocks: number; + blocksPerDay: number; +} + +export interface SchemaField { + name: string; + type: string; +} + +export interface NetworkBinding { + network: NetworkConfig; + firstBlock: number; + contracts: string[]; +} + +export interface ContractConfig { + tableId: string; + schema: SchemaField[]; + abi: readonly any[]; + networkBindings: NetworkBinding[]; + decodeToRow: ( + eventName: string, + args: any, + logCtx: LogContext, + networkName: string + ) => Record | null; +} + +export interface LogContext { + blockNumber: number; + blockHash: string; + blockTimestamp: number; // Unix seconds + txHash: string; + txIndex: number; + logIndex: number; + contractAddress: string; +} + +export interface DecodedRow { + [key: string]: any; +} + +export interface PipelineOpts { + mode: "daily" | "backfill"; + contracts?: string[]; + fromBlock?: number; + toBlock?: number; +} + +export interface PipelineResult { + succeeded: number; + failed: number; + totalRows: number; +} + +export interface IngestionRecord { + network: string; + tableId: string; + ingestionDate: string; + status: "success" | "failed" | "partial"; + lastBlock: number; + rowCount: number; + startedAt: string; + completedAt: string; + errorMessage?: string; + runId: string; +} + +export interface PipelineRunRecord { + runId: string; + mode: string; + startedAt: string; + completedAt: string; + exitCode: number; + totalRowsMerged: number; + contractsProcessed: number; + contractsFailed: number; + host: string; + errorMessage?: string; +} diff --git a/projects/onchain-analytics/pipeline-v5/tsconfig.json b/projects/onchain-analytics/pipeline-v5/tsconfig.json new file mode 100644 index 0000000..e7fe927 --- /dev/null +++ b/projects/onchain-analytics/pipeline-v5/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/projects/onchain-analytics/pipeline/index.ts b/projects/onchain-analytics/pipeline/index.ts index 447568d..c2ab604 100644 --- a/projects/onchain-analytics/pipeline/index.ts +++ b/projects/onchain-analytics/pipeline/index.ts @@ -188,6 +188,9 @@ const CONTRACT_CONFIGS: Record = { const VALID_MODES = ["backfill", "append"] as const; type Mode = (typeof VALID_MODES)[number]; +// P2: Configurable inter-batch delay to prevent HyperSync 429 cascade during catchup +const BATCH_DELAY_MS = Number(process.env.BATCH_DELAY_MS ?? 200); + // ============================================================ // BigQuery helpers // ============================================================ @@ -195,7 +198,17 @@ type Mode = (typeof VALID_MODES)[number]; const bigquery = new BigQuery({ projectId: GCP_PROJECT_ID }); const dataset = bigquery.dataset(DATASET_ID, { projectId: GCP_PROJECT_ID }); -async function insertWithRetry(rows: any[], tableId: string, retries = 3): Promise { +// P7: Exponential backoff with jitter for BQ insert retries +const MAX_RETRIES = 5; +const BASE_DELAY_MS = 1000; + +function backoffDelay(attempt: number): number { + const exponential = BASE_DELAY_MS * Math.pow(2, attempt - 1); + const jitter = Math.random() * exponential * 0.3; + return exponential + jitter; +} + +async function insertWithRetry(rows: any[], tableId: string, retries = MAX_RETRIES): Promise { const table = dataset.table(tableId); const rowsWithInsertId = rows.map((row) => ({ insertId: `${row.network}:${row.tx_hash}:${row.log_index}`, @@ -210,12 +223,13 @@ async function insertWithRetry(rows: any[], tableId: string, retries = 3): Promi // Log it clearly so schema mismatches are immediately obvious. const firstRowErr = e.errors?.[0]; if (firstRowErr?.errors?.length > 0) { - console.error(" BQ rejection — first row errors:", JSON.stringify(firstRowErr.errors)); - console.error(" BQ rejection — first row data: ", JSON.stringify(firstRowErr.row)); + console.error(" BQ rejection -- first row errors:", JSON.stringify(firstRowErr.errors)); + console.error(" BQ rejection -- first row data: ", JSON.stringify(firstRowErr.row)); } if (attempt === retries) throw e; - console.warn(` Insert failed (attempt ${attempt}/${retries}), retrying in 3s...`); - await new Promise((r) => setTimeout(r, 3000)); + const delay = backoffDelay(attempt); + console.warn(` Insert failed (attempt ${attempt}/${retries}), retrying in ${Math.round(delay)}ms...`); + await new Promise((r) => setTimeout(r, delay)); } } } @@ -226,7 +240,8 @@ async function getChainTip(networkUrl: string, finalityBlocks: number): Promise< const height = await client.getHeight(); return Math.max(0, height - finalityBlocks); } catch (e: any) { - console.warn(`Could not get chain tip: ${e.message}; fetching to latest`); + // P3: Explicit warning -- finality guard is dropped, pipeline will fetch to chain tip + console.warn(`[WARN] getChainTip failed: ${e.message}. Finality guard DISABLED for this run -- fetching to latest block. Data may include unfinalized blocks.`); return undefined; } } @@ -247,6 +262,9 @@ async function getLastBlockForNetwork(tableId: string, network: string): Promise // Stream and ingest events for one (contract, network) pair // ============================================================ +// P1: Track rows actually written to BQ (survives throws) +let globalInsertedCount = 0; + async function syncEvents( cfg: ContractConfig, network: NetworkConfig, @@ -293,6 +311,7 @@ async function syncEvents( const stream = await client.stream(query, {}); let totalDecoded = 0; let totalSkipped = 0; + let totalInserted = 0; const BATCH_SIZE = 1000; let pendingRows: any[] = []; const ingestedAt = new Date().toISOString(); @@ -367,17 +386,23 @@ async function syncEvents( while (pendingRows.length >= BATCH_SIZE) { const chunk = pendingRows.splice(0, BATCH_SIZE); await insertWithRetry(chunk, cfg.tableId); - console.log(`[${network.name}] Inserted ${chunk.length} rows (total: ${totalDecoded})`); + totalInserted += chunk.length; + globalInsertedCount += chunk.length; + console.log(`[${network.name}] Inserted ${chunk.length} rows (total inserted: ${totalInserted})`); + // P2: Inter-batch delay to prevent HyperSync 429 rate-limiting + if (BATCH_DELAY_MS > 0) await new Promise((r) => setTimeout(r, BATCH_DELAY_MS)); } } if (pendingRows.length > 0) { await insertWithRetry(pendingRows, cfg.tableId); - console.log(`[${network.name}] Inserted final ${pendingRows.length} rows (total: ${totalDecoded})`); + totalInserted += pendingRows.length; + globalInsertedCount += pendingRows.length; + console.log(`[${network.name}] Inserted final ${pendingRows.length} rows (total inserted: ${totalInserted})`); } - console.log(`[${network.name}] Done. Decoded: ${totalDecoded}, skipped: ${totalSkipped}.`); - return totalDecoded; + console.log(`[${network.name}] Done. Decoded: ${totalDecoded}, inserted: ${totalInserted}, skipped: ${totalSkipped}.`); + return totalInserted; } // ============================================================ @@ -396,7 +421,8 @@ async function main() { const contractArg = (process.argv[3] || "all").toLowerCase(); if (!VALID_MODES.includes(modeArg as Mode)) { - throw new Error(`Unknown mode: "${modeArg}". Valid modes: ${VALID_MODES.join(", ")}`); + console.error(`Unknown mode: "${modeArg}". Valid modes: ${VALID_MODES.join(", ")}`); + process.exit(2); } const allKeys = Object.keys(CONTRACT_CONFIGS); @@ -405,41 +431,61 @@ async function main() { : contractArg.split(",").map((s) => s.trim()).filter((k) => k in CONTRACT_CONFIGS); if (contractKeys.length === 0) { - throw new Error(`Unknown contract(s): "${contractArg}". Valid: ${allKeys.join(", ")}, all`); + console.error(`Unknown contract(s): "${contractArg}". Valid: ${allKeys.join(", ")}, all`); + process.exit(2); } const mode = modeArg as Mode; console.log(`\nMode: ${mode} | Contracts: ${contractKeys.join(", ")}`); - console.log(`Project: ${GCP_PROJECT_ID} | Dataset: ${DATASET_ID}\n`); + console.log(`Project: ${GCP_PROJECT_ID} | Dataset: ${DATASET_ID}`); + console.log(`Batch delay: ${BATCH_DELAY_MS}ms\n`); - let grandTotal = 0; + // P6: Track per-contract success/failure for structured exit codes + let succeededContracts = 0; + let failedContracts = 0; for (const key of contractKeys) { try { const cfg = CONTRACT_CONFIGS[key]; - console.log(`\n=== ${key.toUpperCase()} → ${DATASET_ID}.${cfg.tableId} ===`); + console.log(`\n=== ${key.toUpperCase()} -> ${DATASET_ID}.${cfg.tableId} ===`); for (const network of cfg.networks) { if (mode === "backfill") { console.log(`\n--- BACKFILL: ${network.name} (chainId ${network.chainId}) from block ${network.firstBlock} ---`); - grandTotal += await syncEvents(cfg, network, network.firstBlock); + await syncEvents(cfg, network, network.firstBlock); } else { const lastBlock = await getLastBlockForNetwork(cfg.tableId, network.name); const startBlock = lastBlock > 0 ? lastBlock + 1 : network.firstBlock; const safeTip = await getChainTip(network.url, network.finalityBlocks); console.log(`\n--- APPEND: ${network.name} (chainId ${network.chainId}) from block ${startBlock}${safeTip ? ` to ${safeTip}` : ""} ---`); - grandTotal += await syncEvents(cfg, network, startBlock, safeTip); + await syncEvents(cfg, network, startBlock, safeTip); } } + succeededContracts++; } catch (e: any) { console.error(`[${key}] Error processing contract: ${e.message}`); + failedContracts++; } } - console.log(`\n=== DONE. Total events inserted: ${grandTotal} ===`); + // P1: Report actual rows written (globalInsertedCount survives throws) + console.log(`\n=== DONE. Total events inserted: ${globalInsertedCount} (${succeededContracts} contracts succeeded, ${failedContracts} failed) ===`); + + // P6: Structured exit codes + if (failedContracts === 0) { + process.exit(0); // Full success + } else if (succeededContracts > 0) { + process.exit(1); // Partial -- some contracts failed + } else { + process.exit(2); // Complete failure + } } main().catch((error) => { console.error(error); - process.exit(1); + // P1: Even on fatal crash, report what was written + if (globalInsertedCount > 0) { + console.log(`(Partial progress: ${globalInsertedCount} rows were inserted before crash)`); + } + process.exit(2); }); diff --git a/projects/superfluid-s4/dashboard/demo.html b/projects/superfluid-s4/dashboard/demo.html deleted file mode 100644 index b26d336..0000000 --- a/projects/superfluid-s4/dashboard/demo.html +++ /dev/null @@ -1,955 +0,0 @@ - - - - - - GoodDollar SUP Rewards -- Season 6 - - - - - -
- -
-

GoodDollar SUP Rewards -- Campaign Dashboard S5 TEST DATA

-

Testing with Season 5 live data (campaign 511, 2,611 participants)

-
-
-
622,000
-
Total SUP
-
-
-
--
-
Participants
-
-
-
--
-
Points Earned
-
-
-
--
-
Total Events
-
-
-
-
-
-
- Loading... - -
-
- - -
-
-

Pool 1: GoodDollar Actions

-
217,700 SUP
-
-
-
--
-
Total Points
-
-
-
--
-
Participants
-
-
-
--
-
Events
-
-
-
    -
  • Claim UBI daily1 pt/claim
    • -
  • Successful invite5 pts
    • -
  • Vote on FlowState5 pts/epoch
    • -
-
Source: Superfluid Points API, Campaign --
-
-
-

Pool 2: Ecosystem Funding

-
404,300 SUP
-
-
-
--
-
Total Points
-
-
-
--
-
Participants
-
-
-
--
-
Events
-
-
-
    -
  • Stream to GoodBuilders S4 council2 pts/$1 G$
    • -
  • Donate to Gardens pool1 pt/$1 G$
    • -
  • Stream to Gardens pool2 pts/$1 G$
    • -
-
Source: Superfluid Points API, Campaign --
-
-
- - -
-

Leaderboard

-
- - -
-
-
- - Loading leaderboard... -
-
- -
- - -
-

Reward Calculator

-

- Enter your wallet address to see your points and estimated SUP rewards. -

-
- - -
-
-
-
-
0
-
Pool 1 Points
-
-
-
0
-
Pool 2 Points
-
-
-
0
-
Est. Total SUP
-
-
-
--
-
Rank (Pool 1)
-
-
-
- - -
-

How to Participate

-

- Earn points by completing actions below. More points = more SUP rewards. -

-
- - Claim G$ UBI - Open GoodWallet and claim your daily UBI - 1 point per claim - - - Invite Users - Share your invite link and earn points for each new user - 5 points per successful invite - - - Stream to GoodBuilders S4 - Open a stream on FlowState council - 2 points per $1 G$ streamed - - - Donate to Gardens Pool - One-time donation to the Gardens pool - 1 point per $1 G$ donated - - - Stream to Gardens Pool - Open a stream to the Gardens pool - 2 points per $1 G$ streamed - - - Claim SUP Rewards - Claim your earned SUP tokens once available - Claim portal - -
-
- - -
-

Campaign Analytics

-
-
-
-

Daily Events (Full Season)

- -
-
-
-
-

Points Distribution (Top 100)

- -
-
-

Cumulative Participants

- -
-
-
-
- - Loading analytics (fetching full season events)... -
- -
-
- - - - diff --git a/projects/superfluid-s4/dashboard/index.html b/projects/superfluid-s4/dashboard/index.html index 8bb4ad1..4754de5 100644 --- a/projects/superfluid-s4/dashboard/index.html +++ b/projects/superfluid-s4/dashboard/index.html @@ -3,7 +3,7 @@ - GoodDollar SUP Rewards -- Season 6 + GoodDollar SUP Rewards Campaign Dashboard