Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions .github/workflows/pipeline-daily.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
name: pipeline-daily

# Runs the onchain analytics pipeline daily at 01:00 UTC (1h after midnight
# to ensure yesterday's data is fully finalized on all chains), then refreshes
# the dbt models so dashboards show the latest data.
#
# Required secrets:
# ENVIO_API_TOKEN -- HyperSync API token
# GCP_SA_KEY -- GCP service account JSON key (BigQuery access)
# SLACK_WEBHOOK_URL -- Slack incoming webhook for alerts (optional)
#
# Manual trigger: use workflow_dispatch to run on demand.

on:
schedule:
- cron: '0 1 * * *' # 01:00 UTC daily
workflow_dispatch: # Manual trigger from GitHub UI

concurrency:
group: pipeline-daily
cancel-in-progress: false # Let running pipeline finish; don't start another

defaults:
run:
working-directory: projects/onchain-analytics/pipeline-v5

jobs:
ingest:
runs-on: ubuntu-latest
timeout-minutes: 30

steps:
- uses: actions/checkout@v4

- uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: projects/onchain-analytics/pipeline-v5/package-lock.json

- name: Install dependencies
run: npm ci

- name: Authenticate to GCP
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_SA_KEY }}

- name: Run pipeline (daily)
env:
ENVIO_API_TOKEN: ${{ secrets.ENVIO_API_TOKEN }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
run: npx tsx src/index.ts daily

# dbt refresh runs after successful ingestion
# Uncomment when dbt is ready for automated runs:
#
# transform:
# needs: ingest
# runs-on: ubuntu-latest
# timeout-minutes: 15
# defaults:
# run:
# working-directory: projects/onchain-analytics/gd_dbt
# steps:
# - uses: actions/checkout@v4
# - uses: actions/setup-python@v5
# with:
# python-version: '3.11'
# - name: Install dbt
# run: pip install dbt-bigquery
# - name: Authenticate to GCP
# uses: google-github-actions/auth@v2
# with:
# credentials_json: ${{ secrets.GCP_SA_KEY }}
# - name: dbt run
# run: dbt run --select marts
53 changes: 52 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ projects/dashboard-scripts/handoff.md
docs/data-dictionary/
docs/datasources/
docs/other/

# Internal strategy / planning docs â€" local only, never published
projects/onchain-analytics/docs/antseed-tracking-plan.md
projects/onchain-analytics/docs/tracking-plan-research.md
docs/utm-tracking-architecture.md
# Agent context & process files — local only, never published
# This is a public repo: instructions files of any kind do not belong here,
# only working code and reader-facing documentation. Keep this list ahead of
Expand All @@ -61,3 +64,51 @@ CLAUDE.md
**/plan.md
**/audit.md
**/specs/

# Private working files - local only, never committed
_internal/

# Investigation working files - local only
investigations/jul15-spike/
investigations/sybil-farm/amplitude-prompts.md
investigations/sybil-farm/cross_reference_results.json
queries/dune/reserve-governance/cross_reference.py
queries/dune/sybil-monitor/BUILD-GUIDE.md

investigations/sybil-farm/startup-prompt.md

# ---------------------------------------------------------------------------
# THE UNDERSCORE CONVENTION (added 2026-07-30)
# A leading "_" means LOCAL-ONLY working file: raw dumps, exports, scratch
# analysis, and anything candid about people. One rule to remember instead of
# listing every future filename. Prefix a file with "_" and it can never be
# committed to this public repo by accident.
# NOTE: *.yml is deliberately NOT matched — dbt uses _sources.yml / _schema.yml
# as real tracked code.
_*.md
_*.txt
_*.json
_*.csv
_*.tsv
**/_*.md
**/_*.txt
**/_*.json
**/_*.csv
**/_*.tsv

# Agent kickoff / seed prompts (same family as the handoff + prompt rules above)
**/*kickoff*.md
**/*startup-prompt*.md
**/*-primer.md

# Internal meeting material + working digests — never reader-facing
**/meeting-prep*.md
**/meeting-notes*.md
**/project-checklist-meeting.md
docs/monday-digest-*.md
docs/monday-digest-pull-guide.md
docs/monday-digest-template.md

# Candid docs about colleagues / stakeholders — NEVER in a repo others can read
**/*-alignment-analysis.md
**/*-profile.md
Loading