cloud-data-platform/Makefile at main · Hediii-coder/cloud-data-platform · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
# ============================================================
# Cloud Data Platform — Makefile
# One-command operations for the data platform
# ============================================================

.PHONY: help build up down restart logs status clean test dbt-run dbt-test dbt-docs

help: ## Show this help message
	@echo "╔══════════════════════════════════════════════════════╗"
	@echo "║        Cloud Data Platform — Commands               ║"
	@echo "╠══════════════════════════════════════════════════════╣"
	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
		awk 'BEGIN {FS = ":.*?## "}; {printf "║  \033[36m%-18s\033[0m %s\n", $$1, $$2}'
	@echo "╚══════════════════════════════════════════════════════╝"

# ---- Infrastructure ----

setup: ## First-time setup: copy env, build, and start everything
	@echo "🚀 Setting up Cloud Data Platform..."
	@cp -n .env.example .env 2>/dev/null || true
	@chmod +x infrastructure/minio/create-buckets.sh
	@docker compose build
	@docker compose up -d
	@echo "✅ Platform is starting up!"
	@echo "   Airflow UI: http://localhost:8080 (airflow/airflow)"
	@echo "   MinIO UI:   http://localhost:9001 (minio_admin/minio_password)"
	@echo "   Postgres:   localhost:5432 (warehouse/warehouse123)"

build: ## Build all Docker images
	docker compose build

up: ## Start all services
	docker compose up -d

down: ## Stop all services
	docker compose down

restart: ## Restart all services
	docker compose down && docker compose up -d

logs: ## Tail logs from all services
	docker compose logs -f

logs-airflow: ## Tail Airflow logs only
	docker compose logs -f airflow-webserver airflow-scheduler airflow-worker

status: ## Show status of all services
	docker compose ps

# ---- dbt ----

dbt-run: ## Run all dbt models
	docker compose exec airflow-worker bash -c "cd /opt/dbt && dbt run"

dbt-test: ## Run all dbt tests
	docker compose exec airflow-worker bash -c "cd /opt/dbt && dbt test"

dbt-build: ## Run dbt build (run + test)
	docker compose exec airflow-worker bash -c "cd /opt/dbt && dbt build"

dbt-docs: ## Generate and serve dbt documentation
	docker compose exec airflow-worker bash -c "cd /opt/dbt && dbt docs generate"
	@echo "📖 dbt docs generated. Run 'make dbt-docs-serve' to view."

dbt-compile: ## Compile dbt models (dry run)
	docker compose exec airflow-worker bash -c "cd /opt/dbt && dbt compile"

dbt-deps: ## Install dbt packages
	docker compose exec airflow-worker bash -c "cd /opt/dbt && dbt deps"

dbt-seed: ## Load seed data
	docker compose exec airflow-worker bash -c "cd /opt/dbt && dbt seed"

dbt-snapshot: ## Run dbt snapshots (SCD Type-2)
	docker compose exec airflow-worker bash -c "cd /opt/dbt && dbt snapshot"

# ---- Testing ----

test: ## Run all tests (DAG integrity + unit tests)
	docker compose exec airflow-worker python -m pytest /opt/airflow/tests/ -v

test-dags: ## Validate DAGs parse without errors
	docker compose exec airflow-worker python -m pytest /opt/airflow/tests/test_dag_integrity.py -v

# ---- Maintenance ----

clean: ## Stop services and remove all volumes (DESTRUCTIVE!)
	@echo "⚠️  This will delete all data (Postgres, MinIO, logs)!"
	@read -p "Are you sure? [y/N] " confirm && [ "$$confirm" = "y" ] && \
		docker compose down -v --remove-orphans || echo "Cancelled."

shell-airflow: ## Open a shell in the Airflow worker
	docker compose exec airflow-worker bash

shell-postgres: ## Open psql in the warehouse database
	docker compose exec postgres psql -U warehouse -d ecommerce_dw

reset-airflow: ## Reset Airflow database (DESTRUCTIVE!)
	docker compose exec airflow-webserver airflow db reset --yes