diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..0414fe9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 diff --git a/.github/workflows/docker-deploy-dev.yml b/.github/workflows/docker-deploy-dev.yml index 3d54b47..a1d43ee 100644 --- a/.github/workflows/docker-deploy-dev.yml +++ b/.github/workflows/docker-deploy-dev.yml @@ -12,23 +12,25 @@ jobs: steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v7 + with: + persist-credentials: false - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v4.6.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v4.1.0 with: driver: docker - - name: Build letsencrypt - id: build-letsencrypt - uses: docker/build-push-action@v2 + - name: Build mantis + id: build-mantis + uses: docker/build-push-action@v7.1.0 with: push: true tags: sublimesec/nginx-letsencrypt:latest diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml deleted file mode 100644 index 3134b37..0000000 --- a/.github/workflows/docker-tests.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Platform PR CI - -on: - push: - branches: [ "**" ] - workflow_dispatch: {} - -concurrency: - group: ${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - tests: - name: Validate Platform Installation - runs-on: ubuntu-22.04 - - permissions: - id-token: write - contents: read - - steps: - - name: Checkout - uses: actions/checkout@v3 - - # main should never require private images - - name: Configure AWS Credentials - if: github.ref != 'refs/heads/main' - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::948971135452:role/ci-sublime-platform-ecr-read - aws-region: us-east-1 - - - name: Login to Amazon ECR - if: github.ref != 'refs/heads/main' - uses: aws-actions/amazon-ecr-login@v2 - with: - registries: "948971135452" - - - name: Make ECR credentials available to sudo - if: github.ref != 'refs/heads/main' - run: sudo cp -r $HOME/.docker /root/ - - - name: Install Platform - run: | - interactive=false clone_platform=false ./install-and-launch.sh - - - name: Check Health - run: | - ./postflight_checks.sh diff --git a/.gitignore b/.gitignore index b3e9a4d..5333d47 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ certbot.env sublime.env *.swp +.DS_Store diff --git a/LICENSE b/LICENSE index 90933c4..574d1b1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Sublime Security +Copyright (c) 2021-2026 Sublime Security Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index e18fce1..14ebf1e 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,43 @@ - Sublime Logo + Sublime Logo Sublime Platform ========== -by Sublime Security +by [Sublime Security](https://sublime.security/) Overview --------- -An open, adaptable email security platform for writing, running, and sharing custom detection and response rules to block phishing attacks, hunt for threats, and more. +A free and open platform for detecting and preventing email attacks like BEC, malware, and credential phishing. Gain visibility and control, hunt for advanced threats, and collaborate with the community. -Why? +Sublime uses Message Query Language (MQL), a domain-specific language purpose-built for describing behavior in email. MQL is email provider agnostic, enabling defenders to write, run, and share Detections-as-Code. + +Learn more about MQL: [Introduction to Message Query Language](https://sublime.security/blog/introduction-to-message-query-language-mql) + +Docker Usage & Caveats ---------- -Traditional email security is a one-size-fits-all black box. +This Docker deployment is intended for small-medium size deployments and for testing purposes ONLY (limited to 100 active mailboxes). For the best Sublime experience, we recommend the [AWS Cloud-native deployment](https://docs.sublime.security/docs/aws-cloudformation) or [Sublime Managed Cloud](https://docs.sublime.security/docs/sublime-managed), which can support any number of mailboxes, is resilient, and has the latest features. The docker deployment allows you to gain hands on experience, but will only receive best effort support (no long term support). + +[Learn more about feature restrictions for Docker Compose](https://docs.sublime.security/docs/docker-requirements-and-limitations) -The Sublime Platform **gives defenders control over their email environment** and uses an intuitive, interoperable, purpose-built domain-specific language (DSL). +The Sublime Platform Docker Compose ships as an entire setup. Modifying the docker-compose file or using our docker images within your own implementation is not supported. Setup ---------- ```console -curl -sL https://sublime.security/install.sh | sh +curl -sL https://raw.githubusercontent.com/sublime-security/sublime-platform/main/install-and-launch.sh | sh ``` [View Docker Quickstart](https://docs.sublimesecurity.com/docs/quickstart-docker) +[View other deployment methods](https://sublime.security/start) + Detection rules ---------- -Open-source detection rules are maintained in the [sublime-rules repo](https://github.com/sublime-security/sublime-rules). - +Open-source detection rules and links to community Feeds are maintained in the [sublime-rules repo](https://github.com/sublime-security/sublime-rules). Learn more ---------- -- [Sublime overview](https://sublime.security) - [Docs](https://docs.sublimesecurity.com) -- [Message Query Language (MQL) reference](https://docs.sublimesecurity.com/docs/message-query-language) - Sublime's DSL purpose-built for email analysis +- [API](https://docs.sublimesecurity.com/reference/introduction) - [Release log](https://new.sublimesecurity.com) +- [Message Query Language (MQL)](https://docs.sublimesecurity.com/docs/message-query-language) diff --git a/assets/sublime-logo.png b/assets/sublime-logo.png new file mode 100644 index 0000000..b7a666a Binary files /dev/null and b/assets/sublime-logo.png differ diff --git a/docker-compose.yml b/docker-compose.yml index 38ac189..2c6d95f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: sublime_postgres: image: postgres:13.2 + command: -c 'max_connections=200' restart: unless-stopped container_name: sublime_postgres environment: @@ -9,52 +10,35 @@ services: PGDATA: /data/postgres env_file: sublime.env volumes: - - postgres:/data/postgres - ports: - - "5432:5432" + - postgres_e2e:/data/postgres # different volume than dev-be to avoid overwriting data networks: - net + ports: + - 5432:5432 sublime_dashboard: image: sublimesec/dashboard:dev restart: unless-stopped container_name: sublime_dashboard ports: - - "0.0.0.0:3000:80" + - 0.0.0.0:3000:80 networks: - net env_file: sublime.env environment: - MANTIS_HOST_URL: "http://host.docker.internal:8000" + MANTIS_HOST_URL: http://host.docker.internal:8000 sublime_redis: image: redis:6.2 restart: unless-stopped container_name: sublime_redis command: redis-server --loglevel warning - ports: - - "6379:6379" networks: - net - sublime_nipper: - image: 948971135452.dkr.ecr.us-east-1.amazonaws.com/nipper-python-dev:dev - restart: unless-stopped - container_name: sublime_nipper - env_file: sublime.env ports: - - "8300:8000" - networks: - - net - environment: - NIPPER_BUCKET: "nipper-artifacts" - BUCKET_ENDPOINT_URL: "http://sublimes3:8110" - DD_TRACE_ENABLED: false - depends_on: - - sublime_create_buckets + - 6379:6379 sublime_strelka_frontend: image: sublimesec/strelka-frontend:0.3 restart: unless-stopped command: strelka-frontend - ports: - - "57314:57314" networks: - net volumes: @@ -64,6 +48,8 @@ services: - sublime_strelka_coordinator deploy: replicas: 1 + ports: + - 57314:57314 sublime_strelka_backend: image: sublimesec/strelka-backend:0.3 restart: unless-stopped @@ -96,92 +82,125 @@ services: networks: - net sublime_screenshot_service: - image: sublimesec/render-email-html:0.1 + image: sublimesec/render-email-html:0.2 restart: unless-stopped - ports: - - "8100:8100" environment: - S3_ENDPOINT=http://sublimes3:8110 - SCREENSHOT_BUCKET=email-screenshots - AWS_REGION=us-east-1 - DISABLE_DD=true container_name: sublime_screenshot_service - env_file: sublime.env networks: - net + ports: + - 8100:8100 + env_file: sublime.env depends_on: - sublime_create_buckets - sublime_azurite: - container_name: sublime_azurite - image: mcr.microsoft.com/azure-storage/azurite - restart: unless-stopped - networks: - - net - ports: - - "10000:10000" # Blob service - - "10001:10001" # Queue service - - "10002:10002" # Table service - volumes: - - azurite_data:/data - command: azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --location /data --skipApiVersionCheck - sublime_create_azure_blob_containers: - image: mcr.microsoft.com/azure-cli - depends_on: - - sublime_azurite + # cgr.dev/chainguard/minio has no shell, so anything needing one - fixing up volume + # ownership for minio's non-root UID, and writing MINIO_ROOT_USER/PASSWORD out to files from + # the AWS_* vars already in sublime.env - happens here instead, before minio itself starts. + sublimes3_init: + image: busybox + container_name: sublimes3_init networks: - net - environment: - AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://sublime_azurite:10000/devstoreaccount1;" - command: > - bash -c " - az storage container create --name email-screenshots && - az storage container create --name events && - az storage container create --name message-storage && - az storage container create --name message-export && - az storage container create --name nipper-artifacts - " + env_file: sublime.env + volumes: + - s3_data:/data + - s3_creds:/creds + command: + - sh + - -c + - | + printf '%s' "$$AWS_ACCESS_KEY_ID" > /creds/root_user && printf '%s' "$$AWS_SECRET_ACCESS_KEY" > /creds/root_password && chmod 400 /creds/root_user /creds/root_password && chown -R 65532:65532 /creds /data # Keep this name as sublimes3 because underscores don't play nice with certain endpoint validation sublimes3: container_name: sublimes3 - image: minio/minio + image: cgr.dev/chainguard/minio:latest restart: unless-stopped networks: - net - ports: - - "8110:8110" volumes: - s3_data:/data - env_file: sublime.env - entrypoint: > - /bin/sh -c " - export MINIO_ROOT_USER=$$AWS_ACCESS_KEY_ID; - export MINIO_ROOT_PASSWORD=$$AWS_SECRET_ACCESS_KEY; - minio server --address 0.0.0.0:8110 --console-address 0.0.0.0:8111 /data; - " - sublime_create_buckets: - image: minio/mc + - s3_creds:/creds:ro + environment: + MINIO_ROOT_USER_FILE: /creds/root_user + MINIO_ROOT_PASSWORD_FILE: /creds/root_password + depends_on: + sublimes3_init: + condition: service_completed_successfully + command: + - server + - --address + - 0.0.0.0:8110 + - --console-address + - 0.0.0.0:8111 + - /data + ports: + - 8110:8110 + # cgr.dev/chainguard/minio-client has no shell, so it can't run `mc alias set` (which needs + # shell interpolation of the AWS_* secrets into CLI args), and we don't want that secret ever + # needing to land in sublime.env either. This throwaway busybox step hand-writes mc's + # config.json straight from the AWS_* vars already in sublime.env; chainguard's mc then just + # reads it via MC_CONFIG_DIR below. mc still needs to create its own share/certs scratch dirs + # under this path at runtime, so the volume can't be mounted read-only. + sublime_mc_config: + image: busybox depends_on: - sublimes3 networks: - net env_file: sublime.env - entrypoint: > - /bin/sh -c " - sleep 15; - /usr/bin/mc alias set myminio http://sublimes3:8110 $$AWS_ACCESS_KEY_ID $$AWS_SECRET_ACCESS_KEY; - /usr/bin/mc mb myminio/email-screenshots; - /usr/bin/mc mb myminio/events; - /usr/bin/mc mb myminio/message-storage; - /usr/bin/mc mb myminio/message-export; - /usr/bin/mc mb myminio/nipper-artifacts; - /usr/bin/mc ls myminio; - exit 0; - " + volumes: + - mc_config:/config + command: + - sh + - -c + - | + cat > /config/config.json <&2; exit 1; fi; sleep 2; done + sublime_nipper: + image: 948971135452.dkr.ecr.us-east-1.amazonaws.com/nipper-python-dev:dev + restart: unless-stopped + container_name: sublime_nipper + env_file: sublime.env + ports: + - 8300:8000 + networks: + - net + environment: + NIPPER_BUCKET: nipper-artifacts + BUCKET_ENDPOINT_URL: http://sublimes3:8110 + TIKA_ENDPOINT: http://sublime_tika:9998 + DD_TRACE_ENABLED: false + depends_on: + - sublime_create_buckets + - sublime_tika + sublime_tika: + image: apache/tika:3.3.1.0-full + restart: unless-stopped + container_name: sublime_tika + ports: + - 9998:9998 + networks: + - net + sublime_azurite: + container_name: sublime_azurite + image: mcr.microsoft.com/azure-storage/azurite + restart: unless-stopped + networks: + - net + ports: + - 10000:10000 # Blob service + - 10001:10001 # Queue service + - 10002:10002 # Table service + volumes: + - azurite_data:/data + command: azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --location /data --skipApiVersionCheck + sublime_create_azure_blob_containers: + image: mcr.microsoft.com/azure-cli + depends_on: + - sublime_azurite + networks: + - net + environment: + # Azurite's well-known, publicly documented emulator account key - not a real secret. + AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://sublime_azurite:10000/devstoreaccount1; + command: | + bash -c " az storage container create --name email-screenshots && az storage container create --name events && az storage container create --name message-storage && az storage container create --name message-export && az storage container create --name nipper-artifacts " sublime_localstack: container_name: sublime_localstack image: localstack/localstack:4.10.0 ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - - "127.0.0.1:4510-4559:4510-4559" # external services port range + - 127.0.0.1:4566:4566 # LocalStack Gateway + - 127.0.0.1:4510-4559:4510-4559 # external services port range networks: - net environment: @@ -202,13 +287,32 @@ services: - DEBUG=0 volumes: - localstack:/var/lib/localstack - + # docker compose up --wait treats any container it doesn't already expect to exit as failed the + # moment it exits, even with code 0 (https://github.com/docker/compose/issues/10596). Neither + # sublime_create_buckets nor sublime_create_azure_blob_containers has a dependent declared with + # condition: service_completed_successfully, so --wait has no way to know their exit is expected. + # This long-running sentinel supplies that dependent so --wait treats their completion as success. + sublime_storage_ready: + image: alpine:3.20 + command: + - tail + - -f + - /dev/null + networks: + - net + depends_on: + sublime_create_buckets: + condition: service_completed_successfully + sublime_create_azure_blob_containers: + condition: service_completed_successfully networks: net: driver: bridge volumes: - postgres: logs: s3_data: + s3_creds: + mc_config: localstack: azurite_data: + postgres_e2e: diff --git a/install-and-launch.sh b/install-and-launch.sh index b68692c..6021048 100755 --- a/install-and-launch.sh +++ b/install-and-launch.sh @@ -43,10 +43,10 @@ set -e # : ----------------------------------------- -: Branch - default: dev +: Branch - default: main : ----------------------------------------- -# By default, this script assumes that it should pull dependencies from branch `dev`. If you wish to get dependencies +# By default, this script assumes that it should pull dependencies from branch `main`. If you wish to get dependencies # from another branch, you can specify it here. # : curl -sL https://sublimesecurity.com/install.sh | remote_branch=custom-branch sh @@ -80,7 +80,7 @@ if [ -z "$interactive" ]; then # ascii art # credit: https://patorjk.com/ # font: Cyberlarge - cat </dev/null 2>&1; then - print_error "docker compose appears to be brought down. Will not proceed to avoid relaunching." + print_error "Sublime Platform appears to have been manually shut down. Will not proceed to avoid relaunching." + print_warning "If you wish to relaunch, please refer to the documentation here:" + print_warning "https://docs.sublimesecurity.com/docs/quickstart-docker#how-to-update" exit 0 fi fi @@ -23,7 +29,7 @@ if [ -z "$(git status --porcelain)" ]; then echo "git working dir clean. Proceeding with git updates." old_ref=$(git rev-parse HEAD) - git pull + logrun git pull new_ref=$(git rev-parse HEAD) if [ "${old_ref}" != "${new_ref}" ]; then diff --git a/utils.sh b/utils.sh index 982d534..5610818 100644 --- a/utils.sh +++ b/utils.sh @@ -48,3 +48,8 @@ print_info() { print_warning() { print_color "\n$1\n" "warning" } + +logrun() { + echo >&2 "+ $*" + "$@" +}