tasks.py

import json
from os import listdir
from os.path import isfile, join
from typing import List
from invoke import exceptions, task
from deployment.inv.commands.docker import docker_build_and_tag
from deployment.inv.commands.gcloud import gcloud_push
from deployment.inv.commands.jinja import j2_template
from deployment.inv.commands.kubernetes import k8s_apply, k8s_runjob, k8s_push_secret, k8s_wait_for_pod_status, k8s_is_pod_running, k8s_get_contextname
from deployment.inv.utils import get_config_value, get_filename_no_extension


@task
def transformyaml(ctx):
    config_path = get_config_value(ctx, 'bbconfig', 'global', 'config_path')
    if not config_path:
        config_path = resolve_config_path(ctx)

    section_name = 'k8s_template_vars'
    j2_template(
        ctx,
        'deployment/k8s/templates/bbapi.yaml.j2',
        config_path,
        'deployment/k8s/yaml/generated/bbapi.yaml',
        section=section_name)
    j2_template(
        ctx,
        'deployment/k8s/templates/postgres-pv.yaml.j2',
        config_path,
        'deployment/k8s/yaml/generated/postgres-pv.yaml',
        section=section_name)


@task
def deploy_config(ctx, base_name=None):
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    warn_not_local(ctx)
    config_yaml = 'deployment/k8s/yaml/*-config.yaml'
    if base_name is not None:
        config_yaml = 'deployment/k8s/yaml/{0}-config.yaml'.format(base_name)
    k8s_apply(ctx, config_yaml)


@task
def deploy_bbapi_service(ctx):
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    warn_not_local(ctx)
    k8s_apply(ctx, 'deployment/k8s/yaml/bbapi-service.yaml')


@task
def deploy_bbapi_pods(ctx):
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    warn_not_local(ctx)

    image_prefix = 'gcr.io/battlebrain-api/'
    transformyaml(ctx)
    docker_build_and_tag(ctx, image_prefix,
                         'deployment/docker/bbapi.Dockerfile')
    docker_build_and_tag(ctx, image_prefix,
                         'deployment/docker/bbapi-headless.Dockerfile')
    k8s_apply(ctx, 'deployment/k8s/yaml/generated/bbapi.yaml')


@task
def deploy_update_leaderboards(ctx):
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    warn_not_local(ctx)

    image_prefix = 'gcr.io/battlebrain-api/'
    transformyaml(ctx)
    docker_build_and_tag(ctx, image_prefix,
                         'deployment/docker/pybox.Dockerfile')
    k8s_apply(ctx, 'deployment/k8s/yaml/cronjobs/updateleaderboards.yaml')


@task
def deploy(ctx):
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    warn_not_local(ctx)

    # If secrets are not on the cluster, deploy them
    k8s_push_secret(ctx, 'db-secret', ['user', 'password'])
    k8s_push_secret(ctx, 'auth-secret', ['secret'])

    # If we are creating postgres server for the first time, ask if user wants to createdb also
    firstdbserver = False
    createdb = False
    if not k8s_is_pod_running(ctx, pod_label='app=postgres'):
        firstdbserver = True
        yorn = input(
            'Postgres is not yet on the kubernetes cluster. Do you want to create the db after the server is created?\n(Y/n): '
        )
        if yorn == 'Y':
            createdb = True

    print('User interaction complete. Starting deployment...')

    image_prefix = 'gcr.io/battlebrain-api/'
    transformyaml(ctx)
    docker_build_and_tag(ctx, image_prefix, 'deployment/docker/*.Dockerfile')
    k8s_apply(ctx, 'deployment/k8s/yaml/generated/*.yaml',
              'deployment/k8s/yaml/*.yaml')

    # Wait for db server to spin up
    k8s_wait_for_pod_status(ctx, 'Running', pod_label='app=postgres')

    # Create the database if necessary, and/or run migrations
    if createdb:
        k8s_runjob(ctx, image_prefix, 'bbapi',
                   'deployment/k8s/yaml/jobs/createdb.yaml')
    if not firstdbserver or createdb:
        k8s_runjob(ctx, image_prefix, 'bbapi',
                   'deployment/k8s/yaml/jobs/migration.yaml')

    # Deploy cron jobs afterwards since they might depend on the db
    k8s_apply(ctx, 'deployment/k8s/yaml/cronjobs/*.yaml')


@task(help={
    'name': "Name of the job to run in the jobs dir without the .yaml extension"
})
def runjob(ctx, name):
    warningjobs = ['migration', 'createsystemusers']
    if name in warningjobs:
        yorn = input(
            'Warning! The "{0}" job might depend on a previous step and you should run a top-level task instead of "runjob". Are you sure you want to do this?\n(Y/n): '.
            format(name))
        if yorn != 'Y':
            raise exceptions.Exit(1)
    image_prefix = 'gcr.io/battlebrain-api/'
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    k8s_runjob(ctx, image_prefix, 'bbapi',
               'deployment/k8s/yaml/jobs/{0}.yaml'.format(name))


@task
def deploy_cronjobs(ctx):
    image_prefix = 'gcr.io/battlebrain-api/'
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    docker_build_and_tag(ctx, image_prefix,
                         'deployment/docker/pybox.Dockerfile')
    k8s_apply(ctx, 'deployment/k8s/yaml/cronjobs/updateleaderboards.yaml')


@task
def createsystemusers_db(ctx):
    image_prefix = 'gcr.io/battlebrain-api/'
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    k8s_runjob(ctx, image_prefix, 'bbapi-headless',
               'deployment/k8s/yaml/jobs/createsystemusers.yaml')


@task
def seed_db(ctx):
    image_prefix = 'gcr.io/battlebrain-api/'
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    docker_build_and_tag(ctx, image_prefix,
                         'deployment/docker/psql.Dockerfile')
    k8s_runjob(ctx, image_prefix, 'bbapi',
               'deployment/k8s/yaml/jobs/seeddata.yaml', True)


@task
def migrate_db(ctx):
    image_prefix = 'gcr.io/battlebrain-api/'
    ctx.config['bbconfig'] = load_config(resolve_config_path(ctx))
    docker_build_and_tag(ctx, image_prefix,
                         'deployment/docker/flyway.Dockerfile')
    k8s_runjob(ctx, image_prefix, 'bbapi',
               'deployment/k8s/yaml/jobs/migration.yaml')


def warn_not_local(ctx):
    # Warn if we are going to push to gcloud
    if not get_config_value(ctx, 'bbconfig', 'global', 'local'):
        config_name = get_filename_no_extension(
            get_config_value(ctx, 'bbconfig', 'global', 'config_path'))
        yorn = input(
            'This deployment will deploy externally, using "{0}" config. Are you sure you want to do this?\n(Y/n): '
            .format(get_filename_no_extension(config_name)))
        if yorn != 'Y':
            raise exceptions.Exit(1)


def load_config(config_path) -> object:
    with open(config_path) as configdata:
        config = json.load(configdata)
    extrapolate_global_config(config, config_path)
    extrapolate_docker_config(config)
    return config


def extrapolate_global_config(config: object, config_path: str):
    config['global']['config_path'] = config_path


def extrapolate_docker_config(config: object):
    if config['global']['gcloud_push_tag']:
        if 'docker' not in config:
            config['docker'] = {}
        config['docker']['tag_post_action'] = gcloud_push


def resolve_config_path(ctx) -> str:
    k8s_contextname: str = k8s_get_contextname(ctx)
    paths = get_config_paths_matching(k8s_contextname)
    if len(paths) != 1:
        print('ERROR: Found multiple possible config names: {0}'.format(
            ', '.join(paths)))
        raise exceptions.Exit(1)
    else:
        configpath = paths[0]
        print('Using config named: {0}'.format(configpath))

    return configpath


def get_config_paths_matching(name_to_match) -> List[str]:
    # get the list of available configs
    configdir = 'deployment/config/'
    names_and_paths = [(f.split('.')[0], join(configdir, f))
                       for f in listdir(configdir)
                       if isfile(join(configdir, f))]
    candidatepaths = [
        path for name, path in names_and_paths
        if name_to_match.startswith(name)
    ]
    return candidatepaths