diff --git a/kcidev/_data/kci-dev.toml.example b/kcidev/_data/kci-dev.toml.example index aaeb0a7..45836af 100644 --- a/kcidev/_data/kci-dev.toml.example +++ b/kcidev/_data/kci-dev.toml.example @@ -11,10 +11,14 @@ api="https://staging.kernelci.org:9000/" token="example" kcidb_rest_url="https://staging.kcidb.kernelci.org/submit" kcidb_token="your-kcidb-token-here" +storage_url="https://files-staging.kernelci.org" +storage_token="your-storage-jwt-token" [production] pipeline="https://kernelci-pipeline.westus3.cloudapp.azure.com/" api="https://kernelci-api.westus3.cloudapp.azure.com/" token="example" kcidb_rest_url="https://db.kernelci.org/submit" -kcidb_token="your-kcidb-token-here" \ No newline at end of file +kcidb_token="your-kcidb-token-here" +storage_url="https://files.kernelci.org" +storage_token="your-storage-jwt-token" \ No newline at end of file diff --git a/kcidev/libs/common.py b/kcidev/libs/common.py index d5a4f9d..de5ccf1 100644 --- a/kcidev/libs/common.py +++ b/kcidev/libs/common.py @@ -78,7 +78,7 @@ def load_toml(settings, subcommand): return config # config and results subcommand work without a config file - if subcommand not in ("config", "results", "submit"): + if subcommand not in ("config", "results", "submit", "storage"): if not config: logging.warning(f"No config file found for subcommand {subcommand}") kci_err( diff --git a/kcidev/libs/storage.py b/kcidev/libs/storage.py new file mode 100644 index 0000000..e4fb91a --- /dev/null +++ b/kcidev/libs/storage.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import logging +import os + +import click +import requests + +from kcidev.libs.common import kci_err, kci_msg, kcidev_session + + +def resolve_storage_config(cfg, instance, cli_url, cli_token): + """ + Resolve storage URL and token. Priority: + 1. CLI flags (--storage-url, --storage-token) + 2. Environment variables (KCI_STORAGE_URL, KCI_STORAGE_TOKEN) + 3. Instance config (storage_url, storage_token in TOML) + + Returns (storage_url, token) tuple. + Raises click.Abort if no valid credentials found. + """ + # 1. CLI flags + if cli_url or cli_token: + if not cli_url or not cli_token: + kci_err("Both --storage-url and --storage-token must be provided together") + raise click.Abort() + logging.debug(f"Using storage config from CLI flags: {cli_url}") + return cli_url, cli_token + + # 2. Environment variables + env_url = os.environ.get("KCI_STORAGE_URL") + env_token = os.environ.get("KCI_STORAGE_TOKEN") + if env_url or env_token: + if env_url and env_token: + logging.debug(f"Using storage config from env vars: {env_url}") + return env_url, env_token + kci_err( + "Both KCI_STORAGE_URL and KCI_STORAGE_TOKEN env vars must be set together" + ) + raise click.Abort() + + # 3. Instance config + if cfg and instance and instance in cfg: + inst_cfg = cfg[instance] + storage_url = inst_cfg.get("storage_url") + storage_token = inst_cfg.get("storage_token") + if storage_url and storage_token: + logging.debug( + f"Using storage config from instance '{instance}': {storage_url}" + ) + return storage_url, storage_token + + kci_err( + "No storage credentials found. Provide --storage-url and --storage-token, " + "set KCI_STORAGE_URL/KCI_STORAGE_TOKEN env vars, " + "or configure storage_url/storage_token in config file" + ) + raise click.Abort() + + +def upload_file(storage_url, token, remote_path, local_file_path, timeout=120): + """ + Upload a file to kernelci-storage. + + POST /v1/file with multipart form: + - path: remote directory path + - file0: file content + """ + url = storage_url.rstrip("/") + "/v1/file" + headers = { + "Authorization": f"Bearer {token}", + } + + logging.info(f"Uploading {local_file_path} to {remote_path}/") + logging.debug(f"POST request to: {url}") + + try: + with open(local_file_path, "rb") as f: + files = {"file0": (os.path.basename(local_file_path), f)} + data = {"path": remote_path} + response = kcidev_session.post( + url, headers=headers, files=files, data=data, timeout=timeout + ) + logging.debug(f"Upload response status: {response.status_code}") + except requests.exceptions.RequestException as e: + logging.error(f"Upload request failed: {e}") + kci_err(f"Storage connection error: {e}") + raise click.Abort() + + if response.status_code == 200: + logging.info(f"Upload successful: {os.path.basename(local_file_path)}") + return response.text + elif response.status_code == 401: + kci_err("Authentication failed: invalid or expired token") + raise click.Abort() + else: + kci_err(f"Upload failed (HTTP {response.status_code}): {response.text}") + raise click.Abort() + + +def check_auth(storage_url, token, timeout=30): + """ + Validate a JWT token against the storage server. + + GET /v1/checkauth + Returns the response text on success. + """ + url = storage_url.rstrip("/") + "/v1/checkauth" + headers = { + "Authorization": f"Bearer {token}", + } + + logging.info("Checking storage authentication") + logging.debug(f"GET request to: {url}") + + try: + response = kcidev_session.get(url, headers=headers, timeout=timeout) + logging.debug(f"Auth check response status: {response.status_code}") + except requests.exceptions.RequestException as e: + logging.error(f"Auth check request failed: {e}") + kci_err(f"Storage connection error: {e}") + raise click.Abort() + + if response.status_code == 200: + logging.info(f"Authentication valid: {response.text}") + return response.text + elif response.status_code == 401: + kci_err("Authentication failed: invalid or expired token") + raise click.Abort() + else: + kci_err(f"Auth check failed (HTTP {response.status_code}): {response.text}") + raise click.Abort() diff --git a/kcidev/main.py b/kcidev/main.py index 305b19a..5ac9dbd 100755 --- a/kcidev/main.py +++ b/kcidev/main.py @@ -13,6 +13,7 @@ config, maestro, results, + storage, submit, testretry, watch, @@ -44,7 +45,7 @@ def cli(ctx, settings, instance, debug): if subcommand not in ("results", "config"): if instance: ctx.obj["INSTANCE"] = instance - elif subcommand != "submit": + elif subcommand not in ("submit", "storage"): ctx.obj["INSTANCE"] = ctx.obj["CFG"].get("default_instance") fconfig = config_path(settings) if not ctx.obj["INSTANCE"]: @@ -63,6 +64,7 @@ def run(): cli.add_command(maestro.maestro) cli.add_command(testretry.testretry) cli.add_command(results.results) + cli.add_command(storage.storage) cli.add_command(submit.submit) cli.add_command(watch.watch) cli() diff --git a/kcidev/subcommands/storage.py b/kcidev/subcommands/storage.py new file mode 100644 index 0000000..56f7efc --- /dev/null +++ b/kcidev/subcommands/storage.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import sys + +import click + +from kcidev.libs.common import kci_err, kci_msg +from kcidev.libs.storage import check_auth, resolve_storage_config, upload_file + + +@click.group( + help="""Interact with KernelCI storage. + +This command group provides access to KernelCI storage operations including +uploading files and validating authentication tokens. + +\b +Examples: + # Upload a file (path recommended to match your origin) + kci-dev storage upload --path myci/build-123 ./some-files.tar.xz + # Upload multiple files + kci-dev storage upload --path myci/build-123 log.txt config.gz + # Validate authentication token + kci-dev storage checkauth +""", + invoke_without_command=True, +) +@click.pass_context +def storage(ctx): + """Commands related to KernelCI storage.""" + cfg = ctx.obj.get("CFG") + if cfg: + instance = ctx.obj.get("INSTANCE") + if not instance: + instance = cfg.get("default_instance") + ctx.obj["INSTANCE"] = instance + + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + sys.exit(0) + + +@storage.command() +@click.option( + "--storage-url", + help="Storage server URL (overrides config and env)", +) +@click.option( + "--storage-token", + help="Storage JWT token (overrides config and env)", +) +@click.option( + "--path", + "remote_path", + required=True, + help="Remote directory path, recommended to match your origin (e.g. origin/build-123)", +) +@click.argument( + "files", + nargs=-1, + required=True, + type=click.Path(exists=True), +) +@click.pass_context +def upload(ctx, storage_url, storage_token, remote_path, files): + """Upload files to KernelCI storage.""" + cfg = ctx.obj.get("CFG") + instance = ctx.obj.get("INSTANCE") + + url, token = resolve_storage_config(cfg, instance, storage_url, storage_token) + + for file_path in files: + if os.path.isdir(file_path): + kci_err(f"Skipping directory: {file_path}") + continue + filename = os.path.basename(file_path) + kci_msg(f"Uploading {filename} to {remote_path}/...") + result = upload_file(url, token, remote_path, file_path) + kci_msg(f"Uploaded: {filename} - {result}") + + +@storage.command() +@click.option( + "--storage-url", + help="Storage server URL (overrides config and env)", +) +@click.option( + "--storage-token", + help="Storage JWT token (overrides config and env)", +) +@click.pass_context +def checkauth(ctx, storage_url, storage_token): + """Validate storage authentication token.""" + cfg = ctx.obj.get("CFG") + instance = ctx.obj.get("INSTANCE") + + url, token = resolve_storage_config(cfg, instance, storage_url, storage_token) + + result = check_auth(url, token) + kci_msg(result)