diff --git a/src/repo_manager/docs/catalog_operations.md b/src/repo_manager/docs/catalog_operations.md new file mode 100644 index 0000000000..181c0ef0bb --- /dev/null +++ b/src/repo_manager/docs/catalog_operations.md @@ -0,0 +1,306 @@ +# Catalog Operations Guide + +This document describes how to use the catalog management system in repo_manager. + +## Overview + +The catalog management system provides operations to: +- **Generate**: Create a new catalog from an input text file +- **Add**: Add or update packages in an existing catalog (upsert semantics) +- **Delete**: Remove packages from a catalog +- **Validate**: Validate a catalog against the schema and business rules + +## Quick Start + +```bash +cd /path/to/repo_manager/playbooks + +# Generate a new catalog +ansible-playbook repo_manager.yml --tags catalog_generate \ + -e "input_file=input/packages.txt" + +# Add packages to existing catalog +ansible-playbook repo_manager.yml --tags catalog_add \ + -e "input_file=input/additions.txt" + +# Delete packages from catalog +ansible-playbook repo_manager.yml --tags catalog_delete \ + -e "input_file=input/removals.txt" + +# Validate a catalog +ansible-playbook repo_manager.yml --tags catalog_validate +``` + +## Input File Format + +### For Generate and Add Operations + +The input file uses an INI-like format with optional defaults header: + +```ini +# Optional defaults section (applies to all packages unless overridden) +[defaults] +arch=x86_64, os=rhel, os_version=10.0 + +# Group headers with optional metadata +# Format: [group_key | key=value, key=value, ...] +# Supported metadata: type, description, os, os_version +# 'type' defaults to "group" if omitted + +[baseos_group_10.0 | type=base_os, description=base os packages, os=rhel, os_version=10.0] +systemd, rpm, systemd, baseos +wget, rpm, wget, appstream +glibc_langpack_en, rpm, glibc-langpack-en, baseos + +[slurm_custom_group | description=slurm custom packages] +clustershell, rpm, clustershell, epel +papi, tarball, papi, https://github.com/icl-utk-edu/papi/releases/download/papi-7-2-0-t/papi-7.2.0.tar.gz +curl, image, docker.io/curlimages/curl, docker.io, 8.17.0 + +# Override arch for a specific package +doca_ofed, rpm_repo, doca-ofed, doca, arch=aarch64 +``` + +### Package Line Formats + +| Type | Format | Example | +|------|--------|---------| +| `rpm` | `key, rpm, name, reponame` | `wget, rpm, wget, appstream` | +| `rpm_repo` | `key, rpm_repo, name, reponame` | `doca_ofed, rpm_repo, doca-ofed, doca` | +| `tarball` | `key, tarball, name, url` | `papi, tarball, papi, https://...` | +| `image` | `key, image, image_path, registry, tag` | `curl, image, docker.io/curl, docker.io, 8.17.0` | + +### Trailing Overrides + +Any package line can have trailing key=value overrides: +- `arch=aarch64` - Override architecture +- `os=rhel` - Override OS +- `os_version=9.4` - Override OS version + +### For Delete Operations + +The delete input file is simpler - just group headers and package keys: + +```ini +[baseos_group_10.0] +wget +glibc_langpack_en + +[slurm_custom_group] +papi +``` + +## Operations Reference + +### catalog_generate + +Creates a new catalog from an input file. + +```bash +ansible-playbook repo_manager.yml --tags catalog_generate \ + -e "input_file=input/packages.txt" +``` + +**Parameters:** + +| Parameter | Required | Default | Description | +|-----------|----------|---------|-------------| +| `input_file` | Yes | - | Path to input text file | +| `catalog_file` | No | `catalogs/catalog.json` | Output catalog path | +| `catalog_name` | No | `default` | Name for the catalog | +| `force` | No | `false` | Overwrite existing file | +| `validate_after` | No | `true` | Run validation after generate | + +**Behavior:** +- Fails if output file exists (unless `force=true`) +- Creates parent directories automatically + +### catalog_add + +Adds or updates packages in an existing catalog. + +```bash +ansible-playbook repo_manager.yml --tags catalog_add \ + -e "input_file=input/additions.txt" +``` + +**Parameters:** + +| Parameter | Required | Default | Description | +|-----------|----------|---------|-------------| +| `input_file` | Yes | - | Path to input text file | +| `catalog_file` | No | `catalogs/catalog.json` | Source catalog path | +| `output_file` | No | Same as `catalog_file` | Output path (preserves source if different) | +| `validate_after` | No | `true` | Run validation after add | + +**Behavior:** +- **Upsert semantics**: Updates existing packages, adds new ones +- Auto-creates groups that don't exist +- No duplicates in `components[]` arrays + +### catalog_delete + +Removes packages from a catalog. + +```bash +ansible-playbook repo_manager.yml --tags catalog_delete \ + -e "input_file=input/removals.txt" +``` + +**Parameters:** + +| Parameter | Required | Default | Description | +|-----------|----------|---------|-------------| +| `input_file` | Yes | - | Path to delete input file | +| `catalog_file` | No | `catalogs/catalog.json` | Source catalog path | +| `output_file` | No | Same as `catalog_file` | Output path | +| `validate_after` | No | `true` | Run validation after delete | + +**Behavior:** +- Removes packages from specified groups +- Deletes packages entirely when unreferenced by any group +- Removes empty groups automatically +- Skips (with warning) missing groups/packages + +### catalog_validate + +Validates a catalog against schema and business rules. + +```bash +ansible-playbook repo_manager.yml --tags catalog_validate +``` + +**Parameters:** + +| Parameter | Required | Default | Description | +|-----------|----------|---------|-------------| +| `catalog_file` | No | `catalogs/catalog.json` | Catalog to validate | +| `schema_file` | No | `schemas/catalog_schema.json` | JSON schema file | + +**Validation Layers:** + +1. **Structural (JSON Schema)**: Required fields, types, patterns +2. **Referential Integrity**: FunctionalLayer → Groups → Packages +3. **Business Rules**: + - No duplicate components in arrays + - Valid package types + - Type-specific required fields (reponame for rpm, url for tarball, etc.) + - base_os groups must have os and os_version +4. **Warnings**: + - Orphan packages (unreferenced by any group) + - Orphan groups (unreferenced by any functional layer) + +## Catalog JSON Structure + +```json +{ + "catalog": { + "name": "default", + "version": "1.0", + "identifier": "default", + "description": "", + "functionallayer": [ + { + "name": "layer_name", + "components": ["group_key_1", "group_key_2"] + } + ], + "groups": { + "group_key": { + "name": "group_key", + "type": "group", + "description": "Group description", + "components": ["pkg_key_1", "pkg_key_2"] + } + }, + "packages": { + "pkg_key": { + "name": "package_name", + "packagetype": "rpm", + "sources": [ + { + "architecture": "x86_64", + "reponame": "baseos", + "name": "rhel", + "version": ["10.0"] + } + ] + } + } + } +} +``` + +## Default Values + +| Variable | Default | Description | +|----------|---------|-------------| +| `default_arch` | `x86_64` | Default architecture | +| `default_os` | `rhel` | Default OS | +| `default_os_version` | `10.0` | Default OS version | +| `catalog_file` | `catalogs/catalog.json` | Default catalog path | +| `schema_file` | `schemas/catalog_schema.json` | Default schema path | + +## Logging + +All operations write logs to `$OMNIA_DATA_PATH/repo_manager/log/catalog/`. + +## Examples + +### Complete Workflow + +```bash +# 1. Create input file +cat > input/my_catalog.txt <<'EOF' +[defaults] +arch=x86_64, os=rhel, os_version=10.0 + +[baseos_group_10.0 | type=base_os, description=base os packages, os=rhel, os_version=10.0] +systemd, rpm, systemd, baseos +wget, rpm, wget, appstream + +[slurm_group | description=slurm packages] +clustershell, rpm, clustershell, epel +EOF + +# 2. Generate catalog +ansible-playbook repo_manager.yml --tags catalog_generate \ + -e "input_file=input/my_catalog.txt" + +# 3. Add more packages +cat > input/add_packages.txt <<'EOF' +[slurm_group] +geopm, tarball, geopm, https://github.com/geopm/geopm/releases/geopm-3.1.0.tar.gz +EOF + +ansible-playbook repo_manager.yml --tags catalog_add \ + -e "input_file=input/add_packages.txt" + +# 4. Validate +ansible-playbook repo_manager.yml --tags catalog_validate + +# 5. Delete a package +cat > input/remove.txt <<'EOF' +[baseos_group_10.0] +wget +EOF + +ansible-playbook repo_manager.yml --tags catalog_delete \ + -e "input_file=input/remove.txt" +``` + +### Using Custom Paths + +```bash +# Custom output path +ansible-playbook repo_manager.yml --tags catalog_generate \ + -e "input_file=input/packages.txt" \ + -e "catalog_file=catalogs/custom.json" \ + -e "catalog_name=my_custom_catalog" + +# Preserve source catalog +ansible-playbook repo_manager.yml --tags catalog_add \ + -e "input_file=input/additions.txt" \ + -e "catalog_file=catalogs/v1.json" \ + -e "output_file=catalogs/v2.json" +``` diff --git a/src/repo_manager/playbooks/repo_manager.yml b/src/repo_manager/playbooks/repo_manager.yml index fc0f4f608e..ec9ded6172 100644 --- a/src/repo_manager/playbooks/repo_manager.yml +++ b/src/repo_manager/playbooks/repo_manager.yml @@ -49,6 +49,12 @@ # # # Rollback repo_manager (placeholder) # ansible-playbook repo_manager.yml --tags rollback +# +# # Catalog operations +# ansible-playbook repo_manager.yml --tags catalog_generate -e "input_file=input/packages.txt" +# ansible-playbook repo_manager.yml --tags catalog_add -e "input_file=input/additions.txt" +# ansible-playbook repo_manager.yml --tags catalog_delete -e "input_file=input/removals.txt" +# ansible-playbook repo_manager.yml --tags catalog_validate # ============================================================================= # ------------------------------------------------------------------------- @@ -149,3 +155,21 @@ tags: - never - rollback + +# ------------------------------------------------------------------------- +# CATALOG OPERATIONS +# ------------------------------------------------------------------------- +- name: Catalog Operations + hosts: localhost + connection: local + gather_facts: false + vars_files: + - "{{ playbook_dir | dirname }}/vars/default.yml" + roles: + - role: catalog + tags: + - never + - catalog_generate + - catalog_add + - catalog_delete + - catalog_validate diff --git a/src/repo_manager/plugins/module_utils/catalog/__init__.py b/src/repo_manager/plugins/module_utils/catalog/__init__.py new file mode 100644 index 0000000000..fd0adfd89b --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Catalog management module for repo_manager. + +Provides CLI-based operations for generating, modifying, and validating +service catalogs used by the repo_manager domain. +""" diff --git a/src/repo_manager/plugins/module_utils/catalog/catalog_io.py b/src/repo_manager/plugins/module_utils/catalog/catalog_io.py new file mode 100644 index 0000000000..4191546caa --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/catalog_io.py @@ -0,0 +1,113 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Catalog I/O operations: read and write catalog JSON files. +""" + +import os +import json +import re +import logging + +logger = logging.getLogger(__name__) + + +def slugify(text): + """Convert text to a valid identifier (lowercase, underscores).""" + slug = text.lower() + slug = re.sub(r'[^a-z0-9]+', '_', slug) + slug = slug.strip('_') + return slug or 'catalog' + + +def read_catalog(filepath): + """ + Read a catalog from a JSON file. + + Args: + filepath: Path to the catalog JSON file. + + Returns: + dict: Catalog data with 'catalog' as the root key. + + Raises: + FileNotFoundError: If the file doesn't exist. + json.JSONDecodeError: If the file is not valid JSON. + ValueError: If the catalog structure is invalid. + """ + logger.info("Reading catalog from: %s", filepath) + with open(filepath, 'r', encoding='utf-8') as fh: + data = json.load(fh) + + if 'catalog' not in data: + raise ValueError(f"Invalid catalog file: missing 'catalog' key in {filepath}") + + return data + + +def write_catalog(catalog, filepath): + """ + Write a catalog to a JSON file. + + Args: + catalog: Catalog data dict (must have 'catalog' root key). + filepath: Path to write the catalog JSON file. + + Raises: + OSError: If the file cannot be written. + """ + # Ensure parent directory exists + parent_dir = os.path.dirname(filepath) + if parent_dir: + os.makedirs(parent_dir, exist_ok=True) + + logger.info("Writing catalog to: %s", filepath) + with open(filepath, 'w', encoding='utf-8') as fh: + json.dump(catalog, fh, indent=2) + + +def new_catalog(name, groups, packages, functional_layers=None, description='', version='1.0'): + """ + Create a new catalog structure. + + Args: + name: Catalog name. + groups: Dict of group_key -> group_entry. + packages: Dict of pkg_key -> package_entry. + functional_layers: List of functional layer entries (optional). + description: Optional catalog description. + version: Catalog version string. + + Returns: + dict: Complete catalog structure. + """ + if functional_layers is None: + functional_layers = [] + + return { + "catalog": { + "name": name, + "version": version, + "identifier": slugify(name), + "description": description, + "functionallayer": functional_layers, + "groups": groups, + "packages": packages + } + } + + +def catalog_exists(filepath): + """Check if a catalog file exists.""" + return os.path.isfile(filepath) diff --git a/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py b/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py new file mode 100644 index 0000000000..57805425e6 --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Catalog Manager CLI - Entry point for catalog operations. + +Usage: + catalog_manager.py generate --input --output [options] + catalog_manager.py add --input --catalog [--output ] [options] + catalog_manager.py delete --input --catalog [--output ] + catalog_manager.py validate --catalog [--schema ] +""" + +import argparse +import logging +import os +import sys + +# Add parent directory to path for module imports +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from catalog.parser import parse_input_file, parse_delete_file +from catalog.catalog_io import read_catalog, write_catalog, new_catalog, catalog_exists +from catalog.mutator import upsert_packages, delete_packages +from catalog.validator import validate_catalog, format_issues + + +def setup_logging(log_dir=None, log_file='catalog_manager.log'): + """Setup logging configuration.""" + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + # Console handler + console = logging.StreamHandler(sys.stdout) + console.setLevel(logging.INFO) + console.setFormatter(logging.Formatter('%(message)s')) + logger.addHandler(console) + + # File handler if log_dir provided + if log_dir: + os.makedirs(log_dir, exist_ok=True) + log_path = os.path.join(log_dir, log_file) + file_handler = logging.FileHandler(log_path) + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(logging.Formatter( + '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s' + )) + logger.addHandler(file_handler) + logging.info("Log file: %s", log_path) + + return logger + + +def cmd_generate(args): + """Generate a new catalog from input file.""" + logger = logging.getLogger(__name__) + + if catalog_exists(args.output) and not args.force: + logger.error("Output file '%s' already exists. Use --force to overwrite.", args.output) + return 1 + + try: + parsed = parse_input_file( + args.input, + default_arch=args.default_arch, + default_os=args.default_os, + default_os_version=args.default_os_version + ) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to parse input file: %s", e) + return 1 + + catalog = new_catalog( + args.name, + parsed['groups'], + parsed['packages'], + functional_layers=parsed.get('functional_layers', []) + ) + write_catalog(catalog, args.output) + + # Optional validation + if args.validate and args.schema: + issues = validate_catalog(catalog, args.schema) + if issues: + print(format_issues(issues)) + errors = [i for i in issues if i['severity'] == 'error'] + if errors: + logger.warning("Catalog generated with validation errors") + + fl_count = len(parsed.get('functional_layers', [])) + group_count = len(parsed['groups']) + pkg_count = len(parsed['packages']) + print(f"Catalog generated: {fl_count} functional layers, {group_count} groups, {pkg_count} packages -> {args.output}") + return 0 + + +def cmd_add(args): + """Add packages to existing catalog.""" + logger = logging.getLogger(__name__) + + try: + catalog = read_catalog(args.catalog) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to read catalog: %s", e) + return 1 + + try: + parsed = parse_input_file( + args.input, + default_arch=args.default_arch, + default_os=args.default_os, + default_os_version=args.default_os_version + ) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to parse input file: %s", e) + return 1 + + summary = upsert_packages(catalog, parsed) + output_file = args.output or args.catalog + write_catalog(catalog, output_file) + + # Optional validation + if args.validate and args.schema: + issues = validate_catalog(catalog, args.schema) + if issues: + print(format_issues(issues)) + + print(f"Added: {summary['added']}, Updated: {summary['updated']}, " + f"Groups created: {summary['groups_created']} -> {output_file}") + return 0 + + +def cmd_delete(args): + """Delete packages from catalog.""" + logger = logging.getLogger(__name__) + + try: + catalog = read_catalog(args.catalog) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to read catalog: %s", e) + return 1 + + try: + parsed_delete = parse_delete_file(args.input) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to parse delete file: %s", e) + return 1 + + summary = delete_packages(catalog, parsed_delete) + output_file = args.output or args.catalog + write_catalog(catalog, output_file) + + # Optional validation + if args.validate and args.schema: + issues = validate_catalog(catalog, args.schema) + if issues: + print(format_issues(issues)) + + print(f"Deleted: {summary['deleted']}, Groups removed: {summary['groups_removed']}, " + f"Skipped: {summary['skipped']} -> {output_file}") + return 0 + + +def cmd_validate(args): + """Validate a catalog.""" + logger = logging.getLogger(__name__) + + try: + catalog = read_catalog(args.catalog) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to read catalog: %s", e) + return 1 + + issues = validate_catalog(catalog, args.schema) + print(format_issues(issues)) + + errors = [i for i in issues if i['severity'] == 'error'] + return 1 if errors else 0 + + +def main(): + """Main entry point.""" + # Get CATALOG_FILE_PATH environment variable as default + catalog_file_path = os.environ.get('CATALOG_FILE_PATH', '') + + parser = argparse.ArgumentParser( + description='Catalog Manager - Generate, modify, and validate service catalogs', + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('--log-dir', help='Directory for log files') + + subparsers = parser.add_subparsers(dest='command', required=True) + + # Generate command + gen_parser = subparsers.add_parser('generate', help='Generate new catalog from input file') + gen_parser.add_argument('--input', '-i', help='Input file path') + gen_parser.add_argument('--output', '-o', default=catalog_file_path, + help=f'Output catalog file path (default: $CATALOG_FILE_PATH={catalog_file_path})') + gen_parser.add_argument('--name', '-n', default='default', help='Catalog name') + gen_parser.add_argument('--force', '-f', action='store_true', help='Overwrite existing file') + gen_parser.add_argument('--default-arch', default='x86_64', help='Default architecture') + gen_parser.add_argument('--default-os', default='rhel', help='Default OS') + gen_parser.add_argument('--default-os-version', default='10.0', help='Default OS version') + gen_parser.add_argument('--schema', help='Schema file for validation') + gen_parser.add_argument('--validate', action='store_true', default=True, + help='Validate after generation') + gen_parser.set_defaults(func=cmd_generate) + + # Add command + add_parser = subparsers.add_parser('add', help='Add packages to existing catalog') + add_parser.add_argument('--input', '-i', help='Input file with packages to add') + add_parser.add_argument('--catalog', '-c', default=catalog_file_path, + help=f'Existing catalog file (default: $CATALOG_FILE_PATH={catalog_file_path})') + add_parser.add_argument('--output', '-o', help='Output file (default: overwrite catalog)') + add_parser.add_argument('--default-arch', default='x86_64', help='Default architecture') + add_parser.add_argument('--default-os', default='rhel', help='Default OS') + add_parser.add_argument('--default-os-version', default='10.0', help='Default OS version') + add_parser.add_argument('--schema', help='Schema file for validation') + add_parser.add_argument('--validate', action='store_true', default=True, + help='Validate after add') + add_parser.set_defaults(func=cmd_add) + + # Delete command + del_parser = subparsers.add_parser('delete', help='Delete packages from catalog') + del_parser.add_argument('--input', '-i', help='Input file with packages to delete') + del_parser.add_argument('--catalog', '-c', default=catalog_file_path, + help=f'Existing catalog file (default: $CATALOG_FILE_PATH={catalog_file_path})') + del_parser.add_argument('--output', '-o', help='Output file (default: overwrite catalog)') + del_parser.add_argument('--schema', help='Schema file for validation') + del_parser.add_argument('--validate', action='store_true', default=True, + help='Validate after delete') + del_parser.set_defaults(func=cmd_delete) + + # Validate command + val_parser = subparsers.add_parser('validate', help='Validate a catalog') + val_parser.add_argument('--catalog', '-c', default=catalog_file_path, + help=f'Catalog file to validate (default: $CATALOG_FILE_PATH={catalog_file_path})') + val_parser.add_argument('--schema', '-s', help='JSON schema file') + val_parser.set_defaults(func=cmd_validate) + + args = parser.parse_args() + + # Validate required arguments + if args.command == 'generate': + if not args.input: + parser.error("generate: --input is required") + if not args.output: + parser.error("generate: --output is required (set CATALOG_FILE_PATH or use -o)") + elif args.command in ('add', 'delete'): + if not args.input: + parser.error(f"{args.command}: --input is required") + if not args.catalog: + parser.error(f"{args.command}: --catalog is required (set CATALOG_FILE_PATH or use -c)") + elif args.command == 'validate': + if not args.catalog: + parser.error("validate: --catalog is required (set CATALOG_FILE_PATH or use -c)") + + setup_logging(args.log_dir) + + return args.func(args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/repo_manager/plugins/module_utils/catalog/mutator.py b/src/repo_manager/plugins/module_utils/catalog/mutator.py new file mode 100644 index 0000000000..18089a3e73 --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/mutator.py @@ -0,0 +1,164 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Catalog mutation operations: add (upsert) and delete packages/groups. +""" + +import logging + +logger = logging.getLogger(__name__) + + +def upsert_packages(catalog, parsed): + """ + Add or update packages in a catalog (upsert semantics). + + Args: + catalog: Catalog dict (with 'catalog' root key). + parsed: Parsed input dict with 'functional_layers', 'groups', and 'packages'. + + Returns: + dict: Summary with counts {'added', 'updated', 'groups_created', 'fl_created'}. + """ + cat = catalog['catalog'] + functional_layers = cat.setdefault('functionallayer', []) + groups = cat.setdefault('groups', {}) + packages = cat.setdefault('packages', {}) + + summary = {'added': 0, 'updated': 0, 'groups_created': 0, 'fl_created': 0} + + # Process functional layers + input_fl = parsed.get('functional_layers', []) + existing_fl_names = {fl['name'] for fl in functional_layers} + for fl_entry in input_fl: + fl_name = fl_entry.get('name') + if fl_name not in existing_fl_names: + # Add new functional layer + functional_layers.append(fl_entry) + summary['fl_created'] += 1 + logger.info("Created new functional layer: %s", fl_name) + else: + # Merge components into existing functional layer + for existing_fl in functional_layers: + if existing_fl['name'] == fl_name: + existing_comps = set(existing_fl.get('components', [])) + for comp in fl_entry.get('components', []): + if comp not in existing_comps: + existing_fl['components'].append(comp) + logger.debug("Added %s to functional layer %s", comp, fl_name) + break + + # Process groups + for group_key, group_entry in parsed['groups'].items(): + if group_key not in groups: + # Create new group + groups[group_key] = { + 'name': group_entry['name'], + 'type': group_entry.get('type', 'group'), + 'description': group_entry.get('description', ''), + 'components': [] + } + if group_entry.get('type') == 'base_os': + groups[group_key]['os'] = group_entry.get('os', '') + groups[group_key]['os_version'] = group_entry.get('os_version', '') + summary['groups_created'] += 1 + logger.info("Created new group: %s", group_key) + + # Process packages + for pkg_key, pkg_entry in parsed['packages'].items(): + if pkg_key in packages: + # Update existing package + packages[pkg_key] = pkg_entry + summary['updated'] += 1 + logger.info("Updated package: %s", pkg_key) + else: + # Add new package + packages[pkg_key] = pkg_entry + summary['added'] += 1 + logger.info("Added package: %s", pkg_key) + + # Ensure package keys are in their group's components list + for group_key, group_entry in parsed['groups'].items(): + if group_key in groups: + existing_components = set(groups[group_key].get('components', [])) + for pkg_key in group_entry.get('components', []): + if pkg_key not in existing_components: + groups[group_key]['components'].append(pkg_key) + logger.debug("Added %s to group %s components", pkg_key, group_key) + + return summary + + +def delete_packages(catalog, parsed_delete): + """ + Delete packages from a catalog. + + Args: + catalog: Catalog dict (with 'catalog' root key). + parsed_delete: Dict {group_key: [pkg_key, ...]}. + + Returns: + dict: Summary with counts {'deleted', 'groups_removed', 'skipped'}. + """ + cat = catalog['catalog'] + groups = cat.get('groups', {}) + packages = cat.get('packages', {}) + functional_layers = cat.get('functionallayer', []) + + summary = {'deleted': 0, 'groups_removed': 0, 'skipped': 0} + + for group_key, pkg_keys in parsed_delete.items(): + if group_key not in groups: + logger.warning("Group [%s] not found - skipping", group_key) + summary['skipped'] += len(pkg_keys) + continue + + group = groups[group_key] + components = group.get('components', []) + + for pkg_key in pkg_keys: + if pkg_key not in components: + logger.warning("Package '%s' not in group [%s] - skipping", pkg_key, group_key) + summary['skipped'] += 1 + continue + + # Remove from group components + components.remove(pkg_key) + logger.info("Removed '%s' from group [%s]", pkg_key, group_key) + + # Check if package is still referenced by any group + still_referenced = any( + pkg_key in g.get('components', []) + for g in groups.values() + ) + + if not still_referenced and pkg_key in packages: + del packages[pkg_key] + logger.info("Deleted package '%s' from catalog (no remaining references)", pkg_key) + elif still_referenced: + logger.info("Package '%s' retained (still referenced by other groups)", pkg_key) + + summary['deleted'] += 1 + + # Remove empty groups + if not components: + del groups[group_key] + # Also remove from functional layers + for layer in functional_layers: + if group_key in layer.get('components', []): + layer['components'].remove(group_key) + summary['groups_removed'] += 1 + logger.info("Removed empty group [%s]", group_key) + + return summary diff --git a/src/repo_manager/plugins/module_utils/catalog/parser.py b/src/repo_manager/plugins/module_utils/catalog/parser.py new file mode 100644 index 0000000000..14a37868ae --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/parser.py @@ -0,0 +1,289 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=too-many-branches,too-many-locals,too-many-statements +""" +Input file parser for catalog operations. + +Parses the INI-like input format: + [defaults] + arch=x86_64, os=rhel, os_version=10.0 + + [group_key | type=base_os, description=..., os=rhel, os_version=10.0] + pkg_key, rpm, name, reponame + pkg_key, tarball, name, url + pkg_key, image, image_path, registry, tag +""" + +import re +import logging + +logger = logging.getLogger(__name__) + +# Regex patterns +DEFAULTS_HEADER = re.compile(r'^\[defaults\]\s*$', re.IGNORECASE) +GROUP_HEADER = re.compile(r'^\[([^\]|]+)(?:\s*\|\s*([^\]]*))?\]\s*$') +KV_PAIR = re.compile(r'(\w+)\s*=\s*([^,\]]+)') + + +def _parse_kv_pairs(text): + """Parse key=value pairs from a string.""" + return {m.group(1).lower().strip(): m.group(2).strip() for m in KV_PAIR.finditer(text)} + + +def _parse_trailing_overrides(fields, start_index): + """Parse trailing key=value overrides from positional fields.""" + overrides = {} + for field in fields[start_index:]: + if '=' in field: + kv = _parse_kv_pairs(field) + overrides.update(kv) + return overrides + + +def _build_package_entry(pkg_type, fields, defaults, overrides): + """Build a package entry dict based on package type.""" + arch = overrides.get('arch', defaults['arch']) + os_name = overrides.get('os', defaults['os']) + os_version = overrides.get('os_version', defaults['os_version']) + + if pkg_type in ('rpm', 'rpm_repo'): + # fields: [pkg_key, type, name, reponame, ...] + name = fields[2] if len(fields) > 2 else fields[0] + reponame = fields[3] if len(fields) > 3 else '' + return { + "name": name, + "packagetype": pkg_type, + "sources": [{ + "architecture": arch, + "reponame": reponame, + "name": os_name, + "version": [os_version] + }] + } + elif pkg_type == 'tarball': + # fields: [pkg_key, tarball, name, url, ...] + name = fields[2] if len(fields) > 2 else fields[0] + url = fields[3] if len(fields) > 3 else '' + return { + "name": name, + "packagetype": "tarball", + "sources": [{ + "architecture": arch, + "name": os_name, + "version": [os_version], + "url": url + }] + } + elif pkg_type == 'image': + # fields: [pkg_key, image, image_path, registry, tag, ...] + image_path = fields[2] if len(fields) > 2 else fields[0] + registry = fields[3] if len(fields) > 3 else '' + tag = fields[4] if len(fields) > 4 else 'latest' + return { + "name": image_path, + "packagetype": "image", + "tag": tag, + "sources": [{ + "architecture": arch, + "registry": registry + }] + } + else: + raise ValueError(f"Unknown package type: {pkg_type}") + + +def parse_input_file(filepath, default_arch='x86_64', default_os='rhel', default_os_version='10.0'): + """ + Parse an input file for catalog generate/add operations. + + Args: + filepath: Path to the input file. + default_arch: Default architecture if not specified. + default_os: Default OS if not specified. + default_os_version: Default OS version if not specified. + + Returns: + dict: {'functional_layers': [...], 'groups': {...}, 'packages': {...}} + + Raises: + ValueError: On parse errors (duplicate groups, package before group, etc.) + FileNotFoundError: If input file doesn't exist. + """ + defaults = { + 'arch': default_arch, + 'os': default_os, + 'os_version': default_os_version + } + functional_layers = [] + groups = {} + packages = {} + current_section = None + current_section_type = None + line_num = 0 + + with open(filepath, 'r', encoding='utf-8') as fh: + for line in fh: + line_num += 1 + line = line.strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + # Check for [defaults] header + if DEFAULTS_HEADER.match(line): + current_section = '__defaults__' + current_section_type = 'defaults' + continue + + # Check for section header (group or functional layer) + section_match = GROUP_HEADER.match(line) + if section_match: + section_key = section_match.group(1).strip() + metadata_str = section_match.group(2) or '' + + # Parse section metadata + meta = _parse_kv_pairs(metadata_str) + section_type = meta.get('type', 'group') + section_desc = meta.get('description', '') + + if section_type == 'functional_layer': + # Functional layer: components are group references + fl_entry = { + "name": section_key, + "components": [] + } + functional_layers.append(fl_entry) + current_section = fl_entry + current_section_type = 'functional_layer' + else: + # Regular group or base_os group + if section_key in groups: + raise ValueError(f"Line {line_num}: Duplicate group '{section_key}'") + + group_entry = { + "name": section_key, + "type": section_type, + "description": section_desc, + "components": [] + } + if section_type == 'base_os': + group_entry['os'] = meta.get('os', defaults['os']) + group_entry['os_version'] = meta.get('os_version', defaults['os_version']) + + groups[section_key] = group_entry + current_section = section_key + current_section_type = 'group' + continue + + # If we're in the defaults section, parse key=value pairs + if current_section_type == 'defaults': + # Line like: arch=x86_64, os=rhel, os_version=10.0 + kv = _parse_kv_pairs(line) + if 'arch' in kv: + defaults['arch'] = kv['arch'] + if 'os' in kv: + defaults['os'] = kv['os'] + if 'os_version' in kv: + defaults['os_version'] = kv['os_version'] + continue + + # Content line + if current_section is None: + raise ValueError(f"Line {line_num}: Content before any section header") + + # If we're in a functional layer, the line is a group reference + if current_section_type == 'functional_layer': + # Remove quotes and trailing comma if present + group_ref = line.strip().strip(',').strip('"').strip("'") + if group_ref: + current_section['components'].append(group_ref) + continue + + # Otherwise, it's a package line in a group + # Split by comma, strip each field + fields = [f.strip() for f in line.split(',')] + if len(fields) < 2: + raise ValueError(f"Line {line_num}: Package line needs at least key and type") + + pkg_key = fields[0] + pkg_type = fields[1].lower() + + if pkg_type not in ('rpm', 'rpm_repo', 'tarball', 'image'): + raise ValueError(f"Line {line_num}: Unknown package type '{pkg_type}'") + + # Determine where trailing overrides start + override_start = 4 # Default for rpm/rpm_repo/tarball + if pkg_type == 'image': + override_start = 5 + + overrides = _parse_trailing_overrides(fields, override_start) + pkg_entry = _build_package_entry(pkg_type, fields, defaults, overrides) + packages[pkg_key] = pkg_entry + groups[current_section]['components'].append(pkg_key) + + logger.info("Parsed input file: %d functional layers, %d groups, %d packages", + len(functional_layers), len(groups), len(packages)) + return {'functional_layers': functional_layers, 'groups': groups, 'packages': packages} + + +def parse_delete_file(filepath): + """ + Parse a delete input file (simplified format). + + Format: + [group_key] + pkg_key1 + pkg_key2 + + Args: + filepath: Path to the delete input file. + + Returns: + dict: {group_key: [pkg_key, pkg_key, ...], ...} + + Raises: + FileNotFoundError: If input file doesn't exist. + """ + result = {} + current_group = None + line_num = 0 + + with open(filepath, 'r', encoding='utf-8') as fh: + for line in fh: + line_num += 1 + line = line.strip() + + if not line or line.startswith('#'): + continue + + # Group header: [group_key] + group_match = GROUP_HEADER.match(line) + if group_match: + current_group = group_match.group(1).strip() + if current_group not in result: + result[current_group] = [] + continue + + # Package key line + if current_group is None: + raise ValueError(f"Line {line_num}: Package key before any group header") + + # The line is just a package key + pkg_key = line.split(',')[0].strip() # Handle trailing commas gracefully + if pkg_key: + result[current_group].append(pkg_key) + + logger.info("Parsed delete file: %d groups", len(result)) + return result diff --git a/src/repo_manager/plugins/module_utils/catalog/validator.py b/src/repo_manager/plugins/module_utils/catalog/validator.py new file mode 100644 index 0000000000..4810589c92 --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/validator.py @@ -0,0 +1,293 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=too-many-branches,too-many-locals +""" +Catalog validation: JSON schema validation and business rule checks. +""" + +import json +import logging + +logger = logging.getLogger(__name__) + +VALID_PACKAGE_TYPES = {'rpm', 'tarball', 'image', 'rpm_repo'} + + +def _validate_schema(catalog, schema): + """ + Validate catalog against JSON schema. + + Returns: + list: List of issue dicts {'severity': 'error', 'message': ...} + """ + issues = [] + try: + import jsonschema + jsonschema.validate(instance=catalog, schema=schema) + except ImportError: + logger.warning("jsonschema not installed, skipping schema validation") + issues.append({ + 'severity': 'warning', + 'message': 'jsonschema library not installed, schema validation skipped' + }) + except jsonschema.ValidationError as e: + issues.append({ + 'severity': 'error', + 'message': f"Schema validation failed: {e.message}" + }) + except jsonschema.SchemaError as e: + issues.append({ + 'severity': 'error', + 'message': f"Invalid schema: {e.message}" + }) + return issues + + +def _validate_referential_integrity(catalog): + """ + Check referential integrity between layers, groups, and packages. + + Returns: + list: List of issue dicts. + """ + issues = [] + cat = catalog.get('catalog', {}) + functional_layers = cat.get('functionallayer', []) + groups = cat.get('groups', {}) + packages = cat.get('packages', {}) + + # Check functional layer references + for layer in functional_layers: + layer_name = layer.get('name', '') + for ref in layer.get('components', []): + if ref not in groups: + issues.append({ + 'severity': 'error', + 'message': f"FunctionalLayer '{layer_name}' references missing group '{ref}'" + }) + + # Check group references + for group_key, group in groups.items(): + for ref in group.get('components', []): + if ref not in packages: + issues.append({ + 'severity': 'error', + 'message': f"Group '{group_key}' references missing package '{ref}'" + }) + + return issues + + +def _validate_business_rules(catalog): + """ + Check business rules for functional layers, packages, and groups. + + Returns: + list: List of issue dicts. + """ + issues = [] + cat = catalog.get('catalog', {}) + functional_layers = cat.get('functionallayer', []) + groups = cat.get('groups', {}) + packages = cat.get('packages', {}) + + # Check functional layers are not empty + if not functional_layers: + issues.append({ + 'severity': 'error', + 'message': 'Catalog must have at least one functional layer' + }) + + # Check each functional layer + for layer in functional_layers: + layer_name = layer.get('name', '') + components = layer.get('components', []) + + # Check components are not empty + if not components: + issues.append({ + 'severity': 'error', + 'message': f"Functional layer '{layer_name}' has no components" + }) + + # Check for exactly one base_os group in components + base_os_count = 0 + for comp_ref in components: + comp_group = groups.get(comp_ref, {}) + if comp_group.get('type') == 'base_os': + base_os_count += 1 + + if base_os_count == 0: + issues.append({ + 'severity': 'error', + 'message': f"Functional layer '{layer_name}' must have exactly one base_os group (found 0)" + }) + elif base_os_count > 1: + issues.append({ + 'severity': 'error', + 'message': f"Functional layer '{layer_name}' must have exactly one base_os group (found {base_os_count})" + }) + + # Check for duplicate entries in group components + for group_key, group in groups.items(): + components = group.get('components', []) + seen = set() + for comp in components: + if comp in seen: + issues.append({ + 'severity': 'error', + 'message': f"Group '{group_key}' has duplicate component '{comp}'" + }) + seen.add(comp) + + # Check base_os groups have os and os_version + for group_key, group in groups.items(): + if group.get('type') == 'base_os': + if not group.get('os'): + issues.append({ + 'severity': 'error', + 'message': f"base_os group '{group_key}' missing 'os' field" + }) + if not group.get('os_version'): + issues.append({ + 'severity': 'error', + 'message': f"base_os group '{group_key}' missing 'os_version' field" + }) + + # Check packages + for pkg_key, pkg in packages.items(): + pkg_type = pkg.get('packagetype', '') + sources = pkg.get('sources', []) + + # Every package must have at least one source + if not sources: + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' has no sources" + }) + + # Package type must be valid + if pkg_type not in VALID_PACKAGE_TYPES: + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' has invalid packagetype '{pkg_type}'" + }) + + # Type-specific checks + if pkg_type in ('rpm', 'rpm_repo'): + for src in sources: + if not src.get('reponame'): + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' ({pkg_type}) source missing 'reponame'" + }) + + if pkg_type == 'tarball': + for src in sources: + if not src.get('url'): + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' (tarball) source missing 'url'" + }) + + if pkg_type == 'image': + if not pkg.get('tag'): + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' (image) missing 'tag'" + }) + for src in sources: + if not src.get('registry'): + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' (image) source missing 'registry'" + }) + + # Check for orphan packages (in packages{} but unreferenced by any group) + all_referenced = set() + for group in groups.values(): + all_referenced.update(group.get('components', [])) + for pkg_key in packages: + if pkg_key not in all_referenced: + issues.append({ + 'severity': 'warning', + 'message': f"Orphan package '{pkg_key}' not referenced by any group" + }) + + # Check for orphan groups (in groups{} but unreferenced by any functional layer) + fl_referenced = set() + for layer in functional_layers: + fl_referenced.update(layer.get('components', [])) + for group_key in groups: + if group_key not in fl_referenced: + issues.append({ + 'severity': 'warning', + 'message': f"Orphan group '{group_key}' not referenced by any functional layer" + }) + + return issues + + +def validate_catalog(catalog, schema_path=None): + """ + Validate a catalog with all validation layers. + + Args: + catalog: Catalog dict (with 'catalog' root key). + schema_path: Optional path to JSON schema file. + + Returns: + list: List of issue dicts {'severity': 'error'|'warning', 'message': ...} + """ + issues = [] + + # Layer 1: Schema validation + if schema_path: + try: + with open(schema_path, 'r', encoding='utf-8') as fh: + schema = json.load(fh) + issues.extend(_validate_schema(catalog, schema)) + except FileNotFoundError: + issues.append({ + 'severity': 'warning', + 'message': f"Schema file not found: {schema_path}" + }) + except json.JSONDecodeError as e: + issues.append({ + 'severity': 'error', + 'message': f"Invalid schema JSON: {e}" + }) + + # Layer 2: Referential integrity + issues.extend(_validate_referential_integrity(catalog)) + + # Layer 3: Business rules + issues.extend(_validate_business_rules(catalog)) + + return issues + + +def format_issues(issues): + """Format issues for display.""" + lines = [] + errors = [i for i in issues if i['severity'] == 'error'] + warnings = [i for i in issues if i['severity'] == 'warning'] + + for issue in errors: + lines.append(f"[ERROR] {issue['message']}") + for issue in warnings: + lines.append(f"[WARNING] {issue['message']}") + + lines.append(f"\nSummary: {len(errors)} error(s), {len(warnings)} warning(s)") + return '\n'.join(lines) diff --git a/src/repo_manager/roles/catalog/defaults/main.yml b/src/repo_manager/roles/catalog/defaults/main.yml new file mode 100644 index 0000000000..a381d40d5d --- /dev/null +++ b/src/repo_manager/roles/catalog/defaults/main.yml @@ -0,0 +1,36 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG ROLE DEFAULTS +# ============================================================================= + +# ── Paths ── +catalog_scripts_dir: "{{ playbook_dir | dirname }}/plugins/module_utils/catalog" +# Use CATALOG_FILE_PATH env var if set, otherwise use default location +catalog_file: "{{ lookup('env', 'CATALOG_FILE_PATH') | default('/opt/omnia/catalog/catalog_rhel.json', true) }}" +schema_file: "{{ playbook_dir | dirname }}/schemas/catalog_schema.json" +catalog_log_dir: "{{ lookup('env', 'OMNIA_DATA_PATH') | default('/opt/omnia', true) }}/repo_manager/log/catalog" + +# ── Generate defaults ── +catalog_name: "default" +force: false + +# ── Package defaults (used when [defaults] header absent in input file) ── +default_arch: "x86_64" +default_os: "rhel" +default_os_version: "10.0" + +# ── Post-operation validation ── +validate_after: true diff --git a/src/repo_manager/roles/catalog/meta/main.yml b/src/repo_manager/roles/catalog/meta/main.yml new file mode 100644 index 0000000000..5b1217f5e8 --- /dev/null +++ b/src/repo_manager/roles/catalog/meta/main.yml @@ -0,0 +1,29 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +galaxy_info: + author: Dell Inc. + description: Catalog management operations for repo_manager + license: Apache-2.0 + min_ansible_version: "2.14" + platforms: + - name: EL + versions: + - "10" + galaxy_tags: + - catalog + - repomanager + - omnia + +dependencies: [] diff --git a/src/repo_manager/roles/catalog/tasks/add.yml b/src/repo_manager/roles/catalog/tasks/add.yml new file mode 100644 index 0000000000..8916699097 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/add.yml @@ -0,0 +1,54 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG ADD OPERATION +# ============================================================================= + +- name: "Assert input_file is provided for add" + ansible.builtin.assert: + that: input_file is defined and input_file | length > 0 + fail_msg: "{{ catalog_input_file_required_msg }}" + +- name: "Ensure log directory exists" + ansible.builtin.file: + path: "{{ catalog_log_dir }}" + state: directory + mode: "0755" + +- name: "Set catalog input file (use catalog_input if provided, otherwise catalog_file)" + ansible.builtin.set_fact: + _catalog_input: "{{ catalog_input | default(catalog_file, true) }}" + _output_file: "{{ output_file | default(catalog_file, true) }}" + +- name: "Add packages to catalog" + ansible.builtin.command: + cmd: >- + python3 {{ catalog_scripts_dir }}/catalog_manager.py + --log-dir {{ catalog_log_dir }} + add + --input {{ input_file }} + --catalog {{ _catalog_input }} + --output {{ _output_file }} + --default-arch {{ default_arch }} + --default-os {{ default_os }} + --default-os-version {{ default_os_version }} + {{ '--schema ' + schema_file if validate_after | bool else '' }} + register: add_result + changed_when: add_result.rc == 0 + +- name: "Show add result" + ansible.builtin.debug: + msg: "{{ add_result.stdout_lines }}" + when: add_result is defined and add_result.stdout_lines is defined diff --git a/src/repo_manager/roles/catalog/tasks/delete.yml b/src/repo_manager/roles/catalog/tasks/delete.yml new file mode 100644 index 0000000000..fa1a88c4d4 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/delete.yml @@ -0,0 +1,51 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG DELETE OPERATION +# ============================================================================= + +- name: "Assert input_file is provided for delete" + ansible.builtin.assert: + that: input_file is defined and input_file | length > 0 + fail_msg: "{{ catalog_input_file_required_msg }}" + +- name: "Ensure log directory exists" + ansible.builtin.file: + path: "{{ catalog_log_dir }}" + state: directory + mode: "0755" + +- name: "Set catalog input file (use catalog_input if provided, otherwise catalog_file)" + ansible.builtin.set_fact: + _catalog_input: "{{ catalog_input | default(catalog_file, true) }}" + _output_file: "{{ output_file | default(catalog_file, true) }}" + +- name: "Delete packages from catalog" + ansible.builtin.command: + cmd: >- + python3 {{ catalog_scripts_dir }}/catalog_manager.py + --log-dir {{ catalog_log_dir }} + delete + --input {{ input_file }} + --catalog {{ _catalog_input }} + --output {{ _output_file }} + {{ '--schema ' + schema_file if validate_after | bool else '' }} + register: del_result + changed_when: del_result.rc == 0 + +- name: "Show delete result" + ansible.builtin.debug: + msg: "{{ del_result.stdout_lines }}" + when: del_result is defined and del_result.stdout_lines is defined diff --git a/src/repo_manager/roles/catalog/tasks/generate.yml b/src/repo_manager/roles/catalog/tasks/generate.yml new file mode 100644 index 0000000000..48eec1f380 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/generate.yml @@ -0,0 +1,54 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG GENERATE OPERATION +# ============================================================================= + +- name: "Assert input_file is provided for generate" + ansible.builtin.assert: + that: input_file is defined and input_file | length > 0 + fail_msg: "{{ catalog_input_file_required_msg }}" + +- name: "Ensure log directory exists" + ansible.builtin.file: + path: "{{ catalog_log_dir }}" + state: directory + mode: "0755" + +- name: "Set output file (use output_file if provided, otherwise catalog_file)" + ansible.builtin.set_fact: + _output_file: "{{ output_file | default(catalog_file, true) }}" + +- name: "Generate catalog from input file" + ansible.builtin.command: + cmd: >- + python3 {{ catalog_scripts_dir }}/catalog_manager.py + --log-dir {{ catalog_log_dir }} + generate + --input {{ input_file }} + --output {{ _output_file }} + --name {{ catalog_name }} + --default-arch {{ default_arch }} + --default-os {{ default_os }} + --default-os-version {{ default_os_version }} + {{ '--force' if force | bool else '' }} + {{ '--schema ' + schema_file if validate_after | bool else '' }} + register: gen_result + changed_when: gen_result.rc == 0 + +- name: "Show generate result" + ansible.builtin.debug: + msg: "{{ gen_result.stdout_lines }}" + when: gen_result is defined and gen_result.stdout_lines is defined diff --git a/src/repo_manager/roles/catalog/tasks/main.yml b/src/repo_manager/roles/catalog/tasks/main.yml new file mode 100644 index 0000000000..836acfb959 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/main.yml @@ -0,0 +1,36 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG ROLE - MAIN ENTRY POINT +# ============================================================================= +# Routes to specific operation based on tags. +# Each operation is in a separate file for isolation. +# ============================================================================= + +- name: Include catalog generate tasks + ansible.builtin.include_tasks: generate.yml + when: "'catalog_generate' in ansible_run_tags" + +- name: Include catalog add tasks + ansible.builtin.include_tasks: add.yml + when: "'catalog_add' in ansible_run_tags" + +- name: Include catalog delete tasks + ansible.builtin.include_tasks: delete.yml + when: "'catalog_delete' in ansible_run_tags" + +- name: Include catalog validate tasks + ansible.builtin.include_tasks: validate.yml + when: "'catalog_validate' in ansible_run_tags" diff --git a/src/repo_manager/roles/catalog/tasks/validate.yml b/src/repo_manager/roles/catalog/tasks/validate.yml new file mode 100644 index 0000000000..af12fe66ed --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/validate.yml @@ -0,0 +1,49 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG VALIDATE OPERATION +# ============================================================================= + +- name: "Ensure log directory exists" + ansible.builtin.file: + path: "{{ catalog_log_dir }}" + state: directory + mode: "0755" + +- name: "Set catalog input file (use catalog_input if provided, otherwise catalog_file)" + ansible.builtin.set_fact: + _catalog_input: "{{ catalog_input | default(catalog_file, true) }}" + +- name: "Validate catalog" + ansible.builtin.command: + cmd: >- + python3 {{ catalog_scripts_dir }}/catalog_manager.py + --log-dir {{ catalog_log_dir }} + validate + --catalog {{ _catalog_input }} + --schema {{ schema_file }} + register: val_result + changed_when: false + failed_when: false + +- name: "Show validation result" + ansible.builtin.debug: + msg: "{{ val_result.stdout_lines }}" + when: val_result is defined and val_result.stdout_lines is defined + +- name: "Fail on validation errors" + ansible.builtin.fail: + msg: "{{ catalog_validation_failed_msg }}" + when: val_result is defined and val_result.rc == 1 diff --git a/src/repo_manager/roles/catalog/vars/main.yml b/src/repo_manager/roles/catalog/vars/main.yml new file mode 100644 index 0000000000..8b6d4d92d5 --- /dev/null +++ b/src/repo_manager/roles/catalog/vars/main.yml @@ -0,0 +1,49 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG ROLE MESSAGES +# ============================================================================= + +# ── Error messages ── +catalog_input_file_required_msg: >- + Required: -e "input_file=path/to/packages.txt" + +catalog_generate_exists_msg: >- + Output file '{{ catalog_file }}' already exists. + Use -e "force=true" to overwrite. + +catalog_file_not_found_msg: >- + Catalog file '{{ catalog_file }}' not found. + Generate a catalog first with --tags catalog_generate. + +catalog_parse_error_msg: >- + Failed to parse input file. Check the format and try again. + See docs/catalog_operations.md for format details. + +catalog_validation_failed_msg: >- + Catalog validation failed. See errors above. + +# ── Success messages ── +catalog_generate_success_msg: >- + Successfully generated catalog at {{ catalog_file }}. + +catalog_add_success_msg: >- + Successfully added packages to catalog. + +catalog_delete_success_msg: >- + Successfully deleted packages from catalog. + +catalog_validate_success_msg: >- + Catalog validation passed. diff --git a/src/repo_manager/samples/catalog_generator_input.txt b/src/repo_manager/samples/catalog_generator_input.txt new file mode 100644 index 0000000000..9fb51041de --- /dev/null +++ b/src/repo_manager/samples/catalog_generator_input.txt @@ -0,0 +1,36 @@ +# Test catalog input file with functional layers +[defaults] +arch=x86_64, os=rhel, os_version=10.0 + +[baseos_group_10.0 | type=base_os, description=base os packages for rhel cluster nodes, os=rhel, os_version=10.0] +systemd, rpm, systemd, baseos +systemd_udev, rpm, systemd-udev, baseos +wget, rpm, wget, appstream +glibc_langpack_en, rpm, glibc-langpack-en, baseos + +[slurm_custom_group | description=slurm custom packages] +clustershell, rpm, clustershell, epel +papi, tarball, papi, https://github.com/icl-utk-edu/papi/releases/download/papi-7-2-0-t/papi-7.2.0.tar.gz +curl_image, image, docker.io/curlimages/curl, docker.io, 8.17.0 + +[openldap_group | description=OpenLDAP packages] +openldap, rpm, openldap, baseos +openldap_clients, rpm, openldap-clients, baseos + +[slurm_control_node_group | description=Slurm controller specific packages] +slurm_slurmctld, rpm, slurm-slurmctld, slurm_custom + +[slurm_node_group | description=Slurm compute node packages] +slurm_slurmd, rpm, slurm-slurmd, slurm_custom + +[slurm_control_node_rhel_10_0_x86_64 | type=functional_layer] +"baseos_group_10.0", +"slurm_custom_group", +"openldap_group", +"slurm_control_node_group" + +[slurm_node_rhel_10_0_x86_64 | type=functional_layer] +"baseos_group_10.0", +"slurm_custom_group", +"openldap_group", +"slurm_node_group" diff --git a/src/repo_manager/schemas/catalog_schema.json b/src/repo_manager/schemas/catalog_schema.json new file mode 100644 index 0000000000..30ea12cac2 --- /dev/null +++ b/src/repo_manager/schemas/catalog_schema.json @@ -0,0 +1,118 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "HPC Catalog Schema", + "description": "Schema for the omnia services catalog format", + "type": "object", + "required": ["catalog"], + "additionalProperties": false, + "properties": { + "catalog": { + "type": "object", + "required": ["name", "version", "identifier", "description", "functionallayer", "groups", "packages"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "pattern": "^\\d+\\.\\d+(\\.\\d+)?$" }, + "identifier": { "type": "string", "pattern": "^[a-z0-9][a-z0-9_-]*$" }, + "description": { "type": "string" }, + "functionallayer": { + "type": "array", + "items": { "$ref": "#/definitions/functional_layer" } + }, + "groups": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^[a-zA-Z0-9_][a-zA-Z0-9_.+-]*$": { "$ref": "#/definitions/group" } + }, + "additionalProperties": false + }, + "packages": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^[a-zA-Z0-9_][a-zA-Z0-9_-]*$": { "$ref": "#/definitions/package" } + }, + "additionalProperties": false + } + } + } + }, + "definitions": { + "functional_layer": { + "type": "object", + "required": ["name", "components"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "components": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1, + "uniqueItems": true + } + } + }, + "group": { + "type": "object", + "required": ["name", "type", "description", "components"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "type": { "type": "string", "enum": ["base_os", "group"] }, + "os": { "type": "string" }, + "os_version": { "type": "string" }, + "description": { "type": "string" }, + "components": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "uniqueItems": true + } + }, + "if": { + "properties": { "type": { "const": "base_os" } } + }, + "then": { + "required": ["name", "type", "description", "components", "os", "os_version"] + } + }, + "package": { + "type": "object", + "required": ["name", "packagetype", "sources"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "packagetype": { "type": "string", "enum": ["rpm", "tarball", "image", "rpm_repo"] }, + "tag": { "type": "string" }, + "sources": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/source" } + } + }, + "allOf": [ + { + "if": { "properties": { "packagetype": { "const": "image" } } }, + "then": { "required": ["name", "packagetype", "tag", "sources"] } + } + ] + }, + "source": { + "type": "object", + "required": ["architecture"], + "additionalProperties": false, + "properties": { + "architecture": { "type": "string", "enum": ["x86_64", "aarch64", "ppc64le", "noarch"] }, + "reponame": { "type": "string" }, + "registry": { "type": "string" }, + "name": { "type": "string" }, + "version": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "url": { "type": "string", "format": "uri" } + } + } + } +}