From 60d881075398cdf5c5b4c519c1c5720358f8c569 Mon Sep 17 00:00:00 2001 From: venu <236371043+Venu-p1@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:49:00 +0530 Subject: [PATCH 1/4] feat(repo_manager): Add catalog management feature Implement catalog operations (generate, add, delete, validate) with: - Python backend: parser, I/O, mutator, validator, CLI dispatcher - Ansible role with operation-specific tasks - JSON schema for validation - INI-like input format with [defaults] section - Upsert semantics and automatic orphan cleanup - Multi-layer validation and logging - Comprehensive documentation Signed-off-by: venu <236371043+Venu-p1@users.noreply.github.com> --- src/repo_manager/docs/catalog_operations.md | 306 ++++++++++++++++++ src/repo_manager/playbooks/repo_manager.yml | 24 ++ .../plugins/module_utils/catalog/__init__.py | 19 ++ .../module_utils/catalog/catalog_io.py | 109 +++++++ .../module_utils/catalog/catalog_manager.py | 246 ++++++++++++++ .../plugins/module_utils/catalog/mutator.py | 142 ++++++++ .../plugins/module_utils/catalog/parser.py | 264 +++++++++++++++ .../plugins/module_utils/catalog/validator.py | 256 +++++++++++++++ .../roles/catalog/defaults/main.yml | 35 ++ src/repo_manager/roles/catalog/meta/main.yml | 29 ++ src/repo_manager/roles/catalog/tasks/add.yml | 49 +++ .../roles/catalog/tasks/delete.yml | 46 +++ .../roles/catalog/tasks/generate.yml | 50 +++ src/repo_manager/roles/catalog/tasks/main.yml | 36 +++ .../roles/catalog/tasks/validate.yml | 45 +++ src/repo_manager/roles/catalog/vars/main.yml | 49 +++ src/repo_manager/schemas/catalog_schema.json | 118 +++++++ 17 files changed, 1823 insertions(+) create mode 100644 src/repo_manager/docs/catalog_operations.md create mode 100644 src/repo_manager/plugins/module_utils/catalog/__init__.py create mode 100644 src/repo_manager/plugins/module_utils/catalog/catalog_io.py create mode 100644 src/repo_manager/plugins/module_utils/catalog/catalog_manager.py create mode 100644 src/repo_manager/plugins/module_utils/catalog/mutator.py create mode 100644 src/repo_manager/plugins/module_utils/catalog/parser.py create mode 100644 src/repo_manager/plugins/module_utils/catalog/validator.py create mode 100644 src/repo_manager/roles/catalog/defaults/main.yml create mode 100644 src/repo_manager/roles/catalog/meta/main.yml create mode 100644 src/repo_manager/roles/catalog/tasks/add.yml create mode 100644 src/repo_manager/roles/catalog/tasks/delete.yml create mode 100644 src/repo_manager/roles/catalog/tasks/generate.yml create mode 100644 src/repo_manager/roles/catalog/tasks/main.yml create mode 100644 src/repo_manager/roles/catalog/tasks/validate.yml create mode 100644 src/repo_manager/roles/catalog/vars/main.yml create mode 100644 src/repo_manager/schemas/catalog_schema.json diff --git a/src/repo_manager/docs/catalog_operations.md b/src/repo_manager/docs/catalog_operations.md new file mode 100644 index 0000000000..181c0ef0bb --- /dev/null +++ b/src/repo_manager/docs/catalog_operations.md @@ -0,0 +1,306 @@ +# Catalog Operations Guide + +This document describes how to use the catalog management system in repo_manager. + +## Overview + +The catalog management system provides operations to: +- **Generate**: Create a new catalog from an input text file +- **Add**: Add or update packages in an existing catalog (upsert semantics) +- **Delete**: Remove packages from a catalog +- **Validate**: Validate a catalog against the schema and business rules + +## Quick Start + +```bash +cd /path/to/repo_manager/playbooks + +# Generate a new catalog +ansible-playbook repo_manager.yml --tags catalog_generate \ + -e "input_file=input/packages.txt" + +# Add packages to existing catalog +ansible-playbook repo_manager.yml --tags catalog_add \ + -e "input_file=input/additions.txt" + +# Delete packages from catalog +ansible-playbook repo_manager.yml --tags catalog_delete \ + -e "input_file=input/removals.txt" + +# Validate a catalog +ansible-playbook repo_manager.yml --tags catalog_validate +``` + +## Input File Format + +### For Generate and Add Operations + +The input file uses an INI-like format with optional defaults header: + +```ini +# Optional defaults section (applies to all packages unless overridden) +[defaults] +arch=x86_64, os=rhel, os_version=10.0 + +# Group headers with optional metadata +# Format: [group_key | key=value, key=value, ...] +# Supported metadata: type, description, os, os_version +# 'type' defaults to "group" if omitted + +[baseos_group_10.0 | type=base_os, description=base os packages, os=rhel, os_version=10.0] +systemd, rpm, systemd, baseos +wget, rpm, wget, appstream +glibc_langpack_en, rpm, glibc-langpack-en, baseos + +[slurm_custom_group | description=slurm custom packages] +clustershell, rpm, clustershell, epel +papi, tarball, papi, https://github.com/icl-utk-edu/papi/releases/download/papi-7-2-0-t/papi-7.2.0.tar.gz +curl, image, docker.io/curlimages/curl, docker.io, 8.17.0 + +# Override arch for a specific package +doca_ofed, rpm_repo, doca-ofed, doca, arch=aarch64 +``` + +### Package Line Formats + +| Type | Format | Example | +|------|--------|---------| +| `rpm` | `key, rpm, name, reponame` | `wget, rpm, wget, appstream` | +| `rpm_repo` | `key, rpm_repo, name, reponame` | `doca_ofed, rpm_repo, doca-ofed, doca` | +| `tarball` | `key, tarball, name, url` | `papi, tarball, papi, https://...` | +| `image` | `key, image, image_path, registry, tag` | `curl, image, docker.io/curl, docker.io, 8.17.0` | + +### Trailing Overrides + +Any package line can have trailing key=value overrides: +- `arch=aarch64` - Override architecture +- `os=rhel` - Override OS +- `os_version=9.4` - Override OS version + +### For Delete Operations + +The delete input file is simpler - just group headers and package keys: + +```ini +[baseos_group_10.0] +wget +glibc_langpack_en + +[slurm_custom_group] +papi +``` + +## Operations Reference + +### catalog_generate + +Creates a new catalog from an input file. + +```bash +ansible-playbook repo_manager.yml --tags catalog_generate \ + -e "input_file=input/packages.txt" +``` + +**Parameters:** + +| Parameter | Required | Default | Description | +|-----------|----------|---------|-------------| +| `input_file` | Yes | - | Path to input text file | +| `catalog_file` | No | `catalogs/catalog.json` | Output catalog path | +| `catalog_name` | No | `default` | Name for the catalog | +| `force` | No | `false` | Overwrite existing file | +| `validate_after` | No | `true` | Run validation after generate | + +**Behavior:** +- Fails if output file exists (unless `force=true`) +- Creates parent directories automatically + +### catalog_add + +Adds or updates packages in an existing catalog. + +```bash +ansible-playbook repo_manager.yml --tags catalog_add \ + -e "input_file=input/additions.txt" +``` + +**Parameters:** + +| Parameter | Required | Default | Description | +|-----------|----------|---------|-------------| +| `input_file` | Yes | - | Path to input text file | +| `catalog_file` | No | `catalogs/catalog.json` | Source catalog path | +| `output_file` | No | Same as `catalog_file` | Output path (preserves source if different) | +| `validate_after` | No | `true` | Run validation after add | + +**Behavior:** +- **Upsert semantics**: Updates existing packages, adds new ones +- Auto-creates groups that don't exist +- No duplicates in `components[]` arrays + +### catalog_delete + +Removes packages from a catalog. + +```bash +ansible-playbook repo_manager.yml --tags catalog_delete \ + -e "input_file=input/removals.txt" +``` + +**Parameters:** + +| Parameter | Required | Default | Description | +|-----------|----------|---------|-------------| +| `input_file` | Yes | - | Path to delete input file | +| `catalog_file` | No | `catalogs/catalog.json` | Source catalog path | +| `output_file` | No | Same as `catalog_file` | Output path | +| `validate_after` | No | `true` | Run validation after delete | + +**Behavior:** +- Removes packages from specified groups +- Deletes packages entirely when unreferenced by any group +- Removes empty groups automatically +- Skips (with warning) missing groups/packages + +### catalog_validate + +Validates a catalog against schema and business rules. + +```bash +ansible-playbook repo_manager.yml --tags catalog_validate +``` + +**Parameters:** + +| Parameter | Required | Default | Description | +|-----------|----------|---------|-------------| +| `catalog_file` | No | `catalogs/catalog.json` | Catalog to validate | +| `schema_file` | No | `schemas/catalog_schema.json` | JSON schema file | + +**Validation Layers:** + +1. **Structural (JSON Schema)**: Required fields, types, patterns +2. **Referential Integrity**: FunctionalLayer → Groups → Packages +3. **Business Rules**: + - No duplicate components in arrays + - Valid package types + - Type-specific required fields (reponame for rpm, url for tarball, etc.) + - base_os groups must have os and os_version +4. **Warnings**: + - Orphan packages (unreferenced by any group) + - Orphan groups (unreferenced by any functional layer) + +## Catalog JSON Structure + +```json +{ + "catalog": { + "name": "default", + "version": "1.0", + "identifier": "default", + "description": "", + "functionallayer": [ + { + "name": "layer_name", + "components": ["group_key_1", "group_key_2"] + } + ], + "groups": { + "group_key": { + "name": "group_key", + "type": "group", + "description": "Group description", + "components": ["pkg_key_1", "pkg_key_2"] + } + }, + "packages": { + "pkg_key": { + "name": "package_name", + "packagetype": "rpm", + "sources": [ + { + "architecture": "x86_64", + "reponame": "baseos", + "name": "rhel", + "version": ["10.0"] + } + ] + } + } + } +} +``` + +## Default Values + +| Variable | Default | Description | +|----------|---------|-------------| +| `default_arch` | `x86_64` | Default architecture | +| `default_os` | `rhel` | Default OS | +| `default_os_version` | `10.0` | Default OS version | +| `catalog_file` | `catalogs/catalog.json` | Default catalog path | +| `schema_file` | `schemas/catalog_schema.json` | Default schema path | + +## Logging + +All operations write logs to `$OMNIA_DATA_PATH/repo_manager/log/catalog/`. + +## Examples + +### Complete Workflow + +```bash +# 1. Create input file +cat > input/my_catalog.txt <<'EOF' +[defaults] +arch=x86_64, os=rhel, os_version=10.0 + +[baseos_group_10.0 | type=base_os, description=base os packages, os=rhel, os_version=10.0] +systemd, rpm, systemd, baseos +wget, rpm, wget, appstream + +[slurm_group | description=slurm packages] +clustershell, rpm, clustershell, epel +EOF + +# 2. Generate catalog +ansible-playbook repo_manager.yml --tags catalog_generate \ + -e "input_file=input/my_catalog.txt" + +# 3. Add more packages +cat > input/add_packages.txt <<'EOF' +[slurm_group] +geopm, tarball, geopm, https://github.com/geopm/geopm/releases/geopm-3.1.0.tar.gz +EOF + +ansible-playbook repo_manager.yml --tags catalog_add \ + -e "input_file=input/add_packages.txt" + +# 4. Validate +ansible-playbook repo_manager.yml --tags catalog_validate + +# 5. Delete a package +cat > input/remove.txt <<'EOF' +[baseos_group_10.0] +wget +EOF + +ansible-playbook repo_manager.yml --tags catalog_delete \ + -e "input_file=input/remove.txt" +``` + +### Using Custom Paths + +```bash +# Custom output path +ansible-playbook repo_manager.yml --tags catalog_generate \ + -e "input_file=input/packages.txt" \ + -e "catalog_file=catalogs/custom.json" \ + -e "catalog_name=my_custom_catalog" + +# Preserve source catalog +ansible-playbook repo_manager.yml --tags catalog_add \ + -e "input_file=input/additions.txt" \ + -e "catalog_file=catalogs/v1.json" \ + -e "output_file=catalogs/v2.json" +``` diff --git a/src/repo_manager/playbooks/repo_manager.yml b/src/repo_manager/playbooks/repo_manager.yml index fc0f4f608e..ec9ded6172 100644 --- a/src/repo_manager/playbooks/repo_manager.yml +++ b/src/repo_manager/playbooks/repo_manager.yml @@ -49,6 +49,12 @@ # # # Rollback repo_manager (placeholder) # ansible-playbook repo_manager.yml --tags rollback +# +# # Catalog operations +# ansible-playbook repo_manager.yml --tags catalog_generate -e "input_file=input/packages.txt" +# ansible-playbook repo_manager.yml --tags catalog_add -e "input_file=input/additions.txt" +# ansible-playbook repo_manager.yml --tags catalog_delete -e "input_file=input/removals.txt" +# ansible-playbook repo_manager.yml --tags catalog_validate # ============================================================================= # ------------------------------------------------------------------------- @@ -149,3 +155,21 @@ tags: - never - rollback + +# ------------------------------------------------------------------------- +# CATALOG OPERATIONS +# ------------------------------------------------------------------------- +- name: Catalog Operations + hosts: localhost + connection: local + gather_facts: false + vars_files: + - "{{ playbook_dir | dirname }}/vars/default.yml" + roles: + - role: catalog + tags: + - never + - catalog_generate + - catalog_add + - catalog_delete + - catalog_validate diff --git a/src/repo_manager/plugins/module_utils/catalog/__init__.py b/src/repo_manager/plugins/module_utils/catalog/__init__.py new file mode 100644 index 0000000000..fd0adfd89b --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Catalog management module for repo_manager. + +Provides CLI-based operations for generating, modifying, and validating +service catalogs used by the repo_manager domain. +""" diff --git a/src/repo_manager/plugins/module_utils/catalog/catalog_io.py b/src/repo_manager/plugins/module_utils/catalog/catalog_io.py new file mode 100644 index 0000000000..c80878c110 --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/catalog_io.py @@ -0,0 +1,109 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Catalog I/O operations: read and write catalog JSON files. +""" + +import os +import json +import re +import logging + +logger = logging.getLogger(__name__) + + +def slugify(text): + """Convert text to a valid identifier (lowercase, underscores).""" + slug = text.lower() + slug = re.sub(r'[^a-z0-9]+', '_', slug) + slug = slug.strip('_') + return slug or 'catalog' + + +def read_catalog(filepath): + """ + Read a catalog from a JSON file. + + Args: + filepath: Path to the catalog JSON file. + + Returns: + dict: Catalog data with 'catalog' as the root key. + + Raises: + FileNotFoundError: If the file doesn't exist. + json.JSONDecodeError: If the file is not valid JSON. + ValueError: If the catalog structure is invalid. + """ + logger.info("Reading catalog from: %s", filepath) + with open(filepath, 'r', encoding='utf-8') as fh: + data = json.load(fh) + + if 'catalog' not in data: + raise ValueError(f"Invalid catalog file: missing 'catalog' key in {filepath}") + + return data + + +def write_catalog(catalog, filepath): + """ + Write a catalog to a JSON file. + + Args: + catalog: Catalog data dict (must have 'catalog' root key). + filepath: Path to write the catalog JSON file. + + Raises: + OSError: If the file cannot be written. + """ + # Ensure parent directory exists + parent_dir = os.path.dirname(filepath) + if parent_dir: + os.makedirs(parent_dir, exist_ok=True) + + logger.info("Writing catalog to: %s", filepath) + with open(filepath, 'w', encoding='utf-8') as fh: + json.dump(catalog, fh, indent=2) + + +def new_catalog(name, groups, packages, description='', version='1.0'): + """ + Create a new catalog structure. + + Args: + name: Catalog name. + groups: Dict of group_key -> group_entry. + packages: Dict of pkg_key -> package_entry. + description: Optional catalog description. + version: Catalog version string. + + Returns: + dict: Complete catalog structure. + """ + return { + "catalog": { + "name": name, + "version": version, + "identifier": slugify(name), + "description": description, + "functionallayer": [], + "groups": groups, + "packages": packages + } + } + + +def catalog_exists(filepath): + """Check if a catalog file exists.""" + return os.path.isfile(filepath) diff --git a/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py b/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py new file mode 100644 index 0000000000..ded23193e5 --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Catalog Manager CLI - Entry point for catalog operations. + +Usage: + catalog_manager.py generate --input --output [options] + catalog_manager.py add --input --catalog [--output ] [options] + catalog_manager.py delete --input --catalog [--output ] + catalog_manager.py validate --catalog [--schema ] +""" + +import argparse +import logging +import os +import sys + +# Add parent directory to path for module imports +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from catalog.parser import parse_input_file, parse_delete_file +from catalog.catalog_io import read_catalog, write_catalog, new_catalog, catalog_exists +from catalog.mutator import upsert_packages, delete_packages +from catalog.validator import validate_catalog, format_issues + + +def setup_logging(log_dir=None, log_file='catalog_manager.log'): + """Setup logging configuration.""" + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + # Console handler + console = logging.StreamHandler(sys.stdout) + console.setLevel(logging.INFO) + console.setFormatter(logging.Formatter('%(message)s')) + logger.addHandler(console) + + # File handler if log_dir provided + if log_dir: + os.makedirs(log_dir, exist_ok=True) + log_path = os.path.join(log_dir, log_file) + file_handler = logging.FileHandler(log_path) + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(logging.Formatter( + '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s' + )) + logger.addHandler(file_handler) + logging.info("Log file: %s", log_path) + + return logger + + +def cmd_generate(args): + """Generate a new catalog from input file.""" + logger = logging.getLogger(__name__) + + if catalog_exists(args.output) and not args.force: + logger.error("Output file '%s' already exists. Use --force to overwrite.", args.output) + return 1 + + try: + parsed = parse_input_file( + args.input, + default_arch=args.default_arch, + default_os=args.default_os, + default_os_version=args.default_os_version + ) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to parse input file: %s", e) + return 1 + + catalog = new_catalog(args.name, parsed['groups'], parsed['packages']) + write_catalog(catalog, args.output) + + # Optional validation + if args.validate and args.schema: + issues = validate_catalog(catalog, args.schema) + if issues: + print(format_issues(issues)) + errors = [i for i in issues if i['severity'] == 'error'] + if errors: + logger.warning("Catalog generated with validation errors") + + group_count = len(parsed['groups']) + pkg_count = len(parsed['packages']) + print(f"Catalog generated: {group_count} groups, {pkg_count} packages -> {args.output}") + return 0 + + +def cmd_add(args): + """Add packages to existing catalog.""" + logger = logging.getLogger(__name__) + + try: + catalog = read_catalog(args.catalog) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to read catalog: %s", e) + return 1 + + try: + parsed = parse_input_file( + args.input, + default_arch=args.default_arch, + default_os=args.default_os, + default_os_version=args.default_os_version + ) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to parse input file: %s", e) + return 1 + + summary = upsert_packages(catalog, parsed) + output_file = args.output or args.catalog + write_catalog(catalog, output_file) + + # Optional validation + if args.validate and args.schema: + issues = validate_catalog(catalog, args.schema) + if issues: + print(format_issues(issues)) + + print(f"Added: {summary['added']}, Updated: {summary['updated']}, " + f"Groups created: {summary['groups_created']} -> {output_file}") + return 0 + + +def cmd_delete(args): + """Delete packages from catalog.""" + logger = logging.getLogger(__name__) + + try: + catalog = read_catalog(args.catalog) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to read catalog: %s", e) + return 1 + + try: + parsed_delete = parse_delete_file(args.input) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to parse delete file: %s", e) + return 1 + + summary = delete_packages(catalog, parsed_delete) + output_file = args.output or args.catalog + write_catalog(catalog, output_file) + + # Optional validation + if args.validate and args.schema: + issues = validate_catalog(catalog, args.schema) + if issues: + print(format_issues(issues)) + + print(f"Deleted: {summary['deleted']}, Groups removed: {summary['groups_removed']}, " + f"Skipped: {summary['skipped']} -> {output_file}") + return 0 + + +def cmd_validate(args): + """Validate a catalog.""" + logger = logging.getLogger(__name__) + + try: + catalog = read_catalog(args.catalog) + except (FileNotFoundError, ValueError) as e: + logger.error("Failed to read catalog: %s", e) + return 1 + + issues = validate_catalog(catalog, args.schema) + print(format_issues(issues)) + + errors = [i for i in issues if i['severity'] == 'error'] + return 1 if errors else 0 + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description='Catalog Manager - Generate, modify, and validate service catalogs', + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('--log-dir', help='Directory for log files') + + subparsers = parser.add_subparsers(dest='command', required=True) + + # Generate command + gen_parser = subparsers.add_parser('generate', help='Generate new catalog from input file') + gen_parser.add_argument('--input', '-i', required=True, help='Input file path') + gen_parser.add_argument('--output', '-o', required=True, help='Output catalog file path') + gen_parser.add_argument('--name', '-n', default='default', help='Catalog name') + gen_parser.add_argument('--force', '-f', action='store_true', help='Overwrite existing file') + gen_parser.add_argument('--default-arch', default='x86_64', help='Default architecture') + gen_parser.add_argument('--default-os', default='rhel', help='Default OS') + gen_parser.add_argument('--default-os-version', default='10.0', help='Default OS version') + gen_parser.add_argument('--schema', help='Schema file for validation') + gen_parser.add_argument('--validate', action='store_true', default=True, + help='Validate after generation') + gen_parser.set_defaults(func=cmd_generate) + + # Add command + add_parser = subparsers.add_parser('add', help='Add packages to existing catalog') + add_parser.add_argument('--input', '-i', required=True, help='Input file with packages to add') + add_parser.add_argument('--catalog', '-c', required=True, help='Existing catalog file') + add_parser.add_argument('--output', '-o', help='Output file (default: overwrite catalog)') + add_parser.add_argument('--default-arch', default='x86_64', help='Default architecture') + add_parser.add_argument('--default-os', default='rhel', help='Default OS') + add_parser.add_argument('--default-os-version', default='10.0', help='Default OS version') + add_parser.add_argument('--schema', help='Schema file for validation') + add_parser.add_argument('--validate', action='store_true', default=True, + help='Validate after add') + add_parser.set_defaults(func=cmd_add) + + # Delete command + del_parser = subparsers.add_parser('delete', help='Delete packages from catalog') + del_parser.add_argument('--input', '-i', required=True, help='Input file with packages to delete') + del_parser.add_argument('--catalog', '-c', required=True, help='Existing catalog file') + del_parser.add_argument('--output', '-o', help='Output file (default: overwrite catalog)') + del_parser.add_argument('--schema', help='Schema file for validation') + del_parser.add_argument('--validate', action='store_true', default=True, + help='Validate after delete') + del_parser.set_defaults(func=cmd_delete) + + # Validate command + val_parser = subparsers.add_parser('validate', help='Validate a catalog') + val_parser.add_argument('--catalog', '-c', required=True, help='Catalog file to validate') + val_parser.add_argument('--schema', '-s', help='JSON schema file') + val_parser.set_defaults(func=cmd_validate) + + args = parser.parse_args() + setup_logging(args.log_dir) + + return args.func(args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/repo_manager/plugins/module_utils/catalog/mutator.py b/src/repo_manager/plugins/module_utils/catalog/mutator.py new file mode 100644 index 0000000000..38cf7e0c4b --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/mutator.py @@ -0,0 +1,142 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Catalog mutation operations: add (upsert) and delete packages/groups. +""" + +import logging + +logger = logging.getLogger(__name__) + + +def upsert_packages(catalog, parsed): + """ + Add or update packages in a catalog (upsert semantics). + + Args: + catalog: Catalog dict (with 'catalog' root key). + parsed: Parsed input dict with 'groups' and 'packages'. + + Returns: + dict: Summary with counts {'added', 'updated', 'groups_created'}. + """ + cat = catalog['catalog'] + groups = cat.setdefault('groups', {}) + packages = cat.setdefault('packages', {}) + + summary = {'added': 0, 'updated': 0, 'groups_created': 0} + + # Process groups + for group_key, group_entry in parsed['groups'].items(): + if group_key not in groups: + # Create new group + groups[group_key] = { + 'name': group_entry['name'], + 'type': group_entry.get('type', 'group'), + 'description': group_entry.get('description', ''), + 'components': [] + } + if group_entry.get('type') == 'base_os': + groups[group_key]['os'] = group_entry.get('os', '') + groups[group_key]['os_version'] = group_entry.get('os_version', '') + summary['groups_created'] += 1 + logger.info("Created new group: %s", group_key) + + # Process packages + for pkg_key, pkg_entry in parsed['packages'].items(): + if pkg_key in packages: + # Update existing package + packages[pkg_key] = pkg_entry + summary['updated'] += 1 + logger.info("Updated package: %s", pkg_key) + else: + # Add new package + packages[pkg_key] = pkg_entry + summary['added'] += 1 + logger.info("Added package: %s", pkg_key) + + # Ensure package keys are in their group's components list + for group_key, group_entry in parsed['groups'].items(): + if group_key in groups: + existing_components = set(groups[group_key].get('components', [])) + for pkg_key in group_entry.get('components', []): + if pkg_key not in existing_components: + groups[group_key]['components'].append(pkg_key) + logger.debug("Added %s to group %s components", pkg_key, group_key) + + return summary + + +def delete_packages(catalog, parsed_delete): + """ + Delete packages from a catalog. + + Args: + catalog: Catalog dict (with 'catalog' root key). + parsed_delete: Dict {group_key: [pkg_key, ...]}. + + Returns: + dict: Summary with counts {'deleted', 'groups_removed', 'skipped'}. + """ + cat = catalog['catalog'] + groups = cat.get('groups', {}) + packages = cat.get('packages', {}) + functional_layers = cat.get('functionallayer', []) + + summary = {'deleted': 0, 'groups_removed': 0, 'skipped': 0} + + for group_key, pkg_keys in parsed_delete.items(): + if group_key not in groups: + logger.warning("Group [%s] not found - skipping", group_key) + summary['skipped'] += len(pkg_keys) + continue + + group = groups[group_key] + components = group.get('components', []) + + for pkg_key in pkg_keys: + if pkg_key not in components: + logger.warning("Package '%s' not in group [%s] - skipping", pkg_key, group_key) + summary['skipped'] += 1 + continue + + # Remove from group components + components.remove(pkg_key) + logger.info("Removed '%s' from group [%s]", pkg_key, group_key) + + # Check if package is still referenced by any group + still_referenced = any( + pkg_key in g.get('components', []) + for g in groups.values() + ) + + if not still_referenced and pkg_key in packages: + del packages[pkg_key] + logger.info("Deleted package '%s' from catalog (no remaining references)", pkg_key) + elif still_referenced: + logger.info("Package '%s' retained (still referenced by other groups)", pkg_key) + + summary['deleted'] += 1 + + # Remove empty groups + if not components: + del groups[group_key] + # Also remove from functional layers + for layer in functional_layers: + if group_key in layer.get('components', []): + layer['components'].remove(group_key) + summary['groups_removed'] += 1 + logger.info("Removed empty group [%s]", group_key) + + return summary diff --git a/src/repo_manager/plugins/module_utils/catalog/parser.py b/src/repo_manager/plugins/module_utils/catalog/parser.py new file mode 100644 index 0000000000..e68d9223d3 --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/parser.py @@ -0,0 +1,264 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=too-many-branches,too-many-locals,too-many-statements +""" +Input file parser for catalog operations. + +Parses the INI-like input format: + [defaults] + arch=x86_64, os=rhel, os_version=10.0 + + [group_key | type=base_os, description=..., os=rhel, os_version=10.0] + pkg_key, rpm, name, reponame + pkg_key, tarball, name, url + pkg_key, image, image_path, registry, tag +""" + +import re +import logging + +logger = logging.getLogger(__name__) + +# Regex patterns +DEFAULTS_HEADER = re.compile(r'^\[defaults\]\s*$', re.IGNORECASE) +GROUP_HEADER = re.compile(r'^\[([^\]|]+)(?:\s*\|\s*([^\]]*))?\]\s*$') +KV_PAIR = re.compile(r'(\w+)\s*=\s*([^,\]]+)') + + +def _parse_kv_pairs(text): + """Parse key=value pairs from a string.""" + return {m.group(1).lower().strip(): m.group(2).strip() for m in KV_PAIR.finditer(text)} + + +def _parse_trailing_overrides(fields, start_index): + """Parse trailing key=value overrides from positional fields.""" + overrides = {} + for field in fields[start_index:]: + if '=' in field: + kv = _parse_kv_pairs(field) + overrides.update(kv) + return overrides + + +def _build_package_entry(pkg_type, fields, defaults, overrides): + """Build a package entry dict based on package type.""" + arch = overrides.get('arch', defaults['arch']) + os_name = overrides.get('os', defaults['os']) + os_version = overrides.get('os_version', defaults['os_version']) + + if pkg_type in ('rpm', 'rpm_repo'): + # fields: [pkg_key, type, name, reponame, ...] + name = fields[2] if len(fields) > 2 else fields[0] + reponame = fields[3] if len(fields) > 3 else '' + return { + "name": name, + "packagetype": pkg_type, + "sources": [{ + "architecture": arch, + "reponame": reponame, + "name": os_name, + "version": [os_version] + }] + } + elif pkg_type == 'tarball': + # fields: [pkg_key, tarball, name, url, ...] + name = fields[2] if len(fields) > 2 else fields[0] + url = fields[3] if len(fields) > 3 else '' + return { + "name": name, + "packagetype": "tarball", + "sources": [{ + "architecture": arch, + "name": os_name, + "version": [os_version], + "url": url + }] + } + elif pkg_type == 'image': + # fields: [pkg_key, image, image_path, registry, tag, ...] + image_path = fields[2] if len(fields) > 2 else fields[0] + registry = fields[3] if len(fields) > 3 else '' + tag = fields[4] if len(fields) > 4 else 'latest' + return { + "name": image_path, + "packagetype": "image", + "tag": tag, + "sources": [{ + "architecture": arch, + "registry": registry + }] + } + else: + raise ValueError(f"Unknown package type: {pkg_type}") + + +def parse_input_file(filepath, default_arch='x86_64', default_os='rhel', default_os_version='10.0'): + """ + Parse an input file for catalog generate/add operations. + + Args: + filepath: Path to the input file. + default_arch: Default architecture if not specified. + default_os: Default OS if not specified. + default_os_version: Default OS version if not specified. + + Returns: + dict: {'groups': {...}, 'packages': {...}} + + Raises: + ValueError: On parse errors (duplicate groups, package before group, etc.) + FileNotFoundError: If input file doesn't exist. + """ + defaults = { + 'arch': default_arch, + 'os': default_os, + 'os_version': default_os_version + } + groups = {} + packages = {} + current_group = None + line_num = 0 + + with open(filepath, 'r', encoding='utf-8') as fh: + for line in fh: + line_num += 1 + line = line.strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + # Check for [defaults] header + if DEFAULTS_HEADER.match(line): + current_group = '__defaults__' # Mark that we're in defaults section + continue + + # Check for group header + group_match = GROUP_HEADER.match(line) + if group_match: + group_key = group_match.group(1).strip() + metadata_str = group_match.group(2) or '' + + if group_key in groups: + raise ValueError(f"Line {line_num}: Duplicate group '{group_key}'") + + # Parse group metadata + meta = _parse_kv_pairs(metadata_str) + group_type = meta.get('type', 'group') + group_desc = meta.get('description', '') + + group_entry = { + "name": group_key, + "type": group_type, + "description": group_desc, + "components": [] + } + if group_type == 'base_os': + group_entry['os'] = meta.get('os', defaults['os']) + group_entry['os_version'] = meta.get('os_version', defaults['os_version']) + + groups[group_key] = group_entry + current_group = group_key + continue + + # If we're in the defaults section, parse key=value pairs + if current_group == '__defaults__': + # Line like: arch=x86_64, os=rhel, os_version=10.0 + kv = _parse_kv_pairs(line) + if 'arch' in kv: + defaults['arch'] = kv['arch'] + if 'os' in kv: + defaults['os'] = kv['os'] + if 'os_version' in kv: + defaults['os_version'] = kv['os_version'] + continue + + # Package line + if current_group is None: + raise ValueError(f"Line {line_num}: Package line before any group header") + + # Split by comma, strip each field + fields = [f.strip() for f in line.split(',')] + if len(fields) < 2: + raise ValueError(f"Line {line_num}: Package line needs at least key and type") + + pkg_key = fields[0] + pkg_type = fields[1].lower() + + if pkg_type not in ('rpm', 'rpm_repo', 'tarball', 'image'): + raise ValueError(f"Line {line_num}: Unknown package type '{pkg_type}'") + + # Determine where trailing overrides start + override_start = 4 # Default for rpm/rpm_repo/tarball + if pkg_type == 'image': + override_start = 5 + + overrides = _parse_trailing_overrides(fields, override_start) + pkg_entry = _build_package_entry(pkg_type, fields, defaults, overrides) + packages[pkg_key] = pkg_entry + groups[current_group]['components'].append(pkg_key) + + logger.info("Parsed input file: %d groups, %d packages", len(groups), len(packages)) + return {'groups': groups, 'packages': packages} + + +def parse_delete_file(filepath): + """ + Parse a delete input file (simplified format). + + Format: + [group_key] + pkg_key1 + pkg_key2 + + Args: + filepath: Path to the delete input file. + + Returns: + dict: {group_key: [pkg_key, pkg_key, ...], ...} + + Raises: + FileNotFoundError: If input file doesn't exist. + """ + result = {} + current_group = None + line_num = 0 + + with open(filepath, 'r', encoding='utf-8') as fh: + for line in fh: + line_num += 1 + line = line.strip() + + if not line or line.startswith('#'): + continue + + # Group header: [group_key] + group_match = GROUP_HEADER.match(line) + if group_match: + current_group = group_match.group(1).strip() + if current_group not in result: + result[current_group] = [] + continue + + # Package key line + if current_group is None: + raise ValueError(f"Line {line_num}: Package key before any group header") + + # The line is just a package key + pkg_key = line.split(',')[0].strip() # Handle trailing commas gracefully + if pkg_key: + result[current_group].append(pkg_key) + + logger.info("Parsed delete file: %d groups", len(result)) + return result diff --git a/src/repo_manager/plugins/module_utils/catalog/validator.py b/src/repo_manager/plugins/module_utils/catalog/validator.py new file mode 100644 index 0000000000..19175b72df --- /dev/null +++ b/src/repo_manager/plugins/module_utils/catalog/validator.py @@ -0,0 +1,256 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=too-many-branches,too-many-locals +""" +Catalog validation: JSON schema validation and business rule checks. +""" + +import json +import logging + +logger = logging.getLogger(__name__) + +VALID_PACKAGE_TYPES = {'rpm', 'tarball', 'image', 'rpm_repo'} + + +def _validate_schema(catalog, schema): + """ + Validate catalog against JSON schema. + + Returns: + list: List of issue dicts {'severity': 'error', 'message': ...} + """ + issues = [] + try: + import jsonschema + jsonschema.validate(instance=catalog, schema=schema) + except ImportError: + logger.warning("jsonschema not installed, skipping schema validation") + issues.append({ + 'severity': 'warning', + 'message': 'jsonschema library not installed, schema validation skipped' + }) + except jsonschema.ValidationError as e: + issues.append({ + 'severity': 'error', + 'message': f"Schema validation failed: {e.message}" + }) + except jsonschema.SchemaError as e: + issues.append({ + 'severity': 'error', + 'message': f"Invalid schema: {e.message}" + }) + return issues + + +def _validate_referential_integrity(catalog): + """ + Check referential integrity between layers, groups, and packages. + + Returns: + list: List of issue dicts. + """ + issues = [] + cat = catalog.get('catalog', {}) + functional_layers = cat.get('functionallayer', []) + groups = cat.get('groups', {}) + packages = cat.get('packages', {}) + + # Check functional layer references + for layer in functional_layers: + layer_name = layer.get('name', '') + for ref in layer.get('components', []): + if ref not in groups: + issues.append({ + 'severity': 'error', + 'message': f"FunctionalLayer '{layer_name}' references missing group '{ref}'" + }) + + # Check group references + for group_key, group in groups.items(): + for ref in group.get('components', []): + if ref not in packages: + issues.append({ + 'severity': 'error', + 'message': f"Group '{group_key}' references missing package '{ref}'" + }) + + return issues + + +def _validate_business_rules(catalog): + """ + Check business rules for packages and groups. + + Returns: + list: List of issue dicts. + """ + issues = [] + cat = catalog.get('catalog', {}) + functional_layers = cat.get('functionallayer', []) + groups = cat.get('groups', {}) + packages = cat.get('packages', {}) + + # Check for duplicate entries in group components + for group_key, group in groups.items(): + components = group.get('components', []) + seen = set() + for comp in components: + if comp in seen: + issues.append({ + 'severity': 'error', + 'message': f"Group '{group_key}' has duplicate component '{comp}'" + }) + seen.add(comp) + + # Check base_os groups have os and os_version + for group_key, group in groups.items(): + if group.get('type') == 'base_os': + if not group.get('os'): + issues.append({ + 'severity': 'error', + 'message': f"base_os group '{group_key}' missing 'os' field" + }) + if not group.get('os_version'): + issues.append({ + 'severity': 'error', + 'message': f"base_os group '{group_key}' missing 'os_version' field" + }) + + # Check packages + for pkg_key, pkg in packages.items(): + pkg_type = pkg.get('packagetype', '') + sources = pkg.get('sources', []) + + # Every package must have at least one source + if not sources: + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' has no sources" + }) + + # Package type must be valid + if pkg_type not in VALID_PACKAGE_TYPES: + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' has invalid packagetype '{pkg_type}'" + }) + + # Type-specific checks + if pkg_type in ('rpm', 'rpm_repo'): + for src in sources: + if not src.get('reponame'): + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' ({pkg_type}) source missing 'reponame'" + }) + + if pkg_type == 'tarball': + for src in sources: + if not src.get('url'): + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' (tarball) source missing 'url'" + }) + + if pkg_type == 'image': + if not pkg.get('tag'): + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' (image) missing 'tag'" + }) + for src in sources: + if not src.get('registry'): + issues.append({ + 'severity': 'error', + 'message': f"Package '{pkg_key}' (image) source missing 'registry'" + }) + + # Check for orphan packages (in packages{} but unreferenced by any group) + all_referenced = set() + for group in groups.values(): + all_referenced.update(group.get('components', [])) + for pkg_key in packages: + if pkg_key not in all_referenced: + issues.append({ + 'severity': 'warning', + 'message': f"Orphan package '{pkg_key}' not referenced by any group" + }) + + # Check for orphan groups (in groups{} but unreferenced by any functional layer) + fl_referenced = set() + for layer in functional_layers: + fl_referenced.update(layer.get('components', [])) + for group_key in groups: + if group_key not in fl_referenced: + issues.append({ + 'severity': 'warning', + 'message': f"Orphan group '{group_key}' not referenced by any functional layer" + }) + + return issues + + +def validate_catalog(catalog, schema_path=None): + """ + Validate a catalog with all validation layers. + + Args: + catalog: Catalog dict (with 'catalog' root key). + schema_path: Optional path to JSON schema file. + + Returns: + list: List of issue dicts {'severity': 'error'|'warning', 'message': ...} + """ + issues = [] + + # Layer 1: Schema validation + if schema_path: + try: + with open(schema_path, 'r', encoding='utf-8') as fh: + schema = json.load(fh) + issues.extend(_validate_schema(catalog, schema)) + except FileNotFoundError: + issues.append({ + 'severity': 'warning', + 'message': f"Schema file not found: {schema_path}" + }) + except json.JSONDecodeError as e: + issues.append({ + 'severity': 'error', + 'message': f"Invalid schema JSON: {e}" + }) + + # Layer 2: Referential integrity + issues.extend(_validate_referential_integrity(catalog)) + + # Layer 3: Business rules + issues.extend(_validate_business_rules(catalog)) + + return issues + + +def format_issues(issues): + """Format issues for display.""" + lines = [] + errors = [i for i in issues if i['severity'] == 'error'] + warnings = [i for i in issues if i['severity'] == 'warning'] + + for issue in errors: + lines.append(f"[ERROR] {issue['message']}") + for issue in warnings: + lines.append(f"[WARNING] {issue['message']}") + + lines.append(f"\nSummary: {len(errors)} error(s), {len(warnings)} warning(s)") + return '\n'.join(lines) diff --git a/src/repo_manager/roles/catalog/defaults/main.yml b/src/repo_manager/roles/catalog/defaults/main.yml new file mode 100644 index 0000000000..9767cbe72c --- /dev/null +++ b/src/repo_manager/roles/catalog/defaults/main.yml @@ -0,0 +1,35 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG ROLE DEFAULTS +# ============================================================================= + +# ── Paths ── +catalog_scripts_dir: "{{ playbook_dir | dirname }}/plugins/module_utils/catalog" +catalog_file: "{{ playbook_dir | dirname }}/catalogs/catalog.json" +schema_file: "{{ playbook_dir | dirname }}/schemas/catalog_schema.json" +catalog_log_dir: "{{ repo_manager_log_dir | default('/opt/omnia/repo_manager/log') }}/catalog" + +# ── Generate defaults ── +catalog_name: "default" +force: false + +# ── Package defaults (used when [defaults] header absent in input file) ── +default_arch: "x86_64" +default_os: "rhel" +default_os_version: "10.0" + +# ── Post-operation validation ── +validate_after: true diff --git a/src/repo_manager/roles/catalog/meta/main.yml b/src/repo_manager/roles/catalog/meta/main.yml new file mode 100644 index 0000000000..001ad8657a --- /dev/null +++ b/src/repo_manager/roles/catalog/meta/main.yml @@ -0,0 +1,29 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +galaxy_info: + author: Dell Inc. + description: Catalog management operations for repo_manager + license: Apache-2.0 + min_ansible_version: "2.14" + platforms: + - name: EL + versions: + - "10" + galaxy_tags: + - catalog + - repo_manager + - omnia + +dependencies: [] diff --git a/src/repo_manager/roles/catalog/tasks/add.yml b/src/repo_manager/roles/catalog/tasks/add.yml new file mode 100644 index 0000000000..04edaeacd2 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/add.yml @@ -0,0 +1,49 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG ADD OPERATION +# ============================================================================= + +- name: "Assert input_file is provided for add" + ansible.builtin.assert: + that: input_file is defined and input_file | length > 0 + fail_msg: "{{ catalog_input_file_required_msg }}" + +- name: "Ensure log directory exists" + ansible.builtin.file: + path: "{{ catalog_log_dir }}" + state: directory + mode: "0755" + +- name: "Add packages to catalog" + ansible.builtin.command: + cmd: >- + python3 {{ catalog_scripts_dir }}/catalog_manager.py + --log-dir {{ catalog_log_dir }} + add + --input {{ input_file }} + --catalog {{ catalog_file }} + --output {{ output_file | default(catalog_file) }} + --default-arch {{ default_arch }} + --default-os {{ default_os }} + --default-os-version {{ default_os_version }} + {{ '--schema ' + schema_file if validate_after | bool else '' }} + register: add_result + changed_when: add_result.rc == 0 + +- name: "Show add result" + ansible.builtin.debug: + msg: "{{ add_result.stdout_lines }}" + when: add_result is defined and add_result.stdout_lines is defined diff --git a/src/repo_manager/roles/catalog/tasks/delete.yml b/src/repo_manager/roles/catalog/tasks/delete.yml new file mode 100644 index 0000000000..6c7a0fc2d7 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/delete.yml @@ -0,0 +1,46 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG DELETE OPERATION +# ============================================================================= + +- name: "Assert input_file is provided for delete" + ansible.builtin.assert: + that: input_file is defined and input_file | length > 0 + fail_msg: "{{ catalog_input_file_required_msg }}" + +- name: "Ensure log directory exists" + ansible.builtin.file: + path: "{{ catalog_log_dir }}" + state: directory + mode: "0755" + +- name: "Delete packages from catalog" + ansible.builtin.command: + cmd: >- + python3 {{ catalog_scripts_dir }}/catalog_manager.py + --log-dir {{ catalog_log_dir }} + delete + --input {{ input_file }} + --catalog {{ catalog_file }} + --output {{ output_file | default(catalog_file) }} + {{ '--schema ' + schema_file if validate_after | bool else '' }} + register: del_result + changed_when: del_result.rc == 0 + +- name: "Show delete result" + ansible.builtin.debug: + msg: "{{ del_result.stdout_lines }}" + when: del_result is defined and del_result.stdout_lines is defined diff --git a/src/repo_manager/roles/catalog/tasks/generate.yml b/src/repo_manager/roles/catalog/tasks/generate.yml new file mode 100644 index 0000000000..1cb53a71a2 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/generate.yml @@ -0,0 +1,50 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG GENERATE OPERATION +# ============================================================================= + +- name: "Assert input_file is provided for generate" + ansible.builtin.assert: + that: input_file is defined and input_file | length > 0 + fail_msg: "{{ catalog_input_file_required_msg }}" + +- name: "Ensure log directory exists" + ansible.builtin.file: + path: "{{ catalog_log_dir }}" + state: directory + mode: "0755" + +- name: "Generate catalog from input file" + ansible.builtin.command: + cmd: >- + python3 {{ catalog_scripts_dir }}/catalog_manager.py + --log-dir {{ catalog_log_dir }} + generate + --input {{ input_file }} + --output {{ catalog_file }} + --name {{ catalog_name }} + --default-arch {{ default_arch }} + --default-os {{ default_os }} + --default-os-version {{ default_os_version }} + {{ '--force' if force | bool else '' }} + {{ '--schema ' + schema_file if validate_after | bool else '' }} + register: gen_result + changed_when: gen_result.rc == 0 + +- name: "Show generate result" + ansible.builtin.debug: + msg: "{{ gen_result.stdout_lines }}" + when: gen_result is defined and gen_result.stdout_lines is defined diff --git a/src/repo_manager/roles/catalog/tasks/main.yml b/src/repo_manager/roles/catalog/tasks/main.yml new file mode 100644 index 0000000000..836acfb959 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/main.yml @@ -0,0 +1,36 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG ROLE - MAIN ENTRY POINT +# ============================================================================= +# Routes to specific operation based on tags. +# Each operation is in a separate file for isolation. +# ============================================================================= + +- name: Include catalog generate tasks + ansible.builtin.include_tasks: generate.yml + when: "'catalog_generate' in ansible_run_tags" + +- name: Include catalog add tasks + ansible.builtin.include_tasks: add.yml + when: "'catalog_add' in ansible_run_tags" + +- name: Include catalog delete tasks + ansible.builtin.include_tasks: delete.yml + when: "'catalog_delete' in ansible_run_tags" + +- name: Include catalog validate tasks + ansible.builtin.include_tasks: validate.yml + when: "'catalog_validate' in ansible_run_tags" diff --git a/src/repo_manager/roles/catalog/tasks/validate.yml b/src/repo_manager/roles/catalog/tasks/validate.yml new file mode 100644 index 0000000000..028651eda4 --- /dev/null +++ b/src/repo_manager/roles/catalog/tasks/validate.yml @@ -0,0 +1,45 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG VALIDATE OPERATION +# ============================================================================= + +- name: "Ensure log directory exists" + ansible.builtin.file: + path: "{{ catalog_log_dir }}" + state: directory + mode: "0755" + +- name: "Validate catalog" + ansible.builtin.command: + cmd: >- + python3 {{ catalog_scripts_dir }}/catalog_manager.py + --log-dir {{ catalog_log_dir }} + validate + --catalog {{ catalog_file }} + --schema {{ schema_file }} + register: val_result + changed_when: false + failed_when: false + +- name: "Show validation result" + ansible.builtin.debug: + msg: "{{ val_result.stdout_lines }}" + when: val_result is defined and val_result.stdout_lines is defined + +- name: "Fail on validation errors" + ansible.builtin.fail: + msg: "{{ catalog_validation_failed_msg }}" + when: val_result is defined and val_result.rc == 1 diff --git a/src/repo_manager/roles/catalog/vars/main.yml b/src/repo_manager/roles/catalog/vars/main.yml new file mode 100644 index 0000000000..8b6d4d92d5 --- /dev/null +++ b/src/repo_manager/roles/catalog/vars/main.yml @@ -0,0 +1,49 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# CATALOG ROLE MESSAGES +# ============================================================================= + +# ── Error messages ── +catalog_input_file_required_msg: >- + Required: -e "input_file=path/to/packages.txt" + +catalog_generate_exists_msg: >- + Output file '{{ catalog_file }}' already exists. + Use -e "force=true" to overwrite. + +catalog_file_not_found_msg: >- + Catalog file '{{ catalog_file }}' not found. + Generate a catalog first with --tags catalog_generate. + +catalog_parse_error_msg: >- + Failed to parse input file. Check the format and try again. + See docs/catalog_operations.md for format details. + +catalog_validation_failed_msg: >- + Catalog validation failed. See errors above. + +# ── Success messages ── +catalog_generate_success_msg: >- + Successfully generated catalog at {{ catalog_file }}. + +catalog_add_success_msg: >- + Successfully added packages to catalog. + +catalog_delete_success_msg: >- + Successfully deleted packages from catalog. + +catalog_validate_success_msg: >- + Catalog validation passed. diff --git a/src/repo_manager/schemas/catalog_schema.json b/src/repo_manager/schemas/catalog_schema.json new file mode 100644 index 0000000000..30ea12cac2 --- /dev/null +++ b/src/repo_manager/schemas/catalog_schema.json @@ -0,0 +1,118 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "HPC Catalog Schema", + "description": "Schema for the omnia services catalog format", + "type": "object", + "required": ["catalog"], + "additionalProperties": false, + "properties": { + "catalog": { + "type": "object", + "required": ["name", "version", "identifier", "description", "functionallayer", "groups", "packages"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "pattern": "^\\d+\\.\\d+(\\.\\d+)?$" }, + "identifier": { "type": "string", "pattern": "^[a-z0-9][a-z0-9_-]*$" }, + "description": { "type": "string" }, + "functionallayer": { + "type": "array", + "items": { "$ref": "#/definitions/functional_layer" } + }, + "groups": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^[a-zA-Z0-9_][a-zA-Z0-9_.+-]*$": { "$ref": "#/definitions/group" } + }, + "additionalProperties": false + }, + "packages": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^[a-zA-Z0-9_][a-zA-Z0-9_-]*$": { "$ref": "#/definitions/package" } + }, + "additionalProperties": false + } + } + } + }, + "definitions": { + "functional_layer": { + "type": "object", + "required": ["name", "components"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "components": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1, + "uniqueItems": true + } + } + }, + "group": { + "type": "object", + "required": ["name", "type", "description", "components"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "type": { "type": "string", "enum": ["base_os", "group"] }, + "os": { "type": "string" }, + "os_version": { "type": "string" }, + "description": { "type": "string" }, + "components": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "uniqueItems": true + } + }, + "if": { + "properties": { "type": { "const": "base_os" } } + }, + "then": { + "required": ["name", "type", "description", "components", "os", "os_version"] + } + }, + "package": { + "type": "object", + "required": ["name", "packagetype", "sources"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "packagetype": { "type": "string", "enum": ["rpm", "tarball", "image", "rpm_repo"] }, + "tag": { "type": "string" }, + "sources": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/source" } + } + }, + "allOf": [ + { + "if": { "properties": { "packagetype": { "const": "image" } } }, + "then": { "required": ["name", "packagetype", "tag", "sources"] } + } + ] + }, + "source": { + "type": "object", + "required": ["architecture"], + "additionalProperties": false, + "properties": { + "architecture": { "type": "string", "enum": ["x86_64", "aarch64", "ppc64le", "noarch"] }, + "reponame": { "type": "string" }, + "registry": { "type": "string" }, + "name": { "type": "string" }, + "version": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "url": { "type": "string", "format": "uri" } + } + } + } +} From 9edbfcc59c1b4626d57b46172217365b5033f126 Mon Sep 17 00:00:00 2001 From: venu <236371043+Venu-p1@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:38:10 +0530 Subject: [PATCH 2/4] feat(repo_manager): Add functional layer support and CATALOG_FILE_PATH integration - Add functional layer parsing to catalog parser - Support type=functional_layer sections in input files - Implement functional layer validation rules: * Catalog must have at least one functional layer * Each functional layer must have components * Each functional layer must have exactly one base_os group * All group references must exist - Update catalog_io to store functional layers in catalog JSON - Enhance mutator to merge functional layers on add/delete operations - Integrate CATALOG_FILE_PATH environment variable: * Prioritize CATALOG_FILE_PATH as default for all catalog operations * Support command-line overrides (-i, -o, -c flags) * Update role defaults and tasks to use CATALOG_FILE_PATH - All validation tests pass successfully Generated with Devin Signed-off-by: venu <236371043+Venu-p1@users.noreply.github.com> --- .../module_utils/catalog/catalog_io.py | 8 +- .../module_utils/catalog/catalog_manager.py | 47 ++++++++-- .../plugins/module_utils/catalog/mutator.py | 28 +++++- .../plugins/module_utils/catalog/parser.py | 93 ++++++++++++------- .../plugins/module_utils/catalog/validator.py | 39 +++++++- .../roles/catalog/defaults/main.yml | 5 +- src/repo_manager/roles/catalog/tasks/add.yml | 9 +- .../roles/catalog/tasks/delete.yml | 9 +- .../roles/catalog/tasks/generate.yml | 6 +- .../roles/catalog/tasks/validate.yml | 6 +- 10 files changed, 193 insertions(+), 57 deletions(-) diff --git a/src/repo_manager/plugins/module_utils/catalog/catalog_io.py b/src/repo_manager/plugins/module_utils/catalog/catalog_io.py index c80878c110..4191546caa 100644 --- a/src/repo_manager/plugins/module_utils/catalog/catalog_io.py +++ b/src/repo_manager/plugins/module_utils/catalog/catalog_io.py @@ -77,7 +77,7 @@ def write_catalog(catalog, filepath): json.dump(catalog, fh, indent=2) -def new_catalog(name, groups, packages, description='', version='1.0'): +def new_catalog(name, groups, packages, functional_layers=None, description='', version='1.0'): """ Create a new catalog structure. @@ -85,19 +85,23 @@ def new_catalog(name, groups, packages, description='', version='1.0'): name: Catalog name. groups: Dict of group_key -> group_entry. packages: Dict of pkg_key -> package_entry. + functional_layers: List of functional layer entries (optional). description: Optional catalog description. version: Catalog version string. Returns: dict: Complete catalog structure. """ + if functional_layers is None: + functional_layers = [] + return { "catalog": { "name": name, "version": version, "identifier": slugify(name), "description": description, - "functionallayer": [], + "functionallayer": functional_layers, "groups": groups, "packages": packages } diff --git a/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py b/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py index ded23193e5..57805425e6 100644 --- a/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py +++ b/src/repo_manager/plugins/module_utils/catalog/catalog_manager.py @@ -81,7 +81,12 @@ def cmd_generate(args): logger.error("Failed to parse input file: %s", e) return 1 - catalog = new_catalog(args.name, parsed['groups'], parsed['packages']) + catalog = new_catalog( + args.name, + parsed['groups'], + parsed['packages'], + functional_layers=parsed.get('functional_layers', []) + ) write_catalog(catalog, args.output) # Optional validation @@ -93,9 +98,10 @@ def cmd_generate(args): if errors: logger.warning("Catalog generated with validation errors") + fl_count = len(parsed.get('functional_layers', [])) group_count = len(parsed['groups']) pkg_count = len(parsed['packages']) - print(f"Catalog generated: {group_count} groups, {pkg_count} packages -> {args.output}") + print(f"Catalog generated: {fl_count} functional layers, {group_count} groups, {pkg_count} packages -> {args.output}") return 0 @@ -185,6 +191,9 @@ def cmd_validate(args): def main(): """Main entry point.""" + # Get CATALOG_FILE_PATH environment variable as default + catalog_file_path = os.environ.get('CATALOG_FILE_PATH', '') + parser = argparse.ArgumentParser( description='Catalog Manager - Generate, modify, and validate service catalogs', formatter_class=argparse.RawDescriptionHelpFormatter @@ -195,8 +204,9 @@ def main(): # Generate command gen_parser = subparsers.add_parser('generate', help='Generate new catalog from input file') - gen_parser.add_argument('--input', '-i', required=True, help='Input file path') - gen_parser.add_argument('--output', '-o', required=True, help='Output catalog file path') + gen_parser.add_argument('--input', '-i', help='Input file path') + gen_parser.add_argument('--output', '-o', default=catalog_file_path, + help=f'Output catalog file path (default: $CATALOG_FILE_PATH={catalog_file_path})') gen_parser.add_argument('--name', '-n', default='default', help='Catalog name') gen_parser.add_argument('--force', '-f', action='store_true', help='Overwrite existing file') gen_parser.add_argument('--default-arch', default='x86_64', help='Default architecture') @@ -209,8 +219,9 @@ def main(): # Add command add_parser = subparsers.add_parser('add', help='Add packages to existing catalog') - add_parser.add_argument('--input', '-i', required=True, help='Input file with packages to add') - add_parser.add_argument('--catalog', '-c', required=True, help='Existing catalog file') + add_parser.add_argument('--input', '-i', help='Input file with packages to add') + add_parser.add_argument('--catalog', '-c', default=catalog_file_path, + help=f'Existing catalog file (default: $CATALOG_FILE_PATH={catalog_file_path})') add_parser.add_argument('--output', '-o', help='Output file (default: overwrite catalog)') add_parser.add_argument('--default-arch', default='x86_64', help='Default architecture') add_parser.add_argument('--default-os', default='rhel', help='Default OS') @@ -222,8 +233,9 @@ def main(): # Delete command del_parser = subparsers.add_parser('delete', help='Delete packages from catalog') - del_parser.add_argument('--input', '-i', required=True, help='Input file with packages to delete') - del_parser.add_argument('--catalog', '-c', required=True, help='Existing catalog file') + del_parser.add_argument('--input', '-i', help='Input file with packages to delete') + del_parser.add_argument('--catalog', '-c', default=catalog_file_path, + help=f'Existing catalog file (default: $CATALOG_FILE_PATH={catalog_file_path})') del_parser.add_argument('--output', '-o', help='Output file (default: overwrite catalog)') del_parser.add_argument('--schema', help='Schema file for validation') del_parser.add_argument('--validate', action='store_true', default=True, @@ -232,11 +244,28 @@ def main(): # Validate command val_parser = subparsers.add_parser('validate', help='Validate a catalog') - val_parser.add_argument('--catalog', '-c', required=True, help='Catalog file to validate') + val_parser.add_argument('--catalog', '-c', default=catalog_file_path, + help=f'Catalog file to validate (default: $CATALOG_FILE_PATH={catalog_file_path})') val_parser.add_argument('--schema', '-s', help='JSON schema file') val_parser.set_defaults(func=cmd_validate) args = parser.parse_args() + + # Validate required arguments + if args.command == 'generate': + if not args.input: + parser.error("generate: --input is required") + if not args.output: + parser.error("generate: --output is required (set CATALOG_FILE_PATH or use -o)") + elif args.command in ('add', 'delete'): + if not args.input: + parser.error(f"{args.command}: --input is required") + if not args.catalog: + parser.error(f"{args.command}: --catalog is required (set CATALOG_FILE_PATH or use -c)") + elif args.command == 'validate': + if not args.catalog: + parser.error("validate: --catalog is required (set CATALOG_FILE_PATH or use -c)") + setup_logging(args.log_dir) return args.func(args) diff --git a/src/repo_manager/plugins/module_utils/catalog/mutator.py b/src/repo_manager/plugins/module_utils/catalog/mutator.py index 38cf7e0c4b..18089a3e73 100644 --- a/src/repo_manager/plugins/module_utils/catalog/mutator.py +++ b/src/repo_manager/plugins/module_utils/catalog/mutator.py @@ -26,16 +26,38 @@ def upsert_packages(catalog, parsed): Args: catalog: Catalog dict (with 'catalog' root key). - parsed: Parsed input dict with 'groups' and 'packages'. + parsed: Parsed input dict with 'functional_layers', 'groups', and 'packages'. Returns: - dict: Summary with counts {'added', 'updated', 'groups_created'}. + dict: Summary with counts {'added', 'updated', 'groups_created', 'fl_created'}. """ cat = catalog['catalog'] + functional_layers = cat.setdefault('functionallayer', []) groups = cat.setdefault('groups', {}) packages = cat.setdefault('packages', {}) - summary = {'added': 0, 'updated': 0, 'groups_created': 0} + summary = {'added': 0, 'updated': 0, 'groups_created': 0, 'fl_created': 0} + + # Process functional layers + input_fl = parsed.get('functional_layers', []) + existing_fl_names = {fl['name'] for fl in functional_layers} + for fl_entry in input_fl: + fl_name = fl_entry.get('name') + if fl_name not in existing_fl_names: + # Add new functional layer + functional_layers.append(fl_entry) + summary['fl_created'] += 1 + logger.info("Created new functional layer: %s", fl_name) + else: + # Merge components into existing functional layer + for existing_fl in functional_layers: + if existing_fl['name'] == fl_name: + existing_comps = set(existing_fl.get('components', [])) + for comp in fl_entry.get('components', []): + if comp not in existing_comps: + existing_fl['components'].append(comp) + logger.debug("Added %s to functional layer %s", comp, fl_name) + break # Process groups for group_key, group_entry in parsed['groups'].items(): diff --git a/src/repo_manager/plugins/module_utils/catalog/parser.py b/src/repo_manager/plugins/module_utils/catalog/parser.py index e68d9223d3..14a37868ae 100644 --- a/src/repo_manager/plugins/module_utils/catalog/parser.py +++ b/src/repo_manager/plugins/module_utils/catalog/parser.py @@ -114,7 +114,7 @@ def parse_input_file(filepath, default_arch='x86_64', default_os='rhel', default default_os_version: Default OS version if not specified. Returns: - dict: {'groups': {...}, 'packages': {...}} + dict: {'functional_layers': [...], 'groups': {...}, 'packages': {...}} Raises: ValueError: On parse errors (duplicate groups, package before group, etc.) @@ -125,9 +125,11 @@ def parse_input_file(filepath, default_arch='x86_64', default_os='rhel', default 'os': default_os, 'os_version': default_os_version } + functional_layers = [] groups = {} packages = {} - current_group = None + current_section = None + current_section_type = None line_num = 0 with open(filepath, 'r', encoding='utf-8') as fh: @@ -141,39 +143,52 @@ def parse_input_file(filepath, default_arch='x86_64', default_os='rhel', default # Check for [defaults] header if DEFAULTS_HEADER.match(line): - current_group = '__defaults__' # Mark that we're in defaults section + current_section = '__defaults__' + current_section_type = 'defaults' continue - # Check for group header - group_match = GROUP_HEADER.match(line) - if group_match: - group_key = group_match.group(1).strip() - metadata_str = group_match.group(2) or '' - - if group_key in groups: - raise ValueError(f"Line {line_num}: Duplicate group '{group_key}'") + # Check for section header (group or functional layer) + section_match = GROUP_HEADER.match(line) + if section_match: + section_key = section_match.group(1).strip() + metadata_str = section_match.group(2) or '' - # Parse group metadata + # Parse section metadata meta = _parse_kv_pairs(metadata_str) - group_type = meta.get('type', 'group') - group_desc = meta.get('description', '') - - group_entry = { - "name": group_key, - "type": group_type, - "description": group_desc, - "components": [] - } - if group_type == 'base_os': - group_entry['os'] = meta.get('os', defaults['os']) - group_entry['os_version'] = meta.get('os_version', defaults['os_version']) - - groups[group_key] = group_entry - current_group = group_key + section_type = meta.get('type', 'group') + section_desc = meta.get('description', '') + + if section_type == 'functional_layer': + # Functional layer: components are group references + fl_entry = { + "name": section_key, + "components": [] + } + functional_layers.append(fl_entry) + current_section = fl_entry + current_section_type = 'functional_layer' + else: + # Regular group or base_os group + if section_key in groups: + raise ValueError(f"Line {line_num}: Duplicate group '{section_key}'") + + group_entry = { + "name": section_key, + "type": section_type, + "description": section_desc, + "components": [] + } + if section_type == 'base_os': + group_entry['os'] = meta.get('os', defaults['os']) + group_entry['os_version'] = meta.get('os_version', defaults['os_version']) + + groups[section_key] = group_entry + current_section = section_key + current_section_type = 'group' continue # If we're in the defaults section, parse key=value pairs - if current_group == '__defaults__': + if current_section_type == 'defaults': # Line like: arch=x86_64, os=rhel, os_version=10.0 kv = _parse_kv_pairs(line) if 'arch' in kv: @@ -184,10 +199,19 @@ def parse_input_file(filepath, default_arch='x86_64', default_os='rhel', default defaults['os_version'] = kv['os_version'] continue - # Package line - if current_group is None: - raise ValueError(f"Line {line_num}: Package line before any group header") + # Content line + if current_section is None: + raise ValueError(f"Line {line_num}: Content before any section header") + + # If we're in a functional layer, the line is a group reference + if current_section_type == 'functional_layer': + # Remove quotes and trailing comma if present + group_ref = line.strip().strip(',').strip('"').strip("'") + if group_ref: + current_section['components'].append(group_ref) + continue + # Otherwise, it's a package line in a group # Split by comma, strip each field fields = [f.strip() for f in line.split(',')] if len(fields) < 2: @@ -207,10 +231,11 @@ def parse_input_file(filepath, default_arch='x86_64', default_os='rhel', default overrides = _parse_trailing_overrides(fields, override_start) pkg_entry = _build_package_entry(pkg_type, fields, defaults, overrides) packages[pkg_key] = pkg_entry - groups[current_group]['components'].append(pkg_key) + groups[current_section]['components'].append(pkg_key) - logger.info("Parsed input file: %d groups, %d packages", len(groups), len(packages)) - return {'groups': groups, 'packages': packages} + logger.info("Parsed input file: %d functional layers, %d groups, %d packages", + len(functional_layers), len(groups), len(packages)) + return {'functional_layers': functional_layers, 'groups': groups, 'packages': packages} def parse_delete_file(filepath): diff --git a/src/repo_manager/plugins/module_utils/catalog/validator.py b/src/repo_manager/plugins/module_utils/catalog/validator.py index 19175b72df..4810589c92 100644 --- a/src/repo_manager/plugins/module_utils/catalog/validator.py +++ b/src/repo_manager/plugins/module_utils/catalog/validator.py @@ -91,7 +91,7 @@ def _validate_referential_integrity(catalog): def _validate_business_rules(catalog): """ - Check business rules for packages and groups. + Check business rules for functional layers, packages, and groups. Returns: list: List of issue dicts. @@ -101,6 +101,43 @@ def _validate_business_rules(catalog): functional_layers = cat.get('functionallayer', []) groups = cat.get('groups', {}) packages = cat.get('packages', {}) + + # Check functional layers are not empty + if not functional_layers: + issues.append({ + 'severity': 'error', + 'message': 'Catalog must have at least one functional layer' + }) + + # Check each functional layer + for layer in functional_layers: + layer_name = layer.get('name', '') + components = layer.get('components', []) + + # Check components are not empty + if not components: + issues.append({ + 'severity': 'error', + 'message': f"Functional layer '{layer_name}' has no components" + }) + + # Check for exactly one base_os group in components + base_os_count = 0 + for comp_ref in components: + comp_group = groups.get(comp_ref, {}) + if comp_group.get('type') == 'base_os': + base_os_count += 1 + + if base_os_count == 0: + issues.append({ + 'severity': 'error', + 'message': f"Functional layer '{layer_name}' must have exactly one base_os group (found 0)" + }) + elif base_os_count > 1: + issues.append({ + 'severity': 'error', + 'message': f"Functional layer '{layer_name}' must have exactly one base_os group (found {base_os_count})" + }) # Check for duplicate entries in group components for group_key, group in groups.items(): diff --git a/src/repo_manager/roles/catalog/defaults/main.yml b/src/repo_manager/roles/catalog/defaults/main.yml index 9767cbe72c..a381d40d5d 100644 --- a/src/repo_manager/roles/catalog/defaults/main.yml +++ b/src/repo_manager/roles/catalog/defaults/main.yml @@ -18,9 +18,10 @@ # ── Paths ── catalog_scripts_dir: "{{ playbook_dir | dirname }}/plugins/module_utils/catalog" -catalog_file: "{{ playbook_dir | dirname }}/catalogs/catalog.json" +# Use CATALOG_FILE_PATH env var if set, otherwise use default location +catalog_file: "{{ lookup('env', 'CATALOG_FILE_PATH') | default('/opt/omnia/catalog/catalog_rhel.json', true) }}" schema_file: "{{ playbook_dir | dirname }}/schemas/catalog_schema.json" -catalog_log_dir: "{{ repo_manager_log_dir | default('/opt/omnia/repo_manager/log') }}/catalog" +catalog_log_dir: "{{ lookup('env', 'OMNIA_DATA_PATH') | default('/opt/omnia', true) }}/repo_manager/log/catalog" # ── Generate defaults ── catalog_name: "default" diff --git a/src/repo_manager/roles/catalog/tasks/add.yml b/src/repo_manager/roles/catalog/tasks/add.yml index 04edaeacd2..8916699097 100644 --- a/src/repo_manager/roles/catalog/tasks/add.yml +++ b/src/repo_manager/roles/catalog/tasks/add.yml @@ -27,6 +27,11 @@ state: directory mode: "0755" +- name: "Set catalog input file (use catalog_input if provided, otherwise catalog_file)" + ansible.builtin.set_fact: + _catalog_input: "{{ catalog_input | default(catalog_file, true) }}" + _output_file: "{{ output_file | default(catalog_file, true) }}" + - name: "Add packages to catalog" ansible.builtin.command: cmd: >- @@ -34,8 +39,8 @@ --log-dir {{ catalog_log_dir }} add --input {{ input_file }} - --catalog {{ catalog_file }} - --output {{ output_file | default(catalog_file) }} + --catalog {{ _catalog_input }} + --output {{ _output_file }} --default-arch {{ default_arch }} --default-os {{ default_os }} --default-os-version {{ default_os_version }} diff --git a/src/repo_manager/roles/catalog/tasks/delete.yml b/src/repo_manager/roles/catalog/tasks/delete.yml index 6c7a0fc2d7..fa1a88c4d4 100644 --- a/src/repo_manager/roles/catalog/tasks/delete.yml +++ b/src/repo_manager/roles/catalog/tasks/delete.yml @@ -27,6 +27,11 @@ state: directory mode: "0755" +- name: "Set catalog input file (use catalog_input if provided, otherwise catalog_file)" + ansible.builtin.set_fact: + _catalog_input: "{{ catalog_input | default(catalog_file, true) }}" + _output_file: "{{ output_file | default(catalog_file, true) }}" + - name: "Delete packages from catalog" ansible.builtin.command: cmd: >- @@ -34,8 +39,8 @@ --log-dir {{ catalog_log_dir }} delete --input {{ input_file }} - --catalog {{ catalog_file }} - --output {{ output_file | default(catalog_file) }} + --catalog {{ _catalog_input }} + --output {{ _output_file }} {{ '--schema ' + schema_file if validate_after | bool else '' }} register: del_result changed_when: del_result.rc == 0 diff --git a/src/repo_manager/roles/catalog/tasks/generate.yml b/src/repo_manager/roles/catalog/tasks/generate.yml index 1cb53a71a2..48eec1f380 100644 --- a/src/repo_manager/roles/catalog/tasks/generate.yml +++ b/src/repo_manager/roles/catalog/tasks/generate.yml @@ -27,6 +27,10 @@ state: directory mode: "0755" +- name: "Set output file (use output_file if provided, otherwise catalog_file)" + ansible.builtin.set_fact: + _output_file: "{{ output_file | default(catalog_file, true) }}" + - name: "Generate catalog from input file" ansible.builtin.command: cmd: >- @@ -34,7 +38,7 @@ --log-dir {{ catalog_log_dir }} generate --input {{ input_file }} - --output {{ catalog_file }} + --output {{ _output_file }} --name {{ catalog_name }} --default-arch {{ default_arch }} --default-os {{ default_os }} diff --git a/src/repo_manager/roles/catalog/tasks/validate.yml b/src/repo_manager/roles/catalog/tasks/validate.yml index 028651eda4..af12fe66ed 100644 --- a/src/repo_manager/roles/catalog/tasks/validate.yml +++ b/src/repo_manager/roles/catalog/tasks/validate.yml @@ -22,13 +22,17 @@ state: directory mode: "0755" +- name: "Set catalog input file (use catalog_input if provided, otherwise catalog_file)" + ansible.builtin.set_fact: + _catalog_input: "{{ catalog_input | default(catalog_file, true) }}" + - name: "Validate catalog" ansible.builtin.command: cmd: >- python3 {{ catalog_scripts_dir }}/catalog_manager.py --log-dir {{ catalog_log_dir }} validate - --catalog {{ catalog_file }} + --catalog {{ _catalog_input }} --schema {{ schema_file }} register: val_result changed_when: false From e7c868094c39f7a839129ebd3e6af2619738399a Mon Sep 17 00:00:00 2001 From: venu <236371043+Venu-p1@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:52:35 +0530 Subject: [PATCH 3/4] Adding a sample input file for catalog_generate Signed-off-by: venu <236371043+Venu-p1@users.noreply.github.com> --- .../samples/catalog_generator_input.txt | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/repo_manager/samples/catalog_generator_input.txt diff --git a/src/repo_manager/samples/catalog_generator_input.txt b/src/repo_manager/samples/catalog_generator_input.txt new file mode 100644 index 0000000000..9fb51041de --- /dev/null +++ b/src/repo_manager/samples/catalog_generator_input.txt @@ -0,0 +1,36 @@ +# Test catalog input file with functional layers +[defaults] +arch=x86_64, os=rhel, os_version=10.0 + +[baseos_group_10.0 | type=base_os, description=base os packages for rhel cluster nodes, os=rhel, os_version=10.0] +systemd, rpm, systemd, baseos +systemd_udev, rpm, systemd-udev, baseos +wget, rpm, wget, appstream +glibc_langpack_en, rpm, glibc-langpack-en, baseos + +[slurm_custom_group | description=slurm custom packages] +clustershell, rpm, clustershell, epel +papi, tarball, papi, https://github.com/icl-utk-edu/papi/releases/download/papi-7-2-0-t/papi-7.2.0.tar.gz +curl_image, image, docker.io/curlimages/curl, docker.io, 8.17.0 + +[openldap_group | description=OpenLDAP packages] +openldap, rpm, openldap, baseos +openldap_clients, rpm, openldap-clients, baseos + +[slurm_control_node_group | description=Slurm controller specific packages] +slurm_slurmctld, rpm, slurm-slurmctld, slurm_custom + +[slurm_node_group | description=Slurm compute node packages] +slurm_slurmd, rpm, slurm-slurmd, slurm_custom + +[slurm_control_node_rhel_10_0_x86_64 | type=functional_layer] +"baseos_group_10.0", +"slurm_custom_group", +"openldap_group", +"slurm_control_node_group" + +[slurm_node_rhel_10_0_x86_64 | type=functional_layer] +"baseos_group_10.0", +"slurm_custom_group", +"openldap_group", +"slurm_node_group" From f6439c1d29cad5f0ce37ab2d7dd116f64286760b Mon Sep 17 00:00:00 2001 From: venu <236371043+Venu-p1@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:57:52 +0530 Subject: [PATCH 4/4] fix(repo_manager): Fix ansible-lint galaxy_tags violation - Change 'repo_manager' tag to 'repomanager' (no underscores allowed) - Complies with ansible-lint meta-no-tags rule - Tags must contain lowercase letters and digits only Signed-off-by: venu <236371043+Venu-p1@users.noreply.github.com> --- src/repo_manager/roles/catalog/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repo_manager/roles/catalog/meta/main.yml b/src/repo_manager/roles/catalog/meta/main.yml index 001ad8657a..5b1217f5e8 100644 --- a/src/repo_manager/roles/catalog/meta/main.yml +++ b/src/repo_manager/roles/catalog/meta/main.yml @@ -23,7 +23,7 @@ galaxy_info: - "10" galaxy_tags: - catalog - - repo_manager + - repomanager - omnia dependencies: []