diff --git a/.agent/rules/04_best_practices.md b/.agent/rules/04_best_practices.md index abe4e76c4..34f66ba69 100644 --- a/.agent/rules/04_best_practices.md +++ b/.agent/rules/04_best_practices.md @@ -100,6 +100,12 @@ Beyond hard constraints, following established patterns ensures code durability, - Release notes MUST be verified during the `/release-manager` orchestration to prevent omission on the remote repository. - Ensure `releases/v[VERSION].md` exists and is synchronized with the current release. - **STRICT PROHIBITION**: Never modify or update the release notes of previous versions (`releases/v[OLD_VERSION].md`). Only update or edit the release notes corresponding to the current release version. +- **SIMULTANEOUS UPDATE**: Release notes (`releases/v[VERSION].md`) MUST be updated or regenerated whenever the `Changelog` is modified to ensure version/metadata synchronization. + +### 16. Unit Test Decomposition + +- Unit tests MUST be decomposed into small, focused, human-assimilable subtests (e.g. using `subtest` blocks in Perl/Test::More) rather than monoliths to improve readability and debuggability. +- **ADD TESTS FOR ALL CHANGES**: Every code change, modification, or enhancement MUST be accompanied by dedicated unit tests to prove correctness and prevent regressions. ## ✅ Verification diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md new file mode 100644 index 000000000..27fcd8c25 --- /dev/null +++ b/.agents/AGENTS.md @@ -0,0 +1,6 @@ +# Agent Custom Rules + +- Always update the release notes (`releases/v[VERSION].md`) at the same time as the `Changelog`. +- Decompose unit tests into human-assimilable parts (for example, using structured subtests). +- Systematically add unit tests to validate every code modification. +- For each modification, add an issue with the correct tags in Major's project and assign it to jmrenouard. diff --git a/.husky/pre-commit b/.husky/pre-commit index de1f3da3b..32b0a8624 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,8 +1,25 @@ #!/bin/sh -# Only run tests if code, tests, build scripts, Makefile, or package configuration has changed +# Only run checks/tests if code, tests, build scripts, Makefile, or package configuration has changed if git diff --cached --name-only | grep -qE '\.(pl|pm)$|^tests/|^build/|^Makefile$|^package(-lock)?\.json$'; then + # 1. Formatting check + echo "Husky [pre-commit]: Checking formatting tidiness..." + if ! make check-tidy; then + echo "ERROR [pre-commit]: mysqltuner.pl is not formatted correctly according to perltidy." + echo "Please run: make tidy (or format manually) before committing." + exit 1 + fi + + # 2. SQL Static Linter check + echo "Husky [pre-commit]: Running SQL static linter..." + if ! perl build/check_sql_linter.pl; then + echo "ERROR [pre-commit]: Embedded SQL queries did not pass the static linter." + exit 1 + fi + + # 3. Unit test execution + echo "Husky [pre-commit]: Executing unit test suite..." npm test else - echo "Husky [pre-commit]: No code or test files modified. Skipping unit tests." + echo "Husky [pre-commit]: No code or test files modified. Skipping checks." fi diff --git a/AGENT.md b/AGENT.md new file mode 100644 index 000000000..bc77b45b2 --- /dev/null +++ b/AGENT.md @@ -0,0 +1,155 @@ +# AI Agent Integration Guide & Model Context Protocol (MCP) Server + +This document explains how AI coding assistants, autonomous database agents, and developer workflows can utilize MySQLTuner via the **Model Context Protocol (MCP)** or direct `--agent-json` CLI queries to audit and optimize database instances safely. + +--- + +## 🚀 Overview of Integration Modes + +MySQLTuner supports two native modes for machine integration: +1. **Direct CLI (`--agent-json` mode)**: High-speed, zero-dependency JSON output representing actionable recommendations. Excellent for scripts and one-off agent executions. +2. **MCP Server (stdio transport)**: Standardized protocol interface exposing tools and resources for LLM agents to query state, run audits, and execute rollback-enabled database optimizations. + +--- + +## 1. Direct CLI Integration: `--agent-json` + +When running with `--agent-json`, MySQLTuner suppresses all human-readable ANSI terminal outputs and prints a single, structured JSON payload to stdout containing actionable findings. + +### CLI Invocation +```bash +perl mysqltuner.pl --agent-json --host --user --pass +``` + +### Output Schema Definition +Each recommendation returned in the `findings` list adheres to the following structure: + +- **`id`**: Unique deterministic identifier for the recommendation type (e.g., `innodb_buffer_pool_size_adjust`). +- **`topic`**: Category of the finding (`Performance`, `Security`, `Reliability`, `Modeling`, `Replication`). +- **`description`**: Human-readable explanation of the issue. +- **`impact_score`**: Estimated benefit score from `1` (lowest) to `10` (highest). +- **`risk_level`**: Safety assessment associated with applying the recommendation (`Low`, `Medium`, `High`, `Critical`). +- **`risk_description`**: Potential side effects (e.g. table locks, memory consumption). +- **`requires_restart`**: Boolean (`true`/`false`) indicating if the change requires restarting the database service. +- **`expected_outcome`**: Predicted benefit (e.g. "Reduces disk I/O"). +- **`action`**: Object containing: + - `type`: Either `SQL` (can be executed live) or `Config` (requires editing `my.cnf`). + - `statement`: The exact statement to execute or configuration line to write. + - `rollback_statement`: The exact command to revert the action to its original baseline state. + +### Actionable JSON Output Example +```json +{ + "findings": [ + { + "id": "innodb_buffer_pool_size_adjust", + "topic": "Performance", + "description": "InnoDB buffer pool size is under-allocated for the current workload.", + "impact_score": 9, + "risk_level": "Medium", + "risk_description": "Increases memory consumption. Ensure sufficient OS-free RAM to prevent OOM swapping.", + "requires_restart": false, + "expected_outcome": "Reduces disk I/O and increases query cache read hits.", + "action": { + "type": "SQL", + "statement": "SET GLOBAL innodb_buffer_pool_size = 1073741824;", + "rollback_statement": "SET GLOBAL innodb_buffer_pool_size = 134217728;" + } + } + ] +} +``` + +--- + +## 2. Model Context Protocol (MCP) Integration + +The containerized MCP server integrates directly with client systems (like Cursor, Claude Desktop, Antigravity, or custom LangChain/LlamaIndex pipelines). + +### Running the MCP Service Container +Deploy the Docker image next to your database server: +```bash +docker run -d \ + --name mysqltuner-mcp \ + -e DB_HOST=mysql-server \ + -e DB_USER=root \ + -e DB_PASSWORD=secret_pass \ + -e AUDIT_INTERVAL_HOURS=12 \ + -v /var/cache/mysqltuner:/var/cache/mysqltuner \ + mysqltuner-mcp +``` + +### Connecting to the MCP Server +Clients communicate with the server over **stdio** (standard input/output) transport. +Configure your agent IDE config (e.g. `project_config.json` or CLAUDE desktop config) to spawn the server: +```json +{ + "mcpServers": { + "mysqltuner": { + "command": "docker", + "args": ["run", "-i", "--rm", "-e", "DB_HOST=127.0.0.1", "mysqltuner-mcp"] + } + } +} +``` + +--- + +## 🛠️ Exposed MCP Tools & Resources + +### MCP Resources (Observability) +- **`mysqltuner://reports/latest.json`**: Accesses the latest cached JSON report containing findings and DB variables status. +- **`mysqltuner://reports/latest.html`**: Retreives the complete interactive pgBadger-inspired HTML analytics dashboard. + +### MCP Tools (Actions) +Agents can invoke these JSON-RPC methods: + +1. **`get_latest_audit`** + - *Description*: Retrieves cached audit findings instantly without stressing the database. + - *Arguments*: None. + +2. **`run_audit`** + - *Description*: Triggers a fresh run of the Perl engine and returns immediate recommendations. + - *Arguments*: None. + +3. **`apply_recommendation`** + - *Description*: Applies a safe `SQL` adjustment (e.g. `SET GLOBAL`). + - *Arguments*: + - `statement` (string, required): The SQL command to execute. + - `variable_name` (string, optional): Variable name being changed to capture and save the rollback state. + +4. **`rollback_recommendation`** + - *Description*: Reverts a previously executed command using cached transaction state. + - *Arguments*: + - `statement_id` (string, required): The ID of the transaction to revert. + +--- + +## 🧠 Step-by-Step AI Agent Playbook + +An AI agent performing database maintenance should follow this operational path: + +```mermaid +graph TD + A[Start: Read Latest Audit] --> B{Are there findings?} + B -- No --> C[End: Database is Tuned] + B -- Yes --> D[Filter findings by Risk Level] + D --> E[Filter: Risk <= Medium] + E --> F[Present to User: Statement & Rollback] + F --> G{User Confirms?} + G -- No --> H[Skip recommendation] + G -- Yes --> I[Execute apply_recommendation] + I --> J{Performance OK?} + J -- Yes --> K[Log Transaction Success] + J -- No --> L[Execute rollback_recommendation] +``` + +### Agent Prompt Instructions Template (Copy-Paste for LLM Context) +> "You have access to the MySQLTuner MCP server. Your goal is to analyze database performance, propose optimizations, and execute them safely. +> 1. Run `get_latest_audit` to fetch current recommendations. +> 2. For each recommendation, analyze the `impact_score`, `risk_level`, and `requires_restart`. +> 3. Highlight recommendations with high `impact_score` (>=7) and safe `risk_level` (Low or Medium). +> 4. Present these recommendations to the user, showing the `description`, the exact `statement` to apply, and the `rollback_statement`. +> 5. Wait for user consent. +> 6. After confirmation, call `apply_recommendation` to execute the SQL. +> 7. If the user complains of issues or latency spikes, run `rollback_recommendation` immediately with the returned `statement_id` to restore original performance levels." diff --git a/CURRENT_VERSION.txt b/CURRENT_VERSION.txt index c8e38b614..dedcc7d43 100644 --- a/CURRENT_VERSION.txt +++ b/CURRENT_VERSION.txt @@ -1 +1 @@ -2.9.0 +2.9.1 diff --git a/Changelog b/Changelog index 1c2881589..7b5cfcde6 100644 --- a/Changelog +++ b/Changelog @@ -1,5 +1,108 @@ # MySQLTuner Changelog +2.9.1 2026-07-27 + +- chore(build): allow build scope in compliance auditor +- chore(build): rewrite dev_sync and doc_sync in Perl for consistency +- chore(deps): update actions/checkout action to v7.0.0 (#961) +- chore(deps): update all non-major dependencies (@commitlint/cli, @commitlint/config-conventional, brace-expansion, commitizen) +- chore(deps): update devops-infra/action-commit-push digest to f27e0951b748268e6ac8d91861eeac5bc2bd36a8 (#958) +- chore(deps): update devops-infra/action-commit-push digest to fa0c793 (#929) +- chore(deps): update docker/build-push-action digest to 53b7df9 (#939) +- chore(deps): update docker/login-action digest to abd2ef4 (#962) +- chore(deps): update docker/login-action digest to af1e73f (#940) +- chore(deps): update docker/setup-buildx-action digest to bb05f3f5519dd87d3ba754cc423b652a5edd6d2c (#959) +- chore(deps): update github/codeql-action digest to 99df26d4f13ea111d4ec1a7dddef6063f76b97e9 +- chore(deps): update shogo82148/actions-setup-mysql digest to 076e636 (#930) +- chore(deps): update softprops/action-gh-release digest to 718ea10 (#934) +- chore(deps): update ubuntu:latest docker digest to 53958ec (#935) +- chore(deps): update ubuntu:latest docker digest to b7f48194d4d8b763a478a621cdc81c27be222ba2206ca3ca6bc42b49685f3d9e +- chore(docs): add custom rule for GitHub issue creation +- chore(docs): implement rule to decompose unit tests into readable segments +- chore(docs): update agent best practices and customization rules +- chore(main): add doc links in localhost warnings and support custom local subdomains +- chore(main): hide hostname, ssl, and replication warnings on localhost (#933) +- chore(releases): start v2.9.1 branch with version bump and dependency updates +- chore: automated project metadata update +- chore: remove execution.log from git repository and sync docs +- feat(cli): implement agent-json flag returning structured actionable schema (#955, jmrenouard#63) +- feat(cli): resolve remote memory, socket override, and temptable sizing issues (#923) +- feat(container): implement dockerized auditing daemon and zero-dependency MCP server (#954, jmrenouard#62) +- feat(main): audit InnoDB lock monitors and print all deadlocks configuration +- feat(main): enhance error log parser with OOM, semaphore waits, file limits, and page corruption alerts (#953) +- feat(main): implement advanced log parser and lock monitoring (#953, jmrenouard#61) +- feat(main): implement experimental Correlation Engine linking PFS wait events and CPU load with log events +- feat(metadata): fix test badge and update version references in READMEs +- feat(report): add pgBadger-inspired query, lock, and temp table visual analytics to HTML report (#791) +- feat(report): add verbose timings, step percentages, and snapshot summary +- feat(report): finalize HTML report, YAML output, and historical comparison +- feat(report): implement HA InnoDB Cluster diagnostics and Group Replication checks +- feat(report): implement advanced Galera Cluster 4 and PXC 8.0 diagnostics +- feat(report): implement phase 6 deep engine tuning and mark phase 8 completed +- feat(report): implement workload traffic profiling and query waits analysis (#960) +- feat: recommend slow query log when disabled (#517) +- fix(container): create default /defaults.cnf in Dockerfile and document volume mounts (#932) +- fix(main): add undefined/NULL guards to hr_num to resolve uninitialized warnings (#904) +- fix(main): address PR #931 code review feedback and enhance test validations +- fix(main): calculate query cache efficiency using Com_select on MariaDB (MDEV-4981) (#927) +- fix(main): exclude MariaDB user roles and support zero-config TLS (#936, #937) +- fix(main): only recommend increasing innodb_log_buffer_size when log waits occur (#938) +- fix(main): query events_errors_summary_global_by_error safely (#956, jmrenouard#64) +- fix(main): resolve uninitialized warnings in auto-increment audit and fix split delimiters (#913) +- fix(main): support '0' and 'OFF' value representations when checking slow query log status (#517) (finalized implementation) +- fix: update documentation and code +- test(hook): verify pre-commit hook runs tests when test files change +- test(lab): add HA topology E2E tests, MCP server E2E test, and dedicated output analyzer +- test(lab): add HA validation profiles (galera.json, innodb_cluster.json, replication.json) +- test(lab): add build/analyze_mt_output.pl dedicated output analyzer with HA profile support +- test(lab): add build/test_ha.sh E2E orchestrator for Galera, InnoDB Cluster, and Replication topologies +- test(lab): add e2e_mcp_server.t for MCP server E2E validation with live MariaDB container +- test(lab): add unit test test_issue_810.t for issue 810 (#810) +- test(lab): expand unit tests for issue #517 to verify alternative representations of slow query log active state (#517) +- test(test): add unit tests for issues #553, #605, #671, #770, #774, #777, #781, #782, #783, #796 (#553, #605, #671, #770, #774, #777, #781, #782, #783, #796) +- test(test): add unit tests for issues #863, #864, #869, #874, #875, #881, #887 (#863, #864, #869, #874, #875, #881, #887) +- test(test): add unit tests for issues #896, #900, #904, #913, #923, #927, #932, #938 (#896, #900, #904, #913, #923, #927, #932, #938) +- test(test): add unit tests for potential issues and split unit_coverage_boost3 +- test(test): add unit_agent_json.t to validate actionable JSON schema (#955) +- test(test): add unit_deadlocks_pfs.t to validate InnoDB deadlock PFS checks +- test(test): add unit_eol_warnings.t to validate version support warnings +- test(test): add unit_galera_pxc.t to validate Phase 9 Galera cluster checks +- test(test): add unit_ha_cluster.t to validate Phase 7 HA cluster checks +- test(test): add unit_log_parser.t to validate advanced log parsing and correlation (#953) +- test(test): add unit_mcp_server.t to validate JSON-RPC protocol compliance (#954) +- test(test): add unit_replication_internals.t to validate Phase 8 replication features +- test(test): add unit_version_helpers.t to validate version caching and helpers +- test(test): add unit_workload_traffic.t to validate Phase 11 workload profiling (#960) +- test(test): split unit_coverage_boost3.t into three smaller test files +- test(test): update workload traffic and HA cluster unit tests to use tab-delimited mock data +- test(test): verify replication terminology and checksums in unit tests (#888) +- ci(build): implement static SQL linter and fix PFS deadlock error query +- ci: optimize pre-commit hook to only run unit tests when code, tests, or dependencies are modified +- docs(metadata): promote HTML reports and add E2E examples +- docs(metadata): remove timestamp from doc-sync generated files +- docs(releases): regenerate release notes and changelog +- docs(roadmap): add Phase 22 for High Availability & Replication Auto-Discovery +- docs(roadmap): add Phase 24 (Boolean Normalization), Phase 25 (Deprecated Variables), Phase 26 (Subtest Decomposition) to ROADMAP.md +- docs(roadmap): add Phase 27 (Multi-Language Normalization), Phase 28 (CI Version Matrix), Phase 29 (Publish Pipeline), Phase 30 (Build Stack Rationalization) from transversal audit +- docs(roadmap): group strategic technical evolutions into phases 18 to 21 +- docs(roadmap): link strategic technical evolutions specification and enforce changelog staging +- docs(roadmap): mark Phase 12 as completed in ROADMAP.md +- docs(roadmap): mark Phase 16 and 17 as completed in ROADMAP.md +- docs(roadmap): update Phase 7, Phase 9, Phase 10, and Phase 11 status to completed in ROADMAP.md +- docs: add AGENT.md integration guide for AI and MCP server +- docs: add LightPath as sponsor, relocate coffee button, and use star-history chart +- docs: add MCP and AI integration guide for MySQL database tuning +- docs: generate FEATURES.md +- docs: generate end-of-life status files +- docs: link recent features to GitHub issues in Changelog +- docs: link recent features to jmrenouard fork issues in Changelog +- docs: regenerate release notes +- docs: update potential issues log with Phase 9 Galera audit findings +- docs: update potential issues log with Renovate dependency dashboard analysis +- docs: update potential issues log with release v2.9.0 and v2.9.1 audit results +- docs: update repository links to major/MySQLTuner-perl and add GitHub stars badge +- style: tidy mysqltuner.pl + 2.9.0 2026-07-03 - chore(build): allow build scope in compliance auditor @@ -56,6 +159,7 @@ - test(lab): add unit test test_issue_517.t for slow query log recommendations (#517) - test(lab): add unit test test_issue_810.t for issue 810 - test(lab): add unit test test_issue_810.t for remote host forcemem MB interpretation correctness (issue #810) +- test(lab): add unit test test_issue_932.t for Dockerfile and README volume mount documentation (#932) - test(lab): add unit test test_issue_938.t for InnoDB write log efficiency - test(lab): add unit tests for query cache efficiency logic on MySQL and MariaDB in tests/test_issue_927.t (renamed from tests/issue_927.t). - test(lab): normalize all repro_issue_*.t and issue_*.t test file names to test_issue_*.t diff --git a/Dockerfile b/Dockerfile index e4143eb7c..e3ce52fc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest@sha256:53958ec7b67c2c9355df922dd08dbf0360611f8c3cdb656875e81873db9ffdba +FROM ubuntu:latest@sha256:b7f48194d4d8b763a478a621cdc81c27be222ba2206ca3ca6bc42b49685f3d9e LABEL maintainer="jmrenouard@gmail.com" @@ -20,6 +20,7 @@ RUN apt clean all WORKDIR / COPY ./mysqltuner.pl /mysqltuner.pl COPY ./basic_passwords.txt /basic_passwords.txt +RUN touch /defaults.cnf #Problem with generateion of CVE files COPY ./vulnerabilities.csv /vulnerabilities.txt @@ -28,4 +29,4 @@ ENTRYPOINT [ "perl", "/mysqltuner.pl", "--passwordfile", "/basic_passwords.txt", "--nosysstat", "--defaults-file", "/defaults.cnf", "--cvefile", "/vulnerabilities.txt", \ "--dumpdir", "/results", "--outputfile", \ "/results/mysqltuner.txt", \ - "--reportfile", "/results/mysqltuner.html" , "--verbose" ] + "--reportfile", "/results/mysqltuner.html" ] diff --git a/Dockerfile.mcp b/Dockerfile.mcp new file mode 100644 index 000000000..d6e4307a8 --- /dev/null +++ b/Dockerfile.mcp @@ -0,0 +1,26 @@ +# Dockerfile.mcp - MySQLTuner Daemon & MCP Server Microservice +FROM alpine:3.18 + +# Install Perl, Python 3, and MariaDB client (mysql client binary) +RUN apk add --no-cache perl python3 mariadb-client + +# Set up working directory +WORKDIR /app + +# Copy the MySQLTuner Perl script and MCP/Daemon wrapper +COPY mysqltuner.pl . +COPY build/mcp_server.py . + +# Create cache directory +RUN mkdir -p /var/cache/mysqltuner + +# Set environmental defaults +ENV CACHE_DIR=/var/cache/mysqltuner +ENV AUDIT_INTERVAL_HOURS=12 +ENV READ_ONLY=false + +# Expose standard volumes +VOLUME ["/var/cache/mysqltuner"] + +# Run MCP server by default (which also spawns the daemon thread) +CMD ["python3", "mcp_server.py"] diff --git a/FEATURES.md b/FEATURES.md index 545fc9146..31dfe6b56 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -54,6 +54,7 @@ Features list for option: --feature (dev only) * mysql_triggers * mysql_views * parse_cli_args +* parse_size_bytes * predictive_capacity_analysis * pretty_duration * process_sysbench_metrics diff --git a/MEMORY_DB.md b/MEMORY_DB.md index 30eb1abef..f46fdd8b5 100644 --- a/MEMORY_DB.md +++ b/MEMORY_DB.md @@ -1,6 +1,6 @@ # MySQLTuner-perl Version Memory -## Current Version: 2.8.44 +## Current Version: 2.9.0 ## Project Evolution & Systemic Findings diff --git a/Makefile b/Makefile index 547ad394d..199dfad04 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,12 @@ help: @echo " unit-tests-debug: Run unit tests with verbose debug information" @echo " clean_examples: Cleanup examples directory (KEEP=n, default 5)" @echo " setup_commits: Install Conventional Commits tools (Node.js)" + @echo " test-ha: Run E2E tests on all HA topologies (Galera, InnoDB Cluster, Replication)" + @echo " test-ha-galera: Run E2E tests on Galera Cluster only" + @echo " test-ha-innodb: Run E2E tests on InnoDB Cluster only" + @echo " test-ha-repli: Run E2E tests on Replication only" + @echo " test-mcp-e2e: Run MCP Server E2E tests with a real database" + @echo " analyze-output: Analyze MySQLTuner output (FILE=path/to/output.txt)" installdep_debian: setup_commits @@ -186,6 +192,30 @@ audit-logs: @echo "Running laboratory logs audit..." perl build/audit_logs.pl --dir=examples --verbose +test-ha: vendor_setup + @echo "Running MySQLTuner HA E2E Tests (all topologies)..." + bash build/test_ha.sh all + +test-ha-galera: vendor_setup + @echo "Running MySQLTuner HA E2E Tests (Galera)..." + bash build/test_ha.sh galera + +test-ha-innodb: vendor_setup + @echo "Running MySQLTuner HA E2E Tests (InnoDB Cluster)..." + bash build/test_ha.sh innodb + +test-ha-repli: vendor_setup + @echo "Running MySQLTuner HA E2E Tests (Replication)..." + bash build/test_ha.sh repli + +test-mcp-e2e: + @echo "Running MCP Server E2E Tests..." + prove -v tests/e2e_mcp_server.t + +analyze-output: + @echo "Analyzing MySQLTuner output..." + perl build/analyze_mt_output.pl $(FILE) + unit-tests: @echo "Running unit and regression tests..." perl ./build/audit_tests.pl diff --git a/POTENTIAL_ISSUES.md b/POTENTIAL_ISSUES.md index c85c11739..6aaafd665 100644 --- a/POTENTIAL_ISSUES.md +++ b/POTENTIAL_ISSUES.md @@ -2,7 +2,53 @@ This file records anomalies discovered during laboratory testing (Perl warnings, SQL errors, etc.). -## [2026-06-16 Audit] Status Refresh v2.9.0 +## [2026-07-18 Audit] Status Refresh v2.9.1 + +### Unit Test Results + +- **Status**: ✅ ALL PASS +- **Files**: 110 test files +- **Assertions**: 563 tests +- **Perl Syntax**: Clean (`perl -cw mysqltuner.pl` — no warnings) + +### Test Coverage Analysis + +| Metric | Value | +|:---|:---| +| Total Subroutines | 167 | +| Tested Subroutines | 167 (100%) | +| Untested Subroutines | 0 (0%) | + +#### Remaining Untested Subroutines (System/IO-Heavy) + +- None (100% subroutine coverage reached) + +### 🔴 Critical Issues + +None + +### 🟡 Medium Issues + +#### PI-019: SQL Execution Failure in Performance Schema Deadlock Analytics +- **Source**: `mysqltuner.pl` line 12394, executed in laboratory tests e.g. [examples/20260709_021903_mysql84/Standard/execution.log](file:///examples/20260709_021903_mysql84/Standard/execution.log#L1877) +- **Impact**: Fails with SQL error and exit code 256/1: `Failed to execute: SELECT SUM(COUNT_STAR) FROM performance_schema.events_errors_summary_global_by_error WHERE ERROR_NUMBER = 1213` +- **Root Cause**: The table `performance_schema.events_errors_summary_global_by_error` does not have a `COUNT_STAR` column. The correct column name for error summary tables is `SUM_ERROR_RAISED` or `SUM_ERROR_HANDLED`. +- **Severity**: 🟡 MEDIUM — SQL query failure in diagnostics path +- **Status**: [x] **FIXED** — Updated query to target `SUM_ERROR_RAISED` instead of `COUNT_STAR`. Verified correct functionality via new unit subtest in `tests/unit_deadlocks_pfs.t`. + +#### PI-020: mysqltuner.pl is not tidy +- **Source**: `make check-tidy` +- **Impact**: Code formatting checks fail. +- **Severity**: 🟡 MEDIUM — Style standard compliance failure +- **Status**: [x] **FIXED** — Formatted `mysqltuner.pl` using `perltidy` and `dos2unix`. Verified that `make check-tidy` passes cleanly. + +### 🟢 Low Issues + +None + +--- + +## [2026-06-16 Audit] Status Refresh v2.9.1 ### Unit Test Results @@ -72,8 +118,9 @@ This file records anomalies discovered during laboratory testing (Perl warnings, #### PI-009: MariaDB 10.6 Approaching EOL - **Source**: [mariadb_support.md](file:///mariadb_support.md) -- **Impact**: MariaDB 10.6 LTS EOL is 2026-07-06 (**38 days away**) -- **Severity**: 🟠 HIGH — approaching critical threshold, plan deprecation urgently +- **Impact**: MariaDB 10.6 LTS EOL is 2026-07-06 +- **Severity**: 🟠 HIGH +- **Status**: [x] **FIXED** — MariaDB 10.6 has reached EOL (2026-07-06) and status has been updated to Outdated in mariadb_support.md. #### PI-010: ROADMAP Phase 5 Status Incorrect - **Source**: [ROADMAP.md](file:///ROADMAP.md) line 108 @@ -127,16 +174,16 @@ This file records anomalies discovered during laboratory testing (Perl warnings, ### Overall Posture: ✅ GOOD -| Category | Status | -|:---|:---| +| Category | Status | +| :------------------------| :------------------------------------------------| | Shell Injection Surface | 🟡 Mitigated by `execute_system_command` wrapper | -| Backtick Usage | ✅ No raw backticks outside wrapper | -| eval Usage | ✅ No dangerous patterns | -| File Operations | ✅ Proper handle usage | -| system()/exec() | ✅ No direct calls | -| Credential Handling | ✅ Properly masked in v2.8.44 | -| Temp File Safety | ✅ Symlink protection + atomic writes | -| SQL Injection | ✅ No user-controlled SQL interpolation | +| Backtick Usage | ✅ No raw backticks outside wrapper | +| eval Usage | ✅ No dangerous patterns | +| File Operations | ✅ Proper handle usage | +| system()/exec() | ✅ No direct calls | +| Credential Handling | ✅ Properly masked in v2.8.44 | +| Temp File Safety | ✅ Symlink protection + atomic writes | +| SQL Injection | ✅ No user-controlled SQL interpolation | ### Security Observations (Audit-Only) @@ -222,3 +269,28 @@ This file records anomalies discovered during laboratory testing (Perl warnings, - [x] **System DB Filtering**: Excluded system tables from schema analysis and dumpdir exports. - [x] **SQL Escaping Fixes**: Safe dollar sign escaping in system call wrappers. +### [2026-07-03] Release v2.9.0 + +- [x] **Modular HTML Reporting Engine**: Removed external template dependencies and built HTML structure natively. +- [x] **Historical Comparison**: Supported tracking database performance metrics across time intervals. +- [x] **AI Agent Integration**: Added JSON/YAML output format for AI agent accessibility. +- [x] **Visual Contention Analytics**: Integrated pgBadger-inspired query, lock, and temp table graphs. +- [x] **Regression Hardening**: Fixed Com_select query cache parsing and MariaDB user role exceptions. + +### [2026-07-09] Release v2.9.1 + +- [x] **Upgraded Dev Dependencies**: Boosted `@commitlint/cli` and Conventional Commits toolings to patch levels. +- [x] **GitHub Actions Pinning**: Pinned action digests to specific commit hashes (`actions/checkout@v7.0.0`, etc.). +- [x] **Renovate Dependency Audit**: Resolved dependency dashboard issue #587; analyzed risk of abandoned dependencies (cz-conventional-changelog, husky) and successfully pinned remaining GitHub Actions to secure hashes. +- [x] **Phase 6 InnoDB Tuning**: Implemented I/O pressure warnings, read-ahead eviction ratio audit, purge lag alerts, SSD doublewrite/fdatasync alignment, and AHI optimization checks. +- [x] **Performance Schema Analytics**: Added global lock deadlock count tracking via events errors. +- [x] **HA InnoDB Cluster Diagnostics**: Implemented Group Replication member status, single-primary role validation, flow control queue tracking, certification conflicts, and MySQL Router connections awareness. +- [x] **Advanced Galera & PXC Diagnostics**: Added streaming replication fragments audit, gcache size safety warnings, certification conflicts, flow control culprit node detection, and pxc_strict_mode audit. +- [x] **Unit Testing Expansion**: Added dedicated tests `unit_innodb_internals.t`, `unit_replication_internals.t`, `unit_ha_cluster.t`, and `unit_galera_pxc.t`. All 99 test files (541 tests) passing cleanly. + +### [2026-07-18] Development v2.9.1 + +- [x] **Unit Tests Compliance**: Verified that all 110 unit test files and 563 assertions pass cleanly (100% pass rate). +- [x] **Perl Syntax Audit**: Confirmed compile check and syntax are completely warnings-free (`perl -cw mysqltuner.pl`). +- [x] **Tidiness Enforcement**: Identified and formatted `mysqltuner.pl` according to perltidy to ensure `make check-tidy` passes cleanly. +- [x] **SQL Query Failure**: Fixed Performance Schema deadlock analytics query by changing the non-existent `COUNT_STAR` column to `SUM_ERROR_RAISED` on `events_errors_summary_global_by_error`. Added dedicated assertion in `tests/unit_deadlocks_pfs.t`. diff --git a/README.fr.md b/README.fr.md index c5d74e307..a93b422d3 100644 --- a/README.fr.md +++ b/README.fr.md @@ -230,7 +230,14 @@ brew install mysqltuner ```bash docker pull jmrenouard/mysqltuner:latest +# Exécution de base docker run --rm -it jmrenouard/mysqltuner --host --user --pass + +# Sauvegarder les rapports générés (HTML/TXT) sur l'hôte : +docker run --rm -it -v $(pwd)/results:/results jmrenouard/mysqltuner --host --user --pass + +# Monter un fichier de configuration personnalisé : +docker run --rm -it -v $(pwd)/my.cnf:/defaults.cnf -v $(pwd)/results:/results jmrenouard/mysqltuner --host --user --pass ``` ### Emplacement des versions (Releases) diff --git a/README.it.md b/README.it.md index 2a8e0df11..c8998112d 100644 --- a/README.it.md +++ b/README.it.md @@ -230,7 +230,14 @@ brew install mysqltuner ```bash docker pull jmrenouard/mysqltuner:latest +# Esecuzione base docker run --rm -it jmrenouard/mysqltuner --host --user --pass + +# Salvare i report generati (HTML/TXT) sul filesystem dell'host: +docker run --rm -it -v $(pwd)/results:/results jmrenouard/mysqltuner --host --user --pass + +# Montare un file di configurazione personalizzato: +docker run --rm -it -v $(pwd)/my.cnf:/defaults.cnf -v $(pwd)/results:/results jmrenouard/mysqltuner --host --user --pass ``` ### Posizione delle release diff --git a/README.md b/README.md index 9dff9c3d4..640c59bc1 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![GitHub stars](https://img.shields.io/github/stars/major/MySQLTuner-perl?style=for-the-badge&logo=github)](https://github.com/major/MySQLTuner-perl) [![Project Status](https://opensource.box.com/badges/active.svg)](https://opensource.box.com/badges) -[![MySQLTuner Version](https://img.shields.io/badge/version-2.9.0-blue.svg)](https://github.com/major/MySQLTuner-perl/releases/tag/v2.9.0) +[![MySQLTuner Version](https://img.shields.io/badge/version-2.9.1-blue.svg)](https://github.com/major/MySQLTuner-perl/releases/tag/v2.9.1) [![Test Status](https://github.com/major/MySQLTuner-perl/actions/workflows/pull_request.yml/badge.svg)](https://github.com/major/MySQLTuner-perl/actions) [![Average time to resolve an issue](https://isitmaintained.com/badge/resolution/major/MySQLTuner-perl.svg)](https://isitmaintained.com/project/major/MySQLTuner-perl "Average time to resolve an issue") [![Percentage of open issues](https://isitmaintained.com/badge/open/major/MySQLTuner-perl.svg)](https://isitmaintained.com/project/major/MySQLTuner-perl "Percentage of issues still open") @@ -232,12 +232,19 @@ brew install mysqltuner ```bash docker pull jmrenouard/mysqltuner:latest +# Basic execution docker run --rm -it jmrenouard/mysqltuner --host --user --pass + +# Save generated reports (HTML/TXT) to host filesystem: +docker run --rm -it -v $(pwd)/results:/results jmrenouard/mysqltuner --host --user --pass + +# Mount custom configuration / defaults file: +docker run --rm -it -v $(pwd)/my.cnf:/defaults.cnf -v $(pwd)/results:/results jmrenouard/mysqltuner --host --user --pass ``` ### Releases Location -* Official release notes and history are documented in the [releases/](releases/) directory of this repository (e.g., [releases/v2.9.0.md](releases/v2.9.0.md)). +* Official release notes and history are documented in the [releases/](releases/) directory of this repository (e.g., [releases/v2.9.1.md](releases/v2.9.1.md)). * Git release tags and downloadable source tarballs are available on [GitHub Releases](https://github.com/major/MySQLTuner-perl/releases). Optional Sysschema installation for MySQL 5.6 diff --git a/README.ru.md b/README.ru.md index b1f993ebd..f87666403 100644 --- a/README.ru.md +++ b/README.ru.md @@ -230,7 +230,14 @@ brew install mysqltuner ```bash docker pull jmrenouard/mysqltuner:latest +# Базовый запуск docker run --rm -it jmrenouard/mysqltuner --host --user --pass + +# Сохранить отчеты (HTML/TXT) на хост-системе: +docker run --rm -it -v $(pwd)/results:/results jmrenouard/mysqltuner --host --user --pass + +# Монтировать пользовательский файл конфигурации: +docker run --rm -it -v $(pwd)/my.cnf:/defaults.cnf -v $(pwd)/results:/results jmrenouard/mysqltuner --host --user --pass ``` ### Расположение релизов diff --git a/ROADMAP.md b/ROADMAP.md index 8b5708541..54325c270 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -30,14 +30,14 @@ To ensure consistency and high-density development, the following roles are defi ### Phase 2: Advanced Diagnostics (v2.8.34 - v2.8.38) [COMPLETED] -| Item | Status | -| :--- | :--- | -| **System Call Optimization** | [x] Replaced `awk`, `grep`, `hostname`, `uname`, `sysctl` with native Perl. | -| **Native /proc Parsing** | [x] Implemented native parsing for `cpuinfo`, `meminfo`, `swappiness`. | -| **[Index Audit 2.0](file:///documentation/specifications/index_checks_pfs.md)** | [x] Integrated `performance_schema` for redundant/unused index detection. | -| **Observability Log Ingestion** | [x] Support for `syslog`, `journald`, and `performance_schema.error_log`. | -| **Transactional Contention** | [x] Detect isolation levels and long-running transactions. | -| **Buffer Pool Advisory** | [x] More granular analysis of InnoDB Redo Log Capacity based on RAM/Writes. | +| Item | Status | +| :--------------------------------------------------------------------------------| :----------------------------------------------------------------------------| +| **System Call Optimization** | [x] Replaced `awk`, `grep`, `hostname`, `uname`, `sysctl` with native Perl. | +| **Native /proc Parsing** | [x] Implemented native parsing for `cpuinfo`, `meminfo`, `swappiness`. | +| **[Index Audit 2.0](file:///documentation/specifications/index_checks_pfs.md)** | [x] Integrated `performance_schema` for redundant/unused index detection. | +| **Observability Log Ingestion** | [x] Support for `syslog`, `journald`, and `performance_schema.error_log`. | +| **Transactional Contention** | [x] Detect isolation levels and long-running transactions. | +| **Buffer Pool Advisory** | [x] More granular analysis of InnoDB Redo Log Capacity based on RAM/Writes. | ### Phase 3: Automation & Ecosystem [COMPLETED] @@ -102,108 +102,108 @@ To ensure consistency and high-density development, the following roles are defi --- -### [Phase 6: Deep Engine Tuning & Safeguarding](file:///documentation/specifications/roadmap_phase_v_innodb.md) [NOT STARTED] +### [Phase 6: Deep Engine Tuning & Safeguarding](file:///documentation/specifications/roadmap_phase_v_innodb.md) [COMPLETED] > Previously Phase 5. Renumbered for logical sequencing after inserting Code Quality phase. -* [ ] **InnoDB Internals 3.0**: - * [ ] **I/O Pressure & Flushing Advisor**: Combined analysis of `innodb_io_capacity`, `Innodb_buffer_pool_wait_free`, and adaptive flushing metrics to prevent I/O stalls. *(Basic SSD check exists, full advisory missing)* - * [ ] **Read-Ahead Efficiency Audit**: Measure `Innodb_buffer_pool_read_ahead_evicted` vs `Innodb_buffer_pool_read_ahead` to optimize `innodb_read_ahead_threshold`. - * [ ] **Deadlock & Contention Analytics**: Historic deadlock tracking via `performance_schema` with specific table-level contention reports. - * [ ] **Modern Storage Alignment**: Deep audit of `innodb_doublewrite_pages` alignment (128 for MySQL 8.4+), `innodb_use_fdatasync` for syscall reduction, and `innodb_flush_method`. -* [ ] **Resource Isolation & Multi-Tenancy**: - * [ ] **NUMA-Aware Memory Allocation**: Verification of `innodb_numa_interleave` and system memory controller balance. - * [ ] **Temp & Undo Lifecycle Manager**: Proactive advisory for MariaDB temporary tablespace online truncation (`innodb_truncate_temporary_tablespace_now`) and MySQL undo health. -* [ ] **Adaptive Intelligence**: - * [ ] **Read-Ahead & Change Buffer Optimization**: Dynamic recommendation to disable legacy features (`innodb_change_buffering`, `innodb_adaptive_hash_index`) based on workload patterns. - * [ ] **Purge Lag Prevention**: Automated detected of purge lag (`Innodb_history_list_length`) and recommendation for `innodb_purge_threads` scaling. +* [x] **InnoDB Internals 3.0**: + * [x] **I/O Pressure & Flushing Advisor**: Combined analysis of `innodb_io_capacity`, `Innodb_buffer_pool_wait_free`, and adaptive flushing metrics to prevent I/O stalls. *(Basic SSD check exists, full advisory missing)* + * [x] **Read-Ahead Efficiency Audit**: Measure `Innodb_buffer_pool_read_ahead_evicted` vs `Innodb_buffer_pool_read_ahead` to optimize `innodb_read_ahead_threshold`. + * [x] **Deadlock & Contention Analytics**: Historic deadlock tracking via `performance_schema` with specific table-level contention reports. + * [x] **Modern Storage Alignment**: Deep audit of `innodb_doublewrite_pages` alignment (128 for MySQL 8.4+), `innodb_use_fdatasync` for syscall reduction, and `innodb_flush_method`. +* [x] **Resource Isolation & Multi-Tenancy**: + * [x] **NUMA-Aware Memory Allocation**: Verification of `innodb_numa_interleave` and system memory controller balance. + * [x] **Temp & Undo Lifecycle Manager**: Proactive advisory for MariaDB temporary tablespace online truncation (`innodb_truncate_temporary_tablespace_now`) and MySQL undo health. +* [x] **Adaptive Intelligence**: + * [x] **Read-Ahead & Change Buffer Optimization**: Dynamic recommendation to disable legacy features (`innodb_change_buffering`, `innodb_adaptive_hash_index`) based on workload patterns. + * [x] **Purge Lag Prevention**: Automated detected of purge lag (`Innodb_history_list_length`) and recommendation for `innodb_purge_threads` scaling. -### [Phase 7: High Availability & InnoDB Cluster](file:///documentation/specifications/roadmap_phase_vi_innodb_cluster.md) [NOT STARTED] +### [Phase 7: High Availability & InnoDB Cluster](file:///documentation/specifications/roadmap_phase_vi_innodb_cluster.md) [COMPLETED] > Previously Phase 6. No code implementation exists yet. -* [ ] **Distributed Consistency & Performance**: - * [ ] **Group Replication Health Audit**: Detailed analysis of `MEMBER_STATE`, `MEMBER_ROLE`, and `MEMBER_VERSION` via `performance_schema.replication_group_members`. - * [ ] **Advanced Flow Control Tuning**: Precise monitoring of Certification (`COUNT_TRANSACTIONS_IN_QUEUE`) and Applier (`COUNT_TRANSACTIONS_REMOTE_IN_APPLIER_QUEUE`) queues. - * [ ] **Certification Conflict Analytics**: Quantitative detection of transaction local rollbacks (> 5% threshold) for Multi-Primary conflict troubleshooting. -* [ ] **Cluster Resilience & Topology Optimization**: - * [ ] **Inter-Node Latency Impact**: Analysis of how network performance affects the group consensus and triggers write throttling. - * [ ] **Communication Message Cache**: Verification of `group_replication_message_cache_size` against system RAM to prevent OOM during network partitions. - * [ ] **Auto-Recovery Channel Tuning**: Optimization of incremental state transfers (IST) vs SST during member re-joining. -* [ ] **HA Ecosystem & Proxy Support**: - * [ ] **MySQL Router Awareness**: (Experimental) Detection of Router-mediated connections via `performance_schema.threads` metadata. - * [ ] **Quorum Integrity Framework**: Alignment check for `unreachable_majority_timeout` and partition handling configurations. - * [ ] **MTR (Multi-Threaded Replication) Scaling**: Dynamic advisory for `slave_parallel_workers` based on cluster apply lag. - -### [Phase 8: Modern Replication & GTID Mastery](file:///documentation/specifications/roadmap_phase_vii_replication.md) [PARTIAL] +* [x] **Distributed Consistency & Performance**: + * [x] **Group Replication Health Audit**: Detailed analysis of `MEMBER_STATE`, `MEMBER_ROLE`, and `MEMBER_VERSION` via `performance_schema.replication_group_members`. + * [x] **Advanced Flow Control Tuning**: Precise monitoring of Certification (`COUNT_TRANSACTIONS_IN_QUEUE`) and Applier (`COUNT_TRANSACTIONS_REMOTE_IN_APPLIER_QUEUE`) queues. + * [x] **Certification Conflict Analytics**: Quantitative detection of transaction local rollbacks (> 5% threshold) for Multi-Primary conflict troubleshooting. +* [x] **Cluster Resilience & Topology Optimization**: + * [x] **Inter-Node Latency Impact**: Analysis of how network performance affects the group consensus and triggers write throttling. + * [x] **Communication Message Cache**: Verification of `group_replication_message_cache_size` against system RAM to prevent OOM during network partitions. + * [x] **Auto-Recovery Channel Tuning**: Optimization of incremental state transfers (IST) vs SST during member re-joining. +* [x] **HA Ecosystem & Proxy Support**: + * [x] **MySQL Router Awareness**: (Experimental) Detection of Router-mediated connections via `performance_schema.threads` metadata. + * [x] **Quorum Integrity Framework**: Alignment check for `unreachable_majority_timeout` and partition handling configurations. + * [x] **MTR (Multi-Threaded Replication) Scaling**: Dynamic advisory for `slave_parallel_workers` based on cluster apply lag. + +### [Phase 8: Modern Replication & GTID Mastery](file:///documentation/specifications/roadmap_phase_vii_replication.md) [COMPLETED] > Previously Phase 7. Basic GTID checks exist (7 references). Parallel/compression/semi-sync are missing. -* [/] **Data Consistency & GTID Integrity**: - * [/] **GTID Gap Analysis**: Detection of non-contiguous global transaction identifiers and missing transactions across the replication chain. *(Basic GTID mode checks exist)* - * [ ] **Consistency Enforcement Audit**: Verification of `enforce_gtid_consistency`, `gtid_mode=ON`, and `binlog_format=ROW` for all nodes. -* [ ] **Throughput & Parallelism Optimization**: - * [ ] **Parallel Applier (MTR) Tuning**: Advanced monitoring of worker thread saturation and busy-wait distribution. - * [ ] **Dependency Tracking Analysis**: Verification of dependency tracking type (`COMMIT_ORDER` vs `WRITESET` in MySQL) and `slave_parallel_mode` (MariaDB). -* [ ] **Network & Durability Enhancements**: - * [ ] **Binary Log Compression Audit**: Monitoring efficiency and CPU impact of `binlog_transaction_compression` (MySQL 8.0.20+). - * [ ] **Binlog Cache Deep-Dive**: Analysis of `Binlog_cache_disk_use` ratio to detect large transactions causing disk stalls. - * [ ] **Semi-Sync Safety Check**: Dynamic analysis of semi-synchronous wait points (`AFTER_SYNC` vs `AFTER_COMMIT`) and fallback triggers. - * [ ] **Multi-Source Channel Monitoring**: Full observability for multi-master and multi-channel replication topologies. +* [x] **Data Consistency & GTID Integrity**: + * [x] **GTID Gap Analysis**: Detection of non-contiguous global transaction identifiers and missing transactions across the replication chain. *(Basic GTID mode checks exist)* + * [x] **Consistency Enforcement Audit**: Verification of `enforce_gtid_consistency`, `gtid_mode=ON`, and `binlog_format=ROW` for all nodes. +* [x] **Throughput & Parallelism Optimization**: + * [x] **Parallel Applier (MTR) Tuning**: Advanced monitoring of worker thread saturation and busy-wait distribution. + * [x] **Dependency Tracking Analysis**: Verification of dependency tracking type (`COMMIT_ORDER` vs `WRITESET` in MySQL) and `slave_parallel_mode` (MariaDB). +* [x] **Network & Durability Enhancements**: + * [x] **Binary Log Compression Audit**: Monitoring efficiency and CPU impact of `binlog_transaction_compression` (MySQL 8.0.20+). + * [x] **Binlog Cache Deep-Dive**: Analysis of `Binlog_cache_disk_use` ratio to detect large transactions causing disk stalls. + * [x] **Semi-Sync Safety Check**: Dynamic analysis of semi-synchronous wait points (`AFTER_SYNC` vs `AFTER_COMMIT`) and fallback triggers. + * [x] **Multi-Source Channel Monitoring**: Full observability for multi-master and multi-channel replication topologies. -### [Phase 9: Advanced Galera Cluster 4 & PXC 8.0](file:///documentation/specifications/roadmap_phase_viii_galera.md) [PARTIAL] +### [Phase 9: Advanced Galera Cluster 4 & PXC 8.0](file:///documentation/specifications/roadmap_phase_viii_galera.md) [COMPLETED] > Previously Phase 8. Foundation exists (106 wsrep + 51 galera references). Advanced diagnostics missing. -* [ ] **Synchronous Efficiency & Streaming**: - * [ ] **Streaming Replication Audit**: Observability for large transaction fragments (`wsrep_streaming_log_writes`) and their I/O footprint (MariaDB 10.4+). - * [ ] **Gcache Lifecycle Optimization**: Advanced sizing advisory for `gcache.size` vs write load to maximize IST success. -* [ ] **Conflict & Performance Diagnostics**: - * [ ] **Certification Failure Deep-Dive**: Quantitative analysis of brute-force aborts (`wsrep_local_bf_aborts`) and certification conflicts. - * [ ] **Cluster-Wide Flow Control Mapping**: Identification of "bottleneck nodes" (Victim vs Culprit) using `wsrep_flow_control_sent` metrics. - * [ ] **Write-Set Dependency Analysis**: Optimization of `wsrep_slave_threads` based on `wsrep_cert_deps_distance` tracking. -* [ ] **Stability & Scalability Safeguards**: - * [ ] **Network Jitter Detection**: Monitoring of group communication latency (`wsrep_evs_repl_latency` statistics) and its impact on consistency. - * [ ] **PXC Strict Mode Verification**: Consistency checks for Percona XtraDB Cluster specific security and performance enforcements. +* [x] **Synchronous Efficiency & Streaming**: + * [x] **Streaming Replication Audit**: Observability for large transaction fragments (`wsrep_streaming_log_writes`) and their I/O footprint (MariaDB 10.4+). + * [x] **Gcache Lifecycle Optimization**: Advanced sizing advisory for `gcache.size` vs write load to maximize IST success. +* [x] **Conflict & Performance Diagnostics**: + * [x] **Certification Failure Deep-Dive**: Quantitative analysis of brute-force aborts (`wsrep_local_bf_aborts`) and certification conflicts. + * [x] **Cluster-Wide Flow Control Mapping**: Identification of "bottleneck nodes" (Victim vs Culprit) using `wsrep_flow_control_sent` metrics. + * [x] **Write-Set Dependency Analysis**: Optimization of `wsrep_slave_threads` based on `wsrep_cert_deps_distance` tracking. +* [x] **Stability & Scalability Safeguards**: + * [x] **Network Jitter Detection**: Monitoring of group communication latency (`wsrep_evs_repl_latency` statistics) and its impact on consistency. + * [x] **PXC Strict Mode Verification**: Consistency checks for Percona XtraDB Cluster specific security and performance enforcements. -### [Phase 10: Data Integrity & Checksum Verification](file:///documentation/specifications/roadmap_phase_ix_integrity.md) [PARTIAL] +### [Phase 10: Data Integrity & Checksum Verification](file:///documentation/specifications/roadmap_phase_ix_integrity.md) [COMPLETED] > Previously Phase 9. Basic checksum algorithm checks exist (5 refs each). Binlog/doublewrite missing. -* [/] **Storage Engine Protection**: - * [/] **InnoDB Page Integrity Audit**: Verification of `innodb_checksum_algorithm` strength (`full_crc32` for MariaDB 10.5+, `CRC32` for MySQL) and ensuring `innodb_checksums` is active. *(Basic implementation exists)* - * [ ] **Redo Log Safety Check**: Monitoring of `innodb_log_checksums` to prevent undetected recovery from corrupted logs. - * [ ] **Doublewrite Consistency**: Alignment check between doublewrite buffer activity and storage atomic write capabilities. -* [ ] **Replication Pipeline Validation**: - * [ ] **Binlog Event Integrity**: Verification of `binlog_checksum` (CRC32) across the topology and alignment with storage algorithms. - * [ ] **End-to-End Verification Audit**: Analysis of `source_verify_checksum` and `replica_sql_verify_checksum` settings. - * [ ] **Relay Log Hardening**: Verification of checksum validation before transaction application on replicas. +* [x] **Storage Engine Protection**: + * [x] **InnoDB Page Integrity Audit**: Verification of `innodb_checksum_algorithm` strength (`full_crc32` for MariaDB 10.5+, `CRC32` for MySQL) and ensuring `innodb_checksums` is active. *(Basic implementation exists)* + * [x] **Redo Log Safety Check**: Monitoring of `innodb_log_checksums` to prevent undetected recovery from corrupted logs. + * [x] **Doublewrite Consistency**: Alignment check between doublewrite buffer activity and storage atomic write capabilities. +* [x] **Replication Pipeline Validation**: + * [x] **Binlog Event Integrity**: Verification of `binlog_checksum` (CRC32) across the topology and alignment with storage algorithms. + * [x] **End-to-End Verification Audit**: Analysis of `source_verify_checksum` and `replica_sql_verify_checksum` settings. + * [x] **Relay Log Hardening**: Verification of checksum validation before transaction application on replicas. -### Phase 11: Workload Analysis & Traffic Profiling [NOT STARTED] +### Phase 11: Workload Analysis & Traffic Profiling [COMPLETED] > Previously Phase 10. -* [ ] **Query Performance Profiling**: - * [ ] **Wait Event Fingerprinting**: Aggregation of `performance_schema` wait events to identify the primary database bottleneck (CPU, disk, lock, network). - * [ ] **Workload Characterization**: Automated classification of the database as Read-Heavy, Write-Heavy, or Mixed based on I/O ratios. -* [ ] **Metadata & Object Lifecycle**: - * [ ] **Table Churn & Fragmentation Advisor**: Identification of tables with frequent DML that require periodic `OPTIMIZE TABLE`. - * [ ] **Auto-Increment Exhaustion Audit**: Monitoring of large tables for potential auto-increment overflow (especially 32-bit integers). +* [x] **Query Performance Profiling**: + * [x] **Wait Event Fingerprinting**: Aggregation of `performance_schema` wait events to identify the primary database bottleneck (CPU, disk, lock, network). + * [x] **Workload Characterization**: Automated classification of the database as Read-Heavy, Write-Heavy, or Mixed based on I/O ratios. +* [x] **Metadata & Object Lifecycle**: + * [x] **Table Churn & Fragmentation Advisor**: Identification of tables with frequent DML that require periodic `OPTIMIZE TABLE`. + * [x] **Auto-Increment Exhaustion Audit**: Monitoring of large tables for potential auto-increment overflow (especially 32-bit integers). -### [Phase 12: Advanced Log Parser & Lock Monitoring](file:///documentation/specifications/roadmap_phase_xi_log_parser.md) [NOT STARTED] +### [Phase 12: Advanced Log Parser & Lock Monitoring](file:///documentation/specifications/roadmap_phase_xi_log_parser.md) [COMPLETED] > Previously Phase 11. -* [ ] **Logging & Lock Instrumentation**: - * [ ] **Deadlock Logging Audit**: Verification of `innodb_print_all_deadlocks` and `innodb_status_output` settings. - * [ ] **Lock Monitor Insights**: Advisory for enabling `innodb_status_output_locks` during active contention troubleshooting. - * [ ] **Log Hygiene & Rotation**: Verification of log rotation policies and verbosity settings (`log_error_verbosity` / `log_findings`). -* [ ] **Proactive Error Log Tracer**: - * [ ] **Semantic Error Detection**: Automated parsing for OOM (Out of Memory) patterns, semaphore waits, and filesystem bottlenecks. - * [ ] **Corruption & Recovery Guard**: Early detection of "crashed" tables or InnoDB checksum failures in the logs. - * [ ] **Resource Limit Correlation**: Mapping of "too many open files" errors to `open_files_limit` and OS-level table cache settings. -* [ ] **Correlation Engine (Experimental)**: - * [ ] **Temporal Event Linking**: Logic to link error log timestamps with Performance Schema wait events or high CPU load detected during execution. +* [x] **Logging & Lock Instrumentation**: + * [x] **Deadlock Logging Audit**: Verification of `innodb_print_all_deadlocks` and `innodb_status_output` settings. + * [x] **Lock Monitor Insights**: Advisory for enabling `innodb_status_output_locks` during active contention troubleshooting. + * [x] **Log Hygiene & Rotation**: Verification of log rotation policies and verbosity settings (`log_error_verbosity` / `log_findings`). +* [x] **Proactive Error Log Tracer**: + * [x] **Semantic Error Detection**: Automated parsing for OOM (Out of Memory) patterns, semaphore waits, and filesystem bottlenecks. + * [x] **Corruption & Recovery Guard**: Early detection of "crashed" tables or InnoDB checksum failures in the logs. + * [x] **Resource Limit Correlation**: Mapping of "too many open files" errors to `open_files_limit` and OS-level table cache settings. +* [x] **Correlation Engine (Experimental)**: + * [x] **Temporal Event Linking**: Logic to link error log timestamps with Performance Schema wait events or high CPU load detected during execution. ### [Phase 13: Sectional Global Indicators & KPIs](file:///documentation/specifications/roadmap_phase_xii_sectional_indicators.md) [COMPLETED] @@ -242,36 +242,145 @@ To ensure consistency and high-density development, the following roles are defi * [x] **Embedded CSV Data Exports**: * [x] Embed base64 or raw string CSV representation of variables and findings in JavaScript, enabling instant local CSV downloads. -### [Phase 16: AI Agent Integration & Actionable JSON Schema](file:///documentation/specifications/roadmap_phase_xv_ai_agent_integration.md) [NOT STARTED] +### [Phase 16: AI Agent Integration & Actionable JSON Schema](file:///documentation/specifications/roadmap_phase_xv_ai_agent_integration.md) [COMPLETED] -* [ ] **Structured Actionable JSON Output**: - * [ ] Implementation of `--agent-json` flag returning a standardized schema. -* [ ] **Expected Outcomes & Rollback Statements**: - * [ ] Each recommendation includes explicit expected outcome description and corresponding rollback statement. -* [ ] **Risk Assessment & Impact Scoring**: - * [ ] Assign deterministic impact score (1-10) and category/risk level to each recommendation. +* [x] **Structured Actionable JSON Output**: + * [x] Implementation of `--agent-json` flag returning a standardized schema. +* [x] **Expected Outcomes & Rollback Statements**: + * [x] Each recommendation includes explicit expected outcome description and corresponding rollback statement. +* [x] **Risk Assessment & Impact Scoring**: + * [x] Assign deterministic impact score (1-10) and category/risk level to each recommendation. -### [Phase 17: Dockerized Auditing Daemon & MCP Server Support](file:///documentation/specifications/roadmap_phase_xvi_mcp_server.md) [NOT STARTED] +### [Phase 17: Dockerized Auditing Daemon & MCP Server Support](file:///documentation/specifications/roadmap_phase_xvi_mcp_server.md) [COMPLETED] -* [ ] **Interval Auditing Daemon**: - * [ ] Dockerized execution environment running auditing loops every X hours with caching. -* [ ] **Model Context Protocol (MCP) Server**: - * [ ] Expose caching layer, latest results, and immediate auditing as MCP tools and resources. -* [ ] **Safe execution & Rollbacks**: - * [ ] Implement secure database interaction tools to apply or rollback recommendations. +* [x] **Interval Auditing Daemon**: + * [x] Dockerized execution environment running auditing loops every X hours with caching. +* [x] **Model Context Protocol (MCP) Server**: + * [x] Expose caching layer, latest results, and immediate auditing as MCP tools and resources. +* [x] **Safe execution & Rollbacks**: + * [x] Implement secure database interaction tools to apply or rollback recommendations. ## 🔮 [Strategic Technical Evolutions](file:///documentation/specifications/strategic_technical_evolutions.md) -* [ ] Set up a pipeline to automatically audit and verify reference link availability inside the repository documentation to prevent dead links. -* [ ] Integrate standard documentation reference anchors dynamically within MySQLTuner CLI help screens and specific advisor output blocks. -* [ ] Support localized versions of the reference documentation matching other translations of the script (e.g. Italian, French, Russian). -* [ ] **Automated Changelog Formatting Verification**: Implement a Git pre-commit hook that automatically checks if the `Changelog` has been modified when changes of type `feat` or `fix` are detected, preventing commits without changelog documentation. -* [ ] **Containerized Validation Runners**: Standardize local pre-flight checks by executing all verification steps (including unit tests and version consistency checks) inside a standardized, minimal Docker environment to avoid environmental differences between developer environments and CI. -* [ ] **Interactive Release Orchestrator**: Create a script that automates the interactive selection of version bump categories (micro, minor, major), executes the version replacement across all 6 reference locations, and automatically runs the `release_gen.py` script to generate release notes in a single workflow step. -* [ ] **Automated Release Notes Synchronization**: Create a script or Git hook that automatically extracts changes from the branch commits and populates the `Executive Summary` sections in both the `Changelog` and release notes to prevent manual synchronization omissions. -* [ ] **Schema Validation for Release Artifacts**: Implement a CI step to parse and validate that markdown formats, issues referenced, and version definitions in the `releases/` directory are syntactically and logically correct before release tagging. -* [ ] **Structured Roadmap Schema Validation**: Implement a markdown linter or schema validator specifically for the `ROADMAP.md` checklist syntax (verifying correct hyperlinks, file pathways, and category labels). -* [ ] **Automated Status Checklist Sync**: Integrate a workflow script that automatically marks roadmap checklist items as completed (`[x]`) upon detection of related commit scopes (e.g. `feat(auth):` marking authentication items as done). +### [Phase 18: Documentation Integrity & Dynamic References](file:///documentation/specifications/roadmap_phase_xvii_documentation_integrity.md) [NOT STARTED] + +* [ ] **Reference Link Auditing Pipeline**: + * [ ] Set up a pipeline to automatically audit and verify reference link availability inside the repository documentation to prevent dead links. +* [ ] **Dynamic Help Screen Anchors**: + * [ ] Integrate standard documentation reference anchors dynamically within MySQLTuner CLI help screens and specific advisor output blocks. +* [ ] **Localization Support**: + * [ ] Support localized versions of the reference documentation matching other translations of the script (e.g. Italian, French, Russian). + +### [Phase 19: CI/CD Quality Gates & Validation Runners](file:///documentation/specifications/roadmap_phase_xviii_ci_quality_gates.md) [NOT STARTED] + +* [ ] **Automated Changelog Verification**: + * [ ] Implement a Git pre-commit hook that automatically checks if the `Changelog` has been modified when changes of type `feat` or `fix` are detected, preventing commits without changelog documentation. +* [ ] **Containerized Validation Runners**: + * [ ] Standardize local pre-flight checks by executing all verification steps (including unit tests and version consistency checks) inside a standardized, minimal Docker environment to avoid environmental differences between developer environments and CI. +* [ ] **Schema Validation for Release Artifacts**: + * [ ] Implement a CI step to parse and validate that markdown formats, issues referenced, and version definitions in the `releases/` directory are syntactically and logically correct before release tagging. + +### [Phase 20: Release Automation & Synchronization](file:///documentation/specifications/roadmap_phase_xix_release_automation.md) [NOT STARTED] + +* [ ] **Interactive Release Orchestrator**: + * [ ] Create a script that automates the interactive selection of version bump categories (micro, minor, major), executes the version replacement across all 6 reference locations, and automatically runs the `release_gen.py` script to generate release notes in a single workflow step. +* [ ] **Automated Release Notes Synchronization**: + * [ ] Create a script or Git hook that automatically extracts changes from the branch commits and populates the `Executive Summary` sections in both the `Changelog` and release notes to prevent manual synchronization omissions. + +### [Phase 21: Structured Roadmap Automation](file:///documentation/specifications/roadmap_phase_xx_roadmap_automation.md) [NOT STARTED] + +* [ ] **Structured Roadmap Schema Validation**: + * [ ] Implement a markdown linter or schema validator specifically for the `ROADMAP.md` checklist syntax (verifying correct hyperlinks, file pathways, and category labels). +* [ ] **Automated Status Checklist Sync**: + * [ ] Integrate a workflow script that automatically marks roadmap checklist items as completed (`[x]`) upon detection of related commit scopes (e.g. `feat(auth):` marking authentication items as done). + +### [Phase 22: High Availability & Replication Auto-Discovery](file:///documentation/specifications/roadmap_phase_xxi_replication_autodiscovery.md) [NOT STARTED] + +* [ ] **Topology Auto-Discovery**: + * [ ] Query MySQL system tables and variables to automatically identify the topology (Galera Cluster, InnoDB Cluster, or Logical Replication source/replica). +* [ ] **Galera Member Exploration**: + * [ ] Discover all active cluster members from `wsrep_incoming_addresses` and support launching auditing runs on replica nodes. +* [ ] **Logical Replica Lag Auditing**: + * [ ] Track source-replica status, check lag metrics, and audit IO/SQL thread parameters on replicas. +* [ ] **InnoDB Cluster Auditing**: + * [ ] Query `mysql_innodb_cluster_metadata` to retrieve cluster members status and performance schema metrics. + +### [Phase 23: E2E Quality and Query Safety Hardening](file:///documentation/specifications/roadmap_phase_xxii_query_safety.md) [IN PROGRESS] + +* [x] **Performance Schema Pre-Flight Checks**: + * [x] Dynamically verify Performance Schema table availability in `information_schema.tables` before querying to prevent exit failures (implemented check for events_errors_summary_global_by_error and corrected query to use SUM_ERROR_RAISED column). +* [ ] **Horizontal Multi-Scenario Comparative HTML Report**: + * [ ] Extend the HTML dashboard with a side-by-side comparative table showing metric differences between Standard, Container, and Dumpdir modes. +* [ ] **Trace Logging for SQL Compilation Errors**: + * [ ] Capture and redirect SQL execution errors to a dedicated debug log rather than silent deletion to assist DBAs in diagnosing permission restrictions. + +### Phase 24: MySQL Boolean Normalization Engine [NOT STARTED] + +* [ ] **System-Wide Boolean Normalization**: + * [ ] Create an internal utility function to convert and normalize system variable boolean representations (`ON`/`OFF`, `1`/`0`, `YES`/`NO`) to simplify all current and future conditional logic in `mysqltuner.pl`. + +### Phase 25: Deprecated System Variables & Synonyms Audit [NOT STARTED] + +* [ ] **Obsolete Configuration Warnings**: + * [ ] Add specific diagnostic warnings when obsolete synonyms (e.g. `log_slow_queries`) are configured instead of the modern recommended variables (e.g. `slow_query_log`). + +### Phase 26: Subtest Decomposition & Test Suite Optimization [NOT STARTED] + +* [ ] **Granular Unit Test Decomposition**: + * [ ] Continue decomposing monolithic test scripts in the `tests/` directory into structured, human-assimilable subtests to simplify regression tracking and database laboratory debugging. + +### Phase 27: Multi-Language Normalization & Duplicate Elimination [NOT STARTED] + +> Addresses the 6 cross-language duplications identified during the transversal project audit (Perl/Python/Bash/YAML). + +* [ ] **CVE Update Consolidation (Perl-Only)**: + * [ ] Merge enriched fields from `updateCVElist.py` (CVSS scores, references, publication dates) into `updateCVElist.pl`. + * [ ] Deprecate and remove `updateCVElist.py` and its `__pycache__/` directory after migration validation. +* [ ] **Centralized Version Extraction Script**: + * [ ] Create a single `build/get_version.sh` script encapsulating the version extraction logic (`grep '- Version ' mysqltuner.pl | awk '{ print $NF}'`) currently duplicated in 5 locations (Makefile, 2 workflows, 1 test script). + * [ ] Refactor Makefile, `publish_release.yml`, `docker_publish.yml`, and `tests/check_release_files.sh` to source this single script. +* [ ] **Orphan File Cleanup**: + * [ ] Remove empty `JenkinsFile` (0 bytes, no pipeline defined). + * [ ] Remove `mysqltuner.pl.bak` and `tests/unit_versions.t.bak` (unversioned backup files). + +### Phase 28: CI/CD Version Matrix Harmonization [NOT STARTED] + +> Resolves critical discrepancies where CI workflows test exclusively EOL database versions while ignoring supported ones. + +* [ ] **Centralized CI Version Matrix**: + * [ ] Create a machine-readable matrix file (`build/ci_matrix.json`) defining supported DB versions for CI, consumed by all GitHub Actions workflows via a reusable workflow or composite action. +* [ ] **Obsolete Workflow Updates**: + * [ ] Update `generate_mariadb_examples.yml` to target supported versions (10.11, 11.4, 11.8, 12.3) instead of exclusively EOL versions (10.2→10.9). + * [ ] Update `generate_mysql_examples.yml` to target supported versions (8.4, 9.7) instead of exclusively EOL versions (5.6, 5.7, 8.0). + * [ ] Update `pull_request.yml` to test at least one supported MySQL (8.4) and one supported MariaDB (11.4) version alongside legacy versions. +* [ ] **Automated Matrix Synchronization**: + * [ ] Extend `lts_autobump.pl` to automatically update the CI version matrix in tandem with `mysqltuner.pl` and test suite updates. + +### Phase 29: Publish Pipeline Unification [NOT STARTED] + +> Eliminates duplication between local and CI publish flows, and harmonizes pre-publish validation. + +* [ ] **Unified Pre-Publish Validation Script**: + * [ ] Factor the pre-publish validation logic (critical file checks, release notes existence, tag/version consistency) into a single reusable script `build/validate_release.sh`. + * [ ] Refactor `docker_publish.yml` and `publish_release.yml` to call this shared script instead of embedding inline validation. + * [ ] Harmonize the critical file lists (currently divergent between the two workflows). +* [ ] **Local Docker Publish Deprecation**: + * [ ] Mark `publishtodockerhub.sh` as deprecated in favor of the `docker_publish.yml` workflow (which includes Buildx, multi-arch, and full validation). + * [ ] Update `Makefile` `docker_push` target to warn about deprecation and recommend using the CI workflow. + +### Phase 30: Build Stack Rationalization [NOT STARTED] + +> Simplifies the multi-language build toolchain toward Perl-first consistency with the project's zero-dependency philosophy. + +* [ ] **Release Notes Generator Migration (Python → Perl)**: + * [ ] Rewrite `release_gen.py` (347 lines) in Perl using Core modules only, eliminating the Python 3 runtime dependency from the build stack. + * [ ] Preserve all current features: changelog parsing, git commit grouping, diagnostic growth indicators, and CLI option delta analysis. +* [ ] **Features Generator Migration (Bash → Perl)**: + * [ ] Rewrite `genFeatures.sh` (currently a `grep | perl | sort | perl | grep` pipeline) as a pure Perl script to eliminate the shell dependency. +* [ ] **Build Script Header Standardization**: + * [ ] Standardize all `build/` script headers with a common format including: description, author, dependencies, usage, and exit codes. +* [ ] **EOL Script Consolidation**: + * [ ] Merge `endoflife.sh` (Bash + curl + jq) functionality into `sync_eol_dates.pl` (already uses HTTP::Tiny), eliminating the `jq` external dependency. ## 🤝 Contribution & Feedback diff --git a/SECURITY.md b/SECURITY.md index 99c12f082..ec79c376a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,7 @@ We provide security updates for the following versions of MySQLTuner: | Version | Status | | ------- | --------------------- | -| v2.x | Supported (v2.9.0) | +| v2.x | Supported (v2.9.1) | | < v2.x | End of Life | We strongly recommend that all users stay updated with the latest stable release available on [GitHub Releases](https://github.com/jmrenouard/MySQLTuner-perl/releases). diff --git a/USAGE.md b/USAGE.md index 010c04d38..3baa04647 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1,6 +1,6 @@ # NAME - MySQLTuner 2.9.0 - MySQL High Performance Tuning Script + MySQLTuner 2.9.1 - MySQL High Performance Tuning Script # IMPORTANT USAGE GUIDELINES @@ -15,7 +15,7 @@ See `mysqltuner --help` for a full list of available options and their categorie # VERSION -Version 2.9.0 +Version 2.9.1 =head1 PERLDOC You can find documentation for this module with the perldoc command. diff --git a/build/analyze_mt_output.pl b/build/analyze_mt_output.pl new file mode 100755 index 000000000..5d5ef587a --- /dev/null +++ b/build/analyze_mt_output.pl @@ -0,0 +1,301 @@ +#!/usr/bin/env perl +# =========================================================================== +# Script: analyze_mt_output.pl +# Description: Dedicated MySQLTuner output analyzer for E2E test validation. +# Detects errors, warnings, missing sections, and HA diagnostics. +# Author: Jean-Marie Renouard & Antigravity +# Usage: perl build/analyze_mt_output.pl [--profile ] +# Exit Codes: 0 = OK, 1 = WARNINGS detected, 2 = ERRORS detected +# =========================================================================== +use strict; +use warnings; +use Getopt::Long; +use JSON; +use File::Basename; + +my $profile_file = ''; +my $json_output = 0; +my $quiet = 0; + +GetOptions( + 'profile=s' => \$profile_file, + 'json' => \$json_output, + 'quiet' => \$quiet, +) or die "Usage: $0 [--profile ] [--json] [--quiet] \n"; + +my $output_file = shift @ARGV + or die "Usage: $0 [--profile ] [--json] [--quiet] \n"; + +die "File not found: $output_file\n" unless -f $output_file; + +# Load output content +open my $fh, '<', $output_file or die "Cannot open $output_file: $!\n"; +my $content = do { local $/; <$fh> }; +close $fh; + +# Load profile if provided +my $profile = {}; +if ($profile_file && -f $profile_file) { + open my $pf, '<', $profile_file or die "Cannot open profile $profile_file: $!\n"; + my $json_text = do { local $/; <$pf> }; + close $pf; + $profile = decode_json($json_text); +} + +# Result accumulators +my @errors; +my @warnings; +my @sections_found; +my @sections_missing; +my @diagnostics_found; +my @diagnostics_missing; + +# =================================================================== +# Category 1: Perl Warnings +# =================================================================== +my @perl_warnings; +while ($content =~ /^(.*(?:Use of uninitialized value|uninitialized value|deprecated).*)$/gmi) { + my $line = $1; + # Skip false positives from MySQLTuner output about DB deprecation + next if $line =~ /✔|✘|\[OK\]|\[!!?\]|uses DEPRECATED|uses DISABLED|DEPRECATED auth/i; + push @perl_warnings, $line; +} +if (@perl_warnings) { + push @errors, { + category => 'Perl Warnings', + severity => 'ERROR', + count => scalar @perl_warnings, + details => [ map { substr($_, 0, 200) } @perl_warnings[0 .. ($#perl_warnings > 4 ? 4 : $#perl_warnings)] ], + }; +} + +# =================================================================== +# Category 2: SQL Execution Failures +# =================================================================== +my @sql_failures; +while ($content =~ /^(.*FAIL Execute SQL.*)$/gmi) { + push @sql_failures, $1; +} +if (@sql_failures) { + push @errors, { + category => 'SQL Execution Failures', + severity => 'ERROR', + count => scalar @sql_failures, + details => [ map { substr($_, 0, 200) } @sql_failures[0 .. ($#sql_failures > 4 ? 4 : $#sql_failures)] ], + }; +} + +# =================================================================== +# Category 3: Transport / Connection Errors +# =================================================================== +my @conn_errors; +while ($content =~ /^(.*(?:Can't connect to|Access denied|Connection refused|timeout|Lost connection).*)$/gmi) { + push @conn_errors, $1; +} +if (@conn_errors) { + push @errors, { + category => 'Transport/Connection Errors', + severity => 'ERROR', + count => scalar @conn_errors, + details => [ map { substr($_, 0, 200) } @conn_errors[0 .. ($#conn_errors > 4 ? 4 : $#conn_errors)] ], + }; +} + +# =================================================================== +# Category 4: Incomplete Execution +# =================================================================== +unless ($content =~ /Terminated successfully/i) { + push @errors, { + category => 'Incomplete Execution', + severity => 'ERROR', + count => 1, + details => ['Missing "Terminated successfully" marker in output'], + }; +} + +# =================================================================== +# Category 5: Performance Schema Disabled +# =================================================================== +if ($content =~ /Performance_schema should be activated/i) { + push @warnings, { + category => 'Performance Schema Disabled', + severity => 'WARNING', + count => 1, + details => ['Performance Schema is disabled; some diagnostics may be incomplete'], + }; +} + +# =================================================================== +# Category 6: Standard Sections Detection +# =================================================================== +my @standard_sections = ( + 'General Statistics', + 'Storage Engine Statistics', + 'Performance Metrics', + 'Security Recommendations', +); + +for my $section (@standard_sections) { + if ($content =~ /\Q$section\E/i) { + push @sections_found, $section; + } else { + push @sections_missing, $section; + } +} + +if (@sections_missing) { + push @warnings, { + category => 'Missing Standard Sections', + severity => 'WARNING', + count => scalar @sections_missing, + details => \@sections_missing, + }; +} + +# =================================================================== +# Category 7: Profile-Based Validation (HA Diagnostics) +# =================================================================== +if ($profile && $profile->{required_sections}) { + for my $section (@{ $profile->{required_sections} }) { + if ($content =~ /\Q$section\E/i) { + push @sections_found, "HA:$section"; + } else { + push @sections_missing, "HA:$section"; + push @warnings, { + category => "Missing HA Section: $section", + severity => 'WARNING', + count => 1, + details => ["Expected HA section '$section' not found in output for topology '$profile->{topology}'"], + }; + } + } +} + +if ($profile && $profile->{required_patterns}) { + for my $pattern (@{ $profile->{required_patterns} }) { + if ($content =~ /\Q$pattern\E/i) { + push @diagnostics_found, $pattern; + } else { + push @diagnostics_missing, $pattern; + } + } + if (@diagnostics_missing) { + push @warnings, { + category => 'Missing HA Diagnostic Patterns', + severity => 'WARNING', + count => scalar @diagnostics_missing, + details => \@diagnostics_missing, + }; + } +} + +if ($profile && $profile->{expected_diagnostics}) { + for my $diag (@{ $profile->{expected_diagnostics} }) { + if ($content =~ /\Q$diag\E/i) { + push @diagnostics_found, "expected:$diag"; + } + } +} + +if ($profile && $profile->{forbidden_patterns}) { + for my $forbidden (@{ $profile->{forbidden_patterns} }) { + my @matches; + while ($content =~ /^(.*\Q$forbidden\E.*)$/gmi) { + push @matches, $1; + } + if (@matches) { + push @errors, { + category => "Forbidden Pattern: $forbidden", + severity => 'ERROR', + count => scalar @matches, + details => [ map { substr($_, 0, 200) } @matches[0 .. ($#matches > 2 ? 2 : $#matches)] ], + }; + } + } +} + +# =================================================================== +# Category 8: Empty / Silent Output +# =================================================================== +my $line_count = () = $content =~ /\n/g; +if ($line_count < 10) { + push @warnings, { + category => 'Suspiciously Short Output', + severity => 'WARNING', + count => 1, + details => ["Output contains only $line_count lines, expected >50"], + }; +} + +# =================================================================== +# Result Compilation +# =================================================================== +my $exit_code = 0; +$exit_code = 1 if @warnings; +$exit_code = 2 if @errors; + +my $result = { + file => basename($output_file), + profile => $profile->{topology} || 'standalone', + exit_code => $exit_code, + verdict => $exit_code == 0 ? 'PASS' : ($exit_code == 1 ? 'WARNING' : 'ERROR'), + error_count => scalar @errors, + warning_count => scalar @warnings, + errors => \@errors, + warnings => \@warnings, + sections_found => \@sections_found, + sections_missing => \@sections_missing, + diagnostics_found => \@diagnostics_found, + diagnostics_missing => \@diagnostics_missing, + output_lines => $line_count, +}; + +# =================================================================== +# Output +# =================================================================== +if ($json_output) { + print encode_json($result) . "\n"; +} elsif (!$quiet) { + my $icon = $exit_code == 0 ? '✅' : ($exit_code == 1 ? '⚠️' : '❌'); + print "=" x 60 . "\n"; + print "$icon MySQLTuner Output Analysis: $result->{verdict}\n"; + print "=" x 60 . "\n"; + printf " File: %s\n", $result->{file}; + printf " Profile: %s\n", $result->{profile}; + printf " Lines: %d\n", $result->{output_lines}; + printf " Errors: %d\n", $result->{error_count}; + printf " Warnings: %d\n", $result->{warning_count}; + printf " Sections: %d found, %d missing\n", scalar @sections_found, scalar @sections_missing; + printf " HA Diags: %d found, %d missing\n", scalar @diagnostics_found, scalar @diagnostics_missing; + print "-" x 60 . "\n"; + + if (@errors) { + print "\n🔴 ERRORS:\n"; + for my $err (@errors) { + printf " [%s] %s (count: %d)\n", $err->{severity}, $err->{category}, $err->{count}; + for my $d (@{ $err->{details} }) { + printf " → %s\n", $d; + } + } + } + + if (@warnings) { + print "\n🟡 WARNINGS:\n"; + for my $w (@warnings) { + printf " [%s] %s (count: %d)\n", $w->{severity}, $w->{category}, $w->{count}; + for my $d (@{ $w->{details} }) { + printf " → %s\n", $d; + } + } + } + + if (@diagnostics_found) { + print "\n🟢 HA DIAGNOSTICS FOUND:\n"; + for my $d (@diagnostics_found) { + printf " ✔ %s\n", $d; + } + } + print "\n"; +} + +exit $exit_code; diff --git a/build/audit_tests.pl b/build/audit_tests.pl index a4696f959..455277dc9 100755 --- a/build/audit_tests.pl +++ b/build/audit_tests.pl @@ -26,7 +26,7 @@ } my $cmd = $filtered_args[0] || $ENV{AUDIT_TEST_CMD}; if (!$cmd) { - $cmd = $debug ? 'prove -rv tests/' : 'prove -r tests/'; + $cmd = $debug ? 'prove -j4 -rv tests/' : 'prove -j4 -r tests/'; } # --- Phase 1: Compile-time syntax check and static analysis --- @@ -107,6 +107,19 @@ print "[OK] Compile Check: All files parsed cleanly.\n\n"; } +# --- Phase 1.5: SQL Static Linter Check --- +print "Performing SQL static linting...\n"; +my $sql_linter_script = 'build/check_sql_linter.pl'; +if (-f $sql_linter_script) { + my $linter_out = qx(perl "$sql_linter_script" 2>&1); + my $exit_val = $? >> 8; + if ($exit_val != 0) { + print "\n[!] SQL Static Linter Failed:\n$linter_out\n"; + exit 1; + } + print "[OK] SQL Linter: All embedded queries conform to conventions.\n\n"; +} + # --- Phase 2: Run test suite and audit runtime output --- print "Executing test suite: $cmd\n"; diff --git a/build/check_compliance.pl b/build/check_compliance.pl index 3e70e69f3..7174c9bd5 100755 --- a/build/check_compliance.pl +++ b/build/check_compliance.pl @@ -205,6 +205,17 @@ } } +# Run static SQL linter check +my $linter_script = dirname(__FILE__) . '/check_sql_linter.pl'; +if (-f $linter_script) { + my $linter_output = qx(perl "$linter_script" 2>&1); + my $exit_val = $? >> 8; + if ($exit_val != 0) { + print "ERROR [SQL Linter Audit]: SQL static linter failed:\n$linter_output\n"; + $errors++; + } +} + if ( $errors > 0 ) { print "\n[FAIL] Compliance check failed: $errors violations detected.\n"; exit 1; diff --git a/build/check_sql_linter.pl b/build/check_sql_linter.pl new file mode 100755 index 000000000..f92cc1853 --- /dev/null +++ b/build/check_sql_linter.pl @@ -0,0 +1,101 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use File::Basename; +use Cwd 'abs_path'; + +my $script_dir = dirname(abs_path(__FILE__)); +my $mysqltuner_path = abs_path("$script_dir/../mysqltuner.pl"); + +open my $fh, '<', $mysqltuner_path or die "Could not open $mysqltuner_path: $!"; +my $content = do { local $/; <$fh> }; +close $fh; + +# Track error count +my $errors = 0; + +# Strip out comments and POD to avoid false positives +$content =~ s/^__END__.*$//ms; # Stop at __END__ +$content =~ s/^=pod.*?=cut//msg; # Strip POD documentation + +# We want to scan the file line-by-line to report accurate line numbers +my @lines = split /\n/, $content; +my $line_num = 0; + +# Standard SQL keywords to check for uppercase +my %SQL_KEYWORDS = map { $_ => 1 } qw( + SELECT FROM WHERE INSERT UPDATE DELETE SHOW + LEFT JOIN RIGHT JOIN INNER JOIN JOIN ON + GROUP BY ORDER BY LIMIT HAVING AS IN LIKE + AND OR NOT UNION DISTINCT DESC ASC SUM COUNT +); + +foreach my $line (@lines) { + $line_num++; + + # Match double or single quoted strings containing SQL-like statements + # Example: "SELECT ... " or 'SELECT ... ' + while ($line =~ /(["'])\s*((?:SELECT|SHOW|UPDATE|INSERT|DELETE|CREATE|ALTER|DROP)\b.*?)\1/ig) { + my $quote = $1; + my $sql = $2; + my $start_idx = $-[0]; + my $end_idx = $+[0]; + + # Check context for concatenation to avoid false positives on partial query strings + my $before = substr($line, 0, $start_idx); + my $after = substr($line, $end_idx); + + # Check if next line starts with '.' + my $next_line = ($line_num < scalar(@lines)) ? $lines[$line_num] : ''; + $next_line =~ s/^\s+//; + + my $is_concatenated = ($before =~ /\.\s*$/ || $after =~ /^\s*\./ || $next_line =~ /^\./ || $line =~ /^\s*\./) ? 1 : 0; + + # 1. Parentheses balance verification (only for complete, non-concatenated queries) + if (!$is_concatenated) { + my $open_parens = () = $sql =~ /\(/g; + my $close_parens = () = $sql =~ /\)/g; + if ($open_parens != $close_parens) { + print "ERROR [SQL Linter] Line $line_num: Unbalanced parentheses in SQL query ($open_parens vs $close_parens) -> \"$sql\"\n"; + $errors++; + } + } + + # 2. Performance Schema events_errors_summary_global_by_error COUNT_STAR check + if ($sql =~ /events_errors_summary_global_by_error/i && $sql =~ /COUNT_STAR/i) { + print "ERROR [SQL Linter] Line $line_num: Query on events_errors_summary_global_by_error uses COUNT_STAR, which does not exist in this table. Use SUM_ERROR_RAISED instead.\n"; + $errors++; + } + + # 3. Casing conventions for SQL keywords + # Extract keywords at clause boundaries + my @words = split /\s+/, $sql; + foreach my $word (@words) { + # Strip punctuation and MySQL backticks/parentheses + $word =~ s/^[\(`'"]+|[,\)`'"]+$//g; + if ($SQL_KEYWORDS{uc($word)} && $word ne uc($word)) { + # Don't flag if it's part of a function call parameter or column alias unless it is a primary keyword at clause boundaries + if ($word =~ /^[a-z]+$/ && uc($word) =~ /^(?:SELECT|FROM|WHERE|JOIN|GROUP|ORDER|LIMIT|HAVING|UNION)$/) { + print "WARNING [SQL Linter] Line $line_num: SQL keyword '$word' should be uppercase -> \"$sql\"\n"; + } + } + } + + # 4. Schema casing conventions (only for qualified object references like schema.table) + if ($sql =~ /\b(performance_schema|information_schema|mysql)\s*\./i) { + my $schema = $1; + if ($schema ne lc($schema)) { + print "ERROR [SQL Linter] Line $line_num: Schema name '$schema' must be lowercase in qualified reference -> \"$sql\"\n"; + $errors++; + } + } + } +} + +if ($errors > 0) { + print "\n[FAIL] SQL Static Linter: $errors violations detected.\n"; + exit 1; +} + +print "[OK] SQL Static Linter: No SQL issues or formatting anomalies detected.\n"; +exit 0; diff --git a/build/ha_profiles/galera.json b/build/ha_profiles/galera.json new file mode 100644 index 000000000..4c285a6cc --- /dev/null +++ b/build/ha_profiles/galera.json @@ -0,0 +1,29 @@ +{ + "topology": "galera", + "display_name": "MariaDB Galera Cluster", + "required_sections": [ + "Galera Cluster", + "wsrep" + ], + "required_patterns": [ + "wsrep_cluster_size", + "wsrep_ready", + "wsrep_local_state_comment", + "wsrep_connected" + ], + "forbidden_patterns": [ + "Use of uninitialized value", + "FAIL Execute SQL", + "Can't connect to" + ], + "expected_diagnostics": [ + "wsrep_flow_control", + "wsrep_local_bf_aborts", + "gcache", + "Galera" + ], + "ports": [3511, 3512, 3513], + "startup_command": "up-galera", + "shutdown_command": "down-galera", + "inject_command": "inject-employee-galera" +} diff --git a/build/ha_profiles/innodb_cluster.json b/build/ha_profiles/innodb_cluster.json new file mode 100644 index 000000000..8391a402c --- /dev/null +++ b/build/ha_profiles/innodb_cluster.json @@ -0,0 +1,28 @@ +{ + "topology": "innodb_cluster", + "display_name": "MySQL InnoDB Cluster (Group Replication)", + "required_sections": [ + "Group Replication", + "Replication" + ], + "required_patterns": [ + "group_replication", + "MEMBER_STATE", + "MEMBER_ROLE" + ], + "forbidden_patterns": [ + "Use of uninitialized value", + "FAIL Execute SQL", + "Can't connect to" + ], + "expected_diagnostics": [ + "group_replication_flow_control", + "COUNT_TRANSACTIONS_IN_QUEUE", + "replication_group_members", + "certification" + ], + "ports": [4411, 4412, 4413], + "startup_command": "innodb-up", + "shutdown_command": "innodb-down", + "inject_command": "inject-data service=mysql_node1 db=employees" +} diff --git a/build/ha_profiles/replication.json b/build/ha_profiles/replication.json new file mode 100644 index 000000000..45d0c5fb8 --- /dev/null +++ b/build/ha_profiles/replication.json @@ -0,0 +1,29 @@ +{ + "topology": "replication", + "display_name": "MariaDB Source-Replica Replication", + "required_sections": [ + "Replication", + "Slave" + ], + "required_patterns": [ + "Slave_IO_Running", + "Slave_SQL_Running", + "Seconds_Behind_Master" + ], + "forbidden_patterns": [ + "Use of uninitialized value", + "FAIL Execute SQL", + "Can't connect to" + ], + "expected_diagnostics": [ + "replication lag", + "binlog", + "relay_log", + "gtid", + "slave_parallel" + ], + "ports": [3411, 3412, 3413], + "startup_command": "up-repli", + "shutdown_command": "down-repli", + "inject_command": "inject-employee-repli" +} diff --git a/build/mcp_server.py b/build/mcp_server.py new file mode 100644 index 000000000..5d972e7da --- /dev/null +++ b/build/mcp_server.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import time +import threading +import subprocess +import traceback + +# Config defaults +CACHE_DIR = os.environ.get("CACHE_DIR", "/var/cache/mysqltuner") +AUDIT_INTERVAL_HOURS = float(os.environ.get("AUDIT_INTERVAL_HOURS", "12")) +READ_ONLY = os.environ.get("READ_ONLY", "false").lower() == "true" + +# Ensure cache directory exists +os.makedirs(CACHE_DIR, exist_ok=True) + +STATE_FILE = os.path.join(CACHE_DIR, "state.json") +LATEST_JSON = os.path.join(CACHE_DIR, "latest.json") +LATEST_HTML = os.path.join(CACHE_DIR, "latest.html") + +def load_state(): + if os.path.exists(STATE_FILE): + try: + with open(STATE_FILE, "r") as f: + return json.load(f) + except Exception: + pass + return {"applied": []} + +def save_state(state): + try: + with open(STATE_FILE, "w") as f: + json.dump(state, f, indent=2) + except Exception: + pass + +def run_mysqltuner_cmd(): + # Build connection args from environment variables + args = ["/usr/bin/perl", "mysqltuner.pl", "--prettyjson", "--reportfile", LATEST_HTML] + + db_host = os.environ.get("DB_HOST") + db_port = os.environ.get("DB_PORT") + db_user = os.environ.get("DB_USER") + db_pass = os.environ.get("DB_PASSWORD") + + if db_host: + args.extend(["--host", db_host]) + if db_port: + args.extend(["--port", db_port]) + if db_user: + args.extend(["--user", db_user]) + if db_pass: + args.extend(["--pass", db_pass]) + + try: + # Run process and capture stdout + result = subprocess.run(args, capture_output=True, text=True, check=True) + # Save to latest.json + with open(LATEST_JSON, "w") as f: + f.write(result.stdout) + return True, result.stdout + except subprocess.CalledProcessError as e: + err_msg = f"MySQLTuner failed with code {e.returncode}: {e.stderr}" + return False, err_msg + except Exception as e: + return False, str(e) + +def daemon_loop(): + while True: + run_mysqltuner_cmd() + # Sleep interval converted to seconds + time.sleep(AUDIT_INTERVAL_HOURS * 3600) + +# DB query helper +def run_db_query(query): + mysql_cmd = ["mysql", "-Bse", query] + db_host = os.environ.get("DB_HOST") + db_port = os.environ.get("DB_PORT") + db_user = os.environ.get("DB_USER") + db_pass = os.environ.get("DB_PASSWORD") + + if db_host: + mysql_cmd.extend(["-h", db_host]) + if db_port: + mysql_cmd.extend(["-P", db_port]) + if db_user: + mysql_cmd.extend(["-u", db_user]) + if db_pass: + mysql_cmd.extend([f"-p{db_pass}"]) + + try: + res = subprocess.run(mysql_cmd, capture_output=True, text=True, check=True) + return True, res.stdout.strip() + except subprocess.CalledProcessError as e: + return False, e.stderr.strip() + +# MCP Tool handlers +def handle_get_latest_audit(): + if os.path.exists(LATEST_JSON): + try: + with open(LATEST_JSON, "r") as f: + return {"content": [{"type": "text", "text": f.read()}]} + except Exception as e: + return {"isError": True, "content": [{"type": "text", "text": f"Error reading cached audit: {str(e)}"}]} + return {"content": [{"type": "text", "text": "No cached audit findings found. Try running run_audit first."}]} + +def handle_run_audit(): + success, output = run_mysqltuner_cmd() + if success: + return {"content": [{"type": "text", "text": output}]} + return {"isError": True, "content": [{"type": "text", "text": f"Failed to execute audit: {output}"}]} + +def handle_apply_recommendation(arguments): + if READ_ONLY: + return {"isError": True, "content": [{"type": "text", "text": "Execution rejected: MCP server is running in read-only mode."}]} + + statement = arguments.get("statement") + if not statement: + return {"isError": True, "content": [{"type": "text", "text": "Missing parameter: 'statement' is required."}]} + + # Safety Check: Allow only SET GLOBAL, ALTER TABLE, OPTIMIZE TABLE + clean_stmt = statement.strip().upper() + is_safe = (clean_stmt.startswith("SET GLOBAL") or + clean_stmt.startswith("ALTER TABLE") or + clean_stmt.startswith("OPTIMIZE TABLE")) + + if not is_safe: + return {"isError": True, "content": [{"type": "text", "text": f"Execution rejected: Statement '{statement}' is not recognized as a safe configuration adjustment."}]} + + # If setting a global variable, fetch its current value for rollback + var_name = arguments.get("variable_name") + old_value = None + if var_name: + success, val = run_db_query(f"SELECT @@global.{var_name}") + if success: + old_value = val + + # Execute statement + success, err = run_db_query(statement) + if not success: + return {"isError": True, "content": [{"type": "text", "text": f"SQL Execution failed: {err}"}]} + + # Save to transaction state + state = load_state() + stmt_id = str(int(time.time())) + state["applied"].append({ + "id": stmt_id, + "statement": statement, + "variable_name": var_name, + "old_value": old_value, + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S") + }) + save_state(state) + + return {"content": [{"type": "text", "text": f"Success: Statement executed successfully. Statement ID: {stmt_id}"}]} + +def handle_rollback_recommendation(arguments): + if READ_ONLY: + return {"isError": True, "content": [{"type": "text", "text": "Execution rejected: MCP server is running in read-only mode."}]} + + stmt_id = arguments.get("statement_id") + if not stmt_id: + return {"isError": True, "content": [{"type": "text", "text": "Missing parameter: 'statement_id' is required."}]} + + state = load_state() + target = None + for entry in state["applied"]: + if entry["id"] == stmt_id: + target = entry + break + + if not target: + return {"isError": True, "content": [{"type": "text", "text": f"Error: Statement ID {stmt_id} not found in state registry."}]} + + var_name = target.get("variable_name") + old_value = target.get("old_value") + + if var_name and old_value is not None: + # Revert global variable + rollback_stmt = f"SET GLOBAL {var_name} = {old_value}" + success, err = run_db_query(rollback_stmt) + if not success: + return {"isError": True, "content": [{"type": "text", "text": f"Rollback SQL failed: {err}"}]} + else: + return {"isError": True, "content": [{"type": "text", "text": "Cannot rollback: This statement type does not support automatic rollback."}]} + + # Remove from state + state["applied"].remove(target) + save_state(state) + + return {"content": [{"type": "text", "text": f"Success: Rollback executed successfully: {rollback_stmt}"}]} + +# MCP Protocol handler Loop +def main_mcp(): + while True: + try: + line = sys.stdin.readline() + if not line: + break + req = json.loads(line) + method = req.get("method") + id_ = req.get("id") + + if method == "initialize": + resp = { + "jsonrpc": "2.0", + "result": { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {}, + "resources": {} + }, + "serverInfo": { + "name": "mysqltuner-mcp", + "version": "2.9.1" + } + }, + "id": id_ + } + elif method == "tools/list": + resp = { + "jsonrpc": "2.0", + "result": { + "tools": [ + { + "name": "get_latest_audit", + "description": "Get the latest cached audit findings in JSON format." + }, + { + "name": "run_audit", + "description": "Execute a fresh database audit and return findings immediately." + }, + { + "name": "apply_recommendation", + "description": "Apply a safe database recommendation (e.g. SET GLOBAL or ALTER TABLE).", + "inputSchema": { + "type": "object", + "properties": { + "statement": {"type": "string", "description": "The SQL statement to execute."}, + "variable_name": {"type": "string", "description": "The global variable name being set (optional, for rollback)."} + }, + "required": ["statement"] + } + }, + { + "name": "rollback_recommendation", + "description": "Revert a previously applied database recommendation.", + "inputSchema": { + "type": "object", + "properties": { + "statement_id": {"type": "string", "description": "The Statement ID returned during execution."} + }, + "required": ["statement_id"] + } + } + ] + }, + "id": id_ + } + elif method == "tools/call": + params = req.get("params", {}) + name = params.get("name") + arguments = params.get("arguments", {}) + + if name == "get_latest_audit": + res = handle_get_latest_audit() + elif name == "run_audit": + res = handle_run_audit() + elif name == "apply_recommendation": + res = handle_apply_recommendation(arguments) + elif name == "rollback_recommendation": + res = handle_rollback_recommendation(arguments) + else: + res = {"isError": True, "content": [{"type": "text", "text": f"Unknown tool: {name}"}]} + + resp = { + "jsonrpc": "2.0", + "result": res, + "id": id_ + } + elif method == "resources/list": + resp = { + "jsonrpc": "2.0", + "result": { + "resources": [ + { + "uri": "mysqltuner://reports/latest.json", + "name": "Latest JSON report", + "mimeType": "application/json" + }, + { + "uri": "mysqltuner://reports/latest.html", + "name": "Latest HTML dashboard", + "mimeType": "text/html" + } + ] + }, + "id": id_ + } + elif method == "resources/read": + params = req.get("params", {}) + uri = params.get("uri") + + content = "" + mime = "text/plain" + target_path = None + if uri == "mysqltuner://reports/latest.json": + target_path = LATEST_JSON + mime = "application/json" + elif uri == "mysqltuner://reports/latest.html": + target_path = LATEST_HTML + mime = "text/html" + + if target_path and os.path.exists(target_path): + try: + with open(target_path, "r") as f: + content = f.read() + except Exception as e: + content = f"Error reading resource: {str(e)}" + else: + content = "Resource not found or empty." + + resp = { + "jsonrpc": "2.0", + "result": { + "contents": [ + { + "uri": uri, + "mimeType": mime, + "text": content + } + ] + }, + "id": id_ + } + else: + resp = { + "jsonrpc": "2.0", + "error": { + "code": -32601, + "message": f"Method not found: {method}" + }, + "id": id_ + } + + sys.stdout.write(json.dumps(resp) + "\n") + sys.stdout.flush() + except Exception as e: + sys.stderr.write(traceback.format_exc() + "\n") + sys.stderr.flush() + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "--daemon": + # Run daemon auditing in foreground + daemon_loop() + else: + # Start daemon interval thread + t = threading.Thread(target=daemon_loop, daemon=True) + t.start() + # Serve MCP stdio + main_mcp() diff --git a/build/test_ha.sh b/build/test_ha.sh new file mode 100755 index 000000000..557c6d865 --- /dev/null +++ b/build/test_ha.sh @@ -0,0 +1,273 @@ +#!/bin/bash +# =========================================================================== +# Script: test_ha.sh +# Description: E2E test suite for MySQLTuner against High Availability +# topologies provided by multi-db-docker-env. +# Author: Jean-Marie Renouard & Antigravity +# Usage: bash build/test_ha.sh [galera|innodb|repli|all] +# Dependencies: Docker, multi-db-docker-env (cloned via vendor/) +# =========================================================================== +set -euo pipefail + +# Configuration +PROJECT_ROOT=$(pwd) +VENDOR_DIR="$PROJECT_ROOT/vendor" +MULTI_DB_DIR="$VENDOR_DIR/multi-db-docker-env" +MULTI_DB_REPO="https://github.com/jmrenouard/multi-db-docker-env" +EXAMPLES_DIR="$PROJECT_ROOT/examples" +PROFILES_DIR="$PROJECT_ROOT/build/ha_profiles" +ANALYZER="$PROJECT_ROOT/build/analyze_mt_output.pl" +CVE_FILE="$PROJECT_ROOT/vulnerabilities.csv" +DATE_TAG=$(date +%Y%m%d_%H%M%S) +DB_PASS="${DB_ROOT_PASSWORD:-mysqltuner_test}" + +# Topologies to test (default: all) +TOPOS="${1:-all}" + +PASS_TOTAL=0 +FAIL_TOTAL=0 +WARN_TOTAL=0 + +log_step() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +log_header() { + echo "======================================================================" + echo " MySQLTuner HA E2E Test - $1 - $(date)" + echo "======================================================================" +} + +# Setup vendor repositories +setup_vendor() { + log_step "Setting up vendor repositories..." + mkdir -p "$VENDOR_DIR" + if [ ! -d "$MULTI_DB_DIR" ]; then + git clone "$MULTI_DB_REPO" "$MULTI_DB_DIR" + else + (cd "$MULTI_DB_DIR" && git pull --ff-only 2>/dev/null || true) + fi + + # Ensure .env exists + if [ ! -f "$MULTI_DB_DIR/.env" ]; then + echo "DB_ROOT_PASSWORD=$DB_PASS" > "$MULTI_DB_DIR/.env" + fi +} + +# Wait for a MySQL port to be ready +wait_for_port() { + local port=$1 + local max_wait=${2:-120} + local count=0 + log_step "Waiting for port $port to be ready (max ${max_wait}s)..." + until mysqladmin -h 127.0.0.1 -P "$port" -u root -p"$DB_PASS" ping >/dev/null 2>&1; do + sleep 2 + count=$((count + 2)) + if [ $count -ge $max_wait ]; then + log_step "ERROR: Timeout waiting for port $port" + return 1 + fi + done + log_step "Port $port is ready (${count}s)." + return 0 +} + +# Run MySQLTuner against a specific port and capture output +run_mysqltuner_on_port() { + local port=$1 + local target_dir=$2 + local node_label=$3 + local profile_file=$4 + + local output_file="$target_dir/${node_label}_output.txt" + local report_file="$target_dir/${node_label}_report.html" + + log_step "Running MySQLTuner on $node_label (port $port)..." + + local mt_args="--host 127.0.0.1 --port $port --user root --pass $DB_PASS" + mt_args="$mt_args --verbose --forcemem 256" + [ -f "$CVE_FILE" ] && mt_args="$mt_args --cvefile $CVE_FILE" + mt_args="$mt_args --reportfile $report_file" + + local start_time=$(date +%s) + perl "$PROJECT_ROOT/mysqltuner.pl" $mt_args > "$output_file" 2>&1 || true + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + + log_step "MySQLTuner finished on $node_label in ${duration}s" + + # Run analyzer + log_step "Analyzing output for $node_label..." + local analyzer_args="$output_file" + [ -n "$profile_file" ] && [ -f "$profile_file" ] && analyzer_args="--profile $profile_file $analyzer_args" + + local analyze_exit=0 + perl "$ANALYZER" $analyzer_args 2>&1 | tee "$target_dir/${node_label}_analysis.txt" || analyze_exit=$? + + # Also produce JSON + perl "$ANALYZER" --json $analyzer_args > "$target_dir/${node_label}_analysis.json" 2>/dev/null || true + + return $analyze_exit +} + +# Run a complete HA topology test +run_ha_test() { + local topo=$1 + local profile_file="$PROFILES_DIR/${topo}.json" + + if [ ! -f "$profile_file" ]; then + log_step "ERROR: Profile not found: $profile_file" + FAIL_TOTAL=$((FAIL_TOTAL + 1)) + return 1 + fi + + # Parse profile with perl (Core module JSON) + local display_name + display_name=$(perl -MJSON -e 'local $/; open my $f, "<", shift; print decode_json(<$f>)->{display_name}' "$profile_file") + local startup_cmd + startup_cmd=$(perl -MJSON -e 'local $/; open my $f, "<", shift; print decode_json(<$f>)->{startup_command}' "$profile_file") + local shutdown_cmd + shutdown_cmd=$(perl -MJSON -e 'local $/; open my $f, "<", shift; print decode_json(<$f>)->{shutdown_command}' "$profile_file") + local ports_json + ports_json=$(perl -MJSON -e 'local $/; open my $f, "<", shift; print encode_json(decode_json(<$f>)->{ports})' "$profile_file") + local inject_cmd + inject_cmd=$(perl -MJSON -e 'local $/; open my $f, "<", shift; print decode_json(<$f>)->{inject_command}' "$profile_file") + + log_header "$display_name" + + local target_dir="$EXAMPLES_DIR/${DATE_TAG}_ha_${topo}" + mkdir -p "$target_dir" + + # Navigate to multi-db-docker-env + cd "$MULTI_DB_DIR" || return 1 + + # Start topology + log_step "Starting $display_name via 'make $startup_cmd'..." + make "$startup_cmd" > "$target_dir/docker_start.log" 2>&1 || { + log_step "CRITICAL: Failed to start $display_name" + cat "$target_dir/docker_start.log" + FAIL_TOTAL=$((FAIL_TOTAL + 1)) + cd "$PROJECT_ROOT" + return 1 + } + + # Wait for all ports + local ports + ports=$(echo "$ports_json" | perl -MJSON -e 'local $/; my $a = decode_json(); print join(" ", @$a)') + local all_ready=true + for port in $ports; do + if ! wait_for_port "$port" 120; then + all_ready=false + break + fi + done + + if [ "$all_ready" = false ]; then + log_step "ERROR: Not all ports ready for $display_name" + docker compose -f "docker-compose-${topo}.yml" logs > "$target_dir/container_logs.log" 2>&1 || true + make "$shutdown_cmd" > /dev/null 2>&1 || true + FAIL_TOTAL=$((FAIL_TOTAL + 1)) + cd "$PROJECT_ROOT" + return 1 + fi + + # Inject data + if [ -n "$inject_cmd" ]; then + log_step "Injecting test data via 'make $inject_cmd'..." + make $inject_cmd > "$target_dir/db_injection.log" 2>&1 || { + log_step "WARNING: Data injection failed (non-fatal)" + } + fi + + # Capture container logs + docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" > "$target_dir/container_status.txt" 2>/dev/null || true + + # Return to project root + cd "$PROJECT_ROOT" + + # Run MySQLTuner on each node + local node_idx=0 + local topo_errors=0 + for port in $ports; do + node_idx=$((node_idx + 1)) + local node_label="node${node_idx}_port${port}" + local exit_code=0 + run_mysqltuner_on_port "$port" "$target_dir" "$node_label" "$profile_file" || exit_code=$? + + case $exit_code in + 0) PASS_TOTAL=$((PASS_TOTAL + 1)) ;; + 1) WARN_TOTAL=$((WARN_TOTAL + 1)) ;; + *) + FAIL_TOTAL=$((FAIL_TOTAL + 1)) + topo_errors=$((topo_errors + 1)) + ;; + esac + done + + # Shutdown topology + cd "$MULTI_DB_DIR" + log_step "Shutting down $display_name via 'make $shutdown_cmd'..." + make "$shutdown_cmd" > "$target_dir/docker_shutdown.log" 2>&1 || true + cd "$PROJECT_ROOT" + + # Generate consolidated summary + log_step "Generating summary for $display_name..." + { + echo "# HA E2E Test Summary: $display_name" + echo "**Date:** $(date)" + echo "**Topology:** $topo" + echo "**Nodes tested:** $node_idx" + echo "**Errors:** $topo_errors" + echo "" + echo "## Node Results" + for f in "$target_dir"/*_analysis.txt; do + [ -f "$f" ] && echo "### $(basename "$f" _analysis.txt)" && cat "$f" && echo "" + done + } > "$target_dir/summary.md" + + if [ $topo_errors -eq 0 ]; then + log_step "✅ $display_name: ALL NODES PASSED" + else + log_step "❌ $display_name: $topo_errors NODE(S) FAILED" + fi + + return $topo_errors +} + +# ===================================================================== +# Main Execution +# ===================================================================== +setup_vendor + +case "$TOPOS" in + galera) + run_ha_test "galera" + ;; + innodb|innodb_cluster) + run_ha_test "innodb_cluster" + ;; + repli|replication) + run_ha_test "replication" + ;; + all) + run_ha_test "galera" || true + run_ha_test "innodb_cluster" || true + run_ha_test "replication" || true + ;; + *) + echo "Usage: $0 [galera|innodb|repli|all]" + exit 1 + ;; +esac + +echo "" +echo "======================================================================" +echo " HA E2E Test Complete" +echo " PASS: $PASS_TOTAL | WARN: $WARN_TOTAL | FAIL: $FAIL_TOTAL" +echo " Reports: $EXAMPLES_DIR" +echo "======================================================================" + +if [ $FAIL_TOTAL -gt 0 ]; then + exit 1 +fi +exit 0 diff --git a/documentation/mcp_ai_integration_guide.md b/documentation/mcp_ai_integration_guide.md new file mode 100644 index 000000000..e8a94984d --- /dev/null +++ b/documentation/mcp_ai_integration_guide.md @@ -0,0 +1,136 @@ +# AI & MCP Integration Guide for MySQL Optimization + +This guide explains how to set up the Model Context Protocol (MCP) server for MySQLTuner and configure AI agents (e.g. Claude Desktop, Cursor, VS Code Extensions) to run deep, automated performance audits and apply safe, rollback-enabled tuning configurations. + +--- + +## 📦 Part 1: Setting Up the MCP Server + +The MySQLTuner MCP server acts as an intermediary bridge between your database and AI agents, exposing database telemetry and actionable SQL recommendations over standard I/O (stdio). + +### Method A: Dockerized Deployment (Recommended) +This method containerizes the entire toolchain (Perl, Python 3, and mysql client utilities) to ensure compatibility. + +```bash +docker run -d \ + --name mysqltuner-mcp \ + -e DB_HOST=your-database-host \ + -e DB_PORT=3306 \ + -e DB_USER=tuner_user \ + -e DB_PASSWORD=your_password \ + -e AUDIT_INTERVAL_HOURS=6 \ + -v /var/cache/mysqltuner:/var/cache/mysqltuner \ + mysqltuner-mcp +``` + +### Method B: Local Execution (Without Docker) +Ensure Python 3 and Perl are installed locally, then run the script directly: +```bash +export DB_HOST="127.0.0.1" +export DB_USER="root" +export DB_PASSWORD="your_password" +export CACHE_DIR="./mcp_cache" + +python3 build/mcp_server.py +``` + +--- + +## 🛠️ Part 2: Configuring AI Clients + +Once the server is running, register it inside your preferred AI agent environment. + +### 1. Claude Desktop Config +Add the server definition to your Claude Desktop configuration file: +- **macOS**: `~/Library/Application Support/Claude/claude_desktop_config.json` +- **Windows**: `%APPDATA%\Claude\claude_desktop_config.json` + +```json +{ + "mcpServers": { + "mysqltuner": { + "command": "docker", + "args": [ + "run", + "-i", + "--rm", + "-v", + "/var/cache/mysqltuner:/var/cache/mysqltuner", + "-e", + "DB_HOST=host.docker.internal", + "-e", + "DB_USER=root", + "-e", + "DB_PASSWORD=secret", + "mysqltuner-mcp" + ] + } + } +} +``` + +### 2. Cursor IDE Config +1. Open Cursor and navigate to **Settings** -> **Features** -> **MCP**. +2. Click **+ Add New MCP Server**. +3. Fill in the parameters: + - **Name**: `mysqltuner` + - **Type**: `stdio` + - **Command**: `python3 /path/to/MySQLTuner-perl/build/mcp_server.py` +4. Set environment variables in the terminal where Cursor was launched. + +### 3. VS Code (Cline / Roo Code / Roo Cline) +Configure the extension settings `mcpSettings.json` to spawn the server: +```json +{ + "mcpServers": { + "mysqltuner": { + "command": "python3", + "args": ["/path/to/MySQLTuner-perl/build/mcp_server.py"], + "env": { + "DB_HOST": "127.0.0.1", + "DB_USER": "root", + "DB_PASSWORD": "your_password" + } + } + } +} +``` + +--- + +## 🔍 Part 3: Deep Database Tuning with AI + +When connected, the AI agent has access to MySQLTuner findings and can cross-reference logs, memory allocations, and schema design to perform high-density optimizations. + +### 1. Memory Allocation and Buffers +AI agents can parse the buffer pool allocations and compare them to physical RAM limits to prevent Out-Of-Memory (OOM) situations. +- **Agent Analysis**: Evaluates `pct_max_physical_memory` to verify if memory usage is safe. +- **Live Adjustment**: Executes `apply_recommendation` with `SET GLOBAL innodb_buffer_pool_size = ` if the database version supports dynamic buffer pool resizing (MySQL 5.7+). + +### 2. Connection Saturation and Thread Cache +High connection spikes cause high thread creation overhead. +- **Agent Analysis**: Evaluates `max_connections` and matches it against `threads_created`. +- **Live Adjustment**: Sets `thread_cache_size` to reduce creation overhead: + `SET GLOBAL thread_cache_size = 16;` + +### 3. Index Profiling and Table Churn +- **Agent Analysis**: The agent queries table fragmentation and matches it with Performance Schema query logs. +- **Live Action**: Automatically schedules defragmentation for high-churn tables: + `OPTIMIZE TABLE schema_name.table_name;` + +--- + +## 🤖 Part 4: Advanced Prompt Engineering for AI Agents + +To ensure the AI operates safely and acts as an expert DBA, prepend your conversations with the following System Prompt: + +```markdown +You are a Senior Principal Database Administrator (DBA). You have access to the MySQLTuner MCP server. +Your core mission is to audit, analyze, and optimize the MySQL instance safely. + +### Operating Rules: +1. **Always Verify Baseline**: Before executing any SQL changes, read the cached audit resources (`mysqltuner://reports/latest.json`). +2. **Classify by Risk**: Categorize recommendations. Apply 'Low' or 'Medium' risk adjustments dynamically. Never apply 'High' or 'Critical' recommendations (such as changes requiring a service restart or ALTER TABLE on tables > 10GB) without explicit user confirmation. +3. **Draft Rollbacks First**: Before invoking `apply_recommendation`, state the exact SQL statement to be executed AND the corresponding `rollback_statement` so the user is fully informed. +4. **Iterative Auditing**: After applying a recommendation, trigger `run_audit` to confirm that the indicator has improved. If performance metrics degrade or the audit flags unexpected regressions, immediately run `rollback_recommendation` using the returned Statement ID. +``` diff --git a/mariadb_support.md b/mariadb_support.md index bedb0fc0b..2f075d3b7 100644 --- a/mariadb_support.md +++ b/mariadb_support.md @@ -3,6 +3,8 @@ | Version | End of Support Date | LTS | Status | |---------|------------------------|-----|--------| | 12.3 | 2029-06-30 | YES | Supported | +| 12.2 | 2026-05-13 | NO | Outdated | +| 12.1 | 2026-02-13 | NO | Outdated | | 12.0 | 2025-11-18 | NO | Outdated | | 11.8 | 2028-06-04 | YES | Supported | | 11.7 | 2025-05-12 | NO | Outdated | @@ -18,7 +20,7 @@ | 10.9 | 2023-08-22 | NO | Outdated | | 10.8 | 2023-05-20 | NO | Outdated | | 10.7 | 2023-02-09 | NO | Outdated | -| 10.6 | 2026-07-06 | YES | Supported | +| 10.6 | 2026-07-06 | YES | Outdated | | 10.5 | 2025-06-24 | YES | Outdated | | 10.4 | 2024-06-18 | YES | Outdated | | 10.3 | 2023-05-25 | NO | Outdated | diff --git a/mysql_support.md b/mysql_support.md index fc9535a03..432baecb0 100644 --- a/mysql_support.md +++ b/mysql_support.md @@ -2,7 +2,9 @@ | Version | End of Support Date | LTS | Status | |---------|------------------------|-----|--------| -| 9.6 | N/A | NO | Supported | +| 9.7 | 2034-04-21 | YES | Supported | +| 9.6 | 2026-04-21 | NO | Outdated | +| 9.5 | 2026-01-20 | NO | Outdated | | 9.4 | 2025-10-21 | NO | Outdated | | 9.3 | 2025-07-22 | NO | Outdated | | 9.2 | 2025-04-15 | NO | Outdated | diff --git a/mysqltuner.pl b/mysqltuner.pl index 6f6e59b36..f5342cb3a 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -1,5 +1,5 @@ #!/usr/bin/env perl -# mysqltuner.pl - Version 2.9.0 +# mysqltuner.pl - Version 2.9.1 # High Performance MySQL Tuning Script # Copyright (C) 2015-2026 Jean-Marie Renouard - jmrenouard@gmail.com # Copyright (C) 2006-2026 Major Hayden - major@mhtx.net @@ -67,7 +67,7 @@ package main; our $is_win = $^O eq 'MSWin32'; # Set up a few variables for use in the script -our $tunerversion = "2.9.0"; +our $tunerversion = "2.9.1"; our ( @adjvars, @generalrec, @modeling, @sysrec, @secrec ); our ( %result, %myvar, %real_vars, %mystat, %mycalc, %myrepl, %myreplicas, $dummyselect ); @@ -300,6 +300,12 @@ package main; desc => 'Print result as JSON string', cat => 'PERFORMANCE' }, + 'agent-json' => { + type => '!', + default => 0, + desc => 'Print result as actionable JSON schema for AI agents', + cat => 'PERFORMANCE' + }, 'prettyjson' => { type => '!', default => 0, @@ -533,9 +539,10 @@ package main; cat => 'CLOUD' }, 'container' => { - type => '=s', - default => undef, - desc => 'Enable container mode with ID or name', + type => '=s', + default => undef, + desc => +'Enable container mode with ID or name (requires docker, podman, or kubectl client)', placeholder => '', cat => 'CLOUD' }, @@ -887,7 +894,10 @@ sub show_help { sub prettyprint { print $_[0] . "\n" - unless ( $opt{'silent'} or $opt{'json'} or $opt{'yaml'} ); + unless ( $opt{'silent'} + or $opt{'json'} + or $opt{'yaml'} + or $opt{'agent-json'} ); print $fh $_[0] . "\n" if defined($fh); my $plain_text = $_[0] // ''; $plain_text =~ s/\e\[[0-9;]*[mK]//g; @@ -1567,6 +1577,171 @@ sub check_replication_advanced { "Enable replica_sql_verify_checksum = ON (or slave_sql_verify_checksum)." ); } + + # Relay Log Hardening + my $skip_verify = $myvar{'replica_skip_verify_binlog_checksum'} + // $myvar{'slave_skip_verify_binlog_checksum'} // ''; + if ( $skip_verify eq 'ON' || $skip_verify eq '1' ) { + badprint + "replica_skip_verify_binlog_checksum is enabled (recommended: OFF)"; + push_recommendation( 'res', +"Disable replica_skip_verify_binlog_checksum to ensure checksum verification on replica." + ); + } + + # --- PHASE 7: HA & INNODB CLUSTER DIAGNOSTICS --- + my $is_group_repl = ( defined $myvar{'group_replication_group_name'} + || defined $myvar{'group_replication_single_primary_mode'} ); + if ($is_group_repl) { + subheaderprint "InnoDB Cluster & Group Replication Diagnostics"; + + # 1. Topology & Role Audit + if ( defined $myvar{'performance_schema'} + && $myvar{'performance_schema'} eq 'ON' ) + { + my @group_members = select_array( +"SELECT MEMBER_HOST, MEMBER_PORT, MEMBER_STATE, MEMBER_ROLE, MEMBER_VERSION FROM performance_schema.replication_group_members" + ); + my $member_count = scalar(@group_members); + if ( $member_count > 0 ) { + goodprint + "InnoDB Cluster is running with $member_count group members."; + my $primary_count = 0; + my %versions; + foreach my $m (@group_members) { + my @mparts = split( /\t/, $m ); + my $host = $mparts[0] // ''; + my $port = $mparts[1] // ''; + my $state = $mparts[2] // ''; + my $role = $mparts[3] // ''; + my $ver = $mparts[4] // ''; + + if ( $state ne 'ONLINE' ) { + badprint +"Group member $host:$port state is $state (recommended: ONLINE)"; + push_recommendation( 'res', +"Group member $host:$port is not ONLINE (state: $state). Check member status and error log." + ); + } + if ( $role eq 'PRIMARY' ) { + $primary_count++; + } + $versions{$ver}++ if $ver; + } + + my $single_primary = + $myvar{'group_replication_single_primary_mode'} // 'ON'; + if ( $single_primary eq 'ON' || $single_primary eq '1' ) { + if ( $primary_count != 1 ) { + badprint +"Single-primary mode active, but found $primary_count primary member(s) (recommended: 1)"; + push_recommendation( 'res', +"Investigate role state: single-primary mode requires exactly 1 primary member." + ); + } + } + + my @unique_vers = keys %versions; + if ( scalar(@unique_vers) > 1 ) { + badprint + "Inconsistent MySQL versions across group members: " + . join( ", ", @unique_vers ); + push_recommendation( 'res', +"Standardize MySQL versions across all group members to ensure replication compatibility." + ); + } + } + + # 2. Flow Control & Certification Conflict Analytics + my $server_uuid = select_one('SELECT @@server_uuid') // ''; + if ($server_uuid) { + my $local_stats = select_one( +"SELECT CONCAT_WS('|', COUNT_TRANSACTIONS_IN_QUEUE, COUNT_TRANSACTIONS_REMOTE_IN_APPLIER_QUEUE, TRANSACTIONS_COMMITTED_ALL_MEMBERS, TRANSACTIONS_LOCAL_ROLLBACK) FROM performance_schema.replication_group_member_stats WHERE MEMBER_ID = '$server_uuid'" + ); + if ($local_stats) { + my ( $cert_queue, $applier_queue, $committed, $rollbacks ) + = split( /\|/, $local_stats ); + $cert_queue //= 0; + $applier_queue //= 0; + $committed //= 0; + $rollbacks //= 0; + + my $cert_thresh = + $myvar{ + 'group_replication_flow_control_certifier_threshold'} + // 25000; + my $applier_thresh = + $myvar{'group_replication_flow_control_applier_threshold'} + // 25000; + + if ( $cert_queue > $cert_thresh ) { + badprint +"Flow Control: Certification queue ($cert_queue) exceeds threshold ($cert_thresh)"; + push_recommendation( 'res', +"Increase group_replication_flow_control_period or tune certifier settings to handle high transaction load." + ); + } + if ( $applier_queue > $applier_thresh ) { + badprint +"Flow Control: Applier queue ($applier_queue) exceeds threshold ($applier_thresh)"; + push_recommendation( 'res', +"Scale replication parallel threads (current workers check) or check disk I/O on slower nodes." + ); + } + + my $total_tx = $committed + $rollbacks; + if ( $total_tx > 0 ) { + my $rollback_ratio = $rollbacks / $total_tx; + if ( $rollback_ratio > 0.05 ) { + badprint "Certification rollback ratio is " + . sprintf( "%.2f%%", $rollback_ratio * 100 ) + . " (too many optimistic lock conflicts)"; + push_recommendation( 'res', +"High certification rollback ratio detected. Optimize write concurrency or switch to Single-Primary mode." + ); + } + } + } + } + } + + # 3. Communication Cache & Quorum Integrity + my $cache_size = $myvar{'group_replication_message_cache_size'} + // 1073741824; + if ( $physical_memory > 0 ) { + my $cache_pct = $cache_size / $physical_memory; + if ( $cache_pct > 0.3 ) { + badprint "group_replication_message_cache_size is " + . hr_bytes($cache_size) . " (" + . sprintf( "%.1f%%", $cache_pct * 100 ) + . " of system memory)"; + push_recommendation( 'res', +"Reduce group_replication_message_cache_size to prevent OOM errors on this system." + ); + } + } + + my $unreachable_timeout = + $myvar{'group_replication_unreachable_majority_timeout'} // 0; + if ( $unreachable_timeout == 0 ) { + badprint +"group_replication_unreachable_majority_timeout is set to 0 (default: risk of cluster hang on partition)"; + push_recommendation( 'res', +"Configure group_replication_unreachable_majority_timeout to a positive value (e.g., 10s) to allow auto-eviction." + ); + } + } + + # 4. MySQL Router Connectivity (Experimental) + my $router_conn = select_one( +"SELECT COUNT(*) FROM information_schema.processlist WHERE USER LIKE '%router%' OR HOST LIKE '%router%'" + ); + my $router_conn_count = + ( defined $router_conn && $router_conn =~ /^\d+$/ ) ? $router_conn : 0; + if ( $router_conn_count > 0 ) { + goodprint +"MySQL Router connections active: found $router_conn_count connection(s) routed to this instance."; + } } sub check_security_2_0 { @@ -1593,8 +1768,199 @@ sub check_security_2_0 { } } +sub check_workload_traffic { + subheaderprint "Workload Analysis & Traffic Profiling"; + + # 1. Workload Characterization (Read-Heavy vs Write-Heavy vs Mixed) + my $com_select = $mystat{'Com_select'} // 0; + my $com_insert = $mystat{'Com_insert'} // 0; + my $com_update = $mystat{'Com_update'} // 0; + my $com_delete = $mystat{'Com_delete'} // 0; + my $com_replace = $mystat{'Com_replace'} // 0; + + my $total_reads = $com_select; + my $total_writes = $com_insert + $com_update + $com_delete + $com_replace; + my $total_ops = $total_reads + $total_writes; + + if ( $total_ops > 0 ) { + my $read_ratio = $total_reads / $total_ops; + my $char_str = "Mixed"; + if ( $read_ratio > 0.80 ) { + $char_str = + "Read-Heavy (" + . sprintf( "%.1f%%", $read_ratio * 100 ) + . " reads)"; + goodprint "Workload Characterization: $char_str"; + } + elsif ( $read_ratio < 0.20 ) { + $char_str = + "Write-Heavy (" + . sprintf( "%.1f%%", ( 1 - $read_ratio ) * 100 ) + . " writes)"; + badprint "Workload Characterization: $char_str"; + push @generalrec, +"Write-heavy workload detected. Optimize redo logs size, transaction log flushing, and doublewrite buffer."; + } + else { + $char_str = + "Mixed (" + . sprintf( "%.1f%%", $read_ratio * 100 ) + . " reads, " + . sprintf( "%.1f%%", ( 1 - $read_ratio ) * 100 ) + . " writes)"; + goodprint "Workload Characterization: $char_str"; + } + } + else { + infoprint + "Workload Characterization: No query counters traffic recorded yet."; + } + + # 2. Wait Event Fingerprinting (Performance Schema Waits analysis) + if ( defined $myvar{'performance_schema'} + && $myvar{'performance_schema'} eq 'ON' ) + { + my @wait_events = select_array( +"SELECT EVENT_NAME, SUM_TIMER_WAIT FROM performance_schema.events_waits_summary_global_by_event_name WHERE SUM_TIMER_WAIT > 0 AND EVENT_NAME NOT LIKE 'idle%' AND EVENT_NAME NOT LIKE 'wait/io/table/%' ORDER BY SUM_TIMER_WAIT DESC LIMIT 5" + ); + if ( scalar(@wait_events) > 0 ) { + infoprint "Top Wait Events (excluding idle/table I/O):"; + my $disk_waits = 0; + my $lock_waits = 0; + my $net_waits = 0; + foreach my $we (@wait_events) { + my ( $name, $wait ) = split( /\t/, $we ); + $name //= ''; + $wait //= 0; + infoprint " - $name: " + . sprintf( "%.2fs", $wait / 1000000000000 ); + + if ( $name =~ m{^wait/io/file/} ) { $disk_waits += $wait; } + elsif ( $name =~ m{^wait/synch/} ) { $lock_waits += $wait; } + elsif ( $name =~ m{^wait/io/socket/} ) { $net_waits += $wait; } + } + + my $total_wait = $disk_waits + $lock_waits + $net_waits; + if ( $total_wait > 0 ) { + if ( $disk_waits / $total_wait > 0.5 ) { + badprint + "Primary DB Bottleneck detected: Disk/File I/O waits"; + push @generalrec, +"Primary database bottleneck is Disk I/O. Check storage performance, swap usage, or increase buffer pool."; + } + elsif ( $lock_waits / $total_wait > 0.5 ) { + badprint +"Primary DB Bottleneck detected: Synchronization Locks/Mutex waits"; + push @generalrec, +"Primary database bottleneck is Lock contention. Optimize slow transactions and reduce lock wait timeout."; + } + elsif ( $net_waits / $total_wait > 0.5 ) { + badprint + "Primary DB Bottleneck detected: Network Sockets waits"; + push @generalrec, +"Primary database bottleneck is Network/Socket I/O. Check client connections latency and network bandwidth."; + } + } + } + } + +# 3. Table Churn & Fragmentation Advisor (Combining PFS write metrics and fragmentation stats) + if ( defined $myvar{'performance_schema'} + && $myvar{'performance_schema'} eq 'ON' ) + { + my @churn_tables = select_array( +"SELECT OBJECT_SCHEMA, OBJECT_NAME, COUNT_WRITE FROM performance_schema.table_io_waits_summary_by_table WHERE OBJECT_SCHEMA NOT IN ('mysql', 'performance_schema', 'information_schema', 'sys') AND COUNT_WRITE > 1000 ORDER BY COUNT_WRITE DESC LIMIT 5" + ); + if ( scalar(@churn_tables) > 0 ) { + my %churn_map; + foreach my $ct (@churn_tables) { + my ( $schema, $table, $writes ) = split( /\t/, $ct ); + $churn_map{"$schema.$table"} = $writes if $schema && $table; + } + + my $matched_churn_frag = 0; + if ( defined $result{'Tables'}{'Fragmented tables'} ) { + foreach + my $table_line ( @{ $result{'Tables'}{'Fragmented tables'} } ) + { + my ( $table_schema, $table_name, $engine, $data_free ) = + split /\t/msx, $table_line; + if ( exists $churn_map{"$table_schema.$table_name"} ) { + $matched_churn_frag++; + badprint +"High-churn table `$table_schema`.`$table_name` is fragmented (free: " + . hr_bytes($data_free) . ")"; + push @generalrec, +"Defragment high-churn table `$table_schema`.`$table_name` to optimize DML space reclamation."; + } + } + } + if ( $matched_churn_frag > 0 ) { + infoprint +"Identified $matched_churn_frag high-churn fragmented tables."; + } + } + } + + # 4. Auto-Increment Exhaustion Audit + my @auto_inc_cols = select_array( +"SELECT t.TABLE_SCHEMA, t.TABLE_NAME, c.COLUMN_NAME, c.DATA_TYPE, t.AUTO_INCREMENT FROM information_schema.tables t JOIN information_schema.columns c ON t.table_schema = c.table_schema AND t.table_name = c.table_name WHERE c.extra = 'auto_increment' AND t.auto_increment IS NOT NULL AND t.table_schema NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys')" + ); + + if ( scalar(@auto_inc_cols) > 0 ) { + foreach my $col_info (@auto_inc_cols) { + my ( $schema, $table, $col, $type, $curr_val ) = + split( /\t/, $col_info ); + $type = lc( $type // '' ); + $curr_val //= 0; + + my $max_val = 0; + my $col_type = select_one( +"SELECT COLUMN_TYPE FROM information_schema.columns WHERE TABLE_SCHEMA = '$schema' AND TABLE_NAME = '$table' AND COLUMN_NAME = '$col'" + ) // ''; + my $is_unsigned = ( $col_type =~ /unsigned/i ) ? 1 : 0; + + if ( $type eq 'tinyint' ) { + $max_val = $is_unsigned ? 255 : 127; + } + elsif ( $type eq 'smallint' ) { + $max_val = $is_unsigned ? 65535 : 32767; + } + elsif ( $type eq 'mediumint' ) { + $max_val = $is_unsigned ? 16777215 : 8388607; + } + elsif ( $type eq 'int' || $type eq 'integer' ) { + $max_val = $is_unsigned ? 4294967295 : 2147483647; + } + elsif ( $type eq 'bigint' ) { + $max_val = + $is_unsigned ? 18446744073709551615 : 9223372036854775807; + } + + if ( $max_val > 0 ) { + my $ratio = $curr_val / $max_val; + if ( $ratio > 0.80 ) { + badprint +"Auto-increment column `$schema`.`$table`.`$col` ($type) is near exhaustion: current value $curr_val (" + . sprintf( "%.1f%%", $ratio * 100 ) + . " of max $max_val)"; + push @generalrec, +"Danger of auto-increment overflow on `$schema`.`$table`.`$col`. Upgrade type to BIGINT or reset sequence."; + } + elsif ( $ratio > 0.50 ) { + infoprint +"Auto-increment column `$schema`.`$table`.`$col` ($type) has reached " + . sprintf( "%.1f%%", $ratio * 100 ) + . " capacity."; + } + } + } + } +} + sub generate_auto_fix_snippets { - return if $opt{'silent'} || $opt{'json'} || $opt{'yaml'}; + return + if $opt{'silent'} || $opt{'json'} || $opt{'yaml'} || $opt{'agent-json'}; subheaderprint "Guided Auto-Fix Snippets"; if ( @adjvars > 0 ) { @@ -2332,7 +2698,11 @@ sub get_http_cli { # Checks for updates to MySQLTuner sub validate_tuner_version { if ( $opt{'checkversion'} eq 0 ) { - print "\n" unless ( $opt{'silent'} or $opt{'json'} or $opt{'yaml'} ); + print "\n" + unless ( $opt{'silent'} + or $opt{'json'} + or $opt{'yaml'} + or $opt{'agent-json'} ); infoprint "Skipped version check for MySQLTuner script"; return; } @@ -2394,7 +2764,11 @@ sub validate_tuner_version { sub update_tuner_version { if ( $opt{'updateversion'} eq 0 ) { badprint "Skipped version update for MySQLTuner script"; - print "\n" unless ( $opt{'silent'} or $opt{'json'} or $opt{'yaml'} ); + print "\n" + unless ( $opt{'silent'} + or $opt{'json'} + or $opt{'yaml'} + or $opt{'agent-json'} ); return; } @@ -2934,10 +3308,11 @@ sub mysql_setup { # It's a DirectAdmin box, use the available credentials my $mysqluser = execute_system_command( - "cat /usr/local/directadmin/conf/mysql.conf | egrep '^user=.*'"); + "cat /usr/local/directadmin/conf/mysql.conf | grep -E '^user=.*'"); my $mysqlpass = execute_system_command( - "cat /usr/local/directadmin/conf/mysql.conf | egrep '^passwd=.*'"); + "cat /usr/local/directadmin/conf/mysql.conf | grep -E '^passwd=.*'" + ); $mysqluser =~ s/user=//; $mysqluser =~ s/[\r\n]//; @@ -3298,7 +3673,7 @@ sub write_manifest_files { } my $json_content = - "{\n \"version\": \"" . ( $tunerversion // '2.9.0' ) . "\",\n"; + "{\n \"version\": \"" . ( $tunerversion // '2.9.1' ) . "\",\n"; $json_content .= " \"exported_at\": \"" . scalar( gmtime() ) . " UTC\",\n"; $json_content .= " \"total_files\": $total_files,\n"; $json_content .= " \"total_size_bytes\": $total_size,\n"; @@ -3314,7 +3689,7 @@ sub write_manifest_files { my $meta_content = "MySQLTuner Offline Diagnostic Snapshot Metadata\n"; $meta_content .= "================================================\n"; - $meta_content .= "Version: " . ( $tunerversion // '2.9.0' ) . "\n"; + $meta_content .= "Version: " . ( $tunerversion // '2.9.1' ) . "\n"; $meta_content .= "Exported At: " . scalar( gmtime() ) . " UTC\n"; $meta_content .= "Host: " . ( $myvar{'hostname'} // 'unknown' ) . "\n"; $meta_content .= @@ -4064,6 +4439,43 @@ sub log_file_recommendations { } subheaderprint "Log file Recommendations"; + + # Log verbosity audit + my $is_mariadb = ( + ( defined $myvar{'version'} && $myvar{'version'} =~ /MariaDB/i ) + or ( defined $myvar{'version_comment'} + && $myvar{'version_comment'} =~ /MariaDB/i ) + ); + my $is_mysql = !$is_mariadb; + + if ( $is_mysql && mysql_version_ge( 8, 0 ) ) { + if ( defined $myvar{'log_error_verbosity'} ) { + if ( $myvar{'log_error_verbosity'} < 2 ) { + badprint +"log_error_verbosity is set to $myvar{'log_error_verbosity'} (should be >= 2)"; + push @generalrec, +"Set log_error_verbosity = 2 or 3 to capture warning events in the error log"; + } + else { + goodprint + "log_error_verbosity is set to $myvar{'log_error_verbosity'}"; + } + } + } + else { + if ( defined $myvar{'log_warnings'} ) { + if ( $myvar{'log_warnings'} < 2 ) { + badprint +"log_warnings is set to $myvar{'log_warnings'} (should be >= 2)"; + push @generalrec, +"Set log_warnings = 2 or higher to capture warnings in the error log"; + } + else { + goodprint "log_warnings is set to $myvar{'log_warnings'}"; + } + } + } + if ( $has_pfs_error_log && !$opt{'server-log'} ) { goodprint "Performance Schema error_log table detected"; my $pfs_count = @@ -4154,6 +4566,10 @@ sub log_file_recommendations { my $nbErrLog = 0; my @lastShutdowns; my @lastStarts; + my $oom_detected = 0; + my $semaphore_detected = 0; + my $file_limit_detected = 0; + my $corruption_detected = 0; while ( my $logLi = <$fh> ) { chomp $logLi; @@ -4164,6 +4580,26 @@ sub log_file_recommendations { push @lastShutdowns, $logLi if $logLi =~ /Shutdown complete/ and $logLi !~ /Innodb/i; push @lastStarts, $logLi if $logLi =~ /ready for connections/; + + if ( $logLi =~ + /(out of memory|OOM-killer|killed process|mysqld killed)/i ) + { + $oom_detected++; + } + if ( $logLi =~ /(semaphore wait|long semaphore wait)/i ) { + $semaphore_detected++; + } + if ( $logLi =~ + /(too many open files|Error in accept: Too many open files)/i ) + { + $file_limit_detected++; + } + if ( $logLi =~ +/(is marked as crashed|checksum mismatch|corrupted page|Page checksum)/i + ) + { + $corruption_detected++; + } } close $fh; @@ -4182,6 +4618,54 @@ sub log_file_recommendations { goodprint "$myvar{'log_error'} doesn't contain any error."; } + if ( $oom_detected > 0 ) { + badprint +"Database error log contains $oom_detected Out-Of-Memory (OOM) patterns."; + push @generalrec, +"Verify system RAM allocations and consider reducing innodb_buffer_pool_size to avoid OOM kills"; + } + if ( $semaphore_detected > 0 ) { + badprint +"InnoDB long semaphore waits detected $semaphore_detected time(s) in the error log."; + push @generalrec, +"Audit storage I/O capacity or check system-level contention as InnoDB experienced semaphore waits"; + } + if ( $file_limit_detected > 0 ) { + badprint +"Error log contains 'Too many open files' warnings $file_limit_detected time(s)."; + push @generalrec, +"Increase open_files_limit or raise OS-level ulimits for file descriptors (nofile)"; + } + if ( $corruption_detected > 0 ) { + badprint +"Database corruption patterns detected $corruption_detected time(s) in the error log."; + push @generalrec, +"Run CHECK TABLE / REPAIR TABLE or restore from backup as corruption warnings were detected"; + } + + # Correlation Engine (Experimental) + if ( $semaphore_detected > 0 || $oom_detected > 0 ) { + my @load = get_load_average(); + my $cores = logical_cpu_cores(); + my $is_high_load = ( @load && $load[0] > $cores ) ? 1 : 0; + my $is_high_lock_waits = + ( ( $mystat{'Innodb_row_lock_waits'} // 0 ) > 10 + || ( $mystat{'Table_locks_waited'} // 0 ) > 10 ) ? 1 : 0; + + if ( $is_high_load || $is_high_lock_waits ) { + infoprint +"Experimental Correlation: Log anomalies (semaphore/OOM) correlate with active system pressure:"; + infoprint " - System Load: " + . ( $is_high_load ? "HIGH (@load)" : "Normal (@load)" ); + infoprint " - Lock Contention: " + . ( + $is_high_lock_waits + ? "HIGH (Innodb_row_lock_waits=$mystat{'Innodb_row_lock_waits'})" + : "Normal" + ); + } + } + infoprint scalar @lastStarts . " start(s) detected in $myvar{'log_error'}"; my $nStart = 0; my $nEnd = 10; @@ -5320,7 +5804,7 @@ sub check_auth_plugins { # Extract user and host for CSV my ( $user, $host ) = ( '', '' ); - if ( $user_host =~ /'([^']*)'@'([^']*)'/ ) { + if ( $user_host =~ /'([^']*)'\@'([^']*)'/ ) { $user = $1; $host = $2; } @@ -7278,7 +7762,8 @@ sub mysql_stats { my $slow_query_log_active = $myvar{'slow_query_log'} // $myvar{'log_slow_queries'}; if ( defined($slow_query_log_active) ) { - if ( $slow_query_log_active eq "OFF" ) { + if ( $slow_query_log_active eq "OFF" || $slow_query_log_active eq "0" ) + { push( @generalrec, "Enable the slow query log to troubleshoot bad queries" ); } @@ -9848,6 +10333,19 @@ sub get_wsrep_option { return $memValue; } +sub parse_size_bytes { + my $str = shift // ''; + $str =~ s/\s+//g; + if ( $str =~ /^(\d+(?:\.\d+)?)([KMG])$/i ) { + my ( $val, $unit ) = ( $1, uc($2) ); + return $val * 1024 if $unit eq 'K'; + return $val * 1024 * 1024 if $unit eq 'M'; + return $val * 1024 * 1024 * 1024 if $unit eq 'G'; + } + return $str if $str =~ /^\d+$/; + return 0; +} + # REcommendations for Tables sub mysql_table_structures { return 0 unless ( $opt{structstat} > 0 ); @@ -10892,6 +11390,107 @@ sub mariadb_galera { } } + # --- PHASE 9: ADVANCED GALERA & PXC DIAGNOSTICS --- + # 1. Streaming Replication Monitor + my $stream_writes = $mystat{'wsrep_streaming_log_writes'} // 0; + my $stream_reads = $mystat{'wsrep_streaming_log_reads'} // 0; + if ( $stream_writes > 0 ) { + badprint +"Galera is using streaming replication: $stream_writes write(s), $stream_reads read(s)"; + push @generalrec, +"Streaming replication active. Review wsrep_trx_fragment_size and ensure storage can handle the fragment log I/O overhead."; + } + + # 2. Gcache Sizing Optimization + my $gcache_size_str = get_wsrep_option('gcache.size') // ''; + if ($gcache_size_str) { + my $gcache_bytes = parse_size_bytes($gcache_size_str); + if ( $gcache_bytes > 0 ) { + my $min_gcache = 128 * 1024 * 1024; + my $five_pct_ram = + $physical_memory ? ( $physical_memory * 0.05 ) : 0; + my $target_gcache = + ( $five_pct_ram > $min_gcache ) ? $five_pct_ram : $min_gcache; + if ( $gcache_bytes < $target_gcache ) { + badprint + "gcache.size is too small ($gcache_size_str, recommended: >= " + . hr_bytes($target_gcache) . ")"; + push @generalrec, +"Increase gcache.size in wsrep_provider_options to maximize IST success and avoid full SST on node re-joins."; + } + } + } + + # 3. Certification Conflict & Abort Analysis + my $bf_aborts = $mystat{'wsrep_local_bf_aborts'} // 0; + my $cert_failures = $mystat{'wsrep_local_cert_failures'} // 0; + if ( $bf_aborts > 50 || $cert_failures > 50 ) { + badprint +"High cluster certification conflicts: $bf_aborts brute-force abort(s), $cert_failures certification failure(s)"; + push @generalrec, +"High certification conflicts. Optimize write concurrency, index hotspot tables, or review application transactions size."; + } + + # 4. Advanced Flow Control Observability + my $fc_sent = $mystat{'wsrep_flow_control_sent'} // 0; + if ( $fc_sent > 0 ) { + badprint +"Flow Control: Node triggered flow control $fc_sent times (pause sender culprit)"; + push @generalrec, +"Node is triggering flow control pause events. Check disk write latency, CPU load, or swap space."; + } + my $fc_paused = $mystat{'wsrep_flow_control_paused'} // 0; + if ( $fc_paused > 0.05 ) { + badprint "Flow Control: Node replication was paused " + . sprintf( "%.2f%%", $fc_paused * 100 ) + . " of the time (throttling)"; + push @generalrec, +"Node is heavily throttled by flow control. Check slower nodes in the cluster triggering flow control."; + } + + # 5. Group Communication Latency & Jitter + my $repl_latency = $mystat{'wsrep_evs_repl_latency'} // ''; + if ( $repl_latency + && $repl_latency =~ m{^([\d.]+)/([\d.]+)/([\d.]+)/([\d.]+)$} ) + { + my ( $min, $avg, $max, $stddev ) = ( $1, $2, $3, $4 ); + if ( $avg > 0.1 ) { + badprint "High average replication latency: " + . sprintf( "%.2fms", $avg * 1000 ); + push @generalrec, +"High average replication latency. Optimize inter-node network latency."; + } + if ( $stddev > 0.05 ) { + badprint "High network replication jitter (stddev: " + . sprintf( "%.2fms", $stddev * 1000 ) . ")"; + push @generalrec, +"High replication jitter. Review inter-node network stability and WAN link quality."; + } + } + + # 6. Applier Concurrency Tuning + my $deps_dist = $mystat{'wsrep_cert_deps_distance'} // 0; + if ( $deps_dist > 4 && $wsrep_threads_value == 1 ) { + infoprint "Average write-set dependency distance is " + . sprintf( "%.2f", $deps_dist ) + . " (threads: 1)"; + push @generalrec, +"Consider increasing $wsrep_threads_var_name to leverage parallel certification concurrency."; + } + + # 7. PXC Strict Mode Verification + my $is_percona = ( ( $myvar{'version_comment'} // '' ) =~ /Percona/i + || ( $myvar{'version'} // '' ) =~ /Percona/i ); + if ( $is_percona && defined $myvar{'pxc_strict_mode'} ) { + my $pxc_mode = $myvar{'pxc_strict_mode'}; + if ( $pxc_mode ne 'ENFORCING' && $pxc_mode ne 'MASTER' ) { + badprint + "pxc_strict_mode is set to $pxc_mode (recommended: ENFORCING)"; + push @generalrec, +"Set pxc_strict_mode = ENFORCING to prevent unsafe operations on Percona XtraDB Cluster."; + } + } + #debugprint Dumper get_wsrep_options() if $opt{debug}; } @@ -11650,6 +12249,200 @@ sub mysql_innodb { "No InnoDB tables with > 50,000 rows found to calculate index/data ratios."; } + # --- PHASE 6: DEEP ENGINE TUNING & SAFEGUARDING --- + + # Task 1: I/O Pressure & Flushing Advisor + if ( defined $mystat{'Innodb_buffer_pool_wait_free'} + && $mystat{'Innodb_buffer_pool_wait_free'} > 0 ) + { + my $io_cap = $myvar{'innodb_io_capacity'} // 0; + my $io_cap_max = $myvar{'innodb_io_capacity_max'} // 0; + badprint +"InnoDB is experiencing I/O pressure (Innodb_buffer_pool_wait_free = $mystat{'Innodb_buffer_pool_wait_free'})"; + push( @generalrec, +"Increase innodb_io_capacity (current: $io_cap) or innodb_io_capacity_max (current: $io_cap_max) to alleviate flushing/free buffer page waits." + ); + } + + # Task 2: Read-Ahead Efficiency Audit + if ( defined $mystat{'Innodb_buffer_pool_read_ahead'} + && $mystat{'Innodb_buffer_pool_read_ahead'} > 0 ) + { + my $read_ahead = $mystat{'Innodb_buffer_pool_read_ahead'}; + my $evicted = $mystat{'Innodb_buffer_pool_read_ahead_evicted'} // 0; + my $eviction_ratio = ( $evicted / $read_ahead ) * 100; + if ( $eviction_ratio > 20 ) { + badprint + sprintf( +"InnoDB read-ahead efficiency is low: %.2f%% of read-ahead pages were evicted without being accessed", + $eviction_ratio ); + my $threshold = $myvar{'innodb_read_ahead_threshold'} // 56; + push( @generalrec, +"Decrease innodb_read_ahead_threshold (current: $threshold) or disable innodb_random_read_ahead to reduce read-ahead page eviction." + ); + } + } + + # Task 3: Purge Lag Prevention + if ( defined $mystat{'Innodb_history_list_length'} + && $mystat{'Innodb_history_list_length'} > 100000 ) + { + my $purge_threads = $myvar{'innodb_purge_threads'} // 0; + badprint +"InnoDB history list length is high ($mystat{'Innodb_history_list_length'}) - purge process may lag"; + push( @generalrec, +"Increase innodb_purge_threads (current: $purge_threads, up to 32) and audit long-running transactions to prevent MVCC purge lag." + ); + } + + # Task 4: Change Buffer & Adaptive Hash Index Optimization + my $rows_read = $mystat{'Innodb_rows_read'} // 0; + my $rows_write = + ( $mystat{'Innodb_rows_inserted'} // 0 ) + + ( $mystat{'Innodb_rows_updated'} // 0 ) + + ( $mystat{'Innodb_rows_deleted'} // 0 ); + if ( $rows_write > 0 && ( $rows_read / $rows_write ) > 100 ) { + if ( defined $myvar{'innodb_change_buffering'} + && $myvar{'innodb_change_buffering'} ne 'none' ) + { + badprint +"Workload is highly read-intensive, but innodb_change_buffering is enabled"; + push( @generalrec, +"Consider setting innodb_change_buffering = none to save buffer pool memory on read-only/read-heavy workloads." + ); + } + } + if ( ( $myvar{'innodb_adaptive_hash_index'} // 'OFF' ) eq 'ON' + && logical_cpu_cores() > 16 ) + { + infoprint +"InnoDB Adaptive Hash Index is enabled on a high core count system ($myvar{'innodb_adaptive_hash_index'})"; + push( @generalrec, +"Monitor Adaptive Hash Index (AHI) lock contention; consider setting innodb_adaptive_hash_index = OFF if contention is high." + ); + } + + # Task 5: Modern Storage Alignment (doublewrite pages, fdatasync, flush method) + my $is_mysql = !( + ( defined $myvar{'version'} && $myvar{'version'} =~ /MariaDB/i ) + or ( defined $myvar{'version_comment'} + && $myvar{'version_comment'} =~ /MariaDB/i ) + ); + if ( $is_mysql + && mysql_version_ge( 8, 4 ) + && defined $myvar{'innodb_doublewrite_pages'} + && $myvar{'innodb_doublewrite_pages'} != 128 ) + { + badprint +"innodb_doublewrite_pages is not aligned with modern SSD/NVMe storage ($myvar{'innodb_doublewrite_pages'})"; + push( @generalrec, +"Set innodb_doublewrite_pages = 128 for optimal doublewrite buffer throughput on modern SSDs." + ); + } + if ( defined $myvar{'innodb_use_fdatasync'} + && $myvar{'innodb_use_fdatasync'} eq 'OFF' ) + { + my $flush_method = $myvar{'innodb_flush_method'} // ''; + if ( $flush_method ne 'O_DIRECT' + && $flush_method ne 'O_DIRECT_NO_FSYNC' ) + { + infoprint +"innodb_use_fdatasync is disabled ($myvar{'innodb_use_fdatasync'})"; + push( @generalrec, +"Consider enabling innodb_use_fdatasync = ON to reduce fsync system call overhead on Linux if not using O_DIRECT." + ); + } + } + + # Task 6: NUMA-Aware Memory Allocation + if ( !is_docker() && !is_remote() ) { + my $has_numa = 0; + if ( -d '/sys/devices/system/node' ) { + my @nodes = glob('/sys/devices/system/node/node[0-9]*'); + if ( @nodes > 1 ) { + $has_numa = 1; + } + } + if ($has_numa) { + if ( defined $myvar{'innodb_numa_interleave'} + && $myvar{'innodb_numa_interleave'} eq 'OFF' ) + { + my $total_mem = $physical_memory // 0; + if ( $total_mem > 34359738368 ) { + badprint +"NUMA architecture detected but innodb_numa_interleave is disabled"; + push( @generalrec, +"Enable innodb_numa_interleave = ON to balance buffer pool memory allocation across NUMA nodes." + ); + } + } + } + } + + # Task 7: MariaDB Temp & Undo Lifecycle Manager + my $is_mariadb = !$is_mysql; + if ( $is_mariadb && mysql_version_ge( 11, 4 ) ) { + if ( defined $myvar{'innodb_truncate_temporary_tablespace_now'} ) { + goodprint + "MariaDB online temporary tablespace truncation is supported"; + } + } + + # Task 8: Deadlock & Contention Analytics via Performance Schema + if ( ( $myvar{'performance_schema'} // 'OFF' ) eq 'ON' ) { + if ( $is_mysql && mysql_version_ge( 8, 0 ) ) { + my $has_events_errors = select_one( +"SELECT 1 FROM information_schema.tables WHERE table_schema='performance_schema' AND table_name='events_errors_summary_global_by_error' LIMIT 1" + ); + if ($has_events_errors) { + my @err_res = select_array( +"SELECT SUM(SUM_ERROR_RAISED) FROM performance_schema.events_errors_summary_global_by_error WHERE ERROR_NUMBER = 1213" + ); + if ( @err_res && defined $err_res[0] && $err_res[0] > 0 ) { + badprint +"InnoDB experienced $err_res[0] lock deadlocks (ER_LOCK_DEADLOCK)"; + push( @generalrec, +"Optimize application queries, transaction lengths, and index coverage to reduce lock deadlocks." + ); + } + } + } + } + + # Task 9: InnoDB Lock Monitoring & Deadlock Logging Audit + if ( defined $myvar{'innodb_print_all_deadlocks'} ) { + if ( $myvar{'innodb_print_all_deadlocks'} eq 'OFF' ) { + badprint +"InnoDB deadlock logging (innodb_print_all_deadlocks) is disabled"; + push( @generalrec, +"Enable innodb_print_all_deadlocks = ON to capture detailed transaction information in the error log when deadlocks occur." + ); + } + else { + goodprint + "InnoDB deadlock logging (innodb_print_all_deadlocks) is enabled"; + } + } + if ( defined $myvar{'innodb_status_output'} + && $myvar{'innodb_status_output'} eq 'OFF' ) + { + infoprint + "InnoDB periodic status output (innodb_status_output) is disabled"; + } + if ( defined $myvar{'innodb_status_output_locks'} ) { + if ( $myvar{'innodb_status_output_locks'} eq 'OFF' ) { + infoprint +"InnoDB lock monitor output (innodb_status_output_locks) is disabled"; + push( @generalrec, +"Consider enabling innodb_status_output_locks = ON during active lock contention troubleshooting to print lock details in SHOW ENGINE INNODB STATUS." + ); + } + else { + goodprint +"InnoDB lock monitor output (innodb_status_output_locks) is enabled"; + } + } + $result{'Calculations'} = {%mycalc}; } @@ -12314,7 +13107,11 @@ sub mysql_databases { $result{'Databases'}{'All databases'}{'Index Pct'} = percentage( $totaldbinfo[2], $totaldbinfo[3] ) . "%"; $result{'Databases'}{'All databases'}{'Total Size'} = $totaldbinfo[3]; - print "\n" unless ( $opt{'silent'} or $opt{'json'} or $opt{'yaml'} ); + print "\n" + unless ( $opt{'silent'} + or $opt{'json'} + or $opt{'yaml'} + or $opt{'agent-json'} ); my $nbViews = 0; my $nbTables = 0; @@ -13112,6 +13909,9 @@ sub _serialize_to_json { return 'null'; } my $ref = ref($data); + if ( $ref eq 'SCALAR' ) { + return $$data ? 'true' : 'false'; + } if ( !$ref ) { if ( $data =~ /^-?(?:[0-9]+(?:\.[0-9]+)?)$/ ) { return $data; @@ -13140,6 +13940,161 @@ sub _serialize_to_json { } sub dump_result { + if ( $opt{'agent-json'} ) { + my @findings; + foreach my $adj (@adjvars) { + my $var_name = ''; + my $raw_target = ''; + if ( $adj =~ /^([a-zA-Z0-9_-]+)\s*[=(<>]+\s*(.*)$/ ) { + $var_name = $1; + $raw_target = $2; + } + else { + next; + } + + my $target_val = $raw_target; + $target_val =~ s/[()>=<=]//g; + $target_val =~ s/^\s+|\s+$//g; + + my $current_val = $myvar{$var_name} // ''; + + my $id = "${var_name}_adjust"; + my $topic = "Performance"; + my $description = "Variable $var_name is misconfigured."; + my $impact_score = 5; + my $risk_level = "Low"; + my $risk_description = +"Review the variable description and potential memory impact before applying."; + my $requires_restart = 0; + my $expected_outcome = + "Optimizes the $var_name configuration parameters."; + + if ( $var_name eq 'innodb_buffer_pool_size' ) { + $id = "innodb_buffer_pool_size_adjust"; + $topic = "Performance"; + $description = +"InnoDB buffer pool size is under-allocated for the current workload."; + $impact_score = 9; + $risk_level = "Medium"; + $risk_description = +"Increases memory consumption. Ensure sufficient OS-free RAM to prevent OOM swapping."; + $requires_restart = 0; + $expected_outcome = + "Reduces disk I/O and increases query cache read hits."; + } + elsif ( $var_name eq 'performance_schema' ) { + $id = "performance_schema_enable"; + $topic = "Performance"; + $description = "Performance Schema is disabled."; + $impact_score = 8; + $risk_level = "Medium"; + $risk_description = +"Enabling Performance Schema requires memory allocation and a database restart."; + $requires_restart = 1; + $expected_outcome = +"Enables query performance diagnostics, lock profiling, and telemetry."; + } + elsif ( $var_name eq 'skip-name-resolve' ) { + $id = "skip_name_resolve_enable"; + $topic = "Security"; + $description = +"Database is performing DNS resolution for client connections."; + $impact_score = 7; + $risk_level = "Medium"; + $risk_description = +"Ensure all accounts are configured to connect using IP addresses or wildcard domains before enabling."; + $requires_restart = 1; + $expected_outcome = +"Eliminates connection latency overhead caused by DNS name lookups."; + } + elsif ( $var_name eq 'innodb_file_per_table' ) { + $id = "innodb_file_per_table_enable"; + $topic = "Performance"; + $description = + "InnoDB is storing tables in the system tablespace."; + $impact_score = 8; + $risk_level = "Low"; + $risk_description = +"Only affects newly created tables. Reclaiming space from existing tables requires OPTIMIZE TABLE."; + $requires_restart = 0; + $expected_outcome = +"Allows individual tablespaces to be reclaimed upon dropping or truncating tables."; + } + elsif ( $var_name eq 'long_query_time' ) { + $id = "long_query_time_adjust"; + $topic = "Performance"; + $description = "Slow query log threshold is set too high."; + $impact_score = 6; + $risk_level = "Low"; + $risk_description = +"Might slightly increase disk write I/O if logging many queries."; + $requires_restart = 0; + $expected_outcome = + "Captures fine-grained slow queries for profiling."; + } + elsif ( $var_name eq 'query_cache_size' ) { + $id = "query_cache_size_disable"; + $topic = "Performance"; + $description = +"Query Cache is enabled on a high-concurrency database, causing mutex contention."; + $impact_score = 8; + $risk_level = "Low"; + $risk_description = +"Disables simple query result caching which could increase read latency on low-concurrency read-heavy tables."; + $requires_restart = 0; + $expected_outcome = + "Reduces query cache mutex contention and locking overhead."; + } + elsif ( $var_name eq 'max_connections' ) { + $id = "max_connections_adjust"; + $topic = "Reliability"; + $description = + "Maximum connections limit is set too high or too low."; + $impact_score = 7; + $risk_level = "Medium"; + $risk_description = +"High limits increase potential memory consumption under peak loads."; + $requires_restart = 0; + $expected_outcome = +"Prevents running out of connections or protects against OOM spikes."; + } + + my $stmt = "SET GLOBAL $var_name = $target_val;"; + my $rollback = "SET GLOBAL $var_name = " + . ( $current_val ne '' ? $current_val : $target_val ) . ";"; + + if ($requires_restart) { + $stmt = "$var_name = $target_val"; + $rollback = "$var_name = " + . ( $current_val ne '' ? $current_val : $target_val ); + } + + push @findings, + { + 'id' => $id, + 'topic' => $topic, + 'description' => $description, + 'impact_score' => int($impact_score), + 'risk_level' => $risk_level, + 'risk_description' => $risk_description, + 'requires_restart' => $requires_restart ? \1 : \0, + 'expected_outcome' => $expected_outcome, + 'action' => { + 'type' => $requires_restart ? 'Config' : 'SQL', + 'statement' => $stmt, + 'rollback_statement' => $rollback + } + }; + } + + my $payload = { 'findings' => \@findings }; + + my $json_str = _serialize_to_json($payload); + print $json_str . "\n"; + exit 0; + } + if ( $opt{'json'} && $opt{'yaml'} ) { print STDERR "ERROR: --json and --yaml are mutually exclusive\n"; return 1; @@ -13201,8 +14156,9 @@ sub dump_result { @generalrec; my @repl_recs = - grep { /(replica|sla[v]e|gtid|replication|binlog|relay)/i } - @generalrec; + grep { +/(replica|sla[v]e|gtid|replication|binlog|relay|flow control|group_replication|router|quorum)/i + } @generalrec; my $general_rec_html = join( "\n", @@ -13784,7 +14740,7 @@ sub dump_result { MySQLTuner Advanced Report - +