diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 5d9a2ab5d9..07dc171469 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -103,7 +103,6 @@ jobs:
with:
name: lint-python${{ matrix.python-versions }}
path: |
- .lint.txt
.lint.json
include-hidden-files: true
diff --git a/.gitignore b/.gitignore
index 648a2db892..bfe5173af3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,4 @@
.lint.json
-.lint.txt
.security.json
odbcconfig/odbcinst.ini
diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md
index fb4737052b..5cd75f42be 100644
--- a/doc/changes/unreleased.md
+++ b/doc/changes/unreleased.md
@@ -1,3 +1,7 @@
# Unreleased
## Summary
+
+## Refactoring
+
+* #800: Removed tbx security pretty-print, tbx lint pretty-print, and creation of .lint.txt, as superseded by Sonar and .lint.json usage
diff --git a/doc/user_guide/features/metrics/collecting_metrics.rst b/doc/user_guide/features/metrics/collecting_metrics.rst
index e6104c4af1..2545d37068 100644
--- a/doc/user_guide/features/metrics/collecting_metrics.rst
+++ b/doc/user_guide/features/metrics/collecting_metrics.rst
@@ -61,8 +61,8 @@ sessions collect the artifacts from various jobs:
| | (unit, integration ...) |
| | * Copies downloaded artifacts to their parent directory |
+--------------------------+----------------------------------------------------------+
-| ``artifacts:validate`` | * Verifies that the ``.lint.json``, ``.lint.txt``, |
-| | ``.security.json``, and ``.coverage`` are present |
+| ``artifacts:validate`` | * Verifies that the ``.lint.json``, ``.security.json``, |
+| | and ``.coverage`` are present |
| | * Checks that each file contains the expected attributes |
| | for that file type |
+--------------------------+----------------------------------------------------------+
diff --git a/exasol/toolbox/nox/_artifacts.py b/exasol/toolbox/nox/_artifacts.py
index e72324383e..19eab7d9da 100644
--- a/exasol/toolbox/nox/_artifacts.py
+++ b/exasol/toolbox/nox/_artifacts.py
@@ -1,6 +1,5 @@
import json
import os
-import re
import shutil
import sqlite3
import subprocess # nosec
@@ -17,10 +16,9 @@
COVERAGE_DB = ".coverage"
COVERAGE_XML = "ci-coverage.xml"
LINT_JSON = ".lint.json"
-LINT_TXT = ".lint.txt"
SECURITY_JSON = ".security.json"
-ALL_LINT_FILES = {COVERAGE_DB, LINT_JSON, LINT_TXT, SECURITY_JSON}
+ALL_LINT_FILES = {COVERAGE_DB, LINT_JSON, SECURITY_JSON}
COVERAGE_TABLES = {"coverage_schema", "meta", "file", "line_bits"}
LINT_JSON_ATTRIBUTES = {
"type",
@@ -48,7 +46,6 @@ def check_artifacts(session: Session) -> None:
sys.exit(1)
all_is_valid_checks = [
- _is_valid_lint_txt(Path(PROJECT_CONFIG.root_path, LINT_TXT)),
_is_valid_lint_json(Path(PROJECT_CONFIG.root_path, LINT_JSON)),
_is_valid_security_json(Path(PROJECT_CONFIG.root_path, SECURITY_JSON)),
_is_valid_coverage(Path(PROJECT_CONFIG.root_path, COVERAGE_DB)),
@@ -62,16 +59,6 @@ def _handle_validation_error(file: Path, message: str) -> bool:
return False
-def _is_valid_lint_txt(file: Path) -> bool:
- content = file.read_text()
- expr = re.compile(r"^Your code has been rated at (\d+.\d+)/.*", re.MULTILINE)
- matches = expr.search(content)
- if not matches:
- _handle_validation_error(file, "Could not find a rating")
- return False
- return True
-
-
def _is_valid_lint_json(file: Path) -> bool:
try:
content = file.read_text()
@@ -147,7 +134,6 @@ def copy_artifacts(session: Session) -> None:
artifact_dir,
artifact_dir.parent,
[
- f"lint{suffix}/{LINT_TXT}",
f"lint{suffix}/{LINT_JSON}",
f"security{suffix}/{SECURITY_JSON}",
],
diff --git a/exasol/toolbox/nox/_lint.py b/exasol/toolbox/nox/_lint.py
index a601fbd636..b4e5508cd6 100644
--- a/exasol/toolbox/nox/_lint.py
+++ b/exasol/toolbox/nox/_lint.py
@@ -17,12 +17,11 @@
def _pylint(session: Session, files: Iterable[str]) -> None:
json_file = PROJECT_CONFIG.root_path / ".lint.json"
- txt_file = PROJECT_CONFIG.root_path / ".lint.txt"
session.run(
"pylint",
"--output-format",
- f"colorized,json:{json_file},text:{txt_file}",
+ f"colorized,json:{json_file}",
*files,
)
diff --git a/exasol/toolbox/templates/github/workflows/checks.yml b/exasol/toolbox/templates/github/workflows/checks.yml
index 4b21172376..d81b8f7599 100644
--- a/exasol/toolbox/templates/github/workflows/checks.yml
+++ b/exasol/toolbox/templates/github/workflows/checks.yml
@@ -103,7 +103,6 @@ jobs:
with:
name: lint-python${{ matrix.python-versions }}
path: |
- .lint.txt
.lint.json
include-hidden-files: true
diff --git a/exasol/toolbox/tools/lint.py b/exasol/toolbox/tools/lint.py
deleted file mode 100644
index acde78cc3c..0000000000
--- a/exasol/toolbox/tools/lint.py
+++ /dev/null
@@ -1,78 +0,0 @@
-import json
-from collections.abc import Iterable
-from dataclasses import dataclass
-from inspect import cleandoc
-from pathlib import Path
-
-import typer
-
-CLI = typer.Typer()
-
-
-@dataclass(frozen=True)
-class Finding:
- type: str
- module: str
- obj: str
- line: int
- column: int
- endLine: int
- endColumn: int
- path: str
- symbol: str
- message: str
- message_id: str
-
-
-def lint_issue_from_json(data: str) -> Iterable[Finding]:
- issues = json.loads(data)
- for issue in issues:
- yield Finding(
- type=issue["type"],
- module=issue["module"],
- obj=issue["obj"],
- line=issue["line"],
- column=issue["column"],
- endLine=issue["endLine"],
- endColumn=issue["endColumn"],
- path=issue["path"],
- symbol=issue["symbol"],
- message=issue["message"],
- message_id=issue["message-id"],
- )
-
-
-def lint_issue_to_markdown(lint_issues: Iterable[Finding]) -> str:
- def _header() -> str:
- header = "# Static Code Analysis\n\n"
- header += "|File|line/
column|id|message|\n"
- header += "|---|:-:|:-:|---|\n"
- return header
-
- def _rows(findings: Iterable[Finding]) -> str:
- rows = ""
- for finding in findings:
- rows += f"|{finding.path}"
- rows += f"|line: {finding.line}/
column: {finding.column}"
- rows += f"|{finding.message_id}"
- rows += f"|{finding.message}|\n"
- return rows
-
- template = cleandoc("""
- {header}{rows}
- """)
- lint_issues = sorted(lint_issues, key=lambda i: (i.path, i.message_id, i.line))
- return template.format(header=_header(), rows=_rows(lint_issues))
-
-
-@CLI.command(name="pretty-print")
-def lint_json_to_markdown(
- path: Path = typer.Argument(default=Path(".lint.json"), help="path to lint.json"),
-) -> None:
- """converts the lint json to a Markdown table"""
- issues = lint_issue_from_json(path.read_text())
- print(lint_issue_to_markdown(issues))
-
-
-if __name__ == "__main__":
- CLI()
diff --git a/exasol/toolbox/tools/security.py b/exasol/toolbox/tools/security.py
index 4ddec6e2bf..9fc30161b4 100644
--- a/exasol/toolbox/tools/security.py
+++ b/exasol/toolbox/tools/security.py
@@ -17,7 +17,6 @@
from enum import Enum
from functools import partial
from inspect import cleandoc
-from pathlib import Path
import typer
@@ -168,61 +167,6 @@ def from_pip_audit(report: str) -> Iterable[Issue]:
)
-@dataclass(frozen=True)
-class SecurityIssue:
- file_name: str
- line: int
- column: int
- cwe: str
- test_id: str
- description: str
- references: tuple
-
-
-def from_json(report_str: str, prefix: Path) -> Iterable[SecurityIssue]:
- report = json.loads(report_str)
- issues = report.get("results", {})
- for issue in issues:
- references = []
- if issue["more_info"]:
- references.append(issue["more_info"])
- if issue.get("issue_cwe", {}).get("link", None):
- references.append(issue["issue_cwe"]["link"])
- yield SecurityIssue(
- file_name=issue["filename"].replace(str(prefix) + "/", ""),
- line=issue["line_number"],
- column=issue["col_offset"],
- cwe=str(issue["issue_cwe"].get("id", "")),
- test_id=issue["test_id"],
- description=issue["issue_text"],
- references=tuple(references),
- )
-
-
-def issues_to_markdown(issues: Iterable[SecurityIssue]) -> str:
- template = cleandoc("""
- {header}{rows}
- """)
-
- def _header():
- header = "# Security\n\n"
- header += "|File|line/
column|Cwe|Test ID|Details|\n"
- header += "|---|:-:|:-:|:-:|---|\n"
- return header
-
- def _row(issue):
- row = "|" + issue.file_name + "|"
- row += f"line: {issue.line}
column: {issue.column}|"
- row += issue.cwe + "|"
- row += issue.test_id + "|"
- for element in issue.references:
- row += element + " ,
"
- row = row[:-5] + "|"
- return row
-
- return template.format(header=_header(), rows="\n".join(_row(i) for i in issues))
-
-
def security_issue_title(issue: Issue) -> str:
return f"{issue.cve}: {issue.coordinates}"
@@ -384,23 +328,6 @@ def create(
stdout(format_jsonl(issue_url, issue))
-class PPrintFormats(str, Enum):
- markdown = "markdown"
-
-
-@CLI.command(name="pretty-print")
-def json_issue_to_markdown(
- json_file: typer.FileText = typer.Argument(
- mode="r", help="json file with issues to convert"
- ),
- path: Path = typer.Argument(default=Path("."), help="path to project root"),
-) -> None:
- content = json_file.read()
- issues = from_json(content, path.absolute())
- issues = sorted(issues, key=lambda i: (i.file_name, i.cwe, i.test_id))
- print(issues_to_markdown(issues))
-
-
def format_jsonl(issue_url: str, issue: Issue) -> str:
issue_json = asdict(issue)
issue_json["issue_url"] = issue_url.strip()
diff --git a/exasol/toolbox/tools/tbx.py b/exasol/toolbox/tools/tbx.py
index dd9cc96673..178759c663 100644
--- a/exasol/toolbox/tools/tbx.py
+++ b/exasol/toolbox/tools/tbx.py
@@ -2,14 +2,12 @@
from exasol.toolbox.tools import (
issue,
- lint,
security,
)
CLI = typer.Typer()
CLI.add_typer(security.CLI, name="security", help="Security related helpers")
CLI.add_typer(issue.CLI, name="issue", help="Manage issue templates")
-CLI.add_typer(lint.CLI, name="lint", help="linting related helpers")
if __name__ == "__main__":
CLI()
diff --git a/project-template/{{cookiecutter.repo_name}}/.gitignore b/project-template/{{cookiecutter.repo_name}}/.gitignore
index 5f3190fb40..13503bf0c3 100644
--- a/project-template/{{cookiecutter.repo_name}}/.gitignore
+++ b/project-template/{{cookiecutter.repo_name}}/.gitignore
@@ -148,6 +148,5 @@ itde/
# PTB
/.lint.json
-/.lint.txt
/.security.json
/.sonar
diff --git a/test/integration/tools/security_integration_test.py b/test/integration/tools/security_integration_test.py
index 29d4dc017b..6532c013ec 100644
--- a/test/integration/tools/security_integration_test.py
+++ b/test/integration/tools/security_integration_test.py
@@ -1,8 +1,6 @@
-import json
from unittest.mock import patch
from exasol.toolbox.tools.security import (
- CLI,
CVE_CLI,
Filter,
Format,
@@ -132,43 +130,3 @@ def test_works_as_expected_with_mocked_create_security_issue(
assert result.exit_code == 0
assert result.output.strip() == sample_maven_vulnerabilities.create_issues_json
-
-
-class TestJsonIssueToMarkdown:
- @staticmethod
- def test_with_filled_file(cli_runner, tmp_path):
- json_path = tmp_path / "test.json"
- json_path.write_text(json.dumps(JSON_RESULTS))
-
- result = cli_runner.invoke(CLI, ["pretty-print", str(json_path)])
-
- assert result.exit_code == 0
- assert result.output.strip() == (
- "# Security\n\n"
- "|File|line/
column|Cwe|Test ID|Details|\n"
- "|---|:-:|:-:|:-:|---|\n"
- "|exasol/toolbox/sphinx/multiversion/git.py|line: 160
column: "
- "12|22|B202|https://bandit.readthedocs.io/en/1.7.10/plugins/b202_tarfile_unsafe_members.html "
- ",
https://cwe.mitre.org/data/definitions/22.html |\n"
- "|exasol/toolbox/sphinx/multiversion/git.py|line: 157
column: "
- "8|78|B603|https://bandit.readthedocs.io/en/1.7.10/plugins/b603_subprocess_without_shell_equals_true.html "
- ",
https://cwe.mitre.org/data/definitions/78.html |\n"
- "|exasol/toolbox/sphinx/multiversion/main.py|line: 556
column: "
- "16|78|B602|https://bandit.readthedocs.io/en/1.7.10/plugins/b602_subprocess_popen_with_shell_equals_true.html "
- ",
https://cwe.mitre.org/data/definitions/78.html |"
- )
-
- @staticmethod
- def test_with_empty_file(cli_runner, tmp_path):
- contents = {"result": []}
- json_path = tmp_path / "test.json"
- json_path.write_text(json.dumps(contents))
-
- result = cli_runner.invoke(CLI, ["pretty-print", str(json_path)])
-
- assert result.exit_code == 0
- assert result.output.strip() == (
- "# Security\n\n"
- "|File|line/
column|Cwe|Test ID|Details|\n"
- "|---|:-:|:-:|:-:|---|"
- )
diff --git a/test/unit/lint_test.py b/test/unit/lint_test.py
deleted file mode 100644
index ded3e52239..0000000000
--- a/test/unit/lint_test.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import json
-
-import pytest
-
-from exasol.toolbox.tools import lint
-
-
-@pytest.mark.parametrize(
- "data",
- [
- {
- "type": "test_type",
- "module": "test_module",
- "obj": "test_obj",
- "line": 0,
- "column": 1,
- "endLine": 2,
- "endColumn": 3,
- "path": "test_path",
- "symbol": "test_symbol",
- "message": "test_message",
- "message-id": "test_message_id",
- },
- ],
-)
-def test_lint_issue_from_json(data):
- actual = lint.lint_issue_from_json(json.dumps([data]))
- expected = lint.Finding(
- type=data["type"],
- module=data["module"],
- obj=data["obj"],
- line=data["line"],
- column=data["column"],
- endLine=data["endLine"],
- endColumn=data["endColumn"],
- path=data["path"],
- symbol=data["symbol"],
- message=data["message"],
- message_id=data["message-id"],
- )
- assert list(actual) == [expected]
-
-
-@pytest.mark.parametrize(
- "data,expected",
- [
- (
- {
- "type": "test_type",
- "module": "test_module",
- "obj": "test_obj",
- "line": 0,
- "column": 1,
- "endLine": 2,
- "endColumn": 3,
- "path": "test_path",
- "symbol": "test_symbol",
- "message": "test_message",
- "message-id": "test_message_id",
- },
- """# Static Code Analysis
-
-|File|line/
column|id|message|
-|---|:-:|:-:|---|
-|test_path|line: 0/
column: 1|test_message_id|test_message|
-""",
- )
- ],
-)
-def test_lint_issue_to_markdown(data, expected):
- issue = lint.Finding(
- type=data["type"],
- module=data["module"],
- obj=data["obj"],
- line=data["line"],
- column=data["column"],
- endLine=data["endLine"],
- endColumn=data["endColumn"],
- path=data["path"],
- symbol=data["symbol"],
- message=data["message"],
- message_id=data["message-id"],
- )
- issues = [issue]
- actual = lint.lint_issue_to_markdown(issues)
- assert actual == expected
diff --git a/test/unit/nox/_artifacts_test.py b/test/unit/nox/_artifacts_test.py
index 34173eb428..e086ded340 100644
--- a/test/unit/nox/_artifacts_test.py
+++ b/test/unit/nox/_artifacts_test.py
@@ -26,12 +26,10 @@
COVERAGE_XML,
LINT_JSON,
LINT_JSON_ATTRIBUTES,
- LINT_TXT,
SECURITY_JSON,
SECURITY_JSON_ATTRIBUTES,
_is_valid_coverage,
_is_valid_lint_json,
- _is_valid_lint_txt,
_is_valid_security_json,
_prepare_coverage_xml,
check_artifacts,
@@ -83,7 +81,6 @@ def _create_artifact_files(path: Path, existing_files: set):
for file in existing_files:
Path(path, file).touch()
- @mock.patch("exasol.toolbox.nox._artifacts._is_valid_lint_txt", return_value=True)
@mock.patch("exasol.toolbox.nox._artifacts._is_valid_lint_json", return_value=True)
@mock.patch(
"exasol.toolbox.nox._artifacts._is_valid_security_json", return_value=True
@@ -94,7 +91,6 @@ def test_passes_when_as_expected(
mock_coverage,
mock_security,
mock_lint_json,
- mock_lint_txt,
test_project_config_factory,
tmp_path,
):
@@ -136,30 +132,6 @@ def test_fails_when_check_fails(
assert "error in [" in capsys.readouterr().err
-class TestIsValidLintTxt:
- @staticmethod
- def _create_json_txt(path: Path, text: str) -> None:
- path.touch()
- path.write_text(text)
-
- def test_passes_when_as_expected(self, tmp_path):
- path = Path(tmp_path, LINT_TXT)
- text = "Your code has been rated at 7.85/10 (previous run: 7.83/10, +0.02"
- self._create_json_txt(path, text)
-
- assert _is_valid_lint_txt(path)
-
- def test_fails_when_rating_not_found(self, tmp_path, capsys):
- path = Path(tmp_path, LINT_TXT)
- text = "dummy_text"
- self._create_json_txt(path, text)
-
- result = _is_valid_lint_txt(path)
-
- assert not result
- assert "Could not find a rating" in capsys.readouterr().err
-
-
class TestIsValidLintJson:
@staticmethod
def _create_expected_json_file(path: Path, attributes: set) -> None:
@@ -282,7 +254,6 @@ def test_missing_files(tmp_path, capsys):
assert re.match(
cleandoc("""
Could not find any file .*/coverage-python9.9\\*/.coverage
- File not found .*/lint-python9.9/.lint.txt
File not found .*/lint-python9.9/.lint.json
File not found .*/security-python9.9/.security.json
"""),
@@ -296,7 +267,6 @@ def test_all_files(tmp_path, capsys):
"9.9",
"coverage-python9.9-fast/.coverage",
"coverage-python9.9-slow/.coverage",
- "lint-python9.9/.lint.txt",
"lint-python9.9/.lint.json",
"security-python9.9/.security.json",
) as session:
@@ -312,13 +282,12 @@ def test_all_files(tmp_path, capsys):
)
assert re.match(
cleandoc("""
- Copying file .*/lint-python9.9/.lint.txt
Copying file .*/lint-python9.9/.lint.json
Copying file .*/security-python9.9/.security.json
"""),
captured.err,
)
- for f in [".lint.txt", ".lint.json", ".security.json"]:
+ for f in [".lint.json", ".security.json"]:
assert (tmp_path / f).exists()
diff --git a/test/unit/nox/_lint_test.py b/test/unit/nox/_lint_test.py
index 1fc18e9e3e..837382c95a 100644
--- a/test/unit/nox/_lint_test.py
+++ b/test/unit/nox/_lint_test.py
@@ -53,10 +53,8 @@ def test_lint(nox_session, config, file_with_multiple_problems):
lint(session=nox_session)
json_file = config.root_path / ".lint.json"
- txt_file = config.root_path / ".lint.txt"
assert json_file.exists()
- assert txt_file.exists()
contents = json_file.read_text()
errors = {row["message-id"] for row in json.loads(contents)}
diff --git a/test/unit/security_test.py b/test/unit/security_test.py
index e9624c562c..60052de5a9 100644
--- a/test/unit/security_test.py
+++ b/test/unit/security_test.py
@@ -1,6 +1,5 @@
import json
import os
-import pathlib
import subprocess
from contextlib import contextmanager
from inspect import cleandoc
@@ -272,62 +271,6 @@ def test_format_jsonl_removes_newline():
assert actual == expected
-@pytest.mark.parametrize(
- "json_input,expected",
- [
- (
- {
- "results": [
- {
- "code": "1 import subprocess\\n2 from typing import Iterable\\n3 \\n",
- "col_offset": 12,
- "end_col_offset": 17,
- "filename": "/home/test/python-toolbox/exasol/toolbox/git.py",
- "issue_confidence": "HIGH",
- "issue_cwe": {
- "id": 78,
- "link": "https://cwe.mitre.org/data/definitions/78.html",
- },
- "issue_severity": "LOW",
- "issue_text": "Consider possible security implications associated with the subprocess module.",
- "line_number": 53,
- "line_range": [1],
- "more_info": "https://bandit.readthedocs.io/en/1.7.10/blacklists/blacklist_imports.html#b404-import-subprocess",
- "test_id": "B404",
- "test_name": "blacklist",
- }
- ]
- },
- {
- "file_name": "exasol/toolbox/git.py",
- "line": 53,
- "column": 12,
- "cwe": "78",
- "test_id": "B404",
- "description": "Consider possible security implications associated with the subprocess module.",
- "references": (
- "https://bandit.readthedocs.io/en/1.7.10/blacklists/blacklist_imports.html#b404-import-subprocess",
- "https://cwe.mitre.org/data/definitions/78.html",
- ),
- },
- )
- ],
-)
-def test_from_json(json_input, expected):
- json_file = json.dumps(json_input)
- actual = security.from_json(json_file, pathlib.Path("/home/test/python-toolbox"))
- expected_issue = security.SecurityIssue(
- file_name=expected["file_name"],
- line=expected["line"],
- column=expected["column"],
- cwe=expected["cwe"],
- test_id=expected["test_id"],
- description=expected["description"],
- references=expected["references"],
- )
- assert list(actual) == [expected_issue]
-
-
@pytest.mark.parametrize(
"reference, expected",
[