diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3171dad..b5f4151c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,6 +353,7 @@ jobs: - name: Upload to Codecov uses: codecov/codecov-action@v5 + continue-on-error: ${{ github.repository != 'pondpilot/flowscope' }} with: disable_search: true fail_ci_if_error: true diff --git a/.github/workflows/publish-core.yml b/.github/workflows/publish-core.yml index 1a7f0622..53b22e31 100644 --- a/.github/workflows/publish-core.yml +++ b/.github/workflows/publish-core.yml @@ -21,6 +21,9 @@ env: jobs: publish-packages: name: Publish crates and core npm package + # Forks may use this workflow to test CLI release assets, but must never + # publish packages to the public registries. + if: ${{ github.repository == 'pondpilot/flowscope' }} runs-on: ubuntu-latest environment: prod env: @@ -147,3 +150,273 @@ jobs: fi env: NODE_AUTH_TOKEN: '' + + build-cli-linux-binaries: + name: Build CLI (${{ matrix.platform.name }}) + runs-on: ${{ matrix.platform.runs-on }} + # Build against an older glibc so the published binaries run on common + # long-term-support distributions. The image supports both amd64 and arm64. + container: + image: rust:1.95-bullseye@sha256:28afaeb8445f2a2e7d878bd34ed39ba02bb517efb29986188cbd59b7cf4f2fdf + permissions: + contents: read + strategy: + fail-fast: false + matrix: + platform: + - name: Linux x86_64 + runs-on: ubuntu-24.04 + target: x86_64-unknown-linux-gnu + - name: Linux aarch64 + runs-on: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + + steps: + - uses: actions/checkout@v4 + + - name: Resolve CLI version + id: cli_version + shell: bash + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import pathlib + import re + + data = pathlib.Path('Cargo.toml').read_text() + match = re.search(r'^version\s*=\s*"([^"]+)"', data, re.MULTILINE) + if not match: + raise SystemExit('Could not find workspace version in Cargo.toml') + print(f"version={match.group(1)}") + PY + + - name: Verify release tag + if: ${{ github.ref_type == 'tag' }} + shell: bash + env: + CLI_VERSION: ${{ steps.cli_version.outputs.version }} + run: | + set -euo pipefail + expected_tag="v${CLI_VERSION}" + if [[ "$GITHUB_REF_NAME" != "$expected_tag" ]]; then + echo "Release tag '$GITHUB_REF_NAME' does not match CLI version '$CLI_VERSION'." + exit 1 + fi + + - name: Build CLI binary + run: | + cargo build -p flowscope-cli --release --locked + strip target/release/flowscope + + - name: Smoke test CLI on glibc 2.31 + run: target/release/flowscope --version + + - name: Package CLI archive + uses: houseabsolute/actions-rust-release@v1 + with: + executable-name: flowscope + archive-name: flowscope-v${{ steps.cli_version.outputs.version }}-${{ matrix.platform.target }} + changes-file: '' + extra-files: README.md + + build-cli-other-binaries: + name: Build CLI (${{ matrix.platform.name }}) + runs-on: ${{ matrix.platform.runs-on }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + platform: + - name: macOS x86_64 + runs-on: macos-15-intel + target: x86_64-apple-darwin + - name: macOS aarch64 + runs-on: macos-15 + target: aarch64-apple-darwin + - name: Windows x86_64 + runs-on: windows-2022 + target: x86_64-pc-windows-msvc + + steps: + - uses: actions/checkout@v4 + + - name: Resolve CLI version + id: cli_version + shell: bash + run: | + python - <<'PY' >> "$GITHUB_OUTPUT" + import pathlib + import re + + data = pathlib.Path('Cargo.toml').read_text() + match = re.search(r'^version\s*=\s*"([^"]+)"', data, re.MULTILINE) + if not match: + raise SystemExit('Could not find workspace version in Cargo.toml') + print(f"version={match.group(1)}") + PY + + - name: Verify release tag + if: ${{ github.ref_type == 'tag' }} + shell: bash + env: + CLI_VERSION: ${{ steps.cli_version.outputs.version }} + run: | + set -euo pipefail + expected_tag="v${CLI_VERSION}" + if [[ "$GITHUB_REF_NAME" != "$expected_tag" ]]; then + echo "Release tag '$GITHUB_REF_NAME' does not match CLI version '$CLI_VERSION'." + exit 1 + fi + + - name: Build CLI binary + uses: houseabsolute/actions-rust-cross@v1 + with: + command: build + target: ${{ matrix.platform.target }} + args: '--package flowscope-cli --locked --release' + strip: true + + - name: Package CLI archive + uses: houseabsolute/actions-rust-release@v1 + with: + executable-name: flowscope + target: ${{ matrix.platform.target }} + archive-name: flowscope-v${{ steps.cli_version.outputs.version }}-${{ matrix.platform.target }} + changes-file: '' + extra-files: README.md + + publish-cli-binaries: + name: Publish CLI binaries + needs: + - publish-packages + - build-cli-linux-binaries + - build-cli-other-binaries + if: >- + ${{ + always() && + github.ref_type == 'tag' && + (github.event_name == 'push' || inputs.dry_run != 'true') && + needs['build-cli-linux-binaries'].result == 'success' && + needs['build-cli-other-binaries'].result == 'success' && + ( + github.repository != 'pondpilot/flowscope' || + needs['publish-packages'].result == 'success' + ) + }} + runs-on: ubuntu-24.04 + permissions: + actions: read + attestations: write + contents: write + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Download packaged CLI assets + uses: actions/download-artifact@v4 + with: + pattern: flowscope-v* + path: release-assets + merge-multiple: true + + - name: Generate aggregate SHA-256 checksums + shell: bash + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + mapfile -t archives < <( + find release-assets -maxdepth 1 -type f \ + \( -name 'flowscope-*.tar.gz' -o -name 'flowscope-*.zip' \) \ + -printf '%f\n' | sort + ) + if [[ "${#archives[@]}" -ne 5 ]]; then + echo "Expected five CLI archives, found ${#archives[@]}." + printf '%s\n' "${archives[@]}" + exit 1 + fi + ( + cd release-assets + sha256sum "${archives[@]}" > "flowscope-${RELEASE_TAG}-SHA256SUMS" + ) + + - name: Upload aggregate checksum artifact + uses: actions/upload-artifact@v4 + with: + name: flowscope-${{ github.ref_name }}-SHA256SUMS + path: release-assets/flowscope-${{ github.ref_name }}-SHA256SUMS + if-no-files-found: error + + - name: Attest CLI release assets + uses: actions/attest-build-provenance@v4 + with: + subject-path: | + release-assets/*.tar.gz + release-assets/*.zip + release-assets/*.sha256 + release-assets/*SHA256SUMS + + - name: Publish CLI release + uses: houseabsolute/actions-rust-release/publish@v1 + with: + executable-name: flowscope + artifact-regex: '\Aflowscope-v.*(\.tar\.gz|\.zip|-SHA256SUMS)\Z' + changes-file: '' + generate-release-notes: true + + publish-fork-core-package: + name: Publish fork core package asset + needs: publish-cli-binaries + if: ${{ github.repository != 'pondpilot/flowscope' && github.ref_type == 'tag' && needs.publish-cli-binaries.result == 'success' }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.95.0 + with: + targets: wasm32-unknown-unknown + + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: yarn + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Build core package + run: yarn workspace @pondpilot/flowscope-core build + + - name: Run core package tests + run: yarn workspace @pondpilot/flowscope-core test --silent + + - name: Pack core package + working-directory: packages/core + run: | + mkdir -p ../../release-assets + npm pack --workspaces=false --pack-destination ../../release-assets + package=$(find ../../release-assets -maxdepth 1 -name '*.tgz' -print -quit) + test -n "$package" + mv "$package" "../../release-assets/flowscope-core-${GITHUB_REF_NAME}.tgz" + + - name: Generate package checksum + run: | + set -euo pipefail + sha256sum "release-assets/flowscope-core-${GITHUB_REF_NAME}.tgz" \ + > "release-assets/flowscope-core-${GITHUB_REF_NAME}-SHA256SUMS" + + - name: Upload core package assets + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "$GITHUB_REF_NAME" \ + "release-assets/flowscope-core-${GITHUB_REF_NAME}.tgz" \ + "release-assets/flowscope-core-${GITHUB_REF_NAME}-SHA256SUMS" \ + --clobber diff --git a/CHANGELOG.md b/CHANGELOG.md index 5456b0dd..83b95812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Kept `@pondpilot/flowscope-react` as a private monorepo workspace and removed it from the npm release pipeline +## [0.9.1] - 2026-09-16 + +### Fixed + +- Improved MSSQL batch handling, multi-file diagnostic attribution, CLI lint metadata, and HTML/CSV/XLSX issue exports. +- Preserved WASM HTML and filename export metadata from TypeScript callers. + +### Added + +- QUALIFY filter attribution and parsed OPENJSON/XMLTABLE source extraction. +- Fork release assets for the CLI and the built core npm package. + ## [0.9.0] - 2026-08-12 ### Added diff --git a/Cargo.lock b/Cargo.lock index 223b9503..67d7339e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -964,7 +964,7 @@ dependencies = [ [[package]] name = "flowscope-cli" -version = "0.9.0" +version = "0.9.1" dependencies = [ "anyhow", "axum", @@ -998,7 +998,7 @@ dependencies = [ [[package]] name = "flowscope-core" -version = "0.9.0" +version = "0.9.1" dependencies = [ "chrono", "indexmap", @@ -1018,7 +1018,7 @@ dependencies = [ [[package]] name = "flowscope-export" -version = "0.9.0" +version = "0.9.1" dependencies = [ "chrono", "csv", @@ -1034,7 +1034,7 @@ dependencies = [ [[package]] name = "flowscope-wasm" -version = "0.9.0" +version = "0.9.1" dependencies = [ "chrono", "console_error_panic_hook", diff --git a/Cargo.toml b/Cargo.toml index e0c52b0f..f50771b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,15 +8,15 @@ members = [ resolver = "2" [workspace.package] -version = "0.9.0" +version = "0.9.1" authors = ["PondPilot Team"] edition = "2021" license = "Apache-2.0" repository = "https://github.com/pondpilot/flowscope" [workspace.dependencies] -flowscope-core = { version = "0.9.0", path = "crates/flowscope-core", default-features = false } -flowscope-export = { version = "0.9.0", path = "crates/flowscope-export", default-features = false } +flowscope-core = { version = "0.9.1", path = "crates/flowscope-core", default-features = false } +flowscope-export = { version = "0.9.1", path = "crates/flowscope-export", default-features = false } sqlparser = "0.61" serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0" diff --git a/crates/flowscope-cli/src/main.rs b/crates/flowscope-cli/src/main.rs index 6fff00e2..297e851e 100644 --- a/crates/flowscope-cli/src/main.rs +++ b/crates/flowscope-cli/src/main.rs @@ -486,6 +486,18 @@ fn to_file_lint_result( code: i.code.clone(), message: i.message.clone(), severity: i.severity, + metadata: { + let mut metadata = serde_json::to_value(i).unwrap_or(serde_json::Value::Null); + if let Some(object) = metadata.as_object_mut() { + object.insert( + "sourceName".to_string(), + serde_json::Value::String( + i.source_name.clone().unwrap_or_else(|| source.name.clone()), + ), + ); + } + metadata + }, } }) .collect(); diff --git a/crates/flowscope-cli/src/output/lint.rs b/crates/flowscope-cli/src/output/lint.rs index 625a4b1f..d50e89e4 100644 --- a/crates/flowscope-cli/src/output/lint.rs +++ b/crates/flowscope-cli/src/output/lint.rs @@ -19,6 +19,8 @@ pub struct LintIssue { pub code: String, pub message: String, pub severity: Severity, + /// Structured issue metadata preserved for JSON consumers. + pub metadata: serde_json::Value, } /// Convert a byte offset into a 1-based (line, col) pair. @@ -204,17 +206,42 @@ pub fn format_lint_json(results: &[FileLintResult], compact: bool) -> String { .issues .iter() .map(|issue| { - serde_json::json!({ - "line": issue.line, - "column": issue.col, - "code": sqlfluff_display_code(&issue.code), - "message": issue.message, - "severity": match issue.severity { - Severity::Error => "error", - Severity::Warning => "warning", - Severity::Info => "info", + let mut violation = serde_json::Map::from_iter([ + ("line".to_string(), serde_json::json!(issue.line)), + ("column".to_string(), serde_json::json!(issue.col)), + ( + "code".to_string(), + serde_json::json!(sqlfluff_display_code(&issue.code)), + ), + ("message".to_string(), serde_json::json!(issue.message)), + ( + "severity".to_string(), + serde_json::json!(match issue.severity { + Severity::Error => "error", + Severity::Warning => "warning", + Severity::Info => "info", + }), + ), + ]); + + for key in [ + "sourceName", + "statementIndex", + "span", + "sqlfluffName", + "lintEngine", + "lintConfidence", + "lintFallbackSource", + "autofix", + ] { + if let Some(value) = issue.metadata.get(key) { + if !value.is_null() { + violation.insert(key.to_string(), value.clone()); + } } - }) + } + + serde_json::Value::Object(violation) }) .collect(); @@ -302,6 +329,7 @@ mod tests { code: "LINT_AM_007".to_string(), message: "Use UNION DISTINCT or UNION ALL instead of bare UNION.".to_string(), severity: Severity::Info, + metadata: serde_json::Value::Null, }, LintIssue { line: 7, @@ -309,6 +337,7 @@ mod tests { code: "LINT_ST_006".to_string(), message: "CTE 'unused' is defined but never referenced.".to_string(), severity: Severity::Info, + metadata: serde_json::Value::Null, }, ], }]; @@ -348,6 +377,7 @@ mod tests { code: "LINT_AM_007".to_string(), message: "test".to_string(), severity: Severity::Info, + metadata: serde_json::Value::Null, }], }, ]; @@ -370,6 +400,7 @@ mod tests { code: "LINT_AM_007".to_string(), message: "Use UNION DISTINCT or UNION ALL.".to_string(), severity: Severity::Info, + metadata: serde_json::Value::Null, }], }]; diff --git a/crates/flowscope-cli/tests/lint_cli.rs b/crates/flowscope-cli/tests/lint_cli.rs index cf0b7385..c806449f 100644 --- a/crates/flowscope-cli/tests/lint_cli.rs +++ b/crates/flowscope-cli/tests/lint_cli.rs @@ -310,6 +310,17 @@ fn test_lint_json_format() { let arr = parsed.as_array().expect("Expected JSON array"); assert_eq!(arr.len(), 1); assert!(!arr[0]["violations"].as_array().unwrap().is_empty()); + + let violation = &arr[0]["violations"][0]; + assert_eq!(violation["sourceName"], sql_path.to_string_lossy().as_ref()); + assert_eq!(violation["statementIndex"], 0); + assert!(violation["span"]["start"].is_number()); + assert!(violation["span"]["end"].is_number()); + assert_eq!(violation["sqlfluffName"], "ambiguous.union"); + assert_eq!(violation["lintEngine"], "semantic"); + assert_eq!(violation["lintConfidence"], "high"); + assert_eq!(violation["autofix"]["applicability"], "safe"); + assert!(violation["autofix"]["edits"].is_array()); } #[test] diff --git a/crates/flowscope-core/src/analyzer.rs b/crates/flowscope-core/src/analyzer.rs index 46ba3b95..74b7a3fe 100644 --- a/crates/flowscope-core/src/analyzer.rs +++ b/crates/flowscope-core/src/analyzer.rs @@ -439,7 +439,17 @@ impl<'a> Analyzer<'a> { parser_fallback_used, Some(source_statement_ranges), ); - self.issues.extend(linter.check_document(&document)); + let mut lint_issues = linter.check_document(&document); + for issue in &mut lint_issues { + if let Some(local_index) = issue.statement_index { + issue.statement_index = + (start + local_index < end).then_some(start + local_index); + } + if issue.source_name.is_none() { + issue.source_name = source_name_key.map(str::to_owned); + } + } + self.issues.extend(lint_issues); start = end; } @@ -456,14 +466,28 @@ impl<'a> Analyzer<'a> { } for file in files { let document = LintDocument::new(&file.content, self.request.dialect, Vec::new()); - self.issues.extend(linter.check_document(&document)); + let mut lint_issues = linter.check_document(&document); + for issue in &mut lint_issues { + issue.statement_index = None; + if issue.source_name.is_none() { + issue.source_name = Some(file.name.clone()); + } + } + self.issues.extend(lint_issues); } return; } if !self.request.sql.is_empty() { let document = LintDocument::new(&self.request.sql, self.request.dialect, Vec::new()); - self.issues.extend(linter.check_document(&document)); + let mut lint_issues = linter.check_document(&document); + for issue in &mut lint_issues { + issue.statement_index = None; + if issue.source_name.is_none() { + issue.source_name = self.request.source_name.clone(); + } + } + self.issues.extend(lint_issues); } } diff --git a/crates/flowscope-core/src/analyzer/global.rs b/crates/flowscope-core/src/analyzer/global.rs index 7b216ad0..8363276f 100644 --- a/crates/flowscope-core/src/analyzer/global.rs +++ b/crates/flowscope-core/src/analyzer/global.rs @@ -49,11 +49,21 @@ impl<'a> Analyzer<'a> { let summary = self.build_summary(&nodes); let resolved_schema = self.build_resolved_schema(); + let mut issues = self.issues.clone(); + for issue in &mut issues { + if issue.source_name.is_none() { + issue.source_name = issue + .statement_index + .and_then(|index| statements.get(index)) + .and_then(|statement| statement.source_name.clone()); + } + } + crate::AnalyzeResult { statements, nodes, edges, - issues: self.issues.clone(), + issues, summary, resolved_schema, } diff --git a/crates/flowscope-core/src/analyzer/input.rs b/crates/flowscope-core/src/analyzer/input.rs index e9d43c49..b7a2c581 100644 --- a/crates/flowscope-core/src/analyzer/input.rs +++ b/crates/flowscope-core/src/analyzer/input.rs @@ -653,13 +653,14 @@ fn split_ranges_on_mssql_go_separators(sql: &str, ranges: Vec>) -> for range in ranges { let mut cursor = range.start; for go_range in &go_line_ranges { - if go_range.start < range.start || go_range.end > range.end || go_range.start < cursor { + if go_range.end <= cursor || go_range.start >= range.end { continue; } - if let Some(chunk) = trim_statement_range(sql, cursor, go_range.start) { + let separator_start = go_range.start.max(cursor); + if let Some(chunk) = trim_statement_range(sql, cursor, separator_start) { out.push(chunk); } - cursor = go_range.end; + cursor = go_range.end.min(range.end); } if let Some(chunk) = trim_statement_range(sql, cursor, range.end) { @@ -1212,6 +1213,31 @@ mod tests { assert_eq!(&sql[ranges[1].clone()], "CREATE TABLE test (id INT)"); } + #[test] + fn mssql_statement_ranges_split_trailing_go_batch_separators() { + for sql in [ + "SELECT 1;\nGO\nSELECT 2;\nGO\n", + "SELECT 1;\r\n go \r\nSELECT 2;\r\nGO\r\n", + "SELECT 1\nGO\nGO\nSELECT 2\nGO\n", + ] { + let ranges = compute_statement_ranges_for_dialect(sql, Dialect::Mssql); + assert_eq!(ranges.len(), 2, "unexpected ranges for {sql:?}"); + assert_eq!(&sql[ranges[0].clone()], "SELECT 1"); + assert_eq!(&sql[ranges[1].clone()], "SELECT 2"); + } + } + + #[test] + fn mssql_statement_ranges_ignore_go_inside_strings_comments_and_identifiers() { + let sql = "SELECT 'GO' AS literal;\nSELECT [GO] FROM [source];\n-- GO\n/* GO */\nGO\nSELECT 'inside\nGO\nstring' AS literal;"; + let ranges = compute_statement_ranges_for_dialect(sql, Dialect::Mssql); + + assert_eq!(ranges.len(), 3); + assert_eq!(&sql[ranges[0].clone()], "SELECT 'GO' AS literal"); + assert_eq!(&sql[ranges[1].clone()], "SELECT [GO] FROM [source]"); + assert!(sql[ranges[2].clone()].contains("inside\nGO\nstring")); + } + #[test] fn collect_statements_mssql_go_batch_without_final_semicolon_parses_statements() { let mut request = base_request(); @@ -1226,6 +1252,21 @@ mod tests { assert_eq!(statements.len(), 2); } + #[test] + fn collect_statements_mssql_go_batch_with_trailing_separator_has_no_parse_error() { + let mut request = base_request(); + request.dialect = Dialect::Mssql; + request.sql = "SELECT 1;\nGO\nSELECT 2;\nGO\n".to_string(); + + let (statements, issues) = collect_statements(&request); + + assert_eq!(statements.len(), 2); + assert!( + issues.is_empty(), + "MSSQL trailing GO should not produce parse errors: {issues:?}" + ); + } + #[test] fn parses_procedure_with_inner_semicolons() { let mut request = base_request(); diff --git a/crates/flowscope-core/src/analyzer/select_analyzer.rs b/crates/flowscope-core/src/analyzer/select_analyzer.rs index a1b55bfe..cc1b6c51 100644 --- a/crates/flowscope-core/src/analyzer/select_analyzer.rs +++ b/crates/flowscope-core/src/analyzer/select_analyzer.rs @@ -43,6 +43,7 @@ impl<'a, 'b> SelectAnalyzer<'a, 'b> { self.analyze_projection(&select.projection); self.analyze_selection(&select.selection); self.analyze_having(&select.having); + self.analyze_qualify(&select.qualify); } /// Analyzes GROUP BY expressions to track grouping columns. @@ -357,6 +358,14 @@ impl<'a, 'b> SelectAnalyzer<'a, 'b> { } } + fn analyze_qualify(&mut self, qualify: &Option) { + if let Some(qualify_expr) = qualify { + let mut ea = ExpressionAnalyzer::new(self.analyzer, self.ctx); + ea.analyze(qualify_expr); + ea.capture_filter_predicates(qualify_expr, FilterClauseType::Qualify); + } + } + /// Checks if an expression references any output column aliases and emits a warning. /// /// Used by HAVING and can be extended to other clauses that need alias checking. diff --git a/crates/flowscope-core/src/analyzer/tests.rs b/crates/flowscope-core/src/analyzer/tests.rs index ba677995..07945737 100644 --- a/crates/flowscope-core/src/analyzer/tests.rs +++ b/crates/flowscope-core/src/analyzer/tests.rs @@ -332,6 +332,7 @@ fn file_statements_produce_spans() { .span .expect("span should be present for file statement"); assert_eq!(&file_sql[span.start..span.end], "missing_table"); + assert_eq!(issue.source_name.as_deref(), Some("file.sql")); } #[test] @@ -360,14 +361,71 @@ fn lint_document_rules_apply_to_each_file_in_multi_file_request() { .collect(); assert_eq!(st012_issues.len(), 2, "expected one ST_012 issue per file"); - assert!( - st012_issues + assert_eq!(st012_issues[0].statement_index, Some(0)); + assert_eq!(st012_issues[0].source_name.as_deref(), Some("first.sql")); + assert_eq!(st012_issues[1].statement_index, Some(1)); + assert_eq!(st012_issues[1].source_name.as_deref(), Some("second.sql")); +} + +#[test] +fn lint_issues_keep_global_statement_indices_within_each_file() { + let mut request = make_request(""); + request.files = Some(vec![ + FileSource { + name: "first.sql".to_string(), + content: "SELECT 1 UNION SELECT 2; SELECT 3 UNION SELECT 4;".to_string(), + }, + FileSource { + name: "second.sql".to_string(), + content: "SELECT 5 UNION SELECT 6;".to_string(), + }, + ]); + request.options = Some(AnalysisOptions { + lint: Some(LintConfig::default()), + ..Default::default() + }); + + let result = analyze(&request); + let union_issues: Vec<_> = result + .issues + .iter() + .filter(|issue| issue.code == issue_codes::LINT_AM_002) + .collect(); + + assert_eq!(union_issues.len(), 3); + assert_eq!( + union_issues .iter() - .all(|issue| issue.statement_index == Some(0)), - "document-level lint rules should run with per-document statement indices" + .map(|issue| (issue.source_name.as_deref(), issue.statement_index)) + .collect::>(), + vec![ + (Some("first.sql"), Some(0)), + (Some("first.sql"), Some(1)), + (Some("second.sql"), Some(2)), + ] ); } +#[test] +fn statementless_file_lint_issues_do_not_claim_another_statement() { + let mut request = make_request(""); + request.files = Some(vec![FileSource { + name: "empty.sql".to_string(), + content: "-- noqa: disable=all\n".to_string(), + }]); + request.options = Some(AnalysisOptions { + lint: Some(LintConfig::default()), + ..Default::default() + }); + + let result = analyze(&request); + + for issue in result.issues { + assert_eq!(issue.source_name.as_deref(), Some("empty.sql")); + assert_eq!(issue.statement_index, None); + } +} + #[test] fn parser_fallback_metadata_is_attached_to_lint_issues() { let mut request = diff --git a/crates/flowscope-core/src/extractors/mod.rs b/crates/flowscope-core/src/extractors/mod.rs index 3674f5ae..7046fe0b 100644 --- a/crates/flowscope-core/src/extractors/mod.rs +++ b/crates/flowscope-core/src/extractors/mod.rs @@ -187,15 +187,80 @@ fn extract_tables_from_table_factor(table_factor: &TableFactor, tables: &mut Vec TableFactor::Unpivot { .. } => {} TableFactor::MatchRecognize { .. } => {} TableFactor::JsonTable { .. } => {} - // TODO: Implement table extraction for OPENJSON (SQL Server) - TableFactor::OpenJsonTable { .. } => {} + TableFactor::OpenJsonTable { json_expr, .. } => { + extract_tables_from_expr(json_expr, tables); + } // TODO: Implement table extraction for XMLTABLE - TableFactor::XmlTable { .. } => {} + TableFactor::XmlTable { passing, .. } => { + for argument in &passing.arguments { + extract_tables_from_expr(&argument.expr, tables); + } + } // TODO: Implement table extraction for semantic views TableFactor::SemanticView { .. } => {} } } +fn extract_tables_from_expr(expr: &sqlparser::ast::Expr, tables: &mut Vec) { + use sqlparser::ast::{Expr, FunctionArg, FunctionArgExpr, FunctionArguments}; + + match expr { + Expr::CompoundIdentifier(parts) if parts.len() > 1 => { + tables.push( + parts[..parts.len() - 1] + .iter() + .map(|ident| ident.value.clone()) + .collect::>() + .join("."), + ); + } + Expr::Function(function) => { + if let FunctionArguments::List(arguments) = &function.args { + for argument in &arguments.args { + if let FunctionArg::Unnamed(FunctionArgExpr::Expr(argument)) = argument { + extract_tables_from_expr(argument, tables); + } + } + } + } + Expr::Nested(inner) => extract_tables_from_expr(inner, tables), + Expr::Subquery(query) => extract_tables_from_query_body(&query.body, tables), + Expr::BinaryOp { left, right, .. } => { + extract_tables_from_expr(left, tables); + extract_tables_from_expr(right, tables); + } + Expr::UnaryOp { expr, .. } => extract_tables_from_expr(expr, tables), + Expr::Cast { expr, .. } => extract_tables_from_expr(expr, tables), + Expr::Extract { expr, .. } => extract_tables_from_expr(expr, tables), + Expr::Case { + operand, + conditions, + else_result, + .. + } => { + if let Some(operand) = operand { + extract_tables_from_expr(operand, tables); + } + for condition in conditions { + extract_tables_from_expr(&condition.condition, tables); + extract_tables_from_expr(&condition.result, tables); + } + if let Some(else_result) = else_result { + extract_tables_from_expr(else_result, tables); + } + } + Expr::Exists { subquery, .. } => extract_tables_from_query_body(&subquery.body, tables), + Expr::InSubquery { subquery, expr, .. } => { + extract_tables_from_expr(expr, tables); + extract_tables_from_query_body(&subquery.body, tables); + } + _ => { + // Literals and expression forms without nested SQL relations do not + // contribute table names to the legacy extractor. + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/flowscope-core/src/types/response.rs b/crates/flowscope-core/src/types/response.rs index 6425cd23..cbd90727 100644 --- a/crates/flowscope-core/src/types/response.rs +++ b/crates/flowscope-core/src/types/response.rs @@ -640,7 +640,7 @@ impl Edge { } } -/// A filter predicate from a WHERE, HAVING, or JOIN ON clause. +/// A filter predicate from a WHERE, HAVING, QUALIFY, or JOIN ON clause. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct FilterPredicate { @@ -659,6 +659,8 @@ pub enum FilterClauseType { Where, /// HAVING clause (after GROUP BY) Having, + /// QUALIFY clause (after window evaluation) + Qualify, /// JOIN ... ON clause JoinOn, } diff --git a/crates/flowscope-core/tests/lineage_engine.rs b/crates/flowscope-core/tests/lineage_engine.rs index 645d3ec9..3a8b888e 100644 --- a/crates/flowscope-core/tests/lineage_engine.rs +++ b/crates/flowscope-core/tests/lineage_engine.rs @@ -3454,13 +3454,17 @@ fn snowflake_qualify_clause_filters_window_results() { let result = run_analysis(sql, Dialect::Snowflake, None); - // QUALIFY is Snowflake-specific and may have limited support - // This test documents current behavior - // TODO: Verify QUALIFY clause support in Snowflake dialect - assert!( - result.summary.statement_count >= 1, - "QUALIFY clause should parse in Snowflake" - ); + assert_eq!(result.summary.statement_count, 1); + assert!(!result.summary.has_errors); + let orders = result + .nodes + .iter() + .find(|node| node.label.as_ref().eq_ignore_ascii_case("orders")) + .expect("QUALIFY source table should be present"); + assert!(orders.filters.iter().any(|filter| { + filter.clause_type == flowscope_core::FilterClauseType::Qualify + && filter.expression.contains("rn = 1") + })); } #[test] diff --git a/crates/flowscope-core/tests/snapshots/golden__golden_multi_file_keeps_sources.snap b/crates/flowscope-core/tests/snapshots/golden__golden_multi_file_keeps_sources.snap index da24ff42..3e0a37dc 100644 --- a/crates/flowscope-core/tests/snapshots/golden__golden_multi_file_keeps_sources.snap +++ b/crates/flowscope-core/tests/snapshots/golden__golden_multi_file_keeps_sources.snap @@ -122,13 +122,15 @@ expression: cleaned "severity": "info", "code": "APPROXIMATE_LINEAGE", "message": "SELECT * from 'alpha_table' - column list unknown without schema metadata", - "statementIndex": 0 + "statementIndex": 0, + "sourceName": "alpha.sql" }, { "severity": "info", "code": "APPROXIMATE_LINEAGE", "message": "SELECT * from 'beta_table' - column list unknown without schema metadata", - "statementIndex": 1 + "statementIndex": 1, + "sourceName": "beta.sql" } ], "summary": { diff --git a/crates/flowscope-export/src/csv.rs b/crates/flowscope-export/src/csv.rs index 4ac12b5a..7fe334ed 100644 --- a/crates/flowscope-export/src/csv.rs +++ b/crates/flowscope-export/src/csv.rs @@ -9,6 +9,7 @@ use zip::CompressionMethod; use crate::extract::{ extract_column_mappings, extract_script_info, extract_table_dependencies, extract_table_info, }; +use crate::issue_export::{issue_rows, ISSUE_HEADERS}; use crate::ExportError; pub fn export_csv_bundle(result: &AnalyzeResult) -> Result, ExportError> { @@ -202,34 +203,26 @@ fn export_issues_csv(result: &AnalyzeResult) -> Result, ExportError> { .from_writer(Vec::new()); writer - .write_record([ - "Severity", - "Code", - "Message", - "Statement", - "Span Start", - "Span End", - ]) + .write_record(ISSUE_HEADERS) .map_err(|err| ExportError::Csv(err.to_string()))?; - for issue in &result.issues { - let statement = issue - .statement_index - .map(|idx| idx.to_string()) - .unwrap_or_default(); - let (start, end) = issue - .span - .map(|span| (span.start.to_string(), span.end.to_string())) - .unwrap_or_default(); - + for issue in issue_rows(result) { writer .write_record([ - format!("{:?}", issue.severity).to_lowercase(), + issue.severity.to_string(), issue.code.clone(), issue.message.clone(), - statement, - start, - end, + issue.statement, + issue.span_start, + issue.span_end, + issue.source_name, + issue.sqlfluff_name, + issue.lint_engine, + issue.lint_confidence, + issue.lint_fallback_source, + issue.autofix_applicability, + issue.autofix_edit_count, + issue.autofix_json, ]) .map_err(|err| ExportError::Csv(err.to_string()))?; } diff --git a/crates/flowscope-export/src/html.rs b/crates/flowscope-export/src/html.rs index 5d4bb24a..1f5652b6 100644 --- a/crates/flowscope-export/src/html.rs +++ b/crates/flowscope-export/src/html.rs @@ -3,6 +3,7 @@ use chrono::{DateTime, Utc}; use flowscope_core::AnalyzeResult; use crate::extract::{extract_column_mappings, extract_script_info, extract_table_info}; +use crate::issue_export::issue_rows; use crate::mermaid::{export_mermaid, MermaidView}; pub fn export_html( @@ -18,24 +19,38 @@ pub fn export_html( let scripts = extract_script_info(result); let tables = extract_table_info(result); let mappings = extract_column_mappings(result); - let issues = &result.issues; - let export_date = exported_at.format("%Y-%m-%d %H:%M:%S UTC"); - let issues_section = if issues.is_empty() { + let issues_section = if result.issues.is_empty() { String::new() } else { - let rows = issues + let rows = issue_rows(result) .iter() .map(|issue| { - let severity_class = severity_class(issue.severity); - let severity_label = severity_label(issue.severity); + let autofix = if issue.autofix_json.is_empty() { + String::new() + } else { + format!( + "
{} ({} edits){}
", + escape_html(&issue.autofix_applicability), + escape_html(&issue.autofix_edit_count), + escape_html(&issue.autofix_json) + ) + }; format!( - "{}{}{}", - severity_class, - escape_html(severity_label), + "{}{}{}{}{}{}{}{}{}{}{}", + issue.severity, + escape_html(issue.severity), escape_html(&issue.code), - escape_html(&issue.message) + escape_html(&issue.message), + escape_html(&issue.source_name), + escape_html(&issue.statement), + escape_html(&format!("{}..{}", issue.span_start, issue.span_end)), + escape_html(&issue.sqlfluff_name), + escape_html(&issue.lint_engine), + escape_html(&issue.lint_confidence), + escape_html(&issue.lint_fallback_source), + autofix, ) }) .collect::>() @@ -44,7 +59,7 @@ pub fn export_html( format!( "
Issues
\ \ - \ + \ {rows}\
SeverityCodeMessage
SeverityCodeMessageSourceStatementSpanSQLFluff RuleLint EngineConfidenceFallbackAutofix
" ) @@ -294,19 +309,3 @@ fn escape_html(value: &str) -> String { .replace('"', """) .replace('\'', "'") } - -fn severity_class(severity: flowscope_core::Severity) -> &'static str { - match severity { - flowscope_core::Severity::Error => "error", - flowscope_core::Severity::Warning => "warning", - flowscope_core::Severity::Info => "info", - } -} - -fn severity_label(severity: flowscope_core::Severity) -> &'static str { - match severity { - flowscope_core::Severity::Error => "ERROR", - flowscope_core::Severity::Warning => "WARNING", - flowscope_core::Severity::Info => "INFO", - } -} diff --git a/crates/flowscope-export/src/issue_export.rs b/crates/flowscope-export/src/issue_export.rs new file mode 100644 index 00000000..b002f3af --- /dev/null +++ b/crates/flowscope-export/src/issue_export.rs @@ -0,0 +1,148 @@ +use flowscope_core::{ + types::{LintConfidence, LintEngine, LintFallbackSource}, + AnalyzeResult, Issue, IssueAutofixApplicability, Severity, +}; + +pub(crate) const ISSUE_HEADERS: [&str; 14] = [ + "Severity", + "Code", + "Message", + "Statement", + "Span Start", + "Span End", + "Source Name", + "SQLFluff Name", + "Lint Engine", + "Lint Confidence", + "Lint Fallback Source", + "Autofix Applicability", + "Autofix Edit Count", + "Autofix JSON", +]; + +#[derive(Debug, Clone)] +pub(crate) struct IssueExportRow { + pub(crate) severity: &'static str, + pub(crate) code: String, + pub(crate) message: String, + pub(crate) statement: String, + pub(crate) span_start: String, + pub(crate) span_end: String, + pub(crate) source_name: String, + pub(crate) sqlfluff_name: String, + pub(crate) lint_engine: String, + pub(crate) lint_confidence: String, + pub(crate) lint_fallback_source: String, + pub(crate) autofix_applicability: String, + pub(crate) autofix_edit_count: String, + pub(crate) autofix_json: String, +} + +pub(crate) fn issue_rows(result: &AnalyzeResult) -> Vec { + result + .issues + .iter() + .map(|issue| issue_row(result, issue)) + .collect() +} + +fn issue_row(result: &AnalyzeResult, issue: &Issue) -> IssueExportRow { + let statement = issue.statement_index.map(|index| index.to_string()); + let source_name = issue + .source_name + .clone() + .or_else(|| { + issue.statement_index.and_then(|index| { + result + .statements + .iter() + .find(|statement| statement.statement_index == index) + .and_then(|statement| statement.source_name.clone()) + }) + }) + .unwrap_or_default(); + let (span_start, span_end) = issue + .span + .map(|span| (span.start.to_string(), span.end.to_string())) + .unwrap_or_default(); + let (autofix_applicability, autofix_edit_count, autofix_json) = issue + .autofix + .as_ref() + .map(|autofix| { + ( + autofix_applicability(autofix.applicability).to_string(), + autofix.edits.len().to_string(), + serde_json::to_string(autofix).unwrap_or_default(), + ) + }) + .unwrap_or_default(); + + IssueExportRow { + severity: severity(issue.severity), + code: issue.code.clone(), + message: issue.message.clone(), + statement: statement.unwrap_or_default(), + span_start, + span_end, + source_name, + sqlfluff_name: issue.sqlfluff_name.clone().unwrap_or_default(), + lint_engine: issue + .lint_engine + .map(lint_engine) + .unwrap_or_default() + .to_string(), + lint_confidence: issue + .lint_confidence + .map(lint_confidence) + .unwrap_or_default() + .to_string(), + lint_fallback_source: issue + .lint_fallback_source + .map(lint_fallback_source) + .unwrap_or_default() + .to_string(), + autofix_applicability, + autofix_edit_count, + autofix_json, + } +} + +fn severity(value: Severity) -> &'static str { + match value { + Severity::Error => "error", + Severity::Warning => "warning", + Severity::Info => "info", + } +} + +fn lint_engine(value: LintEngine) -> &'static str { + match value { + LintEngine::Semantic => "semantic", + LintEngine::Lexical => "lexical", + LintEngine::Document => "document", + } +} + +fn lint_confidence(value: LintConfidence) -> &'static str { + match value { + LintConfidence::High => "high", + LintConfidence::Medium => "medium", + LintConfidence::Low => "low", + } +} + +fn lint_fallback_source(value: LintFallbackSource) -> &'static str { + match value { + LintFallbackSource::ParserFallback => "parser_fallback", + LintFallbackSource::TokenizerFallback => "tokenizer_fallback", + LintFallbackSource::HeuristicRule => "heuristic_rule", + } +} + +fn autofix_applicability(value: IssueAutofixApplicability) -> &'static str { + match value { + IssueAutofixApplicability::Safe => "safe", + IssueAutofixApplicability::Unsafe => "unsafe", + IssueAutofixApplicability::DisplayOnly => "displayOnly", + } +} diff --git a/crates/flowscope-export/src/lib.rs b/crates/flowscope-export/src/lib.rs index ff2a062b..6cc749c8 100644 --- a/crates/flowscope-export/src/lib.rs +++ b/crates/flowscope-export/src/lib.rs @@ -25,6 +25,7 @@ pub mod dali_compat; mod error; mod extract; mod html; +mod issue_export; mod join_export; mod json; mod mermaid; diff --git a/crates/flowscope-export/src/xlsx.rs b/crates/flowscope-export/src/xlsx.rs index 05a40fc5..1e509202 100644 --- a/crates/flowscope-export/src/xlsx.rs +++ b/crates/flowscope-export/src/xlsx.rs @@ -3,6 +3,7 @@ use rust_xlsxwriter::{Workbook, Worksheet}; use crate::extract::{ extract_column_mappings, extract_script_info, extract_table_dependencies, extract_table_info, }; +use crate::issue_export::{issue_rows, ISSUE_HEADERS}; use crate::ExportError; use flowscope_core::AnalyzeResult; use std::collections::{BTreeSet, HashMap}; @@ -40,6 +41,12 @@ pub fn export_xlsx(result: &AnalyzeResult) -> Result, ExportError> { .map_err(|err| ExportError::Xlsx(err.to_string()))?; write_dependency_matrix_sheet(dependency_sheet, result)?; + let issues_sheet = workbook.add_worksheet(); + issues_sheet + .set_name("Issues") + .map_err(|err| ExportError::Xlsx(err.to_string()))?; + write_issues_sheet(issues_sheet, result)?; + workbook .save_to_buffer() .map_err(|err| ExportError::Xlsx(err.to_string())) @@ -220,6 +227,34 @@ fn write_dependency_matrix_sheet( Ok(()) } +fn write_issues_sheet(sheet: &mut Worksheet, result: &AnalyzeResult) -> Result<(), ExportError> { + write_row(sheet, 0, &ISSUE_HEADERS)?; + + for (index, issue) in issue_rows(result).iter().enumerate() { + let row = (index + 1) as u32; + let values = [ + issue.severity.to_string(), + sanitize_xlsx_value(&issue.code), + sanitize_xlsx_value(&issue.message), + sanitize_xlsx_value(&issue.statement), + sanitize_xlsx_value(&issue.span_start), + sanitize_xlsx_value(&issue.span_end), + sanitize_xlsx_value(&issue.source_name), + sanitize_xlsx_value(&issue.sqlfluff_name), + sanitize_xlsx_value(&issue.lint_engine), + sanitize_xlsx_value(&issue.lint_confidence), + sanitize_xlsx_value(&issue.lint_fallback_source), + sanitize_xlsx_value(&issue.autofix_applicability), + sanitize_xlsx_value(&issue.autofix_edit_count), + sanitize_xlsx_value(&issue.autofix_json), + ]; + let values: Vec<&str> = values.iter().map(String::as_str).collect(); + write_row(sheet, row, &values)?; + } + + Ok(()) +} + fn write_row(sheet: &mut Worksheet, row: u32, values: &[&str]) -> Result<(), ExportError> { for (col, value) in values.iter().enumerate() { sheet diff --git a/crates/flowscope-export/tests/export_formats.rs b/crates/flowscope-export/tests/export_formats.rs index eb974160..a66bd3a7 100644 --- a/crates/flowscope-export/tests/export_formats.rs +++ b/crates/flowscope-export/tests/export_formats.rs @@ -1,13 +1,20 @@ +#[cfg(feature = "duckdb")] use duckdb::Connection; use flowscope_core::{ - analyze, AggregationInfo, AnalyzeRequest, AnalyzeResult, Dialect, FilterClauseType, - FilterPredicate, Issue, Node, NodeType, Span, StatementMeta, Summary, -}; -use flowscope_export::{ - export_csv_bundle, export_html, export_json, export_mermaid, export_sql, export_xlsx, - ExportNaming, MermaidView, + analyze, + types::{LintConfidence, LintEngine, LintFallbackSource}, + AnalyzeRequest, AnalyzeResult, Dialect, Issue, IssueAutofixApplicability, IssuePatchEdit, Span, + StatementMeta, }; +#[cfg(feature = "duckdb")] +use flowscope_core::{AggregationInfo, FilterClauseType, FilterPredicate, Node, NodeType, Summary}; +#[cfg(feature = "duckdb")] +use flowscope_export::export_sql; +use flowscope_export::{export_csv_bundle, export_html, export_json, export_xlsx, ExportNaming}; +use flowscope_export::{export_mermaid, MermaidView}; +#[cfg(feature = "duckdb")] use serde_json::json; +#[cfg(feature = "duckdb")] use std::collections::HashMap; use std::io::Read; @@ -63,6 +70,16 @@ fn exports_csv_archive() { let mut content = String::new(); file.read_to_string(&mut content).expect("read csv content"); assert!(content.contains("Source Table")); + drop(file); + + let mut issues = archive.by_name("issues.csv").expect("issues file"); + let mut issues_content = String::new(); + issues + .read_to_string(&mut issues_content) + .expect("read issues content"); + assert!(issues_content.starts_with( + "Severity,Code,Message,Statement,Span Start,Span End,Source Name,SQLFluff Name,Lint Engine,Lint Confidence,Lint Fallback Source,Autofix Applicability,Autofix Edit Count,Autofix JSON" + )); } #[test] @@ -72,6 +89,77 @@ fn exports_xlsx_bytes() { assert!(!bytes.is_empty()); } +#[test] +fn exports_issue_metadata_in_html_and_csv() { + let issue = Issue::warning("LINT_TEST", ", \"message\"\nnext") + .with_statement(7) + .with_span(Span::new(11, 16)) + .with_sqlfluff_name("layout.test") + .with_lint_engine(LintEngine::Lexical) + .with_lint_confidence(LintConfidence::Medium) + .with_lint_fallback_source(LintFallbackSource::TokenizerFallback) + .with_autofix_edits( + IssueAutofixApplicability::Safe, + vec![ + IssuePatchEdit::new(Span::new(11, 13), " "), + IssuePatchEdit::new(Span::new(14, 16), "=value"), + ], + ); + let mut result = AnalyzeResult { + statements: vec![StatementMeta { + statement_index: 7, + statement_type: "SELECT".to_string(), + source_name: Some("models/orders.sql".to_string()), + span: Some(Span::new(0, 20)), + join_count: 0, + complexity_score: 1, + resolved_sql: None, + }], + issues: vec![issue], + ..Default::default() + }; + result.summary.issue_count.warnings = 1; + + let html = export_html( + &result, + "Test Project", + ExportNaming::new("Test Project").exported_at(), + ) + .expect("html export"); + assert!(html.contains("Source")); + assert!(html.contains("models/orders.sql")); + assert!(html.contains("layout.test")); + assert!(html.contains("<unsafe>")); + assert!(!html.contains("")); + + let bytes = export_csv_bundle(&result).expect("csv bundle"); + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes)).expect("zip archive"); + let mut file = archive.by_name("issues.csv").expect("issues file"); + let mut content = String::new(); + file.read_to_string(&mut content) + .expect("read issues content"); + let mut reader = csv::Reader::from_reader(content.as_bytes()); + let headers = reader.headers().expect("csv headers").clone(); + assert_eq!(headers.get(0), Some("Severity")); + assert_eq!(headers.get(6), Some("Source Name")); + assert_eq!(headers.get(13), Some("Autofix JSON")); + let row = reader + .records() + .next() + .expect("issue row") + .expect("csv row"); + assert_eq!(&row[1], "LINT_TEST"); + assert_eq!(&row[3], "7"); + assert_eq!(&row[6], "models/orders.sql"); + assert_eq!(&row[8], "lexical"); + assert_eq!(&row[9], "medium"); + assert_eq!(&row[10], "tokenizer_fallback"); + assert_eq!(&row[11], "safe"); + assert_eq!(&row[12], "2"); + let autofix: serde_json::Value = serde_json::from_str(&row[13]).expect("autofix JSON"); + assert_eq!(autofix["edits"][1]["replacement"], "=value"); +} + /// Multi-statement regression for `representative_join_edge_ids`. /// /// The same logical `users JOIN orders` appears in two statements, each @@ -81,6 +169,7 @@ fn exports_xlsx_bytes() { /// representative logic should emit exactly one join row *per statement* /// (dedup across column-level edges sharing the same relation pair + join /// metadata) — so with two statements we expect exactly two join rows. +#[cfg(feature = "duckdb")] #[test] fn sql_export_dedups_column_level_joins_and_preserves_per_statement_rows() { let sql = "SELECT u.id, o.total FROM users u JOIN orders o ON u.id = o.user_id;\n\ @@ -121,6 +210,7 @@ fn sql_export_dedups_column_level_joins_and_preserves_per_statement_rows() { } } +#[cfg(feature = "duckdb")] #[test] fn sql_export_preserves_statement_scoped_filters_and_aggregations() { let mut table_metadata = HashMap::new(); @@ -226,6 +316,7 @@ fn sql_export_preserves_statement_scoped_filters_and_aggregations() { assert_eq!(aggregation_rows, vec![(0, Some("COUNT".to_string()))]); } +#[cfg(feature = "duckdb")] #[test] fn sql_export_reindexes_statement_references_and_preserves_occurrence_spans() { let mut column_metadata = HashMap::new(); diff --git a/crates/flowscope-wasm/src/lib.rs b/crates/flowscope-wasm/src/lib.rs index b4386a13..cc56dff6 100644 --- a/crates/flowscope-wasm/src/lib.rs +++ b/crates/flowscope-wasm/src/lib.rs @@ -40,6 +40,7 @@ struct ExportMermaidRequest { } #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] struct ExportHtmlRequest { result: AnalyzeResult, #[serde(default = "default_project_name")] @@ -59,6 +60,7 @@ struct ExportXlsxRequest { } #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] struct ExportFilenameRequest { #[serde(default = "default_project_name")] project_name: String, @@ -601,6 +603,36 @@ mod tests { assert!(!version.is_empty()); } + #[test] + fn export_requests_deserialize_typescript_camel_case_fields() { + let html_request: ExportHtmlRequest = serde_json::from_value(serde_json::json!({ + "result": AnalyzeResult::default(), + "projectName": "Metadata Case", + "exportedAt": "2026-01-18T12:30:05Z" + })) + .expect("HTML export request should deserialize"); + + assert_eq!(html_request.project_name, "Metadata Case"); + assert_eq!( + html_request.exported_at.as_deref(), + Some("2026-01-18T12:30:05Z") + ); + + let filename_request: ExportFilenameRequest = serde_json::from_value(serde_json::json!({ + "projectName": "Metadata Case", + "exportedAt": "2026-01-18T12:30:05Z", + "format": { "type": "xlsx" } + })) + .expect("filename export request should deserialize"); + + assert_eq!(filename_request.project_name, "Metadata Case"); + assert_eq!( + filename_request.exported_at.as_deref(), + Some("2026-01-18T12:30:05Z") + ); + assert!(matches!(filename_request.format, ExportFormatRequest::Xlsx)); + } + // Note: Tests for export_to_duckdb_sql and analyze_and_export_sql cannot run // on native targets because they return Result<_, JsValue> which only works // on wasm32. These functions are tested via wasm-pack test. diff --git a/crates/flowscope-wasm/tests/analysis.rs b/crates/flowscope-wasm/tests/analysis.rs new file mode 100644 index 00000000..6877fa1b --- /dev/null +++ b/crates/flowscope-wasm/tests/analysis.rs @@ -0,0 +1,48 @@ +use flowscope_wasm::{analyze_sql_json, split_statements_json}; +use serde_json::Value; + +#[test] +fn analyze_sql_json_handles_mssql_go_batch_separators() { + let request = serde_json::json!({ + "sql": "SELECT 1;\nGO\nSELECT 2;\nGO\n", + "dialect": "mssql" + }); + + let result: Value = serde_json::from_str(&analyze_sql_json(&request.to_string())) + .expect("analysis result should be valid JSON"); + let statements = result + .get("statements") + .and_then(Value::as_array) + .expect("analysis result should contain statements"); + let issues = result + .get("issues") + .and_then(Value::as_array) + .expect("analysis result should contain issues"); + + assert_eq!(statements.len(), 2); + assert!(!issues + .iter() + .any(|issue| { issue.get("code") == Some(&Value::String("PARSE_ERROR".to_string())) })); +} + +#[test] +fn split_statements_json_handles_mssql_go_batch_separators() { + let sql = "SELECT 1;\nGO\nSELECT 2;\nGO\n"; + let request = serde_json::json!({ + "sql": sql, + "dialect": "mssql" + }); + + let result: Value = serde_json::from_str(&split_statements_json(&request.to_string())) + .expect("statement split result should be valid JSON"); + let statements = result + .get("statements") + .and_then(Value::as_array) + .expect("statement split result should contain statements"); + + assert_eq!(statements.len(), 2); + assert_eq!(statements[0]["start"], 0); + assert_eq!(statements[0]["end"], 8); + assert_eq!(statements[1]["start"], 13); + assert_eq!(statements[1]["end"], 21); +} diff --git a/docs/api-types.md b/docs/api-types.md index b584f744..944f0966 100644 --- a/docs/api-types.md +++ b/docs/api-types.md @@ -206,6 +206,22 @@ export interface Issue { message: string; span?: Span; statementIndex?: number; + sourceName?: string; + sqlfluffName?: string; + lintEngine?: 'semantic' | 'lexical' | 'document'; + lintConfidence?: 'high' | 'medium' | 'low'; + lintFallbackSource?: 'parser_fallback' | 'tokenizer_fallback' | 'heuristic_rule'; + autofix?: IssueAutofix; +} + +export interface IssueAutofix { + applicability: 'safe' | 'unsafe' | 'displayOnly'; + edits: IssuePatchEdit[]; +} + +export interface IssuePatchEdit { + span: Span; + replacement: string; } export interface Summary { diff --git a/docs/api_schema.json b/docs/api_schema.json index 72f5b1ed..635355d2 100644 --- a/docs/api_schema.json +++ b/docs/api_schema.json @@ -677,7 +677,7 @@ ] }, "FilterPredicate": { - "description": "A filter predicate from a WHERE, HAVING, or JOIN ON clause.", + "description": "A filter predicate from a WHERE, HAVING, QUALIFY, or JOIN ON clause.", "type": "object", "properties": { "expression": { @@ -708,6 +708,11 @@ "type": "string", "const": "HAVING" }, + { + "description": "QUALIFY clause (after window evaluation)", + "type": "string", + "const": "QUALIFY" + }, { "description": "JOIN ... ON clause", "type": "string", diff --git a/packages/core/package.json b/packages/core/package.json index 498fa8b9..836f256f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@pondpilot/flowscope-core", - "version": "0.9.0", + "version": "0.9.1", "description": "SQL lineage analysis engine for the browser", "type": "module", "main": "dist/index.js", diff --git a/packages/core/src/generated/api-types.ts b/packages/core/src/generated/api-types.ts index d1a58fdd..fbf13b3d 100644 --- a/packages/core/src/generated/api-types.ts +++ b/packages/core/src/generated/api-types.ts @@ -448,7 +448,7 @@ export interface CanonicalName { export type ResolutionSource = 'imported' | 'implied' | 'unknown'; /** - * A filter predicate from a WHERE, HAVING, or JOIN ON clause. + * A filter predicate from a WHERE, HAVING, QUALIFY, or JOIN ON clause. */ export interface FilterPredicate { /** @@ -464,7 +464,7 @@ export interface FilterPredicate { /** * The type of SQL clause where a filter predicate appears. */ -export type FilterClauseType = 'WHERE' | 'HAVING' | 'JOIN_ON'; +export type FilterClauseType = 'WHERE' | 'HAVING' | 'QUALIFY' | 'JOIN_ON'; /** * Information about aggregation applied to a column. diff --git a/packages/core/tests/export.test.ts b/packages/core/tests/export.test.ts index b1265730..5c76823b 100644 --- a/packages/core/tests/export.test.ts +++ b/packages/core/tests/export.test.ts @@ -130,3 +130,50 @@ describe('exportToDuckDbSql', () => { ); }); }); + +describe('export metadata payloads', () => { + beforeEach(() => { + wasmModuleMock.default.mockClear(); + wasmModuleMock.default.mockImplementation(async () => undefined); + wasmModuleMock.export_html.mockClear(); + wasmModuleMock.export_html.mockImplementation(() => ''); + wasmModuleMock.export_filename.mockClear(); + wasmModuleMock.export_filename.mockImplementation(() => 'flowscope_export.xlsx'); + }); + + afterEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + it('serializes camelCase HTML export metadata', async () => { + const { exportHtml } = await loadAnalyzer(); + const exportedAt = new Date('2026-01-18T12:30:05Z'); + + await exportHtml(baseResult, { projectName: 'Metadata Case', exportedAt }); + + const payload = JSON.parse(wasmModuleMock.export_html.mock.calls[0][0]); + expect(payload.projectName).toBe('Metadata Case'); + expect(payload.exportedAt).toBe(exportedAt.toISOString()); + expect(payload.project_name).toBeUndefined(); + expect(payload.exported_at).toBeUndefined(); + }); + + it('serializes camelCase filename metadata and format', async () => { + const { exportFilename } = await loadAnalyzer(); + const exportedAt = new Date('2026-01-18T12:30:05Z'); + + await exportFilename({ + projectName: 'Metadata Case', + exportedAt, + format: 'xlsx', + }); + + const payload = JSON.parse(wasmModuleMock.export_filename.mock.calls[0][0]); + expect(payload.projectName).toBe('Metadata Case'); + expect(payload.exportedAt).toBe(exportedAt.toISOString()); + expect(payload.format).toEqual({ type: 'xlsx' }); + expect(payload.project_name).toBeUndefined(); + expect(payload.exported_at).toBeUndefined(); + }); +}); diff --git a/packages/core/wasm/package.json b/packages/core/wasm/package.json index 4778fd1b..3eac41b5 100644 --- a/packages/core/wasm/package.json +++ b/packages/core/wasm/package.json @@ -5,7 +5,7 @@ "PondPilot Team" ], "description": "WASM bindings for flowscope-core", - "version": "0.9.0", + "version": "0.9.1", "license": "Apache-2.0", "repository": { "type": "git", diff --git a/packages/react/package.json b/packages/react/package.json index a751797b..ba8242ba 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -1,6 +1,6 @@ { "name": "@pondpilot/flowscope-react", - "version": "0.9.0", + "version": "0.9.1", "private": true, "description": "React components for FlowScope SQL lineage visualization", "type": "module", @@ -32,7 +32,7 @@ "lint:fix": "eslint src --fix" }, "peerDependencies": { - "@pondpilot/flowscope-core": "^0.9.0", + "@pondpilot/flowscope-core": "^0.9.1", "react": ">=19.0.0", "react-dom": ">=19.0.0" }, diff --git a/scripts/test_wasm_browser.mjs b/scripts/test_wasm_browser.mjs index 80cd1d1c..55b235ed 100644 --- a/scripts/test_wasm_browser.mjs +++ b/scripts/test_wasm_browser.mjs @@ -43,6 +43,15 @@ const harness = ` .map((node) => node.label); const errors = result.issues.filter((issue) => issue.severity === 'error'); + const mssqlRequest = { + sql: 'SELECT 1;\nGO\nSELECT 2;\nGO\n', + dialect: 'mssql', + }; + const mssqlResult = JSON.parse(analyze_sql_json(JSON.stringify(mssqlRequest))); + const mssqlParseErrors = mssqlResult.issues.filter( + (issue) => issue.code === 'PARSE_ERROR' + ); + if (result.statements.length !== 1 || result.summary.statementCount !== 1) { throw new Error('Expected one analyzed statement'); } @@ -52,12 +61,18 @@ const harness = ` if (errors.length > 0) { throw new Error('Analysis returned errors: ' + JSON.stringify(errors)); } + if (mssqlResult.statements.length !== 2 || mssqlParseErrors.length > 0) { + throw new Error( + 'MSSQL GO batch analysis failed: ' + JSON.stringify(mssqlResult.issues) + ); + } body.dataset.status = 'passed'; body.textContent = JSON.stringify({ version: get_version(), statementCount: result.summary.statementCount, - tableLabels + tableLabels, + mssqlStatementCount: mssqlResult.summary.statementCount, }); } catch (error) { body.dataset.status = 'failed'; diff --git a/yarn.lock b/yarn.lock index 5d6f309a..b73154a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -704,10 +704,10 @@ integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== "@pondpilot/flowscope-core@file:packages/core": - version "0.9.0" + version "0.9.1" "@pondpilot/flowscope-react@file:packages/react": - version "0.9.0" + version "0.9.1" dependencies: "@codemirror/autocomplete" "^6.18.0" "@codemirror/lang-sql" "^6.8.0"