Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions core/actions/assertion.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos";
import { ActionBuilder } from "df/core/actions";
import { IActionContext, Resolvable } from "df/core/contextables";
import { IActionContext, JitContextable, Resolvable } from "df/core/contextables";
import * as Path from "df/core/path";
import { Session } from "df/core/session";
import {
Expand Down Expand Up @@ -32,6 +32,9 @@ interface ILegacyAssertionConfig extends dataform.ActionConfig.AssertionConfig {
/** @hidden */
export type AContextable<T> = T | ((ctx: AssertionContext) => T);

/** JiT compilation stage result for assertions. */
export type JitAssertionResult = string;

/**
* An assertion is a data quality test query that finds rows that violate one or more conditions
* specified in the query. If the query returns any rows, the assertion fails.
Expand Down Expand Up @@ -90,6 +93,9 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
/** @hidden We delay contextification until the final compile step, so hold these here for now. */
private contextableQuery: AContextable<string>;

/** @hidden */
private contextableJitCode: JitContextable<AssertionContext, JitAssertionResult> | undefined;

/** @hidden */
constructor(session?: Session, unverifiedConfig?: any, configPath?: string) {
super(session);
Expand Down Expand Up @@ -166,6 +172,15 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
return this;
}

public jitCode(jitCode: JitContextable<AssertionContext, JitAssertionResult>) {
if (!this.proto.actionDescriptor) {
this.proto.actionDescriptor = {};
}
this.proto.actionDescriptor.compilationMode = dataform.ActionCompilationMode.ACTION_COMPILATION_MODE_JIT;
this.contextableJitCode = jitCode;
return this;
}

/**
* @deprecated Deprecated in favor of
* [AssertionConfig.dependencies](configs#dataform-ActionConfig-AssertionConfig).
Expand Down Expand Up @@ -293,8 +308,15 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
public compile() {
const context = new AssertionContext(this);

this.proto.query = context.apply(this.contextableQuery);
validateQueryString(this.session, this.proto.query, this.proto.fileName);
if (this.contextableJitCode) {
if (!this.proto.actionDescriptor) {
this.proto.actionDescriptor = {};
}
this.proto.jitCode = this.contextableJitCode.toString();
} else {
this.proto.query = context.apply(this.contextableQuery);
validateQueryString(this.session, this.proto.query, this.proto.fileName);
}

return verifyObjectMatchesProto(
dataform.Assertion,
Expand Down
37 changes: 37 additions & 0 deletions core/main_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1537,6 +1537,43 @@ assert("name", {
]);
});

test("jitCode correctly populates the jitCode field", () => {
const projectDir = tmpDirFixture.createNewTmpDir();
fs.writeFileSync(
path.join(projectDir, "workflow_settings.yaml"),
VALID_WORKFLOW_SETTINGS_YAML
);
fs.mkdirSync(path.join(projectDir, "definitions"));
fs.writeFileSync(
path.join(projectDir, "definitions/assert.js"),
`
assert("name").jitCode(ctx => "jit");`
);

const result = runMainInVm(coreExecutionRequestFromPath(projectDir));

expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals([
{
actionDescriptor: {
compilationMode: "ACTION_COMPILATION_MODE_JIT"
},
canonicalTarget: {
database: "defaultProject",
name: "name",
schema: "defaultDataset"
},
fileName: "definitions/assert.js",
jitCode: 'ctx => "jit"',
target: {
database: "defaultProject",
name: "name",
schema: "defaultDataset"
}
}
]);
});

test("assert API returns disabled assertions when disableAssertions is true", () => {
const projectDir = tmpDirFixture.createNewTmpDir();
fs.writeFileSync(
Expand Down
25 changes: 17 additions & 8 deletions core/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,20 @@ import { default as TarjanGraphConstructor, Graph as TarjanGraph } from "tarjan-

import { encode64, unknownToValue, verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos";
import { Action, ActionProto, ILegacyTableConfig, TableType } from "df/core/actions";
import { AContextable, Assertion, AssertionContext } from "df/core/actions/assertion";
import { AContextable, Assertion, AssertionContext, JitAssertionResult } from "df/core/actions/assertion";
import {
DataPreparation,
DataPreparationContext,
} from "df/core/actions/data_preparation";
import { Declaration } from "df/core/actions/declaration";
import { IncrementalTable } from "df/core/actions/incremental_table";
import { Notebook } from "df/core/actions/notebook";
import { Operation, OperationContext } from "df/core/actions/operation";
import { Table, TableContext } from "df/core/actions/table";
import { JitOperationResult, Operation, OperationContext } from "df/core/actions/operation";
import { JitTableResult, Table, TableContext } from "df/core/actions/table";
import { Test } from "df/core/actions/test";
import { View } from "df/core/actions/view";
import { JitViewResult, View } from "df/core/actions/view";
import { CompilationSql } from "df/core/compilation_sql";
import { Contextable, IActionContext, ITableContext, Resolvable } from "df/core/contextables";
import { Contextable, IActionContext, ITableContext, JitContextable, Resolvable } from "df/core/contextables";
import { targetAsReadableString, targetStringifier } from "df/core/targets";
import * as utils from "df/core/utils";
import { ResolvableMap, toResolvable } from "df/core/utils";
Expand Down Expand Up @@ -113,6 +113,13 @@ export class Session {
contextable: (ctx: IActionContext) => string;
}
];
jitContextable?:
| JitContextable<TableContext, JitTableResult>
| JitContextable<TableContext, JitViewResult>
| JitContextable<AssertionContext, JitAssertionResult>
| JitContextable<OperationContext, JitOperationResult>
// Fallback for context types without explicit JIT results (e.g. DataPreparation)
| JitContextable<any, any>;
}) {
const { sqlxConfig } = actionOptions;
const actionType = sqlxConfig.hasOwnProperty("type") ? sqlxConfig.type : "operations";
Expand Down Expand Up @@ -190,9 +197,11 @@ export class Session {
this.actions.push(table);
break;
case "assertion":
this.actions.push(
new Assertion(this, sqlxConfig).query(ctx => actionOptions.sqlContextable(ctx)[0])
);
const assertion = new Assertion(this, sqlxConfig).query(ctx => actionOptions.sqlContextable(ctx)[0]);
if (actionOptions.jitContextable) {
assertion.jitCode(actionOptions.jitContextable as JitContextable<AssertionContext, JitAssertionResult>);
}
this.actions.push(assertion);
break;
case "dataPreparation":
const dataPreparation = new DataPreparation(this, sqlxConfig).query(
Expand Down
2 changes: 2 additions & 0 deletions protos/core.proto
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,8 @@ message Assertion {
// Only present for auto assertions.
Target parent_action = 15;

string jit_code = 16;

// Generated.
string file_name = 7;

Expand Down
Loading