CLI: add --workflows flag to merge and deploy

.changeset/few-cobras-pick.md

@@ -0,0 +1,7 @@
+---
+'@openfn/cli': minor
+---
+
+Allow users to specifiy which workflows to deploy or merge by passing `-w`.
+
+NOTE: the `-w` alias has been repurposed from `--workspace` to `--workflow`. This may affect your local development environment. If so, just expand `-w` to `--workspace`.

.changeset/some-squids-follow.md

@@ -0,0 +1,5 @@
+---
+'@openfn/lightning-mock': patch
+---
+
+Ensure that projects added as JSON are deeply cloned, preventing scribbles

integration-tests/cli/test/project-v1.test.ts

@@ -108,7 +108,9 @@ workflows:
 const projectsPath = path.resolve(TMP_DIR);
 
 test.before(async () => {
-  // await rm(TMP_DIR, { recursive: true });
+  try {
+    await rm(TMP_DIR, { recursive: true });
+  } catch (e) {}
   await mkdir(`${TMP_DIR}/.projects`, { recursive: true });
 
   await writeFile(`${TMP_DIR}/openfn.yaml`, '');
@@ -120,7 +122,7 @@ test.before(async () => {
 });
 
 test.serial('list available projects', async (t) => {
-  const { stdout } = await run(`openfn projects -w ${projectsPath}`);
+  const { stdout } = await run(`openfn projects --workspace ${projectsPath}`);
 
   t.regex(stdout, /hello-world/);
   t.regex(stdout, /8dbc4349-52b4-4bf2-be10-fdf06da52c46/);
@@ -130,7 +132,7 @@ test.serial('list available projects', async (t) => {
 
 // checkout a project from a yaml file
 test.serial('Checkout a project', async (t) => {
-  await run(`openfn checkout hello-world -w ${projectsPath}`);
+  await run(`openfn checkout hello-world --workspace ${projectsPath}`);
 
   // check workflow.yaml
   const workflowYaml = await readFile(
@@ -167,7 +169,7 @@ steps:
 // note: order of tests is important here
 test.serial('execute a workflow from the checked out project', async (t) => {
   // cheeky bonus test of checkout by alias
-  await run(`openfn checkout main -w ${projectsPath}`);
+  await run(`openfn checkout main --workspace ${projectsPath}`);
 
   // execute a workflow
   const { stdout } = await run(
@@ -191,7 +193,9 @@ test.serial('merge a project', async (t) => {
   t.is(initial, 'fn(() => ({ x: 1}))');
 
   // Run the merge
-  await run(`openfn merge hello-world-staging -w ${projectsPath} --force`);
+  await run(
+    `openfn merge hello-world-staging --workspace ${projectsPath} --force`
+  );
 
   // Check the step is updated
   const merged = await readStep();

integration-tests/cli/test/project-v2.test.ts

@@ -131,15 +131,15 @@ test.before(async () => {
 });
 
 test.serial('list available projects', async (t) => {
-  const { stdout } = await run(`openfn projects -w ${TMP_DIR}`);
+  const { stdout } = await run(`openfn projects --workspace ${TMP_DIR}`);
   t.regex(stdout, /sandboxing-simple/);
   t.regex(stdout, /a272a529-716a-4de7-a01c-a082916c6d23/);
   t.regex(stdout, /staging/);
   t.regex(stdout, /bc6629fb-7dc8-4b28-93af-901e2bd58dc4/);
 });
 
 test.serial('Checkout a project', async (t) => {
-  await run(`openfn checkout staging -w ${TMP_DIR}`);
+  await run(`openfn checkout staging --workspace ${TMP_DIR}`);
 
   // check workflow.yaml
   const workflowYaml = await readFile(
@@ -174,7 +174,7 @@ steps:
 
 test.serial('execute a workflow from the checked out project', async (t) => {
   // cheeky bonus test of checkout by alias
-  await run(`openfn checkout main -w ${TMP_DIR}`);
+  await run(`openfn checkout main --workspace ${TMP_DIR}`);
 
   // execute a workflow
   await run(
@@ -189,7 +189,7 @@ test.serial('execute a workflow from the checked out project', async (t) => {
 test.serial(
   'execute a workflow from the checked out project with a credential map',
   async (t) => {
-    await run(`openfn checkout main --log debug -w ${TMP_DIR}`);
+    await run(`openfn checkout main --log debug --workspace ${TMP_DIR}`);
 
     // Modify the checked out workflow code
     await writeFile(
@@ -248,7 +248,7 @@ test.serial(
     // Important: the collection value MUST be as string
     server.collections.upsert('stuff', 'x', JSON.stringify({ id: 'x' }));
 
-    await run(`openfn checkout main --log debug -w ${TMP_DIR}`);
+    await run(`openfn checkout main --log debug --workspace ${TMP_DIR}`);
 
     // Modify the checked out workflow code
     await writeFile(
@@ -313,7 +313,7 @@ workspace:
 );
 
 test.serial('merge a project', async (t) => {
-  await run(`openfn checkout main -w ${TMP_DIR}`);
+  await run(`openfn checkout main --workspace ${TMP_DIR}`);
 
   const readStep = () =>
     readFile(
@@ -326,7 +326,9 @@ test.serial('merge a project', async (t) => {
   t.is(initial, 'fn(() => ({ x: 1}))');
 
   // Run the merge
-  const { stdout } = await run(`openfn merge staging -w ${TMP_DIR} --force`);
+  const { stdout } = await run(
+    `openfn merge staging --workspace ${TMP_DIR} --force`
+  );
 
   // Check the step is updated
   const merged = await readStep();

packages/cli/src/projects/deploy.ts

@@ -1,5 +1,9 @@
 import yargs from 'yargs';
-import Project, { versionsEqual, Workspace } from '@openfn/project';
+import Project, {
+  MergeProjectOptions,
+  versionsEqual,
+  Workspace,
+} from '@openfn/project';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
 
@@ -42,6 +46,7 @@ export type DeployOptions = Pick<
   name?: string;
   alias?: string;
   jsonDiff?: boolean;
+  workflow?: string[];
 };
 
 const options = [
@@ -53,6 +58,7 @@ const options = [
   o2.name,
   o2.alias,
   o2.jsonDiff,
+  o2.workflow,
 
   // general options
   o.apiKey,
@@ -170,14 +176,26 @@ const syncProjects = async (
     // this will actually happen later
   }
 
-  const locallyChangedWorkflows = await findLocallyChangedWorkflows(
-    ws,
-    localProject
-  );
+  let mergeCandidates: string[];
+  if (options.workflow?.length) {
+    const missing = options.workflow.filter(
+      (id) => !localProject.workflows.some((w) => w.id === id)
+    );
+    if (missing.length) {
+      throw new Error(
+        `The following workflows were not found in local project ${
+          localProject.id
+        }: ${missing.join(', ')}`
+      );
+    }
+    mergeCandidates = options.workflow;
+  } else {
+    mergeCandidates = await findLocallyChangedWorkflows(ws, localProject);
+  }
 
   // TODO: what if remote diff and the version checked disagree for some reason?
-  const diffs = locallyChangedWorkflows.length
-    ? remoteProject.diff(localProject, locallyChangedWorkflows)
+  const diffs = mergeCandidates.length
+    ? remoteProject.diff(localProject, mergeCandidates)
     : [];
 
   if (!diffs.length) {
@@ -203,7 +221,7 @@ const syncProjects = async (
     const divergentWorkflows = hasRemoteDiverged(
       localProject,
       remoteProject!,
-      locallyChangedWorkflows
+      mergeCandidates
     );
     if (divergentWorkflows) {
       logger.warn(
@@ -231,16 +249,24 @@ const syncProjects = async (
   }
 
   logger.info('Merging changes into remote project');
-  // TODO I would like to log which workflows are being updated
-  const merged = Project.merge(localProject, remoteProject!, {
+  const mergeOptions: MergeProjectOptions = {
     // If pushing the same project, we use a replace strategy
     // Otherwise, use the sandbox strategy to preserve UUIDs
     mode: localProject.uuid === remoteProject.uuid ? 'replace' : 'sandbox',
     force: true,
-    onlyUpdated: true,
-  });
+  };
+  if (options.workflow?.length) {
+    // If --workflow is passed, force-include exactly the listed workflows via workflowMappings
+    mergeOptions.workflowMappings = Object.fromEntries(
+      options.workflow.map((id) => [id, id])
+    );
+  } else {
+    // Otherwise only merge locally updated workflows
+    mergeOptions.onlyUpdated = true;
+  }
+  const merged = Project.merge(localProject, remoteProject!, mergeOptions);
 
-  return { merged, remoteProject, locallyChangedWorkflows };
+  return { merged, remoteProject, locallyChangedWorkflows: mergeCandidates };
 };
 
 export async function handler(options: DeployOptions, logger: Logger) {

packages/cli/src/projects/merge.ts

@@ -17,11 +17,12 @@ export type MergeOptions = Required<
     'command' | 'project' | 'workspace' | 'removeUnmapped' | 'workflowMappings'
   >
 > &
-  Pick<Opts, 'log' | 'force' | 'outputPath'> & { base?: string };
+  Pick<Opts, 'log' | 'force' | 'outputPath' | 'workflow'> & { base?: string };
 
 const options = [
   po.removeUnmapped,
   po.workflowMappings,
+  po.workflow,
   po.workspace,
   o.log,
   // custom output because we don't want defaults or anything
@@ -109,6 +110,29 @@ export const handler = async (options: MergeOptions, logger: Logger) => {
     logger.error('The checked out project has no id');
     return;
   }
+
+  let workflowMappings = options.workflowMappings;
+  if (options.workflow?.length) {
+    if (workflowMappings && Object.keys(workflowMappings).length) {
+      logger.error('--workflow and --workflow-mappings are mutually exclusive');
+      return;
+    }
+    const missing = options.workflow.filter(
+      (id) => !sourceProject.workflows.some((w) => w.id === id)
+    );
+    if (missing.length) {
+      logger.error(
+        `The following workflows were not found in source project ${
+          sourceProject.id
+        }: ${missing.join(', ')}`
+      );
+      return;
+    }
+    workflowMappings = Object.fromEntries(
+      options.workflow.map((id) => [id, id])
+    );
+  }
+
   const finalPath =
     options.outputPath ?? workspace.getProjectPath(targetProject.id);
   if (!finalPath) {
@@ -117,7 +141,7 @@ export const handler = async (options: MergeOptions, logger: Logger) => {
   }
   const final = Project.merge(sourceProject, targetProject, {
     removeUnmapped: options.removeUnmapped,
-    workflowMappings: options.workflowMappings,
+    workflowMappings,
     force: options.force,
   });
 

packages/cli/src/projects/options.ts

@@ -8,6 +8,7 @@ export type Opts = BaseOpts & {
   workspace?: string;
   removeUnmapped?: boolean | undefined;
   workflowMappings?: Record<string, string> | undefined;
+  workflow?: string[];
   project?: string;
   format?: 'yaml' | 'json' | 'state';
   clean?: boolean;
@@ -86,6 +87,22 @@ export const workflowMappings: CLIOption = {
   },
 };
 
+export const workflow: CLIOption = {
+  name: 'workflow',
+  yargs: {
+    alias: ['w'],
+    array: true,
+    description:
+      'Restrict merge/deploy to the given workflow ids. Pass multiple times to include multiple workflows. Listed workflows are force-included from the source and will overwrite the target/remote even if unchanged locally. Mutually exclusive with --workflow-mappings.',
+  },
+  ensure: (opts: any) => {
+    if (opts.workflow?.length) {
+      opts.workflow = Array.from(new Set(opts.workflow));
+    }
+    delete opts.w;
+  },
+};
+
 // We declare a new output path here, overriding the default cli one,
 // because default rules are different
 export const outputPath: CLIOption = {
@@ -100,7 +117,6 @@ export const outputPath: CLIOption = {
 export const workspace: CLIOption = {
   name: 'workspace',
   yargs: {
-    alias: ['w'],
     description: 'Path to the project workspace (ie, path to openfn.yaml)',
   },
   ensure: (opts: any) => {