feat: redirect pull & deploy command to projects

.changeset/vast-weeks-tap.md

@@ -0,0 +1,13 @@
+---
+'@openfn/cli': minor
+---
+
+Support github sync with new `project` commands
+
+GitHub Sync is currently designed with a pair of actions which trigger the legacy `openfn deploy` and `openfn pull` commands.
+
+This update adds support for the `openfn.yaml` file to the commands: so if you run `openfn project pull` in a github synced repo, the repo goes into v2 mode. The legacy deploy and pull commands will "redirect" to v2. You should leave the legacy `config.json` file, but state.json and spec.yaml files can be removed.
+
+Initaliasing GitHub sync from the app will continue to use the legacy file structure for the initial commit. If you want to switch to v2, either create an empty openfn.yaml and trigger a sync, or locally run `openfn project pull` and commit changes.
+
+Be aware that v2 sync only supports a single `openfn.yaml` and `workflows` folder at a time - so a sync which pulls from multiple connected apps will not work well. It should however be safe to push changes to multiple apps.

packages/cli/src/deploy/command.ts

@@ -1,12 +1,13 @@
 import yargs from 'yargs';
 import { build, ensure, override } from '../util/command-builders';
 import { Opts } from '../options';
+import { Opts as POpts } from '../projects/options';
 import * as o from '../options';
 import * as o2 from '../projects/options';
 
 export type DeployOptions = Required<
   Pick<
-    Opts,
+    Opts & POpts,
     | 'beta'
     | 'command'
     | 'configPath'
@@ -15,6 +16,7 @@ export type DeployOptions = Required<
     | 'logJson'
     | 'projectPath'
     | 'statePath'
+    | 'workspace'
   >
 >;
 

packages/cli/src/deploy/handler.ts

@@ -8,6 +8,8 @@ import {
 import type { Logger } from '../util/logger';
 import { DeployOptions } from './command';
 import * as beta from '../projects/deploy';
+import path from 'node:path';
+import { fileExists } from '../util/file-exists';
 
 export type DeployFn = typeof deploy;
 
@@ -32,6 +34,17 @@ async function deployHandler(
   try {
     const config = mergeOverrides(await getConfig(options.configPath), options);
 
+    const v2ConfigPath = path.join(
+      options.workspace || process.cwd(),
+      'openfn.yaml'
+    );
+    if (await fileExists(v2ConfigPath)) {
+      logger.always(
+        'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy)'
+      );
+      return beta.handler({ ...options, force: true }, logger);
+    }
+
     logger.debug('Deploying with config', JSON.stringify(config, null, 2));
 
     if (options.confirm === false) {
@@ -76,13 +89,18 @@ function mergeOverrides(
   config: DeployConfig,
   options: DeployOptions
 ): DeployConfig {
+  const workspace = options.workspace || process.cwd();
+  const resolveRelative = (p: string) =>
+    path.isAbsolute(p) ? p : path.join(workspace, p);
+  const specPath = pickFirst(options.projectPath, config.specPath);
+  const statePath = pickFirst(options.statePath, config.statePath);
   return {
     ...config,
     apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey),
     endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint),
-    statePath: pickFirst(options.statePath, config.statePath),
-    specPath: pickFirst(options.projectPath, config.specPath),
-    configPath: options.configPath,
+    statePath: resolveRelative(statePath),
+    specPath: resolveRelative(specPath),
+    configPath: resolveRelative(options.configPath),
     requireConfirmation: pickFirst(options.confirm, config.requireConfirmation),
   };
 }

packages/cli/src/projects/pull.ts

@@ -23,6 +23,7 @@ export type PullOptions = Pick<
   | 'project'
   | 'confirm'
   | 'snapshots'
+  | 'force'
 >;
 
 const options = [
@@ -61,6 +62,7 @@ export const command: yargs.CommandModule<PullOptions> = {
 };
 
 export async function handler(options: PullOptions, logger: Logger) {
+  options.workspace ??= process.cwd();
   ensureProjectId(options, logger);
 
   await fetch(options, logger);

packages/cli/src/pull/command.ts

@@ -1,12 +1,13 @@
 import yargs from 'yargs';
 import { build, ensure, override } from '../util/command-builders';
 import { Opts } from '../options';
+import { Opts as POpts } from '../projects/options';
 import * as o from '../options';
 import * as po from '../projects/options';
 
 export type PullOptions = Required<
   Pick<
-    Opts,
+    Opts & POpts,
     | 'beta'
     | 'command'
     | 'log'
@@ -17,6 +18,7 @@ export type PullOptions = Required<
     | 'projectId'
     | 'confirm'
     | 'snapshots'
+    | 'workspace'
   >
 >;
 

packages/cli/src/pull/handler.ts

@@ -11,6 +11,7 @@ import {
 import type { Logger } from '../util/logger';
 import { PullOptions } from '../pull/command';
 import beta from '../projects/pull';
+import { fileExists } from '../util/file-exists';
 
 async function pullHandler(options: PullOptions, logger: Logger) {
   if (options.beta) {
@@ -21,6 +22,20 @@ async function pullHandler(options: PullOptions, logger: Logger) {
   try {
     const config = mergeOverrides(await getConfig(options.configPath), options);
 
+    const v2ConfigPath = path.join(
+      options.workspace || process.cwd(),
+      'openfn.yaml'
+    );
+    if (await fileExists(v2ConfigPath)) {
+      logger.always(
+        'Detected openfn.yaml file - switching to v2 pull (openfn project pull)'
+      );
+      return beta(
+        { ...options, project: options.projectId, force: true },
+        logger
+      );
+    }
+
     if (process.env['OPENFN_API_KEY']) {
       logger.info('Using OPENFN_API_KEY environment variable');
       config.apiKey = process.env['OPENFN_API_KEY'];
@@ -127,11 +142,20 @@ function mergeOverrides(
   config: DeployConfig,
   options: PullOptions
 ): DeployConfig {
+  const workspace = options.workspace || process.cwd();
   return {
     ...config,
     apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey),
     endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint),
-    configPath: options.configPath,
+    configPath: path.isAbsolute(options.configPath)
+      ? options.configPath
+      : path.join(workspace, options.configPath),
+    specPath: path.isAbsolute(config.specPath)
+      ? config.specPath
+      : path.join(workspace, config.specPath),
+    statePath: path.isAbsolute(config.statePath)
+      ? config.statePath
+      : path.join(workspace, config.statePath),
     requireConfirmation: pickFirst(options.confirm, config.requireConfirmation),
   };
 }

packages/cli/src/util/file-exists.ts

@@ -0,0 +1,10 @@
+import fs from 'fs/promises';
+
+export async function fileExists(filePath: string) {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.isFile();
+  } catch (error) {
+    return false;
+  }
+}

packages/cli/test/pull/handler.test.ts

@@ -0,0 +1,38 @@
+import test from 'ava';
+import mockfs from 'mock-fs';
+import { createMockLogger } from '@openfn/logger';
+import pullHandler from '../../src/pull/handler';
+import { PullOptions } from '../../src/pull/command';
+
+test.beforeEach(() => {
+  mockfs.restore();
+});
+
+test.afterEach(() => {
+  mockfs.restore();
+});
+
+const options: PullOptions = {
+  beta: false,
+  command: 'pull',
+  projectPath: './project.yaml',
+  configPath: './config.json',
+  projectId: 'abc-123',
+  confirm: false,
+  snapshots: [],
+};
+
+test.serial(
+  'redirects to beta handler when openfn.yaml exists in cwd',
+  async (t) => {
+    const logger = createMockLogger('', { level: 'debug' });
+    mockfs({
+      ['./config.json']: `{"apiKey": "123"}`,
+      ['./openfn.yaml']: '',
+    });
+
+    await t.throwsAsync(() => pullHandler(options, logger));
+
+    t.truthy(logger._find('always', /Detected openfn.yaml file/i));
+  }
+);

packages/cli/test/util/file-exists.test.ts

@@ -0,0 +1,22 @@
+import test from 'ava';
+import mockfs from 'mock-fs';
+import { fileExists } from '../../src/util/file-exists';
+
+test.afterEach(() => {
+  mockfs.restore();
+});
+
+test('returns true for an existing file', async (t) => {
+  mockfs({ './test.txt': 'content' });
+  t.true(await fileExists('./test.txt'));
+});
+
+test('returns false for a non-existent path', async (t) => {
+  mockfs({});
+  t.false(await fileExists('./nonexistent.txt'));
+});
+
+test('returns false for a directory', async (t) => {
+  mockfs({ './mydir': {} });
+  t.false(await fileExists('./mydir'));
+});