diff --git a/.changeset/vast-weeks-tap.md b/.changeset/vast-weeks-tap.md new file mode 100644 index 000000000..953c4560a --- /dev/null +++ b/.changeset/vast-weeks-tap.md @@ -0,0 +1,13 @@ +--- +'@openfn/cli': minor +--- + +Support github sync with new `project` commands + +GitHub Sync is currently designed with a pair of actions which trigger the legacy `openfn deploy` and `openfn pull` commands. + +This update adds support for the `openfn.yaml` file to the commands: so if you run `openfn project pull` in a github synced repo, the repo goes into v2 mode. The legacy deploy and pull commands will "redirect" to v2. You should leave the legacy `config.json` file, but state.json and spec.yaml files can be removed. + +Initaliasing GitHub sync from the app will continue to use the legacy file structure for the initial commit. If you want to switch to v2, either create an empty openfn.yaml and trigger a sync, or locally run `openfn project pull` and commit changes. + +Be aware that v2 sync only supports a single `openfn.yaml` and `workflows` folder at a time - so a sync which pulls from multiple connected apps will not work well. It should however be safe to push changes to multiple apps. diff --git a/packages/cli/src/deploy/command.ts b/packages/cli/src/deploy/command.ts index 79ed15cc4..6b144f8ee 100644 --- a/packages/cli/src/deploy/command.ts +++ b/packages/cli/src/deploy/command.ts @@ -1,12 +1,13 @@ import yargs from 'yargs'; import { build, ensure, override } from '../util/command-builders'; import { Opts } from '../options'; +import { Opts as POpts } from '../projects/options'; import * as o from '../options'; import * as o2 from '../projects/options'; export type DeployOptions = Required< Pick< - Opts, + Opts & POpts, | 'beta' | 'command' | 'configPath' @@ -15,6 +16,7 @@ export type DeployOptions = Required< | 'logJson' | 'projectPath' | 'statePath' + | 'workspace' > >; diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index c61b70ba9..4e834a0fb 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -8,6 +8,8 @@ import { import type { Logger } from '../util/logger'; import { DeployOptions } from './command'; import * as beta from '../projects/deploy'; +import path from 'node:path'; +import { fileExists } from '../util/file-exists'; export type DeployFn = typeof deploy; @@ -32,6 +34,17 @@ async function deployHandler( try { const config = mergeOverrides(await getConfig(options.configPath), options); + const v2ConfigPath = path.join( + options.workspace || process.cwd(), + 'openfn.yaml' + ); + if (await fileExists(v2ConfigPath)) { + logger.always( + 'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy)' + ); + return beta.handler({ ...options, force: true }, logger); + } + logger.debug('Deploying with config', JSON.stringify(config, null, 2)); if (options.confirm === false) { @@ -76,13 +89,18 @@ function mergeOverrides( config: DeployConfig, options: DeployOptions ): DeployConfig { + const workspace = options.workspace || process.cwd(); + const resolveRelative = (p: string) => + path.isAbsolute(p) ? p : path.join(workspace, p); + const specPath = pickFirst(options.projectPath, config.specPath); + const statePath = pickFirst(options.statePath, config.statePath); return { ...config, apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey), endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint), - statePath: pickFirst(options.statePath, config.statePath), - specPath: pickFirst(options.projectPath, config.specPath), - configPath: options.configPath, + statePath: resolveRelative(statePath), + specPath: resolveRelative(specPath), + configPath: resolveRelative(options.configPath), requireConfirmation: pickFirst(options.confirm, config.requireConfirmation), }; } diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts index 6a3492561..1df4af557 100644 --- a/packages/cli/src/projects/pull.ts +++ b/packages/cli/src/projects/pull.ts @@ -23,6 +23,7 @@ export type PullOptions = Pick< | 'project' | 'confirm' | 'snapshots' + | 'force' >; const options = [ @@ -61,6 +62,7 @@ export const command: yargs.CommandModule = { }; export async function handler(options: PullOptions, logger: Logger) { + options.workspace ??= process.cwd(); ensureProjectId(options, logger); await fetch(options, logger); diff --git a/packages/cli/src/pull/command.ts b/packages/cli/src/pull/command.ts index ba6dd6620..314164623 100644 --- a/packages/cli/src/pull/command.ts +++ b/packages/cli/src/pull/command.ts @@ -1,12 +1,13 @@ import yargs from 'yargs'; import { build, ensure, override } from '../util/command-builders'; import { Opts } from '../options'; +import { Opts as POpts } from '../projects/options'; import * as o from '../options'; import * as po from '../projects/options'; export type PullOptions = Required< Pick< - Opts, + Opts & POpts, | 'beta' | 'command' | 'log' @@ -17,6 +18,7 @@ export type PullOptions = Required< | 'projectId' | 'confirm' | 'snapshots' + | 'workspace' > >; diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts index fe96e7188..2981ce5ed 100644 --- a/packages/cli/src/pull/handler.ts +++ b/packages/cli/src/pull/handler.ts @@ -11,6 +11,7 @@ import { import type { Logger } from '../util/logger'; import { PullOptions } from '../pull/command'; import beta from '../projects/pull'; +import { fileExists } from '../util/file-exists'; async function pullHandler(options: PullOptions, logger: Logger) { if (options.beta) { @@ -21,6 +22,20 @@ async function pullHandler(options: PullOptions, logger: Logger) { try { const config = mergeOverrides(await getConfig(options.configPath), options); + const v2ConfigPath = path.join( + options.workspace || process.cwd(), + 'openfn.yaml' + ); + if (await fileExists(v2ConfigPath)) { + logger.always( + 'Detected openfn.yaml file - switching to v2 pull (openfn project pull)' + ); + return beta( + { ...options, project: options.projectId, force: true }, + logger + ); + } + if (process.env['OPENFN_API_KEY']) { logger.info('Using OPENFN_API_KEY environment variable'); config.apiKey = process.env['OPENFN_API_KEY']; @@ -127,11 +142,20 @@ function mergeOverrides( config: DeployConfig, options: PullOptions ): DeployConfig { + const workspace = options.workspace || process.cwd(); return { ...config, apiKey: pickFirst(process.env['OPENFN_API_KEY'], config.apiKey), endpoint: pickFirst(process.env['OPENFN_ENDPOINT'], config.endpoint), - configPath: options.configPath, + configPath: path.isAbsolute(options.configPath) + ? options.configPath + : path.join(workspace, options.configPath), + specPath: path.isAbsolute(config.specPath) + ? config.specPath + : path.join(workspace, config.specPath), + statePath: path.isAbsolute(config.statePath) + ? config.statePath + : path.join(workspace, config.statePath), requireConfirmation: pickFirst(options.confirm, config.requireConfirmation), }; } diff --git a/packages/cli/src/util/file-exists.ts b/packages/cli/src/util/file-exists.ts new file mode 100644 index 000000000..97c6e61d9 --- /dev/null +++ b/packages/cli/src/util/file-exists.ts @@ -0,0 +1,10 @@ +import fs from 'fs/promises'; + +export async function fileExists(filePath: string) { + try { + const stats = await fs.stat(filePath); + return stats.isFile(); + } catch (error) { + return false; + } +} diff --git a/packages/cli/test/pull/handler.test.ts b/packages/cli/test/pull/handler.test.ts new file mode 100644 index 000000000..3ef838dff --- /dev/null +++ b/packages/cli/test/pull/handler.test.ts @@ -0,0 +1,38 @@ +import test from 'ava'; +import mockfs from 'mock-fs'; +import { createMockLogger } from '@openfn/logger'; +import pullHandler from '../../src/pull/handler'; +import { PullOptions } from '../../src/pull/command'; + +test.beforeEach(() => { + mockfs.restore(); +}); + +test.afterEach(() => { + mockfs.restore(); +}); + +const options: PullOptions = { + beta: false, + command: 'pull', + projectPath: './project.yaml', + configPath: './config.json', + projectId: 'abc-123', + confirm: false, + snapshots: [], +}; + +test.serial( + 'redirects to beta handler when openfn.yaml exists in cwd', + async (t) => { + const logger = createMockLogger('', { level: 'debug' }); + mockfs({ + ['./config.json']: `{"apiKey": "123"}`, + ['./openfn.yaml']: '', + }); + + await t.throwsAsync(() => pullHandler(options, logger)); + + t.truthy(logger._find('always', /Detected openfn.yaml file/i)); + } +); diff --git a/packages/cli/test/util/file-exists.test.ts b/packages/cli/test/util/file-exists.test.ts new file mode 100644 index 000000000..a6c8ce7b4 --- /dev/null +++ b/packages/cli/test/util/file-exists.test.ts @@ -0,0 +1,22 @@ +import test from 'ava'; +import mockfs from 'mock-fs'; +import { fileExists } from '../../src/util/file-exists'; + +test.afterEach(() => { + mockfs.restore(); +}); + +test('returns true for an existing file', async (t) => { + mockfs({ './test.txt': 'content' }); + t.true(await fileExists('./test.txt')); +}); + +test('returns false for a non-existent path', async (t) => { + mockfs({}); + t.false(await fileExists('./nonexistent.txt')); +}); + +test('returns false for a directory', async (t) => { + mockfs({ './mydir': {} }); + t.false(await fileExists('./mydir')); +});