diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml index 0d60eed6..e02fdb0d 100644 --- a/.github/workflows/generate.yml +++ b/.github/workflows/generate.yml @@ -76,36 +76,36 @@ jobs: fail-fast: false matrix: include: - - target: '@node-core/doc-kit/generators/man-page' + - target: man-page input: './node/doc/api/cli.md' - - target: '@node-core/doc-kit/generators/addon-verify' + - target: addon-verify input: './node/doc/api/addons.md' - - target: '@node-core/doc-kit/generators/api-links' + - target: api-links input: './node/lib/*.js' compare: object-assertion - - target: '@node-core/doc-kit/generators/orama-db' + - target: orama-db input: './node/doc/api/*.md' compare: file-size - - target: '@node-core/doc-kit/generators/json-simple' + - target: json-simple input: './node/doc/api/*.md' - - target: '@node-core/doc-kit/generators/legacy-json' + - target: legacy-json input: './node/doc/api/*.md' compare: object-assertion - - target: '@node-core/doc-kit/generators/legacy-html' + - target: legacy-html input: './node/doc/api/*.md' compare: file-size - - target: '@node-core/doc-kit/generators/web' + - target: web input: './node/doc/api/*.md' compare: file-size - - target: '@node-core/doc-kit/generators/llms-txt' + - target: llms-txt input: './node/doc/api/*.md' compare: file-size steps: @@ -140,14 +140,10 @@ jobs: - name: Install dependencies run: npm ci - - name: Get generator name - id: generator - run: echo "name=$(node -e "import('${{ matrix.target }}').then(m => console.log(m.name))")" >> "$GITHUB_OUTPUT" - - name: Create output directory - run: mkdir -p out/${{ steps.generator.outputs.name }} + run: mkdir -p out/${{ matrix.target }} - - name: Generate ${{ steps.generator.outputs.name }} + - name: Generate ${{ matrix.target }} run: | node packages/core/bin/cli.mjs generate \ -t ${{ matrix.target }} \ @@ -161,7 +157,7 @@ jobs: if: ${{ matrix.compare && needs.prepare.outputs.base-run }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ steps.generator.outputs.name }} + name: ${{ matrix.target }} path: base run-id: ${{ needs.prepare.outputs.base-run }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -173,8 +169,8 @@ jobs: run: | node scripts/comparators/${{ matrix.compare }}.mjs > out/comparison.txt - - name: Upload ${{ steps.generator.outputs.name }} artifacts + - name: Upload ${{ matrix.target }} artifacts uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: ${{ steps.generator.outputs.name }} + name: ${{ matrix.target }} path: out diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b9b29890..78f3337c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,7 +89,7 @@ The steps below will give you a general idea of how to prepare your local enviro For fast iteration during development, target a single Markdown file instead of all API docs: ```bash - node packages/core/bin/cli.mjs generate \ + node bin/cli.mjs generate \ -t legacy-html \ -i ../node/doc/api/fs.md \ -o out \ @@ -110,7 +110,7 @@ The steps below will give you a general idea of how to prepare your local enviro Add `--log-level debug` before the `generate` subcommand to see the full pipeline trace: ```bash - node packages/core/bin/cli.mjs --log-level debug generate -t legacy-html -i ../node/doc/api/fs.md -o out + node bin/cli.mjs --log-level debug generate -t legacy-html -i ../node/doc/api/fs.md -o out ``` > [!TIP] diff --git a/README.md b/README.md index 47a3c9c2..607cf69f 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Options: -v, --version Target Node.js version (default: "v22.14.0") -c, --changelog Changelog URL or path (default: "https://raw.githubusercontent.com/nodejs/node/HEAD/CHANGELOG.md") --git-ref Git ref/commit URL (default: "https://github.com/nodejs/node/tree/HEAD") - -t, --target Target generator(s) (e.g. @node-core/doc-kit/generators/web) + -t, --target [modes...] Target generator modes (choices: "json-simple", "legacy-html", "legacy-html-all", "man-page", "legacy-json", "legacy-json-all", "addon-verify", "api-links", "orama-db", "llms-txt") -h, --help display help for command ``` @@ -91,8 +91,8 @@ To generate a 1:1 match with the [legacy tooling](https://github.com/nodejs/node ```sh npx doc-kit generate \ - -t @node-core/doc-kit/generators/legacy-html \ - -t @node-core/doc-kit/generators/legacy-json \ + -t legacy-html \ + -t legacy-json \ -i "path/to/node/doc/api/*.md" \ -o out \ --index path/to/node/doc/api/index.md @@ -104,8 +104,8 @@ To generate [our redesigned documentation pages](https://nodejs-api-docs-tooling ```sh npx doc-kit generate \ - -t @node-core/doc-kit/generators/web \ - -t @node-core/doc-kit/generators/orama-db \ + -t web \ + -t orama-db \ -i "path/to/node/doc/api/*.md" \ -o out \ --index path/to/node/doc/api/index.md diff --git a/docs/commands.md b/docs/commands.md index 1703aeed..9a78f7af 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -69,7 +69,7 @@ export default [ ### Step 3: Update CLI Entry Point -The CLI in `packages/core/bin/cli.mjs` automatically loads commands from `bin/commands/index.mjs`, so no changes are needed there if you followed step 2. +The CLI in `bin/cli.mjs` automatically loads commands from `bin/commands/index.mjs`, so no changes are needed there if you followed step 2. ## Command Options diff --git a/docs/configuration.md b/docs/configuration.md index 26353b43..9a073e74 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -100,17 +100,17 @@ Configurations are merged in the following order (earlier sources take precedenc CLI options map to configuration properties: -| CLI Option | Config Property | Example | -| ---------------------- | ------------------ | ---------------------------------------------------- | -| `--input ` | `global.input` | `--input src/` | -| `--output ` | `global.output` | `--output dist/` | -| `--ignore ` | `global.ignore[]` | `--ignore test/` | -| `--minify` | `global.minify` | `--minify` | -| `--git-ref ` | `global.ref` | `--git-ref v20.0.0` | -| `--version ` | `global.version` | `--version 20.0.0` | -| `--changelog ` | `global.changelog` | `--changelog https://...` | -| `--index ` | `global.index` | `--index file://...` | -| `--type-map ` | `metadata.typeMap` | `--type-map file://...` | -| `--target ` | `target` | `--target @node-core/doc-kit/generators/legacy-json` | -| `--threads ` | `threads` | `--threads 4` | -| `--chunk-size ` | `chunkSize` | `--chunk-size 10` | +| CLI Option | Config Property | Example | +| ---------------------- | ------------------ | ------------------------- | +| `--input ` | `global.input` | `--input src/` | +| `--output ` | `global.output` | `--output dist/` | +| `--ignore ` | `global.ignore[]` | `--ignore test/` | +| `--minify` | `global.minify` | `--minify` | +| `--git-ref ` | `global.ref` | `--git-ref v20.0.0` | +| `--version ` | `global.version` | `--version 20.0.0` | +| `--changelog ` | `global.changelog` | `--changelog https://...` | +| `--index ` | `global.index` | `--index file://...` | +| `--type-map ` | `metadata.typeMap` | `--type-map file://...` | +| `--target ` | `target` | `--target json` | +| `--threads ` | `threads` | `--threads 4` | +| `--chunk-size ` | `chunkSize` | `--chunk-size 10` | diff --git a/docs/generators.md b/docs/generators.md index 1c64c928..d86f9465 100644 --- a/docs/generators.md +++ b/docs/generators.md @@ -24,27 +24,22 @@ Raw Markdown Files [web] - Generate HTML/CSS/JS bundles ``` -Each generator declares its dependency using the `dependsOn` export, allowing automatic pipeline construction. +Each generator declares its dependency using the `dependsOn` field, allowing automatic pipeline construction. ## Generator Structure -A generator is a single module (`index.mjs`) that exports its metadata and logic as named exports: - -- `name` - The generator's short name (used for config keys and logging) -- `generate` - The main generation function (required) -- `processChunk` - Worker thread processing function (optional — presence enables parallel processing) -- `dependsOn` - Import specifier of the dependency generator (optional) -- `defaultConfiguration` - Default config values (optional) +A generator is defined as a module exporting an object conforming to the `GeneratorMetadata` interface. ## Creating a Basic Generator -### Step 1: Create the Generator Directory +### Step 1: Create the Generator Files Create a new directory in `src/generators/`: ``` src/generators/my-format/ -├── index.mjs # Generator entry point (required) +├── index.mjs # Generator metadata (required) +├── generate.mjs # Generator implementation (required) ├── constants.mjs # Constants (optional) ├── types.d.ts # TypeScript types (required) └── utils/ # Utility functions (optional) @@ -72,25 +67,51 @@ export type Generator = GeneratorMetadata< >; ``` -### Step 3: Implement the Generator +### Step 3: Define Generator Metadata -Create `index.mjs` with your generator's metadata and logic: +Create the generator metadata in `index.mjs` using `createLazyGenerator`: ```javascript // src/generators/my-format/index.mjs -'use strict'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + +/** + * Generates output in MyFormat. + * + * @type {import('./types').Generator} + */ +export default createLazyGenerator({ + name: 'my-format', + + version: '1.0.0', + + description: 'Generates documentation in MyFormat', + + // This generator depends on the metadata generator + dependsOn: 'metadata', + + defaultConfiguration: { + // If your generator supports a custom configuration, define the defaults here + myCustomOption: 'myDefaultValue', + // All generators support options in the GlobalConfiguration object + // To override the defaults, they can be specified here + ref: 'overriddenRef', + }, +}); +``` + +### Step 4: Implement the Generator Logic + +Create the generator implementation in `generate.mjs`: + +```javascript +// src/generators/my-format/generate.mjs import { writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import getConfig from '../../utils/configuration/index.mjs'; -export const name = 'my-format'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; -export const defaultConfiguration = { - myCustomOption: 'myDefaultValue', -}; - /** * Main generation function * @@ -128,22 +149,62 @@ function transformToMyFormat(entries, version) { } ``` -### Step 4: Register the Generator +### Step 5: Register the Generator -Add an entry to the `exports` map in `packages/core/package.json`. If you follow the `index.mjs` convention, the wildcard pattern `"./generators/*": "./src/generators/*/index.mjs"` handles this automatically — no changes needed. +Add your generator to the exports in `src/generators/index.mjs`: + +```javascript +// For public generators (available via CLI) +import myFormat from './my-format/index.mjs'; + +export const publicGenerators = { + 'json-simple': jsonSimple, + 'my-format': myFormat, // Add this + // ... other generators +}; + +// For internal generators (used only as dependencies) +const internalGenerators = { + ast, + metadata, + // ... internal generators +}; +``` ## Parallel Processing with Workers -For generators processing large datasets, implement parallel processing using worker threads. Export a `processChunk` function from your `index.mjs` — its presence automatically enables parallel processing. +For generators processing large datasets, implement parallel processing using worker threads. ### Implementing Worker-Based Processing +First, define the generator metadata in `index.mjs`: + ```javascript // src/generators/parallel-generator/index.mjs -import getConfig from '../../utils/configuration/index.mjs'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + +/** + * @type {import('./types').Generator} + */ +export default createLazyGenerator({ + name: 'parallel-generator', + + version: '1.0.0', + + description: 'Processes data in parallel', + + dependsOn: 'metadata', + + // Indicates this generator has a processChunk implementation + hasParallelProcessor: true, +}); +``` -export const name = 'parallel-generator'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; +Then, implement both `processChunk` and `generate` in `generate.mjs`: + +```javascript +// src/generators/parallel-generator/generate.mjs +import getConfig from '../../utils/configuration/index.mjs'; /** * Process a chunk of items in a worker thread. @@ -171,7 +232,7 @@ export async function processChunk(fullInput, itemIndices, deps) { */ export async function* generate(input, worker) { // Configuration for this generator is based on its name - const config = getConfig('parallel-generator'); + const config = getConfig('my-format'); // Prepare serializable dependencies const deps = { @@ -210,13 +271,34 @@ Don't use workers when: ## Streaming Results -Generators can yield results as they're produced using async generators. Export `processChunk` to enable parallel processing, then use `async function*` for `generate`: +Generators can yield results as they're produced using async generators. + +Define the generator metadata in `index.mjs`: ```javascript // src/generators/streaming-generator/index.mjs -export const name = 'streaming-generator'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; +import { createLazyGenerator } from '../../utils/generators.mjs'; +/** + * @type {import('./types').Generator} + */ +export default createLazyGenerator({ + name: 'streaming-generator', + + version: '1.0.0', + + description: 'Streams results as they are ready', + + dependsOn: 'metadata', + + hasParallelProcessor: true, +}); +``` + +Implement the generator in `generate.mjs`: + +```javascript +// src/generators/streaming-generator/generate.mjs /** * Process a chunk of data * @@ -249,13 +331,32 @@ export async function* generate(input, worker) { ### Non-Streaming Generators -Some generators must collect all input before processing: +Some generators must collect all input before processing. + +Generator metadata in `index.mjs`: ```javascript // src/generators/batch-generator/index.mjs -export const name = 'batch-generator'; -export const dependsOn = '@node-core/doc-kit/generators/jsx-ast'; +import { createLazyGenerator } from '../../utils/generators.mjs'; +/** + * @type {import('./types').Generator} + */ +export default createLazyGenerator({ + name: 'batch-generator', + + version: '1.0.0', + + description: 'Requires all input at once', + + dependsOn: 'jsx-ast', +}); +``` + +Implementation in `generate.mjs`: + +```javascript +// src/generators/batch-generator/generate.mjs /** * Non-streaming - returns Promise instead of AsyncGenerator * @@ -282,33 +383,54 @@ Use non-streaming when: ### Declaring Dependencies +In `index.mjs`: + ```javascript -// src/generators/my-generator/index.mjs -export const name = 'my-generator'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; +import { createLazyGenerator } from '../../utils/generators.mjs'; + +export default createLazyGenerator({ + name: 'my-generator', + + dependsOn: 'metadata', // This generator requires metadata output + + // ... other metadata +}); +``` + +In `generate.mjs`: +```javascript export async function generate(input, worker) { - // input contains the output from the metadata generator + // input contains the output from 'metadata' generator } ``` ### Dependency Chain Example ```javascript -// Step 1: Parse markdown to AST (no dependency) +// Step 1: Parse markdown to AST // src/generators/ast/index.mjs -export const name = 'ast'; -// No dependsOn — processes raw markdown files +export default createLazyGenerator({ + name: 'ast', + dependsOn: undefined, // No dependency + // Processes raw markdown files +}); // Step 2: Extract metadata from AST // src/generators/metadata/index.mjs -export const name = 'metadata'; -export const dependsOn = '@node-core/doc-kit/generators/ast'; +export default createLazyGenerator({ + name: 'metadata', + dependsOn: 'ast', // Depends on AST + // Processes AST output +}); // Step 3: Generate HTML from metadata // src/generators/html-generator/index.mjs -export const name = 'html-generator'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; +export default createLazyGenerator({ + name: 'html-generator', + dependsOn: 'metadata', // Depends on metadata + // Processes metadata output +}); ``` ### Multiple Consumers @@ -327,6 +449,8 @@ The framework ensures `metadata` runs once and its output is cached for all cons ### Writing Output Files +In `generate.mjs`: + ```javascript import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; diff --git a/package-lock.json b/package-lock.json index 2b4014ee..1871058c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8791,7 +8791,7 @@ "yaml": "^2.8.3" }, "bin": { - "doc-kit": "packages/core/bin/cli.mjs" + "doc-kit": "bin/cli.mjs" }, "devDependencies": { "@reporters/github": "^1.12.0", diff --git a/packages/core/bin/commands/generate.mjs b/packages/core/bin/commands/generate.mjs index d15bcbbb..d4586663 100644 --- a/packages/core/bin/commands/generate.mjs +++ b/packages/core/bin/commands/generate.mjs @@ -1,10 +1,12 @@ import { Command, Option } from 'commander'; +import { publicGenerators } from '../../src/generators/index.mjs'; import createGenerator from '../../src/generators.mjs'; -import { resolveGeneratorGraph } from '../../src/loader.mjs'; import { setConfig } from '../../src/utils/configuration/index.mjs'; import { errorWrap } from '../utils.mjs'; +const { runGenerators } = createGenerator(); + /** * @typedef {Object} CLIOptions * @property {string[]} input @@ -29,7 +31,11 @@ export default new Command('generate') .addOption( new Option('-i, --input ', 'Input file patterns (glob)') ) - .addOption(new Option('-t, --target ', 'Target generator(s)')) + .addOption( + new Option('-t, --target ', 'Target generator(s)').choices( + Object.keys(publicGenerators) + ) + ) .addOption( new Option('--ignore ', 'Ignore file patterns (glob)') ) @@ -55,12 +61,7 @@ export default new Command('generate') .action( errorWrap(async opts => { - const targets = opts.target ?? []; - const loadedGenerators = await resolveGeneratorGraph(targets); - - const config = await setConfig(opts, loadedGenerators); - - const { runGenerators } = createGenerator(loadedGenerators); - await runGenerators(config, targets); + const config = await setConfig(opts); + await runGenerators(config); }) ); diff --git a/packages/core/package.json b/packages/core/package.json index 7f08821e..c8358237 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -15,15 +15,12 @@ "test:ci": "c8 --reporter=lcov node --test --experimental-test-module-mocks \"src/**/*.test.mjs\" --test-reporter=@reporters/github --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=junit.xml --test-reporter=spec --test-reporter-destination=stdout", "test:update-snapshots": "node --test --experimental-test-module-mocks --test-update-snapshots \"src/**/*.test.mjs\"", "test:watch": "node --test --experimental-test-module-mocks --watch \"src/**/*.test.mjs\"", - "run": "node packages/core/bin/cli.mjs", - "watch": "node --watch packages/core/bin/cli.mjs" + "run": "node bin/cli.mjs", + "watch": "node --watch bin/cli.mjs" }, "main": "./src/generators.mjs", - "exports": { - "./generators/*": "./src/generators/*/index.mjs" - }, "bin": { - "doc-kit": "./packages/core/bin/cli.mjs" + "doc-kit": "./bin/cli.mjs" }, "dependencies": { "@actions/core": "^3.0.0", diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index c3cc7b2c..451c39b5 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -1,5 +1,6 @@ 'use strict'; +import { allGenerators } from './generators/index.mjs'; import logger from './logger/index.mjs'; import { isAsyncGenerator, createStreamingCache } from './streaming.mjs'; import createWorkerPool from './threading/index.mjs'; @@ -11,10 +12,8 @@ const generatorsLogger = logger.child('generators'); * Creates a generator orchestration system that manages the execution of * documentation generators in dependency order, with support for parallel * processing and streaming results. - * - * @param {Map} loadedGenerators - Map of specifier → loaded generator */ -const createGenerator = loadedGenerators => { +const createGenerator = () => { /** @type {{ [key: string]: Promise | AsyncGenerator }} */ const cachedGenerators = {}; @@ -26,7 +25,7 @@ const createGenerator = loadedGenerators => { /** * Gets the collected input from a dependency generator. * - * @param {string | undefined} dependsOn - Dependency generator specifier + * @param {string | undefined} dependsOn - Dependency generator name * @returns {Promise} */ const getDependencyInput = async dependsOn => { @@ -46,36 +45,36 @@ const createGenerator = loadedGenerators => { /** * Schedules a generator and its dependencies for execution. * - * @param {string} specifier - Generator specifier to schedule - * @param {object} configuration - Runtime options + * @param {string} generatorName - Generator to schedule + * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options */ - const scheduleGenerator = async (specifier, configuration) => { - if (specifier in cachedGenerators) { + const scheduleGenerator = async (generatorName, configuration) => { + if (generatorName in cachedGenerators) { return; } - const generator = loadedGenerators.get(specifier); - const { dependsOn, generate, processChunk, name } = generator; + const { dependsOn, generate, hasParallelProcessor } = + allGenerators[generatorName]; // Schedule dependency first if (dependsOn && !(dependsOn in cachedGenerators)) { await scheduleGenerator(dependsOn, configuration); } - generatorsLogger.debug(`Scheduling "${name}"`, { + generatorsLogger.debug(`Scheduling "${generatorName}"`, { dependsOn: dependsOn || 'none', - streaming: !!processChunk, + streaming: hasParallelProcessor, }); // Schedule the generator - cachedGenerators[specifier] = (async () => { + cachedGenerators[generatorName] = (async () => { const dependencyInput = await getDependencyInput(dependsOn); - generatorsLogger.debug(`Starting "${name}"`); + generatorsLogger.debug(`Starting "${generatorName}"`); // Create parallel worker for streaming generators - const worker = processChunk - ? createParallelWorker(specifier, generator, pool, configuration) + const worker = hasParallelProcessor + ? createParallelWorker(generatorName, pool, configuration) : Promise.resolve(null); const result = await generate(dependencyInput, await worker); @@ -83,7 +82,7 @@ const createGenerator = loadedGenerators => { // For streaming generators, "Completed" is logged when collection finishes // (in streamingCache.getOrCollect), not here when the generator returns if (!isAsyncGenerator(result)) { - generatorsLogger.debug(`Completed "${name}"`); + generatorsLogger.debug(`Completed "${generatorName}"`); } return result; @@ -93,17 +92,14 @@ const createGenerator = loadedGenerators => { /** * Runs all requested generators with their dependencies. * - * @param {object} configuration - Runtime options - * @param {string[]} targetSpecifiers - Resolved target specifiers + * @param {import('./utils/configuration/types').Configuration} options - Runtime options * @returns {Promise} Results of all requested generators */ - const runGenerators = async (configuration, targetSpecifiers) => { - const { threads } = configuration; + const runGenerators = async configuration => { + const { target: generators, threads } = configuration; generatorsLogger.debug(`Starting pipeline`, { - generators: targetSpecifiers - .map(s => loadedGenerators.get(s).name) - .join(', '), + generators: generators.join(', '), threads, }); @@ -111,16 +107,16 @@ const createGenerator = loadedGenerators => { pool = createWorkerPool(threads); // Schedule all generators - for (const specifier of targetSpecifiers) { - await scheduleGenerator(specifier, configuration); + for (const name of generators) { + await scheduleGenerator(name, configuration); } // Start all collections in parallel (don't await sequentially) - const resultPromises = targetSpecifiers.map(async specifier => { - let result = await cachedGenerators[specifier]; + const resultPromises = generators.map(async name => { + let result = await cachedGenerators[name]; if (isAsyncGenerator(result)) { - result = await streamingCache.getOrCollect(specifier, result); + result = await streamingCache.getOrCollect(name, result); } return result; diff --git a/packages/core/src/generators/__tests__/index.test.mjs b/packages/core/src/generators/__tests__/index.test.mjs new file mode 100644 index 00000000..8efd3c9a --- /dev/null +++ b/packages/core/src/generators/__tests__/index.test.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import semver from 'semver'; + +import { allGenerators } from '../index.mjs'; + +const validDependencies = Object.keys(allGenerators); + +const allGeneratorsEntries = Object.entries(allGenerators); + +describe('All Generators', () => { + it('should have keys matching their name property', () => { + allGeneratorsEntries.forEach(([key, generator]) => { + assert.equal( + key, + generator.name, + `Generator key "${key}" does not match its name property "${generator.name}"` + ); + }); + }); + + it('should have valid semver versions', () => { + allGeneratorsEntries.forEach(([key, generator]) => { + const isValid = semver.valid(generator.version); + assert.ok( + isValid, + `Generator "${key}" has invalid semver version: "${generator.version}"` + ); + }); + }); + + it('should have valid dependsOn references', () => { + allGeneratorsEntries.forEach(([key, generator]) => { + if (generator.dependsOn) { + assert.ok( + validDependencies.includes(generator.dependsOn), + `Generator "${key}" depends on "${generator.dependsOn}" which is not a valid generator` + ); + } + }); + }); + + it('should have ast generator as a top-level generator with no dependencies', () => { + const ast = allGenerators.ast; + assert.ok(ast, 'ast generator should exist'); + assert.equal( + ast.dependsOn, + undefined, + 'ast generator should have no dependencies' + ); + }); +}); diff --git a/packages/core/src/generators/addon-verify/generate.mjs b/packages/core/src/generators/addon-verify/generate.mjs new file mode 100644 index 00000000..91f9b463 --- /dev/null +++ b/packages/core/src/generators/addon-verify/generate.mjs @@ -0,0 +1,82 @@ +'use strict'; + +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { visit } from 'unist-util-visit'; + +import { EXTRACT_CODE_FILENAME_COMMENT } from './constants.mjs'; +import { generateFileList } from './utils/generateFileList.mjs'; +import { + generateSectionFolderName, + isBuildableSection, + normalizeSectionName, +} from './utils/section.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +/** + * Generates a file list from code blocks. + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('addon-verify'); + + const sectionsCodeBlocks = input.reduce((addons, node) => { + const sectionName = node.heading.data.name; + + const content = node.content; + + visit(content, childNode => { + if (childNode.type === 'code') { + const filename = childNode.value.match(EXTRACT_CODE_FILENAME_COMMENT); + + if (filename === null) { + return; + } + + if (!addons[sectionName]) { + addons[sectionName] = []; + } + + addons[sectionName].push({ + name: filename[1], + content: childNode.value, + }); + } + }); + + return addons; + }, {}); + + const files = await Promise.all( + Object.entries(sectionsCodeBlocks) + .filter(([, codeBlocks]) => isBuildableSection(codeBlocks)) + .flatMap(async ([sectionName, codeBlocks], index) => { + const files = generateFileList(codeBlocks); + + if (config.output) { + const normalizedSectionName = normalizeSectionName(sectionName); + + const folderName = generateSectionFolderName( + normalizedSectionName, + index + ); + + await mkdir(join(config.output, folderName), { recursive: true }); + + for (const file of files) { + await writeFile( + join(config.output, folderName, file.name), + file.content + ); + } + } + + return files; + }) + ); + + return files; +} diff --git a/packages/core/src/generators/addon-verify/index.mjs b/packages/core/src/generators/addon-verify/index.mjs index 8b888999..fab78599 100644 --- a/packages/core/src/generators/addon-verify/index.mjs +++ b/packages/core/src/generators/addon-verify/index.mjs @@ -1,84 +1,21 @@ 'use strict'; -import { mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; +import { createLazyGenerator } from '../../utils/generators.mjs'; -import { visit } from 'unist-util-visit'; - -import { EXTRACT_CODE_FILENAME_COMMENT } from './constants.mjs'; -import { generateFileList } from './utils/generateFileList.mjs'; -import { - generateSectionFolderName, - isBuildableSection, - normalizeSectionName, -} from './utils/section.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -export const name = 'addon-verify'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; /** - * Generates a file list from code blocks. + * This generator generates a file list from code blocks extracted from + * `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime + * validations. * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function generate(input) { - const config = getConfig('addon-verify'); - - const sectionsCodeBlocks = input.reduce((addons, node) => { - const sectionName = node.heading.data.name; - - const content = node.content; - - visit(content, childNode => { - if (childNode.type === 'code') { - const filename = childNode.value.match(EXTRACT_CODE_FILENAME_COMMENT); - - if (filename === null) { - return; - } - - if (!addons[sectionName]) { - addons[sectionName] = []; - } - - addons[sectionName].push({ - name: filename[1], - content: childNode.value, - }); - } - }); - - return addons; - }, {}); - - const files = await Promise.all( - Object.entries(sectionsCodeBlocks) - .filter(([, codeBlocks]) => isBuildableSection(codeBlocks)) - .flatMap(async ([sectionName, codeBlocks], index) => { - const files = generateFileList(codeBlocks); - - if (config.output) { - const normalizedSectionName = normalizeSectionName(sectionName); - - const folderName = generateSectionFolderName( - normalizedSectionName, - index - ); - - await mkdir(join(config.output, folderName), { recursive: true }); +export default await createLazyGenerator({ + name: 'addon-verify', - for (const file of files) { - await writeFile( - join(config.output, folderName, file.name), - file.content - ); - } - } + version: '1.0.0', - return files; - }) - ); + description: + 'Generates a file list from code blocks extracted from `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime validations', - return files; -} + dependsOn: 'metadata', +}); diff --git a/packages/core/src/generators/api-links/__tests__/fixtures.test.mjs b/packages/core/src/generators/api-links/__tests__/fixtures.test.mjs index 669e3dac..6259aeb3 100644 --- a/packages/core/src/generators/api-links/__tests__/fixtures.test.mjs +++ b/packages/core/src/generators/api-links/__tests__/fixtures.test.mjs @@ -3,12 +3,11 @@ import { after, before, describe, it } from 'node:test'; import { globSync } from 'tinyglobby'; -import { loadGenerator } from '../../../loader.mjs'; import createWorkerPool from '../../../threading/index.mjs'; import createParallelWorker from '../../../threading/parallel.mjs'; import { setConfig } from '../../../utils/configuration/index.mjs'; -import { generate as astJsGenerate } from '../../ast-js/index.mjs'; -import { generate as apiLinksGenerate } from '../index.mjs'; +import { generate as astJsGenerate } from '../../ast-js/generate.mjs'; +import { generate as apiLinksGenerate } from '../generate.mjs'; const relativePath = relative(process.cwd(), import.meta.dirname); @@ -16,17 +15,7 @@ const sourceFiles = globSync('*.js', { cwd: new URL(import.meta.resolve('./fixtures')), }); -const astJsSpecifier = '@node-core/doc-kit/generators/ast-js'; -const astJsGenerator = await loadGenerator(astJsSpecifier); -const apiLinksSpecifier = '@node-core/doc-kit/generators/api-links'; -const apiLinksGenerator = await loadGenerator(apiLinksSpecifier); - -const loadedGenerators = new Map([ - [astJsSpecifier, astJsGenerator], - [apiLinksSpecifier, apiLinksGenerator], -]); - -const config = await setConfig({}, loadedGenerators); +const config = await setConfig({}); describe('api links', () => { let pool; @@ -46,12 +35,7 @@ describe('api links', () => { join(relativePath, 'fixtures', sourceFile).replaceAll(sep, '/'), ]; - const worker = await createParallelWorker( - astJsSpecifier, - astJsGenerator, - pool, - config - ); + const worker = await createParallelWorker('ast-js', pool, config); // Collect results from the async generator const astJsResults = []; diff --git a/packages/core/src/generators/api-links/generate.mjs b/packages/core/src/generators/api-links/generate.mjs new file mode 100644 index 00000000..3890a308 --- /dev/null +++ b/packages/core/src/generators/api-links/generate.mjs @@ -0,0 +1,67 @@ +'use strict'; + +import { basename, join } from 'node:path'; + +import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs'; +import { extractExports } from './utils/extractExports.mjs'; +import { findDefinitions } from './utils/findDefinitions.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { populate } from '../../utils/configuration/templates.mjs'; +import { withExt, writeFile } from '../../utils/file.mjs'; + +/** + * Generates the `apilinks.json` file. + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('api-links'); + /** + * @type Record + */ + const definitions = {}; + + input.forEach(program => { + /** + * Mapping of definitions to their line number + * + * @type {Record} + * @example { 'someclass.foo': 10 } + */ + const nameToLineNumberMap = {}; + + const fileName = basename(program.path); + const baseName = withExt(fileName); + + const exports = extractExports(program, baseName, nameToLineNumberMap); + + findDefinitions(program, baseName, nameToLineNumberMap, exports); + + checkIndirectReferences(program, exports, nameToLineNumberMap); + + const fullGitUrl = populate(config.sourceURL, { + ...config, + fileName, + }); + + // Add the exports we found in this program to our output + Object.keys(nameToLineNumberMap).forEach(key => { + const lineNumber = nameToLineNumberMap[key]; + + definitions[key] = `${fullGitUrl}#L${lineNumber}`; + }); + }); + + if (config.output) { + const out = join(config.output, 'apilinks.json'); + + await writeFile( + out, + config.minify + ? JSON.stringify(definitions) + : JSON.stringify(definitions, null, 2) + ); + } + + return definitions; +} diff --git a/packages/core/src/generators/api-links/index.mjs b/packages/core/src/generators/api-links/index.mjs index e967b76d..dca3f908 100644 --- a/packages/core/src/generators/api-links/index.mjs +++ b/packages/core/src/generators/api-links/index.mjs @@ -1,76 +1,31 @@ 'use strict'; -import { basename, join } from 'node:path'; - -import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs'; -import { extractExports } from './utils/extractExports.mjs'; -import { findDefinitions } from './utils/findDefinitions.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { - GITHUB_BLOB_URL, - populate, -} from '../../utils/configuration/templates.mjs'; -import { withExt, writeFile } from '../../utils/file.mjs'; - -export const name = 'api-links'; -export const dependsOn = '@node-core/doc-kit/generators/ast-js'; -export const defaultConfiguration = { - sourceURL: `${GITHUB_BLOB_URL}lib/{fileName}`, -}; +import { GITHUB_BLOB_URL } from '../../utils/configuration/templates.mjs'; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Generates the `apilinks.json` file. + * This generator is responsible for mapping publicly accessible functions in + * Node.js to their source locations in the Node.js repository. + * + * This is a top-level generator. It takes in the raw AST tree of the JavaScript + * source files. It outputs a `apilinks.json` file into the specified output + * directory. * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function generate(input) { - const config = getConfig('api-links'); - /** - * @type Record - */ - const definitions = {}; - - input.forEach(program => { - /** - * Mapping of definitions to their line number - * - * @type {Record} - * @example { 'someclass.foo': 10 } - */ - const nameToLineNumberMap = {}; - - const fileName = basename(program.path); - const baseName = withExt(fileName); - - const exports = extractExports(program, baseName, nameToLineNumberMap); - - findDefinitions(program, baseName, nameToLineNumberMap, exports); - - checkIndirectReferences(program, exports, nameToLineNumberMap); - - const fullGitUrl = populate(config.sourceURL, { - ...config, - fileName, - }); - - // Add the exports we found in this program to our output - Object.keys(nameToLineNumberMap).forEach(key => { - const lineNumber = nameToLineNumberMap[key]; +export default createLazyGenerator({ + name: 'api-links', - definitions[key] = `${fullGitUrl}#L${lineNumber}`; - }); - }); + version: '1.0.0', - if (config.output) { - const out = join(config.output, 'apilinks.json'); + description: + 'Creates a mapping of publicly accessible functions to their source locations in the Node.js repository.', - await writeFile( - out, - config.minify - ? JSON.stringify(definitions) - : JSON.stringify(definitions, null, 2) - ); - } + // Unlike the rest of the generators, this utilizes Javascript sources being + // passed into the input field rather than Markdown. + dependsOn: 'ast-js', - return definitions; -} + defaultConfiguration: { + sourceURL: `${GITHUB_BLOB_URL}lib/{fileName}`, + }, +}); diff --git a/packages/core/src/generators/ast-js/generate.mjs b/packages/core/src/generators/ast-js/generate.mjs new file mode 100644 index 00000000..303e5ec0 --- /dev/null +++ b/packages/core/src/generators/ast-js/generate.mjs @@ -0,0 +1,56 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { extname } from 'node:path'; + +import { parse } from 'acorn'; +import { globSync } from 'tinyglobby'; + +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Process a chunk of JavaScript files in a worker thread. + * Parses JS source files into AST representations. + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(inputSlice, itemIndices) { + const filePaths = itemIndices.map(idx => inputSlice[idx]); + + const results = []; + + for (const path of filePaths) { + const value = await readFile(path, 'utf-8'); + + const parsed = parse(value, { + allowReturnOutsideFunction: true, + ecmaVersion: 'latest', + locations: true, + }); + + parsed.path = path; + + results.push(parsed); + } + + return results; +} + +/** + * Generates a JavaScript AST from the input files. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(_, worker) { + const config = getConfig('ast-js'); + + const files = globSync(config.input, { ignore: config.ignore }).filter( + p => extname(p) === '.js' + ); + + // Parse the Javascript sources into ASTs in parallel using worker threads + // source is both the items list and the fullInput since we use sliceInput + for await (const chunkResult of worker.stream(files)) { + yield chunkResult; + } +} diff --git a/packages/core/src/generators/ast-js/index.mjs b/packages/core/src/generators/ast-js/index.mjs index 3e666101..06652167 100644 --- a/packages/core/src/generators/ast-js/index.mjs +++ b/packages/core/src/generators/ast-js/index.mjs @@ -1,58 +1,23 @@ 'use strict'; -import { readFile } from 'node:fs/promises'; -import { extname } from 'node:path'; - -import { parse } from 'acorn'; -import { globSync } from 'tinyglobby'; - -import getConfig from '../../utils/configuration/index.mjs'; - -export const name = 'ast-js'; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Process a chunk of JavaScript files in a worker thread. - * Parses JS source files into AST representations. + * This generator parses Javascript sources passed into the generator's input + * field. This is separate from the Markdown parsing step since it's not as + * commonly used and can take up a significant amount of memory. * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(inputSlice, itemIndices) { - const filePaths = itemIndices.map(idx => inputSlice[idx]); - - const results = []; - - for (const path of filePaths) { - const value = await readFile(path, 'utf-8'); - - const parsed = parse(value, { - allowReturnOutsideFunction: true, - ecmaVersion: 'latest', - locations: true, - }); - - parsed.path = path; - - results.push(parsed); - } - - return results; -} - -/** - * Generates a JavaScript AST from the input files. + * Putting this with the rest of the generators allows it to be lazily loaded + * so we're only parsing the Javascript sources when we need to. * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function* generate(_, worker) { - const config = getConfig('ast-js'); +export default createLazyGenerator({ + name: 'ast-js', + + version: '1.0.0', - const files = globSync(config.input, { ignore: config.ignore }).filter( - p => extname(p) === '.js' - ); + description: 'Parses Javascript source files passed into the input.', - // Parse the Javascript sources into ASTs in parallel using worker threads - // source is both the items list and the fullInput since we use sliceInput - for await (const chunkResult of worker.stream(files)) { - yield chunkResult; - } -} + hasParallelProcessor: true, +}); diff --git a/packages/core/src/generators/ast/generate.mjs b/packages/core/src/generators/ast/generate.mjs new file mode 100644 index 00000000..97724e03 --- /dev/null +++ b/packages/core/src/generators/ast/generate.mjs @@ -0,0 +1,73 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { relative, sep } from 'node:path/posix'; + +import globParent from 'glob-parent'; +import { globSync } from 'tinyglobby'; + +import { STABILITY_INDEX_URL } from './constants.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { withExt } from '../../utils/file.mjs'; +import { QUERIES } from '../../utils/queries/index.mjs'; +import { getRemark } from '../../utils/remark.mjs'; + +const remarkProcessor = getRemark(); + +/** + * Process a chunk of markdown files in a worker thread. + * Loads and parses markdown files into AST representations. + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(inputSlice, itemIndices) { + const filePaths = itemIndices.map(idx => inputSlice[idx]); + + const results = []; + + for (const [path, parent] of filePaths) { + const content = await readFile(path, 'utf-8'); + const value = content + .replace( + QUERIES.standardYamlFrontmatter, + (_, yaml) => '\n' + ) + .replace( + QUERIES.stabilityIndexPrefix, + match => `[${match}](${STABILITY_INDEX_URL})` + ); + + const relativePath = sep + withExt(relative(parent, path)); + + results.push({ + tree: remarkProcessor.parse(value), + // The path is the relative path minus the extension + path: relativePath, + }); + } + + return results; +} + +/** + * Generates AST trees from markdown input files. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(_, worker) { + const { ast: config } = getConfig(); + + const files = config.input.flatMap(input => { + const parent = globParent(input); + + return globSync(input, { ignore: config.ignore }).map(child => [ + child, + parent, + ]); + }); + + // Parse markdown files in parallel using worker threads + for await (const chunkResult of worker.stream(files)) { + yield chunkResult; + } +} diff --git a/packages/core/src/generators/ast/index.mjs b/packages/core/src/generators/ast/index.mjs index 46ef0e50..4c34b637 100644 --- a/packages/core/src/generators/ast/index.mjs +++ b/packages/core/src/generators/ast/index.mjs @@ -1,75 +1,19 @@ 'use strict'; -import { readFile } from 'node:fs/promises'; -import { relative, sep } from 'node:path/posix'; - -import globParent from 'glob-parent'; -import { globSync } from 'tinyglobby'; - -import { STABILITY_INDEX_URL } from './constants.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { withExt } from '../../utils/file.mjs'; -import { QUERIES } from '../../utils/queries/index.mjs'; -import { getRemark } from '../../utils/remark.mjs'; - -export const name = 'ast'; - -const remarkProcessor = getRemark(); - -/** - * Process a chunk of markdown files in a worker thread. - * Loads and parses markdown files into AST representations. - * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(inputSlice, itemIndices) { - const filePaths = itemIndices.map(idx => inputSlice[idx]); - - const results = []; - - for (const [path, parent] of filePaths) { - const content = await readFile(path, 'utf-8'); - const value = content - .replace( - QUERIES.standardYamlFrontmatter, - (_, yaml) => '\n' - ) - .replace( - QUERIES.stabilityIndexPrefix, - match => `[${match}](${STABILITY_INDEX_URL})` - ); - - const relativePath = sep + withExt(relative(parent, path)); - - results.push({ - tree: remarkProcessor.parse(value), - // The path is the relative path minus the extension - path: relativePath, - }); - } - - return results; -} +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Generates AST trees from markdown input files. + * This generator parses Markdown API doc files into AST trees. + * It parallelizes the parsing across worker threads for better performance. * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function* generate(_, worker) { - const { ast: config } = getConfig(); +export default createLazyGenerator({ + name: 'ast', - const files = config.input.flatMap(input => { - const parent = globParent(input); + version: '1.0.0', - return globSync(input, { ignore: config.ignore }).map(child => [ - child, - parent, - ]); - }); + description: 'Parses Markdown API doc files into AST trees', - // Parse markdown files in parallel using worker threads - for await (const chunkResult of worker.stream(files)) { - yield chunkResult; - } -} + hasParallelProcessor: true, +}); diff --git a/packages/core/src/generators/index.mjs b/packages/core/src/generators/index.mjs new file mode 100644 index 00000000..24294f92 --- /dev/null +++ b/packages/core/src/generators/index.mjs @@ -0,0 +1,47 @@ +'use strict'; + +import addonVerify from './addon-verify/index.mjs'; +import apiLinks from './api-links/index.mjs'; +import ast from './ast/index.mjs'; +import astJs from './ast-js/index.mjs'; +import jsonSimple from './json-simple/index.mjs'; +import jsxAst from './jsx-ast/index.mjs'; +import legacyHtml from './legacy-html/index.mjs'; +import legacyHtmlAll from './legacy-html-all/index.mjs'; +import legacyJson from './legacy-json/index.mjs'; +import legacyJsonAll from './legacy-json-all/index.mjs'; +import llmsTxt from './llms-txt/index.mjs'; +import manPage from './man-page/index.mjs'; +import metadata from './metadata/index.mjs'; +import oramaDb from './orama-db/index.mjs'; +import sitemap from './sitemap/index.mjs'; +import web from './web/index.mjs'; + +export const publicGenerators = { + 'json-simple': jsonSimple, + 'legacy-html': legacyHtml, + 'legacy-html-all': legacyHtmlAll, + 'man-page': manPage, + 'legacy-json': legacyJson, + 'legacy-json-all': legacyJsonAll, + 'addon-verify': addonVerify, + 'api-links': apiLinks, + 'orama-db': oramaDb, + 'llms-txt': llmsTxt, + sitemap, + web, +}; + +// These ones are special since they don't produce standard output, +// and hence, we don't expose them to the CLI. +const internalGenerators = { + ast, + metadata, + 'jsx-ast': jsxAst, + 'ast-js': astJs, +}; + +export const allGenerators = { + ...publicGenerators, + ...internalGenerators, +}; diff --git a/packages/core/src/generators/json-simple/generate.mjs b/packages/core/src/generators/json-simple/generate.mjs new file mode 100644 index 00000000..82d06a99 --- /dev/null +++ b/packages/core/src/generators/json-simple/generate.mjs @@ -0,0 +1,43 @@ +'use strict'; + +import { join } from 'node:path'; + +import { remove } from 'unist-util-remove'; + +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; +import { UNIST } from '../../utils/queries/index.mjs'; + +/** + * Generates the simplified JSON version of the API docs + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('json-simple'); + + // Iterates the input (MetadataEntry) and performs a few changes + const mappedInput = input.map(node => { + // Deep clones the content nodes to avoid affecting upstream nodes + const content = JSON.parse(JSON.stringify(node.content)); + + // Removes numerous nodes from the content that should not be on the "body" + // of the JSON version of the API docs as they are already represented in the metadata + remove(content, [UNIST.isStabilityNode, UNIST.isHeading]); + + return { ...node, content }; + }); + + if (config.output) { + // Writes all the API docs stringified content into one file + // Note: The full JSON generator in the future will create one JSON file per top-level API doc file + await writeFile( + join(config.output, 'api-docs.json'), + config.minify + ? JSON.stringify(mappedInput) + : JSON.stringify(mappedInput, null, 2) + ); + } + + return mappedInput; +} diff --git a/packages/core/src/generators/json-simple/index.mjs b/packages/core/src/generators/json-simple/index.mjs index 41c04ff0..9cd62537 100644 --- a/packages/core/src/generators/json-simple/index.mjs +++ b/packages/core/src/generators/json-simple/index.mjs @@ -1,46 +1,23 @@ 'use strict'; -import { join } from 'node:path'; - -import { remove } from 'unist-util-remove'; - -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; -import { UNIST } from '../../utils/queries/index.mjs'; - -export const name = 'json-simple'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Generates the simplified JSON version of the API docs + * This generator generates a simplified JSON version of the API docs and returns it as a string + * this is not meant to be used for the final API docs, but for debugging and testing purposes * - * @type {import('./types').Generator['generate']} + * This generator is a top-level generator, and it takes the raw AST tree of the API doc files + * and returns a stringified JSON version of the API docs. + * + * @type {import('./types').Generator} */ -export async function generate(input) { - const config = getConfig('json-simple'); - - // Iterates the input (MetadataEntry) and performs a few changes - const mappedInput = input.map(node => { - // Deep clones the content nodes to avoid affecting upstream nodes - const content = JSON.parse(JSON.stringify(node.content)); - - // Removes numerous nodes from the content that should not be on the "body" - // of the JSON version of the API docs as they are already represented in the metadata - remove(content, [UNIST.isStabilityNode, UNIST.isHeading]); +export default createLazyGenerator({ + name: 'json-simple', - return { ...node, content }; - }); + version: '1.0.0', - if (config.output) { - // Writes all the API docs stringified content into one file - // Note: The full JSON generator in the future will create one JSON file per top-level API doc file - await writeFile( - join(config.output, 'api-docs.json'), - config.minify - ? JSON.stringify(mappedInput) - : JSON.stringify(mappedInput, null, 2) - ); - } + description: + 'Generates the simple JSON version of the API docs, and returns it as a string', - return mappedInput; -} + dependsOn: 'metadata', +}); diff --git a/packages/core/src/generators/jsx-ast/generate.mjs b/packages/core/src/generators/jsx-ast/generate.mjs new file mode 100644 index 00000000..160971fd --- /dev/null +++ b/packages/core/src/generators/jsx-ast/generate.mjs @@ -0,0 +1,70 @@ +import { buildSideBarProps } from './utils/buildBarProps.mjs'; +import buildContent from './utils/buildContent.mjs'; +import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { getRemarkRecma } from '../../utils/remark.mjs'; +import { relative } from '../../utils/url.mjs'; + +const remarkRecma = getRemarkRecma(); + +/** + * Process a chunk of items in a worker thread. + * Transforms metadata entries into JSX AST nodes. + * + * Each item is a SlicedModuleInput containing the head node + * and all entries for that module - no need to recompute grouping. + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(slicedInput, itemIndices, docPages) { + const results = []; + + for (const idx of itemIndices) { + const { head, entries } = slicedInput[idx]; + + const sideBarProps = buildSideBarProps( + head, + docPages.map(([heading, path]) => [ + heading, + head.path === path + ? `${head.basename}.html` + : `${relative(path, head.path)}.html`, + ]) + ); + + const content = await buildContent( + entries, + head, + sideBarProps, + remarkRecma + ); + + results.push(content); + } + + return results; +} + +/** + * Generates a JSX AST + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input, worker) { + const groupedModules = groupNodesByModule(input); + + const headNodes = getSortedHeadNodes(input); + + const docPages = headNodes.map(node => [node.heading.data.name, node.path]); + + // Create sliced input: each item contains head + its module's entries + // This avoids sending all 4700+ entries to every worker + const entries = headNodes.map(head => ({ + head, + entries: groupedModules.get(head.api), + })); + + for await (const chunkResult of worker.stream(entries, docPages)) { + yield chunkResult; + } +} diff --git a/packages/core/src/generators/jsx-ast/index.mjs b/packages/core/src/generators/jsx-ast/index.mjs index a11baab6..a83839f4 100644 --- a/packages/core/src/generators/jsx-ast/index.mjs +++ b/packages/core/src/generators/jsx-ast/index.mjs @@ -1,79 +1,27 @@ -import { buildSideBarProps } from './utils/buildBarProps.mjs'; -import buildContent from './utils/buildContent.mjs'; -import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; -import { GITHUB_EDIT_URL } from '../../utils/configuration/templates.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { getRemarkRecma } from '../../utils/remark.mjs'; -import { relative } from '../../utils/url.mjs'; - -export const name = 'jsx-ast'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; -export const defaultConfiguration = { - ref: 'main', - pageURL: '{baseURL}/latest-{version}/api{path}.html', - editURL: `${GITHUB_EDIT_URL}/doc/api{path}.md`, -}; +'use strict'; -const remarkRecma = getRemarkRecma(); +import { GITHUB_EDIT_URL } from '../../utils/configuration/templates.mjs'; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Process a chunk of items in a worker thread. - * Transforms metadata entries into JSX AST nodes. + * Generator for converting MDAST to JSX AST. * - * Each item is a SlicedModuleInput containing the head node - * and all entries for that module - no need to recompute grouping. - * - * @type {import('./types').Generator['processChunk']} + * @type {import('./types').Generator} */ -export async function processChunk(slicedInput, itemIndices, docPages) { - const results = []; - - for (const idx of itemIndices) { - const { head, entries } = slicedInput[idx]; - - const sideBarProps = buildSideBarProps( - head, - docPages.map(([heading, path]) => [ - heading, - head.path === path - ? `${head.basename}.html` - : `${relative(path, head.path)}.html`, - ]) - ); +export default createLazyGenerator({ + name: 'jsx-ast', - const content = await buildContent( - entries, - head, - sideBarProps, - remarkRecma - ); - - results.push(content); - } - - return results; -} - -/** - * Generates a JSX AST - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(input, worker) { - const groupedModules = groupNodesByModule(input); + version: '1.0.0', - const headNodes = getSortedHeadNodes(input); + description: 'Generates JSX AST from the input MDAST', - const docPages = headNodes.map(node => [node.heading.data.name, node.path]); + dependsOn: 'metadata', - // Create sliced input: each item contains head + its module's entries - // This avoids sending all 4700+ entries to every worker - const entries = headNodes.map(head => ({ - head, - entries: groupedModules.get(head.api), - })); + defaultConfiguration: { + ref: 'main', + pageURL: '{baseURL}/latest-{version}/api{path}.html', + editURL: `${GITHUB_EDIT_URL}/doc/api{path}.md`, + }, - for await (const chunkResult of worker.stream(entries, docPages)) { - yield chunkResult; - } -} + hasParallelProcessor: true, +}); diff --git a/packages/core/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs b/packages/core/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs index 3e5db799..e91ba049 100644 --- a/packages/core/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs +++ b/packages/core/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs @@ -3,7 +3,6 @@ import { describe, it, mock } from 'node:test'; import { SemVer } from 'semver'; -import { loadGenerator } from '../../../../loader.mjs'; import { setConfig } from '../../../../utils/configuration/index.mjs'; import * as generatorsExports from '../../../../utils/generators.mjs'; @@ -31,19 +30,13 @@ const { buildSideBarProps, } = await import('../buildBarProps.mjs'); -const jsxAstSpecifier = '@node-core/doc-kit/generators/jsx-ast'; -const jsxAstGenerator = await loadGenerator(jsxAstSpecifier); - -await setConfig( - { - version: 'v17.0.0', - changelog: [ - { version: new SemVer('16.0.0'), isLts: true, isCurrent: false }, - { version: new SemVer('17.0.0'), isLts: false, isCurrent: true }, - ], - }, - new Map([[jsxAstSpecifier, jsxAstGenerator]]) -); +await setConfig({ + version: 'v17.0.0', + changelog: [ + { version: new SemVer('16.0.0'), isLts: true, isCurrent: false }, + { version: new SemVer('17.0.0'), isLts: false, isCurrent: true }, + ], +}); describe('extractTextContent', () => { it('combines text and code node values from entries', () => { diff --git a/packages/core/src/generators/legacy-html-all/generate.mjs b/packages/core/src/generators/legacy-html-all/generate.mjs new file mode 100644 index 00000000..ad4e2ca1 --- /dev/null +++ b/packages/core/src/generators/legacy-html-all/generate.mjs @@ -0,0 +1,75 @@ +'use strict'; + +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import getConfig from '../../utils/configuration/index.mjs'; +import { minifyHTML } from '../../utils/html-minifier.mjs'; +import { getRemarkRehype } from '../../utils/remark.mjs'; +import { replaceTemplateValues } from '../legacy-html/utils/replaceTemplateValues.mjs'; +import tableOfContents from '../legacy-html/utils/tableOfContents.mjs'; + +/** + * Generates the `all.html` file from the `legacy-html` generator + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('legacy-html-all'); + + // Gets a Remark Processor that parses Markdown to minified HTML + const remarkWithRehype = getRemarkRehype(); + + // Reads the API template.html file to be used as a base for the HTML files + const apiTemplate = await readFile(config.templatePath, 'utf-8'); + + // Filter out index entries and extract needed properties + const entries = input.filter(entry => entry.api !== 'index'); + + // Aggregates all individual Table of Contents into one giant string + const aggregatedToC = entries.map(entry => entry.toc).join('\n'); + + // Aggregates all individual content into one giant string + const aggregatedContent = entries.map(entry => entry.content).join('\n'); + + // Creates a "mimic" of an `MetadataEntry` which fulfils the requirements + // for generating the `tableOfContents` with the `tableOfContents.parseNavigationNode` parser + const sideNavigationFromValues = entries.map(entry => ({ + api: entry.api, + heading: { data: { depth: 1, name: entry.section } }, + })); + + // Generates the global Table of Contents (Sidebar Navigation) + const parsedSideNav = remarkWithRehype.processSync( + tableOfContents(sideNavigationFromValues, { + maxDepth: 1, + parser: tableOfContents.parseNavigationNode, + }) + ); + + const templateValues = { + api: 'all', + path: 'all', + added: '', + section: 'All', + version: `v${config.version.version}`, + toc: aggregatedToC, + nav: String(parsedSideNav), + content: aggregatedContent, + }; + + let result = replaceTemplateValues(apiTemplate, templateValues, config, { + skipGitHub: true, + skipGtocPicker: true, + }); + + if (config.minify) { + result = await minifyHTML(result); + } + + if (config.output) { + await writeFile(join(config.output, 'all.html'), result); + } + + return result; +} diff --git a/packages/core/src/generators/legacy-html-all/index.mjs b/packages/core/src/generators/legacy-html-all/index.mjs index 43527413..2edea628 100644 --- a/packages/core/src/generators/legacy-html-all/index.mjs +++ b/packages/core/src/generators/legacy-html-all/index.mjs @@ -1,82 +1,28 @@ 'use strict'; -import { readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import getConfig from '../../utils/configuration/index.mjs'; -import { minifyHTML } from '../../utils/html-minifier.mjs'; -import { getRemarkRehype } from '../../utils/remark.mjs'; -import { defaultConfiguration as legacyHtmlDefaults } from '../legacy-html/index.mjs'; -import { replaceTemplateValues } from '../legacy-html/utils/replaceTemplateValues.mjs'; -import tableOfContents from '../legacy-html/utils/tableOfContents.mjs'; - -export const name = 'legacy-html-all'; -export const dependsOn = '@node-core/doc-kit/generators/legacy-html'; -export const defaultConfiguration = { - templatePath: legacyHtmlDefaults.templatePath, -}; +import { createLazyGenerator } from '../../utils/generators.mjs'; +import legacyHtml from '../legacy-html/index.mjs'; /** - * Generates the `all.html` file from the `legacy-html` generator + * This generator generates the legacy HTML pages of the legacy API docs + * for retro-compatibility and while we are implementing the new 'react' and 'html' generators. + * + * This generator is a top-level generator, and it takes the raw AST tree of the API doc files + * and generates the HTML files to the specified output directory from the configuration settings * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function generate(input) { - const config = getConfig('legacy-html-all'); - - // Gets a Remark Processor that parses Markdown to minified HTML - const remarkWithRehype = getRemarkRehype(); - - // Reads the API template.html file to be used as a base for the HTML files - const apiTemplate = await readFile(config.templatePath, 'utf-8'); - - // Filter out index entries and extract needed properties - const entries = input.filter(entry => entry.api !== 'index'); - - // Aggregates all individual Table of Contents into one giant string - const aggregatedToC = entries.map(entry => entry.toc).join('\n'); - - // Aggregates all individual content into one giant string - const aggregatedContent = entries.map(entry => entry.content).join('\n'); - - // Creates a "mimic" of an `MetadataEntry` which fulfils the requirements - // for generating the `tableOfContents` with the `tableOfContents.parseNavigationNode` parser - const sideNavigationFromValues = entries.map(entry => ({ - api: entry.api, - heading: { data: { depth: 1, name: entry.section } }, - })); - - // Generates the global Table of Contents (Sidebar Navigation) - const parsedSideNav = remarkWithRehype.processSync( - tableOfContents(sideNavigationFromValues, { - maxDepth: 1, - parser: tableOfContents.parseNavigationNode, - }) - ); - - const templateValues = { - api: 'all', - path: 'all', - added: '', - section: 'All', - version: `v${config.version.version}`, - toc: aggregatedToC, - nav: String(parsedSideNav), - content: aggregatedContent, - }; +export default createLazyGenerator({ + name: 'legacy-html-all', - let result = replaceTemplateValues(apiTemplate, templateValues, config, { - skipGitHub: true, - skipGtocPicker: true, - }); + version: '1.0.0', - if (config.minify) { - result = await minifyHTML(result); - } + description: + 'Generates the `all.html` file from the `legacy-html` generator, which includes all the modules in one single file', - if (config.output) { - await writeFile(join(config.output, 'all.html'), result); - } + dependsOn: 'legacy-html', - return result; -} + defaultConfiguration: { + templatePath: legacyHtml.defaultConfiguration.templatePath, + }, +}); diff --git a/packages/core/src/generators/legacy-html/generate.mjs b/packages/core/src/generators/legacy-html/generate.mjs new file mode 100644 index 00000000..cff2698d --- /dev/null +++ b/packages/core/src/generators/legacy-html/generate.mjs @@ -0,0 +1,140 @@ +'use strict'; + +import { readFile, cp } from 'node:fs/promises'; +import { basename, join } from 'node:path'; + +import buildContent from './utils/buildContent.mjs'; +import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs'; +import tableOfContents from './utils/tableOfContents.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { minifyHTML } from '../../utils/html-minifier.mjs'; +import { getRemarkRehypeWithShiki } from '../../utils/remark.mjs'; + +/** + * Creates a heading object with the given name. + * @param {string} name - The name of the heading + * @returns {HeadingMetadataEntry} The heading object + */ +const getHeading = name => ({ depth: 1, data: { name } }); + +const remarkRehypeProcessor = getRemarkRehypeWithShiki(); + +/** + * Process a chunk of items in a worker thread. + * Builds HTML template objects - FS operations happen in generate(). + * + * Each item is pre-grouped {head, nodes, headNodes} - no need to + * recompute groupNodesByModule for every chunk. + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(slicedInput, itemIndices, navigation) { + const results = []; + + for (const idx of itemIndices) { + const { head, nodes, headNodes } = slicedInput[idx]; + + const nav = navigation.replace( + `class="nav-${head.api}`, + `class="nav-${head.api} active` + ); + + const toc = String( + remarkRehypeProcessor.processSync( + tableOfContents(nodes, { + maxDepth: 5, + parser: tableOfContents.parseToCNode, + }) + ) + ); + + const content = buildContent(headNodes, nodes, remarkRehypeProcessor); + + const apiAsHeading = head.api.charAt(0).toUpperCase() + head.api.slice(1); + + const template = { + api: head.api, + path: head.path, + added: head.introduced_in ?? '', + section: head.heading.data.name || apiAsHeading, + toc, + nav, + content, + }; + + results.push(template); + } + + return results; +} + +/** + * Generates the legacy version of the API docs in HTML + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input, worker) { + const config = getConfig('legacy-html'); + + const apiTemplate = await readFile(config.templatePath, 'utf-8'); + + const groupedModules = groupNodesByModule(input); + + const headNodes = input + .filter(node => node.heading.depth === 1) + .toSorted((a, b) => a.heading.data.name.localeCompare(b.heading.data.name)); + + const indexOfFiles = config.index + ? config.index.map(({ api, section }) => ({ + api, + heading: getHeading(section), + })) + : headNodes; + + const navigation = String( + remarkRehypeProcessor.processSync( + tableOfContents(indexOfFiles, { + maxDepth: 1, + parser: tableOfContents.parseNavigationNode, + }) + ) + ); + + if (config.output) { + for (const path of config.additionalPathsToCopy) { + // Define the output folder for API docs assets + const assetsFolder = join(config.output, basename(path)); + + // Copy all files from assets folder to output + await cp(path, assetsFolder, { recursive: true }); + } + } + + // Create sliced input: each item contains head + its module's entries + headNodes reference + // This avoids sending all ~4900 entries to every worker and recomputing groupings + const entries = headNodes.map(head => ({ + head, + nodes: groupedModules.get(head.api), + headNodes, + })); + + // Stream chunks as they complete - HTML files are written immediately + for await (const chunkResult of worker.stream(entries, navigation)) { + // Write files for this chunk in the generate method (main thread) + if (config.output) { + for (const template of chunkResult) { + let result = replaceTemplateValues(apiTemplate, template, config); + + if (config.minify) { + result = await minifyHTML(result); + } + + await writeFile(join(config.output, `${template.api}.html`), result); + } + } + + yield chunkResult; + } +} diff --git a/packages/core/src/generators/legacy-html/index.mjs b/packages/core/src/generators/legacy-html/index.mjs index 332d0df1..32003ec1 100644 --- a/packages/core/src/generators/legacy-html/index.mjs +++ b/packages/core/src/generators/legacy-html/index.mjs @@ -1,151 +1,37 @@ 'use strict'; -import { readFile, cp } from 'node:fs/promises'; -import { basename, join } from 'node:path'; +import { join } from 'node:path'; -import buildContent from './utils/buildContent.mjs'; -import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs'; -import tableOfContents from './utils/tableOfContents.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; import { GITHUB_EDIT_URL } from '../../utils/configuration/templates.mjs'; -import { writeFile } from '../../utils/file.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { minifyHTML } from '../../utils/html-minifier.mjs'; -import { getRemarkRehypeWithShiki } from '../../utils/remark.mjs'; - -export const name = 'legacy-html'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; -export const defaultConfiguration = { - templatePath: join(import.meta.dirname, 'template.html'), - additionalPathsToCopy: [join(import.meta.dirname, 'assets')], - ref: 'main', - pageURL: '{baseURL}/latest-{version}/api{path}.html', - editURL: `${GITHUB_EDIT_URL}/doc/api{path}.md`, -}; - -/** - * Creates a heading object with the given name. - * @param {string} name - The name of the heading - * @returns {HeadingMetadataEntry} The heading object - */ -const getHeading = name => ({ depth: 1, data: { name } }); - -const remarkRehypeProcessor = getRemarkRehypeWithShiki(); +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Process a chunk of items in a worker thread. - * Builds HTML template objects - FS operations happen in generate(). * - * Each item is pre-grouped {head, nodes, headNodes} - no need to - * recompute groupNodesByModule for every chunk. + * This generator generates the legacy HTML pages of the legacy API docs + * for retro-compatibility and while we are implementing the new 'react' and 'html' generators. * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(slicedInput, itemIndices, navigation) { - const results = []; - - for (const idx of itemIndices) { - const { head, nodes, headNodes } = slicedInput[idx]; - - const nav = navigation.replace( - `class="nav-${head.api}`, - `class="nav-${head.api} active` - ); - - const toc = String( - remarkRehypeProcessor.processSync( - tableOfContents(nodes, { - maxDepth: 5, - parser: tableOfContents.parseToCNode, - }) - ) - ); - - const content = buildContent(headNodes, nodes, remarkRehypeProcessor); - - const apiAsHeading = head.api.charAt(0).toUpperCase() + head.api.slice(1); - - const template = { - api: head.api, - path: head.path, - added: head.introduced_in ?? '', - section: head.heading.data.name || apiAsHeading, - toc, - nav, - content, - }; - - results.push(template); - } - - return results; -} - -/** - * Generates the legacy version of the API docs in HTML + * This generator is a top-level generator, and it takes the raw AST tree of the API doc files + * and generates the HTML files to the specified output directory from the configuration settings * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function* generate(input, worker) { - const config = getConfig('legacy-html'); - - const apiTemplate = await readFile(config.templatePath, 'utf-8'); - - const groupedModules = groupNodesByModule(input); - - const headNodes = input - .filter(node => node.heading.depth === 1) - .toSorted((a, b) => a.heading.data.name.localeCompare(b.heading.data.name)); - - const indexOfFiles = config.index - ? config.index.map(({ api, section }) => ({ - api, - heading: getHeading(section), - })) - : headNodes; - - const navigation = String( - remarkRehypeProcessor.processSync( - tableOfContents(indexOfFiles, { - maxDepth: 1, - parser: tableOfContents.parseNavigationNode, - }) - ) - ); - - if (config.output) { - for (const path of config.additionalPathsToCopy) { - // Define the output folder for API docs assets - const assetsFolder = join(config.output, basename(path)); - - // Copy all files from assets folder to output - await cp(path, assetsFolder, { recursive: true }); - } - } +export default createLazyGenerator({ + name: 'legacy-html', - // Create sliced input: each item contains head + its module's entries + headNodes reference - // This avoids sending all ~4900 entries to every worker and recomputing groupings - const entries = headNodes.map(head => ({ - head, - nodes: groupedModules.get(head.api), - headNodes, - })); + version: '1.0.0', - // Stream chunks as they complete - HTML files are written immediately - for await (const chunkResult of worker.stream(entries, navigation)) { - // Write files for this chunk in the generate method (main thread) - if (config.output) { - for (const template of chunkResult) { - let result = replaceTemplateValues(apiTemplate, template, config); + description: + 'Generates the legacy version of the API docs in HTML, with the assets and styles included as files', - if (config.minify) { - result = await minifyHTML(result); - } + dependsOn: 'metadata', - await writeFile(join(config.output, `${template.api}.html`), result); - } - } + defaultConfiguration: { + templatePath: join(import.meta.dirname, 'template.html'), + additionalPathsToCopy: [join(import.meta.dirname, 'assets')], + ref: 'main', + pageURL: '{baseURL}/latest-{version}/api{path}.html', + editURL: `${GITHUB_EDIT_URL}/doc/api{path}.md`, + }, - yield chunkResult; - } -} + hasParallelProcessor: true, +}); diff --git a/packages/core/src/generators/legacy-json-all/generate.mjs b/packages/core/src/generators/legacy-json-all/generate.mjs new file mode 100644 index 00000000..bc04f9aa --- /dev/null +++ b/packages/core/src/generators/legacy-json-all/generate.mjs @@ -0,0 +1,80 @@ +'use strict'; + +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import getConfig from '../../utils/configuration/index.mjs'; +import { legacyToJSON } from '../../utils/generators.mjs'; + +/** + * Generates the legacy JSON `all.json` file. + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('legacy-json-all'); + + /** + * The consolidated output object that will contain + * combined data from all sections in the input. + * + * @type {import('./types.d.ts').Output} + */ + const generatedValue = { + miscs: [], + modules: [], + classes: [], + globals: [], + methods: [], + }; + + /** + * The properties to copy from each section in the input + */ + const propertiesToCopy = Object.keys(generatedValue); + + // Create a map of api name to index position for sorting + const indexOrder = new Map( + config.index?.map(({ api }, position) => [`doc/api/${api}.md`, position]) ?? + [] + ); + + // Sort input by index order (documents not in index go to the end) + const sortedInput = input.toSorted((a, b) => { + const aOrder = indexOrder.get(a.source) ?? Infinity; + const bOrder = indexOrder.get(b.source) ?? Infinity; + + return aOrder - bOrder; + }); + + // Aggregate all sections into the output + for (const section of sortedInput) { + // Skip index.json - it has no useful content, just navigation + if (section.api === 'index') { + continue; + } + + for (const property of propertiesToCopy) { + const items = section[property]; + + if (Array.isArray(items)) { + const enrichedItems = section.source + ? items.map(item => ({ ...item, source: section.source })) + : items; + + generatedValue[property].push(...enrichedItems); + } + } + } + + if (config.output) { + await writeFile( + join(config.output, 'all.json'), + config.minify + ? legacyToJSON(generatedValue) + : legacyToJSON(generatedValue, null, 2) + ); + } + + return generatedValue; +} diff --git a/packages/core/src/generators/legacy-json-all/index.mjs b/packages/core/src/generators/legacy-json-all/index.mjs index ae11fddf..dcfe0fab 100644 --- a/packages/core/src/generators/legacy-json-all/index.mjs +++ b/packages/core/src/generators/legacy-json-all/index.mjs @@ -1,86 +1,24 @@ 'use strict'; -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import getConfig from '../../utils/configuration/index.mjs'; -import { legacyToJSON } from '../../utils/generators.mjs'; - -export const name = 'legacy-json-all'; -export const dependsOn = '@node-core/doc-kit/generators/legacy-json'; -export const defaultConfiguration = { - minify: false, -}; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Generates the legacy JSON `all.json` file. + * This generator consolidates data from the `legacy-json` generator into a single + * JSON file (`all.json`). * - * @type {import('./types').Generator['generate']} + * @type {import('./types.d.ts').Generator} */ -export async function generate(input) { - const config = getConfig('legacy-json-all'); - - /** - * The consolidated output object that will contain - * combined data from all sections in the input. - * - * @type {import('./types.d.ts').Output} - */ - const generatedValue = { - miscs: [], - modules: [], - classes: [], - globals: [], - methods: [], - }; - - /** - * The properties to copy from each section in the input - */ - const propertiesToCopy = Object.keys(generatedValue); - - // Create a map of api name to index position for sorting - const indexOrder = new Map( - config.index?.map(({ api }, position) => [`doc/api/${api}.md`, position]) ?? - [] - ); - - // Sort input by index order (documents not in index go to the end) - const sortedInput = input.toSorted((a, b) => { - const aOrder = indexOrder.get(a.source) ?? Infinity; - const bOrder = indexOrder.get(b.source) ?? Infinity; - - return aOrder - bOrder; - }); - - // Aggregate all sections into the output - for (const section of sortedInput) { - // Skip index.json - it has no useful content, just navigation - if (section.api === 'index') { - continue; - } - - for (const property of propertiesToCopy) { - const items = section[property]; +export default createLazyGenerator({ + name: 'legacy-json-all', - if (Array.isArray(items)) { - const enrichedItems = section.source - ? items.map(item => ({ ...item, source: section.source })) - : items; + version: '1.0.0', - generatedValue[property].push(...enrichedItems); - } - } - } + description: + 'Generates the `all.json` file from the `legacy-json` generator, which includes all the modules in one single file.', - if (config.output) { - await writeFile( - join(config.output, 'all.json'), - config.minify - ? legacyToJSON(generatedValue) - : legacyToJSON(generatedValue, null, 2) - ); - } + dependsOn: 'legacy-json', - return generatedValue; -} + defaultConfiguration: { + minify: false, + }, +}); diff --git a/packages/core/src/generators/legacy-json/generate.mjs b/packages/core/src/generators/legacy-json/generate.mjs new file mode 100644 index 00000000..d5f5fb82 --- /dev/null +++ b/packages/core/src/generators/legacy-json/generate.mjs @@ -0,0 +1,66 @@ +'use strict'; + +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { createSectionBuilder } from './utils/buildSection.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { groupNodesByModule, legacyToJSON } from '../../utils/generators.mjs'; + +const buildSection = createSectionBuilder(); + +/** + * Process a chunk of items in a worker thread. + * Builds JSON sections - FS operations happen in generate(). + * + * Each item is pre-grouped {head, nodes} - no need to + * recompute groupNodesByModule for every chunk. + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(slicedInput, itemIndices) { + const results = []; + + for (const idx of itemIndices) { + const { head, nodes } = slicedInput[idx]; + + results.push(buildSection(head, nodes)); + } + + return results; +} + +/** + * Generates a legacy JSON file. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input, worker) { + const config = getConfig('legacy-json'); + + const groupedModules = groupNodesByModule(input); + + const headNodes = input.filter(node => node.heading.depth === 1); + + // Create sliced input: each item contains head + its module's entries + // This avoids sending all 4900+ entries to every worker + const entries = headNodes.map(head => ({ + head, + nodes: groupedModules.get(head.api), + })); + + for await (const chunkResult of worker.stream(entries)) { + if (config.output) { + for (const section of chunkResult) { + const out = join(config.output, `${section.api}.json`); + + await writeFile( + out, + config.minify ? legacyToJSON(section) : legacyToJSON(section, null, 2) + ); + } + } + + yield chunkResult; + } +} diff --git a/packages/core/src/generators/legacy-json/index.mjs b/packages/core/src/generators/legacy-json/index.mjs index 69ec68ff..a2b5f5d9 100644 --- a/packages/core/src/generators/legacy-json/index.mjs +++ b/packages/core/src/generators/legacy-json/index.mjs @@ -1,73 +1,31 @@ 'use strict'; -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { createSectionBuilder } from './utils/buildSection.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { groupNodesByModule, legacyToJSON } from '../../utils/generators.mjs'; - -export const name = 'legacy-json'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; -export const defaultConfiguration = { - ref: 'main', - minify: false, -}; - -const buildSection = createSectionBuilder(); +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Process a chunk of items in a worker thread. - * Builds JSON sections - FS operations happen in generate(). + * This generator is responsible for generating the legacy JSON files for the + * legacy API docs for retro-compatibility. It is to be replaced while we work + * on the new schema for this file. * - * Each item is pre-grouped {head, nodes} - no need to - * recompute groupNodesByModule for every chunk. + * This is a top-level generator, intaking the raw AST tree of the api docs. + * It generates JSON files to the specified output directory given by the + * config. * - * @type {import('./types').Generator['processChunk']} + * @type {import('./types').Generator} */ -export async function processChunk(slicedInput, itemIndices) { - const results = []; - - for (const idx of itemIndices) { - const { head, nodes } = slicedInput[idx]; - - results.push(buildSection(head, nodes)); - } - - return results; -} - -/** - * Generates a legacy JSON file. - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(input, worker) { - const config = getConfig('legacy-json'); - - const groupedModules = groupNodesByModule(input); +export default createLazyGenerator({ + name: 'legacy-json', - const headNodes = input.filter(node => node.heading.depth === 1); + version: '1.0.0', - // Create sliced input: each item contains head + its module's entries - // This avoids sending all 4900+ entries to every worker - const entries = headNodes.map(head => ({ - head, - nodes: groupedModules.get(head.api), - })); + description: 'Generates the legacy version of the JSON API docs.', - for await (const chunkResult of worker.stream(entries)) { - if (config.output) { - for (const section of chunkResult) { - const out = join(config.output, `${section.api}.json`); + dependsOn: 'metadata', - await writeFile( - out, - config.minify ? legacyToJSON(section) : legacyToJSON(section, null, 2) - ); - } - } + defaultConfiguration: { + ref: 'main', + minify: false, + }, - yield chunkResult; - } -} + hasParallelProcessor: true, +}); diff --git a/packages/core/src/generators/llms-txt/generate.mjs b/packages/core/src/generators/llms-txt/generate.mjs new file mode 100644 index 00000000..00852c6a --- /dev/null +++ b/packages/core/src/generators/llms-txt/generate.mjs @@ -0,0 +1,32 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { buildApiDocLink } from './utils/buildApiDocLink.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +/** + * Generates a llms.txt file + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('llms-txt'); + + const template = await readFile(config.templatePath, 'utf-8'); + + const apiDocsLinks = input + .filter(entry => entry.heading.depth === 1) + .map(entry => `- ${buildApiDocLink(entry, config)}`) + .join('\n'); + + const filledTemplate = `${template}${apiDocsLinks}`; + + if (config.output) { + await writeFile(join(config.output, 'llms.txt'), filledTemplate); + } + + return filledTemplate; +} diff --git a/packages/core/src/generators/llms-txt/index.mjs b/packages/core/src/generators/llms-txt/index.mjs index d37bd214..078ec7f1 100644 --- a/packages/core/src/generators/llms-txt/index.mjs +++ b/packages/core/src/generators/llms-txt/index.mjs @@ -1,39 +1,27 @@ 'use strict'; -import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { buildApiDocLink } from './utils/buildApiDocLink.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -export const name = 'llms-txt'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; -export const defaultConfiguration = { - templatePath: join(import.meta.dirname, 'template.txt'), - pageURL: '{baseURL}/latest/api{path}.md', -}; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Generates a llms.txt file + * This generator generates a llms.txt file to provide information to LLMs at + * inference time * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function generate(input) { - const config = getConfig('llms-txt'); - - const template = await readFile(config.templatePath, 'utf-8'); +export default createLazyGenerator({ + name: 'llms-txt', - const apiDocsLinks = input - .filter(entry => entry.heading.depth === 1) - .map(entry => `- ${buildApiDocLink(entry, config)}`) - .join('\n'); + version: '1.0.0', - const filledTemplate = `${template}${apiDocsLinks}`; + description: + 'Generates a llms.txt file to provide information to LLMs at inference time', - if (config.output) { - await writeFile(join(config.output, 'llms.txt'), filledTemplate); - } + dependsOn: 'metadata', - return filledTemplate; -} + defaultConfiguration: { + templatePath: join(import.meta.dirname, 'template.txt'), + pageURL: '{baseURL}/latest/api{path}.md', + }, +}); diff --git a/packages/core/src/generators/man-page/generate.mjs b/packages/core/src/generators/man-page/generate.mjs new file mode 100644 index 00000000..4f779f71 --- /dev/null +++ b/packages/core/src/generators/man-page/generate.mjs @@ -0,0 +1,78 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { + convertOptionToMandoc, + convertEnvVarToMandoc, +} from './utils/converter.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +/** + * @param {Array} components + * @param {number} start + * @param {number} end + * @param {(element: import('../metadata/types').MetadataEntry) => string} convert + * @returns {string} + */ +function extractMandoc(components, start, end, convert) { + return components + .slice(start, end) + .filter(({ heading }) => heading.depth === 3) + .map(convert) + .join(''); +} + +/** + * Generates the Node.js man-page + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('man-page'); + + // Find the appropriate headers + const optionsStart = input.findIndex( + ({ heading }) => heading.data.slug === config.cliOptionsHeaderSlug + ); + + const environmentStart = input.findIndex( + ({ heading }) => heading.data.slug === config.envVarsHeaderSlug + ); + + // The first header that is <3 in depth after environmentStart + const environmentEnd = input.findIndex( + ({ heading }, index) => heading.depth < 3 && index > environmentStart + ); + + const output = { + // Extract the CLI options. + options: extractMandoc( + input, + optionsStart + 1, + environmentStart, + convertOptionToMandoc + ), + // Extract the environment variables. + env: extractMandoc( + input, + environmentStart + 1, + environmentEnd, + convertEnvVarToMandoc + ), + }; + + const template = await readFile(config.templatePath, 'utf-8'); + + const filledTemplate = template + .replace('__OPTIONS__', output.options) + .replace('__ENVIRONMENT__', output.env); + + if (config.output) { + await writeFile(join(config.output, config.fileName), filledTemplate); + } + + return filledTemplate; +} diff --git a/packages/core/src/generators/man-page/index.mjs b/packages/core/src/generators/man-page/index.mjs index 89c83337..1128720e 100644 --- a/packages/core/src/generators/man-page/index.mjs +++ b/packages/core/src/generators/man-page/index.mjs @@ -1,87 +1,28 @@ 'use strict'; -import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { - convertOptionToMandoc, - convertEnvVarToMandoc, -} from './utils/converter.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -export const name = 'man-page'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; -export const defaultConfiguration = { - fileName: 'node.1', - cliOptionsHeaderSlug: 'options', - envVarsHeaderSlug: 'environment-variables-1', - templatePath: join(import.meta.dirname, 'template.1'), -}; - -/** - * @param {Array} components - * @param {number} start - * @param {number} end - * @param {(element: import('../metadata/types').MetadataEntry) => string} convert - * @returns {string} - */ -function extractMandoc(components, start, end, convert) { - return components - .slice(start, end) - .filter(({ heading }) => heading.depth === 3) - .map(convert) - .join(''); -} +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Generates the Node.js man-page + * This generator generates a man page version of the CLI.md file. + * See https://man.openbsd.org/mdoc.7 for the formatting. * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function generate(input) { - const config = getConfig('man-page'); - - // Find the appropriate headers - const optionsStart = input.findIndex( - ({ heading }) => heading.data.slug === config.cliOptionsHeaderSlug - ); - - const environmentStart = input.findIndex( - ({ heading }) => heading.data.slug === config.envVarsHeaderSlug - ); - - // The first header that is <3 in depth after environmentStart - const environmentEnd = input.findIndex( - ({ heading }, index) => heading.depth < 3 && index > environmentStart - ); - - const output = { - // Extract the CLI options. - options: extractMandoc( - input, - optionsStart + 1, - environmentStart, - convertOptionToMandoc - ), - // Extract the environment variables. - env: extractMandoc( - input, - environmentStart + 1, - environmentEnd, - convertEnvVarToMandoc - ), - }; +export default createLazyGenerator({ + name: 'man-page', - const template = await readFile(config.templatePath, 'utf-8'); + version: '1.0.0', - const filledTemplate = template - .replace('__OPTIONS__', output.options) - .replace('__ENVIRONMENT__', output.env); + description: 'Generates the Node.js man-page.', - if (config.output) { - await writeFile(join(config.output, config.fileName), filledTemplate); - } + dependsOn: 'metadata', - return filledTemplate; -} + defaultConfiguration: { + fileName: 'node.1', + cliOptionsHeaderSlug: 'options', + envVarsHeaderSlug: 'environment-variables-1', + templatePath: join(import.meta.dirname, 'template.1'), + }, +}); diff --git a/packages/core/src/generators/metadata/generate.mjs b/packages/core/src/generators/metadata/generate.mjs new file mode 100644 index 00000000..ecb830bb --- /dev/null +++ b/packages/core/src/generators/metadata/generate.mjs @@ -0,0 +1,38 @@ +'use strict'; + +import { parseApiDoc } from './utils/parse.mjs'; +import { parseTypeMap } from '../../parsers/json.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; + +/** + * Process a chunk of API doc files in a worker thread. + * Called by chunk-worker.mjs for parallel processing. + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(fullInput, itemIndices, typeMap) { + const results = []; + + for (const idx of itemIndices) { + results.push(...parseApiDoc(fullInput[idx], typeMap)); + } + + return results; +} + +/** + * Generates a flattened list of metadata entries from API docs. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(inputs, worker) { + const { metadata: config } = getConfig(); + + const typeMap = await parseTypeMap(config.typeMap); + + // Stream chunks as they complete - allows dependent generators + // to start collecting/preparing while we're still processing + for await (const chunkResult of worker.stream(inputs, typeMap)) { + yield chunkResult.flat(); + } +} diff --git a/packages/core/src/generators/metadata/index.mjs b/packages/core/src/generators/metadata/index.mjs index 97c23b11..53c426d4 100644 --- a/packages/core/src/generators/metadata/index.mjs +++ b/packages/core/src/generators/metadata/index.mjs @@ -1,44 +1,20 @@ 'use strict'; -import { parseApiDoc } from './utils/parse.mjs'; -import { parseTypeMap } from '../../parsers/json.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; - -export const name = 'metadata'; -export const dependsOn = '@node-core/doc-kit/generators/ast'; -export const defaultConfiguration = { - typeMap: import.meta.resolve('./typeMap.json'), -}; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Process a chunk of API doc files in a worker thread. - * Called by chunk-worker.mjs for parallel processing. + * This generator generates a flattened list of metadata entries from a API doc * - * @type {import('./types').Generator['processChunk']} + * @type {import('./types').Generator} */ -export async function processChunk(fullInput, itemIndices, typeMap) { - const results = []; - - for (const idx of itemIndices) { - results.push(...parseApiDoc(fullInput[idx], typeMap)); - } +export default createLazyGenerator({ + name: 'metadata', - return results; -} + version: '1.0.0', -/** - * Generates a flattened list of metadata entries from API docs. - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(inputs, worker) { - const { metadata: config } = getConfig(); + description: 'generates a flattened list of API doc metadata entries', - const typeMap = await parseTypeMap(config.typeMap); + dependsOn: 'ast', - // Stream chunks as they complete - allows dependent generators - // to start collecting/preparing while we're still processing - for await (const chunkResult of worker.stream(inputs, typeMap)) { - yield chunkResult.flat(); - } -} + hasParallelProcessor: true, +}); diff --git a/packages/core/src/generators/orama-db/generate.mjs b/packages/core/src/generators/orama-db/generate.mjs new file mode 100644 index 00000000..39134009 --- /dev/null +++ b/packages/core/src/generators/orama-db/generate.mjs @@ -0,0 +1,60 @@ +'use strict'; + +import { join } from 'node:path'; + +import { create, save, insertMultiple } from '@orama/orama'; + +import { SCHEMA } from './constants.mjs'; +import { buildHierarchicalTitle } from './utils/title.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { transformNodeToString } from '../../utils/unist.mjs'; + +/** + * Generates the Orama database. + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('orama-db'); + + const db = create({ schema: SCHEMA }); + + const apiGroups = groupNodesByModule(input); + + // Process all API groups and flatten into a single document array + const documents = Array.from(apiGroups.values()).flatMap(headings => + headings.map((entry, index) => { + const hierarchicalTitle = buildHierarchicalTitle(headings, index); + + const paragraph = entry.content.children.find( + child => child.type === 'paragraph' + ); + + return { + title: hierarchicalTitle, + description: paragraph + ? transformNodeToString(paragraph, true) + : undefined, + href: `${entry.path.slice(1)}.html#${entry.heading.data.slug}`, + siteSection: headings[0].heading.data.name, + }; + }) + ); + + // Insert all documents + await insertMultiple(db, documents); + + const result = save(db); + + // Persist + if (config.output) { + await writeFile( + join(config.output, 'orama-db.json'), + config.minify ? JSON.stringify(result) : JSON.stringify(result, null, 2) + ); + } + + return result; +} diff --git a/packages/core/src/generators/orama-db/index.mjs b/packages/core/src/generators/orama-db/index.mjs index 390c08a7..bb6c2755 100644 --- a/packages/core/src/generators/orama-db/index.mjs +++ b/packages/core/src/generators/orama-db/index.mjs @@ -1,62 +1,19 @@ 'use strict'; -import { join } from 'node:path'; +import { createLazyGenerator } from '../../utils/generators.mjs'; -import { create, save, insertMultiple } from '@orama/orama'; - -import { SCHEMA } from './constants.mjs'; -import { buildHierarchicalTitle } from './utils/title.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { transformNodeToString } from '../../utils/unist.mjs'; - -export const name = 'orama-db'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; /** - * Generates the Orama database. + * This generator is responsible for generating the Orama database for the + * API docs. It is based on the legacy-json generator. * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function generate(input) { - const config = getConfig('orama-db'); - - const db = create({ schema: SCHEMA }); - - const apiGroups = groupNodesByModule(input); - - // Process all API groups and flatten into a single document array - const documents = Array.from(apiGroups.values()).flatMap(headings => - headings.map((entry, index) => { - const hierarchicalTitle = buildHierarchicalTitle(headings, index); - - const paragraph = entry.content.children.find( - child => child.type === 'paragraph' - ); - - return { - title: hierarchicalTitle, - description: paragraph - ? transformNodeToString(paragraph, true) - : undefined, - href: `${entry.path.slice(1)}.html#${entry.heading.data.slug}`, - siteSection: headings[0].heading.data.name, - }; - }) - ); - - // Insert all documents - await insertMultiple(db, documents); +export default createLazyGenerator({ + name: 'orama-db', - const result = save(db); + version: '1.0.0', - // Persist - if (config.output) { - await writeFile( - join(config.output, 'orama-db.json'), - config.minify ? JSON.stringify(result) : JSON.stringify(result, null, 2) - ); - } + description: 'Generates the Orama database for the API docs.', - return result; -} + dependsOn: 'metadata', +}); diff --git a/packages/core/src/generators/sitemap/generate.mjs b/packages/core/src/generators/sitemap/generate.mjs new file mode 100644 index 00000000..9ec9786a --- /dev/null +++ b/packages/core/src/generators/sitemap/generate.mjs @@ -0,0 +1,66 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { createPageSitemapEntry } from './utils/createPageSitemapEntry.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { populate } from '../../utils/configuration/templates.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +/** + * Generates a sitemap.xml file + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(entries) { + const { sitemap: config } = getConfig(); + + const template = await readFile( + join(import.meta.dirname, 'template.xml'), + 'utf-8' + ); + + const entryTemplate = await readFile( + join(import.meta.dirname, 'entry-template.xml'), + 'utf-8' + ); + + const lastmod = new Date().toISOString().split('T')[0]; + + const apiPages = entries + .filter(entry => entry.heading.depth === 1) + .map(entry => createPageSitemapEntry(entry, config, lastmod)); + + const loc = populate(config.indexURL, config); + + /** + * @typedef {import('./types').SitemapEntry} + */ + const mainPage = { + loc, + lastmod, + changefreq: 'daily', + priority: '1.0', + }; + + apiPages.push(mainPage); + + const urlset = apiPages + .map(page => + entryTemplate + .replace('__LOC__', page.loc) + .replace('__LASTMOD__', page.lastmod) + .replace('__CHANGEFREQ__', page.changefreq) + .replace('__PRIORITY__', page.priority) + ) + .join(''); + + const sitemap = template.replace('__URLSET__', urlset); + + if (config.output) { + await writeFile(join(config.output, 'sitemap.xml'), sitemap, 'utf-8'); + } + + return sitemap; +} diff --git a/packages/core/src/generators/sitemap/index.mjs b/packages/core/src/generators/sitemap/index.mjs index 729961e6..1f983423 100644 --- a/packages/core/src/generators/sitemap/index.mjs +++ b/packages/core/src/generators/sitemap/index.mjs @@ -1,73 +1,23 @@ 'use strict'; -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { createPageSitemapEntry } from './utils/createPageSitemapEntry.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { populate } from '../../utils/configuration/templates.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -export const name = 'sitemap'; -export const dependsOn = '@node-core/doc-kit/generators/metadata'; -export const defaultConfiguration = { - indexURL: '{baseURL}/latest/api/', - pageURL: '{indexURL}{path}.html', -}; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Generates a sitemap.xml file + * This generator generates a sitemap.xml file for search engine optimization * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function generate(entries) { - const { sitemap: config } = getConfig(); - - const template = await readFile( - join(import.meta.dirname, 'template.xml'), - 'utf-8' - ); - - const entryTemplate = await readFile( - join(import.meta.dirname, 'entry-template.xml'), - 'utf-8' - ); - - const lastmod = new Date().toISOString().split('T')[0]; - - const apiPages = entries - .filter(entry => entry.heading.depth === 1) - .map(entry => createPageSitemapEntry(entry, config, lastmod)); - - const loc = populate(config.indexURL, config); - - /** - * @typedef {import('./types').SitemapEntry} - */ - const mainPage = { - loc, - lastmod, - changefreq: 'daily', - priority: '1.0', - }; - - apiPages.push(mainPage); +export default createLazyGenerator({ + name: 'sitemap', - const urlset = apiPages - .map(page => - entryTemplate - .replace('__LOC__', page.loc) - .replace('__LASTMOD__', page.lastmod) - .replace('__CHANGEFREQ__', page.changefreq) - .replace('__PRIORITY__', page.priority) - ) - .join(''); + version: '1.0.0', - const sitemap = template.replace('__URLSET__', urlset); + description: 'Generates a sitemap.xml file for search engine optimization', - if (config.output) { - await writeFile(join(config.output, 'sitemap.xml'), sitemap, 'utf-8'); - } + dependsOn: 'metadata', - return sitemap; -} + defaultConfiguration: { + indexURL: '{baseURL}/latest/api/', + pageURL: '{indexURL}{path}.html', + }, +}); diff --git a/packages/core/src/generators/types.d.ts b/packages/core/src/generators/types.d.ts index 1265d77e..6374d800 100644 --- a/packages/core/src/generators/types.d.ts +++ b/packages/core/src/generators/types.d.ts @@ -1,4 +1,12 @@ +import type { publicGenerators, allGenerators } from './index.mjs'; + declare global { + // Public generators exposed to the CLI + export type AvailableGenerators = typeof publicGenerators; + + // All generators including internal ones (metadata, jsx-ast, ast-js) + export type AllGenerators = typeof allGenerators; + /** * ParallelWorker interface for distributing work across Node.js worker threads. * Streams results as chunks complete, enabling pipeline parallelism. @@ -20,7 +28,7 @@ declare global { } export interface ParallelTaskOptions { - generatorSpecifier: string; + generatorName: keyof AllGenerators; input: unknown[]; itemIndices: number[]; } @@ -52,18 +60,40 @@ declare global { G extends Generate, P extends ProcessChunk | undefined = undefined, > = { - readonly defaultConfiguration?: C; + readonly defaultConfiguration: C; + + // The name of the Generator. Must match the Key in AllGenerators + name: keyof AllGenerators; + + version: string; + + description: string; - // The name of the Generator - name: string; + hasParallelProcessor: boolean; /** - * The import specifier of the generator this one depends on. - * For example, '@node-core/doc-kit/generators/metadata'. + * The immediate generator that this generator depends on. + * For example, the `html` generator depends on the `react` generator. + * + * If a given generator has no "before" generator, it will be considered a top-level + * generator, and run in parallel. + * + * Assume you pass to the `createGenerator`: ['json', 'html'] as the generators, + * this means both the 'json' and the 'html' generators will be executed and generate their + * own outputs in parallel. If the 'html' generator depends on the 'react' generator, then + * the 'react' generator will be executed first, then the 'html' generator. + * + * But both 'json' and 'html' generators will be executed in parallel. + * + * If you pass `createGenerator` with ['react', 'html'], the 'react' generator will be executed first, + * as it is a top level generator and then the 'html' generator would be executed after the 'react' generator. + * + * The 'ast' generator is the top-level parser for markdown files. It has no dependencies. * - * If undefined, this is a top-level generator with no dependencies. + * The `ast-js` generator is the top-level parser for JavaScript files. It + * passes the ASTs for any JavaScript files given in the input. */ - dependsOn?: string; + dependsOn: keyof AllGenerators | undefined; /** * Generators are abstract and the different generators have different sort of inputs and outputs. diff --git a/packages/core/src/generators/web/generate.mjs b/packages/core/src/generators/web/generate.mjs new file mode 100644 index 00000000..57ca2a75 --- /dev/null +++ b/packages/core/src/generators/web/generate.mjs @@ -0,0 +1,54 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; + +import createASTBuilder from './utils/generate.mjs'; +import { processJSXEntries } from './utils/processing.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +/** + * Main generation function that processes JSX AST entries into web bundles. + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input) { + const config = getConfig('web'); + + const template = await readFile(config.templatePath, 'utf-8'); + + // Create AST builders for server and client programs + const astBuilders = createASTBuilder(); + + // Create require function for resolving external packages in server code + const requireFn = createRequire(import.meta.url); + + // Process all entries: convert JSX to HTML/CSS/JS + const { results, css, chunks } = await processJSXEntries( + input, + template, + astBuilders, + requireFn, + config + ); + + // Process all entries together (required for code-split bundles) + if (config.output) { + // Write HTML files + for (const { html, path } of results) { + await writeFile(join(config.output, `${path}.html`), html, 'utf-8'); + } + + // Write code-split JavaScript chunks + for (const chunk of chunks) { + await writeFile(join(config.output, chunk.fileName), chunk.code, 'utf-8'); + } + + // Write CSS bundle + await writeFile(join(config.output, 'styles.css'), css, 'utf-8'); + } + + return results.map(({ html }) => ({ html: html.toString(), css })); +} diff --git a/packages/core/src/generators/web/index.mjs b/packages/core/src/generators/web/index.mjs index 293fd177..f881bebf 100644 --- a/packages/core/src/generators/web/index.mjs +++ b/packages/core/src/generators/web/index.mjs @@ -1,69 +1,41 @@ 'use strict'; -import { readFile } from 'node:fs/promises'; -import { createRequire } from 'node:module'; import { join } from 'node:path'; -import createASTBuilder from './utils/generate.mjs'; -import { processJSXEntries } from './utils/processing.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -export const name = 'web'; -export const dependsOn = '@node-core/doc-kit/generators/jsx-ast'; -export const defaultConfiguration = { - templatePath: join(import.meta.dirname, 'template.html'), - title: 'Node.js', - imports: { - '#theme/Logo': '@node-core/ui-components/Common/NodejsLogo', - '#theme/Navigation': join(import.meta.dirname, './ui/components/NavBar'), - '#theme/Sidebar': join(import.meta.dirname, './ui/components/SideBar'), - '#theme/Metabar': join(import.meta.dirname, './ui/components/MetaBar'), - '#theme/Footer': join(import.meta.dirname, './ui/components/NoOp'), - '#theme/Layout': join(import.meta.dirname, './ui/components/Layout'), - }, -}; +import { createLazyGenerator } from '../../utils/generators.mjs'; /** - * Main generation function that processes JSX AST entries into web bundles. + * Web generator - transforms JSX AST entries into complete web bundles. + * + * This generator processes JSX AST entries and produces: + * - Server-side rendered HTML pages + * - Client-side JavaScript with code splitting + * - Bundled CSS styles + * + * Note: This generator does NOT support streaming/chunked processing because + * processJSXEntries needs all entries together to generate code-split bundles. * - * @type {import('./types').Generator['generate']} + * @type {import('./types').Generator} */ -export async function generate(input) { - const config = getConfig('web'); - - const template = await readFile(config.templatePath, 'utf-8'); - - // Create AST builders for server and client programs - const astBuilders = createASTBuilder(); - - // Create require function for resolving external packages in server code - const requireFn = createRequire(import.meta.url); - - // Process all entries: convert JSX to HTML/CSS/JS - const { results, css, chunks } = await processJSXEntries( - input, - template, - astBuilders, - requireFn, - config - ); - - // Process all entries together (required for code-split bundles) - if (config.output) { - // Write HTML files - for (const { html, path } of results) { - await writeFile(join(config.output, `${path}.html`), html, 'utf-8'); - } - - // Write code-split JavaScript chunks - for (const chunk of chunks) { - await writeFile(join(config.output, chunk.fileName), chunk.code, 'utf-8'); - } - - // Write CSS bundle - await writeFile(join(config.output, 'styles.css'), css, 'utf-8'); - } - - return results.map(({ html }) => ({ html: html.toString(), css })); -} +export default createLazyGenerator({ + name: 'web', + + version: '1.0.0', + + description: 'Generates HTML/CSS/JS bundles from JSX AST entries', + + dependsOn: 'jsx-ast', + + defaultConfiguration: { + templatePath: join(import.meta.dirname, 'template.html'), + title: 'Node.js', + imports: { + '#theme/Logo': '@node-core/ui-components/Common/NodejsLogo', + '#theme/Navigation': join(import.meta.dirname, './ui/components/NavBar'), + '#theme/Sidebar': join(import.meta.dirname, './ui/components/SideBar'), + '#theme/Metabar': join(import.meta.dirname, './ui/components/MetaBar'), + '#theme/Footer': join(import.meta.dirname, './ui/components/NoOp'), + '#theme/Layout': join(import.meta.dirname, './ui/components/Layout'), + }, + }, +}); diff --git a/packages/core/src/loader.mjs b/packages/core/src/loader.mjs deleted file mode 100644 index d0904cc2..00000000 --- a/packages/core/src/loader.mjs +++ /dev/null @@ -1,64 +0,0 @@ -'use strict'; - -/** @type {Map} */ -const cache = new Map(); - -/** - * Loads a generator by its import specifier. - * Imports the single entry point which exports generate, processChunk, - * name, dependsOn, and defaultConfiguration. - * - * @param {string} specifier - Full import specifier (e.g. '@node-core/doc-kit/generators/ast') - * @returns {Promise} The loaded generator - */ -export const loadGenerator = async specifier => { - if (cache.has(specifier)) { - return cache.get(specifier); - } - - const module = await import(specifier); - - const generator = { - specifier, - ...module, - }; - - cache.set(specifier, generator); - - return generator; -}; - -/** - * Resolves the full dependency chain for a set of target specifiers. - * Recursively follows `dependsOn` chains, loading each generator. - * - * @param {string[]} targets - Target generator specifiers - * @returns {Promise>} All generators needed, keyed by specifier - */ -export const resolveGeneratorGraph = async targets => { - /** @type {Map} */ - const loaded = new Map(); - - /** - * Resolve a generator via it's specifier - * @param {string} specifier - */ - const resolve = async specifier => { - if (loaded.has(specifier)) { - return; - } - - const generator = await loadGenerator(specifier); - loaded.set(specifier, generator); - - if (generator.dependsOn) { - await resolve(generator.dependsOn); - } - }; - - for (const target of targets) { - await resolve(target); - } - - return loaded; -}; diff --git a/packages/core/src/threading/__tests__/parallel.test.mjs b/packages/core/src/threading/__tests__/parallel.test.mjs index 3d0d56d3..766e3d03 100644 --- a/packages/core/src/threading/__tests__/parallel.test.mjs +++ b/packages/core/src/threading/__tests__/parallel.test.mjs @@ -1,7 +1,6 @@ import { deepStrictEqual, ok, strictEqual } from 'node:assert'; import { describe, it } from 'node:test'; -import { loadGenerator } from '../../loader.mjs'; import createWorkerPool from '../index.mjs'; import createParallelWorker from '../parallel.mjs'; @@ -39,20 +38,10 @@ async function collectChunks(generator) { return chunks; } -const metadataSpecifier = '@node-core/doc-kit/generators/metadata'; -const metadataGenerator = await loadGenerator(metadataSpecifier); -const astJsSpecifier = '@node-core/doc-kit/generators/ast-js'; -const astJsGenerator = await loadGenerator(astJsSpecifier); - describe('createParallelWorker', () => { it('should create a ParallelWorker with stream method', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker( - metadataSpecifier, - metadataGenerator, - pool, - { threads: 2 } - ); + const worker = createParallelWorker('metadata', pool, { threads: 2 }); ok(worker); strictEqual(typeof worker.stream, 'function'); @@ -62,7 +51,7 @@ describe('createParallelWorker', () => { it('should handle empty items array', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker(astJsSpecifier, astJsGenerator, pool, { + const worker = createParallelWorker('ast-js', pool, { threads: 2, chunkSize: 10, }); @@ -76,15 +65,10 @@ describe('createParallelWorker', () => { it('should distribute items to multiple worker threads', async () => { const pool = createWorkerPool(4); - const worker = createParallelWorker( - metadataSpecifier, - metadataGenerator, - pool, - { - threads: 4, - chunkSize: 1, - } - ); + const worker = createParallelWorker('metadata', pool, { + threads: 4, + chunkSize: 1, + }); const mockInput = [ { @@ -118,15 +102,10 @@ describe('createParallelWorker', () => { it('should yield results as chunks complete', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker( - metadataSpecifier, - metadataGenerator, - pool, - { - threads: 2, - chunkSize: 1, - } - ); + const worker = createParallelWorker('metadata', pool, { + threads: 2, + chunkSize: 1, + }); const mockInput = [ { @@ -148,15 +127,10 @@ describe('createParallelWorker', () => { it('should work with single thread and items', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker( - metadataSpecifier, - metadataGenerator, - pool, - { - threads: 2, - chunkSize: 5, - } - ); + const worker = createParallelWorker('metadata', pool, { + threads: 2, + chunkSize: 5, + }); const mockInput = [ { @@ -175,15 +149,10 @@ describe('createParallelWorker', () => { it('should use sliceInput for metadata generator', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker( - metadataSpecifier, - metadataGenerator, - pool, - { - threads: 2, - chunkSize: 1, - } - ); + const worker = createParallelWorker('metadata', pool, { + threads: 2, + chunkSize: 1, + }); const mockInput = [ { diff --git a/packages/core/src/threading/chunk-worker.mjs b/packages/core/src/threading/chunk-worker.mjs index 03e5a92b..08e363ac 100644 --- a/packages/core/src/threading/chunk-worker.mjs +++ b/packages/core/src/threading/chunk-worker.mjs @@ -1,8 +1,6 @@ +import { allGenerators } from '../generators/index.mjs'; import { setConfig } from '../utils/configuration/index.mjs'; -/** @type {Map>} */ -const generatorCache = new Map(); - /** * Processes a chunk of items using the specified generator's processChunk method. * This is the worker entry point for Piscina. @@ -11,7 +9,7 @@ const generatorCache = new Map(); * @returns {Promise} The processed result */ export default async ({ - generatorSpecifier, + generatorName, input, itemIndices, extra, @@ -19,11 +17,7 @@ export default async ({ }) => { await setConfig(configuration); - if (!generatorCache.has(generatorSpecifier)) { - generatorCache.set(generatorSpecifier, import(generatorSpecifier)); - } - - const { processChunk } = await generatorCache.get(generatorSpecifier); + const generator = allGenerators[generatorName]; - return processChunk(input, itemIndices, extra); + return generator.processChunk(input, itemIndices, extra); }; diff --git a/packages/core/src/threading/parallel.mjs b/packages/core/src/threading/parallel.mjs index f369cbcf..d74ba92e 100644 --- a/packages/core/src/threading/parallel.mjs +++ b/packages/core/src/threading/parallel.mjs @@ -1,5 +1,6 @@ 'use strict'; +import { allGenerators } from '../generators/index.mjs'; import logger from '../logger/index.mjs'; const parallelLogger = logger.child('parallel'); @@ -29,9 +30,8 @@ const createChunks = (count, size) => { * @param {unknown[]} fullInput - Full input array * @param {number[]} indices - Indices to process * @param {Object} extra - Stuff to pass to the worker - * @param {object} configuration - Serialized options - * @param {string} generatorSpecifier - Import specifier of the generator - * @param {string} generatorName - Short name of the generator + * @param {import('../utils/configuration/types').Configuration} configuration - Serialized options + * @param {string} generatorName - Name of the generator * @returns {ParallelTaskOptions} Task data for Piscina */ const createTask = ( @@ -39,11 +39,10 @@ const createTask = ( indices, extra, configuration, - generatorSpecifier, generatorName ) => { return { - generatorSpecifier, + generatorName, // Only send the items needed for this chunk (reduces serialization overhead) input: indices.map(i => fullInput[i]), // Remap indices to 0-based for the sliced array @@ -59,20 +58,19 @@ const createTask = ( /** * Creates a parallel worker that distributes work across a Piscina thread pool. * - * @param {string} generatorSpecifier - Import specifier for the generator - * @param {object} generator - The loaded generator object + * @param {keyof AllGenerators} generatorName - Generator name * @param {import('piscina').Piscina} pool - Piscina instance - * @param {object} configuration - Generator options + * @param {import('../utils/configuration/types').Configuration} configuration - Generator options * @returns {ParallelWorker} */ export default function createParallelWorker( - generatorSpecifier, - generator, + generatorName, pool, configuration ) { const { threads, chunkSize } = configuration; - const { name: generatorName } = generator; + + const generator = allGenerators[generatorName]; return { /** @@ -92,12 +90,7 @@ export default function createParallelWorker( parallelLogger.debug( `Distributing ${items.length} items across ${chunks.length} chunks`, - { - generator: generatorName, - chunks: chunks.length, - chunkSize, - threads, - } + { generator: generatorName, chunks: chunks.length, chunkSize, threads } ); const runInOneGo = threads <= 1 || items.length <= 2; @@ -115,14 +108,7 @@ export default function createParallelWorker( const promise = pool .run( - createTask( - items, - indices, - extra, - configuration, - generatorSpecifier, - generatorName - ) + createTask(items, indices, extra, configuration, generatorName) ) .then(result => ({ promise, result })); diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index cd0c48eb..e4c9e1a8 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; +import { describe, it, mock, afterEach } from 'node:test'; import { groupNodesByModule, @@ -7,6 +7,7 @@ import { coerceSemVer, getCompatibleVersions, legacyToJSON, + createLazyGenerator, } from '../generators.mjs'; describe('groupNodesByModule', () => { @@ -123,3 +124,34 @@ describe('legacyToJSON', () => { assert.ok(result.includes('\n')); }); }); + +describe('createLazyGenerator', () => { + afterEach(() => mock.restoreAll()); + + it('spreads metadata properties onto the returned object', () => { + const metadata = { name: 'ast', version: '1.0.0', dependsOn: undefined }; + const gen = createLazyGenerator(metadata); + assert.equal(gen.name, 'ast'); + assert.equal(gen.version, '1.0.0'); + }); + + it('exposes generate and processChunk functions that delegate to the lazily loaded module', async () => { + // Both exports are mocked in a single mock.module() call to avoid ESM import + // cache collisions that occur when re-mocking the same specifier across two it() blocks. + const specifier = import.meta.resolve('../../generators/ast/generate.mjs'); + const fakeGenerate = async input => `processed:${input}`; + const fakeProcessChunk = async (input, indices) => + indices.map(i => input[i]); + mock.module(specifier, { + namedExports: { generate: fakeGenerate, processChunk: fakeProcessChunk }, + }); + + const gen = createLazyGenerator({ name: 'ast' }); + + const generateResult = await gen.generate('hello'); + assert.equal(generateResult, 'processed:hello'); + + const processChunkResult = await gen.processChunk(['a', 'b', 'c'], [0, 2]); + assert.deepStrictEqual(processChunkResult, ['a', 'c']); + }); +}); diff --git a/packages/core/src/utils/configuration/__tests__/index.test.mjs b/packages/core/src/utils/configuration/__tests__/index.test.mjs index 942255dc..293a468f 100644 --- a/packages/core/src/utils/configuration/__tests__/index.test.mjs +++ b/packages/core/src/utils/configuration/__tests__/index.test.mjs @@ -12,6 +12,15 @@ const createMockConfig = (overrides = {}) => ({ }); // Mock modules +mock.module('../../../generators/index.mjs', { + namedExports: { + allGenerators: { + json: { defaultConfiguration: { format: 'json' } }, + html: { defaultConfiguration: { format: 'html' } }, + markdown: {}, + }, + }, +}); mock.module('../../../parsers/markdown.mjs', { namedExports: { parseChangelog: mockParseChangelog, @@ -31,20 +40,6 @@ const { default: getConfig, } = await import('../index.mjs'); -// Create a mock loaded generators map for tests -const createMockGenerators = () => - new Map([ - [ - '@node-core/doc-kit/generators/json', - { name: 'json', defaultConfiguration: { format: 'json' } }, - ], - [ - '@node-core/doc-kit/generators/html', - { name: 'html', defaultConfiguration: { format: 'html' } }, - ], - ['@node-core/doc-kit/generators/markdown', { name: 'markdown' }], - ]); - // Helper to reset all mocks const resetAllMocks = () => { [mockParseChangelog, mockParseIndex, mockImportFromURL].forEach(m => @@ -136,14 +131,11 @@ describe('config.mjs', () => { createMockConfig({ global: { input: 'custom-src/' } }) ); - const config = await createRunConfiguration( - { - configFile: 'config.mjs', - output: 'custom-dist/', - threads: 2, - }, - createMockGenerators() - ); + const config = await createRunConfiguration({ + configFile: 'config.mjs', + output: 'custom-dist/', + threads: 2, + }); assert.strictEqual(config.global.input, 'custom-src/'); assert.strictEqual(config.global.output, 'custom-dist/'); @@ -165,10 +157,7 @@ describe('config.mjs', () => { ); resetAllMocks(); // Clear calls from getDefaultConfig - await createRunConfiguration( - { configFile: 'config.mjs' }, - createMockGenerators() - ); + await createRunConfiguration({ configFile: 'config.mjs' }); // Each should be called at least once for the string value assert.ok( @@ -183,26 +172,20 @@ describe('config.mjs', () => { }); it('should enforce minimum constraints', async () => { - const config = await createRunConfiguration( - { - threads: -5, - chunkSize: 0, - }, - createMockGenerators() - ); + const config = await createRunConfiguration({ + threads: -5, + chunkSize: 0, + }); assert.strictEqual(config.threads, 1); assert.strictEqual(config.chunkSize, 1); }); it('should work without config file', async () => { - const config = await createRunConfiguration( - { - version: '20.0.0', - threads: 4, - }, - createMockGenerators() - ); + const config = await createRunConfiguration({ + version: '20.0.0', + threads: 4, + }); assert.ok(config); assert.strictEqual(config.threads, 4); @@ -217,12 +200,9 @@ describe('config.mjs', () => { }) ); - const config = await createRunConfiguration( - { - configFile: 'config.mjs', - }, - createMockGenerators() - ); + const config = await createRunConfiguration({ + configFile: 'config.mjs', + }); assert.ok(config.json); assert.ok(config.html); @@ -232,10 +212,7 @@ describe('config.mjs', () => { describe('setConfig and getConfig', () => { it('should persist config across calls', async () => { - const config = await setConfig( - { version: '20.0.0', threads: 2 }, - createMockGenerators() - ); + const config = await setConfig({ version: '20.0.0', threads: 2 }); const retrieved = getConfig(); assert.strictEqual(config, retrieved); @@ -266,10 +243,7 @@ describe('config.mjs', () => { ); resetAllMocks(); - await createRunConfiguration( - { configFile: 'config.mjs' }, - createMockGenerators() - ); + await createRunConfiguration({ configFile: 'config.mjs' }); assert.ok(countCallsMatching(mockFn, ([arg]) => arg === value) >= 1); }); diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs index 42990300..acedda6c 100644 --- a/packages/core/src/utils/configuration/index.mjs +++ b/packages/core/src/utils/configuration/index.mjs @@ -4,37 +4,45 @@ import { isMainThread } from 'node:worker_threads'; import { coerce } from 'semver'; import { CHANGELOG_URL, populate } from './templates.mjs'; +import { allGenerators } from '../../generators/index.mjs'; import logger from '../../logger/index.mjs'; import { parseChangelog, parseIndex } from '../../parsers/markdown.mjs'; import { enforceArray } from '../array.mjs'; import { leftHandAssign } from '../generators.mjs'; -import { deepMerge } from '../misc.mjs'; +import { deepMerge, lazy } from '../misc.mjs'; import { importFromURL } from '../url.mjs'; /** - * Get's the default configuration (global/structural defaults only). - * Per-generator defaults are merged separately via mergeGeneratorDefaults. + * Get's the default configuration */ -export const getDefaultConfig = () => ({ - global: { - version: process.version, - minify: true, - repository: 'nodejs/node', - ref: 'HEAD', - baseURL: 'https://nodejs.org/docs', - changelog: populate(CHANGELOG_URL, { - repository: 'nodejs/node', - ref: 'HEAD', - }), - }, - - // The number of wasm memory instances is severely limited on - // riscv64 with sv39. Running multiple generators that use wasm in - // parallel could cause failures to allocate new wasm instance. - // See also https://github.com/nodejs/node/pull/60591 - threads: process.arch === 'riscv64' ? 1 : cpus().length, - chunkSize: 10, -}); +export const getDefaultConfig = lazy(() => + Object.keys(allGenerators).reduce( + (acc, k) => { + acc[k] = allGenerators[k].defaultConfiguration ?? {}; + return acc; + }, + /** @type {import('./types').Configuration} */ ({ + global: { + version: process.version, + minify: true, + repository: 'nodejs/node', + ref: 'HEAD', + baseURL: 'https://nodejs.org/docs', + changelog: populate(CHANGELOG_URL, { + repository: 'nodejs/node', + ref: 'HEAD', + }), + }, + + // The number of wasm memory instances is severely limited on + // riscv64 with sv39. Running multiple generators that use wasm in + // parallel could cause failures to allocate new wasm instance. + // See also https://github.com/nodejs/node/pull/60591 + threads: process.arch === 'riscv64' ? 1 : cpus().length, + chunkSize: 10, + }) + ) +); /** * Loads a configuration file from a URL or file path. @@ -101,10 +109,9 @@ export const createConfigFromCLIOptions = options => ({ * and constraint enforcement for threads and chunk size. * * @param {import('../../../bin/commands/generate.mjs').CLIOptions} options - User-provided configuration options - * @param {Map} loadedGenerators - Map of specifier → loaded generator * @returns {Promise} The configuration */ -export const createRunConfiguration = async (options, loadedGenerators) => { +export const createRunConfiguration = async options => { const config = await loadConfigFile(options.configFile); config.target &&= enforceArray(config.target); @@ -130,30 +137,18 @@ export const createRunConfiguration = async (options, loadedGenerators) => { // Transform global config if it wasn't already done await transformConfig(merged.global); - // Merge per-generator defaults from loaded generators and apply global config - if (loadedGenerators) { - await Promise.all( - [...loadedGenerators.values()].map(async generator => { - const { name, defaultConfiguration } = generator; - - // Initialize generator config section if it doesn't exist - if (!merged[name]) { - merged[name] = {}; - } + // Now assign to each generator config (they inherit from global) + await Promise.all( + Object.keys(allGenerators).map(async k => { + const value = merged[k]; - // Merge generator's default configuration - if (defaultConfiguration) { - leftHandAssign(merged[name], defaultConfiguration); - } + // Transform generator-specific overrides + await transformConfig(value); - // Transform generator-specific overrides - await transformConfig(merged[name]); - - // Assign from global (this populates missing values from global) - leftHandAssign(merged[name], merged.global); - }) - ); - } + // Assign from global (this populates missing values from global) + leftHandAssign(value, merged.global); + }) + ); return merged; }; @@ -164,13 +159,10 @@ let config; /** * Configuration setter * @param {import('./types').Configuration | import('../../../bin/commands/generate.mjs').CLIOptions} options - * @param {Map} [loadedGenerators] * @returns {Promise} */ -export const setConfig = async (options, loadedGenerators) => - (config = isMainThread - ? await createRunConfiguration(options, loadedGenerators) - : options); +export const setConfig = async options => + (config = isMainThread ? await createRunConfiguration(options) : options); /** * Configuration getter @@ -178,7 +170,6 @@ export const setConfig = async (options, loadedGenerators) => * @param {T} generator * @returns {T extends keyof import('./types').Configuration ? import('./types').Configuration[T] : import('./types').Configuration} */ -const getConfig = generator => - generator ? (config[generator] ?? config.global) : config; +const getConfig = generator => (generator ? config[generator] : config); export default getConfig; diff --git a/packages/core/src/utils/configuration/types.d.ts b/packages/core/src/utils/configuration/types.d.ts index 35c0f452..3536ba05 100644 --- a/packages/core/src/utils/configuration/types.d.ts +++ b/packages/core/src/utils/configuration/types.d.ts @@ -8,14 +8,17 @@ export type Configuration = { // This is considered a "sorted" list of generators, in the sense that // if the last entry of this list contains a generated value, we will return // the value of the last generator in the list, if any. - target: Array; + target: Array; // The number of threads the process is allowed to use threads: number; // Number of items to process per worker thread chunkSize: number; -} & Record>; +} & { + [K in keyof AllGenerators]: GlobalConfiguration & + AllGenerators[K]['defaultConfiguration']; +}; export type GlobalConfiguration = { // The repository diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index 837c345a..955619b7 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -2,6 +2,8 @@ import { coerce, major } from 'semver'; +import { lazy } from './misc.mjs'; + /** * Groups all the API metadata nodes by module (`api` property) so that we can process each different file * based on the module it belongs to. @@ -133,3 +135,30 @@ export const legacyToJSON = ( }, ...args ); + +/** + * Creates a generator with the provided metadata. + * @template T + * @param {T} metadata - The metadata object + * @returns {Promise} The metadata object with generator methods + */ +export const createLazyGenerator = metadata => { + const generator = lazy( + () => import(`../generators/${metadata.name}/generate.mjs`) + ); + return { + ...metadata, + /** + * Processes a chunk using the lazily-loaded generator. + * @param {...any} args - Arguments to pass to the processChunk method + * @returns {Promise} Result from the generator's processChunk method + */ + processChunk: async (...args) => (await generator()).processChunk(...args), + /** + * Generates output using the lazily-loaded generator. + * @param {...any} args - Arguments to pass to the generate method + * @returns {Promise} Result from the generator's generate method + */ + generate: async (...args) => (await generator()).generate(...args), + }; +}; diff --git a/scripts/vercel-build.sh b/scripts/vercel-build.sh index 4628191a..f8dc7e58 100755 --- a/scripts/vercel-build.sh +++ b/scripts/vercel-build.sh @@ -1,8 +1,8 @@ node packages/core/bin/cli.mjs generate \ - -t @node-core/doc-kit/generators/orama-db \ - -t @node-core/doc-kit/generators/legacy-json \ - -t @node-core/doc-kit/generators/llms-txt \ - -t @node-core/doc-kit/generators/web \ + -t orama-db \ + -t legacy-json \ + -t llms-txt \ + -t web \ -i "./node/doc/api/*.md" \ -o "./out" \ -c "./node/CHANGELOG.md" \