Tools: Queue concurrent sub-package publish runs instead of cancelling them

Ref #16659

.github/workflows/publish-sub-packages.yml

@@ -0,0 +1,47 @@
+name: Publish sub-packages
+
+# The tag is created and pushed by `yarn publishAll`. Publishing uses npm trusted publishing, so no
+# token is needed, but each package must have this workflow registered as a trusted publisher on
+# npmjs.com.
+on:
+  push:
+    tags:
+      - 'sub-packages-v*'
+
+permissions:
+  contents: read
+  id-token: write
+
+# Two concurrent runs could both see a version as unpublished and then fail on the second publish
+concurrency:
+  group: publish-sub-packages
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  Publish:
+    if: github.repository == 'laurent22/joplin'
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v6
+
+      # Trusted publishing requires registry-url, and npm >= 11.5.1, which is bundled with Node 24.18
+      - name: Setup build environment
+        uses: ./.github/workflows/shared/setup-build-environment
+        with:
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Install and build
+        env:
+          IS_CONTINUOUS_INTEGRATION: 1
+          BUILD_SEQUENCIAL: 1
+        run: |
+          source "${GITHUB_WORKSPACE}/.github/scripts/retry.sh"
+          retry yarn install
+
+      # Not part of the regular build, but plugin-repo-cli's entry point is dist/index.js
+      - name: Bundle plugin-repo-cli
+        run: yarn workspace @joplin/plugin-repo-cli dist
+
+      - name: Publish
+        run: node packages/tools/releaseSubPackages.js publish

.github/workflows/shared/setup-build-environment/action.yml

@@ -1,5 +1,10 @@
 name: 'Setup build environment'
 description: 'Install Joplin build dependencies'
+inputs:
+  registry-url:
+    description: 'npm registry URL, required for npm trusted publishing'
+    required: false
+    default: ''
 runs:
   using: 'composite'
   steps:
@@ -55,6 +60,7 @@ runs:
     - uses: actions/setup-node@v6
       with:
         node-version: '24.18'
+        registry-url: ${{ inputs.registry-url }}
         # Disable the cache on ARM runners. For now, we don't run "yarn install" on these
         # environments and this breaks actions/setup-node.
         # See https://github.com/laurent22/joplin/commit/47d0d3eb9e89153a609fb5441344da10904c6308#commitcomment-159577783.

.gitignore

@@ -2258,6 +2258,8 @@ packages/tools/release-ios.js
 packages/tools/release-plugin-repo-cli.js
 packages/tools/release-server.js
 packages/tools/release-transcribe.js
+packages/tools/releaseSubPackages.test.js
+packages/tools/releaseSubPackages.js
 packages/tools/saveClaConsentRecords.js
 packages/tools/setupNewRelease.js
 packages/tools/spellcheck.js

.ignore.eslint

@@ -2284,6 +2284,8 @@ packages/tools/release-ios.js
 packages/tools/release-plugin-repo-cli.js
 packages/tools/release-server.js
 packages/tools/release-transcribe.js
+packages/tools/releaseSubPackages.test.js
+packages/tools/releaseSubPackages.js
 packages/tools/saveClaConsentRecords.js
 packages/tools/setupNewRelease.js
 packages/tools/spellcheck.js

gulpfile.js

@@ -1,6 +1,7 @@
 const gulp = require('gulp');
 const execa = require('execa');
 const { stdout } = require('process');
+const { readFileSync } = require('fs');
 
 const execCommand = async (executableName, args, options = null) => {
 	options = {
@@ -32,21 +33,15 @@ const execCommand = async (executableName, args, options = null) => {
 const tasks = {
 	completePublishAll: {
 		fn: async () => {
+			await execCommand('yarn', ['install', '--mode=update-lockfile']);
 			await execCommand('git', ['add', '-A']);
 			await execCommand('git', ['commit', '-m', 'Releasing sub-packages']);
 
-			// Lerna does some unnecessary auth check that doesn't work with
-			// automation tokens, thus the --no-verify-access. Automation token
-			// is still used for access when publishing even with this flag
-			// (publishing would fail otherwise).
-			// https://github.com/lerna/lerna/issues/2788
-			await execCommand('lerna', ['publish', 'from-package', '-y', '--no-verify-access']);
-
-			await execCommand('yarn', ['install']);
-			await execCommand('git', ['add', '-A']);
-			await execCommand('git', ['commit', '-m', 'Lock file']);
-
-			await execCommand('git', ['push']);
+			// Pushing the tag triggers the publish-sub-packages workflow, which publishes the packages
+			const libVersion = JSON.parse(readFileSync('packages/lib/package.json', 'utf8')).version;
+			const tagName = `sub-packages-v${libVersion}`;
+			await execCommand('git', ['tag', tagName]);
+			await execCommand('git', ['push', '--atomic', 'origin', 'HEAD', tagName]);
 		},
 	},
 	build: {

lerna.json

@@ -1,7 +0,0 @@
-{
-  "packages": [
-    "packages/*"
-  ],
-  "version": "independent",
-  "useWorkspaces": true
-}

package.json

@@ -42,7 +42,7 @@
     "syncFuzzer": "node ./packages/tools/fuzzer/cli.js",
     "postinstall": "husky && gulp build",
     "postPreReleasesToForum": "node ./packages/tools/postPreReleasesToForum",
-    "publishAll": "git pull && yarn buildParallel && lerna version --yes --no-private --no-git-tag-version && gulp completePublishAll",
+    "publishAll": "git pull && yarn tsc && node packages/tools/releaseSubPackages.js version && gulp completePublishAll",
     "releaseAndroid": "PATH=\"/usr/local/opt/openjdk@17/bin:$PATH\" node packages/tools/release-android.js",
     "releaseAndroidClean": "node packages/tools/release-android.js",
     "releaseCli": "node packages/tools/release-cli.js",
@@ -92,7 +92,6 @@
     "globals": "17.7.0",
     "gulp": "5.0.1",
     "husky": "9.1.7",
-    "lerna": "3.22.1",
     "lint-staged": "16.4.0",
     "madge": "8.0.0",
     "npm-package-json-lint": "10.4.1",

packages/fork-htmlparser2/package.json

@@ -18,7 +18,8 @@
   ],
   "repository": {
     "type": "git",
-    "url": "git://github.com/fb55/htmlparser2.git"
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/fork-htmlparser2"
   },
   "directories": {
     "lib": "lib/"

packages/fork-sax/package.json

@@ -8,7 +8,11 @@
     "access": "public"
   },
   "license": "ISC",
-  "repository": "git://github.com/isaacs/sax-js.git",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/fork-sax"
+  },
   "files": [
     "lib/sax.js",
     "LICENSE",

packages/fork-uslug/package.json

@@ -23,7 +23,8 @@
   },
   "repository": {
     "type": "git",
-    "url": "http://github.com/jeremys/uslug.git"
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/fork-uslug"
   },
   "main": "./index",
   "exports": {

packages/htmlpack/package.json

@@ -2,6 +2,11 @@
   "name": "@joplin/htmlpack",
   "version": "3.7.3",
   "description": "Pack an HTML file and all its linked resources into a single HTML file",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/htmlpack"
+  },
   "main": "dist/index.js",
   "types": "index.ts",
   "exports": {

packages/lib/package.json

@@ -2,6 +2,11 @@
   "name": "@joplin/lib",
   "version": "3.7.3",
   "description": "Joplin Core library",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/lib"
+  },
   "author": "Laurent Cozic",
   "homepage": "",
   "license": "AGPL-3.0-or-later",

packages/onenote-converter/package.json

@@ -6,7 +6,11 @@
   "description": "Used to import a OneNote archive into Joplin",
   "version": "3.7.3",
   "license": "MPL-2.0-no-copyleft-exception",
-  "repository": "https://github.com/laurent22/joplin/tree/dev/packages/onenote-converter",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/onenote-converter"
+  },
   "main": "./renderer/pkg/renderer.js",
   "scripts": {
     "build": "node ./tools/build.js --profile=release",

packages/plugin-repo-cli/package.json

@@ -2,6 +2,11 @@
   "name": "@joplin/plugin-repo-cli",
   "version": "3.7.6",
   "description": "",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/plugin-repo-cli"
+  },
   "main": "./dist/index.js",
   "bin": "./dist/index.js",
   "publishConfig": {

packages/react-native-saf-x/package.json

@@ -36,7 +36,11 @@
     "storage-access-framework",
     "fs"
   ],
-  "repository": "https://github.com/jd1378/react-native-saf-x",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/react-native-saf-x"
+  },
   "author": "Javad Mnjd (https://github.com/jd1378)",
   "license": "AGPL-3.0-or-later",
   "homepage": "https://github.com/laurent22/joplin/tree/dev/packages/react-native-saf-x",

packages/renderer/package.json

@@ -2,7 +2,11 @@
   "name": "@joplin/renderer",
   "version": "3.7.3",
   "description": "The Joplin note renderer, used the mobile and desktop application",
-  "repository": "https://github.com/laurent22/joplin/tree/dev/packages/renderer",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/renderer"
+  },
   "main": "index.js",
   "types": "index.d.ts",
   "publishConfig": {

packages/tools/package.json

@@ -17,7 +17,8 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/laurent22/joplin.git"
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/tools"
   },
   "license": "AGPL-3.0-or-later",
   "dependencies": {

packages/tools/releaseSubPackages.test.ts

@@ -0,0 +1,41 @@
+import { bumpVersions, PackageJson, sortByDependencies } from './releaseSubPackages';
+
+const toPackages = (manifests: PackageJson[]) => manifests.map(manifest => ({ dir: '', manifest }));
+
+describe('releaseSubPackages', () => {
+
+	test('should bump all versions and update ranges between public packages', () => {
+		const result = bumpVersions([
+			{ name: 'utils', version: '3.7.3' },
+			{ name: 'fork', version: '4.1.66' },
+			{
+				name: 'lib',
+				version: '3.7.3',
+				dependencies: { 'utils': '^3.7.3', 'fork': '~4.1.60', 'external': '^3.7.3' },
+				devDependencies: { 'utils': '~3.7', 'other': '1.0.0' },
+			},
+		]);
+
+		expect(result.map(m => m.version)).toEqual(['3.7.4', '4.1.67', '3.7.4']);
+		expect(result[2].dependencies).toEqual({ 'utils': '^3.7.4', 'fork': '~4.1.67', 'external': '^3.7.3' });
+		expect(result[2].devDependencies).toEqual({ 'utils': '~3.7', 'other': '1.0.0' });
+	});
+
+	test('should sort packages so that dependencies come first', () => {
+		const result = sortByDependencies(toPackages([
+			{ name: 'tools', version: '1.0.0', dependencies: { 'lib': '^1.0.0' } },
+			{ name: 'lib', version: '1.0.0', dependencies: { 'utils': '^1.0.0' } },
+			{ name: 'utils', version: '1.0.0', devDependencies: { 'tools': '^1.0.0' } },
+		]));
+
+		expect(result.map(p => p.manifest.name)).toEqual(['utils', 'lib', 'tools']);
+	});
+
+	test('should throw on circular dependencies', () => {
+		expect(() => sortByDependencies(toPackages([
+			{ name: 'a', version: '1.0.0', dependencies: { 'b': '^1.0.0' } },
+			{ name: 'b', version: '1.0.0', dependencies: { 'a': '^1.0.0' } },
+		]))).toThrow('Circular dependency');
+	});
+
+});

packages/tools/releaseSubPackages.ts

@@ -0,0 +1,165 @@
+import { execCommand, getRootDir } from '@joplin/utils';
+import { satisfies } from 'compare-versions';
+import { copy, pathExists, readdir, readFile, remove, writeFile } from 'fs-extra';
+import { join } from 'path';
+import { chdir } from 'process';
+import yargs = require('yargs');
+import { hideBin } from 'yargs/helpers';
+
+type Dependencies = Record<string, string>;
+
+export interface PackageJson {
+	name: string;
+	version: string;
+	private?: boolean;
+	dependencies?: Dependencies;
+	devDependencies?: Dependencies;
+	peerDependencies?: Dependencies;
+	optionalDependencies?: Dependencies;
+}
+
+export interface PackageInfo {
+	dir: string;
+	manifest: PackageJson;
+}
+
+const dependencyTypes = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies'] as const;
+
+export const bumpPatch = (version: string) => {
+	const match = version.match(/^(\d+)\.(\d+)\.(\d+)$/);
+	if (!match) throw new Error(`Unsupported version format: ${version}`);
+	return `${match[1]}.${match[2]}.${Number(match[3]) + 1}`;
+};
+
+// Same behaviour as `lerna version --no-private`: every public package gets a patch bump, and ranges
+// pointing to another public package are moved to its new version, keeping the `^` or `~` prefix.
+export const bumpVersions = (manifests: PackageJson[]) => {
+	const oldVersions = new Map(manifests.map(manifest => [manifest.name, manifest.version]));
+
+	const updateRange = (name: string, range: string) => {
+		const oldVersion = oldVersions.get(name);
+		const match = range.match(/^([\^~]?)\d+\.\d+\.\d+$/);
+		if (!oldVersion || !match || !satisfies(oldVersion, range)) return range;
+		return `${match[1]}${bumpPatch(oldVersion)}`;
+	};
+
+	return manifests.map(manifest => {
+		const output: PackageJson = { ...manifest, version: bumpPatch(manifest.version) };
+		for (const type of dependencyTypes) {
+			const dependencies = manifest[type];
+			if (!dependencies) continue;
+			output[type] = Object.fromEntries(Object.entries(dependencies).map(([name, range]) => [name, updateRange(name, range)]));
+		}
+		return output;
+	});
+};
+
+// Dev dependencies are ignored because they don't affect consumers, and including them could create
+// cycles.
+export const sortByDependencies = (packages: PackageInfo[]) => {
+	const remaining = new Map(packages.map(p => [p.manifest.name, p]));
+	const output: PackageInfo[] = [];
+
+	const runtimeDependencies = ({ manifest }: PackageInfo) => {
+		return Object.keys({ ...manifest.dependencies, ...manifest.peerDependencies, ...manifest.optionalDependencies });
+	};
+
+	while (remaining.size) {
+		const ready = [...remaining.values()].filter(p => !runtimeDependencies(p).some(name => remaining.has(name)));
+		if (!ready.length) throw new Error(`Circular dependency between: ${[...remaining.keys()].join(', ')}`);
+		for (const p of ready) {
+			output.push(p);
+			remaining.delete(p.manifest.name);
+		}
+	}
+
+	return output;
+};
+
+const loadPublicPackages = async (rootDir: string) => {
+	const packagesDir = join(rootDir, 'packages');
+	const output: PackageInfo[] = [];
+	for (const name of await readdir(packagesDir)) {
+		const dir = join(packagesDir, name);
+		const path = join(dir, 'package.json');
+		if (!await pathExists(path)) continue;
+		const manifest: PackageJson = JSON.parse(await readFile(path, 'utf8'));
+		if (!manifest.private) output.push({ dir, manifest });
+	}
+	return output;
+};
+
+const isPublished = async ({ name, version }: PackageJson) => {
+	try {
+		return await execCommand(['npm', 'view', `${name}@${version}`, 'version'], { quiet: true }) !== '';
+	} catch (error) {
+		if (String(error).includes('E404')) return false;
+		throw error;
+	}
+};
+
+const hasLicenseFile = async (dir: string) => {
+	return (await readdir(dir)).some(name => /^(license|licence)(\..*)?$/i.test(name));
+};
+
+// Like lerna, the root licence is temporarily copied to packages that don't have their own.
+const publishPackage = async (dir: string, rootLicensePath: string, dryRun: boolean) => {
+	const tempLicensePath = await hasLicenseFile(dir) ? '' : join(dir, 'LICENSE');
+	if (tempLicensePath) await copy(rootLicensePath, tempLicensePath);
+	const previousDir = process.cwd();
+	try {
+		chdir(dir);
+		// Without --workspaces=false, npm applies the root .gitignore to the package, which excludes
+		// required files such as packages/lib/vendor/
+		await execCommand(['npm', 'publish', '--workspaces=false', ...(dryRun ? ['--dry-run'] : [])]);
+	} finally {
+		chdir(previousDir);
+		if (tempLicensePath) await remove(tempLicensePath);
+	}
+};
+
+const versionAll = async () => {
+	const packages = await loadPublicPackages(await getRootDir());
+	const bumped = bumpVersions(packages.map(p => p.manifest));
+	for (const [index, { dir, manifest }] of packages.entries()) {
+		await writeFile(join(dir, 'package.json'), `${JSON.stringify(bumped[index], null, 2)}\n`, 'utf8');
+		console.info(`${manifest.name}: ${manifest.version} => ${bumped[index].version}`);
+	}
+};
+
+// Packages are published one at a time, dependencies first, so that a failure leaves the registry in a
+// consistent state. Already published versions are skipped, so the command can be re-run after a failure.
+const publishAll = async (dryRun: boolean) => {
+	const rootDir = await getRootDir();
+	for (const { dir, manifest } of sortByDependencies(await loadPublicPackages(rootDir))) {
+		const id = `${manifest.name}@${manifest.version}`;
+		if (await isPublished(manifest)) {
+			console.info(`Skipping ${id}: already published`);
+			continue;
+		}
+		console.info(`Publishing ${id}...`);
+		await publishPackage(dir, join(rootDir, 'LICENSE'), dryRun);
+	}
+};
+
+if (require.main === module) {
+	void (async () => {
+		try {
+			await yargs(hideBin(process.argv))
+				.command('version', 'Bumps the patch version of all public packages', {}, () => versionAll())
+				.command('publish', 'Publishes all public packages whose version is not on the registry yet', yargs => {
+					return yargs.option('dry-run', {
+						type: 'boolean',
+						default: false,
+					});
+				}, args => publishAll(args['dry-run']))
+				.demandCommand(1)
+				.strict()
+				.parseAsync();
+		} catch (error) {
+			console.error('Fatal error');
+			console.error(error);
+			process.exit(1);
+		}
+	})();
+}

packages/turndown-plugin-gfm/package.json

@@ -30,7 +30,8 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/laurent22/joplin-turndown-plugin-gfm.git"
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/turndown-plugin-gfm"
   },
   "scripts": {
     "build-all": "npm run build-cjs && npm run build-es && npm run build-iife",

packages/turndown/package.json

@@ -34,7 +34,7 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/laurent22/joplin.git",
+    "url": "git+https://github.com/laurent22/joplin.git",
     "directory": "packages/turndown"
   },
   "scripts": {

packages/utils/package.json

@@ -2,7 +2,11 @@
   "name": "@joplin/utils",
   "version": "3.7.3",
   "description": "Utilities for Joplin",
-  "repository": "https://github.com/laurent22/joplin/tree/dev/packages/utils",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/utils"
+  },
   "exports": {
     ".": {
       "types": "./index.ts",

readme/dev/DEPLOY.md

@@ -34,7 +34,9 @@ Unlike the mobile or desktop application, the CLI app doesn't bundle its depende
 
 	yarn publishAll
 
-This is going to publish all the Joplin libraries, such as `@joplin/lib`, `@joplin/tools`, etc.
+This is going to bump the version of all the Joplin libraries, such as `@joplin/lib`, `@joplin/tools`, etc., then commit and push a `sub-packages-v*` tag. The tag triggers the `publish-sub-packages.yml` workflow, which builds and publishes the libraries to npm using [trusted publishing](https://docs.npmjs.com/trusted-publishers). Wait for that workflow to complete before releasing the CLI app.
+
+Each library must have `laurent22/joplin` and `publish-sub-packages.yml` registered as a trusted publisher on npmjs.com, with "Allow npm publish" enabled. When adding a new library, it must be published once manually before this can be configured.
 
 Then in `app-cli/package.json`, all `@joplin` dependencies and devdependencies must be set to the last major/minor version. For example: