Tools: Queue concurrent sub-package publish runs instead of cancelling them

Ref #16659

.github/workflows/publish-sub-packages.yml

@@ -0,0 +1,47 @@
+name: Publish sub-packages
+
+# The tag is created and pushed by `yarn publishAll`. Publishing uses npm trusted publishing, so no
+# token is needed, but each package must have this workflow registered as a trusted publisher on
+# npmjs.com.
+on:
+  push:
+    tags:
+      - 'sub-packages-v*'
+
+permissions:
+  contents: read
+  id-token: write
+
+# Two concurrent runs could both see a version as unpublished and then fail on the second publish
+concurrency:
+  group: publish-sub-packages
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  Publish:
+    if: github.repository == 'laurent22/joplin'
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v6
+
+      # Trusted publishing requires registry-url, and npm >= 11.5.1, which is bundled with Node 24.18
+      - name: Setup build environment
+        uses: ./.github/workflows/shared/setup-build-environment
+        with:
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Install and build
+        env:
+          IS_CONTINUOUS_INTEGRATION: 1
+          BUILD_SEQUENCIAL: 1
+        run: |
+          source "${GITHUB_WORKSPACE}/.github/scripts/retry.sh"
+          retry yarn install
+
+      # Not part of the regular build, but plugin-repo-cli's entry point is dist/index.js
+      - name: Bundle plugin-repo-cli
+        run: yarn workspace @joplin/plugin-repo-cli dist
+
+      - name: Publish
+        run: node packages/tools/releaseSubPackages.js publish

.github/workflows/shared/setup-build-environment/action.yml

@@ -1,5 +1,10 @@
 name: 'Setup build environment'
 description: 'Install Joplin build dependencies'
+inputs:
+  registry-url:
+    description: 'npm registry URL, required for npm trusted publishing'
+    required: false
+    default: ''
 runs:
   using: 'composite'
   steps:
@@ -55,6 +60,7 @@ runs:
     - uses: actions/setup-node@v6
       with:
         node-version: '24.18'
+        registry-url: ${{ inputs.registry-url }}
         # Disable the cache on ARM runners. For now, we don't run "yarn install" on these
         # environments and this breaks actions/setup-node.
         # See https://github.com/laurent22/joplin/commit/47d0d3eb9e89153a609fb5441344da10904c6308#commitcomment-159577783.

.gitignore

@@ -2258,6 +2258,8 @@ packages/tools/release-ios.js
 packages/tools/release-plugin-repo-cli.js
 packages/tools/release-server.js
 packages/tools/release-transcribe.js
+packages/tools/releaseSubPackages.test.js
+packages/tools/releaseSubPackages.js
 packages/tools/saveClaConsentRecords.js
 packages/tools/setupNewRelease.js
 packages/tools/spellcheck.js

.ignore.eslint

@@ -2284,6 +2284,8 @@ packages/tools/release-ios.js
 packages/tools/release-plugin-repo-cli.js
 packages/tools/release-server.js
 packages/tools/release-transcribe.js
+packages/tools/releaseSubPackages.test.js
+packages/tools/releaseSubPackages.js
 packages/tools/saveClaConsentRecords.js
 packages/tools/setupNewRelease.js
 packages/tools/spellcheck.js

gulpfile.js

@@ -1,6 +1,7 @@
 const gulp = require('gulp');
 const execa = require('execa');
 const { stdout } = require('process');
+const { readFileSync } = require('fs');
 
 const execCommand = async (executableName, args, options = null) => {
 	options = {
@@ -32,21 +33,15 @@ const execCommand = async (executableName, args, options = null) => {
 const tasks = {
 	completePublishAll: {
 		fn: async () => {
+			await execCommand('yarn', ['install', '--mode=update-lockfile']);
 			await execCommand('git', ['add', '-A']);
 			await execCommand('git', ['commit', '-m', 'Releasing sub-packages']);
 
-			// Lerna does some unnecessary auth check that doesn't work with
-			// automation tokens, thus the --no-verify-access. Automation token
-			// is still used for access when publishing even with this flag
-			// (publishing would fail otherwise).
-			// https://github.com/lerna/lerna/issues/2788
-			await execCommand('lerna', ['publish', 'from-package', '-y', '--no-verify-access']);
-
-			await execCommand('yarn', ['install']);
-			await execCommand('git', ['add', '-A']);
-			await execCommand('git', ['commit', '-m', 'Lock file']);
-
-			await execCommand('git', ['push']);
+			// Pushing the tag triggers the publish-sub-packages workflow, which publishes the packages
+			const libVersion = JSON.parse(readFileSync('packages/lib/package.json', 'utf8')).version;
+			const tagName = `sub-packages-v${libVersion}`;
+			await execCommand('git', ['tag', tagName]);
+			await execCommand('git', ['push', '--atomic', 'origin', 'HEAD', tagName]);
 		},
 	},
 	build: {

lerna.json

@@ -1,7 +0,0 @@
-{
-  "packages": [
-    "packages/*"
-  ],
-  "version": "independent",
-  "useWorkspaces": true
-}

package.json

@@ -42,7 +42,7 @@
     "syncFuzzer": "node ./packages/tools/fuzzer/cli.js",
     "postinstall": "husky && gulp build",
     "postPreReleasesToForum": "node ./packages/tools/postPreReleasesToForum",
-    "publishAll": "git pull && yarn buildParallel && lerna version --yes --no-private --no-git-tag-version && gulp completePublishAll",
+    "publishAll": "git pull && yarn tsc && node packages/tools/releaseSubPackages.js version && gulp completePublishAll",
     "releaseAndroid": "PATH=\"/usr/local/opt/openjdk@17/bin:$PATH\" node packages/tools/release-android.js",
     "releaseAndroidClean": "node packages/tools/release-android.js",
     "releaseCli": "node packages/tools/release-cli.js",
@@ -92,7 +92,6 @@
     "globals": "17.7.0",
     "gulp": "5.0.1",
     "husky": "9.1.7",
-    "lerna": "3.22.1",
     "lint-staged": "16.4.0",
     "madge": "8.0.0",
     "npm-package-json-lint": "10.4.1",

packages/app-desktop/gui/ChatPanel/ChatPanel.test.tsx

@@ -6,7 +6,7 @@ import Note from '@joplin/lib/models/Note';
 import CommandService from '@joplin/lib/services/CommandService';
 import { runNoteChat } from '@joplin/lib/services/ai/noteChat';
 import { ChatRole } from '@joplin/lib/services/ai/types';
-import { setupDatabaseAndSynchronizer, switchClient, withWarningSilenced } from '@joplin/lib/testing/test-utils';
+import { setupDatabaseAndSynchronizer, switchClient } from '@joplin/lib/testing/test-utils';
 import appReducer, { AiChatMessage, createAppDefaultState, createAppDefaultWindowState } from '../../app.reducer';
 import ChatPanel from './ChatPanel';
 import { WindowIdContext } from '../NewWindowOrIFrame';
@@ -186,10 +186,8 @@ describe('ChatPanel', () => {
 		jest.mocked(runNoteChat).mockRejectedValue(new Error('Network down'));
 		const view = renderPanel({ conversationId: 'chat-1', noteId: 'note-a', noteTitle: 'A', dispatch: jest.fn() });
 		fireEvent.change(view.getByRole('textbox'), { target: { value: 'Question' } });
-		await withWarningSilenced(/Chat failed/, async () => {
-			await act(async () => { fireEvent.click(view.getByRole('button', { name: 'Send' })); });
-			await waitFor(async () => expect(await ChatConversation.messages('chat-1')).toMatchObject([{ role: 'error', text: 'Network down' }]));
-		}, { requireWarning: true });
+		await act(async () => { fireEvent.click(view.getByRole('button', { name: 'Send' })); });
+		await waitFor(async () => expect(await ChatConversation.messages('chat-1')).toMatchObject([{ role: 'error', text: 'Network down' }]));
 	});
 
 	it('should assign an ID before messages are added to a new conversation', async () => {

packages/app-desktop/gui/PromptDialog.test.ts

@@ -17,8 +17,6 @@ describe('PromptDialog', () => {
 			answer: null,
 		});
 		instance.state = { visible: true, answer: 'some-answer' };
-		// The instance is never mounted, so a real setState() call would trigger a React warning
-		instance.setState = jest.fn();
 		// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Accessing private property for test setup
 		(instance as unknown as any).menuIsOpened_ = menuIsOpened;
 

packages/fork-htmlparser2/package.json

@@ -18,7 +18,8 @@
   ],
   "repository": {
     "type": "git",
-    "url": "git://github.com/fb55/htmlparser2.git"
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/fork-htmlparser2"
   },
   "directories": {
     "lib": "lib/"

packages/fork-sax/package.json

@@ -8,7 +8,11 @@
     "access": "public"
   },
   "license": "ISC",
-  "repository": "git://github.com/isaacs/sax-js.git",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/fork-sax"
+  },
   "files": [
     "lib/sax.js",
     "LICENSE",

packages/fork-uslug/package.json

@@ -23,7 +23,8 @@
   },
   "repository": {
     "type": "git",
-    "url": "http://github.com/jeremys/uslug.git"
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/fork-uslug"
   },
   "main": "./index",
   "exports": {

packages/htmlpack/package.json

@@ -2,6 +2,11 @@
   "name": "@joplin/htmlpack",
   "version": "3.7.3",
   "description": "Pack an HTML file and all its linked resources into a single HTML file",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/htmlpack"
+  },
   "main": "dist/index.js",
   "types": "index.ts",
   "exports": {

packages/lib/package.json

@@ -2,6 +2,11 @@
   "name": "@joplin/lib",
   "version": "3.7.3",
   "description": "Joplin Core library",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/lib"
+  },
   "author": "Laurent Cozic",
   "homepage": "",
   "license": "AGPL-3.0-or-later",

packages/onenote-converter/package.json

@@ -6,7 +6,11 @@
   "description": "Used to import a OneNote archive into Joplin",
   "version": "3.7.3",
   "license": "MPL-2.0-no-copyleft-exception",
-  "repository": "https://github.com/laurent22/joplin/tree/dev/packages/onenote-converter",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/onenote-converter"
+  },
   "main": "./renderer/pkg/renderer.js",
   "scripts": {
     "build": "node ./tools/build.js --profile=release",

packages/plugin-repo-cli/package.json

@@ -2,6 +2,11 @@
   "name": "@joplin/plugin-repo-cli",
   "version": "3.7.6",
   "description": "",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/plugin-repo-cli"
+  },
   "main": "./dist/index.js",
   "bin": "./dist/index.js",
   "publishConfig": {

packages/react-native-saf-x/package.json

@@ -36,7 +36,11 @@
     "storage-access-framework",
     "fs"
   ],
-  "repository": "https://github.com/jd1378/react-native-saf-x",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/react-native-saf-x"
+  },
   "author": "Javad Mnjd (https://github.com/jd1378)",
   "license": "AGPL-3.0-or-later",
   "homepage": "https://github.com/laurent22/joplin/tree/dev/packages/react-native-saf-x",

packages/renderer/package.json

@@ -2,7 +2,11 @@
   "name": "@joplin/renderer",
   "version": "3.7.3",
   "description": "The Joplin note renderer, used the mobile and desktop application",
-  "repository": "https://github.com/laurent22/joplin/tree/dev/packages/renderer",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/renderer"
+  },
   "main": "index.js",
   "types": "index.d.ts",
   "publishConfig": {

packages/server/src/models/ItemModel.test.ts

@@ -632,38 +632,6 @@ describe('ItemModel', () => {
 		expect((await models().userItem().byUserId(user1.id)).length).toBe(5);
 	});
 
-	test('should process orphaned items mixed with valid items across batches', async () => {
-		const { user: user1 } = await createUserAndSession(1);
-		const { user: user2 } = await createUserAndSession(2);
-
-		await createItemTree3(user1.id, '', '', [
-			{ id: '000000000000000000000000000000F1' },
-		]);
-
-		await createItemTree3(user2.id, '', '', [
-			{ id: '000000000000000000000000000000F2' },
-			{ id: '000000000000000000000000000000F3' },
-			{ id: '000000000000000000000000000000F4' },
-			{ id: '000000000000000000000000000000F5' },
-		]);
-
-		const f4 = await models().item().loadByJopId(user2.id, '000000000000000000000000000000F4');
-		await db()('user_items').where('item_id', '=', f4.id).delete();
-		await db()('users').where('id', '=', user1.id).delete();
-		await db()('user_items').where('user_id', '=', user1.id).delete();
-
-		await models().item().processOrphanedItems({ batchSize: 2 });
-
-		const items = await models().item().all();
-		expect(items.map(i => i.name).sort()).toEqual([
-			'000000000000000000000000000000F2.md',
-			'000000000000000000000000000000F3.md',
-			'000000000000000000000000000000F4.md',
-			'000000000000000000000000000000F5.md',
-		]);
-		expect((await models().userItem().byUserId(user2.id)).length).toBe(4);
-	});
-
 	test('should return multiple item contents', async () => {
 		const { user: user1 } = await createUserAndSession(1);
 

packages/server/src/models/ItemModel.ts

@@ -1102,10 +1102,6 @@ export default class ItemModel extends BaseModel<Item> {
 		// then look up which of those ids exist in user_items. The orphan
 		// filter happens in TS. Per-query cost is bounded by the window size,
 		// not by the table size, so no statement can blow past the timeout.
-		//
-		// The window query only selects `id` so that it is an index-only scan.
-		// IDs are random, so any other column costs one heap page read per row,
-		// and walking the whole heap evicts the buffer cache used by the app.
 		const batchSize = options.batchSize ?? 10000;
 		let batchNum = 0;
 		let totalProcessed = 0;
@@ -1118,7 +1114,7 @@ export default class ItemModel extends BaseModel<Item> {
 			const batchStartTime = Date.now();
 
 			const windowItems: Item[] = await this.db(this.tableName)
-				.select(['items.id'])
+				.select(['items.id', 'items.name', 'items.owner_id'])
 				.where('items.id', '>', lastId)
 				.orderBy('items.id')
 				.limit(batchSize);
@@ -1137,18 +1133,14 @@ export default class ItemModel extends BaseModel<Item> {
 				.select('item_id')
 				.whereIn('item_id', windowItemIds);
 			const userItemIds = new Set(existingUserItems.map(u => u.item_id));
-			const orphanedItemIds = windowItemIds.filter(id => !userItemIds.has(id));
+			const orphanedItems = windowItems.filter(i => !userItemIds.has(i.id));
 
-			if (!orphanedItemIds.length) {
+			if (!orphanedItems.length) {
 				const batchDuration = Date.now() - batchStartTime;
 				modelLogger.info(`processOrphanedItems: Batch ${batchNum} - No orphans in window of ${windowItems.length} items (${batchDuration}ms)`);
 				continue;
 			}
 
-			const orphanedItems: Item[] = await this.db(this.tableName)
-				.select(['id', 'name', 'owner_id'])
-				.whereIn('id', orphanedItemIds);
-
 			modelLogger.info(`processOrphanedItems: Batch ${batchNum} - Found ${orphanedItems.length} orphaned items`);
 			for (const o of orphanedItems) {
 				modelLogger.info(`OrphanTrace: found orphan item=${o.id} owner=${o.owner_id} name=${o.name}`);
@@ -1180,7 +1172,7 @@ export default class ItemModel extends BaseModel<Item> {
 				const restoreCount = Array.from(itemsToRestoreByUser.values()).reduce((sum, items) => sum + items.length, 0);
 				modelLogger.info(`processOrphanedItems: Batch ${batchNum} - Restoring ${restoreCount} items for ${itemsToRestoreByUser.size} users`);
 				for (const [userId, items] of itemsToRestoreByUser) {
-					await this.models().userItem().addMulti(userId, items, { ignoreAlreadyExists: true });
+					await this.models().userItem().addMulti(userId, items);
 				}
 			}
 

packages/server/src/routes/api/shares.test.ts

@@ -87,10 +87,6 @@ describe('shares', () => {
 		const result = await getApi<Share>('', `shares/${share.id}`);
 		expect(result.id).toBe(share.id);
 		expect(result.type).toBe(ShareType.PublishedFolder);
-		expect(result.owner_id).toBeUndefined();
-
-		const ownerResult = await getApi<Share>(session.id, `shares/${share.id}`);
-		expect(ownerResult.owner_id).toBe(share.owner_id);
 	});
 
 	test('should not reveal whether an email has an account', async () => {

packages/server/src/routes/api/shares.ts

@@ -132,10 +132,7 @@ router.get('api/shares/:id', async (path: SubPath, ctx: AppContext) => {
 	if (share && (share.type === ShareType.Note || share.type === ShareType.PublishedFolder)) {
 		// No authentication is necessary - anyone who knows the share ID is allowed
 		// to access the file. It is essentially public.
-		const output = await shareModel.toApiOutput(share) as Share;
-		// Published links are indexed by search engines, and the owner's ID would let anyone target their account
-		if (ctx.joplin.owner?.id !== share.owner_id) delete output.owner_id;
-		return output;
+		return shareModel.toApiOutput(share);
 	}
 
 	throw new ErrorNotFound();

packages/server/src/routes/index/users.test.ts

@@ -222,9 +222,6 @@ describe('index/users', () => {
 		// "confirm_email" is set to 0.
 		user1 = await models().user().load(user1.id);
 		expect(user1.email_confirmed).toBe(0);
-
-		// The token is still required
-		await expectHttpError(() => execRequest('', 'GET', path, null, { query: { token: 'invalid', confirm_email: '0' } }), ErrorForbidden.httpCode);
 	});
 
 	test('should allow user to verify their email', async () => {

packages/server/src/routes/index/users.ts

@@ -166,12 +166,7 @@ router.get('users/:id/confirm', async (path: SubPath, ctx: AppContext, error: Er
 		}
 	};
 
-	if (ctx.query.confirm_email !== '0') {
-		await models.user().processEmailConfirmation(userId, token, beforeChangingEmailHandler);
-	} else {
-		// Otherwise this page would reveal whether an account exists, and the email of an account without a password
-		await models.token().checkToken(userId, token, TokenPurpose.AccountConfirmation);
-	}
+	if (ctx.query.confirm_email !== '0') await models.user().processEmailConfirmation(userId, token, beforeChangingEmailHandler);
 
 	const user = await models.user().load(userId);
 	if (!user) throw new ErrorNotFound(`No such user: ${userId}`);

packages/server/src/utils/setupTaskService.ts

@@ -63,12 +63,10 @@ export default async function(env: Env, models: Models, config: Config, services
 			run: (models: Models) => models.user().handleOversizedAccounts(),
 		},
 
-		// Walks the whole items table, so it runs once a day rather than
-		// hourly. Orphans are rare and clients work around them meanwhile.
 		{
 			id: TaskId.ProcessOrphanedItems,
 			description: taskIdToLabel(TaskId.ProcessOrphanedItems),
-			schedule: '15 4 * * *',
+			schedule: '15 * * * *',
 			run: (models: Models) => models.item().processOrphanedItems(),
 		},
 

packages/tools/package.json

@@ -17,7 +17,8 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/laurent22/joplin.git"
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/tools"
   },
   "license": "AGPL-3.0-or-later",
   "dependencies": {

packages/tools/releaseSubPackages.test.ts

@@ -0,0 +1,41 @@
+import { bumpVersions, PackageJson, sortByDependencies } from './releaseSubPackages';
+
+const toPackages = (manifests: PackageJson[]) => manifests.map(manifest => ({ dir: '', manifest }));
+
+describe('releaseSubPackages', () => {
+
+	test('should bump all versions and update ranges between public packages', () => {
+		const result = bumpVersions([
+			{ name: 'utils', version: '3.7.3' },
+			{ name: 'fork', version: '4.1.66' },
+			{
+				name: 'lib',
+				version: '3.7.3',
+				dependencies: { 'utils': '^3.7.3', 'fork': '~4.1.60', 'external': '^3.7.3' },
+				devDependencies: { 'utils': '~3.7', 'other': '1.0.0' },
+			},
+		]);
+
+		expect(result.map(m => m.version)).toEqual(['3.7.4', '4.1.67', '3.7.4']);
+		expect(result[2].dependencies).toEqual({ 'utils': '^3.7.4', 'fork': '~4.1.67', 'external': '^3.7.3' });
+		expect(result[2].devDependencies).toEqual({ 'utils': '~3.7', 'other': '1.0.0' });
+	});
+
+	test('should sort packages so that dependencies come first', () => {
+		const result = sortByDependencies(toPackages([
+			{ name: 'tools', version: '1.0.0', dependencies: { 'lib': '^1.0.0' } },
+			{ name: 'lib', version: '1.0.0', dependencies: { 'utils': '^1.0.0' } },
+			{ name: 'utils', version: '1.0.0', devDependencies: { 'tools': '^1.0.0' } },
+		]));
+
+		expect(result.map(p => p.manifest.name)).toEqual(['utils', 'lib', 'tools']);
+	});
+
+	test('should throw on circular dependencies', () => {
+		expect(() => sortByDependencies(toPackages([
+			{ name: 'a', version: '1.0.0', dependencies: { 'b': '^1.0.0' } },
+			{ name: 'b', version: '1.0.0', dependencies: { 'a': '^1.0.0' } },
+		]))).toThrow('Circular dependency');
+	});
+
+});

packages/tools/releaseSubPackages.ts

@@ -0,0 +1,165 @@
+import { execCommand, getRootDir } from '@joplin/utils';
+import { satisfies } from 'compare-versions';
+import { copy, pathExists, readdir, readFile, remove, writeFile } from 'fs-extra';
+import { join } from 'path';
+import { chdir } from 'process';
+import yargs = require('yargs');
+import { hideBin } from 'yargs/helpers';
+
+type Dependencies = Record<string, string>;
+
+export interface PackageJson {
+	name: string;
+	version: string;
+	private?: boolean;
+	dependencies?: Dependencies;
+	devDependencies?: Dependencies;
+	peerDependencies?: Dependencies;
+	optionalDependencies?: Dependencies;
+}
+
+export interface PackageInfo {
+	dir: string;
+	manifest: PackageJson;
+}
+
+const dependencyTypes = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies'] as const;
+
+export const bumpPatch = (version: string) => {
+	const match = version.match(/^(\d+)\.(\d+)\.(\d+)$/);
+	if (!match) throw new Error(`Unsupported version format: ${version}`);
+	return `${match[1]}.${match[2]}.${Number(match[3]) + 1}`;
+};
+
+// Same behaviour as `lerna version --no-private`: every public package gets a patch bump, and ranges
+// pointing to another public package are moved to its new version, keeping the `^` or `~` prefix.
+export const bumpVersions = (manifests: PackageJson[]) => {
+	const oldVersions = new Map(manifests.map(manifest => [manifest.name, manifest.version]));
+
+	const updateRange = (name: string, range: string) => {
+		const oldVersion = oldVersions.get(name);
+		const match = range.match(/^([\^~]?)\d+\.\d+\.\d+$/);
+		if (!oldVersion || !match || !satisfies(oldVersion, range)) return range;
+		return `${match[1]}${bumpPatch(oldVersion)}`;
+	};
+
+	return manifests.map(manifest => {
+		const output: PackageJson = { ...manifest, version: bumpPatch(manifest.version) };
+		for (const type of dependencyTypes) {
+			const dependencies = manifest[type];
+			if (!dependencies) continue;
+			output[type] = Object.fromEntries(Object.entries(dependencies).map(([name, range]) => [name, updateRange(name, range)]));
+		}
+		return output;
+	});
+};
+
+// Dev dependencies are ignored because they don't affect consumers, and including them could create
+// cycles.
+export const sortByDependencies = (packages: PackageInfo[]) => {
+	const remaining = new Map(packages.map(p => [p.manifest.name, p]));
+	const output: PackageInfo[] = [];
+
+	const runtimeDependencies = ({ manifest }: PackageInfo) => {
+		return Object.keys({ ...manifest.dependencies, ...manifest.peerDependencies, ...manifest.optionalDependencies });
+	};
+
+	while (remaining.size) {
+		const ready = [...remaining.values()].filter(p => !runtimeDependencies(p).some(name => remaining.has(name)));
+		if (!ready.length) throw new Error(`Circular dependency between: ${[...remaining.keys()].join(', ')}`);
+		for (const p of ready) {
+			output.push(p);
+			remaining.delete(p.manifest.name);
+		}
+	}
+
+	return output;
+};
+
+const loadPublicPackages = async (rootDir: string) => {
+	const packagesDir = join(rootDir, 'packages');
+	const output: PackageInfo[] = [];
+	for (const name of await readdir(packagesDir)) {
+		const dir = join(packagesDir, name);
+		const path = join(dir, 'package.json');
+		if (!await pathExists(path)) continue;
+		const manifest: PackageJson = JSON.parse(await readFile(path, 'utf8'));
+		if (!manifest.private) output.push({ dir, manifest });
+	}
+	return output;
+};
+
+const isPublished = async ({ name, version }: PackageJson) => {
+	try {
+		return await execCommand(['npm', 'view', `${name}@${version}`, 'version'], { quiet: true }) !== '';
+	} catch (error) {
+		if (String(error).includes('E404')) return false;
+		throw error;
+	}
+};
+
+const hasLicenseFile = async (dir: string) => {
+	return (await readdir(dir)).some(name => /^(license|licence)(\..*)?$/i.test(name));
+};
+
+// Like lerna, the root licence is temporarily copied to packages that don't have their own.
+const publishPackage = async (dir: string, rootLicensePath: string, dryRun: boolean) => {
+	const tempLicensePath = await hasLicenseFile(dir) ? '' : join(dir, 'LICENSE');
+	if (tempLicensePath) await copy(rootLicensePath, tempLicensePath);
+	const previousDir = process.cwd();
+	try {
+		chdir(dir);
+		// Without --workspaces=false, npm applies the root .gitignore to the package, which excludes
+		// required files such as packages/lib/vendor/
+		await execCommand(['npm', 'publish', '--workspaces=false', ...(dryRun ? ['--dry-run'] : [])]);
+	} finally {
+		chdir(previousDir);
+		if (tempLicensePath) await remove(tempLicensePath);
+	}
+};
+
+const versionAll = async () => {
+	const packages = await loadPublicPackages(await getRootDir());
+	const bumped = bumpVersions(packages.map(p => p.manifest));
+	for (const [index, { dir, manifest }] of packages.entries()) {
+		await writeFile(join(dir, 'package.json'), `${JSON.stringify(bumped[index], null, 2)}\n`, 'utf8');
+		console.info(`${manifest.name}: ${manifest.version} => ${bumped[index].version}`);
+	}
+};
+
+// Packages are published one at a time, dependencies first, so that a failure leaves the registry in a
+// consistent state. Already published versions are skipped, so the command can be re-run after a failure.
+const publishAll = async (dryRun: boolean) => {
+	const rootDir = await getRootDir();
+	for (const { dir, manifest } of sortByDependencies(await loadPublicPackages(rootDir))) {
+		const id = `${manifest.name}@${manifest.version}`;
+		if (await isPublished(manifest)) {
+			console.info(`Skipping ${id}: already published`);
+			continue;
+		}
+		console.info(`Publishing ${id}...`);
+		await publishPackage(dir, join(rootDir, 'LICENSE'), dryRun);
+	}
+};
+
+if (require.main === module) {
+	void (async () => {
+		try {
+			await yargs(hideBin(process.argv))
+				.command('version', 'Bumps the patch version of all public packages', {}, () => versionAll())
+				.command('publish', 'Publishes all public packages whose version is not on the registry yet', yargs => {
+					return yargs.option('dry-run', {
+						type: 'boolean',
+						default: false,
+					});
+				}, args => publishAll(args['dry-run']))
+				.demandCommand(1)
+				.strict()
+				.parseAsync();
+		} catch (error) {
+			console.error('Fatal error');
+			console.error(error);
+			process.exit(1);
+		}
+	})();
+}

packages/turndown-plugin-gfm/package.json

@@ -30,7 +30,8 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/laurent22/joplin-turndown-plugin-gfm.git"
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/turndown-plugin-gfm"
   },
   "scripts": {
     "build-all": "npm run build-cjs && npm run build-es && npm run build-iife",

packages/turndown/package.json

@@ -34,7 +34,7 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/laurent22/joplin.git",
+    "url": "git+https://github.com/laurent22/joplin.git",
     "directory": "packages/turndown"
   },
   "scripts": {

packages/utils/ipc.test.ts

@@ -1,35 +1,20 @@
 import { readFile } from 'fs/promises';
 import { createTempDir } from './fs.test';
-import { IpcMessageHandler, IpcServer, newHttpError, sendMessage, startServer as startServerBase, stopServer } from './ipc';
+import { newHttpError, sendMessage, startServer, stopServer } from './ipc';
 
 const getRandomPort = () => {
-	// Stay below the OS ephemeral port ranges (Linux: 32768+, macOS/Windows: 49152+), otherwise an outgoing
-	// connection may hold the port, which `tcp-port-used` reports as free but `listen()` rejects with EADDRINUSE.
-	return Math.floor(Math.random() * 20000) + 10000;
-};
-
-let startedServers: IpcServer[] = [];
-
-const startServer = async (startPort: number, secretKeyFilePath: string, messageHandler: IpcMessageHandler) => {
-	const server = await startServerBase(startPort, secretKeyFilePath, messageHandler);
-	startedServers.push(server);
-	return server;
+	return Math.floor(Math.random() * (65535 - 20000 + 1)) + 20000;
 };
 
 describe('ipc', () => {
 
-	afterEach(async () => {
-		for (const server of startedServers) await stopServer(server);
-		startedServers = [];
-	});
-
 	it('should send and receive messages', async () => {
 		const tempDir = await createTempDir();
 		const secretFilePath = `${tempDir}/secret.txt`;
 		const serverPort1 = getRandomPort();
 		const serverPort2 = serverPort1 + 5;
 
-		await startServer(serverPort1, secretFilePath, async (request) => {
+		const server1 = await startServer(serverPort1, secretFilePath, async (request) => {
 			if (request.action === 'testing') {
 				return {
 					text: 'hello1',
@@ -39,7 +24,7 @@ describe('ipc', () => {
 			throw newHttpError(404);
 		});
 
-		await startServer(serverPort2, secretFilePath, async (request) => {
+		const server2 = await startServer(serverPort2, secretFilePath, async (request) => {
 			if (request.action === 'testing') {
 				return {
 					text: 'hello2',
@@ -98,6 +83,9 @@ describe('ipc', () => {
 				{ port: serverPort2, response: { text: 'hello2' } },
 			]);
 		}
+
+		await stopServer(server1);
+		await stopServer(server2);
 	});
 
 	it('should not process message if secret is invalid', async () => {
@@ -105,7 +93,7 @@ describe('ipc', () => {
 		const secretFilePath = `${tempDir}/secret.txt`;
 		const serverPort = getRandomPort();
 
-		await startServer(serverPort, secretFilePath, async (request) => {
+		const server = await startServer(serverPort, secretFilePath, async (request) => {
 			if (request.action === 'testing') {
 				return {
 					text: 'hello1',
@@ -136,6 +124,8 @@ describe('ipc', () => {
 
 			expect(responses.length).toBe(1);
 		}
+
+		await stopServer(server);
 	});
 
 });

packages/utils/package.json

@@ -2,7 +2,11 @@
   "name": "@joplin/utils",
   "version": "3.7.3",
   "description": "Utilities for Joplin",
-  "repository": "https://github.com/laurent22/joplin/tree/dev/packages/utils",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laurent22/joplin.git",
+    "directory": "packages/utils"
+  },
   "exports": {
     ".": {
       "types": "./index.ts",

readme/dev/DEPLOY.md

@@ -34,7 +34,9 @@ Unlike the mobile or desktop application, the CLI app doesn't bundle its depende
 
 	yarn publishAll
 
-This is going to publish all the Joplin libraries, such as `@joplin/lib`, `@joplin/tools`, etc.
+This is going to bump the version of all the Joplin libraries, such as `@joplin/lib`, `@joplin/tools`, etc., then commit and push a `sub-packages-v*` tag. The tag triggers the `publish-sub-packages.yml` workflow, which builds and publishes the libraries to npm using [trusted publishing](https://docs.npmjs.com/trusted-publishers). Wait for that workflow to complete before releasing the CLI app.
+
+Each library must have `laurent22/joplin` and `publish-sub-packages.yml` registered as a trusted publisher on npmjs.com, with "Allow npm publish" enabled. When adding a new library, it must be published once manually before this can be configured.
 
 Then in `app-cli/package.json`, all `@joplin` dependencies and devdependencies must be set to the last major/minor version. For example: