Desktop, Cli: Migrate from node-fetch to Undici (#16566)

packages/lib/BaseApplication.ts

@@ -29,11 +29,10 @@ import { setDateFormat, setTimeFormat, setTimeLocale } from '@joplin/utils/time'
 import { reg } from './registry';
 import time from './time';
 import BaseSyncTarget from './BaseSyncTarget';
-import reduxSharedMiddleware from './components/shared/reduxSharedMiddleware';
 import dns = require('dns');
+import reduxSharedMiddleware from './components/shared/reduxSharedMiddleware';
 import fs = require('fs-extra');
 import { EventEmitter } from 'events';
-const syswidecas = require('./vendor/syswide-cas');
 import SyncTargetRegistry from './SyncTargetRegistry';
 import SyncTargetFilesystem from './SyncTargetFilesystem';
 import SyncTargetNextcloud from './SyncTargetNextcloud';
@@ -76,6 +75,7 @@ import NoteLockKey from './services/noteLock/NoteLockKey';
 import isNoteLockEnabled from './services/noteLock/isNoteLockEnabled';
 import NoteLockSession from './services/noteLock/NoteLockSession';
 import NoteLockService from './services/noteLock/NoteLockService';
+import setExtraRootCertificates from './utils/tls/setExtraRootCertificates';
 import { BuiltInMetadataKeys } from './models/settings/builtInMetadata';
 
 const appLogger: LoggerWrapper = Logger.create('App');
@@ -388,11 +388,13 @@ export default class BaseApplication {
 				process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = Setting.value('net.ignoreTlsErrors') ? '0' : '1';
 			},
 			'net.customCertificates': async () => {
-				const caPaths = Setting.value('net.customCertificates').split(',');
-				for (let i = 0; i < caPaths.length; i++) {
-					const f = caPaths[i].trim();
-					if (!f) continue;
-					syswidecas.addCAs(f);
+				const caPaths = Setting.value('net.customCertificates')
+					.split(',')
+					.filter(path => !!path.trim());
+				try {
+					await setExtraRootCertificates(caPaths.map(path => ({ path })));
+				} catch (error) {
+					this.logger().error('Failed to add extra CA certificates:', error);
 				}
 			},
 			'net.proxyEnabled': async () => {

packages/lib/package.json

@@ -28,7 +28,7 @@
     "@types/jsdom": "21.1.7",
     "@types/markdown-it": "13.0.9",
     "@types/mustache": "4.2.6",
-    "@types/node": "18.19.130",
+    "@types/node": "22.19.0",
     "@types/node-rsa": "1.1.4",
     "@types/react": "19.1.10",
     "@types/react-dom": "19.1.7",
@@ -73,9 +73,7 @@
     "fast-xml-parser": "3.21.1",
     "file-type": "16.5.4",
     "follow-redirects": "1.16.0",
-    "form-data": "4.0.6",
     "fs-extra": "11.3.5",
-    "hpagent": "1.2.0",
     "html-entities": "1.4.0",
     "html-minifier": "4.0.0",
     "image-data-uri": "2.0.1",
@@ -89,7 +87,6 @@
     "mustache": "4.2.0",
     "nanoid": "5.1.15",
     "node-diff3": "3.2.1",
-    "node-fetch": "2.6.7",
     "node-notifier": "10.0.1",
     "node-persist": "3.1.3",
     "node-rsa": "1.1.1",
@@ -107,6 +104,7 @@
     "tar": "7.5.16",
     "tcp-port-used": "1.0.3",
     "uglifycss": "0.0.29",
+    "undici": "8.10.0",
     "url-parse": "1.5.10",
     "uuid": "11.1.1",
     "word-wrap": "1.2.5",

packages/lib/shim-init-node.test.ts

@@ -1,8 +1,9 @@
 import { shimInit } from './shim-init-node';
 import shim from './shim';
 import { createTempDir, setupDatabaseAndSynchronizer, supportDir, withExtraRootCa } from './testing/test-utils';
-import { copyFile } from 'fs-extra';
+import { copyFile, readFile, remove } from 'fs-extra';
 import createLocalhostServer from './testing/createLocalhostServer';
+import { join } from 'path';
 
 describe('shim-init-node', () => {
 
@@ -51,12 +52,43 @@ describe('shim-init-node', () => {
 		});
 	});
 
-	test('should expose an agent per protocol', async () => {
-		const agents = shim.httpAgents();
-		expect(agents.http).toBeTruthy();
-		expect(agents.https).toBeTruthy();
-		expect(agents.http).not.toBe(agents.https);
-		expect(shim.httpAgent('http://example.com')).toBe(agents.http);
-		expect(shim.httpAgent('https://example.com')).toBe(agents.https);
+	test('should handle https requests', async () => {
+		await using httpsServer = await createLocalhostServer((_req, res) => {
+			res.writeHead(200);
+			res.end('test!');
+		}, { https: true });
+
+		await withExtraRootCa(httpsServer.cert, async () => {
+			const response = await shim.fetch(`${httpsServer.baseUrl}`);
+			expect(response.status).toBe(200);
+			expect(await response.text()).toBe('test!');
+		});
+	});
+
+	test.each([
+		200,
+		// Should still download the file even for error responses.
+		// (Behavior of fetchBlob before migrating to Undici from node-fetch)
+		400,
+	])('fetchBlob should download a file to disk when the server responds with code %d', async (code) => {
+		await using httpServer = await createLocalhostServer((req, res) => {
+			if (req.url?.endsWith('/file.txt')) {
+				res.writeHead(code, { 'content-type': 'text/plain' });
+				res.end('File content');
+			} else {
+				res.writeHead(404);
+				res.end('Not found');
+			}
+		}, { https: false });
+
+		const tempDir = await createTempDir();
+		try {
+			const path = join(tempDir, 'test.txt');
+			const response = await shim.fetchBlob(`${httpServer.baseUrl}/file.txt`, { path });
+			expect(response.status).toBe(code);
+			expect(await readFile(path, 'utf-8')).toBe('File content');
+		} finally {
+			await remove(tempDir);
+		}
 	});
 });

packages/lib/shim-init-node.ts

@@ -1,4 +1,4 @@
-import shim, { CreatePdfFromImagesOptions, CreateResourceFromPathOptions, PdfInfo, PdfPageImage } from './shim';
+import shim, { CreatePdfFromImagesOptions, CreateResourceFromPathOptions, HttpAgentOptions, PdfInfo, PdfPageImage } from './shim';
 import createAccessiblePdf from './services/ocr/utils/createAccessiblePdf';
 import GeolocationNode from './geolocation-node';
 import { setLocale, defaultLocale, closestSupportedLocale } from './locale';
@@ -13,6 +13,7 @@ import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters';
 import { FetchBlobOptions } from './types';
 import { fromFile as fileTypeFromFile } from 'file-type';
 import crypto from './services/e2ee/crypto';
+import fastDeepEqual = require('fast-deep-equal');
 
 import FileApiDriverLocal from './file-api-driver-local';
 import * as mimeUtils from './mime-utils';
@@ -20,16 +21,17 @@ import BaseItem from './models/BaseItem';
 import { Size } from '@joplin/utils/types';
 import { cpus } from 'os';
 import { pathToFileURL } from 'url';
+// Use fetch from undici rather than the built-in fetch: Undici's fetch provides
+// more information when fetch fails.
+import { Agent, Request, Response, Headers, fetch, FormData, ProxyAgent, interceptors } from 'undici';
 import tls from 'tls';
 import type PdfJs from './utils/types/pdfJs';
 import { _ } from './locale';
-import http from 'http';
-import https from 'https';
-const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent');
 const toRelative = require('relative');
 import timers from 'timers';
-import zlib from 'zlib';
 import dgram from 'dgram';
+import { pipeline } from 'stream/promises';
+import { Second } from '@joplin/utils/time';
 
 interface ProxySettings {
 	maxConcurrentConnections?: number;
@@ -39,18 +41,6 @@ interface ProxySettings {
 }
 const proxySettings: ProxySettings = {};
 
-function fileExists(filePath: string) {
-	try {
-		return fs.statSync(filePath).isFile();
-	} catch (error) {
-		return false;
-	}
-}
-
-function isUrlHttps(url: string) {
-	return url.startsWith('https');
-}
-
 function resolveProxyUrl(proxyUrl: string) {
 	return (
 		proxyUrl ||
@@ -70,34 +60,6 @@ function callsites(): NodeJS.CallSite[] {
 	return stack;
 }
 
-const gunzipFile = function(source: string, destination: string) {
-	if (!fileExists(source)) {
-		throw new Error(`No such file: ${source}`);
-	}
-
-	return new Promise((resolve, reject) => {
-		// prepare streams
-		const src = fs.createReadStream(source);
-		const dest = fs.createWriteStream(destination);
-
-		// extract the archive
-		src.pipe(zlib.createGunzip()).pipe(dest);
-
-		// callback on extract completion
-		dest.on('close', () => {
-			resolve(null);
-		});
-
-		src.on('error', () => {
-			reject();
-		});
-
-		dest.on('error', () => {
-			reject();
-		});
-	});
-};
-
 function setupProxySettings(options: ProxySettings) {
 	proxySettings.maxConcurrentConnections = options.maxConcurrentConnections;
 	proxySettings.proxyTimeout = options.proxyTimeout;
@@ -156,7 +118,7 @@ function shimInit(options: ShimInitOptions = null) {
 	};
 	shim.FileApiDriverLocal = FileApiDriverLocal;
 	shim.Geolocation = GeolocationNode;
-	shim.FormData = require('form-data');
+	shim.FormData = FormData as unknown as typeof shim.FormData;
 	shim.sjclModule = require('./vendor/sjcl.js');
 	shim.crypto = crypto;
 	shim.electronBridge_ = options.electronBridge;
@@ -538,36 +500,39 @@ function shimInit(options: ShimInitOptions = null) {
 		}
 	};
 
-	const nodeFetch = require('node-fetch');
-
 	// Not used??
 	shim.readLocalFileBase64 = path => {
 		const data = fs.readFileSync(path);
 		return new Buffer(data).toString('base64');
 	};
 
+	const mapFetchError = (error: Error & { cause?: unknown }) => {
+		// When error is a TypeError, information about the error failure is in
+		// error.cause:
+		const cause = error.cause;
+		if (error instanceof TypeError && cause instanceof Error) {
+			return cause;
+		}
+		return error;
+	};
+
 	shim.fetch = async function(url, options = {}) {
 		try { // Check if the url is valid
 			new URL(url);
 		} catch (error) { // If the url is not valid, a TypeError will be thrown
 			throw new Error(`Not a valid URL: ${url}`);
 		}
-		const resolvedProxyUrl = resolveProxyUrl(proxySettings.proxyUrl);
-		if (resolvedProxyUrl && proxySettings.proxyEnabled) {
-			options.agent = shim.proxyAgent(url, resolvedProxyUrl);
-		} else {
-			// node-fetch calls this for every request, including each redirect hop, so the agent
-			// always matches the protocol actually being used.
-			const agents = shim.httpAgents();
-			options.agent = (parsedUrl: URL) => parsedUrl.protocol === 'https:' ? agents.https : agents.http;
-		}
-		return shim.fetchWithRetry(() => {
-			return nodeFetch(url, options);
+		const agent = shim.httpAgent(url);
+		return shim.fetchWithRetry(async () => {
+			try {
+				return await fetch(url, { ...options, dispatcher: agent });
+			} catch (error) {
+				throw mapFetchError(error);
+			}
 		}, options);
 	};
 
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- url is passed as a string but reassigned to a parsed UrlWithStringQuery by urlParse below
-	shim.fetchBlob = async function(url: any, options: FetchBlobOptions) {
+	shim.fetchBlob = async function(url: string, options: FetchBlobOptions) {
 		if (!options || !options.path) throw new Error('fetchBlob: target file path is missing');
 		if (!options.method) options.method = 'GET';
 		// if (!('maxRetry' in options)) options.maxRetry = 5;
@@ -577,136 +542,86 @@ function shimInit(options: ShimInitOptions = null) {
 		if (!options.maxRedirects) options.maxRedirects = 21;
 		if (!options.timeout) options.timeout = undefined;
 
-		const urlParse = require('url').parse;
-
-		url = urlParse(url.trim());
 		const method = options.method ? options.method : 'GET';
-		const http = url.protocol.toLowerCase() === 'http:' ? require('follow-redirects').http : require('follow-redirects').https;
 		const headers = options.headers ? options.headers : {};
 		const filePath = options.path;
 		const downloadController = options.downloadController;
 
-		function makeResponse(response: { statusCode: number; statusMessage: string; headers: Record<string, string | string[]> }) {
+		function makeResponse(response: Response) {
 			return {
-				ok: response.statusCode < 400,
+				ok: response.status < 400,
 				path: filePath,
 				text: () => {
-					return response.statusMessage;
+					return response.statusText;
 				},
 				json: () => {
-					return { message: `${response.statusCode}: ${response.statusMessage}` };
+					return { message: `${response.status}: ${response.statusText}` };
 				},
-				status: response.statusCode,
+				status: response.status,
 				headers: response.headers,
 			};
 		}
 
-		// eslint-disable-next-line @typescript-eslint/no-explicit-any -- requestOptions is the node http/https request options bag plus a runtime-set `agent` field
-		const requestOptions: any = {
-			protocol: url.protocol,
-			host: url.hostname,
-			port: url.port,
-			method: method,
-			path: url.pathname + (url.query ? `?${url.query}` : ''),
-			headers: headers,
-			timeout: options.timeout,
-			maxRedirects: options.maxRedirects,
-		};
-
-		const resolvedProxyUrl = resolveProxyUrl(proxySettings.proxyUrl);
-		if (resolvedProxyUrl && proxySettings.proxyEnabled) {
-			requestOptions.agent = shim.proxyAgent(url.href, resolvedProxyUrl);
-		} else {
-			// follow-redirects re-picks from this map on each hop, so a redirect that switches
-			// protocol gets the matching agent.
-			requestOptions.agents = shim.httpAgents();
-		}
-
 		const doFetchOperation = async () => {
-			return new Promise((resolve, reject) => {
-				let file: fs.WriteStream | null = null;
+			const agent: Agent = shim.httpAgent(url, options);
+			const dispatcher = agent.compose([
+				interceptors.redirect({ maxRedirections: options.maxRedirects }),
+				interceptors.decompress(),
+			]);
 
-				const cleanUpOnError = (error: Error) => {
-					// We ignore any unlink error as we only want to report on the main error
-					void fs.unlink(filePath)
-					// eslint-disable-next-line promise/prefer-await-to-then -- Old code before rule was applied
-						.catch(() => {})
-					// eslint-disable-next-line promise/prefer-await-to-then -- Old code before rule was applied
-						.then(() => {
-							if (file) {
-								file.close(() => {
-									file = null;
-									reject(error);
-								});
-							} else {
-								reject(error);
-							}
-						});
-				};
-
-				try {
-					// Note: relative paths aren't supported
-					file = fs.createWriteStream(filePath);
-
-					file.on('error', (error: Error) => {
-						cleanUpOnError(error);
-					});
-
-					const requestStart = new Date();
-					// eslint-disable-next-line @typescript-eslint/no-explicit-any -- http is dynamically required from follow-redirects (http or https variant); response is its IncomingMessage
-					const request = http.request(requestOptions, (response: any) => {
-
-						if (downloadController) {
-							response.on('data', downloadController.handleChunk(request));
-						}
-
-						response.pipe(file);
-
-						const isGzipped = response.headers['content-encoding'] === 'gzip';
-
-						file.on('finish', () => {
-							file.close(async () => {
-								if (isGzipped) {
-									const gzipFilePath = `${filePath}.gzip`;
-									await shim.fsDriver().move(filePath, gzipFilePath);
-
-									try {
-										await gunzipFile(gzipFilePath, filePath);
-										// Calling request.destroy() within the downloadController can cause problems.
-										// The response.pipe(file) will continue even after request.destroy() is called,
-										// potentially causing the same promise to resolve while the cleanUpOnError
-										// is removing the file that have been downloaded by this function.
-										if (request.destroyed) return;
-										resolve(makeResponse(response));
-									} catch (error) {
-										cleanUpOnError(error);
-									}
-
-									await shim.fsDriver().remove(gzipFilePath);
-								} else {
-									if (request.destroyed) return;
-									resolve(makeResponse(response));
-								}
-							});
-						});
-					});
-
-					request.on('timeout', () => {
-						// We choose to not destroy the request when a timeout value is not specified to keep
-						// the behavior we had before the addition of this event handler.
-						if (!requestOptions.timeout) return;
-						request.destroy(new Error(`Request timed out. Timeout value: ${requestOptions.timeout}ms. Actual connection time: ${new Date().getTime() - requestStart.getTime()}ms`));
-					});
-
-					request.on('error', (error: Error) => {
-						cleanUpOnError(error);
-					});
-
-					request.end();
-				} catch (error) {
-					cleanUpOnError(error);
-				}
+			const abortController = new AbortController();
+			const requestOptions = new Request(url, {
+				method: method,
+				headers: new Headers(headers),
+				dispatcher,
+				signal: abortController.signal,
 			});
+			try {
+				const response = await fetch(requestOptions);
+
+				const notifyController = async function*(source: AsyncIterable<Buffer>) {
+					let cancelWithError: Error|null = null;
+					const chunkHandler = downloadController.handleChunk({
+						destroy: (error) => {
+							if (!cancelWithError) {
+								cancelWithError = error ?? new Error('Cancelled');
+								abortController.abort(error);
+							}
+						},
+					});
+
+					for await (const chunk of source) {
+						chunkHandler(chunk);
+
+						if (cancelWithError) throw cancelWithError;
+
+						yield chunk;
+					}
+				};
+				if (downloadController) {
+					await pipeline(
+						response.body,
+						notifyController,
+						fs.createWriteStream(filePath),
+					);
+				} else {
+					await pipeline(
+						response.body,
+						fs.createWriteStream(filePath),
+					);
+				}
+				return makeResponse(response);
+			} catch (error) {
+				if (await fs.exists(filePath)) {
+					try {
+						await fs.unlink(filePath);
+					} catch {
+						// Ignore. Report the original error
+					}
+				}
+
+				throw mapFetchError(error);
+			}
 		};
 
 		return shim.fetchWithRetry(doFetchOperation, options);
@@ -742,47 +657,60 @@ function shimInit(options: ShimInitOptions = null) {
 		tlsEcdhCurve = 'auto';
 	}
 
-	shim.httpAgents = () => {
-		if (!shim.httpAgent_) {
-			const AgentSettings = {
-				keepAlive: true,
-				maxSockets: 1,
-				keepAliveMsecs: 5000,
+	const agentSettingsBase = (options: HttpAgentOptions|undefined) => {
+		return {
+			headersTimeout: options?.timeout,
+			bodyTimeout: options?.timeout,
+			connectTimeout: options?.timeout,
+			keepAliveTimeout: 5000,
+
+			connect: {
 				ecdhCurve: tlsEcdhCurve,
-			};
-			shim.httpAgent_ = {
-				http: new http.Agent(AgentSettings),
-				https: new https.Agent(AgentSettings),
-			};
-		}
-		return shim.httpAgent_;
+			},
+		} satisfies Agent.Options;
 	};
 
-	shim.httpAgent = url => {
-		const agents = shim.httpAgents();
-		return url.startsWith('https') ? agents.https : agents.http;
-	};
+	shim.httpAgent = (_url, options) => {
+		const resolvedProxyUrl = resolveProxyUrl(proxySettings.proxyUrl);
+		const lastSettings = shim.httpAgent_?.lastSettings;
 
-	shim.proxyAgent = (serverUrl: string, proxyUrl: string) => {
-		const proxyAgentConfig = {
-			keepAlive: true,
-			maxSockets: proxySettings.maxConcurrentConnections,
-			keepAliveMsecs: 5000,
-			proxy: proxyUrl,
-			timeout: proxySettings.proxyTimeout * 1000,
-			ecdhCurve: tlsEcdhCurve,
-		};
+		if (resolvedProxyUrl && proxySettings.proxyEnabled) {
+			const baseSettings = agentSettingsBase(options);
+			const { connect: proxyConnectSettings } = agentSettingsBase(options);
 
-		// Based on https://github.com/delvedor/hpagent#usage
-		if (!isUrlHttps(proxyUrl) && !isUrlHttps(serverUrl)) {
-			return new HttpProxyAgent(proxyAgentConfig);
-		} else if (isUrlHttps(proxyUrl) && !isUrlHttps(serverUrl)) {
-			return new HttpProxyAgent(proxyAgentConfig);
-		} else if (!isUrlHttps(proxyUrl) && isUrlHttps(serverUrl)) {
-			return new HttpsProxyAgent(proxyAgentConfig);
+			const agentSettings = {
+				...baseSettings,
+				requestTls: baseSettings.connect,
+				proxyTls: {
+					...proxyConnectSettings,
+					timeout: (proxySettings.proxyTimeout ?? 5) * Second,
+				},
+
+				connections: proxySettings.maxConcurrentConnections ?? null,
+				uri: resolvedProxyUrl,
+			} satisfies ProxyAgent.Options;
+			delete agentSettings.connect;
+
+			if (!fastDeepEqual(lastSettings, agentSettings)) {
+				shim.httpAgent_ = {
+					lastSettings: agentSettings,
+					agent: new ProxyAgent(agentSettings),
+				};
+			}
 		} else {
-			return new HttpsProxyAgent(proxyAgentConfig);
+			const agentSettings: Agent.Options = {
+				...agentSettingsBase(options),
+				connections: 1,
+			} satisfies Agent.Options;
+			if (!fastDeepEqual(lastSettings, agentSettings)) {
+				shim.httpAgent_ = {
+					lastSettings: agentSettings,
+					agent: new Agent(agentSettings),
+				};
+			}
 		}
+
+		return shim.httpAgent_.agent;
 	};
 
 	shim.openOrCreateFile = (filepath, defaultContents) => {

packages/lib/shim.ts

@@ -5,6 +5,7 @@ import type FsDriverBase from './fs-driver-base';
 import type FileApiDriverLocal from './file-api-driver-local';
 import type { Crypto } from './services/e2ee/types';
 import type { MarkupLanguage } from '@joplin/renderer';
+import { FetchBlobOptions } from './types';
 
 export interface CreateResourceFromPathOptions {
 	resizeLargeImages?: 'always' | 'never' | 'ask';
@@ -79,6 +80,17 @@ export enum MobilePlatform {
 	Web = 'web',
 }
 
+export type SetClientCertificateOptions = {
+	certPath: string;
+	keyPath: string;
+	keyPassword: string;
+	domains: string[];
+};
+
+export interface HttpAgentOptions {
+	timeout?: number;
+}
+
 let isTestingEnv_ = false;
 
 // We need to ensure that there's only one instance of React being used by all
@@ -116,8 +128,6 @@ const shim = {
 	fsDriver_: null as any,
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- http agent set per-platform (node https.Agent, RN noop); accessed structurally
 	httpAgent_: null as any,
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- proxy agent set per-platform; accessed structurally
-	proxyAgent: null as any,
 
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- See electronBridge_
 	electronBridge: (): any => {
@@ -286,6 +296,10 @@ const shim = {
 		// Code: ETIMEDOUT
 		if (error.code === 'ETIMEDOUT') return true;
 
+		// Failure to establish a connection
+		// See https://undici.nodejs.org/api/Errors#class-connecttimeouterror
+		if (error.code === 'UND_ERR_CONNECT_TIMEOUT') return true;
+
 		// ECONNREFUSED is generally temporary
 		if (error.code === 'ECONNREFUSED') return true;
 
@@ -399,7 +413,7 @@ const shim = {
 	},
 
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- options flow into FetchBlobOptions; return shape varies (node fetch Response vs RN blob)
-	fetchBlob: function(_url: string, _options: any = null): any {
+	fetchBlob: function(_url: string, _options: FetchBlobOptions = null): any {
 		throw new Error('Not implemented: fetchBlob');
 	},
 
@@ -431,18 +445,10 @@ const shim = {
 	},
 
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- See httpAgent_ above
-	httpAgent: (_url: string): any => {
+	httpAgent: (_url: string, _options?: HttpAgentOptions): any => {
 		throw new Error('Not implemented: httpAgent');
 	},
 
-	// Returns the agents for both protocols, so that callers which follow redirects can pick the
-	// right one for each hop. Binding a single agent up-front breaks when a redirect switches
-	// between http: and https: - see https://github.com/laurent22/joplin/issues/16486
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- See httpAgent_ above
-	httpAgents: (): { http: any; https: any } => {
-		throw new Error('Not implemented: httpAgents');
-	},
-
 	openOrCreateFile: (_path: string, _defaultContents: string): string => {
 		throw new Error('Not implemented: openOrCreateFile');
 	},

packages/lib/testing/createLocalhostServer.ts

@@ -26,9 +26,13 @@ interface Options {
 	https: boolean;
 }
 
+let caIndex = 0;
 const createLocalhostCerts = async () => {
+	// Using the same common name for multiple root CAs breaks certificate handling.
+	// Change the CA name to allow running multiple localhost https servers with different CAs:
+	const caCommonName = `ca${++caIndex}.localhost`;
 	const rootCa = await selfsigned.generate(
-		[{ name: 'commonName', value: 'ca.localhost' }],
+		[{ name: 'commonName', value: caCommonName }],
 		{
 			extensions: [
 				{

packages/lib/testing/test-utils.ts

@@ -72,6 +72,8 @@ import { dirname } from '@joplin/utils/path';
 import SyncTargetJoplinServerSAML from '../SyncTargetJoplinServerSAML';
 import { MarkupLanguage } from '@joplin/renderer';
 import SearchEngine from '../services/search/SearchEngine';
+import { getCACertificates } from 'node:tls';
+import setExtraRootCertificates from '../utils/tls/setExtraRootCertificates';
 
 // Each suite has its own separate data and temp directory so that multiple
 // suites can be run at the same time. suiteName is what is used to
@@ -1186,15 +1188,13 @@ export const mockFetch = (requestHandler: MockFetchRequestHandler) => {
 };
 
 export const withExtraRootCa = async <T> (caPemData: string, task: ()=> Promise<T>) => {
-	// getCACertificates requires a newer NodeJS version than the current @types/node version.
-	// Dynamically import tls to work around the missing types:
-	const tls = require('node:tls');
-	const trustedCas = tls.getCACertificates();
+	const trustedCas = getCACertificates();
 	try {
-		tls.setDefaultCACertificates([...trustedCas, caPemData]);
+		await setExtraRootCertificates([...trustedCas, caPemData].map(cert => ({ pem: cert })));
+
 		await task();
 	} finally {
-		tls.setDefaultCACertificates([...trustedCas]);
+		await setExtraRootCertificates(trustedCas.map(cert => ({ pem: cert })));
 	}
 };
 

packages/lib/types.ts

@@ -17,11 +17,17 @@ export enum ApplicationType {
 	Cli = 3,
 }
 
+export type DownloadHandler = Pick<DownloadController, 'handleChunk'>;
+
 export type FetchBlobOptions = {
 	path?: string;
 	method?: string;
 	maxRedirects?: number;
+	maxRetry?: number;
 	timeout?: number;
+	ignoreTlsErrors?: boolean;
 	headers?: Record<string, string>;
-	downloadController?: DownloadController;
+	downloadController?: DownloadHandler;
+	// Defaults to true, only respected on mobile:
+	overwrite?: boolean;
 };

packages/lib/utils/tls/setExtraRootCertificates.test.ts

@@ -0,0 +1,36 @@
+import { remove, writeFile } from 'fs-extra';
+import createLocalhostServer from '../../testing/createLocalhostServer';
+import { createTempDir, expectNotThrow, expectThrow } from '../../testing/test-utils';
+import { join } from 'path';
+import shim from '../../shim';
+import setExtraRootCertificates from './setExtraRootCertificates';
+
+describe('setExtraRootCertificates', () => {
+	it('should support loading extra root certificates from a directory', async () => {
+		await using httpsServer1 = await createLocalhostServer((_req, res) => {
+			res.writeHead(200);
+			res.end('test 1!');
+		}, { https: true });
+		await using httpsServer2 = await createLocalhostServer((_req, res) => {
+			res.writeHead(200);
+			res.end('test 2!');
+		}, { https: true });
+
+		const tempDir = await createTempDir();
+		try {
+			await expectThrow(async () => await shim.fetch(httpsServer1.baseUrl));
+			await expectThrow(async () => await shim.fetch(httpsServer2.baseUrl));
+
+			await writeFile(join(tempDir, 'cert-1.pem'), httpsServer1.cert, 'utf-8');
+			await writeFile(join(tempDir, 'cert-2.pem'), httpsServer2.cert, 'utf-8');
+
+			await setExtraRootCertificates([{ path: tempDir }]);
+
+			await expectNotThrow(async () => await shim.fetch(httpsServer2.baseUrl));
+			await expectNotThrow(async () => await shim.fetch(httpsServer1.baseUrl));
+		} finally {
+			await remove(tempDir);
+			await setExtraRootCertificates([]);
+		}
+	});
+});

packages/lib/utils/tls/setExtraRootCertificates.ts

@@ -0,0 +1,50 @@
+import { join } from 'node:path';
+import shim from '../../shim';
+import { getCACertificates, setDefaultCACertificates } from 'node:tls';
+
+let defaultCaCerts_: string[]|null = null;
+const defaultCaCerts = () => {
+	defaultCaCerts_ ??= getCACertificates();
+	return defaultCaCerts_;
+};
+
+type Cert = {
+	path: string; pem?: undefined;
+} | {
+	pem: string; path?: undefined;
+};
+
+let cacheKey = '[]';
+const setExtraRootCertificates = async (certs: Cert[]) => {
+	const newCacheKey = JSON.stringify(certs);
+	if (newCacheKey === cacheKey) return;
+
+	const cas = [...defaultCaCerts(), ...await readCertData(certs)];
+	setDefaultCACertificates(cas);
+	cacheKey = newCacheKey;
+};
+
+const readCertData = async (certs: Cert[]) => {
+	const certData: string[] = [];
+	for (const cert of certs) {
+		if (cert.pem) {
+			certData.push(cert.pem);
+		} else {
+			let filePaths;
+			if (await shim.fsDriver().isDirectory(cert.path)) {
+				filePaths = (await shim.fsDriver().readDirStats(cert.path))
+					.filter(stat => !stat.isDirectory())
+					.map(stat => join(cert.path, stat.path));
+			} else {
+				filePaths = [cert.path];
+			}
+
+			for (const filePath of filePaths) {
+				certData.push(await shim.fsDriver().readFile(filePath, 'utf-8'));
+			}
+		}
+	}
+	return certData;
+};
+
+export default setExtraRootCertificates;

packages/lib/vendor/syswide-cas.js

@@ -1,106 +0,0 @@
-// From https://github.com/capriza/syswide-cas/tree/e0a214f23a072866abf6af540a5b4b88b892a109
-// With a patch to make it work within Jest environment
-
-const fs = require("fs");
-const path = require("path");
-const tls = require("tls");
-
-const rootCAs = [];
-
-// for node 7.2 and up, trapping method must be used.
-var useTrap = false;
-const parts = process.versions.node.split(".");
-const major = parseInt(parts[0]);
-const minor = parseInt(parts[1]);
-if (major > 7 || (major == 7 && minor >= 2) || (major === 6 && minor >= 10)) {
-  useTrap = true;
-}
-
-// create an empty secure context loaded with the root CAs
-const rootSecureContext = tls.createSecureContext ? tls.createSecureContext() : require("crypto").createCredentials();
-
-function addDefaultCA(file) {
-  try {
-    var cert, match;
-    var content = fs.readFileSync(file, { encoding: "ascii" }).trim();
-    content = content.replace(/\r\n/g, "\n"); // Handles certificates that have been created in Windows
-    var regex = /-----BEGIN CERTIFICATE-----\n[\s\S]+?\n-----END CERTIFICATE-----/g;
-    var results = content.match(regex);
-    if (!results) throw new Error("Could not parse certificate");
-    results.forEach(function(match) {
-      var cert = match.trim();
-      rootCAs.push(cert);
-      // this will add the cert to the root certificate authorities list
-      // which will be used by all subsequent secure contexts with root CAs.
-      // this only works up to node 6. node 7 and up it has no affect.
-      if (!useTrap) {
-        rootSecureContext.context.addCACert(cert);
-      }
-    });
-  } catch (e) {
-    if (e.code !== "ENOENT") {
-      console.log("failed reading file " + file + ": " + e.message);
-    }
-  }
-}
-
-exports.addCAs = function(dirs) {
-  if (!dirs) {
-    return;
-  }
-
-  if (typeof dirs === "string") {
-    dirs = dirs.split(",").map(function(dir) {
-      return dir.trim();
-    });
-  }
-
-  var files, stat, file, i, j;
-  for (i = 0; i < dirs.length; ++i) {
-    try {
-      stat = fs.statSync(dirs[i]);
-      if (stat.isDirectory()) {
-        files = fs.readdirSync(dirs[i]);
-        for (j = 0; j < files.length; ++j) {
-          file = path.resolve(dirs[i], files[j]);
-          try {
-            stat = fs.statSync(file);
-            if (stat.isFile()) {
-              addDefaultCA(file);
-            }
-          } catch (e) {
-            if (e.code !== "ENOENT") {
-              console.log("failed reading " + file + ": " + e.message);
-            }
-          }
-        }
-      } else {
-        addDefaultCA(dirs[i]);
-      }
-    } catch (e) {
-      if (e.code !== "ENOENT") {
-        console.log("failed reading " + dirs[i] + ": " + e.message);
-      }
-    }
-  }
-};
-
-if (useTrap) {
-  // trap the createSecureContext method and inject custom root CAs whenever invoked
-  const origCreateSecureContext = tls.createSecureContext;
-  tls.createSecureContext = function(options) {
-	options = options || {}; // PATCH - in Jest, this is null
-    var c = origCreateSecureContext.apply(null, arguments);
-    if (!options.ca && rootCAs.length > 0) {
-      rootCAs.forEach(function(ca) {
-        // add to the created context our own root CAs
-        c.context.addCACert(ca);
-      });
-    }
-    return c;
-  };
-}
-
-const defaultCALocations = ["/etc/ssl/ca-node.pem"];
-
-exports.addCAs(defaultCALocations);