diff --git a/.gitignore b/.gitignore index c8a3ceb472..5d437a3235 100644 --- a/.gitignore +++ b/.gitignore @@ -2050,6 +2050,8 @@ packages/lib/utils/replaceUnsupportedCharacters.test.js packages/lib/utils/replaceUnsupportedCharacters.js packages/lib/utils/resolvePathWithinDir.test.js packages/lib/utils/resolvePathWithinDir.js +packages/lib/utils/tls/setExtraRootCertificates.test.js +packages/lib/utils/tls/setExtraRootCertificates.js packages/lib/utils/types/pdfJs.js packages/lib/utils/userFetcher.js packages/lib/utils/webDAVUtils.test.js diff --git a/.ignore.eslint b/.ignore.eslint index 1dd66f4ef3..03a4dd8128 100644 --- a/.ignore.eslint +++ b/.ignore.eslint @@ -2076,6 +2076,8 @@ packages/lib/utils/replaceUnsupportedCharacters.test.js packages/lib/utils/replaceUnsupportedCharacters.js packages/lib/utils/resolvePathWithinDir.test.js packages/lib/utils/resolvePathWithinDir.js +packages/lib/utils/tls/setExtraRootCertificates.test.js +packages/lib/utils/tls/setExtraRootCertificates.js packages/lib/utils/types/pdfJs.js packages/lib/utils/userFetcher.js packages/lib/utils/webDAVUtils.test.js diff --git a/jest.base-setup.js b/jest.base-setup.js index 2d622dc3c2..8b575acc05 100644 --- a/jest.base-setup.js +++ b/jest.base-setup.js @@ -33,3 +33,33 @@ if (typeof document !== 'undefined') { return range; }; } +if (typeof window !== 'undefined') { + const { Buffer, Blob, File } = require('node:buffer'); + const { TextDecoder, TextEncoder } = require('node:util'); + const { URLSearchParams, URL } = require('node:url'); + const { ReadableStream } = require('node:stream/web'); + const { MessageChannel, MessagePort } = require('node:worker_threads'); + const { setTimeout, setInterval, clearTimeout, clearInterval, setImmediate, clearImmediate } = require('node:timers'); + const { performance } = require('node:perf_hooks'); + + window.performance.markResourceTiming = performance.markResourceTiming; + + // Override some of the JSDom globals. Some libraries (e.g. Undici) require the + // original Node.js versions of these globals: + globalThis.Buffer = Buffer; + globalThis.Blob = Blob; + globalThis.File = File; + globalThis.URLSearchParams = URLSearchParams; + globalThis.URL = URL; + globalThis.TextDecoder = TextDecoder; + globalThis.TextEncoder = TextEncoder; + globalThis.ReadableStream = ReadableStream; + globalThis.MessageChannel = MessageChannel; + globalThis.MessagePort = MessagePort; + globalThis.setTimeout = setTimeout; + globalThis.setInterval = setInterval; + globalThis.clearTimeout = clearTimeout; + globalThis.clearInterval = clearInterval; + globalThis.setImmediate = setImmediate; + globalThis.clearImmediate = clearImmediate; +} diff --git a/package.json b/package.json index 7a183a24ee..7afd96e2d1 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "url": "git+https://github.com/laurent22/joplin.git" }, "engines": { - "node": ">=22.12", + "node": ">=22.19", "yarn": "4.14.1" }, "scripts": { diff --git a/packages/app-cli/app/cli-utils.ts b/packages/app-cli/app/cli-utils.ts index 4a288dc2a8..ad0abd1456 100644 --- a/packages/app-cli/app/cli-utils.ts +++ b/packages/app-cli/app/cli-utils.ts @@ -2,7 +2,7 @@ import { _ } from '@joplin/lib/locale'; import time from '@joplin/lib/time'; import Logger, { TargetType } from '@joplin/utils/Logger'; import * as readline from 'readline'; -import { Writable } from 'stream'; +import { Stream, Writable } from 'stream'; import BaseCommand from './base-command'; import yargParser = require('yargs-parser'); const stringPadding = require('string-padding'); @@ -236,7 +236,7 @@ cliUtils.prompt = function(_initialText = '', promptString = ':', options: Promp if (!this.muted) process.stdout.write(chunk, encoding); callback(); }, - }) as MutableStdout; + } as Stream.WritableOptions) as MutableStdout; const rl = readline.createInterface({ input: process.stdin, diff --git a/packages/app-cli/app/command-config.ts b/packages/app-cli/app/command-config.ts index b2f880826b..5bff507461 100644 --- a/packages/app-cli/app/command-config.ts +++ b/packages/app-cli/app/command-config.ts @@ -58,7 +58,7 @@ class Command extends BaseCommand { } }); - inputStream.on('error', (error) => { + inputStream.on('error', (error: Error) => { if (!isSettled) { isSettled = true; reject(error); diff --git a/packages/app-cli/package.json b/packages/app-cli/package.json index 059d7d05ba..0593ba4ad3 100644 --- a/packages/app-cli/package.json +++ b/packages/app-cli/package.json @@ -38,7 +38,7 @@ "version": "3.7.1", "bin": "./main.js", "engines": { - "node": ">=10.0.0" + "node": ">=22.19.0" }, "dependencies": { "@joplin/lib": "~3.7", @@ -73,7 +73,7 @@ "@joplin/tools": "~3.7", "@types/fs-extra": "11.0.4", "@types/jest": "29.5.14", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "@types/proper-lockfile": "^4.1.2", "gulp": "4.0.2", "jest": "29.7.0", diff --git a/packages/app-desktop/jest.setup.js b/packages/app-desktop/jest.setup.js index a333b094a6..c8cc418d65 100644 --- a/packages/app-desktop/jest.setup.js +++ b/packages/app-desktop/jest.setup.js @@ -1,9 +1,9 @@ /* eslint-disable jest/require-top-level-describe */ +require('../../jest.base-setup.js')(); const { shimInit } = require('@joplin/lib/shim-init-node'); const sqlite3 = require('sqlite3'); const SyncTargetNone = require('@joplin/lib/SyncTargetNone').default; -require('../../jest.base-setup.js')(); // Mock the S3 sync target -- the @aws-s3 libraries depend on an old version // of uuid that doesn't work with jest without additional configuration. diff --git a/packages/app-desktop/package.json b/packages/app-desktop/package.json index 39421efd0a..73f42fc89a 100644 --- a/packages/app-desktop/package.json +++ b/packages/app-desktop/package.json @@ -175,7 +175,7 @@ "@types/jest": "29.5.14", "@types/markdown-it": "13.0.9", "@types/mustache": "4.2.6", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "@types/react": "19.1.10", "@types/react-dom": "19.1.7", "@types/react-redux": "7.1.34", @@ -231,7 +231,8 @@ "tinymce": "6.8.5", "ts-jest": "29.4.11", "ts-node": "10.9.2", - "typescript": "5.9.3" + "typescript": "5.9.3", + "undici": "8.10.0" }, "dependencies": { "@electron/remote": "2.1.3", @@ -240,7 +241,6 @@ "@xyflow/react": "12.10.2", "fs-extra": "11.3.5", "keytar": "7.9.0", - "node-fetch": "2.6.7", "onnxruntime-node": "1.26.0", "sqlite-vec": "0.1.9", "sqlite3": "5.1.6" diff --git a/packages/app-desktop/tools/bundleJs.ts b/packages/app-desktop/tools/bundleJs.ts index b745ea2392..8ee2579ce0 100644 --- a/packages/app-desktop/tools/bundleJs.ts +++ b/packages/app-desktop/tools/bundleJs.ts @@ -1,15 +1,17 @@ import { filename, toForwardSlashes } from '@joplin/utils/path'; import * as esbuild from 'esbuild'; import { existsSync, readFileSync } from 'fs'; -import { writeFile } from 'fs/promises'; +import { readFile, writeFile } from 'fs/promises'; import { dirname, join, relative } from 'path'; const baseDir = dirname(__dirname); const baseNodeModules = join(baseDir, 'node_modules'); +const undiciPath = join(__dirname, '../build/undici.bundle.js'); + // Note: Roughly based on js-draw's use of esbuild: // https://github.com/personalizedrefrigerator/js-draw/blob/6fe6d6821402a08a8d17f15a8f48d95e5d7b084f/packages/build-tool/src/BundledFile.ts#L64 -const makeBuildContext = (entryPoint: string, renderer: boolean, addDebugStats: boolean) => { +const makeBuildContext = (entryPoint: string, renderer: boolean, addDebugStats: boolean, overrides?: Partial) => { return esbuild.context({ entryPoints: [entryPoint], outfile: `${filename(entryPoint)}.bundle.js`, @@ -23,6 +25,7 @@ const makeBuildContext = (entryPoint: string, renderer: boolean, addDebugStats: platform: 'node', target: ['node20.0'], mainFields: renderer ? ['browser', 'main'] : ['main'], + ...overrides, plugins: [ { // Configures ESBuild to require(...) certain libraries that cause issues if included directly @@ -117,11 +120,51 @@ const makeBuildContext = (entryPoint: string, renderer: boolean, addDebugStats: } }, }, + { + name: 'joplin--fix-undici', + setup: build => { + build.onResolve({ filter: /^undici$/ }, () => { + return { + path: undiciPath, + }; + }); + }, + }, + ...(overrides?.plugins ?? []), ], }); }; +const bundleUndici = async () => { + // By default, Undici is broken in Electron renderer environments and can cause renderer process + // crashes when fetching certain URLs. + // + // Undici relies on several NodeJS global objects that are replaced in the Electron renderer process. + // For example, with the Electron globals, there's no .unref() on timer handles returned by setTimeout(). + // + // As a workaround, add imports for many of the built-in objects so that Undici uses the NodeJS versions + // of the built-ins: + const undiciBuilder = await makeBuildContext( + require.resolve('undici'), true, false, { outfile: undiciPath, format: 'cjs', sourcemap: false, minify: false }, + ); + await undiciBuilder.rebuild(); + await writeFile(undiciPath, ` + const { Buffer, Blob, File } = require('node:buffer'); + const { TextDecoder, TextEncoder } = require('node:util'); + const { URLSearchParams, URL } = require('node:url'); + const { ReadableStream } = require('node:stream/web'); + const { MessageChannel, MessagePort } = require('node:worker_threads'); + const { setTimeout, setInterval, clearTimeout, clearInterval } = require('node:timers'); + const { performance } = require('node:perf_hooks'); + + ${await readFile(undiciPath, 'utf-8')} + `, 'utf-8'); + await undiciBuilder.dispose(); +}; + const bundleJs = async (writeStats: boolean) => { + await bundleUndici(); + const entryPoints = [ { fileName: 'main.ts', renderer: false }, { fileName: 'main-html.ts', renderer: true }, diff --git a/packages/app-mobile/jest.setup.js b/packages/app-mobile/jest.setup.js index 1fc204bf81..baf71b86d9 100644 --- a/packages/app-mobile/jest.setup.js +++ b/packages/app-mobile/jest.setup.js @@ -1,5 +1,6 @@ /* eslint-disable jest/require-top-level-describe */ +require('../../jest.base-setup.js')(); const { afterEachCleanUp, afterAllCleanUp } = require('@joplin/lib/testing/test-utils.js'); const shim = require('@joplin/lib/shim').default; const { shimInit } = require('@joplin/lib/shim-init-node.js'); @@ -12,7 +13,6 @@ const uuid = require('@joplin/lib/uuid').default; const Setting = require('@joplin/lib/models/Setting').default; const sqlite3 = require('sqlite3'); const React = require('react'); -require('../../jest.base-setup.js')(); import { setImmediate } from 'timers'; diff --git a/packages/app-mobile/package.json b/packages/app-mobile/package.json index 2892adf236..b35ddaf286 100644 --- a/packages/app-mobile/package.json +++ b/packages/app-mobile/package.json @@ -115,7 +115,7 @@ "@testing-library/react-native": "13.2.0", "@types/fs-extra": "11.0.4", "@types/jest": "29.5.14", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "@types/react": "19.1.10", "@types/react-redux": "7.1.34", "@types/serviceworker": "0.0.197", diff --git a/packages/app-mobile/utils/shim-init-react/index.ts b/packages/app-mobile/utils/shim-init-react/index.ts index c7c24a0491..e98a24d750 100644 --- a/packages/app-mobile/utils/shim-init-react/index.ts +++ b/packages/app-mobile/utils/shim-init-react/index.ts @@ -100,10 +100,11 @@ export default function shimInit() { delete options.overwrite; const doFetchBlob = () => { + type Method = 'POST' | 'GET' | 'DELETE' | 'PUT'; return RNFetchBlob.config({ path: localFilePath, trusty: options.ignoreTlsErrors, - }).fetch(method, url, headers); + }).fetch(method as Method, url, headers); }; try { diff --git a/packages/app-mobile/utils/shim-init-react/index.web.ts b/packages/app-mobile/utils/shim-init-react/index.web.ts index 11a1db63b6..e0b648a22d 100644 --- a/packages/app-mobile/utils/shim-init-react/index.web.ts +++ b/packages/app-mobile/utils/shim-init-react/index.web.ts @@ -91,10 +91,6 @@ const shimInit = () => { return null; }; - shim.httpAgents = () => { - return { http: null, https: null }; - }; - shim.waitForFrame = () => { return new Promise((resolve) => { requestAnimationFrame(() => { diff --git a/packages/app-mobile/utils/shim-init-react/shimInitShared.ts b/packages/app-mobile/utils/shim-init-react/shimInitShared.ts index 1d4538e2e2..58704321c3 100644 --- a/packages/app-mobile/utils/shim-init-react/shimInitShared.ts +++ b/packages/app-mobile/utils/shim-init-react/shimInitShared.ts @@ -20,7 +20,6 @@ const shimInitShared = () => { }; shim.httpAgent = () => null; - shim.httpAgents = () => ({ http: null, https: null }); shim.fetch = async function(url, options = null) { // The native fetch() throws an uncatchable error that crashes the diff --git a/packages/editor/package.json b/packages/editor/package.json index 2a3647e3fc..24d5b670ff 100644 --- a/packages/editor/package.json +++ b/packages/editor/package.json @@ -18,7 +18,7 @@ "@joplin/utils": "~3.7", "@testing-library/react-hooks": "8.0.1", "@types/jest": "29.5.14", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "@types/react": "19.1.10", "@types/react-redux": "7.1.34", "@types/styled-components": "5.1.32", diff --git a/packages/fork-htmlparser2/package.json b/packages/fork-htmlparser2/package.json index bd1af214b0..1f4eca9b37 100644 --- a/packages/fork-htmlparser2/package.json +++ b/packages/fork-htmlparser2/package.json @@ -46,7 +46,7 @@ }, "devDependencies": { "@types/jest": "29.5.14", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "@typescript-eslint/eslint-plugin": "6.21.0", "@typescript-eslint/parser": "6.21.0", "coveralls": "3.1.1", diff --git a/packages/fork-uslug/package.json b/packages/fork-uslug/package.json index 7791d471a7..1ccd1958df 100644 --- a/packages/fork-uslug/package.json +++ b/packages/fork-uslug/package.json @@ -17,7 +17,7 @@ }, "devDependencies": { "@types/jest": "29.5.14", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "jest": "29.7.0", "typescript": "5.9.3" }, diff --git a/packages/lib/BaseApplication.ts b/packages/lib/BaseApplication.ts index db7ea9ce48..1bbd09faad 100644 --- a/packages/lib/BaseApplication.ts +++ b/packages/lib/BaseApplication.ts @@ -29,11 +29,10 @@ import { setDateFormat, setTimeFormat, setTimeLocale } from '@joplin/utils/time' import { reg } from './registry'; import time from './time'; import BaseSyncTarget from './BaseSyncTarget'; -import reduxSharedMiddleware from './components/shared/reduxSharedMiddleware'; import dns = require('dns'); +import reduxSharedMiddleware from './components/shared/reduxSharedMiddleware'; import fs = require('fs-extra'); import { EventEmitter } from 'events'; -const syswidecas = require('./vendor/syswide-cas'); import SyncTargetRegistry from './SyncTargetRegistry'; import SyncTargetFilesystem from './SyncTargetFilesystem'; import SyncTargetNextcloud from './SyncTargetNextcloud'; @@ -76,6 +75,7 @@ import NoteLockKey from './services/noteLock/NoteLockKey'; import isNoteLockEnabled from './services/noteLock/isNoteLockEnabled'; import NoteLockSession from './services/noteLock/NoteLockSession'; import NoteLockService from './services/noteLock/NoteLockService'; +import setExtraRootCertificates from './utils/tls/setExtraRootCertificates'; import { BuiltInMetadataKeys } from './models/settings/builtInMetadata'; const appLogger: LoggerWrapper = Logger.create('App'); @@ -388,11 +388,13 @@ export default class BaseApplication { process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = Setting.value('net.ignoreTlsErrors') ? '0' : '1'; }, 'net.customCertificates': async () => { - const caPaths = Setting.value('net.customCertificates').split(','); - for (let i = 0; i < caPaths.length; i++) { - const f = caPaths[i].trim(); - if (!f) continue; - syswidecas.addCAs(f); + const caPaths = Setting.value('net.customCertificates') + .split(',') + .filter(path => !!path.trim()); + try { + await setExtraRootCertificates(caPaths.map(path => ({ path }))); + } catch (error) { + this.logger().error('Failed to add extra CA certificates:', error); } }, 'net.proxyEnabled': async () => { diff --git a/packages/lib/package.json b/packages/lib/package.json index eef5843bce..b12eb59d04 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -28,7 +28,7 @@ "@types/jsdom": "21.1.7", "@types/markdown-it": "13.0.9", "@types/mustache": "4.2.6", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "@types/node-rsa": "1.1.4", "@types/react": "19.1.10", "@types/react-dom": "19.1.7", @@ -73,9 +73,7 @@ "fast-xml-parser": "3.21.1", "file-type": "16.5.4", "follow-redirects": "1.16.0", - "form-data": "4.0.6", "fs-extra": "11.3.5", - "hpagent": "1.2.0", "html-entities": "1.4.0", "html-minifier": "4.0.0", "image-data-uri": "2.0.1", @@ -89,7 +87,6 @@ "mustache": "4.2.0", "nanoid": "5.1.15", "node-diff3": "3.2.1", - "node-fetch": "2.6.7", "node-notifier": "10.0.1", "node-persist": "3.1.3", "node-rsa": "1.1.1", @@ -107,6 +104,7 @@ "tar": "7.5.16", "tcp-port-used": "1.0.3", "uglifycss": "0.0.29", + "undici": "8.10.0", "url-parse": "1.5.10", "uuid": "11.1.1", "word-wrap": "1.2.5", diff --git a/packages/lib/shim-init-node.test.ts b/packages/lib/shim-init-node.test.ts index 8e0019267b..17b69efd39 100644 --- a/packages/lib/shim-init-node.test.ts +++ b/packages/lib/shim-init-node.test.ts @@ -1,8 +1,9 @@ import { shimInit } from './shim-init-node'; import shim from './shim'; import { createTempDir, setupDatabaseAndSynchronizer, supportDir, withExtraRootCa } from './testing/test-utils'; -import { copyFile } from 'fs-extra'; +import { copyFile, readFile, remove } from 'fs-extra'; import createLocalhostServer from './testing/createLocalhostServer'; +import { join } from 'path'; describe('shim-init-node', () => { @@ -51,12 +52,43 @@ describe('shim-init-node', () => { }); }); - test('should expose an agent per protocol', async () => { - const agents = shim.httpAgents(); - expect(agents.http).toBeTruthy(); - expect(agents.https).toBeTruthy(); - expect(agents.http).not.toBe(agents.https); - expect(shim.httpAgent('http://example.com')).toBe(agents.http); - expect(shim.httpAgent('https://example.com')).toBe(agents.https); + test('should handle https requests', async () => { + await using httpsServer = await createLocalhostServer((_req, res) => { + res.writeHead(200); + res.end('test!'); + }, { https: true }); + + await withExtraRootCa(httpsServer.cert, async () => { + const response = await shim.fetch(`${httpsServer.baseUrl}`); + expect(response.status).toBe(200); + expect(await response.text()).toBe('test!'); + }); + }); + + test.each([ + 200, + // Should still download the file even for error responses. + // (Behavior of fetchBlob before migrating to Undici from node-fetch) + 400, + ])('fetchBlob should download a file to disk when the server responds with code %d', async (code) => { + await using httpServer = await createLocalhostServer((req, res) => { + if (req.url?.endsWith('/file.txt')) { + res.writeHead(code, { 'content-type': 'text/plain' }); + res.end('File content'); + } else { + res.writeHead(404); + res.end('Not found'); + } + }, { https: false }); + + const tempDir = await createTempDir(); + try { + const path = join(tempDir, 'test.txt'); + const response = await shim.fetchBlob(`${httpServer.baseUrl}/file.txt`, { path }); + expect(response.status).toBe(code); + expect(await readFile(path, 'utf-8')).toBe('File content'); + } finally { + await remove(tempDir); + } }); }); diff --git a/packages/lib/shim-init-node.ts b/packages/lib/shim-init-node.ts index cbebfec814..fa97d1dba3 100644 --- a/packages/lib/shim-init-node.ts +++ b/packages/lib/shim-init-node.ts @@ -1,4 +1,4 @@ -import shim, { CreatePdfFromImagesOptions, CreateResourceFromPathOptions, PdfInfo, PdfPageImage } from './shim'; +import shim, { CreatePdfFromImagesOptions, CreateResourceFromPathOptions, HttpAgentOptions, PdfInfo, PdfPageImage } from './shim'; import createAccessiblePdf from './services/ocr/utils/createAccessiblePdf'; import GeolocationNode from './geolocation-node'; import { setLocale, defaultLocale, closestSupportedLocale } from './locale'; @@ -13,6 +13,7 @@ import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters'; import { FetchBlobOptions } from './types'; import { fromFile as fileTypeFromFile } from 'file-type'; import crypto from './services/e2ee/crypto'; +import fastDeepEqual = require('fast-deep-equal'); import FileApiDriverLocal from './file-api-driver-local'; import * as mimeUtils from './mime-utils'; @@ -20,16 +21,17 @@ import BaseItem from './models/BaseItem'; import { Size } from '@joplin/utils/types'; import { cpus } from 'os'; import { pathToFileURL } from 'url'; +// Use fetch from undici rather than the built-in fetch: Undici's fetch provides +// more information when fetch fails. +import { Agent, Request, Response, Headers, fetch, FormData, ProxyAgent, interceptors } from 'undici'; import tls from 'tls'; import type PdfJs from './utils/types/pdfJs'; import { _ } from './locale'; -import http from 'http'; -import https from 'https'; -const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent'); const toRelative = require('relative'); import timers from 'timers'; -import zlib from 'zlib'; import dgram from 'dgram'; +import { pipeline } from 'stream/promises'; +import { Second } from '@joplin/utils/time'; interface ProxySettings { maxConcurrentConnections?: number; @@ -39,18 +41,6 @@ interface ProxySettings { } const proxySettings: ProxySettings = {}; -function fileExists(filePath: string) { - try { - return fs.statSync(filePath).isFile(); - } catch (error) { - return false; - } -} - -function isUrlHttps(url: string) { - return url.startsWith('https'); -} - function resolveProxyUrl(proxyUrl: string) { return ( proxyUrl || @@ -70,34 +60,6 @@ function callsites(): NodeJS.CallSite[] { return stack; } -const gunzipFile = function(source: string, destination: string) { - if (!fileExists(source)) { - throw new Error(`No such file: ${source}`); - } - - return new Promise((resolve, reject) => { - // prepare streams - const src = fs.createReadStream(source); - const dest = fs.createWriteStream(destination); - - // extract the archive - src.pipe(zlib.createGunzip()).pipe(dest); - - // callback on extract completion - dest.on('close', () => { - resolve(null); - }); - - src.on('error', () => { - reject(); - }); - - dest.on('error', () => { - reject(); - }); - }); -}; - function setupProxySettings(options: ProxySettings) { proxySettings.maxConcurrentConnections = options.maxConcurrentConnections; proxySettings.proxyTimeout = options.proxyTimeout; @@ -156,7 +118,7 @@ function shimInit(options: ShimInitOptions = null) { }; shim.FileApiDriverLocal = FileApiDriverLocal; shim.Geolocation = GeolocationNode; - shim.FormData = require('form-data'); + shim.FormData = FormData as unknown as typeof shim.FormData; shim.sjclModule = require('./vendor/sjcl.js'); shim.crypto = crypto; shim.electronBridge_ = options.electronBridge; @@ -538,36 +500,39 @@ function shimInit(options: ShimInitOptions = null) { } }; - const nodeFetch = require('node-fetch'); - // Not used?? shim.readLocalFileBase64 = path => { const data = fs.readFileSync(path); return new Buffer(data).toString('base64'); }; + const mapFetchError = (error: Error & { cause?: unknown }) => { + // When error is a TypeError, information about the error failure is in + // error.cause: + const cause = error.cause; + if (error instanceof TypeError && cause instanceof Error) { + return cause; + } + return error; + }; + shim.fetch = async function(url, options = {}) { try { // Check if the url is valid new URL(url); } catch (error) { // If the url is not valid, a TypeError will be thrown throw new Error(`Not a valid URL: ${url}`); } - const resolvedProxyUrl = resolveProxyUrl(proxySettings.proxyUrl); - if (resolvedProxyUrl && proxySettings.proxyEnabled) { - options.agent = shim.proxyAgent(url, resolvedProxyUrl); - } else { - // node-fetch calls this for every request, including each redirect hop, so the agent - // always matches the protocol actually being used. - const agents = shim.httpAgents(); - options.agent = (parsedUrl: URL) => parsedUrl.protocol === 'https:' ? agents.https : agents.http; - } - return shim.fetchWithRetry(() => { - return nodeFetch(url, options); + const agent = shim.httpAgent(url); + return shim.fetchWithRetry(async () => { + try { + return await fetch(url, { ...options, dispatcher: agent }); + } catch (error) { + throw mapFetchError(error); + } }, options); }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- url is passed as a string but reassigned to a parsed UrlWithStringQuery by urlParse below - shim.fetchBlob = async function(url: any, options: FetchBlobOptions) { + shim.fetchBlob = async function(url: string, options: FetchBlobOptions) { if (!options || !options.path) throw new Error('fetchBlob: target file path is missing'); if (!options.method) options.method = 'GET'; // if (!('maxRetry' in options)) options.maxRetry = 5; @@ -577,136 +542,86 @@ function shimInit(options: ShimInitOptions = null) { if (!options.maxRedirects) options.maxRedirects = 21; if (!options.timeout) options.timeout = undefined; - const urlParse = require('url').parse; - - url = urlParse(url.trim()); const method = options.method ? options.method : 'GET'; - const http = url.protocol.toLowerCase() === 'http:' ? require('follow-redirects').http : require('follow-redirects').https; const headers = options.headers ? options.headers : {}; const filePath = options.path; const downloadController = options.downloadController; - function makeResponse(response: { statusCode: number; statusMessage: string; headers: Record }) { + function makeResponse(response: Response) { return { - ok: response.statusCode < 400, + ok: response.status < 400, path: filePath, text: () => { - return response.statusMessage; + return response.statusText; }, json: () => { - return { message: `${response.statusCode}: ${response.statusMessage}` }; + return { message: `${response.status}: ${response.statusText}` }; }, - status: response.statusCode, + status: response.status, headers: response.headers, }; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- requestOptions is the node http/https request options bag plus a runtime-set `agent` field - const requestOptions: any = { - protocol: url.protocol, - host: url.hostname, - port: url.port, - method: method, - path: url.pathname + (url.query ? `?${url.query}` : ''), - headers: headers, - timeout: options.timeout, - maxRedirects: options.maxRedirects, - }; - - const resolvedProxyUrl = resolveProxyUrl(proxySettings.proxyUrl); - if (resolvedProxyUrl && proxySettings.proxyEnabled) { - requestOptions.agent = shim.proxyAgent(url.href, resolvedProxyUrl); - } else { - // follow-redirects re-picks from this map on each hop, so a redirect that switches - // protocol gets the matching agent. - requestOptions.agents = shim.httpAgents(); - } - const doFetchOperation = async () => { - return new Promise((resolve, reject) => { - let file: fs.WriteStream | null = null; + const agent: Agent = shim.httpAgent(url, options); + const dispatcher = agent.compose([ + interceptors.redirect({ maxRedirections: options.maxRedirects }), + interceptors.decompress(), + ]); - const cleanUpOnError = (error: Error) => { - // We ignore any unlink error as we only want to report on the main error - void fs.unlink(filePath) - // eslint-disable-next-line promise/prefer-await-to-then -- Old code before rule was applied - .catch(() => {}) - // eslint-disable-next-line promise/prefer-await-to-then -- Old code before rule was applied - .then(() => { - if (file) { - file.close(() => { - file = null; - reject(error); - }); - } else { - reject(error); - } - }); - }; - - try { - // Note: relative paths aren't supported - file = fs.createWriteStream(filePath); - - file.on('error', (error: Error) => { - cleanUpOnError(error); - }); - - const requestStart = new Date(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- http is dynamically required from follow-redirects (http or https variant); response is its IncomingMessage - const request = http.request(requestOptions, (response: any) => { - - if (downloadController) { - response.on('data', downloadController.handleChunk(request)); - } - - response.pipe(file); - - const isGzipped = response.headers['content-encoding'] === 'gzip'; - - file.on('finish', () => { - file.close(async () => { - if (isGzipped) { - const gzipFilePath = `${filePath}.gzip`; - await shim.fsDriver().move(filePath, gzipFilePath); - - try { - await gunzipFile(gzipFilePath, filePath); - // Calling request.destroy() within the downloadController can cause problems. - // The response.pipe(file) will continue even after request.destroy() is called, - // potentially causing the same promise to resolve while the cleanUpOnError - // is removing the file that have been downloaded by this function. - if (request.destroyed) return; - resolve(makeResponse(response)); - } catch (error) { - cleanUpOnError(error); - } - - await shim.fsDriver().remove(gzipFilePath); - } else { - if (request.destroyed) return; - resolve(makeResponse(response)); - } - }); - }); - }); - - request.on('timeout', () => { - // We choose to not destroy the request when a timeout value is not specified to keep - // the behavior we had before the addition of this event handler. - if (!requestOptions.timeout) return; - request.destroy(new Error(`Request timed out. Timeout value: ${requestOptions.timeout}ms. Actual connection time: ${new Date().getTime() - requestStart.getTime()}ms`)); - }); - - request.on('error', (error: Error) => { - cleanUpOnError(error); - }); - - request.end(); - } catch (error) { - cleanUpOnError(error); - } + const abortController = new AbortController(); + const requestOptions = new Request(url, { + method: method, + headers: new Headers(headers), + dispatcher, + signal: abortController.signal, }); + try { + const response = await fetch(requestOptions); + + const notifyController = async function*(source: AsyncIterable) { + let cancelWithError: Error|null = null; + const chunkHandler = downloadController.handleChunk({ + destroy: (error) => { + if (!cancelWithError) { + cancelWithError = error ?? new Error('Cancelled'); + abortController.abort(error); + } + }, + }); + + for await (const chunk of source) { + chunkHandler(chunk); + + if (cancelWithError) throw cancelWithError; + + yield chunk; + } + }; + if (downloadController) { + await pipeline( + response.body, + notifyController, + fs.createWriteStream(filePath), + ); + } else { + await pipeline( + response.body, + fs.createWriteStream(filePath), + ); + } + return makeResponse(response); + } catch (error) { + if (await fs.exists(filePath)) { + try { + await fs.unlink(filePath); + } catch { + // Ignore. Report the original error + } + } + + throw mapFetchError(error); + } }; return shim.fetchWithRetry(doFetchOperation, options); @@ -742,47 +657,60 @@ function shimInit(options: ShimInitOptions = null) { tlsEcdhCurve = 'auto'; } - shim.httpAgents = () => { - if (!shim.httpAgent_) { - const AgentSettings = { - keepAlive: true, - maxSockets: 1, - keepAliveMsecs: 5000, + const agentSettingsBase = (options: HttpAgentOptions|undefined) => { + return { + headersTimeout: options?.timeout, + bodyTimeout: options?.timeout, + connectTimeout: options?.timeout, + keepAliveTimeout: 5000, + + connect: { ecdhCurve: tlsEcdhCurve, - }; - shim.httpAgent_ = { - http: new http.Agent(AgentSettings), - https: new https.Agent(AgentSettings), - }; - } - return shim.httpAgent_; + }, + } satisfies Agent.Options; }; - shim.httpAgent = url => { - const agents = shim.httpAgents(); - return url.startsWith('https') ? agents.https : agents.http; - }; + shim.httpAgent = (_url, options) => { + const resolvedProxyUrl = resolveProxyUrl(proxySettings.proxyUrl); + const lastSettings = shim.httpAgent_?.lastSettings; - shim.proxyAgent = (serverUrl: string, proxyUrl: string) => { - const proxyAgentConfig = { - keepAlive: true, - maxSockets: proxySettings.maxConcurrentConnections, - keepAliveMsecs: 5000, - proxy: proxyUrl, - timeout: proxySettings.proxyTimeout * 1000, - ecdhCurve: tlsEcdhCurve, - }; + if (resolvedProxyUrl && proxySettings.proxyEnabled) { + const baseSettings = agentSettingsBase(options); + const { connect: proxyConnectSettings } = agentSettingsBase(options); - // Based on https://github.com/delvedor/hpagent#usage - if (!isUrlHttps(proxyUrl) && !isUrlHttps(serverUrl)) { - return new HttpProxyAgent(proxyAgentConfig); - } else if (isUrlHttps(proxyUrl) && !isUrlHttps(serverUrl)) { - return new HttpProxyAgent(proxyAgentConfig); - } else if (!isUrlHttps(proxyUrl) && isUrlHttps(serverUrl)) { - return new HttpsProxyAgent(proxyAgentConfig); + const agentSettings = { + ...baseSettings, + requestTls: baseSettings.connect, + proxyTls: { + ...proxyConnectSettings, + timeout: (proxySettings.proxyTimeout ?? 5) * Second, + }, + + connections: proxySettings.maxConcurrentConnections ?? null, + uri: resolvedProxyUrl, + } satisfies ProxyAgent.Options; + delete agentSettings.connect; + + if (!fastDeepEqual(lastSettings, agentSettings)) { + shim.httpAgent_ = { + lastSettings: agentSettings, + agent: new ProxyAgent(agentSettings), + }; + } } else { - return new HttpsProxyAgent(proxyAgentConfig); + const agentSettings: Agent.Options = { + ...agentSettingsBase(options), + connections: 1, + } satisfies Agent.Options; + if (!fastDeepEqual(lastSettings, agentSettings)) { + shim.httpAgent_ = { + lastSettings: agentSettings, + agent: new Agent(agentSettings), + }; + } } + + return shim.httpAgent_.agent; }; shim.openOrCreateFile = (filepath, defaultContents) => { diff --git a/packages/lib/shim.ts b/packages/lib/shim.ts index 2e1bbd859e..20e9888068 100644 --- a/packages/lib/shim.ts +++ b/packages/lib/shim.ts @@ -5,6 +5,7 @@ import type FsDriverBase from './fs-driver-base'; import type FileApiDriverLocal from './file-api-driver-local'; import type { Crypto } from './services/e2ee/types'; import type { MarkupLanguage } from '@joplin/renderer'; +import { FetchBlobOptions } from './types'; export interface CreateResourceFromPathOptions { resizeLargeImages?: 'always' | 'never' | 'ask'; @@ -79,6 +80,17 @@ export enum MobilePlatform { Web = 'web', } +export type SetClientCertificateOptions = { + certPath: string; + keyPath: string; + keyPassword: string; + domains: string[]; +}; + +export interface HttpAgentOptions { + timeout?: number; +} + let isTestingEnv_ = false; // We need to ensure that there's only one instance of React being used by all @@ -116,8 +128,6 @@ const shim = { fsDriver_: null as any, // eslint-disable-next-line @typescript-eslint/no-explicit-any -- http agent set per-platform (node https.Agent, RN noop); accessed structurally httpAgent_: null as any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- proxy agent set per-platform; accessed structurally - proxyAgent: null as any, // eslint-disable-next-line @typescript-eslint/no-explicit-any -- See electronBridge_ electronBridge: (): any => { @@ -286,6 +296,10 @@ const shim = { // Code: ETIMEDOUT if (error.code === 'ETIMEDOUT') return true; + // Failure to establish a connection + // See https://undici.nodejs.org/api/Errors#class-connecttimeouterror + if (error.code === 'UND_ERR_CONNECT_TIMEOUT') return true; + // ECONNREFUSED is generally temporary if (error.code === 'ECONNREFUSED') return true; @@ -399,7 +413,7 @@ const shim = { }, // eslint-disable-next-line @typescript-eslint/no-explicit-any -- options flow into FetchBlobOptions; return shape varies (node fetch Response vs RN blob) - fetchBlob: function(_url: string, _options: any = null): any { + fetchBlob: function(_url: string, _options: FetchBlobOptions = null): any { throw new Error('Not implemented: fetchBlob'); }, @@ -431,18 +445,10 @@ const shim = { }, // eslint-disable-next-line @typescript-eslint/no-explicit-any -- See httpAgent_ above - httpAgent: (_url: string): any => { + httpAgent: (_url: string, _options?: HttpAgentOptions): any => { throw new Error('Not implemented: httpAgent'); }, - // Returns the agents for both protocols, so that callers which follow redirects can pick the - // right one for each hop. Binding a single agent up-front breaks when a redirect switches - // between http: and https: - see https://github.com/laurent22/joplin/issues/16486 - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- See httpAgent_ above - httpAgents: (): { http: any; https: any } => { - throw new Error('Not implemented: httpAgents'); - }, - openOrCreateFile: (_path: string, _defaultContents: string): string => { throw new Error('Not implemented: openOrCreateFile'); }, diff --git a/packages/lib/testing/createLocalhostServer.ts b/packages/lib/testing/createLocalhostServer.ts index 3af4c90e69..588602cc57 100644 --- a/packages/lib/testing/createLocalhostServer.ts +++ b/packages/lib/testing/createLocalhostServer.ts @@ -26,9 +26,13 @@ interface Options { https: boolean; } +let caIndex = 0; const createLocalhostCerts = async () => { + // Using the same common name for multiple root CAs breaks certificate handling. + // Change the CA name to allow running multiple localhost https servers with different CAs: + const caCommonName = `ca${++caIndex}.localhost`; const rootCa = await selfsigned.generate( - [{ name: 'commonName', value: 'ca.localhost' }], + [{ name: 'commonName', value: caCommonName }], { extensions: [ { diff --git a/packages/lib/testing/test-utils.ts b/packages/lib/testing/test-utils.ts index ef01c03f76..f9cea7e040 100644 --- a/packages/lib/testing/test-utils.ts +++ b/packages/lib/testing/test-utils.ts @@ -72,6 +72,8 @@ import { dirname } from '@joplin/utils/path'; import SyncTargetJoplinServerSAML from '../SyncTargetJoplinServerSAML'; import { MarkupLanguage } from '@joplin/renderer'; import SearchEngine from '../services/search/SearchEngine'; +import { getCACertificates } from 'node:tls'; +import setExtraRootCertificates from '../utils/tls/setExtraRootCertificates'; // Each suite has its own separate data and temp directory so that multiple // suites can be run at the same time. suiteName is what is used to @@ -1186,15 +1188,13 @@ export const mockFetch = (requestHandler: MockFetchRequestHandler) => { }; export const withExtraRootCa = async (caPemData: string, task: ()=> Promise) => { - // getCACertificates requires a newer NodeJS version than the current @types/node version. - // Dynamically import tls to work around the missing types: - const tls = require('node:tls'); - const trustedCas = tls.getCACertificates(); + const trustedCas = getCACertificates(); try { - tls.setDefaultCACertificates([...trustedCas, caPemData]); + await setExtraRootCertificates([...trustedCas, caPemData].map(cert => ({ pem: cert }))); + await task(); } finally { - tls.setDefaultCACertificates([...trustedCas]); + await setExtraRootCertificates(trustedCas.map(cert => ({ pem: cert }))); } }; diff --git a/packages/lib/types.ts b/packages/lib/types.ts index 846945f2d1..546760e5ea 100644 --- a/packages/lib/types.ts +++ b/packages/lib/types.ts @@ -17,11 +17,17 @@ export enum ApplicationType { Cli = 3, } +export type DownloadHandler = Pick; + export type FetchBlobOptions = { path?: string; method?: string; maxRedirects?: number; + maxRetry?: number; timeout?: number; + ignoreTlsErrors?: boolean; headers?: Record; - downloadController?: DownloadController; + downloadController?: DownloadHandler; + // Defaults to true, only respected on mobile: + overwrite?: boolean; }; diff --git a/packages/lib/utils/tls/setExtraRootCertificates.test.ts b/packages/lib/utils/tls/setExtraRootCertificates.test.ts new file mode 100644 index 0000000000..87d5f8b8dd --- /dev/null +++ b/packages/lib/utils/tls/setExtraRootCertificates.test.ts @@ -0,0 +1,36 @@ +import { remove, writeFile } from 'fs-extra'; +import createLocalhostServer from '../../testing/createLocalhostServer'; +import { createTempDir, expectNotThrow, expectThrow } from '../../testing/test-utils'; +import { join } from 'path'; +import shim from '../../shim'; +import setExtraRootCertificates from './setExtraRootCertificates'; + +describe('setExtraRootCertificates', () => { + it('should support loading extra root certificates from a directory', async () => { + await using httpsServer1 = await createLocalhostServer((_req, res) => { + res.writeHead(200); + res.end('test 1!'); + }, { https: true }); + await using httpsServer2 = await createLocalhostServer((_req, res) => { + res.writeHead(200); + res.end('test 2!'); + }, { https: true }); + + const tempDir = await createTempDir(); + try { + await expectThrow(async () => await shim.fetch(httpsServer1.baseUrl)); + await expectThrow(async () => await shim.fetch(httpsServer2.baseUrl)); + + await writeFile(join(tempDir, 'cert-1.pem'), httpsServer1.cert, 'utf-8'); + await writeFile(join(tempDir, 'cert-2.pem'), httpsServer2.cert, 'utf-8'); + + await setExtraRootCertificates([{ path: tempDir }]); + + await expectNotThrow(async () => await shim.fetch(httpsServer2.baseUrl)); + await expectNotThrow(async () => await shim.fetch(httpsServer1.baseUrl)); + } finally { + await remove(tempDir); + await setExtraRootCertificates([]); + } + }); +}); diff --git a/packages/lib/utils/tls/setExtraRootCertificates.ts b/packages/lib/utils/tls/setExtraRootCertificates.ts new file mode 100644 index 0000000000..4ec02a9ff0 --- /dev/null +++ b/packages/lib/utils/tls/setExtraRootCertificates.ts @@ -0,0 +1,50 @@ +import { join } from 'node:path'; +import shim from '../../shim'; +import { getCACertificates, setDefaultCACertificates } from 'node:tls'; + +let defaultCaCerts_: string[]|null = null; +const defaultCaCerts = () => { + defaultCaCerts_ ??= getCACertificates(); + return defaultCaCerts_; +}; + +type Cert = { + path: string; pem?: undefined; +} | { + pem: string; path?: undefined; +}; + +let cacheKey = '[]'; +const setExtraRootCertificates = async (certs: Cert[]) => { + const newCacheKey = JSON.stringify(certs); + if (newCacheKey === cacheKey) return; + + const cas = [...defaultCaCerts(), ...await readCertData(certs)]; + setDefaultCACertificates(cas); + cacheKey = newCacheKey; +}; + +const readCertData = async (certs: Cert[]) => { + const certData: string[] = []; + for (const cert of certs) { + if (cert.pem) { + certData.push(cert.pem); + } else { + let filePaths; + if (await shim.fsDriver().isDirectory(cert.path)) { + filePaths = (await shim.fsDriver().readDirStats(cert.path)) + .filter(stat => !stat.isDirectory()) + .map(stat => join(cert.path, stat.path)); + } else { + filePaths = [cert.path]; + } + + for (const filePath of filePaths) { + certData.push(await shim.fsDriver().readFile(filePath, 'utf-8')); + } + } + } + return certData; +}; + +export default setExtraRootCertificates; diff --git a/packages/lib/vendor/syswide-cas.js b/packages/lib/vendor/syswide-cas.js deleted file mode 100644 index 434ece7649..0000000000 --- a/packages/lib/vendor/syswide-cas.js +++ /dev/null @@ -1,106 +0,0 @@ -// From https://github.com/capriza/syswide-cas/tree/e0a214f23a072866abf6af540a5b4b88b892a109 -// With a patch to make it work within Jest environment - -const fs = require("fs"); -const path = require("path"); -const tls = require("tls"); - -const rootCAs = []; - -// for node 7.2 and up, trapping method must be used. -var useTrap = false; -const parts = process.versions.node.split("."); -const major = parseInt(parts[0]); -const minor = parseInt(parts[1]); -if (major > 7 || (major == 7 && minor >= 2) || (major === 6 && minor >= 10)) { - useTrap = true; -} - -// create an empty secure context loaded with the root CAs -const rootSecureContext = tls.createSecureContext ? tls.createSecureContext() : require("crypto").createCredentials(); - -function addDefaultCA(file) { - try { - var cert, match; - var content = fs.readFileSync(file, { encoding: "ascii" }).trim(); - content = content.replace(/\r\n/g, "\n"); // Handles certificates that have been created in Windows - var regex = /-----BEGIN CERTIFICATE-----\n[\s\S]+?\n-----END CERTIFICATE-----/g; - var results = content.match(regex); - if (!results) throw new Error("Could not parse certificate"); - results.forEach(function(match) { - var cert = match.trim(); - rootCAs.push(cert); - // this will add the cert to the root certificate authorities list - // which will be used by all subsequent secure contexts with root CAs. - // this only works up to node 6. node 7 and up it has no affect. - if (!useTrap) { - rootSecureContext.context.addCACert(cert); - } - }); - } catch (e) { - if (e.code !== "ENOENT") { - console.log("failed reading file " + file + ": " + e.message); - } - } -} - -exports.addCAs = function(dirs) { - if (!dirs) { - return; - } - - if (typeof dirs === "string") { - dirs = dirs.split(",").map(function(dir) { - return dir.trim(); - }); - } - - var files, stat, file, i, j; - for (i = 0; i < dirs.length; ++i) { - try { - stat = fs.statSync(dirs[i]); - if (stat.isDirectory()) { - files = fs.readdirSync(dirs[i]); - for (j = 0; j < files.length; ++j) { - file = path.resolve(dirs[i], files[j]); - try { - stat = fs.statSync(file); - if (stat.isFile()) { - addDefaultCA(file); - } - } catch (e) { - if (e.code !== "ENOENT") { - console.log("failed reading " + file + ": " + e.message); - } - } - } - } else { - addDefaultCA(dirs[i]); - } - } catch (e) { - if (e.code !== "ENOENT") { - console.log("failed reading " + dirs[i] + ": " + e.message); - } - } - } -}; - -if (useTrap) { - // trap the createSecureContext method and inject custom root CAs whenever invoked - const origCreateSecureContext = tls.createSecureContext; - tls.createSecureContext = function(options) { - options = options || {}; // PATCH - in Jest, this is null - var c = origCreateSecureContext.apply(null, arguments); - if (!options.ca && rootCAs.length > 0) { - rootCAs.forEach(function(ca) { - // add to the created context our own root CAs - c.context.addCACert(ca); - }); - } - return c; - }; -} - -const defaultCALocations = ["/etc/ssl/ca-node.pem"]; - -exports.addCAs(defaultCALocations); diff --git a/packages/plugin-repo-cli/package.json b/packages/plugin-repo-cli/package.json index 38d66ddcc3..c1f10bbdb1 100644 --- a/packages/plugin-repo-cli/package.json +++ b/packages/plugin-repo-cli/package.json @@ -30,7 +30,7 @@ "@joplin/utils": "^3.7.1", "@types/fs-extra": "11.0.4", "@types/jest": "29.5.14", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "jest": "29.7.0", "source-map-loader": "5.0.0", "typescript": "5.9.3", diff --git a/packages/renderer/package.json b/packages/renderer/package.json index 4c04f157a1..6402374ae6 100644 --- a/packages/renderer/package.json +++ b/packages/renderer/package.json @@ -22,7 +22,7 @@ "devDependencies": { "@types/jest": "29.5.14", "@types/markdown-it": "13.0.9", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "jest": "29.7.0", "jest-environment-jsdom": "29.7.0", "mermaid": "11.10.1", diff --git a/packages/server/package.json b/packages/server/package.json index 36fe562f3f..3f6693a9a5 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -76,7 +76,7 @@ "@types/koa": "2.15.0", "@types/markdown-it": "13.0.9", "@types/mustache": "4.2.6", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "@types/nodemailer": "6.4.24", "@types/yargs": "17.0.35", "@types/zxcvbn": "4.4.5", diff --git a/packages/tools/cspell/dictionary4.txt b/packages/tools/cspell/dictionary4.txt index 6dae254b1d..602c145fc5 100644 --- a/packages/tools/cspell/dictionary4.txt +++ b/packages/tools/cspell/dictionary4.txt @@ -228,6 +228,7 @@ dpkg onepkg ATTLIST Keycloak +undici abcjs Peacherine tablatures @@ -341,6 +342,7 @@ Socialboom AABB Koofr undecryptable +Interceptable wikilink wikilinks cssclasses diff --git a/packages/tools/package.json b/packages/tools/package.json index 32450267c9..7463bebb68 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -52,7 +52,7 @@ "@types/js-yaml": "4.0.9", "@types/markdown-it": "13.0.9", "@types/mustache": "4.2.6", - "@types/node": "18.19.130", + "@types/node": "22.19.0", "@types/node-fetch": "2.6.13", "@types/yargs": "17.0.35", "gettext-extractor": "4.0.6", diff --git a/packages/utils/ipc.ts b/packages/utils/ipc.ts index 9d936d6bda..22ca9d3157 100644 --- a/packages/utils/ipc.ts +++ b/packages/utils/ipc.ts @@ -1,5 +1,4 @@ import { createServer, IncomingMessage, ServerResponse } from 'http'; -import fetch from 'node-fetch'; import { Server } from 'http'; import Logger, { LoggerWrapper } from './Logger'; import { pathExists } from 'fs-extra'; diff --git a/packages/utils/net.ts b/packages/utils/net.ts index 5b04b9bcb4..a08d284919 100644 --- a/packages/utils/net.ts +++ b/packages/utils/net.ts @@ -1,7 +1,6 @@ /* eslint-disable import/prefer-default-export */ import { msleep } from './time'; -import fetch, { RequestInit } from 'node-fetch'; interface FetchWithRetryOptions extends RequestInit { retry?: number; diff --git a/packages/utils/package.json b/packages/utils/package.json index f09dda86c1..70890732de 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -99,7 +99,6 @@ "html-entities": "1.4.0", "markdown-it": "13.0.2", "moment": "2.30.1", - "node-fetch": "2.6.7", "sprintf-js": "1.1.3", "tcp-port-used": "1.0.3" }, @@ -107,7 +106,6 @@ "@types/fs-extra": "11.0.4", "@types/jest": "29.5.14", "@types/markdown-it": "13.0.9", - "@types/node-fetch": "2.6.13", "jest": "29.7.0", "ts-jest": "29.4.11", "typescript": "5.9.3" diff --git a/yarn.lock b/yarn.lock index aa40467094..55d62aa60f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12503,7 +12503,7 @@ __metadata: "@types/jest": "npm:29.5.14" "@types/markdown-it": "npm:13.0.9" "@types/mustache": "npm:4.2.6" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" "@types/react": "npm:19.1.10" "@types/react-dom": "npm:19.1.7" "@types/react-redux": "npm:7.1.34" @@ -12539,7 +12539,6 @@ __metadata: moment: "npm:2.30.1" mustache: "npm:4.2.0" nan: "npm:2.27.0" - node-fetch: "npm:2.6.7" node-notifier: "npm:10.0.1" node-rsa: "npm:1.1.1" onnxruntime-node: "npm:1.26.0" @@ -12567,6 +12566,7 @@ __metadata: ts-jest: "npm:29.4.11" ts-node: "npm:10.9.2" typescript: "npm:5.9.3" + undici: "npm:8.10.0" languageName: unknown linkType: soft @@ -12612,7 +12612,7 @@ __metadata: "@testing-library/react-native": "npm:13.2.0" "@types/fs-extra": "npm:11.0.4" "@types/jest": "npm:29.5.14" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" "@types/react": "npm:19.1.10" "@types/react-redux": "npm:7.1.34" "@types/serviceworker": "npm:0.0.197" @@ -12761,7 +12761,7 @@ __metadata: "@replit/codemirror-vim": "npm:6.2.1" "@testing-library/react-hooks": "npm:8.0.1" "@types/jest": "npm:29.5.14" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" "@types/react": "npm:19.1.10" "@types/react-redux": "npm:7.1.34" "@types/styled-components": "npm:5.1.32" @@ -12793,7 +12793,7 @@ __metadata: resolution: "@joplin/fork-htmlparser2@workspace:packages/fork-htmlparser2" dependencies: "@types/jest": "npm:29.5.14" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" "@typescript-eslint/eslint-plugin": "npm:6.21.0" "@typescript-eslint/parser": "npm:6.21.0" coveralls: "npm:3.1.1" @@ -12823,7 +12823,7 @@ __metadata: resolution: "@joplin/fork-uslug@workspace:packages/fork-uslug" dependencies: "@types/jest": "npm:29.5.14" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" jest: "npm:29.7.0" node-emoji: "npm:1.11.0" typescript: "npm:5.9.3" @@ -12873,7 +12873,7 @@ __metadata: "@types/jsdom": "npm:21.1.7" "@types/markdown-it": "npm:13.0.9" "@types/mustache": "npm:4.2.6" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" "@types/node-rsa": "npm:1.1.4" "@types/react": "npm:19.1.10" "@types/react-dom": "npm:19.1.7" @@ -12892,9 +12892,7 @@ __metadata: fast-xml-parser: "npm:3.21.1" file-type: "npm:16.5.4" follow-redirects: "npm:1.16.0" - form-data: "npm:4.0.6" fs-extra: "npm:11.3.5" - hpagent: "npm:1.2.0" html-entities: "npm:1.4.0" html-minifier: "npm:4.0.0" image-data-uri: "npm:2.0.1" @@ -12911,7 +12909,6 @@ __metadata: mustache: "npm:4.2.0" nanoid: "npm:5.1.15" node-diff3: "npm:3.2.1" - node-fetch: "npm:2.6.7" node-notifier: "npm:10.0.1" node-persist: "npm:3.1.3" node-rsa: "npm:1.1.1" @@ -12938,6 +12935,7 @@ __metadata: tesseract.js: "npm:7.0.0" typescript: "npm:5.9.3" uglifycss: "npm:0.0.29" + undici: "npm:8.10.0" url-parse: "npm:1.5.10" uuid: "npm:11.1.1" word-wrap: "npm:1.2.5" @@ -12994,7 +12992,7 @@ __metadata: "@joplin/utils": "npm:^3.7.1" "@types/fs-extra": "npm:11.0.4" "@types/jest": "npm:29.5.14" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" fs-extra: "npm:11.3.5" gh-release-assets: "npm:2.0.1" jest: "npm:29.7.0" @@ -13049,7 +13047,7 @@ __metadata: "@types/jest": "npm:29.5.14" "@types/json5": "npm:2.2.0" "@types/markdown-it": "npm:13.0.9" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" abcjs: "npm:6.6.3" font-awesome-filetypes: "npm:2.1.0" fs-extra: "npm:11.3.5" @@ -13102,7 +13100,7 @@ __metadata: "@types/koa": "npm:2.15.0" "@types/markdown-it": "npm:13.0.9" "@types/mustache": "npm:4.2.6" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" "@types/nodemailer": "npm:6.4.24" "@types/qrcode": "npm:1.5.6" "@types/uuid": "npm:11.0.0" @@ -13164,7 +13162,7 @@ __metadata: "@types/js-yaml": "npm:4.0.9" "@types/markdown-it": "npm:13.0.9" "@types/mustache": "npm:4.2.6" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" "@types/node-fetch": "npm:2.6.13" "@types/yargs": "npm:17.0.35" compare-versions: "npm:6.1.1" @@ -13264,7 +13262,6 @@ __metadata: "@types/fs-extra": "npm:11.0.4" "@types/jest": "npm:29.5.14" "@types/markdown-it": "npm:13.0.9" - "@types/node-fetch": "npm:2.6.13" async-mutex: "npm:0.5.0" dayjs: "npm:1.11.21" execa: "npm:5.1.1" @@ -13274,7 +13271,6 @@ __metadata: jest: "npm:29.7.0" markdown-it: "npm:13.0.2" moment: "npm:2.30.1" - node-fetch: "npm:2.6.7" sprintf-js: "npm:1.1.3" tcp-port-used: "npm:1.0.3" ts-jest: "npm:29.4.11" @@ -19141,12 +19137,12 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:18.19.130": - version: 18.19.130 - resolution: "@types/node@npm:18.19.130" +"@types/node@npm:22.19.0": + version: 22.19.0 + resolution: "@types/node@npm:22.19.0" dependencies: - undici-types: "npm:~5.26.4" - checksum: 10/ebb85c6edcec78df926de27d828ecbeb1b3d77c165ceef95bfc26e171edbc1924245db4eb2d7d6230206fe6b1a1f7665714fe1c70739e9f5980d8ce31af6ef82 + undici-types: "npm:~6.21.0" + checksum: 10/fdb16593b54fcc82f16064da089f89e80df3da70162a379dc9dc91fd65234704d738f31a5479ab1affc45ca531333ccdb4fad5546e5fcd40327987b9539f173f languageName: node linkType: hard @@ -33111,19 +33107,6 @@ __metadata: languageName: node linkType: hard -"form-data@npm:4.0.6": - version: 4.0.6 - resolution: "form-data@npm:4.0.6" - dependencies: - asynckit: "npm:^0.4.0" - combined-stream: "npm:^1.0.8" - es-set-tostringtag: "npm:^2.1.0" - hasown: "npm:^2.0.4" - mime-types: "npm:^2.1.35" - checksum: 10/de6614c8537c92fa5fa3ee7e827758f98f5a9c033f348b7de81855ef36e5cb867e75d9f405d9483ab8d724a4a20d4e79926a299fa8dbba38f530eb659f0884e4 - languageName: node - linkType: hard - "form-data@npm:^3.0.0": version: 3.0.1 resolution: "form-data@npm:3.0.1" @@ -33147,15 +33130,15 @@ __metadata: linkType: hard "form-data@npm:^4.0.4": - version: 4.0.4 - resolution: "form-data@npm:4.0.4" + version: 4.0.6 + resolution: "form-data@npm:4.0.6" dependencies: asynckit: "npm:^0.4.0" combined-stream: "npm:^1.0.8" es-set-tostringtag: "npm:^2.1.0" - hasown: "npm:^2.0.2" - mime-types: "npm:^2.1.12" - checksum: 10/a4b62e21932f48702bc468cc26fb276d186e6b07b557e3dd7cc455872bdbb82db7db066844a64ad3cf40eaf3a753c830538183570462d3649fdfd705601cbcfb + hasown: "npm:^2.0.4" + mime-types: "npm:^2.1.35" + checksum: 10/de6614c8537c92fa5fa3ee7e827758f98f5a9c033f348b7de81855ef36e5cb867e75d9f405d9483ab8d724a4a20d4e79926a299fa8dbba38f530eb659f0884e4 languageName: node linkType: hard @@ -35430,13 +35413,6 @@ __metadata: languageName: node linkType: hard -"hpagent@npm:1.2.0": - version: 1.2.0 - resolution: "hpagent@npm:1.2.0" - checksum: 10/bad186449da7e3456788a8cbae459fc6c0a855d5872a7f460c48ce4a613020d8d914839dad10047297099299c4f9e6c65a0eec3f5886af196c0a516e4ad8a845 - languageName: node - linkType: hard - "html-encoding-sniffer@npm:^2.0.1": version: 2.0.1 resolution: "html-encoding-sniffer@npm:2.0.1" @@ -38826,7 +38802,7 @@ __metadata: "@joplin/utils": "npm:~3.7" "@types/fs-extra": "npm:11.0.4" "@types/jest": "npm:29.5.14" - "@types/node": "npm:18.19.130" + "@types/node": "npm:22.19.0" "@types/proper-lockfile": "npm:^4.1.2" aws-sdk: "npm:2.1340.0" chalk: "npm:4.1.2" @@ -57603,10 +57579,10 @@ __metadata: languageName: node linkType: hard -"undici-types@npm:~5.26.4": - version: 5.26.5 - resolution: "undici-types@npm:5.26.5" - checksum: 10/0097779d94bc0fd26f0418b3a05472410408877279141ded2bd449167be1aed7ea5b76f756562cb3586a07f251b90799bab22d9019ceba49c037c76445f7cddd +"undici-types@npm:~6.21.0": + version: 6.21.0 + resolution: "undici-types@npm:6.21.0" + checksum: 10/ec8f41aa4359d50f9b59fa61fe3efce3477cc681908c8f84354d8567bb3701fafdddf36ef6bff307024d3feb42c837cf6f670314ba37fc8145e219560e473d14 languageName: node linkType: hard @@ -57617,10 +57593,17 @@ __metadata: languageName: node linkType: hard +"undici@npm:8.10.0": + version: 8.10.0 + resolution: "undici@npm:8.10.0" + checksum: 10/254219966d4a2fb110f565ffe73131caa82e949f207b9dd88930ebc23dfb6561db72ca187391cbd408ceeba7c647cb47d4110e8ae00df44c0c66e8a21e091dc0 + languageName: node + linkType: hard + "undici@npm:^6.18.2": - version: 6.21.2 - resolution: "undici@npm:6.21.2" - checksum: 10/9cd9ead22599c23aa2a7dfa5b80fa1491bebb294bf1dc64c9c0f90ea4ec8e272a4db2810e2565d65b952249f105523d2929d7cc951f88cf0a1f082143bae8d75 + version: 6.28.0 + resolution: "undici@npm:6.28.0" + checksum: 10/672a7a53bd1f6c80edb73b357ab36e98ef9e474024c442aab2b8ea2bc32f1103cab05525c78661ae724f3e1340e23d03efb9730fba28e76218ab0ec52e15d75a languageName: node linkType: hard @@ -57632,9 +57615,9 @@ __metadata: linkType: hard "undici@npm:^7.24.4": - version: 7.26.0 - resolution: "undici@npm:7.26.0" - checksum: 10/c55b8f8e378dce6e645853faf0834d66b9f470c6ee1430c6e2b7971d831de36081f7e2e21ac1fc11297657d7c856241b0bf746b2a24f1277f8d156df9c181ba7 + version: 7.29.0 + resolution: "undici@npm:7.29.0" + checksum: 10/ca73639071bdcbc41e57be1800847fa18f86f8fbf7a5641ff9899f7bbb6f5cecdd593e225f5fd94e31703dcbfaea4a722a3b81cab90c5c4e2b406aec1ae55732 languageName: node linkType: hard