mirror of
https://github.com/laurent22/joplin.git
synced 2026-11-11 19:16:16 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Laurent Cozic
|
98a55d175a | ||
|
Laurent Cozic
|
54a645bfec | ||
|
Laurent Cozic
|
34696bc462 | ||
|
Laurent Cozic
|
dfe4a2424e | ||
|
Laurent Cozic
|
fbf73e5f6a |
No files matched your search
+1
-4
@@ -686,7 +686,6 @@ packages/app-desktop/services/plugins/hooks/useThemeCss.test.js
|
||||
packages/app-desktop/services/plugins/hooks/useThemeCss.js
|
||||
packages/app-desktop/services/plugins/hooks/useViewIsReady.js
|
||||
packages/app-desktop/services/plugins/hooks/useWebviewToPluginMessages.js
|
||||
packages/app-desktop/services/plugins/startBlockedPluginsCheck.js
|
||||
packages/app-desktop/services/plugins/types.js
|
||||
packages/app-desktop/services/restart.js
|
||||
packages/app-desktop/services/spellChecker/SpellCheckerServiceDriverNative.js
|
||||
@@ -1840,6 +1839,7 @@ packages/lib/services/ocr/drivers/OcrDriverTranscribe.js
|
||||
packages/lib/services/ocr/utils/createAccessiblePdf.js
|
||||
packages/lib/services/ocr/utils/filterOcrText.test.js
|
||||
packages/lib/services/ocr/utils/filterOcrText.js
|
||||
packages/lib/services/ocr/utils/glyphLessFont.js
|
||||
packages/lib/services/ocr/utils/types.js
|
||||
packages/lib/services/plugins/BasePlatformImplementation.js
|
||||
packages/lib/services/plugins/BasePluginRunner.js
|
||||
@@ -1882,11 +1882,8 @@ packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.js
|
||||
packages/lib/services/plugins/loadPlugins.test.js
|
||||
packages/lib/services/plugins/loadPlugins.js
|
||||
packages/lib/services/plugins/reducer.js
|
||||
packages/lib/services/plugins/refreshBlockedPluginVersions.js
|
||||
packages/lib/services/plugins/testing/MockPlatformImplementation.js
|
||||
packages/lib/services/plugins/testing/MockPluginRunner.js
|
||||
packages/lib/services/plugins/utils/blockedVersions.test.js
|
||||
packages/lib/services/plugins/utils/blockedVersions.js
|
||||
packages/lib/services/plugins/utils/createViewHandle.js
|
||||
packages/lib/services/plugins/utils/executeSandboxCall.js
|
||||
packages/lib/services/plugins/utils/getActivePluginEditorView.js
|
||||
|
||||
+1
-4
@@ -712,7 +712,6 @@ packages/app-desktop/services/plugins/hooks/useThemeCss.test.js
|
||||
packages/app-desktop/services/plugins/hooks/useThemeCss.js
|
||||
packages/app-desktop/services/plugins/hooks/useViewIsReady.js
|
||||
packages/app-desktop/services/plugins/hooks/useWebviewToPluginMessages.js
|
||||
packages/app-desktop/services/plugins/startBlockedPluginsCheck.js
|
||||
packages/app-desktop/services/plugins/types.js
|
||||
packages/app-desktop/services/restart.js
|
||||
packages/app-desktop/services/spellChecker/SpellCheckerServiceDriverNative.js
|
||||
@@ -1866,6 +1865,7 @@ packages/lib/services/ocr/drivers/OcrDriverTranscribe.js
|
||||
packages/lib/services/ocr/utils/createAccessiblePdf.js
|
||||
packages/lib/services/ocr/utils/filterOcrText.test.js
|
||||
packages/lib/services/ocr/utils/filterOcrText.js
|
||||
packages/lib/services/ocr/utils/glyphLessFont.js
|
||||
packages/lib/services/ocr/utils/types.js
|
||||
packages/lib/services/plugins/BasePlatformImplementation.js
|
||||
packages/lib/services/plugins/BasePluginRunner.js
|
||||
@@ -1908,11 +1908,8 @@ packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.js
|
||||
packages/lib/services/plugins/loadPlugins.test.js
|
||||
packages/lib/services/plugins/loadPlugins.js
|
||||
packages/lib/services/plugins/reducer.js
|
||||
packages/lib/services/plugins/refreshBlockedPluginVersions.js
|
||||
packages/lib/services/plugins/testing/MockPlatformImplementation.js
|
||||
packages/lib/services/plugins/testing/MockPluginRunner.js
|
||||
packages/lib/services/plugins/utils/blockedVersions.test.js
|
||||
packages/lib/services/plugins/utils/blockedVersions.js
|
||||
packages/lib/services/plugins/utils/createViewHandle.js
|
||||
packages/lib/services/plugins/utils/executeSandboxCall.js
|
||||
packages/lib/services/plugins/utils/getActivePluginEditorView.js
|
||||
|
||||
@@ -1,26 +1,14 @@
|
||||
import { AppType } from '@joplin/lib/models/Setting';
|
||||
import RepositoryApi, { AppInfo, InstallMode } from '@joplin/lib/services/plugins/RepositoryApi';
|
||||
import Setting from '@joplin/lib/models/Setting';
|
||||
import shim from '@joplin/lib/shim';
|
||||
import refreshBlockedPluginVersions from '@joplin/lib/services/plugins/refreshBlockedPluginVersions';
|
||||
import { setupDatabaseAndSynchronizer, switchClient, supportDir, createTempDir } from '@joplin/lib/testing/test-utils';
|
||||
import { copy, mkdirp, readJson, remove, writeFile, writeJson } from 'fs-extra';
|
||||
import { remove } from 'fs-extra';
|
||||
|
||||
let tempDirs: string[] = [];
|
||||
async function newRepoApi(appInfo: AppInfo = { type: AppType.Desktop, version: '3.0.0' }, blockedVersions: Record<string, string[]> = null): Promise<RepositoryApi> {
|
||||
async function newRepoApi(appInfo: AppInfo = { type: AppType.Desktop, version: '3.0.0' }): Promise<RepositoryApi> {
|
||||
const tempDir = await createTempDir();
|
||||
tempDirs.push(tempDir);
|
||||
|
||||
let repoDir = `${supportDir}/pluginRepo`;
|
||||
if (blockedVersions) {
|
||||
repoDir = `${tempDir}/pluginRepo`;
|
||||
await copy(`${supportDir}/pluginRepo`, repoDir);
|
||||
const manifests = await readJson(`${repoDir}/manifests.json`);
|
||||
for (const [id, versions] of Object.entries(blockedVersions)) manifests[id]._blocked_versions = versions;
|
||||
await writeJson(`${repoDir}/manifests.json`, manifests);
|
||||
}
|
||||
|
||||
const repo = new RepositoryApi(repoDir, tempDir, appInfo, InstallMode.Default);
|
||||
const repo = new RepositoryApi(`${supportDir}/pluginRepo`, tempDir, appInfo, InstallMode.Default);
|
||||
await repo.initialize();
|
||||
return repo;
|
||||
}
|
||||
@@ -79,33 +67,4 @@ describe('services_plugins_RepositoryApi', () => {
|
||||
expect(await api.pluginCanBeUpdated(id, installedVersion)).toBe(shouldBeUpdatable);
|
||||
}));
|
||||
|
||||
it.each([
|
||||
['1.0.2'],
|
||||
['>=1.0.1'],
|
||||
['*'],
|
||||
])('should not update to or download a blocked version (%s)', (async (blockedVersion) => {
|
||||
const id = 'org.joplinapp.plugins.ToggleSidebars';
|
||||
const api = await newRepoApi(undefined, { [id]: [blockedVersion] });
|
||||
expect(await api.pluginCanBeUpdated(id, '1.0.0')).toBe(false);
|
||||
await expect(api.downloadPlugin(id)).rejects.toThrow('has been blocked');
|
||||
}));
|
||||
|
||||
it('should cache the blocked versions only when plugins are installed', (async () => {
|
||||
const blockedVersions = { 'org.joplinapp.plugins.ToggleSidebars': ['1.0.1', '1.0.2'] };
|
||||
const api = await newRepoApi(undefined, blockedVersions);
|
||||
|
||||
await refreshBlockedPluginVersions(api);
|
||||
expect(Setting.value('plugins.blockedVersions')).toEqual({});
|
||||
|
||||
const pluginPath = `${Setting.value('pluginDir')}/test.jpl`;
|
||||
await mkdirp(Setting.value('pluginDir'));
|
||||
await writeFile(pluginPath, '');
|
||||
try {
|
||||
await refreshBlockedPluginVersions(api);
|
||||
expect(Setting.value('plugins.blockedVersions')).toEqual(blockedVersions);
|
||||
} finally {
|
||||
await remove(pluginPath);
|
||||
}
|
||||
}));
|
||||
|
||||
});
|
||||
@@ -18,7 +18,6 @@ import { _, setLocale } from '@joplin/lib/locale';
|
||||
import SpellCheckerService from '@joplin/lib/services/spellChecker/SpellCheckerService';
|
||||
import SpellCheckerServiceDriverNative from './services/spellChecker/SpellCheckerServiceDriverNative';
|
||||
import bridge from './services/bridge';
|
||||
import startBlockedPluginsCheck from './services/plugins/startBlockedPluginsCheck';
|
||||
import menuCommandNames from './gui/menuCommandNames';
|
||||
import ResourceService from '@joplin/lib/services/ResourceService';
|
||||
import ExternalEditWatcher from '@joplin/lib/services/ExternalEditWatcher';
|
||||
@@ -386,8 +385,6 @@ class Application extends BaseApplication {
|
||||
ipcRenderer.send('startup-plugins-loaded');
|
||||
|
||||
void afterDefaultPluginsLoaded(service.plugins, getDefaultPluginsInfo(), pluginSettings);
|
||||
|
||||
startBlockedPluginsCheck(this.store());
|
||||
}
|
||||
}, 500);
|
||||
}
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
import Setting, { AppType } from '@joplin/lib/models/Setting';
|
||||
import RepositoryApi, { InstallMode } from '@joplin/lib/services/plugins/RepositoryApi';
|
||||
import PluginService from '@joplin/lib/services/plugins/PluginService';
|
||||
import refreshBlockedPluginVersions from '@joplin/lib/services/plugins/refreshBlockedPluginVersions';
|
||||
import { State, stateUtils } from '@joplin/lib/reducer';
|
||||
import shim from '@joplin/lib/shim';
|
||||
import time from '@joplin/lib/time';
|
||||
import { _ } from '@joplin/lib/locale';
|
||||
import Logger from '@joplin/utils/Logger';
|
||||
import { Store } from 'redux';
|
||||
import bridge from '../bridge';
|
||||
import restart from '../restart';
|
||||
|
||||
const logger = Logger.create('startBlockedPluginsCheck');
|
||||
|
||||
const waitForNotesSaved = async (store: Store<State>) => {
|
||||
const startTime = Date.now();
|
||||
while (stateUtils.hasNotesBeingSaved(store.getState()) && Date.now() - startTime < 5000) {
|
||||
await time.msleep(100);
|
||||
}
|
||||
};
|
||||
|
||||
// A blocked plugin can't be fully unloaded on desktop since its content scripts run in the main
|
||||
// window, so we restart straight away rather than wait for the user, who may not be at their computer.
|
||||
const restartForBlockedPlugins = async (store: Store<State>, pluginIds: string[]) => {
|
||||
logger.warn('Restarting because these running plugins have been blocked:', pluginIds);
|
||||
const service = PluginService.instance();
|
||||
Setting.setValue('plugins.blockedRestartNotice', pluginIds.map(id => service.pluginById(id).manifest.name));
|
||||
await waitForNotesSaved(store);
|
||||
await restart();
|
||||
};
|
||||
|
||||
const showBlockedRestartNotice = () => {
|
||||
const pluginNames: string[] = Setting.value('plugins.blockedRestartNotice');
|
||||
if (!pluginNames.length) return;
|
||||
Setting.setValue('plugins.blockedRestartNotice', []);
|
||||
bridge().showInfoMessageBox(_('Joplin was restarted because the following plugins have been disabled by the Joplin team for security reasons: %s\n\nWe recommend uninstalling them.', pluginNames.join(', ')));
|
||||
};
|
||||
|
||||
const startBlockedPluginsCheck = (store: Store<State>) => {
|
||||
showBlockedRestartNotice();
|
||||
|
||||
const repoApi = RepositoryApi.ofDefaultJoplinRepo(
|
||||
Setting.value('tempDir'), { type: AppType.Desktop, version: PluginService.instance().appVersion }, InstallMode.Default,
|
||||
);
|
||||
|
||||
const check = async () => {
|
||||
try {
|
||||
const blockedRunningPluginIds = await refreshBlockedPluginVersions(repoApi);
|
||||
if (blockedRunningPluginIds.length) await restartForBlockedPlugins(store, blockedRunningPluginIds);
|
||||
} catch (error) {
|
||||
logger.warn('Could not refresh blocked plugin versions:', error);
|
||||
}
|
||||
};
|
||||
|
||||
void check();
|
||||
shim.setInterval(() => { void check(); }, 60 * 60 * 1000);
|
||||
};
|
||||
|
||||
export default startBlockedPluginsCheck;
|
||||
@@ -9,7 +9,6 @@ import { connect, useStore } from 'react-redux';
|
||||
import Logger from '@joplin/utils/Logger';
|
||||
import PluginService, { PluginSettings, SerializedPluginSettings } from '@joplin/lib/services/plugins/PluginService';
|
||||
import { PluginHtmlContents, PluginStates } from '@joplin/lib/services/plugins/reducer';
|
||||
import { BlockedPluginVersions } from '@joplin/lib/services/plugins/utils/blockedVersions';
|
||||
import useAsyncEffect from '@joplin/lib/hooks/useAsyncEffect';
|
||||
import PluginDialogManager from './dialogs/PluginDialogManager';
|
||||
import { AppState } from '../../utils/types';
|
||||
@@ -34,7 +33,6 @@ const usePlugins = (
|
||||
pluginSettings: PluginSettings,
|
||||
pluginSupportEnabled: boolean,
|
||||
devPluginPath: string,
|
||||
blockedPluginVersions: BlockedPluginVersions,
|
||||
) => {
|
||||
const store = useStore<AppState>();
|
||||
const lastPluginRunner = usePrevious(pluginRunner);
|
||||
@@ -73,7 +71,7 @@ const usePlugins = (
|
||||
if (!event.cancelled) {
|
||||
reloadAllRef.current = false;
|
||||
}
|
||||
}, [pluginRunner, store, webviewLoaded, pluginSettings, reloadCounter, blockedPluginVersions]);
|
||||
}, [pluginRunner, store, webviewLoaded, pluginSettings, reloadCounter]);
|
||||
};
|
||||
|
||||
const useUnloadPluginsOnGlobalDisable = (
|
||||
@@ -97,7 +95,6 @@ interface Props {
|
||||
pluginSupportEnabled: boolean;
|
||||
pluginStates: PluginStates;
|
||||
devPluginPath: string;
|
||||
blockedPluginVersions: BlockedPluginVersions;
|
||||
pluginHtmlContents: PluginHtmlContents;
|
||||
themeId: number;
|
||||
}
|
||||
@@ -128,7 +125,7 @@ const PluginRunnerWebViewComponent: React.FC<Props> = props => {
|
||||
}, [webviewReloadCounter]);
|
||||
|
||||
const pluginSettings = usePluginSettings(props.serializedPluginSettings);
|
||||
usePlugins(pluginRunner, webviewLoaded, pluginSettings, props.pluginSupportEnabled, props.devPluginPath, props.blockedPluginVersions);
|
||||
usePlugins(pluginRunner, webviewLoaded, pluginSettings, props.pluginSupportEnabled, props.devPluginPath);
|
||||
useUnloadPluginsOnGlobalDisable(props.pluginStates, props.pluginSupportEnabled);
|
||||
|
||||
const onLoadStart = useCallback(() => {
|
||||
@@ -216,7 +213,6 @@ export default connect((state: AppState) => {
|
||||
serializedPluginSettings: state.settings['plugins.states'],
|
||||
pluginSupportEnabled: state.settings['plugins.pluginSupportEnabled'],
|
||||
devPluginPath: state.settings['plugins.devPluginPaths'],
|
||||
blockedPluginVersions: state.settings['plugins.blockedVersions'],
|
||||
pluginStates: state.pluginService.plugins,
|
||||
pluginHtmlContents: state.pluginService.pluginHtmlContents,
|
||||
themeId: state.settings.theme,
|
||||
|
||||
@@ -8,8 +8,6 @@ import ResourceService from '@joplin/lib/services/ResourceService';
|
||||
import KvStore from '@joplin/lib/services/KvStore';
|
||||
import Setting, { AppType, Env } from '@joplin/lib/models/Setting';
|
||||
import PoorManIntervals from '@joplin/lib/PoorManIntervals';
|
||||
import RepositoryApi, { InstallMode } from '@joplin/lib/services/plugins/RepositoryApi';
|
||||
import refreshBlockedPluginVersions from '@joplin/lib/services/plugins/refreshBlockedPluginVersions';
|
||||
import { parseNotesParent } from '@joplin/lib/reducer';
|
||||
import uuid from '@joplin/lib/uuid';
|
||||
import { loadKeychainServiceAndSettings } from '@joplin/lib/services/SettingUtils';
|
||||
@@ -517,21 +515,6 @@ const buildStartupTasks = (
|
||||
|
||||
const updatedSettings = pluginService.clearUpdateState(pluginSettings);
|
||||
Setting.setValue('plugins.states', updatedSettings);
|
||||
|
||||
// Running plugins that are now blocked are unloaded by PluginRunnerWebView when the setting changes
|
||||
const repoApi = RepositoryApi.ofDefaultJoplinRepo(
|
||||
Setting.value('tempDir'), { type: AppType.Mobile, version: platformImplementation.versionInfo.version }, InstallMode.Default,
|
||||
);
|
||||
const checkBlockedPlugins = async () => {
|
||||
if (!Setting.value('plugins.pluginSupportEnabled')) return;
|
||||
try {
|
||||
await refreshBlockedPluginVersions(repoApi);
|
||||
} catch (error) {
|
||||
logger.warn('Could not refresh blocked plugin versions:', error);
|
||||
}
|
||||
};
|
||||
void checkBlockedPlugins();
|
||||
PoorManIntervals.setInterval(() => { void checkBlockedPlugins(); }, 60 * 60 * 1000);
|
||||
});
|
||||
addTask('buildStartupTasks/run startup tests', async () => {
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
@@ -7,7 +7,6 @@ import type SettingType from '../Setting';
|
||||
import { AppType, SettingItemSubType, SettingItemType, SettingStorage, SyncStartupOperation, SettingItem, SettingButtonStyle } from './types';
|
||||
import { defaultListColumns } from '../../services/plugins/api/noteListType';
|
||||
import type { PluginSettings } from '../../services/plugins/PluginService';
|
||||
import type { BlockedPluginVersions } from '../../services/plugins/utils/blockedVersions';
|
||||
import type { PublicPrivateKeyPair } from '../../services/e2ee/ppk/ppk';
|
||||
import type { NoteLockKeyConflict } from '../../services/synchronizer/syncInfoUtils';
|
||||
import { EmptyObject } from '@joplin/utils/types';
|
||||
@@ -1448,20 +1447,6 @@ const builtInMetadata = (Setting: typeof SettingType) => {
|
||||
autoSave: true,
|
||||
},
|
||||
|
||||
'plugins.blockedVersions': {
|
||||
value: {} as BlockedPluginVersions,
|
||||
type: SettingItemType.Object,
|
||||
public: false,
|
||||
appTypes: [AppType.Desktop, AppType.Mobile],
|
||||
},
|
||||
|
||||
'plugins.blockedRestartNotice': {
|
||||
value: [] as string[],
|
||||
type: SettingItemType.Array,
|
||||
public: false,
|
||||
appTypes: [AppType.Desktop],
|
||||
},
|
||||
|
||||
'plugins.enableWebviewDebugging': {
|
||||
value: false,
|
||||
type: SettingItemType.Bool,
|
||||
|
||||
@@ -397,6 +397,33 @@ describe('OcrService', () => {
|
||||
expect(pdfBytes.length).toBeGreaterThan(jpegBuffer.length * 1.5);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ words: ['ı', 'Мир', '日本語'], rtl: false },
|
||||
{ words: ['אור', 'חדש'], rtl: true },
|
||||
{ words: ['نور', 'باب', 'عام2024'], rtl: true },
|
||||
])('should create an accessible PDF with text that can be extracted: $words', async ({ words, rtl }) => {
|
||||
const jpegBuffer = await fs.readFile(`${supportDir}/photo.jpg`);
|
||||
|
||||
// Like Tesseract, words are in logical order, so right-to-left words are laid out from the right
|
||||
const ocrDetails: PdfOcrDetails = {
|
||||
version: 1,
|
||||
pages: [{
|
||||
lines: [{ words: words.map((t, i) => {
|
||||
const x = rtl ? 700 - (i + 1) * 150 : 20 + i * 150;
|
||||
return { t, bb: [x, x + 130, 20, 60] };
|
||||
}) }],
|
||||
}],
|
||||
};
|
||||
|
||||
const pdfBytes = await createAccessiblePdf([{ buffer: jpegBuffer, width: 800, height: 400 }], JSON.stringify(ocrDetails));
|
||||
const pdfPath = join(Setting.value('tempDir'), 'accessible.pdf');
|
||||
await fs.writeFile(pdfPath, pdfBytes);
|
||||
|
||||
const textByPage = await shim.pdfExtractEmbeddedText(pdfPath);
|
||||
expect(textByPage.length).toBe(1);
|
||||
expect(textByPage[0].split(/\s+/)).toEqual(words);
|
||||
});
|
||||
|
||||
it('should truncate long OCR text', async () => {
|
||||
const service = newOcrService();
|
||||
service.testing_setOcrMaxSize(10);
|
||||
|
||||
@@ -1,60 +1,188 @@
|
||||
import { PDFDocument, PDFFont, PDFPage, rgb, StandardFonts } from 'pdf-lib';
|
||||
import { beginText, endText, PDFDocument, PDFHexString, PDFOperator, PDFPage, PDFRef, PDFString, setCharacterSqueeze, setFontAndSize, setTextMatrix, setTextRenderingMode, showText, TextRenderingMode } from 'pdf-lib';
|
||||
import { PdfOcrDetails, RecognizeResultLine } from './types';
|
||||
import glyphLessFontBase64 from './glyphLessFont';
|
||||
|
||||
// The PDF OCR images are created at 2x scale by pdfToImages()
|
||||
const OCR_SCALE_FACTOR = 2;
|
||||
|
||||
// Metrics of the glyphless font, in 1/1000 of the font size
|
||||
const GLYPH_WIDTH = 500;
|
||||
const FONT_ASCENT = 800;
|
||||
const FONT_DESCENT = -200;
|
||||
|
||||
export interface PageImageWithDimensions {
|
||||
buffer: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
// Maps each UTF-16 code unit (used as CID) to itself. A bfrange may only vary in its last byte and a
|
||||
// block may contain at most 100 entries.
|
||||
const makeToUnicodeCMap = () => {
|
||||
const ranges: string[] = [];
|
||||
for (let i = 0; i < 256; i++) {
|
||||
const high = i.toString(16).toUpperCase().padStart(2, '0');
|
||||
ranges.push(`<${high}00> <${high}FF> <${high}00>`);
|
||||
}
|
||||
|
||||
// cSpell:disable
|
||||
const blocks: string[] = [];
|
||||
for (let i = 0; i < ranges.length; i += 100) {
|
||||
const chunk = ranges.slice(i, i + 100);
|
||||
blocks.push(`${chunk.length} beginbfrange\n${chunk.join('\n')}\nendbfrange`);
|
||||
}
|
||||
|
||||
return [
|
||||
'/CIDInit /ProcSet findresource begin',
|
||||
'12 dict begin',
|
||||
'begincmap',
|
||||
'/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def',
|
||||
'/CMapName /Adobe-Identity-UCS def',
|
||||
'/CMapType 2 def',
|
||||
'1 begincodespacerange',
|
||||
'<0000> <FFFF>',
|
||||
'endcodespacerange',
|
||||
...blocks,
|
||||
'endcmap',
|
||||
'CMapName currentdict /CMap defineresource pop',
|
||||
'end',
|
||||
'end',
|
||||
].join('\n');
|
||||
// cSpell:enable
|
||||
};
|
||||
|
||||
// Standard fonts only support WinAnsi, so any other character (Cyrillic, CJK, etc.) would make
|
||||
// pdf-lib throw. Instead, like Tesseract, we use a Type0 font whose CIDs are UTF-16 code units, all
|
||||
// mapped to the same empty glyph, with a ToUnicode CMap so that the text can be extracted.
|
||||
const embedGlyphLessFont = (pdfDoc: PDFDocument) => {
|
||||
const context = pdfDoc.context;
|
||||
|
||||
const fontFile = Buffer.from(glyphLessFontBase64, 'base64');
|
||||
const fontFileRef = context.register(context.flateStream(fontFile, { Length1: fontFile.length }));
|
||||
|
||||
// Two bytes per CID, all pointing to glyph 1
|
||||
const cidToGidMap = new Uint8Array(0x10000 * 2);
|
||||
for (let i = 1; i < cidToGidMap.length; i += 2) cidToGidMap[i] = 1;
|
||||
const cidToGidMapRef = context.register(context.flateStream(cidToGidMap));
|
||||
|
||||
const fontDescriptorRef = context.register(context.obj({
|
||||
Type: 'FontDescriptor',
|
||||
FontName: 'GlyphLessFont',
|
||||
Flags: 5,
|
||||
FontBBox: [0, FONT_DESCENT, GLYPH_WIDTH, FONT_ASCENT],
|
||||
ItalicAngle: 0,
|
||||
Ascent: FONT_ASCENT,
|
||||
Descent: FONT_DESCENT,
|
||||
CapHeight: FONT_ASCENT,
|
||||
StemV: 80,
|
||||
FontFile2: fontFileRef,
|
||||
}));
|
||||
|
||||
const cidFontRef = context.register(context.obj({
|
||||
Type: 'Font',
|
||||
Subtype: 'CIDFontType2',
|
||||
BaseFont: 'GlyphLessFont',
|
||||
CIDSystemInfo: {
|
||||
Registry: PDFString.of('Adobe'),
|
||||
Ordering: PDFString.of('Identity'),
|
||||
Supplement: 0,
|
||||
},
|
||||
FontDescriptor: fontDescriptorRef,
|
||||
CIDToGIDMap: cidToGidMapRef,
|
||||
DW: GLYPH_WIDTH,
|
||||
}));
|
||||
|
||||
const toUnicodeRef = context.register(context.flateStream(makeToUnicodeCMap()));
|
||||
|
||||
return context.register(context.obj({
|
||||
Type: 'Font',
|
||||
Subtype: 'Type0',
|
||||
BaseFont: 'GlyphLessFont',
|
||||
Encoding: 'Identity-H',
|
||||
DescendantFonts: [cidFontRef],
|
||||
ToUnicode: toUnicodeRef,
|
||||
}));
|
||||
};
|
||||
|
||||
const rtlScriptRegex = /[\p{Script=Hebrew}\p{Script=Arabic}\p{Script=Syriac}\p{Script=Thaana}\p{Script=Nko}]/u;
|
||||
const isRtlLetter = (char: string) => rtlScriptRegex.test(char) && !/\p{N}/u.test(char);
|
||||
const isLtrStrong = (char: string) => /[\p{L}\p{N}]/u.test(char) && !isRtlLetter(char);
|
||||
|
||||
// OCR text is in logical order, but viewers expect PDF text in visual (left-to-right) order and
|
||||
// reorder it when extracting. So in right-to-left words, the runs are reversed, as well as the
|
||||
// characters within right-to-left runs (numbers and Latin text remain left-to-right).
|
||||
const toVisualOrder = (text: string) => {
|
||||
const chars = Array.from(text);
|
||||
if (!chars.some(isRtlLetter)) return text;
|
||||
|
||||
const runs: string[][] = [];
|
||||
for (const char of chars) {
|
||||
const lastRun = runs[runs.length - 1];
|
||||
if (lastRun && isLtrStrong(lastRun[0]) === isLtrStrong(char)) {
|
||||
lastRun.push(char);
|
||||
} else {
|
||||
runs.push([char]);
|
||||
}
|
||||
}
|
||||
|
||||
return runs.reverse().map(run => isLtrStrong(run[0]) ? run.join('') : run.reverse().join('')).join('');
|
||||
};
|
||||
|
||||
const toUtf16Hex = (text: string) => {
|
||||
let output = '';
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
output += text.charCodeAt(i).toString(16).padStart(4, '0');
|
||||
}
|
||||
return output;
|
||||
};
|
||||
|
||||
// Adds an invisible text layer to a PDF page based on OCR word positions.
|
||||
// The text is rendered in "invisible" mode (render mode 3) so it doesn't
|
||||
// appear visually but can be selected, copied, and read by screen readers.
|
||||
const addInvisibleTextLayer = (
|
||||
page: PDFPage,
|
||||
lines: RecognizeResultLine[],
|
||||
font: PDFFont,
|
||||
_pageWidth: number,
|
||||
fontRef: PDFRef,
|
||||
pageHeight: number,
|
||||
): void => {
|
||||
) => {
|
||||
const fontName = page.node.newFontDictionary('GlyphLessFont', fontRef);
|
||||
const operators: PDFOperator[] = [beginText(), setTextRenderingMode(TextRenderingMode.Invisible)];
|
||||
|
||||
for (const line of lines) {
|
||||
for (const word of line.words) {
|
||||
const text = word.t;
|
||||
if (!text || !text.trim()) continue;
|
||||
|
||||
// Bounding box format from Tesseract: [x0, x1, y0, y1]
|
||||
const [x0, , y0, y1] = word.bb;
|
||||
const [x0, x1, y0, y1] = word.bb;
|
||||
|
||||
// Convert from OCR coordinates (2x scale, origin top-left)
|
||||
// to PDF coordinates (1x scale, origin bottom-left)
|
||||
const pdfX = x0 / OCR_SCALE_FACTOR;
|
||||
const pdfY = pageHeight - (y1 / OCR_SCALE_FACTOR); // Flip Y axis and use bottom of bbox
|
||||
|
||||
// Calculate word height in PDF coordinates (width not currently used)
|
||||
const wordWidth = (x1 - x0) / OCR_SCALE_FACTOR;
|
||||
const wordHeight = (y1 - y0) / OCR_SCALE_FACTOR;
|
||||
|
||||
// Estimate font size based on word height
|
||||
// We want the text to fit within the bounding box
|
||||
const fontSize = Math.max(1, wordHeight * 0.85);
|
||||
// The font's ascent-to-descent height is one em, so this makes the glyphs span the word's
|
||||
// bounding box, with the baseline placed above its bottom by the descent.
|
||||
const fontSize = Math.max(1, wordHeight);
|
||||
const pdfX = x0 / OCR_SCALE_FACTOR;
|
||||
const pdfY = pageHeight - (y1 / OCR_SCALE_FACTOR) - fontSize * FONT_DESCENT / 1000;
|
||||
|
||||
// Draw the text as invisible (using transparent color)
|
||||
// PDF.js and screen readers can still detect and read this text
|
||||
page.drawText(text, {
|
||||
x: pdfX,
|
||||
y: pdfY,
|
||||
size: fontSize,
|
||||
font: font,
|
||||
color: rgb(0, 0, 0),
|
||||
opacity: 0, // Make text invisible
|
||||
// Note: pdf-lib doesn't directly support render mode 3 (invisible),
|
||||
// but opacity: 0 achieves the same visual effect while keeping
|
||||
// the text selectable
|
||||
});
|
||||
// Stretch the text horizontally so that its selection matches the word on the image
|
||||
const hex = toUtf16Hex(toVisualOrder(text));
|
||||
const naturalWidth = (hex.length / 4) * fontSize * GLYPH_WIDTH / 1000;
|
||||
const horizontalScale = wordWidth > 0 ? wordWidth / naturalWidth * 100 : 100;
|
||||
|
||||
operators.push(
|
||||
setFontAndSize(fontName, fontSize),
|
||||
setCharacterSqueeze(horizontalScale),
|
||||
setTextMatrix(1, 0, 0, 1, pdfX, pdfY),
|
||||
showText(PDFHexString.of(hex)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
operators.push(endText());
|
||||
page.pushOperators(...operators);
|
||||
};
|
||||
|
||||
// Creates an accessible PDF by overlaying invisible text on top of page images.
|
||||
@@ -78,8 +206,7 @@ const createAccessiblePdf = async (
|
||||
|
||||
const pdfDoc = await PDFDocument.create();
|
||||
|
||||
// Embed a standard font for the invisible text layer
|
||||
const font = await pdfDoc.embedFont(StandardFonts.Helvetica);
|
||||
const fontRef = embedGlyphLessFont(pdfDoc);
|
||||
|
||||
for (let pageIndex = 0; pageIndex < pageImages.length; pageIndex++) {
|
||||
const pageImage = pageImages[pageIndex];
|
||||
@@ -104,7 +231,7 @@ const createAccessiblePdf = async (
|
||||
});
|
||||
|
||||
// Add invisible text layer on top
|
||||
addInvisibleTextLayer(page, pageOcr.lines, font, pageWidth, pageHeight);
|
||||
addInvisibleTextLayer(page, pageOcr.lines, fontRef, pageHeight);
|
||||
}
|
||||
|
||||
return pdfDoc.save();
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
// Minimal TrueType font generated with fontTools' FontBuilder. Same idea as Tesseract's GlyphLessFont,
|
||||
// used for invisible OCR text. Glyph 1 is a 500x1000 rectangle, with ascent 800 and descent -200.
|
||||
// It's never drawn (render mode 3), but macOS PDFKit (Preview) needs a non-zero descent and a glyph
|
||||
// outline to compute text bounds, otherwise the text can't be selected.
|
||||
const glyphLessFontBase64 = 'AAEAAAAKAIAAAwAgT1MvMkUAQ7AAAAEoAAAAYGNtYXAADABzAAABkAAAADRnbHlmA24B+AAAAcwAAAAaaGVhZC9AAeoAAACsAAAANmhoZWEFFgEuAAAA5AAAACRobXR4AfQAAAAAAYgAAAAGbG9jYQANAAAAAAHEAAAABm1heHAABAAGAAABCAAAACBuYW1lGZ8ZNAAAAegAAABycG9zdJ5/ds8AAAJcAAAALQABAAAAAQAAN7DqNF8PPPUAAwPoAAAAAOcaX1kAAAAA5xpfWQAA/zgB9AMgAAAAAwACAAAAAAAAAAEAAAMg/zgAAAH0AAAAAAH0AAEAAAAAAAAAAAAAAAAAAAABAAEAAAACAAQAAQAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAwH0AZAABQAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAPz8/PwAAACAAIAMg/zgAAAMgAMgAAAAAAAAAAAAAAAAAAAAgAAAB9AAAAAAAAAAAAAIAAAADAAAAFAADAAEAAAAUAAQAIAAAAAQABAABAAAAIP//AAAAIP///+EAAQAAAAAAAAAAAA0AAAABAAD/OAH0AyAAAwAAFREhEQH0yAPo/BgAAAAAAAAEADYAAQAAAAAAAQANAAAAAQAAAAAAAgAHAA0AAwABBAkAAQAaABQAAwABBAkAAgAOAC5HbHlwaExlc3NGb250UmVndWxhcgBHAGwAeQBwAGgATABlAHMAcwBGAG8AbgB0AFIAZQBnAHUAbABhAHIAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAECBmdseXBoMQAAAA==';
|
||||
|
||||
export default glyphLessFontBase64;
|
||||
@@ -10,7 +10,6 @@ import Logger from '@joplin/utils/Logger';
|
||||
import RepositoryApi from './RepositoryApi';
|
||||
import { produce } from 'immer';
|
||||
import { PluginManifest } from './utils/types';
|
||||
import { BlockedPluginVersions, isRepoManifestBlocked, isVersionBlocked } from './utils/blockedVersions';
|
||||
import isCompatible from './utils/isCompatible';
|
||||
import { AppType } from './api/types';
|
||||
import minVersionForPlatform from './utils/isCompatible/minVersionForPlatform';
|
||||
@@ -512,9 +511,7 @@ export default class PluginService extends BaseService {
|
||||
// Load only the manifest first to check if the plugin is
|
||||
// enabled before doing the expensive full load.
|
||||
let plugin = await this.loadPluginFromPath(pluginPath, true);
|
||||
const blocked = this.isBlocked(plugin.manifest);
|
||||
if (blocked) logger.warn(`Plugin version has been blocked: ${plugin.id}@${plugin.manifest.version}`);
|
||||
const enabled = this.pluginEnabled(settings, plugin.id) && !blocked;
|
||||
const enabled = this.pluginEnabled(settings, plugin.id);
|
||||
if (enabled) {
|
||||
logger.info(`Loading full plugin: ${plugin.id}`);
|
||||
plugin = await this.loadPluginFromPath(pluginPath, false);
|
||||
@@ -586,13 +583,8 @@ export default class PluginService extends BaseService {
|
||||
return shim.mobilePlatform() ? AppType.Mobile : AppType.Desktop;
|
||||
}
|
||||
|
||||
public isBlocked(manifest: PluginManifest) {
|
||||
const blockedVersions: BlockedPluginVersions = Setting.value('plugins.blockedVersions');
|
||||
return isRepoManifestBlocked(manifest) || isVersionBlocked(blockedVersions?.[manifest.id], manifest.version);
|
||||
}
|
||||
|
||||
public isCompatible(manifest: PluginManifest): boolean {
|
||||
return isCompatible(this.appVersion_, this.appType_, manifest) && !this.isBlocked(manifest);
|
||||
return isCompatible(this.appVersion_, this.appType_, manifest);
|
||||
}
|
||||
|
||||
private validateManifest(manifest: unknown): void {
|
||||
@@ -608,8 +600,6 @@ export default class PluginService extends BaseService {
|
||||
return _('Invalid plugin manifest: %s', message);
|
||||
}
|
||||
|
||||
if (this.isBlocked(manifest)) return _('This version of the plugin has been disabled by the Joplin team for security reasons.');
|
||||
|
||||
if (this.isCompatible(manifest)) return null;
|
||||
|
||||
const minVersion = minVersionForPlatform(this.appType_, manifest);
|
||||
|
||||
@@ -4,7 +4,6 @@ import { PluginManifest } from './utils/types';
|
||||
const md5 = require('md5');
|
||||
import { compareVersions } from 'compare-versions';
|
||||
import isCompatible from './utils/isCompatible';
|
||||
import { isRepoManifestBlocked } from './utils/blockedVersions';
|
||||
import { AppType } from '../../models/Setting';
|
||||
|
||||
const logger = Logger.create('RepositoryApi');
|
||||
@@ -93,14 +92,6 @@ export default class RepositoryApi {
|
||||
}
|
||||
|
||||
public async initialize() {
|
||||
await this.initializeManifestsOnly();
|
||||
await this.loadRelease();
|
||||
|
||||
this.lastInitializedTime_ = Date.now();
|
||||
}
|
||||
|
||||
// Skips the GitHub release API call, which is rate-limited and only needed to download plugins
|
||||
public async initializeManifestsOnly() {
|
||||
// https://github.com/joplin/plugins
|
||||
// https://api.github.com/repos/joplin/plugins/releases
|
||||
this.githubApiUrl_ = this.baseUrl_.replace(/^(https:\/\/)(github\.com\/)(.*)$/, '$1api.$2repos/$3');
|
||||
@@ -111,6 +102,9 @@ export default class RepositoryApi {
|
||||
this.isUsingDefaultContentUrl_ = this.contentBaseUrl_ === defaultContentBaseUrl;
|
||||
|
||||
await this.loadManifests();
|
||||
await this.loadRelease();
|
||||
|
||||
this.lastInitializedTime_ = Date.now();
|
||||
}
|
||||
|
||||
public async reinitialize() {
|
||||
@@ -251,7 +245,6 @@ export default class RepositoryApi {
|
||||
const manifest = manifests.find(m => m.id === pluginId);
|
||||
if (!manifest) throw new Error(`No manifest for plugin ID "${pluginId}"`);
|
||||
if (this.isBlockedByInstallMode(manifest)) throw new Error(`Plugin is blocked by intstallation policy. ID "${pluginId}"`);
|
||||
if (isRepoManifestBlocked(manifest)) throw new Error(`This version of the plugin has been blocked: ${pluginId}@${manifest.version}`);
|
||||
|
||||
const fileUrl = this.assetFileUrl(manifest.id); // this.repoFileUrl(`plugins/${manifest.id}/plugin.jpl`);
|
||||
const hash = md5(Date.now() + Math.random());
|
||||
@@ -291,7 +284,7 @@ export default class RepositoryApi {
|
||||
if (!manifest) return false;
|
||||
|
||||
const supportsCurrentAppVersion = compareVersions(installedVersion, manifest.version) < 0 && isCompatible(this.appVersion_, this.appType_, manifest);
|
||||
return supportsCurrentAppVersion && !this.isBlockedByInstallMode(manifest) && !isRepoManifestBlocked(manifest);
|
||||
return supportsCurrentAppVersion && !this.isBlockedByInstallMode(manifest);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -254,34 +254,4 @@ describe('loadPlugins', () => {
|
||||
// The bad plugin should not be running
|
||||
expect(failingPluginRunner.runningPluginIds).not.toContain(badPluginId);
|
||||
});
|
||||
|
||||
test('should stop and not run blocked plugin versions', async () => {
|
||||
const blockedId = 'joplin.test.plugin.blocked';
|
||||
const otherId = 'joplin.test.plugin.other';
|
||||
await createTestPlugin({ ...defaultManifestProperties, id: blockedId, name: 'Blocked' });
|
||||
await createTestPlugin({ ...defaultManifestProperties, id: otherId, name: 'Other' });
|
||||
|
||||
const pluginRunner = new MockPluginRunner();
|
||||
const loadPluginsOptions: LoadPluginsProps = {
|
||||
pluginRunner,
|
||||
pluginSettings: Setting.value('plugins.states'),
|
||||
platformImplementation,
|
||||
store: createMockReduxStore(),
|
||||
reloadAll: false,
|
||||
cancelEvent: { cancelled: false },
|
||||
};
|
||||
|
||||
await loadPlugins(loadPluginsOptions);
|
||||
await pluginRunner.waitForAllToBeRunning([blockedId, otherId]);
|
||||
|
||||
Setting.setValue('plugins.blockedVersions', { [blockedId]: [defaultManifestProperties.version] });
|
||||
await withWarningSilenced(/Plugin version has been blocked/, async () => {
|
||||
await loadPlugins(loadPluginsOptions);
|
||||
});
|
||||
|
||||
expect(pluginRunner.runningPluginIds).toEqual([otherId]);
|
||||
const blockedManifest = PluginService.instance().pluginById(blockedId).manifest;
|
||||
expect(PluginService.instance().isCompatible(blockedManifest)).toBe(false);
|
||||
expect(PluginService.instance().describeIncompatibility(blockedManifest)).toContain('security reasons');
|
||||
});
|
||||
});
|
||||
@@ -1,31 +0,0 @@
|
||||
import Setting from '../../models/Setting';
|
||||
import shim from '../../shim';
|
||||
import PluginService from './PluginService';
|
||||
import RepositoryApi from './RepositoryApi';
|
||||
import { blockedVersionsFromManifests, mergeBlockedVersions } from './utils/blockedVersions';
|
||||
|
||||
const hasUserInstalledPlugins = async () => {
|
||||
const pluginDir = Setting.value('pluginDir');
|
||||
if (!(await shim.fsDriver().exists(pluginDir))) return false;
|
||||
return (await shim.fsDriver().readDirStats(pluginDir)).length > 0;
|
||||
};
|
||||
|
||||
// Returns the IDs of the running plugins that are now blocked
|
||||
const refreshBlockedPluginVersions = async (repoApi: RepositoryApi) => {
|
||||
if (!(await hasUserInstalledPlugins())) return [];
|
||||
|
||||
await repoApi.initializeManifestsOnly();
|
||||
const current = Setting.value('plugins.blockedVersions') ?? {};
|
||||
let fetched = blockedVersionsFromManifests(await repoApi.manifests());
|
||||
if (!repoApi.isUsingDefaultContentUrl) fetched = mergeBlockedVersions(current, fetched);
|
||||
// Avoids a needless settings change, which on mobile reloads all plugins
|
||||
if (JSON.stringify(fetched) !== JSON.stringify(current)) Setting.setValue('plugins.blockedVersions', fetched);
|
||||
|
||||
const service = PluginService.instance();
|
||||
return service.pluginIds.filter(id => {
|
||||
const plugin = service.pluginById(id);
|
||||
return plugin.running && service.isBlocked(plugin.manifest);
|
||||
});
|
||||
};
|
||||
|
||||
export default refreshBlockedPluginVersions;
|
||||
@@ -1,49 +0,0 @@
|
||||
import { isValidBlockedVersionEntry, isVersionBlocked, mergeBlockedVersions } from './blockedVersions';
|
||||
|
||||
describe('blockedVersions', () => {
|
||||
|
||||
test.each([
|
||||
[['1.2.3'], '1.2.3', true],
|
||||
[['1.2.3'], '1.2.4', false],
|
||||
[['>=1.2.3'], '1.2.2', false],
|
||||
[['>=1.2.3'], '1.2.4', true],
|
||||
[['>=1.2.3'], '2.0.0', true],
|
||||
[['1.2.3', '1.2.4'], '1.2.5', false],
|
||||
[['*'], '0.0.1', true],
|
||||
[['not a version'], '1.2.3', true],
|
||||
[[], '1.2.3', false],
|
||||
[undefined, '1.2.3', false],
|
||||
])('should tell if a version is blocked (%j, %s)', (blockedVersions, version, expected) => {
|
||||
expect(isVersionBlocked(blockedVersions, version)).toBe(expected);
|
||||
});
|
||||
|
||||
test.each([
|
||||
['*', true],
|
||||
['1.2.3', true],
|
||||
['>=1.2.3', true],
|
||||
['>=1.2.3 <1.2.5', true],
|
||||
['1.2.3 - 1.2.4', true],
|
||||
['1.2.3 || >=2.0.0', true],
|
||||
['<1.0.0 || garbage', false],
|
||||
['>=1.2.3 garbage', false],
|
||||
['>=1.2.3\t<2.0.0', false],
|
||||
['1.2', false],
|
||||
['v1.2.3', false],
|
||||
['>=x', false],
|
||||
['', false],
|
||||
[123, false],
|
||||
])('should validate blocked version entries (%j)', (entry, expected) => {
|
||||
expect(isValidBlockedVersionEntry(entry)).toBe(expected);
|
||||
});
|
||||
|
||||
test('should only add blocks when merging', () => {
|
||||
const current = { 'plugin.a': ['>=1.2.3'], 'plugin.b': ['*'] };
|
||||
const fetched = { 'plugin.a': ['1.2.3', '1.2.4'], 'plugin.c': ['2.0.0'] };
|
||||
expect(mergeBlockedVersions(current, fetched)).toEqual({
|
||||
'plugin.a': ['>=1.2.3', '1.2.3', '1.2.4'],
|
||||
'plugin.b': ['*'],
|
||||
'plugin.c': ['2.0.0'],
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
@@ -1,54 +0,0 @@
|
||||
import { satisfies, validateStrict } from 'compare-versions';
|
||||
import { PluginManifest } from './types';
|
||||
|
||||
// Maps a plugin ID to its blocked versions. Each entry is "*", an exact version, or a semver range
|
||||
// such as ">=1.2.3", so that versions published by an attacker after the block are also covered.
|
||||
export type BlockedPluginVersions = Record<string, string[]>;
|
||||
|
||||
const entryMatches = (entry: string, version: string) => {
|
||||
if (entry === '*') return true;
|
||||
try {
|
||||
return satisfies(version, entry);
|
||||
} catch (error) {
|
||||
// Fail closed: a malformed entry still blocks the plugin
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
export const isVersionBlocked = (blockedVersions: string[] | undefined, version: string) => {
|
||||
if (!Array.isArray(blockedVersions)) return false;
|
||||
return blockedVersions.some(entry => entryMatches(entry, version));
|
||||
};
|
||||
|
||||
// Each comparator is checked on its own because satisfies() short-circuits, so "<1.0.0 || garbage"
|
||||
// would pass. This also rejects partial versions such as "1.2", which silently match nothing.
|
||||
// Only spaces are accepted as separators since satisfies() throws on tabs.
|
||||
export const isValidBlockedVersionEntry = (entry: unknown) => {
|
||||
if (typeof entry !== 'string') return false;
|
||||
if (entry === '*') return true;
|
||||
const comparators = entry.split('||').flatMap(part => part.trim().split(/ +- +| +/));
|
||||
return comparators.every(c => validateStrict(c.replace(/^(<=|>=|<|>|=|\^|~)/, '')));
|
||||
};
|
||||
|
||||
export const isRepoManifestBlocked = (manifest: PluginManifest) => {
|
||||
return isVersionBlocked(manifest._blocked_versions, manifest.version);
|
||||
};
|
||||
|
||||
// A mirror may serve a stale copy of the manifests, so it can add blocks but never lift them
|
||||
export const mergeBlockedVersions = (current: BlockedPluginVersions, fetched: BlockedPluginVersions) => {
|
||||
const output = { ...current };
|
||||
for (const [pluginId, versions] of Object.entries(fetched)) {
|
||||
output[pluginId] = [...new Set([...(output[pluginId] ?? []), ...versions])];
|
||||
}
|
||||
return output;
|
||||
};
|
||||
|
||||
export const blockedVersionsFromManifests = (manifests: PluginManifest[]) => {
|
||||
const output: BlockedPluginVersions = {};
|
||||
for (const manifest of manifests) {
|
||||
if (Array.isArray(manifest._blocked_versions) && manifest._blocked_versions.length) {
|
||||
output[manifest.id] = manifest._blocked_versions.filter(v => typeof v === 'string');
|
||||
}
|
||||
}
|
||||
return output;
|
||||
};
|
||||
@@ -40,5 +40,4 @@ export interface PluginManifest {
|
||||
_npm_package_name?: string;
|
||||
_obsolete?: boolean;
|
||||
_recommended?: boolean;
|
||||
_blocked_versions?: string[];
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { applyManifestOverrides, getObsoleteManifests, getSupersededPackages, ManifestOverrides, validateManifestOverrides } from './overrideUtils';
|
||||
import { applyManifestOverrides, getObsoleteManifests, getSupersededPackages, ManifestOverrides } from './overrideUtils';
|
||||
|
||||
describe('overrideUtils', () => {
|
||||
|
||||
@@ -104,23 +104,4 @@ describe('overrideUtils', () => {
|
||||
const supersededPackages = getSupersededPackages(manifestOverrides);
|
||||
expect(supersededPackages).toMatchObject(['joplin-plugin-benji-favorites']);
|
||||
});
|
||||
|
||||
test.each([
|
||||
[['>=1.2.3', '1.0.0'], true],
|
||||
[['1.2'], false],
|
||||
[[] as string[], false],
|
||||
['1.2.3', false],
|
||||
])('should validate blocked versions (%j)', (blockedVersions, isValid) => {
|
||||
const overrides = { 'org.joplinapp.plugins.ToggleSidebars': { _blocked_versions: blockedVersions } } as ManifestOverrides;
|
||||
if (isValid) {
|
||||
expect(() => validateManifestOverrides(overrides)).not.toThrow();
|
||||
} else {
|
||||
expect(() => validateManifestOverrides(overrides)).toThrow('Invalid _blocked_versions');
|
||||
}
|
||||
});
|
||||
|
||||
test('should not allow blocking a plugin that is not in the manifests', () => {
|
||||
const overrides: ManifestOverrides = { 'does.not.exist': { _blocked_versions: ['*'] } };
|
||||
expect(() => applyManifestOverrides({}, overrides)).toThrow('Cannot block plugin does.not.exist');
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,11 @@
|
||||
import * as path from 'path';
|
||||
import { readJsonFile } from './utils';
|
||||
import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
|
||||
import { isValidBlockedVersionEntry } from '@joplin/lib/services/plugins/utils/blockedVersions';
|
||||
|
||||
export interface ManifestOverride {
|
||||
_obsolete?: boolean;
|
||||
_recommended?: boolean;
|
||||
_superseded_package?: string;
|
||||
_blocked_versions?: string[];
|
||||
}
|
||||
|
||||
export type ManifestOverrides = Record<string, ManifestOverride>;
|
||||
@@ -18,10 +16,6 @@ export function applyManifestOverrides(manifests: PluginManifests, overrides: Ma
|
||||
const manifest = manifests[pluginId] as PluginManifest & Record<string, unknown>;
|
||||
|
||||
if (!manifest) {
|
||||
if (override._blocked_versions) {
|
||||
throw new Error(`Cannot block plugin ${pluginId}: it is not in manifests.json`);
|
||||
}
|
||||
|
||||
// If the manifest does not exist in the destination, it means the
|
||||
// plugin has been taken out, so the obsolete property should be set
|
||||
// to `true`. If not, it's an error.
|
||||
@@ -55,20 +49,8 @@ function pluginManifestOverridesPath(repoDir: string): string {
|
||||
return path.resolve(repoDir, 'manifestOverrides.json');
|
||||
}
|
||||
|
||||
export const validateManifestOverrides = (overrides: ManifestOverrides) => {
|
||||
for (const [pluginId, override] of Object.entries(overrides)) {
|
||||
const blockedVersions = override._blocked_versions;
|
||||
if (blockedVersions === undefined) continue;
|
||||
if (!Array.isArray(blockedVersions) || !blockedVersions.length || !blockedVersions.every(isValidBlockedVersionEntry)) {
|
||||
throw new Error(`Invalid _blocked_versions for plugin ${pluginId}: ${JSON.stringify(blockedVersions)}. It must be a non-empty array of "*", exact versions or semver ranges.`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export async function readManifestOverrides(repoDir: string): Promise<ManifestOverrides> {
|
||||
const overrides: ManifestOverrides = await readJsonFile(pluginManifestOverridesPath(repoDir), {});
|
||||
validateManifestOverrides(overrides);
|
||||
return overrides;
|
||||
return readJsonFile(pluginManifestOverridesPath(repoDir), {});
|
||||
}
|
||||
|
||||
export const getSupersededPackages = (manifestOverrides: ManifestOverrides): string[] => {
|
||||
|
||||
@@ -105,19 +105,6 @@ describe('validateUntrustedManifest', () => {
|
||||
).toThrow('mark itself as recommended');
|
||||
});
|
||||
|
||||
test('should not allow plugin authors to set their own blocked versions', () => {
|
||||
const newManifest = {
|
||||
id: 'joplin-plugin.this.is.another.test',
|
||||
_blocked_versions: [] as string[],
|
||||
_npm_package_name: 'joplin-plugin-another-test',
|
||||
version: '1.2.3',
|
||||
};
|
||||
|
||||
expect(
|
||||
() => validateUntrustedManifest(newManifest as unknown as PluginManifest, originalManifests, { source: 'npm' }),
|
||||
).toThrow('cannot set its own blocked versions');
|
||||
});
|
||||
|
||||
test('should not allow changing the NPM package for an existing plugin', () => {
|
||||
const newManifest = {
|
||||
id: 'joplin-plugin.this.is.a.test',
|
||||
|
||||
@@ -26,10 +26,6 @@ const validateUntrustedManifest = (
|
||||
throw new Error(`Plugin ${manifest.id} cannot mark itself as recommended.`);
|
||||
}
|
||||
|
||||
if (typeof manifest._blocked_versions !== 'undefined') {
|
||||
throw new Error(`Plugin ${manifest.id} cannot set its own blocked versions.`);
|
||||
}
|
||||
|
||||
if (options.source === 'repository') {
|
||||
if (!manifest.repository_url?.trim()) {
|
||||
throw new Error(`Plugin ${manifest.id} must specify repository_url when published from a repository.`);
|
||||
|
||||
@@ -632,6 +632,38 @@ describe('ItemModel', () => {
|
||||
expect((await models().userItem().byUserId(user1.id)).length).toBe(5);
|
||||
});
|
||||
|
||||
test('should process orphaned items mixed with valid items across batches', async () => {
|
||||
const { user: user1 } = await createUserAndSession(1);
|
||||
const { user: user2 } = await createUserAndSession(2);
|
||||
|
||||
await createItemTree3(user1.id, '', '', [
|
||||
{ id: '000000000000000000000000000000F1' },
|
||||
]);
|
||||
|
||||
await createItemTree3(user2.id, '', '', [
|
||||
{ id: '000000000000000000000000000000F2' },
|
||||
{ id: '000000000000000000000000000000F3' },
|
||||
{ id: '000000000000000000000000000000F4' },
|
||||
{ id: '000000000000000000000000000000F5' },
|
||||
]);
|
||||
|
||||
const f4 = await models().item().loadByJopId(user2.id, '000000000000000000000000000000F4');
|
||||
await db()('user_items').where('item_id', '=', f4.id).delete();
|
||||
await db()('users').where('id', '=', user1.id).delete();
|
||||
await db()('user_items').where('user_id', '=', user1.id).delete();
|
||||
|
||||
await models().item().processOrphanedItems({ batchSize: 2 });
|
||||
|
||||
const items = await models().item().all();
|
||||
expect(items.map(i => i.name).sort()).toEqual([
|
||||
'000000000000000000000000000000F2.md',
|
||||
'000000000000000000000000000000F3.md',
|
||||
'000000000000000000000000000000F4.md',
|
||||
'000000000000000000000000000000F5.md',
|
||||
]);
|
||||
expect((await models().userItem().byUserId(user2.id)).length).toBe(4);
|
||||
});
|
||||
|
||||
test('should return multiple item contents', async () => {
|
||||
const { user: user1 } = await createUserAndSession(1);
|
||||
|
||||
|
||||
@@ -1102,6 +1102,10 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
// then look up which of those ids exist in user_items. The orphan
|
||||
// filter happens in TS. Per-query cost is bounded by the window size,
|
||||
// not by the table size, so no statement can blow past the timeout.
|
||||
//
|
||||
// The window query only selects `id` so that it is an index-only scan.
|
||||
// IDs are random, so any other column costs one heap page read per row,
|
||||
// and walking the whole heap evicts the buffer cache used by the app.
|
||||
const batchSize = options.batchSize ?? 10000;
|
||||
let batchNum = 0;
|
||||
let totalProcessed = 0;
|
||||
@@ -1114,7 +1118,7 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
const batchStartTime = Date.now();
|
||||
|
||||
const windowItems: Item[] = await this.db(this.tableName)
|
||||
.select(['items.id', 'items.name', 'items.owner_id'])
|
||||
.select(['items.id'])
|
||||
.where('items.id', '>', lastId)
|
||||
.orderBy('items.id')
|
||||
.limit(batchSize);
|
||||
@@ -1133,14 +1137,18 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
.select('item_id')
|
||||
.whereIn('item_id', windowItemIds);
|
||||
const userItemIds = new Set(existingUserItems.map(u => u.item_id));
|
||||
const orphanedItems = windowItems.filter(i => !userItemIds.has(i.id));
|
||||
const orphanedItemIds = windowItemIds.filter(id => !userItemIds.has(id));
|
||||
|
||||
if (!orphanedItems.length) {
|
||||
if (!orphanedItemIds.length) {
|
||||
const batchDuration = Date.now() - batchStartTime;
|
||||
modelLogger.info(`processOrphanedItems: Batch ${batchNum} - No orphans in window of ${windowItems.length} items (${batchDuration}ms)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const orphanedItems: Item[] = await this.db(this.tableName)
|
||||
.select(['id', 'name', 'owner_id'])
|
||||
.whereIn('id', orphanedItemIds);
|
||||
|
||||
modelLogger.info(`processOrphanedItems: Batch ${batchNum} - Found ${orphanedItems.length} orphaned items`);
|
||||
for (const o of orphanedItems) {
|
||||
modelLogger.info(`OrphanTrace: found orphan item=${o.id} owner=${o.owner_id} name=${o.name}`);
|
||||
@@ -1172,7 +1180,7 @@ export default class ItemModel extends BaseModel<Item> {
|
||||
const restoreCount = Array.from(itemsToRestoreByUser.values()).reduce((sum, items) => sum + items.length, 0);
|
||||
modelLogger.info(`processOrphanedItems: Batch ${batchNum} - Restoring ${restoreCount} items for ${itemsToRestoreByUser.size} users`);
|
||||
for (const [userId, items] of itemsToRestoreByUser) {
|
||||
await this.models().userItem().addMulti(userId, items);
|
||||
await this.models().userItem().addMulti(userId, items, { ignoreAlreadyExists: true });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -87,6 +87,10 @@ describe('shares', () => {
|
||||
const result = await getApi<Share>('', `shares/${share.id}`);
|
||||
expect(result.id).toBe(share.id);
|
||||
expect(result.type).toBe(ShareType.PublishedFolder);
|
||||
expect(result.owner_id).toBeUndefined();
|
||||
|
||||
const ownerResult = await getApi<Share>(session.id, `shares/${share.id}`);
|
||||
expect(ownerResult.owner_id).toBe(share.owner_id);
|
||||
});
|
||||
|
||||
test('should not reveal whether an email has an account', async () => {
|
||||
|
||||
@@ -132,7 +132,10 @@ router.get('api/shares/:id', async (path: SubPath, ctx: AppContext) => {
|
||||
if (share && (share.type === ShareType.Note || share.type === ShareType.PublishedFolder)) {
|
||||
// No authentication is necessary - anyone who knows the share ID is allowed
|
||||
// to access the file. It is essentially public.
|
||||
return shareModel.toApiOutput(share);
|
||||
const output = await shareModel.toApiOutput(share) as Share;
|
||||
// Published links are indexed by search engines, and the owner's ID would let anyone target their account
|
||||
if (ctx.joplin.owner?.id !== share.owner_id) delete output.owner_id;
|
||||
return output;
|
||||
}
|
||||
|
||||
throw new ErrorNotFound();
|
||||
|
||||
@@ -222,6 +222,9 @@ describe('index/users', () => {
|
||||
// "confirm_email" is set to 0.
|
||||
user1 = await models().user().load(user1.id);
|
||||
expect(user1.email_confirmed).toBe(0);
|
||||
|
||||
// The token is still required
|
||||
await expectHttpError(() => execRequest('', 'GET', path, null, { query: { token: 'invalid', confirm_email: '0' } }), ErrorForbidden.httpCode);
|
||||
});
|
||||
|
||||
test('should allow user to verify their email', async () => {
|
||||
|
||||
@@ -166,7 +166,12 @@ router.get('users/:id/confirm', async (path: SubPath, ctx: AppContext, error: Er
|
||||
}
|
||||
};
|
||||
|
||||
if (ctx.query.confirm_email !== '0') await models.user().processEmailConfirmation(userId, token, beforeChangingEmailHandler);
|
||||
if (ctx.query.confirm_email !== '0') {
|
||||
await models.user().processEmailConfirmation(userId, token, beforeChangingEmailHandler);
|
||||
} else {
|
||||
// Otherwise this page would reveal whether an account exists, and the email of an account without a password
|
||||
await models.token().checkToken(userId, token, TokenPurpose.AccountConfirmation);
|
||||
}
|
||||
|
||||
const user = await models.user().load(userId);
|
||||
if (!user) throw new ErrorNotFound(`No such user: ${userId}`);
|
||||
|
||||
@@ -63,10 +63,12 @@ export default async function(env: Env, models: Models, config: Config, services
|
||||
run: (models: Models) => models.user().handleOversizedAccounts(),
|
||||
},
|
||||
|
||||
// Walks the whole items table, so it runs once a day rather than
|
||||
// hourly. Orphans are rare and clients work around them meanwhile.
|
||||
{
|
||||
id: TaskId.ProcessOrphanedItems,
|
||||
description: taskIdToLabel(TaskId.ProcessOrphanedItems),
|
||||
schedule: '15 * * * *',
|
||||
schedule: '15 4 * * *',
|
||||
run: (models: Models) => models.item().processOrphanedItems(),
|
||||
},
|
||||
|
||||
|
||||
@@ -378,3 +378,7 @@ instr
|
||||
mixmark
|
||||
Hrvatski
|
||||
Defense
|
||||
glyphless
|
||||
bfrange
|
||||
flate
|
||||
Thaana
|
||||
Reference in new issue
Block a user