Compare commits

...
24 changed files with 613 additions and 61 deletions

No files matched your search

+3
View File
@@ -1640,8 +1640,10 @@ packages/lib/services/ai/providers/JoplinCloud.js
packages/lib/services/ai/providers/OpenAiCompatible.test.js
packages/lib/services/ai/providers/OpenAiCompatible.js
packages/lib/services/ai/providers/TestProvider.js
packages/lib/services/ai/providers/openAiResponses.js
packages/lib/services/ai/testing/TestEmbeddingProvider.test.js
packages/lib/services/ai/testing/TestEmbeddingProvider.js
packages/lib/services/ai/testing/mockOpenAiServer.js
packages/lib/services/ai/tools/ToolIndex.js
packages/lib/services/ai/tools/buildEditorTools.js
packages/lib/services/ai/tools/global/createNote.js
@@ -1670,6 +1672,7 @@ packages/lib/services/ai/types.js
packages/lib/services/ai/utils/extractReasoning.test.js
packages/lib/services/ai/utils/extractReasoning.js
packages/lib/services/ai/utils/findFencedBlock.js
packages/lib/services/ai/utils/parseToolArguments.js
packages/lib/services/commands/MenuUtils.js
packages/lib/services/commands/ToolbarButtonUtils.test.js
packages/lib/services/commands/ToolbarButtonUtils.js
+3
View File
@@ -1666,8 +1666,10 @@ packages/lib/services/ai/providers/JoplinCloud.js
packages/lib/services/ai/providers/OpenAiCompatible.test.js
packages/lib/services/ai/providers/OpenAiCompatible.js
packages/lib/services/ai/providers/TestProvider.js
packages/lib/services/ai/providers/openAiResponses.js
packages/lib/services/ai/testing/TestEmbeddingProvider.test.js
packages/lib/services/ai/testing/TestEmbeddingProvider.js
packages/lib/services/ai/testing/mockOpenAiServer.js
packages/lib/services/ai/tools/ToolIndex.js
packages/lib/services/ai/tools/buildEditorTools.js
packages/lib/services/ai/tools/global/createNote.js
@@ -1696,6 +1698,7 @@ packages/lib/services/ai/types.js
packages/lib/services/ai/utils/extractReasoning.test.js
packages/lib/services/ai/utils/extractReasoning.js
packages/lib/services/ai/utils/findFencedBlock.js
packages/lib/services/ai/utils/parseToolArguments.js
packages/lib/services/commands/MenuUtils.js
packages/lib/services/commands/ToolbarButtonUtils.test.js
packages/lib/services/commands/ToolbarButtonUtils.js
@@ -13,6 +13,8 @@
<updated>20260914T100000Z</updated>
<note-attributes>
<application-data key="evernote.caption">Phase 1: Mobilize & Measure, which involves forming an advocacy group.</application-data>
<application-data key="evernote.caption">Caption truncated after Polars &</application-data>
<application-data key="evernote.captionSource">AI</application-data>
</note-attributes>
</note>
<note>
@@ -1,2 +1,2 @@
<img src/>
<img src/>
<img src=""/>
<img src=""/>
@@ -1,9 +1,9 @@
<span class="not-loaded-resource not-loaded-image-resource resource-status-notDownloaded" data-resource-id="a1test2a1test2a1test2a1test22345" data-original-alt data-original-title="test" contenteditable="false"><img src="data:image/svg+xml;utf8,
<span class="not-loaded-resource not-loaded-image-resource resource-status-notDownloaded" data-resource-id="a1test2a1test2a1test2a1test22345" data-original-alt="" data-original-title="test" contenteditable="false"><img src="data:image/svg+xml;utf8,
&Tab;&Tab;&lt;svg width=&quot;1700&quot; height=&quot;1536&quot; xmlns=&quot;http://www.w3.org/2000/svg&quot;&gt;
&Tab;&Tab; &lt;path d=&quot;M1280 1344c0-35-29-64-64-64s-64 29-64 64 29 64 64 64 64-29 64-64zm256 0c0-35-29-64-64-64s-64 29-64 64 29 64 64 64 64-29 64-64zm128-224v320c0 53-43 96-96 96H96c-53 0-96-43-96-96v-320c0-53 43-96 96-96h465l135 136c37 36 85 56 136 56s99-20 136-56l136-136h464c53 0 96 43 96 96zm-325-569c10 24 5 52-14 70l-448 448c-12 13-29 19-45 19s-33-6-45-19L339 621c-19-18-24-46-14-70 10-23 33-39 59-39h256V64c0-35 29-64 64-64h256c35 0 64 29 64 64v448h256c26 0 49 16 59 39z&quot;/&gt;
&Tab;&Tab;&lt;/svg&gt;
&Tab;"/></span>
<span class="not-loaded-resource not-loaded-image-resource resource-status-notDownloaded" data-resource-id="a1test2a1test2a1test2a1test22346" data-original-alt="test" data-original-title contenteditable="false"><img src="data:image/svg+xml;utf8,
<span class="not-loaded-resource not-loaded-image-resource resource-status-notDownloaded" data-resource-id="a1test2a1test2a1test2a1test22346" data-original-alt="test" data-original-title="" contenteditable="false"><img src="data:image/svg+xml;utf8,
&Tab;&Tab;&lt;svg width=&quot;1700&quot; height=&quot;1536&quot; xmlns=&quot;http://www.w3.org/2000/svg&quot;&gt;
&Tab;&Tab; &lt;path d=&quot;M1280 1344c0-35-29-64-64-64s-64 29-64 64 29 64 64 64 64-29 64-64zm256 0c0-35-29-64-64-64s-64 29-64 64 29 64 64 64 64-29 64-64zm128-224v320c0 53-43 96-96 96H96c-53 0-96-43-96-96v-320c0-53 43-96 96-96h465l135 136c37 36 85 56 136 56s99-20 136-56l136-136h464c53 0 96 43 96 96zm-325-569c10 24 5 52-14 70l-448 448c-12 13-29 19-45 19s-33-6-45-19L339 621c-19-18-24-46-14-70 10-23 33-39 59-39h256V64c0-35 29-64 64-64h256c35 0 64 29 64 64v448h256c26 0 49 16 59 39z&quot;/&gt;
&Tab;&Tab;&lt;/svg&gt;
@@ -1,2 +1,2 @@
<img src class="jop-noMdConv"/>
<img src class="jop-noMdConv"/>
<img src="" class="jop-noMdConv"/>
<img src="" class="jop-noMdConv"/>
@@ -0,0 +1 @@
<div class="jop-noMdConv"><img src="x" a="" class="jop-noMdConv"/></div>
@@ -0,0 +1 @@
<div><img src="x" a="" =b="x onerror=alert(1) c"></div>
+7
View File
@@ -1494,6 +1494,13 @@
parser.state = returnState
} else if (isMatch(parser.entity.length ? entityBody : entityStart, c)) {
parser.entity += c
} else if (c === '<' && parser.state === S.TEXT_ENTITY) {
// Joplin: Treat "<" as the start of a tag, otherwise the tag would be swallowed as text
strictFail(parser, 'Invalid character in entity name')
parser[buffer] += '&' + parser.entity
parser.entity = ''
parser.state = S.OPEN_WAKA
parser.startTagPosition = parser.position
} else {
strictFail(parser, 'Invalid character in entity name')
parser[buffer] += '&' + parser.entity + c
+1 -1
View File
@@ -40,7 +40,7 @@ describe('renderMarkup', () => {
const renderedNote = await await command.execute(null, MarkupLanguage.Markdown, noteBody);
expect(renderedNote.html).toContain(`<div id="rendered-md"><p><img data-from-md data-resource-id="${resource.id}" src="`);
expect(renderedNote.html).toContain(`/resources-1/${resource.id}.jpg?t=`);
expect(renderedNote.html).toContain('" title alt="photo.jpg" /></p>');
expect(renderedNote.html).toContain('" title="" alt="photo.jpg" /></p>');
}
});
@@ -6,6 +6,7 @@ import Note from '../../models/Note';
import Tag from '../../models/Tag';
import { NoteEntity } from '../database/types';
import { NoteContext, ToolDefinition } from './tools/types';
import mockOpenAiServer, { jsonResponse } from './testing/mockOpenAiServer';
const runChatForNote = (note: NoteEntity, history: ChatMessage[], message: string) => {
const getContext = async () => {
@@ -117,4 +118,30 @@ describe('noteChat.tools', () => {
expect(infoMessage).toBeTruthy();
expect(infoMessage.content).toContain('Tool `manage_tags` is disabled in Joplin\'s settings');
});
test('should keep the model reasoning between the requests of a tool loop', async () => {
const baseUrl = 'http://localhost:1234/v1';
Setting.setValue('ai.chat.providerType', 'openai-compatible');
Setting.setValue('ai.chat.baseUrl', baseUrl);
Setting.setValue('ai.chat.model', 'test-model');
const reasoningItem = { type: 'reasoning', id: 'rs_1', summary: [{ type: 'summary_text', text: 'Thinking...' }], encrypted_content: 'encrypted' };
const functionCallItem = { type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'editor_readNoteBody', arguments: '{}' };
const requests = mockOpenAiServer(baseUrl, {
'/responses': () => {
const output = requests.length === 1 ? [reasoningItem, functionCallItem] : [{ type: 'message', content: [{ type: 'output_text', text: 'Done' }] }];
return jsonResponse({ status: 'completed', output });
},
});
try {
const note = await Note.save({ title: 'test' });
await runChatForNote(note, [{ role: ChatRole.User, content: 'testing' }], 'Read the note');
} finally {
jest.restoreAllMocks();
}
expect(requests).toHaveLength(2);
expect(requests[1].body.input).toEqual(expect.arrayContaining([reasoningItem, functionCallItem]));
});
});
+1
View File
@@ -203,6 +203,7 @@ const stepNoteChat = async ({
role: ChatRole.Assistant,
content: chatResult.text ?? '',
toolCalls: allowTools ? chatResult.toolCalls : [],
responsesState: allowTools ? chatResult.responsesState : undefined,
});
onHistoryChanged([...messages]);
@@ -85,6 +85,22 @@ describe('ai/providers/JoplinCloud', () => {
expect(error.message).toMatch(/reached your AI usage budget/i);
});
it('should only use the Chat Completions API', async () => {
const urls: string[] = [];
const mock = mockFetch((request) => {
urls.push(request.url);
return new Response(request.url.endsWith('ai/chat/completions') ? '{ "choices": [] }' : '{ }', { status: 200 });
});
try {
await new JoplinCloudProvider().chat([]);
} finally {
mock.reset();
}
expect(urls.filter(url => url.endsWith('ai/chat/completions'))).toHaveLength(1);
expect(urls.filter(url => url.includes('responses'))).toHaveLength(0);
});
it('should retry rate-limited requests', async () => {
const { mock, requestCount } = mockRateLimitedChatApi();
try {
@@ -87,6 +87,10 @@ export default class JoplinCloudProvider extends OpenAiCompatibleProvider {
});
}
protected override canTryResponsesApi() {
return false;
}
private async api() {
// Fetch the API from the registry, rather than caching the API locally.
// This avoids authentication drift if the user changes the sync target credentials.
@@ -1,21 +1,51 @@
import { mockFetch, setupDatabaseAndSynchronizer, switchClient } from '../../../testing/test-utils';
import { setupDatabaseAndSynchronizer, switchClient } from '../../../testing/test-utils';
import OpenAiCompatibleProvider from './OpenAiCompatible';
import { ChatRole } from '../types';
import { ChatMessage, ChatResult, ChatRole } from '../types';
import { ToolImageResponse } from '../tools/types';
import mockOpenAiServer, { jsonResponse } from '../testing/mockOpenAiServer';
const baseUrl = 'http://localhost:1234/v1';
const mockChatResponse = (message: Record<string, unknown>, finishReason: string|undefined) => {
return mockFetch(() => {
return new Response(JSON.stringify({
return mockOpenAiServer(baseUrl, {
'/chat/completions': () => jsonResponse({
choices: [{ message, finish_reason: finishReason }],
usage: { prompt_tokens: 10, completion_tokens: 20 },
}), { status: 200 });
}),
});
};
const newProvider = () => {
const responsesBody = (output: unknown[], extra: Record<string, unknown> = {}) => {
return {
status: 'completed',
output,
usage: { input_tokens: 10, output_tokens: 20 },
...extra,
};
};
const mockResponsesOutput = (output: unknown[], extra: Record<string, unknown> = {}) => {
return mockOpenAiServer(baseUrl, {
'/responses': () => jsonResponse(responsesBody(output, extra)),
});
};
const messageItem = (text: string) => ({ type: 'message', role: 'assistant', content: [{ type: 'output_text', text }] });
const reasoningItem = { type: 'reasoning', id: 'rs_1', summary: [{ type: 'summary_text', text: 'Thinking...' }], encrypted_content: 'encrypted' };
const reasoningContentItem = { type: 'reasoning', id: 'rs_1', summary: [] as unknown[], content: [{ type: 'reasoning_text', text: 'Thinking...' }] };
const functionCallItem = { type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'get_weather', arguments: '{"city":"Paris"}' };
const weatherTool = {
id: 'get_weather',
description: 'Gets the weather',
inputSchema: { type: 'object', properties: { city: { type: 'string' } } },
};
const newProvider = (model = 'test-model', apiKey = 'test-key') => {
return new OpenAiCompatibleProvider({
baseUrl: 'http://localhost:1234/v1',
apiKey: 'test-key',
model: 'test-model',
baseUrl,
apiKey,
model,
classification: 'local',
});
};
@@ -24,6 +54,22 @@ const chat = async () => {
return newProvider().chat([{ role: ChatRole.User, content: 'Hello' }]);
};
const toolLoopMessages = (firstResult: ChatResult, toolOutput: ChatMessage['content']): ChatMessage[] => {
return [
{ role: ChatRole.User, content: 'Weather in Paris?' },
{ role: ChatRole.Assistant, content: firstResult.text, toolCalls: firstResult.toolCalls, responsesState: firstResult.responsesState },
{
role: ChatRole.Tool,
content: toolOutput,
toolName: 'get_weather',
toolCallId: 'call_1',
isError: false,
userDescription: '',
isEdit: false,
},
];
};
describe('ai/providers/OpenAiCompatible', () => {
beforeEach(async () => {
@@ -31,6 +77,10 @@ describe('ai/providers/OpenAiCompatible', () => {
await switchClient(0);
});
afterEach(() => {
jest.restoreAllMocks();
});
test.each([
['length', 'length'],
['max_tokens', 'length'],
@@ -61,4 +111,188 @@ describe('ai/providers/OpenAiCompatible', () => {
expect(result.reasoningText).toBeUndefined();
});
it('should use the Responses API when the server supports it', async () => {
const requests = mockResponsesOutput([reasoningItem, messageItem('Hi!')]);
const provider = newProvider();
await provider.chat([{ role: ChatRole.User, content: 'Hello' }]);
const result = await provider.chat([{ role: ChatRole.User, content: 'Hello' }]);
expect(requests.map(request => request.path)).toEqual(['/responses', '/responses']);
expect(result).toMatchObject({
text: 'Hi!',
toolCalls: [],
usage: { inputTokens: 10, outputTokens: 20 },
finishReason: 'stop',
reasoningText: 'Thinking...',
responsesState: undefined,
});
});
test.each([
['content', [reasoningContentItem, messageItem('Hi!')], 'Hi!'],
['content without a message', [reasoningContentItem], ''],
])('should read the reasoning text from %s', async (_label, output, expectedText) => {
mockResponsesOutput(output);
const result = await chat();
expect(result.text).toBe(expectedText);
expect(result.reasoningText).toBe('Thinking...');
});
test.each([
['returns 404', () => new Response('404 page not found', { status: 404 })],
// Older LM Studio versions answer unknown endpoints like this
['returns 200 without output', () => jsonResponse({ error: 'Unexpected endpoint or method. (POST /v1/responses)' })],
])('should fall back to Chat Completions once when the Responses API %s', async (_label, responsesHandler) => {
const requests = mockOpenAiServer(baseUrl, {
'/responses': responsesHandler,
'/chat/completions': () => jsonResponse({ choices: [{ message: { content: 'OK' }, finish_reason: 'stop' }] }),
});
const provider = newProvider();
await provider.chat([{ role: ChatRole.User, content: 'Hello' }]);
const result = await provider.chat([{ role: ChatRole.User, content: 'Hello' }]);
expect(requests.map(request => request.path)).toEqual(['/responses', '/chat/completions', '/chat/completions']);
expect(result.text).toBe('OK');
});
it('should try the Responses API again when the fallback also fails', async () => {
let modelInstalled = false;
const modelNotFound = () => jsonResponse({ error: { message: 'model \'test-model\' not found' } }, 404);
const requests = mockOpenAiServer(baseUrl, {
'/responses': () => modelInstalled ? jsonResponse(responsesBody([messageItem('Hi!')])) : modelNotFound(),
'/chat/completions': modelNotFound,
});
const provider = newProvider();
await expect(provider.chat([{ role: ChatRole.User, content: 'Hello' }])).rejects.toThrow('AI provider returned 404: model \'test-model\' not found');
modelInstalled = true;
const result = await provider.chat([{ role: ChatRole.User, content: 'Hello' }]);
expect(requests.map(request => request.path)).toEqual(['/responses', '/chat/completions', '/responses']);
expect(result.text).toBe('Hi!');
});
test.each([400, 401])('should not fall back to Chat Completions on error %s', async (status) => {
const requests = mockOpenAiServer(baseUrl, {
'/responses': () => jsonResponse({ error: { message: 'Something failed' } }, status),
'/chat/completions': () => jsonResponse({ choices: [{ message: { content: 'OK' } }] }),
});
await expect(chat()).rejects.toThrow(`AI provider returned ${status}: Something failed`);
expect(requests.map(request => request.path)).toEqual(['/responses']);
});
it('should not fall back to Chat Completions after the Responses API has worked', async () => {
let responsesCallCount = 0;
const requests = mockOpenAiServer(baseUrl, {
'/responses': () => {
responsesCallCount++;
return responsesCallCount === 1 ? jsonResponse(responsesBody([messageItem('Hi!')])) : new Response('', { status: 404 });
},
'/chat/completions': () => jsonResponse({ choices: [{ message: { content: 'OK' } }] }),
});
const provider = newProvider();
await provider.chat([{ role: ChatRole.User, content: 'Hello' }]);
await expect(provider.chat([{ role: ChatRole.User, content: 'Hello' }])).rejects.toThrow('AI provider returned 404');
expect(requests.map(request => request.path)).toEqual(['/responses', '/responses']);
});
it('should map the request options to the Responses API', async () => {
const requests = mockResponsesOutput([messageItem('{"answer":"Paris"}')]);
await newProvider().chat([
{ role: ChatRole.System, content: 'Be brief' },
{ role: ChatRole.User, content: 'Capital of France?' },
], {
maxTokens: 100,
tools: [weatherTool],
responseFormat: { type: 'json_schema', json_schema: { name: 'answer', strict: true, schema: { type: 'object' } } },
});
expect(requests[0].body).toEqual({
model: 'test-model',
input: [
{ role: 'system', content: 'Be brief' },
{ role: 'user', content: 'Capital of France?' },
],
stream: false,
store: false,
include: ['reasoning.encrypted_content'],
max_output_tokens: 100,
text: { format: { type: 'json_schema', name: 'answer', strict: true, schema: { type: 'object' } } },
tools: [{ type: 'function', name: 'get_weather', description: 'Gets the weather', parameters: weatherTool.inputSchema }],
});
});
it('should retry without the schema when the model rejects structured output', async () => {
const requests = mockOpenAiServer(baseUrl, {
'/responses': body => body.text
? jsonResponse({ error: { message: 'Invalid parameter: \'text.format\' of type \'json_schema\' is not supported with this model.' } }, 400)
: jsonResponse(responsesBody([messageItem('Hi!')])),
});
const result = await newProvider().chat([{ role: ChatRole.User, content: 'Hello' }], {
responseFormat: { type: 'json_schema', json_schema: { name: 'answer', strict: true, schema: { type: 'object' } } },
});
expect(requests.map(request => request.path)).toEqual(['/responses', '/responses']);
expect(requests[1].body.text).toBeUndefined();
expect(result.text).toBe('Hi!');
});
test.each([
[{ status: 'completed' }, 'stop'],
[{ status: 'incomplete', incomplete_details: { reason: 'max_output_tokens' } }, 'length'],
[{ status: 'incomplete', incomplete_details: { reason: 'content_filter' } }, 'other'],
])('should map Responses API status %j to finish reason %s', async (status, expected) => {
mockResponsesOutput([messageItem('Hi!')], status);
const result = await chat();
expect(result.finishReason).toBe(expected);
});
it('should send the output items back in their original order to the same model and API key only', async () => {
const textItem = messageItem('Let me check.');
const requests = mockResponsesOutput([reasoningItem, textItem, functionCallItem]);
const provider = newProvider();
const firstResult = await provider.chat([{ role: ChatRole.User, content: 'Weather in Paris?' }], { tools: [weatherTool] });
expect(firstResult.toolCalls).toEqual([{ toolName: 'get_weather', callId: 'call_1', arguments: { city: 'Paris' }, parseError: null }]);
expect(firstResult.finishReason).toBe('tool_calls');
const messages = toolLoopMessages(firstResult, 'Sunny');
await provider.chat(messages, { tools: [weatherTool] });
const toolOutputItem = { type: 'function_call_output', call_id: 'call_1', output: 'Sunny' };
expect(requests[1].body.input).toEqual([
{ role: 'user', content: 'Weather in Paris?' },
reasoningItem,
textItem,
functionCallItem,
toolOutputItem,
]);
await newProvider('other-model').chat(messages, { tools: [weatherTool] });
await newProvider('test-model', 'other-key').chat(messages, { tools: [weatherTool] });
expect(requests).toHaveLength(4);
for (const request of requests.slice(2)) {
expect(request.body.input).toEqual([
{ role: 'user', content: 'Weather in Paris?' },
{ role: 'assistant', content: 'Let me check.' },
{ type: 'function_call', call_id: 'call_1', name: 'get_weather', arguments: '{"city":"Paris"}' },
toolOutputItem,
]);
}
});
it('should send image tool results as images', async () => {
const requests = mockResponsesOutput([functionCallItem]);
const provider = newProvider();
const firstResult = await provider.chat([{ role: ChatRole.User, content: 'Weather in Paris?' }], { tools: [weatherTool] });
const dataUrl = 'data:image/png;base64,AAAA';
const image = new ToolImageResponse({ dataUrl, mimeType: 'image/png', id: '0123456789abcdef0123456789abcdef' });
await provider.chat(toolLoopMessages(firstResult, image), { tools: [weatherTool] });
expect(requests[1].body.input).toContainEqual({
type: 'function_call_output',
call_id: 'call_1',
output: [{ type: 'input_image', image_url: dataUrl }],
});
});
});
@@ -6,6 +6,9 @@ import { ChatFinishReason, ChatMessage, ChatOptions, ChatResult, ChatToolCall, P
import ChatProviderBase from './ChatProviderBase';
import { ToolSpec } from '../tools/types';
import extractReasoning from '../utils/extractReasoning';
import parseToolArguments from '../utils/parseToolArguments';
import { fromResponsesJson, ResponsesApiResponse, toResponsesBody } from './openAiResponses';
const md5 = require('md5');
const logger = Logger.create('OpenAiCompatibleProvider');
@@ -118,19 +121,6 @@ const convertMessage = (message: ChatMessage) => {
}
};
const describeJsonParseFailure = (rawContent: string) => {
const parseError = ['Failed to parse JSON.'];
// With certain providers (e.g. Joplin Cloud), long messages are truncated. Include this information in the error message so that
// the model knows to retry with a shorter message:
const suggestedRetryLengthLimit = 1000;
if (rawContent.startsWith('{') && rawContent.length > suggestedRetryLengthLimit) {
parseError.push(`It's likely that the tool call JSON is too long. Please try again with a message shorter than ${suggestedRetryLengthLimit} characters.`);
}
return parseError.join(' ');
};
export interface ChatRequestOptions {
signal?: AbortSignal;
}
@@ -140,6 +130,8 @@ export interface OpenAiChatResponse {
json: OpenAiResponse;
}
type ChatApi = 'responses' | 'chat-completions';
export default class OpenAiCompatibleProvider extends ChatProviderBase {
public id = 'openai-compatible';
@@ -147,6 +139,7 @@ export default class OpenAiCompatibleProvider extends ChatProviderBase {
private baseUrl_: string;
private apiKey_: string;
private model_: string;
private api_: ChatApi|null = null;
public constructor(options: Options) {
super();
@@ -156,9 +149,72 @@ export default class OpenAiCompatibleProvider extends ChatProviderBase {
this.classification = options.classification;
}
protected canTryResponsesApi() {
return true;
}
protected async doChat(messages: ChatMessage[], options?: ChatOptions): Promise<ChatResult> {
if (!this.model_) throw new JoplinError('OpenAI-compatible provider has no model configured', 'aiProviderNotConfigured');
if (this.canTryResponsesApi() && this.api_ !== 'chat-completions') {
const result = await this.doResponsesChat(messages, options);
if (result) {
this.api_ = 'responses';
return result;
}
}
const chatCompletionsResult = await this.doChatCompletions(messages, options);
// Some servers like Ollama also return 404 when the model is not found so we only
// remember the fallback after /chat/completions works
this.api_ = 'chat-completions';
return chatCompletionsResult;
}
private async doResponsesChat(messages: ChatMessage[], options?: ChatOptions): Promise<ChatResult|null> {
// Encrypted reasoning only works with the account that created it. The key is
// hashed because the replay key is saved with the chat history
const replayKey = `${this.baseUrl_}|${this.model_}|${md5(this.apiKey_)}`;
const body = toResponsesBody(this.model_, messages, options, replayKey);
let { status, json } = await this.sendResponsesRequest(body, { signal: options?.signal });
if (status === 404 && this.api_ === null) {
logger.info(`${this.baseUrl_}/responses returned 404. Using /chat/completions instead.`);
return null;
}
// Same compatibility retry as `response_format` in doChatCompletions
if (status === 400 && 'text' in body && /json_schema|response_format/i.test(json?.error?.message ?? '')) {
logger.warn(`Model ${this.model_} rejected text.format; retrying without structured output schema.`);
delete body.text;
({ status, json } = await this.sendResponsesRequest(body, { signal: options?.signal }));
}
if (status >= 400) {
const detail = json?.error?.message ? `: ${json.error.message}` : '';
throw new JoplinError(`AI provider returned ${status}${detail}`, status);
}
if (!Array.isArray(json?.output)) {
// Older LM Studio versions return 200 with an error body for unknown endpoints
if (this.api_ === null) {
logger.info(`${this.baseUrl_}/responses returned ${status} with no output. Using /chat/completions instead.`);
return null;
}
throw this.badResponseShapeError();
}
return fromResponsesJson(json, replayKey);
}
private badResponseShapeError() {
return new JoplinError(
`AI provider returned an unexpected response shape. The base URL is likely wrong — for OpenAI, Ollama, and LM Studio the URL must end with "/v1" (got ${this.baseUrl_}).`,
'aiProviderBadResponse',
);
}
private async doChatCompletions(messages: ChatMessage[], options?: ChatOptions): Promise<ChatResult> {
const body: Record<string, unknown> = {
model: this.model_,
messages: messages.flatMap((message): unknown[] => convertMessage(message)),
@@ -210,12 +266,7 @@ export default class OpenAiCompatibleProvider extends ChatProviderBase {
// is wrong (e.g. user forgot the `/v1` suffix on a local server) — many
// such servers reply 200 with an empty body rather than 404. Surface
// this rather than returning an empty string the caller can't diagnose.
if (!Array.isArray(json.choices)) {
throw new JoplinError(
`AI provider returned an unexpected response shape. The base URL is likely wrong — for OpenAI, Ollama, and LM Studio the URL must end with "/v1" (got ${this.baseUrl_}).`,
'aiProviderBadResponse',
);
}
if (!Array.isArray(json.choices)) throw this.badResponseShapeError();
const choice = json.choices?.[0];
const responseMessage = choice?.message;
@@ -229,16 +280,7 @@ export default class OpenAiCompatibleProvider extends ChatProviderBase {
const toolCalls: ChatToolCall[] = (responseMessage?.tool_calls ?? []).map(call => {
if (!call.function) return null;
let args;
let parseError: string|null = null;
const argumentString = call.function.arguments;
try {
args = JSON.parse(argumentString);
} catch (error) {
args = {};
parseError = describeJsonParseFailure(argumentString);
logger.error('JSON parse failed', error, parseError);
}
const { args, parseError } = parseToolArguments(call.function.arguments);
return {
toolName: call.function.name,
@@ -257,29 +299,44 @@ export default class OpenAiCompatibleProvider extends ChatProviderBase {
};
}
protected async sendChatRequest(body: Record<string, unknown>, options: ChatRequestOptions): Promise<OpenAiChatResponse> {
private async post(path: string, body: Record<string, unknown>, options: ChatRequestOptions) {
if (!this.baseUrl_) throw new JoplinError('OpenAI-compatible provider has no base URL configured', 'aiProviderNotConfigured');
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (this.apiKey_) headers['Authorization'] = `Bearer ${this.apiKey_}`;
const response = await shim.fetch(`${this.baseUrl_}/chat/completions`, {
const response = await shim.fetch(`${this.baseUrl_}${path}`, {
method: 'POST',
headers,
body: JSON.stringify(body),
signal: options.signal,
});
const text = await response.text();
return { status: response.status, text: await response.text() };
}
protected async sendChatRequest(body: Record<string, unknown>, options: ChatRequestOptions): Promise<OpenAiChatResponse> {
const { status, text } = await this.post('/chat/completions', body, options);
let json: OpenAiResponse;
try {
json = JSON.parse(text) as OpenAiResponse;
} catch {
throw new JoplinError(`AI provider returned non-JSON response: ${text.slice(0, 200)}`, response.status);
throw new JoplinError(`AI provider returned non-JSON response: ${text.slice(0, 200)}`, status);
}
return {
response: { status: response.status },
response: { status },
json,
};
}
private async sendResponsesRequest(body: Record<string, unknown>, options: ChatRequestOptions) {
const { status, text } = await this.post('/responses', body, options);
let json: ResponsesApiResponse|null = null;
try {
json = JSON.parse(text) as ResponsesApiResponse;
} catch {
if (status !== 404) throw new JoplinError(`AI provider returned non-JSON response: ${text.slice(0, 200)}`, status);
}
return { status, json };
}
}
@@ -0,0 +1,123 @@
import { ChatFinishReason, ChatMessage, ChatOptions, ChatResult, ChatRole, ChatToolCall } from '../types';
import { ToolSpec } from '../tools/types';
import extractReasoning from '../utils/extractReasoning';
import parseToolArguments from '../utils/parseToolArguments';
interface ResponsesContentPart {
type: string;
text?: string;
}
interface ResponsesOutputItem {
type: string;
content?: ResponsesContentPart[];
summary?: ResponsesContentPart[];
call_id?: string;
name?: string;
arguments?: string;
}
export interface ResponsesApiResponse {
output?: ResponsesOutputItem[];
status?: string;
incomplete_details?: { reason?: string }|null;
usage?: { input_tokens?: number; output_tokens?: number };
error?: { message?: string }|null;
}
const convertTool = (tool: ToolSpec) => {
return {
type: 'function',
name: tool.id,
description: tool.description,
parameters: tool.inputSchema,
};
};
const convertMessage = (message: ChatMessage, replayKey: string): unknown[] => {
if (message.role === ChatRole.Tool) {
const content = message.content;
return [{
type: 'function_call_output',
call_id: message.toolCallId,
output: typeof content === 'string' ? content : [{ type: 'input_image', image_url: content.dataUrl }],
}];
}
if (message.responsesState?.key === replayKey) {
return message.responsesState.items;
}
const textItems = message.content || message.role !== ChatRole.Assistant ? [{ role: message.role, content: message.content }] : [];
return [
...textItems,
...(message.toolCalls ?? []).map(call => ({
type: 'function_call',
call_id: call.callId,
name: call.toolName,
arguments: JSON.stringify(call.arguments),
})),
];
};
export const toResponsesBody = (model: string, messages: ChatMessage[], options: ChatOptions|undefined, replayKey: string) => {
const body: Record<string, unknown> = {
model,
input: messages.flatMap(message => convertMessage(message, replayKey)),
stream: false,
store: false,
include: ['reasoning.encrypted_content'],
};
if (options?.temperature !== undefined) body.temperature = options.temperature;
if (options?.maxTokens !== undefined) body.max_output_tokens = options.maxTokens;
if (options?.responseFormat !== undefined) body.text = { format: { type: 'json_schema', ...options.responseFormat.json_schema } };
if (options?.tools?.length) body.tools = options.tools.map(convertTool);
return body;
};
const toChatFinishReason = (json: ResponsesApiResponse, hasToolCalls: boolean): ChatFinishReason|undefined => {
if (json.status === 'incomplete') return json.incomplete_details?.reason === 'max_output_tokens' ? 'length' : 'other';
if (hasToolCalls) return 'tool_calls';
if (json.status === 'completed') return 'stop';
return json.status ? 'other' : undefined;
};
const partsText = (parts: ResponsesContentPart[], type: string) => {
return parts.filter(part => part.type === type).map(part => part.text ?? '');
};
export const fromResponsesJson = (json: ResponsesApiResponse, replayKey: string): ChatResult => {
const output = json.output ?? [];
const rawText = partsText(output.filter(item => item.type === 'message').flatMap(item => item.content ?? []), 'output_text').join('');
const { text, reasoning: inlineReasoning } = extractReasoning(rawText);
const reasoningItems = output.filter(item => item.type === 'reasoning');
const summaryText = partsText(reasoningItems.flatMap(item => item.summary ?? []), 'summary_text').join('\n\n');
// LM Studio sends the reasoning in `content` instead of `summary`
const reasoningContentText = partsText(reasoningItems.flatMap(item => item.content ?? []), 'reasoning_text').join('\n\n');
const toolCalls: ChatToolCall[] = output.filter(item => item.type === 'function_call').map(call => {
const { args, parseError } = parseToolArguments(call.arguments ?? '');
return {
toolName: call.name,
callId: call.call_id,
arguments: args,
parseError,
};
});
const replayItems = output.filter(item => item.type === 'reasoning' || item.type === 'message' || item.type === 'function_call');
return {
text,
toolCalls,
usage: {
inputTokens: json.usage?.input_tokens ?? 0,
outputTokens: json.usage?.output_tokens ?? 0,
},
finishReason: toChatFinishReason(json, toolCalls.length > 0),
reasoningText: summaryText || reasoningContentText || inlineReasoning || undefined,
responsesState: toolCalls.length ? { key: replayKey, items: replayItems } : undefined,
};
};
@@ -0,0 +1,27 @@
import shim from '../../../shim';
type RequestBody = Record<string, unknown>;
type RequestHandler = (body: RequestBody)=> Response;
export interface MockServerRequest {
path: string;
body: RequestBody;
}
export const jsonResponse = (json: unknown, status = 200) => {
return new Response(JSON.stringify(json), { status });
};
const mockOpenAiServer = (baseUrl: string, handlers: Record<string, RequestHandler>) => {
const requests: MockServerRequest[] = [];
jest.spyOn(shim, 'fetch').mockImplementation(async (url, options) => {
const path = url.substring(baseUrl.length);
const body: RequestBody = JSON.parse(options.body);
requests.push({ path, body });
const handler = handlers[path];
return handler ? handler(body) : new Response('404 page not found', { status: 404 });
});
return requests;
};
export default mockOpenAiServer;
+7
View File
@@ -15,6 +15,7 @@ export interface ChatStandardMessage extends ChatBaseMessage {
content: string;
hide?: boolean;
toolCalls?: ChatToolCall[];
responsesState?: ResponsesReplayState;
}
export interface ChatToolMessage extends ChatBaseMessage {
@@ -67,6 +68,11 @@ export interface ChatToolCall {
parseError: string|null;
}
export interface ResponsesReplayState {
key: string;
items: unknown[];
}
export type ChatFinishReason = 'stop' | 'length' | 'tool_calls' | 'other';
export interface ChatResult {
@@ -79,6 +85,7 @@ export interface ChatResult {
// Model reasoning, from a dedicated provider field or extracted from inline
// markers. Kept separate so it doesn't pollute the assistant reply.
reasoningText?: string;
responsesState?: ResponsesReplayState;
// Joplin Cloud degradation / budget signals. Populated only by the
// joplin-cloud provider; other providers leave them undefined. Consumed
// internally to drive the aiStatus Redux slice — plugins receive only
@@ -0,0 +1,29 @@
import Logger from '@joplin/utils/Logger';
const logger = Logger.create('ai/parseToolArguments');
const describeJsonParseFailure = (rawContent: string) => {
const parseError = ['Failed to parse JSON.'];
// With certain providers (e.g. Joplin Cloud), long messages are truncated. Include this information in the error message so that
// the model knows to retry with a shorter message:
const suggestedRetryLengthLimit = 1000;
if (rawContent.startsWith('{') && rawContent.length > suggestedRetryLengthLimit) {
parseError.push(`It's likely that the tool call JSON is too long. Please try again with a message shorter than ${suggestedRetryLengthLimit} characters.`);
}
return parseError.join(' ');
};
const parseToolArguments = (argumentString: string) => {
try {
const args: Record<string, unknown> = JSON.parse(argumentString);
return { args, parseError: null as string|null };
} catch (error) {
const parseError = describeJsonParseFailure(argumentString);
logger.error('JSON parse failed', error, parseError);
return { args: {}, parseError };
}
};
export default parseToolArguments;
@@ -70,7 +70,7 @@ describe('InteropService_Importer_EnexToMd', () => {
expect(notes[1].body).toContain('Mobilize & Measure');
// Reported, but flagged so it's not presented as a failed import.
expect(errors.length).toBe(1);
expect(isRecoverableError(errors[0])).toBe(true);
expect(errors.length).toBe(2);
expect(errors.every(isRecoverableError)).toBe(true);
});
});
+8
View File
@@ -29,6 +29,14 @@ describe('htmlUtils', () => {
}
});
it.each([
'<img src="x" a="" =b="x onerror=alert(1) c">',
'<img src="x" a =b="x onerror=alert(1) c">',
'<div><img src="x" a="" =b="x onerror=alert(1) c"></div>',
])('should not let an invalid attribute name bring in an event handler: %s', (html) => {
expect(htmlUtils.sanitizeHtml(html)).not.toContain('onerror');
});
test('should extract the HTML body', () => {
const testCases: [string, string][] = [
[
+7 -5
View File
@@ -35,17 +35,19 @@ interface SanitizeHtmlOptions {
allowedFilePrefixes?: string[];
}
// htmlparser2 accepts attribute names that browsers parse differently, such as one starting with "=". Together with a
// value-less attribute before it (`a =b="x onerror=..."`), the browser would see an attribute the sanitiser never did.
const validAttributeName = /^[a-zA-Z_:][-a-zA-Z0-9_:.]*$/;
export const attributesHtml = (attr: Record<string, string>) => {
const output = [];
for (const n in attr) {
if (!attr.hasOwnProperty(n)) continue;
if (!validAttributeName.test(n)) continue;
if (!attr[n]) {
output.push(n);
} else {
output.push(`${n}="${htmlentities(attr[n])}"`);
}
// Always with a value, even an empty one, so that nothing that follows can be read as this attribute's value
output.push(`${n}="${htmlentities(attr[n] || '')}"`);
}
return output.join(' ');
@@ -9,26 +9,26 @@ describe('applyTranslations', () => {
translations: {
'Translate me': ['Traduis moi'],
},
htmlTranslated: '<div>\n<span translate>\nTraduis moi\n</span>\n</div>',
htmlTranslated: '<div>\n<span translate="">\nTraduis moi\n</span>\n</div>',
},
{
html: '<div><span translate>Missing translation</span></div>',
translations: {},
htmlTranslated: '<div>\n<span translate>\nMissing translation\n</span>\n</div>',
htmlTranslated: '<div>\n<span translate="">\nMissing translation\n</span>\n</div>',
},
{
html: '<h1 translate class="text-center">\nFree your <span class="frame-bg frame-bg-blue">notes</span>\n</h1>',
translations: {
'Free your <span class="frame-bg frame-bg-blue">notes</span>': ['Libérez vos <span class="frame-bg frame-bg-blue">notes</span>'],
},
htmlTranslated: '<h1 translate class="text-center">\nLibérez vos <span class="frame-bg frame-bg-blue">notes</span>\n</h1>',
htmlTranslated: '<h1 translate="" class="text-center">\nLibérez vos <span class="frame-bg frame-bg-blue">notes</span>\n</h1>',
},
{
html: '<div translate>Save <span class="frame-bg frame-bg-blue">web pages</span> <br />as notes</div>',
translations: {
'Save <span class="frame-bg frame-bg-blue">web pages</span> <br>as notes': ['Sauvegardez vos <span class="frame-bg frame-bg-blue">pages web</span> <br>en notes'],
},
htmlTranslated: '<div translate>\nSauvegardez vos <span class="frame-bg frame-bg-blue">pages web</span> <br>en notes\n</div>',
htmlTranslated: '<div translate="">\nSauvegardez vos <span class="frame-bg frame-bg-blue">pages web</span> <br>en notes\n</div>',
},
];