diff --git a/.gitignore b/.gitignore index 25e05b4e2e..93d614398d 100644 --- a/.gitignore +++ b/.gitignore @@ -1622,6 +1622,7 @@ packages/lib/services/ai/tools/global/index.js packages/lib/services/ai/tools/global/listNotebooks.js packages/lib/services/ai/tools/global/listTags.js packages/lib/services/ai/tools/global/manageTags.js +packages/lib/services/ai/tools/global/readImage.js packages/lib/services/ai/tools/global/readNote.js packages/lib/services/ai/tools/global/searchNotes.js packages/lib/services/ai/tools/global/semanticSearchNotes.js @@ -1630,6 +1631,7 @@ packages/lib/services/ai/tools/types.js packages/lib/services/ai/tools/utils/applyNoteEdits.test.js packages/lib/services/ai/tools/utils/applyNoteEdits.js packages/lib/services/ai/tools/utils/buildTool.js +packages/lib/services/ai/tools/utils/serializeToolOutput.js packages/lib/services/ai/tools/utils/settings.js packages/lib/services/ai/types.js packages/lib/services/ai/utils/findFencedBlock.js diff --git a/.ignore.eslint b/.ignore.eslint index f856cc925a..d75a138376 100644 --- a/.ignore.eslint +++ b/.ignore.eslint @@ -1648,6 +1648,7 @@ packages/lib/services/ai/tools/global/index.js packages/lib/services/ai/tools/global/listNotebooks.js packages/lib/services/ai/tools/global/listTags.js packages/lib/services/ai/tools/global/manageTags.js +packages/lib/services/ai/tools/global/readImage.js packages/lib/services/ai/tools/global/readNote.js packages/lib/services/ai/tools/global/searchNotes.js packages/lib/services/ai/tools/global/semanticSearchNotes.js @@ -1656,6 +1657,7 @@ packages/lib/services/ai/tools/types.js packages/lib/services/ai/tools/utils/applyNoteEdits.test.js packages/lib/services/ai/tools/utils/applyNoteEdits.js packages/lib/services/ai/tools/utils/buildTool.js +packages/lib/services/ai/tools/utils/serializeToolOutput.js packages/lib/services/ai/tools/utils/settings.js packages/lib/services/ai/types.js packages/lib/services/ai/utils/findFencedBlock.js diff --git a/packages/lib/models/settings/builtInMetadata.ts b/packages/lib/models/settings/builtInMetadata.ts index c3e76aa4ca..4cb6fb17f0 100644 --- a/packages/lib/models/settings/builtInMetadata.ts +++ b/packages/lib/models/settings/builtInMetadata.ts @@ -903,6 +903,17 @@ const builtInMetadata = (Setting: typeof SettingType) => { storage: SettingStorage.File, }, + 'ai.tool.read_image.enabled': { + value: false, + type: SettingItemType.Bool, + public: true, + section: 'ai.tools', + appTypes: [AppType.Desktop], + show: showAiTools, + label: () => _('Allow reading images'), + storage: SettingStorage.File, + }, + 'ai.tool.list_notebooks.enabled': { value: false, type: SettingItemType.Bool, diff --git a/packages/lib/services/ai/noteChat.ts b/packages/lib/services/ai/noteChat.ts index 2ee516dfed..07bc4cad17 100644 --- a/packages/lib/services/ai/noteChat.ts +++ b/packages/lib/services/ai/noteChat.ts @@ -4,8 +4,9 @@ import JoplinError from '../../JoplinError'; import Logger from '@joplin/utils/Logger'; import { _ } from '../../locale'; import { EditOp, isEditorToolCall } from './tools/buildEditorTools'; -import { NoteContext, ToolError } from './tools/types'; +import { NoteContext, ToolError, ToolImageResponse, ToolOutput } from './tools/types'; import ToolIndex from './tools/ToolIndex'; +import serializeToolOutput from './tools/utils/serializeToolOutput'; const logger = Logger.create('noteChat'); @@ -112,7 +113,19 @@ const createHistory = (history: ChatMessage[], newMessage: string, context: Note return history; }; -const estimateTokens = (text: string) => Math.ceil(text.length / charsPerToken); +const estimateTokens = (text: string|ToolImageResponse) => { + // In the case of images, this is a *very* rough estimate: + if (text instanceof ToolImageResponse) { + const approximateByteLength = text.base64Only.length * 3 / 4; + // Assuming a 50% compression ratio + const approximatePixelCount = approximateByteLength * 2; + // Assuming a patch size of 24 + // See https://developers.openai.com/api/docs/guides/images-vision#patch-based-image-tokenization + const approximateTokens = Math.ceil(approximatePixelCount / 24 / 24); + return approximateTokens; + } + return Math.ceil(text.length / charsPerToken); +}; export interface ChatCommands { replaceSelection: (text: string, originalText: string)=> Promise; @@ -294,7 +307,7 @@ const runTools = async ( role: ChatRole.Tool, toolName: tool.id, toolCallId: toolCall.callId, - content: typeof output === 'string' ? output : JSON.stringify(output), + content: serializeToolOutput(output as ToolOutput), userDescription: tool.userDescription(toolCall.arguments, output), isError: false, isEdit: isEdit(tool.id), diff --git a/packages/lib/services/ai/providers/Anthropic.ts b/packages/lib/services/ai/providers/Anthropic.ts index ceda1f8622..2cfe16cf1c 100644 --- a/packages/lib/services/ai/providers/Anthropic.ts +++ b/packages/lib/services/ai/providers/Anthropic.ts @@ -1,7 +1,7 @@ import shim from '../../../shim'; import JoplinError from '../../../JoplinError'; import Logger from '@joplin/utils/Logger'; -import { ChatMessage, ChatOptions, ChatResult, ChatRole, ChatToolCall, ProviderClassification } from '../types'; +import { ChatMessage, ChatOptions, ChatResult, ChatRole, ChatToolCall, ChatToolMessage, ProviderClassification } from '../types'; import ChatProviderBase from './ChatProviderBase'; const logger = Logger.create('AnthropicProvider'); @@ -23,10 +23,19 @@ interface AnthropicToolUseContentBlock { input: Record; } +interface AnthropicImageContent { + type: 'image'; + source: { + type: 'base64'; + media_type: string; + data: string; + }; +} + interface AnthropicToolResultContentBlock { type: 'tool_result'; tool_use_id: string; - content: string; + content: string|AnthropicImageContent[]; } type AnthropicContentBlock = AnthropicToolUseContentBlock|AnthropicToolResultContentBlock|AnthropicTextContentBlock; @@ -98,21 +107,39 @@ const convertMessages = (messages: ChatMessage[]) => { }; }; + const convertToolResponse = (message: ChatToolMessage): AnthropicMessage => { + let content; + if (typeof message.content === 'string') { + content = message.content; + } else { + content = [{ + type: 'image' as const, + source: { + type: 'base64' as const, + media_type: message.content.mimeType, + data: message.content.base64Only, + }, + }]; + } + + return { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: message.toolCallId, + content, + ...(message.isError ? { is_error: true } : {}), + }, + ], + }; + }; + const result = messages .map((message): AnthropicMessage => { if (message.role === ChatRole.System) return null; if (message.role === ChatRole.Tool) { - return { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: message.toolCallId, - content: message.content, - ...(message.isError ? { is_error: true } : {}), - }, - ], - }; + return convertToolResponse(message); } else if (message.role === ChatRole.Assistant || message.role === ChatRole.User) { return { role: message.role, diff --git a/packages/lib/services/ai/providers/OpenAiCompatible.ts b/packages/lib/services/ai/providers/OpenAiCompatible.ts index ebfccec2ca..9a48b05d31 100644 --- a/packages/lib/services/ai/providers/OpenAiCompatible.ts +++ b/packages/lib/services/ai/providers/OpenAiCompatible.ts @@ -59,14 +59,34 @@ const convertTool = (tool: ToolSpec) => { const convertMessage = (message: ChatMessage) => { if (message.role === 'tool') { - return { - role: 'tool', - name: message.toolName, - content: message.content, - tool_call_id: message.toolCallId, - }; + const content = message.content; + if (typeof content === 'string') { + return [{ + role: 'tool', + name: message.toolName, + content, + tool_call_id: message.toolCallId, + }]; + } else { + // Joplin currently uses the older OpenAI chat responses API, which does not support + // images in tool results. Attach the image in a user message instead: + return [{ + role: 'tool', + name: message.toolName, + content: 'success: will be attached in user message', + tool_call_id: message.toolCallId, + }, { + role: 'user', + content: [ + { + type: 'image_url', + image_url: { url: content.dataUrl }, + }, + ], + }]; + } } else { - return { + return [{ role: message.role, content: message.content, ...(message.toolCalls ? { @@ -81,7 +101,7 @@ const convertMessage = (message: ChatMessage) => { }; }), } : {}), - }; + }]; } }; @@ -128,7 +148,7 @@ export default class OpenAiCompatibleProvider extends ChatProviderBase { const body: Record = { model: this.model_, - messages: messages.map(convertMessage), + messages: messages.flatMap((message): unknown[] => convertMessage(message)), stream: false, }; if (options?.temperature !== undefined) body.temperature = options.temperature; diff --git a/packages/lib/services/ai/providers/TestProvider.ts b/packages/lib/services/ai/providers/TestProvider.ts index e76e25c17a..050d941378 100644 --- a/packages/lib/services/ai/providers/TestProvider.ts +++ b/packages/lib/services/ai/providers/TestProvider.ts @@ -1,8 +1,8 @@ import { ToolSpec } from '../tools/types'; -import { ChatMessage, ChatOptions, ChatResult, ChatRole, ChatToolCall, ProviderClassification } from '../types'; +import { ChatMessage, ChatOptions, ChatResult, ChatRole, ChatStandardMessage, ChatToolCall, ProviderClassification } from '../types'; import ChatProviderBase from './ChatProviderBase'; -const parseUserCommand = (message: ChatMessage, availableTools: ToolSpec[]) => { +const parseUserCommand = (message: ChatStandardMessage, availableTools: ToolSpec[]) => { const toolCalls: ChatToolCall[] = []; const reply = []; let repeat = 0; @@ -61,11 +61,12 @@ export default class TestProvider extends ChatProviderBase { let lastUserMessage; let replyCount = 0; for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].role === ChatRole.User) { - lastUserMessage = messages[i]; + const message = messages[i]; + if (message.role === ChatRole.User) { + lastUserMessage = message; break; } - if (messages[i].role === ChatRole.Assistant) { + if (message.role === ChatRole.Assistant) { replyCount ++; } } @@ -80,7 +81,8 @@ export default class TestProvider extends ChatProviderBase { } const output = content.join(' '); - const inputTokens = lastMessage.content.length; + const lastContent = lastMessage.content; + const inputTokens = typeof lastContent === 'string' ? lastContent.length : lastContent.dataUrl.length; const outputTokens = output.length + toolCalls.length; return { text: output, toolCalls, usage: { inputTokens, outputTokens } }; } diff --git a/packages/lib/services/ai/tools/global/index.ts b/packages/lib/services/ai/tools/global/index.ts index 513e0ebe2b..017c5998f7 100644 --- a/packages/lib/services/ai/tools/global/index.ts +++ b/packages/lib/services/ai/tools/global/index.ts @@ -8,6 +8,7 @@ import updateNote from './updateNote'; import deleteNote from './deleteNote'; import manageTags from './manageTags'; import createNotebook from './createNotebook'; +import readImage from './readImage'; export default [ searchNotes, @@ -20,4 +21,5 @@ export default [ deleteNote, manageTags, createNotebook, + readImage, ]; diff --git a/packages/lib/services/ai/tools/global/readImage.ts b/packages/lib/services/ai/tools/global/readImage.ts new file mode 100644 index 0000000000..485a22c0cb --- /dev/null +++ b/packages/lib/services/ai/tools/global/readImage.ts @@ -0,0 +1,87 @@ +import { _ } from '../../../../locale'; +import buildTool from '../utils/buildTool'; +import { ToolError, ToolImageResponse } from '../types'; +import Resource from '../../../../models/Resource'; +import shim from '../../../../shim'; +import { ResourceEntity } from '@joplin/renderer/types'; +import isItemId from '../../../../models/utils/isItemId'; + +interface Input { + id?: string; + resolution?: 'low'|'medium'|'high'; +} + +const dataUrlToMimeType = (url: string) => { + const dataUrlMatch = url.match(/^data:(.*);/); + if (!dataUrlMatch) throw new Error('Invalid data URL'); + return dataUrlMatch[1]; +}; + +class ReadImageResponse extends ToolImageResponse { + public constructor(dataUrl: string, public readonly resource: ResourceEntity) { + super({ + dataUrl, + // Prefer the mime type from the data URL over the one in the resource, in case the + // resource was re-encoded while converting to a data URL: + mimeType: dataUrlToMimeType(dataUrl), + id: resource.id, + }); + } +} + +const tool = buildTool({ + id: 'read_image', + userDescription: (_input, output) => { + return _('Read image: %s', output?.resource?.title ?? _('(untitled)')); + }, + description: 'View an image attachment. Use this to describe image content or text. Returns the image data.', + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'The attachment id (32-character hex).' }, + resolution: { + type: 'string', + enum: ['low', 'medium', 'high'], + default: 'medium', + description: 'The quality of the image. High resolution is better for OCR, but uses more tokens.', + }, + }, + required: ['id'], + }, + handler: async (input: Input): Promise => { + if (!input.id) throw new ToolError('Missing "id" parameter'); + // Models sometimes try to pass the full resource link. Handle this gracefully: + const id = input.id.replace(/^:?\//, ''); + if (!isItemId(id)) throw new ToolError(`Invalid ID: ${id}`); + + const resource = await Resource.load(id); + if (!resource || resource.is_locked) { + throw new ToolError(`Resource not found: ${id}`); + } + if (resource.encryption_applied) { + throw new ToolError(`Resource is encrypted: ${id}`); + } + if (!resource.mime.startsWith('image/')) { + throw new ToolError(`Unsupported image MIME type: ${resource.mime}`); + } + + const resolution = input.resolution ?? 'medium'; + if (!['low', 'medium', 'high'].includes(resolution)) { + throw new ToolError(`Invalid resolution: ${JSON.stringify(resolution)}`); + } + + let maximumSize = 128; + if (resolution === 'medium') { + maximumSize = 256; + } else if (resolution === 'high') { + maximumSize = 512; + } + + const fullPath = Resource.fullPath(resource); + // Use a small default image size: Images can use a large number of tokens. + const url = await shim.imageToDataUrl(fullPath, maximumSize); + return new ReadImageResponse(url, resource); + }, +}); + +export default tool; diff --git a/packages/lib/services/ai/tools/types.ts b/packages/lib/services/ai/tools/types.ts index f5e157a9f3..486002705a 100644 --- a/packages/lib/services/ai/tools/types.ts +++ b/packages/lib/services/ai/tools/types.ts @@ -1,3 +1,4 @@ +import isItemId from '../../../models/utils/isItemId'; import { JsonSchema } from '../types'; export interface EditorCommands { @@ -25,7 +26,35 @@ export type ToolInput = Record; // Plain Errors are treated as internal bugs. export class ToolError extends Error {} -export type ToolOutput = string|Record; +export interface ImageProperties { + dataUrl: string; + mimeType: string; + // A unique identifier, should be the same if the image is included in the chat + // transcript again. + id: string; +} + +export class ToolImageResponse { + public readonly dataUrl: string; + public readonly mimeType: string; + public readonly id: string; + + public constructor(props: ImageProperties) { + if (!isItemId(props.id)) { + throw new Error('Invalid image ID: Must be compatible with Joplin IDs'); + } + + this.dataUrl = props.dataUrl; + this.mimeType = props.mimeType; + this.id = props.id; + } + + public get base64Only() { + return this.dataUrl.replace(/^data:[^;]+;base64,/, ''); + } +} + +export type ToolOutput = string|Record|ToolImageResponse; export interface ToolSpec { id: string; diff --git a/packages/lib/services/ai/tools/utils/serializeToolOutput.ts b/packages/lib/services/ai/tools/utils/serializeToolOutput.ts new file mode 100644 index 0000000000..b3dca7190c --- /dev/null +++ b/packages/lib/services/ai/tools/utils/serializeToolOutput.ts @@ -0,0 +1,9 @@ +import { ToolImageResponse, ToolOutput } from '../types'; + +const serializeToolOutput = (output: ToolOutput) => { + if (output instanceof ToolImageResponse) return output; + return typeof output === 'string' ? output : JSON.stringify(output); +}; + +export default serializeToolOutput; + diff --git a/packages/lib/services/ai/types.ts b/packages/lib/services/ai/types.ts index cce3a7f06a..5d4e1736d4 100644 --- a/packages/lib/services/ai/types.ts +++ b/packages/lib/services/ai/types.ts @@ -1,4 +1,4 @@ -import { ToolSpec } from './tools/types'; +import { ToolImageResponse, ToolSpec } from './tools/types'; export enum ChatRole { System = 'system', @@ -8,17 +8,18 @@ export enum ChatRole { } interface ChatBaseMessage { - content: string; } export interface ChatStandardMessage extends ChatBaseMessage { role: ChatRole.System | ChatRole.User | ChatRole.Assistant; + content: string; hide?: boolean; toolCalls?: ChatToolCall[]; } export interface ChatToolMessage extends ChatBaseMessage { role: ChatRole.Tool; + content: string|ToolImageResponse; toolName: string; toolCallId: string; isError: boolean; diff --git a/packages/lib/services/mcp/McpServer.test.ts b/packages/lib/services/mcp/McpServer.test.ts index aefc29a735..40c2f35fdb 100644 --- a/packages/lib/services/mcp/McpServer.test.ts +++ b/packages/lib/services/mcp/McpServer.test.ts @@ -3,9 +3,11 @@ import Note from '../../models/Note'; import Folder from '../../models/Folder'; import Tag from '../../models/Tag'; import SearchEngine from '../search/SearchEngine'; -import { db, setupDatabaseAndSynchronizer, switchClient, withWarningSilenced } from '../../testing/test-utils'; +import { db, setupDatabaseAndSynchronizer, supportDir, switchClient, withWarningSilenced } from '../../testing/test-utils'; import McpServer from './McpServer'; import { McpProtocolVersion } from './types'; +import shim from '../../shim'; +import { join } from 'path'; const allToolSettings = [ 'ai.tool.search_notes.enabled', @@ -13,6 +15,7 @@ const allToolSettings = [ 'ai.tool.read_note.enabled', 'ai.tool.list_notebooks.enabled', 'ai.tool.list_tags.enabled', + 'ai.tool.read_image.enabled', 'ai.tool.create_note.enabled', 'ai.tool.update_note.enabled', 'ai.tool.delete_note.enabled', @@ -357,4 +360,22 @@ describe('McpServer', () => { expect(updated.title).toBe('New'); expect(updated.body).toBe('Keep'); }); + + test('read_image returns image data as base64', async () => { + const folder = await Folder.save({ title: 'Folder' }); + let note = await Note.save({ title: 'Test', body: 'Test', parent_id: folder.id }); + note = await shim.attachFileToNote(note, join(supportDir, 'photo.jpg')); + const resourceId = Note.linkedItemIds(note.body)[0]; + + const response = await McpServer.instance().handleRequest({ + jsonrpc: '2.0', id: 1, method: 'tools/call', + params: { name: 'read_image', arguments: { id: resourceId, resolution: 'low' } }, + }); + + expect(response.error).toBeFalsy(); + const content = response.result.content[0]; + expect(content.type).toBe('image'); + expect(content.mimeType).toContain('image/'); + expect(content.data).toBeTruthy(); + }); }); diff --git a/packages/lib/services/mcp/McpServer.ts b/packages/lib/services/mcp/McpServer.ts index fbeb5aa18c..b44ce7a4ad 100644 --- a/packages/lib/services/mcp/McpServer.ts +++ b/packages/lib/services/mcp/McpServer.ts @@ -1,8 +1,8 @@ import Logger from '@joplin/utils/Logger'; import Setting from '../../models/Setting'; import ToolIndex from '../ai/tools/ToolIndex'; -import { JsonRpcRequest, JsonRpcResponse, JsonRpcErrorCodes, McpProtocolVersion, ToolCallResult } from './types'; -import { ToolError } from '../ai/tools/types'; +import { JsonRpcRequest, JsonRpcResponse, JsonRpcErrorCodes, McpProtocolVersion, ToolCallResult, ToolContent, ToolTextContent } from './types'; +import { ToolError, ToolImageResponse, ToolOutput } from '../ai/tools/types'; const logger = Logger.create('McpServer'); @@ -96,9 +96,10 @@ export default class McpServer { } const input = params.arguments ?? {}; try { - const payload = await tool.handler(input, {}); + const payload = await tool.handler(input, {}) as ToolOutput; + return { - content: [{ type: 'text', text: serialisePayload(payload) }], + content: [serialisePayload(payload)], }; } catch (error) { if (error instanceof ToolError) { @@ -129,8 +130,18 @@ const toolErrorResult = (message: string): ToolCallResult => ({ // MCP content is always text, so we JSON-serialise objects/arrays and pass // strings through unchanged. null/undefined collapse to an empty string. -const serialisePayload = (payload: unknown) => { - if (payload === null || payload === undefined) return ''; - if (typeof payload === 'string') return payload; - return JSON.stringify(payload, null, 2); +const serialisePayload = (payload: unknown): ToolContent => { + const textResponse = (text: string): ToolTextContent => ({ + type: 'text', text, + }); + if (payload === null || payload === undefined) return textResponse(''); + if (typeof payload === 'string') return textResponse(payload); + if (payload instanceof ToolImageResponse) { + return { + type: 'image', + data: payload.base64Only, + mimeType: payload.mimeType, + }; + } + return textResponse(JSON.stringify(payload, null, 2)); }; diff --git a/packages/lib/services/mcp/types.ts b/packages/lib/services/mcp/types.ts index b009a60f3e..71c23e5b4f 100644 --- a/packages/lib/services/mcp/types.ts +++ b/packages/lib/services/mcp/types.ts @@ -5,8 +5,13 @@ export interface ToolTextContent { text: string; } -// MCP also allows image and resource content; v1 ships text only. -export type ToolContent = ToolTextContent; +export interface ToolImageContent { + type: 'image'; + data: string; + mimeType: string; +} + +export type ToolContent = ToolTextContent|ToolImageContent; export interface ToolCallResult { content: ToolContent[]; diff --git a/packages/lib/shim-init-node.ts b/packages/lib/shim-init-node.ts index 9694ab3c20..9427993c81 100644 --- a/packages/lib/shim-init-node.ts +++ b/packages/lib/shim-init-node.ts @@ -492,6 +492,19 @@ function shimInit(options: ShimInitOptions = null) { } return image.toDataURL(); + } else if (shim.sharpEnabled()) { + let image = sharp(filePath); + const metadata = await image.metadata(); + + const maxDimensionIsWidth = metadata.width > metadata.height; + if (metadata.width > maxSize && maxDimensionIsWidth) { + image = image.resize({ width: maxSize }); + } else if (metadata.height > maxSize) { + image = image.resize({ height: maxSize }); + } + + const base64 = (await image.png().toBuffer()).toString('base64'); + return `data:image/png;base64,${base64}`; } else { throw new Error('Unsupported method'); }