Transcribe: Resolves #14253: Add optional GPU support to Transcribe server (#15182)

Dockerfile.transcribe.gpu

@@ -0,0 +1,91 @@
+# GPU-enabled variant of the Transcribe server image (NVIDIA / CUDA).
+#
+# This Dockerfile is the same as Dockerfile.transcribe, but:
+#   - The base image is nvidia/cuda (so the CUDA runtime libs are present).
+#   - Node.js 24 is installed on top of the CUDA base image.
+#   - The CUDA build of llama-mtmd-cli is copied from the official llama.cpp
+#     CUDA image (ghcr.io/ggml-org/llama.cpp:full-cuda-b5449). We use a
+#     multi-stage COPY here because llama.cpp does not publish a prebuilt
+#     Linux/CUDA zip on its release page — only Windows CUDA zips and the
+#     official Docker images.
+#
+# To run with GPU:
+#   docker run --gpus all -e HTR_CLI_GPU_LAYERS=9999 \
+#       --rm --env-file .env-transcribe -p 4567:4567 \
+#       -v ./data:/data joplin/transcribe:gpu-latest
+#
+# Requires the NVIDIA Container Toolkit on the host.
+# Apple Silicon / Metal is not supported through Docker (Docker Desktop on
+# macOS cannot expose the GPU to containers) — see packages/transcribe/README.md
+# for the native-run instructions.
+
+# Stage 1: pull the CUDA-built llama.cpp binaries from the official image.
+FROM ghcr.io/ggml-org/llama.cpp:full-cuda-b5449 AS llama-cuda
+
+# Stage 2: our runtime image.
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+
+RUN apt-get update \
+    && apt-get install -y \
+    ca-certificates curl wget \
+    python3 tini gnupg \
+    && curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
+    && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV NODE_ENV=production
+
+RUN corepack enable
+
+# Copy the CUDA-built llama-mtmd-cli binary and its runtime shared libraries
+# from the official llama.cpp CUDA image. The /app directory of that image
+# contains the built binaries and their .so dependencies.
+RUN mkdir -p /opt/llama/build/bin
+COPY --from=llama-cuda /app/llama-mtmd-cli /opt/llama/build/bin/llama-mtmd-cli
+COPY --from=llama-cuda /app/*.so /opt/llama/build/bin/
+RUN chmod +x /opt/llama/build/bin/llama-mtmd-cli
+
+# Create non-root user for security
+RUN groupadd -r transcribe && useradd -r -g transcribe -m transcribe
+
+WORKDIR /app
+
+COPY .yarn/releases ./.yarn/releases
+COPY .yarn/patches ./.yarn/patches
+COPY package.json .
+COPY .yarnrc.yml .
+COPY yarn.lock .
+COPY gulpfile.js .
+COPY tsconfig.json .
+COPY packages/lib ./packages/lib
+COPY packages/utils ./packages/utils
+COPY packages/tools ./packages/tools
+COPY packages/renderer ./packages/renderer
+COPY packages/htmlpack ./packages/htmlpack
+COPY packages/transcribe ./packages/transcribe
+
+# We don't want to build onenote-converter since it is not used by the server
+RUN sed --in-place '/onenote-converter/d' ./packages/lib/package.json
+
+RUN BUILD_SEQUENCIAL=1 yarn install --inline-builds \
+    && yarn cache clean \
+    && rm -rf .yarn/berry
+
+# Create data directory and set permissions
+RUN mkdir -p /data/images \
+    && chown -R transcribe:transcribe /data
+
+WORKDIR /app/packages/transcribe
+
+# Switch to non-root user
+USER transcribe
+
+# Set environment variables
+ENV HTR_CLI_BINARY_PATH=/opt/llama/build/bin/llama-mtmd-cli
+ENV LD_LIBRARY_PATH=/opt/llama/build/bin
+ENV DATA_DIR=/data
+ENV QUEUE_DRIVER=sqlite
+
+# Start the Node.js application
+ENTRYPOINT ["/usr/bin/tini", "--"]
+CMD ["yarn", "start"]

packages/tools/cspell/dictionary4.txt

@@ -255,6 +255,7 @@ mtmd
 gguf
 armor
 clearsign
+CUDA
 ligne
 payant
 llamacpp

packages/transcribe/README.md

@@ -91,6 +91,40 @@ The following paths are automatically derived from `DATA_DIR`:
 - `$DATA_DIR/models` - AI models
 - `$DATA_DIR/queue.sqlite3` - SQLite database (when using sqlite driver)
 
+### GPU support
+
+The server uses CPU by default. For NVIDIA GPU, run a CUDA-capable image with `--gpus all` and set `HTR_CLI_GPU_LAYERS=9999`.
+
+```shell
+docker run --rm --gpus all --env-file .env-transcribe -p 4567:4567 \
+	-e HTR_CLI_GPU_LAYERS=9999 \
+	-v ./data:/data \
+	joplin/transcribe:gpu-latest
+```
+
+This needs NVIDIA Container Toolkit on the host. Use `0` or leave it unset for CPU.
+
+### Native Windows without Docker
+
+On Windows, CUDA GPU acceleration can run in Docker using the NVIDIA instructions above, or natively on the host. For a native run, use a Windows x64 CUDA-enabled build of llama-mtmd-cli.exe, then set HTR_CLI_GPU_LAYERS to the number of layers you want offloaded to the GPU.
+
+### Native macOS/Metal
+
+On Apple Silicon, GPU acceleration runs natively on the host (Docker can't expose Metal to containers). Use a macOS ARM64 build of llama-mtmd-cli with Metal support, then set HTR_CLI_GPU_LAYERS to the number of layers you want offloaded to the GPU.
+
+### Native GPU configuration
+
+You can download the binary from the official llama.cpp releases page: <https://github.com/ggml-org/llama.cpp/releases>.
+
+Native GPU settings:
+
+```env
+HTR_CLI_BINARY_PATH=/path/to/llama-mtmd-cli
+HTR_CLI_GPU_LAYERS=9999
+```
+
+With this setup, the server passes `-ngl 9999` to `llama-mtmd-cli`. GPU support comes from the selected binary.
+
 # API Endpoints
 
 All requests must include the `Authorization` header with the value set to your `API_KEY`.

packages/transcribe/src/api/app.ts

@@ -38,6 +38,7 @@ const init = async (logger: LoggerWrapper) => {
 		htrCliImagesFolder: envVariables.HTR_CLI_IMAGES_FOLDER,
 		binaryPath: envVariables.HTR_CLI_BINARY_PATH,
 		modelsFolder: envVariables.HTR_CLI_MODELS_FOLDER,
+		gpuLayers: envVariables.HTR_CLI_GPU_LAYERS,
 	});
 
 	const jobProcessor = new JobProcessor(queue, htrCli, fileStorage);

packages/transcribe/src/core/HtrCli.test.ts

@@ -2,7 +2,7 @@ import { readFile } from 'fs-extra';
 import HtrCli from './HtrCli';
 
 describe('HtrCli', () => {
-	const dt = new HtrCli({ htrCliImagesFolder: '', binaryPath: '', modelsFolder: '' });
+	const dt = new HtrCli({ htrCliImagesFolder: '', binaryPath: '', modelsFolder: '', gpuLayers: 0 });
 	it('should parse multiline result', async () => {
 		const testCase = await readFile('./test-cases/1.txt');
 		const result = dt.cleanUpResult(testCase.toString());
@@ -33,4 +33,18 @@ describe('HtrCli', () => {
 		const result = dt.cleanUpResult(testCase.toString());
 		expect(result).toMatchSnapshot();
 	});
+
+	it('should not pass -ngl when gpuLayers is 0 (CPU only)', () => {
+		const cpuCli = new HtrCli({ htrCliImagesFolder: '/img', binaryPath: '/bin/llama', modelsFolder: '/models', gpuLayers: 0 });
+		const command = cpuCli.buildCommand('sample.jpg');
+		expect(command).not.toContain('-ngl');
+	});
+
+	it('should pass -ngl N when gpuLayers > 0 (GPU offload)', () => {
+		const gpuCli = new HtrCli({ htrCliImagesFolder: '/img', binaryPath: '/bin/llama', modelsFolder: '/models', gpuLayers: 9999 });
+		const command = gpuCli.buildCommand('sample.jpg');
+		const nglIndex = command.indexOf('-ngl');
+		expect(nglIndex).toBeGreaterThanOrEqual(0);
+		expect(command[nglIndex + 1]).toBe('9999');
+	});
 });

packages/transcribe/src/core/HtrCli.ts

@@ -11,6 +11,8 @@ export interface HtrCliOptions {
 	htrCliImagesFolder: string;
 	binaryPath: string;
 	modelsFolder: string;
+	// Number of model layers to offload to the GPU. 0 = CPU only.
+	gpuLayers: number;
 }
 
 export default class HtrCli implements WorkHandler {
@@ -43,9 +45,9 @@ export default class HtrCli implements WorkHandler {
 		return this.cleanUpResult(result);
 	}
 
-	private buildCommand(imageName: string): string[] {
-		const { binaryPath, modelsFolder, htrCliImagesFolder } = this.options;
-		return [
+	public buildCommand(imageName: string): string[] {
+		const { binaryPath, modelsFolder, htrCliImagesFolder, gpuLayers } = this.options;
+		const args = [
 			binaryPath,
 			'-m', `${modelsFolder}/Model-7.6B-Q4_K_M.gguf`,
 			'--mmproj', `${modelsFolder}/mmproj-model-f16.gguf`,
@@ -57,6 +59,8 @@ export default class HtrCli implements WorkHandler {
 			'--image', `${htrCliImagesFolder}/${imageName}`,
 			'-p', systemPrompt,
 		];
+		if (gpuLayers > 0) args.push('-ngl', String(gpuLayers));
+		return args;
 	}
 
 	public cleanUpResult(transcriptionAndLogs: string) {

packages/transcribe/src/env.ts

@@ -16,6 +16,7 @@ export const defaultEnvValues: EnvVariables = {
 	FILE_STORAGE_TTL: 7 * Day,
 	QUEUE_DATABASE_HOST: 'localhost',
 	IMAGE_MAX_DIMENSION: 400,
+	HTR_CLI_GPU_LAYERS: 0,
 };
 
 export interface EnvVariables {
@@ -34,6 +35,9 @@ export interface EnvVariables {
 	FILE_STORAGE_TTL: number;
 	QUEUE_DATABASE_HOST: string;
 	IMAGE_MAX_DIMENSION: number;
+	// Number of model layers to offload to the GPU. 0 = CPU only (default).
+	// Set to a high value (e.g. 9999) to offload all layers.
+	HTR_CLI_GPU_LAYERS: number;
 }
 
 export interface ComputedEnvVariables extends EnvVariables {

packages/transcribe/src/workers/JobProcessor.test.ts

@@ -36,7 +36,7 @@ describe('JobProcessor', () => {
 
 	skipByDefault('should execute work on job in the queue', async () => {
 		jest.useRealTimers();
-		const tw = new JobProcessor(queue, new HtrCli({ htrCliImagesFolder: join(process.cwd(), 'images'), binaryPath: '/opt/llama/build/bin/llama-mtmd-cli', modelsFolder: '/opt/models' }), new FileStorage(), 1000);
+		const tw = new JobProcessor(queue, new HtrCli({ htrCliImagesFolder: join(process.cwd(), 'images'), binaryPath: '/opt/llama/build/bin/llama-mtmd-cli', modelsFolder: '/opt/models', gpuLayers: 0 }), new FileStorage(), 1000);
 		await tw.init();
 
 		await copy(join('images', 'htr_sample.png'), join('images', 'htr_sample_copy.png'));
@@ -59,7 +59,7 @@ describe('JobProcessor', () => {
 
 	skipByDefault('should execute work on job in the queue even if one fails', async () => {
 		jest.useRealTimers();
-		const tw = new JobProcessor(queue, new HtrCli({ htrCliImagesFolder: join(process.cwd(), 'images'), binaryPath: '/opt/llama/build/bin/llama-mtmd-cli', modelsFolder: '/opt/models' }), new FileStorage(), 1000);
+		const tw = new JobProcessor(queue, new HtrCli({ htrCliImagesFolder: join(process.cwd(), 'images'), binaryPath: '/opt/llama/build/bin/llama-mtmd-cli', modelsFolder: '/opt/models', gpuLayers: 0 }), new FileStorage(), 1000);
 		await tw.init();
 		await copy(join('images', 'htr_sample.png'), join('images', 'htr_sample_copy_2.png'));
 
@@ -84,7 +84,7 @@ describe('JobProcessor', () => {
 
 	skipByDefault('should remove file sent to queue if job is completed', async () => {
 		jest.useRealTimers();
-		const tw = new JobProcessor(queue, new HtrCli({ htrCliImagesFolder: join(process.cwd(), 'images'), binaryPath: '/opt/llama/build/bin/llama-mtmd-cli', modelsFolder: '/opt/models' }), new FileStorage(), 1000);
+		const tw = new JobProcessor(queue, new HtrCli({ htrCliImagesFolder: join(process.cwd(), 'images'), binaryPath: '/opt/llama/build/bin/llama-mtmd-cli', modelsFolder: '/opt/models', gpuLayers: 0 }), new FileStorage(), 1000);
 		await tw.init();
 		const imagePath = join('images', 'htr_sample_copy_3.png');
 		await copy(join('images', 'htr_sample.png'), imagePath);
@@ -112,7 +112,7 @@ describe('JobProcessor', () => {
 		const fileStorage = new FileStorage();
 		const mockedFileStorageRemove = jest.fn();
 		fileStorage.remove = mockedFileStorageRemove;
-		const tw = new JobProcessor(queue, new HtrCli({ htrCliImagesFolder: join(process.cwd(), 'images'), binaryPath: '/opt/llama/build/bin/llama-mtmd-cli', modelsFolder: '/opt/models' }), fileStorage, 1000);
+		const tw = new JobProcessor(queue, new HtrCli({ htrCliImagesFolder: join(process.cwd(), 'images'), binaryPath: '/opt/llama/build/bin/llama-mtmd-cli', modelsFolder: '/opt/models', gpuLayers: 0 }), fileStorage, 1000);
 		await tw.init();
 
 		// file doesn't exist to force a fail, but the call to remove the file should still exist