update

packages/server/src/models/ItemModel.test.ts

@@ -632,6 +632,38 @@ describe('ItemModel', () => {
 		expect((await models().userItem().byUserId(user1.id)).length).toBe(5);
 	});
 
+	test('should process orphaned items mixed with valid items across batches', async () => {
+		const { user: user1 } = await createUserAndSession(1);
+		const { user: user2 } = await createUserAndSession(2);
+
+		await createItemTree3(user1.id, '', '', [
+			{ id: '000000000000000000000000000000F1' },
+		]);
+
+		await createItemTree3(user2.id, '', '', [
+			{ id: '000000000000000000000000000000F2' },
+			{ id: '000000000000000000000000000000F3' },
+			{ id: '000000000000000000000000000000F4' },
+			{ id: '000000000000000000000000000000F5' },
+		]);
+
+		const f4 = await models().item().loadByJopId(user2.id, '000000000000000000000000000000F4');
+		await db()('user_items').where('item_id', '=', f4.id).delete();
+		await db()('users').where('id', '=', user1.id).delete();
+		await db()('user_items').where('user_id', '=', user1.id).delete();
+
+		await models().item().processOrphanedItems({ batchSize: 2 });
+
+		const items = await models().item().all();
+		expect(items.map(i => i.name).sort()).toEqual([
+			'000000000000000000000000000000F2.md',
+			'000000000000000000000000000000F3.md',
+			'000000000000000000000000000000F4.md',
+			'000000000000000000000000000000F5.md',
+		]);
+		expect((await models().userItem().byUserId(user2.id)).length).toBe(4);
+	});
+
 	test('should return multiple item contents', async () => {
 		const { user: user1 } = await createUserAndSession(1);
 

packages/server/src/models/ItemModel.ts

@@ -1102,6 +1102,10 @@ export default class ItemModel extends BaseModel<Item> {
 		// then look up which of those ids exist in user_items. The orphan
 		// filter happens in TS. Per-query cost is bounded by the window size,
 		// not by the table size, so no statement can blow past the timeout.
+		//
+		// The window query only selects `id` so that it is an index-only scan.
+		// IDs are random, so any other column costs one heap page read per row,
+		// and walking the whole heap evicts the buffer cache used by the app.
 		const batchSize = options.batchSize ?? 10000;
 		let batchNum = 0;
 		let totalProcessed = 0;
@@ -1114,7 +1118,7 @@ export default class ItemModel extends BaseModel<Item> {
 			const batchStartTime = Date.now();
 
 			const windowItems: Item[] = await this.db(this.tableName)
-				.select(['items.id', 'items.name', 'items.owner_id'])
+				.select(['items.id'])
 				.where('items.id', '>', lastId)
 				.orderBy('items.id')
 				.limit(batchSize);
@@ -1133,14 +1137,18 @@ export default class ItemModel extends BaseModel<Item> {
 				.select('item_id')
 				.whereIn('item_id', windowItemIds);
 			const userItemIds = new Set(existingUserItems.map(u => u.item_id));
-			const orphanedItems = windowItems.filter(i => !userItemIds.has(i.id));
+			const orphanedItemIds = windowItemIds.filter(id => !userItemIds.has(id));
 
-			if (!orphanedItems.length) {
+			if (!orphanedItemIds.length) {
 				const batchDuration = Date.now() - batchStartTime;
 				modelLogger.info(`processOrphanedItems: Batch ${batchNum} - No orphans in window of ${windowItems.length} items (${batchDuration}ms)`);
 				continue;
 			}
 
+			const orphanedItems: Item[] = await this.db(this.tableName)
+				.select(['id', 'name', 'owner_id'])
+				.whereIn('id', orphanedItemIds);
+
 			modelLogger.info(`processOrphanedItems: Batch ${batchNum} - Found ${orphanedItems.length} orphaned items`);
 			for (const o of orphanedItems) {
 				modelLogger.info(`OrphanTrace: found orphan item=${o.id} owner=${o.owner_id} name=${o.name}`);

packages/server/src/utils/setupTaskService.ts

@@ -63,10 +63,12 @@ export default async function(env: Env, models: Models, config: Config, services
 			run: (models: Models) => models.user().handleOversizedAccounts(),
 		},
 
+		// Walks the whole items table, so it runs once a day rather than
+		// hourly. Orphans are rare and clients work around them meanwhile.
 		{
 			id: TaskId.ProcessOrphanedItems,
 			description: taskIdToLabel(TaskId.ProcessOrphanedItems),
-			schedule: '15 * * * *',
+			schedule: '15 4 * * *',
 			run: (models: Models) => models.item().processOrphanedItems(),
 		},