From e0fb40baaabd486bc66d1b2fa3fd3e2fd7945629 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Thu, 19 Nov 2026 17:57:25 -0500 Subject: [PATCH] fix: Halve indexed document text until the search vector fits (#14646) --- ...ocument-search-text-until-tsvector-fits.js | 54 +++++++++++++++++++ server/models/Document.test.ts | 22 ++++++++ 2 files changed, 76 insertions(+) create mode 100644 server/migrations/20261118120000-halve-document-search-text-until-tsvector-fits.js diff --git a/server/migrations/20261118120000-halve-document-search-text-until-tsvector-fits.js b/server/migrations/20261118120000-halve-document-search-text-until-tsvector-fits.js new file mode 100644 index 0000000000..52ff17c980 --- /dev/null +++ b/server/migrations/20261118120000-halve-document-search-text-until-tsvector-fits.js @@ -0,0 +1,54 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface) { + await queryInterface.sequelize.query(` + CREATE OR REPLACE FUNCTION documents_search_trigger() RETURNS trigger AS $$ + declare + "textLength" integer := 1000000; + begin + loop + begin + new."searchVector" := + setweight(to_tsvector('english', coalesce(new.title, '')),'A') || + setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') || + setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, "textLength")), 'D'); + return new; + exception when program_limit_exceeded then + if "textLength" = 0 then + raise; + end if; + + -- The tsvector limit is on the output size, so index a shorter prefix. + "textLength" := "textLength" / 2; + end; + end loop; + end + $$ LANGUAGE plpgsql; + `); + }, + + async down(queryInterface) { + await queryInterface.sequelize.query(` + CREATE OR REPLACE FUNCTION documents_search_trigger() RETURNS trigger AS $$ + begin + begin + new."searchVector" := + setweight(to_tsvector('english', coalesce(new.title, '')),'A') || + setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') || + setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, 1000000)), 'D'); + exception when program_limit_exceeded then + -- The tsvector limit is on the output size, so text made of mostly + -- unique tokens can overflow it. Index a shorter prefix instead. + new."searchVector" := + setweight(to_tsvector('english', coalesce(new.title, '')),'A') || + setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') || + setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, 500000)), 'D'); + end; + return new; + end + $$ LANGUAGE plpgsql; + `); + }, +}; diff --git a/server/models/Document.test.ts b/server/models/Document.test.ts index 2b808423cf..5356f96830 100644 --- a/server/models/Document.test.ts +++ b/server/models/Document.test.ts @@ -185,6 +185,28 @@ describe("#save", () => { expect(rows.searchVector).toMatch(/'origin':\d+C/); expect(rows.searchVector).toContain(tokens[0]); }); + + it("should index text whose shorter prefix still exceeds the tsvector limit", async () => { + const document = await buildDocument({ title: "Original" }); + const tokens: string[] = []; + for (let i = 0; i < 120_000; i++) { + tokens.push( + String.fromCharCode(19_968 + (i % 20_000)) + + String.fromCharCode(19_968 + Math.floor(i / 20_000)) + ); + } + document.title = "Unique tokens"; + document.text = tokens.join(" "); + await document.save(); + + const [rows] = await sequelize.query<{ searchVector: string }>( + `SELECT "searchVector" FROM documents WHERE id = :id`, + { replacements: { id: document.id }, type: QueryTypes.SELECT } + ); + expect(rows.searchVector).toContain("'uniqu':1A"); + expect(rows.searchVector).toMatch(/'origin':\d+C/); + expect(rows.searchVector).toContain(`'${tokens[0]}'`); + }); }); describe("#findAllChildDocumentIds", () => {