fix: Halve indexed document text until the search vector fits (#14646)

This commit is contained in:
Tom Moor authored and GitHub committed 2026-11-19 17:57:25 -05:00
1 parent 97a00a37ce
commit e0fb40baaa
2 files changed
+76

No files matched your search

@@ -0,0 +1,54 @@
"use strict";
/** @type {import('sequelize-cli').Migration} */
module.exports = {
async up(queryInterface) {
await queryInterface.sequelize.query(`
CREATE OR REPLACE FUNCTION documents_search_trigger() RETURNS trigger AS $$
declare
"textLength" integer := 1000000;
begin
loop
begin
new."searchVector" :=
setweight(to_tsvector('english', coalesce(new.title, '')),'A') ||
setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') ||
setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, "textLength")), 'D');
return new;
exception when program_limit_exceeded then
if "textLength" = 0 then
raise;
end if;
-- The tsvector limit is on the output size, so index a shorter prefix.
"textLength" := "textLength" / 2;
end;
end loop;
end
$$ LANGUAGE plpgsql;
`);
},
async down(queryInterface) {
await queryInterface.sequelize.query(`
CREATE OR REPLACE FUNCTION documents_search_trigger() RETURNS trigger AS $$
begin
begin
new."searchVector" :=
setweight(to_tsvector('english', coalesce(new.title, '')),'A') ||
setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') ||
setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, 1000000)), 'D');
exception when program_limit_exceeded then
-- The tsvector limit is on the output size, so text made of mostly
-- unique tokens can overflow it. Index a shorter prefix instead.
new."searchVector" :=
setweight(to_tsvector('english', coalesce(new.title, '')),'A') ||
setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') ||
setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, 500000)), 'D');
end;
return new;
end
$$ LANGUAGE plpgsql;
`);
},
};
+22
View File
@@ -185,6 +185,28 @@ describe("#save", () => {
expect(rows.searchVector).toMatch(/'origin':\d+C/);
expect(rows.searchVector).toContain(tokens[0]);
});
it("should index text whose shorter prefix still exceeds the tsvector limit", async () => {
const document = await buildDocument({ title: "Original" });
const tokens: string[] = [];
for (let i = 0; i < 120_000; i++) {
tokens.push(
String.fromCharCode(19_968 + (i % 20_000)) +
String.fromCharCode(19_968 + Math.floor(i / 20_000))
);
}
document.title = "Unique tokens";
document.text = tokens.join(" ");
await document.save();
const [rows] = await sequelize.query<{ searchVector: string }>(
`SELECT "searchVector" FROM documents WHERE id = :id`,
{ replacements: { id: document.id }, type: QueryTypes.SELECT }
);
expect(rows.searchVector).toContain("'uniqu':1A");
expect(rows.searchVector).toMatch(/'origin':\d+C/);
expect(rows.searchVector).toContain(`'${tokens[0]}'`);
});
});
describe("#findAllChildDocumentIds", () => {