mirror of
https://github.com/outline/outline.git
synced 2026-11-22 05:51:43 +03:00
fix: Halve indexed document text until the search vector fits (#14646)
This commit is contained in:
1 parent
97a00a37ce
commit
e0fb40baaa
2 files changed
+76
No files matched your search
@@ -0,0 +1,54 @@
|
||||
"use strict";
|
||||
|
||||
/** @type {import('sequelize-cli').Migration} */
|
||||
module.exports = {
|
||||
async up(queryInterface) {
|
||||
await queryInterface.sequelize.query(`
|
||||
CREATE OR REPLACE FUNCTION documents_search_trigger() RETURNS trigger AS $$
|
||||
declare
|
||||
"textLength" integer := 1000000;
|
||||
begin
|
||||
loop
|
||||
begin
|
||||
new."searchVector" :=
|
||||
setweight(to_tsvector('english', coalesce(new.title, '')),'A') ||
|
||||
setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') ||
|
||||
setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, "textLength")), 'D');
|
||||
return new;
|
||||
exception when program_limit_exceeded then
|
||||
if "textLength" = 0 then
|
||||
raise;
|
||||
end if;
|
||||
|
||||
-- The tsvector limit is on the output size, so index a shorter prefix.
|
||||
"textLength" := "textLength" / 2;
|
||||
end;
|
||||
end loop;
|
||||
end
|
||||
$$ LANGUAGE plpgsql;
|
||||
`);
|
||||
},
|
||||
|
||||
async down(queryInterface) {
|
||||
await queryInterface.sequelize.query(`
|
||||
CREATE OR REPLACE FUNCTION documents_search_trigger() RETURNS trigger AS $$
|
||||
begin
|
||||
begin
|
||||
new."searchVector" :=
|
||||
setweight(to_tsvector('english', coalesce(new.title, '')),'A') ||
|
||||
setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') ||
|
||||
setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, 1000000)), 'D');
|
||||
exception when program_limit_exceeded then
|
||||
-- The tsvector limit is on the output size, so text made of mostly
|
||||
-- unique tokens can overflow it. Index a shorter prefix instead.
|
||||
new."searchVector" :=
|
||||
setweight(to_tsvector('english', coalesce(new.title, '')),'A') ||
|
||||
setweight(to_tsvector('english', coalesce(array_to_string(new."previousTitles", ' , '),'')),'C') ||
|
||||
setweight(to_tsvector('english', substring(coalesce(new.text, ''), 1, 500000)), 'D');
|
||||
end;
|
||||
return new;
|
||||
end
|
||||
$$ LANGUAGE plpgsql;
|
||||
`);
|
||||
},
|
||||
};
|
||||
@@ -185,6 +185,28 @@ describe("#save", () => {
|
||||
expect(rows.searchVector).toMatch(/'origin':\d+C/);
|
||||
expect(rows.searchVector).toContain(tokens[0]);
|
||||
});
|
||||
|
||||
it("should index text whose shorter prefix still exceeds the tsvector limit", async () => {
|
||||
const document = await buildDocument({ title: "Original" });
|
||||
const tokens: string[] = [];
|
||||
for (let i = 0; i < 120_000; i++) {
|
||||
tokens.push(
|
||||
String.fromCharCode(19_968 + (i % 20_000)) +
|
||||
String.fromCharCode(19_968 + Math.floor(i / 20_000))
|
||||
);
|
||||
}
|
||||
document.title = "Unique tokens";
|
||||
document.text = tokens.join(" ");
|
||||
await document.save();
|
||||
|
||||
const [rows] = await sequelize.query<{ searchVector: string }>(
|
||||
`SELECT "searchVector" FROM documents WHERE id = :id`,
|
||||
{ replacements: { id: document.id }, type: QueryTypes.SELECT }
|
||||
);
|
||||
expect(rows.searchVector).toContain("'uniqu':1A");
|
||||
expect(rows.searchVector).toMatch(/'origin':\d+C/);
|
||||
expect(rows.searchVector).toContain(`'${tokens[0]}'`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#findAllChildDocumentIds", () => {
|
||||
|
||||
Reference in new issue
Block a user