mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-11-20 21:23:37 +03:00
fix(server): doc parse cycle (#15794)
#### PR Dependency Tree * **PR #15794** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> <!-- coderabbit-ai-generated:v1 --> ## Summary by CodeRabbit * **Bug Fixes** * Improved search reliability when documents are changing, including more consistent handling of invalid or incomplete document data. * Prevented overlapping scheduled search updates, helping keep background processing consistent. * Improved coordination between search projections and payment processing when both operate on document snapshots. * Search processing now better isolates failures, helping other documents continue to be processed and allowing recovery after source data changes. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
1 parent
e1261c40bf
commit
4db3ac8d6b
14 files changed
+252
-121
No files matched your search
@@ -190,7 +190,18 @@ jobs:
|
||||
};
|
||||
for (const target of plan.targets) {
|
||||
if (target.name === 'server') continue;
|
||||
const artifact = artifacts.find(a => a.name === `candidate-${target.name}` && !a.expired);
|
||||
const arches = plan.buildMatrix.include.filter(row => row.target === target.name && target.name.startsWith('desktop-'));
|
||||
const missing = arches.filter(row => !artifacts.some(a =>
|
||||
a.name === `candidate-desktop-${row.platform}-${row.arch}` && !a.expired
|
||||
));
|
||||
const artifact = missing.length === 0
|
||||
? artifacts.find(a => a.name === `candidate-${target.name}` && !a.expired)
|
||||
: undefined;
|
||||
if (arches.length && !artifact) {
|
||||
const reason = missing.length ? `missing ${missing.map(row => row.arch).join(', ')}` : 'missing merged artifact';
|
||||
if (target.required) throw new Error(`${target.name}: ${reason}`);
|
||||
core.warning(`Omitting ${target.name}: ${reason}`);
|
||||
}
|
||||
if (artifact) candidate.artifacts[target.name] = {
|
||||
id: artifact.id, runId: context.runId, repository: candidate.repository,
|
||||
name: artifact.name, stage: 'unsigned',
|
||||
|
||||
Generated
+19
-52
@@ -93,9 +93,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "affine_doc_loader"
|
||||
version = "0.1.10"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f1d81b2efb500e1b000db2148ccdc196f76b1dede5ff29a1fdae49c49410bdf"
|
||||
checksum = "52f78576f0d8862c41e56a15ecc63b5424630b126a41160e7fcd3fdbb441f4ff"
|
||||
dependencies = [
|
||||
"affine_merge",
|
||||
"chrono",
|
||||
@@ -110,9 +110,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "affine_importer"
|
||||
version = "0.1.3"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9264989137add695376d62d01d4df4d6bb91a3b7b4f418130adc8470e85c7aa"
|
||||
checksum = "7b1cf38c45c12d6e981adc554264d5c3318cabde658719d138b497e87d8bbf49"
|
||||
dependencies = [
|
||||
"affine_doc_loader",
|
||||
"bytes",
|
||||
@@ -126,7 +126,7 @@ dependencies = [
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.18",
|
||||
"typed-path",
|
||||
"zip 8.6.0",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -214,7 +214,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"uuid",
|
||||
"y-octo",
|
||||
"zip 9.0.5",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1530,12 +1530,6 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.6.1"
|
||||
@@ -2279,7 +2273,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users 0.5.2",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2617,7 +2611,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3126,8 +3120,8 @@ dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"rustversion",
|
||||
"windows-link 0.2.1",
|
||||
"windows-result 0.4.1",
|
||||
"windows-link 0.1.3",
|
||||
"windows-result 0.3.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3874,7 +3868,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core 0.62.2",
|
||||
"windows-core 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4381,7 +4375,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5617,7 +5611,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5870,7 +5864,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
"zip 9.0.5",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7506,7 +7500,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7565,7 +7559,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8942,7 +8936,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10714,7 +10708,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11575,33 +11569,6 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "8.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
||||
dependencies = [
|
||||
"aes 0.9.1",
|
||||
"bzip2",
|
||||
"constant_time_eq 0.4.2",
|
||||
"crc32fast",
|
||||
"deflate64",
|
||||
"flate2",
|
||||
"getrandom 0.4.2",
|
||||
"hmac 0.13.0",
|
||||
"indexmap",
|
||||
"lzma-rust2 0.16.4",
|
||||
"memchr",
|
||||
"pbkdf2",
|
||||
"ppmd-rust",
|
||||
"sha1 0.11.0",
|
||||
"time",
|
||||
"typed-path",
|
||||
"zeroize",
|
||||
"zopfli",
|
||||
"zstd 0.13.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.0.5"
|
||||
@@ -11610,7 +11577,7 @@ checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
|
||||
dependencies = [
|
||||
"aes 0.9.1",
|
||||
"bzip2",
|
||||
"constant_time_eq 0.6.1",
|
||||
"constant_time_eq",
|
||||
"crc32fast",
|
||||
"deflate64",
|
||||
"flate2",
|
||||
|
||||
+2
-2
@@ -16,8 +16,8 @@ resolver = "3"
|
||||
[workspace.dependencies]
|
||||
aes-gcm = "0.10"
|
||||
affine_common = { path = "./packages/common/native" }
|
||||
affine_doc_loader = "0.1.9"
|
||||
affine_importer = "0.1.2"
|
||||
affine_doc_loader = "0.1.11"
|
||||
affine_importer = "0.1.4"
|
||||
affine_nbstore = { path = "./packages/frontend/native/nbstore" }
|
||||
affine_preview = { version = "0.1.0", default-features = false }
|
||||
anyhow = "1"
|
||||
|
||||
@@ -617,8 +617,8 @@ async fn session_kernel_serializes_refresh_replay_revocation_and_key_rotation()
|
||||
.id;
|
||||
assert_eq!(predecessor, (None, None, Some(child_id.clone())));
|
||||
let child_material: bool = sqlx::query_scalar(
|
||||
"SELECT successor_ciphertext IS NOT NULL AND successor_expires_at IS NOT NULL AND refresh_attempt_hash IS \
|
||||
NOT NULL FROM auth_refresh_tokens WHERE id=$1",
|
||||
"SELECT successor_ciphertext IS NOT NULL AND successor_expires_at IS NOT NULL AND refresh_attempt_hash IS NOT \
|
||||
NULL FROM auth_refresh_tokens WHERE id=$1",
|
||||
)
|
||||
.bind(&child_id)
|
||||
.fetch_one(&pool)
|
||||
|
||||
@@ -17,7 +17,7 @@ use tokio::{
|
||||
};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::*;
|
||||
use super::{super::search::project_current_doc, *};
|
||||
use crate::runtime::{Deployment, RevenueCatRuntimeConfig, StripeRuntimeConfig};
|
||||
|
||||
async fn apply_payment_snapshot(
|
||||
@@ -1248,6 +1248,38 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
|
||||
"t={timestamp},v1={}",
|
||||
hex::encode(hmac::Mac::finalize(signature).into_bytes())
|
||||
);
|
||||
sqlx::query("INSERT INTO snapshots(workspace_id,guid,blob,updated_at) VALUES($1,$1,$2,now())")
|
||||
.bind(&marker)
|
||||
.bind(affine_doc_loader::build_full_doc("title", "body", &marker).unwrap())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let (started, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, mut release_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let projection_pool = pool.clone();
|
||||
let projection_doc = marker.clone();
|
||||
let projection = tokio::spawn(async move {
|
||||
project_current_doc(&projection_pool, &projection_doc, &projection_doc, move |_| {
|
||||
started.send(()).unwrap();
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(15);
|
||||
while matches!(
|
||||
release_rx.try_recv(),
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
|
||||
) {
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"payment did not complete while search was busy"
|
||||
);
|
||||
std::hint::spin_loop();
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let first_receipt = super::webhook::capture_stripe(&pool, &webhook_client, raw_event.as_bytes(), &signature)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1345,6 +1377,13 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(processed_receipt.get("status"), Some(&json!("processed")));
|
||||
release.send(()).expect("payment completed after search stopped");
|
||||
projection.await.unwrap().unwrap().unwrap();
|
||||
sqlx::query("DELETE FROM snapshots WHERE workspace_id=$1 AND guid=$1")
|
||||
.bind(&marker)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(super::worker::claim_receipt(&webhook_runtime).await.unwrap().is_none());
|
||||
let mut connection = PaymentConnection::try_acquire(&pool, lock_scopes)
|
||||
.await
|
||||
|
||||
@@ -9,6 +9,7 @@ mod types;
|
||||
mod worker;
|
||||
|
||||
use generation::{ActiveGeneration, activate, cleanup_retired_generation, config_hash, ensure, fail, load_active};
|
||||
pub(super) use projection::project_current_doc;
|
||||
use projection::{ProjectionInput, project_document};
|
||||
use provider::{SearchChange, SearchProvider, projection_external_id};
|
||||
use query::{compile, compile_aggregate};
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::{PgPool, Row};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::{
|
||||
permission::doc_role_allows,
|
||||
runtime::{RuntimeError, RuntimeResult, storage_runtime::load_current_doc},
|
||||
runtime::{
|
||||
RuntimeError, RuntimeResult,
|
||||
storage_runtime::{CurrentDoc, load_current_doc_parts, merge_current_doc},
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
@@ -13,24 +17,46 @@ pub(super) struct ProjectionInput {
|
||||
pub(super) payload: Value,
|
||||
}
|
||||
|
||||
static SEARCH_PROJECTION_SLOT: Semaphore = Semaphore::const_new(1);
|
||||
|
||||
pub(in crate::runtime::backend_runtime) async fn project_current_doc<T: Send + 'static>(
|
||||
pool: &PgPool,
|
||||
workspace_id: &str,
|
||||
doc_id: &str,
|
||||
project: impl FnOnce(CurrentDoc) -> RuntimeResult<T> + Send + 'static,
|
||||
) -> RuntimeResult<Option<T>> {
|
||||
let (snapshot, updates) = load_current_doc_parts(pool, workspace_id, doc_id).await?;
|
||||
let permit = SEARCH_PROJECTION_SLOT
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|error| RuntimeError::invalid_state(format!("search projection admission failed: {error}")))?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _permit = permit;
|
||||
let current = merge_current_doc(snapshot, updates).map_err(|error| match error {
|
||||
RuntimeError::InvalidState(message) => RuntimeError::SearchSourceInvalid(message),
|
||||
error => error,
|
||||
})?;
|
||||
current.map(project).transpose()
|
||||
})
|
||||
.await
|
||||
.map_err(|error| RuntimeError::invalid_state(format!("search projection task failed: {error}")))?
|
||||
}
|
||||
|
||||
pub(super) async fn project_document(
|
||||
pool: &PgPool,
|
||||
workspace_id: &str,
|
||||
doc_id: &str,
|
||||
) -> RuntimeResult<Option<(ProjectionInput, Vec<ProjectionInput>)>> {
|
||||
let Some(current) = load_current_doc(pool, workspace_id, doc_id)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
RuntimeError::InvalidState(message) => RuntimeError::SearchSourceInvalid(message),
|
||||
error => error,
|
||||
})?
|
||||
let projection_doc_id = doc_id.to_string();
|
||||
let Some(projection) = project_current_doc(pool, workspace_id, doc_id, move |current| {
|
||||
let revision = current.updated_at.timestamp_millis();
|
||||
affine_doc_loader::project_document_search(current.blob, projection_doc_id, revision.to_string())
|
||||
.map_err(|error| RuntimeError::SearchSourceInvalid(format!("document projection failed: {error}")))
|
||||
})
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let revision = current.updated_at.timestamp_millis();
|
||||
let projection =
|
||||
affine_doc_loader::project_document_search(current.blob, doc_id.to_string(), revision.to_string())
|
||||
.map_err(|error| RuntimeError::SearchSourceInvalid(format!("document projection failed: {error}")))?;
|
||||
let Some(metadata) = sqlx::query(
|
||||
r#"SELECT snapshot.created_at,snapshot.updated_at,snapshot.created_by,snapshot.updated_by,
|
||||
coalesce(doc_policy.visibility,'private') AS visibility,
|
||||
@@ -197,6 +223,65 @@ fn input(workspace_id: &str, doc_id: &str, payload: Value) -> ProjectionInput {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_caller_keeps_projection_slot_until_computation_finishes() {
|
||||
let _guard = super::super::SEARCH_TEST_LOCK.lock().await;
|
||||
let Ok(database_url) = std::env::var("DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
let pool = PgPool::connect(&database_url).await.unwrap();
|
||||
let id = uuid::Uuid::new_v4().to_string();
|
||||
sqlx::query("INSERT INTO snapshots(workspace_id,guid,blob,updated_at) VALUES($1,$1,$2,now())")
|
||||
.bind(&id)
|
||||
.bind(affine_doc_loader::build_full_doc("test", "body", &id).unwrap())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (started, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let first_pool = pool.clone();
|
||||
let first_id = id.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
project_current_doc(&first_pool, &first_id, &first_id, move |_| {
|
||||
started.send(()).unwrap();
|
||||
release_rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
started_rx.await.unwrap();
|
||||
first.abort();
|
||||
assert!(first.await.unwrap_err().is_cancelled());
|
||||
|
||||
let (next_started, mut next_started_rx) = tokio::sync::oneshot::channel();
|
||||
let next_pool = pool.clone();
|
||||
let next_id = id.clone();
|
||||
let next = tokio::spawn(async move {
|
||||
project_current_doc(&next_pool, &next_id, &next_id, move |_| {
|
||||
next_started.send(()).unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(200), &mut next_started_rx)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
release.send(()).unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), next_started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(next.await.unwrap().unwrap(), Some(()));
|
||||
sqlx::query("DELETE FROM snapshots WHERE workspace_id=$1 AND guid=$1")
|
||||
.bind(&id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_acl_role_is_a_permanent_search_source_error() {
|
||||
let error = readable_role("future-role", "grant").unwrap_err();
|
||||
|
||||
@@ -37,5 +37,5 @@ use workspace_state::{
|
||||
|
||||
use super::{
|
||||
ActiveGeneration, DOCUMENT_PROJECTION_FAILED, ProjectionInput, SearchChange, SearchProvider, SearchTable,
|
||||
WORKSPACE_RECONCILE_FAILED, project_document, projection_external_id, provider_payload,
|
||||
WORKSPACE_RECONCILE_FAILED, project_current_doc, project_document, projection_external_id, provider_payload,
|
||||
};
|
||||
@@ -2,9 +2,9 @@ use sqlx::{PgPool, Row};
|
||||
|
||||
use super::{
|
||||
DOCUMENT_PROJECTION_FAILED, ProjectionExpectation, RECONCILE_BATCH, WorkspacePhase, WorkspaceReconcileContext,
|
||||
WorkspaceStep, provider_projection_matches, renew_workspace_lease, upsert_document,
|
||||
WorkspaceStep, project_current_doc, provider_projection_matches, renew_workspace_lease, upsert_document,
|
||||
};
|
||||
use crate::runtime::{RuntimeError, RuntimeResult, storage_runtime::load_current_doc};
|
||||
use crate::runtime::{RuntimeError, RuntimeResult};
|
||||
|
||||
pub(super) const ROOT_DOCUMENT_BATCH_SQL: &str = r#"WITH root_document AS (
|
||||
SELECT unnest($5::text[]) AS doc_id
|
||||
@@ -44,24 +44,21 @@ impl RootReconcilePhase {
|
||||
}
|
||||
|
||||
pub(super) async fn load_root_document_ids(pool: &PgPool, workspace_id: &str) -> RuntimeResult<Vec<String>> {
|
||||
let root = load_current_doc(pool, workspace_id, workspace_id)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
RuntimeError::InvalidState(message) => RuntimeError::SearchSourceInvalid(message),
|
||||
error => error,
|
||||
})?
|
||||
.ok_or_else(|| RuntimeError::SearchSourceInvalid("workspace root doc is missing".to_string()))?;
|
||||
let projection = affine_doc_loader::project_workspace_root(root.blob, false)
|
||||
.map_err(|error| RuntimeError::SearchSourceInvalid(format!("workspace root projection failed: {error}")))?;
|
||||
if !projection.complete {
|
||||
return Err(RuntimeError::SearchSourceInvalid(
|
||||
"workspace root projection is incomplete".to_string(),
|
||||
));
|
||||
}
|
||||
let mut doc_ids = projection.doc_ids;
|
||||
doc_ids.sort();
|
||||
doc_ids.dedup();
|
||||
Ok(doc_ids)
|
||||
project_current_doc(pool, workspace_id, workspace_id, |root| {
|
||||
let projection = affine_doc_loader::project_workspace_root(root.blob, false)
|
||||
.map_err(|error| RuntimeError::SearchSourceInvalid(format!("workspace root projection failed: {error}")))?;
|
||||
if !projection.complete {
|
||||
return Err(RuntimeError::SearchSourceInvalid(
|
||||
"workspace root projection is incomplete".to_string(),
|
||||
));
|
||||
}
|
||||
let mut doc_ids = projection.doc_ids;
|
||||
doc_ids.sort();
|
||||
doc_ids.dedup();
|
||||
Ok(doc_ids)
|
||||
})
|
||||
.await?
|
||||
.ok_or_else(|| RuntimeError::SearchSourceInvalid("workspace root doc is missing".to_string()))
|
||||
}
|
||||
|
||||
async fn converge_document_membership(
|
||||
|
||||
@@ -718,6 +718,21 @@ mod tests {
|
||||
.unwrap();
|
||||
let doc_blob = if corrupt_update {
|
||||
affine_doc_loader::build_full_doc("title", "body", &doc_id).unwrap()
|
||||
} else if phase_name == "source" {
|
||||
let doc = Doc::default();
|
||||
let mut blocks = doc.get_or_create_map("blocks").unwrap();
|
||||
for (id, child) in [("a", "b"), ("b", "a")] {
|
||||
let mut block = doc.create_map().unwrap();
|
||||
let mut children = doc.create_array().unwrap();
|
||||
children.push(child).unwrap();
|
||||
block.insert("sys:id".into(), id).unwrap();
|
||||
block.insert("sys:flavour".into(), "affine:paragraph").unwrap();
|
||||
block
|
||||
.insert("sys:children".into(), y_octo::Value::Array(children))
|
||||
.unwrap();
|
||||
blocks.insert(id.to_string(), y_octo::Value::Map(block)).unwrap();
|
||||
}
|
||||
doc.encode_update_v1().unwrap()
|
||||
} else {
|
||||
vec![0]
|
||||
};
|
||||
|
||||
@@ -21,6 +21,15 @@ pub(in crate::runtime) async fn load_current_doc(
|
||||
workspace_id: &str,
|
||||
doc_id: &str,
|
||||
) -> RuntimeResult<Option<CurrentDoc>> {
|
||||
let (snapshot, updates) = load_current_doc_parts(pool, workspace_id, doc_id).await?;
|
||||
merge_current_doc(snapshot, updates)
|
||||
}
|
||||
|
||||
pub(in crate::runtime) async fn load_current_doc_parts(
|
||||
pool: &PgPool,
|
||||
workspace_id: &str,
|
||||
doc_id: &str,
|
||||
) -> RuntimeResult<(Option<CurrentDoc>, Vec<CurrentDocUpdate>)> {
|
||||
let snapshot = sqlx::query_as::<_, CurrentDoc>(
|
||||
r#"
|
||||
SELECT blob, updated_at
|
||||
@@ -46,7 +55,7 @@ pub(in crate::runtime) async fn load_current_doc(
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|err| RuntimeError::database("Current doc updates load failed", err))?;
|
||||
merge_current_doc(snapshot, updates)
|
||||
Ok((snapshot, updates))
|
||||
}
|
||||
|
||||
pub(super) async fn load_canonical_doc<'a>(
|
||||
|
||||
@@ -15,7 +15,7 @@ pub use capabilities::StorageProviderCapabilities;
|
||||
use capabilities::storage_provider_capabilities;
|
||||
pub(in crate::runtime) use config::StorageRuntimeConfig;
|
||||
pub(super) use current_doc::load_current_doc;
|
||||
pub(in crate::runtime) use current_doc::{CurrentDoc, CurrentDocUpdate, merge_current_doc};
|
||||
pub(in crate::runtime) use current_doc::{CurrentDoc, CurrentDocUpdate, load_current_doc_parts, merge_current_doc};
|
||||
use current_doc::{load_canonical_doc, load_workspace_canonical_doc_ids, load_workspace_live_doc_ids};
|
||||
use napi::bindgen_prelude::Buffer;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -279,7 +279,7 @@ export class BackendRuntimeHousekeepingJob {
|
||||
export class BackendRuntimeSearchJob {
|
||||
constructor(private readonly rt: BackendRuntimeProvider) {}
|
||||
|
||||
@Cron(CronExpression.EVERY_30_SECONDS)
|
||||
@Cron(CronExpression.EVERY_30_SECONDS, { waitForCompletion: true })
|
||||
async reconcileProjection(limit = 100) {
|
||||
if (!this.rt.searchEnabled()) return 0;
|
||||
const startedAt = performance.now();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
|
||||
import { Package } from '@affine-tools/utils/workspace';
|
||||
@@ -80,43 +81,46 @@ async function submit() {
|
||||
) {
|
||||
throw new Error('Invalid Google Play candidate metadata');
|
||||
}
|
||||
const hash = createHash('sha256');
|
||||
for await (const chunk of fs.createReadStream(bundle)) {
|
||||
hash.update(chunk);
|
||||
}
|
||||
const digest = hash.digest('hex');
|
||||
const editId = await createEdit(packageName);
|
||||
const tracks = await publisher.edits.tracks.list({ packageName, editId });
|
||||
const submitted = tracks.data.tracks?.some(track =>
|
||||
const uploaded = await publisher.edits.bundles.list({ packageName, editId });
|
||||
const existing = uploaded.data.bundles?.find(
|
||||
item => item.versionCode === candidate.versionCode
|
||||
);
|
||||
const onTrack = tracks.data.tracks?.some(track =>
|
||||
track.releases?.some(release =>
|
||||
release.versionCodes?.includes(String(candidate.versionCode))
|
||||
)
|
||||
);
|
||||
if (process.env.STORE_QUERY_ONLY === '1') {
|
||||
if ((onTrack || existing) && !existing?.sha256) {
|
||||
await publisher.edits.delete({ packageName, editId });
|
||||
if (!submitted && process.env.STORE_ALLOW_ABSENT !== '1') {
|
||||
throw new Error('Google Play version has no verifiable bundle digest');
|
||||
}
|
||||
const conflict = existing && existing.sha256 !== digest;
|
||||
const submitted = onTrack && !conflict;
|
||||
const queryOnly = process.env.STORE_QUERY_ONLY === '1';
|
||||
if (queryOnly || submitted || conflict) {
|
||||
await publisher.edits.delete({ packageName, editId });
|
||||
if (conflict && !queryOnly) {
|
||||
throw new Error('Google Play version code belongs to a different bundle');
|
||||
}
|
||||
if (
|
||||
queryOnly &&
|
||||
!submitted &&
|
||||
!conflict &&
|
||||
process.env.STORE_ALLOW_ABSENT !== '1'
|
||||
) {
|
||||
throw new Error(
|
||||
'No visible store result; the previous submission step ran, so its result remains unresolved'
|
||||
);
|
||||
}
|
||||
fs.writeFileSync(
|
||||
receipt,
|
||||
JSON.stringify({
|
||||
...candidate,
|
||||
store: 'google-play',
|
||||
status: submitted ? 'submitted' : 'not-submitted',
|
||||
...operation,
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (submitted) {
|
||||
await publisher.edits.delete({ packageName, editId });
|
||||
} else {
|
||||
const uploaded = await publisher.edits.bundles.list({
|
||||
packageName,
|
||||
editId,
|
||||
});
|
||||
if (
|
||||
!uploaded.data.bundles?.some(
|
||||
item => item.versionCode === candidate.versionCode
|
||||
)
|
||||
) {
|
||||
if (!existing) {
|
||||
const result = await publisher.edits.bundles.upload({
|
||||
packageName,
|
||||
editId,
|
||||
@@ -125,9 +129,12 @@ async function submit() {
|
||||
body: fs.createReadStream(bundle),
|
||||
},
|
||||
});
|
||||
if (result.data.versionCode !== candidate.versionCode) {
|
||||
if (
|
||||
result.data.versionCode !== candidate.versionCode ||
|
||||
result.data.sha256 !== digest
|
||||
) {
|
||||
throw new Error(
|
||||
'Uploaded bundle version does not match the prepared candidate'
|
||||
'Uploaded bundle does not match the prepared candidate'
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -149,7 +156,7 @@ async function submit() {
|
||||
...candidate,
|
||||
...operation,
|
||||
store: 'google-play',
|
||||
status: 'submitted',
|
||||
status: !queryOnly || submitted ? 'submitted' : 'not-submitted',
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user