feat(server): improve config loading (#15649)

fix #15650
fix #15530
fix #15534
fix #15506


#### PR Dependency Tree


* **PR #15649** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Summary

* **New Features**
* Admin settings show configuration sources and secret status, conceal
secret values, and allow database-level overrides to be cleared.
* Self-hosted deployments can automatically upgrade existing
configuration files.
* **Improvements**
* Configuration changes apply without restarting the server, and storage
settings reload automatically.
* Uploads support runtime-managed proxy settings and signed upload
tokens.
  * Incomplete managed Copilot profiles are disabled during migration.
* Saving an AI response as a document more reliably targets the matching
editor.
* **Documentation**
* The CLI clarifies that imported configuration changes take effect
after the next server start.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky authored and GitHub committed 2026-09-30 00:39:36 +08:00
1 parent 164a84ffe0
commit 7afb62d5b4
159 files changed
+9477 -4352

No files matched your search

+3
View File
@@ -1,5 +1,8 @@
{
"$schema": "https://github.com/toeverything/affine/releases/latest/download/config.schema.json",
"deployment": {
"type": "selfhosted"
},
"server": {
"name": "AFFiNE Self Hosted Server"
},
+3
View File
@@ -1,5 +1,8 @@
{
"$schema": "https://github.com/toeverything/affine/releases/latest/download/config.schema.json",
"deployment": {
"type": "selfhosted"
},
"server": {
"name": "AFFiNE Self-hosted",
"externalUrl": "http://localhost:3010"
+3226 -894
View File
File diff suppressed because it is too large. Load diff
+9 -10
View File
@@ -19,16 +19,6 @@ runs:
env:
PGPASSWORD: affine
- name: Run init-db script
shell: bash
env:
NODE_ENV: test
run: |
yarn affine @affine/server prisma generate
yarn affine @affine/server data-migration admit-legacy-context-blobs
yarn affine @affine/server prisma migrate deploy
yarn affine @affine/server data-migration run
- name: Import config
shell: bash
run: |
@@ -55,3 +45,12 @@ runs:
else
jq -n --argjson indexer "$default_indexer" '{indexer: $indexer}' > ./packages/backend/server/config.json
fi
- name: Run init-db script
shell: bash
env:
NODE_ENV: test
run: |
yarn affine @affine/server prisma generate
yarn affine @affine/server prisma migrate deploy
yarn affine @affine/server data-migration run
+1 -1
View File
@@ -32,4 +32,4 @@ ENV LD_PRELOAD=libjemalloc.so.2
EXPOSE 3010
CMD ["node", "./dist/main.js"]
CMD ["sh", "-c", "node ./scripts/upgrade-config.js && exec node ./dist/main.js"]
@@ -12,7 +12,7 @@ spec:
template:
metadata:
annotations:
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
checksum/server-config: {{ include "affine.serverConfig" . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
@@ -12,7 +12,7 @@ spec:
template:
metadata:
annotations:
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
checksum/server-config: {{ include "affine.serverConfig" . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
+3 -3
View File
@@ -62,8 +62,8 @@ Create the name of the service account to use
{{- end }}
{{/*
Render the indexer runtime configuration shared by the Secret and Pod checksums.
Render the server runtime configuration shared by the Secret and Pod checksums.
*/}}
{{- define "affine.indexerConfig" -}}
{{- dict "indexer" (dict "enabled" .Values.global.indexer.enabled "provider" (dict "type" .Values.global.indexer.provider "endpoint" .Values.global.indexer.endpoint "apiKey" .Values.global.indexer.apiKey "username" .Values.global.indexer.username "password" .Values.global.indexer.password)) | toJson -}}
{{- define "affine.serverConfig" -}}
{{- dict "deployment" (dict "type" (ternary "selfhosted" "cloud" (eq .Values.global.deployment.type "selfhosted"))) "indexer" (dict "enabled" .Values.global.indexer.enabled "provider" (dict "type" .Values.global.indexer.provider "endpoint" .Values.global.indexer.endpoint "apiKey" .Values.global.indexer.apiKey "username" .Values.global.indexer.username "password" .Values.global.indexer.password)) | toJson -}}
{{- end }}
@@ -8,4 +8,4 @@ metadata:
"helm.sh/hook-delete-policy": before-hook-creation
type: Opaque
data:
config.json: {{ include "affine.indexerConfig" . | b64enc | quote }}
config.json: {{ include "affine.serverConfig" . | b64enc | quote }}
@@ -17,7 +17,7 @@ spec:
template:
metadata:
annotations:
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
checksum/server-config: {{ include "affine.serverConfig" . | sha256sum }}
{{- with .Values.worker.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
+1
View File
@@ -3,6 +3,7 @@ exclude = [
"target/**/*.toml",
"packages/frontend/apps/ios/App/Packages/AffineGraphQL/**/*.toml",
"packages/frontend/apps/ios/App/Packages/Intelligents/**/*.toml",
"packages/frontend/apps/ios/App/Packages/AffinePaywall/**/*.toml",
]
# https://taplo.tamasfe.dev/configuration/formatter-options.html
Generated
+2 -3
View File
@@ -76,9 +76,9 @@ dependencies = [
[[package]]
name = "affine_core"
version = "0.0.4"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5626fc812421da8dd685e58d130ff6823d098ea4f412b9fb6350f8d14878c19"
checksum = "acda52a0c7f4ee390c1a05814ffe5ddaaa86a4f877b611505281d4d47858d721"
dependencies = [
"base64 0.23.1",
"chrono",
@@ -179,7 +179,6 @@ dependencies = [
"homedir",
"lru 0.16.3",
"objc2",
"objc2-foundation",
"serde_json",
"thiserror 2.0.18",
"tokio",
-2
View File
@@ -58,7 +58,6 @@ resolver = "3"
memory-indexer = "0.4.0"
mimalloc = "0.1"
mp4parse = "0.17"
nanoid = "0.4"
napi = { version = "3.7.0", features = [
"async",
"chrono_date",
@@ -76,7 +75,6 @@ resolver = "3"
"onepkg",
] }
p256 = { version = "0.13", features = ["ecdsa", "pem"] }
pulldown-cmark = "0.13"
rand = "0.9"
rayon = "1.10"
redis = { version = "1.6", features = ["tokio-comp", "tokio-rustls-comp"] }
+1 -1
View File
@@ -11,7 +11,7 @@ crate-type = ["cdylib"]
[dependencies]
aes-gcm = { workspace = true }
affine_common = { workspace = true, features = ["hashcash", "napi"] }
affine_core = "0.0.4"
affine_core = "0.0.5"
affine_doc_loader = { workspace = true }
anyhow = { workspace = true }
argon2 = "0.6"
+44 -5
View File
@@ -22,6 +22,7 @@ export declare class BackendRuntime {
deleteByokProfile(workspaceId: string, profileId: string): Promise<boolean>
reorderByokProfiles(input: ReorderByokProfilesInput): Promise<Array<ByokProfileOutput>>
createByokLocalLease(input: CreateByokLocalLeaseInput): Promise<ByokLocalLeaseOutput>
saveAppConfig(actor: string | undefined | null, commands: Array<AppConfigCommand>): Promise<Array<string>>
getUserQuotaStateV1(userId: string): Promise<RuntimeUserQuotaState>
getWorkspaceQuotaStateV1(workspaceId: string): Promise<RuntimeWorkspaceQuotaState>
getSyncPermissionGenerationV1(workspaceId: string): Promise<number>
@@ -29,6 +30,9 @@ export declare class BackendRuntime {
runMigrations(): Promise<void>
searchAuthorized(actorUserId: string, workspaceId: string, request: RuntimeSearchRequest): Promise<SearchOperationOutput>
aggregateAuthorized(actorUserId: string, workspaceId: string, request: RuntimeAggregateRequest): Promise<SearchOperationOutput>
searchEnabled(): boolean
copilotEnabled(): boolean
nodeCryptoPrivateKey(): string
reconcileSearchProjection(limit?: number | undefined | null): Promise<number>
authorizePermissionV1(input: any): Promise<any>
executeDomainCommandV1(input: any): Promise<any>
@@ -79,11 +83,12 @@ export declare class BackendRuntime {
cleanupExpiredRuntimeGates(limit: number): Promise<number>
cleanupExpiredUserSessions(limit: number): Promise<number>
cleanupExpiredSnapshotHistories(limit: number): Promise<number>
configureObjectStorage(configJson: string): void
start(): Promise<void>
stop(): Promise<void>
reloadConfig(privateKey?: string | undefined | null, objectStorageConfig?: string | undefined | null, inlineConfig?: string | undefined | null): Promise<void>
reloadConfig(): Promise<void>
health(): Promise<BackendRuntimeHealth>
paymentEnabled(): boolean
stripeEnabled(): boolean
executePaymentCommandV1(input: any): Promise<any>
createPaymentCustomerPortalV1(actorUserId: string): Promise<string>
createLicenseCustomerPortalV1(licenseKey: string, validateKey?: string | undefined | null): Promise<string>
@@ -119,13 +124,28 @@ export declare class BackendRuntime {
activateWorkspaceSeatV1(input: RuntimeSeatActivationInput): Promise<boolean>
reserveWorkspaceSeatsV1(input: RuntimeSeatReservationInput): Promise<RuntimeSeatReservationDecision>
reserveStorageQuotaV1(input: RuntimeStorageReservationInput): Promise<RuntimeStorageReservationDecision>
constructor(privateKey?: string | undefined | null, configPaths?: Array<string> | undefined | null, permissionTelemetry?: (((err: Error | null, arg: string) => void)) | undefined | null, inlineConfig?: string | undefined | null, invalidationEvents?: (((err: Error | null, arg: string) => void)) | undefined | null)
constructor(serverConfig: ServerConfigHandle, privateKey?: string | undefined | null, permissionTelemetry?: (((err: Error | null, arg: string) => void)) | undefined | null, invalidationEvents?: (((err: Error | null, arg: string) => void)) | undefined | null)
}
export declare class CopilotStreamHandle {
abort(): void
}
export declare class ServerConfigHandle {
constructor(path: string, legacyDeploymentType?: string | undefined | null)
get path(): string
get deploymentType(): string
nodeOwnedJson(): string
publicNativeBaselineJson(): string
hasBaselineConfigKey(key: string): boolean
baselineConfigKeyConfigured(key: string): boolean
nativeAppConfigKeys(): Array<string>
nativeSecretAppConfigKeys(): Array<string>
staticAppConfigKeys(): Array<string>
redisUrl(): string | null
redisNodeOptionsJson(): string
}
export declare class StorageRuntime {
cleanupUnreferencedWorkspaceBlobs(workspaceId: string, gracePeriodDays: number, limit: number): Promise<RuntimeBlobCleanupResult>
backfillMissingBlobMetadata(workspaceId: string | undefined | null, limit: number): Promise<RuntimeBlobMetadataBackfillResult>
@@ -135,13 +155,17 @@ export declare class StorageRuntime {
executeDocumentCleanupCandidates(workspaceId: string | undefined | null, gracePeriodDays: number, limit: number): Promise<RuntimeDocumentCleanupExecuteResult>
deleteWorkspaceObjects(workspaceId: string, userIds?: Array<string> | undefined | null): Promise<number>
reconcileWorkspaceStorage(limit: number): Promise<RuntimeWorkspaceStorageReconcileResult>
constructor()
constructor(serverConfig: ServerConfigHandle)
start(): Promise<void>
configure(configJson: string): void
reloadConfig(): Promise<void>
stop(): Promise<void>
runMigrations(): Promise<void>
health(): Promise<StorageRuntimeHealth>
providerCapabilities(scope: string): Promise<StorageProviderCapabilities>
isLocalStorage(scope: string): boolean
uploadUrlConfig(scope: string): StorageUploadUrlConfig | null
signUploadToken(scope: string, path: string, fields: Array<any>, expiresAt: number): string | null
verifyUploadToken(scope: string, path: string, fields: Array<any>, expiresAt: number, token: string): boolean
putObject(scope: string, key: string, body: Buffer, metadata?: RuntimeObjectStoragePutOptions | undefined | null): Promise<RuntimeObjectMetadata>
headObject(scope: string, key: string): Promise<RuntimeObjectMetadata | null>
getObject(scope: string, key: string): Promise<RuntimeObjectGetResult | null>
@@ -187,12 +211,22 @@ export interface AggregateOptions {
pagination: SearchPagination
}
export interface AppConfigCommand {
key: string
owner: string
operation: string
valueJson?: string
}
export interface AppConfigDescriptor {
key: string
description: string
defaultValue: any
schema: any
internal: boolean
envName?: string
envType?: string
link?: string
}
export declare function appConfigDescriptors(module: string): Array<AppConfigDescriptor>
@@ -1768,6 +1802,11 @@ export interface StorageRuntimeHealth {
bucket?: string
}
export interface StorageUploadUrlConfig {
proxyUpload: boolean
urlPrefix?: string
}
export interface SyncEmbeddingStateInput {
workspaceId: string
enabled: boolean
@@ -330,7 +330,11 @@ impl ArtifactService {
.await
.map_err(|error| RuntimeError::database("complete workspace artifact failed", error))?;
if updated.rows_affected() != 1 {
return Err(RuntimeError::invalid_state("artifact_reservation_changed"));
let row = self.get(workspace_id, content_hash).await?;
if row.status != "ready" || row.storage_scope != locator.scope.as_str() || row.storage_key != locator.key.as_str()
{
return Err(RuntimeError::invalid_state("artifact_reservation_changed"));
}
}
Ok(())
}
@@ -18,9 +18,13 @@ struct StoredKey {
id: String,
secret: String,
status: String,
// TODO(0.27.5): Remove old-reader serialization constraints after 0.27.4 exits.
#[serde(skip_serializing_if = "Option::is_none")]
created_at: Option<DateTime<Utc>>,
source: String,
#[serde(skip_serializing_if = "Option::is_none")]
retired_at: Option<DateTime<Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
verify_until: Option<DateTime<Utc>>,
}
@@ -703,6 +703,21 @@ async fn session_kernel_serializes_refresh_replay_revocation_and_key_rotation()
.unwrap()
.to_string();
keyring::rotate(&pool, &config, &user_id, &old_key).await.unwrap();
let stored_after_rotation: serde_json::Value =
sqlx::query_scalar("SELECT value FROM app_configs WHERE id='auth.session.signingKeys'")
.fetch_one(&pool)
.await
.unwrap();
let active = stored_after_rotation
.as_array()
.unwrap()
.iter()
.find(|key| key["status"] == "active")
.unwrap()
.as_object()
.unwrap();
assert!(!active.contains_key("retiredAt"));
assert!(!active.contains_key("verifyUntil"));
let old_access = rotated["accessToken"].as_str().unwrap();
let old_principal = principal::resolve(
&pool,
@@ -380,8 +380,15 @@ async fn doc_append_invalidation_fences_inflight_refresh_and_stamp_covers_redis_
let publisher = InvalidationRuntime::start(&config, false, Arc::new(NoopInvalidationTarget)).await;
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
let redis_update =
affine_doc_loader::update_doc(&initial_blob, "![redis](blob://redis-key)", &fixture.doc_id).unwrap();
let initial_markdown = affine_doc_loader::export_markdown_source(&initial_blob, &fixture.doc_id, None)
.unwrap()
.markdown;
let redis_update = affine_doc_loader::update_doc(
&initial_blob,
&format!("{initial_markdown}\n![redis](blob://redis-key)"),
&fixture.doc_id,
)
.unwrap();
append_updates(
&fixture.pool,
Some(publisher.clone()),
@@ -413,8 +420,15 @@ async fn doc_append_invalidation_fences_inflight_refresh_and_stamp_covers_redis_
.authorize_and_load(Some(&fixture.member_user_id), current(&fixture))
.await
.unwrap();
let loss_update =
affine_doc_loader::update_doc(&current_source.blob, "![loss](blob://loss-key)", &fixture.doc_id).unwrap();
let current_markdown = affine_doc_loader::export_markdown_source(&current_source.blob, &fixture.doc_id, None)
.unwrap()
.markdown;
let loss_update = affine_doc_loader::update_doc(
&current_source.blob,
&format!("{current_markdown}\n![loss](blob://loss-key)"),
&fixture.doc_id,
)
.unwrap();
append_updates(
&fixture.pool,
None,
@@ -0,0 +1,56 @@
use napi::Result;
use super::{AppConfigChange, BackendRuntime, napi_error, save_app_config_changes, to_napi_error};
#[napi_derive::napi(object)]
pub struct AppConfigCommand {
pub key: String,
pub owner: String,
pub operation: String,
pub value_json: Option<String>,
}
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub async fn save_app_config(&self, actor: Option<String>, commands: Vec<AppConfigCommand>) -> Result<Vec<String>> {
let changes = commands
.into_iter()
.map(|command| {
let native = match command.owner.as_str() {
"native" => true,
"node" => false,
_ => return Err(napi_error("app config owner must be native or node")),
};
let value = match command.operation.as_str() {
"set" => Some(
serde_json::from_str(
command
.value_json
.as_deref()
.ok_or_else(|| napi_error("app config set requires a value"))?,
)
.map_err(|_| napi_error("invalid app config JSON value"))?,
),
"clear" if command.value_json.is_none() => None,
"clear" => return Err(napi_error("app config clear must not include a value")),
_ => return Err(napi_error("app config operation must be set or clear")),
};
Ok(AppConfigChange {
key: command.key,
value,
native,
})
})
.collect::<Result<Vec<_>>>()?;
save_app_config_changes(
&self.pool().await.map_err(to_napi_error)?,
&self.server_config,
self.bootstrap_private_key.as_ref().map(|key| key.to_string()),
actor.as_deref(),
&changes,
)
.await
.map_err(to_napi_error)
}
}
@@ -92,6 +92,21 @@ impl BackendRuntime {
Ok(search_operation_output(result))
}
#[napi]
pub fn search_enabled(&self) -> Result<bool> {
Ok(self.config()?.search.enabled)
}
#[napi]
pub fn copilot_enabled(&self) -> Result<bool> {
Ok(self.config()?.copilot.enabled)
}
#[napi]
pub fn node_crypto_private_key(&self) -> Result<String> {
Ok(self.config()?.private_key.as_str().to_string())
}
#[napi]
pub async fn reconcile_search_projection(&self, limit: Option<i32>) -> Result<i32> {
self.require_background()?;
@@ -2,7 +2,9 @@ use super::{
license::{RuntimeLicenseInstallInput, RuntimeLicenseRefreshInput},
*,
};
use crate::runtime::{backend_runtime::tests::runtime_from_database_url, migrations::DATABASE_TEST_LOCK};
use crate::runtime::{
BackendRuntimeConfig, backend_runtime::tests::runtime_from_database_url, migrations::DATABASE_TEST_LOCK,
};
pub(super) async fn fixture(runtime: &BackendRuntime) -> (String, String) {
let owner = uuid::Uuid::new_v4().to_string();
@@ -171,9 +173,10 @@ async fn license_upgrade_recovers_installed_files_and_online_renewals_before_wor
}
}
runtime.stop().await.unwrap();
std::sync::Arc::get_mut(&mut runtime.config.write().unwrap())
.unwrap()
.deployment = Deployment::SelfHosted;
let directory = tempfile::tempdir().unwrap();
let config_path = directory.path().join("config.json");
std::fs::write(&config_path, r#"{"deployment":{"type":"selfhosted"}}"#).unwrap();
runtime.server_config = std::sync::Arc::new(crate::runtime::config::ServerConfig::open(&config_path, None).unwrap());
runtime.start().await.unwrap();
assert!(runtime.license_health_worker.lock().await.is_none());
assert_eq!(remote.requests().len(), 1);
@@ -455,10 +458,9 @@ async fn license_install_revoke_rollback_and_stale_fences() {
.unwrap();
assert!(
!runtime
.get_byok_entitlement_v1(workspace.clone(), None)
.has_workspace_commercial_entitlement_v1(workspace.clone())
.await
.unwrap()
.server
);
let status: String = sqlx::query_scalar("SELECT status FROM entitlements WHERE subject_id=$1")
.bind(&key)
@@ -774,11 +776,30 @@ async fn authoritative_byok_and_mixed_owner_storage_use_current_facts() {
assert!(access.local && access.server);
assert!(
!runtime
.get_byok_entitlement_v1("missing".into(), Some(owner))
.get_byok_entitlement_v1("missing".into(), Some(owner.clone()))
.await
.unwrap()
.local
);
let cloud_config = runtime.config().unwrap();
*runtime.config.write().unwrap() = std::sync::Arc::new(BackendRuntimeConfig {
database_url: cloud_config.database_url.clone(),
auth: cloud_config.auth.clone(),
invite_quota: cloud_config.invite_quota.clone(),
private_key: std::sync::Arc::clone(&cloud_config.private_key),
deployment: Deployment::SelfHosted,
copilot: cloud_config.copilot.clone(),
search: cloud_config.search.clone(),
redis: cloud_config.redis.clone(),
payment: cloud_config.payment.clone(),
});
let access = runtime.get_byok_entitlement_v1(workspace.clone(), None).await.unwrap();
assert!(access.server && access.local);
let access = runtime
.get_byok_entitlement_v1(workspace.clone(), Some(owner))
.await
.unwrap();
assert!(access.server && access.local);
sqlx::query("DELETE FROM workspace_members WHERE workspace_id=$1 AND role='owner'")
.bind(&workspace)
.execute(&pool)
@@ -4,8 +4,8 @@ use napi::Result;
use sqlx::{Row, postgres::PgPoolOptions};
use super::{
BackendRuntime, BackendRuntimeConfig, BackendRuntimeHealth, EmbeddingHealth, ObjectStorageService, RuntimeError,
RuntimeResult, SearchRuntime, blob_access, embedding_schema_health, invalidation,
BackendRuntime, BackendRuntimeConfig, BackendRuntimeHealth, EmbeddingHealth, RuntimeError, RuntimeResult,
SearchRuntime, blob_access, embedding_schema_health, invalidation,
invalidation::{InvalidationHintV1, InvalidationTarget},
napi_error,
payment::PaymentRuntime,
@@ -14,16 +14,6 @@ use super::{
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub fn configure_object_storage(&self, config_json: String) -> Result<()> {
let object_storage = ObjectStorageService::from_config_json(&config_json).map_err(to_napi_error)?;
*self
.object_storage
.write()
.map_err(|_| napi_error("object storage service lock poisoned"))? = Arc::new(object_storage);
Ok(())
}
#[napi]
pub async fn start(&self) -> Result<()> {
self.start_inner().await.map_err(to_napi_error)
@@ -50,14 +40,12 @@ impl BackendRuntime {
.map_err(|err| RuntimeError::database("BackendRuntime postgres health check failed", err))?;
let initialized = async {
let inline_config = self
.inline_config
.read()
.map_err(|_| RuntimeError::invalid_state("BackendRuntime inline config lock poisoned"))?
.clone();
let config = self
.config()?
.with_db_overrides(&pool, &self.config_source, inline_config.as_ref())
let baseline = BackendRuntimeConfig::from_server_config(
self.bootstrap_private_key.as_ref().map(|key| key.to_string()),
&self.server_config,
)?;
let config = baseline
.with_db_overrides_from_server_config(&pool, &self.server_config)
.await?;
let deployment = config.deployment;
let redis = config.redis.clone();
@@ -77,7 +65,8 @@ impl BackendRuntime {
None
};
*self.payment.lock().await = payment;
let object_storage = self.object_storage()?.with_db_overrides(&pool).await?;
let storage_baseline = self.server_config.object_storage().clone();
let object_storage = storage_baseline.with_db_overrides(&pool).await?;
*self
.object_storage
.write()
@@ -231,36 +220,24 @@ impl BackendRuntime {
}
#[napi]
pub async fn reload_config(
&self,
private_key: Option<String>,
object_storage_config: Option<String>,
inline_config: Option<String>,
) -> Result<()> {
pub async fn reload_config(&self) -> Result<()> {
let _reload = self.config_reload.lock().await;
let pool = self.pool().await.map_err(to_napi_error)?;
let active_private_key = self.config().map_err(to_napi_error)?.private_key.to_string();
let inline_config = inline_config
.map(|value| serde_json::from_str(&value))
.transpose()
.map_err(|error| to_napi_error(RuntimeError::json("decode inline runtime config", error)))?
.or_else(|| self.inline_config.read().ok().and_then(|value| value.clone()));
let config = BackendRuntimeConfig::from_config_source_with_inline(
private_key.or(Some(active_private_key)),
&self.config_source,
inline_config.as_ref(),
let config = BackendRuntimeConfig::from_server_config(
self.bootstrap_private_key.as_ref().map(|key| key.to_string()),
&self.server_config,
)
.map_err(to_napi_error)?
.with_db_overrides(&pool, &self.config_source, inline_config.as_ref())
.await
.map_err(to_napi_error)?;
let object_storage = match object_storage_config {
Some(config) => ObjectStorageService::from_config_json(&config).map_err(to_napi_error)?,
None => self.object_storage().map_err(to_napi_error)?.as_ref().clone(),
}
.with_db_overrides(&pool)
.with_db_overrides_from_server_config(&pool, &self.server_config)
.await
.map_err(to_napi_error)?;
let object_storage = self
.server_config
.object_storage()
.clone()
.with_db_overrides(&pool)
.await
.map_err(to_napi_error)?;
let search = if !self.script_mode && config.search.enabled {
if config.search.provider == "embedded" && !self.role.allows_embedded_search() {
return Err(napi_error(format!(
@@ -314,11 +291,6 @@ impl BackendRuntime {
} else {
None
};
*self
.inline_config
.write()
.map_err(|_| napi_error("BackendRuntime inline config lock poisoned"))? = inline_config;
let embedding = self.embedding.lock().await.as_ref().cloned();
if let Some(embedding) = embedding {
embedding
@@ -4,6 +4,7 @@ mod auth_session;
mod blob_access;
mod byok;
mod byok_api;
mod config_command;
mod constants;
mod control_plane;
mod copilot;
@@ -44,6 +45,7 @@ use search::SearchRuntime;
use sha2::{Digest, Sha256};
use sqlx::{PgPool, Row};
use tokio::sync::Mutex;
use zeroize::Zeroizing;
use self::{
blob_access::SourceIdentity,
@@ -59,14 +61,17 @@ use self::{
use super::object_storage::ObjectStorageService;
pub(crate) use super::types;
pub(super) use super::{
BackendRuntimeConfig, ConfigSource, InviteQuotaConfig, RedisRuntimeConfig, RuntimeError, RuntimeResult,
BackendRuntimeConfig, InviteQuotaConfig, RedisRuntimeConfig, RuntimeError, RuntimeResult,
migrations::{embedding_schema_health, migrate_all_tables},
napi_error, to_napi_error, webpki_tls_config,
};
use crate::llm::{
ByokLocalLeaseOutput, ByokPolicyOutput, ByokProbeResultOutput, ByokProfileOutput, CreateByokLocalLeaseInput,
CreateByokProfileInput, ProbeByokDraftInput, ProbeByokProfileInput, ReorderByokProfilesInput,
ReplaceByokProfileInput, RotateByokCredentialInput,
use crate::{
llm::{
ByokLocalLeaseOutput, ByokPolicyOutput, ByokProbeResultOutput, ByokProfileOutput, CreateByokLocalLeaseInput,
CreateByokProfileInput, ProbeByokDraftInput, ProbeByokProfileInput, ReorderByokProfilesInput,
ReplaceByokProfileInput, RotateByokCredentialInput,
},
runtime::config::{AppConfigChange, ServerConfig, ServerConfigHandle, save_app_config_changes},
};
pub(super) fn token_hash(token: &str) -> String {
@@ -104,8 +109,8 @@ fn search_operation_output(result: RuntimeResult<serde_json::Value>) -> SearchOp
#[derive(Clone)]
#[napi_derive::napi]
pub struct BackendRuntime {
config_source: ConfigSource,
inline_config: Arc<RwLock<Option<serde_json::Value>>>,
server_config: Arc<ServerConfig>,
bootstrap_private_key: Option<Arc<Zeroizing<String>>>,
role: ServerRole,
script_mode: bool,
config: Arc<RwLock<Arc<BackendRuntimeConfig>>>,
@@ -130,25 +135,18 @@ pub struct BackendRuntime {
impl BackendRuntime {
#[napi(constructor)]
pub fn new(
server_config: &ServerConfigHandle,
private_key: Option<String>,
config_paths: Option<Vec<String>>,
permission_telemetry: Option<ThreadsafeFunction<String, (), String, Status, true, true, 1024>>,
inline_config: Option<String>,
invalidation_events: Option<ThreadsafeFunction<String, (), String, Status, true, true, 1024>>,
) -> Result<Self> {
let config_source = ConfigSource::new(config_paths);
let (role, script_mode) = ServerRole::from_environment().map_err(napi_error)?;
let inline_config = inline_config
.map(|value| serde_json::from_str(&value))
.transpose()
.map_err(|error| to_napi_error(RuntimeError::json("decode inline runtime config", error)))?;
let config =
BackendRuntimeConfig::from_config_source_with_inline(private_key, &config_source, inline_config.as_ref())
.map_err(to_napi_error)?;
let object_storage = ObjectStorageService::from_config_source(&config_source).map_err(to_napi_error)?;
BackendRuntimeConfig::from_server_config(private_key.clone(), &server_config.inner).map_err(to_napi_error)?;
let object_storage = server_config.inner.object_storage().clone();
Ok(Self {
config_source,
inline_config: Arc::new(RwLock::new(inline_config)),
server_config: Arc::clone(&server_config.inner),
bootstrap_private_key: private_key.map(|key| Arc::new(Zeroizing::new(key))),
role,
script_mode,
config: Arc::new(RwLock::new(Arc::new(config))),
@@ -48,6 +48,7 @@ impl PaymentRuntime {
.get::<Option<String>, _>("validate_key")
.unwrap_or_else(|| operation_id.to_string());
let envelope = issue_license(
self.license_issuer_private_key.as_ref().map(|key| key.as_str()),
license_key,
workspace_id,
subscription.quantity,
@@ -145,6 +146,7 @@ impl PaymentRuntime {
}
let subscription = active_license_subscription(&mut tx, license_key, &namespace).await?;
let envelope = issue_license(
self.license_issuer_private_key.as_ref().map(|key| key.as_str()),
license_key,
workspace_id,
subscription.quantity,
@@ -304,14 +306,14 @@ fn decode_subscription(row: sqlx::postgres::PgRow) -> RuntimeResult<ActiveLicens
}
fn issue_license(
private_key: Option<&str>,
license_id: &str,
workspace_id: &str,
seat_quantity: i32,
subscription_end: Option<DateTime<Utc>>,
now: DateTime<Utc>,
) -> RuntimeResult<Vec<u8>> {
let private_key = std::env::var("AFFINE_PRO_LICENSE_PRIVATE_KEY")
.map_err(|_| RuntimeError::invalid_state("license_private_key_missing"))?;
let private_key = private_key.ok_or_else(|| RuntimeError::invalid_state("license_private_key_missing"))?;
LicenseIssuer::issue(
LicenseIssuance {
license_id,
@@ -320,7 +322,7 @@ fn issue_license(
subscription_end,
now,
},
&private_key,
private_key,
)
.map_err(|_| RuntimeError::invalid_state("license_issue_failed"))
}
@@ -5,6 +5,7 @@ use serde_json::Value;
use sha2::{Digest, Sha256};
use sqlx::{PgPool, postgres::PgPoolOptions};
use tokio::sync::{Mutex, Semaphore};
use zeroize::Zeroizing;
use super::{
super::{BackendRuntime, to_napi_error},
@@ -22,6 +23,7 @@ pub(in crate::runtime::backend_runtime) struct PaymentRuntime {
pub(super) deployment: Deployment,
pub(super) revenuecat_config: Option<crate::runtime::RevenueCatRuntimeConfig>,
pub(super) mail_hash_key: [u8; 32],
pub(super) license_issuer_private_key: Option<Arc<Zeroizing<String>>>,
pub(super) worker: Mutex<Option<super::PaymentWorker>>,
}
@@ -51,6 +53,9 @@ impl PaymentRuntime {
deployment,
revenuecat_config: config.revenuecat.clone(),
mail_hash_key: Sha256::digest(private_key.as_bytes()).into(),
license_issuer_private_key: std::env::var("AFFINE_PRO_LICENSE_PRIVATE_KEY")
.ok()
.map(|key| Arc::new(Zeroizing::new(key))),
worker: Mutex::new(None),
})
}
@@ -223,6 +228,24 @@ impl PaymentRuntime {
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub fn payment_enabled(&self) -> napi::Result<bool> {
let config = self
.config
.read()
.map_err(|_| napi::Error::from_reason("BackendRuntime config lock poisoned"))?;
Ok(config.payment.enabled)
}
#[napi]
pub fn stripe_enabled(&self) -> napi::Result<bool> {
let config = self
.config
.read()
.map_err(|_| napi::Error::from_reason("BackendRuntime config lock poisoned"))?;
Ok(config.payment.enabled && config.payment.stripe.is_some())
}
#[napi]
pub async fn execute_payment_command_v1(&self, input: Value) -> napi::Result<Value> {
let outcome = self
@@ -777,6 +777,7 @@ async fn lock_set_expansion_discards_old_snapshots() {
deployment: Deployment::Cloud,
revenuecat_config: None,
mail_hash_key: [0; 32],
license_issuer_private_key: None,
worker: tokio::sync::Mutex::new(None),
};
runtime.apply_with_connection(connection, complete).await.unwrap();
@@ -981,6 +982,7 @@ async fn operation_intent_is_frozen_before_send_and_blocks_overlapping_work() {
deployment: Deployment::Cloud,
revenuecat_config: None,
mail_hash_key: [0; 32],
license_issuer_private_key: None,
worker: tokio::sync::Mutex::new(None),
};
assert_eq!(
@@ -1086,6 +1088,7 @@ async fn operation_intent_is_frozen_before_send_and_blocks_overlapping_work() {
deployment: Deployment::Cloud,
revenuecat_config: None,
mail_hash_key: [0; 32],
license_issuer_private_key: None,
worker: tokio::sync::Mutex::new(None),
};
let provisioned = catalog_runtime
@@ -1223,6 +1226,7 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
deployment: Deployment::Cloud,
revenuecat_config: None,
mail_hash_key: [0; 32],
license_issuer_private_key: None,
worker: tokio::sync::Mutex::new(None),
};
let raw_event = json!({
@@ -1774,6 +1778,7 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
deployment: Deployment::Cloud,
revenuecat_config: None,
mail_hash_key: [0; 32],
license_issuer_private_key: None,
worker: tokio::sync::Mutex::new(None),
};
assert!(
@@ -1986,15 +1991,11 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
deployment: Deployment::Cloud,
revenuecat_config: None,
mail_hash_key: [0; 32],
license_issuer_private_key: Some(Arc::new(zeroize::Zeroizing::new(
crate::entitlement::tests::TEST_PRIVATE_KEY.to_string(),
))),
worker: tokio::sync::Mutex::new(None),
};
let previous_private_key = std::env::var_os("AFFINE_PRO_LICENSE_PRIVATE_KEY");
unsafe {
std::env::set_var(
"AFFINE_PRO_LICENSE_PRIVATE_KEY",
crate::entitlement::tests::TEST_PRIVATE_KEY,
);
}
for legacy in [true, false] {
let license_key = format!("rfc12-{license_marker}-{legacy}");
let license_source = format!("sub-{license_marker}-{legacy}");
@@ -2196,13 +2197,6 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
.unwrap();
assert_eq!(binding, (Some("new-workspace".into()), Some(new_generation)));
}
unsafe {
if let Some(previous_private_key) = previous_private_key {
std::env::set_var("AFFINE_PRO_LICENSE_PRIVATE_KEY", previous_private_key);
} else {
std::env::remove_var("AFFINE_PRO_LICENSE_PRIVATE_KEY");
}
}
cleanup(&pool, &license_namespace_key, &license_marker).await;
}
@@ -3210,6 +3204,7 @@ async fn revenuecat_access_and_missing_source_rules_fail_closed() {
deployment: Deployment::Cloud,
revenuecat_config: Some(config),
mail_hash_key: [0; 32],
license_issuer_private_key: None,
worker: tokio::sync::Mutex::new(None),
};
let revenuecat_event = json!({
@@ -124,6 +124,8 @@ pub(super) fn final_storage_locator(
#[cfg(test)]
mod tests {
use std::sync::Arc;
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use sha2::{Digest, Sha256};
@@ -132,13 +134,19 @@ mod tests {
#[tokio::test]
async fn promotes_valid_blob_and_rejects_late_overwrite() {
let temp = tempfile::tempdir().unwrap();
let runtime = BackendRuntime::new(None, None, None, None, None).unwrap();
runtime
.configure_object_storage(format!(
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"promotion","config":{{"path":{}}}}}}}}}"#,
let config_path = temp.path().join("config.json");
std::fs::write(
&config_path,
format!(
r#"{{"deployment":{{"type":"cloud"}},"storages":{{"blob.storage":{{"provider":"fs","bucket":"promotion","config":{{"path":{}}}}}}}}}"#,
serde_json::to_string(temp.path()).unwrap()
))
.unwrap();
),
)
.unwrap();
let config = crate::runtime::config::ServerConfigHandle {
inner: Arc::new(crate::runtime::config::ServerConfig::open(&config_path, None).unwrap()),
};
let runtime = BackendRuntime::new(&config, None, None, None).unwrap();
let body = b"immutable blob".to_vec();
let key = URL_SAFE_NO_PAD.encode(Sha256::digest(&body));
let reservation_id = Uuid::new_v4();
@@ -1,5 +1,5 @@
use anyhow::{Context, Result as AnyResult, anyhow};
use sqlx::postgres::PgPoolOptions;
use anyhow::{Context, Result as AnyResult, anyhow, ensure};
use sqlx::{Row, postgres::PgPoolOptions};
use super::{
super::migrations::{RUNTIME_MIGRATIONS, migrate_runtime_tables},
@@ -10,6 +10,167 @@ pub(super) fn pg_test_lock() -> &'static tokio::sync::Mutex<()> {
&crate::runtime::migrations::DATABASE_TEST_LOCK
}
fn test_server_config() -> AnyResult<Arc<ServerConfig>> {
let directory = tempfile::tempdir()?;
let path = directory.path().join("config.json");
std::fs::write(&path, r#"{"deployment":{"type":"cloud"}}"#)?;
Ok(Arc::new(
ServerConfig::open(&path, None).map_err(|error| anyhow!(error.to_string()))?,
))
}
fn configure_test_storage(runtime: &BackendRuntime, bucket: &str, path: &std::path::Path) {
let config = serde_json::json!({
"storages": {
"blob.storage": {
"provider": "fs",
"bucket": bucket,
"config": { "path": path.to_string_lossy() }
}
}
});
let storage = ObjectStorageService::from_config_json(&config.to_string()).unwrap();
*runtime.object_storage.write().unwrap() = Arc::new(storage);
}
#[tokio::test]
async fn shared_config_file_database_schema_and_runtime_harness() -> AnyResult<()> {
let _guard = pg_test_lock().lock().await;
let Ok(database_url) = std::env::var("DATABASE_URL") else {
return Ok(());
};
let directory = tempfile::tempdir()?;
let config_path = directory.path().join("config.json");
let file_storage_path = directory.path().join("file-storage");
let db_storage_path = directory.path().join("db-storage");
std::fs::write(
&config_path,
serde_json::json!({
"deployment": { "type": "cloud" },
"copilot": { "enabled": true },
"storages": {
"blob.storage": {
"provider": "fs",
"bucket": "file-bucket",
"config": { "path": file_storage_path }
}
}
})
.to_string(),
)?;
let handle = ServerConfigHandle::new(config_path.to_string_lossy().into_owned(), None)?;
ensure!(handle.inner.path() == config_path.canonicalize()?);
ensure!(
crate::runtime::app_config_descriptors("copilot".to_string())?
.iter()
.any(|descriptor| descriptor.key == "enabled" && descriptor.default_value == serde_json::json!(false))
);
ensure!(
crate::runtime::validate_app_config_value("copilot".to_string(), "enabled".to_string(), serde_json::json!(true))?
.is_empty()
);
ensure!(
!crate::runtime::validate_app_config_value(
"copilot".to_string(),
"enabled".to_string(),
serde_json::json!("true")
)?
.is_empty()
);
let pool = PgPoolOptions::new().max_connections(2).connect(&database_url).await?;
let keys = ["copilot.enabled", "indexer.enabled", "storages.blob.storage"];
let originals =
sqlx::query("SELECT id, value, created_at, updated_at, last_updated_by FROM app_configs WHERE id = ANY($1)")
.bind(keys)
.fetch_all(&pool)
.await?
.into_iter()
.map(|row| {
(
row.get::<String, _>("id"),
row.get::<serde_json::Value, _>("value"),
row.get::<chrono::DateTime<chrono::Utc>, _>("created_at"),
row.get::<chrono::DateTime<chrono::Utc>, _>("updated_at"),
row.get::<Option<String>, _>("last_updated_by"),
)
})
.collect::<Vec<_>>();
for (key, value) in [
("copilot.enabled", serde_json::json!(false)),
("indexer.enabled", serde_json::json!(false)),
(
"storages.blob.storage",
serde_json::json!({
"provider": "fs",
"bucket": "db-bucket",
"config": { "path": db_storage_path }
}),
),
] {
sqlx::query(
"INSERT INTO app_configs(id,value,created_at,updated_at) VALUES($1,$2,now(),now()) ON CONFLICT(id) DO UPDATE \
SET value=$2,updated_at=now()",
)
.bind(key)
.bind(value)
.execute(&pool)
.await?;
}
let mut runtime = BackendRuntime::new(&handle, Some("test-private-key".to_string()), None, None)?;
runtime.role = ServerRole::Frontend;
let storage = crate::runtime::storage_runtime::StorageRuntime::new(&handle)?;
let result: AnyResult<()> = async {
runtime.start().await.map_err(|error| anyhow!(error.to_string()))?;
storage.start().await.map_err(|error| anyhow!(error.to_string()))?;
let backend_health = runtime.health().await?;
ensure!(backend_health.started && backend_health.database_connected);
ensure!(!runtime.config()?.copilot.enabled);
let storage_health = storage.health().await?;
ensure!(storage_health.started && storage_health.database_connected);
ensure!(storage_health.bucket.as_deref() == Some("db-bucket"));
sqlx::query("DELETE FROM app_configs WHERE id = ANY($1)")
.bind(keys)
.execute(&pool)
.await?;
runtime
.reload_config()
.await
.map_err(|error| anyhow!(error.to_string()))?;
storage
.reload_config()
.await
.map_err(|error| anyhow!(error.to_string()))?;
ensure!(runtime.config()?.copilot.enabled);
ensure!(storage.health().await?.bucket.as_deref() == Some("file-bucket"));
Ok(())
}
.await;
let storage_stop = storage.stop().await;
let runtime_stop = runtime.stop().await;
sqlx::query("DELETE FROM app_configs WHERE id = ANY($1)")
.bind(keys)
.execute(&pool)
.await?;
for (key, value, created_at, updated_at, last_updated_by) in originals {
sqlx::query("INSERT INTO app_configs(id,value,created_at,updated_at,last_updated_by) VALUES($1,$2,$3,$4,$5)")
.bind(key)
.bind(value)
.bind(created_at)
.bind(updated_at)
.bind(last_updated_by)
.execute(&pool)
.await?;
}
pool.close().await;
result?;
storage_stop.map_err(|error| anyhow!(error.to_string()))?;
runtime_stop.map_err(|error| anyhow!(error.to_string()))?;
Ok(())
}
#[test]
fn migrations_include_runtime_tables_without_worker_heartbeats() {
assert!(RUNTIME_MIGRATIONS.contains("runtime_states"));
@@ -57,25 +218,13 @@ async fn failed_start_rolls_back_resources_and_can_restart() -> AnyResult<()> {
pool.close().await;
}
runtime.role = ServerRole::Frontend;
{
let mut config = runtime.config.write().unwrap();
let current = config.as_ref();
*config = Arc::new(BackendRuntimeConfig {
database_url: current.database_url.clone(),
auth: current.auth.clone(),
invite_quota: current.invite_quota.clone(),
private_key: Arc::clone(&current.private_key),
deployment: crate::runtime::Deployment::SelfHosted,
copilot: current.copilot.clone(),
search: crate::runtime::config::SearchRuntimeConfig {
enabled: true,
provider: "embedded".to_string(),
..Default::default()
},
redis: current.redis.clone(),
payment: current.payment.clone(),
});
}
let directory = tempfile::tempdir()?;
let config_path = directory.path().join("config.json");
std::fs::write(
&config_path,
r#"{"deployment":{"type":"selfhosted"},"indexer":{"enabled":true,"provider":{"type":"embedded"}}}"#,
)?;
runtime.server_config = Arc::new(ServerConfig::open(&config_path, None)?);
let error = runtime.start_inner().await.unwrap_err();
assert!(error.to_string().contains("embedded search is only available"));
@@ -87,21 +236,8 @@ async fn failed_start_rolls_back_resources_and_can_restart() -> AnyResult<()> {
assert!(runtime.embedding.lock().await.is_none());
assert!(runtime.embedding_worker.lock().await.is_none());
{
let mut config = runtime.config.write().unwrap();
let current = config.as_ref();
*config = Arc::new(BackendRuntimeConfig {
database_url: current.database_url.clone(),
auth: current.auth.clone(),
invite_quota: current.invite_quota.clone(),
private_key: Arc::clone(&current.private_key),
deployment: current.deployment,
copilot: current.copilot.clone(),
search: Default::default(),
redis: current.redis.clone(),
payment: current.payment.clone(),
});
}
std::fs::write(&config_path, r#"{"deployment":{"type":"selfhosted"}}"#)?;
runtime.server_config = Arc::new(ServerConfig::open(&config_path, None)?);
runtime.role = ServerRole::AllInOne;
runtime
.start_inner()
@@ -128,12 +264,15 @@ async fn failed_config_reload_keeps_active_resources() -> AnyResult<()> {
};
runtime.role = ServerRole::Frontend;
let config_path = std::env::temp_dir().join(format!("affine-runtime-config-{}.json", uuid::Uuid::new_v4()));
std::fs::write(&config_path, r#"{"indexer":{"enabled":true,"provider":"embedded"}}"#)?;
runtime.config_source = ConfigSource::new(Some(vec![config_path.to_string_lossy().into_owned()]));
std::fs::write(
&config_path,
r#"{"deployment":{"type":"cloud"},"indexer":{"enabled":true,"provider":{"type":"embedded"}}}"#,
)?;
runtime.server_config = Arc::new(ServerConfig::open(&config_path, None)?);
let active_config = runtime.config()?;
let active_storage = runtime.object_storage()?;
let result = runtime.reload_config(None, None, None).await;
let result = runtime.reload_config().await;
std::fs::remove_file(config_path)?;
assert!(result.is_err());
@@ -209,9 +348,10 @@ pub(super) async fn runtime_from_database_url() -> AnyResult<Option<BackendRunti
.await
.context("cleanup invite abuse subjects for backend runtime tests")?;
let server_config = test_server_config()?;
Ok(Some(BackendRuntime {
config_source: Default::default(),
inline_config: Arc::new(RwLock::new(None)),
server_config: Arc::clone(&server_config),
bootstrap_private_key: None,
role: ServerRole::AllInOne,
script_mode: false,
config: Arc::new(RwLock::new(Arc::new(BackendRuntimeConfig {
@@ -228,9 +368,7 @@ pub(super) async fn runtime_from_database_url() -> AnyResult<Option<BackendRunti
config_reload: Arc::new(Mutex::new(())),
pool: Arc::new(Mutex::new(Some(pool))),
embedding_health: Arc::new(RwLock::new(super::EmbeddingHealth::disabled("test", None))),
object_storage: Arc::new(RwLock::new(Arc::new(
crate::runtime::object_storage::ObjectStorageService::from_config_files()?,
))),
object_storage: Arc::new(RwLock::new(Arc::new(server_config.object_storage().clone()))),
embedding: Arc::new(Mutex::new(None)),
embedding_worker: Arc::new(Mutex::new(None)),
search: Arc::new(Mutex::new(None)),
@@ -407,8 +545,8 @@ async fn runtime_gate_sql_semantics_are_atomic_and_ttl_bound() {
let mut tasks = Vec::new();
for _ in 0..16 {
let runtime = BackendRuntime {
config_source: Default::default(),
inline_config: Arc::new(RwLock::new(None)),
server_config: Arc::clone(&runtime.server_config),
bootstrap_private_key: None,
role: ServerRole::AllInOne,
script_mode: false,
config: Arc::new(RwLock::new(runtime.config().unwrap())),
@@ -807,12 +945,7 @@ async fn strict_storage_reservation_serializes_last_bytes_and_recovers_ledger_ro
};
let pool = runtime.pool().await.unwrap();
let temp = tempfile::tempdir().unwrap();
runtime
.configure_object_storage(format!(
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"strict-storage","config":{{"path":{}}}}}}}}}"#,
serde_json::to_string(temp.path()).unwrap()
))
.unwrap();
configure_test_storage(&runtime, "strict-storage", temp.path());
let user_id = "rust-test:strict-storage:user";
let workspace_id = "rust-test:strict-storage:workspace";
sqlx::query("DELETE FROM workspaces WHERE id=$1")
@@ -1248,12 +1381,7 @@ async fn strict_blob_management_authorizes_inventory_and_denies_before_object_cl
return;
};
let temp = tempfile::tempdir().unwrap();
runtime
.configure_object_storage(format!(
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"blob-management","config":{{"path":{}}}}}}}}}"#,
serde_json::to_string(temp.path()).unwrap()
))
.unwrap();
configure_test_storage(&runtime, "blob-management", temp.path());
let pool = runtime.pool().await.unwrap();
let workspace_id = "rust-test:blob-management:workspace";
let owner_id = "rust-test:blob-management:owner";
@@ -1581,12 +1709,7 @@ async fn strict_comment_reservation_rejects_hijack_and_cleans_expired_generation
.unwrap();
}
let temp = tempfile::tempdir().unwrap();
runtime
.configure_object_storage(format!(
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"strict-comment","config":{{"path":{}}}}}}}}}"#,
serde_json::to_string(temp.path()).unwrap()
))
.unwrap();
configure_test_storage(&runtime, "strict-comment", temp.path());
while runtime.cleanup_expired_storage_reservations_v1(100).await.unwrap() > 0 {}
let input = |user_id: &str| types::RuntimeStorageReservationInput {
workspace_id: workspace_id.to_string(),
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,918 @@
use jsonschema::Draft;
use napi::{Error, Result, Status};
use schemars::generate::SchemaSettings;
use serde_json::{Value, from_value, json, to_value};
use super::{
AuthConfigFile, CopilotRuntimeConfig, CopilotRuntimeConfigFile, CryptoConfigFile, MANAGED_PROFILE_REQUIREMENTS,
OAuthProviderConfigFile, PaymentRuntimeConfigFile, RedisRuntimeConfigFile, RuntimeError, SUPPORTED_BYOK_PROVIDERS,
SearchRuntimeConfigFile, auth_limit, insert_flat_override, valid_search_endpoint, validate_copilot_config,
};
use crate::runtime::object_storage::{
StorageBackendConfig, default_storage_provider_config, storage_provider_schema, types::StorageProviderConfig,
};
const AUTH_MODULE: &str = "auth";
const COPILOT_MODULE: &str = "copilot";
const CRYPTO_MODULE: &str = "crypto";
const INDEXER_MODULE: &str = "indexer";
const OAUTH_MODULE: &str = "oauth";
const PAYMENT_MODULE: &str = "payment";
const REDIS_MODULE: &str = "redis";
const STORAGES_MODULE: &str = "storages";
#[napi_derive::napi(object)]
pub struct AppConfigDescriptor {
pub key: String,
pub description: String,
pub default_value: Value,
pub schema: Value,
pub internal: bool,
pub env_name: Option<String>,
pub env_type: Option<String>,
pub link: Option<String>,
}
fn invalid_config(message: impl Into<String>) -> Error {
Error::new(Status::InvalidArg, message.into())
}
fn field<'a>(root: &'a Value, key: &str) -> &'a Value {
let mut current = root;
for segment in key.split('.') {
if let Some(reference) = current
.get("$ref")
.or_else(|| current.pointer("/allOf/0/$ref"))
.and_then(Value::as_str)
{
current = root
.pointer(reference.strip_prefix('#').expect("local schema reference"))
.expect("generated schema reference");
}
current = &current["properties"][segment];
}
current
}
fn descriptors() -> Vec<AppConfigDescriptor> {
let defaults = to_value(CopilotRuntimeConfigFile::default()).expect("copilot defaults should serialize");
let root = to_value(
SchemaSettings::draft07()
.into_generator()
.into_root_schema_for::<CopilotRuntimeConfigFile>(),
)
.expect("copilot schema should serialize");
let definitions = root.get("$defs").or_else(|| root.get("definitions"));
[
(
"enabled",
"Enable AI features. Workspace owners configure provider keys in Workspace Settings → Integrations → AI BYOK.",
false,
),
(
"byok.enabled",
"Allow workspace owners and admins to configure AI provider keys through AI BYOK.",
false,
),
(
"byok.allowedProviders",
"AI providers that workspace owners and admins may add through AI BYOK.",
false,
),
(
"byok.allowCustomEndpoint",
"Allow AI BYOK keys to use a custom provider endpoint.",
false,
),
(
"byok.allowPrivateEndpoint",
"Whether workspace BYOK custom endpoints may resolve to private network targets. Enabling this allows workspace \
owners and admins to send provider probe requests to the private network.",
false,
),
("providers.profiles", "The profile list for copilot providers.", true),
]
.into_iter()
.map(|(key, description, internal)| {
let pointer = format!("/{}", key.replace('.', "/"));
let mut schema = field(&root, key).clone();
if let Some(definitions) = definitions {
let name = if root.get("$defs").is_some() {
"$defs"
} else {
"definitions"
};
schema[name] = definitions.clone();
}
if key == "byok.allowedProviders" {
schema["items"]["enum"] = json!(SUPPORTED_BYOK_PROVIDERS);
}
if key == "providers.profiles" {
let profile = &mut schema["definitions"]["CopilotManagedProfileConfigFile"];
let requirements = MANAGED_PROFILE_REQUIREMENTS
.iter()
.map(|(provider, fields)| {
let properties = fields
.iter()
.map(|field| ((*field).to_string(), json!({ "type": "string", "pattern": "\\S" })))
.collect::<serde_json::Map<_, _>>();
json!({
"if": { "properties": { "type": { "const": provider } }, "required": ["type"] },
"then": { "properties": { "config": { "required": fields, "properties": properties } } }
})
})
.collect::<Vec<_>>();
profile["allOf"] = json!([{
"if": { "properties": { "enabled": { "const": false } }, "required": ["enabled"] },
"else": { "allOf": requirements }
}]);
}
AppConfigDescriptor {
key: key.to_string(),
description: description.to_string(),
default_value: defaults.pointer(&pointer).expect("copilot default field").clone(),
schema,
internal,
env_name: None,
env_type: None,
link: None,
}
})
.chain(std::iter::once(storage_descriptor(
"storage",
"copilot",
"The config for the storage provider.",
)))
.collect()
}
fn storage_descriptor(key: &str, bucket: &str, description: &str) -> AppConfigDescriptor {
AppConfigDescriptor {
key: key.to_string(),
description: description.to_string(),
default_value: default_storage_provider_config(bucket),
schema: storage_provider_schema(),
internal: false,
env_name: None,
env_type: None,
link: None,
}
}
fn storage_descriptors() -> Vec<AppConfigDescriptor> {
vec![
storage_descriptor("avatar.storage", "avatars", "The config of storage for user avatars."),
storage_descriptor(
"blob.storage",
"blobs",
"The config of storage for all uploaded blobs(images, videos, etc.).",
),
]
}
fn payment_descriptors() -> Vec<AppConfigDescriptor> {
let defaults = to_value(PaymentRuntimeConfigFile::default()).expect("payment defaults should serialize");
let root = to_value(
SchemaSettings::draft07()
.into_generator()
.into_root_schema_for::<PaymentRuntimeConfigFile>(),
)
.expect("payment schema should serialize");
let definitions = root.get("$defs").or_else(|| root.get("definitions"));
[
("enabled", "Whether to enable payment integration."),
("stripe", "Stripe SDK options and credentials."),
("revenuecat", "RevenueCat integration configuration."),
]
.into_iter()
.map(|(key, description)| {
let mut schema = field(&root, key).clone();
if let Some(definitions) = definitions {
let name = if root.get("$defs").is_some() {
"$defs"
} else {
"definitions"
};
schema[name] = definitions.clone();
}
AppConfigDescriptor {
key: key.to_string(),
description: description.to_string(),
default_value: defaults.get(key).expect("payment default field").clone(),
schema,
internal: false,
env_name: None,
env_type: None,
link: None,
}
})
.collect()
}
fn auth_descriptors() -> Vec<AppConfigDescriptor> {
let defaults = to_value(AuthConfigFile::default()).expect("auth defaults should serialize");
let root = to_value(
SchemaSettings::draft07()
.into_generator()
.into_root_schema_for::<AuthConfigFile>(),
)
.expect("auth schema should serialize");
let definitions = root.get("$defs").or_else(|| root.get("definitions"));
[
("allowSignup", "Allow new registrations."),
(
"allowSignupForOauth",
"Allow new registrations through configured OAuth providers.",
),
(
"requireEmailDomainVerification",
"Require email domain verification for restricted resources.",
),
(
"newAccountActionDelay",
"Minimum account age in seconds for invites and document publishing.",
),
("session.ttl", "Application auth expiration in seconds."),
("session.ttr", "Application auth refresh interval in seconds."),
("token.accessTokenTtl", "Access JWT expiration in seconds."),
(
"token.refreshIdleTtl",
"Refresh session inactivity expiration in seconds.",
),
(
"token.refreshAbsoluteTtl",
"Refresh session absolute expiration in seconds.",
),
(
"token.refreshGracePeriod",
"Refresh rotation concurrency grace period in seconds.",
),
(
"token.refreshRetention",
"Expired refresh generation retention in seconds.",
),
]
.into_iter()
.map(|(key, description)| {
let pointer = format!("/{}", key.replace('.', "/"));
let mut schema = field(&root, key).clone();
if let Some(definitions) = definitions {
let name = if root.get("$defs").is_some() {
"$defs"
} else {
"definitions"
};
schema[name] = definitions.clone();
}
if let Some((min, max)) = auth_limit(key) {
schema["minimum"] = json!(min);
if max != i64::MAX {
schema["maximum"] = json!(max);
}
}
AppConfigDescriptor {
key: key.to_string(),
description: description.to_string(),
default_value: defaults.pointer(&pointer).expect("auth default field").clone(),
schema,
internal: false,
env_name: None,
env_type: None,
link: None,
}
})
.collect()
}
fn indexer_descriptors() -> Vec<AppConfigDescriptor> {
let defaults = to_value(SearchRuntimeConfigFile::default()).expect("indexer defaults should serialize");
let root = to_value(
SchemaSettings::draft07()
.into_generator()
.into_root_schema_for::<SearchRuntimeConfigFile>(),
)
.expect("indexer schema should serialize");
let definitions = root.get("$defs").or_else(|| root.get("definitions"));
[
("enabled", "Enable the indexer."),
("provider.type", "Search provider type."),
("provider.endpoint", "Remote search provider endpoint."),
("provider.apiKey", "Remote search provider API key."),
("provider.username", "Remote search provider username."),
("provider.password", "Remote search provider password."),
]
.into_iter()
.map(|(key, description)| {
let pointer = format!("/{}", key.replace('.', "/"));
let mut schema = field(&root, key).clone();
if let Some(definitions) = definitions {
let name = if root.get("$defs").is_some() {
"$defs"
} else {
"definitions"
};
schema[name] = definitions.clone();
}
AppConfigDescriptor {
key: key.to_string(),
description: description.to_string(),
default_value: defaults.pointer(&pointer).expect("indexer default field").clone(),
schema,
internal: false,
env_name: None,
env_type: None,
link: None,
}
})
.collect()
}
fn redis_descriptors() -> Vec<AppConfigDescriptor> {
let defaults = to_value(RedisRuntimeConfigFile::default()).expect("Redis defaults should serialize");
let root = to_value(
SchemaSettings::draft07()
.into_generator()
.into_root_schema_for::<RedisRuntimeConfigFile>(),
)
.expect("Redis schema should serialize");
[
("host", "Redis host."),
("port", "Redis port."),
("db", "Redis database index."),
("username", "Redis username."),
("password", "Redis password."),
]
.into_iter()
.map(|(key, description)| {
let mut schema = field(&root, key).clone();
if key == "port" {
schema["minimum"] = json!(1);
}
if key == "db" {
schema["maximum"] = json!(10);
}
AppConfigDescriptor {
key: key.to_string(),
description: description.to_string(),
default_value: defaults.get(key).expect("Redis default field").clone(),
schema,
internal: false,
env_name: Some(format!(
"REDIS_SERVER_{}",
match key {
"db" => "DATABASE",
"host" => "HOST",
"port" => "PORT",
"username" => "USERNAME",
"password" => "PASSWORD",
_ => unreachable!(),
}
)),
env_type: Some(
if matches!(key, "db" | "port") {
"integer"
} else {
"string"
}
.to_string(),
),
link: None,
}
})
.collect()
}
fn crypto_descriptors() -> Vec<AppConfigDescriptor> {
let defaults = to_value(CryptoConfigFile::default()).expect("crypto defaults should serialize");
let root = to_value(
SchemaSettings::draft07()
.into_generator()
.into_root_schema_for::<CryptoConfigFile>(),
)
.expect("crypto schema should serialize");
vec![AppConfigDescriptor {
key: "privateKey".to_string(),
description: "Private key for Node signing and encryption plus native runtime identity.".to_string(),
default_value: defaults["privateKey"].clone(),
schema: field(&root, "privateKey").clone(),
internal: false,
env_name: Some("AFFINE_PRIVATE_KEY".to_string()),
env_type: Some("string".to_string()),
link: None,
}]
}
fn oauth_descriptors() -> Vec<AppConfigDescriptor> {
let default = to_value(OAuthProviderConfigFile::default()).expect("OAuth defaults should serialize");
let root = to_value(
SchemaSettings::draft07()
.into_generator()
.into_root_schema_for::<OAuthProviderConfigFile>(),
)
.expect("OAuth schema should serialize");
[
(
"google",
"Google OAuth provider config",
"https://developers.google.com/identity/protocols/oauth2/web-server",
),
(
"github",
"GitHub OAuth provider config",
"https://docs.github.com/en/apps/oauth-apps",
),
(
"oidc",
"OIDC OAuth provider config. Private network access requires allowPrivateNetwork: true",
"https://openid.net/specs/openid-connect-core-1_0.html",
),
(
"apple",
"Apple OAuth provider config",
"https://developer.apple.com/documentation/sign_in_with_apple/sign_in_with_apple_js/implementing_sign_in_with_apple_in_your_app",
),
]
.into_iter()
.map(|(name, description, link)| {
let mut schema = root.clone();
let mut default_value = default.clone();
if name == "oidc" {
schema["properties"]["issuer"]["description"] = json!("OIDC issuer HTTP(S) URL");
schema["properties"]["allowPrivateNetwork"]["description"] =
json!("Allow the OIDC issuer origin to resolve to private network addresses");
} else {
for key in ["issuer", "allowPrivateNetwork"] {
schema["properties"].as_object_mut().expect("OAuth schema properties").remove(key);
default_value.as_object_mut().expect("OAuth defaults object").remove(key);
}
}
AppConfigDescriptor {
key: format!("providers.{name}"),
description: description.to_string(),
default_value,
schema,
internal: false,
env_name: None,
env_type: None,
link: Some(link.to_string()),
}
})
.collect()
}
fn validate_leaf(key: &str, value: Value) -> std::result::Result<(), RuntimeError> {
let mut raw = to_value(CopilotRuntimeConfigFile::default())
.map_err(|error| RuntimeError::json("serialize copilot defaults", error))?;
insert_flat_override(raw.as_object_mut().expect("copilot defaults are an object"), key, value);
let config: CopilotRuntimeConfigFile =
from_value(raw).map_err(|error| RuntimeError::json("invalid copilot config", error))?;
let config = CopilotRuntimeConfig::try_from(config)?;
validate_copilot_config(&config)
}
#[napi_derive::napi(catch_unwind)]
pub fn app_config_descriptors(module: String) -> Result<Vec<AppConfigDescriptor>> {
match module.as_str() {
AUTH_MODULE => Ok(auth_descriptors()),
COPILOT_MODULE => Ok(descriptors()),
CRYPTO_MODULE => Ok(crypto_descriptors()),
INDEXER_MODULE => Ok(indexer_descriptors()),
OAUTH_MODULE => Ok(oauth_descriptors()),
PAYMENT_MODULE => Ok(payment_descriptors()),
REDIS_MODULE => Ok(redis_descriptors()),
STORAGES_MODULE => Ok(storage_descriptors()),
_ => Err(invalid_config(format!("unknown native app config module: {module}"))),
}
}
#[napi_derive::napi(catch_unwind)]
pub fn validate_app_config_value(module: String, key: String, value: Value) -> Result<Vec<String>> {
let descriptors = app_config_descriptors(module.clone())?;
let descriptor = descriptors
.into_iter()
.find(|descriptor| descriptor.key == key)
.ok_or_else(|| invalid_config(format!("unknown native app config key: {module}.{key}")))?;
let schema = jsonschema::options()
.with_draft(Draft::Draft7)
.build(&descriptor.schema)
.map_err(|error| invalid_config(format!("failed to compile app config schema: {error}")))?;
let errors = schema
.iter_errors(&value)
.map(|error| error.to_string())
.collect::<Vec<_>>();
if !errors.is_empty() {
return Ok(errors);
}
if module == INDEXER_MODULE
&& key == "provider.endpoint"
&& !valid_search_endpoint(value.as_str().unwrap_or_default())
{
return Ok(vec!["invalid search provider endpoint".to_string()]);
}
if module == OAUTH_MODULE && key == "providers.oidc" {
let issuer = value.get("issuer").and_then(Value::as_str).unwrap_or_default();
if !valid_search_endpoint(issuer) {
return Ok(vec!["invalid OIDC issuer URL".to_string()]);
}
}
if module == COPILOT_MODULE {
if key == "storage" {
return Ok(validate_storage(value));
}
Ok(
validate_leaf(&key, value)
.err()
.map(|error| vec![error.to_string()])
.unwrap_or_default(),
)
} else {
Ok(if module == STORAGES_MODULE {
validate_storage(value)
} else {
Vec::new()
})
}
}
fn validate_storage(value: Value) -> Vec<String> {
let config: StorageProviderConfig = match from_value(value) {
Ok(config) => config,
Err(error) => return vec![error.to_string()],
};
StorageBackendConfig::from_provider_config(Some(config))
.err()
.map(|error| vec![error.to_string()])
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::{app_config_descriptors, json, validate_app_config_value};
#[test]
fn native_descriptors_and_validation_share_runtime_contract() {
let storages = app_config_descriptors("storages".to_string()).unwrap();
let copilot_storage = app_config_descriptors("copilot".to_string())
.unwrap()
.into_iter()
.find(|descriptor| descriptor.key == "storage")
.unwrap();
for (module, descriptor) in storages
.iter()
.map(|descriptor| ("storages", descriptor))
.chain(std::iter::once(("copilot", &copilot_storage)))
{
assert!(
validate_app_config_value(
module.to_string(),
descriptor.key.clone(),
descriptor.default_value.clone()
)
.unwrap()
.is_empty(),
"{}.{}",
module,
descriptor.key
);
assert!(
!validate_app_config_value(
module.to_string(),
descriptor.key.clone(),
json!({ "provider": "fs", "bucket": "blobs", "config": {} })
)
.unwrap()
.is_empty()
);
assert!(
!validate_app_config_value(module.to_string(), descriptor.key.clone(), json!({ "provider": "cloudflare-r2", "bucket": "blobs", "config": { "accountId": "x", "usePresignedURL": { "enabled": "yes" } } }))
.unwrap()
.is_empty()
);
}
let crypto = app_config_descriptors("crypto".to_string()).unwrap();
assert_eq!(crypto.len(), 1);
assert_eq!(crypto[0].default_value, "");
assert_eq!(crypto[0].env_name.as_deref(), Some("AFFINE_PRIVATE_KEY"));
assert!(
!validate_app_config_value("crypto".to_string(), "privateKey".to_string(), json!(true))
.unwrap()
.is_empty()
);
let oauth = app_config_descriptors("oauth".to_string()).unwrap();
assert_eq!(oauth.len(), 4);
for descriptor in &oauth {
assert!(descriptor.link.is_some());
assert!(
validate_app_config_value(
"oauth".to_string(),
descriptor.key.clone(),
descriptor.default_value.clone()
)
.unwrap()
.is_empty()
);
}
assert!(
validate_app_config_value(
"oauth".to_string(),
"providers.oidc".to_string(),
json!({ "clientId": "id", "clientSecret": "secret", "issuer": "https://example.com", "args": {} })
)
.unwrap()
.is_empty()
);
assert!(
!validate_app_config_value(
"oauth".to_string(),
"providers.oidc".to_string(),
json!({ "clientId": "id", "clientSecret": "secret", "issuer": "file:///etc/passwd", "args": {} })
)
.unwrap()
.is_empty()
);
let redis = app_config_descriptors("redis".to_string()).unwrap();
assert_eq!(
redis
.iter()
.map(|descriptor| descriptor.key.as_str())
.collect::<Vec<_>>(),
["host", "port", "db", "username", "password"]
);
assert_eq!(redis[0].default_value, "localhost");
assert_eq!(redis[1].default_value, 6379);
assert_eq!(redis[0].env_name.as_deref(), Some("REDIS_SERVER_HOST"));
assert_eq!(redis[2].env_name.as_deref(), Some("REDIS_SERVER_DATABASE"));
for descriptor in &redis {
assert!(
validate_app_config_value(
"redis".to_string(),
descriptor.key.clone(),
descriptor.default_value.clone()
)
.unwrap()
.is_empty(),
"{}",
descriptor.key
);
}
assert!(
!validate_app_config_value("redis".to_string(), "port".to_string(), json!(0))
.unwrap()
.is_empty()
);
assert!(
!validate_app_config_value("redis".to_string(), "db".to_string(), json!(11))
.unwrap()
.is_empty()
);
let auth = app_config_descriptors("auth".to_string()).unwrap();
assert_eq!(auth.len(), 11);
assert_eq!(
auth
.iter()
.find(|descriptor| descriptor.key == "allowSignup")
.unwrap()
.default_value,
true
);
assert_eq!(
auth
.iter()
.find(|descriptor| descriptor.key == "token.accessTokenTtl")
.unwrap()
.default_value,
15 * 60
);
for descriptor in &auth {
assert!(
validate_app_config_value(
"auth".to_string(),
descriptor.key.clone(),
descriptor.default_value.clone()
)
.unwrap()
.is_empty(),
"{}",
descriptor.key
);
}
assert!(
!validate_app_config_value("auth".to_string(), "token.accessTokenTtl".to_string(), json!(30))
.unwrap()
.is_empty()
);
assert!(
!validate_app_config_value("auth".to_string(), "newAccountActionDelay".to_string(), json!(-1))
.unwrap()
.is_empty()
);
let descriptors = app_config_descriptors("copilot".to_string()).unwrap();
assert_eq!(
descriptors
.iter()
.map(|descriptor| descriptor.key.as_str())
.collect::<Vec<_>>(),
[
"enabled",
"byok.enabled",
"byok.allowedProviders",
"byok.allowCustomEndpoint",
"byok.allowPrivateEndpoint",
"providers.profiles",
"storage",
]
);
assert_eq!(descriptors[0].default_value, json!(false));
assert_eq!(descriptors[1].default_value, json!(true));
assert!(descriptors[5].internal);
let profile_schema = jsonschema::options()
.with_draft(jsonschema::Draft::Draft7)
.build(&descriptors[5].schema)
.unwrap();
for (profile, accepted) in [
(
json!({"id":"openai","type":"openai","models":["gpt-5.6-luna"],"config":{"apiKey":"test"}}),
true,
),
(
json!({"id":"openai","type":"openai","models":["gpt-5.6-luna"],"config":{}}),
false,
),
(
json!({"id":"openai","type":"openai","enabled":false,"models":["gpt-5.6-luna"],"config":{}}),
true,
),
(
json!({"id":"cloudflare","type":"cloudflareWorkersAi","models":["@cf/baai/bge-reranker-base"],"config":{"apiToken":"test"}}),
false,
),
(
json!({"id":"vertex","type":"geminiVertex","models":["gemini-3.7-flash"],"config":{"project":"test","location":"global"}}),
true,
),
] {
let value = json!([profile]);
assert_eq!(profile_schema.is_valid(&value), accepted);
assert_eq!(
validate_app_config_value("copilot".to_string(), "providers.profiles".to_string(), value)
.unwrap()
.is_empty(),
accepted
);
}
for descriptor in &descriptors {
assert!(
validate_app_config_value(
"copilot".to_string(),
descriptor.key.clone(),
descriptor.default_value.clone()
)
.unwrap()
.is_empty(),
"{}",
descriptor.key
);
}
assert!(
validate_app_config_value(
"copilot".to_string(),
"providers.profiles".to_string(),
json!([{
"id": "managed-openai",
"type": "openai",
"displayName": "OpenAI",
"priority": 1,
"enabled": true,
"models": ["gpt-5.6-luna"],
"middleware": {
"rust": { "request": ["normalize_messages"] },
"node": { "text": ["citation_footnote"] }
},
"config": { "apiKey": "test" }
}]),
)
.unwrap()
.is_empty()
);
let payment = app_config_descriptors("payment".to_string()).unwrap();
assert_eq!(
payment
.iter()
.map(|descriptor| descriptor.key.as_str())
.collect::<Vec<_>>(),
["enabled", "stripe", "revenuecat"]
);
assert_eq!(payment[0].default_value, json!(false));
assert_eq!(payment[1].default_value["environment"], "test");
assert_eq!(payment[2].default_value["environment"], "production");
for descriptor in &payment {
assert!(
validate_app_config_value(
"payment".to_string(),
descriptor.key.clone(),
descriptor.default_value.clone()
)
.unwrap()
.is_empty(),
"{}",
descriptor.key
);
}
assert!(
!validate_app_config_value("payment".to_string(), "enabled".to_string(), json!("true"))
.unwrap()
.is_empty()
);
assert!(
!validate_app_config_value("payment".to_string(), "stripe".to_string(), json!("invalid"))
.unwrap()
.is_empty()
);
let indexer = app_config_descriptors("indexer".to_string()).unwrap();
assert_eq!(
indexer
.iter()
.map(|descriptor| descriptor.key.as_str())
.collect::<Vec<_>>(),
[
"enabled",
"provider.type",
"provider.endpoint",
"provider.apiKey",
"provider.username",
"provider.password",
]
);
assert_eq!(indexer[1].default_value, "embedded");
for descriptor in &indexer {
assert!(
validate_app_config_value(
"indexer".to_string(),
descriptor.key.clone(),
descriptor.default_value.clone()
)
.unwrap()
.is_empty(),
"{}",
descriptor.key
);
}
assert!(
!validate_app_config_value("indexer".to_string(), "provider.type".to_string(), json!("unknown"))
.unwrap()
.is_empty()
);
assert!(
!validate_app_config_value(
"indexer".to_string(),
"provider.endpoint".to_string(),
json!("not a URL")
)
.unwrap()
.is_empty()
);
}
#[test]
fn copilot_validation_rejects_invalid_leaf_values() {
for (key, value) in [
("enabled", json!("yes")),
("byok.enabled", json!("yes")),
("byok.allowedProviders", json!(["openai", "openai"])),
(
"providers.profiles",
json!([{
"id": "invalid id",
"type": "openai",
"models": ["gpt-5.6-luna"],
"config": {}
}]),
),
(
"providers.profiles",
json!([{
"id": "managed-openai",
"type": "openai",
"models": ["gpt-5.6-luna"],
"middleware": { "node": { "text": ["unknown"] } },
"config": {}
}]),
),
] {
assert!(
!validate_app_config_value("copilot".to_string(), key.to_string(), value)
.unwrap()
.is_empty(),
"{key}"
);
}
}
}
@@ -0,0 +1,129 @@
use serde::{Deserialize, Serialize};
use super::{AuthRuntimeConfig, InviteQuotaConfig, RuntimeError, RuntimeResult};
#[derive(Deserialize, Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(super) struct AuthConfigFile {
pub(super) new_account_action_delay: i64,
pub(super) allow_signup: bool,
pub(super) allow_signup_for_oauth: bool,
pub(super) require_email_domain_verification: bool,
pub(super) session: AuthSessionConfigFile,
pub(super) token: AuthTokenConfigFile,
}
#[derive(Deserialize, Serialize, schemars::JsonSchema)]
#[serde(default)]
pub(super) struct AuthSessionConfigFile {
ttl: i64,
ttr: i64,
}
#[derive(Deserialize, Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(super) struct AuthTokenConfigFile {
access_token_ttl: i64,
refresh_idle_ttl: i64,
refresh_absolute_ttl: i64,
refresh_grace_period: i64,
refresh_retention: i64,
}
impl Default for AuthConfigFile {
fn default() -> Self {
let runtime = AuthRuntimeConfig::default();
Self {
new_account_action_delay: InviteQuotaConfig::default().new_account_action_delay_seconds,
allow_signup: runtime.allow_signup,
allow_signup_for_oauth: runtime.allow_signup_for_oauth,
require_email_domain_verification: runtime.require_email_domain_verification,
session: AuthSessionConfigFile {
ttl: runtime.session_ttl_seconds,
ttr: runtime.session_ttr_seconds,
},
token: AuthTokenConfigFile {
access_token_ttl: runtime.access_token_ttl_seconds,
refresh_idle_ttl: runtime.refresh_idle_ttl_seconds,
refresh_absolute_ttl: runtime.refresh_absolute_ttl_seconds,
refresh_grace_period: runtime.refresh_grace_seconds,
refresh_retention: runtime.refresh_retention_seconds,
},
}
}
}
impl Default for AuthSessionConfigFile {
fn default() -> Self {
Self {
ttl: AuthRuntimeConfig::default().session_ttl_seconds,
ttr: AuthRuntimeConfig::default().session_ttr_seconds,
}
}
}
impl Default for AuthTokenConfigFile {
fn default() -> Self {
let runtime = AuthRuntimeConfig::default();
Self {
access_token_ttl: runtime.access_token_ttl_seconds,
refresh_idle_ttl: runtime.refresh_idle_ttl_seconds,
refresh_absolute_ttl: runtime.refresh_absolute_ttl_seconds,
refresh_grace_period: runtime.refresh_grace_seconds,
refresh_retention: runtime.refresh_retention_seconds,
}
}
}
pub(super) fn auth_limit(key: &str) -> Option<(i64, i64)> {
match key {
"newAccountActionDelay" => Some((0, i64::MAX)),
"token.accessTokenTtl" => Some((60, 60 * 60)),
"token.refreshIdleTtl" => Some((60 * 60, 60 * 60 * 24 * 365)),
"token.refreshAbsoluteTtl" => Some((60 * 60, 60 * 60 * 24 * 730)),
"token.refreshGracePeriod" => Some((0, 60)),
"token.refreshRetention" => Some((60 * 60, 60 * 60 * 24 * 365)),
_ => None,
}
}
impl AuthConfigFile {
pub(super) fn validate(&self) -> RuntimeResult<()> {
for (key, value) in [
("newAccountActionDelay", self.new_account_action_delay),
("token.accessTokenTtl", self.token.access_token_ttl),
("token.refreshIdleTtl", self.token.refresh_idle_ttl),
("token.refreshAbsoluteTtl", self.token.refresh_absolute_ttl),
("token.refreshGracePeriod", self.token.refresh_grace_period),
("token.refreshRetention", self.token.refresh_retention),
] {
let (min, max) = auth_limit(key).expect("auth range must be defined");
if !(min..=max).contains(&value) {
return Err(RuntimeError::config(format!("auth.{key} is out of range")));
}
}
Ok(())
}
pub(super) fn invite_quota_config(&self) -> InviteQuotaConfig {
InviteQuotaConfig {
new_account_action_delay_seconds: self.new_account_action_delay,
}
}
pub(super) fn runtime_config(&self) -> AuthRuntimeConfig {
AuthRuntimeConfig {
allow_signup: self.allow_signup,
allow_signup_for_oauth: self.allow_signup_for_oauth,
require_email_domain_verification: self.require_email_domain_verification,
session_ttl_seconds: self.session.ttl,
session_ttr_seconds: self.session.ttr,
access_token_ttl_seconds: self.token.access_token_ttl,
refresh_idle_ttl_seconds: self.token.refresh_idle_ttl,
refresh_absolute_ttl_seconds: self.token.refresh_absolute_ttl,
refresh_grace_seconds: self.token.refresh_grace_period,
refresh_retention_seconds: self.token.refresh_retention,
oauth: Default::default(),
}
}
}
@@ -0,0 +1,341 @@
use llm_adapter::capability::provider_default_capability_upper_bound;
use serde::Deserialize;
use serde_json::Map;
use super::{RuntimeError, RuntimeResult};
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub(crate) struct CopilotRuntimeConfig {
pub(crate) enabled: bool,
pub(crate) byok: CopilotByokRuntimeConfig,
pub(crate) providers: CopilotProvidersRuntimeConfig,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(crate) struct CopilotByokRuntimeConfig {
pub(crate) enabled: bool,
#[serde(default = "default_allowed_providers")]
pub(crate) allowed_providers: Vec<String>,
pub(crate) allow_custom_endpoint: bool,
pub(crate) allow_private_endpoint: bool,
}
impl Default for CopilotByokRuntimeConfig {
fn default() -> Self {
Self {
enabled: true,
allowed_providers: default_allowed_providers(),
allow_custom_endpoint: false,
allow_private_endpoint: false,
}
}
}
pub(in crate::runtime) const SUPPORTED_BYOK_PROVIDERS: [&str; 4] = ["openai", "anthropic", "gemini", "fal"];
pub(super) const MANAGED_PROFILE_REQUIREMENTS: [(&str, &[&str]); 7] = [
("openai", &["apiKey"]),
("anthropic", &["apiKey"]),
("gemini", &["apiKey"]),
("fal", &["apiKey"]),
("cloudflareWorkersAi", &["apiToken", "accountId"]),
("geminiVertex", &["project", "location"]),
("anthropicVertex", &["project", "location"]),
];
fn default_allowed_providers() -> Vec<String> {
SUPPORTED_BYOK_PROVIDERS.into_iter().map(str::to_string).collect()
}
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub(crate) struct CopilotProvidersRuntimeConfig {
pub(crate) profiles: Vec<CopilotManagedProfileConfig>,
}
#[derive(Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CopilotManagedProfileConfig {
pub(crate) id: String,
#[serde(rename = "type")]
pub(crate) provider: String,
#[serde(default = "enabled_by_default")]
pub(crate) enabled: bool,
#[serde(default)]
pub(crate) models: Vec<String>,
pub(crate) config: serde_json::Value,
}
fn enabled_by_default() -> bool {
true
}
#[derive(Clone, Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(crate) struct CopilotRuntimeConfigFile {
pub(in crate::runtime) enabled: bool,
pub(in crate::runtime) byok: CopilotByokRuntimeConfig,
pub(in crate::runtime) providers: CopilotProvidersRuntimeConfigFile,
}
#[derive(Clone, Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(in crate::runtime) struct CopilotProvidersRuntimeConfigFile {
pub(in crate::runtime) profiles: Vec<CopilotManagedProfileConfigFile>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CopilotManagedProfileConfigFile {
id: String,
#[serde(rename = "type")]
provider: CopilotManagedProvider,
display_name: Option<String>,
priority: Option<f64>,
#[serde(default = "enabled_by_default")]
enabled: bool,
models: Vec<String>,
middleware: Option<CopilotProviderMiddlewareConfigFile>,
config: CopilotManagedProviderSettingsFile,
}
#[derive(Clone, Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase")]
struct CopilotManagedProviderSettingsFile {
#[serde(skip_serializing_if = "Option::is_none")]
api_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
api_token: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
account_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
project: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
location: Option<String>,
#[serde(rename = "baseURL", skip_serializing_if = "Option::is_none")]
base_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
google_auth_options: Option<CopilotGoogleAuthOptionsFile>,
#[serde(flatten)]
additional: Map<String, serde_json::Value>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase")]
struct CopilotGoogleAuthOptionsFile {
#[serde(skip_serializing_if = "Option::is_none")]
credentials: Option<CopilotGoogleCredentialsFile>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
struct CopilotGoogleCredentialsFile {
client_email: String,
private_key: String,
#[serde(flatten)]
additional: Map<String, serde_json::Value>,
}
#[derive(Clone, Copy, Deserialize, serde::Serialize, schemars::JsonSchema)]
enum CopilotManagedProvider {
#[serde(rename = "anthropic")]
Anthropic,
#[serde(rename = "anthropicVertex")]
AnthropicVertex,
#[serde(rename = "cloudflareWorkersAi")]
CloudflareWorkersAi,
#[serde(rename = "fal")]
Fal,
#[serde(rename = "gemini")]
Gemini,
#[serde(rename = "geminiVertex")]
GeminiVertex,
#[serde(rename = "openai")]
OpenAi,
}
impl CopilotManagedProvider {
fn as_str(self) -> &'static str {
match self {
Self::Anthropic => "anthropic",
Self::AnthropicVertex => "anthropicVertex",
Self::CloudflareWorkersAi => "cloudflareWorkersAi",
Self::Fal => "fal",
Self::Gemini => "gemini",
Self::GeminiVertex => "geminiVertex",
Self::OpenAi => "openai",
}
}
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
struct CopilotProviderMiddlewareConfigFile {
rust: Option<CopilotRustMiddlewareConfigFile>,
node: Option<CopilotNodeMiddlewareConfigFile>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
struct CopilotRustMiddlewareConfigFile {
request: Option<Vec<CopilotRustRequestMiddleware>>,
stream: Option<Vec<CopilotRustStreamMiddleware>>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
struct CopilotNodeMiddlewareConfigFile {
text: Option<Vec<CopilotNodeTextMiddleware>>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
enum CopilotRustRequestMiddleware {
NormalizeMessages,
ClampMaxTokens,
ToolSchemaRewrite,
OpenaiRequestCompat,
OmitToolChoice,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
enum CopilotRustStreamMiddleware {
StreamEventNormalize,
CitationIndexing,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
enum CopilotNodeTextMiddleware {
CitationFootnote,
Callout,
ThinkingFormat,
}
impl TryFrom<CopilotRuntimeConfigFile> for CopilotRuntimeConfig {
type Error = RuntimeError;
fn try_from(value: CopilotRuntimeConfigFile) -> Result<Self, Self::Error> {
Ok(Self {
enabled: value.enabled,
byok: value.byok,
providers: CopilotProvidersRuntimeConfig {
profiles: value
.providers
.profiles
.into_iter()
.map(TryInto::try_into)
.collect::<RuntimeResult<_>>()?,
},
})
}
}
impl TryFrom<CopilotManagedProfileConfigFile> for CopilotManagedProfileConfig {
type Error = RuntimeError;
fn try_from(value: CopilotManagedProfileConfigFile) -> Result<Self, Self::Error> {
if value.id.is_empty()
|| !value
.id
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
{
return Err(RuntimeError::invalid_state(
"managed copilot profile id must contain only letters, numbers, hyphens, and underscores",
));
}
Ok(Self {
id: value.id,
provider: value.provider.as_str().to_string(),
enabled: value.enabled,
models: value.models,
config: serde_json::to_value(value.config)
.map_err(|error| RuntimeError::json("serialize managed copilot config failed", error))?,
})
}
}
pub(in crate::runtime) fn validate_copilot_config(config: &CopilotRuntimeConfig) -> RuntimeResult<()> {
let mut allowed_providers = std::collections::HashSet::new();
for provider in &config.byok.allowed_providers {
if !SUPPORTED_BYOK_PROVIDERS.contains(&provider.as_str()) || !allowed_providers.insert(provider.as_str()) {
return Err(RuntimeError::invalid_state(
"copilot BYOK allowed providers must be supported and unique",
));
}
}
let mut profile_ids = std::collections::HashSet::new();
let mut managed_models = std::collections::HashMap::new();
for profile in &config.providers.profiles {
if profile.id.trim().is_empty() || !profile_ids.insert(profile.id.as_str()) {
return Err(RuntimeError::invalid_state(
"managed copilot profile ids must be non-empty and unique",
));
}
if profile.provider.trim().is_empty() {
return Err(RuntimeError::invalid_state(
"managed copilot profile provider is required",
));
}
if profile.models.is_empty() {
return Err(RuntimeError::invalid_state(
"managed copilot profile models must be non-empty",
));
}
if profile.enabled {
let required = MANAGED_PROFILE_REQUIREMENTS
.iter()
.find(|(provider, _)| *provider == profile.provider)
.map(|(_, fields)| *fields)
.ok_or_else(|| RuntimeError::invalid_state("unsupported managed copilot provider"))?;
for field in required {
if profile
.config
.get(*field)
.and_then(serde_json::Value::as_str)
.is_none_or(|value| value.trim().is_empty())
{
return Err(RuntimeError::invalid_state(format!(
"managed copilot profile requires {field}"
)));
}
}
if let Some(base_url) = profile.config.get("baseURL").and_then(serde_json::Value::as_str) {
llm_adapter::target::canonicalize_endpoint(base_url)
.map_err(|error| RuntimeError::invalid_state(error.to_string()))?;
}
if let Some(credentials) = profile.config.pointer("/googleAuthOptions/credentials") {
for field in ["client_email", "private_key"] {
if credentials
.get(field)
.and_then(serde_json::Value::as_str)
.is_none_or(|value| value.trim().is_empty())
{
return Err(RuntimeError::invalid_state(format!(
"managed Vertex credentials require {field}"
)));
}
}
}
}
let mut models = std::collections::HashSet::new();
for model in &profile.models {
if model.trim().is_empty() || !models.insert(model.as_str()) {
return Err(RuntimeError::invalid_state(
"managed copilot profile models must be non-empty and unique",
));
}
provider_default_capability_upper_bound(&profile.provider, model)
.ok_or_else(|| RuntimeError::invalid_state("managed copilot profile model is unsupported"))?;
if profile.enabled
&& let Some(existing_profile) = managed_models.insert(model.as_str(), profile.id.as_str())
{
return Err(RuntimeError::invalid_state(format!(
"managed copilot model {model} is assigned to both {existing_profile} and {}",
profile.id
)));
}
}
}
Ok(())
}
@@ -0,0 +1,178 @@
use std::{
fs,
path::{Path, PathBuf},
};
use serde_json::{Map, Value};
use super::{
Deployment, NATIVE_APP_CONFIG_KEYS, RedisRuntimeConfig, RedisRuntimeConfigFile, RuntimeError, RuntimeResult,
deserialize_app_config, insert_flat_override, is_secret_native_app_config_key,
};
use crate::runtime::object_storage::ObjectStorageService;
pub(crate) struct ServerConfig {
path: PathBuf,
baseline: serde_json::Value,
deployment: Deployment,
object_storage: ObjectStorageService,
}
impl ServerConfig {
pub(crate) fn open(path: &Path, legacy_deployment_type: Option<&str>) -> RuntimeResult<Self> {
let path = fs::canonicalize(path).map_err(|error| RuntimeError::io("failed to locate config file", error))?;
let raw = fs::read_to_string(&path).map_err(|error| RuntimeError::io("failed to read config file", error))?;
let source: Value =
serde_json::from_str(&raw).map_err(|error| RuntimeError::json("failed to parse config file", error))?;
let baseline = expand_module_config_paths(source.clone());
let deployment = match baseline.pointer("/deployment/type").and_then(serde_json::Value::as_str) {
Some("cloud") => Deployment::Cloud,
Some("selfhosted") => Deployment::SelfHosted,
// TODO(0.27.5): Remove the old deployment env fallback after self-host config files are upgraded.
None => match legacy_deployment_type {
Some("cloud") => Deployment::Cloud,
Some("selfhosted") => Deployment::SelfHosted,
_ => return Err(RuntimeError::config("deployment.type must be cloud or selfhosted")),
},
_ => return Err(RuntimeError::config("deployment.type must be cloud or selfhosted")),
};
deserialize_app_config(baseline.clone())?;
let object_storage = ObjectStorageService::from_config_value(&source)?;
Ok(Self {
path,
baseline,
deployment,
object_storage,
})
}
pub(crate) fn path(&self) -> &Path {
&self.path
}
pub(crate) fn baseline(&self) -> &serde_json::Value {
&self.baseline
}
pub(crate) fn has_baseline_config_key(&self, key: &str) -> bool {
key
.split('.')
.try_fold(&self.baseline, |value, segment| value.get(segment))
.is_some()
}
pub(crate) fn baseline_config_key_configured(&self, key: &str) -> bool {
fn configured(value: &Value) -> bool {
match value {
Value::Null => false,
Value::String(value) => !value.is_empty(),
Value::Array(values) => !values.is_empty(),
Value::Object(values) => values.values().any(configured),
Value::Bool(_) | Value::Number(_) => true,
}
}
key
.split('.')
.try_fold(&self.baseline, |value, segment| value.get(segment))
.is_some_and(configured)
}
pub(crate) fn redis_url(&self) -> RuntimeResult<Option<String>> {
let file = self
.baseline
.get("redis")
.cloned()
.map(serde_json::from_value::<RedisRuntimeConfigFile>)
.transpose()
.map_err(|error| RuntimeError::json("invalid Redis config", error))?;
Ok(RedisRuntimeConfig::from_sources(file)?.url)
}
pub(crate) fn redis_node_options(&self) -> Value {
self
.baseline
.pointer("/redis/ioredis")
.cloned()
.unwrap_or_else(|| Value::Object(Map::new()))
}
pub(crate) fn node_owned(&self) -> Value {
let baseline = self.baseline();
let mut modules = Map::new();
for name in [
"calendar",
"captcha",
"client",
"doc",
"flags",
"graphql",
"mailer",
"metrics",
"server",
"telemetry",
"throttle",
"websocket",
"worker",
] {
if let Some(value) = baseline.get(name) {
modules.insert(name.to_string(), value.clone());
}
}
for (module, key) in [
("auth", "passwordRequirements"),
("auth", "signInRateLimit"),
("auth", "trustedCloudflareHeaders"),
("copilot", "exa"),
("copilot", "unsplash"),
("db", "prisma"),
("payment", "showLifetimePrice"),
("storages", "avatar.publicPath"),
] {
if let Some(value) = baseline.pointer(&format!("/{}/{}", module, key.replace('.', "/"))) {
let entry = modules
.entry(module.to_string())
.or_insert_with(|| Value::Object(Map::new()));
insert_flat_override(
entry.as_object_mut().expect("node projection module must be an object"),
key,
value.clone(),
);
}
}
Value::Object(modules)
}
pub(crate) fn public_native_baseline(&self) -> Value {
let mut modules = Map::new();
for key in NATIVE_APP_CONFIG_KEYS {
if is_secret_native_app_config_key(key) {
continue;
}
if let Some(value) = key.split('.').try_fold(&self.baseline, |item, part| item.get(part)) {
insert_flat_override(&mut modules, key, value.clone());
}
}
Value::Object(modules)
}
pub(crate) fn deployment(&self) -> Deployment {
self.deployment
}
pub(crate) fn object_storage(&self) -> &ObjectStorageService {
&self.object_storage
}
}
pub(super) fn expand_module_config_paths(mut value: serde_json::Value) -> serde_json::Value {
if let Some(root) = value.as_object_mut() {
for module in root.values_mut().filter_map(serde_json::Value::as_object_mut) {
let entries = std::mem::take(module);
for (path, value) in entries {
insert_flat_override(module, &path, value);
}
}
}
value
}
@@ -0,0 +1,471 @@
use std::{env, sync::Arc};
use serde::Deserialize;
use sqlx::PgPool;
use zeroize::Zeroizing;
use super::{RuntimeError, RuntimeResult};
mod admin;
mod auth;
mod copilot;
mod file;
mod napi;
mod payment;
use payment::PaymentRuntimeConfigFile;
mod store;
mod types;
pub use admin::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value};
use auth::{AuthConfigFile, auth_limit};
pub(crate) use copilot::{CopilotByokRuntimeConfig, CopilotManagedProfileConfig, CopilotRuntimeConfig};
use copilot::{
CopilotRuntimeConfigFile, MANAGED_PROFILE_REQUIREMENTS, SUPPORTED_BYOK_PROVIDERS, validate_copilot_config,
};
pub(crate) use file::ServerConfig;
#[cfg(test)]
use file::expand_module_config_paths;
pub use napi::ServerConfigHandle;
pub(crate) use store::{AppConfigChange, save_app_config_changes};
use store::{
NATIVE_APP_CONFIG_KEYS, insert_flat_override, is_secret_native_app_config_key, is_static_app_config_key,
load_app_config_overrides_from_db,
};
#[cfg(test)]
use store::{app_config_from_flat_overrides, app_config_value_from_flat_overrides};
pub(crate) use types::{
AuthRuntimeConfig, BackendRuntimeConfig, Deployment, InviteQuotaConfig, OAuthProviderRuntimeConfig,
PaymentProductConfig, PaymentRuntimeConfig, RedisRuntimeConfig, RevenueCatRuntimeConfig, SearchRuntimeConfig,
StripeRuntimeConfig,
};
use crate::llm::byok::ByokPolicy;
impl BackendRuntimeConfig {
pub(crate) fn byok_policy(&self) -> ByokPolicy {
ByokPolicy::from(self.deployment, &self.copilot.byok)
}
pub(crate) fn from_server_config(private_key: Option<String>, config: &ServerConfig) -> RuntimeResult<Self> {
Self::from_value(private_key, config.baseline().clone(), config.deployment())
}
fn from_value(
private_key: Option<String>,
app_config_value: serde_json::Value,
deployment: Deployment,
) -> RuntimeResult<Self> {
let mut app_config = deserialize_app_config(app_config_value)?;
let database_url = database_url_from_env()
.or(app_config.database_url())
.unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string());
Self {
database_url,
auth: app_config.auth_runtime_config(),
invite_quota: app_config.invite_quota_config(),
private_key: Arc::new(Zeroizing::new(
private_key
.filter(|key| !key.trim().is_empty())
.or_else(|| {
app_config
.crypto
.as_ref()
.and_then(|crypto| non_empty_string(crypto.private_key.clone()))
})
.or_else(private_key_from_env)
.unwrap_or_default(),
)),
deployment,
copilot: app_config
.copilot
.take()
.map(TryInto::try_into)
.transpose()?
.unwrap_or_default(),
search: app_config.indexer.map(Into::into).unwrap_or_default(),
redis: RedisRuntimeConfig::from_sources(app_config.redis.take())?,
payment: PaymentRuntimeConfig::from_file(app_config.payment.take()),
}
.validated()
}
pub(crate) async fn with_db_overrides_from_server_config(
&self,
pool: &PgPool,
config: &ServerConfig,
) -> RuntimeResult<Self> {
let db_overrides = load_app_config_overrides_from_db(pool).await?;
self.apply_db_overrides(config.baseline().clone(), db_overrides)
}
fn apply_db_overrides(
&self,
mut app_config_value: serde_json::Value,
db_overrides: serde_json::Value,
) -> RuntimeResult<Self> {
let db_private_key = db_overrides
.pointer("/crypto/privateKey")
.and_then(serde_json::Value::as_str)
.map(str::to_string)
.and_then(non_empty_string);
merge_config_value(&mut app_config_value, db_overrides);
let mut app_config = deserialize_app_config(app_config_value)?;
Self {
// The DB override is loaded after this connection already exists, so it
// must not rewrite the active datasource URL.
database_url: self.database_url.clone(),
auth: app_config.auth_runtime_config(),
invite_quota: app_config.invite_quota_config(),
private_key: db_private_key
.map(|key| Arc::new(Zeroizing::new(key)))
.unwrap_or_else(|| Arc::clone(&self.private_key)),
deployment: self.deployment,
copilot: app_config
.copilot
.take()
.map(TryInto::try_into)
.transpose()?
.unwrap_or_else(|| self.copilot.clone()),
search: app_config
.indexer
.map(Into::into)
.unwrap_or_else(|| self.search.clone()),
redis: RedisRuntimeConfig::from_sources(app_config.redis.take())?.or_else(|| self.redis.clone()),
payment: app_config
.payment
.take()
.map(PaymentRuntimeConfig::from_file_value)
.unwrap_or_else(|| self.payment.clone()),
}
.validated()
}
fn validated(self) -> RuntimeResult<Self> {
if self.copilot.enabled && self.copilot.byok.enabled && self.private_key.is_empty() {
return Err(RuntimeError::invalid_state(
"stable crypto.privateKey is required when persistent BYOK is enabled",
));
}
validate_copilot_config(&self.copilot)?;
self.payment.validate()?;
if self.search.provider != "embedded" && self.search.endpoint.is_empty() {
return Err(RuntimeError::config("remote search provider requires an endpoint"));
}
if !valid_search_endpoint(&self.search.endpoint) {
return Err(RuntimeError::config("invalid search provider endpoint"));
}
Ok(self)
}
}
fn valid_search_endpoint(endpoint: &str) -> bool {
if endpoint.is_empty() {
return true;
}
url::Url::parse(endpoint)
.map(|url| matches!(url.scheme(), "http" | "https") && url.host_str().is_some())
.unwrap_or(false)
}
#[derive(Default, Deserialize)]
struct AppConfigFile {
auth: Option<AuthConfigFile>,
oauth: Option<OAuthConfigFile>,
db: Option<DbConfigFile>,
crypto: Option<CryptoConfigFile>,
copilot: Option<CopilotRuntimeConfigFile>,
indexer: Option<SearchRuntimeConfigFile>,
redis: Option<RedisRuntimeConfigFile>,
payment: Option<PaymentRuntimeConfigFile>,
}
#[derive(Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(super) struct RedisRuntimeConfigFile {
host: String,
port: u16,
db: u8,
username: String,
password: String,
ioredis: RedisIoRuntimeConfigFile,
}
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
struct RedisIoRuntimeConfigFile {
tls: Option<serde_json::Value>,
}
impl Default for RedisRuntimeConfigFile {
fn default() -> Self {
Self {
host: "localhost".to_string(),
port: 6379,
db: 0,
username: String::new(),
password: String::new(),
ioredis: Default::default(),
}
}
}
impl RedisRuntimeConfigFile {
fn validate(&self) -> RuntimeResult<()> {
if self.port == 0 || self.db > 10 {
return Err(RuntimeError::config(
"redis.port must be positive and redis.db must be at most 10",
));
}
Ok(())
}
}
impl RedisRuntimeConfig {
fn from_sources(file: Option<RedisRuntimeConfigFile>) -> RuntimeResult<Self> {
if let Some(url) = env::var("REDIS_SERVER_URL").ok().and_then(non_empty_string) {
let url = url::Url::parse(&url).map_err(|_| RuntimeError::config("invalid Redis URL"))?;
return Ok(Self {
url: Some(url.to_string()),
});
}
let file = file.unwrap_or_default();
let host = env::var("REDIS_SERVER_HOST")
.ok()
.and_then(non_empty_string)
.or_else(|| non_empty_string(file.host));
let Some(host) = host else {
return Ok(Self::default());
};
let port = env::var("REDIS_SERVER_PORT")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(if file.port == 0 { 6379 } else { file.port });
let db = env::var("REDIS_SERVER_DATABASE")
.ok()
.and_then(|value| value.parse::<u8>().ok())
.unwrap_or(file.db);
let username = env::var("REDIS_SERVER_USERNAME")
.ok()
.and_then(non_empty_string)
.or_else(|| env::var("REDIS_SERVER_USER").ok().and_then(non_empty_string))
.unwrap_or(file.username);
let password = env::var("REDIS_SERVER_PASSWORD")
.ok()
.and_then(non_empty_string)
.unwrap_or(file.password);
let scheme = if file.ioredis.tls.is_some() { "rediss" } else { "redis" };
let mut url = url::Url::parse(&format!("{scheme}://{host}:{port}/{db}"))
.map_err(|_| RuntimeError::config("invalid Redis host"))?;
if !username.is_empty() {
let _ = url.set_username(&username);
}
if !password.is_empty() {
let _ = url.set_password(Some(&password));
}
Ok(Self {
url: Some(url.to_string()),
})
}
fn or_else(self, fallback: impl FnOnce() -> Self) -> Self {
if self.url.is_some() { self } else { fallback() }
}
}
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(default)]
pub(super) struct OAuthConfigFile {
providers: std::collections::BTreeMap<String, OAuthProviderConfigFile>,
}
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(super) struct OAuthProviderConfigFile {
client_id: String,
client_secret: String,
args: std::collections::BTreeMap<String, String>,
issuer: String,
allow_private_network: bool,
}
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
struct SearchRuntimeConfigFile {
enabled: bool,
provider: SearchProviderConfigFile,
}
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
struct SearchProviderConfigFile {
#[serde(rename = "type")]
provider: SearchProviderType,
endpoint: String,
api_key: String,
username: String,
password: String,
}
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "lowercase")]
enum SearchProviderType {
#[default]
Embedded,
Elasticsearch,
Manticoresearch,
}
impl SearchProviderType {
fn as_str(&self) -> &'static str {
match self {
Self::Embedded => "embedded",
Self::Elasticsearch => "elasticsearch",
Self::Manticoresearch => "manticoresearch",
}
}
}
impl From<SearchRuntimeConfigFile> for SearchRuntimeConfig {
fn from(value: SearchRuntimeConfigFile) -> Self {
Self {
enabled: value.enabled,
provider: value.provider.provider.as_str().to_string(),
endpoint: value.provider.endpoint,
api_key: value.provider.api_key,
username: value.provider.username,
password: value.provider.password,
}
}
}
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(super) struct CryptoConfigFile {
private_key: String,
}
#[derive(Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct DbConfigFile {
datasource_url: Option<String>,
}
impl AppConfigFile {
fn database_url(&self) -> Option<String> {
self
.db
.as_ref()
.and_then(|db| db.datasource_url.clone())
.and_then(non_empty_string)
}
fn invite_quota_config(&self) -> InviteQuotaConfig {
self
.auth
.as_ref()
.map(AuthConfigFile::invite_quota_config)
.unwrap_or_default()
}
fn auth_runtime_config(&self) -> AuthRuntimeConfig {
let mut config = self
.auth
.as_ref()
.map(AuthConfigFile::runtime_config)
.unwrap_or_default();
if let Some(oauth) = &self.oauth {
for (name, provider) in &oauth.providers {
if provider.client_id.trim().is_empty() {
continue;
}
let mut args = provider.args.clone();
let apple_private_key = args.remove("privateKey").map(|value| Arc::new(Zeroizing::new(value)));
let apple_key_id = args.remove("keyId");
let apple_team_id = args.remove("teamId");
if provider.client_secret.trim().is_empty()
&& (name != "apple"
|| apple_private_key.is_none()
|| apple_key_id.as_deref().is_none_or(str::is_empty)
|| apple_team_id.as_deref().is_none_or(str::is_empty))
{
continue;
}
config.oauth.providers.insert(
name.clone(),
OAuthProviderRuntimeConfig {
client_id: provider.client_id.clone(),
client_secret: Arc::new(Zeroizing::new(provider.client_secret.clone())),
args,
issuer: non_empty_string(provider.issuer.clone()),
allow_private_network: provider.allow_private_network,
apple_private_key,
apple_key_id,
apple_team_id,
},
);
}
}
config
}
}
fn database_url_from_env() -> Option<String> {
env::var("DATABASE_URL").ok().and_then(non_empty_string)
}
fn private_key_from_env() -> Option<String> {
env::var("AFFINE_PRIVATE_KEY").ok().and_then(non_empty_string)
}
fn non_empty_string(value: String) -> Option<String> {
if value.trim().is_empty() { None } else { Some(value) }
}
#[cfg(test)]
fn app_config_from_module_json(value: serde_json::Value) -> RuntimeResult<AppConfigFile> {
deserialize_app_config(expand_module_config_paths(value))
}
fn deserialize_app_config(value: serde_json::Value) -> RuntimeResult<AppConfigFile> {
let config: AppConfigFile =
serde_json::from_value(value).map_err(|err| RuntimeError::json("failed to parse config file", err))?;
if let Some(auth) = &config.auth {
auth.validate()?;
}
if let Some(redis) = &config.redis {
redis.validate()?;
}
if let Some(oauth) = &config.oauth {
oauth.validate()?;
}
Ok(config)
}
impl OAuthConfigFile {
fn validate(&self) -> RuntimeResult<()> {
if let Some(oidc) = self.providers.get("oidc")
&& !oidc.issuer.is_empty()
&& !valid_search_endpoint(&oidc.issuer)
{
return Err(RuntimeError::config("invalid OIDC issuer URL"));
}
Ok(())
}
}
fn merge_config_value(base: &mut serde_json::Value, overrides: serde_json::Value) {
match (base, overrides) {
(serde_json::Value::Object(base), serde_json::Value::Object(overrides)) => {
for (key, value) in overrides {
if let Some(existing) = base.get_mut(&key) {
merge_config_value(existing, value);
} else {
base.insert(key, value);
}
}
}
(base, overrides) => *base = overrides,
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,92 @@
use std::{path::Path, sync::Arc};
use super::{
Deployment, NATIVE_APP_CONFIG_KEYS, ServerConfig, is_secret_native_app_config_key, is_static_app_config_key,
};
use crate::runtime::to_napi_error;
#[napi_derive::napi]
pub struct ServerConfigHandle {
pub(crate) inner: Arc<ServerConfig>,
}
#[napi_derive::napi]
impl ServerConfigHandle {
#[napi(constructor)]
pub fn new(path: String, legacy_deployment_type: Option<String>) -> napi::Result<Self> {
// TODO(0.27.5): Remove this argument after legacy self-host config files
// have deployment.type.
let config = ServerConfig::open(Path::new(&path), legacy_deployment_type.as_deref()).map_err(to_napi_error)?;
Ok(Self {
inner: Arc::new(config),
})
}
#[napi(getter)]
pub fn path(&self) -> String {
self.inner.path().to_string_lossy().into_owned()
}
#[napi(getter)]
pub fn deployment_type(&self) -> String {
match self.inner.deployment() {
Deployment::Cloud => "cloud",
Deployment::SelfHosted => "selfhosted",
}
.to_string()
}
#[napi]
pub fn node_owned_json(&self) -> napi::Result<String> {
serde_json::to_string(&self.inner.node_owned()).map_err(|error| napi::Error::from_reason(error.to_string()))
}
#[napi]
pub fn public_native_baseline_json(&self) -> napi::Result<String> {
serde_json::to_string(&self.inner.public_native_baseline())
.map_err(|error| napi::Error::from_reason(error.to_string()))
}
#[napi]
pub fn has_baseline_config_key(&self, key: String) -> bool {
self.inner.has_baseline_config_key(&key)
}
#[napi]
pub fn baseline_config_key_configured(&self, key: String) -> bool {
self.inner.baseline_config_key_configured(&key)
}
#[napi]
pub fn native_app_config_keys(&self) -> Vec<String> {
NATIVE_APP_CONFIG_KEYS.iter().map(|key| (*key).to_string()).collect()
}
#[napi]
pub fn native_secret_app_config_keys(&self) -> Vec<String> {
NATIVE_APP_CONFIG_KEYS
.iter()
.filter(|key| is_secret_native_app_config_key(key))
.map(|key| (*key).to_string())
.collect()
}
#[napi]
pub fn static_app_config_keys(&self) -> Vec<String> {
NATIVE_APP_CONFIG_KEYS
.iter()
.filter(|key| is_static_app_config_key(key))
.map(|key| (*key).to_string())
.collect()
}
#[napi]
pub fn redis_url(&self) -> napi::Result<Option<String>> {
self.inner.redis_url().map_err(to_napi_error)
}
#[napi]
pub fn redis_node_options_json(&self) -> napi::Result<String> {
serde_json::to_string(&self.inner.redis_node_options()).map_err(|error| napi::Error::from_reason(error.to_string()))
}
}
@@ -0,0 +1,163 @@
use std::sync::Arc;
use serde::Deserialize;
use zeroize::Zeroizing;
use super::{
PaymentProductConfig, PaymentRuntimeConfig, RevenueCatRuntimeConfig, RuntimeError, RuntimeResult,
StripeRuntimeConfig, non_empty_string,
};
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
pub(super) struct PaymentRuntimeConfigFile {
enabled: bool,
stripe: StripeRuntimeConfigFile,
revenuecat: RevenueCatRuntimeConfigFile,
}
#[derive(Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
struct StripeRuntimeConfigFile {
api_key: String,
webhook_key: String,
account_id: String,
environment: String,
}
#[derive(Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase", default)]
struct RevenueCatRuntimeConfigFile {
enabled: bool,
api_key: String,
webhook_auth: String,
project_id: String,
environment: String,
product_map: std::collections::BTreeMap<String, PaymentProductConfig>,
}
impl Default for StripeRuntimeConfigFile {
fn default() -> Self {
Self {
api_key: String::new(),
webhook_key: String::new(),
account_id: String::new(),
environment: "test".to_string(),
}
}
}
impl Default for RevenueCatRuntimeConfigFile {
fn default() -> Self {
Self {
enabled: false,
api_key: String::new(),
webhook_auth: String::new(),
project_id: String::new(),
environment: "production".to_string(),
product_map: Default::default(),
}
}
}
impl PaymentRuntimeConfig {
pub(super) fn from_file(file: Option<PaymentRuntimeConfigFile>) -> Self {
file.map(Self::from_file_value).unwrap_or_default()
}
pub(super) fn from_file_value(file: PaymentRuntimeConfigFile) -> Self {
let stripe = non_empty_string(file.stripe.api_key).map(|api_key| StripeRuntimeConfig {
api_key: Arc::new(Zeroizing::new(api_key)),
webhook_key: Arc::new(Zeroizing::new(file.stripe.webhook_key)),
account_id: file.stripe.account_id,
live: file.stripe.environment == "live",
});
let revenuecat = (file.revenuecat.enabled || !file.revenuecat.api_key.trim().is_empty()).then(|| {
let mut product_map = std::collections::BTreeMap::from([
(
"app.affine.pro.Monthly".to_string(),
PaymentProductConfig {
plan: "pro".to_string(),
recurring: "monthly".to_string(),
},
),
(
"app.affine.pro.Annual".to_string(),
PaymentProductConfig {
plan: "pro".to_string(),
recurring: "yearly".to_string(),
},
),
(
"app.affine.pro.ai.Annual".to_string(),
PaymentProductConfig {
plan: "ai".to_string(),
recurring: "yearly".to_string(),
},
),
]);
product_map.extend(file.revenuecat.product_map);
RevenueCatRuntimeConfig {
api_key: Arc::new(Zeroizing::new(file.revenuecat.api_key)),
webhook_auth: Arc::new(Zeroizing::new(file.revenuecat.webhook_auth)),
project_id: file.revenuecat.project_id,
production: file.revenuecat.environment == "production",
product_map,
}
});
Self {
enabled: file.enabled,
stripe,
revenuecat,
}
}
pub(super) fn validate(&self) -> RuntimeResult<()> {
if self.enabled {
let stripe = self
.stripe
.as_ref()
.ok_or_else(|| RuntimeError::config("payment.stripe.apiKey is required when payment is enabled"))?;
if stripe.account_id.trim().is_empty() || stripe.account_id != stripe.account_id.trim() {
return Err(RuntimeError::config(
"payment.stripe.accountId is required when payment is enabled",
));
}
if stripe.webhook_key.trim().is_empty() {
return Err(RuntimeError::config(
"payment.stripe.webhookKey is required when payment is enabled",
));
}
}
if let Some(stripe) = &self.stripe {
let expected_prefixes = if stripe.live {
["sk_live_", "rk_live_"]
} else {
["sk_test_", "rk_test_"]
};
if !expected_prefixes
.iter()
.any(|prefix| stripe.api_key.starts_with(prefix))
{
return Err(RuntimeError::config(
"payment.stripe.environment does not match the API key",
));
}
}
if let Some(revenuecat) = &self.revenuecat {
if revenuecat.api_key.is_empty() || revenuecat.project_id.trim().is_empty() {
return Err(RuntimeError::config(
"payment.revenuecat apiKey and projectId are required when RevenueCat is enabled",
));
}
for mapping in revenuecat.product_map.values() {
if affine_core::access_control::Plan::parse(&mapping.plan).is_none()
|| affine_core::payment::SubscriptionRecurring::parse(&mapping.recurring).is_none()
{
return Err(RuntimeError::config("invalid payment.revenuecat productMap"));
}
}
}
Ok(())
}
}
@@ -0,0 +1,441 @@
use std::collections::{BTreeMap, BTreeSet};
use serde_json::{Map, Value};
use sqlx::{PgPool, Row};
#[cfg(test)]
use super::{AppConfigFile, deserialize_app_config};
use super::{BackendRuntimeConfig, RuntimeError, RuntimeResult, ServerConfig, merge_config_value};
use crate::runtime::object_storage::{ObjectStorageService, backends_from_flat_overrides};
pub(crate) struct AppConfigChange {
pub(crate) key: String,
pub(crate) value: Option<Value>,
pub(crate) native: bool,
}
pub(super) const NATIVE_APP_CONFIG_KEYS: &[&str] = &[
"auth.allowSignup",
"auth.allowSignupForOauth",
"auth.newAccountActionDelay",
"auth.requireEmailDomainVerification",
"auth.session.ttl",
"auth.session.ttr",
"auth.token.accessTokenTtl",
"auth.token.refreshAbsoluteTtl",
"auth.token.refreshGracePeriod",
"auth.token.refreshIdleTtl",
"auth.token.refreshRetention",
"copilot.byok.allowCustomEndpoint",
"copilot.byok.allowedProviders",
"copilot.byok.allowPrivateEndpoint",
"copilot.byok.enabled",
"copilot.enabled",
"copilot.providers.profiles",
"copilot.storage",
"crypto.privateKey",
// TODO(0.27.5): Remove legacy DB override loading after old instances exit and existing rows are cleared.
"db.datasourceUrl",
"indexer.enabled",
"indexer.provider.apiKey",
"indexer.provider.endpoint",
"indexer.provider.password",
"indexer.provider.type",
"indexer.provider.username",
"oauth.providers.apple",
"oauth.providers.github",
"oauth.providers.google",
"oauth.providers.oidc",
"payment.enabled",
"payment.revenuecat",
"payment.stripe",
// TODO(0.27.5): Remove legacy Redis DB override loading after old instances exit and existing rows are cleared.
"redis.db",
"redis.host",
"redis.ioredis",
"redis.password",
"redis.port",
"redis.username",
"storages.avatar.storage",
"storages.blob.storage",
];
pub(super) fn is_secret_native_app_config_key(key: &str) -> bool {
matches!(
key,
"copilot.providers.profiles"
| "copilot.storage"
| "crypto.privateKey"
| "db.datasourceUrl"
| "indexer.provider.apiKey"
| "indexer.provider.password"
| "oauth.providers.apple"
| "oauth.providers.github"
| "oauth.providers.google"
| "oauth.providers.oidc"
| "payment.revenuecat"
| "payment.stripe"
| "redis.ioredis"
| "redis.password"
| "storages.avatar.storage"
| "storages.blob.storage"
)
}
pub(super) fn is_static_app_config_key(key: &str) -> bool {
key == "db.datasourceUrl" || key.starts_with("redis.")
}
pub(super) async fn load_app_config_overrides_from_db(pool: &PgPool) -> RuntimeResult<serde_json::Value> {
let rows = match sqlx::query("SELECT id, value FROM app_configs WHERE id = ANY($1) ORDER BY id ASC")
.bind(NATIVE_APP_CONFIG_KEYS)
.fetch_all(pool)
.await
{
Ok(rows) => rows,
Err(sqlx::Error::Database(err)) if err.code().as_deref() == Some("42P01") => {
return Ok(serde_json::Value::Object(Map::new()));
}
Err(err) => return Err(RuntimeError::database("failed to load app config overrides", err)),
};
Ok(app_config_value_from_flat_overrides(rows.into_iter().map(|row| {
let id: String = row.get("id");
let value: serde_json::Value = row.get("value");
(id, value)
})))
}
pub(crate) async fn save_app_config_changes(
pool: &PgPool,
config: &ServerConfig,
bootstrap_private_key: Option<String>,
actor: Option<&str>,
changes: &[AppConfigChange],
) -> RuntimeResult<Vec<String>> {
let mut transaction = pool
.begin()
.await
.map_err(|error| RuntimeError::database("begin app config update", error))?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended('app-config-paths', 0))")
.execute(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("lock app config paths", error))?;
let rows = sqlx::query("SELECT id, value FROM app_configs ORDER BY id")
.fetch_all(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("load app config paths", error))?;
let mut prospective = rows
.into_iter()
.map(|row| (row.get::<String, _>("id"), row.get::<Value, _>("value")))
.collect::<BTreeMap<_, _>>();
let mut changed = BTreeSet::new();
for change in changes {
if is_static_app_config_key(&change.key) && change.value.is_some() {
return Err(RuntimeError::config(
"database and Redis connection settings are bootstrap values and cannot be changed through app_configs",
));
}
if !changed.insert(change.key.clone()) {
return Err(RuntimeError::config(format!(
"duplicate app config path: {}",
change.key
)));
}
if NATIVE_APP_CONFIG_KEYS.contains(&change.key.as_str()) != change.native
|| change.key == "auth.session.signingKeys"
{
return Err(RuntimeError::config(format!(
"invalid app config owner: {}",
change.key
)));
}
match &change.value {
Some(value) => {
prospective.insert(change.key.clone(), value.clone());
}
None => {
prospective.remove(&change.key);
}
}
}
let keys = prospective.keys().collect::<Vec<_>>();
for (index, key) in keys.iter().enumerate() {
if let Some(overlap) = keys[index + 1..]
.iter()
.find(|candidate| candidate.starts_with(&format!("{key}.")))
{
return Err(RuntimeError::config(format!(
"app config paths must not overlap: {key} and {overlap}"
)));
}
}
let mut native = config.baseline().clone();
merge_config_value(
&mut native,
app_config_value_from_flat_overrides(
prospective
.iter()
.filter(|(key, _)| NATIVE_APP_CONFIG_KEYS.contains(&key.as_str()))
.map(|(key, value)| (key.as_str(), value.clone())),
),
);
BackendRuntimeConfig::from_value(bootstrap_private_key, native.clone(), config.deployment())?;
ObjectStorageService::from_config_value(&native)?;
backends_from_flat_overrides(
prospective
.iter()
.filter(|(key, _)| matches!(key.as_str(), "storages.blob.storage" | "storages.avatar.storage"))
.map(|(key, value)| (key.as_str(), value.clone())),
)?;
for change in changes {
match &change.value {
Some(value) => {
sqlx::query(
"INSERT INTO app_configs(id,value,last_updated_by,created_at,updated_at) \
VALUES($1,$2,$3,clock_timestamp(),clock_timestamp()) ON CONFLICT(id) DO UPDATE SET \
value=$2,last_updated_by=$3,updated_at=clock_timestamp()",
)
.bind(&change.key)
.bind(value)
.bind(actor)
.execute(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("write app config", error))?;
}
None => {
sqlx::query("DELETE FROM app_configs WHERE id=$1")
.bind(&change.key)
.execute(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("clear app config", error))?;
}
}
}
transaction
.commit()
.await
.map_err(|error| RuntimeError::database("commit app config update", error))?;
Ok(changed.into_iter().collect())
}
#[cfg(test)]
pub(super) fn app_config_from_flat_overrides<I, S>(rows: I) -> RuntimeResult<AppConfigFile>
where
I: IntoIterator<Item = (S, serde_json::Value)>,
S: AsRef<str>,
{
deserialize_app_config(app_config_value_from_flat_overrides(rows))
}
pub(super) fn app_config_value_from_flat_overrides<I, S>(rows: I) -> serde_json::Value
where
I: IntoIterator<Item = (S, serde_json::Value)>,
S: AsRef<str>,
{
let mut root = Map::new();
let mut rows = rows.into_iter().collect::<Vec<_>>();
rows.sort_by(|(left, _), (right, _)| left.as_ref().cmp(right.as_ref()));
for (path, value) in rows {
insert_flat_override(&mut root, path.as_ref(), value);
}
serde_json::Value::Object(root)
}
pub(in crate::runtime) fn insert_flat_override(
root: &mut Map<String, serde_json::Value>,
path: &str,
value: serde_json::Value,
) {
let mut parts = path.split('.').peekable();
let mut current = root;
while let Some(part) = parts.next() {
if parts.peek().is_none() {
current.insert(part.to_string(), value);
return;
}
let entry = current
.entry(part.to_string())
.or_insert_with(|| serde_json::Value::Object(Map::new()));
if !entry.is_object() {
*entry = serde_json::Value::Object(Map::new());
}
current = entry.as_object_mut().expect("override node must be an object");
}
}
#[cfg(test)]
mod tests {
use sqlx::postgres::PgPoolOptions;
use super::{AppConfigChange, ServerConfig, load_app_config_overrides_from_db, save_app_config_changes};
#[tokio::test]
async fn reads_only_native_app_config_keys() {
let Ok(database_url) = std::env::var("DATABASE_URL") else {
return;
};
let pool = PgPoolOptions::new()
.max_connections(1)
.connect(&database_url)
.await
.unwrap();
sqlx::query("CREATE TEMP TABLE app_configs (id text PRIMARY KEY, value jsonb NOT NULL)")
.execute(&pool)
.await
.unwrap();
sqlx::query("INSERT INTO app_configs (id, value) VALUES ($1, $2), ($3, $4)")
.bind("copilot.enabled")
.bind(serde_json::json!(false))
.bind("server.name")
.bind(serde_json::json!("node-only"))
.execute(&pool)
.await
.unwrap();
let overrides = load_app_config_overrides_from_db(&pool).await.unwrap();
assert_eq!(overrides, serde_json::json!({ "copilot": { "enabled": false } }));
}
#[tokio::test]
async fn app_config_set_clear_and_failed_batch_are_atomic() {
let Ok(database_url) = std::env::var("DATABASE_URL") else {
return;
};
let pool = PgPoolOptions::new()
.max_connections(1)
.connect(&database_url)
.await
.unwrap();
sqlx::query(
"CREATE TEMP TABLE app_configs (id text PRIMARY KEY, value jsonb NOT NULL, last_updated_by text, created_at \
timestamptz DEFAULT clock_timestamp(), updated_at timestamptz DEFAULT clock_timestamp(), CONSTRAINT \
fail_second CHECK (id <> 'server.second'))",
)
.execute(&pool)
.await
.unwrap();
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("config.json");
std::fs::write(&path, r#"{"deployment":{"type":"cloud"},"copilot":{"enabled":false}}"#).unwrap();
let config = ServerConfig::open(&path, None).unwrap();
let node = |key: &str, value| AppConfigChange {
key: key.to_string(),
value,
native: false,
};
let native = |key: &str, value| AppConfigChange {
key: key.to_string(),
value,
native: true,
};
save_app_config_changes(
&pool,
&config,
None,
None,
&[
node("server.name", Some(serde_json::json!("configured"))),
native("copilot.enabled", Some(serde_json::json!(false))),
],
)
.await
.unwrap();
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
.fetch_one(&pool)
.await
.unwrap(),
2
);
save_app_config_changes(
&pool,
&config,
None,
None,
&[node("server.name", None), native("copilot.enabled", None)],
)
.await
.unwrap();
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
.fetch_one(&pool)
.await
.unwrap(),
0
);
assert!(
save_app_config_changes(
&pool,
&config,
None,
None,
&[
node("server.first", Some(serde_json::json!(true))),
node("server.second", Some(serde_json::json!(true))),
],
)
.await
.is_err()
);
assert!(
save_app_config_changes(
&pool,
&config,
None,
None,
&[native("redis.host", Some(serde_json::json!("redis.example")))],
)
.await
.is_err()
);
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
.fetch_one(&pool)
.await
.unwrap(),
0
);
assert!(
save_app_config_changes(&pool, &config, None, None, &[node("copilot.enabled", None)])
.await
.is_err()
);
assert!(
save_app_config_changes(
&pool,
&config,
None,
None,
&[native(
"db.datasourceUrl",
Some(serde_json::json!("postgresql://unreachable"))
)],
)
.await
.is_err()
);
assert!(
save_app_config_changes(
&pool,
&config,
None,
None,
&[native(
"storages.blob.storage",
Some(serde_json::json!({"provider":"invalid","bucket":"blobs","config":{}}))
)],
)
.await
.is_err()
);
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
.fetch_one(&pool)
.await
.unwrap(),
0
);
}
}
@@ -0,0 +1,318 @@
use std::fs;
use serde_json::Map;
use super::*;
#[test]
fn server_config_uses_one_explicit_path() {
let directory = tempfile::tempdir().unwrap();
let config_path = directory.path().join("config.json");
fs::write(
&config_path,
r#"{"deployment":{"type":"selfhosted"},"copilot":{"enabled":false,"exa":{"key":"node"}},"crypto":{"privateKey":"native-only"},"redis":{"ioredis":{"enableAutoPipelining":true}},"server":{"name":"example"}}"#,
)
.unwrap();
let config = ServerConfig::open(&config_path, None).unwrap();
assert!(ServerConfig::open(&config_path, Some("unknown")).is_ok());
assert_eq!(config.path(), config_path.canonicalize().unwrap());
assert!(config.deployment() == Deployment::SelfHosted);
assert_eq!(config.baseline()["copilot"]["enabled"], false);
assert_eq!(
config.node_owned(),
serde_json::json!({"copilot":{"exa":{"key":"node"}},"server":{"name":"example"}})
);
assert_eq!(
config.redis_node_options(),
serde_json::json!({"enableAutoPipelining":true})
);
assert_eq!(
config.public_native_baseline(),
serde_json::json!({"copilot":{"enabled":false}})
);
let native = BackendRuntimeConfig::from_server_config(None, &config).unwrap();
assert!(native.deployment == Deployment::SelfHosted);
assert_eq!(native.private_key.as_str(), "native-only");
assert_eq!(native.redis.url, config.redis_url().unwrap());
let storage = crate::runtime::storage_runtime::StorageRuntimeConfig::from_server_config(&config).unwrap();
assert_eq!(native.database_url, storage.database_url);
assert!(storage.object_storage.backends.contains_key("blob"));
fs::write(&config_path, r#"{"copilot":{"enabled":false}}"#).unwrap();
assert!(ServerConfig::open(&config_path, None).is_err());
let upgraded = ServerConfig::open(&config_path, Some("selfhosted")).unwrap();
assert!(upgraded.deployment() == Deployment::SelfHosted);
assert!(ServerConfig::open(&config_path, Some("unknown")).is_err());
fs::write(
&config_path,
r#"{"deployment":{"type":"selfhosted"},"copilot":{"enabled":"yes"}}"#,
)
.unwrap();
assert!(ServerConfig::open(&config_path, None).is_err());
fs::write(
&config_path,
r#"{"deployment":{"type":"selfhosted"},"indexer":{"provider":{"type":"unknown"}}}"#,
)
.unwrap();
assert!(ServerConfig::open(&config_path, None).is_err());
fs::write(
&config_path,
r#"{"deployment":{"type":"selfhosted"},"indexer":{"provider":{"type":"elasticsearch","endpoint":"invalid"}}}"#,
)
.unwrap();
let config = ServerConfig::open(&config_path, None).unwrap();
assert!(BackendRuntimeConfig::from_server_config(None, &config).is_err());
fs::write(
&config_path,
r#"{"deployment":{"type":"selfhosted"},"auth":{"token":{"accessTokenTtl":30}}}"#,
)
.unwrap();
assert!(ServerConfig::open(&config_path, None).is_err());
fs::write(
&config_path,
r#"{"deployment":{"type":"selfhosted"},"redis":{"port":0}}"#,
)
.unwrap();
assert!(ServerConfig::open(&config_path, None).is_err());
fs::write(
&config_path,
r#"{"deployment":{"type":"selfhosted"},"oauth":{"providers":{"oidc":{"issuer":"file:///etc/passwd"}}}}"#,
)
.unwrap();
assert!(ServerConfig::open(&config_path, None).is_err());
}
#[test]
fn blank_database_urls_are_ignored() {
assert_eq!(non_empty_string("".to_string()), None);
assert_eq!(non_empty_string(" ".to_string()), None);
assert_eq!(
non_empty_string("postgresql://affine:affine@localhost:5432/affine".to_string()),
Some("postgresql://affine:affine@localhost:5432/affine".to_string())
);
}
#[test]
fn ignores_storage_app_config_values() {
let app_config = app_config_from_flat_overrides([
(
"storages.blob.storage",
serde_json::json!({"provider": "cloudflare-r2"}),
),
("db.datasourceUrl", serde_json::json!("postgresql://example/runtime")),
])
.unwrap();
assert_eq!(
app_config.database_url().as_deref(),
Some("postgresql://example/runtime")
);
}
#[test]
fn expands_module_config_paths_from_json_files() {
let app_config = app_config_from_module_json(serde_json::json!({
"copilot": {
"enabled": true,
"byok.enabled": false,
"providers.profiles": [{
"id": "managed-openai",
"type": "openai",
"models": ["gpt-5.6-luna"],
"config": { "apiKey": "test" }
}]
}
}))
.unwrap();
let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap();
assert!(copilot.enabled);
assert!(!copilot.byok.enabled);
assert_eq!(copilot.providers.profiles.len(), 1);
assert_eq!(copilot.providers.profiles[0].id, "managed-openai");
let missing_models = app_config_from_flat_overrides([(
"copilot.providers.profiles",
serde_json::json!([{
"id": "managed-openai",
"type": "openai",
"config": {}
}]),
)]);
assert!(missing_models.is_err());
let app_config = app_config_from_flat_overrides([(
"copilot.providers.profiles",
serde_json::json!([{
"id": "managed-openai",
"type": "openai",
"models": [],
"config": {}
}]),
)])
.unwrap();
let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap();
assert!(validate_copilot_config(&copilot).is_err());
let app_config = app_config_from_flat_overrides([(
"copilot.providers.profiles",
serde_json::json!([
{
"id": "anthropic-direct",
"type": "anthropic",
"models": ["claude-sonnet-4-6"],
"config": {}
},
{
"id": "anthropic-vertex",
"type": "anthropicVertex",
"models": ["claude-sonnet-4-6"],
"config": {}
}
]),
)])
.unwrap();
let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap();
assert!(validate_copilot_config(&copilot).is_err());
let directory = tempfile::tempdir().unwrap();
let config_path = directory.path().join("config.json");
fs::write(
&config_path,
r#"{"deployment":{"type":"cloud"},"copilot":{"enabled":true,"byok.enabled":false,"byok.allowCustomEndpoint":true}}"#,
)
.unwrap();
let config = ServerConfig::open(&config_path, None).unwrap();
let copilot = BackendRuntimeConfig::from_server_config(None, &config).unwrap().copilot;
assert!(!copilot.byok.enabled);
assert!(copilot.byok.allow_custom_endpoint);
}
#[test]
fn search_config_keeps_disabled_state_separate_from_embedded_provider() {
let disabled = app_config_from_flat_overrides([
("indexer.enabled", serde_json::json!(false)),
("indexer.provider.type", serde_json::json!("embedded")),
])
.unwrap();
let disabled: SearchRuntimeConfig = disabled.indexer.unwrap().into();
assert!(!disabled.enabled);
assert_eq!(disabled.provider, "embedded");
let enabled = app_config_from_flat_overrides([
("indexer.enabled", serde_json::json!(true)),
("indexer.provider.type", serde_json::json!("elasticsearch")),
])
.unwrap();
let enabled: SearchRuntimeConfig = enabled.indexer.unwrap().into();
assert!(enabled.enabled);
assert_eq!(enabled.provider, "elasticsearch");
let enabled_without_provider = app_config_from_module_json(serde_json::json!({
"indexer": { "enabled": true }
}))
.unwrap();
let enabled_without_provider: SearchRuntimeConfig = enabled_without_provider.indexer.unwrap().into();
assert!(enabled_without_provider.enabled);
assert_eq!(enabled_without_provider.provider, "embedded");
let manticore = app_config_from_flat_overrides([
("indexer.enabled", serde_json::json!(true)),
("indexer.provider.type", serde_json::json!("manticoresearch")),
("indexer.provider.endpoint", serde_json::json!("http://localhost:9308")),
])
.unwrap();
let manticore: SearchRuntimeConfig = manticore.indexer.unwrap().into();
assert!(manticore.enabled);
assert_eq!(manticore.provider, "manticoresearch");
}
#[test]
fn partial_database_config_preserves_file_config_siblings() {
let mut file_config = expand_module_config_paths(serde_json::json!({
"copilot": {
"enabled": true,
"byok": { "enabled": true, "allowCustomEndpoint": true },
"providers": {
"profiles": [{
"id": "managed-openai",
"type": "openai",
"models": ["gpt-5.6-luna"],
"config": { "apiKey": "test" }
}]
}
}
}));
let database_config = app_config_value_from_flat_overrides([("copilot.byok.enabled", serde_json::json!(false))]);
merge_config_value(&mut file_config, database_config);
let copilot: CopilotRuntimeConfig = deserialize_app_config(file_config)
.unwrap()
.copilot
.unwrap()
.try_into()
.unwrap();
assert!(copilot.enabled);
assert!(!copilot.byok.enabled);
assert!(copilot.byok.allow_custom_endpoint);
assert_eq!(copilot.providers.profiles.len(), 1);
assert_eq!(copilot.providers.profiles[0].id, "managed-openai");
}
#[test]
fn nested_database_config_overrides_are_order_independent() {
let app_config = app_config_from_flat_overrides([
("copilot.byok.enabled", serde_json::json!(false)),
(
"copilot.byok",
serde_json::json!({ "enabled": true, "allowCustomEndpoint": true }),
),
])
.unwrap();
let byok = CopilotRuntimeConfig::try_from(app_config.copilot.unwrap())
.unwrap()
.byok;
assert!(!byok.enabled);
assert!(byok.allow_custom_endpoint);
}
#[test]
fn database_config_only_replaces_an_active_private_key_explicitly() {
let active = BackendRuntimeConfig {
database_url: "postgresql://active".to_string(),
auth: AuthRuntimeConfig::default(),
invite_quota: InviteQuotaConfig::default(),
private_key: Arc::new(Zeroizing::new("active-private-key".to_string())),
deployment: Deployment::Cloud,
copilot: CopilotRuntimeConfig::default(),
search: SearchRuntimeConfig::default(),
redis: RedisRuntimeConfig::default(),
payment: PaymentRuntimeConfig::default(),
};
let empty = serde_json::Value::Object(Map::new());
let unchanged = active.apply_db_overrides(empty.clone(), empty.clone()).unwrap();
assert_eq!(unchanged.private_key.as_str(), "active-private-key");
let overridden = active
.apply_db_overrides(
empty,
app_config_value_from_flat_overrides([("crypto.privateKey", serde_json::json!("database-private-key"))]),
)
.unwrap();
assert_eq!(overridden.private_key.as_str(), "database-private-key");
}
#[test]
fn invite_abuse_policy_is_internal_while_action_delay_is_configurable() {
let app_config = app_config_from_flat_overrides([
("auth.newAccountActionDelay", serde_json::json!(123)),
("auth.untrustedPolicyOverride", serde_json::json!("runtime-salt-v2")),
("auth.untrustedDomainList", serde_json::json!(["Example.COM."])),
])
.unwrap();
let config = app_config.invite_quota_config();
assert_eq!(config.new_account_action_delay_seconds, 123);
}
@@ -0,0 +1,146 @@
use std::sync::Arc;
use serde::Deserialize;
use zeroize::Zeroizing;
use super::CopilotRuntimeConfig;
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Deployment {
Cloud,
SelfHosted,
}
pub(crate) struct BackendRuntimeConfig {
pub(crate) database_url: String,
pub(crate) auth: AuthRuntimeConfig,
pub(crate) invite_quota: InviteQuotaConfig,
pub(crate) private_key: Arc<Zeroizing<String>>,
pub(crate) deployment: Deployment,
pub(crate) copilot: CopilotRuntimeConfig,
pub(crate) search: SearchRuntimeConfig,
pub(crate) redis: RedisRuntimeConfig,
pub(crate) payment: PaymentRuntimeConfig,
}
#[derive(Clone)]
pub(crate) struct AuthRuntimeConfig {
pub(crate) allow_signup: bool,
pub(crate) allow_signup_for_oauth: bool,
pub(crate) require_email_domain_verification: bool,
pub(crate) session_ttl_seconds: i64,
pub(crate) session_ttr_seconds: i64,
pub(crate) access_token_ttl_seconds: i64,
pub(crate) refresh_idle_ttl_seconds: i64,
pub(crate) refresh_absolute_ttl_seconds: i64,
pub(crate) refresh_grace_seconds: i64,
pub(crate) refresh_retention_seconds: i64,
pub(crate) oauth: OAuthRuntimeConfig,
}
impl Default for AuthRuntimeConfig {
fn default() -> Self {
Self {
allow_signup: true,
allow_signup_for_oauth: true,
require_email_domain_verification: false,
session_ttl_seconds: 15 * 24 * 60 * 60,
session_ttr_seconds: 7 * 24 * 60 * 60,
access_token_ttl_seconds: 15 * 60,
refresh_idle_ttl_seconds: 30 * 24 * 60 * 60,
refresh_absolute_ttl_seconds: 180 * 24 * 60 * 60,
refresh_grace_seconds: 30,
refresh_retention_seconds: 30 * 24 * 60 * 60,
oauth: OAuthRuntimeConfig::default(),
}
}
}
#[derive(Clone, Default)]
pub(crate) struct OAuthRuntimeConfig {
pub(crate) providers: std::collections::BTreeMap<String, OAuthProviderRuntimeConfig>,
}
#[derive(Clone)]
pub(crate) struct OAuthProviderRuntimeConfig {
pub(crate) client_id: String,
pub(crate) client_secret: Arc<Zeroizing<String>>,
pub(crate) args: std::collections::BTreeMap<String, String>,
pub(crate) issuer: Option<String>,
pub(crate) allow_private_network: bool,
pub(crate) apple_private_key: Option<Arc<Zeroizing<String>>>,
pub(crate) apple_key_id: Option<String>,
pub(crate) apple_team_id: Option<String>,
}
#[derive(Clone, Default)]
pub(crate) struct PaymentRuntimeConfig {
pub(crate) enabled: bool,
pub(crate) stripe: Option<StripeRuntimeConfig>,
pub(crate) revenuecat: Option<RevenueCatRuntimeConfig>,
}
#[derive(Clone)]
pub(crate) struct StripeRuntimeConfig {
pub(crate) api_key: Arc<Zeroizing<String>>,
pub(crate) webhook_key: Arc<Zeroizing<String>>,
pub(crate) account_id: String,
pub(crate) live: bool,
}
#[derive(Clone)]
pub(crate) struct RevenueCatRuntimeConfig {
pub(crate) api_key: Arc<Zeroizing<String>>,
pub(crate) webhook_auth: Arc<Zeroizing<String>>,
pub(crate) project_id: String,
pub(crate) production: bool,
pub(crate) product_map: std::collections::BTreeMap<String, PaymentProductConfig>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase")]
pub(crate) struct PaymentProductConfig {
pub(crate) plan: String,
pub(crate) recurring: String,
}
#[derive(Clone, Debug, Default)]
pub(crate) struct RedisRuntimeConfig {
pub(crate) url: Option<String>,
}
#[derive(Clone, Debug)]
pub(crate) struct SearchRuntimeConfig {
pub(crate) enabled: bool,
pub(crate) provider: String,
pub(crate) endpoint: String,
pub(crate) api_key: String,
pub(crate) username: String,
pub(crate) password: String,
}
impl Default for SearchRuntimeConfig {
fn default() -> Self {
Self {
enabled: false,
provider: "embedded".to_string(),
endpoint: String::new(),
api_key: String::new(),
username: String::new(),
password: String::new(),
}
}
}
#[derive(Clone, Debug)]
pub(crate) struct InviteQuotaConfig {
pub(crate) new_account_action_delay_seconds: i64,
}
impl Default for InviteQuotaConfig {
fn default() -> Self {
Self {
new_account_action_delay_seconds: 24 * 60 * 60,
}
}
}
@@ -1,221 +0,0 @@
use jsonschema::Draft;
use napi::{Error, Result, Status};
use schemars::{JsonSchema, generate::SchemaSettings};
use serde_json::{Value, from_value, json, to_value};
use super::{
CopilotManagedProfileConfigFile, CopilotRuntimeConfig, CopilotRuntimeConfigFile, RuntimeError,
SUPPORTED_BYOK_PROVIDERS, validate_copilot_config,
};
const COPILOT_MODULE: &str = "copilot";
#[napi_derive::napi(object)]
pub struct AppConfigDescriptor {
pub key: String,
pub description: String,
pub default_value: Value,
pub schema: Value,
pub internal: bool,
}
fn invalid_config(message: impl Into<String>) -> Error {
Error::new(Status::InvalidArg, message.into())
}
fn schema_for<T: JsonSchema>() -> Value {
let schema = SchemaSettings::draft07().into_generator().into_root_schema_for::<T>();
to_value(schema).expect("config schema should serialize")
}
fn descriptors() -> Vec<AppConfigDescriptor> {
let defaults = CopilotRuntimeConfigFile::default();
let mut allowed_providers_schema = schema_for::<Vec<String>>();
allowed_providers_schema["items"]["enum"] = json!(SUPPORTED_BYOK_PROVIDERS);
vec![
AppConfigDescriptor {
key: "byok.enabled".to_string(),
description: "Allow workspace owners and admins to configure AI provider keys through AI BYOK.".to_string(),
default_value: json!(defaults.byok.enabled),
schema: schema_for::<bool>(),
internal: false,
},
AppConfigDescriptor {
key: "byok.allowedProviders".to_string(),
description: "AI providers that workspace owners and admins may add through AI BYOK.".to_string(),
default_value: json!(defaults.byok.allowed_providers),
schema: allowed_providers_schema,
internal: false,
},
AppConfigDescriptor {
key: "byok.allowCustomEndpoint".to_string(),
description: "Allow AI BYOK keys to use a custom provider endpoint.".to_string(),
default_value: json!(defaults.byok.allow_custom_endpoint),
schema: schema_for::<bool>(),
internal: false,
},
AppConfigDescriptor {
key: "byok.allowPrivateEndpoint".to_string(),
description: "Whether workspace BYOK custom endpoints may resolve to private network targets. Enabling this \
allows workspace owners and admins to send provider probe requests to the private network."
.to_string(),
default_value: json!(defaults.byok.allow_private_endpoint),
schema: schema_for::<bool>(),
internal: false,
},
AppConfigDescriptor {
key: "providers.profiles".to_string(),
description: "The profile list for copilot providers.".to_string(),
default_value: json!(defaults.providers.profiles),
schema: schema_for::<Vec<CopilotManagedProfileConfigFile>>(),
internal: true,
},
]
}
fn validate_leaf(key: &str, value: Value) -> std::result::Result<(), RuntimeError> {
let mut config = CopilotRuntimeConfigFile::default();
match key {
"byok.enabled" => {
config.byok.enabled =
from_value(value).map_err(|error| RuntimeError::json("invalid copilot BYOK enabled config", error))?;
}
"byok.allowedProviders" => {
config.byok.allowed_providers = from_value(value)
.map_err(|error| RuntimeError::json("invalid copilot BYOK allowed providers config", error))?;
}
"byok.allowCustomEndpoint" => {
config.byok.allow_custom_endpoint =
from_value(value).map_err(|error| RuntimeError::json("invalid copilot BYOK custom endpoint config", error))?;
}
"byok.allowPrivateEndpoint" => {
config.byok.allow_private_endpoint =
from_value(value).map_err(|error| RuntimeError::json("invalid copilot BYOK private endpoint config", error))?;
}
"providers.profiles" => {
config.providers.profiles =
from_value(value).map_err(|error| RuntimeError::json("invalid managed copilot profiles config", error))?;
}
_ => return Err(RuntimeError::config(format!("unknown copilot app config key: {key}"))),
}
let config = CopilotRuntimeConfig::try_from(config)?;
validate_copilot_config(&config)
}
#[napi_derive::napi(catch_unwind)]
pub fn app_config_descriptors(module: String) -> Result<Vec<AppConfigDescriptor>> {
if module != COPILOT_MODULE {
return Err(invalid_config(format!("unknown native app config module: {module}")));
}
Ok(descriptors())
}
#[napi_derive::napi(catch_unwind)]
pub fn validate_app_config_value(module: String, key: String, value: Value) -> Result<Vec<String>> {
if module != COPILOT_MODULE {
return Err(invalid_config(format!("unknown native app config module: {module}")));
}
let descriptor = descriptors()
.into_iter()
.find(|descriptor| descriptor.key == key)
.ok_or_else(|| invalid_config(format!("unknown native app config key: {module}.{key}")))?;
let schema = jsonschema::options()
.with_draft(Draft::Draft7)
.build(&descriptor.schema)
.map_err(|error| invalid_config(format!("failed to compile app config schema: {error}")))?;
let errors = schema
.iter_errors(&value)
.map(|error| error.to_string())
.collect::<Vec<_>>();
if !errors.is_empty() {
return Ok(errors);
}
Ok(
validate_leaf(&key, value)
.err()
.map(|error| vec![error.to_string()])
.unwrap_or_default(),
)
}
#[cfg(test)]
mod tests {
use super::{app_config_descriptors, json, validate_app_config_value};
#[test]
fn copilot_descriptors_and_validation_share_runtime_contract() {
let descriptors = app_config_descriptors("copilot".to_string()).unwrap();
assert_eq!(
descriptors
.iter()
.map(|descriptor| descriptor.key.as_str())
.collect::<Vec<_>>(),
[
"byok.enabled",
"byok.allowedProviders",
"byok.allowCustomEndpoint",
"byok.allowPrivateEndpoint",
"providers.profiles",
]
);
assert_eq!(descriptors[0].default_value, json!(true));
assert!(descriptors[4].internal);
assert!(
validate_app_config_value(
"copilot".to_string(),
"providers.profiles".to_string(),
json!([{
"id": "managed-openai",
"type": "openai",
"displayName": "OpenAI",
"priority": 1,
"enabled": true,
"models": ["gpt-5.6-luna"],
"middleware": {
"rust": { "request": ["normalize_messages"] },
"node": { "text": ["citation_footnote"] }
},
"config": { "apiKey": "test" }
}]),
)
.unwrap()
.is_empty()
);
}
#[test]
fn copilot_validation_rejects_invalid_leaf_values() {
for (key, value) in [
("byok.enabled", json!("yes")),
("byok.allowedProviders", json!(["openai", "openai"])),
(
"providers.profiles",
json!([{
"id": "invalid id",
"type": "openai",
"models": ["gpt-5.6-luna"],
"config": {}
}]),
),
(
"providers.profiles",
json!([{
"id": "managed-openai",
"type": "openai",
"models": ["gpt-5.6-luna"],
"middleware": { "node": { "text": ["unknown"] } },
"config": {}
}]),
),
] {
assert!(
!validate_app_config_value("copilot".to_string(), key.to_string(), value)
.unwrap()
.is_empty(),
"{key}"
);
}
}
}
+5 -7
View File
@@ -2,7 +2,7 @@ pub mod backend_runtime;
pub mod storage_runtime;
pub(crate) mod config;
mod config_descriptor;
pub use config::ServerConfigHandle;
pub(crate) mod error;
mod http;
pub(crate) mod migrations;
@@ -14,14 +14,12 @@ pub(crate) mod types;
pub(super) use affine_doc_loader::blob_refs::{BlobRefProjectionError, extract_blob_refs};
#[cfg(test)]
pub(crate) use config::AuthRuntimeConfig;
pub use config::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value};
pub(crate) use config::{
BackendRuntimeConfig, ConfigSource, CopilotManagedProfileConfig, CopilotManagedProfileConfigFile,
CopilotRuntimeConfig, CopilotRuntimeConfigFile, Deployment, InviteQuotaConfig, OAuthProviderRuntimeConfig,
PaymentProductConfig, PaymentRuntimeConfig, RedisRuntimeConfig, RevenueCatRuntimeConfig, SearchRuntimeConfig,
StripeRuntimeConfig,
BackendRuntimeConfig, CopilotManagedProfileConfig, CopilotRuntimeConfig, Deployment, InviteQuotaConfig,
OAuthProviderRuntimeConfig, PaymentProductConfig, PaymentRuntimeConfig, RedisRuntimeConfig, RevenueCatRuntimeConfig,
SearchRuntimeConfig, StripeRuntimeConfig,
};
use config::{SUPPORTED_BYOK_PROVIDERS, validate_copilot_config};
pub use config_descriptor::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value};
pub(crate) use error::{RuntimeError, RuntimeResult, napi_error, to_napi_error};
pub(in crate::runtime) use http::webpki_tls_config;
@@ -1,16 +1,16 @@
use std::{collections::HashMap, fs};
use std::collections::HashMap;
use serde::Deserialize;
use serde_json::{Map, Value};
use sqlx::{PgPool, Row};
use super::{config::ObjectStorageConfig, types::StorageProviderConfig};
use crate::runtime::{ConfigSource, RuntimeError, RuntimeResult};
use crate::runtime::{RuntimeError, RuntimeResult};
#[derive(Clone, Debug)]
pub(in crate::runtime) enum StorageBackendConfig {
Fs(FsStorageConfig),
S3(ObjectStorageConfig),
S3(Box<ObjectStorageConfig>),
Assetpack(FsStorageConfig),
}
@@ -39,7 +39,9 @@ struct CopilotConfigFile {
}
impl StorageBackendConfig {
fn from_provider_config(storage: Option<StorageProviderConfig>) -> RuntimeResult<Option<Self>> {
pub(in crate::runtime) fn from_provider_config(
storage: Option<StorageProviderConfig>,
) -> RuntimeResult<Option<Self>> {
let Some(storage) = storage else {
return Ok(None);
};
@@ -60,7 +62,7 @@ impl StorageBackendConfig {
}))
}
"aws-s3" | "cloudflare-r2" => ObjectStorageConfig::from_provider_config(Some(storage))
.map(|config| config.map(Self::S3))
.map(|config| config.map(|config| Self::S3(Box::new(config))))
.map_err(Into::into),
provider => Err(RuntimeError::config(format!(
"unsupported object storage provider: {provider}"
@@ -124,56 +126,34 @@ impl ObjectStorageAppConfig {
}
fn default_object_storage_config() -> ObjectStorageAppConfig {
let storage = |bucket: &str| {
serde_json::json!({
"provider": "fs",
"bucket": bucket,
"config": { "path": "~/.affine/storage" }
})
};
ObjectStorageAppConfig {
storages: Some(HashMap::from([
("blob.storage".to_string(), storage("blobs")),
("avatar.storage".to_string(), storage("avatars")),
("blob.storage".to_string(), default_storage_provider_config("blobs")),
("avatar.storage".to_string(), default_storage_provider_config("avatars")),
])),
copilot: Some(CopilotConfigFile {
storage: Some(StorageProviderConfig {
provider: "fs".to_string(),
bucket: "copilot".to_string(),
config: serde_json::json!({ "path": "~/.affine/storage" }),
}),
storage: Some(serde_json::from_value(default_storage_provider_config("copilot")).expect("storage default")),
}),
}
}
pub(super) fn backends_from_config_files() -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
backends_from_config_source(&ConfigSource::default())
}
pub(super) fn backends_from_config_source(
source: &ConfigSource,
) -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
let mut merged = default_object_storage_config();
for path in source.paths() {
if !path.exists() {
if source.required(&path) {
return Err(RuntimeError::config(format!(
"config file does not exist: {}",
path.display()
)));
}
continue;
}
let raw = fs::read_to_string(&path).map_err(|err| RuntimeError::io("failed to read config file", err))?;
let config = serde_json::from_str(&raw).map_err(|err| RuntimeError::json("failed to parse config file", err))?;
merged.merge(config);
}
merged.storage_backends()
pub(in crate::runtime) fn default_storage_provider_config(bucket: &str) -> Value {
serde_json::json!({
"provider": "fs",
"bucket": bucket,
"config": { "path": "~/.affine/storage" }
})
}
#[cfg(test)]
pub(super) fn backends_from_config_json(config_json: &str) -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
let config = serde_json::from_str::<ObjectStorageAppConfig>(config_json)
let value =
serde_json::from_str(config_json).map_err(|err| RuntimeError::json("invalid object storage config", err))?;
backends_from_config_value(&value)
}
pub(super) fn backends_from_config_value(value: &Value) -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
let config = serde_json::from_value::<ObjectStorageAppConfig>(value.clone())
.map_err(|err| RuntimeError::json("invalid object storage config", err))?;
let mut merged = default_object_storage_config();
merged.merge(config);
@@ -181,15 +161,34 @@ pub(super) fn backends_from_config_json(config_json: &str) -> RuntimeResult<Hash
}
pub(super) async fn backends_from_db(pool: &PgPool) -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
let rows = match sqlx::query("SELECT id, value FROM app_configs").fetch_all(pool).await {
let rows = match sqlx::query(
"SELECT id, value FROM app_configs WHERE id IN ('storages.blob.storage', 'storages.avatar.storage', \
'copilot.storage')",
)
.fetch_all(pool)
.await
{
Ok(rows) => rows,
Err(sqlx::Error::Database(err)) if err.code().as_deref() == Some("42P01") => return Ok(HashMap::new()),
Err(err) => return Err(RuntimeError::database("failed to load app config overrides", err)),
};
backends_from_flat_overrides(
rows
.into_iter()
.map(|row| (row.get::<String, _>("id"), row.get::<Value, _>("value"))),
)
}
pub(in crate::runtime) fn backends_from_flat_overrides<I, S>(
rows: I,
) -> RuntimeResult<HashMap<String, StorageBackendConfig>>
where
I: IntoIterator<Item = (S, Value)>,
S: AsRef<str>,
{
let mut root = Map::new();
for row in rows {
let path: String = row.get("id");
let value: Value = row.get("value");
for (path, value) in rows {
let path = path.as_ref();
let Some((module, key)) = path.split_once('.') else {
continue;
};
@@ -1,9 +1,14 @@
use std::time::{SystemTime, UNIX_EPOCH};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use base64::{
Engine as _,
engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD},
};
use hmac::{Hmac, KeyInit, Mac};
use rusty_s3::{Bucket, Credentials, UrlStyle};
use schemars::{JsonSchema, generate::SchemaSettings};
use serde::Deserialize;
use serde_json::{Value, json, to_value};
use sha2::Sha256;
use url::Url;
@@ -31,15 +36,17 @@ pub(crate) struct ObjectStorageConfig {
pub(crate) presign_sign_content_type_for_put: Option<bool>,
pub(crate) use_presigned_url: bool,
pub(crate) proxy_upload: bool,
pub(crate) upload_url_prefix: Option<String>,
pub(crate) proxy_sign_key: Option<String>,
pub(crate) custom_get_url_prefix: Option<String>,
pub(crate) custom_get_sign_key: Option<String>,
}
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
struct S3ConfigFile {
endpoint: Option<String>,
region: Option<String>,
region: String,
credentials: Option<S3CredentialsConfigFile>,
force_path_style: Option<bool>,
request_timeout_ms: Option<u64>,
@@ -49,7 +56,7 @@ struct S3ConfigFile {
use_presigned_url: Option<UsePresignedUrlConfigFile>,
}
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
struct R2ConfigFile {
account_id: String,
@@ -63,14 +70,14 @@ struct R2ConfigFile {
use_presigned_url: Option<UsePresignedUrlConfigFile>,
}
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "lowercase")]
enum R2Jurisdiction {
Default,
Eu,
}
#[derive(Debug, Deserialize, Default)]
#[derive(Debug, Deserialize, Default, JsonSchema)]
#[serde(rename_all = "camelCase")]
struct S3CredentialsConfigFile {
access_key_id: Option<String>,
@@ -78,14 +85,14 @@ struct S3CredentialsConfigFile {
session_token: Option<String>,
}
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
struct S3PresignConfigFile {
expires_in_seconds: Option<u64>,
sign_content_type_for_put: Option<bool>,
}
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
struct UsePresignedUrlConfigFile {
enabled: bool,
@@ -93,6 +100,59 @@ struct UsePresignedUrlConfigFile {
sign_key: Option<String>,
}
pub(in crate::runtime) fn storage_provider_schema() -> Value {
let mut settings = SchemaSettings::draft07();
settings.inline_subschemas = true;
let s3 = to_value(settings.clone().into_generator().into_root_schema_for::<S3ConfigFile>())
.expect("S3 schema should serialize");
let r2 =
to_value(settings.into_generator().into_root_schema_for::<R2ConfigFile>()).expect("R2 schema should serialize");
let storage = |provider: &str, config: Value| {
json!({
"type": "object",
"required": ["provider", "bucket", "config"],
"properties": {
"provider": { "const": provider },
"bucket": { "type": "string" },
"config": config
}
})
};
json!({
"oneOf": [
storage("fs", json!({ "type": "object", "required": ["path"], "properties": { "path": { "type": "string" } } })),
storage("assetpack", json!({ "type": "object", "required": ["path"], "properties": { "path": { "type": "string" } } })),
storage("aws-s3", s3),
storage("cloudflare-r2", r2)
]
})
}
impl UsePresignedUrlConfigFile {
fn settings(
self,
) -> (
bool,
bool,
Option<String>,
Option<String>,
Option<String>,
Option<String>,
) {
let prefix = self.url_prefix.filter(|value| !value.is_empty());
let key = self.sign_key.filter(|value| !value.is_empty());
let custom_get = self.enabled && prefix.is_some() && key.is_some();
(
self.enabled,
self.enabled && key.is_some(),
prefix.clone(),
key.clone(),
custom_get.then_some(prefix).flatten(),
custom_get.then_some(key).flatten(),
)
}
}
impl ObjectStorageConfig {
pub(crate) fn from_provider_config(storage: Option<StorageProviderConfig>) -> ObjectStorageResult<Option<Self>> {
let Some(storage) = storage else {
@@ -112,11 +172,13 @@ impl ObjectStorageConfig {
pub(crate) fn from_s3_config(storage: StorageProviderConfig) -> ObjectStorageResult<Option<Self>> {
let config: S3ConfigFile = serde_json::from_value(storage.config)
.map_err(|err| ObjectStorageError::Config(format!("invalid aws-s3 blob storage config: {err}")))?;
let region = config
.region
.ok_or_else(|| ObjectStorageError::Config("aws-s3 blob storage config requires region".to_string()))?;
let region = config.region;
let endpoint = config.endpoint.or_else(|| Some(resolve_s3_endpoint(&region)));
let credentials = config.credentials.unwrap_or_default();
let (use_presigned_url, proxy_upload, upload_url_prefix, proxy_sign_key, _, _) = config
.use_presigned_url
.map(UsePresignedUrlConfigFile::settings)
.unwrap_or((false, false, None, None, None, None));
Ok(Some(Self {
provider: storage.provider,
@@ -131,8 +193,10 @@ impl ObjectStorageConfig {
min_part_size: config.min_part_size,
presign_expires_in_seconds: config.presign.as_ref().and_then(|v| v.expires_in_seconds),
presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put),
use_presigned_url: config.use_presigned_url.map(|v| v.enabled).unwrap_or(false),
proxy_upload: false,
use_presigned_url,
proxy_upload,
upload_url_prefix,
proxy_sign_key,
custom_get_url_prefix: None,
custom_get_sign_key: None,
}))
@@ -146,25 +210,17 @@ impl ObjectStorageConfig {
Some(R2Jurisdiction::Default) | None => config.account_id,
};
let credentials = config.credentials.unwrap_or_default();
let (use_presigned_url, proxy_upload, custom_get_url_prefix, custom_get_sign_key) = config
let (
use_presigned_url,
proxy_upload,
upload_url_prefix,
proxy_sign_key,
custom_get_url_prefix,
custom_get_sign_key,
) = config
.use_presigned_url
.map(|value| {
let url_prefix = value.url_prefix.filter(|prefix| !prefix.is_empty());
let sign_key = value.sign_key.filter(|key| !key.is_empty());
let custom_get_enabled = value.enabled && url_prefix.is_some() && sign_key.is_some();
let (custom_get_url_prefix, custom_get_sign_key) = if custom_get_enabled {
(url_prefix, sign_key)
} else {
(None, None)
};
(
value.enabled,
custom_get_enabled,
custom_get_url_prefix,
custom_get_sign_key,
)
})
.unwrap_or((false, false, None, None));
.map(UsePresignedUrlConfigFile::settings)
.unwrap_or((false, false, None, None, None, None));
Ok(Some(Self {
provider: storage.provider,
@@ -181,11 +237,59 @@ impl ObjectStorageConfig {
presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put),
use_presigned_url,
proxy_upload,
upload_url_prefix,
proxy_sign_key,
custom_get_url_prefix,
custom_get_sign_key,
}))
}
pub(crate) fn proxy_upload_token(
&self,
path: &str,
fields: &[Value],
expires_at: i64,
) -> ObjectStorageResult<Option<String>> {
// TODO(0.27.5): Review 0.27.4 upload URL compatibility after old instances
// and URLs expire.
let Some(key) = self.proxy_sign_key.as_ref().filter(|_| self.proxy_upload) else {
return Ok(None);
};
let mut canonical = vec![json!("affine-storage-upload"), json!(1), json!("PUT"), json!(path)];
canonical.extend_from_slice(fields);
canonical.push(json!(expires_at));
let payload = serde_json::to_vec(&canonical)
.map_err(|err| ObjectStorageError::Config(format!("invalid upload token payload: {err}")))?;
let mut mac = HmacSha256::new_from_slice(key.as_bytes())
.map_err(|err| ObjectStorageError::Config(format!("invalid upload signing key: {err}")))?;
mac.update(&payload);
Ok(Some(URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes())))
}
pub(crate) fn verify_proxy_upload_token(
&self,
path: &str,
fields: &[Value],
expires_at: i64,
token: &str,
) -> ObjectStorageResult<bool> {
let Some(key) = self.proxy_sign_key.as_ref().filter(|_| self.proxy_upload) else {
return Ok(false);
};
let Ok(signature) = URL_SAFE_NO_PAD.decode(token) else {
return Ok(false);
};
let mut canonical = vec![json!("affine-storage-upload"), json!(1), json!("PUT"), json!(path)];
canonical.extend_from_slice(fields);
canonical.push(json!(expires_at));
let payload = serde_json::to_vec(&canonical)
.map_err(|err| ObjectStorageError::Config(format!("invalid upload token payload: {err}")))?;
let mut mac = HmacSha256::new_from_slice(key.as_bytes())
.map_err(|err| ObjectStorageError::Config(format!("invalid upload signing key: {err}")))?;
mac.update(&payload);
Ok(mac.verify_slice(&signature).is_ok())
}
pub(crate) fn custom_presign_get(&self, key: &ObjectKey) -> ObjectStorageResult<Option<PresignedObjectRequest>> {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -9,9 +9,12 @@ mod service;
mod tests;
pub(crate) mod types;
pub(in crate::runtime) use backend::{FsStorageConfig, StorageBackendConfig};
pub(in crate::runtime) use backend::{
FsStorageConfig, StorageBackendConfig, backends_from_flat_overrides, default_storage_provider_config,
};
#[cfg(test)]
pub(in crate::runtime) use config::ObjectStorageConfig;
pub(in crate::runtime) use config::storage_provider_schema;
pub(crate) use service::ObjectStorageService;
pub(super) use super::webpki_tls_config;
@@ -1,18 +1,19 @@
use std::collections::HashMap;
use serde_json::Value;
use sqlx::PgPool;
use tokio::task::JoinSet;
use super::{
StorageBackendConfig, assetpack,
backend::{backends_from_config_files, backends_from_config_json, backends_from_config_source, backends_from_db},
backend::{backends_from_config_value, backends_from_db},
fs::{delete_many_fs, fs_delete, fs_get, fs_get_range, fs_head, fs_list_page, fs_put},
types::{
MultipartUploadInitResult, MultipartUploadPart, ObjectDeleteOutcome, ObjectGetResult, ObjectKey, ObjectListPage,
ObjectLocator, ObjectMetadata, ObjectPrefix, ObjectPutMetadata, PresignedObjectRequest, StorageScope,
},
};
use crate::runtime::{ConfigSource, RuntimeError, RuntimeResult};
use crate::runtime::{RuntimeError, RuntimeResult};
const DELETE_MANY_CHUNK_SIZE: usize = 500;
const DELETE_MANY_CONCURRENCY: usize = 3;
@@ -23,21 +24,16 @@ pub(crate) struct ObjectStorageService {
}
impl ObjectStorageService {
pub(crate) fn from_config_files() -> RuntimeResult<Self> {
Ok(Self {
backends: backends_from_config_files()?,
})
}
pub(crate) fn from_config_source(source: &ConfigSource) -> RuntimeResult<Self> {
Ok(Self {
backends: backends_from_config_source(source)?,
})
}
#[cfg(test)]
pub(in crate::runtime) fn from_config_json(config_json: &str) -> RuntimeResult<Self> {
Ok(Self {
backends: backends_from_config_json(config_json)?,
backends: super::backend::backends_from_config_json(config_json)?,
})
}
pub(crate) fn from_config_value(value: &Value) -> RuntimeResult<Self> {
Ok(Self {
backends: backends_from_config_value(value)?,
})
}
@@ -1,7 +1,7 @@
use reqwest::StatusCode;
use super::{
backend::backends_from_config_json,
backend::{backends_from_config_json, backends_from_flat_overrides},
config::ObjectStorageConfig,
error::ObjectStorageError,
types::{
@@ -114,6 +114,13 @@ fn resolves_storage_config_from_config_json_shape() {
.unwrap();
assert_eq!(configured.get("blob").unwrap().bucket(), "custom-blobs");
assert_eq!(configured.get("copilot").unwrap().bucket(), "copilot");
assert!(
backends_from_flat_overrides([(
"storages.blob.storage",
serde_json::json!({"provider":"invalid","bucket":"blobs","config":{}})
)])
.is_err()
);
let storage = StorageProviderConfig {
provider: "cloudflare-r2".to_string(),
@@ -273,6 +280,29 @@ fn resolves_r2_proxy_upload_capability_from_config_json_shape() {
assert!(config.use_presigned_url);
assert!(config.proxy_upload);
let fields = [
serde_json::json!("ws"),
serde_json::json!("file"),
serde_json::json!("text/plain"),
serde_json::json!(7),
];
let token = config
.proxy_upload_token("/api/storage/upload", &fields, 1_700_000_000)
.unwrap()
.unwrap();
// TODO(0.27.5): Remove this old Node token-format comparison after 0.27.4
// upload URLs expire.
assert_eq!(token, "NXaxFbKQjOOnqo5ggDsuK1jf8w5LlM7tLRWRoj_EmtY");
assert!(
config
.verify_proxy_upload_token("/api/storage/upload", &fields, 1_700_000_000, &token)
.unwrap()
);
assert!(
!config
.verify_proxy_upload_token("/api/storage/upload", &fields, 1_700_000_001, &token)
.unwrap()
);
let request = config
.custom_presign_get_at(&ObjectKey::new("workspace/blob.m4a").unwrap(), 1_700_000_000)
.unwrap()
@@ -453,7 +453,10 @@ impl StorageRuntime {
#[cfg(test)]
mod tests {
use std::{collections::HashMap, sync::RwLock};
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use anyhow::{Context, Result as AnyResult};
use napi::bindgen_prelude::Buffer;
@@ -465,8 +468,9 @@ mod tests {
use super::*;
use crate::runtime::{
migrations::migrate_runtime_tables,
object_storage::{FsStorageConfig, StorageBackendConfig},
config::ServerConfig,
migrations::{EMBEDDING_TEST_LOCK, migrate_runtime_tables},
object_storage::{FsStorageConfig, ObjectStorageService, StorageBackendConfig},
storage_runtime::StorageRuntimeConfig,
};
@@ -512,10 +516,13 @@ mod tests {
.await
.map_err(|err| anyhow::anyhow!(err.to_string()))?;
let object_root = tempfile::tempdir()?;
let config_dir = tempfile::tempdir()?;
let config_path = config_dir.path().join("config.json");
std::fs::write(&config_path, r#"{"deployment":{"type":"cloud"}}"#)?;
let runtime = StorageRuntime {
config: RwLock::new(StorageRuntimeConfig {
database_url,
object_storage: crate::runtime::object_storage::ObjectStorageService {
object_storage: ObjectStorageService {
backends: HashMap::from([(
"blob".to_string(),
StorageBackendConfig::Fs(FsStorageConfig {
@@ -526,6 +533,7 @@ mod tests {
)]),
},
}),
server_config: Arc::new(ServerConfig::open(&config_path, None)?),
pool: Mutex::new(Some(pool.clone())),
};
let suffix = Uuid::new_v4().simple().to_string();
@@ -629,7 +637,7 @@ mod tests {
let Ok(database_url) = std::env::var("DATABASE_URL") else {
return;
};
let _guard = crate::runtime::migrations::EMBEDDING_TEST_LOCK.lock().await;
let _guard = EMBEDDING_TEST_LOCK.lock().await;
let pool = PgPool::connect(&database_url).await.unwrap();
let suffix = Uuid::new_v4().simple().to_string();
let workspace_id = format!("blob-cleanup-ws-{suffix}");
@@ -682,7 +690,7 @@ mod tests {
#[tokio::test]
#[ignore = "requires DATABASE_URL and a migrated PostgreSQL database"]
async fn blob_cleanup_projection_reprojects_reference_removal_before_deletion() -> AnyResult<()> {
let _guard = crate::runtime::migrations::EMBEDDING_TEST_LOCK.lock().await;
let _guard = EMBEDDING_TEST_LOCK.lock().await;
let fixture = blob_cleanup_fixture(true).await?;
let _object_root = &fixture.object_root;
fixture
@@ -758,7 +766,7 @@ mod tests {
#[tokio::test]
#[ignore = "requires DATABASE_URL and a migrated PostgreSQL database"]
async fn blob_cleanup_projection_outdated_pending_and_failed_projections_fail_closed() -> AnyResult<()> {
let _guard = crate::runtime::migrations::EMBEDDING_TEST_LOCK.lock().await;
let _guard = EMBEDDING_TEST_LOCK.lock().await;
for status in ["fresh", "pending", "failed"] {
let fixture = blob_cleanup_fixture(false).await?;
let _object_root = &fixture.object_root;
@@ -81,7 +81,7 @@ mod tests {
#[test]
fn capabilities_enable_presign_get_for_presigned_s3_provider() {
let capabilities = storage_provider_capabilities(&StorageBackendConfig::S3(ObjectStorageConfig {
let capabilities = storage_provider_capabilities(&StorageBackendConfig::S3(Box::new(ObjectStorageConfig {
provider: "cloudflare-r2".to_string(),
bucket: "blob".to_string(),
endpoint: Some("https://account.r2.cloudflarestorage.com".to_string()),
@@ -96,9 +96,11 @@ mod tests {
presign_sign_content_type_for_put: Some(true),
use_presigned_url: true,
proxy_upload: false,
upload_url_prefix: None,
proxy_sign_key: None,
custom_get_url_prefix: None,
custom_get_sign_key: None,
}));
})));
assert!(capabilities.presign_put);
assert!(capabilities.presign_get);
@@ -108,7 +110,7 @@ mod tests {
#[test]
fn capabilities_expose_r2_proxy_upload() {
let capabilities = storage_provider_capabilities(&StorageBackendConfig::S3(ObjectStorageConfig {
let capabilities = storage_provider_capabilities(&StorageBackendConfig::S3(Box::new(ObjectStorageConfig {
provider: "cloudflare-r2".to_string(),
bucket: "blob".to_string(),
endpoint: Some("https://account.r2.cloudflarestorage.com".to_string()),
@@ -123,9 +125,11 @@ mod tests {
presign_sign_content_type_for_put: Some(true),
use_presigned_url: true,
proxy_upload: true,
upload_url_prefix: None,
proxy_sign_key: Some("key".to_string()),
custom_get_url_prefix: None,
custom_get_sign_key: None,
}));
})));
assert!(capabilities.proxy_upload);
assert!(capabilities.presign_put);
@@ -1,14 +1,15 @@
use std::{env, fs};
use std::env;
use serde::Deserialize;
use sqlx::PgPool;
use super::{ObjectStorageService, RuntimeError, RuntimeResult};
use crate::runtime::config::ServerConfig;
#[derive(Clone, Debug)]
pub(super) struct StorageRuntimeConfig {
pub(super) database_url: String,
pub(super) object_storage: ObjectStorageService,
pub(in crate::runtime) struct StorageRuntimeConfig {
pub(in crate::runtime) database_url: String,
pub(in crate::runtime) object_storage: ObjectStorageService,
}
#[derive(Debug, Default, Deserialize)]
@@ -23,17 +24,19 @@ struct DbConfigFile {
}
impl StorageRuntimeConfig {
pub(super) fn from_config_files() -> RuntimeResult<Self> {
let app_config = storage_runtime_config_from_files()?;
pub(in crate::runtime) fn from_server_config(config: &ServerConfig) -> RuntimeResult<Self> {
let app_config: StorageRuntimeAppConfig = serde_json::from_value(config.baseline().clone())
.map_err(|err| RuntimeError::json("invalid storage runtime config", err))?;
let database_url = database_url_from_env()
.or(app_config.database_url())
.unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string());
Ok(Self {
database_url,
object_storage: ObjectStorageService::from_config_files()?,
object_storage: config.object_storage().clone(),
})
}
#[cfg(test)]
pub(super) fn from_config_json(config_json: &str) -> RuntimeResult<Self> {
let app_config: StorageRuntimeAppConfig =
serde_json::from_str(config_json).map_err(|err| RuntimeError::json("invalid storage runtime config", err))?;
@@ -61,12 +64,6 @@ impl StorageRuntimeAppConfig {
.and_then(|db| db.datasource_url.clone())
.and_then(non_empty_string)
}
fn merge(&mut self, config: Self) {
if config.db.is_some() {
self.db = config.db;
}
}
}
fn database_url_from_env() -> Option<String> {
@@ -76,16 +73,3 @@ fn database_url_from_env() -> Option<String> {
fn non_empty_string(value: String) -> Option<String> {
if value.trim().is_empty() { None } else { Some(value) }
}
fn storage_runtime_config_from_files() -> RuntimeResult<StorageRuntimeAppConfig> {
let mut merged = StorageRuntimeAppConfig::default();
for path in crate::runtime::config::config_json_paths() {
if !path.exists() {
continue;
}
let raw = fs::read_to_string(&path).map_err(|err| RuntimeError::io("failed to read config file", err))?;
let config = serde_json::from_str(&raw).map_err(|err| RuntimeError::json("failed to parse config file", err))?;
merged.merge(config);
}
Ok(merged)
}
@@ -273,7 +273,10 @@ impl StorageRuntime {
#[cfg(test)]
mod tests {
use std::{collections::HashMap, sync::RwLock};
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use anyhow::{Context, Result as AnyResult};
use napi::bindgen_prelude::Buffer;
@@ -285,7 +288,9 @@ mod tests {
use super::*;
use crate::runtime::{
backend_runtime::SEARCH_TEST_LOCK,
config::ServerConfig,
migrations::{migrate_runtime_tables, migrate_search_tables},
object_storage::{FsStorageConfig, ObjectStorageService, StorageBackendConfig},
storage_runtime::StorageRuntimeConfig,
};
@@ -301,13 +306,17 @@ mod tests {
migrate_runtime_tables(&pool)
.await
.map_err(|err| anyhow::anyhow!(err.to_string()))?;
let config_dir = tempfile::tempdir()?;
let config_path = config_dir.path().join("config.json");
std::fs::write(&config_path, r#"{"deployment":{"type":"cloud"}}"#)?;
let runtime = StorageRuntime {
config: RwLock::new(StorageRuntimeConfig {
database_url,
object_storage: crate::runtime::object_storage::ObjectStorageService {
object_storage: ObjectStorageService {
backends: HashMap::new(),
},
}),
server_config: Arc::new(ServerConfig::open(&config_path, None)?),
pool: Mutex::new(Some(pool.clone())),
};
Ok(Some((runtime, pool)))
@@ -768,7 +777,7 @@ mod tests {
let object_root = tempfile::tempdir()?;
runtime.config.write().unwrap().object_storage.backends.insert(
"blob".to_string(),
crate::runtime::object_storage::StorageBackendConfig::Fs(crate::runtime::object_storage::FsStorageConfig {
StorageBackendConfig::Fs(FsStorageConfig {
provider: "fs".to_string(),
root: object_root.path().to_string_lossy().to_string(),
bucket: "document-cleanup-test".to_string(),
@@ -9,15 +9,16 @@ mod document_cleanup_execution;
mod workspace_cleanup;
mod workspace_cleanup_checkpoint;
mod workspace_cleanup_namespace;
use std::sync::RwLock;
use std::sync::{Arc, RwLock};
pub use capabilities::StorageProviderCapabilities;
use capabilities::storage_provider_capabilities;
use config::StorageRuntimeConfig;
pub(in crate::runtime) use config::StorageRuntimeConfig;
pub(super) use current_doc::load_current_doc;
pub(in crate::runtime) use current_doc::{CurrentDoc, CurrentDocUpdate, merge_current_doc};
use current_doc::{load_canonical_doc, load_workspace_canonical_doc_ids, load_workspace_live_doc_ids};
use napi::bindgen_prelude::Buffer;
use serde_json::Value;
use sqlx::{PgPool, Row, postgres::PgPoolOptions};
use tokio::sync::Mutex;
@@ -39,6 +40,7 @@ use super::{
types::{ObjectDeleteOutcome, ObjectKey, ObjectLocator, ObjectPrefix, StorageScope},
},
};
use crate::runtime::config::{ServerConfig, ServerConfigHandle};
type Result<T> = RuntimeResult<T>;
@@ -67,18 +69,27 @@ pub struct StorageRuntimeHealth {
pub bucket: Option<String>,
}
#[napi_derive::napi(object)]
pub struct StorageUploadUrlConfig {
pub proxy_upload: bool,
pub url_prefix: Option<String>,
}
#[napi_derive::napi]
pub struct StorageRuntime {
config: RwLock<StorageRuntimeConfig>,
server_config: Arc<ServerConfig>,
pool: Mutex<Option<PgPool>>,
}
#[napi_derive::napi]
impl StorageRuntime {
#[napi(constructor)]
pub fn new() -> napi::Result<Self> {
pub fn new(server_config: &ServerConfigHandle) -> napi::Result<Self> {
let config = StorageRuntimeConfig::from_server_config(&server_config.inner).map_err(to_napi_error)?;
Ok(Self {
config: RwLock::new(StorageRuntimeConfig::from_config_files().map_err(to_napi_error)?),
config: RwLock::new(config),
server_config: Arc::clone(&server_config.inner),
pool: Mutex::new(None),
})
}
@@ -88,12 +99,6 @@ impl StorageRuntime {
self.start_inner().await.map_err(to_napi_error)
}
#[napi]
pub fn configure(&self, config_json: String) -> napi::Result<()> {
let config = StorageRuntimeConfig::from_config_json(&config_json).map_err(to_napi_error)?;
self.update_config(config).map_err(to_napi_error)
}
async fn start_inner(&self) -> RuntimeResult<()> {
let mut guard = self.pool.lock().await;
if guard.is_some() {
@@ -113,12 +118,21 @@ impl StorageRuntime {
.await
.map_err(|err| RuntimeError::database("StorageRuntime postgres health check failed", err))?;
let config = self.config()?.with_db_overrides(&pool).await?;
let baseline = StorageRuntimeConfig::from_server_config(&self.server_config)?;
let config = baseline.with_db_overrides(&pool).await?;
self.update_config(config)?;
*guard = Some(pool);
Ok(())
}
#[napi]
pub async fn reload_config(&self) -> napi::Result<()> {
let pool = self.pool().await.map_err(to_napi_error)?;
let baseline = StorageRuntimeConfig::from_server_config(&self.server_config).map_err(to_napi_error)?;
let config = baseline.with_db_overrides(&pool).await.map_err(to_napi_error)?;
self.update_config(config).map_err(to_napi_error)
}
#[napi]
pub async fn stop(&self) -> napi::Result<()> {
let pool = self.pool.lock().await.take();
@@ -169,6 +183,63 @@ impl StorageRuntime {
.map_err(to_napi_error)
}
#[napi]
pub fn is_local_storage(&self, scope: String) -> napi::Result<bool> {
self
.backend_for_scope(&scope)
.map(|backend| {
matches!(
backend,
StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_)
)
})
.map_err(to_napi_error)
}
#[napi]
pub fn upload_url_config(&self, scope: String) -> napi::Result<Option<StorageUploadUrlConfig>> {
match self.backend_for_scope(&scope).map_err(to_napi_error)? {
StorageBackendConfig::S3(config) if config.use_presigned_url => Ok(Some(StorageUploadUrlConfig {
proxy_upload: config.proxy_upload,
url_prefix: config.upload_url_prefix,
})),
_ => Ok(None),
}
}
#[napi]
pub fn sign_upload_token(
&self,
scope: String,
path: String,
fields: Vec<Value>,
expires_at: i64,
) -> napi::Result<Option<String>> {
match self.backend_for_scope(&scope).map_err(to_napi_error)? {
StorageBackendConfig::S3(config) => config
.proxy_upload_token(&path, &fields, expires_at)
.map_err(|err| to_napi_error(err.into())),
_ => Ok(None),
}
}
#[napi]
pub fn verify_upload_token(
&self,
scope: String,
path: String,
fields: Vec<Value>,
expires_at: i64,
token: String,
) -> napi::Result<bool> {
match self.backend_for_scope(&scope).map_err(to_napi_error)? {
StorageBackendConfig::S3(config) => config
.verify_proxy_upload_token(&path, &fields, expires_at, &token)
.map_err(|err| to_napi_error(err.into())),
_ => Ok(false),
}
}
#[napi]
pub async fn put_object(
&self,
@@ -531,18 +531,25 @@ fn record_failure(result: &mut RuntimeWorkspaceStorageReconcileResult, scope: &s
#[cfg(test)]
mod tests {
use std::{fs::write, sync::Arc};
use super::*;
use crate::runtime::object_storage::types::{ObjectKey, ObjectLocator, ObjectPutMetadata};
fn runtime(root: &std::path::Path) -> StorageRuntime {
let runtime = StorageRuntime::new().unwrap();
let root = serde_json::to_string(root).unwrap();
runtime
.configure(format!(
let config = super::super::StorageRuntimeConfig::from_config_json(&format!(
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"workspace-cleanup","config":{{"path":{root}}}}},"avatar.storage":{{"provider":"fs","bucket":"workspace-cleanup-avatar","config":{{"path":{root}}}}}}},"copilot":{{"storage":{{"provider":"fs","bucket":"workspace-cleanup-copilot","config":{{"path":{root}}}}}}}}}"#,
))
.unwrap();
runtime
.unwrap();
let config_dir = tempfile::tempdir().unwrap();
let config_path = config_dir.path().join("config.json");
write(&config_path, r#"{"deployment":{"type":"cloud"}}"#).unwrap();
StorageRuntime {
config: std::sync::RwLock::new(config),
server_config: Arc::new(crate::runtime::config::ServerConfig::open(&config_path, None).unwrap()),
pool: tokio::sync::Mutex::new(None),
}
}
async fn put(runtime: &StorageRuntime, key: String) {
+23 -10
View File
@@ -11,6 +11,13 @@ import {
} from '../src/base/config/register';
const IGNORED_MODULES = new Set(['db', 'redis', 'graphql']);
const SELFHOSTED_IGNORED_MODULES = new Set([
'payment',
'telemetry',
'captcha',
'metrics',
]);
const SELFHOSTED_IGNORED_KEYS = new Set(['auth.trustedCloudflareHeaders']);
function getDescriptors() {
return getAllDescriptors()
@@ -49,18 +56,23 @@ function generateJsonSchema(outputPath: string) {
properties: {},
};
getDescriptors().forEach(({ module, descriptors }) => {
schema.properties[module] = {
type: 'object',
description: `Configuration for ${module} module`,
properties: {},
};
getDescriptors()
.filter(({ module }) => !SELFHOSTED_IGNORED_MODULES.has(module))
.forEach(({ module, descriptors }) => {
schema.properties[module] = {
type: 'object',
description: `Configuration for ${module} module`,
properties: {},
};
descriptors.forEach(({ key, descriptor }) => {
schema.properties[module].properties[key] =
convertDescriptorToSchemaProperty(descriptor);
descriptors.forEach(({ key, descriptor }) => {
if (SELFHOSTED_IGNORED_KEYS.has(`${module}.${key}`)) {
return;
}
schema.properties[module].properties[key] =
convertDescriptorToSchemaProperty(descriptor);
});
});
});
fs.writeFileSync(outputPath, JSON.stringify(schema, null, 2));
@@ -76,6 +88,7 @@ function generateAdminConfigJson(outputPath: string) {
let type: string;
switch (descriptor.schema?.type) {
case 'number':
case 'integer':
type = 'Number';
break;
case 'boolean':
@@ -4,6 +4,8 @@ import fs from 'node:fs';
import { homedir } from 'node:os';
import path from 'node:path';
import { upgradeConfig } from './upgrade-config.js';
const SELF_HOST_CONFIG_DIR = `${homedir()}/.affine/config`;
function generatePrivateKey() {
@@ -29,6 +31,8 @@ const files = [{ to: 'private.key', generator: generatePrivateKey }];
function prepare() {
fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
upgradeConfig();
for (const { to, generator } of files) {
const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, to);
if (!fs.existsSync(targetFilePath)) {
@@ -0,0 +1,111 @@
import fs from 'node:fs';
import { homedir } from 'node:os';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { config as loadEnv } from 'dotenv';
const aliases = [
['mailer', 'SMTP.host', 'MAILER_HOST'],
['mailer', 'SMTP.ignoreTLS', 'MAILER_IGNORE_TLS', 'boolean'],
['mailer', 'SMTP.name', 'MAILER_SERVERNAME'],
['mailer', 'SMTP.password', 'MAILER_PASSWORD'],
['mailer', 'SMTP.port', 'MAILER_PORT', 'integer'],
['mailer', 'SMTP.sender', 'MAILER_SENDER'],
['mailer', 'SMTP.username', 'MAILER_USER'],
['redis', 'db', 'REDIS_SERVER_DATABASE', 'integer'],
['redis', 'host', 'REDIS_SERVER_HOST'],
['redis', 'password', 'REDIS_SERVER_PASSWORD'],
['redis', 'port', 'REDIS_SERVER_PORT', 'integer'],
['redis', 'username', 'REDIS_SERVER_USERNAME'],
['server', 'externalUrl', 'AFFINE_SERVER_EXTERNAL_URL'],
['server', 'host', 'AFFINE_SERVER_HOST'],
['server', 'https', 'AFFINE_SERVER_HTTPS', 'boolean'],
['server', 'listenAddr', 'LISTEN_ADDR'],
['server', 'path', 'AFFINE_SERVER_SUB_PATH'],
['server', 'port', 'AFFINE_SERVER_PORT', 'integer'],
['telemetry', 'ga4.apiSecret', 'GA4_API_SECRET'],
['telemetry', 'ga4.measurementId', 'GA4_MEASUREMENT_ID'],
];
// TODO(0.27.5): Remove this upgrade after old self-host config files and mounts are retired.
export function upgradeConfig(
appRoot = process.cwd(),
homeDir = homedir(),
deploymentType
) {
loadEnv({ path: path.join(appRoot, '.env'), quiet: true });
loadEnv({ path: path.join(homeDir, '.affine/config/.env'), quiet: true });
deploymentType ??= process.env.DEPLOYMENT_TYPE;
const appPath = path.join(appRoot, 'config.json');
const homePath = path.join(homeDir, '.affine/config/config.json');
const configPath = fs.existsSync(appPath)
? appPath
: fs.existsSync(homePath) || fs.existsSync(path.dirname(homePath))
? homePath
: appPath;
const config = fs.existsSync(configPath)
? JSON.parse(fs.readFileSync(configPath, 'utf-8'))
: {};
let changed = false;
if (!config.deployment?.type) {
if (deploymentType && !['affine', 'selfhosted'].includes(deploymentType)) {
throw new Error(`Invalid DEPLOYMENT_TYPE: ${deploymentType}`);
}
config.deployment = {
...config.deployment,
type: deploymentType === 'affine' ? 'cloud' : 'selfhosted',
};
changed = true;
}
for (const [module, key, alias, type] of aliases) {
const value = process.env[alias];
if (!value) continue;
const current = config[module] ?? {};
const parts = key.split('.');
const nested = parts
.slice(0, -1)
.reduce((value, part) => value?.[part], current);
if (
Object.hasOwn(current, key) ||
Object.hasOwn(nested ?? {}, parts.at(-1))
) {
continue;
}
const parsed =
type === 'boolean'
? value === '1' || value.toLowerCase() === 'true'
: type === 'integer'
? Number.parseInt(value)
: value;
if (type === 'integer' && Number.isNaN(parsed)) continue;
config[module] = { ...current, [key]: parsed };
changed = true;
}
if (!changed) return;
fs.mkdirSync(path.dirname(configPath), { recursive: true });
const tempPath = `${configPath}.${process.pid}.tmp`;
try {
fs.writeFileSync(tempPath, JSON.stringify(config, null, 2) + '\n', {
mode: 0o600,
});
fs.renameSync(tempPath, configPath);
} catch (error) {
if (
configPath === appPath &&
['EACCES', 'EROFS', 'EBUSY'].includes(error.code)
) {
return;
}
throw error;
} finally {
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
}
}
if (
process.argv[1] &&
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
) {
upgradeConfig();
}
@@ -839,14 +839,17 @@ test('should return methods unavailable for disabled users', async t => {
});
test('should return magic link unavailable for unknown users when signup is disabled', async t => {
const { app, config } = t.context;
const { app, runtime } = t.context;
config.override({
auth: {
allowSignup: false,
await runtime.saveAppConfig(null, [
{
key: 'auth.allowSignup',
owner: 'native',
operation: 'set',
valueJson: 'false',
},
});
await t.context.runtime.onConfigChanged({
]);
await runtime.onConfigChanged({
updates: { auth: { allowSignup: false } },
});
@@ -860,14 +863,17 @@ test('should return magic link unavailable for unknown users when signup is disa
});
test('should return magic link unavailable when domain verification rejects signup email', async t => {
const { app, config } = t.context;
const { app, runtime } = t.context;
config.override({
auth: {
requireEmailDomainVerification: true,
await runtime.saveAppConfig(null, [
{
key: 'auth.requireEmailDomainVerification',
owner: 'native',
operation: 'set',
valueJson: 'true',
},
});
await t.context.runtime.onConfigChanged({
]);
await runtime.onConfigChanged({
updates: { auth: { requireEmailDomainVerification: true } },
});
@@ -13,6 +13,7 @@ import { createTestingModule } from '../utils';
let m: TestingModule;
let db: PrismaClient;
const runtime = {
nodeCryptoPrivateKey: Sinon.stub().returns(''),
cleanupExpiredUserSessions: Sinon.stub(),
executeAuthSessionCommandV1: Sinon.stub(),
};
@@ -4,7 +4,7 @@ import ava from 'ava';
import Sinon from 'sinon';
import { z } from 'zod';
import { Config, CopilotQuotaExceeded } from '../../base';
import { CopilotQuotaExceeded } from '../../base';
import type { BackendRuntimeProvider } from '../../core/backend-runtime';
import { StorageRuntimeProvider } from '../../core/storage-runtime';
import type { Models } from '../../models';
@@ -33,6 +33,7 @@ function runtimeFixture(streamError?: string, enabled = true) {
targetOverride?: { profileId: string; modelId: string };
}> = [];
const backend = {
copilotEnabled: () => enabled,
executeCopilot: async (input: {
slot: string;
request: unknown;
@@ -99,7 +100,6 @@ function runtimeFixture(streamError?: string, enabled = true) {
const consumer = {
consume: async () => {},
} as unknown as CopilotRuntimeEventConsumer;
const config = { copilot: { enabled } } as Config;
const storageRuntime = Sinon.createStubInstance(StorageRuntimeProvider);
storageRuntime.getObject.callsFake(async () => ({
body: Readable.from(Buffer.from('image')),
@@ -126,7 +126,6 @@ function runtimeFixture(streamError?: string, enabled = true) {
conversation,
tools,
consumer,
config,
new AttachmentAdmissionHost(materializer, storage)
),
};
@@ -14,10 +14,8 @@ import { McpAccessMode, PrismaClient } from '@prisma/client';
import type { TestFn } from 'ava';
import ava from 'ava';
import { Config } from '../../base';
import { ServerFeature, ServerService } from '../../core';
import { Models } from '../../models';
import { CopilotFeatureService } from '../../plugins/copilot/feature';
import { McpCredentialService } from '../../plugins/copilot/mcp/credential';
import { WorkspaceMcpProvider } from '../../plugins/copilot/mcp/provider';
import { installMockCopilotRuntime } from '../mocks';
@@ -58,14 +56,13 @@ test.after.always(async t => {
test('disabled copilot hides its server feature and rejects every API transport', async t => {
const { app } = t.context;
const config = app.get(Config);
const feature = app.get(CopilotFeatureService);
const server = app.get(ServerService);
await app.signupV1();
const workspace = await createWorkspace(app);
config.copilot.enabled = false;
feature.onConfigChanged({ updates: { copilot: { enabled: false } } });
await server.updateConfig(null, [
{ module: 'copilot', key: 'enabled', value: false },
]);
try {
t.false(server.features.includes(ServerFeature.Copilot));
await t.throwsAsync(
@@ -117,8 +114,9 @@ test('disabled copilot hides its server feature and rejects every API transport'
})
);
} finally {
config.copilot.enabled = true;
feature.onConfigChanged({ updates: { copilot: { enabled: true } } });
await server.updateConfig(null, [
{ module: 'copilot', key: 'enabled', clear: true },
]);
}
});
@@ -15,6 +15,7 @@ import {
SearchProviderUnavailable,
} from '../../base';
import { ServerFeature, type ServerService } from '../../core';
import type { BackendRuntimeProvider } from '../../core/backend-runtime';
import type { DocReader } from '../../core/doc';
import type { PermissionAccess } from '../../core/permission';
import { type RealtimePublisher, RealtimeRegistry } from '../../core/realtime';
@@ -958,27 +959,30 @@ test('document tools enforce the user-selected hard scope', async t => {
);
});
test('copilot config controls the server feature and request admission', t => {
const config = { copilot: { enabled: false } } as Config;
test('copilot runtime controls the server feature and request admission', t => {
let enabled = false;
const runtime = {
copilotEnabled: () => enabled,
} as unknown as BackendRuntimeProvider;
const features = new Set<ServerFeature>();
const server = {
enableFeature: (feature: ServerFeature) => features.add(feature),
disableFeature: (feature: ServerFeature) => features.delete(feature),
} as unknown as ServerService;
const feature = new CopilotFeatureService(config, server);
const feature = new CopilotFeatureService(runtime, server);
const guard = new CopilotFeatureGuard(feature);
feature.onConfigInit();
t.false(features.has(ServerFeature.Copilot));
t.throws(() => guard.canActivate(), { message: 'Copilot is disabled.' });
config.copilot.enabled = true;
feature.onConfigChanged({ updates: { copilot: { enabled: true } } });
enabled = true;
feature.onConfigApplied({ updates: { copilot: { enabled: true } } });
t.true(features.has(ServerFeature.Copilot));
t.true(guard.canActivate());
config.copilot.enabled = false;
feature.onConfigChanged({ updates: { copilot: { enabled: false } } });
enabled = false;
feature.onConfigApplied({ updates: { copilot: { enabled: false } } });
t.false(features.has(ServerFeature.Copilot));
});
@@ -7,15 +7,18 @@ import {
import { PrismaClient } from '@prisma/client';
import { FunctionalityModules } from '../app.module';
import { AFFiNELogger, ConfigFactory, EventBus } from '../base';
import {
BACKEND_RUNTIME_CONFIG_PATHS,
BackendRuntimeProvider,
} from '../core/backend-runtime';
import { AFFiNELogger, EventBus } from '../base';
import { OVERRIDE_CONFIG_TOKEN } from '../base/config/factory';
import { getDefaultConfig } from '../base/config/register';
import { BackendRuntimeProvider } from '../core/backend-runtime';
import { StorageRuntimeProvider } from '../core/storage-runtime';
import { ServerConfigHandle } from '../native';
import { createFactory, MockEventBus } from './mocks';
import { TEST_LOG_LEVEL } from './utils';
import { createTestRuntimeConfig } from './utils/runtime-config';
import {
applyTestConfigOverrides,
createTestRuntimeConfig,
} from './utils/runtime-config';
interface TestingModuleMetadata extends ModuleMetadata {
tapModule?(m: TestingModuleBuilder): void;
@@ -30,7 +33,7 @@ export interface TestingModule extends NestjsTestingModule {
export async function createModule(
metadata: TestingModuleMetadata = {}
): Promise<TestingModule> {
const config = new ConfigFactory().config;
const config = getDefaultConfig();
const runtimeConfig = await createTestRuntimeConfig(
config.db.datasourceUrl,
config.indexer
@@ -43,11 +46,14 @@ export async function createModule(
imports: [...functionalityModules, ...(meta.imports ?? [])],
});
builder
.overrideProvider(EventBus)
.useValue(new MockEventBus())
.overrideProvider(BACKEND_RUNTIME_CONFIG_PATHS)
.useValue([runtimeConfig.configPath]);
builder.overrideProvider(EventBus).useValue(new MockEventBus());
builder.overrideProvider(ServerConfigHandle).useFactory({
factory: (overrides?: DeepPartial<AppConfig>) => {
applyTestConfigOverrides(runtimeConfig.configPath, overrides);
return new ServerConfigHandle(runtimeConfig.configPath);
},
inject: [{ token: OVERRIDE_CONFIG_TOKEN, optional: true }],
});
// when custom override happens
if (tapModule) {
@@ -13,6 +13,7 @@ interface Context {
db: PrismaClient;
cronJob: DocStorageCronJob;
runtime: {
nodeCryptoPrivateKey: Sinon.SinonStub;
cleanupExpiredSnapshotHistories: Sinon.SinonStub;
executeAuthSessionCommandV1: Sinon.SinonStub;
};
@@ -23,6 +24,7 @@ const test = ava as TestFn<Context>;
// cleanup database before each test
test.before(async t => {
t.context.runtime = {
nodeCryptoPrivateKey: Sinon.stub().returns(''),
cleanupExpiredSnapshotHistories: Sinon.stub(),
executeAuthSessionCommandV1: Sinon.stub().resolves({}),
};
@@ -1,10 +1,10 @@
import { serverConfigQuery, ServerFeature } from '@affine/graphql';
import { Config } from '../../../base';
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
import { app, e2e } from '../test';
e2e('should expose the indexer feature when it is enabled', async t => {
const enabled = app.get(Config).indexer.enabled;
const enabled = app.get(BackendRuntimeProvider).searchEnabled();
const { serverConfig } = await app.gql({ query: serverConfigQuery });
t.is(
@@ -22,17 +22,17 @@ import {
GlobalExceptionFilter,
OneMB,
} from '../../base';
import { OVERRIDE_CONFIG_TOKEN } from '../../base/config/factory';
import { getDefaultConfig } from '../../base/config/register';
import { ThrottlerStorage } from '../../base/throttler';
import { SocketIoAdapter } from '../../base/websocket';
import { AuthGuard, AuthService } from '../../core/auth';
import {
BACKEND_RUNTIME_CONFIG_PATHS,
BackendRuntimeProvider,
} from '../../core/backend-runtime';
import { BackendRuntimeProvider } from '../../core/backend-runtime';
import { Mailer } from '../../core/mail';
import { StorageRuntimeProvider } from '../../core/storage-runtime';
import { ServerRole } from '../../env';
import { Models } from '../../models';
import { ServerConfigHandle } from '../../native';
import { IndexerService } from '../../plugins/indexer/service';
import {
createFactory,
@@ -42,7 +42,10 @@ import {
MockUserInput,
} from '../mocks';
import { parseCookies, TEST_LOG_LEVEL } from '../utils';
import { createTestRuntimeConfig } from '../utils/runtime-config';
import {
applyTestConfigOverrides,
createTestRuntimeConfig,
} from '../utils/runtime-config';
interface TestingAppMetadata {
tapModule?(m: TestingModuleBuilder): void;
@@ -252,7 +255,7 @@ export class TestingApp extends NestApplication {
export async function createApp(
metadata: TestingAppMetadata = {}
): Promise<TestingApp> {
const config = new ConfigFactory().config;
const config = getDefaultConfig();
const runtimeConfig = await createTestRuntimeConfig(
config.db.datasourceUrl,
config.indexer
@@ -265,9 +268,13 @@ export async function createApp(
});
builder.overrideProvider(Mailer).useValue(new MockMailer());
builder
.overrideProvider(BACKEND_RUNTIME_CONFIG_PATHS)
.useValue([runtimeConfig.configPath]);
builder.overrideProvider(ServerConfigHandle).useFactory({
factory: (overrides?: DeepPartial<AppConfig>) => {
applyTestConfigOverrides(runtimeConfig.configPath, overrides);
return new ServerConfigHandle(runtimeConfig.configPath);
},
inject: [{ token: OVERRIDE_CONFIG_TOKEN, optional: true }],
});
// when custom override happens
if (tapModule) {
@@ -4,7 +4,7 @@ import {
SearchTable,
} from '@affine/graphql';
import { Config } from '../../../base';
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
import { createDocWithMarkdown } from '../../../native';
import { Mockers } from '../../mocks';
import {
@@ -14,7 +14,9 @@ import {
reconcileSearchProjection,
} from '../test';
const indexerE2e = app.get(Config).indexer.enabled ? e2e.serial : e2e.skip;
const indexerE2e = app.get(BackendRuntimeProvider).searchEnabled()
? e2e.serial
: e2e.skip;
indexerE2e('should aggregate by docId', async t => {
const owner = await app.signup();
@@ -5,7 +5,7 @@ import {
SearchTable,
} from '@affine/graphql';
import { Config } from '../../../base';
import { ServerService } from '../../../core/config';
import { DocRole } from '../../../models';
import { createDocWithMarkdown } from '../../../native';
import { Mockers } from '../../mocks';
@@ -16,9 +16,10 @@ import {
reconcileSearchProjection,
} from '../test';
const indexer = (await app.get(ServerService).getEffectiveAdminConfig())
.indexer;
const manticoreSearchEnabled =
app.get(Config).indexer.enabled &&
app.get(Config).indexer.provider.type === 'manticoresearch';
indexer?.enabled && indexer.provider?.type === 'manticoresearch';
const manticoreSearchE2e = manticoreSearchEnabled ? e2e.serial : e2e.skip;
async function indexDocument(
@@ -4,7 +4,7 @@ import {
SearchTable,
} from '@affine/graphql';
import { Config } from '../../../base';
import { ServerService } from '../../../core/config';
import { createDocWithMarkdown } from '../../../native';
import { Mockers } from '../../mocks';
import {
@@ -14,9 +14,10 @@ import {
reconcileSearchProjection,
} from '../test';
const indexer = app.get(Config).indexer;
const indexer = (await app.get(ServerService).getEffectiveAdminConfig())
.indexer;
const remoteE2e =
indexer.enabled && indexer.provider.type === 'elasticsearch'
indexer?.enabled && indexer.provider?.type === 'elasticsearch'
? e2e.serial
: e2e.skip;
@@ -1,6 +1,6 @@
import { indexerSearchDocsQuery } from '@affine/graphql';
import { Config } from '../../../base';
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
import { createDocWithMarkdown } from '../../../native';
import { Mockers } from '../../mocks';
import {
@@ -10,7 +10,9 @@ import {
reconcileSearchProjection,
} from '../test';
const indexerE2e = app.get(Config).indexer.enabled ? e2e.serial : e2e.skip;
const indexerE2e = app.get(BackendRuntimeProvider).searchEnabled()
? e2e.serial
: e2e.skip;
indexerE2e('should search docs by keyword', async t => {
const owner = await app.signup();
@@ -4,7 +4,7 @@ import {
SearchTable,
} from '@affine/graphql';
import { Config } from '../../../base';
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
import { DocRole } from '../../../models';
import { createDocWithMarkdown } from '../../../native';
import { Mockers } from '../../mocks';
@@ -15,7 +15,9 @@ import {
reconcileSearchProjection,
} from '../test';
const indexerE2e = app.get(Config).indexer.enabled ? e2e.serial : e2e.skip;
const indexerE2e = app.get(BackendRuntimeProvider).searchEnabled()
? e2e.serial
: e2e.skip;
async function indexDoc(
workspaceId: string,
@@ -1,9 +1,7 @@
import { createHash } from 'node:crypto';
import { createHash, createHmac } from 'node:crypto';
import { mock } from 'node:test';
import {
Config,
ConfigFactory,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
type R2StorageConfig,
@@ -37,8 +35,54 @@ class MockStorageRuntime {
contentLength?: number;
}[] = [];
uploadUrlConfig() {
const storage = currentBlobStorage;
if (storage.provider !== 'cloudflare-r2' && storage.provider !== 'aws-s3') {
return null;
}
const value = (storage.config as R2StorageConfig).usePresignedURL;
if (!value?.enabled) return null;
return {
proxyUpload: !!value.signKey,
urlPrefix: value.urlPrefix,
};
}
signUploadToken(
_scope: string,
path: string,
fields: (string | number)[],
expiresAt: number
) {
const storage = currentBlobStorage;
const key = (storage.config as R2StorageConfig).usePresignedURL?.signKey;
if (!key) return null;
return createHmac('sha256', key)
.update(
JSON.stringify([
'affine-storage-upload',
1,
'PUT',
path,
...fields,
expiresAt,
])
)
.digest('base64url');
}
verifyUploadToken(
scope: string,
path: string,
fields: (string | number)[],
expiresAt: number,
token: string
) {
return this.signUploadToken(scope, path, fields, expiresAt) === token;
}
async providerCapabilities() {
const storage = app.get(Config).storages.blob.storage;
const storage = currentBlobStorage;
if (storage.provider !== 'cloudflare-r2') {
return {
put: true,
@@ -187,17 +231,16 @@ const baseR2Storage: StorageProviderConfig = {
},
};
let defaultBlobStorage: StorageProviderConfig;
let currentBlobStorage: StorageProviderConfig;
let runtime: MockStorageRuntime;
e2e.before(() => {
defaultBlobStorage = structuredClone(app.get(Config).storages.blob.storage);
});
e2e.beforeEach(async () => {
runtime = new MockStorageRuntime();
const rt = app.get(StorageRuntimeProvider);
for (const method of [
'uploadUrlConfig',
'signUploadToken',
'verifyUploadToken',
'providerCapabilities',
'presignPut',
'createMultipartUpload',
@@ -215,13 +258,11 @@ e2e.beforeEach(async () => {
});
e2e.afterEach.always(async () => {
await setBlobStorage(defaultBlobStorage);
mock.reset();
});
async function setBlobStorage(storage: StorageProviderConfig) {
const configFactory = app.get(ConfigFactory);
configFactory.override({ storages: { blob: { storage } } });
currentBlobStorage = storage;
}
async function useR2Storage(
@@ -1,6 +1,10 @@
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import test from 'ava';
import { Env } from '../env';
import { DeploymentType, Env } from '../env';
const envs = { ...process.env };
test.beforeEach(() => {
@@ -64,6 +68,24 @@ test('should read DEPLOYMENT_TYPE', t => {
process.env.DEPLOYMENT_TYPE = 'unknown';
new Env();
});
const directory = mkdtempSync(join(tmpdir(), 'affine-env-config-'));
try {
const configPath = join(directory, 'config.json');
writeFileSync(
configPath,
JSON.stringify({ deployment: { type: 'selfhosted' } })
);
process.env.DEPLOYMENT_TYPE = 'affine';
const selected = new Env(configPath);
t.is(selected.serverConfigHandle.deploymentType, 'selfhosted');
t.is(selected.DEPLOYMENT_TYPE, DeploymentType.Selfhosted);
t.true(selected.selfhosted);
process.env.DEPLOYMENT_TYPE = 'unknown';
t.is(new Env(configPath).DEPLOYMENT_TYPE, DeploymentType.Selfhosted);
} finally {
rmSync(directory, { recursive: true, force: true });
}
});
test('should read FLAVOR', t => {
@@ -1,13 +1,19 @@
import { resolve } from 'node:path';
import test from 'ava';
import { BackendRuntime, resolveEntitlementV1 } from '../native';
import {
BackendRuntime,
resolveEntitlementV1,
ServerConfigHandle,
} from '../native';
test('native entitlement boundary maps invalid input to InvalidArg', async t => {
const runtime = new BackendRuntime(
'test-private-key',
[],
undefined,
undefined
new ServerConfigHandle(
resolve(env.projectRoot, '../../../.docker/selfhost/config.json.example')
),
'test-private-key'
);
t.teardown(() => runtime.stop());
const schemaError = t.throws(() =>
@@ -117,6 +117,22 @@ test('oauth endpoints retain validation and callback error shapes', async t => {
test('configured oauth providers remain visible through the reader', async t => {
const { app } = t.context;
t.deepEqual(app.get(OAuthService).providers.sort(), ['github', 'google']);
t.true(app.get(ServerService).features.includes(ServerFeature.OAuth));
const oauth = app.get(OAuthService);
const server = app.get(ServerService);
t.deepEqual(oauth.providers.sort(), ['github', 'google']);
t.true(server.features.includes(ServerFeature.OAuth));
await server.updateConfig(null, [
{
module: 'oauth',
key: 'providers.github',
value: { clientId: '', clientSecret: '' },
},
]);
t.deepEqual(oauth.providers, ['google']);
await server.updateConfig(null, [
{ module: 'oauth', key: 'providers.github', clear: true },
]);
t.deepEqual(oauth.providers.sort(), ['github', 'google']);
});
@@ -1,7 +1,6 @@
import ava from 'ava';
import Sinon from 'sinon';
import type { Config } from '../../base';
import { BackendRuntimeProvider } from '../../core/backend-runtime';
import { WorkspaceService } from '../../core/workspaces';
import type { Models } from '../../models';
@@ -19,16 +18,14 @@ ava('payment HTTP and event adapters only forward protocol data', async t => {
const execute = runtime.executePaymentCommandV1 as Sinon.SinonStub;
const capture = runtime.capturePaymentWebhookV1 as Sinon.SinonStub;
const workspace = Sinon.createStubInstance(WorkspaceService);
const config = {
payment: { enabled: true, stripe: { apiKey: 'sk_test_protocol' } },
} as Config;
runtime.stripeEnabled.returns(true);
const models = {
workspaceUser: {
chargedCount: Sinon.stub().resolves(3),
getOwner: Sinon.stub().resolves({ id: 'owner-1' }),
},
} as unknown as Models;
const events = new PaymentEventHandlers(workspace, runtime, config, models);
const events = new PaymentEventHandlers(workspace, runtime, models);
execute.resolves({ status: 'pending' });
await events.prepareSubscriptionCancellation({ id: 'user-1' });
@@ -44,6 +41,9 @@ ava('payment HTTP and event adapters only forward protocol data', async t => {
plan: 'team',
quantity: 3,
});
runtime.stripeEnabled.returns(false);
await events.prepareSubscriptionCancellation({ id: 'user-2' });
t.is(execute.callCount, 2);
workspace.isTeamWorkspace.resolves(false);
workspace.sendTeamWorkspaceUpgradedEmail.resolves();
@@ -30,14 +30,16 @@ ava(
const url = Sinon.createStubInstance(URLHelper);
url.safeLink.callsFake(path => new URL(path, 'https://app.example').href);
const config = {
payment: { enabled: false, showLifetimePrice: false },
payment: { showLifetimePrice: false },
} as Config;
const service = new SubscriptionService(runtime, config, server, url);
const paymentEnabled = runtime.paymentEnabled as Sinon.SinonStub;
paymentEnabled.returns(false);
service.onConfigInit();
t.true(server.disableFeature.calledWith(ServerFeature.Payment));
config.payment.enabled = true;
service.onConfigChanged({ updates: { payment: { enabled: true } } });
paymentEnabled.returns(true);
service.onConfigApplied({ updates: { payment: { enabled: true } } });
t.true(server.enableFeature.calledWith(ServerFeature.Payment));
command.resolves([
@@ -1,8 +1,13 @@
import { generateKeyPairSync } from 'node:crypto';
import { readFileSync, writeFileSync } from 'node:fs';
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { set } from 'lodash-es';
import { getDefaultConfig, override } from '../../base/config/register';
const { privateKey } = generateKeyPairSync('ec', { namedCurve: 'P-256' });
const testPrivateKey = privateKey
.export({ format: 'pem', type: 'pkcs8' })
@@ -23,6 +28,7 @@ export async function createTestRuntimeConfig(
await writeFile(
configPath,
JSON.stringify({
deployment: { type: globalThis.env.selfhosted ? 'selfhosted' : 'cloud' },
crypto: { privateKey: testPrivateKey },
db: { datasourceUrl: databaseUrl },
storages: {
@@ -37,7 +43,8 @@ export async function createTestRuntimeConfig(
enabled: indexer.enabled,
provider: indexer.provider,
},
})
}),
{ mode: 0o600 }
);
return {
configPath,
@@ -45,3 +52,31 @@ export async function createTestRuntimeConfig(
cleanup: () => rm(directory, { recursive: true, force: true }),
};
}
export function applyTestConfigOverrides(
configPath: string,
overrides: DeepPartial<AppConfig> = {}
) {
const config = getDefaultConfig();
const fixture = JSON.parse(readFileSync(configPath, 'utf-8'));
for (const [module, values] of Object.entries(fixture)) {
if (module === 'deployment') continue;
for (const [key, value] of Object.entries(
values as Record<string, unknown>
)) {
set(config, `${module}.${key}`, value);
}
}
override(config, overrides);
writeFileSync(
configPath,
JSON.stringify({
...config,
deployment: fixture.deployment,
storages: {
'avatar.storage': config.storages.avatar.storage,
'blob.storage': config.storages.blob.storage,
},
})
);
}
@@ -10,18 +10,23 @@ import { PrismaClient } from '@prisma/client';
import { buildAppModule, FunctionalityModules } from '../../app.module';
import { AFFiNELogger, ConfigFactory } from '../../base';
import { OVERRIDE_CONFIG_TOKEN } from '../../base/config/factory';
import { getDefaultConfig } from '../../base/config/register';
import { GqlModule } from '../../base/graphql';
import { ServerConfigModule } from '../../core';
import { AuthGuard, AuthModule } from '../../core/auth';
import { BACKEND_RUNTIME_CONFIG_PATHS } from '../../core/backend-runtime';
import { Mailer, MailModule } from '../../core/mail';
import { ModelsModule } from '../../models';
import { ServerConfigHandle } from '../../native';
// for jsdoc inference
// oxlint-disable-next-line no-unused-vars
import type { createModule } from '../create-module';
import { createFactory } from '../mocks';
import { MockMailer } from '../mocks/mailer.mock';
import { createTestRuntimeConfig } from './runtime-config';
import {
applyTestConfigOverrides,
createTestRuntimeConfig,
} from './runtime-config';
import { initTestingDB, TEST_LOG_LEVEL } from './utils';
interface TestingModuleMetadata extends ModuleMetadata {
@@ -68,7 +73,7 @@ export async function createTestingModule(
moduleDef: TestingModuleMetadata = {},
autoInitialize = true
): Promise<TestingModule> {
const config = new ConfigFactory().config;
const config = getDefaultConfig();
const runtimeConfig = await createTestRuntimeConfig(
config.db.datasourceUrl,
config.indexer
@@ -103,9 +108,13 @@ export async function createTestingModule(
});
builder.overrideProvider(Mailer).useClass(MockMailer);
builder
.overrideProvider(BACKEND_RUNTIME_CONFIG_PATHS)
.useValue([runtimeConfig.configPath]);
builder.overrideProvider(ServerConfigHandle).useFactory({
factory: (overrides?: DeepPartial<AppConfig>) => {
applyTestConfigOverrides(runtimeConfig.configPath, overrides);
return new ServerConfigHandle(runtimeConfig.configPath);
},
inject: [{ token: OVERRIDE_CONFIG_TOKEN, optional: true }],
});
if (moduleDef.tapModule) {
moduleDef.tapModule(builder);
}
@@ -2,9 +2,13 @@ import { Controller, Get } from '@nestjs/common';
import { SkipThrottle } from './base';
import { Public } from './core/auth';
import { DeploymentType } from './env';
import { ServerConfigHandle } from './native';
@Controller('/info')
export class AppController {
constructor(private readonly serverConfig: ServerConfigHandle) {}
@SkipThrottle()
@Public()
@Get()
@@ -12,7 +16,10 @@ export class AppController {
return {
compatibility: env.version,
message: `AFFiNE ${env.version} Server`,
type: env.DEPLOYMENT_TYPE,
type:
this.serverConfig.deploymentType === 'selfhosted'
? DeploymentType.Selfhosted
: DeploymentType.Affine,
flavor: env.FLAVOR,
};
}
@@ -1,28 +1,184 @@
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import test from 'ava';
import { get, has } from 'lodash-es';
import { createModule } from '../../../__tests__/create-module';
import {
applyTestConfigOverrides,
createTestRuntimeConfig,
} from '../../../__tests__/utils/runtime-config';
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
import { ServerConfigHandle } from '../../../native';
import { InvalidAppConfig } from '../../error';
import { CryptoHelper } from '../../helpers';
import { CacheRedis, SessionRedis, SocketIoRedis } from '../../redis/instances';
import { Config } from '../config';
import { ConfigFactory, ConfigModule } from '../index';
import { override } from '../register';
import { getDefaultConfig, override } from '../register';
const module = await createModule();
test.after.always(async () => {
await module.close();
});
test('should create config', t => {
test('test runtime config applies explicit overrides after fixture values', async t => {
const fixture = await createTestRuntimeConfig(
'postgresql://test:test@localhost/test',
getDefaultConfig().indexer
);
try {
applyTestConfigOverrides(fixture.configPath, {
copilot: { enabled: false },
storages: {
blob: {
storage: {
provider: 'fs',
bucket: 'blobs',
config: { path: fixture.storagePath },
},
},
},
});
const config = JSON.parse(readFileSync(fixture.configPath, 'utf-8'));
t.false(config.copilot.enabled);
t.is(config.storages['blob.storage'].provider, 'fs');
t.is(config.storages['avatar.storage'].provider, 'assetpack');
} finally {
await fixture.cleanup();
}
});
test('should create config', async t => {
const config = module.get(Config);
const crypto = module.get(CryptoHelper);
const runtime = module.get(BackendRuntimeProvider);
crypto.onConfigInit();
t.deepEqual(
crypto.keyPair.sha256.privateKey,
crypto.sha256(runtime.nodeCryptoPrivateKey())
);
t.is(typeof config.auth.passwordRequirements.max, 'number');
t.deepEqual(config.copilot.byok.allowedProviders, [
'openai',
t.deepEqual((await runtime.getByokPolicy()).allowedProviders, [
'anthropic',
'gemini',
'fal',
'gemini',
'openai',
]);
});
test('should read static config from the shared handle', t => {
const directory = mkdtempSync(join(tmpdir(), 'affine-config-handle-'));
try {
const configPath = join(directory, 'config.json');
writeFileSync(
configPath,
JSON.stringify({
deployment: { type: 'selfhosted' },
server: { name: 'shared-config' },
copilot: { enabled: false },
oauth: {
providers: {
github: {
clientId: 'native-client',
clientSecret: 'native-secret',
},
},
},
redis: {
host: '127.0.0.1',
db: 4,
ioredis: { enableAutoPipelining: true },
},
})
);
const handle = new ServerConfigHandle(configPath);
const factory = new ConfigFactory(handle, {
redis: { host: 'ignored.example', db: 1 },
});
const config = factory.config;
t.is(handle.deploymentType, 'selfhosted');
t.true(handle.hasBaselineConfigKey('oauth.providers.github'));
t.false(handle.hasBaselineConfigKey('oauth.providers.oidc'));
t.true(
handle.nativeSecretAppConfigKeys().includes('oauth.providers.github')
);
t.is(JSON.parse(handle.nodeOwnedJson()).oauth, undefined);
for (const key of handle.nativeAppConfigKeys()) {
t.false(has(config, key), key);
}
factory.override({
oauth: {
providers: {
github: {
clientId: 'override-client',
clientSecret: 'override-secret',
},
},
},
});
t.is(get(config, 'oauth.providers.github'), undefined);
const changes = factory.replaceDbOverrides({
oauth: {
providers: {
github: {
clientId: 'database-client',
clientSecret: 'database-secret',
},
},
},
});
t.is(get(config, 'oauth.providers.github'), undefined);
t.is(get(changes, 'oauth.providers.github'), undefined);
t.is(
get(
factory.redactNativeValues({
oauth: {
providers: {
github: {
clientId: 'event-client',
clientSecret: 'event-secret',
},
},
},
}),
'oauth.providers.github'
),
undefined
);
t.is(config.server.name, 'shared-config');
t.false(has(config, 'redis.ioredis'));
t.true(JSON.parse(handle.redisNodeOptionsJson()).enableAutoPipelining);
t.deepEqual(JSON.parse(handle.publicNativeBaselineJson()), {
copilot: { enabled: false },
redis: { host: '127.0.0.1', db: 4 },
});
const cache = new CacheRedis(handle);
const session = new SessionRedis(handle);
const socket = new SocketIoRedis(handle);
try {
const redisUrl = new URL(handle.redisUrl()!);
t.is(cache.options.host, redisUrl.hostname);
t.true(cache.options.enableAutoPipelining);
const baseDb = Number(redisUrl.pathname.slice(1));
t.is(cache.options.db, baseDb);
t.is(session.options.db, baseDb + 2);
t.is(socket.options.db, baseDb + 3);
} finally {
cache.disconnect();
session.disconnect();
socket.disconnect();
}
} finally {
rmSync(directory, { recursive: true, force: true });
}
});
test('should override config', async t => {
await using module = await createModule({
imports: [
@@ -187,11 +343,14 @@ test('should clone from original config without modifications', t => {
const config = module.get(Config);
const configFactory = module.get(ConfigFactory);
config.auth.allowSignup = !config.auth.allowSignup;
config.auth.trustedCloudflareHeaders = !config.auth.trustedCloudflareHeaders;
const newConfig = configFactory.clone();
t.not(newConfig.auth.allowSignup, config.auth.allowSignup);
t.not(
newConfig.auth.trustedCloudflareHeaders,
config.auth.trustedCloudflareHeaders
);
});
test('should override with undefined fields', async t => {
@@ -204,17 +363,14 @@ test('should override with undefined fields', async t => {
configFactory.override({
copilot: {
providers: {
exa: {
key: '',
// @ts-expect-error undefined field
unknown: {
apiKey: '123',
},
unknown: '123',
},
},
});
// @ts-expect-error undefined field
t.deepEqual(config.copilot.providers.unknown, {
apiKey: '123',
});
t.is(config.copilot.exa.unknown, '123');
});
@@ -1,3 +1,4 @@
import { ApplyType } from '../utils';
import { NodeConfig } from './types';
export class Config extends ApplyType<AppConfig>() {}
export class Config extends ApplyType<NodeConfig>() {}
@@ -1,5 +1,7 @@
import { Inject, Injectable, Optional } from '@nestjs/common';
import { get, isEqual, set, unset } from 'lodash-es';
import { ServerConfigHandle } from '../../native';
import { InvalidAppConfig } from '../error';
import { APP_CONFIG_DESCRIPTORS, getDefaultConfig, override } from './register';
@@ -7,18 +9,27 @@ export const OVERRIDE_CONFIG_TOKEN = Symbol('OVERRIDE_CONFIG_TOKEN');
@Injectable()
export class ConfigFactory {
readonly #static: AppConfig;
readonly #original: AppConfig;
readonly #config: AppConfig;
readonly #nativeKeys: ReadonlySet<string>;
readonly #nativeSecretKeys: ReadonlySet<string>;
get config() {
return this.#config;
}
constructor(
private readonly serverConfig: ServerConfigHandle,
@Inject(OVERRIDE_CONFIG_TOKEN)
@Optional()
private readonly overrides: DeepPartial<AppConfig> = {}
) {
this.#original = this.loadDefault();
this.#nativeKeys = new Set(this.serverConfig.nativeAppConfigKeys());
this.#nativeSecretKeys = new Set(
this.serverConfig.nativeSecretAppConfigKeys()
);
this.#static = this.loadDefault();
this.#original = structuredClone(this.#static);
this.#config = structuredClone(this.#original);
}
@@ -28,11 +39,83 @@ export class ConfigFactory {
}
override(updates: DeepPartial<AppConfig>) {
override(this.#original, updates);
override(this.#config, updates);
const visible = this.withoutNativeValues(updates);
override(this.#original, visible);
override(this.#config, visible);
}
validate(updates: Array<{ module: string; key: string; value: any }>) {
private withoutNativeValues(
updates: DeepPartial<AppConfig>
): DeepPartial<AppConfig> {
const visible = structuredClone(updates);
for (const key of this.#nativeKeys) {
unset(visible, key);
}
return visible;
}
redactNativeValues(updates: DeepPartial<AppConfig>): DeepPartial<AppConfig> {
const redacted = structuredClone(updates);
for (const key of this.#nativeKeys) {
if (get(redacted, key) !== undefined) {
set(redacted, key, undefined);
}
}
return redacted;
}
isNativeSecretKey(key: string) {
return this.#nativeSecretKeys.has(key);
}
resetPath(path: string) {
const value = get(this.#static, path);
if (value === undefined) {
unset(this.#original, path);
unset(this.#config, path);
} else {
set(this.#original, path, structuredClone(value));
set(this.#config, path, structuredClone(value));
}
}
replaceDbOverrides(updates: DeepPartial<AppConfig>): DeepPartial<AppConfig> {
const previous = this.clone();
const changed = this.withoutNativeValues(updates);
for (const [module, descriptors] of Object.entries(
APP_CONFIG_DESCRIPTORS
)) {
for (const key of Object.keys(descriptors)) {
const path = `${module}.${key}`;
this.resetPath(path);
}
}
this.override(updates);
for (const [module, descriptors] of Object.entries(
APP_CONFIG_DESCRIPTORS
)) {
for (const key of Object.keys(descriptors)) {
const path = `${module}.${key}`;
const value = get(this.#original, path);
if (!isEqual(get(previous, path), value)) {
set(changed, path, value);
}
}
}
for (const key of this.#nativeKeys) {
set(changed, key, undefined);
}
return changed;
}
validate(
updates: Array<{
module: string;
key: string;
value?: any;
clear?: boolean;
}>
) {
const errors: InvalidAppConfig[] = [];
updates.forEach(update => {
@@ -48,6 +131,21 @@ export class ConfigFactory {
return;
}
if (update.clear) {
return;
}
if (update.value === undefined) {
errors.push(
new InvalidAppConfig({
module: update.module,
key: update.key,
hint: 'A value is required when setting config',
})
);
return;
}
const { success, error } = descriptor.validate(update.value);
if (!success) {
error.issues.forEach(issue => {
@@ -66,8 +164,9 @@ export class ConfigFactory {
}
private loadDefault() {
const config = getDefaultConfig();
override(config, this.overrides);
const config = getDefaultConfig(this.#nativeKeys);
override(config, JSON.parse(this.serverConfig.nodeOwnedJson()));
override(config, this.withoutNativeValues(this.overrides));
return config;
}
}
@@ -1,13 +1,21 @@
import { DynamicModule, Global, Module, Provider } from '@nestjs/common';
import { ServerConfigHandle } from '../../native';
import { Config } from './config';
import { ConfigFactory, OVERRIDE_CONFIG_TOKEN } from './factory';
import { ConfigProvider } from './provider';
@Global()
@Module({
providers: [ConfigProvider, ConfigFactory],
exports: [ConfigProvider, ConfigFactory],
providers: [
{
provide: ServerConfigHandle,
useFactory: () => env.serverConfigHandle,
},
ConfigProvider,
ConfigFactory,
],
exports: [ServerConfigHandle, ConfigProvider, ConfigFactory],
})
export class ConfigModule {
static override(overrides: DeepPartial<AppConfigSchema> = {}): DynamicModule {
@@ -1,7 +1,3 @@
import { existsSync, readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { join } from 'node:path';
import { mergeWith, once, set } from 'lodash-es';
import { z } from 'zod';
@@ -242,6 +238,9 @@ export type NativeAppConfigDescriptor = {
defaultValue: unknown;
schema: JSONSchema;
internal: boolean;
envName?: string | null;
envType?: string | null;
link?: string | null;
};
export function defineNativeModuleConfig<T extends keyof AppConfigSchema>(
@@ -260,6 +259,13 @@ export function defineNativeModuleConfig<T extends keyof AppConfigSchema>(
default: descriptor.defaultValue,
schema: descriptor.schema,
internal: descriptor.internal,
link: descriptor.link ?? undefined,
env:
descriptor.envName &&
(descriptor.envType === 'string' ||
descriptor.envType === 'integer')
? [descriptor.envName, descriptor.envType]
: undefined,
validate: (value: unknown) => {
const errors = validate(module, descriptor.key, value);
return errors.length
@@ -281,33 +287,6 @@ export function defineNativeModuleConfig<T extends keyof AppConfigSchema>(
} as Record<string, ConfigDefineDescriptor<unknown>>);
}
export const CONFIG_JSON_PATHS = [
join(env.projectRoot, 'config.json'),
`${homedir()}/.affine/config/config.json`,
];
function readConfigJSONOverrides(path: string) {
const overrides: DeepPartial<AppConfig> = {};
if (existsSync(path)) {
try {
const config = JSON.parse(readFileSync(path, 'utf-8')) as AppConfig;
Object.entries(config).forEach(([key, value]) => {
if (key === '$schema') {
return;
}
Object.entries(value).forEach(([k, v]) => {
set(overrides, `${key}.${k}`, v);
});
});
} catch (e) {
console.error('Invalid json config file', e);
}
}
return overrides;
}
export function override(config: AppConfig, update: DeepPartial<AppConfig>) {
Object.keys(update).forEach(module => {
const moduleDescriptors = APP_CONFIG_DESCRIPTORS[module];
@@ -346,7 +325,9 @@ export function override(config: AppConfig, update: DeepPartial<AppConfig>) {
});
}
export function getDefaultConfig(): AppConfig {
export function getDefaultConfig(
excludedKeys: ReadonlySet<string> = new Set()
): AppConfig {
const config = {} as AppConfig;
const envs = process.env;
@@ -354,6 +335,9 @@ export function getDefaultConfig(): AppConfig {
const modulizedConfig = {};
for (const [key, desc] of Object.entries(defs)) {
if (excludedKeys.has(`${module}.${key}`)) {
continue;
}
let defaultValue = desc.default;
if (desc.env) {
@@ -385,10 +369,5 @@ Error: ${issue.message}`;
config[module] = modulizedConfig;
}
CONFIG_JSON_PATHS.forEach(path => {
const overrides = readConfigJSONOverrides(path);
override(config, overrides);
});
return config as AppConfigSchema;
}
@@ -0,0 +1,9 @@
export const NODE_SECRET_CONFIG_KEYS = new Set([
'calendar.google',
'captcha.config',
'copilot.exa',
'copilot.unsplash',
'mailer.fallbackSMTP.password',
'mailer.SMTP.password',
'telemetry.ga4.apiSecret',
]);
@@ -18,3 +18,27 @@ export type AppConfigByPath<Module extends keyof AppConfigSchema> =
: never;
}
: never;
export type NodeConfig = Omit<
AppConfig,
| 'auth'
| 'copilot'
| 'crypto'
| 'db'
| 'indexer'
| 'oauth'
| 'payment'
| 'redis'
| 'storages'
> & {
auth: Pick<
AppConfig['auth'],
'passwordRequirements' | 'signInRateLimit' | 'trustedCloudflareHeaders'
>;
copilot: Pick<AppConfig['copilot'], 'exa' | 'unsplash'>;
db: Pick<AppConfig['db'], 'prisma'>;
payment: Pick<AppConfig['payment'], 'showLifetimePrice'>;
storages: {
avatar: Pick<AppConfig['storages']['avatar'], 'publicPath'>;
};
};
@@ -7,6 +7,7 @@ import { CryptoHelper } from '../crypto';
const test = ava as TestFn<{
crypto: CryptoHelper;
setPrivateKey(key: string): void;
}>;
function generateTestPrivateKey(): string {
@@ -25,11 +26,13 @@ const privateKey = generateTestPrivateKey();
const privateKey2 = generateTestPrivateKey();
test.beforeEach(async t => {
let activeKey = privateKey;
t.context.crypto = new CryptoHelper({
crypto: {
privateKey,
},
} as any);
nodeCryptoPrivateKey: () => activeKey,
});
t.context.setPrivateKey = key => {
activeKey = key;
};
t.context.crypto.onConfigInit();
});
@@ -46,14 +49,22 @@ test('should verify signatures across key rotation', t => {
const signatureV1 = t.context.crypto.sign(data);
t.true(t.context.crypto.verify(signatureV1));
(t.context.crypto as any).config.crypto.privateKey = privateKey2;
t.context.crypto.onConfigChanged({
updates: { crypto: { privateKey: privateKey2 } },
} as any);
t.context.setPrivateKey(privateKey2);
t.context.crypto.onConfigApplied({
updates: { crypto: { privateKey: undefined } },
});
const signatureV2 = t.context.crypto.sign(data);
t.true(t.context.crypto.verify(signatureV1));
t.true(t.context.crypto.verify(signatureV2));
const previousKeyHash = t.context.crypto.keyPair.sha256.privateKey;
t.context.setPrivateKey('');
t.context.crypto.onConfigApplied({
updates: { crypto: { privateKey: undefined } },
});
t.notDeepEqual(t.context.crypto.keyPair.sha256.privateKey, previousKeyHash);
t.true(t.context.crypto.verify(signatureV2));
});
test('should same data should get different signature', t => {
@@ -1,4 +1,6 @@
import { defineModuleConfig } from '../config';
import serverNativeModule from '@affine/server-native';
import { defineNativeModuleConfig } from '../config';
declare global {
interface AppConfigSchema {
@@ -8,11 +10,8 @@ declare global {
}
}
defineModuleConfig('crypto', {
privateKey: {
desc: 'The private key for used by the crypto module to create signed tokens or encrypt data.',
env: 'AFFINE_PRIVATE_KEY',
default: '',
schema: { type: 'string' },
},
});
defineNativeModuleConfig(
'crypto',
serverNativeModule.appConfigDescriptors('crypto'),
serverNativeModule.validateAppConfigValue
);
@@ -15,16 +15,17 @@ import {
verify,
} from 'node:crypto';
import { Injectable, OnModuleInit } from '@nestjs/common';
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { hash as hashPassword } from '@node-rs/argon2';
import { AFFINE_PRO_PUBLIC_KEY } from '../../native';
import { Config } from '../config';
import { OnEvent } from '../event';
const NONCE_LENGTH = 12;
const AUTH_TAG_LENGTH = 12;
export const CRYPTO_KEY_SOURCE = Symbol('CRYPTO_KEY_SOURCE');
function generatePrivateKey(): string {
const { privateKey } = generateKeyPairSync('ec', {
namedCurve: 'prime256v1',
@@ -83,23 +84,27 @@ export class CryptoHelper implements OnModuleInit {
}
}
constructor(private readonly config: Config) {}
constructor(
@Inject(CRYPTO_KEY_SOURCE)
private readonly source: { nodeCryptoPrivateKey(): string }
) {}
@OnEvent('config.init')
onConfigInit() {
this.setup();
}
@OnEvent('config.changed')
onConfigChanged(event: Events['config.changed']) {
if (event.updates.crypto?.privateKey) {
@OnEvent('backendRuntime.configApplied')
onConfigApplied(event: Events['backendRuntime.configApplied']) {
if (event.updates.crypto && 'privateKey' in event.updates.crypto) {
this.setup();
}
}
private setup() {
const prevPublicKey = this.keyPair?.publicKey;
const privateKey = this.config.crypto.privateKey || generatePrivateKey();
const privateKey =
this.source.nodeCryptoPrivateKey() || generatePrivateKey();
const { priv, pub } = parseKey(privateKey);
const publicKey = pub
.export({ format: 'pem', type: 'spki' })
@@ -1,7 +1,7 @@
import serverNativeModule from '@affine/server-native';
import { RedisOptions } from 'ioredis';
import { z } from 'zod';
import { defineModuleConfig } from '../config';
import { defineNativeModuleConfig } from '../config';
declare global {
interface AppConfigSchema {
@@ -18,37 +18,15 @@ declare global {
}
}
defineModuleConfig('redis', {
db: {
desc: 'The database index of redis server to be used(Must be less than 10).',
default: 0,
env: ['REDIS_SERVER_DATABASE', 'integer'],
shape: z.number().int().nonnegative().max(10),
},
host: {
desc: 'The host of the redis server.',
default: 'localhost',
env: ['REDIS_SERVER_HOST', 'string'],
},
port: {
desc: 'The port of the redis server.',
default: 6379,
env: ['REDIS_SERVER_PORT', 'integer'],
shape: z.number().positive(),
},
username: {
desc: 'The username of the redis server.',
default: '',
env: ['REDIS_SERVER_USERNAME', 'string'],
},
password: {
desc: 'The password of the redis server.',
default: '',
env: ['REDIS_SERVER_PASSWORD', 'string'],
},
ioredis: {
desc: 'The config for the ioredis client.',
default: {},
link: 'https://github.com/luin/ioredis',
},
});
defineNativeModuleConfig(
'redis',
serverNativeModule.appConfigDescriptors('redis'),
serverNativeModule.validateAppConfigValue,
{
ioredis: {
desc: 'The config for the ioredis client.',
default: {},
link: 'https://github.com/luin/ioredis',
},
}
);
@@ -5,8 +5,9 @@ import {
OnModuleInit,
} from '@nestjs/common';
import { Redis as IORedis, RedisOptions } from 'ioredis';
import { omit } from 'lodash-es';
import { Config } from '../config';
import { ServerConfigHandle } from '../../native';
function redisOptions(options: RedisOptions) {
return {
@@ -15,6 +16,20 @@ function redisOptions(options: RedisOptions) {
};
}
function redisConnection(
handle: ServerConfigHandle,
dbOffset: number
): [string, RedisOptions] {
const url = new URL(handle.redisUrl() ?? 'redis://localhost:6379/0');
const db = Number(url.pathname.slice(1) || 0) + dbOffset;
url.pathname = `/${db}`;
const options = omit(
JSON.parse(handle.redisNodeOptionsJson()) as RedisOptions,
['host', 'port', 'db', 'username', 'password']
);
return [url.toString(), redisOptions(options)];
}
class Redis extends IORedis implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(this.constructor.name);
@@ -53,33 +68,21 @@ class Redis extends IORedis implements OnModuleInit, OnModuleDestroy {
@Injectable()
export class CacheRedis extends Redis {
constructor(config: Config) {
super(redisOptions({ ...config.redis, ...config.redis.ioredis }));
constructor(handle: ServerConfigHandle) {
super(...redisConnection(handle, 0));
}
}
@Injectable()
export class SessionRedis extends Redis {
constructor(config: Config) {
super(
redisOptions({
...config.redis,
...config.redis.ioredis,
db: (config.redis.db ?? 0) + 2,
})
);
constructor(handle: ServerConfigHandle) {
super(...redisConnection(handle, 2));
}
}
@Injectable()
export class SocketIoRedis extends Redis {
constructor(config: Config) {
super(
redisOptions({
...config.redis,
...config.redis.ioredis,
db: (config.redis.db ?? 0) + 3,
})
);
constructor(handle: ServerConfigHandle) {
super(...redisConnection(handle, 3));
}
}
@@ -1,5 +1,3 @@
import { JSONSchema } from '../../config';
export type StorageProviderName =
| 'fs'
| 'aws-s3'
@@ -60,185 +58,6 @@ export type StorageProviderConfig = { bucket: string } & (
}
);
const S3ConfigSchema: JSONSchema = {
type: 'object',
description: 'The config for the S3 compatible storage provider.',
properties: {
endpoint: {
type: 'string',
description:
'The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region.',
},
region: {
type: 'string',
description:
'The region for the storage provider. Example: "us-east-1" or "auto" for R2.',
},
forcePathStyle: {
type: 'boolean',
description: 'Whether to use path-style bucket addressing.',
},
requestTimeoutMs: {
type: 'number',
description: 'Request timeout in milliseconds.',
},
minPartSize: {
type: 'number',
description: 'Minimum multipart part size in bytes.',
},
presign: {
type: 'object',
description: 'Presigned URL behavior configuration.',
properties: {
expiresInSeconds: {
type: 'number',
description: 'Expiration time in seconds for presigned URLs.',
},
signContentTypeForPut: {
type: 'boolean',
description: 'Whether to sign Content-Type for presigned PUT.',
},
},
},
credentials: {
type: 'object',
description: 'The credentials for the s3 compatible storage provider.',
properties: {
accessKeyId: {
type: 'string',
},
secretAccessKey: {
type: 'string',
},
sessionToken: {
type: 'string',
},
},
},
usePresignedURL: {
type: 'object',
description:
'Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.',
properties: {
enabled: {
type: 'boolean',
description: 'Whether browser upload URLs are enabled.',
},
urlPrefix: {
type: 'string',
description:
'Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured.',
},
signKey: {
type: 'string',
description:
'Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix.',
},
},
},
},
};
const S3ConfigPropertiesWithoutEndpoint = Object.fromEntries(
Object.entries(
(
S3ConfigSchema as {
type: 'object';
properties?: Record<string, JSONSchema>;
}
).properties ?? {}
).filter(([key]) => key !== 'endpoint')
) as Record<string, JSONSchema>;
export const StorageJSONSchema: JSONSchema = {
oneOf: [
{
type: 'object',
properties: {
provider: {
type: 'string',
enum: ['fs'],
},
bucket: {
type: 'string',
},
config: {
type: 'object',
properties: {
path: {
type: 'string',
},
},
},
},
},
{
type: 'object',
properties: {
provider: {
type: 'string',
enum: ['aws-s3'],
},
bucket: {
type: 'string',
},
config: S3ConfigSchema,
},
},
{
type: 'object',
properties: {
provider: {
type: 'string',
enum: ['cloudflare-r2'],
},
bucket: {
type: 'string',
},
config: {
...S3ConfigSchema,
properties: {
...S3ConfigPropertiesWithoutEndpoint,
accountId: {
type: 'string' as const,
description:
'The account id for the cloudflare r2 storage provider.',
},
jurisdiction: {
type: 'string' as const,
enum: [...R2_JURISDICTIONS],
description:
'Optional jurisdiction for the cloudflare r2 endpoint. Set to "eu" for EU buckets.',
},
},
},
},
},
{
type: 'object',
properties: {
provider: {
type: 'string',
enum: ['assetpack'],
},
bucket: {
type: 'string',
},
config: {
type: 'object',
properties: {
path: {
type: 'string',
},
},
required: ['path'],
},
},
required: ['provider', 'bucket', 'config'],
},
],
};
export type * from '../types';
export {
applyAttachHeaders,
@@ -1,4 +1,3 @@
import { createHmac } from 'node:crypto';
import { Readable } from 'node:stream';
import type { Response } from 'express';
@@ -69,20 +68,3 @@ export const SIGNED_URL_EXPIRED = 60 * 60; // 1 hour
export const STORAGE_PROXY_ROOT = '/api/storage';
export const PROXY_UPLOAD_PATH = `${STORAGE_PROXY_ROOT}/upload`;
export const PROXY_MULTIPART_PATH = `${STORAGE_PROXY_ROOT}/multipart`;
export function createStorageUploadToken(
path: string,
fields: (string | number)[],
expiresAt: number,
signKey: string
) {
const canonical = JSON.stringify([
'affine-storage-upload',
1,
'PUT',
path,
...fields,
expiresAt,
]);
return createHmac('sha256', signKey).update(canonical).digest('base64url');
}
+6 -1
View File
@@ -100,10 +100,15 @@ function buildProgram(logger: Logger) {
program
.command('import-config [path]')
.description('import config from a file')
.description(
'import config from a file; changes take effect on the next server start'
)
.action(async path => {
await withCliApp(logger, async app => {
await app.get(ImportConfigCommand).execute(path);
logger.log(
'Config imported; changes take effect on the next server start.'
);
});
});
+49 -113
View File
@@ -1,6 +1,7 @@
import serverNativeModule from '@affine/server-native';
import { z } from 'zod';
import { defineModuleConfig } from '../../base';
import { defineNativeModuleConfig } from '../../base';
export interface AuthConfig {
session: {
@@ -17,7 +18,6 @@ export interface AuthConfig {
allowSignup: boolean;
allowSignupForOauth: boolean;
requireEmailDomainVerification: boolean;
requireEmailVerification: boolean;
newAccountActionDelay: number;
trustedCloudflareHeaders: boolean;
signInRateLimit: ConfigItem<{
@@ -37,118 +37,54 @@ declare global {
}
}
defineModuleConfig('auth', {
allowSignup: {
desc: 'Whether allow new registrations.',
default: true,
},
allowSignupForOauth: {
desc: 'Whether allow new registrations via configured oauth.',
default: true,
},
requireEmailDomainVerification: {
desc: 'Whether require email domain record verification before accessing restricted resources.',
default: false,
},
requireEmailVerification: {
desc: 'Whether require email verification before accessing restricted resources(not implemented).',
default: true,
},
newAccountActionDelay: {
desc: 'Minimum account age in seconds before new accounts can invite members, create invite links, or publish documents. Set to 0 to disable.',
default: 24 * 60 * 60,
shape: z.number().int().min(0),
},
trustedCloudflareHeaders: {
desc: 'Whether request abuse source facts should trust Cloudflare headers from the origin edge.',
default: false,
shape: z.boolean(),
},
signInRateLimit: {
desc: 'Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds.',
default: {
ttl: 60_000,
ipLimit: 20,
emailLimit: 5,
defineNativeModuleConfig(
'auth',
serverNativeModule.appConfigDescriptors('auth'),
serverNativeModule.validateAppConfigValue,
{
trustedCloudflareHeaders: {
desc: 'Whether request abuse source facts should trust Cloudflare headers from the origin edge.',
default: false,
shape: z.boolean(),
},
shape: z
.object({
ttl: z.number().int().positive(),
ipLimit: z.number().int().positive(),
emailLimit: z.number().int().positive(),
})
.strict(),
},
passwordRequirements: {
desc: 'The password strength requirements when set new password.',
default: {
min: 8,
max: 32,
signInRateLimit: {
desc: 'Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds.',
default: {
ttl: 60_000,
ipLimit: 20,
emailLimit: 5,
},
shape: z
.object({
ttl: z.number().int().positive(),
ipLimit: z.number().int().positive(),
emailLimit: z.number().int().positive(),
})
.strict(),
},
shape: z
.object({
min: z.number().min(1),
max: z.number().max(100),
})
.strict()
.refine(data => data.min < data.max, {
message: 'Minimum length of password must be less than maximum length',
}),
schema: {
type: 'object',
properties: {
min: { type: 'number' },
max: { type: 'number' },
passwordRequirements: {
desc: 'The password strength requirements when set new password.',
default: {
min: 8,
max: 32,
},
shape: z
.object({
min: z.number().min(1),
max: z.number().max(100),
})
.strict()
.refine(data => data.min < data.max, {
message:
'Minimum length of password must be less than maximum length',
}),
schema: {
type: 'object',
properties: {
min: { type: 'number' },
max: { type: 'number' },
},
},
},
},
'session.ttl': {
desc: 'Application auth expiration time in seconds.',
default: 60 * 60 * 24 * 15, // 15 days
},
'session.ttr': {
desc: 'Application auth time to refresh in seconds.',
default: 60 * 60 * 24 * 7, // 7 days
},
'token.accessTokenTtl': {
desc: 'Access JWT expiration time in seconds.',
default: 15 * 60,
shape: z
.number()
.int()
.min(60)
.max(60 * 60),
},
'token.refreshIdleTtl': {
desc: 'Auth refresh session inactivity expiration in seconds.',
default: 60 * 60 * 24 * 30,
shape: z
.number()
.int()
.min(60 * 60)
.max(60 * 60 * 24 * 365),
},
'token.refreshAbsoluteTtl': {
desc: 'Auth refresh session absolute expiration in seconds.',
default: 60 * 60 * 24 * 180,
shape: z
.number()
.int()
.min(60 * 60)
.max(60 * 60 * 24 * 730),
},
'token.refreshGracePeriod': {
desc: 'One-use refresh rotation concurrency grace period in seconds.',
default: 30,
shape: z.number().int().min(0).max(60),
},
'token.refreshRetention': {
desc: 'Retention for expired auth refresh generations in seconds.',
default: 60 * 60 * 24 * 30,
shape: z
.number()
.int()
.min(60 * 60)
.max(60 * 60 * 24 * 365),
},
});
}
);
@@ -24,15 +24,18 @@ import {
import {
BackendRuntimeEmbeddingService,
BackendRuntimeHousekeepingJob,
BackendRuntimeSearchJob,
} from '../job';
interface Context {
module: TestingModule;
embeddingService: BackendRuntimeEmbeddingService;
job: BackendRuntimeHousekeepingJob;
searchJob: BackendRuntimeSearchJob;
getSnapshot: Sinon.SinonStub;
allowEmbedding: Sinon.SinonStub;
runtime: {
nodeCryptoPrivateKey: Sinon.SinonStub;
cleanupExpiredRuntimeStates: Sinon.SinonStub;
cleanupExpiredRuntimeGates: Sinon.SinonStub;
cleanupExpiredRollingQuota: Sinon.SinonStub;
@@ -40,6 +43,9 @@ interface Context {
embeddingHealth: Sinon.SinonStub;
syncEmbeddingState: Sinon.SinonStub;
executeAuthSessionCommandV1: Sinon.SinonStub;
searchEnabled: Sinon.SinonStub;
reconcileSearchProjection: Sinon.SinonStub;
searchStatus: Sinon.SinonStub;
};
}
@@ -50,6 +56,7 @@ test.before(async t => {
join(process.cwd(), 'src/__tests__/__fixtures__/test-doc.snapshot.bin')
);
t.context.runtime = {
nodeCryptoPrivateKey: Sinon.stub().returns(''),
cleanupExpiredRuntimeStates: Sinon.stub(),
cleanupExpiredRuntimeGates: Sinon.stub(),
cleanupExpiredRollingQuota: Sinon.stub(),
@@ -57,6 +64,9 @@ test.before(async t => {
embeddingHealth: Sinon.stub().resolves({ enabled: true }),
syncEmbeddingState: Sinon.stub(),
executeAuthSessionCommandV1: Sinon.stub().resolves({}),
searchEnabled: Sinon.stub().returns(false),
reconcileSearchProjection: Sinon.stub().resolves(2),
searchStatus: Sinon.stub().resolves({ ready: true, state: 'active' }),
};
t.context.module = await createTestingModule({
imports: [
@@ -92,6 +102,7 @@ test.before(async t => {
BackendRuntimeEmbeddingService
);
t.context.job = t.context.module.get(BackendRuntimeHousekeepingJob);
t.context.searchJob = t.context.module.get(BackendRuntimeSearchJob);
});
test.beforeEach(t => {
@@ -101,6 +112,9 @@ test.beforeEach(t => {
t.context.runtime.cleanupUnreferencedArtifacts.reset();
t.context.runtime.embeddingHealth.resetHistory();
t.context.runtime.syncEmbeddingState.reset();
t.context.runtime.searchEnabled.resetHistory();
t.context.runtime.reconcileSearchProjection.resetHistory();
t.context.runtime.searchStatus.resetHistory();
t.context.getSnapshot.resetHistory();
t.context.allowEmbedding.resetHistory();
});
@@ -209,4 +223,10 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t =>
t.is(t.context.runtime.cleanupExpiredRuntimeGates.callCount, 1);
t.is(t.context.runtime.cleanupExpiredRollingQuota.callCount, 1);
t.is(t.context.runtime.cleanupUnreferencedArtifacts.callCount, 1);
t.is(await t.context.searchJob.reconcileProjection(7), 0);
t.false(t.context.runtime.reconcileSearchProjection.called);
t.context.runtime.searchEnabled.returns(true);
t.is(await t.context.searchJob.reconcileProjection(7), 2);
t.true(t.context.runtime.reconcileSearchProjection.calledOnceWithExactly(7));
});
@@ -1,29 +1,40 @@
import { generateKeyPairSync } from 'node:crypto';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import test from 'ava';
import Sinon from 'sinon';
import type { Config } from '../../../base';
import type { EventBus } from '../../../base';
import { ServerConfigHandle } from '../../../native';
import { BackendRuntimeProvider } from '../provider';
const privateKey = generateKeyPairSync('ec', {
namedCurve: 'P-256',
}).privateKey.export({ format: 'pem', type: 'pkcs8' }) as string;
const storage = {
provider: 'assetpack',
bucket: 'test',
config: { path: '/tmp/affine-backend-runtime-test' },
};
const config = {
crypto: { privateKey },
storages: { blob: { storage }, avatar: { storage } },
copilot: { storage },
} as Config;
const directory = mkdtempSync(join(tmpdir(), 'affine-backend-runtime-'));
const configPath = join(directory, 'config.json');
writeFileSync(
configPath,
JSON.stringify({
deployment: { type: 'selfhosted' },
crypto: { privateKey },
copilot: { enabled: false },
})
);
const serverConfig = new ServerConfigHandle(configPath);
test.after.always(() => rmSync(directory, { recursive: true, force: true }));
test('backend-runtime provider starts without migrations and exposes explicit migration', async t => {
const provider = new BackendRuntimeProvider(config, []);
const event = {
emitAsync: Sinon.stub().resolves(),
};
const provider = new BackendRuntimeProvider(
serverConfig,
event as unknown as EventBus
);
const runtime = {
configureObjectStorage: Sinon.stub(),
start: Sinon.stub().resolves(),
stop: Sinon.stub().resolves(),
runMigrations: Sinon.stub().resolves(),
@@ -49,31 +60,36 @@ test('backend-runtime provider starts without migrations and exposes explicit mi
await provider.onConfigChanged({ updates: { mailer: {} } });
await provider.onConfigChanged({ updates: { copilot: {} } });
await provider.onConfigChanged({ updates: { storages: {} } });
await provider.onConfigChanged({ updates: { oauth: {} } });
const health = await provider.health();
t.is(runtime.reloadConfig.callCount, 3);
t.true(runtime.reloadConfig.alwaysCalledWithExactly());
t.is(event.emitAsync.callCount, 5);
t.true(
event.emitAsync.calledWith('backendRuntime.configApplied', {
updates: { copilot: {} },
})
);
t.true(
event.emitAsync.calledWith('backendRuntime.configApplied', {
updates: { oauth: {} },
})
);
runtime.reloadConfig.rejects(new Error('sensitive native credential'));
await provider.onConfigChanged({ updates: { copilot: {} } });
await provider.stop();
t.is(runtime.start.callCount, 2);
t.is(runtime.runMigrations.callCount, 1);
t.is(runtime.reloadConfig.callCount, 2);
t.true(
runtime.reloadConfig.alwaysCalledWithExactly(
privateKey,
JSON.stringify({
storages: {
'blob.storage': storage,
'avatar.storage': storage,
},
copilot: { storage },
}),
'{}'
)
);
t.is(runtime.reloadConfig.callCount, 4);
t.is(event.emitAsync.callCount, 5);
t.true(health.databaseConnected);
t.is(runtime.stop.callCount, 1);
});
test('backend-runtime provider measures explicit typed methods', async t => {
const provider = new BackendRuntimeProvider(config, []);
const provider = new BackendRuntimeProvider(serverConfig);
const runtime = {
cleanupExpiredRuntimeStates: Sinon.stub().resolves(3),
assertCopilotRoute: Sinon.stub().resolves(),
@@ -98,7 +114,7 @@ test('backend-runtime provider measures explicit typed methods', async t => {
});
test('backend-runtime provider encodes recursive search contracts at the native boundary', async t => {
const provider = new BackendRuntimeProvider(config, []);
const provider = new BackendRuntimeProvider(serverConfig);
const runtime = {
searchAuthorized: Sinon.stub().resolves({
ok: true,
@@ -198,7 +214,7 @@ test('backend-runtime provider encodes recursive search contracts at the native
});
test('backend-runtime provider aborts a stream handle that resolves after iterator cancellation', async t => {
const provider = new BackendRuntimeProvider(config, []);
const provider = new BackendRuntimeProvider(serverConfig);
const abort = Sinon.stub();
let resolveHandle!: (handle: { abort: () => void }) => void;
const runtime = {
@@ -1,26 +1,25 @@
import { Global, Module } from '@nestjs/common';
import { CRYPTO_KEY_SOURCE } from '../../base/helpers/crypto';
import {
BackendRuntimeEmbeddingService,
BackendRuntimeHousekeepingJob,
BackendRuntimeSearchJob,
} from './job';
import {
BACKEND_RUNTIME_CONFIG_PATHS,
BackendRuntimeProvider,
} from './provider';
import { BackendRuntimeProvider } from './provider';
@Global()
@Module({
providers: [
{
provide: BACKEND_RUNTIME_CONFIG_PATHS,
useValue: undefined,
},
BackendRuntimeProvider,
BackendRuntimeEmbeddingService,
{ provide: CRYPTO_KEY_SOURCE, useExisting: BackendRuntimeProvider },
],
exports: [
BackendRuntimeProvider,
BackendRuntimeEmbeddingService,
CRYPTO_KEY_SOURCE,
],
exports: [BackendRuntimeProvider, BackendRuntimeEmbeddingService],
})
export class BackendRuntimeModule {}
@@ -33,7 +32,6 @@ export class BackendRuntimeWorkerModule {}
export { BackendRuntimeEmbeddingService } from './job';
export { BackendRuntimeError, backendRuntimeErrorCode } from './operations';
export {
BACKEND_RUNTIME_CONFIG_PATHS,
BackendRuntimeProvider,
type BlobManifestEntryV1,
type BlobManifestV1,
@@ -2,7 +2,6 @@ import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import {
ConfigFactory,
CopilotSelectedSourcesFailed,
CopilotSelectedSourcesLimitExceeded,
CopilotSelectedSourcesProcessing,
@@ -173,14 +172,11 @@ export class BackendRuntimeHousekeepingJob {
@Injectable()
export class BackendRuntimeSearchJob {
constructor(
private readonly rt: BackendRuntimeProvider,
private readonly config: ConfigFactory
) {}
constructor(private readonly rt: BackendRuntimeProvider) {}
@Cron(CronExpression.EVERY_30_SECONDS)
async reconcileProjection(limit = 100) {
if (!this.config.config.indexer.enabled) return 0;
if (!this.rt.searchEnabled()) return 0;
const startedAt = performance.now();
try {
const reconciled = await this.rt.reconcileSearchProjection(limit);
@@ -41,6 +41,22 @@ export function backendRuntimeErrorCode(error: unknown) {
export class BackendRuntimeCoreOperations {
constructor(protected readonly runtime: RuntimeInstance) {}
stripeEnabled() {
return this.runtime.stripeEnabled();
}
paymentEnabled() {
return this.runtime.paymentEnabled();
}
copilotEnabled() {
return this.runtime.copilotEnabled();
}
nodeCryptoPrivateKey() {
return this.runtime.nodeCryptoPrivateKey();
}
executePaymentCommandV1<T = unknown>(input: Record<string, unknown>) {
return this.measured('executePaymentCommandV1', runtime =>
runtime.executePaymentCommandV1(input)
Loaded 100 of 159 files, more files were not shown because too many files have changed in this diff. Show more