mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-11-20 21:23:37 +03:00
feat(server): improve config loading (#15649)
fix #15650 fix #15530 fix #15534 fix #15506 #### PR Dependency Tree * **PR #15649** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Summary * **New Features** * Admin settings show configuration sources and secret status, conceal secret values, and allow database-level overrides to be cleared. * Self-hosted deployments can automatically upgrade existing configuration files. * **Improvements** * Configuration changes apply without restarting the server, and storage settings reload automatically. * Uploads support runtime-managed proxy settings and signed upload tokens. * Incomplete managed Copilot profiles are disabled during migration. * Saving an AI response as a document more reliably targets the matching editor. * **Documentation** * The CLI clarifies that imported configuration changes take effect after the next server start. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
1 parent
164a84ffe0
commit
7afb62d5b4
159 files changed
+9477
-4352
No files matched your search
@@ -1,5 +1,8 @@
|
||||
{
|
||||
"$schema": "https://github.com/toeverything/affine/releases/latest/download/config.schema.json",
|
||||
"deployment": {
|
||||
"type": "selfhosted"
|
||||
},
|
||||
"server": {
|
||||
"name": "AFFiNE Self Hosted Server"
|
||||
},
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
{
|
||||
"$schema": "https://github.com/toeverything/affine/releases/latest/download/config.schema.json",
|
||||
"deployment": {
|
||||
"type": "selfhosted"
|
||||
},
|
||||
"server": {
|
||||
"name": "AFFiNE Self-hosted",
|
||||
"externalUrl": "http://localhost:3010"
|
||||
|
||||
+3226
-894
File diff suppressed because it is too large.
Load diff
@@ -19,16 +19,6 @@ runs:
|
||||
env:
|
||||
PGPASSWORD: affine
|
||||
|
||||
- name: Run init-db script
|
||||
shell: bash
|
||||
env:
|
||||
NODE_ENV: test
|
||||
run: |
|
||||
yarn affine @affine/server prisma generate
|
||||
yarn affine @affine/server data-migration admit-legacy-context-blobs
|
||||
yarn affine @affine/server prisma migrate deploy
|
||||
yarn affine @affine/server data-migration run
|
||||
|
||||
- name: Import config
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -55,3 +45,12 @@ runs:
|
||||
else
|
||||
jq -n --argjson indexer "$default_indexer" '{indexer: $indexer}' > ./packages/backend/server/config.json
|
||||
fi
|
||||
|
||||
- name: Run init-db script
|
||||
shell: bash
|
||||
env:
|
||||
NODE_ENV: test
|
||||
run: |
|
||||
yarn affine @affine/server prisma generate
|
||||
yarn affine @affine/server prisma migrate deploy
|
||||
yarn affine @affine/server data-migration run
|
||||
@@ -32,4 +32,4 @@ ENV LD_PRELOAD=libjemalloc.so.2
|
||||
|
||||
EXPOSE 3010
|
||||
|
||||
CMD ["node", "./dist/main.js"]
|
||||
CMD ["sh", "-c", "node ./scripts/upgrade-config.js && exec node ./dist/main.js"]
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
|
||||
checksum/server-config: {{ include "affine.serverConfig" . | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
|
||||
checksum/server-config: {{ include "affine.serverConfig" . | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -62,8 +62,8 @@ Create the name of the service account to use
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Render the indexer runtime configuration shared by the Secret and Pod checksums.
|
||||
Render the server runtime configuration shared by the Secret and Pod checksums.
|
||||
*/}}
|
||||
{{- define "affine.indexerConfig" -}}
|
||||
{{- dict "indexer" (dict "enabled" .Values.global.indexer.enabled "provider" (dict "type" .Values.global.indexer.provider "endpoint" .Values.global.indexer.endpoint "apiKey" .Values.global.indexer.apiKey "username" .Values.global.indexer.username "password" .Values.global.indexer.password)) | toJson -}}
|
||||
{{- define "affine.serverConfig" -}}
|
||||
{{- dict "deployment" (dict "type" (ternary "selfhosted" "cloud" (eq .Values.global.deployment.type "selfhosted"))) "indexer" (dict "enabled" .Values.global.indexer.enabled "provider" (dict "type" .Values.global.indexer.provider "endpoint" .Values.global.indexer.endpoint "apiKey" .Values.global.indexer.apiKey "username" .Values.global.indexer.username "password" .Values.global.indexer.password)) | toJson -}}
|
||||
{{- end }}
|
||||
@@ -8,4 +8,4 @@ metadata:
|
||||
"helm.sh/hook-delete-policy": before-hook-creation
|
||||
type: Opaque
|
||||
data:
|
||||
config.json: {{ include "affine.indexerConfig" . | b64enc | quote }}
|
||||
config.json: {{ include "affine.serverConfig" . | b64enc | quote }}
|
||||
@@ -17,7 +17,7 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
|
||||
checksum/server-config: {{ include "affine.serverConfig" . | sha256sum }}
|
||||
{{- with .Values.worker.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -3,6 +3,7 @@ exclude = [
|
||||
"target/**/*.toml",
|
||||
"packages/frontend/apps/ios/App/Packages/AffineGraphQL/**/*.toml",
|
||||
"packages/frontend/apps/ios/App/Packages/Intelligents/**/*.toml",
|
||||
"packages/frontend/apps/ios/App/Packages/AffinePaywall/**/*.toml",
|
||||
]
|
||||
|
||||
# https://taplo.tamasfe.dev/configuration/formatter-options.html
|
||||
|
||||
Generated
+2
-3
@@ -76,9 +76,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "affine_core"
|
||||
version = "0.0.4"
|
||||
version = "0.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5626fc812421da8dd685e58d130ff6823d098ea4f412b9fb6350f8d14878c19"
|
||||
checksum = "acda52a0c7f4ee390c1a05814ffe5ddaaa86a4f877b611505281d4d47858d721"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"chrono",
|
||||
@@ -179,7 +179,6 @@ dependencies = [
|
||||
"homedir",
|
||||
"lru 0.16.3",
|
||||
"objc2",
|
||||
"objc2-foundation",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
|
||||
@@ -58,7 +58,6 @@ resolver = "3"
|
||||
memory-indexer = "0.4.0"
|
||||
mimalloc = "0.1"
|
||||
mp4parse = "0.17"
|
||||
nanoid = "0.4"
|
||||
napi = { version = "3.7.0", features = [
|
||||
"async",
|
||||
"chrono_date",
|
||||
@@ -76,7 +75,6 @@ resolver = "3"
|
||||
"onepkg",
|
||||
] }
|
||||
p256 = { version = "0.13", features = ["ecdsa", "pem"] }
|
||||
pulldown-cmark = "0.13"
|
||||
rand = "0.9"
|
||||
rayon = "1.10"
|
||||
redis = { version = "1.6", features = ["tokio-comp", "tokio-rustls-comp"] }
|
||||
|
||||
@@ -11,7 +11,7 @@ crate-type = ["cdylib"]
|
||||
[dependencies]
|
||||
aes-gcm = { workspace = true }
|
||||
affine_common = { workspace = true, features = ["hashcash", "napi"] }
|
||||
affine_core = "0.0.4"
|
||||
affine_core = "0.0.5"
|
||||
affine_doc_loader = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
argon2 = "0.6"
|
||||
|
||||
Vendored
+44
-5
@@ -22,6 +22,7 @@ export declare class BackendRuntime {
|
||||
deleteByokProfile(workspaceId: string, profileId: string): Promise<boolean>
|
||||
reorderByokProfiles(input: ReorderByokProfilesInput): Promise<Array<ByokProfileOutput>>
|
||||
createByokLocalLease(input: CreateByokLocalLeaseInput): Promise<ByokLocalLeaseOutput>
|
||||
saveAppConfig(actor: string | undefined | null, commands: Array<AppConfigCommand>): Promise<Array<string>>
|
||||
getUserQuotaStateV1(userId: string): Promise<RuntimeUserQuotaState>
|
||||
getWorkspaceQuotaStateV1(workspaceId: string): Promise<RuntimeWorkspaceQuotaState>
|
||||
getSyncPermissionGenerationV1(workspaceId: string): Promise<number>
|
||||
@@ -29,6 +30,9 @@ export declare class BackendRuntime {
|
||||
runMigrations(): Promise<void>
|
||||
searchAuthorized(actorUserId: string, workspaceId: string, request: RuntimeSearchRequest): Promise<SearchOperationOutput>
|
||||
aggregateAuthorized(actorUserId: string, workspaceId: string, request: RuntimeAggregateRequest): Promise<SearchOperationOutput>
|
||||
searchEnabled(): boolean
|
||||
copilotEnabled(): boolean
|
||||
nodeCryptoPrivateKey(): string
|
||||
reconcileSearchProjection(limit?: number | undefined | null): Promise<number>
|
||||
authorizePermissionV1(input: any): Promise<any>
|
||||
executeDomainCommandV1(input: any): Promise<any>
|
||||
@@ -79,11 +83,12 @@ export declare class BackendRuntime {
|
||||
cleanupExpiredRuntimeGates(limit: number): Promise<number>
|
||||
cleanupExpiredUserSessions(limit: number): Promise<number>
|
||||
cleanupExpiredSnapshotHistories(limit: number): Promise<number>
|
||||
configureObjectStorage(configJson: string): void
|
||||
start(): Promise<void>
|
||||
stop(): Promise<void>
|
||||
reloadConfig(privateKey?: string | undefined | null, objectStorageConfig?: string | undefined | null, inlineConfig?: string | undefined | null): Promise<void>
|
||||
reloadConfig(): Promise<void>
|
||||
health(): Promise<BackendRuntimeHealth>
|
||||
paymentEnabled(): boolean
|
||||
stripeEnabled(): boolean
|
||||
executePaymentCommandV1(input: any): Promise<any>
|
||||
createPaymentCustomerPortalV1(actorUserId: string): Promise<string>
|
||||
createLicenseCustomerPortalV1(licenseKey: string, validateKey?: string | undefined | null): Promise<string>
|
||||
@@ -119,13 +124,28 @@ export declare class BackendRuntime {
|
||||
activateWorkspaceSeatV1(input: RuntimeSeatActivationInput): Promise<boolean>
|
||||
reserveWorkspaceSeatsV1(input: RuntimeSeatReservationInput): Promise<RuntimeSeatReservationDecision>
|
||||
reserveStorageQuotaV1(input: RuntimeStorageReservationInput): Promise<RuntimeStorageReservationDecision>
|
||||
constructor(privateKey?: string | undefined | null, configPaths?: Array<string> | undefined | null, permissionTelemetry?: (((err: Error | null, arg: string) => void)) | undefined | null, inlineConfig?: string | undefined | null, invalidationEvents?: (((err: Error | null, arg: string) => void)) | undefined | null)
|
||||
constructor(serverConfig: ServerConfigHandle, privateKey?: string | undefined | null, permissionTelemetry?: (((err: Error | null, arg: string) => void)) | undefined | null, invalidationEvents?: (((err: Error | null, arg: string) => void)) | undefined | null)
|
||||
}
|
||||
|
||||
export declare class CopilotStreamHandle {
|
||||
abort(): void
|
||||
}
|
||||
|
||||
export declare class ServerConfigHandle {
|
||||
constructor(path: string, legacyDeploymentType?: string | undefined | null)
|
||||
get path(): string
|
||||
get deploymentType(): string
|
||||
nodeOwnedJson(): string
|
||||
publicNativeBaselineJson(): string
|
||||
hasBaselineConfigKey(key: string): boolean
|
||||
baselineConfigKeyConfigured(key: string): boolean
|
||||
nativeAppConfigKeys(): Array<string>
|
||||
nativeSecretAppConfigKeys(): Array<string>
|
||||
staticAppConfigKeys(): Array<string>
|
||||
redisUrl(): string | null
|
||||
redisNodeOptionsJson(): string
|
||||
}
|
||||
|
||||
export declare class StorageRuntime {
|
||||
cleanupUnreferencedWorkspaceBlobs(workspaceId: string, gracePeriodDays: number, limit: number): Promise<RuntimeBlobCleanupResult>
|
||||
backfillMissingBlobMetadata(workspaceId: string | undefined | null, limit: number): Promise<RuntimeBlobMetadataBackfillResult>
|
||||
@@ -135,13 +155,17 @@ export declare class StorageRuntime {
|
||||
executeDocumentCleanupCandidates(workspaceId: string | undefined | null, gracePeriodDays: number, limit: number): Promise<RuntimeDocumentCleanupExecuteResult>
|
||||
deleteWorkspaceObjects(workspaceId: string, userIds?: Array<string> | undefined | null): Promise<number>
|
||||
reconcileWorkspaceStorage(limit: number): Promise<RuntimeWorkspaceStorageReconcileResult>
|
||||
constructor()
|
||||
constructor(serverConfig: ServerConfigHandle)
|
||||
start(): Promise<void>
|
||||
configure(configJson: string): void
|
||||
reloadConfig(): Promise<void>
|
||||
stop(): Promise<void>
|
||||
runMigrations(): Promise<void>
|
||||
health(): Promise<StorageRuntimeHealth>
|
||||
providerCapabilities(scope: string): Promise<StorageProviderCapabilities>
|
||||
isLocalStorage(scope: string): boolean
|
||||
uploadUrlConfig(scope: string): StorageUploadUrlConfig | null
|
||||
signUploadToken(scope: string, path: string, fields: Array<any>, expiresAt: number): string | null
|
||||
verifyUploadToken(scope: string, path: string, fields: Array<any>, expiresAt: number, token: string): boolean
|
||||
putObject(scope: string, key: string, body: Buffer, metadata?: RuntimeObjectStoragePutOptions | undefined | null): Promise<RuntimeObjectMetadata>
|
||||
headObject(scope: string, key: string): Promise<RuntimeObjectMetadata | null>
|
||||
getObject(scope: string, key: string): Promise<RuntimeObjectGetResult | null>
|
||||
@@ -187,12 +211,22 @@ export interface AggregateOptions {
|
||||
pagination: SearchPagination
|
||||
}
|
||||
|
||||
export interface AppConfigCommand {
|
||||
key: string
|
||||
owner: string
|
||||
operation: string
|
||||
valueJson?: string
|
||||
}
|
||||
|
||||
export interface AppConfigDescriptor {
|
||||
key: string
|
||||
description: string
|
||||
defaultValue: any
|
||||
schema: any
|
||||
internal: boolean
|
||||
envName?: string
|
||||
envType?: string
|
||||
link?: string
|
||||
}
|
||||
|
||||
export declare function appConfigDescriptors(module: string): Array<AppConfigDescriptor>
|
||||
@@ -1768,6 +1802,11 @@ export interface StorageRuntimeHealth {
|
||||
bucket?: string
|
||||
}
|
||||
|
||||
export interface StorageUploadUrlConfig {
|
||||
proxyUpload: boolean
|
||||
urlPrefix?: string
|
||||
}
|
||||
|
||||
export interface SyncEmbeddingStateInput {
|
||||
workspaceId: string
|
||||
enabled: boolean
|
||||
|
||||
@@ -330,7 +330,11 @@ impl ArtifactService {
|
||||
.await
|
||||
.map_err(|error| RuntimeError::database("complete workspace artifact failed", error))?;
|
||||
if updated.rows_affected() != 1 {
|
||||
return Err(RuntimeError::invalid_state("artifact_reservation_changed"));
|
||||
let row = self.get(workspace_id, content_hash).await?;
|
||||
if row.status != "ready" || row.storage_scope != locator.scope.as_str() || row.storage_key != locator.key.as_str()
|
||||
{
|
||||
return Err(RuntimeError::invalid_state("artifact_reservation_changed"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -18,9 +18,13 @@ struct StoredKey {
|
||||
id: String,
|
||||
secret: String,
|
||||
status: String,
|
||||
// TODO(0.27.5): Remove old-reader serialization constraints after 0.27.4 exits.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
created_at: Option<DateTime<Utc>>,
|
||||
source: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
retired_at: Option<DateTime<Utc>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
verify_until: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
|
||||
@@ -703,6 +703,21 @@ async fn session_kernel_serializes_refresh_replay_revocation_and_key_rotation()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
keyring::rotate(&pool, &config, &user_id, &old_key).await.unwrap();
|
||||
let stored_after_rotation: serde_json::Value =
|
||||
sqlx::query_scalar("SELECT value FROM app_configs WHERE id='auth.session.signingKeys'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let active = stored_after_rotation
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|key| key["status"] == "active")
|
||||
.unwrap()
|
||||
.as_object()
|
||||
.unwrap();
|
||||
assert!(!active.contains_key("retiredAt"));
|
||||
assert!(!active.contains_key("verifyUntil"));
|
||||
let old_access = rotated["accessToken"].as_str().unwrap();
|
||||
let old_principal = principal::resolve(
|
||||
&pool,
|
||||
|
||||
@@ -380,8 +380,15 @@ async fn doc_append_invalidation_fences_inflight_refresh_and_stamp_covers_redis_
|
||||
let publisher = InvalidationRuntime::start(&config, false, Arc::new(NoopInvalidationTarget)).await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
|
||||
let redis_update =
|
||||
affine_doc_loader::update_doc(&initial_blob, "", &fixture.doc_id).unwrap();
|
||||
let initial_markdown = affine_doc_loader::export_markdown_source(&initial_blob, &fixture.doc_id, None)
|
||||
.unwrap()
|
||||
.markdown;
|
||||
let redis_update = affine_doc_loader::update_doc(
|
||||
&initial_blob,
|
||||
&format!("{initial_markdown}\n"),
|
||||
&fixture.doc_id,
|
||||
)
|
||||
.unwrap();
|
||||
append_updates(
|
||||
&fixture.pool,
|
||||
Some(publisher.clone()),
|
||||
@@ -413,8 +420,15 @@ async fn doc_append_invalidation_fences_inflight_refresh_and_stamp_covers_redis_
|
||||
.authorize_and_load(Some(&fixture.member_user_id), current(&fixture))
|
||||
.await
|
||||
.unwrap();
|
||||
let loss_update =
|
||||
affine_doc_loader::update_doc(¤t_source.blob, "", &fixture.doc_id).unwrap();
|
||||
let current_markdown = affine_doc_loader::export_markdown_source(¤t_source.blob, &fixture.doc_id, None)
|
||||
.unwrap()
|
||||
.markdown;
|
||||
let loss_update = affine_doc_loader::update_doc(
|
||||
¤t_source.blob,
|
||||
&format!("{current_markdown}\n"),
|
||||
&fixture.doc_id,
|
||||
)
|
||||
.unwrap();
|
||||
append_updates(
|
||||
&fixture.pool,
|
||||
None,
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
use napi::Result;
|
||||
|
||||
use super::{AppConfigChange, BackendRuntime, napi_error, save_app_config_changes, to_napi_error};
|
||||
|
||||
#[napi_derive::napi(object)]
|
||||
pub struct AppConfigCommand {
|
||||
pub key: String,
|
||||
pub owner: String,
|
||||
pub operation: String,
|
||||
pub value_json: Option<String>,
|
||||
}
|
||||
|
||||
#[napi_derive::napi]
|
||||
impl BackendRuntime {
|
||||
#[napi]
|
||||
pub async fn save_app_config(&self, actor: Option<String>, commands: Vec<AppConfigCommand>) -> Result<Vec<String>> {
|
||||
let changes = commands
|
||||
.into_iter()
|
||||
.map(|command| {
|
||||
let native = match command.owner.as_str() {
|
||||
"native" => true,
|
||||
"node" => false,
|
||||
_ => return Err(napi_error("app config owner must be native or node")),
|
||||
};
|
||||
let value = match command.operation.as_str() {
|
||||
"set" => Some(
|
||||
serde_json::from_str(
|
||||
command
|
||||
.value_json
|
||||
.as_deref()
|
||||
.ok_or_else(|| napi_error("app config set requires a value"))?,
|
||||
)
|
||||
.map_err(|_| napi_error("invalid app config JSON value"))?,
|
||||
),
|
||||
"clear" if command.value_json.is_none() => None,
|
||||
"clear" => return Err(napi_error("app config clear must not include a value")),
|
||||
_ => return Err(napi_error("app config operation must be set or clear")),
|
||||
};
|
||||
Ok(AppConfigChange {
|
||||
key: command.key,
|
||||
value,
|
||||
native,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
save_app_config_changes(
|
||||
&self.pool().await.map_err(to_napi_error)?,
|
||||
&self.server_config,
|
||||
self.bootstrap_private_key.as_ref().map(|key| key.to_string()),
|
||||
actor.as_deref(),
|
||||
&changes,
|
||||
)
|
||||
.await
|
||||
.map_err(to_napi_error)
|
||||
}
|
||||
}
|
||||
@@ -92,6 +92,21 @@ impl BackendRuntime {
|
||||
Ok(search_operation_output(result))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn search_enabled(&self) -> Result<bool> {
|
||||
Ok(self.config()?.search.enabled)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn copilot_enabled(&self) -> Result<bool> {
|
||||
Ok(self.config()?.copilot.enabled)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn node_crypto_private_key(&self) -> Result<String> {
|
||||
Ok(self.config()?.private_key.as_str().to_string())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn reconcile_search_projection(&self, limit: Option<i32>) -> Result<i32> {
|
||||
self.require_background()?;
|
||||
|
||||
@@ -2,7 +2,9 @@ use super::{
|
||||
license::{RuntimeLicenseInstallInput, RuntimeLicenseRefreshInput},
|
||||
*,
|
||||
};
|
||||
use crate::runtime::{backend_runtime::tests::runtime_from_database_url, migrations::DATABASE_TEST_LOCK};
|
||||
use crate::runtime::{
|
||||
BackendRuntimeConfig, backend_runtime::tests::runtime_from_database_url, migrations::DATABASE_TEST_LOCK,
|
||||
};
|
||||
|
||||
pub(super) async fn fixture(runtime: &BackendRuntime) -> (String, String) {
|
||||
let owner = uuid::Uuid::new_v4().to_string();
|
||||
@@ -171,9 +173,10 @@ async fn license_upgrade_recovers_installed_files_and_online_renewals_before_wor
|
||||
}
|
||||
}
|
||||
runtime.stop().await.unwrap();
|
||||
std::sync::Arc::get_mut(&mut runtime.config.write().unwrap())
|
||||
.unwrap()
|
||||
.deployment = Deployment::SelfHosted;
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let config_path = directory.path().join("config.json");
|
||||
std::fs::write(&config_path, r#"{"deployment":{"type":"selfhosted"}}"#).unwrap();
|
||||
runtime.server_config = std::sync::Arc::new(crate::runtime::config::ServerConfig::open(&config_path, None).unwrap());
|
||||
runtime.start().await.unwrap();
|
||||
assert!(runtime.license_health_worker.lock().await.is_none());
|
||||
assert_eq!(remote.requests().len(), 1);
|
||||
@@ -455,10 +458,9 @@ async fn license_install_revoke_rollback_and_stale_fences() {
|
||||
.unwrap();
|
||||
assert!(
|
||||
!runtime
|
||||
.get_byok_entitlement_v1(workspace.clone(), None)
|
||||
.has_workspace_commercial_entitlement_v1(workspace.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.server
|
||||
);
|
||||
let status: String = sqlx::query_scalar("SELECT status FROM entitlements WHERE subject_id=$1")
|
||||
.bind(&key)
|
||||
@@ -774,11 +776,30 @@ async fn authoritative_byok_and_mixed_owner_storage_use_current_facts() {
|
||||
assert!(access.local && access.server);
|
||||
assert!(
|
||||
!runtime
|
||||
.get_byok_entitlement_v1("missing".into(), Some(owner))
|
||||
.get_byok_entitlement_v1("missing".into(), Some(owner.clone()))
|
||||
.await
|
||||
.unwrap()
|
||||
.local
|
||||
);
|
||||
let cloud_config = runtime.config().unwrap();
|
||||
*runtime.config.write().unwrap() = std::sync::Arc::new(BackendRuntimeConfig {
|
||||
database_url: cloud_config.database_url.clone(),
|
||||
auth: cloud_config.auth.clone(),
|
||||
invite_quota: cloud_config.invite_quota.clone(),
|
||||
private_key: std::sync::Arc::clone(&cloud_config.private_key),
|
||||
deployment: Deployment::SelfHosted,
|
||||
copilot: cloud_config.copilot.clone(),
|
||||
search: cloud_config.search.clone(),
|
||||
redis: cloud_config.redis.clone(),
|
||||
payment: cloud_config.payment.clone(),
|
||||
});
|
||||
let access = runtime.get_byok_entitlement_v1(workspace.clone(), None).await.unwrap();
|
||||
assert!(access.server && access.local);
|
||||
let access = runtime
|
||||
.get_byok_entitlement_v1(workspace.clone(), Some(owner))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(access.server && access.local);
|
||||
sqlx::query("DELETE FROM workspace_members WHERE workspace_id=$1 AND role='owner'")
|
||||
.bind(&workspace)
|
||||
.execute(&pool)
|
||||
|
||||
@@ -4,8 +4,8 @@ use napi::Result;
|
||||
use sqlx::{Row, postgres::PgPoolOptions};
|
||||
|
||||
use super::{
|
||||
BackendRuntime, BackendRuntimeConfig, BackendRuntimeHealth, EmbeddingHealth, ObjectStorageService, RuntimeError,
|
||||
RuntimeResult, SearchRuntime, blob_access, embedding_schema_health, invalidation,
|
||||
BackendRuntime, BackendRuntimeConfig, BackendRuntimeHealth, EmbeddingHealth, RuntimeError, RuntimeResult,
|
||||
SearchRuntime, blob_access, embedding_schema_health, invalidation,
|
||||
invalidation::{InvalidationHintV1, InvalidationTarget},
|
||||
napi_error,
|
||||
payment::PaymentRuntime,
|
||||
@@ -14,16 +14,6 @@ use super::{
|
||||
|
||||
#[napi_derive::napi]
|
||||
impl BackendRuntime {
|
||||
#[napi]
|
||||
pub fn configure_object_storage(&self, config_json: String) -> Result<()> {
|
||||
let object_storage = ObjectStorageService::from_config_json(&config_json).map_err(to_napi_error)?;
|
||||
*self
|
||||
.object_storage
|
||||
.write()
|
||||
.map_err(|_| napi_error("object storage service lock poisoned"))? = Arc::new(object_storage);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn start(&self) -> Result<()> {
|
||||
self.start_inner().await.map_err(to_napi_error)
|
||||
@@ -50,14 +40,12 @@ impl BackendRuntime {
|
||||
.map_err(|err| RuntimeError::database("BackendRuntime postgres health check failed", err))?;
|
||||
|
||||
let initialized = async {
|
||||
let inline_config = self
|
||||
.inline_config
|
||||
.read()
|
||||
.map_err(|_| RuntimeError::invalid_state("BackendRuntime inline config lock poisoned"))?
|
||||
.clone();
|
||||
let config = self
|
||||
.config()?
|
||||
.with_db_overrides(&pool, &self.config_source, inline_config.as_ref())
|
||||
let baseline = BackendRuntimeConfig::from_server_config(
|
||||
self.bootstrap_private_key.as_ref().map(|key| key.to_string()),
|
||||
&self.server_config,
|
||||
)?;
|
||||
let config = baseline
|
||||
.with_db_overrides_from_server_config(&pool, &self.server_config)
|
||||
.await?;
|
||||
let deployment = config.deployment;
|
||||
let redis = config.redis.clone();
|
||||
@@ -77,7 +65,8 @@ impl BackendRuntime {
|
||||
None
|
||||
};
|
||||
*self.payment.lock().await = payment;
|
||||
let object_storage = self.object_storage()?.with_db_overrides(&pool).await?;
|
||||
let storage_baseline = self.server_config.object_storage().clone();
|
||||
let object_storage = storage_baseline.with_db_overrides(&pool).await?;
|
||||
*self
|
||||
.object_storage
|
||||
.write()
|
||||
@@ -231,36 +220,24 @@ impl BackendRuntime {
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn reload_config(
|
||||
&self,
|
||||
private_key: Option<String>,
|
||||
object_storage_config: Option<String>,
|
||||
inline_config: Option<String>,
|
||||
) -> Result<()> {
|
||||
pub async fn reload_config(&self) -> Result<()> {
|
||||
let _reload = self.config_reload.lock().await;
|
||||
let pool = self.pool().await.map_err(to_napi_error)?;
|
||||
let active_private_key = self.config().map_err(to_napi_error)?.private_key.to_string();
|
||||
let inline_config = inline_config
|
||||
.map(|value| serde_json::from_str(&value))
|
||||
.transpose()
|
||||
.map_err(|error| to_napi_error(RuntimeError::json("decode inline runtime config", error)))?
|
||||
.or_else(|| self.inline_config.read().ok().and_then(|value| value.clone()));
|
||||
let config = BackendRuntimeConfig::from_config_source_with_inline(
|
||||
private_key.or(Some(active_private_key)),
|
||||
&self.config_source,
|
||||
inline_config.as_ref(),
|
||||
let config = BackendRuntimeConfig::from_server_config(
|
||||
self.bootstrap_private_key.as_ref().map(|key| key.to_string()),
|
||||
&self.server_config,
|
||||
)
|
||||
.map_err(to_napi_error)?
|
||||
.with_db_overrides(&pool, &self.config_source, inline_config.as_ref())
|
||||
.await
|
||||
.map_err(to_napi_error)?;
|
||||
let object_storage = match object_storage_config {
|
||||
Some(config) => ObjectStorageService::from_config_json(&config).map_err(to_napi_error)?,
|
||||
None => self.object_storage().map_err(to_napi_error)?.as_ref().clone(),
|
||||
}
|
||||
.with_db_overrides(&pool)
|
||||
.with_db_overrides_from_server_config(&pool, &self.server_config)
|
||||
.await
|
||||
.map_err(to_napi_error)?;
|
||||
let object_storage = self
|
||||
.server_config
|
||||
.object_storage()
|
||||
.clone()
|
||||
.with_db_overrides(&pool)
|
||||
.await
|
||||
.map_err(to_napi_error)?;
|
||||
let search = if !self.script_mode && config.search.enabled {
|
||||
if config.search.provider == "embedded" && !self.role.allows_embedded_search() {
|
||||
return Err(napi_error(format!(
|
||||
@@ -314,11 +291,6 @@ impl BackendRuntime {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
*self
|
||||
.inline_config
|
||||
.write()
|
||||
.map_err(|_| napi_error("BackendRuntime inline config lock poisoned"))? = inline_config;
|
||||
|
||||
let embedding = self.embedding.lock().await.as_ref().cloned();
|
||||
if let Some(embedding) = embedding {
|
||||
embedding
|
||||
|
||||
@@ -4,6 +4,7 @@ mod auth_session;
|
||||
mod blob_access;
|
||||
mod byok;
|
||||
mod byok_api;
|
||||
mod config_command;
|
||||
mod constants;
|
||||
mod control_plane;
|
||||
mod copilot;
|
||||
@@ -44,6 +45,7 @@ use search::SearchRuntime;
|
||||
use sha2::{Digest, Sha256};
|
||||
use sqlx::{PgPool, Row};
|
||||
use tokio::sync::Mutex;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use self::{
|
||||
blob_access::SourceIdentity,
|
||||
@@ -59,14 +61,17 @@ use self::{
|
||||
use super::object_storage::ObjectStorageService;
|
||||
pub(crate) use super::types;
|
||||
pub(super) use super::{
|
||||
BackendRuntimeConfig, ConfigSource, InviteQuotaConfig, RedisRuntimeConfig, RuntimeError, RuntimeResult,
|
||||
BackendRuntimeConfig, InviteQuotaConfig, RedisRuntimeConfig, RuntimeError, RuntimeResult,
|
||||
migrations::{embedding_schema_health, migrate_all_tables},
|
||||
napi_error, to_napi_error, webpki_tls_config,
|
||||
};
|
||||
use crate::llm::{
|
||||
ByokLocalLeaseOutput, ByokPolicyOutput, ByokProbeResultOutput, ByokProfileOutput, CreateByokLocalLeaseInput,
|
||||
CreateByokProfileInput, ProbeByokDraftInput, ProbeByokProfileInput, ReorderByokProfilesInput,
|
||||
ReplaceByokProfileInput, RotateByokCredentialInput,
|
||||
use crate::{
|
||||
llm::{
|
||||
ByokLocalLeaseOutput, ByokPolicyOutput, ByokProbeResultOutput, ByokProfileOutput, CreateByokLocalLeaseInput,
|
||||
CreateByokProfileInput, ProbeByokDraftInput, ProbeByokProfileInput, ReorderByokProfilesInput,
|
||||
ReplaceByokProfileInput, RotateByokCredentialInput,
|
||||
},
|
||||
runtime::config::{AppConfigChange, ServerConfig, ServerConfigHandle, save_app_config_changes},
|
||||
};
|
||||
|
||||
pub(super) fn token_hash(token: &str) -> String {
|
||||
@@ -104,8 +109,8 @@ fn search_operation_output(result: RuntimeResult<serde_json::Value>) -> SearchOp
|
||||
#[derive(Clone)]
|
||||
#[napi_derive::napi]
|
||||
pub struct BackendRuntime {
|
||||
config_source: ConfigSource,
|
||||
inline_config: Arc<RwLock<Option<serde_json::Value>>>,
|
||||
server_config: Arc<ServerConfig>,
|
||||
bootstrap_private_key: Option<Arc<Zeroizing<String>>>,
|
||||
role: ServerRole,
|
||||
script_mode: bool,
|
||||
config: Arc<RwLock<Arc<BackendRuntimeConfig>>>,
|
||||
@@ -130,25 +135,18 @@ pub struct BackendRuntime {
|
||||
impl BackendRuntime {
|
||||
#[napi(constructor)]
|
||||
pub fn new(
|
||||
server_config: &ServerConfigHandle,
|
||||
private_key: Option<String>,
|
||||
config_paths: Option<Vec<String>>,
|
||||
permission_telemetry: Option<ThreadsafeFunction<String, (), String, Status, true, true, 1024>>,
|
||||
inline_config: Option<String>,
|
||||
invalidation_events: Option<ThreadsafeFunction<String, (), String, Status, true, true, 1024>>,
|
||||
) -> Result<Self> {
|
||||
let config_source = ConfigSource::new(config_paths);
|
||||
let (role, script_mode) = ServerRole::from_environment().map_err(napi_error)?;
|
||||
let inline_config = inline_config
|
||||
.map(|value| serde_json::from_str(&value))
|
||||
.transpose()
|
||||
.map_err(|error| to_napi_error(RuntimeError::json("decode inline runtime config", error)))?;
|
||||
let config =
|
||||
BackendRuntimeConfig::from_config_source_with_inline(private_key, &config_source, inline_config.as_ref())
|
||||
.map_err(to_napi_error)?;
|
||||
let object_storage = ObjectStorageService::from_config_source(&config_source).map_err(to_napi_error)?;
|
||||
BackendRuntimeConfig::from_server_config(private_key.clone(), &server_config.inner).map_err(to_napi_error)?;
|
||||
let object_storage = server_config.inner.object_storage().clone();
|
||||
Ok(Self {
|
||||
config_source,
|
||||
inline_config: Arc::new(RwLock::new(inline_config)),
|
||||
server_config: Arc::clone(&server_config.inner),
|
||||
bootstrap_private_key: private_key.map(|key| Arc::new(Zeroizing::new(key))),
|
||||
role,
|
||||
script_mode,
|
||||
config: Arc::new(RwLock::new(Arc::new(config))),
|
||||
|
||||
@@ -48,6 +48,7 @@ impl PaymentRuntime {
|
||||
.get::<Option<String>, _>("validate_key")
|
||||
.unwrap_or_else(|| operation_id.to_string());
|
||||
let envelope = issue_license(
|
||||
self.license_issuer_private_key.as_ref().map(|key| key.as_str()),
|
||||
license_key,
|
||||
workspace_id,
|
||||
subscription.quantity,
|
||||
@@ -145,6 +146,7 @@ impl PaymentRuntime {
|
||||
}
|
||||
let subscription = active_license_subscription(&mut tx, license_key, &namespace).await?;
|
||||
let envelope = issue_license(
|
||||
self.license_issuer_private_key.as_ref().map(|key| key.as_str()),
|
||||
license_key,
|
||||
workspace_id,
|
||||
subscription.quantity,
|
||||
@@ -304,14 +306,14 @@ fn decode_subscription(row: sqlx::postgres::PgRow) -> RuntimeResult<ActiveLicens
|
||||
}
|
||||
|
||||
fn issue_license(
|
||||
private_key: Option<&str>,
|
||||
license_id: &str,
|
||||
workspace_id: &str,
|
||||
seat_quantity: i32,
|
||||
subscription_end: Option<DateTime<Utc>>,
|
||||
now: DateTime<Utc>,
|
||||
) -> RuntimeResult<Vec<u8>> {
|
||||
let private_key = std::env::var("AFFINE_PRO_LICENSE_PRIVATE_KEY")
|
||||
.map_err(|_| RuntimeError::invalid_state("license_private_key_missing"))?;
|
||||
let private_key = private_key.ok_or_else(|| RuntimeError::invalid_state("license_private_key_missing"))?;
|
||||
LicenseIssuer::issue(
|
||||
LicenseIssuance {
|
||||
license_id,
|
||||
@@ -320,7 +322,7 @@ fn issue_license(
|
||||
subscription_end,
|
||||
now,
|
||||
},
|
||||
&private_key,
|
||||
private_key,
|
||||
)
|
||||
.map_err(|_| RuntimeError::invalid_state("license_issue_failed"))
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
use sqlx::{PgPool, postgres::PgPoolOptions};
|
||||
use tokio::sync::{Mutex, Semaphore};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::{
|
||||
super::{BackendRuntime, to_napi_error},
|
||||
@@ -22,6 +23,7 @@ pub(in crate::runtime::backend_runtime) struct PaymentRuntime {
|
||||
pub(super) deployment: Deployment,
|
||||
pub(super) revenuecat_config: Option<crate::runtime::RevenueCatRuntimeConfig>,
|
||||
pub(super) mail_hash_key: [u8; 32],
|
||||
pub(super) license_issuer_private_key: Option<Arc<Zeroizing<String>>>,
|
||||
pub(super) worker: Mutex<Option<super::PaymentWorker>>,
|
||||
}
|
||||
|
||||
@@ -51,6 +53,9 @@ impl PaymentRuntime {
|
||||
deployment,
|
||||
revenuecat_config: config.revenuecat.clone(),
|
||||
mail_hash_key: Sha256::digest(private_key.as_bytes()).into(),
|
||||
license_issuer_private_key: std::env::var("AFFINE_PRO_LICENSE_PRIVATE_KEY")
|
||||
.ok()
|
||||
.map(|key| Arc::new(Zeroizing::new(key))),
|
||||
worker: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
@@ -223,6 +228,24 @@ impl PaymentRuntime {
|
||||
|
||||
#[napi_derive::napi]
|
||||
impl BackendRuntime {
|
||||
#[napi]
|
||||
pub fn payment_enabled(&self) -> napi::Result<bool> {
|
||||
let config = self
|
||||
.config
|
||||
.read()
|
||||
.map_err(|_| napi::Error::from_reason("BackendRuntime config lock poisoned"))?;
|
||||
Ok(config.payment.enabled)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn stripe_enabled(&self) -> napi::Result<bool> {
|
||||
let config = self
|
||||
.config
|
||||
.read()
|
||||
.map_err(|_| napi::Error::from_reason("BackendRuntime config lock poisoned"))?;
|
||||
Ok(config.payment.enabled && config.payment.stripe.is_some())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn execute_payment_command_v1(&self, input: Value) -> napi::Result<Value> {
|
||||
let outcome = self
|
||||
|
||||
@@ -777,6 +777,7 @@ async fn lock_set_expansion_discards_old_snapshots() {
|
||||
deployment: Deployment::Cloud,
|
||||
revenuecat_config: None,
|
||||
mail_hash_key: [0; 32],
|
||||
license_issuer_private_key: None,
|
||||
worker: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
runtime.apply_with_connection(connection, complete).await.unwrap();
|
||||
@@ -981,6 +982,7 @@ async fn operation_intent_is_frozen_before_send_and_blocks_overlapping_work() {
|
||||
deployment: Deployment::Cloud,
|
||||
revenuecat_config: None,
|
||||
mail_hash_key: [0; 32],
|
||||
license_issuer_private_key: None,
|
||||
worker: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
assert_eq!(
|
||||
@@ -1086,6 +1088,7 @@ async fn operation_intent_is_frozen_before_send_and_blocks_overlapping_work() {
|
||||
deployment: Deployment::Cloud,
|
||||
revenuecat_config: None,
|
||||
mail_hash_key: [0; 32],
|
||||
license_issuer_private_key: None,
|
||||
worker: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
let provisioned = catalog_runtime
|
||||
@@ -1223,6 +1226,7 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
|
||||
deployment: Deployment::Cloud,
|
||||
revenuecat_config: None,
|
||||
mail_hash_key: [0; 32],
|
||||
license_issuer_private_key: None,
|
||||
worker: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
let raw_event = json!({
|
||||
@@ -1774,6 +1778,7 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
|
||||
deployment: Deployment::Cloud,
|
||||
revenuecat_config: None,
|
||||
mail_hash_key: [0; 32],
|
||||
license_issuer_private_key: None,
|
||||
worker: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
assert!(
|
||||
@@ -1986,15 +1991,11 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
|
||||
deployment: Deployment::Cloud,
|
||||
revenuecat_config: None,
|
||||
mail_hash_key: [0; 32],
|
||||
license_issuer_private_key: Some(Arc::new(zeroize::Zeroizing::new(
|
||||
crate::entitlement::tests::TEST_PRIVATE_KEY.to_string(),
|
||||
))),
|
||||
worker: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
let previous_private_key = std::env::var_os("AFFINE_PRO_LICENSE_PRIVATE_KEY");
|
||||
unsafe {
|
||||
std::env::set_var(
|
||||
"AFFINE_PRO_LICENSE_PRIVATE_KEY",
|
||||
crate::entitlement::tests::TEST_PRIVATE_KEY,
|
||||
);
|
||||
}
|
||||
for legacy in [true, false] {
|
||||
let license_key = format!("rfc12-{license_marker}-{legacy}");
|
||||
let license_source = format!("sub-{license_marker}-{legacy}");
|
||||
@@ -2196,13 +2197,6 @@ async fn snapshot_commit_is_atomic_and_transfer_moves_the_entitlement() {
|
||||
.unwrap();
|
||||
assert_eq!(binding, (Some("new-workspace".into()), Some(new_generation)));
|
||||
}
|
||||
unsafe {
|
||||
if let Some(previous_private_key) = previous_private_key {
|
||||
std::env::set_var("AFFINE_PRO_LICENSE_PRIVATE_KEY", previous_private_key);
|
||||
} else {
|
||||
std::env::remove_var("AFFINE_PRO_LICENSE_PRIVATE_KEY");
|
||||
}
|
||||
}
|
||||
cleanup(&pool, &license_namespace_key, &license_marker).await;
|
||||
}
|
||||
|
||||
@@ -3210,6 +3204,7 @@ async fn revenuecat_access_and_missing_source_rules_fail_closed() {
|
||||
deployment: Deployment::Cloud,
|
||||
revenuecat_config: Some(config),
|
||||
mail_hash_key: [0; 32],
|
||||
license_issuer_private_key: None,
|
||||
worker: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
let revenuecat_event = json!({
|
||||
|
||||
@@ -124,6 +124,8 @@ pub(super) fn final_storage_locator(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
@@ -132,13 +134,19 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn promotes_valid_blob_and_rejects_late_overwrite() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let runtime = BackendRuntime::new(None, None, None, None, None).unwrap();
|
||||
runtime
|
||||
.configure_object_storage(format!(
|
||||
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"promotion","config":{{"path":{}}}}}}}}}"#,
|
||||
let config_path = temp.path().join("config.json");
|
||||
std::fs::write(
|
||||
&config_path,
|
||||
format!(
|
||||
r#"{{"deployment":{{"type":"cloud"}},"storages":{{"blob.storage":{{"provider":"fs","bucket":"promotion","config":{{"path":{}}}}}}}}}"#,
|
||||
serde_json::to_string(temp.path()).unwrap()
|
||||
))
|
||||
.unwrap();
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let config = crate::runtime::config::ServerConfigHandle {
|
||||
inner: Arc::new(crate::runtime::config::ServerConfig::open(&config_path, None).unwrap()),
|
||||
};
|
||||
let runtime = BackendRuntime::new(&config, None, None, None).unwrap();
|
||||
let body = b"immutable blob".to_vec();
|
||||
let key = URL_SAFE_NO_PAD.encode(Sha256::digest(&body));
|
||||
let reservation_id = Uuid::new_v4();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use anyhow::{Context, Result as AnyResult, anyhow};
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use anyhow::{Context, Result as AnyResult, anyhow, ensure};
|
||||
use sqlx::{Row, postgres::PgPoolOptions};
|
||||
|
||||
use super::{
|
||||
super::migrations::{RUNTIME_MIGRATIONS, migrate_runtime_tables},
|
||||
@@ -10,6 +10,167 @@ pub(super) fn pg_test_lock() -> &'static tokio::sync::Mutex<()> {
|
||||
&crate::runtime::migrations::DATABASE_TEST_LOCK
|
||||
}
|
||||
|
||||
fn test_server_config() -> AnyResult<Arc<ServerConfig>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let path = directory.path().join("config.json");
|
||||
std::fs::write(&path, r#"{"deployment":{"type":"cloud"}}"#)?;
|
||||
Ok(Arc::new(
|
||||
ServerConfig::open(&path, None).map_err(|error| anyhow!(error.to_string()))?,
|
||||
))
|
||||
}
|
||||
|
||||
fn configure_test_storage(runtime: &BackendRuntime, bucket: &str, path: &std::path::Path) {
|
||||
let config = serde_json::json!({
|
||||
"storages": {
|
||||
"blob.storage": {
|
||||
"provider": "fs",
|
||||
"bucket": bucket,
|
||||
"config": { "path": path.to_string_lossy() }
|
||||
}
|
||||
}
|
||||
});
|
||||
let storage = ObjectStorageService::from_config_json(&config.to_string()).unwrap();
|
||||
*runtime.object_storage.write().unwrap() = Arc::new(storage);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shared_config_file_database_schema_and_runtime_harness() -> AnyResult<()> {
|
||||
let _guard = pg_test_lock().lock().await;
|
||||
let Ok(database_url) = std::env::var("DATABASE_URL") else {
|
||||
return Ok(());
|
||||
};
|
||||
let directory = tempfile::tempdir()?;
|
||||
let config_path = directory.path().join("config.json");
|
||||
let file_storage_path = directory.path().join("file-storage");
|
||||
let db_storage_path = directory.path().join("db-storage");
|
||||
std::fs::write(
|
||||
&config_path,
|
||||
serde_json::json!({
|
||||
"deployment": { "type": "cloud" },
|
||||
"copilot": { "enabled": true },
|
||||
"storages": {
|
||||
"blob.storage": {
|
||||
"provider": "fs",
|
||||
"bucket": "file-bucket",
|
||||
"config": { "path": file_storage_path }
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
)?;
|
||||
let handle = ServerConfigHandle::new(config_path.to_string_lossy().into_owned(), None)?;
|
||||
ensure!(handle.inner.path() == config_path.canonicalize()?);
|
||||
ensure!(
|
||||
crate::runtime::app_config_descriptors("copilot".to_string())?
|
||||
.iter()
|
||||
.any(|descriptor| descriptor.key == "enabled" && descriptor.default_value == serde_json::json!(false))
|
||||
);
|
||||
ensure!(
|
||||
crate::runtime::validate_app_config_value("copilot".to_string(), "enabled".to_string(), serde_json::json!(true))?
|
||||
.is_empty()
|
||||
);
|
||||
ensure!(
|
||||
!crate::runtime::validate_app_config_value(
|
||||
"copilot".to_string(),
|
||||
"enabled".to_string(),
|
||||
serde_json::json!("true")
|
||||
)?
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let pool = PgPoolOptions::new().max_connections(2).connect(&database_url).await?;
|
||||
let keys = ["copilot.enabled", "indexer.enabled", "storages.blob.storage"];
|
||||
let originals =
|
||||
sqlx::query("SELECT id, value, created_at, updated_at, last_updated_by FROM app_configs WHERE id = ANY($1)")
|
||||
.bind(keys)
|
||||
.fetch_all(&pool)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
(
|
||||
row.get::<String, _>("id"),
|
||||
row.get::<serde_json::Value, _>("value"),
|
||||
row.get::<chrono::DateTime<chrono::Utc>, _>("created_at"),
|
||||
row.get::<chrono::DateTime<chrono::Utc>, _>("updated_at"),
|
||||
row.get::<Option<String>, _>("last_updated_by"),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for (key, value) in [
|
||||
("copilot.enabled", serde_json::json!(false)),
|
||||
("indexer.enabled", serde_json::json!(false)),
|
||||
(
|
||||
"storages.blob.storage",
|
||||
serde_json::json!({
|
||||
"provider": "fs",
|
||||
"bucket": "db-bucket",
|
||||
"config": { "path": db_storage_path }
|
||||
}),
|
||||
),
|
||||
] {
|
||||
sqlx::query(
|
||||
"INSERT INTO app_configs(id,value,created_at,updated_at) VALUES($1,$2,now(),now()) ON CONFLICT(id) DO UPDATE \
|
||||
SET value=$2,updated_at=now()",
|
||||
)
|
||||
.bind(key)
|
||||
.bind(value)
|
||||
.execute(&pool)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let mut runtime = BackendRuntime::new(&handle, Some("test-private-key".to_string()), None, None)?;
|
||||
runtime.role = ServerRole::Frontend;
|
||||
let storage = crate::runtime::storage_runtime::StorageRuntime::new(&handle)?;
|
||||
let result: AnyResult<()> = async {
|
||||
runtime.start().await.map_err(|error| anyhow!(error.to_string()))?;
|
||||
storage.start().await.map_err(|error| anyhow!(error.to_string()))?;
|
||||
let backend_health = runtime.health().await?;
|
||||
ensure!(backend_health.started && backend_health.database_connected);
|
||||
ensure!(!runtime.config()?.copilot.enabled);
|
||||
let storage_health = storage.health().await?;
|
||||
ensure!(storage_health.started && storage_health.database_connected);
|
||||
ensure!(storage_health.bucket.as_deref() == Some("db-bucket"));
|
||||
|
||||
sqlx::query("DELETE FROM app_configs WHERE id = ANY($1)")
|
||||
.bind(keys)
|
||||
.execute(&pool)
|
||||
.await?;
|
||||
runtime
|
||||
.reload_config()
|
||||
.await
|
||||
.map_err(|error| anyhow!(error.to_string()))?;
|
||||
storage
|
||||
.reload_config()
|
||||
.await
|
||||
.map_err(|error| anyhow!(error.to_string()))?;
|
||||
ensure!(runtime.config()?.copilot.enabled);
|
||||
ensure!(storage.health().await?.bucket.as_deref() == Some("file-bucket"));
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
let storage_stop = storage.stop().await;
|
||||
let runtime_stop = runtime.stop().await;
|
||||
sqlx::query("DELETE FROM app_configs WHERE id = ANY($1)")
|
||||
.bind(keys)
|
||||
.execute(&pool)
|
||||
.await?;
|
||||
for (key, value, created_at, updated_at, last_updated_by) in originals {
|
||||
sqlx::query("INSERT INTO app_configs(id,value,created_at,updated_at,last_updated_by) VALUES($1,$2,$3,$4,$5)")
|
||||
.bind(key)
|
||||
.bind(value)
|
||||
.bind(created_at)
|
||||
.bind(updated_at)
|
||||
.bind(last_updated_by)
|
||||
.execute(&pool)
|
||||
.await?;
|
||||
}
|
||||
pool.close().await;
|
||||
result?;
|
||||
storage_stop.map_err(|error| anyhow!(error.to_string()))?;
|
||||
runtime_stop.map_err(|error| anyhow!(error.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrations_include_runtime_tables_without_worker_heartbeats() {
|
||||
assert!(RUNTIME_MIGRATIONS.contains("runtime_states"));
|
||||
@@ -57,25 +218,13 @@ async fn failed_start_rolls_back_resources_and_can_restart() -> AnyResult<()> {
|
||||
pool.close().await;
|
||||
}
|
||||
runtime.role = ServerRole::Frontend;
|
||||
{
|
||||
let mut config = runtime.config.write().unwrap();
|
||||
let current = config.as_ref();
|
||||
*config = Arc::new(BackendRuntimeConfig {
|
||||
database_url: current.database_url.clone(),
|
||||
auth: current.auth.clone(),
|
||||
invite_quota: current.invite_quota.clone(),
|
||||
private_key: Arc::clone(¤t.private_key),
|
||||
deployment: crate::runtime::Deployment::SelfHosted,
|
||||
copilot: current.copilot.clone(),
|
||||
search: crate::runtime::config::SearchRuntimeConfig {
|
||||
enabled: true,
|
||||
provider: "embedded".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
redis: current.redis.clone(),
|
||||
payment: current.payment.clone(),
|
||||
});
|
||||
}
|
||||
let directory = tempfile::tempdir()?;
|
||||
let config_path = directory.path().join("config.json");
|
||||
std::fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"selfhosted"},"indexer":{"enabled":true,"provider":{"type":"embedded"}}}"#,
|
||||
)?;
|
||||
runtime.server_config = Arc::new(ServerConfig::open(&config_path, None)?);
|
||||
|
||||
let error = runtime.start_inner().await.unwrap_err();
|
||||
assert!(error.to_string().contains("embedded search is only available"));
|
||||
@@ -87,21 +236,8 @@ async fn failed_start_rolls_back_resources_and_can_restart() -> AnyResult<()> {
|
||||
assert!(runtime.embedding.lock().await.is_none());
|
||||
assert!(runtime.embedding_worker.lock().await.is_none());
|
||||
|
||||
{
|
||||
let mut config = runtime.config.write().unwrap();
|
||||
let current = config.as_ref();
|
||||
*config = Arc::new(BackendRuntimeConfig {
|
||||
database_url: current.database_url.clone(),
|
||||
auth: current.auth.clone(),
|
||||
invite_quota: current.invite_quota.clone(),
|
||||
private_key: Arc::clone(¤t.private_key),
|
||||
deployment: current.deployment,
|
||||
copilot: current.copilot.clone(),
|
||||
search: Default::default(),
|
||||
redis: current.redis.clone(),
|
||||
payment: current.payment.clone(),
|
||||
});
|
||||
}
|
||||
std::fs::write(&config_path, r#"{"deployment":{"type":"selfhosted"}}"#)?;
|
||||
runtime.server_config = Arc::new(ServerConfig::open(&config_path, None)?);
|
||||
runtime.role = ServerRole::AllInOne;
|
||||
runtime
|
||||
.start_inner()
|
||||
@@ -128,12 +264,15 @@ async fn failed_config_reload_keeps_active_resources() -> AnyResult<()> {
|
||||
};
|
||||
runtime.role = ServerRole::Frontend;
|
||||
let config_path = std::env::temp_dir().join(format!("affine-runtime-config-{}.json", uuid::Uuid::new_v4()));
|
||||
std::fs::write(&config_path, r#"{"indexer":{"enabled":true,"provider":"embedded"}}"#)?;
|
||||
runtime.config_source = ConfigSource::new(Some(vec![config_path.to_string_lossy().into_owned()]));
|
||||
std::fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"cloud"},"indexer":{"enabled":true,"provider":{"type":"embedded"}}}"#,
|
||||
)?;
|
||||
runtime.server_config = Arc::new(ServerConfig::open(&config_path, None)?);
|
||||
let active_config = runtime.config()?;
|
||||
let active_storage = runtime.object_storage()?;
|
||||
|
||||
let result = runtime.reload_config(None, None, None).await;
|
||||
let result = runtime.reload_config().await;
|
||||
std::fs::remove_file(config_path)?;
|
||||
|
||||
assert!(result.is_err());
|
||||
@@ -209,9 +348,10 @@ pub(super) async fn runtime_from_database_url() -> AnyResult<Option<BackendRunti
|
||||
.await
|
||||
.context("cleanup invite abuse subjects for backend runtime tests")?;
|
||||
|
||||
let server_config = test_server_config()?;
|
||||
Ok(Some(BackendRuntime {
|
||||
config_source: Default::default(),
|
||||
inline_config: Arc::new(RwLock::new(None)),
|
||||
server_config: Arc::clone(&server_config),
|
||||
bootstrap_private_key: None,
|
||||
role: ServerRole::AllInOne,
|
||||
script_mode: false,
|
||||
config: Arc::new(RwLock::new(Arc::new(BackendRuntimeConfig {
|
||||
@@ -228,9 +368,7 @@ pub(super) async fn runtime_from_database_url() -> AnyResult<Option<BackendRunti
|
||||
config_reload: Arc::new(Mutex::new(())),
|
||||
pool: Arc::new(Mutex::new(Some(pool))),
|
||||
embedding_health: Arc::new(RwLock::new(super::EmbeddingHealth::disabled("test", None))),
|
||||
object_storage: Arc::new(RwLock::new(Arc::new(
|
||||
crate::runtime::object_storage::ObjectStorageService::from_config_files()?,
|
||||
))),
|
||||
object_storage: Arc::new(RwLock::new(Arc::new(server_config.object_storage().clone()))),
|
||||
embedding: Arc::new(Mutex::new(None)),
|
||||
embedding_worker: Arc::new(Mutex::new(None)),
|
||||
search: Arc::new(Mutex::new(None)),
|
||||
@@ -407,8 +545,8 @@ async fn runtime_gate_sql_semantics_are_atomic_and_ttl_bound() {
|
||||
let mut tasks = Vec::new();
|
||||
for _ in 0..16 {
|
||||
let runtime = BackendRuntime {
|
||||
config_source: Default::default(),
|
||||
inline_config: Arc::new(RwLock::new(None)),
|
||||
server_config: Arc::clone(&runtime.server_config),
|
||||
bootstrap_private_key: None,
|
||||
role: ServerRole::AllInOne,
|
||||
script_mode: false,
|
||||
config: Arc::new(RwLock::new(runtime.config().unwrap())),
|
||||
@@ -807,12 +945,7 @@ async fn strict_storage_reservation_serializes_last_bytes_and_recovers_ledger_ro
|
||||
};
|
||||
let pool = runtime.pool().await.unwrap();
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
runtime
|
||||
.configure_object_storage(format!(
|
||||
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"strict-storage","config":{{"path":{}}}}}}}}}"#,
|
||||
serde_json::to_string(temp.path()).unwrap()
|
||||
))
|
||||
.unwrap();
|
||||
configure_test_storage(&runtime, "strict-storage", temp.path());
|
||||
let user_id = "rust-test:strict-storage:user";
|
||||
let workspace_id = "rust-test:strict-storage:workspace";
|
||||
sqlx::query("DELETE FROM workspaces WHERE id=$1")
|
||||
@@ -1248,12 +1381,7 @@ async fn strict_blob_management_authorizes_inventory_and_denies_before_object_cl
|
||||
return;
|
||||
};
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
runtime
|
||||
.configure_object_storage(format!(
|
||||
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"blob-management","config":{{"path":{}}}}}}}}}"#,
|
||||
serde_json::to_string(temp.path()).unwrap()
|
||||
))
|
||||
.unwrap();
|
||||
configure_test_storage(&runtime, "blob-management", temp.path());
|
||||
let pool = runtime.pool().await.unwrap();
|
||||
let workspace_id = "rust-test:blob-management:workspace";
|
||||
let owner_id = "rust-test:blob-management:owner";
|
||||
@@ -1581,12 +1709,7 @@ async fn strict_comment_reservation_rejects_hijack_and_cleans_expired_generation
|
||||
.unwrap();
|
||||
}
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
runtime
|
||||
.configure_object_storage(format!(
|
||||
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"strict-comment","config":{{"path":{}}}}}}}}}"#,
|
||||
serde_json::to_string(temp.path()).unwrap()
|
||||
))
|
||||
.unwrap();
|
||||
configure_test_storage(&runtime, "strict-comment", temp.path());
|
||||
while runtime.cleanup_expired_storage_reservations_v1(100).await.unwrap() > 0 {}
|
||||
let input = |user_id: &str| types::RuntimeStorageReservationInput {
|
||||
workspace_id: workspace_id.to_string(),
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,918 @@
|
||||
use jsonschema::Draft;
|
||||
use napi::{Error, Result, Status};
|
||||
use schemars::generate::SchemaSettings;
|
||||
use serde_json::{Value, from_value, json, to_value};
|
||||
|
||||
use super::{
|
||||
AuthConfigFile, CopilotRuntimeConfig, CopilotRuntimeConfigFile, CryptoConfigFile, MANAGED_PROFILE_REQUIREMENTS,
|
||||
OAuthProviderConfigFile, PaymentRuntimeConfigFile, RedisRuntimeConfigFile, RuntimeError, SUPPORTED_BYOK_PROVIDERS,
|
||||
SearchRuntimeConfigFile, auth_limit, insert_flat_override, valid_search_endpoint, validate_copilot_config,
|
||||
};
|
||||
use crate::runtime::object_storage::{
|
||||
StorageBackendConfig, default_storage_provider_config, storage_provider_schema, types::StorageProviderConfig,
|
||||
};
|
||||
|
||||
const AUTH_MODULE: &str = "auth";
|
||||
const COPILOT_MODULE: &str = "copilot";
|
||||
const CRYPTO_MODULE: &str = "crypto";
|
||||
const INDEXER_MODULE: &str = "indexer";
|
||||
const OAUTH_MODULE: &str = "oauth";
|
||||
const PAYMENT_MODULE: &str = "payment";
|
||||
const REDIS_MODULE: &str = "redis";
|
||||
const STORAGES_MODULE: &str = "storages";
|
||||
|
||||
#[napi_derive::napi(object)]
|
||||
pub struct AppConfigDescriptor {
|
||||
pub key: String,
|
||||
pub description: String,
|
||||
pub default_value: Value,
|
||||
pub schema: Value,
|
||||
pub internal: bool,
|
||||
pub env_name: Option<String>,
|
||||
pub env_type: Option<String>,
|
||||
pub link: Option<String>,
|
||||
}
|
||||
|
||||
fn invalid_config(message: impl Into<String>) -> Error {
|
||||
Error::new(Status::InvalidArg, message.into())
|
||||
}
|
||||
|
||||
fn field<'a>(root: &'a Value, key: &str) -> &'a Value {
|
||||
let mut current = root;
|
||||
for segment in key.split('.') {
|
||||
if let Some(reference) = current
|
||||
.get("$ref")
|
||||
.or_else(|| current.pointer("/allOf/0/$ref"))
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
current = root
|
||||
.pointer(reference.strip_prefix('#').expect("local schema reference"))
|
||||
.expect("generated schema reference");
|
||||
}
|
||||
current = ¤t["properties"][segment];
|
||||
}
|
||||
current
|
||||
}
|
||||
|
||||
fn descriptors() -> Vec<AppConfigDescriptor> {
|
||||
let defaults = to_value(CopilotRuntimeConfigFile::default()).expect("copilot defaults should serialize");
|
||||
let root = to_value(
|
||||
SchemaSettings::draft07()
|
||||
.into_generator()
|
||||
.into_root_schema_for::<CopilotRuntimeConfigFile>(),
|
||||
)
|
||||
.expect("copilot schema should serialize");
|
||||
let definitions = root.get("$defs").or_else(|| root.get("definitions"));
|
||||
[
|
||||
(
|
||||
"enabled",
|
||||
"Enable AI features. Workspace owners configure provider keys in Workspace Settings → Integrations → AI BYOK.",
|
||||
false,
|
||||
),
|
||||
(
|
||||
"byok.enabled",
|
||||
"Allow workspace owners and admins to configure AI provider keys through AI BYOK.",
|
||||
false,
|
||||
),
|
||||
(
|
||||
"byok.allowedProviders",
|
||||
"AI providers that workspace owners and admins may add through AI BYOK.",
|
||||
false,
|
||||
),
|
||||
(
|
||||
"byok.allowCustomEndpoint",
|
||||
"Allow AI BYOK keys to use a custom provider endpoint.",
|
||||
false,
|
||||
),
|
||||
(
|
||||
"byok.allowPrivateEndpoint",
|
||||
"Whether workspace BYOK custom endpoints may resolve to private network targets. Enabling this allows workspace \
|
||||
owners and admins to send provider probe requests to the private network.",
|
||||
false,
|
||||
),
|
||||
("providers.profiles", "The profile list for copilot providers.", true),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(key, description, internal)| {
|
||||
let pointer = format!("/{}", key.replace('.', "/"));
|
||||
let mut schema = field(&root, key).clone();
|
||||
if let Some(definitions) = definitions {
|
||||
let name = if root.get("$defs").is_some() {
|
||||
"$defs"
|
||||
} else {
|
||||
"definitions"
|
||||
};
|
||||
schema[name] = definitions.clone();
|
||||
}
|
||||
if key == "byok.allowedProviders" {
|
||||
schema["items"]["enum"] = json!(SUPPORTED_BYOK_PROVIDERS);
|
||||
}
|
||||
if key == "providers.profiles" {
|
||||
let profile = &mut schema["definitions"]["CopilotManagedProfileConfigFile"];
|
||||
let requirements = MANAGED_PROFILE_REQUIREMENTS
|
||||
.iter()
|
||||
.map(|(provider, fields)| {
|
||||
let properties = fields
|
||||
.iter()
|
||||
.map(|field| ((*field).to_string(), json!({ "type": "string", "pattern": "\\S" })))
|
||||
.collect::<serde_json::Map<_, _>>();
|
||||
json!({
|
||||
"if": { "properties": { "type": { "const": provider } }, "required": ["type"] },
|
||||
"then": { "properties": { "config": { "required": fields, "properties": properties } } }
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
profile["allOf"] = json!([{
|
||||
"if": { "properties": { "enabled": { "const": false } }, "required": ["enabled"] },
|
||||
"else": { "allOf": requirements }
|
||||
}]);
|
||||
}
|
||||
AppConfigDescriptor {
|
||||
key: key.to_string(),
|
||||
description: description.to_string(),
|
||||
default_value: defaults.pointer(&pointer).expect("copilot default field").clone(),
|
||||
schema,
|
||||
internal,
|
||||
env_name: None,
|
||||
env_type: None,
|
||||
link: None,
|
||||
}
|
||||
})
|
||||
.chain(std::iter::once(storage_descriptor(
|
||||
"storage",
|
||||
"copilot",
|
||||
"The config for the storage provider.",
|
||||
)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn storage_descriptor(key: &str, bucket: &str, description: &str) -> AppConfigDescriptor {
|
||||
AppConfigDescriptor {
|
||||
key: key.to_string(),
|
||||
description: description.to_string(),
|
||||
default_value: default_storage_provider_config(bucket),
|
||||
schema: storage_provider_schema(),
|
||||
internal: false,
|
||||
env_name: None,
|
||||
env_type: None,
|
||||
link: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn storage_descriptors() -> Vec<AppConfigDescriptor> {
|
||||
vec![
|
||||
storage_descriptor("avatar.storage", "avatars", "The config of storage for user avatars."),
|
||||
storage_descriptor(
|
||||
"blob.storage",
|
||||
"blobs",
|
||||
"The config of storage for all uploaded blobs(images, videos, etc.).",
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn payment_descriptors() -> Vec<AppConfigDescriptor> {
|
||||
let defaults = to_value(PaymentRuntimeConfigFile::default()).expect("payment defaults should serialize");
|
||||
let root = to_value(
|
||||
SchemaSettings::draft07()
|
||||
.into_generator()
|
||||
.into_root_schema_for::<PaymentRuntimeConfigFile>(),
|
||||
)
|
||||
.expect("payment schema should serialize");
|
||||
let definitions = root.get("$defs").or_else(|| root.get("definitions"));
|
||||
[
|
||||
("enabled", "Whether to enable payment integration."),
|
||||
("stripe", "Stripe SDK options and credentials."),
|
||||
("revenuecat", "RevenueCat integration configuration."),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(key, description)| {
|
||||
let mut schema = field(&root, key).clone();
|
||||
if let Some(definitions) = definitions {
|
||||
let name = if root.get("$defs").is_some() {
|
||||
"$defs"
|
||||
} else {
|
||||
"definitions"
|
||||
};
|
||||
schema[name] = definitions.clone();
|
||||
}
|
||||
AppConfigDescriptor {
|
||||
key: key.to_string(),
|
||||
description: description.to_string(),
|
||||
default_value: defaults.get(key).expect("payment default field").clone(),
|
||||
schema,
|
||||
internal: false,
|
||||
env_name: None,
|
||||
env_type: None,
|
||||
link: None,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn auth_descriptors() -> Vec<AppConfigDescriptor> {
|
||||
let defaults = to_value(AuthConfigFile::default()).expect("auth defaults should serialize");
|
||||
let root = to_value(
|
||||
SchemaSettings::draft07()
|
||||
.into_generator()
|
||||
.into_root_schema_for::<AuthConfigFile>(),
|
||||
)
|
||||
.expect("auth schema should serialize");
|
||||
let definitions = root.get("$defs").or_else(|| root.get("definitions"));
|
||||
[
|
||||
("allowSignup", "Allow new registrations."),
|
||||
(
|
||||
"allowSignupForOauth",
|
||||
"Allow new registrations through configured OAuth providers.",
|
||||
),
|
||||
(
|
||||
"requireEmailDomainVerification",
|
||||
"Require email domain verification for restricted resources.",
|
||||
),
|
||||
(
|
||||
"newAccountActionDelay",
|
||||
"Minimum account age in seconds for invites and document publishing.",
|
||||
),
|
||||
("session.ttl", "Application auth expiration in seconds."),
|
||||
("session.ttr", "Application auth refresh interval in seconds."),
|
||||
("token.accessTokenTtl", "Access JWT expiration in seconds."),
|
||||
(
|
||||
"token.refreshIdleTtl",
|
||||
"Refresh session inactivity expiration in seconds.",
|
||||
),
|
||||
(
|
||||
"token.refreshAbsoluteTtl",
|
||||
"Refresh session absolute expiration in seconds.",
|
||||
),
|
||||
(
|
||||
"token.refreshGracePeriod",
|
||||
"Refresh rotation concurrency grace period in seconds.",
|
||||
),
|
||||
(
|
||||
"token.refreshRetention",
|
||||
"Expired refresh generation retention in seconds.",
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(key, description)| {
|
||||
let pointer = format!("/{}", key.replace('.', "/"));
|
||||
let mut schema = field(&root, key).clone();
|
||||
if let Some(definitions) = definitions {
|
||||
let name = if root.get("$defs").is_some() {
|
||||
"$defs"
|
||||
} else {
|
||||
"definitions"
|
||||
};
|
||||
schema[name] = definitions.clone();
|
||||
}
|
||||
if let Some((min, max)) = auth_limit(key) {
|
||||
schema["minimum"] = json!(min);
|
||||
if max != i64::MAX {
|
||||
schema["maximum"] = json!(max);
|
||||
}
|
||||
}
|
||||
AppConfigDescriptor {
|
||||
key: key.to_string(),
|
||||
description: description.to_string(),
|
||||
default_value: defaults.pointer(&pointer).expect("auth default field").clone(),
|
||||
schema,
|
||||
internal: false,
|
||||
env_name: None,
|
||||
env_type: None,
|
||||
link: None,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn indexer_descriptors() -> Vec<AppConfigDescriptor> {
|
||||
let defaults = to_value(SearchRuntimeConfigFile::default()).expect("indexer defaults should serialize");
|
||||
let root = to_value(
|
||||
SchemaSettings::draft07()
|
||||
.into_generator()
|
||||
.into_root_schema_for::<SearchRuntimeConfigFile>(),
|
||||
)
|
||||
.expect("indexer schema should serialize");
|
||||
let definitions = root.get("$defs").or_else(|| root.get("definitions"));
|
||||
[
|
||||
("enabled", "Enable the indexer."),
|
||||
("provider.type", "Search provider type."),
|
||||
("provider.endpoint", "Remote search provider endpoint."),
|
||||
("provider.apiKey", "Remote search provider API key."),
|
||||
("provider.username", "Remote search provider username."),
|
||||
("provider.password", "Remote search provider password."),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(key, description)| {
|
||||
let pointer = format!("/{}", key.replace('.', "/"));
|
||||
let mut schema = field(&root, key).clone();
|
||||
if let Some(definitions) = definitions {
|
||||
let name = if root.get("$defs").is_some() {
|
||||
"$defs"
|
||||
} else {
|
||||
"definitions"
|
||||
};
|
||||
schema[name] = definitions.clone();
|
||||
}
|
||||
AppConfigDescriptor {
|
||||
key: key.to_string(),
|
||||
description: description.to_string(),
|
||||
default_value: defaults.pointer(&pointer).expect("indexer default field").clone(),
|
||||
schema,
|
||||
internal: false,
|
||||
env_name: None,
|
||||
env_type: None,
|
||||
link: None,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn redis_descriptors() -> Vec<AppConfigDescriptor> {
|
||||
let defaults = to_value(RedisRuntimeConfigFile::default()).expect("Redis defaults should serialize");
|
||||
let root = to_value(
|
||||
SchemaSettings::draft07()
|
||||
.into_generator()
|
||||
.into_root_schema_for::<RedisRuntimeConfigFile>(),
|
||||
)
|
||||
.expect("Redis schema should serialize");
|
||||
[
|
||||
("host", "Redis host."),
|
||||
("port", "Redis port."),
|
||||
("db", "Redis database index."),
|
||||
("username", "Redis username."),
|
||||
("password", "Redis password."),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(key, description)| {
|
||||
let mut schema = field(&root, key).clone();
|
||||
if key == "port" {
|
||||
schema["minimum"] = json!(1);
|
||||
}
|
||||
if key == "db" {
|
||||
schema["maximum"] = json!(10);
|
||||
}
|
||||
AppConfigDescriptor {
|
||||
key: key.to_string(),
|
||||
description: description.to_string(),
|
||||
default_value: defaults.get(key).expect("Redis default field").clone(),
|
||||
schema,
|
||||
internal: false,
|
||||
env_name: Some(format!(
|
||||
"REDIS_SERVER_{}",
|
||||
match key {
|
||||
"db" => "DATABASE",
|
||||
"host" => "HOST",
|
||||
"port" => "PORT",
|
||||
"username" => "USERNAME",
|
||||
"password" => "PASSWORD",
|
||||
_ => unreachable!(),
|
||||
}
|
||||
)),
|
||||
env_type: Some(
|
||||
if matches!(key, "db" | "port") {
|
||||
"integer"
|
||||
} else {
|
||||
"string"
|
||||
}
|
||||
.to_string(),
|
||||
),
|
||||
link: None,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn crypto_descriptors() -> Vec<AppConfigDescriptor> {
|
||||
let defaults = to_value(CryptoConfigFile::default()).expect("crypto defaults should serialize");
|
||||
let root = to_value(
|
||||
SchemaSettings::draft07()
|
||||
.into_generator()
|
||||
.into_root_schema_for::<CryptoConfigFile>(),
|
||||
)
|
||||
.expect("crypto schema should serialize");
|
||||
vec![AppConfigDescriptor {
|
||||
key: "privateKey".to_string(),
|
||||
description: "Private key for Node signing and encryption plus native runtime identity.".to_string(),
|
||||
default_value: defaults["privateKey"].clone(),
|
||||
schema: field(&root, "privateKey").clone(),
|
||||
internal: false,
|
||||
env_name: Some("AFFINE_PRIVATE_KEY".to_string()),
|
||||
env_type: Some("string".to_string()),
|
||||
link: None,
|
||||
}]
|
||||
}
|
||||
|
||||
fn oauth_descriptors() -> Vec<AppConfigDescriptor> {
|
||||
let default = to_value(OAuthProviderConfigFile::default()).expect("OAuth defaults should serialize");
|
||||
let root = to_value(
|
||||
SchemaSettings::draft07()
|
||||
.into_generator()
|
||||
.into_root_schema_for::<OAuthProviderConfigFile>(),
|
||||
)
|
||||
.expect("OAuth schema should serialize");
|
||||
[
|
||||
(
|
||||
"google",
|
||||
"Google OAuth provider config",
|
||||
"https://developers.google.com/identity/protocols/oauth2/web-server",
|
||||
),
|
||||
(
|
||||
"github",
|
||||
"GitHub OAuth provider config",
|
||||
"https://docs.github.com/en/apps/oauth-apps",
|
||||
),
|
||||
(
|
||||
"oidc",
|
||||
"OIDC OAuth provider config. Private network access requires allowPrivateNetwork: true",
|
||||
"https://openid.net/specs/openid-connect-core-1_0.html",
|
||||
),
|
||||
(
|
||||
"apple",
|
||||
"Apple OAuth provider config",
|
||||
"https://developer.apple.com/documentation/sign_in_with_apple/sign_in_with_apple_js/implementing_sign_in_with_apple_in_your_app",
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(name, description, link)| {
|
||||
let mut schema = root.clone();
|
||||
let mut default_value = default.clone();
|
||||
if name == "oidc" {
|
||||
schema["properties"]["issuer"]["description"] = json!("OIDC issuer HTTP(S) URL");
|
||||
schema["properties"]["allowPrivateNetwork"]["description"] =
|
||||
json!("Allow the OIDC issuer origin to resolve to private network addresses");
|
||||
} else {
|
||||
for key in ["issuer", "allowPrivateNetwork"] {
|
||||
schema["properties"].as_object_mut().expect("OAuth schema properties").remove(key);
|
||||
default_value.as_object_mut().expect("OAuth defaults object").remove(key);
|
||||
}
|
||||
}
|
||||
AppConfigDescriptor {
|
||||
key: format!("providers.{name}"),
|
||||
description: description.to_string(),
|
||||
default_value,
|
||||
schema,
|
||||
internal: false,
|
||||
env_name: None,
|
||||
env_type: None,
|
||||
link: Some(link.to_string()),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn validate_leaf(key: &str, value: Value) -> std::result::Result<(), RuntimeError> {
|
||||
let mut raw = to_value(CopilotRuntimeConfigFile::default())
|
||||
.map_err(|error| RuntimeError::json("serialize copilot defaults", error))?;
|
||||
insert_flat_override(raw.as_object_mut().expect("copilot defaults are an object"), key, value);
|
||||
let config: CopilotRuntimeConfigFile =
|
||||
from_value(raw).map_err(|error| RuntimeError::json("invalid copilot config", error))?;
|
||||
let config = CopilotRuntimeConfig::try_from(config)?;
|
||||
validate_copilot_config(&config)
|
||||
}
|
||||
|
||||
#[napi_derive::napi(catch_unwind)]
|
||||
pub fn app_config_descriptors(module: String) -> Result<Vec<AppConfigDescriptor>> {
|
||||
match module.as_str() {
|
||||
AUTH_MODULE => Ok(auth_descriptors()),
|
||||
COPILOT_MODULE => Ok(descriptors()),
|
||||
CRYPTO_MODULE => Ok(crypto_descriptors()),
|
||||
INDEXER_MODULE => Ok(indexer_descriptors()),
|
||||
OAUTH_MODULE => Ok(oauth_descriptors()),
|
||||
PAYMENT_MODULE => Ok(payment_descriptors()),
|
||||
REDIS_MODULE => Ok(redis_descriptors()),
|
||||
STORAGES_MODULE => Ok(storage_descriptors()),
|
||||
_ => Err(invalid_config(format!("unknown native app config module: {module}"))),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi_derive::napi(catch_unwind)]
|
||||
pub fn validate_app_config_value(module: String, key: String, value: Value) -> Result<Vec<String>> {
|
||||
let descriptors = app_config_descriptors(module.clone())?;
|
||||
let descriptor = descriptors
|
||||
.into_iter()
|
||||
.find(|descriptor| descriptor.key == key)
|
||||
.ok_or_else(|| invalid_config(format!("unknown native app config key: {module}.{key}")))?;
|
||||
let schema = jsonschema::options()
|
||||
.with_draft(Draft::Draft7)
|
||||
.build(&descriptor.schema)
|
||||
.map_err(|error| invalid_config(format!("failed to compile app config schema: {error}")))?;
|
||||
let errors = schema
|
||||
.iter_errors(&value)
|
||||
.map(|error| error.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
if !errors.is_empty() {
|
||||
return Ok(errors);
|
||||
}
|
||||
if module == INDEXER_MODULE
|
||||
&& key == "provider.endpoint"
|
||||
&& !valid_search_endpoint(value.as_str().unwrap_or_default())
|
||||
{
|
||||
return Ok(vec!["invalid search provider endpoint".to_string()]);
|
||||
}
|
||||
if module == OAUTH_MODULE && key == "providers.oidc" {
|
||||
let issuer = value.get("issuer").and_then(Value::as_str).unwrap_or_default();
|
||||
if !valid_search_endpoint(issuer) {
|
||||
return Ok(vec!["invalid OIDC issuer URL".to_string()]);
|
||||
}
|
||||
}
|
||||
if module == COPILOT_MODULE {
|
||||
if key == "storage" {
|
||||
return Ok(validate_storage(value));
|
||||
}
|
||||
Ok(
|
||||
validate_leaf(&key, value)
|
||||
.err()
|
||||
.map(|error| vec![error.to_string()])
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
} else {
|
||||
Ok(if module == STORAGES_MODULE {
|
||||
validate_storage(value)
|
||||
} else {
|
||||
Vec::new()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_storage(value: Value) -> Vec<String> {
|
||||
let config: StorageProviderConfig = match from_value(value) {
|
||||
Ok(config) => config,
|
||||
Err(error) => return vec![error.to_string()],
|
||||
};
|
||||
StorageBackendConfig::from_provider_config(Some(config))
|
||||
.err()
|
||||
.map(|error| vec![error.to_string()])
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use super::{app_config_descriptors, json, validate_app_config_value};
|
||||
|
||||
#[test]
|
||||
fn native_descriptors_and_validation_share_runtime_contract() {
|
||||
let storages = app_config_descriptors("storages".to_string()).unwrap();
|
||||
let copilot_storage = app_config_descriptors("copilot".to_string())
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|descriptor| descriptor.key == "storage")
|
||||
.unwrap();
|
||||
for (module, descriptor) in storages
|
||||
.iter()
|
||||
.map(|descriptor| ("storages", descriptor))
|
||||
.chain(std::iter::once(("copilot", &copilot_storage)))
|
||||
{
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
module.to_string(),
|
||||
descriptor.key.clone(),
|
||||
descriptor.default_value.clone()
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"{}.{}",
|
||||
module,
|
||||
descriptor.key
|
||||
);
|
||||
assert!(
|
||||
!validate_app_config_value(
|
||||
module.to_string(),
|
||||
descriptor.key.clone(),
|
||||
json!({ "provider": "fs", "bucket": "blobs", "config": {} })
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!validate_app_config_value(module.to_string(), descriptor.key.clone(), json!({ "provider": "cloudflare-r2", "bucket": "blobs", "config": { "accountId": "x", "usePresignedURL": { "enabled": "yes" } } }))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
let crypto = app_config_descriptors("crypto".to_string()).unwrap();
|
||||
assert_eq!(crypto.len(), 1);
|
||||
assert_eq!(crypto[0].default_value, "");
|
||||
assert_eq!(crypto[0].env_name.as_deref(), Some("AFFINE_PRIVATE_KEY"));
|
||||
assert!(
|
||||
!validate_app_config_value("crypto".to_string(), "privateKey".to_string(), json!(true))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let oauth = app_config_descriptors("oauth".to_string()).unwrap();
|
||||
assert_eq!(oauth.len(), 4);
|
||||
for descriptor in &oauth {
|
||||
assert!(descriptor.link.is_some());
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"oauth".to_string(),
|
||||
descriptor.key.clone(),
|
||||
descriptor.default_value.clone()
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"oauth".to_string(),
|
||||
"providers.oidc".to_string(),
|
||||
json!({ "clientId": "id", "clientSecret": "secret", "issuer": "https://example.com", "args": {} })
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!validate_app_config_value(
|
||||
"oauth".to_string(),
|
||||
"providers.oidc".to_string(),
|
||||
json!({ "clientId": "id", "clientSecret": "secret", "issuer": "file:///etc/passwd", "args": {} })
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let redis = app_config_descriptors("redis".to_string()).unwrap();
|
||||
assert_eq!(
|
||||
redis
|
||||
.iter()
|
||||
.map(|descriptor| descriptor.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["host", "port", "db", "username", "password"]
|
||||
);
|
||||
assert_eq!(redis[0].default_value, "localhost");
|
||||
assert_eq!(redis[1].default_value, 6379);
|
||||
assert_eq!(redis[0].env_name.as_deref(), Some("REDIS_SERVER_HOST"));
|
||||
assert_eq!(redis[2].env_name.as_deref(), Some("REDIS_SERVER_DATABASE"));
|
||||
for descriptor in &redis {
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"redis".to_string(),
|
||||
descriptor.key.clone(),
|
||||
descriptor.default_value.clone()
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"{}",
|
||||
descriptor.key
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!validate_app_config_value("redis".to_string(), "port".to_string(), json!(0))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!validate_app_config_value("redis".to_string(), "db".to_string(), json!(11))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let auth = app_config_descriptors("auth".to_string()).unwrap();
|
||||
assert_eq!(auth.len(), 11);
|
||||
assert_eq!(
|
||||
auth
|
||||
.iter()
|
||||
.find(|descriptor| descriptor.key == "allowSignup")
|
||||
.unwrap()
|
||||
.default_value,
|
||||
true
|
||||
);
|
||||
assert_eq!(
|
||||
auth
|
||||
.iter()
|
||||
.find(|descriptor| descriptor.key == "token.accessTokenTtl")
|
||||
.unwrap()
|
||||
.default_value,
|
||||
15 * 60
|
||||
);
|
||||
for descriptor in &auth {
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"auth".to_string(),
|
||||
descriptor.key.clone(),
|
||||
descriptor.default_value.clone()
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"{}",
|
||||
descriptor.key
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!validate_app_config_value("auth".to_string(), "token.accessTokenTtl".to_string(), json!(30))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!validate_app_config_value("auth".to_string(), "newAccountActionDelay".to_string(), json!(-1))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let descriptors = app_config_descriptors("copilot".to_string()).unwrap();
|
||||
assert_eq!(
|
||||
descriptors
|
||||
.iter()
|
||||
.map(|descriptor| descriptor.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
"enabled",
|
||||
"byok.enabled",
|
||||
"byok.allowedProviders",
|
||||
"byok.allowCustomEndpoint",
|
||||
"byok.allowPrivateEndpoint",
|
||||
"providers.profiles",
|
||||
"storage",
|
||||
]
|
||||
);
|
||||
assert_eq!(descriptors[0].default_value, json!(false));
|
||||
assert_eq!(descriptors[1].default_value, json!(true));
|
||||
assert!(descriptors[5].internal);
|
||||
let profile_schema = jsonschema::options()
|
||||
.with_draft(jsonschema::Draft::Draft7)
|
||||
.build(&descriptors[5].schema)
|
||||
.unwrap();
|
||||
for (profile, accepted) in [
|
||||
(
|
||||
json!({"id":"openai","type":"openai","models":["gpt-5.6-luna"],"config":{"apiKey":"test"}}),
|
||||
true,
|
||||
),
|
||||
(
|
||||
json!({"id":"openai","type":"openai","models":["gpt-5.6-luna"],"config":{}}),
|
||||
false,
|
||||
),
|
||||
(
|
||||
json!({"id":"openai","type":"openai","enabled":false,"models":["gpt-5.6-luna"],"config":{}}),
|
||||
true,
|
||||
),
|
||||
(
|
||||
json!({"id":"cloudflare","type":"cloudflareWorkersAi","models":["@cf/baai/bge-reranker-base"],"config":{"apiToken":"test"}}),
|
||||
false,
|
||||
),
|
||||
(
|
||||
json!({"id":"vertex","type":"geminiVertex","models":["gemini-3.7-flash"],"config":{"project":"test","location":"global"}}),
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let value = json!([profile]);
|
||||
assert_eq!(profile_schema.is_valid(&value), accepted);
|
||||
assert_eq!(
|
||||
validate_app_config_value("copilot".to_string(), "providers.profiles".to_string(), value)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
accepted
|
||||
);
|
||||
}
|
||||
for descriptor in &descriptors {
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"copilot".to_string(),
|
||||
descriptor.key.clone(),
|
||||
descriptor.default_value.clone()
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"{}",
|
||||
descriptor.key
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"copilot".to_string(),
|
||||
"providers.profiles".to_string(),
|
||||
json!([{
|
||||
"id": "managed-openai",
|
||||
"type": "openai",
|
||||
"displayName": "OpenAI",
|
||||
"priority": 1,
|
||||
"enabled": true,
|
||||
"models": ["gpt-5.6-luna"],
|
||||
"middleware": {
|
||||
"rust": { "request": ["normalize_messages"] },
|
||||
"node": { "text": ["citation_footnote"] }
|
||||
},
|
||||
"config": { "apiKey": "test" }
|
||||
}]),
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let payment = app_config_descriptors("payment".to_string()).unwrap();
|
||||
assert_eq!(
|
||||
payment
|
||||
.iter()
|
||||
.map(|descriptor| descriptor.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["enabled", "stripe", "revenuecat"]
|
||||
);
|
||||
assert_eq!(payment[0].default_value, json!(false));
|
||||
assert_eq!(payment[1].default_value["environment"], "test");
|
||||
assert_eq!(payment[2].default_value["environment"], "production");
|
||||
for descriptor in &payment {
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"payment".to_string(),
|
||||
descriptor.key.clone(),
|
||||
descriptor.default_value.clone()
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"{}",
|
||||
descriptor.key
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!validate_app_config_value("payment".to_string(), "enabled".to_string(), json!("true"))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!validate_app_config_value("payment".to_string(), "stripe".to_string(), json!("invalid"))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let indexer = app_config_descriptors("indexer".to_string()).unwrap();
|
||||
assert_eq!(
|
||||
indexer
|
||||
.iter()
|
||||
.map(|descriptor| descriptor.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
"enabled",
|
||||
"provider.type",
|
||||
"provider.endpoint",
|
||||
"provider.apiKey",
|
||||
"provider.username",
|
||||
"provider.password",
|
||||
]
|
||||
);
|
||||
assert_eq!(indexer[1].default_value, "embedded");
|
||||
for descriptor in &indexer {
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"indexer".to_string(),
|
||||
descriptor.key.clone(),
|
||||
descriptor.default_value.clone()
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"{}",
|
||||
descriptor.key
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!validate_app_config_value("indexer".to_string(), "provider.type".to_string(), json!("unknown"))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!validate_app_config_value(
|
||||
"indexer".to_string(),
|
||||
"provider.endpoint".to_string(),
|
||||
json!("not a URL")
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copilot_validation_rejects_invalid_leaf_values() {
|
||||
for (key, value) in [
|
||||
("enabled", json!("yes")),
|
||||
("byok.enabled", json!("yes")),
|
||||
("byok.allowedProviders", json!(["openai", "openai"])),
|
||||
(
|
||||
"providers.profiles",
|
||||
json!([{
|
||||
"id": "invalid id",
|
||||
"type": "openai",
|
||||
"models": ["gpt-5.6-luna"],
|
||||
"config": {}
|
||||
}]),
|
||||
),
|
||||
(
|
||||
"providers.profiles",
|
||||
json!([{
|
||||
"id": "managed-openai",
|
||||
"type": "openai",
|
||||
"models": ["gpt-5.6-luna"],
|
||||
"middleware": { "node": { "text": ["unknown"] } },
|
||||
"config": {}
|
||||
}]),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!validate_app_config_value("copilot".to_string(), key.to_string(), value)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"{key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{AuthRuntimeConfig, InviteQuotaConfig, RuntimeError, RuntimeResult};
|
||||
|
||||
#[derive(Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(super) struct AuthConfigFile {
|
||||
pub(super) new_account_action_delay: i64,
|
||||
pub(super) allow_signup: bool,
|
||||
pub(super) allow_signup_for_oauth: bool,
|
||||
pub(super) require_email_domain_verification: bool,
|
||||
pub(super) session: AuthSessionConfigFile,
|
||||
pub(super) token: AuthTokenConfigFile,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(default)]
|
||||
pub(super) struct AuthSessionConfigFile {
|
||||
ttl: i64,
|
||||
ttr: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(super) struct AuthTokenConfigFile {
|
||||
access_token_ttl: i64,
|
||||
refresh_idle_ttl: i64,
|
||||
refresh_absolute_ttl: i64,
|
||||
refresh_grace_period: i64,
|
||||
refresh_retention: i64,
|
||||
}
|
||||
|
||||
impl Default for AuthConfigFile {
|
||||
fn default() -> Self {
|
||||
let runtime = AuthRuntimeConfig::default();
|
||||
Self {
|
||||
new_account_action_delay: InviteQuotaConfig::default().new_account_action_delay_seconds,
|
||||
allow_signup: runtime.allow_signup,
|
||||
allow_signup_for_oauth: runtime.allow_signup_for_oauth,
|
||||
require_email_domain_verification: runtime.require_email_domain_verification,
|
||||
session: AuthSessionConfigFile {
|
||||
ttl: runtime.session_ttl_seconds,
|
||||
ttr: runtime.session_ttr_seconds,
|
||||
},
|
||||
token: AuthTokenConfigFile {
|
||||
access_token_ttl: runtime.access_token_ttl_seconds,
|
||||
refresh_idle_ttl: runtime.refresh_idle_ttl_seconds,
|
||||
refresh_absolute_ttl: runtime.refresh_absolute_ttl_seconds,
|
||||
refresh_grace_period: runtime.refresh_grace_seconds,
|
||||
refresh_retention: runtime.refresh_retention_seconds,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AuthSessionConfigFile {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
ttl: AuthRuntimeConfig::default().session_ttl_seconds,
|
||||
ttr: AuthRuntimeConfig::default().session_ttr_seconds,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AuthTokenConfigFile {
|
||||
fn default() -> Self {
|
||||
let runtime = AuthRuntimeConfig::default();
|
||||
Self {
|
||||
access_token_ttl: runtime.access_token_ttl_seconds,
|
||||
refresh_idle_ttl: runtime.refresh_idle_ttl_seconds,
|
||||
refresh_absolute_ttl: runtime.refresh_absolute_ttl_seconds,
|
||||
refresh_grace_period: runtime.refresh_grace_seconds,
|
||||
refresh_retention: runtime.refresh_retention_seconds,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn auth_limit(key: &str) -> Option<(i64, i64)> {
|
||||
match key {
|
||||
"newAccountActionDelay" => Some((0, i64::MAX)),
|
||||
"token.accessTokenTtl" => Some((60, 60 * 60)),
|
||||
"token.refreshIdleTtl" => Some((60 * 60, 60 * 60 * 24 * 365)),
|
||||
"token.refreshAbsoluteTtl" => Some((60 * 60, 60 * 60 * 24 * 730)),
|
||||
"token.refreshGracePeriod" => Some((0, 60)),
|
||||
"token.refreshRetention" => Some((60 * 60, 60 * 60 * 24 * 365)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
impl AuthConfigFile {
|
||||
pub(super) fn validate(&self) -> RuntimeResult<()> {
|
||||
for (key, value) in [
|
||||
("newAccountActionDelay", self.new_account_action_delay),
|
||||
("token.accessTokenTtl", self.token.access_token_ttl),
|
||||
("token.refreshIdleTtl", self.token.refresh_idle_ttl),
|
||||
("token.refreshAbsoluteTtl", self.token.refresh_absolute_ttl),
|
||||
("token.refreshGracePeriod", self.token.refresh_grace_period),
|
||||
("token.refreshRetention", self.token.refresh_retention),
|
||||
] {
|
||||
let (min, max) = auth_limit(key).expect("auth range must be defined");
|
||||
if !(min..=max).contains(&value) {
|
||||
return Err(RuntimeError::config(format!("auth.{key} is out of range")));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn invite_quota_config(&self) -> InviteQuotaConfig {
|
||||
InviteQuotaConfig {
|
||||
new_account_action_delay_seconds: self.new_account_action_delay,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn runtime_config(&self) -> AuthRuntimeConfig {
|
||||
AuthRuntimeConfig {
|
||||
allow_signup: self.allow_signup,
|
||||
allow_signup_for_oauth: self.allow_signup_for_oauth,
|
||||
require_email_domain_verification: self.require_email_domain_verification,
|
||||
session_ttl_seconds: self.session.ttl,
|
||||
session_ttr_seconds: self.session.ttr,
|
||||
access_token_ttl_seconds: self.token.access_token_ttl,
|
||||
refresh_idle_ttl_seconds: self.token.refresh_idle_ttl,
|
||||
refresh_absolute_ttl_seconds: self.token.refresh_absolute_ttl,
|
||||
refresh_grace_seconds: self.token.refresh_grace_period,
|
||||
refresh_retention_seconds: self.token.refresh_retention,
|
||||
oauth: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
use llm_adapter::capability::provider_default_capability_upper_bound;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Map;
|
||||
|
||||
use super::{RuntimeError, RuntimeResult};
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(crate) struct CopilotRuntimeConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) byok: CopilotByokRuntimeConfig,
|
||||
pub(crate) providers: CopilotProvidersRuntimeConfig,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(crate) struct CopilotByokRuntimeConfig {
|
||||
pub(crate) enabled: bool,
|
||||
#[serde(default = "default_allowed_providers")]
|
||||
pub(crate) allowed_providers: Vec<String>,
|
||||
pub(crate) allow_custom_endpoint: bool,
|
||||
pub(crate) allow_private_endpoint: bool,
|
||||
}
|
||||
|
||||
impl Default for CopilotByokRuntimeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
allowed_providers: default_allowed_providers(),
|
||||
allow_custom_endpoint: false,
|
||||
allow_private_endpoint: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::runtime) const SUPPORTED_BYOK_PROVIDERS: [&str; 4] = ["openai", "anthropic", "gemini", "fal"];
|
||||
|
||||
pub(super) const MANAGED_PROFILE_REQUIREMENTS: [(&str, &[&str]); 7] = [
|
||||
("openai", &["apiKey"]),
|
||||
("anthropic", &["apiKey"]),
|
||||
("gemini", &["apiKey"]),
|
||||
("fal", &["apiKey"]),
|
||||
("cloudflareWorkersAi", &["apiToken", "accountId"]),
|
||||
("geminiVertex", &["project", "location"]),
|
||||
("anthropicVertex", &["project", "location"]),
|
||||
];
|
||||
|
||||
fn default_allowed_providers() -> Vec<String> {
|
||||
SUPPORTED_BYOK_PROVIDERS.into_iter().map(str::to_string).collect()
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(crate) struct CopilotProvidersRuntimeConfig {
|
||||
pub(crate) profiles: Vec<CopilotManagedProfileConfig>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct CopilotManagedProfileConfig {
|
||||
pub(crate) id: String,
|
||||
#[serde(rename = "type")]
|
||||
pub(crate) provider: String,
|
||||
#[serde(default = "enabled_by_default")]
|
||||
pub(crate) enabled: bool,
|
||||
#[serde(default)]
|
||||
pub(crate) models: Vec<String>,
|
||||
pub(crate) config: serde_json::Value,
|
||||
}
|
||||
|
||||
fn enabled_by_default() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(crate) struct CopilotRuntimeConfigFile {
|
||||
pub(in crate::runtime) enabled: bool,
|
||||
pub(in crate::runtime) byok: CopilotByokRuntimeConfig,
|
||||
pub(in crate::runtime) providers: CopilotProvidersRuntimeConfigFile,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(in crate::runtime) struct CopilotProvidersRuntimeConfigFile {
|
||||
pub(in crate::runtime) profiles: Vec<CopilotManagedProfileConfigFile>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct CopilotManagedProfileConfigFile {
|
||||
id: String,
|
||||
#[serde(rename = "type")]
|
||||
provider: CopilotManagedProvider,
|
||||
display_name: Option<String>,
|
||||
priority: Option<f64>,
|
||||
#[serde(default = "enabled_by_default")]
|
||||
enabled: bool,
|
||||
models: Vec<String>,
|
||||
middleware: Option<CopilotProviderMiddlewareConfigFile>,
|
||||
config: CopilotManagedProviderSettingsFile,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct CopilotManagedProviderSettingsFile {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
api_key: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
api_token: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
account_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
project: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
location: Option<String>,
|
||||
#[serde(rename = "baseURL", skip_serializing_if = "Option::is_none")]
|
||||
base_url: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
google_auth_options: Option<CopilotGoogleAuthOptionsFile>,
|
||||
#[serde(flatten)]
|
||||
additional: Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct CopilotGoogleAuthOptionsFile {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
credentials: Option<CopilotGoogleCredentialsFile>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
struct CopilotGoogleCredentialsFile {
|
||||
client_email: String,
|
||||
private_key: String,
|
||||
#[serde(flatten)]
|
||||
additional: Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
enum CopilotManagedProvider {
|
||||
#[serde(rename = "anthropic")]
|
||||
Anthropic,
|
||||
#[serde(rename = "anthropicVertex")]
|
||||
AnthropicVertex,
|
||||
#[serde(rename = "cloudflareWorkersAi")]
|
||||
CloudflareWorkersAi,
|
||||
#[serde(rename = "fal")]
|
||||
Fal,
|
||||
#[serde(rename = "gemini")]
|
||||
Gemini,
|
||||
#[serde(rename = "geminiVertex")]
|
||||
GeminiVertex,
|
||||
#[serde(rename = "openai")]
|
||||
OpenAi,
|
||||
}
|
||||
|
||||
impl CopilotManagedProvider {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Anthropic => "anthropic",
|
||||
Self::AnthropicVertex => "anthropicVertex",
|
||||
Self::CloudflareWorkersAi => "cloudflareWorkersAi",
|
||||
Self::Fal => "fal",
|
||||
Self::Gemini => "gemini",
|
||||
Self::GeminiVertex => "geminiVertex",
|
||||
Self::OpenAi => "openai",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
struct CopilotProviderMiddlewareConfigFile {
|
||||
rust: Option<CopilotRustMiddlewareConfigFile>,
|
||||
node: Option<CopilotNodeMiddlewareConfigFile>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
struct CopilotRustMiddlewareConfigFile {
|
||||
request: Option<Vec<CopilotRustRequestMiddleware>>,
|
||||
stream: Option<Vec<CopilotRustStreamMiddleware>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
struct CopilotNodeMiddlewareConfigFile {
|
||||
text: Option<Vec<CopilotNodeTextMiddleware>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum CopilotRustRequestMiddleware {
|
||||
NormalizeMessages,
|
||||
ClampMaxTokens,
|
||||
ToolSchemaRewrite,
|
||||
OpenaiRequestCompat,
|
||||
OmitToolChoice,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum CopilotRustStreamMiddleware {
|
||||
StreamEventNormalize,
|
||||
CitationIndexing,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum CopilotNodeTextMiddleware {
|
||||
CitationFootnote,
|
||||
Callout,
|
||||
ThinkingFormat,
|
||||
}
|
||||
|
||||
impl TryFrom<CopilotRuntimeConfigFile> for CopilotRuntimeConfig {
|
||||
type Error = RuntimeError;
|
||||
|
||||
fn try_from(value: CopilotRuntimeConfigFile) -> Result<Self, Self::Error> {
|
||||
Ok(Self {
|
||||
enabled: value.enabled,
|
||||
byok: value.byok,
|
||||
providers: CopilotProvidersRuntimeConfig {
|
||||
profiles: value
|
||||
.providers
|
||||
.profiles
|
||||
.into_iter()
|
||||
.map(TryInto::try_into)
|
||||
.collect::<RuntimeResult<_>>()?,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<CopilotManagedProfileConfigFile> for CopilotManagedProfileConfig {
|
||||
type Error = RuntimeError;
|
||||
|
||||
fn try_from(value: CopilotManagedProfileConfigFile) -> Result<Self, Self::Error> {
|
||||
if value.id.is_empty()
|
||||
|| !value
|
||||
.id
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
|
||||
{
|
||||
return Err(RuntimeError::invalid_state(
|
||||
"managed copilot profile id must contain only letters, numbers, hyphens, and underscores",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
id: value.id,
|
||||
provider: value.provider.as_str().to_string(),
|
||||
enabled: value.enabled,
|
||||
models: value.models,
|
||||
config: serde_json::to_value(value.config)
|
||||
.map_err(|error| RuntimeError::json("serialize managed copilot config failed", error))?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::runtime) fn validate_copilot_config(config: &CopilotRuntimeConfig) -> RuntimeResult<()> {
|
||||
let mut allowed_providers = std::collections::HashSet::new();
|
||||
for provider in &config.byok.allowed_providers {
|
||||
if !SUPPORTED_BYOK_PROVIDERS.contains(&provider.as_str()) || !allowed_providers.insert(provider.as_str()) {
|
||||
return Err(RuntimeError::invalid_state(
|
||||
"copilot BYOK allowed providers must be supported and unique",
|
||||
));
|
||||
}
|
||||
}
|
||||
let mut profile_ids = std::collections::HashSet::new();
|
||||
let mut managed_models = std::collections::HashMap::new();
|
||||
for profile in &config.providers.profiles {
|
||||
if profile.id.trim().is_empty() || !profile_ids.insert(profile.id.as_str()) {
|
||||
return Err(RuntimeError::invalid_state(
|
||||
"managed copilot profile ids must be non-empty and unique",
|
||||
));
|
||||
}
|
||||
if profile.provider.trim().is_empty() {
|
||||
return Err(RuntimeError::invalid_state(
|
||||
"managed copilot profile provider is required",
|
||||
));
|
||||
}
|
||||
if profile.models.is_empty() {
|
||||
return Err(RuntimeError::invalid_state(
|
||||
"managed copilot profile models must be non-empty",
|
||||
));
|
||||
}
|
||||
if profile.enabled {
|
||||
let required = MANAGED_PROFILE_REQUIREMENTS
|
||||
.iter()
|
||||
.find(|(provider, _)| *provider == profile.provider)
|
||||
.map(|(_, fields)| *fields)
|
||||
.ok_or_else(|| RuntimeError::invalid_state("unsupported managed copilot provider"))?;
|
||||
for field in required {
|
||||
if profile
|
||||
.config
|
||||
.get(*field)
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_none_or(|value| value.trim().is_empty())
|
||||
{
|
||||
return Err(RuntimeError::invalid_state(format!(
|
||||
"managed copilot profile requires {field}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
if let Some(base_url) = profile.config.get("baseURL").and_then(serde_json::Value::as_str) {
|
||||
llm_adapter::target::canonicalize_endpoint(base_url)
|
||||
.map_err(|error| RuntimeError::invalid_state(error.to_string()))?;
|
||||
}
|
||||
if let Some(credentials) = profile.config.pointer("/googleAuthOptions/credentials") {
|
||||
for field in ["client_email", "private_key"] {
|
||||
if credentials
|
||||
.get(field)
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_none_or(|value| value.trim().is_empty())
|
||||
{
|
||||
return Err(RuntimeError::invalid_state(format!(
|
||||
"managed Vertex credentials require {field}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut models = std::collections::HashSet::new();
|
||||
for model in &profile.models {
|
||||
if model.trim().is_empty() || !models.insert(model.as_str()) {
|
||||
return Err(RuntimeError::invalid_state(
|
||||
"managed copilot profile models must be non-empty and unique",
|
||||
));
|
||||
}
|
||||
provider_default_capability_upper_bound(&profile.provider, model)
|
||||
.ok_or_else(|| RuntimeError::invalid_state("managed copilot profile model is unsupported"))?;
|
||||
if profile.enabled
|
||||
&& let Some(existing_profile) = managed_models.insert(model.as_str(), profile.id.as_str())
|
||||
{
|
||||
return Err(RuntimeError::invalid_state(format!(
|
||||
"managed copilot model {model} is assigned to both {existing_profile} and {}",
|
||||
profile.id
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::{
|
||||
Deployment, NATIVE_APP_CONFIG_KEYS, RedisRuntimeConfig, RedisRuntimeConfigFile, RuntimeError, RuntimeResult,
|
||||
deserialize_app_config, insert_flat_override, is_secret_native_app_config_key,
|
||||
};
|
||||
use crate::runtime::object_storage::ObjectStorageService;
|
||||
|
||||
pub(crate) struct ServerConfig {
|
||||
path: PathBuf,
|
||||
baseline: serde_json::Value,
|
||||
deployment: Deployment,
|
||||
object_storage: ObjectStorageService,
|
||||
}
|
||||
|
||||
impl ServerConfig {
|
||||
pub(crate) fn open(path: &Path, legacy_deployment_type: Option<&str>) -> RuntimeResult<Self> {
|
||||
let path = fs::canonicalize(path).map_err(|error| RuntimeError::io("failed to locate config file", error))?;
|
||||
let raw = fs::read_to_string(&path).map_err(|error| RuntimeError::io("failed to read config file", error))?;
|
||||
let source: Value =
|
||||
serde_json::from_str(&raw).map_err(|error| RuntimeError::json("failed to parse config file", error))?;
|
||||
let baseline = expand_module_config_paths(source.clone());
|
||||
let deployment = match baseline.pointer("/deployment/type").and_then(serde_json::Value::as_str) {
|
||||
Some("cloud") => Deployment::Cloud,
|
||||
Some("selfhosted") => Deployment::SelfHosted,
|
||||
// TODO(0.27.5): Remove the old deployment env fallback after self-host config files are upgraded.
|
||||
None => match legacy_deployment_type {
|
||||
Some("cloud") => Deployment::Cloud,
|
||||
Some("selfhosted") => Deployment::SelfHosted,
|
||||
_ => return Err(RuntimeError::config("deployment.type must be cloud or selfhosted")),
|
||||
},
|
||||
_ => return Err(RuntimeError::config("deployment.type must be cloud or selfhosted")),
|
||||
};
|
||||
deserialize_app_config(baseline.clone())?;
|
||||
let object_storage = ObjectStorageService::from_config_value(&source)?;
|
||||
Ok(Self {
|
||||
path,
|
||||
baseline,
|
||||
deployment,
|
||||
object_storage,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub(crate) fn baseline(&self) -> &serde_json::Value {
|
||||
&self.baseline
|
||||
}
|
||||
|
||||
pub(crate) fn has_baseline_config_key(&self, key: &str) -> bool {
|
||||
key
|
||||
.split('.')
|
||||
.try_fold(&self.baseline, |value, segment| value.get(segment))
|
||||
.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn baseline_config_key_configured(&self, key: &str) -> bool {
|
||||
fn configured(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Null => false,
|
||||
Value::String(value) => !value.is_empty(),
|
||||
Value::Array(values) => !values.is_empty(),
|
||||
Value::Object(values) => values.values().any(configured),
|
||||
Value::Bool(_) | Value::Number(_) => true,
|
||||
}
|
||||
}
|
||||
key
|
||||
.split('.')
|
||||
.try_fold(&self.baseline, |value, segment| value.get(segment))
|
||||
.is_some_and(configured)
|
||||
}
|
||||
|
||||
pub(crate) fn redis_url(&self) -> RuntimeResult<Option<String>> {
|
||||
let file = self
|
||||
.baseline
|
||||
.get("redis")
|
||||
.cloned()
|
||||
.map(serde_json::from_value::<RedisRuntimeConfigFile>)
|
||||
.transpose()
|
||||
.map_err(|error| RuntimeError::json("invalid Redis config", error))?;
|
||||
Ok(RedisRuntimeConfig::from_sources(file)?.url)
|
||||
}
|
||||
|
||||
pub(crate) fn redis_node_options(&self) -> Value {
|
||||
self
|
||||
.baseline
|
||||
.pointer("/redis/ioredis")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| Value::Object(Map::new()))
|
||||
}
|
||||
|
||||
pub(crate) fn node_owned(&self) -> Value {
|
||||
let baseline = self.baseline();
|
||||
let mut modules = Map::new();
|
||||
for name in [
|
||||
"calendar",
|
||||
"captcha",
|
||||
"client",
|
||||
"doc",
|
||||
"flags",
|
||||
"graphql",
|
||||
"mailer",
|
||||
"metrics",
|
||||
"server",
|
||||
"telemetry",
|
||||
"throttle",
|
||||
"websocket",
|
||||
"worker",
|
||||
] {
|
||||
if let Some(value) = baseline.get(name) {
|
||||
modules.insert(name.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
for (module, key) in [
|
||||
("auth", "passwordRequirements"),
|
||||
("auth", "signInRateLimit"),
|
||||
("auth", "trustedCloudflareHeaders"),
|
||||
("copilot", "exa"),
|
||||
("copilot", "unsplash"),
|
||||
("db", "prisma"),
|
||||
("payment", "showLifetimePrice"),
|
||||
("storages", "avatar.publicPath"),
|
||||
] {
|
||||
if let Some(value) = baseline.pointer(&format!("/{}/{}", module, key.replace('.', "/"))) {
|
||||
let entry = modules
|
||||
.entry(module.to_string())
|
||||
.or_insert_with(|| Value::Object(Map::new()));
|
||||
insert_flat_override(
|
||||
entry.as_object_mut().expect("node projection module must be an object"),
|
||||
key,
|
||||
value.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
Value::Object(modules)
|
||||
}
|
||||
|
||||
pub(crate) fn public_native_baseline(&self) -> Value {
|
||||
let mut modules = Map::new();
|
||||
for key in NATIVE_APP_CONFIG_KEYS {
|
||||
if is_secret_native_app_config_key(key) {
|
||||
continue;
|
||||
}
|
||||
if let Some(value) = key.split('.').try_fold(&self.baseline, |item, part| item.get(part)) {
|
||||
insert_flat_override(&mut modules, key, value.clone());
|
||||
}
|
||||
}
|
||||
Value::Object(modules)
|
||||
}
|
||||
|
||||
pub(crate) fn deployment(&self) -> Deployment {
|
||||
self.deployment
|
||||
}
|
||||
|
||||
pub(crate) fn object_storage(&self) -> &ObjectStorageService {
|
||||
&self.object_storage
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn expand_module_config_paths(mut value: serde_json::Value) -> serde_json::Value {
|
||||
if let Some(root) = value.as_object_mut() {
|
||||
for module in root.values_mut().filter_map(serde_json::Value::as_object_mut) {
|
||||
let entries = std::mem::take(module);
|
||||
for (path, value) in entries {
|
||||
insert_flat_override(module, &path, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
value
|
||||
}
|
||||
@@ -0,0 +1,471 @@
|
||||
use std::{env, sync::Arc};
|
||||
|
||||
use serde::Deserialize;
|
||||
use sqlx::PgPool;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::{RuntimeError, RuntimeResult};
|
||||
|
||||
mod admin;
|
||||
mod auth;
|
||||
mod copilot;
|
||||
mod file;
|
||||
mod napi;
|
||||
mod payment;
|
||||
use payment::PaymentRuntimeConfigFile;
|
||||
mod store;
|
||||
mod types;
|
||||
pub use admin::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value};
|
||||
use auth::{AuthConfigFile, auth_limit};
|
||||
pub(crate) use copilot::{CopilotByokRuntimeConfig, CopilotManagedProfileConfig, CopilotRuntimeConfig};
|
||||
use copilot::{
|
||||
CopilotRuntimeConfigFile, MANAGED_PROFILE_REQUIREMENTS, SUPPORTED_BYOK_PROVIDERS, validate_copilot_config,
|
||||
};
|
||||
pub(crate) use file::ServerConfig;
|
||||
#[cfg(test)]
|
||||
use file::expand_module_config_paths;
|
||||
pub use napi::ServerConfigHandle;
|
||||
pub(crate) use store::{AppConfigChange, save_app_config_changes};
|
||||
use store::{
|
||||
NATIVE_APP_CONFIG_KEYS, insert_flat_override, is_secret_native_app_config_key, is_static_app_config_key,
|
||||
load_app_config_overrides_from_db,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use store::{app_config_from_flat_overrides, app_config_value_from_flat_overrides};
|
||||
pub(crate) use types::{
|
||||
AuthRuntimeConfig, BackendRuntimeConfig, Deployment, InviteQuotaConfig, OAuthProviderRuntimeConfig,
|
||||
PaymentProductConfig, PaymentRuntimeConfig, RedisRuntimeConfig, RevenueCatRuntimeConfig, SearchRuntimeConfig,
|
||||
StripeRuntimeConfig,
|
||||
};
|
||||
|
||||
use crate::llm::byok::ByokPolicy;
|
||||
|
||||
impl BackendRuntimeConfig {
|
||||
pub(crate) fn byok_policy(&self) -> ByokPolicy {
|
||||
ByokPolicy::from(self.deployment, &self.copilot.byok)
|
||||
}
|
||||
|
||||
pub(crate) fn from_server_config(private_key: Option<String>, config: &ServerConfig) -> RuntimeResult<Self> {
|
||||
Self::from_value(private_key, config.baseline().clone(), config.deployment())
|
||||
}
|
||||
|
||||
fn from_value(
|
||||
private_key: Option<String>,
|
||||
app_config_value: serde_json::Value,
|
||||
deployment: Deployment,
|
||||
) -> RuntimeResult<Self> {
|
||||
let mut app_config = deserialize_app_config(app_config_value)?;
|
||||
let database_url = database_url_from_env()
|
||||
.or(app_config.database_url())
|
||||
.unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string());
|
||||
Self {
|
||||
database_url,
|
||||
auth: app_config.auth_runtime_config(),
|
||||
invite_quota: app_config.invite_quota_config(),
|
||||
private_key: Arc::new(Zeroizing::new(
|
||||
private_key
|
||||
.filter(|key| !key.trim().is_empty())
|
||||
.or_else(|| {
|
||||
app_config
|
||||
.crypto
|
||||
.as_ref()
|
||||
.and_then(|crypto| non_empty_string(crypto.private_key.clone()))
|
||||
})
|
||||
.or_else(private_key_from_env)
|
||||
.unwrap_or_default(),
|
||||
)),
|
||||
deployment,
|
||||
copilot: app_config
|
||||
.copilot
|
||||
.take()
|
||||
.map(TryInto::try_into)
|
||||
.transpose()?
|
||||
.unwrap_or_default(),
|
||||
search: app_config.indexer.map(Into::into).unwrap_or_default(),
|
||||
redis: RedisRuntimeConfig::from_sources(app_config.redis.take())?,
|
||||
payment: PaymentRuntimeConfig::from_file(app_config.payment.take()),
|
||||
}
|
||||
.validated()
|
||||
}
|
||||
|
||||
pub(crate) async fn with_db_overrides_from_server_config(
|
||||
&self,
|
||||
pool: &PgPool,
|
||||
config: &ServerConfig,
|
||||
) -> RuntimeResult<Self> {
|
||||
let db_overrides = load_app_config_overrides_from_db(pool).await?;
|
||||
self.apply_db_overrides(config.baseline().clone(), db_overrides)
|
||||
}
|
||||
|
||||
fn apply_db_overrides(
|
||||
&self,
|
||||
mut app_config_value: serde_json::Value,
|
||||
db_overrides: serde_json::Value,
|
||||
) -> RuntimeResult<Self> {
|
||||
let db_private_key = db_overrides
|
||||
.pointer("/crypto/privateKey")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string)
|
||||
.and_then(non_empty_string);
|
||||
merge_config_value(&mut app_config_value, db_overrides);
|
||||
let mut app_config = deserialize_app_config(app_config_value)?;
|
||||
Self {
|
||||
// The DB override is loaded after this connection already exists, so it
|
||||
// must not rewrite the active datasource URL.
|
||||
database_url: self.database_url.clone(),
|
||||
auth: app_config.auth_runtime_config(),
|
||||
invite_quota: app_config.invite_quota_config(),
|
||||
private_key: db_private_key
|
||||
.map(|key| Arc::new(Zeroizing::new(key)))
|
||||
.unwrap_or_else(|| Arc::clone(&self.private_key)),
|
||||
deployment: self.deployment,
|
||||
copilot: app_config
|
||||
.copilot
|
||||
.take()
|
||||
.map(TryInto::try_into)
|
||||
.transpose()?
|
||||
.unwrap_or_else(|| self.copilot.clone()),
|
||||
search: app_config
|
||||
.indexer
|
||||
.map(Into::into)
|
||||
.unwrap_or_else(|| self.search.clone()),
|
||||
redis: RedisRuntimeConfig::from_sources(app_config.redis.take())?.or_else(|| self.redis.clone()),
|
||||
payment: app_config
|
||||
.payment
|
||||
.take()
|
||||
.map(PaymentRuntimeConfig::from_file_value)
|
||||
.unwrap_or_else(|| self.payment.clone()),
|
||||
}
|
||||
.validated()
|
||||
}
|
||||
|
||||
fn validated(self) -> RuntimeResult<Self> {
|
||||
if self.copilot.enabled && self.copilot.byok.enabled && self.private_key.is_empty() {
|
||||
return Err(RuntimeError::invalid_state(
|
||||
"stable crypto.privateKey is required when persistent BYOK is enabled",
|
||||
));
|
||||
}
|
||||
validate_copilot_config(&self.copilot)?;
|
||||
self.payment.validate()?;
|
||||
if self.search.provider != "embedded" && self.search.endpoint.is_empty() {
|
||||
return Err(RuntimeError::config("remote search provider requires an endpoint"));
|
||||
}
|
||||
if !valid_search_endpoint(&self.search.endpoint) {
|
||||
return Err(RuntimeError::config("invalid search provider endpoint"));
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_search_endpoint(endpoint: &str) -> bool {
|
||||
if endpoint.is_empty() {
|
||||
return true;
|
||||
}
|
||||
url::Url::parse(endpoint)
|
||||
.map(|url| matches!(url.scheme(), "http" | "https") && url.host_str().is_some())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
struct AppConfigFile {
|
||||
auth: Option<AuthConfigFile>,
|
||||
oauth: Option<OAuthConfigFile>,
|
||||
db: Option<DbConfigFile>,
|
||||
crypto: Option<CryptoConfigFile>,
|
||||
copilot: Option<CopilotRuntimeConfigFile>,
|
||||
indexer: Option<SearchRuntimeConfigFile>,
|
||||
redis: Option<RedisRuntimeConfigFile>,
|
||||
payment: Option<PaymentRuntimeConfigFile>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(super) struct RedisRuntimeConfigFile {
|
||||
host: String,
|
||||
port: u16,
|
||||
db: u8,
|
||||
username: String,
|
||||
password: String,
|
||||
ioredis: RedisIoRuntimeConfigFile,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
struct RedisIoRuntimeConfigFile {
|
||||
tls: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Default for RedisRuntimeConfigFile {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
host: "localhost".to_string(),
|
||||
port: 6379,
|
||||
db: 0,
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
ioredis: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RedisRuntimeConfigFile {
|
||||
fn validate(&self) -> RuntimeResult<()> {
|
||||
if self.port == 0 || self.db > 10 {
|
||||
return Err(RuntimeError::config(
|
||||
"redis.port must be positive and redis.db must be at most 10",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl RedisRuntimeConfig {
|
||||
fn from_sources(file: Option<RedisRuntimeConfigFile>) -> RuntimeResult<Self> {
|
||||
if let Some(url) = env::var("REDIS_SERVER_URL").ok().and_then(non_empty_string) {
|
||||
let url = url::Url::parse(&url).map_err(|_| RuntimeError::config("invalid Redis URL"))?;
|
||||
return Ok(Self {
|
||||
url: Some(url.to_string()),
|
||||
});
|
||||
}
|
||||
let file = file.unwrap_or_default();
|
||||
let host = env::var("REDIS_SERVER_HOST")
|
||||
.ok()
|
||||
.and_then(non_empty_string)
|
||||
.or_else(|| non_empty_string(file.host));
|
||||
let Some(host) = host else {
|
||||
return Ok(Self::default());
|
||||
};
|
||||
let port = env::var("REDIS_SERVER_PORT")
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
.unwrap_or(if file.port == 0 { 6379 } else { file.port });
|
||||
let db = env::var("REDIS_SERVER_DATABASE")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u8>().ok())
|
||||
.unwrap_or(file.db);
|
||||
let username = env::var("REDIS_SERVER_USERNAME")
|
||||
.ok()
|
||||
.and_then(non_empty_string)
|
||||
.or_else(|| env::var("REDIS_SERVER_USER").ok().and_then(non_empty_string))
|
||||
.unwrap_or(file.username);
|
||||
let password = env::var("REDIS_SERVER_PASSWORD")
|
||||
.ok()
|
||||
.and_then(non_empty_string)
|
||||
.unwrap_or(file.password);
|
||||
let scheme = if file.ioredis.tls.is_some() { "rediss" } else { "redis" };
|
||||
let mut url = url::Url::parse(&format!("{scheme}://{host}:{port}/{db}"))
|
||||
.map_err(|_| RuntimeError::config("invalid Redis host"))?;
|
||||
if !username.is_empty() {
|
||||
let _ = url.set_username(&username);
|
||||
}
|
||||
if !password.is_empty() {
|
||||
let _ = url.set_password(Some(&password));
|
||||
}
|
||||
Ok(Self {
|
||||
url: Some(url.to_string()),
|
||||
})
|
||||
}
|
||||
|
||||
fn or_else(self, fallback: impl FnOnce() -> Self) -> Self {
|
||||
if self.url.is_some() { self } else { fallback() }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(default)]
|
||||
pub(super) struct OAuthConfigFile {
|
||||
providers: std::collections::BTreeMap<String, OAuthProviderConfigFile>,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(super) struct OAuthProviderConfigFile {
|
||||
client_id: String,
|
||||
client_secret: String,
|
||||
args: std::collections::BTreeMap<String, String>,
|
||||
issuer: String,
|
||||
allow_private_network: bool,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
struct SearchRuntimeConfigFile {
|
||||
enabled: bool,
|
||||
provider: SearchProviderConfigFile,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
struct SearchProviderConfigFile {
|
||||
#[serde(rename = "type")]
|
||||
provider: SearchProviderType,
|
||||
endpoint: String,
|
||||
api_key: String,
|
||||
username: String,
|
||||
password: String,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
enum SearchProviderType {
|
||||
#[default]
|
||||
Embedded,
|
||||
Elasticsearch,
|
||||
Manticoresearch,
|
||||
}
|
||||
|
||||
impl SearchProviderType {
|
||||
fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Embedded => "embedded",
|
||||
Self::Elasticsearch => "elasticsearch",
|
||||
Self::Manticoresearch => "manticoresearch",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SearchRuntimeConfigFile> for SearchRuntimeConfig {
|
||||
fn from(value: SearchRuntimeConfigFile) -> Self {
|
||||
Self {
|
||||
enabled: value.enabled,
|
||||
provider: value.provider.provider.as_str().to_string(),
|
||||
endpoint: value.provider.endpoint,
|
||||
api_key: value.provider.api_key,
|
||||
username: value.provider.username,
|
||||
password: value.provider.password,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(super) struct CryptoConfigFile {
|
||||
private_key: String,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct DbConfigFile {
|
||||
datasource_url: Option<String>,
|
||||
}
|
||||
|
||||
impl AppConfigFile {
|
||||
fn database_url(&self) -> Option<String> {
|
||||
self
|
||||
.db
|
||||
.as_ref()
|
||||
.and_then(|db| db.datasource_url.clone())
|
||||
.and_then(non_empty_string)
|
||||
}
|
||||
|
||||
fn invite_quota_config(&self) -> InviteQuotaConfig {
|
||||
self
|
||||
.auth
|
||||
.as_ref()
|
||||
.map(AuthConfigFile::invite_quota_config)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn auth_runtime_config(&self) -> AuthRuntimeConfig {
|
||||
let mut config = self
|
||||
.auth
|
||||
.as_ref()
|
||||
.map(AuthConfigFile::runtime_config)
|
||||
.unwrap_or_default();
|
||||
if let Some(oauth) = &self.oauth {
|
||||
for (name, provider) in &oauth.providers {
|
||||
if provider.client_id.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut args = provider.args.clone();
|
||||
let apple_private_key = args.remove("privateKey").map(|value| Arc::new(Zeroizing::new(value)));
|
||||
let apple_key_id = args.remove("keyId");
|
||||
let apple_team_id = args.remove("teamId");
|
||||
if provider.client_secret.trim().is_empty()
|
||||
&& (name != "apple"
|
||||
|| apple_private_key.is_none()
|
||||
|| apple_key_id.as_deref().is_none_or(str::is_empty)
|
||||
|| apple_team_id.as_deref().is_none_or(str::is_empty))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
config.oauth.providers.insert(
|
||||
name.clone(),
|
||||
OAuthProviderRuntimeConfig {
|
||||
client_id: provider.client_id.clone(),
|
||||
client_secret: Arc::new(Zeroizing::new(provider.client_secret.clone())),
|
||||
args,
|
||||
issuer: non_empty_string(provider.issuer.clone()),
|
||||
allow_private_network: provider.allow_private_network,
|
||||
apple_private_key,
|
||||
apple_key_id,
|
||||
apple_team_id,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
config
|
||||
}
|
||||
}
|
||||
|
||||
fn database_url_from_env() -> Option<String> {
|
||||
env::var("DATABASE_URL").ok().and_then(non_empty_string)
|
||||
}
|
||||
|
||||
fn private_key_from_env() -> Option<String> {
|
||||
env::var("AFFINE_PRIVATE_KEY").ok().and_then(non_empty_string)
|
||||
}
|
||||
|
||||
fn non_empty_string(value: String) -> Option<String> {
|
||||
if value.trim().is_empty() { None } else { Some(value) }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn app_config_from_module_json(value: serde_json::Value) -> RuntimeResult<AppConfigFile> {
|
||||
deserialize_app_config(expand_module_config_paths(value))
|
||||
}
|
||||
|
||||
fn deserialize_app_config(value: serde_json::Value) -> RuntimeResult<AppConfigFile> {
|
||||
let config: AppConfigFile =
|
||||
serde_json::from_value(value).map_err(|err| RuntimeError::json("failed to parse config file", err))?;
|
||||
if let Some(auth) = &config.auth {
|
||||
auth.validate()?;
|
||||
}
|
||||
if let Some(redis) = &config.redis {
|
||||
redis.validate()?;
|
||||
}
|
||||
if let Some(oauth) = &config.oauth {
|
||||
oauth.validate()?;
|
||||
}
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
impl OAuthConfigFile {
|
||||
fn validate(&self) -> RuntimeResult<()> {
|
||||
if let Some(oidc) = self.providers.get("oidc")
|
||||
&& !oidc.issuer.is_empty()
|
||||
&& !valid_search_endpoint(&oidc.issuer)
|
||||
{
|
||||
return Err(RuntimeError::config("invalid OIDC issuer URL"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_config_value(base: &mut serde_json::Value, overrides: serde_json::Value) {
|
||||
match (base, overrides) {
|
||||
(serde_json::Value::Object(base), serde_json::Value::Object(overrides)) => {
|
||||
for (key, value) in overrides {
|
||||
if let Some(existing) = base.get_mut(&key) {
|
||||
merge_config_value(existing, value);
|
||||
} else {
|
||||
base.insert(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
(base, overrides) => *base = overrides,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,92 @@
|
||||
use std::{path::Path, sync::Arc};
|
||||
|
||||
use super::{
|
||||
Deployment, NATIVE_APP_CONFIG_KEYS, ServerConfig, is_secret_native_app_config_key, is_static_app_config_key,
|
||||
};
|
||||
use crate::runtime::to_napi_error;
|
||||
|
||||
#[napi_derive::napi]
|
||||
pub struct ServerConfigHandle {
|
||||
pub(crate) inner: Arc<ServerConfig>,
|
||||
}
|
||||
|
||||
#[napi_derive::napi]
|
||||
impl ServerConfigHandle {
|
||||
#[napi(constructor)]
|
||||
pub fn new(path: String, legacy_deployment_type: Option<String>) -> napi::Result<Self> {
|
||||
// TODO(0.27.5): Remove this argument after legacy self-host config files
|
||||
// have deployment.type.
|
||||
let config = ServerConfig::open(Path::new(&path), legacy_deployment_type.as_deref()).map_err(to_napi_error)?;
|
||||
Ok(Self {
|
||||
inner: Arc::new(config),
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn path(&self) -> String {
|
||||
self.inner.path().to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
#[napi(getter)]
|
||||
pub fn deployment_type(&self) -> String {
|
||||
match self.inner.deployment() {
|
||||
Deployment::Cloud => "cloud",
|
||||
Deployment::SelfHosted => "selfhosted",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn node_owned_json(&self) -> napi::Result<String> {
|
||||
serde_json::to_string(&self.inner.node_owned()).map_err(|error| napi::Error::from_reason(error.to_string()))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn public_native_baseline_json(&self) -> napi::Result<String> {
|
||||
serde_json::to_string(&self.inner.public_native_baseline())
|
||||
.map_err(|error| napi::Error::from_reason(error.to_string()))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn has_baseline_config_key(&self, key: String) -> bool {
|
||||
self.inner.has_baseline_config_key(&key)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn baseline_config_key_configured(&self, key: String) -> bool {
|
||||
self.inner.baseline_config_key_configured(&key)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn native_app_config_keys(&self) -> Vec<String> {
|
||||
NATIVE_APP_CONFIG_KEYS.iter().map(|key| (*key).to_string()).collect()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn native_secret_app_config_keys(&self) -> Vec<String> {
|
||||
NATIVE_APP_CONFIG_KEYS
|
||||
.iter()
|
||||
.filter(|key| is_secret_native_app_config_key(key))
|
||||
.map(|key| (*key).to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn static_app_config_keys(&self) -> Vec<String> {
|
||||
NATIVE_APP_CONFIG_KEYS
|
||||
.iter()
|
||||
.filter(|key| is_static_app_config_key(key))
|
||||
.map(|key| (*key).to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn redis_url(&self) -> napi::Result<Option<String>> {
|
||||
self.inner.redis_url().map_err(to_napi_error)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn redis_node_options_json(&self) -> napi::Result<String> {
|
||||
serde_json::to_string(&self.inner.redis_node_options()).map_err(|error| napi::Error::from_reason(error.to_string()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::Deserialize;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::{
|
||||
PaymentProductConfig, PaymentRuntimeConfig, RevenueCatRuntimeConfig, RuntimeError, RuntimeResult,
|
||||
StripeRuntimeConfig, non_empty_string,
|
||||
};
|
||||
|
||||
#[derive(Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub(super) struct PaymentRuntimeConfigFile {
|
||||
enabled: bool,
|
||||
stripe: StripeRuntimeConfigFile,
|
||||
revenuecat: RevenueCatRuntimeConfigFile,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
struct StripeRuntimeConfigFile {
|
||||
api_key: String,
|
||||
webhook_key: String,
|
||||
account_id: String,
|
||||
environment: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
struct RevenueCatRuntimeConfigFile {
|
||||
enabled: bool,
|
||||
api_key: String,
|
||||
webhook_auth: String,
|
||||
project_id: String,
|
||||
environment: String,
|
||||
product_map: std::collections::BTreeMap<String, PaymentProductConfig>,
|
||||
}
|
||||
|
||||
impl Default for StripeRuntimeConfigFile {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
api_key: String::new(),
|
||||
webhook_key: String::new(),
|
||||
account_id: String::new(),
|
||||
environment: "test".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RevenueCatRuntimeConfigFile {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
api_key: String::new(),
|
||||
webhook_auth: String::new(),
|
||||
project_id: String::new(),
|
||||
environment: "production".to_string(),
|
||||
product_map: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PaymentRuntimeConfig {
|
||||
pub(super) fn from_file(file: Option<PaymentRuntimeConfigFile>) -> Self {
|
||||
file.map(Self::from_file_value).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(super) fn from_file_value(file: PaymentRuntimeConfigFile) -> Self {
|
||||
let stripe = non_empty_string(file.stripe.api_key).map(|api_key| StripeRuntimeConfig {
|
||||
api_key: Arc::new(Zeroizing::new(api_key)),
|
||||
webhook_key: Arc::new(Zeroizing::new(file.stripe.webhook_key)),
|
||||
account_id: file.stripe.account_id,
|
||||
live: file.stripe.environment == "live",
|
||||
});
|
||||
let revenuecat = (file.revenuecat.enabled || !file.revenuecat.api_key.trim().is_empty()).then(|| {
|
||||
let mut product_map = std::collections::BTreeMap::from([
|
||||
(
|
||||
"app.affine.pro.Monthly".to_string(),
|
||||
PaymentProductConfig {
|
||||
plan: "pro".to_string(),
|
||||
recurring: "monthly".to_string(),
|
||||
},
|
||||
),
|
||||
(
|
||||
"app.affine.pro.Annual".to_string(),
|
||||
PaymentProductConfig {
|
||||
plan: "pro".to_string(),
|
||||
recurring: "yearly".to_string(),
|
||||
},
|
||||
),
|
||||
(
|
||||
"app.affine.pro.ai.Annual".to_string(),
|
||||
PaymentProductConfig {
|
||||
plan: "ai".to_string(),
|
||||
recurring: "yearly".to_string(),
|
||||
},
|
||||
),
|
||||
]);
|
||||
product_map.extend(file.revenuecat.product_map);
|
||||
RevenueCatRuntimeConfig {
|
||||
api_key: Arc::new(Zeroizing::new(file.revenuecat.api_key)),
|
||||
webhook_auth: Arc::new(Zeroizing::new(file.revenuecat.webhook_auth)),
|
||||
project_id: file.revenuecat.project_id,
|
||||
production: file.revenuecat.environment == "production",
|
||||
product_map,
|
||||
}
|
||||
});
|
||||
Self {
|
||||
enabled: file.enabled,
|
||||
stripe,
|
||||
revenuecat,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn validate(&self) -> RuntimeResult<()> {
|
||||
if self.enabled {
|
||||
let stripe = self
|
||||
.stripe
|
||||
.as_ref()
|
||||
.ok_or_else(|| RuntimeError::config("payment.stripe.apiKey is required when payment is enabled"))?;
|
||||
if stripe.account_id.trim().is_empty() || stripe.account_id != stripe.account_id.trim() {
|
||||
return Err(RuntimeError::config(
|
||||
"payment.stripe.accountId is required when payment is enabled",
|
||||
));
|
||||
}
|
||||
if stripe.webhook_key.trim().is_empty() {
|
||||
return Err(RuntimeError::config(
|
||||
"payment.stripe.webhookKey is required when payment is enabled",
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(stripe) = &self.stripe {
|
||||
let expected_prefixes = if stripe.live {
|
||||
["sk_live_", "rk_live_"]
|
||||
} else {
|
||||
["sk_test_", "rk_test_"]
|
||||
};
|
||||
if !expected_prefixes
|
||||
.iter()
|
||||
.any(|prefix| stripe.api_key.starts_with(prefix))
|
||||
{
|
||||
return Err(RuntimeError::config(
|
||||
"payment.stripe.environment does not match the API key",
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(revenuecat) = &self.revenuecat {
|
||||
if revenuecat.api_key.is_empty() || revenuecat.project_id.trim().is_empty() {
|
||||
return Err(RuntimeError::config(
|
||||
"payment.revenuecat apiKey and projectId are required when RevenueCat is enabled",
|
||||
));
|
||||
}
|
||||
for mapping in revenuecat.product_map.values() {
|
||||
if affine_core::access_control::Plan::parse(&mapping.plan).is_none()
|
||||
|| affine_core::payment::SubscriptionRecurring::parse(&mapping.recurring).is_none()
|
||||
{
|
||||
return Err(RuntimeError::config("invalid payment.revenuecat productMap"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
use sqlx::{PgPool, Row};
|
||||
|
||||
#[cfg(test)]
|
||||
use super::{AppConfigFile, deserialize_app_config};
|
||||
use super::{BackendRuntimeConfig, RuntimeError, RuntimeResult, ServerConfig, merge_config_value};
|
||||
use crate::runtime::object_storage::{ObjectStorageService, backends_from_flat_overrides};
|
||||
|
||||
pub(crate) struct AppConfigChange {
|
||||
pub(crate) key: String,
|
||||
pub(crate) value: Option<Value>,
|
||||
pub(crate) native: bool,
|
||||
}
|
||||
|
||||
pub(super) const NATIVE_APP_CONFIG_KEYS: &[&str] = &[
|
||||
"auth.allowSignup",
|
||||
"auth.allowSignupForOauth",
|
||||
"auth.newAccountActionDelay",
|
||||
"auth.requireEmailDomainVerification",
|
||||
"auth.session.ttl",
|
||||
"auth.session.ttr",
|
||||
"auth.token.accessTokenTtl",
|
||||
"auth.token.refreshAbsoluteTtl",
|
||||
"auth.token.refreshGracePeriod",
|
||||
"auth.token.refreshIdleTtl",
|
||||
"auth.token.refreshRetention",
|
||||
"copilot.byok.allowCustomEndpoint",
|
||||
"copilot.byok.allowedProviders",
|
||||
"copilot.byok.allowPrivateEndpoint",
|
||||
"copilot.byok.enabled",
|
||||
"copilot.enabled",
|
||||
"copilot.providers.profiles",
|
||||
"copilot.storage",
|
||||
"crypto.privateKey",
|
||||
// TODO(0.27.5): Remove legacy DB override loading after old instances exit and existing rows are cleared.
|
||||
"db.datasourceUrl",
|
||||
"indexer.enabled",
|
||||
"indexer.provider.apiKey",
|
||||
"indexer.provider.endpoint",
|
||||
"indexer.provider.password",
|
||||
"indexer.provider.type",
|
||||
"indexer.provider.username",
|
||||
"oauth.providers.apple",
|
||||
"oauth.providers.github",
|
||||
"oauth.providers.google",
|
||||
"oauth.providers.oidc",
|
||||
"payment.enabled",
|
||||
"payment.revenuecat",
|
||||
"payment.stripe",
|
||||
// TODO(0.27.5): Remove legacy Redis DB override loading after old instances exit and existing rows are cleared.
|
||||
"redis.db",
|
||||
"redis.host",
|
||||
"redis.ioredis",
|
||||
"redis.password",
|
||||
"redis.port",
|
||||
"redis.username",
|
||||
"storages.avatar.storage",
|
||||
"storages.blob.storage",
|
||||
];
|
||||
|
||||
pub(super) fn is_secret_native_app_config_key(key: &str) -> bool {
|
||||
matches!(
|
||||
key,
|
||||
"copilot.providers.profiles"
|
||||
| "copilot.storage"
|
||||
| "crypto.privateKey"
|
||||
| "db.datasourceUrl"
|
||||
| "indexer.provider.apiKey"
|
||||
| "indexer.provider.password"
|
||||
| "oauth.providers.apple"
|
||||
| "oauth.providers.github"
|
||||
| "oauth.providers.google"
|
||||
| "oauth.providers.oidc"
|
||||
| "payment.revenuecat"
|
||||
| "payment.stripe"
|
||||
| "redis.ioredis"
|
||||
| "redis.password"
|
||||
| "storages.avatar.storage"
|
||||
| "storages.blob.storage"
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn is_static_app_config_key(key: &str) -> bool {
|
||||
key == "db.datasourceUrl" || key.starts_with("redis.")
|
||||
}
|
||||
|
||||
pub(super) async fn load_app_config_overrides_from_db(pool: &PgPool) -> RuntimeResult<serde_json::Value> {
|
||||
let rows = match sqlx::query("SELECT id, value FROM app_configs WHERE id = ANY($1) ORDER BY id ASC")
|
||||
.bind(NATIVE_APP_CONFIG_KEYS)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
{
|
||||
Ok(rows) => rows,
|
||||
Err(sqlx::Error::Database(err)) if err.code().as_deref() == Some("42P01") => {
|
||||
return Ok(serde_json::Value::Object(Map::new()));
|
||||
}
|
||||
Err(err) => return Err(RuntimeError::database("failed to load app config overrides", err)),
|
||||
};
|
||||
|
||||
Ok(app_config_value_from_flat_overrides(rows.into_iter().map(|row| {
|
||||
let id: String = row.get("id");
|
||||
let value: serde_json::Value = row.get("value");
|
||||
(id, value)
|
||||
})))
|
||||
}
|
||||
|
||||
pub(crate) async fn save_app_config_changes(
|
||||
pool: &PgPool,
|
||||
config: &ServerConfig,
|
||||
bootstrap_private_key: Option<String>,
|
||||
actor: Option<&str>,
|
||||
changes: &[AppConfigChange],
|
||||
) -> RuntimeResult<Vec<String>> {
|
||||
let mut transaction = pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|error| RuntimeError::database("begin app config update", error))?;
|
||||
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended('app-config-paths', 0))")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::database("lock app config paths", error))?;
|
||||
let rows = sqlx::query("SELECT id, value FROM app_configs ORDER BY id")
|
||||
.fetch_all(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::database("load app config paths", error))?;
|
||||
let mut prospective = rows
|
||||
.into_iter()
|
||||
.map(|row| (row.get::<String, _>("id"), row.get::<Value, _>("value")))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut changed = BTreeSet::new();
|
||||
for change in changes {
|
||||
if is_static_app_config_key(&change.key) && change.value.is_some() {
|
||||
return Err(RuntimeError::config(
|
||||
"database and Redis connection settings are bootstrap values and cannot be changed through app_configs",
|
||||
));
|
||||
}
|
||||
if !changed.insert(change.key.clone()) {
|
||||
return Err(RuntimeError::config(format!(
|
||||
"duplicate app config path: {}",
|
||||
change.key
|
||||
)));
|
||||
}
|
||||
if NATIVE_APP_CONFIG_KEYS.contains(&change.key.as_str()) != change.native
|
||||
|| change.key == "auth.session.signingKeys"
|
||||
{
|
||||
return Err(RuntimeError::config(format!(
|
||||
"invalid app config owner: {}",
|
||||
change.key
|
||||
)));
|
||||
}
|
||||
match &change.value {
|
||||
Some(value) => {
|
||||
prospective.insert(change.key.clone(), value.clone());
|
||||
}
|
||||
None => {
|
||||
prospective.remove(&change.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
let keys = prospective.keys().collect::<Vec<_>>();
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
if let Some(overlap) = keys[index + 1..]
|
||||
.iter()
|
||||
.find(|candidate| candidate.starts_with(&format!("{key}.")))
|
||||
{
|
||||
return Err(RuntimeError::config(format!(
|
||||
"app config paths must not overlap: {key} and {overlap}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let mut native = config.baseline().clone();
|
||||
merge_config_value(
|
||||
&mut native,
|
||||
app_config_value_from_flat_overrides(
|
||||
prospective
|
||||
.iter()
|
||||
.filter(|(key, _)| NATIVE_APP_CONFIG_KEYS.contains(&key.as_str()))
|
||||
.map(|(key, value)| (key.as_str(), value.clone())),
|
||||
),
|
||||
);
|
||||
BackendRuntimeConfig::from_value(bootstrap_private_key, native.clone(), config.deployment())?;
|
||||
ObjectStorageService::from_config_value(&native)?;
|
||||
backends_from_flat_overrides(
|
||||
prospective
|
||||
.iter()
|
||||
.filter(|(key, _)| matches!(key.as_str(), "storages.blob.storage" | "storages.avatar.storage"))
|
||||
.map(|(key, value)| (key.as_str(), value.clone())),
|
||||
)?;
|
||||
for change in changes {
|
||||
match &change.value {
|
||||
Some(value) => {
|
||||
sqlx::query(
|
||||
"INSERT INTO app_configs(id,value,last_updated_by,created_at,updated_at) \
|
||||
VALUES($1,$2,$3,clock_timestamp(),clock_timestamp()) ON CONFLICT(id) DO UPDATE SET \
|
||||
value=$2,last_updated_by=$3,updated_at=clock_timestamp()",
|
||||
)
|
||||
.bind(&change.key)
|
||||
.bind(value)
|
||||
.bind(actor)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::database("write app config", error))?;
|
||||
}
|
||||
None => {
|
||||
sqlx::query("DELETE FROM app_configs WHERE id=$1")
|
||||
.bind(&change.key)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::database("clear app config", error))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|error| RuntimeError::database("commit app config update", error))?;
|
||||
Ok(changed.into_iter().collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn app_config_from_flat_overrides<I, S>(rows: I) -> RuntimeResult<AppConfigFile>
|
||||
where
|
||||
I: IntoIterator<Item = (S, serde_json::Value)>,
|
||||
S: AsRef<str>,
|
||||
{
|
||||
deserialize_app_config(app_config_value_from_flat_overrides(rows))
|
||||
}
|
||||
|
||||
pub(super) fn app_config_value_from_flat_overrides<I, S>(rows: I) -> serde_json::Value
|
||||
where
|
||||
I: IntoIterator<Item = (S, serde_json::Value)>,
|
||||
S: AsRef<str>,
|
||||
{
|
||||
let mut root = Map::new();
|
||||
let mut rows = rows.into_iter().collect::<Vec<_>>();
|
||||
rows.sort_by(|(left, _), (right, _)| left.as_ref().cmp(right.as_ref()));
|
||||
for (path, value) in rows {
|
||||
insert_flat_override(&mut root, path.as_ref(), value);
|
||||
}
|
||||
|
||||
serde_json::Value::Object(root)
|
||||
}
|
||||
|
||||
pub(in crate::runtime) fn insert_flat_override(
|
||||
root: &mut Map<String, serde_json::Value>,
|
||||
path: &str,
|
||||
value: serde_json::Value,
|
||||
) {
|
||||
let mut parts = path.split('.').peekable();
|
||||
let mut current = root;
|
||||
while let Some(part) = parts.next() {
|
||||
if parts.peek().is_none() {
|
||||
current.insert(part.to_string(), value);
|
||||
return;
|
||||
}
|
||||
let entry = current
|
||||
.entry(part.to_string())
|
||||
.or_insert_with(|| serde_json::Value::Object(Map::new()));
|
||||
if !entry.is_object() {
|
||||
*entry = serde_json::Value::Object(Map::new());
|
||||
}
|
||||
current = entry.as_object_mut().expect("override node must be an object");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
|
||||
use super::{AppConfigChange, ServerConfig, load_app_config_overrides_from_db, save_app_config_changes};
|
||||
|
||||
#[tokio::test]
|
||||
async fn reads_only_native_app_config_keys() {
|
||||
let Ok(database_url) = std::env::var("DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("CREATE TEMP TABLE app_configs (id text PRIMARY KEY, value jsonb NOT NULL)")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO app_configs (id, value) VALUES ($1, $2), ($3, $4)")
|
||||
.bind("copilot.enabled")
|
||||
.bind(serde_json::json!(false))
|
||||
.bind("server.name")
|
||||
.bind(serde_json::json!("node-only"))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let overrides = load_app_config_overrides_from_db(&pool).await.unwrap();
|
||||
assert_eq!(overrides, serde_json::json!({ "copilot": { "enabled": false } }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn app_config_set_clear_and_failed_batch_are_atomic() {
|
||||
let Ok(database_url) = std::env::var("DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"CREATE TEMP TABLE app_configs (id text PRIMARY KEY, value jsonb NOT NULL, last_updated_by text, created_at \
|
||||
timestamptz DEFAULT clock_timestamp(), updated_at timestamptz DEFAULT clock_timestamp(), CONSTRAINT \
|
||||
fail_second CHECK (id <> 'server.second'))",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("config.json");
|
||||
std::fs::write(&path, r#"{"deployment":{"type":"cloud"},"copilot":{"enabled":false}}"#).unwrap();
|
||||
let config = ServerConfig::open(&path, None).unwrap();
|
||||
let node = |key: &str, value| AppConfigChange {
|
||||
key: key.to_string(),
|
||||
value,
|
||||
native: false,
|
||||
};
|
||||
let native = |key: &str, value| AppConfigChange {
|
||||
key: key.to_string(),
|
||||
value,
|
||||
native: true,
|
||||
};
|
||||
save_app_config_changes(
|
||||
&pool,
|
||||
&config,
|
||||
None,
|
||||
None,
|
||||
&[
|
||||
node("server.name", Some(serde_json::json!("configured"))),
|
||||
native("copilot.enabled", Some(serde_json::json!(false))),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
save_app_config_changes(
|
||||
&pool,
|
||||
&config,
|
||||
None,
|
||||
None,
|
||||
&[node("server.name", None), native("copilot.enabled", None)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert!(
|
||||
save_app_config_changes(
|
||||
&pool,
|
||||
&config,
|
||||
None,
|
||||
None,
|
||||
&[
|
||||
node("server.first", Some(serde_json::json!(true))),
|
||||
node("server.second", Some(serde_json::json!(true))),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
save_app_config_changes(
|
||||
&pool,
|
||||
&config,
|
||||
None,
|
||||
None,
|
||||
&[native("redis.host", Some(serde_json::json!("redis.example")))],
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert!(
|
||||
save_app_config_changes(&pool, &config, None, None, &[node("copilot.enabled", None)])
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
save_app_config_changes(
|
||||
&pool,
|
||||
&config,
|
||||
None,
|
||||
None,
|
||||
&[native(
|
||||
"db.datasourceUrl",
|
||||
Some(serde_json::json!("postgresql://unreachable"))
|
||||
)],
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
save_app_config_changes(
|
||||
&pool,
|
||||
&config,
|
||||
None,
|
||||
None,
|
||||
&[native(
|
||||
"storages.blob.storage",
|
||||
Some(serde_json::json!({"provider":"invalid","bucket":"blobs","config":{}}))
|
||||
)],
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
use std::fs;
|
||||
|
||||
use serde_json::Map;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn server_config_uses_one_explicit_path() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let config_path = directory.path().join("config.json");
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"selfhosted"},"copilot":{"enabled":false,"exa":{"key":"node"}},"crypto":{"privateKey":"native-only"},"redis":{"ioredis":{"enableAutoPipelining":true}},"server":{"name":"example"}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let config = ServerConfig::open(&config_path, None).unwrap();
|
||||
assert!(ServerConfig::open(&config_path, Some("unknown")).is_ok());
|
||||
assert_eq!(config.path(), config_path.canonicalize().unwrap());
|
||||
assert!(config.deployment() == Deployment::SelfHosted);
|
||||
assert_eq!(config.baseline()["copilot"]["enabled"], false);
|
||||
assert_eq!(
|
||||
config.node_owned(),
|
||||
serde_json::json!({"copilot":{"exa":{"key":"node"}},"server":{"name":"example"}})
|
||||
);
|
||||
assert_eq!(
|
||||
config.redis_node_options(),
|
||||
serde_json::json!({"enableAutoPipelining":true})
|
||||
);
|
||||
assert_eq!(
|
||||
config.public_native_baseline(),
|
||||
serde_json::json!({"copilot":{"enabled":false}})
|
||||
);
|
||||
let native = BackendRuntimeConfig::from_server_config(None, &config).unwrap();
|
||||
assert!(native.deployment == Deployment::SelfHosted);
|
||||
assert_eq!(native.private_key.as_str(), "native-only");
|
||||
assert_eq!(native.redis.url, config.redis_url().unwrap());
|
||||
let storage = crate::runtime::storage_runtime::StorageRuntimeConfig::from_server_config(&config).unwrap();
|
||||
assert_eq!(native.database_url, storage.database_url);
|
||||
assert!(storage.object_storage.backends.contains_key("blob"));
|
||||
fs::write(&config_path, r#"{"copilot":{"enabled":false}}"#).unwrap();
|
||||
assert!(ServerConfig::open(&config_path, None).is_err());
|
||||
let upgraded = ServerConfig::open(&config_path, Some("selfhosted")).unwrap();
|
||||
assert!(upgraded.deployment() == Deployment::SelfHosted);
|
||||
assert!(ServerConfig::open(&config_path, Some("unknown")).is_err());
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"selfhosted"},"copilot":{"enabled":"yes"}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(ServerConfig::open(&config_path, None).is_err());
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"selfhosted"},"indexer":{"provider":{"type":"unknown"}}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(ServerConfig::open(&config_path, None).is_err());
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"selfhosted"},"indexer":{"provider":{"type":"elasticsearch","endpoint":"invalid"}}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let config = ServerConfig::open(&config_path, None).unwrap();
|
||||
assert!(BackendRuntimeConfig::from_server_config(None, &config).is_err());
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"selfhosted"},"auth":{"token":{"accessTokenTtl":30}}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(ServerConfig::open(&config_path, None).is_err());
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"selfhosted"},"redis":{"port":0}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(ServerConfig::open(&config_path, None).is_err());
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"selfhosted"},"oauth":{"providers":{"oidc":{"issuer":"file:///etc/passwd"}}}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(ServerConfig::open(&config_path, None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_database_urls_are_ignored() {
|
||||
assert_eq!(non_empty_string("".to_string()), None);
|
||||
assert_eq!(non_empty_string(" ".to_string()), None);
|
||||
assert_eq!(
|
||||
non_empty_string("postgresql://affine:affine@localhost:5432/affine".to_string()),
|
||||
Some("postgresql://affine:affine@localhost:5432/affine".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_storage_app_config_values() {
|
||||
let app_config = app_config_from_flat_overrides([
|
||||
(
|
||||
"storages.blob.storage",
|
||||
serde_json::json!({"provider": "cloudflare-r2"}),
|
||||
),
|
||||
("db.datasourceUrl", serde_json::json!("postgresql://example/runtime")),
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
app_config.database_url().as_deref(),
|
||||
Some("postgresql://example/runtime")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expands_module_config_paths_from_json_files() {
|
||||
let app_config = app_config_from_module_json(serde_json::json!({
|
||||
"copilot": {
|
||||
"enabled": true,
|
||||
"byok.enabled": false,
|
||||
"providers.profiles": [{
|
||||
"id": "managed-openai",
|
||||
"type": "openai",
|
||||
"models": ["gpt-5.6-luna"],
|
||||
"config": { "apiKey": "test" }
|
||||
}]
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap();
|
||||
|
||||
assert!(copilot.enabled);
|
||||
assert!(!copilot.byok.enabled);
|
||||
assert_eq!(copilot.providers.profiles.len(), 1);
|
||||
assert_eq!(copilot.providers.profiles[0].id, "managed-openai");
|
||||
|
||||
let missing_models = app_config_from_flat_overrides([(
|
||||
"copilot.providers.profiles",
|
||||
serde_json::json!([{
|
||||
"id": "managed-openai",
|
||||
"type": "openai",
|
||||
"config": {}
|
||||
}]),
|
||||
)]);
|
||||
assert!(missing_models.is_err());
|
||||
|
||||
let app_config = app_config_from_flat_overrides([(
|
||||
"copilot.providers.profiles",
|
||||
serde_json::json!([{
|
||||
"id": "managed-openai",
|
||||
"type": "openai",
|
||||
"models": [],
|
||||
"config": {}
|
||||
}]),
|
||||
)])
|
||||
.unwrap();
|
||||
let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap();
|
||||
assert!(validate_copilot_config(&copilot).is_err());
|
||||
|
||||
let app_config = app_config_from_flat_overrides([(
|
||||
"copilot.providers.profiles",
|
||||
serde_json::json!([
|
||||
{
|
||||
"id": "anthropic-direct",
|
||||
"type": "anthropic",
|
||||
"models": ["claude-sonnet-4-6"],
|
||||
"config": {}
|
||||
},
|
||||
{
|
||||
"id": "anthropic-vertex",
|
||||
"type": "anthropicVertex",
|
||||
"models": ["claude-sonnet-4-6"],
|
||||
"config": {}
|
||||
}
|
||||
]),
|
||||
)])
|
||||
.unwrap();
|
||||
let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap();
|
||||
assert!(validate_copilot_config(&copilot).is_err());
|
||||
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let config_path = directory.path().join("config.json");
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"deployment":{"type":"cloud"},"copilot":{"enabled":true,"byok.enabled":false,"byok.allowCustomEndpoint":true}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let config = ServerConfig::open(&config_path, None).unwrap();
|
||||
let copilot = BackendRuntimeConfig::from_server_config(None, &config).unwrap().copilot;
|
||||
assert!(!copilot.byok.enabled);
|
||||
assert!(copilot.byok.allow_custom_endpoint);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_config_keeps_disabled_state_separate_from_embedded_provider() {
|
||||
let disabled = app_config_from_flat_overrides([
|
||||
("indexer.enabled", serde_json::json!(false)),
|
||||
("indexer.provider.type", serde_json::json!("embedded")),
|
||||
])
|
||||
.unwrap();
|
||||
let disabled: SearchRuntimeConfig = disabled.indexer.unwrap().into();
|
||||
assert!(!disabled.enabled);
|
||||
assert_eq!(disabled.provider, "embedded");
|
||||
|
||||
let enabled = app_config_from_flat_overrides([
|
||||
("indexer.enabled", serde_json::json!(true)),
|
||||
("indexer.provider.type", serde_json::json!("elasticsearch")),
|
||||
])
|
||||
.unwrap();
|
||||
let enabled: SearchRuntimeConfig = enabled.indexer.unwrap().into();
|
||||
assert!(enabled.enabled);
|
||||
assert_eq!(enabled.provider, "elasticsearch");
|
||||
|
||||
let enabled_without_provider = app_config_from_module_json(serde_json::json!({
|
||||
"indexer": { "enabled": true }
|
||||
}))
|
||||
.unwrap();
|
||||
let enabled_without_provider: SearchRuntimeConfig = enabled_without_provider.indexer.unwrap().into();
|
||||
assert!(enabled_without_provider.enabled);
|
||||
assert_eq!(enabled_without_provider.provider, "embedded");
|
||||
|
||||
let manticore = app_config_from_flat_overrides([
|
||||
("indexer.enabled", serde_json::json!(true)),
|
||||
("indexer.provider.type", serde_json::json!("manticoresearch")),
|
||||
("indexer.provider.endpoint", serde_json::json!("http://localhost:9308")),
|
||||
])
|
||||
.unwrap();
|
||||
let manticore: SearchRuntimeConfig = manticore.indexer.unwrap().into();
|
||||
assert!(manticore.enabled);
|
||||
assert_eq!(manticore.provider, "manticoresearch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_database_config_preserves_file_config_siblings() {
|
||||
let mut file_config = expand_module_config_paths(serde_json::json!({
|
||||
"copilot": {
|
||||
"enabled": true,
|
||||
"byok": { "enabled": true, "allowCustomEndpoint": true },
|
||||
"providers": {
|
||||
"profiles": [{
|
||||
"id": "managed-openai",
|
||||
"type": "openai",
|
||||
"models": ["gpt-5.6-luna"],
|
||||
"config": { "apiKey": "test" }
|
||||
}]
|
||||
}
|
||||
}
|
||||
}));
|
||||
let database_config = app_config_value_from_flat_overrides([("copilot.byok.enabled", serde_json::json!(false))]);
|
||||
|
||||
merge_config_value(&mut file_config, database_config);
|
||||
let copilot: CopilotRuntimeConfig = deserialize_app_config(file_config)
|
||||
.unwrap()
|
||||
.copilot
|
||||
.unwrap()
|
||||
.try_into()
|
||||
.unwrap();
|
||||
|
||||
assert!(copilot.enabled);
|
||||
assert!(!copilot.byok.enabled);
|
||||
assert!(copilot.byok.allow_custom_endpoint);
|
||||
assert_eq!(copilot.providers.profiles.len(), 1);
|
||||
assert_eq!(copilot.providers.profiles[0].id, "managed-openai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_database_config_overrides_are_order_independent() {
|
||||
let app_config = app_config_from_flat_overrides([
|
||||
("copilot.byok.enabled", serde_json::json!(false)),
|
||||
(
|
||||
"copilot.byok",
|
||||
serde_json::json!({ "enabled": true, "allowCustomEndpoint": true }),
|
||||
),
|
||||
])
|
||||
.unwrap();
|
||||
let byok = CopilotRuntimeConfig::try_from(app_config.copilot.unwrap())
|
||||
.unwrap()
|
||||
.byok;
|
||||
|
||||
assert!(!byok.enabled);
|
||||
assert!(byok.allow_custom_endpoint);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_config_only_replaces_an_active_private_key_explicitly() {
|
||||
let active = BackendRuntimeConfig {
|
||||
database_url: "postgresql://active".to_string(),
|
||||
auth: AuthRuntimeConfig::default(),
|
||||
invite_quota: InviteQuotaConfig::default(),
|
||||
private_key: Arc::new(Zeroizing::new("active-private-key".to_string())),
|
||||
deployment: Deployment::Cloud,
|
||||
copilot: CopilotRuntimeConfig::default(),
|
||||
search: SearchRuntimeConfig::default(),
|
||||
redis: RedisRuntimeConfig::default(),
|
||||
payment: PaymentRuntimeConfig::default(),
|
||||
};
|
||||
let empty = serde_json::Value::Object(Map::new());
|
||||
|
||||
let unchanged = active.apply_db_overrides(empty.clone(), empty.clone()).unwrap();
|
||||
assert_eq!(unchanged.private_key.as_str(), "active-private-key");
|
||||
|
||||
let overridden = active
|
||||
.apply_db_overrides(
|
||||
empty,
|
||||
app_config_value_from_flat_overrides([("crypto.privateKey", serde_json::json!("database-private-key"))]),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(overridden.private_key.as_str(), "database-private-key");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invite_abuse_policy_is_internal_while_action_delay_is_configurable() {
|
||||
let app_config = app_config_from_flat_overrides([
|
||||
("auth.newAccountActionDelay", serde_json::json!(123)),
|
||||
("auth.untrustedPolicyOverride", serde_json::json!("runtime-salt-v2")),
|
||||
("auth.untrustedDomainList", serde_json::json!(["Example.COM."])),
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let config = app_config.invite_quota_config();
|
||||
assert_eq!(config.new_account_action_delay_seconds, 123);
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::Deserialize;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::CopilotRuntimeConfig;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum Deployment {
|
||||
Cloud,
|
||||
SelfHosted,
|
||||
}
|
||||
|
||||
pub(crate) struct BackendRuntimeConfig {
|
||||
pub(crate) database_url: String,
|
||||
pub(crate) auth: AuthRuntimeConfig,
|
||||
pub(crate) invite_quota: InviteQuotaConfig,
|
||||
pub(crate) private_key: Arc<Zeroizing<String>>,
|
||||
pub(crate) deployment: Deployment,
|
||||
pub(crate) copilot: CopilotRuntimeConfig,
|
||||
pub(crate) search: SearchRuntimeConfig,
|
||||
pub(crate) redis: RedisRuntimeConfig,
|
||||
pub(crate) payment: PaymentRuntimeConfig,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct AuthRuntimeConfig {
|
||||
pub(crate) allow_signup: bool,
|
||||
pub(crate) allow_signup_for_oauth: bool,
|
||||
pub(crate) require_email_domain_verification: bool,
|
||||
pub(crate) session_ttl_seconds: i64,
|
||||
pub(crate) session_ttr_seconds: i64,
|
||||
pub(crate) access_token_ttl_seconds: i64,
|
||||
pub(crate) refresh_idle_ttl_seconds: i64,
|
||||
pub(crate) refresh_absolute_ttl_seconds: i64,
|
||||
pub(crate) refresh_grace_seconds: i64,
|
||||
pub(crate) refresh_retention_seconds: i64,
|
||||
pub(crate) oauth: OAuthRuntimeConfig,
|
||||
}
|
||||
|
||||
impl Default for AuthRuntimeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
allow_signup: true,
|
||||
allow_signup_for_oauth: true,
|
||||
require_email_domain_verification: false,
|
||||
session_ttl_seconds: 15 * 24 * 60 * 60,
|
||||
session_ttr_seconds: 7 * 24 * 60 * 60,
|
||||
access_token_ttl_seconds: 15 * 60,
|
||||
refresh_idle_ttl_seconds: 30 * 24 * 60 * 60,
|
||||
refresh_absolute_ttl_seconds: 180 * 24 * 60 * 60,
|
||||
refresh_grace_seconds: 30,
|
||||
refresh_retention_seconds: 30 * 24 * 60 * 60,
|
||||
oauth: OAuthRuntimeConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct OAuthRuntimeConfig {
|
||||
pub(crate) providers: std::collections::BTreeMap<String, OAuthProviderRuntimeConfig>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct OAuthProviderRuntimeConfig {
|
||||
pub(crate) client_id: String,
|
||||
pub(crate) client_secret: Arc<Zeroizing<String>>,
|
||||
pub(crate) args: std::collections::BTreeMap<String, String>,
|
||||
pub(crate) issuer: Option<String>,
|
||||
pub(crate) allow_private_network: bool,
|
||||
pub(crate) apple_private_key: Option<Arc<Zeroizing<String>>>,
|
||||
pub(crate) apple_key_id: Option<String>,
|
||||
pub(crate) apple_team_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct PaymentRuntimeConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) stripe: Option<StripeRuntimeConfig>,
|
||||
pub(crate) revenuecat: Option<RevenueCatRuntimeConfig>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct StripeRuntimeConfig {
|
||||
pub(crate) api_key: Arc<Zeroizing<String>>,
|
||||
pub(crate) webhook_key: Arc<Zeroizing<String>>,
|
||||
pub(crate) account_id: String,
|
||||
pub(crate) live: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct RevenueCatRuntimeConfig {
|
||||
pub(crate) api_key: Arc<Zeroizing<String>>,
|
||||
pub(crate) webhook_auth: Arc<Zeroizing<String>>,
|
||||
pub(crate) project_id: String,
|
||||
pub(crate) production: bool,
|
||||
pub(crate) product_map: std::collections::BTreeMap<String, PaymentProductConfig>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct PaymentProductConfig {
|
||||
pub(crate) plan: String,
|
||||
pub(crate) recurring: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct RedisRuntimeConfig {
|
||||
pub(crate) url: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct SearchRuntimeConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) provider: String,
|
||||
pub(crate) endpoint: String,
|
||||
pub(crate) api_key: String,
|
||||
pub(crate) username: String,
|
||||
pub(crate) password: String,
|
||||
}
|
||||
|
||||
impl Default for SearchRuntimeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
provider: "embedded".to_string(),
|
||||
endpoint: String::new(),
|
||||
api_key: String::new(),
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct InviteQuotaConfig {
|
||||
pub(crate) new_account_action_delay_seconds: i64,
|
||||
}
|
||||
|
||||
impl Default for InviteQuotaConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
new_account_action_delay_seconds: 24 * 60 * 60,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
use jsonschema::Draft;
|
||||
use napi::{Error, Result, Status};
|
||||
use schemars::{JsonSchema, generate::SchemaSettings};
|
||||
use serde_json::{Value, from_value, json, to_value};
|
||||
|
||||
use super::{
|
||||
CopilotManagedProfileConfigFile, CopilotRuntimeConfig, CopilotRuntimeConfigFile, RuntimeError,
|
||||
SUPPORTED_BYOK_PROVIDERS, validate_copilot_config,
|
||||
};
|
||||
|
||||
const COPILOT_MODULE: &str = "copilot";
|
||||
|
||||
#[napi_derive::napi(object)]
|
||||
pub struct AppConfigDescriptor {
|
||||
pub key: String,
|
||||
pub description: String,
|
||||
pub default_value: Value,
|
||||
pub schema: Value,
|
||||
pub internal: bool,
|
||||
}
|
||||
|
||||
fn invalid_config(message: impl Into<String>) -> Error {
|
||||
Error::new(Status::InvalidArg, message.into())
|
||||
}
|
||||
|
||||
fn schema_for<T: JsonSchema>() -> Value {
|
||||
let schema = SchemaSettings::draft07().into_generator().into_root_schema_for::<T>();
|
||||
to_value(schema).expect("config schema should serialize")
|
||||
}
|
||||
|
||||
fn descriptors() -> Vec<AppConfigDescriptor> {
|
||||
let defaults = CopilotRuntimeConfigFile::default();
|
||||
let mut allowed_providers_schema = schema_for::<Vec<String>>();
|
||||
allowed_providers_schema["items"]["enum"] = json!(SUPPORTED_BYOK_PROVIDERS);
|
||||
|
||||
vec![
|
||||
AppConfigDescriptor {
|
||||
key: "byok.enabled".to_string(),
|
||||
description: "Allow workspace owners and admins to configure AI provider keys through AI BYOK.".to_string(),
|
||||
default_value: json!(defaults.byok.enabled),
|
||||
schema: schema_for::<bool>(),
|
||||
internal: false,
|
||||
},
|
||||
AppConfigDescriptor {
|
||||
key: "byok.allowedProviders".to_string(),
|
||||
description: "AI providers that workspace owners and admins may add through AI BYOK.".to_string(),
|
||||
default_value: json!(defaults.byok.allowed_providers),
|
||||
schema: allowed_providers_schema,
|
||||
internal: false,
|
||||
},
|
||||
AppConfigDescriptor {
|
||||
key: "byok.allowCustomEndpoint".to_string(),
|
||||
description: "Allow AI BYOK keys to use a custom provider endpoint.".to_string(),
|
||||
default_value: json!(defaults.byok.allow_custom_endpoint),
|
||||
schema: schema_for::<bool>(),
|
||||
internal: false,
|
||||
},
|
||||
AppConfigDescriptor {
|
||||
key: "byok.allowPrivateEndpoint".to_string(),
|
||||
description: "Whether workspace BYOK custom endpoints may resolve to private network targets. Enabling this \
|
||||
allows workspace owners and admins to send provider probe requests to the private network."
|
||||
.to_string(),
|
||||
default_value: json!(defaults.byok.allow_private_endpoint),
|
||||
schema: schema_for::<bool>(),
|
||||
internal: false,
|
||||
},
|
||||
AppConfigDescriptor {
|
||||
key: "providers.profiles".to_string(),
|
||||
description: "The profile list for copilot providers.".to_string(),
|
||||
default_value: json!(defaults.providers.profiles),
|
||||
schema: schema_for::<Vec<CopilotManagedProfileConfigFile>>(),
|
||||
internal: true,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn validate_leaf(key: &str, value: Value) -> std::result::Result<(), RuntimeError> {
|
||||
let mut config = CopilotRuntimeConfigFile::default();
|
||||
match key {
|
||||
"byok.enabled" => {
|
||||
config.byok.enabled =
|
||||
from_value(value).map_err(|error| RuntimeError::json("invalid copilot BYOK enabled config", error))?;
|
||||
}
|
||||
"byok.allowedProviders" => {
|
||||
config.byok.allowed_providers = from_value(value)
|
||||
.map_err(|error| RuntimeError::json("invalid copilot BYOK allowed providers config", error))?;
|
||||
}
|
||||
"byok.allowCustomEndpoint" => {
|
||||
config.byok.allow_custom_endpoint =
|
||||
from_value(value).map_err(|error| RuntimeError::json("invalid copilot BYOK custom endpoint config", error))?;
|
||||
}
|
||||
"byok.allowPrivateEndpoint" => {
|
||||
config.byok.allow_private_endpoint =
|
||||
from_value(value).map_err(|error| RuntimeError::json("invalid copilot BYOK private endpoint config", error))?;
|
||||
}
|
||||
"providers.profiles" => {
|
||||
config.providers.profiles =
|
||||
from_value(value).map_err(|error| RuntimeError::json("invalid managed copilot profiles config", error))?;
|
||||
}
|
||||
_ => return Err(RuntimeError::config(format!("unknown copilot app config key: {key}"))),
|
||||
}
|
||||
let config = CopilotRuntimeConfig::try_from(config)?;
|
||||
validate_copilot_config(&config)
|
||||
}
|
||||
|
||||
#[napi_derive::napi(catch_unwind)]
|
||||
pub fn app_config_descriptors(module: String) -> Result<Vec<AppConfigDescriptor>> {
|
||||
if module != COPILOT_MODULE {
|
||||
return Err(invalid_config(format!("unknown native app config module: {module}")));
|
||||
}
|
||||
Ok(descriptors())
|
||||
}
|
||||
|
||||
#[napi_derive::napi(catch_unwind)]
|
||||
pub fn validate_app_config_value(module: String, key: String, value: Value) -> Result<Vec<String>> {
|
||||
if module != COPILOT_MODULE {
|
||||
return Err(invalid_config(format!("unknown native app config module: {module}")));
|
||||
}
|
||||
let descriptor = descriptors()
|
||||
.into_iter()
|
||||
.find(|descriptor| descriptor.key == key)
|
||||
.ok_or_else(|| invalid_config(format!("unknown native app config key: {module}.{key}")))?;
|
||||
let schema = jsonschema::options()
|
||||
.with_draft(Draft::Draft7)
|
||||
.build(&descriptor.schema)
|
||||
.map_err(|error| invalid_config(format!("failed to compile app config schema: {error}")))?;
|
||||
let errors = schema
|
||||
.iter_errors(&value)
|
||||
.map(|error| error.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
if !errors.is_empty() {
|
||||
return Ok(errors);
|
||||
}
|
||||
Ok(
|
||||
validate_leaf(&key, value)
|
||||
.err()
|
||||
.map(|error| vec![error.to_string()])
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use super::{app_config_descriptors, json, validate_app_config_value};
|
||||
|
||||
#[test]
|
||||
fn copilot_descriptors_and_validation_share_runtime_contract() {
|
||||
let descriptors = app_config_descriptors("copilot".to_string()).unwrap();
|
||||
assert_eq!(
|
||||
descriptors
|
||||
.iter()
|
||||
.map(|descriptor| descriptor.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
"byok.enabled",
|
||||
"byok.allowedProviders",
|
||||
"byok.allowCustomEndpoint",
|
||||
"byok.allowPrivateEndpoint",
|
||||
"providers.profiles",
|
||||
]
|
||||
);
|
||||
assert_eq!(descriptors[0].default_value, json!(true));
|
||||
assert!(descriptors[4].internal);
|
||||
assert!(
|
||||
validate_app_config_value(
|
||||
"copilot".to_string(),
|
||||
"providers.profiles".to_string(),
|
||||
json!([{
|
||||
"id": "managed-openai",
|
||||
"type": "openai",
|
||||
"displayName": "OpenAI",
|
||||
"priority": 1,
|
||||
"enabled": true,
|
||||
"models": ["gpt-5.6-luna"],
|
||||
"middleware": {
|
||||
"rust": { "request": ["normalize_messages"] },
|
||||
"node": { "text": ["citation_footnote"] }
|
||||
},
|
||||
"config": { "apiKey": "test" }
|
||||
}]),
|
||||
)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copilot_validation_rejects_invalid_leaf_values() {
|
||||
for (key, value) in [
|
||||
("byok.enabled", json!("yes")),
|
||||
("byok.allowedProviders", json!(["openai", "openai"])),
|
||||
(
|
||||
"providers.profiles",
|
||||
json!([{
|
||||
"id": "invalid id",
|
||||
"type": "openai",
|
||||
"models": ["gpt-5.6-luna"],
|
||||
"config": {}
|
||||
}]),
|
||||
),
|
||||
(
|
||||
"providers.profiles",
|
||||
json!([{
|
||||
"id": "managed-openai",
|
||||
"type": "openai",
|
||||
"models": ["gpt-5.6-luna"],
|
||||
"middleware": { "node": { "text": ["unknown"] } },
|
||||
"config": {}
|
||||
}]),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!validate_app_config_value("copilot".to_string(), key.to_string(), value)
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"{key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@ pub mod backend_runtime;
|
||||
pub mod storage_runtime;
|
||||
|
||||
pub(crate) mod config;
|
||||
mod config_descriptor;
|
||||
pub use config::ServerConfigHandle;
|
||||
pub(crate) mod error;
|
||||
mod http;
|
||||
pub(crate) mod migrations;
|
||||
@@ -14,14 +14,12 @@ pub(crate) mod types;
|
||||
pub(super) use affine_doc_loader::blob_refs::{BlobRefProjectionError, extract_blob_refs};
|
||||
#[cfg(test)]
|
||||
pub(crate) use config::AuthRuntimeConfig;
|
||||
pub use config::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value};
|
||||
pub(crate) use config::{
|
||||
BackendRuntimeConfig, ConfigSource, CopilotManagedProfileConfig, CopilotManagedProfileConfigFile,
|
||||
CopilotRuntimeConfig, CopilotRuntimeConfigFile, Deployment, InviteQuotaConfig, OAuthProviderRuntimeConfig,
|
||||
PaymentProductConfig, PaymentRuntimeConfig, RedisRuntimeConfig, RevenueCatRuntimeConfig, SearchRuntimeConfig,
|
||||
StripeRuntimeConfig,
|
||||
BackendRuntimeConfig, CopilotManagedProfileConfig, CopilotRuntimeConfig, Deployment, InviteQuotaConfig,
|
||||
OAuthProviderRuntimeConfig, PaymentProductConfig, PaymentRuntimeConfig, RedisRuntimeConfig, RevenueCatRuntimeConfig,
|
||||
SearchRuntimeConfig, StripeRuntimeConfig,
|
||||
};
|
||||
use config::{SUPPORTED_BYOK_PROVIDERS, validate_copilot_config};
|
||||
pub use config_descriptor::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value};
|
||||
pub(crate) use error::{RuntimeError, RuntimeResult, napi_error, to_napi_error};
|
||||
pub(in crate::runtime) use http::webpki_tls_config;
|
||||
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
use std::{collections::HashMap, fs};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
use sqlx::{PgPool, Row};
|
||||
|
||||
use super::{config::ObjectStorageConfig, types::StorageProviderConfig};
|
||||
use crate::runtime::{ConfigSource, RuntimeError, RuntimeResult};
|
||||
use crate::runtime::{RuntimeError, RuntimeResult};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(in crate::runtime) enum StorageBackendConfig {
|
||||
Fs(FsStorageConfig),
|
||||
S3(ObjectStorageConfig),
|
||||
S3(Box<ObjectStorageConfig>),
|
||||
Assetpack(FsStorageConfig),
|
||||
}
|
||||
|
||||
@@ -39,7 +39,9 @@ struct CopilotConfigFile {
|
||||
}
|
||||
|
||||
impl StorageBackendConfig {
|
||||
fn from_provider_config(storage: Option<StorageProviderConfig>) -> RuntimeResult<Option<Self>> {
|
||||
pub(in crate::runtime) fn from_provider_config(
|
||||
storage: Option<StorageProviderConfig>,
|
||||
) -> RuntimeResult<Option<Self>> {
|
||||
let Some(storage) = storage else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -60,7 +62,7 @@ impl StorageBackendConfig {
|
||||
}))
|
||||
}
|
||||
"aws-s3" | "cloudflare-r2" => ObjectStorageConfig::from_provider_config(Some(storage))
|
||||
.map(|config| config.map(Self::S3))
|
||||
.map(|config| config.map(|config| Self::S3(Box::new(config))))
|
||||
.map_err(Into::into),
|
||||
provider => Err(RuntimeError::config(format!(
|
||||
"unsupported object storage provider: {provider}"
|
||||
@@ -124,56 +126,34 @@ impl ObjectStorageAppConfig {
|
||||
}
|
||||
|
||||
fn default_object_storage_config() -> ObjectStorageAppConfig {
|
||||
let storage = |bucket: &str| {
|
||||
serde_json::json!({
|
||||
"provider": "fs",
|
||||
"bucket": bucket,
|
||||
"config": { "path": "~/.affine/storage" }
|
||||
})
|
||||
};
|
||||
|
||||
ObjectStorageAppConfig {
|
||||
storages: Some(HashMap::from([
|
||||
("blob.storage".to_string(), storage("blobs")),
|
||||
("avatar.storage".to_string(), storage("avatars")),
|
||||
("blob.storage".to_string(), default_storage_provider_config("blobs")),
|
||||
("avatar.storage".to_string(), default_storage_provider_config("avatars")),
|
||||
])),
|
||||
copilot: Some(CopilotConfigFile {
|
||||
storage: Some(StorageProviderConfig {
|
||||
provider: "fs".to_string(),
|
||||
bucket: "copilot".to_string(),
|
||||
config: serde_json::json!({ "path": "~/.affine/storage" }),
|
||||
}),
|
||||
storage: Some(serde_json::from_value(default_storage_provider_config("copilot")).expect("storage default")),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn backends_from_config_files() -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
|
||||
backends_from_config_source(&ConfigSource::default())
|
||||
}
|
||||
|
||||
pub(super) fn backends_from_config_source(
|
||||
source: &ConfigSource,
|
||||
) -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
|
||||
let mut merged = default_object_storage_config();
|
||||
for path in source.paths() {
|
||||
if !path.exists() {
|
||||
if source.required(&path) {
|
||||
return Err(RuntimeError::config(format!(
|
||||
"config file does not exist: {}",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let raw = fs::read_to_string(&path).map_err(|err| RuntimeError::io("failed to read config file", err))?;
|
||||
let config = serde_json::from_str(&raw).map_err(|err| RuntimeError::json("failed to parse config file", err))?;
|
||||
merged.merge(config);
|
||||
}
|
||||
merged.storage_backends()
|
||||
pub(in crate::runtime) fn default_storage_provider_config(bucket: &str) -> Value {
|
||||
serde_json::json!({
|
||||
"provider": "fs",
|
||||
"bucket": bucket,
|
||||
"config": { "path": "~/.affine/storage" }
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn backends_from_config_json(config_json: &str) -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
|
||||
let config = serde_json::from_str::<ObjectStorageAppConfig>(config_json)
|
||||
let value =
|
||||
serde_json::from_str(config_json).map_err(|err| RuntimeError::json("invalid object storage config", err))?;
|
||||
backends_from_config_value(&value)
|
||||
}
|
||||
|
||||
pub(super) fn backends_from_config_value(value: &Value) -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
|
||||
let config = serde_json::from_value::<ObjectStorageAppConfig>(value.clone())
|
||||
.map_err(|err| RuntimeError::json("invalid object storage config", err))?;
|
||||
let mut merged = default_object_storage_config();
|
||||
merged.merge(config);
|
||||
@@ -181,15 +161,34 @@ pub(super) fn backends_from_config_json(config_json: &str) -> RuntimeResult<Hash
|
||||
}
|
||||
|
||||
pub(super) async fn backends_from_db(pool: &PgPool) -> RuntimeResult<HashMap<String, StorageBackendConfig>> {
|
||||
let rows = match sqlx::query("SELECT id, value FROM app_configs").fetch_all(pool).await {
|
||||
let rows = match sqlx::query(
|
||||
"SELECT id, value FROM app_configs WHERE id IN ('storages.blob.storage', 'storages.avatar.storage', \
|
||||
'copilot.storage')",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
{
|
||||
Ok(rows) => rows,
|
||||
Err(sqlx::Error::Database(err)) if err.code().as_deref() == Some("42P01") => return Ok(HashMap::new()),
|
||||
Err(err) => return Err(RuntimeError::database("failed to load app config overrides", err)),
|
||||
};
|
||||
backends_from_flat_overrides(
|
||||
rows
|
||||
.into_iter()
|
||||
.map(|row| (row.get::<String, _>("id"), row.get::<Value, _>("value"))),
|
||||
)
|
||||
}
|
||||
|
||||
pub(in crate::runtime) fn backends_from_flat_overrides<I, S>(
|
||||
rows: I,
|
||||
) -> RuntimeResult<HashMap<String, StorageBackendConfig>>
|
||||
where
|
||||
I: IntoIterator<Item = (S, Value)>,
|
||||
S: AsRef<str>,
|
||||
{
|
||||
let mut root = Map::new();
|
||||
for row in rows {
|
||||
let path: String = row.get("id");
|
||||
let value: Value = row.get("value");
|
||||
for (path, value) in rows {
|
||||
let path = path.as_ref();
|
||||
let Some((module, key)) = path.split_once('.') else {
|
||||
continue;
|
||||
};
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use base64::{
|
||||
Engine as _,
|
||||
engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD},
|
||||
};
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use rusty_s3::{Bucket, Credentials, UrlStyle};
|
||||
use schemars::{JsonSchema, generate::SchemaSettings};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json, to_value};
|
||||
use sha2::Sha256;
|
||||
use url::Url;
|
||||
|
||||
@@ -31,15 +36,17 @@ pub(crate) struct ObjectStorageConfig {
|
||||
pub(crate) presign_sign_content_type_for_put: Option<bool>,
|
||||
pub(crate) use_presigned_url: bool,
|
||||
pub(crate) proxy_upload: bool,
|
||||
pub(crate) upload_url_prefix: Option<String>,
|
||||
pub(crate) proxy_sign_key: Option<String>,
|
||||
pub(crate) custom_get_url_prefix: Option<String>,
|
||||
pub(crate) custom_get_sign_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct S3ConfigFile {
|
||||
endpoint: Option<String>,
|
||||
region: Option<String>,
|
||||
region: String,
|
||||
credentials: Option<S3CredentialsConfigFile>,
|
||||
force_path_style: Option<bool>,
|
||||
request_timeout_ms: Option<u64>,
|
||||
@@ -49,7 +56,7 @@ struct S3ConfigFile {
|
||||
use_presigned_url: Option<UsePresignedUrlConfigFile>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct R2ConfigFile {
|
||||
account_id: String,
|
||||
@@ -63,14 +70,14 @@ struct R2ConfigFile {
|
||||
use_presigned_url: Option<UsePresignedUrlConfigFile>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
enum R2Jurisdiction {
|
||||
Default,
|
||||
Eu,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
#[derive(Debug, Deserialize, Default, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct S3CredentialsConfigFile {
|
||||
access_key_id: Option<String>,
|
||||
@@ -78,14 +85,14 @@ struct S3CredentialsConfigFile {
|
||||
session_token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct S3PresignConfigFile {
|
||||
expires_in_seconds: Option<u64>,
|
||||
sign_content_type_for_put: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct UsePresignedUrlConfigFile {
|
||||
enabled: bool,
|
||||
@@ -93,6 +100,59 @@ struct UsePresignedUrlConfigFile {
|
||||
sign_key: Option<String>,
|
||||
}
|
||||
|
||||
pub(in crate::runtime) fn storage_provider_schema() -> Value {
|
||||
let mut settings = SchemaSettings::draft07();
|
||||
settings.inline_subschemas = true;
|
||||
let s3 = to_value(settings.clone().into_generator().into_root_schema_for::<S3ConfigFile>())
|
||||
.expect("S3 schema should serialize");
|
||||
let r2 =
|
||||
to_value(settings.into_generator().into_root_schema_for::<R2ConfigFile>()).expect("R2 schema should serialize");
|
||||
let storage = |provider: &str, config: Value| {
|
||||
json!({
|
||||
"type": "object",
|
||||
"required": ["provider", "bucket", "config"],
|
||||
"properties": {
|
||||
"provider": { "const": provider },
|
||||
"bucket": { "type": "string" },
|
||||
"config": config
|
||||
}
|
||||
})
|
||||
};
|
||||
json!({
|
||||
"oneOf": [
|
||||
storage("fs", json!({ "type": "object", "required": ["path"], "properties": { "path": { "type": "string" } } })),
|
||||
storage("assetpack", json!({ "type": "object", "required": ["path"], "properties": { "path": { "type": "string" } } })),
|
||||
storage("aws-s3", s3),
|
||||
storage("cloudflare-r2", r2)
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
impl UsePresignedUrlConfigFile {
|
||||
fn settings(
|
||||
self,
|
||||
) -> (
|
||||
bool,
|
||||
bool,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
) {
|
||||
let prefix = self.url_prefix.filter(|value| !value.is_empty());
|
||||
let key = self.sign_key.filter(|value| !value.is_empty());
|
||||
let custom_get = self.enabled && prefix.is_some() && key.is_some();
|
||||
(
|
||||
self.enabled,
|
||||
self.enabled && key.is_some(),
|
||||
prefix.clone(),
|
||||
key.clone(),
|
||||
custom_get.then_some(prefix).flatten(),
|
||||
custom_get.then_some(key).flatten(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectStorageConfig {
|
||||
pub(crate) fn from_provider_config(storage: Option<StorageProviderConfig>) -> ObjectStorageResult<Option<Self>> {
|
||||
let Some(storage) = storage else {
|
||||
@@ -112,11 +172,13 @@ impl ObjectStorageConfig {
|
||||
pub(crate) fn from_s3_config(storage: StorageProviderConfig) -> ObjectStorageResult<Option<Self>> {
|
||||
let config: S3ConfigFile = serde_json::from_value(storage.config)
|
||||
.map_err(|err| ObjectStorageError::Config(format!("invalid aws-s3 blob storage config: {err}")))?;
|
||||
let region = config
|
||||
.region
|
||||
.ok_or_else(|| ObjectStorageError::Config("aws-s3 blob storage config requires region".to_string()))?;
|
||||
let region = config.region;
|
||||
let endpoint = config.endpoint.or_else(|| Some(resolve_s3_endpoint(®ion)));
|
||||
let credentials = config.credentials.unwrap_or_default();
|
||||
let (use_presigned_url, proxy_upload, upload_url_prefix, proxy_sign_key, _, _) = config
|
||||
.use_presigned_url
|
||||
.map(UsePresignedUrlConfigFile::settings)
|
||||
.unwrap_or((false, false, None, None, None, None));
|
||||
|
||||
Ok(Some(Self {
|
||||
provider: storage.provider,
|
||||
@@ -131,8 +193,10 @@ impl ObjectStorageConfig {
|
||||
min_part_size: config.min_part_size,
|
||||
presign_expires_in_seconds: config.presign.as_ref().and_then(|v| v.expires_in_seconds),
|
||||
presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put),
|
||||
use_presigned_url: config.use_presigned_url.map(|v| v.enabled).unwrap_or(false),
|
||||
proxy_upload: false,
|
||||
use_presigned_url,
|
||||
proxy_upload,
|
||||
upload_url_prefix,
|
||||
proxy_sign_key,
|
||||
custom_get_url_prefix: None,
|
||||
custom_get_sign_key: None,
|
||||
}))
|
||||
@@ -146,25 +210,17 @@ impl ObjectStorageConfig {
|
||||
Some(R2Jurisdiction::Default) | None => config.account_id,
|
||||
};
|
||||
let credentials = config.credentials.unwrap_or_default();
|
||||
let (use_presigned_url, proxy_upload, custom_get_url_prefix, custom_get_sign_key) = config
|
||||
let (
|
||||
use_presigned_url,
|
||||
proxy_upload,
|
||||
upload_url_prefix,
|
||||
proxy_sign_key,
|
||||
custom_get_url_prefix,
|
||||
custom_get_sign_key,
|
||||
) = config
|
||||
.use_presigned_url
|
||||
.map(|value| {
|
||||
let url_prefix = value.url_prefix.filter(|prefix| !prefix.is_empty());
|
||||
let sign_key = value.sign_key.filter(|key| !key.is_empty());
|
||||
let custom_get_enabled = value.enabled && url_prefix.is_some() && sign_key.is_some();
|
||||
let (custom_get_url_prefix, custom_get_sign_key) = if custom_get_enabled {
|
||||
(url_prefix, sign_key)
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
(
|
||||
value.enabled,
|
||||
custom_get_enabled,
|
||||
custom_get_url_prefix,
|
||||
custom_get_sign_key,
|
||||
)
|
||||
})
|
||||
.unwrap_or((false, false, None, None));
|
||||
.map(UsePresignedUrlConfigFile::settings)
|
||||
.unwrap_or((false, false, None, None, None, None));
|
||||
|
||||
Ok(Some(Self {
|
||||
provider: storage.provider,
|
||||
@@ -181,11 +237,59 @@ impl ObjectStorageConfig {
|
||||
presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put),
|
||||
use_presigned_url,
|
||||
proxy_upload,
|
||||
upload_url_prefix,
|
||||
proxy_sign_key,
|
||||
custom_get_url_prefix,
|
||||
custom_get_sign_key,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn proxy_upload_token(
|
||||
&self,
|
||||
path: &str,
|
||||
fields: &[Value],
|
||||
expires_at: i64,
|
||||
) -> ObjectStorageResult<Option<String>> {
|
||||
// TODO(0.27.5): Review 0.27.4 upload URL compatibility after old instances
|
||||
// and URLs expire.
|
||||
let Some(key) = self.proxy_sign_key.as_ref().filter(|_| self.proxy_upload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut canonical = vec![json!("affine-storage-upload"), json!(1), json!("PUT"), json!(path)];
|
||||
canonical.extend_from_slice(fields);
|
||||
canonical.push(json!(expires_at));
|
||||
let payload = serde_json::to_vec(&canonical)
|
||||
.map_err(|err| ObjectStorageError::Config(format!("invalid upload token payload: {err}")))?;
|
||||
let mut mac = HmacSha256::new_from_slice(key.as_bytes())
|
||||
.map_err(|err| ObjectStorageError::Config(format!("invalid upload signing key: {err}")))?;
|
||||
mac.update(&payload);
|
||||
Ok(Some(URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes())))
|
||||
}
|
||||
|
||||
pub(crate) fn verify_proxy_upload_token(
|
||||
&self,
|
||||
path: &str,
|
||||
fields: &[Value],
|
||||
expires_at: i64,
|
||||
token: &str,
|
||||
) -> ObjectStorageResult<bool> {
|
||||
let Some(key) = self.proxy_sign_key.as_ref().filter(|_| self.proxy_upload) else {
|
||||
return Ok(false);
|
||||
};
|
||||
let Ok(signature) = URL_SAFE_NO_PAD.decode(token) else {
|
||||
return Ok(false);
|
||||
};
|
||||
let mut canonical = vec![json!("affine-storage-upload"), json!(1), json!("PUT"), json!(path)];
|
||||
canonical.extend_from_slice(fields);
|
||||
canonical.push(json!(expires_at));
|
||||
let payload = serde_json::to_vec(&canonical)
|
||||
.map_err(|err| ObjectStorageError::Config(format!("invalid upload token payload: {err}")))?;
|
||||
let mut mac = HmacSha256::new_from_slice(key.as_bytes())
|
||||
.map_err(|err| ObjectStorageError::Config(format!("invalid upload signing key: {err}")))?;
|
||||
mac.update(&payload);
|
||||
Ok(mac.verify_slice(&signature).is_ok())
|
||||
}
|
||||
|
||||
pub(crate) fn custom_presign_get(&self, key: &ObjectKey) -> ObjectStorageResult<Option<PresignedObjectRequest>> {
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
|
||||
@@ -9,9 +9,12 @@ mod service;
|
||||
mod tests;
|
||||
pub(crate) mod types;
|
||||
|
||||
pub(in crate::runtime) use backend::{FsStorageConfig, StorageBackendConfig};
|
||||
pub(in crate::runtime) use backend::{
|
||||
FsStorageConfig, StorageBackendConfig, backends_from_flat_overrides, default_storage_provider_config,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(in crate::runtime) use config::ObjectStorageConfig;
|
||||
pub(in crate::runtime) use config::storage_provider_schema;
|
||||
pub(crate) use service::ObjectStorageService;
|
||||
|
||||
pub(super) use super::webpki_tls_config;
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::Value;
|
||||
use sqlx::PgPool;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
use super::{
|
||||
StorageBackendConfig, assetpack,
|
||||
backend::{backends_from_config_files, backends_from_config_json, backends_from_config_source, backends_from_db},
|
||||
backend::{backends_from_config_value, backends_from_db},
|
||||
fs::{delete_many_fs, fs_delete, fs_get, fs_get_range, fs_head, fs_list_page, fs_put},
|
||||
types::{
|
||||
MultipartUploadInitResult, MultipartUploadPart, ObjectDeleteOutcome, ObjectGetResult, ObjectKey, ObjectListPage,
|
||||
ObjectLocator, ObjectMetadata, ObjectPrefix, ObjectPutMetadata, PresignedObjectRequest, StorageScope,
|
||||
},
|
||||
};
|
||||
use crate::runtime::{ConfigSource, RuntimeError, RuntimeResult};
|
||||
use crate::runtime::{RuntimeError, RuntimeResult};
|
||||
|
||||
const DELETE_MANY_CHUNK_SIZE: usize = 500;
|
||||
const DELETE_MANY_CONCURRENCY: usize = 3;
|
||||
@@ -23,21 +24,16 @@ pub(crate) struct ObjectStorageService {
|
||||
}
|
||||
|
||||
impl ObjectStorageService {
|
||||
pub(crate) fn from_config_files() -> RuntimeResult<Self> {
|
||||
Ok(Self {
|
||||
backends: backends_from_config_files()?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn from_config_source(source: &ConfigSource) -> RuntimeResult<Self> {
|
||||
Ok(Self {
|
||||
backends: backends_from_config_source(source)?,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::runtime) fn from_config_json(config_json: &str) -> RuntimeResult<Self> {
|
||||
Ok(Self {
|
||||
backends: backends_from_config_json(config_json)?,
|
||||
backends: super::backend::backends_from_config_json(config_json)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn from_config_value(value: &Value) -> RuntimeResult<Self> {
|
||||
Ok(Self {
|
||||
backends: backends_from_config_value(value)?,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use super::{
|
||||
backend::backends_from_config_json,
|
||||
backend::{backends_from_config_json, backends_from_flat_overrides},
|
||||
config::ObjectStorageConfig,
|
||||
error::ObjectStorageError,
|
||||
types::{
|
||||
@@ -114,6 +114,13 @@ fn resolves_storage_config_from_config_json_shape() {
|
||||
.unwrap();
|
||||
assert_eq!(configured.get("blob").unwrap().bucket(), "custom-blobs");
|
||||
assert_eq!(configured.get("copilot").unwrap().bucket(), "copilot");
|
||||
assert!(
|
||||
backends_from_flat_overrides([(
|
||||
"storages.blob.storage",
|
||||
serde_json::json!({"provider":"invalid","bucket":"blobs","config":{}})
|
||||
)])
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let storage = StorageProviderConfig {
|
||||
provider: "cloudflare-r2".to_string(),
|
||||
@@ -273,6 +280,29 @@ fn resolves_r2_proxy_upload_capability_from_config_json_shape() {
|
||||
|
||||
assert!(config.use_presigned_url);
|
||||
assert!(config.proxy_upload);
|
||||
let fields = [
|
||||
serde_json::json!("ws"),
|
||||
serde_json::json!("file"),
|
||||
serde_json::json!("text/plain"),
|
||||
serde_json::json!(7),
|
||||
];
|
||||
let token = config
|
||||
.proxy_upload_token("/api/storage/upload", &fields, 1_700_000_000)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
// TODO(0.27.5): Remove this old Node token-format comparison after 0.27.4
|
||||
// upload URLs expire.
|
||||
assert_eq!(token, "NXaxFbKQjOOnqo5ggDsuK1jf8w5LlM7tLRWRoj_EmtY");
|
||||
assert!(
|
||||
config
|
||||
.verify_proxy_upload_token("/api/storage/upload", &fields, 1_700_000_000, &token)
|
||||
.unwrap()
|
||||
);
|
||||
assert!(
|
||||
!config
|
||||
.verify_proxy_upload_token("/api/storage/upload", &fields, 1_700_000_001, &token)
|
||||
.unwrap()
|
||||
);
|
||||
let request = config
|
||||
.custom_presign_get_at(&ObjectKey::new("workspace/blob.m4a").unwrap(), 1_700_000_000)
|
||||
.unwrap()
|
||||
|
||||
@@ -453,7 +453,10 @@ impl StorageRuntime {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{collections::HashMap, sync::RwLock};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, RwLock},
|
||||
};
|
||||
|
||||
use anyhow::{Context, Result as AnyResult};
|
||||
use napi::bindgen_prelude::Buffer;
|
||||
@@ -465,8 +468,9 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
use crate::runtime::{
|
||||
migrations::migrate_runtime_tables,
|
||||
object_storage::{FsStorageConfig, StorageBackendConfig},
|
||||
config::ServerConfig,
|
||||
migrations::{EMBEDDING_TEST_LOCK, migrate_runtime_tables},
|
||||
object_storage::{FsStorageConfig, ObjectStorageService, StorageBackendConfig},
|
||||
storage_runtime::StorageRuntimeConfig,
|
||||
};
|
||||
|
||||
@@ -512,10 +516,13 @@ mod tests {
|
||||
.await
|
||||
.map_err(|err| anyhow::anyhow!(err.to_string()))?;
|
||||
let object_root = tempfile::tempdir()?;
|
||||
let config_dir = tempfile::tempdir()?;
|
||||
let config_path = config_dir.path().join("config.json");
|
||||
std::fs::write(&config_path, r#"{"deployment":{"type":"cloud"}}"#)?;
|
||||
let runtime = StorageRuntime {
|
||||
config: RwLock::new(StorageRuntimeConfig {
|
||||
database_url,
|
||||
object_storage: crate::runtime::object_storage::ObjectStorageService {
|
||||
object_storage: ObjectStorageService {
|
||||
backends: HashMap::from([(
|
||||
"blob".to_string(),
|
||||
StorageBackendConfig::Fs(FsStorageConfig {
|
||||
@@ -526,6 +533,7 @@ mod tests {
|
||||
)]),
|
||||
},
|
||||
}),
|
||||
server_config: Arc::new(ServerConfig::open(&config_path, None)?),
|
||||
pool: Mutex::new(Some(pool.clone())),
|
||||
};
|
||||
let suffix = Uuid::new_v4().simple().to_string();
|
||||
@@ -629,7 +637,7 @@ mod tests {
|
||||
let Ok(database_url) = std::env::var("DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
let _guard = crate::runtime::migrations::EMBEDDING_TEST_LOCK.lock().await;
|
||||
let _guard = EMBEDDING_TEST_LOCK.lock().await;
|
||||
let pool = PgPool::connect(&database_url).await.unwrap();
|
||||
let suffix = Uuid::new_v4().simple().to_string();
|
||||
let workspace_id = format!("blob-cleanup-ws-{suffix}");
|
||||
@@ -682,7 +690,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL and a migrated PostgreSQL database"]
|
||||
async fn blob_cleanup_projection_reprojects_reference_removal_before_deletion() -> AnyResult<()> {
|
||||
let _guard = crate::runtime::migrations::EMBEDDING_TEST_LOCK.lock().await;
|
||||
let _guard = EMBEDDING_TEST_LOCK.lock().await;
|
||||
let fixture = blob_cleanup_fixture(true).await?;
|
||||
let _object_root = &fixture.object_root;
|
||||
fixture
|
||||
@@ -758,7 +766,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL and a migrated PostgreSQL database"]
|
||||
async fn blob_cleanup_projection_outdated_pending_and_failed_projections_fail_closed() -> AnyResult<()> {
|
||||
let _guard = crate::runtime::migrations::EMBEDDING_TEST_LOCK.lock().await;
|
||||
let _guard = EMBEDDING_TEST_LOCK.lock().await;
|
||||
for status in ["fresh", "pending", "failed"] {
|
||||
let fixture = blob_cleanup_fixture(false).await?;
|
||||
let _object_root = &fixture.object_root;
|
||||
|
||||
@@ -81,7 +81,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn capabilities_enable_presign_get_for_presigned_s3_provider() {
|
||||
let capabilities = storage_provider_capabilities(&StorageBackendConfig::S3(ObjectStorageConfig {
|
||||
let capabilities = storage_provider_capabilities(&StorageBackendConfig::S3(Box::new(ObjectStorageConfig {
|
||||
provider: "cloudflare-r2".to_string(),
|
||||
bucket: "blob".to_string(),
|
||||
endpoint: Some("https://account.r2.cloudflarestorage.com".to_string()),
|
||||
@@ -96,9 +96,11 @@ mod tests {
|
||||
presign_sign_content_type_for_put: Some(true),
|
||||
use_presigned_url: true,
|
||||
proxy_upload: false,
|
||||
upload_url_prefix: None,
|
||||
proxy_sign_key: None,
|
||||
custom_get_url_prefix: None,
|
||||
custom_get_sign_key: None,
|
||||
}));
|
||||
})));
|
||||
|
||||
assert!(capabilities.presign_put);
|
||||
assert!(capabilities.presign_get);
|
||||
@@ -108,7 +110,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn capabilities_expose_r2_proxy_upload() {
|
||||
let capabilities = storage_provider_capabilities(&StorageBackendConfig::S3(ObjectStorageConfig {
|
||||
let capabilities = storage_provider_capabilities(&StorageBackendConfig::S3(Box::new(ObjectStorageConfig {
|
||||
provider: "cloudflare-r2".to_string(),
|
||||
bucket: "blob".to_string(),
|
||||
endpoint: Some("https://account.r2.cloudflarestorage.com".to_string()),
|
||||
@@ -123,9 +125,11 @@ mod tests {
|
||||
presign_sign_content_type_for_put: Some(true),
|
||||
use_presigned_url: true,
|
||||
proxy_upload: true,
|
||||
upload_url_prefix: None,
|
||||
proxy_sign_key: Some("key".to_string()),
|
||||
custom_get_url_prefix: None,
|
||||
custom_get_sign_key: None,
|
||||
}));
|
||||
})));
|
||||
|
||||
assert!(capabilities.proxy_upload);
|
||||
assert!(capabilities.presign_put);
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
use std::{env, fs};
|
||||
use std::env;
|
||||
|
||||
use serde::Deserialize;
|
||||
use sqlx::PgPool;
|
||||
|
||||
use super::{ObjectStorageService, RuntimeError, RuntimeResult};
|
||||
use crate::runtime::config::ServerConfig;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct StorageRuntimeConfig {
|
||||
pub(super) database_url: String,
|
||||
pub(super) object_storage: ObjectStorageService,
|
||||
pub(in crate::runtime) struct StorageRuntimeConfig {
|
||||
pub(in crate::runtime) database_url: String,
|
||||
pub(in crate::runtime) object_storage: ObjectStorageService,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
@@ -23,17 +24,19 @@ struct DbConfigFile {
|
||||
}
|
||||
|
||||
impl StorageRuntimeConfig {
|
||||
pub(super) fn from_config_files() -> RuntimeResult<Self> {
|
||||
let app_config = storage_runtime_config_from_files()?;
|
||||
pub(in crate::runtime) fn from_server_config(config: &ServerConfig) -> RuntimeResult<Self> {
|
||||
let app_config: StorageRuntimeAppConfig = serde_json::from_value(config.baseline().clone())
|
||||
.map_err(|err| RuntimeError::json("invalid storage runtime config", err))?;
|
||||
let database_url = database_url_from_env()
|
||||
.or(app_config.database_url())
|
||||
.unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string());
|
||||
Ok(Self {
|
||||
database_url,
|
||||
object_storage: ObjectStorageService::from_config_files()?,
|
||||
object_storage: config.object_storage().clone(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn from_config_json(config_json: &str) -> RuntimeResult<Self> {
|
||||
let app_config: StorageRuntimeAppConfig =
|
||||
serde_json::from_str(config_json).map_err(|err| RuntimeError::json("invalid storage runtime config", err))?;
|
||||
@@ -61,12 +64,6 @@ impl StorageRuntimeAppConfig {
|
||||
.and_then(|db| db.datasource_url.clone())
|
||||
.and_then(non_empty_string)
|
||||
}
|
||||
|
||||
fn merge(&mut self, config: Self) {
|
||||
if config.db.is_some() {
|
||||
self.db = config.db;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn database_url_from_env() -> Option<String> {
|
||||
@@ -76,16 +73,3 @@ fn database_url_from_env() -> Option<String> {
|
||||
fn non_empty_string(value: String) -> Option<String> {
|
||||
if value.trim().is_empty() { None } else { Some(value) }
|
||||
}
|
||||
|
||||
fn storage_runtime_config_from_files() -> RuntimeResult<StorageRuntimeAppConfig> {
|
||||
let mut merged = StorageRuntimeAppConfig::default();
|
||||
for path in crate::runtime::config::config_json_paths() {
|
||||
if !path.exists() {
|
||||
continue;
|
||||
}
|
||||
let raw = fs::read_to_string(&path).map_err(|err| RuntimeError::io("failed to read config file", err))?;
|
||||
let config = serde_json::from_str(&raw).map_err(|err| RuntimeError::json("failed to parse config file", err))?;
|
||||
merged.merge(config);
|
||||
}
|
||||
Ok(merged)
|
||||
}
|
||||
@@ -273,7 +273,10 @@ impl StorageRuntime {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{collections::HashMap, sync::RwLock};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, RwLock},
|
||||
};
|
||||
|
||||
use anyhow::{Context, Result as AnyResult};
|
||||
use napi::bindgen_prelude::Buffer;
|
||||
@@ -285,7 +288,9 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::runtime::{
|
||||
backend_runtime::SEARCH_TEST_LOCK,
|
||||
config::ServerConfig,
|
||||
migrations::{migrate_runtime_tables, migrate_search_tables},
|
||||
object_storage::{FsStorageConfig, ObjectStorageService, StorageBackendConfig},
|
||||
storage_runtime::StorageRuntimeConfig,
|
||||
};
|
||||
|
||||
@@ -301,13 +306,17 @@ mod tests {
|
||||
migrate_runtime_tables(&pool)
|
||||
.await
|
||||
.map_err(|err| anyhow::anyhow!(err.to_string()))?;
|
||||
let config_dir = tempfile::tempdir()?;
|
||||
let config_path = config_dir.path().join("config.json");
|
||||
std::fs::write(&config_path, r#"{"deployment":{"type":"cloud"}}"#)?;
|
||||
let runtime = StorageRuntime {
|
||||
config: RwLock::new(StorageRuntimeConfig {
|
||||
database_url,
|
||||
object_storage: crate::runtime::object_storage::ObjectStorageService {
|
||||
object_storage: ObjectStorageService {
|
||||
backends: HashMap::new(),
|
||||
},
|
||||
}),
|
||||
server_config: Arc::new(ServerConfig::open(&config_path, None)?),
|
||||
pool: Mutex::new(Some(pool.clone())),
|
||||
};
|
||||
Ok(Some((runtime, pool)))
|
||||
@@ -768,7 +777,7 @@ mod tests {
|
||||
let object_root = tempfile::tempdir()?;
|
||||
runtime.config.write().unwrap().object_storage.backends.insert(
|
||||
"blob".to_string(),
|
||||
crate::runtime::object_storage::StorageBackendConfig::Fs(crate::runtime::object_storage::FsStorageConfig {
|
||||
StorageBackendConfig::Fs(FsStorageConfig {
|
||||
provider: "fs".to_string(),
|
||||
root: object_root.path().to_string_lossy().to_string(),
|
||||
bucket: "document-cleanup-test".to_string(),
|
||||
|
||||
@@ -9,15 +9,16 @@ mod document_cleanup_execution;
|
||||
mod workspace_cleanup;
|
||||
mod workspace_cleanup_checkpoint;
|
||||
mod workspace_cleanup_namespace;
|
||||
use std::sync::RwLock;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
pub use capabilities::StorageProviderCapabilities;
|
||||
use capabilities::storage_provider_capabilities;
|
||||
use config::StorageRuntimeConfig;
|
||||
pub(in crate::runtime) use config::StorageRuntimeConfig;
|
||||
pub(super) use current_doc::load_current_doc;
|
||||
pub(in crate::runtime) use current_doc::{CurrentDoc, CurrentDocUpdate, merge_current_doc};
|
||||
use current_doc::{load_canonical_doc, load_workspace_canonical_doc_ids, load_workspace_live_doc_ids};
|
||||
use napi::bindgen_prelude::Buffer;
|
||||
use serde_json::Value;
|
||||
use sqlx::{PgPool, Row, postgres::PgPoolOptions};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
@@ -39,6 +40,7 @@ use super::{
|
||||
types::{ObjectDeleteOutcome, ObjectKey, ObjectLocator, ObjectPrefix, StorageScope},
|
||||
},
|
||||
};
|
||||
use crate::runtime::config::{ServerConfig, ServerConfigHandle};
|
||||
|
||||
type Result<T> = RuntimeResult<T>;
|
||||
|
||||
@@ -67,18 +69,27 @@ pub struct StorageRuntimeHealth {
|
||||
pub bucket: Option<String>,
|
||||
}
|
||||
|
||||
#[napi_derive::napi(object)]
|
||||
pub struct StorageUploadUrlConfig {
|
||||
pub proxy_upload: bool,
|
||||
pub url_prefix: Option<String>,
|
||||
}
|
||||
|
||||
#[napi_derive::napi]
|
||||
pub struct StorageRuntime {
|
||||
config: RwLock<StorageRuntimeConfig>,
|
||||
server_config: Arc<ServerConfig>,
|
||||
pool: Mutex<Option<PgPool>>,
|
||||
}
|
||||
|
||||
#[napi_derive::napi]
|
||||
impl StorageRuntime {
|
||||
#[napi(constructor)]
|
||||
pub fn new() -> napi::Result<Self> {
|
||||
pub fn new(server_config: &ServerConfigHandle) -> napi::Result<Self> {
|
||||
let config = StorageRuntimeConfig::from_server_config(&server_config.inner).map_err(to_napi_error)?;
|
||||
Ok(Self {
|
||||
config: RwLock::new(StorageRuntimeConfig::from_config_files().map_err(to_napi_error)?),
|
||||
config: RwLock::new(config),
|
||||
server_config: Arc::clone(&server_config.inner),
|
||||
pool: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
@@ -88,12 +99,6 @@ impl StorageRuntime {
|
||||
self.start_inner().await.map_err(to_napi_error)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn configure(&self, config_json: String) -> napi::Result<()> {
|
||||
let config = StorageRuntimeConfig::from_config_json(&config_json).map_err(to_napi_error)?;
|
||||
self.update_config(config).map_err(to_napi_error)
|
||||
}
|
||||
|
||||
async fn start_inner(&self) -> RuntimeResult<()> {
|
||||
let mut guard = self.pool.lock().await;
|
||||
if guard.is_some() {
|
||||
@@ -113,12 +118,21 @@ impl StorageRuntime {
|
||||
.await
|
||||
.map_err(|err| RuntimeError::database("StorageRuntime postgres health check failed", err))?;
|
||||
|
||||
let config = self.config()?.with_db_overrides(&pool).await?;
|
||||
let baseline = StorageRuntimeConfig::from_server_config(&self.server_config)?;
|
||||
let config = baseline.with_db_overrides(&pool).await?;
|
||||
self.update_config(config)?;
|
||||
*guard = Some(pool);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn reload_config(&self) -> napi::Result<()> {
|
||||
let pool = self.pool().await.map_err(to_napi_error)?;
|
||||
let baseline = StorageRuntimeConfig::from_server_config(&self.server_config).map_err(to_napi_error)?;
|
||||
let config = baseline.with_db_overrides(&pool).await.map_err(to_napi_error)?;
|
||||
self.update_config(config).map_err(to_napi_error)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn stop(&self) -> napi::Result<()> {
|
||||
let pool = self.pool.lock().await.take();
|
||||
@@ -169,6 +183,63 @@ impl StorageRuntime {
|
||||
.map_err(to_napi_error)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn is_local_storage(&self, scope: String) -> napi::Result<bool> {
|
||||
self
|
||||
.backend_for_scope(&scope)
|
||||
.map(|backend| {
|
||||
matches!(
|
||||
backend,
|
||||
StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_)
|
||||
)
|
||||
})
|
||||
.map_err(to_napi_error)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn upload_url_config(&self, scope: String) -> napi::Result<Option<StorageUploadUrlConfig>> {
|
||||
match self.backend_for_scope(&scope).map_err(to_napi_error)? {
|
||||
StorageBackendConfig::S3(config) if config.use_presigned_url => Ok(Some(StorageUploadUrlConfig {
|
||||
proxy_upload: config.proxy_upload,
|
||||
url_prefix: config.upload_url_prefix,
|
||||
})),
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn sign_upload_token(
|
||||
&self,
|
||||
scope: String,
|
||||
path: String,
|
||||
fields: Vec<Value>,
|
||||
expires_at: i64,
|
||||
) -> napi::Result<Option<String>> {
|
||||
match self.backend_for_scope(&scope).map_err(to_napi_error)? {
|
||||
StorageBackendConfig::S3(config) => config
|
||||
.proxy_upload_token(&path, &fields, expires_at)
|
||||
.map_err(|err| to_napi_error(err.into())),
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn verify_upload_token(
|
||||
&self,
|
||||
scope: String,
|
||||
path: String,
|
||||
fields: Vec<Value>,
|
||||
expires_at: i64,
|
||||
token: String,
|
||||
) -> napi::Result<bool> {
|
||||
match self.backend_for_scope(&scope).map_err(to_napi_error)? {
|
||||
StorageBackendConfig::S3(config) => config
|
||||
.verify_proxy_upload_token(&path, &fields, expires_at, &token)
|
||||
.map_err(|err| to_napi_error(err.into())),
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn put_object(
|
||||
&self,
|
||||
|
||||
@@ -531,18 +531,25 @@ fn record_failure(result: &mut RuntimeWorkspaceStorageReconcileResult, scope: &s
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{fs::write, sync::Arc};
|
||||
|
||||
use super::*;
|
||||
use crate::runtime::object_storage::types::{ObjectKey, ObjectLocator, ObjectPutMetadata};
|
||||
|
||||
fn runtime(root: &std::path::Path) -> StorageRuntime {
|
||||
let runtime = StorageRuntime::new().unwrap();
|
||||
let root = serde_json::to_string(root).unwrap();
|
||||
runtime
|
||||
.configure(format!(
|
||||
let config = super::super::StorageRuntimeConfig::from_config_json(&format!(
|
||||
r#"{{"storages":{{"blob.storage":{{"provider":"fs","bucket":"workspace-cleanup","config":{{"path":{root}}}}},"avatar.storage":{{"provider":"fs","bucket":"workspace-cleanup-avatar","config":{{"path":{root}}}}}}},"copilot":{{"storage":{{"provider":"fs","bucket":"workspace-cleanup-copilot","config":{{"path":{root}}}}}}}}}"#,
|
||||
))
|
||||
.unwrap();
|
||||
runtime
|
||||
.unwrap();
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let config_path = config_dir.path().join("config.json");
|
||||
write(&config_path, r#"{"deployment":{"type":"cloud"}}"#).unwrap();
|
||||
StorageRuntime {
|
||||
config: std::sync::RwLock::new(config),
|
||||
server_config: Arc::new(crate::runtime::config::ServerConfig::open(&config_path, None).unwrap()),
|
||||
pool: tokio::sync::Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn put(runtime: &StorageRuntime, key: String) {
|
||||
|
||||
@@ -11,6 +11,13 @@ import {
|
||||
} from '../src/base/config/register';
|
||||
|
||||
const IGNORED_MODULES = new Set(['db', 'redis', 'graphql']);
|
||||
const SELFHOSTED_IGNORED_MODULES = new Set([
|
||||
'payment',
|
||||
'telemetry',
|
||||
'captcha',
|
||||
'metrics',
|
||||
]);
|
||||
const SELFHOSTED_IGNORED_KEYS = new Set(['auth.trustedCloudflareHeaders']);
|
||||
|
||||
function getDescriptors() {
|
||||
return getAllDescriptors()
|
||||
@@ -49,18 +56,23 @@ function generateJsonSchema(outputPath: string) {
|
||||
properties: {},
|
||||
};
|
||||
|
||||
getDescriptors().forEach(({ module, descriptors }) => {
|
||||
schema.properties[module] = {
|
||||
type: 'object',
|
||||
description: `Configuration for ${module} module`,
|
||||
properties: {},
|
||||
};
|
||||
getDescriptors()
|
||||
.filter(({ module }) => !SELFHOSTED_IGNORED_MODULES.has(module))
|
||||
.forEach(({ module, descriptors }) => {
|
||||
schema.properties[module] = {
|
||||
type: 'object',
|
||||
description: `Configuration for ${module} module`,
|
||||
properties: {},
|
||||
};
|
||||
|
||||
descriptors.forEach(({ key, descriptor }) => {
|
||||
schema.properties[module].properties[key] =
|
||||
convertDescriptorToSchemaProperty(descriptor);
|
||||
descriptors.forEach(({ key, descriptor }) => {
|
||||
if (SELFHOSTED_IGNORED_KEYS.has(`${module}.${key}`)) {
|
||||
return;
|
||||
}
|
||||
schema.properties[module].properties[key] =
|
||||
convertDescriptorToSchemaProperty(descriptor);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
fs.writeFileSync(outputPath, JSON.stringify(schema, null, 2));
|
||||
|
||||
@@ -76,6 +88,7 @@ function generateAdminConfigJson(outputPath: string) {
|
||||
let type: string;
|
||||
switch (descriptor.schema?.type) {
|
||||
case 'number':
|
||||
case 'integer':
|
||||
type = 'Number';
|
||||
break;
|
||||
case 'boolean':
|
||||
|
||||
@@ -4,6 +4,8 @@ import fs from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import { upgradeConfig } from './upgrade-config.js';
|
||||
|
||||
const SELF_HOST_CONFIG_DIR = `${homedir()}/.affine/config`;
|
||||
|
||||
function generatePrivateKey() {
|
||||
@@ -29,6 +31,8 @@ const files = [{ to: 'private.key', generator: generatePrivateKey }];
|
||||
function prepare() {
|
||||
fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
|
||||
|
||||
upgradeConfig();
|
||||
|
||||
for (const { to, generator } of files) {
|
||||
const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, to);
|
||||
if (!fs.existsSync(targetFilePath)) {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import fs from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
import { config as loadEnv } from 'dotenv';
|
||||
|
||||
const aliases = [
|
||||
['mailer', 'SMTP.host', 'MAILER_HOST'],
|
||||
['mailer', 'SMTP.ignoreTLS', 'MAILER_IGNORE_TLS', 'boolean'],
|
||||
['mailer', 'SMTP.name', 'MAILER_SERVERNAME'],
|
||||
['mailer', 'SMTP.password', 'MAILER_PASSWORD'],
|
||||
['mailer', 'SMTP.port', 'MAILER_PORT', 'integer'],
|
||||
['mailer', 'SMTP.sender', 'MAILER_SENDER'],
|
||||
['mailer', 'SMTP.username', 'MAILER_USER'],
|
||||
['redis', 'db', 'REDIS_SERVER_DATABASE', 'integer'],
|
||||
['redis', 'host', 'REDIS_SERVER_HOST'],
|
||||
['redis', 'password', 'REDIS_SERVER_PASSWORD'],
|
||||
['redis', 'port', 'REDIS_SERVER_PORT', 'integer'],
|
||||
['redis', 'username', 'REDIS_SERVER_USERNAME'],
|
||||
['server', 'externalUrl', 'AFFINE_SERVER_EXTERNAL_URL'],
|
||||
['server', 'host', 'AFFINE_SERVER_HOST'],
|
||||
['server', 'https', 'AFFINE_SERVER_HTTPS', 'boolean'],
|
||||
['server', 'listenAddr', 'LISTEN_ADDR'],
|
||||
['server', 'path', 'AFFINE_SERVER_SUB_PATH'],
|
||||
['server', 'port', 'AFFINE_SERVER_PORT', 'integer'],
|
||||
['telemetry', 'ga4.apiSecret', 'GA4_API_SECRET'],
|
||||
['telemetry', 'ga4.measurementId', 'GA4_MEASUREMENT_ID'],
|
||||
];
|
||||
|
||||
// TODO(0.27.5): Remove this upgrade after old self-host config files and mounts are retired.
|
||||
export function upgradeConfig(
|
||||
appRoot = process.cwd(),
|
||||
homeDir = homedir(),
|
||||
deploymentType
|
||||
) {
|
||||
loadEnv({ path: path.join(appRoot, '.env'), quiet: true });
|
||||
loadEnv({ path: path.join(homeDir, '.affine/config/.env'), quiet: true });
|
||||
deploymentType ??= process.env.DEPLOYMENT_TYPE;
|
||||
const appPath = path.join(appRoot, 'config.json');
|
||||
const homePath = path.join(homeDir, '.affine/config/config.json');
|
||||
const configPath = fs.existsSync(appPath)
|
||||
? appPath
|
||||
: fs.existsSync(homePath) || fs.existsSync(path.dirname(homePath))
|
||||
? homePath
|
||||
: appPath;
|
||||
const config = fs.existsSync(configPath)
|
||||
? JSON.parse(fs.readFileSync(configPath, 'utf-8'))
|
||||
: {};
|
||||
let changed = false;
|
||||
if (!config.deployment?.type) {
|
||||
if (deploymentType && !['affine', 'selfhosted'].includes(deploymentType)) {
|
||||
throw new Error(`Invalid DEPLOYMENT_TYPE: ${deploymentType}`);
|
||||
}
|
||||
config.deployment = {
|
||||
...config.deployment,
|
||||
type: deploymentType === 'affine' ? 'cloud' : 'selfhosted',
|
||||
};
|
||||
changed = true;
|
||||
}
|
||||
for (const [module, key, alias, type] of aliases) {
|
||||
const value = process.env[alias];
|
||||
if (!value) continue;
|
||||
const current = config[module] ?? {};
|
||||
const parts = key.split('.');
|
||||
const nested = parts
|
||||
.slice(0, -1)
|
||||
.reduce((value, part) => value?.[part], current);
|
||||
if (
|
||||
Object.hasOwn(current, key) ||
|
||||
Object.hasOwn(nested ?? {}, parts.at(-1))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const parsed =
|
||||
type === 'boolean'
|
||||
? value === '1' || value.toLowerCase() === 'true'
|
||||
: type === 'integer'
|
||||
? Number.parseInt(value)
|
||||
: value;
|
||||
if (type === 'integer' && Number.isNaN(parsed)) continue;
|
||||
config[module] = { ...current, [key]: parsed };
|
||||
changed = true;
|
||||
}
|
||||
if (!changed) return;
|
||||
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
||||
const tempPath = `${configPath}.${process.pid}.tmp`;
|
||||
try {
|
||||
fs.writeFileSync(tempPath, JSON.stringify(config, null, 2) + '\n', {
|
||||
mode: 0o600,
|
||||
});
|
||||
fs.renameSync(tempPath, configPath);
|
||||
} catch (error) {
|
||||
if (
|
||||
configPath === appPath &&
|
||||
['EACCES', 'EROFS', 'EBUSY'].includes(error.code)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
|
||||
) {
|
||||
upgradeConfig();
|
||||
}
|
||||
@@ -839,14 +839,17 @@ test('should return methods unavailable for disabled users', async t => {
|
||||
});
|
||||
|
||||
test('should return magic link unavailable for unknown users when signup is disabled', async t => {
|
||||
const { app, config } = t.context;
|
||||
const { app, runtime } = t.context;
|
||||
|
||||
config.override({
|
||||
auth: {
|
||||
allowSignup: false,
|
||||
await runtime.saveAppConfig(null, [
|
||||
{
|
||||
key: 'auth.allowSignup',
|
||||
owner: 'native',
|
||||
operation: 'set',
|
||||
valueJson: 'false',
|
||||
},
|
||||
});
|
||||
await t.context.runtime.onConfigChanged({
|
||||
]);
|
||||
await runtime.onConfigChanged({
|
||||
updates: { auth: { allowSignup: false } },
|
||||
});
|
||||
|
||||
@@ -860,14 +863,17 @@ test('should return magic link unavailable for unknown users when signup is disa
|
||||
});
|
||||
|
||||
test('should return magic link unavailable when domain verification rejects signup email', async t => {
|
||||
const { app, config } = t.context;
|
||||
const { app, runtime } = t.context;
|
||||
|
||||
config.override({
|
||||
auth: {
|
||||
requireEmailDomainVerification: true,
|
||||
await runtime.saveAppConfig(null, [
|
||||
{
|
||||
key: 'auth.requireEmailDomainVerification',
|
||||
owner: 'native',
|
||||
operation: 'set',
|
||||
valueJson: 'true',
|
||||
},
|
||||
});
|
||||
await t.context.runtime.onConfigChanged({
|
||||
]);
|
||||
await runtime.onConfigChanged({
|
||||
updates: { auth: { requireEmailDomainVerification: true } },
|
||||
});
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import { createTestingModule } from '../utils';
|
||||
let m: TestingModule;
|
||||
let db: PrismaClient;
|
||||
const runtime = {
|
||||
nodeCryptoPrivateKey: Sinon.stub().returns(''),
|
||||
cleanupExpiredUserSessions: Sinon.stub(),
|
||||
executeAuthSessionCommandV1: Sinon.stub(),
|
||||
};
|
||||
|
||||
@@ -4,7 +4,7 @@ import ava from 'ava';
|
||||
import Sinon from 'sinon';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { Config, CopilotQuotaExceeded } from '../../base';
|
||||
import { CopilotQuotaExceeded } from '../../base';
|
||||
import type { BackendRuntimeProvider } from '../../core/backend-runtime';
|
||||
import { StorageRuntimeProvider } from '../../core/storage-runtime';
|
||||
import type { Models } from '../../models';
|
||||
@@ -33,6 +33,7 @@ function runtimeFixture(streamError?: string, enabled = true) {
|
||||
targetOverride?: { profileId: string; modelId: string };
|
||||
}> = [];
|
||||
const backend = {
|
||||
copilotEnabled: () => enabled,
|
||||
executeCopilot: async (input: {
|
||||
slot: string;
|
||||
request: unknown;
|
||||
@@ -99,7 +100,6 @@ function runtimeFixture(streamError?: string, enabled = true) {
|
||||
const consumer = {
|
||||
consume: async () => {},
|
||||
} as unknown as CopilotRuntimeEventConsumer;
|
||||
const config = { copilot: { enabled } } as Config;
|
||||
const storageRuntime = Sinon.createStubInstance(StorageRuntimeProvider);
|
||||
storageRuntime.getObject.callsFake(async () => ({
|
||||
body: Readable.from(Buffer.from('image')),
|
||||
@@ -126,7 +126,6 @@ function runtimeFixture(streamError?: string, enabled = true) {
|
||||
conversation,
|
||||
tools,
|
||||
consumer,
|
||||
config,
|
||||
new AttachmentAdmissionHost(materializer, storage)
|
||||
),
|
||||
};
|
||||
|
||||
@@ -14,10 +14,8 @@ import { McpAccessMode, PrismaClient } from '@prisma/client';
|
||||
import type { TestFn } from 'ava';
|
||||
import ava from 'ava';
|
||||
|
||||
import { Config } from '../../base';
|
||||
import { ServerFeature, ServerService } from '../../core';
|
||||
import { Models } from '../../models';
|
||||
import { CopilotFeatureService } from '../../plugins/copilot/feature';
|
||||
import { McpCredentialService } from '../../plugins/copilot/mcp/credential';
|
||||
import { WorkspaceMcpProvider } from '../../plugins/copilot/mcp/provider';
|
||||
import { installMockCopilotRuntime } from '../mocks';
|
||||
@@ -58,14 +56,13 @@ test.after.always(async t => {
|
||||
|
||||
test('disabled copilot hides its server feature and rejects every API transport', async t => {
|
||||
const { app } = t.context;
|
||||
const config = app.get(Config);
|
||||
const feature = app.get(CopilotFeatureService);
|
||||
const server = app.get(ServerService);
|
||||
await app.signupV1();
|
||||
const workspace = await createWorkspace(app);
|
||||
|
||||
config.copilot.enabled = false;
|
||||
feature.onConfigChanged({ updates: { copilot: { enabled: false } } });
|
||||
await server.updateConfig(null, [
|
||||
{ module: 'copilot', key: 'enabled', value: false },
|
||||
]);
|
||||
try {
|
||||
t.false(server.features.includes(ServerFeature.Copilot));
|
||||
await t.throwsAsync(
|
||||
@@ -117,8 +114,9 @@ test('disabled copilot hides its server feature and rejects every API transport'
|
||||
})
|
||||
);
|
||||
} finally {
|
||||
config.copilot.enabled = true;
|
||||
feature.onConfigChanged({ updates: { copilot: { enabled: true } } });
|
||||
await server.updateConfig(null, [
|
||||
{ module: 'copilot', key: 'enabled', clear: true },
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
SearchProviderUnavailable,
|
||||
} from '../../base';
|
||||
import { ServerFeature, type ServerService } from '../../core';
|
||||
import type { BackendRuntimeProvider } from '../../core/backend-runtime';
|
||||
import type { DocReader } from '../../core/doc';
|
||||
import type { PermissionAccess } from '../../core/permission';
|
||||
import { type RealtimePublisher, RealtimeRegistry } from '../../core/realtime';
|
||||
@@ -958,27 +959,30 @@ test('document tools enforce the user-selected hard scope', async t => {
|
||||
);
|
||||
});
|
||||
|
||||
test('copilot config controls the server feature and request admission', t => {
|
||||
const config = { copilot: { enabled: false } } as Config;
|
||||
test('copilot runtime controls the server feature and request admission', t => {
|
||||
let enabled = false;
|
||||
const runtime = {
|
||||
copilotEnabled: () => enabled,
|
||||
} as unknown as BackendRuntimeProvider;
|
||||
const features = new Set<ServerFeature>();
|
||||
const server = {
|
||||
enableFeature: (feature: ServerFeature) => features.add(feature),
|
||||
disableFeature: (feature: ServerFeature) => features.delete(feature),
|
||||
} as unknown as ServerService;
|
||||
const feature = new CopilotFeatureService(config, server);
|
||||
const feature = new CopilotFeatureService(runtime, server);
|
||||
const guard = new CopilotFeatureGuard(feature);
|
||||
|
||||
feature.onConfigInit();
|
||||
t.false(features.has(ServerFeature.Copilot));
|
||||
t.throws(() => guard.canActivate(), { message: 'Copilot is disabled.' });
|
||||
|
||||
config.copilot.enabled = true;
|
||||
feature.onConfigChanged({ updates: { copilot: { enabled: true } } });
|
||||
enabled = true;
|
||||
feature.onConfigApplied({ updates: { copilot: { enabled: true } } });
|
||||
t.true(features.has(ServerFeature.Copilot));
|
||||
t.true(guard.canActivate());
|
||||
|
||||
config.copilot.enabled = false;
|
||||
feature.onConfigChanged({ updates: { copilot: { enabled: false } } });
|
||||
enabled = false;
|
||||
feature.onConfigApplied({ updates: { copilot: { enabled: false } } });
|
||||
t.false(features.has(ServerFeature.Copilot));
|
||||
});
|
||||
|
||||
|
||||
@@ -7,15 +7,18 @@ import {
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
import { FunctionalityModules } from '../app.module';
|
||||
import { AFFiNELogger, ConfigFactory, EventBus } from '../base';
|
||||
import {
|
||||
BACKEND_RUNTIME_CONFIG_PATHS,
|
||||
BackendRuntimeProvider,
|
||||
} from '../core/backend-runtime';
|
||||
import { AFFiNELogger, EventBus } from '../base';
|
||||
import { OVERRIDE_CONFIG_TOKEN } from '../base/config/factory';
|
||||
import { getDefaultConfig } from '../base/config/register';
|
||||
import { BackendRuntimeProvider } from '../core/backend-runtime';
|
||||
import { StorageRuntimeProvider } from '../core/storage-runtime';
|
||||
import { ServerConfigHandle } from '../native';
|
||||
import { createFactory, MockEventBus } from './mocks';
|
||||
import { TEST_LOG_LEVEL } from './utils';
|
||||
import { createTestRuntimeConfig } from './utils/runtime-config';
|
||||
import {
|
||||
applyTestConfigOverrides,
|
||||
createTestRuntimeConfig,
|
||||
} from './utils/runtime-config';
|
||||
|
||||
interface TestingModuleMetadata extends ModuleMetadata {
|
||||
tapModule?(m: TestingModuleBuilder): void;
|
||||
@@ -30,7 +33,7 @@ export interface TestingModule extends NestjsTestingModule {
|
||||
export async function createModule(
|
||||
metadata: TestingModuleMetadata = {}
|
||||
): Promise<TestingModule> {
|
||||
const config = new ConfigFactory().config;
|
||||
const config = getDefaultConfig();
|
||||
const runtimeConfig = await createTestRuntimeConfig(
|
||||
config.db.datasourceUrl,
|
||||
config.indexer
|
||||
@@ -43,11 +46,14 @@ export async function createModule(
|
||||
imports: [...functionalityModules, ...(meta.imports ?? [])],
|
||||
});
|
||||
|
||||
builder
|
||||
.overrideProvider(EventBus)
|
||||
.useValue(new MockEventBus())
|
||||
.overrideProvider(BACKEND_RUNTIME_CONFIG_PATHS)
|
||||
.useValue([runtimeConfig.configPath]);
|
||||
builder.overrideProvider(EventBus).useValue(new MockEventBus());
|
||||
builder.overrideProvider(ServerConfigHandle).useFactory({
|
||||
factory: (overrides?: DeepPartial<AppConfig>) => {
|
||||
applyTestConfigOverrides(runtimeConfig.configPath, overrides);
|
||||
return new ServerConfigHandle(runtimeConfig.configPath);
|
||||
},
|
||||
inject: [{ token: OVERRIDE_CONFIG_TOKEN, optional: true }],
|
||||
});
|
||||
|
||||
// when custom override happens
|
||||
if (tapModule) {
|
||||
|
||||
@@ -13,6 +13,7 @@ interface Context {
|
||||
db: PrismaClient;
|
||||
cronJob: DocStorageCronJob;
|
||||
runtime: {
|
||||
nodeCryptoPrivateKey: Sinon.SinonStub;
|
||||
cleanupExpiredSnapshotHistories: Sinon.SinonStub;
|
||||
executeAuthSessionCommandV1: Sinon.SinonStub;
|
||||
};
|
||||
@@ -23,6 +24,7 @@ const test = ava as TestFn<Context>;
|
||||
// cleanup database before each test
|
||||
test.before(async t => {
|
||||
t.context.runtime = {
|
||||
nodeCryptoPrivateKey: Sinon.stub().returns(''),
|
||||
cleanupExpiredSnapshotHistories: Sinon.stub(),
|
||||
executeAuthSessionCommandV1: Sinon.stub().resolves({}),
|
||||
};
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { serverConfigQuery, ServerFeature } from '@affine/graphql';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
|
||||
import { app, e2e } from '../test';
|
||||
|
||||
e2e('should expose the indexer feature when it is enabled', async t => {
|
||||
const enabled = app.get(Config).indexer.enabled;
|
||||
const enabled = app.get(BackendRuntimeProvider).searchEnabled();
|
||||
const { serverConfig } = await app.gql({ query: serverConfigQuery });
|
||||
|
||||
t.is(
|
||||
|
||||
@@ -22,17 +22,17 @@ import {
|
||||
GlobalExceptionFilter,
|
||||
OneMB,
|
||||
} from '../../base';
|
||||
import { OVERRIDE_CONFIG_TOKEN } from '../../base/config/factory';
|
||||
import { getDefaultConfig } from '../../base/config/register';
|
||||
import { ThrottlerStorage } from '../../base/throttler';
|
||||
import { SocketIoAdapter } from '../../base/websocket';
|
||||
import { AuthGuard, AuthService } from '../../core/auth';
|
||||
import {
|
||||
BACKEND_RUNTIME_CONFIG_PATHS,
|
||||
BackendRuntimeProvider,
|
||||
} from '../../core/backend-runtime';
|
||||
import { BackendRuntimeProvider } from '../../core/backend-runtime';
|
||||
import { Mailer } from '../../core/mail';
|
||||
import { StorageRuntimeProvider } from '../../core/storage-runtime';
|
||||
import { ServerRole } from '../../env';
|
||||
import { Models } from '../../models';
|
||||
import { ServerConfigHandle } from '../../native';
|
||||
import { IndexerService } from '../../plugins/indexer/service';
|
||||
import {
|
||||
createFactory,
|
||||
@@ -42,7 +42,10 @@ import {
|
||||
MockUserInput,
|
||||
} from '../mocks';
|
||||
import { parseCookies, TEST_LOG_LEVEL } from '../utils';
|
||||
import { createTestRuntimeConfig } from '../utils/runtime-config';
|
||||
import {
|
||||
applyTestConfigOverrides,
|
||||
createTestRuntimeConfig,
|
||||
} from '../utils/runtime-config';
|
||||
|
||||
interface TestingAppMetadata {
|
||||
tapModule?(m: TestingModuleBuilder): void;
|
||||
@@ -252,7 +255,7 @@ export class TestingApp extends NestApplication {
|
||||
export async function createApp(
|
||||
metadata: TestingAppMetadata = {}
|
||||
): Promise<TestingApp> {
|
||||
const config = new ConfigFactory().config;
|
||||
const config = getDefaultConfig();
|
||||
const runtimeConfig = await createTestRuntimeConfig(
|
||||
config.db.datasourceUrl,
|
||||
config.indexer
|
||||
@@ -265,9 +268,13 @@ export async function createApp(
|
||||
});
|
||||
|
||||
builder.overrideProvider(Mailer).useValue(new MockMailer());
|
||||
builder
|
||||
.overrideProvider(BACKEND_RUNTIME_CONFIG_PATHS)
|
||||
.useValue([runtimeConfig.configPath]);
|
||||
builder.overrideProvider(ServerConfigHandle).useFactory({
|
||||
factory: (overrides?: DeepPartial<AppConfig>) => {
|
||||
applyTestConfigOverrides(runtimeConfig.configPath, overrides);
|
||||
return new ServerConfigHandle(runtimeConfig.configPath);
|
||||
},
|
||||
inject: [{ token: OVERRIDE_CONFIG_TOKEN, optional: true }],
|
||||
});
|
||||
|
||||
// when custom override happens
|
||||
if (tapModule) {
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
SearchTable,
|
||||
} from '@affine/graphql';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
|
||||
import { createDocWithMarkdown } from '../../../native';
|
||||
import { Mockers } from '../../mocks';
|
||||
import {
|
||||
@@ -14,7 +14,9 @@ import {
|
||||
reconcileSearchProjection,
|
||||
} from '../test';
|
||||
|
||||
const indexerE2e = app.get(Config).indexer.enabled ? e2e.serial : e2e.skip;
|
||||
const indexerE2e = app.get(BackendRuntimeProvider).searchEnabled()
|
||||
? e2e.serial
|
||||
: e2e.skip;
|
||||
|
||||
indexerE2e('should aggregate by docId', async t => {
|
||||
const owner = await app.signup();
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
SearchTable,
|
||||
} from '@affine/graphql';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
import { ServerService } from '../../../core/config';
|
||||
import { DocRole } from '../../../models';
|
||||
import { createDocWithMarkdown } from '../../../native';
|
||||
import { Mockers } from '../../mocks';
|
||||
@@ -16,9 +16,10 @@ import {
|
||||
reconcileSearchProjection,
|
||||
} from '../test';
|
||||
|
||||
const indexer = (await app.get(ServerService).getEffectiveAdminConfig())
|
||||
.indexer;
|
||||
const manticoreSearchEnabled =
|
||||
app.get(Config).indexer.enabled &&
|
||||
app.get(Config).indexer.provider.type === 'manticoresearch';
|
||||
indexer?.enabled && indexer.provider?.type === 'manticoresearch';
|
||||
const manticoreSearchE2e = manticoreSearchEnabled ? e2e.serial : e2e.skip;
|
||||
|
||||
async function indexDocument(
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
SearchTable,
|
||||
} from '@affine/graphql';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
import { ServerService } from '../../../core/config';
|
||||
import { createDocWithMarkdown } from '../../../native';
|
||||
import { Mockers } from '../../mocks';
|
||||
import {
|
||||
@@ -14,9 +14,10 @@ import {
|
||||
reconcileSearchProjection,
|
||||
} from '../test';
|
||||
|
||||
const indexer = app.get(Config).indexer;
|
||||
const indexer = (await app.get(ServerService).getEffectiveAdminConfig())
|
||||
.indexer;
|
||||
const remoteE2e =
|
||||
indexer.enabled && indexer.provider.type === 'elasticsearch'
|
||||
indexer?.enabled && indexer.provider?.type === 'elasticsearch'
|
||||
? e2e.serial
|
||||
: e2e.skip;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { indexerSearchDocsQuery } from '@affine/graphql';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
|
||||
import { createDocWithMarkdown } from '../../../native';
|
||||
import { Mockers } from '../../mocks';
|
||||
import {
|
||||
@@ -10,7 +10,9 @@ import {
|
||||
reconcileSearchProjection,
|
||||
} from '../test';
|
||||
|
||||
const indexerE2e = app.get(Config).indexer.enabled ? e2e.serial : e2e.skip;
|
||||
const indexerE2e = app.get(BackendRuntimeProvider).searchEnabled()
|
||||
? e2e.serial
|
||||
: e2e.skip;
|
||||
|
||||
indexerE2e('should search docs by keyword', async t => {
|
||||
const owner = await app.signup();
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
SearchTable,
|
||||
} from '@affine/graphql';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
|
||||
import { DocRole } from '../../../models';
|
||||
import { createDocWithMarkdown } from '../../../native';
|
||||
import { Mockers } from '../../mocks';
|
||||
@@ -15,7 +15,9 @@ import {
|
||||
reconcileSearchProjection,
|
||||
} from '../test';
|
||||
|
||||
const indexerE2e = app.get(Config).indexer.enabled ? e2e.serial : e2e.skip;
|
||||
const indexerE2e = app.get(BackendRuntimeProvider).searchEnabled()
|
||||
? e2e.serial
|
||||
: e2e.skip;
|
||||
|
||||
async function indexDoc(
|
||||
workspaceId: string,
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createHash, createHmac } from 'node:crypto';
|
||||
import { mock } from 'node:test';
|
||||
|
||||
import {
|
||||
Config,
|
||||
ConfigFactory,
|
||||
PROXY_MULTIPART_PATH,
|
||||
PROXY_UPLOAD_PATH,
|
||||
type R2StorageConfig,
|
||||
@@ -37,8 +35,54 @@ class MockStorageRuntime {
|
||||
contentLength?: number;
|
||||
}[] = [];
|
||||
|
||||
uploadUrlConfig() {
|
||||
const storage = currentBlobStorage;
|
||||
if (storage.provider !== 'cloudflare-r2' && storage.provider !== 'aws-s3') {
|
||||
return null;
|
||||
}
|
||||
const value = (storage.config as R2StorageConfig).usePresignedURL;
|
||||
if (!value?.enabled) return null;
|
||||
return {
|
||||
proxyUpload: !!value.signKey,
|
||||
urlPrefix: value.urlPrefix,
|
||||
};
|
||||
}
|
||||
|
||||
signUploadToken(
|
||||
_scope: string,
|
||||
path: string,
|
||||
fields: (string | number)[],
|
||||
expiresAt: number
|
||||
) {
|
||||
const storage = currentBlobStorage;
|
||||
const key = (storage.config as R2StorageConfig).usePresignedURL?.signKey;
|
||||
if (!key) return null;
|
||||
return createHmac('sha256', key)
|
||||
.update(
|
||||
JSON.stringify([
|
||||
'affine-storage-upload',
|
||||
1,
|
||||
'PUT',
|
||||
path,
|
||||
...fields,
|
||||
expiresAt,
|
||||
])
|
||||
)
|
||||
.digest('base64url');
|
||||
}
|
||||
|
||||
verifyUploadToken(
|
||||
scope: string,
|
||||
path: string,
|
||||
fields: (string | number)[],
|
||||
expiresAt: number,
|
||||
token: string
|
||||
) {
|
||||
return this.signUploadToken(scope, path, fields, expiresAt) === token;
|
||||
}
|
||||
|
||||
async providerCapabilities() {
|
||||
const storage = app.get(Config).storages.blob.storage;
|
||||
const storage = currentBlobStorage;
|
||||
if (storage.provider !== 'cloudflare-r2') {
|
||||
return {
|
||||
put: true,
|
||||
@@ -187,17 +231,16 @@ const baseR2Storage: StorageProviderConfig = {
|
||||
},
|
||||
};
|
||||
|
||||
let defaultBlobStorage: StorageProviderConfig;
|
||||
let currentBlobStorage: StorageProviderConfig;
|
||||
let runtime: MockStorageRuntime;
|
||||
|
||||
e2e.before(() => {
|
||||
defaultBlobStorage = structuredClone(app.get(Config).storages.blob.storage);
|
||||
});
|
||||
|
||||
e2e.beforeEach(async () => {
|
||||
runtime = new MockStorageRuntime();
|
||||
const rt = app.get(StorageRuntimeProvider);
|
||||
for (const method of [
|
||||
'uploadUrlConfig',
|
||||
'signUploadToken',
|
||||
'verifyUploadToken',
|
||||
'providerCapabilities',
|
||||
'presignPut',
|
||||
'createMultipartUpload',
|
||||
@@ -215,13 +258,11 @@ e2e.beforeEach(async () => {
|
||||
});
|
||||
|
||||
e2e.afterEach.always(async () => {
|
||||
await setBlobStorage(defaultBlobStorage);
|
||||
mock.reset();
|
||||
});
|
||||
|
||||
async function setBlobStorage(storage: StorageProviderConfig) {
|
||||
const configFactory = app.get(ConfigFactory);
|
||||
configFactory.override({ storages: { blob: { storage } } });
|
||||
currentBlobStorage = storage;
|
||||
}
|
||||
|
||||
async function useR2Storage(
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import test from 'ava';
|
||||
|
||||
import { Env } from '../env';
|
||||
import { DeploymentType, Env } from '../env';
|
||||
|
||||
const envs = { ...process.env };
|
||||
test.beforeEach(() => {
|
||||
@@ -64,6 +68,24 @@ test('should read DEPLOYMENT_TYPE', t => {
|
||||
process.env.DEPLOYMENT_TYPE = 'unknown';
|
||||
new Env();
|
||||
});
|
||||
|
||||
const directory = mkdtempSync(join(tmpdir(), 'affine-env-config-'));
|
||||
try {
|
||||
const configPath = join(directory, 'config.json');
|
||||
writeFileSync(
|
||||
configPath,
|
||||
JSON.stringify({ deployment: { type: 'selfhosted' } })
|
||||
);
|
||||
process.env.DEPLOYMENT_TYPE = 'affine';
|
||||
const selected = new Env(configPath);
|
||||
t.is(selected.serverConfigHandle.deploymentType, 'selfhosted');
|
||||
t.is(selected.DEPLOYMENT_TYPE, DeploymentType.Selfhosted);
|
||||
t.true(selected.selfhosted);
|
||||
process.env.DEPLOYMENT_TYPE = 'unknown';
|
||||
t.is(new Env(configPath).DEPLOYMENT_TYPE, DeploymentType.Selfhosted);
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('should read FLAVOR', t => {
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
import test from 'ava';
|
||||
|
||||
import { BackendRuntime, resolveEntitlementV1 } from '../native';
|
||||
import {
|
||||
BackendRuntime,
|
||||
resolveEntitlementV1,
|
||||
ServerConfigHandle,
|
||||
} from '../native';
|
||||
|
||||
test('native entitlement boundary maps invalid input to InvalidArg', async t => {
|
||||
const runtime = new BackendRuntime(
|
||||
'test-private-key',
|
||||
[],
|
||||
undefined,
|
||||
undefined
|
||||
new ServerConfigHandle(
|
||||
resolve(env.projectRoot, '../../../.docker/selfhost/config.json.example')
|
||||
),
|
||||
'test-private-key'
|
||||
);
|
||||
t.teardown(() => runtime.stop());
|
||||
const schemaError = t.throws(() =>
|
||||
|
||||
@@ -117,6 +117,22 @@ test('oauth endpoints retain validation and callback error shapes', async t => {
|
||||
|
||||
test('configured oauth providers remain visible through the reader', async t => {
|
||||
const { app } = t.context;
|
||||
t.deepEqual(app.get(OAuthService).providers.sort(), ['github', 'google']);
|
||||
t.true(app.get(ServerService).features.includes(ServerFeature.OAuth));
|
||||
const oauth = app.get(OAuthService);
|
||||
const server = app.get(ServerService);
|
||||
t.deepEqual(oauth.providers.sort(), ['github', 'google']);
|
||||
t.true(server.features.includes(ServerFeature.OAuth));
|
||||
|
||||
await server.updateConfig(null, [
|
||||
{
|
||||
module: 'oauth',
|
||||
key: 'providers.github',
|
||||
value: { clientId: '', clientSecret: '' },
|
||||
},
|
||||
]);
|
||||
t.deepEqual(oauth.providers, ['google']);
|
||||
|
||||
await server.updateConfig(null, [
|
||||
{ module: 'oauth', key: 'providers.github', clear: true },
|
||||
]);
|
||||
t.deepEqual(oauth.providers.sort(), ['github', 'google']);
|
||||
});
|
||||
@@ -1,7 +1,6 @@
|
||||
import ava from 'ava';
|
||||
import Sinon from 'sinon';
|
||||
|
||||
import type { Config } from '../../base';
|
||||
import { BackendRuntimeProvider } from '../../core/backend-runtime';
|
||||
import { WorkspaceService } from '../../core/workspaces';
|
||||
import type { Models } from '../../models';
|
||||
@@ -19,16 +18,14 @@ ava('payment HTTP and event adapters only forward protocol data', async t => {
|
||||
const execute = runtime.executePaymentCommandV1 as Sinon.SinonStub;
|
||||
const capture = runtime.capturePaymentWebhookV1 as Sinon.SinonStub;
|
||||
const workspace = Sinon.createStubInstance(WorkspaceService);
|
||||
const config = {
|
||||
payment: { enabled: true, stripe: { apiKey: 'sk_test_protocol' } },
|
||||
} as Config;
|
||||
runtime.stripeEnabled.returns(true);
|
||||
const models = {
|
||||
workspaceUser: {
|
||||
chargedCount: Sinon.stub().resolves(3),
|
||||
getOwner: Sinon.stub().resolves({ id: 'owner-1' }),
|
||||
},
|
||||
} as unknown as Models;
|
||||
const events = new PaymentEventHandlers(workspace, runtime, config, models);
|
||||
const events = new PaymentEventHandlers(workspace, runtime, models);
|
||||
|
||||
execute.resolves({ status: 'pending' });
|
||||
await events.prepareSubscriptionCancellation({ id: 'user-1' });
|
||||
@@ -44,6 +41,9 @@ ava('payment HTTP and event adapters only forward protocol data', async t => {
|
||||
plan: 'team',
|
||||
quantity: 3,
|
||||
});
|
||||
runtime.stripeEnabled.returns(false);
|
||||
await events.prepareSubscriptionCancellation({ id: 'user-2' });
|
||||
t.is(execute.callCount, 2);
|
||||
|
||||
workspace.isTeamWorkspace.resolves(false);
|
||||
workspace.sendTeamWorkspaceUpgradedEmail.resolves();
|
||||
|
||||
@@ -30,14 +30,16 @@ ava(
|
||||
const url = Sinon.createStubInstance(URLHelper);
|
||||
url.safeLink.callsFake(path => new URL(path, 'https://app.example').href);
|
||||
const config = {
|
||||
payment: { enabled: false, showLifetimePrice: false },
|
||||
payment: { showLifetimePrice: false },
|
||||
} as Config;
|
||||
const service = new SubscriptionService(runtime, config, server, url);
|
||||
const paymentEnabled = runtime.paymentEnabled as Sinon.SinonStub;
|
||||
|
||||
paymentEnabled.returns(false);
|
||||
service.onConfigInit();
|
||||
t.true(server.disableFeature.calledWith(ServerFeature.Payment));
|
||||
config.payment.enabled = true;
|
||||
service.onConfigChanged({ updates: { payment: { enabled: true } } });
|
||||
paymentEnabled.returns(true);
|
||||
service.onConfigApplied({ updates: { payment: { enabled: true } } });
|
||||
t.true(server.enableFeature.calledWith(ServerFeature.Payment));
|
||||
|
||||
command.resolves([
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import { generateKeyPairSync } from 'node:crypto';
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { set } from 'lodash-es';
|
||||
|
||||
import { getDefaultConfig, override } from '../../base/config/register';
|
||||
|
||||
const { privateKey } = generateKeyPairSync('ec', { namedCurve: 'P-256' });
|
||||
const testPrivateKey = privateKey
|
||||
.export({ format: 'pem', type: 'pkcs8' })
|
||||
@@ -23,6 +28,7 @@ export async function createTestRuntimeConfig(
|
||||
await writeFile(
|
||||
configPath,
|
||||
JSON.stringify({
|
||||
deployment: { type: globalThis.env.selfhosted ? 'selfhosted' : 'cloud' },
|
||||
crypto: { privateKey: testPrivateKey },
|
||||
db: { datasourceUrl: databaseUrl },
|
||||
storages: {
|
||||
@@ -37,7 +43,8 @@ export async function createTestRuntimeConfig(
|
||||
enabled: indexer.enabled,
|
||||
provider: indexer.provider,
|
||||
},
|
||||
})
|
||||
}),
|
||||
{ mode: 0o600 }
|
||||
);
|
||||
return {
|
||||
configPath,
|
||||
@@ -45,3 +52,31 @@ export async function createTestRuntimeConfig(
|
||||
cleanup: () => rm(directory, { recursive: true, force: true }),
|
||||
};
|
||||
}
|
||||
|
||||
export function applyTestConfigOverrides(
|
||||
configPath: string,
|
||||
overrides: DeepPartial<AppConfig> = {}
|
||||
) {
|
||||
const config = getDefaultConfig();
|
||||
const fixture = JSON.parse(readFileSync(configPath, 'utf-8'));
|
||||
for (const [module, values] of Object.entries(fixture)) {
|
||||
if (module === 'deployment') continue;
|
||||
for (const [key, value] of Object.entries(
|
||||
values as Record<string, unknown>
|
||||
)) {
|
||||
set(config, `${module}.${key}`, value);
|
||||
}
|
||||
}
|
||||
override(config, overrides);
|
||||
writeFileSync(
|
||||
configPath,
|
||||
JSON.stringify({
|
||||
...config,
|
||||
deployment: fixture.deployment,
|
||||
storages: {
|
||||
'avatar.storage': config.storages.avatar.storage,
|
||||
'blob.storage': config.storages.blob.storage,
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -10,18 +10,23 @@ import { PrismaClient } from '@prisma/client';
|
||||
|
||||
import { buildAppModule, FunctionalityModules } from '../../app.module';
|
||||
import { AFFiNELogger, ConfigFactory } from '../../base';
|
||||
import { OVERRIDE_CONFIG_TOKEN } from '../../base/config/factory';
|
||||
import { getDefaultConfig } from '../../base/config/register';
|
||||
import { GqlModule } from '../../base/graphql';
|
||||
import { ServerConfigModule } from '../../core';
|
||||
import { AuthGuard, AuthModule } from '../../core/auth';
|
||||
import { BACKEND_RUNTIME_CONFIG_PATHS } from '../../core/backend-runtime';
|
||||
import { Mailer, MailModule } from '../../core/mail';
|
||||
import { ModelsModule } from '../../models';
|
||||
import { ServerConfigHandle } from '../../native';
|
||||
// for jsdoc inference
|
||||
// oxlint-disable-next-line no-unused-vars
|
||||
import type { createModule } from '../create-module';
|
||||
import { createFactory } from '../mocks';
|
||||
import { MockMailer } from '../mocks/mailer.mock';
|
||||
import { createTestRuntimeConfig } from './runtime-config';
|
||||
import {
|
||||
applyTestConfigOverrides,
|
||||
createTestRuntimeConfig,
|
||||
} from './runtime-config';
|
||||
import { initTestingDB, TEST_LOG_LEVEL } from './utils';
|
||||
|
||||
interface TestingModuleMetadata extends ModuleMetadata {
|
||||
@@ -68,7 +73,7 @@ export async function createTestingModule(
|
||||
moduleDef: TestingModuleMetadata = {},
|
||||
autoInitialize = true
|
||||
): Promise<TestingModule> {
|
||||
const config = new ConfigFactory().config;
|
||||
const config = getDefaultConfig();
|
||||
const runtimeConfig = await createTestRuntimeConfig(
|
||||
config.db.datasourceUrl,
|
||||
config.indexer
|
||||
@@ -103,9 +108,13 @@ export async function createTestingModule(
|
||||
});
|
||||
|
||||
builder.overrideProvider(Mailer).useClass(MockMailer);
|
||||
builder
|
||||
.overrideProvider(BACKEND_RUNTIME_CONFIG_PATHS)
|
||||
.useValue([runtimeConfig.configPath]);
|
||||
builder.overrideProvider(ServerConfigHandle).useFactory({
|
||||
factory: (overrides?: DeepPartial<AppConfig>) => {
|
||||
applyTestConfigOverrides(runtimeConfig.configPath, overrides);
|
||||
return new ServerConfigHandle(runtimeConfig.configPath);
|
||||
},
|
||||
inject: [{ token: OVERRIDE_CONFIG_TOKEN, optional: true }],
|
||||
});
|
||||
if (moduleDef.tapModule) {
|
||||
moduleDef.tapModule(builder);
|
||||
}
|
||||
|
||||
@@ -2,9 +2,13 @@ import { Controller, Get } from '@nestjs/common';
|
||||
|
||||
import { SkipThrottle } from './base';
|
||||
import { Public } from './core/auth';
|
||||
import { DeploymentType } from './env';
|
||||
import { ServerConfigHandle } from './native';
|
||||
|
||||
@Controller('/info')
|
||||
export class AppController {
|
||||
constructor(private readonly serverConfig: ServerConfigHandle) {}
|
||||
|
||||
@SkipThrottle()
|
||||
@Public()
|
||||
@Get()
|
||||
@@ -12,7 +16,10 @@ export class AppController {
|
||||
return {
|
||||
compatibility: env.version,
|
||||
message: `AFFiNE ${env.version} Server`,
|
||||
type: env.DEPLOYMENT_TYPE,
|
||||
type:
|
||||
this.serverConfig.deploymentType === 'selfhosted'
|
||||
? DeploymentType.Selfhosted
|
||||
: DeploymentType.Affine,
|
||||
flavor: env.FLAVOR,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,28 +1,184 @@
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import test from 'ava';
|
||||
import { get, has } from 'lodash-es';
|
||||
|
||||
import { createModule } from '../../../__tests__/create-module';
|
||||
import {
|
||||
applyTestConfigOverrides,
|
||||
createTestRuntimeConfig,
|
||||
} from '../../../__tests__/utils/runtime-config';
|
||||
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
|
||||
import { ServerConfigHandle } from '../../../native';
|
||||
import { InvalidAppConfig } from '../../error';
|
||||
import { CryptoHelper } from '../../helpers';
|
||||
import { CacheRedis, SessionRedis, SocketIoRedis } from '../../redis/instances';
|
||||
import { Config } from '../config';
|
||||
import { ConfigFactory, ConfigModule } from '../index';
|
||||
import { override } from '../register';
|
||||
import { getDefaultConfig, override } from '../register';
|
||||
|
||||
const module = await createModule();
|
||||
test.after.always(async () => {
|
||||
await module.close();
|
||||
});
|
||||
|
||||
test('should create config', t => {
|
||||
test('test runtime config applies explicit overrides after fixture values', async t => {
|
||||
const fixture = await createTestRuntimeConfig(
|
||||
'postgresql://test:test@localhost/test',
|
||||
getDefaultConfig().indexer
|
||||
);
|
||||
try {
|
||||
applyTestConfigOverrides(fixture.configPath, {
|
||||
copilot: { enabled: false },
|
||||
storages: {
|
||||
blob: {
|
||||
storage: {
|
||||
provider: 'fs',
|
||||
bucket: 'blobs',
|
||||
config: { path: fixture.storagePath },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const config = JSON.parse(readFileSync(fixture.configPath, 'utf-8'));
|
||||
t.false(config.copilot.enabled);
|
||||
t.is(config.storages['blob.storage'].provider, 'fs');
|
||||
t.is(config.storages['avatar.storage'].provider, 'assetpack');
|
||||
} finally {
|
||||
await fixture.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('should create config', async t => {
|
||||
const config = module.get(Config);
|
||||
const crypto = module.get(CryptoHelper);
|
||||
const runtime = module.get(BackendRuntimeProvider);
|
||||
|
||||
crypto.onConfigInit();
|
||||
t.deepEqual(
|
||||
crypto.keyPair.sha256.privateKey,
|
||||
crypto.sha256(runtime.nodeCryptoPrivateKey())
|
||||
);
|
||||
|
||||
t.is(typeof config.auth.passwordRequirements.max, 'number');
|
||||
t.deepEqual(config.copilot.byok.allowedProviders, [
|
||||
'openai',
|
||||
t.deepEqual((await runtime.getByokPolicy()).allowedProviders, [
|
||||
'anthropic',
|
||||
'gemini',
|
||||
'fal',
|
||||
'gemini',
|
||||
'openai',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should read static config from the shared handle', t => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'affine-config-handle-'));
|
||||
try {
|
||||
const configPath = join(directory, 'config.json');
|
||||
writeFileSync(
|
||||
configPath,
|
||||
JSON.stringify({
|
||||
deployment: { type: 'selfhosted' },
|
||||
server: { name: 'shared-config' },
|
||||
copilot: { enabled: false },
|
||||
oauth: {
|
||||
providers: {
|
||||
github: {
|
||||
clientId: 'native-client',
|
||||
clientSecret: 'native-secret',
|
||||
},
|
||||
},
|
||||
},
|
||||
redis: {
|
||||
host: '127.0.0.1',
|
||||
db: 4,
|
||||
ioredis: { enableAutoPipelining: true },
|
||||
},
|
||||
})
|
||||
);
|
||||
const handle = new ServerConfigHandle(configPath);
|
||||
const factory = new ConfigFactory(handle, {
|
||||
redis: { host: 'ignored.example', db: 1 },
|
||||
});
|
||||
const config = factory.config;
|
||||
|
||||
t.is(handle.deploymentType, 'selfhosted');
|
||||
t.true(handle.hasBaselineConfigKey('oauth.providers.github'));
|
||||
t.false(handle.hasBaselineConfigKey('oauth.providers.oidc'));
|
||||
t.true(
|
||||
handle.nativeSecretAppConfigKeys().includes('oauth.providers.github')
|
||||
);
|
||||
t.is(JSON.parse(handle.nodeOwnedJson()).oauth, undefined);
|
||||
for (const key of handle.nativeAppConfigKeys()) {
|
||||
t.false(has(config, key), key);
|
||||
}
|
||||
factory.override({
|
||||
oauth: {
|
||||
providers: {
|
||||
github: {
|
||||
clientId: 'override-client',
|
||||
clientSecret: 'override-secret',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
t.is(get(config, 'oauth.providers.github'), undefined);
|
||||
const changes = factory.replaceDbOverrides({
|
||||
oauth: {
|
||||
providers: {
|
||||
github: {
|
||||
clientId: 'database-client',
|
||||
clientSecret: 'database-secret',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
t.is(get(config, 'oauth.providers.github'), undefined);
|
||||
t.is(get(changes, 'oauth.providers.github'), undefined);
|
||||
t.is(
|
||||
get(
|
||||
factory.redactNativeValues({
|
||||
oauth: {
|
||||
providers: {
|
||||
github: {
|
||||
clientId: 'event-client',
|
||||
clientSecret: 'event-secret',
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
'oauth.providers.github'
|
||||
),
|
||||
undefined
|
||||
);
|
||||
t.is(config.server.name, 'shared-config');
|
||||
t.false(has(config, 'redis.ioredis'));
|
||||
t.true(JSON.parse(handle.redisNodeOptionsJson()).enableAutoPipelining);
|
||||
t.deepEqual(JSON.parse(handle.publicNativeBaselineJson()), {
|
||||
copilot: { enabled: false },
|
||||
redis: { host: '127.0.0.1', db: 4 },
|
||||
});
|
||||
const cache = new CacheRedis(handle);
|
||||
const session = new SessionRedis(handle);
|
||||
const socket = new SocketIoRedis(handle);
|
||||
try {
|
||||
const redisUrl = new URL(handle.redisUrl()!);
|
||||
t.is(cache.options.host, redisUrl.hostname);
|
||||
t.true(cache.options.enableAutoPipelining);
|
||||
const baseDb = Number(redisUrl.pathname.slice(1));
|
||||
t.is(cache.options.db, baseDb);
|
||||
t.is(session.options.db, baseDb + 2);
|
||||
t.is(socket.options.db, baseDb + 3);
|
||||
} finally {
|
||||
cache.disconnect();
|
||||
session.disconnect();
|
||||
socket.disconnect();
|
||||
}
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('should override config', async t => {
|
||||
await using module = await createModule({
|
||||
imports: [
|
||||
@@ -187,11 +343,14 @@ test('should clone from original config without modifications', t => {
|
||||
const config = module.get(Config);
|
||||
const configFactory = module.get(ConfigFactory);
|
||||
|
||||
config.auth.allowSignup = !config.auth.allowSignup;
|
||||
config.auth.trustedCloudflareHeaders = !config.auth.trustedCloudflareHeaders;
|
||||
|
||||
const newConfig = configFactory.clone();
|
||||
|
||||
t.not(newConfig.auth.allowSignup, config.auth.allowSignup);
|
||||
t.not(
|
||||
newConfig.auth.trustedCloudflareHeaders,
|
||||
config.auth.trustedCloudflareHeaders
|
||||
);
|
||||
});
|
||||
|
||||
test('should override with undefined fields', async t => {
|
||||
@@ -204,17 +363,14 @@ test('should override with undefined fields', async t => {
|
||||
|
||||
configFactory.override({
|
||||
copilot: {
|
||||
providers: {
|
||||
exa: {
|
||||
key: '',
|
||||
// @ts-expect-error undefined field
|
||||
unknown: {
|
||||
apiKey: '123',
|
||||
},
|
||||
unknown: '123',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// @ts-expect-error undefined field
|
||||
t.deepEqual(config.copilot.providers.unknown, {
|
||||
apiKey: '123',
|
||||
});
|
||||
t.is(config.copilot.exa.unknown, '123');
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { ApplyType } from '../utils';
|
||||
import { NodeConfig } from './types';
|
||||
|
||||
export class Config extends ApplyType<AppConfig>() {}
|
||||
export class Config extends ApplyType<NodeConfig>() {}
|
||||
@@ -1,5 +1,7 @@
|
||||
import { Inject, Injectable, Optional } from '@nestjs/common';
|
||||
import { get, isEqual, set, unset } from 'lodash-es';
|
||||
|
||||
import { ServerConfigHandle } from '../../native';
|
||||
import { InvalidAppConfig } from '../error';
|
||||
import { APP_CONFIG_DESCRIPTORS, getDefaultConfig, override } from './register';
|
||||
|
||||
@@ -7,18 +9,27 @@ export const OVERRIDE_CONFIG_TOKEN = Symbol('OVERRIDE_CONFIG_TOKEN');
|
||||
|
||||
@Injectable()
|
||||
export class ConfigFactory {
|
||||
readonly #static: AppConfig;
|
||||
readonly #original: AppConfig;
|
||||
readonly #config: AppConfig;
|
||||
readonly #nativeKeys: ReadonlySet<string>;
|
||||
readonly #nativeSecretKeys: ReadonlySet<string>;
|
||||
get config() {
|
||||
return this.#config;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly serverConfig: ServerConfigHandle,
|
||||
@Inject(OVERRIDE_CONFIG_TOKEN)
|
||||
@Optional()
|
||||
private readonly overrides: DeepPartial<AppConfig> = {}
|
||||
) {
|
||||
this.#original = this.loadDefault();
|
||||
this.#nativeKeys = new Set(this.serverConfig.nativeAppConfigKeys());
|
||||
this.#nativeSecretKeys = new Set(
|
||||
this.serverConfig.nativeSecretAppConfigKeys()
|
||||
);
|
||||
this.#static = this.loadDefault();
|
||||
this.#original = structuredClone(this.#static);
|
||||
this.#config = structuredClone(this.#original);
|
||||
}
|
||||
|
||||
@@ -28,11 +39,83 @@ export class ConfigFactory {
|
||||
}
|
||||
|
||||
override(updates: DeepPartial<AppConfig>) {
|
||||
override(this.#original, updates);
|
||||
override(this.#config, updates);
|
||||
const visible = this.withoutNativeValues(updates);
|
||||
override(this.#original, visible);
|
||||
override(this.#config, visible);
|
||||
}
|
||||
|
||||
validate(updates: Array<{ module: string; key: string; value: any }>) {
|
||||
private withoutNativeValues(
|
||||
updates: DeepPartial<AppConfig>
|
||||
): DeepPartial<AppConfig> {
|
||||
const visible = structuredClone(updates);
|
||||
for (const key of this.#nativeKeys) {
|
||||
unset(visible, key);
|
||||
}
|
||||
return visible;
|
||||
}
|
||||
|
||||
redactNativeValues(updates: DeepPartial<AppConfig>): DeepPartial<AppConfig> {
|
||||
const redacted = structuredClone(updates);
|
||||
for (const key of this.#nativeKeys) {
|
||||
if (get(redacted, key) !== undefined) {
|
||||
set(redacted, key, undefined);
|
||||
}
|
||||
}
|
||||
return redacted;
|
||||
}
|
||||
|
||||
isNativeSecretKey(key: string) {
|
||||
return this.#nativeSecretKeys.has(key);
|
||||
}
|
||||
|
||||
resetPath(path: string) {
|
||||
const value = get(this.#static, path);
|
||||
if (value === undefined) {
|
||||
unset(this.#original, path);
|
||||
unset(this.#config, path);
|
||||
} else {
|
||||
set(this.#original, path, structuredClone(value));
|
||||
set(this.#config, path, structuredClone(value));
|
||||
}
|
||||
}
|
||||
|
||||
replaceDbOverrides(updates: DeepPartial<AppConfig>): DeepPartial<AppConfig> {
|
||||
const previous = this.clone();
|
||||
const changed = this.withoutNativeValues(updates);
|
||||
for (const [module, descriptors] of Object.entries(
|
||||
APP_CONFIG_DESCRIPTORS
|
||||
)) {
|
||||
for (const key of Object.keys(descriptors)) {
|
||||
const path = `${module}.${key}`;
|
||||
this.resetPath(path);
|
||||
}
|
||||
}
|
||||
this.override(updates);
|
||||
for (const [module, descriptors] of Object.entries(
|
||||
APP_CONFIG_DESCRIPTORS
|
||||
)) {
|
||||
for (const key of Object.keys(descriptors)) {
|
||||
const path = `${module}.${key}`;
|
||||
const value = get(this.#original, path);
|
||||
if (!isEqual(get(previous, path), value)) {
|
||||
set(changed, path, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const key of this.#nativeKeys) {
|
||||
set(changed, key, undefined);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
validate(
|
||||
updates: Array<{
|
||||
module: string;
|
||||
key: string;
|
||||
value?: any;
|
||||
clear?: boolean;
|
||||
}>
|
||||
) {
|
||||
const errors: InvalidAppConfig[] = [];
|
||||
|
||||
updates.forEach(update => {
|
||||
@@ -48,6 +131,21 @@ export class ConfigFactory {
|
||||
return;
|
||||
}
|
||||
|
||||
if (update.clear) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (update.value === undefined) {
|
||||
errors.push(
|
||||
new InvalidAppConfig({
|
||||
module: update.module,
|
||||
key: update.key,
|
||||
hint: 'A value is required when setting config',
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const { success, error } = descriptor.validate(update.value);
|
||||
if (!success) {
|
||||
error.issues.forEach(issue => {
|
||||
@@ -66,8 +164,9 @@ export class ConfigFactory {
|
||||
}
|
||||
|
||||
private loadDefault() {
|
||||
const config = getDefaultConfig();
|
||||
override(config, this.overrides);
|
||||
const config = getDefaultConfig(this.#nativeKeys);
|
||||
override(config, JSON.parse(this.serverConfig.nodeOwnedJson()));
|
||||
override(config, this.withoutNativeValues(this.overrides));
|
||||
return config;
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,21 @@
|
||||
import { DynamicModule, Global, Module, Provider } from '@nestjs/common';
|
||||
|
||||
import { ServerConfigHandle } from '../../native';
|
||||
import { Config } from './config';
|
||||
import { ConfigFactory, OVERRIDE_CONFIG_TOKEN } from './factory';
|
||||
import { ConfigProvider } from './provider';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [ConfigProvider, ConfigFactory],
|
||||
exports: [ConfigProvider, ConfigFactory],
|
||||
providers: [
|
||||
{
|
||||
provide: ServerConfigHandle,
|
||||
useFactory: () => env.serverConfigHandle,
|
||||
},
|
||||
ConfigProvider,
|
||||
ConfigFactory,
|
||||
],
|
||||
exports: [ServerConfigHandle, ConfigProvider, ConfigFactory],
|
||||
})
|
||||
export class ConfigModule {
|
||||
static override(overrides: DeepPartial<AppConfigSchema> = {}): DynamicModule {
|
||||
|
||||
@@ -1,7 +1,3 @@
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { mergeWith, once, set } from 'lodash-es';
|
||||
import { z } from 'zod';
|
||||
|
||||
@@ -242,6 +238,9 @@ export type NativeAppConfigDescriptor = {
|
||||
defaultValue: unknown;
|
||||
schema: JSONSchema;
|
||||
internal: boolean;
|
||||
envName?: string | null;
|
||||
envType?: string | null;
|
||||
link?: string | null;
|
||||
};
|
||||
|
||||
export function defineNativeModuleConfig<T extends keyof AppConfigSchema>(
|
||||
@@ -260,6 +259,13 @@ export function defineNativeModuleConfig<T extends keyof AppConfigSchema>(
|
||||
default: descriptor.defaultValue,
|
||||
schema: descriptor.schema,
|
||||
internal: descriptor.internal,
|
||||
link: descriptor.link ?? undefined,
|
||||
env:
|
||||
descriptor.envName &&
|
||||
(descriptor.envType === 'string' ||
|
||||
descriptor.envType === 'integer')
|
||||
? [descriptor.envName, descriptor.envType]
|
||||
: undefined,
|
||||
validate: (value: unknown) => {
|
||||
const errors = validate(module, descriptor.key, value);
|
||||
return errors.length
|
||||
@@ -281,33 +287,6 @@ export function defineNativeModuleConfig<T extends keyof AppConfigSchema>(
|
||||
} as Record<string, ConfigDefineDescriptor<unknown>>);
|
||||
}
|
||||
|
||||
export const CONFIG_JSON_PATHS = [
|
||||
join(env.projectRoot, 'config.json'),
|
||||
`${homedir()}/.affine/config/config.json`,
|
||||
];
|
||||
function readConfigJSONOverrides(path: string) {
|
||||
const overrides: DeepPartial<AppConfig> = {};
|
||||
if (existsSync(path)) {
|
||||
try {
|
||||
const config = JSON.parse(readFileSync(path, 'utf-8')) as AppConfig;
|
||||
|
||||
Object.entries(config).forEach(([key, value]) => {
|
||||
if (key === '$schema') {
|
||||
return;
|
||||
}
|
||||
|
||||
Object.entries(value).forEach(([k, v]) => {
|
||||
set(overrides, `${key}.${k}`, v);
|
||||
});
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Invalid json config file', e);
|
||||
}
|
||||
}
|
||||
|
||||
return overrides;
|
||||
}
|
||||
|
||||
export function override(config: AppConfig, update: DeepPartial<AppConfig>) {
|
||||
Object.keys(update).forEach(module => {
|
||||
const moduleDescriptors = APP_CONFIG_DESCRIPTORS[module];
|
||||
@@ -346,7 +325,9 @@ export function override(config: AppConfig, update: DeepPartial<AppConfig>) {
|
||||
});
|
||||
}
|
||||
|
||||
export function getDefaultConfig(): AppConfig {
|
||||
export function getDefaultConfig(
|
||||
excludedKeys: ReadonlySet<string> = new Set()
|
||||
): AppConfig {
|
||||
const config = {} as AppConfig;
|
||||
const envs = process.env;
|
||||
|
||||
@@ -354,6 +335,9 @@ export function getDefaultConfig(): AppConfig {
|
||||
const modulizedConfig = {};
|
||||
|
||||
for (const [key, desc] of Object.entries(defs)) {
|
||||
if (excludedKeys.has(`${module}.${key}`)) {
|
||||
continue;
|
||||
}
|
||||
let defaultValue = desc.default;
|
||||
|
||||
if (desc.env) {
|
||||
@@ -385,10 +369,5 @@ Error: ${issue.message}`;
|
||||
config[module] = modulizedConfig;
|
||||
}
|
||||
|
||||
CONFIG_JSON_PATHS.forEach(path => {
|
||||
const overrides = readConfigJSONOverrides(path);
|
||||
override(config, overrides);
|
||||
});
|
||||
|
||||
return config as AppConfigSchema;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
export const NODE_SECRET_CONFIG_KEYS = new Set([
|
||||
'calendar.google',
|
||||
'captcha.config',
|
||||
'copilot.exa',
|
||||
'copilot.unsplash',
|
||||
'mailer.fallbackSMTP.password',
|
||||
'mailer.SMTP.password',
|
||||
'telemetry.ga4.apiSecret',
|
||||
]);
|
||||
@@ -18,3 +18,27 @@ export type AppConfigByPath<Module extends keyof AppConfigSchema> =
|
||||
: never;
|
||||
}
|
||||
: never;
|
||||
|
||||
export type NodeConfig = Omit<
|
||||
AppConfig,
|
||||
| 'auth'
|
||||
| 'copilot'
|
||||
| 'crypto'
|
||||
| 'db'
|
||||
| 'indexer'
|
||||
| 'oauth'
|
||||
| 'payment'
|
||||
| 'redis'
|
||||
| 'storages'
|
||||
> & {
|
||||
auth: Pick<
|
||||
AppConfig['auth'],
|
||||
'passwordRequirements' | 'signInRateLimit' | 'trustedCloudflareHeaders'
|
||||
>;
|
||||
copilot: Pick<AppConfig['copilot'], 'exa' | 'unsplash'>;
|
||||
db: Pick<AppConfig['db'], 'prisma'>;
|
||||
payment: Pick<AppConfig['payment'], 'showLifetimePrice'>;
|
||||
storages: {
|
||||
avatar: Pick<AppConfig['storages']['avatar'], 'publicPath'>;
|
||||
};
|
||||
};
|
||||
@@ -7,6 +7,7 @@ import { CryptoHelper } from '../crypto';
|
||||
|
||||
const test = ava as TestFn<{
|
||||
crypto: CryptoHelper;
|
||||
setPrivateKey(key: string): void;
|
||||
}>;
|
||||
|
||||
function generateTestPrivateKey(): string {
|
||||
@@ -25,11 +26,13 @@ const privateKey = generateTestPrivateKey();
|
||||
const privateKey2 = generateTestPrivateKey();
|
||||
|
||||
test.beforeEach(async t => {
|
||||
let activeKey = privateKey;
|
||||
t.context.crypto = new CryptoHelper({
|
||||
crypto: {
|
||||
privateKey,
|
||||
},
|
||||
} as any);
|
||||
nodeCryptoPrivateKey: () => activeKey,
|
||||
});
|
||||
t.context.setPrivateKey = key => {
|
||||
activeKey = key;
|
||||
};
|
||||
t.context.crypto.onConfigInit();
|
||||
});
|
||||
|
||||
@@ -46,14 +49,22 @@ test('should verify signatures across key rotation', t => {
|
||||
const signatureV1 = t.context.crypto.sign(data);
|
||||
t.true(t.context.crypto.verify(signatureV1));
|
||||
|
||||
(t.context.crypto as any).config.crypto.privateKey = privateKey2;
|
||||
t.context.crypto.onConfigChanged({
|
||||
updates: { crypto: { privateKey: privateKey2 } },
|
||||
} as any);
|
||||
t.context.setPrivateKey(privateKey2);
|
||||
t.context.crypto.onConfigApplied({
|
||||
updates: { crypto: { privateKey: undefined } },
|
||||
});
|
||||
|
||||
const signatureV2 = t.context.crypto.sign(data);
|
||||
t.true(t.context.crypto.verify(signatureV1));
|
||||
t.true(t.context.crypto.verify(signatureV2));
|
||||
|
||||
const previousKeyHash = t.context.crypto.keyPair.sha256.privateKey;
|
||||
t.context.setPrivateKey('');
|
||||
t.context.crypto.onConfigApplied({
|
||||
updates: { crypto: { privateKey: undefined } },
|
||||
});
|
||||
t.notDeepEqual(t.context.crypto.keyPair.sha256.privateKey, previousKeyHash);
|
||||
t.true(t.context.crypto.verify(signatureV2));
|
||||
});
|
||||
|
||||
test('should same data should get different signature', t => {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { defineModuleConfig } from '../config';
|
||||
import serverNativeModule from '@affine/server-native';
|
||||
|
||||
import { defineNativeModuleConfig } from '../config';
|
||||
|
||||
declare global {
|
||||
interface AppConfigSchema {
|
||||
@@ -8,11 +10,8 @@ declare global {
|
||||
}
|
||||
}
|
||||
|
||||
defineModuleConfig('crypto', {
|
||||
privateKey: {
|
||||
desc: 'The private key for used by the crypto module to create signed tokens or encrypt data.',
|
||||
env: 'AFFINE_PRIVATE_KEY',
|
||||
default: '',
|
||||
schema: { type: 'string' },
|
||||
},
|
||||
});
|
||||
defineNativeModuleConfig(
|
||||
'crypto',
|
||||
serverNativeModule.appConfigDescriptors('crypto'),
|
||||
serverNativeModule.validateAppConfigValue
|
||||
);
|
||||
@@ -15,16 +15,17 @@ import {
|
||||
verify,
|
||||
} from 'node:crypto';
|
||||
|
||||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { hash as hashPassword } from '@node-rs/argon2';
|
||||
|
||||
import { AFFINE_PRO_PUBLIC_KEY } from '../../native';
|
||||
import { Config } from '../config';
|
||||
import { OnEvent } from '../event';
|
||||
|
||||
const NONCE_LENGTH = 12;
|
||||
const AUTH_TAG_LENGTH = 12;
|
||||
|
||||
export const CRYPTO_KEY_SOURCE = Symbol('CRYPTO_KEY_SOURCE');
|
||||
|
||||
function generatePrivateKey(): string {
|
||||
const { privateKey } = generateKeyPairSync('ec', {
|
||||
namedCurve: 'prime256v1',
|
||||
@@ -83,23 +84,27 @@ export class CryptoHelper implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
constructor(private readonly config: Config) {}
|
||||
constructor(
|
||||
@Inject(CRYPTO_KEY_SOURCE)
|
||||
private readonly source: { nodeCryptoPrivateKey(): string }
|
||||
) {}
|
||||
|
||||
@OnEvent('config.init')
|
||||
onConfigInit() {
|
||||
this.setup();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
onConfigChanged(event: Events['config.changed']) {
|
||||
if (event.updates.crypto?.privateKey) {
|
||||
@OnEvent('backendRuntime.configApplied')
|
||||
onConfigApplied(event: Events['backendRuntime.configApplied']) {
|
||||
if (event.updates.crypto && 'privateKey' in event.updates.crypto) {
|
||||
this.setup();
|
||||
}
|
||||
}
|
||||
|
||||
private setup() {
|
||||
const prevPublicKey = this.keyPair?.publicKey;
|
||||
const privateKey = this.config.crypto.privateKey || generatePrivateKey();
|
||||
const privateKey =
|
||||
this.source.nodeCryptoPrivateKey() || generatePrivateKey();
|
||||
const { priv, pub } = parseKey(privateKey);
|
||||
const publicKey = pub
|
||||
.export({ format: 'pem', type: 'spki' })
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import serverNativeModule from '@affine/server-native';
|
||||
import { RedisOptions } from 'ioredis';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { defineModuleConfig } from '../config';
|
||||
import { defineNativeModuleConfig } from '../config';
|
||||
|
||||
declare global {
|
||||
interface AppConfigSchema {
|
||||
@@ -18,37 +18,15 @@ declare global {
|
||||
}
|
||||
}
|
||||
|
||||
defineModuleConfig('redis', {
|
||||
db: {
|
||||
desc: 'The database index of redis server to be used(Must be less than 10).',
|
||||
default: 0,
|
||||
env: ['REDIS_SERVER_DATABASE', 'integer'],
|
||||
shape: z.number().int().nonnegative().max(10),
|
||||
},
|
||||
host: {
|
||||
desc: 'The host of the redis server.',
|
||||
default: 'localhost',
|
||||
env: ['REDIS_SERVER_HOST', 'string'],
|
||||
},
|
||||
port: {
|
||||
desc: 'The port of the redis server.',
|
||||
default: 6379,
|
||||
env: ['REDIS_SERVER_PORT', 'integer'],
|
||||
shape: z.number().positive(),
|
||||
},
|
||||
username: {
|
||||
desc: 'The username of the redis server.',
|
||||
default: '',
|
||||
env: ['REDIS_SERVER_USERNAME', 'string'],
|
||||
},
|
||||
password: {
|
||||
desc: 'The password of the redis server.',
|
||||
default: '',
|
||||
env: ['REDIS_SERVER_PASSWORD', 'string'],
|
||||
},
|
||||
ioredis: {
|
||||
desc: 'The config for the ioredis client.',
|
||||
default: {},
|
||||
link: 'https://github.com/luin/ioredis',
|
||||
},
|
||||
});
|
||||
defineNativeModuleConfig(
|
||||
'redis',
|
||||
serverNativeModule.appConfigDescriptors('redis'),
|
||||
serverNativeModule.validateAppConfigValue,
|
||||
{
|
||||
ioredis: {
|
||||
desc: 'The config for the ioredis client.',
|
||||
default: {},
|
||||
link: 'https://github.com/luin/ioredis',
|
||||
},
|
||||
}
|
||||
);
|
||||
@@ -5,8 +5,9 @@ import {
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { Redis as IORedis, RedisOptions } from 'ioredis';
|
||||
import { omit } from 'lodash-es';
|
||||
|
||||
import { Config } from '../config';
|
||||
import { ServerConfigHandle } from '../../native';
|
||||
|
||||
function redisOptions(options: RedisOptions) {
|
||||
return {
|
||||
@@ -15,6 +16,20 @@ function redisOptions(options: RedisOptions) {
|
||||
};
|
||||
}
|
||||
|
||||
function redisConnection(
|
||||
handle: ServerConfigHandle,
|
||||
dbOffset: number
|
||||
): [string, RedisOptions] {
|
||||
const url = new URL(handle.redisUrl() ?? 'redis://localhost:6379/0');
|
||||
const db = Number(url.pathname.slice(1) || 0) + dbOffset;
|
||||
url.pathname = `/${db}`;
|
||||
const options = omit(
|
||||
JSON.parse(handle.redisNodeOptionsJson()) as RedisOptions,
|
||||
['host', 'port', 'db', 'username', 'password']
|
||||
);
|
||||
return [url.toString(), redisOptions(options)];
|
||||
}
|
||||
|
||||
class Redis extends IORedis implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(this.constructor.name);
|
||||
|
||||
@@ -53,33 +68,21 @@ class Redis extends IORedis implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
@Injectable()
|
||||
export class CacheRedis extends Redis {
|
||||
constructor(config: Config) {
|
||||
super(redisOptions({ ...config.redis, ...config.redis.ioredis }));
|
||||
constructor(handle: ServerConfigHandle) {
|
||||
super(...redisConnection(handle, 0));
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class SessionRedis extends Redis {
|
||||
constructor(config: Config) {
|
||||
super(
|
||||
redisOptions({
|
||||
...config.redis,
|
||||
...config.redis.ioredis,
|
||||
db: (config.redis.db ?? 0) + 2,
|
||||
})
|
||||
);
|
||||
constructor(handle: ServerConfigHandle) {
|
||||
super(...redisConnection(handle, 2));
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class SocketIoRedis extends Redis {
|
||||
constructor(config: Config) {
|
||||
super(
|
||||
redisOptions({
|
||||
...config.redis,
|
||||
...config.redis.ioredis,
|
||||
db: (config.redis.db ?? 0) + 3,
|
||||
})
|
||||
);
|
||||
constructor(handle: ServerConfigHandle) {
|
||||
super(...redisConnection(handle, 3));
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import { JSONSchema } from '../../config';
|
||||
|
||||
export type StorageProviderName =
|
||||
| 'fs'
|
||||
| 'aws-s3'
|
||||
@@ -60,185 +58,6 @@ export type StorageProviderConfig = { bucket: string } & (
|
||||
}
|
||||
);
|
||||
|
||||
const S3ConfigSchema: JSONSchema = {
|
||||
type: 'object',
|
||||
description: 'The config for the S3 compatible storage provider.',
|
||||
properties: {
|
||||
endpoint: {
|
||||
type: 'string',
|
||||
description:
|
||||
'The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region.',
|
||||
},
|
||||
region: {
|
||||
type: 'string',
|
||||
description:
|
||||
'The region for the storage provider. Example: "us-east-1" or "auto" for R2.',
|
||||
},
|
||||
forcePathStyle: {
|
||||
type: 'boolean',
|
||||
description: 'Whether to use path-style bucket addressing.',
|
||||
},
|
||||
requestTimeoutMs: {
|
||||
type: 'number',
|
||||
description: 'Request timeout in milliseconds.',
|
||||
},
|
||||
minPartSize: {
|
||||
type: 'number',
|
||||
description: 'Minimum multipart part size in bytes.',
|
||||
},
|
||||
presign: {
|
||||
type: 'object',
|
||||
description: 'Presigned URL behavior configuration.',
|
||||
properties: {
|
||||
expiresInSeconds: {
|
||||
type: 'number',
|
||||
description: 'Expiration time in seconds for presigned URLs.',
|
||||
},
|
||||
signContentTypeForPut: {
|
||||
type: 'boolean',
|
||||
description: 'Whether to sign Content-Type for presigned PUT.',
|
||||
},
|
||||
},
|
||||
},
|
||||
credentials: {
|
||||
type: 'object',
|
||||
description: 'The credentials for the s3 compatible storage provider.',
|
||||
properties: {
|
||||
accessKeyId: {
|
||||
type: 'string',
|
||||
},
|
||||
secretAccessKey: {
|
||||
type: 'string',
|
||||
},
|
||||
sessionToken: {
|
||||
type: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
usePresignedURL: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.',
|
||||
properties: {
|
||||
enabled: {
|
||||
type: 'boolean',
|
||||
description: 'Whether browser upload URLs are enabled.',
|
||||
},
|
||||
urlPrefix: {
|
||||
type: 'string',
|
||||
description:
|
||||
'Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured.',
|
||||
},
|
||||
signKey: {
|
||||
type: 'string',
|
||||
description:
|
||||
'Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix.',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const S3ConfigPropertiesWithoutEndpoint = Object.fromEntries(
|
||||
Object.entries(
|
||||
(
|
||||
S3ConfigSchema as {
|
||||
type: 'object';
|
||||
properties?: Record<string, JSONSchema>;
|
||||
}
|
||||
).properties ?? {}
|
||||
).filter(([key]) => key !== 'endpoint')
|
||||
) as Record<string, JSONSchema>;
|
||||
|
||||
export const StorageJSONSchema: JSONSchema = {
|
||||
oneOf: [
|
||||
{
|
||||
type: 'object',
|
||||
properties: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
enum: ['fs'],
|
||||
},
|
||||
bucket: {
|
||||
type: 'string',
|
||||
},
|
||||
config: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
properties: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
enum: ['aws-s3'],
|
||||
},
|
||||
bucket: {
|
||||
type: 'string',
|
||||
},
|
||||
config: S3ConfigSchema,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
properties: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
enum: ['cloudflare-r2'],
|
||||
},
|
||||
bucket: {
|
||||
type: 'string',
|
||||
},
|
||||
config: {
|
||||
...S3ConfigSchema,
|
||||
properties: {
|
||||
...S3ConfigPropertiesWithoutEndpoint,
|
||||
accountId: {
|
||||
type: 'string' as const,
|
||||
description:
|
||||
'The account id for the cloudflare r2 storage provider.',
|
||||
},
|
||||
jurisdiction: {
|
||||
type: 'string' as const,
|
||||
enum: [...R2_JURISDICTIONS],
|
||||
description:
|
||||
'Optional jurisdiction for the cloudflare r2 endpoint. Set to "eu" for EU buckets.',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
properties: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
enum: ['assetpack'],
|
||||
},
|
||||
bucket: {
|
||||
type: 'string',
|
||||
},
|
||||
config: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string',
|
||||
},
|
||||
},
|
||||
required: ['path'],
|
||||
},
|
||||
},
|
||||
required: ['provider', 'bucket', 'config'],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export type * from '../types';
|
||||
export {
|
||||
applyAttachHeaders,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { createHmac } from 'node:crypto';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import type { Response } from 'express';
|
||||
@@ -69,20 +68,3 @@ export const SIGNED_URL_EXPIRED = 60 * 60; // 1 hour
|
||||
export const STORAGE_PROXY_ROOT = '/api/storage';
|
||||
export const PROXY_UPLOAD_PATH = `${STORAGE_PROXY_ROOT}/upload`;
|
||||
export const PROXY_MULTIPART_PATH = `${STORAGE_PROXY_ROOT}/multipart`;
|
||||
|
||||
export function createStorageUploadToken(
|
||||
path: string,
|
||||
fields: (string | number)[],
|
||||
expiresAt: number,
|
||||
signKey: string
|
||||
) {
|
||||
const canonical = JSON.stringify([
|
||||
'affine-storage-upload',
|
||||
1,
|
||||
'PUT',
|
||||
path,
|
||||
...fields,
|
||||
expiresAt,
|
||||
]);
|
||||
return createHmac('sha256', signKey).update(canonical).digest('base64url');
|
||||
}
|
||||
@@ -100,10 +100,15 @@ function buildProgram(logger: Logger) {
|
||||
|
||||
program
|
||||
.command('import-config [path]')
|
||||
.description('import config from a file')
|
||||
.description(
|
||||
'import config from a file; changes take effect on the next server start'
|
||||
)
|
||||
.action(async path => {
|
||||
await withCliApp(logger, async app => {
|
||||
await app.get(ImportConfigCommand).execute(path);
|
||||
logger.log(
|
||||
'Config imported; changes take effect on the next server start.'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import serverNativeModule from '@affine/server-native';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { defineModuleConfig } from '../../base';
|
||||
import { defineNativeModuleConfig } from '../../base';
|
||||
|
||||
export interface AuthConfig {
|
||||
session: {
|
||||
@@ -17,7 +18,6 @@ export interface AuthConfig {
|
||||
allowSignup: boolean;
|
||||
allowSignupForOauth: boolean;
|
||||
requireEmailDomainVerification: boolean;
|
||||
requireEmailVerification: boolean;
|
||||
newAccountActionDelay: number;
|
||||
trustedCloudflareHeaders: boolean;
|
||||
signInRateLimit: ConfigItem<{
|
||||
@@ -37,118 +37,54 @@ declare global {
|
||||
}
|
||||
}
|
||||
|
||||
defineModuleConfig('auth', {
|
||||
allowSignup: {
|
||||
desc: 'Whether allow new registrations.',
|
||||
default: true,
|
||||
},
|
||||
allowSignupForOauth: {
|
||||
desc: 'Whether allow new registrations via configured oauth.',
|
||||
default: true,
|
||||
},
|
||||
requireEmailDomainVerification: {
|
||||
desc: 'Whether require email domain record verification before accessing restricted resources.',
|
||||
default: false,
|
||||
},
|
||||
requireEmailVerification: {
|
||||
desc: 'Whether require email verification before accessing restricted resources(not implemented).',
|
||||
default: true,
|
||||
},
|
||||
newAccountActionDelay: {
|
||||
desc: 'Minimum account age in seconds before new accounts can invite members, create invite links, or publish documents. Set to 0 to disable.',
|
||||
default: 24 * 60 * 60,
|
||||
shape: z.number().int().min(0),
|
||||
},
|
||||
trustedCloudflareHeaders: {
|
||||
desc: 'Whether request abuse source facts should trust Cloudflare headers from the origin edge.',
|
||||
default: false,
|
||||
shape: z.boolean(),
|
||||
},
|
||||
signInRateLimit: {
|
||||
desc: 'Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds.',
|
||||
default: {
|
||||
ttl: 60_000,
|
||||
ipLimit: 20,
|
||||
emailLimit: 5,
|
||||
defineNativeModuleConfig(
|
||||
'auth',
|
||||
serverNativeModule.appConfigDescriptors('auth'),
|
||||
serverNativeModule.validateAppConfigValue,
|
||||
{
|
||||
trustedCloudflareHeaders: {
|
||||
desc: 'Whether request abuse source facts should trust Cloudflare headers from the origin edge.',
|
||||
default: false,
|
||||
shape: z.boolean(),
|
||||
},
|
||||
shape: z
|
||||
.object({
|
||||
ttl: z.number().int().positive(),
|
||||
ipLimit: z.number().int().positive(),
|
||||
emailLimit: z.number().int().positive(),
|
||||
})
|
||||
.strict(),
|
||||
},
|
||||
passwordRequirements: {
|
||||
desc: 'The password strength requirements when set new password.',
|
||||
default: {
|
||||
min: 8,
|
||||
max: 32,
|
||||
signInRateLimit: {
|
||||
desc: 'Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds.',
|
||||
default: {
|
||||
ttl: 60_000,
|
||||
ipLimit: 20,
|
||||
emailLimit: 5,
|
||||
},
|
||||
shape: z
|
||||
.object({
|
||||
ttl: z.number().int().positive(),
|
||||
ipLimit: z.number().int().positive(),
|
||||
emailLimit: z.number().int().positive(),
|
||||
})
|
||||
.strict(),
|
||||
},
|
||||
shape: z
|
||||
.object({
|
||||
min: z.number().min(1),
|
||||
max: z.number().max(100),
|
||||
})
|
||||
.strict()
|
||||
.refine(data => data.min < data.max, {
|
||||
message: 'Minimum length of password must be less than maximum length',
|
||||
}),
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
min: { type: 'number' },
|
||||
max: { type: 'number' },
|
||||
passwordRequirements: {
|
||||
desc: 'The password strength requirements when set new password.',
|
||||
default: {
|
||||
min: 8,
|
||||
max: 32,
|
||||
},
|
||||
shape: z
|
||||
.object({
|
||||
min: z.number().min(1),
|
||||
max: z.number().max(100),
|
||||
})
|
||||
.strict()
|
||||
.refine(data => data.min < data.max, {
|
||||
message:
|
||||
'Minimum length of password must be less than maximum length',
|
||||
}),
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
min: { type: 'number' },
|
||||
max: { type: 'number' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
'session.ttl': {
|
||||
desc: 'Application auth expiration time in seconds.',
|
||||
default: 60 * 60 * 24 * 15, // 15 days
|
||||
},
|
||||
'session.ttr': {
|
||||
desc: 'Application auth time to refresh in seconds.',
|
||||
default: 60 * 60 * 24 * 7, // 7 days
|
||||
},
|
||||
'token.accessTokenTtl': {
|
||||
desc: 'Access JWT expiration time in seconds.',
|
||||
default: 15 * 60,
|
||||
shape: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60)
|
||||
.max(60 * 60),
|
||||
},
|
||||
'token.refreshIdleTtl': {
|
||||
desc: 'Auth refresh session inactivity expiration in seconds.',
|
||||
default: 60 * 60 * 24 * 30,
|
||||
shape: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60 * 60)
|
||||
.max(60 * 60 * 24 * 365),
|
||||
},
|
||||
'token.refreshAbsoluteTtl': {
|
||||
desc: 'Auth refresh session absolute expiration in seconds.',
|
||||
default: 60 * 60 * 24 * 180,
|
||||
shape: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60 * 60)
|
||||
.max(60 * 60 * 24 * 730),
|
||||
},
|
||||
'token.refreshGracePeriod': {
|
||||
desc: 'One-use refresh rotation concurrency grace period in seconds.',
|
||||
default: 30,
|
||||
shape: z.number().int().min(0).max(60),
|
||||
},
|
||||
'token.refreshRetention': {
|
||||
desc: 'Retention for expired auth refresh generations in seconds.',
|
||||
default: 60 * 60 * 24 * 30,
|
||||
shape: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60 * 60)
|
||||
.max(60 * 60 * 24 * 365),
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
@@ -24,15 +24,18 @@ import {
|
||||
import {
|
||||
BackendRuntimeEmbeddingService,
|
||||
BackendRuntimeHousekeepingJob,
|
||||
BackendRuntimeSearchJob,
|
||||
} from '../job';
|
||||
|
||||
interface Context {
|
||||
module: TestingModule;
|
||||
embeddingService: BackendRuntimeEmbeddingService;
|
||||
job: BackendRuntimeHousekeepingJob;
|
||||
searchJob: BackendRuntimeSearchJob;
|
||||
getSnapshot: Sinon.SinonStub;
|
||||
allowEmbedding: Sinon.SinonStub;
|
||||
runtime: {
|
||||
nodeCryptoPrivateKey: Sinon.SinonStub;
|
||||
cleanupExpiredRuntimeStates: Sinon.SinonStub;
|
||||
cleanupExpiredRuntimeGates: Sinon.SinonStub;
|
||||
cleanupExpiredRollingQuota: Sinon.SinonStub;
|
||||
@@ -40,6 +43,9 @@ interface Context {
|
||||
embeddingHealth: Sinon.SinonStub;
|
||||
syncEmbeddingState: Sinon.SinonStub;
|
||||
executeAuthSessionCommandV1: Sinon.SinonStub;
|
||||
searchEnabled: Sinon.SinonStub;
|
||||
reconcileSearchProjection: Sinon.SinonStub;
|
||||
searchStatus: Sinon.SinonStub;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -50,6 +56,7 @@ test.before(async t => {
|
||||
join(process.cwd(), 'src/__tests__/__fixtures__/test-doc.snapshot.bin')
|
||||
);
|
||||
t.context.runtime = {
|
||||
nodeCryptoPrivateKey: Sinon.stub().returns(''),
|
||||
cleanupExpiredRuntimeStates: Sinon.stub(),
|
||||
cleanupExpiredRuntimeGates: Sinon.stub(),
|
||||
cleanupExpiredRollingQuota: Sinon.stub(),
|
||||
@@ -57,6 +64,9 @@ test.before(async t => {
|
||||
embeddingHealth: Sinon.stub().resolves({ enabled: true }),
|
||||
syncEmbeddingState: Sinon.stub(),
|
||||
executeAuthSessionCommandV1: Sinon.stub().resolves({}),
|
||||
searchEnabled: Sinon.stub().returns(false),
|
||||
reconcileSearchProjection: Sinon.stub().resolves(2),
|
||||
searchStatus: Sinon.stub().resolves({ ready: true, state: 'active' }),
|
||||
};
|
||||
t.context.module = await createTestingModule({
|
||||
imports: [
|
||||
@@ -92,6 +102,7 @@ test.before(async t => {
|
||||
BackendRuntimeEmbeddingService
|
||||
);
|
||||
t.context.job = t.context.module.get(BackendRuntimeHousekeepingJob);
|
||||
t.context.searchJob = t.context.module.get(BackendRuntimeSearchJob);
|
||||
});
|
||||
|
||||
test.beforeEach(t => {
|
||||
@@ -101,6 +112,9 @@ test.beforeEach(t => {
|
||||
t.context.runtime.cleanupUnreferencedArtifacts.reset();
|
||||
t.context.runtime.embeddingHealth.resetHistory();
|
||||
t.context.runtime.syncEmbeddingState.reset();
|
||||
t.context.runtime.searchEnabled.resetHistory();
|
||||
t.context.runtime.reconcileSearchProjection.resetHistory();
|
||||
t.context.runtime.searchStatus.resetHistory();
|
||||
t.context.getSnapshot.resetHistory();
|
||||
t.context.allowEmbedding.resetHistory();
|
||||
});
|
||||
@@ -209,4 +223,10 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t =>
|
||||
t.is(t.context.runtime.cleanupExpiredRuntimeGates.callCount, 1);
|
||||
t.is(t.context.runtime.cleanupExpiredRollingQuota.callCount, 1);
|
||||
t.is(t.context.runtime.cleanupUnreferencedArtifacts.callCount, 1);
|
||||
|
||||
t.is(await t.context.searchJob.reconcileProjection(7), 0);
|
||||
t.false(t.context.runtime.reconcileSearchProjection.called);
|
||||
t.context.runtime.searchEnabled.returns(true);
|
||||
t.is(await t.context.searchJob.reconcileProjection(7), 2);
|
||||
t.true(t.context.runtime.reconcileSearchProjection.calledOnceWithExactly(7));
|
||||
});
|
||||
@@ -1,29 +1,40 @@
|
||||
import { generateKeyPairSync } from 'node:crypto';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import test from 'ava';
|
||||
import Sinon from 'sinon';
|
||||
|
||||
import type { Config } from '../../../base';
|
||||
import type { EventBus } from '../../../base';
|
||||
import { ServerConfigHandle } from '../../../native';
|
||||
import { BackendRuntimeProvider } from '../provider';
|
||||
|
||||
const privateKey = generateKeyPairSync('ec', {
|
||||
namedCurve: 'P-256',
|
||||
}).privateKey.export({ format: 'pem', type: 'pkcs8' }) as string;
|
||||
const storage = {
|
||||
provider: 'assetpack',
|
||||
bucket: 'test',
|
||||
config: { path: '/tmp/affine-backend-runtime-test' },
|
||||
};
|
||||
const config = {
|
||||
crypto: { privateKey },
|
||||
storages: { blob: { storage }, avatar: { storage } },
|
||||
copilot: { storage },
|
||||
} as Config;
|
||||
const directory = mkdtempSync(join(tmpdir(), 'affine-backend-runtime-'));
|
||||
const configPath = join(directory, 'config.json');
|
||||
writeFileSync(
|
||||
configPath,
|
||||
JSON.stringify({
|
||||
deployment: { type: 'selfhosted' },
|
||||
crypto: { privateKey },
|
||||
copilot: { enabled: false },
|
||||
})
|
||||
);
|
||||
const serverConfig = new ServerConfigHandle(configPath);
|
||||
test.after.always(() => rmSync(directory, { recursive: true, force: true }));
|
||||
|
||||
test('backend-runtime provider starts without migrations and exposes explicit migration', async t => {
|
||||
const provider = new BackendRuntimeProvider(config, []);
|
||||
const event = {
|
||||
emitAsync: Sinon.stub().resolves(),
|
||||
};
|
||||
const provider = new BackendRuntimeProvider(
|
||||
serverConfig,
|
||||
event as unknown as EventBus
|
||||
);
|
||||
const runtime = {
|
||||
configureObjectStorage: Sinon.stub(),
|
||||
start: Sinon.stub().resolves(),
|
||||
stop: Sinon.stub().resolves(),
|
||||
runMigrations: Sinon.stub().resolves(),
|
||||
@@ -49,31 +60,36 @@ test('backend-runtime provider starts without migrations and exposes explicit mi
|
||||
await provider.onConfigChanged({ updates: { mailer: {} } });
|
||||
await provider.onConfigChanged({ updates: { copilot: {} } });
|
||||
await provider.onConfigChanged({ updates: { storages: {} } });
|
||||
await provider.onConfigChanged({ updates: { oauth: {} } });
|
||||
const health = await provider.health();
|
||||
t.is(runtime.reloadConfig.callCount, 3);
|
||||
t.true(runtime.reloadConfig.alwaysCalledWithExactly());
|
||||
t.is(event.emitAsync.callCount, 5);
|
||||
t.true(
|
||||
event.emitAsync.calledWith('backendRuntime.configApplied', {
|
||||
updates: { copilot: {} },
|
||||
})
|
||||
);
|
||||
t.true(
|
||||
event.emitAsync.calledWith('backendRuntime.configApplied', {
|
||||
updates: { oauth: {} },
|
||||
})
|
||||
);
|
||||
|
||||
runtime.reloadConfig.rejects(new Error('sensitive native credential'));
|
||||
await provider.onConfigChanged({ updates: { copilot: {} } });
|
||||
await provider.stop();
|
||||
|
||||
t.is(runtime.start.callCount, 2);
|
||||
t.is(runtime.runMigrations.callCount, 1);
|
||||
t.is(runtime.reloadConfig.callCount, 2);
|
||||
t.true(
|
||||
runtime.reloadConfig.alwaysCalledWithExactly(
|
||||
privateKey,
|
||||
JSON.stringify({
|
||||
storages: {
|
||||
'blob.storage': storage,
|
||||
'avatar.storage': storage,
|
||||
},
|
||||
copilot: { storage },
|
||||
}),
|
||||
'{}'
|
||||
)
|
||||
);
|
||||
t.is(runtime.reloadConfig.callCount, 4);
|
||||
t.is(event.emitAsync.callCount, 5);
|
||||
t.true(health.databaseConnected);
|
||||
t.is(runtime.stop.callCount, 1);
|
||||
});
|
||||
|
||||
test('backend-runtime provider measures explicit typed methods', async t => {
|
||||
const provider = new BackendRuntimeProvider(config, []);
|
||||
const provider = new BackendRuntimeProvider(serverConfig);
|
||||
const runtime = {
|
||||
cleanupExpiredRuntimeStates: Sinon.stub().resolves(3),
|
||||
assertCopilotRoute: Sinon.stub().resolves(),
|
||||
@@ -98,7 +114,7 @@ test('backend-runtime provider measures explicit typed methods', async t => {
|
||||
});
|
||||
|
||||
test('backend-runtime provider encodes recursive search contracts at the native boundary', async t => {
|
||||
const provider = new BackendRuntimeProvider(config, []);
|
||||
const provider = new BackendRuntimeProvider(serverConfig);
|
||||
const runtime = {
|
||||
searchAuthorized: Sinon.stub().resolves({
|
||||
ok: true,
|
||||
@@ -198,7 +214,7 @@ test('backend-runtime provider encodes recursive search contracts at the native
|
||||
});
|
||||
|
||||
test('backend-runtime provider aborts a stream handle that resolves after iterator cancellation', async t => {
|
||||
const provider = new BackendRuntimeProvider(config, []);
|
||||
const provider = new BackendRuntimeProvider(serverConfig);
|
||||
const abort = Sinon.stub();
|
||||
let resolveHandle!: (handle: { abort: () => void }) => void;
|
||||
const runtime = {
|
||||
|
||||
@@ -1,26 +1,25 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
|
||||
import { CRYPTO_KEY_SOURCE } from '../../base/helpers/crypto';
|
||||
import {
|
||||
BackendRuntimeEmbeddingService,
|
||||
BackendRuntimeHousekeepingJob,
|
||||
BackendRuntimeSearchJob,
|
||||
} from './job';
|
||||
import {
|
||||
BACKEND_RUNTIME_CONFIG_PATHS,
|
||||
BackendRuntimeProvider,
|
||||
} from './provider';
|
||||
import { BackendRuntimeProvider } from './provider';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [
|
||||
{
|
||||
provide: BACKEND_RUNTIME_CONFIG_PATHS,
|
||||
useValue: undefined,
|
||||
},
|
||||
BackendRuntimeProvider,
|
||||
BackendRuntimeEmbeddingService,
|
||||
{ provide: CRYPTO_KEY_SOURCE, useExisting: BackendRuntimeProvider },
|
||||
],
|
||||
exports: [
|
||||
BackendRuntimeProvider,
|
||||
BackendRuntimeEmbeddingService,
|
||||
CRYPTO_KEY_SOURCE,
|
||||
],
|
||||
exports: [BackendRuntimeProvider, BackendRuntimeEmbeddingService],
|
||||
})
|
||||
export class BackendRuntimeModule {}
|
||||
|
||||
@@ -33,7 +32,6 @@ export class BackendRuntimeWorkerModule {}
|
||||
export { BackendRuntimeEmbeddingService } from './job';
|
||||
export { BackendRuntimeError, backendRuntimeErrorCode } from './operations';
|
||||
export {
|
||||
BACKEND_RUNTIME_CONFIG_PATHS,
|
||||
BackendRuntimeProvider,
|
||||
type BlobManifestEntryV1,
|
||||
type BlobManifestV1,
|
||||
|
||||
@@ -2,7 +2,6 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
|
||||
import {
|
||||
ConfigFactory,
|
||||
CopilotSelectedSourcesFailed,
|
||||
CopilotSelectedSourcesLimitExceeded,
|
||||
CopilotSelectedSourcesProcessing,
|
||||
@@ -173,14 +172,11 @@ export class BackendRuntimeHousekeepingJob {
|
||||
|
||||
@Injectable()
|
||||
export class BackendRuntimeSearchJob {
|
||||
constructor(
|
||||
private readonly rt: BackendRuntimeProvider,
|
||||
private readonly config: ConfigFactory
|
||||
) {}
|
||||
constructor(private readonly rt: BackendRuntimeProvider) {}
|
||||
|
||||
@Cron(CronExpression.EVERY_30_SECONDS)
|
||||
async reconcileProjection(limit = 100) {
|
||||
if (!this.config.config.indexer.enabled) return 0;
|
||||
if (!this.rt.searchEnabled()) return 0;
|
||||
const startedAt = performance.now();
|
||||
try {
|
||||
const reconciled = await this.rt.reconcileSearchProjection(limit);
|
||||
|
||||
@@ -41,6 +41,22 @@ export function backendRuntimeErrorCode(error: unknown) {
|
||||
export class BackendRuntimeCoreOperations {
|
||||
constructor(protected readonly runtime: RuntimeInstance) {}
|
||||
|
||||
stripeEnabled() {
|
||||
return this.runtime.stripeEnabled();
|
||||
}
|
||||
|
||||
paymentEnabled() {
|
||||
return this.runtime.paymentEnabled();
|
||||
}
|
||||
|
||||
copilotEnabled() {
|
||||
return this.runtime.copilotEnabled();
|
||||
}
|
||||
|
||||
nodeCryptoPrivateKey() {
|
||||
return this.runtime.nodeCryptoPrivateKey();
|
||||
}
|
||||
|
||||
executePaymentCommandV1<T = unknown>(input: Record<string, unknown>) {
|
||||
return this.measured('executePaymentCommandV1', runtime =>
|
||||
runtime.executePaymentCommandV1(input)
|
||||
|
||||
Loaded 100 of 159 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user