feat(server): improve copilot module (#15817)

#### PR Dependency Tree


* **PR #15817** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
<!-- coderabbit-ai-generated:v1 -->

## Summary by CodeRabbit

* **New Features**
* Added managed Copilot model routing with route choices and defaults
based on available configuration.
* Expanded model routing support to include OpenRouter and additional
request formats.
* Route options are now available for self-hosted deployments when
configured targets support them.

* **Bug Fixes**
  * Added clearer errors when a selected Copilot target is unavailable.
* Improved configuration migration behavior to preserve custom model
profiles and legacy credentials.
* Cloud deployments now receive a clear configuration error when
attempting to set Copilot provider profiles.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky authored and GitHub committed 2026-11-10 09:42:06 +08:00
1 parent 9c316636e0
commit 63297bca34
43 files changed
+4418 -2275

No files matched your search

+2340 -1040
View File
File diff suppressed because it is too large. Load diff
Generated
+4 -4
View File
@@ -4938,9 +4938,9 @@ dependencies = [
[[package]]
name = "llm_adapter"
version = "0.3.1"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee02e24a06e415066de150ee42c5edec6cf79be0a1680bbbdd17517ede104f9"
checksum = "58f5fa220065e9799a5b93372655414a3af754af39d99436433b9710e874736e"
dependencies = [
"base64 0.22.1",
"jsonschema",
@@ -4956,9 +4956,9 @@ dependencies = [
[[package]]
name = "llm_runtime"
version = "0.3.1"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27a7a79c59311b9e171c370961fd014ff94b560695aa7e44b4ceeca696ad5edd"
checksum = "8c29e217bd14bfadcb2fce5a53a2dab36087ba91857cdf3aa6376116346ae169"
dependencies = [
"jsonschema",
"llm_adapter",
+2 -2
View File
@@ -51,8 +51,8 @@ resolver = "3"
infer = { version = "0.19.0" }
libc = "0.2"
little_exif = "0.6.23"
llm_adapter = { version = "0.3", default-features = false }
llm_runtime = { version = "0.3", default-features = false }
llm_adapter = { version = "0.4", default-features = false }
llm_runtime = { version = "0.4", default-features = false }
lru = "0.16"
matroska = "0.30"
memory-indexer = "0.4.0"
+6 -9
View File
@@ -37,6 +37,7 @@ export declare class BackendRuntime {
authorizePermissionV1(input: any): Promise<any>
executeDomainCommandV1(input: any): Promise<any>
searchStatus(): Promise<any>
getCopilotRouteOptions(name: string): BuiltInRouteOptions | null
executeCopilotStream(input: CopilotExecuteInput, maxSteps: number, callback: ((err: Error | null, arg: string) => void), toolCallback: ((err: Error | null, arg: string) => Promise<string>)): Promise<CopilotStreamHandle>
executeCopilot(input: CopilotExecuteInput): Promise<string>
assertCopilotRoute(input: CopilotRouteCheckInput): Promise<void>
@@ -725,8 +726,6 @@ export interface LlmEmbeddingRequestContract {
export declare function llmGetBuiltInPromptSpec(name: string): BuiltInPromptSpec | null
export declare function llmGetBuiltInRouteOptions(name: string): BuiltInRouteOptions | null
export declare function llmGetByokCatalog(): ByokCatalogOutput
export declare function llmGetContractSchema(name: string): any
@@ -864,7 +863,7 @@ export interface ModelConditionsContract {
}
export interface ModelRegistryMatchRequest {
backendKind: 'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | 'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen'
backendKind: 'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | 'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen' | 'openrouter'
cond: ModelConditionsContract
}
@@ -873,7 +872,7 @@ export interface ModelRegistryMatchResponse {
}
export interface ModelRegistryResolveRequest {
backendKind?: 'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | 'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen'
backendKind?: 'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | 'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen' | 'openrouter'
modelId: string
}
@@ -883,20 +882,18 @@ export interface ModelRegistryResolveResponse {
}
export interface ModelRegistryRouteContract {
protocol?: 'openai_chat' | 'openai_responses' | 'openai_images' | 'anthropic' | 'gemini' | 'fal_image'
requestLayer?: 'anthropic' | 'chat_completions' | 'cloudflare_workers_ai' | 'responses' | 'openai_images' | 'fal' | 'vertex' | 'vertex_anthropic' | 'gemini_api' | 'gemini_vertex'
requestLayers?: Array<string>
}
export interface ModelRegistryVariantContract {
backendKind: 'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | 'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen'
backendKind: 'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | 'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen' | 'openrouter'
canonicalKey: string
rawModelId: string
displayName?: string
aliases: Array<string>
legacyAliases?: Array<string>
capabilities: Array<CapabilityModelCapability>
protocol?: 'openai_chat' | 'openai_responses' | 'openai_images' | 'anthropic' | 'gemini' | 'fal_image'
requestLayer?: 'anthropic' | 'chat_completions' | 'cloudflare_workers_ai' | 'responses' | 'openai_images' | 'fal' | 'vertex' | 'vertex_anthropic' | 'gemini_api' | 'gemini_vertex'
requestLayer?: 'anthropic' | 'chat_completions' | 'cloudflare_workers_ai' | 'responses' | 'openai_images' | 'fal' | 'vertex' | 'vertex_anthropic' | 'gemini_api' | 'gemini_vertex' | 'openai_embeddings' | 'openrouter_responses' | 'openrouter_chat_completions' | 'openrouter_anthropic' | 'perplexity_sonar'
routeOverrides?: Record<string, ModelRegistryRouteContract>
behaviorFlags?: Array<string>
}
@@ -3,8 +3,9 @@
"name": "Transcript audio",
"action": "Transcript audio",
"managedRoute": {
"targets": ["gemini-3.5-flash-lite"],
"premiumTargets": ["gemini-3.8-flash"]
"targets": ["transcript-lite"],
"premiumTargets": ["gemini"],
"operations": ["structured"]
},
"config": {
"requireContent": false,
@@ -17,8 +18,9 @@
"name": "Transcript audio structured",
"action": "Transcript audio structured",
"managedRoute": {
"targets": ["gemini-3.5-flash-lite"],
"premiumTargets": ["gemini-3.8-flash"]
"targets": ["transcript-lite"],
"premiumTargets": ["gemini"],
"operations": ["structured"]
},
"config": {
"requireContent": false,
@@ -39,7 +41,10 @@
{
"name": "Generate a caption",
"action": "Generate a caption",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"config": {
"requireContent": false,
"requireAttachment": true
@@ -54,7 +59,10 @@
{
"name": "Conversation Summary",
"action": "Conversation Summary",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"config": {
"requireContent": false
},
@@ -72,7 +80,10 @@
{
"name": "Summary",
"action": "Summary",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -87,7 +98,10 @@
{
"name": "Summary as title",
"action": "Summary as title",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -102,7 +116,10 @@
{
"name": "Summary the webpage",
"action": "Summary the webpage",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "user",
@@ -113,7 +130,10 @@
{
"name": "Explain this",
"action": "Explain this",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"builtins": ["language"],
"messages": [
{
@@ -129,7 +149,10 @@
{
"name": "Explain this image",
"action": "Explain this image",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"config": {
"requireContent": false,
"requireAttachment": true
@@ -148,7 +171,10 @@
{
"name": "Explain this code",
"action": "Explain this code",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -163,7 +189,10 @@
{
"name": "Translate to",
"action": "Translate",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"params": {
"language": {
"default": "English",
@@ -196,7 +225,10 @@
{
"name": "Summarize the meeting structured",
"action": "Summarize the meeting structured",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["structured"]
},
"messages": [
{
"role": "system",
@@ -211,7 +243,10 @@
{
"name": "Summarize the meeting",
"action": "Summarize the meeting",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -226,7 +261,10 @@
{
"name": "Find action for summary",
"action": "Find action for summary",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -241,7 +279,10 @@
{
"name": "Write an article about this",
"action": "Write an article about this",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -256,7 +297,10 @@
{
"name": "Write a twitter about this",
"action": "Write a twitter about this",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -271,7 +315,10 @@
{
"name": "Write a poem about this",
"action": "Write a poem about this",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -286,7 +333,10 @@
{
"name": "Write a blog post about this",
"action": "Write a blog post about this",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -301,7 +351,10 @@
{
"name": "Write outline",
"action": "Write outline",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -316,7 +369,10 @@
{
"name": "Change tone to",
"action": "Change tone",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"params": {
"tone": {
"default": "professional",
@@ -337,7 +393,10 @@
{
"name": "Brainstorm ideas about this",
"action": "Brainstorm ideas about this",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -352,7 +411,10 @@
{
"name": "Brainstorm mindmap",
"action": "Brainstorm mindmap",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -367,7 +429,10 @@
{
"name": "Expand mind map",
"action": "Expand mind map",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -386,7 +451,10 @@
{
"name": "Improve writing for it",
"action": "Improve writing for it",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -401,7 +469,10 @@
{
"name": "Improve grammar for it",
"action": "Improve grammar for it",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -416,7 +487,10 @@
{
"name": "Fix spelling for it",
"action": "Fix spelling for it",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -431,7 +505,10 @@
{
"name": "Find action items from it",
"action": "Find action items from it",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -446,7 +523,10 @@
{
"name": "Check code error",
"action": "Check code error",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -461,7 +541,10 @@
{
"name": "Create a presentation",
"action": "Create a presentation",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -476,7 +559,10 @@
{
"name": "Create headings",
"action": "Create headings",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -491,7 +577,10 @@
{
"name": "Make it real",
"action": "Make it real",
"managedRoute": { "targets": ["claude-sonnet-4-6"] },
"managedRoute": {
"targets": ["claude"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -506,7 +595,10 @@
{
"name": "Make it real with text",
"action": "Make it real with text",
"managedRoute": { "targets": ["claude-sonnet-4-6"] },
"managedRoute": {
"targets": ["claude"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -521,7 +613,10 @@
{
"name": "Make it longer",
"action": "Make it longer",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -536,7 +631,10 @@
{
"name": "Make it shorter",
"action": "Make it shorter",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -551,7 +649,10 @@
{
"name": "Continue writing",
"action": "Continue writing",
"managedRoute": { "targets": ["gemini-3.8-flash"] },
"managedRoute": {
"targets": ["gemini"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -566,7 +667,10 @@
{
"name": "Section Edit",
"action": "Section Edit",
"managedRoute": { "targets": ["claude-sonnet-4-6"] },
"managedRoute": {
"targets": ["claude"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -581,7 +685,10 @@
{
"name": "Generate image",
"action": "image",
"managedRoute": { "targets": ["gpt-image-2.6-flare"] },
"managedRoute": {
"targets": ["image"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -592,7 +699,10 @@
{
"name": "Convert to Clay style",
"action": "Convert to Clay style",
"managedRoute": { "targets": ["gpt-image-2.6-flare"] },
"managedRoute": {
"targets": ["image"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -603,7 +713,10 @@
{
"name": "Convert to Sketch style",
"action": "Convert to Sketch style",
"managedRoute": { "targets": ["gpt-image-2.6-flare"] },
"managedRoute": {
"targets": ["image"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -614,7 +727,10 @@
{
"name": "Convert to Anime style",
"action": "Convert to Anime style",
"managedRoute": { "targets": ["gpt-image-2.6-flare"] },
"managedRoute": {
"targets": ["image"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -625,7 +741,10 @@
{
"name": "Convert to Pixel style",
"action": "Convert to Pixel style",
"managedRoute": { "targets": ["gpt-image-2.6-flare"] },
"managedRoute": {
"targets": ["image"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -636,7 +755,10 @@
{
"name": "Convert to sticker",
"action": "Convert to sticker",
"managedRoute": { "targets": ["gpt-image-2.6-flare"] },
"managedRoute": {
"targets": ["image"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -647,7 +769,10 @@
{
"name": "Upscale image",
"action": "Upscale image",
"managedRoute": { "targets": ["gpt-image-2.6-flare"] },
"managedRoute": {
"targets": ["image"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -658,7 +783,10 @@
{
"name": "Remove background",
"action": "Remove background",
"managedRoute": { "targets": ["gpt-image-2.6-flare"] },
"managedRoute": {
"targets": ["image"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -669,7 +797,10 @@
{
"name": "debug:action:fal-teed",
"action": "fal-teed",
"managedRoute": { "targets": ["workflowutils/teed"] },
"managedRoute": {
"targets": ["edge-detection"],
"operations": ["image"]
},
"messages": [
{
"role": "user",
@@ -679,7 +810,10 @@
},
{
"name": "Code Artifact",
"managedRoute": { "targets": ["claude-sonnet-4-6"] },
"managedRoute": {
"targets": ["claude"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -694,34 +828,35 @@
{
"name": "Chat With AFFiNE AI",
"managedRoute": {
"targets": ["gpt-6-luna"],
"premiumTargets": ["gpt-6-luna"],
"targets": ["luna"],
"premiumTargets": ["luna"],
"selectableTargets": [
{
"id": "luna",
"modelId": "gpt-6-luna",
"target": "luna",
"displayName": "GPT-6 Luna",
"minimumTier": "standard"
},
{
"id": "terra",
"modelId": "gpt-6.1-terra",
"target": "terra",
"displayName": "GPT-6.1 Terra",
"minimumTier": "premium"
},
{
"id": "gemini",
"modelId": "gemini-3.8-flash",
"target": "gemini",
"displayName": "Gemini 3.8 Flash",
"minimumTier": "premium"
},
{
"id": "claude",
"modelId": "claude-sonnet-4-6",
"target": "claude",
"displayName": "Claude Sonnet 4.6",
"minimumTier": "premium"
}
]
],
"operations": ["chat"]
},
"config": {
"tools": [
@@ -764,7 +899,10 @@
{
"name": "mindmap.generate",
"action": "mindmap.generate",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["structured"]
},
"config": {
"frequencyPenalty": 0.5,
"presencePenalty": 0.5,
@@ -789,7 +927,10 @@
{
"name": "slides.outline",
"action": "slides.outline",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["structured"]
},
"messages": [
{
"role": "system",
@@ -808,7 +949,10 @@
{
"name": "image.filter.sketch",
"action": "image.filter.sketch",
"managedRoute": { "targets": ["lora/image-to-image"] },
"managedRoute": {
"targets": ["image-filter"],
"operations": ["image"]
},
"config": {
"modelName": "stabilityai/stable-diffusion-xl-base-1.0",
"loras": [
@@ -828,7 +972,10 @@
{
"name": "image.filter.clay",
"action": "image.filter.clay",
"managedRoute": { "targets": ["lora/image-to-image"] },
"managedRoute": {
"targets": ["image-filter"],
"operations": ["image"]
},
"config": {
"modelName": "stabilityai/stable-diffusion-xl-base-1.0",
"loras": [
@@ -848,7 +995,10 @@
{
"name": "image.filter.anime",
"action": "image.filter.anime",
"managedRoute": { "targets": ["lora/image-to-image"] },
"managedRoute": {
"targets": ["image-filter"],
"operations": ["image"]
},
"config": {
"modelName": "stabilityai/stable-diffusion-xl-base-1.0",
"loras": [
@@ -868,7 +1018,10 @@
{
"name": "image.filter.pixel",
"action": "image.filter.pixel",
"managedRoute": { "targets": ["lora/image-to-image"] },
"managedRoute": {
"targets": ["image-filter"],
"operations": ["image"]
},
"config": {
"modelName": "stabilityai/stable-diffusion-xl-base-1.0",
"loras": [
@@ -893,7 +1046,10 @@
{
"name": "workflow:presentation:step1",
"action": "workflow:presentation:step1",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"config": {
"temperature": 0.7
},
@@ -911,7 +1067,10 @@
{
"name": "workflow:presentation:step2",
"action": "workflow:presentation:step2",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -930,7 +1089,10 @@
{
"name": "workflow:presentation:step4",
"action": "workflow:presentation:step4",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"messages": [
{
"role": "system",
@@ -954,7 +1116,10 @@
{
"name": "workflow:brainstorm:step1",
"action": "workflow:brainstorm:step1",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"config": {
"temperature": 0.7
},
@@ -972,7 +1137,10 @@
{
"name": "workflow:brainstorm:step2",
"action": "workflow:brainstorm:step2",
"managedRoute": { "targets": ["gpt-6-luna"] },
"managedRoute": {
"targets": ["luna"],
"operations": ["chat"]
},
"config": {
"frequencyPenalty": 0.5,
"presencePenalty": 0.5,
@@ -0,0 +1,47 @@
{
"transcript-lite": {
"apiShapes": ["gemini_api", "gemini_vertex"],
"modelId": "gemini-3.5-flash-lite"
},
"gemini": {
"apiShapes": ["gemini_api", "gemini_vertex"],
"modelId": "gemini-3.8-flash"
},
"luna": {
"apiShapes": ["responses", "chat_completions"],
"modelId": "gpt-6-luna"
},
"claude": {
"apiShapes": [
"vertex_anthropic",
"anthropic",
"responses",
"chat_completions"
],
"modelId": "claude-sonnet-4.6"
},
"image": {
"apiShapes": ["openai_images"],
"modelId": "gpt-image-2.6-flare"
},
"edge-detection": {
"apiShapes": ["fal"],
"modelId": "workflowutils/teed"
},
"terra": {
"apiShapes": ["responses", "chat_completions"],
"modelId": "gpt-6.1-terra"
},
"image-filter": {
"apiShapes": ["fal"],
"modelId": "lora/image-to-image"
},
"embedding": {
"modelId": "gemini-embedding-001",
"apiShapes": ["gemini_api", "gemini_vertex"]
},
"rerank": {
"modelId": "gpt-4o-mini",
"apiShapes": ["chat_completions"]
}
}
@@ -231,7 +231,8 @@ pub struct CapabilityMatchResponse {
pub struct ModelRegistryResolveRequest {
#[napi(
ts_type = "'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | \
'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen'"
'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen' | \
'openrouter'"
)]
#[serde(skip_serializing_if = "Option::is_none")]
pub backend_kind: Option<String>,
@@ -245,7 +246,8 @@ pub struct ModelRegistryResolveRequest {
pub struct ModelRegistryMatchRequest {
#[napi(
ts_type = "'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | \
'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen'"
'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen' | \
'openrouter'"
)]
pub backend_kind: String,
pub cond: ModelConditionsContract,
@@ -258,7 +260,8 @@ pub struct ModelRegistryMatchRequest {
pub struct ModelRegistryVariantContract {
#[napi(
ts_type = "'openai_chat' | 'openai_responses' | 'anthropic' | 'cloudflare_workers_ai' | 'gemini_api' | \
'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen'"
'gemini_vertex' | 'fal' | 'anthropic_vertex' | 'deepseek' | 'kimi' | 'opencode_go' | 'opencode_zen' | \
'openrouter'"
)]
pub backend_kind: String,
pub canonical_key: String,
@@ -269,12 +272,10 @@ pub struct ModelRegistryVariantContract {
#[serde(skip_serializing_if = "Option::is_none")]
pub legacy_aliases: Option<Vec<String>>,
pub capabilities: Vec<CapabilityModelCapability>,
#[napi(ts_type = "'openai_chat' | 'openai_responses' | 'openai_images' | 'anthropic' | 'gemini' | 'fal_image'")]
#[serde(skip_serializing_if = "Option::is_none")]
pub protocol: Option<String>,
#[napi(
ts_type = "'anthropic' | 'chat_completions' | 'cloudflare_workers_ai' | 'responses' | 'openai_images' | 'fal' | \
'vertex' | 'vertex_anthropic' | 'gemini_api' | 'gemini_vertex'"
'vertex' | 'vertex_anthropic' | 'gemini_api' | 'gemini_vertex' | 'openai_embeddings' | \
'openrouter_responses' | 'openrouter_chat_completions' | 'openrouter_anthropic' | 'perplexity_sonar'"
)]
#[serde(skip_serializing_if = "Option::is_none")]
pub request_layer: Option<String>,
@@ -289,15 +290,8 @@ pub struct ModelRegistryVariantContract {
#[serde(rename_all = "camelCase")]
#[serde(deny_unknown_fields)]
pub struct ModelRegistryRouteContract {
#[napi(ts_type = "'openai_chat' | 'openai_responses' | 'openai_images' | 'anthropic' | 'gemini' | 'fal_image'")]
#[serde(skip_serializing_if = "Option::is_none")]
pub protocol: Option<String>,
#[napi(
ts_type = "'anthropic' | 'chat_completions' | 'cloudflare_workers_ai' | 'responses' | 'openai_images' | 'fal' | \
'vertex' | 'vertex_anthropic' | 'gemini_api' | 'gemini_vertex'"
)]
#[serde(skip_serializing_if = "Option::is_none")]
pub request_layer: Option<String>,
pub request_layers: Option<Vec<String>>,
}
#[napi(object)]
@@ -79,6 +79,25 @@ mod tests {
assert_eq!(response.variant.unwrap().raw_model_id, model_id);
}
let response = llm_resolve_model_registry_variant(ModelRegistryResolveRequest {
backend_kind: Some("openrouter".to_string()),
model_id: "gpt-6-luna".to_string(),
})
.unwrap();
let variant = response.variant.unwrap();
assert_eq!(variant.backend_kind, "openrouter");
assert_eq!(variant.raw_model_id, "openai/gpt-6-luna");
assert_eq!(
variant.route_overrides.unwrap()["text"].request_layers.as_deref(),
Some(
&[
"openrouter_responses".to_string(),
"openrouter_chat_completions".to_string(),
"openrouter_anthropic".to_string()
][..]
)
);
let response = llm_resolve_model_registry_variant(ModelRegistryResolveRequest {
backend_kind: Some("gemini_api".to_string()),
model_id: "gemini-3.7-flash".to_string(),
@@ -169,7 +188,6 @@ mod tests {
let variant = response.variant.unwrap();
assert_eq!(variant.raw_model_id, "kimi-k2.7-code-highspeed");
assert_eq!(variant.protocol.as_deref(), Some("openai_chat"));
assert_eq!(variant.request_layer.as_deref(), Some("chat_completions"));
assert_eq!(
variant.behavior_flags.as_deref(),
@@ -221,7 +239,6 @@ mod tests {
let variant = response.variant.unwrap();
assert_eq!(variant.raw_model_id, "gemini-embedding-2");
assert_eq!(variant.protocol.as_deref(), Some("gemini"));
assert_eq!(variant.request_layer.as_deref(), Some("gemini_api"));
assert_eq!(variant.display_name.as_deref(), Some("Gemini Embedding 2"));
}
@@ -263,7 +280,6 @@ mod tests {
.unwrap()
.variant
.unwrap();
assert_eq!(openai.protocol.as_deref(), Some("openai_images"));
assert_eq!(openai.request_layer.as_deref(), Some("openai_images"));
let fal = llm_match_model_registry(ModelRegistryMatchRequest {
@@ -280,7 +296,6 @@ mod tests {
.unwrap()
.variant
.unwrap();
assert_eq!(fal.protocol.as_deref(), Some("fal_image"));
assert_eq!(fal.request_layer.as_deref(), Some("fal"));
let gemini = llm_match_model_registry(ModelRegistryMatchRequest {
@@ -297,7 +312,6 @@ mod tests {
.unwrap()
.variant
.unwrap();
assert_eq!(gemini.protocol.as_deref(), Some("gemini"));
assert_eq!(gemini.request_layer.as_deref(), Some("gemini_api"));
let generic_gemini_image = llm_match_model_registry(ModelRegistryMatchRequest {
+4 -1
View File
@@ -39,7 +39,10 @@ pub use contract_schema::{llm_get_contract_schema, llm_validate_contract};
pub(crate) use ffi::{
LlmDispatchPayload, LlmMiddlewarePayload, LlmRerankDispatchPayload, LlmStructuredDispatchPayload,
};
pub use prompt_catalog::llm_get_built_in_route_options;
pub(crate) use prompt_catalog::{
BuiltInManagedTargetTier, BuiltInRouteOptions, built_in_managed_operations, built_in_managed_target,
llm_get_built_in_route_options,
};
pub use route::{
CopilotAccessProjection, CopilotExecuteInput, CopilotManagedTier, CopilotRouteCheckInput, CopilotTargetOverrideInput,
};
+87 -151
View File
@@ -8,6 +8,11 @@ use napi_derive::napi;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
mod managed;
use managed::{BuiltInManagedRouteSpec, resolve_managed_models, valid_managed_route};
pub(crate) use managed::{BuiltInManagedTargetSpec, ManagedTarget, llm_get_built_in_route_options, managed_model};
pub use managed::{BuiltInManagedTargetTier, BuiltInRouteOptions};
static PROMPT_PARTIALS_SOURCE: &str = include_str!("assets/partials/common.json");
static PROMPT_SPECS_SOURCE: &str = include_str!("assets/prompts/built-in.json");
@@ -78,52 +83,6 @@ struct PromptCatalogSpec {
messages: Vec<PromptSpecMessage>,
}
#[derive(Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct BuiltInManagedRouteSpec {
targets: Vec<String>,
#[serde(default)]
premium_targets: Option<Vec<String>>,
#[serde(default)]
selectable_targets: Vec<BuiltInManagedTargetSpec>,
}
#[derive(Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct BuiltInManagedTargetSpec {
id: String,
model_id: String,
display_name: String,
minimum_tier: BuiltInManagedTargetTier,
}
#[napi(string_enum)]
#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum BuiltInManagedTargetTier {
Standard,
Premium,
}
#[napi(object)]
#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BuiltInManagedTarget {
pub id: String,
pub display_name: String,
pub minimum_tier: BuiltInManagedTargetTier,
}
#[napi(object)]
#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BuiltInRouteOptions {
pub route_id: String,
pub standard_default_target_id: Option<String>,
pub premium_default_target_id: Option<String>,
pub choices: Vec<BuiltInManagedTarget>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct BuiltInPromptMessage {
@@ -139,23 +98,17 @@ pub(crate) struct BuiltInPrompt {
pub(crate) name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) action: Option<String>,
pub(crate) managed_targets: Vec<String>,
pub(crate) managed_premium_targets: Option<Vec<String>>,
#[serde(skip)]
pub(crate) managed_selectable_targets: Vec<BuiltInManagedTargetDefinition>,
pub(crate) managed_operations: Vec<llm_adapter::target::BackendOperation>,
pub(crate) managed_targets: Vec<ManagedTarget>,
pub(crate) managed_premium_targets: Option<Vec<ManagedTarget>>,
#[serde(skip)]
pub(crate) managed_selectable_targets: Vec<BuiltInManagedTargetSpec<ManagedTarget>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) config: Option<Value>,
pub(crate) messages: Vec<BuiltInPromptMessage>,
}
#[derive(Clone)]
pub(crate) struct BuiltInManagedTargetDefinition {
pub(crate) id: String,
pub(crate) model_id: String,
pub(crate) display_name: String,
pub(crate) minimum_tier: BuiltInManagedTargetTier,
}
struct PromptCatalog {
specs: Vec<BuiltInPromptSpec>,
prompts: Vec<BuiltInPrompt>,
@@ -181,7 +134,31 @@ pub(crate) fn built_in_prompt(name: &str) -> Option<&'static BuiltInPrompt> {
.and_then(|index| BUILTIN_PROMPT_CATALOG.prompts.get(*index))
}
pub(crate) fn built_in_managed_targets(name: &str, premium: bool) -> Option<&'static [String]> {
pub(crate) fn built_in_managed_operations(name: &str) -> Option<&'static [llm_adapter::target::BackendOperation]> {
built_in_prompt(name).map(|prompt| prompt.managed_operations.as_slice())
}
pub(crate) fn built_in_managed_catalog() -> Vec<(ManagedTarget, llm_adapter::target::BackendOperation)> {
BUILTIN_PROMPT_CATALOG
.prompts
.iter()
.flat_map(|prompt| {
prompt
.managed_targets
.iter()
.chain(prompt.managed_premium_targets.iter().flatten())
.chain(prompt.managed_selectable_targets.iter().map(|choice| &choice.target))
.flat_map(|target| {
prompt
.managed_operations
.iter()
.map(move |operation| (target.clone(), *operation))
})
})
.collect()
}
pub(crate) fn built_in_managed_targets(name: &str, premium: bool) -> Option<&'static [ManagedTarget]> {
built_in_prompt(name).and_then(|prompt| {
let targets = if premium {
prompt
@@ -199,50 +176,13 @@ pub(crate) fn built_in_managed_target(
name: &str,
target_id: &str,
premium: bool,
) -> Option<&'static BuiltInManagedTargetDefinition> {
) -> Option<&'static BuiltInManagedTargetSpec<ManagedTarget>> {
built_in_prompt(name)?
.managed_selectable_targets
.iter()
.find(|target| target.id == target_id && (premium || target.minimum_tier == BuiltInManagedTargetTier::Standard))
}
#[napi(catch_unwind)]
pub fn llm_get_built_in_route_options(name: String) -> Option<BuiltInRouteOptions> {
let prompt = built_in_prompt(&name)?;
if prompt.managed_selectable_targets.is_empty() {
return None;
}
let target_id_for_model = |model: Option<&String>| {
model.and_then(|model| {
prompt
.managed_selectable_targets
.iter()
.find(|target| &target.model_id == model)
.map(|target| target.id.clone())
})
};
Some(BuiltInRouteOptions {
route_id: prompt.name.clone(),
standard_default_target_id: target_id_for_model(prompt.managed_targets.first()),
premium_default_target_id: target_id_for_model(
prompt
.managed_premium_targets
.as_ref()
.and_then(|targets| targets.first())
.or_else(|| prompt.managed_targets.first()),
),
choices: prompt
.managed_selectable_targets
.iter()
.map(|target| BuiltInManagedTarget {
id: target.id.clone(),
display_name: target.display_name.clone(),
minimum_tier: target.minimum_tier,
})
.collect(),
})
}
impl PromptCatalog {
fn load() -> Result<Self, String> {
let partials: BTreeMap<String, String> =
@@ -338,15 +278,23 @@ fn compile_prompt_spec(spec: &PromptCatalogSpec, partials: &BTreeMap<String, Str
Ok(BuiltInPrompt {
name: spec.name.clone(),
action: spec.action.clone(),
managed_operations: spec
.managed_route
.as_ref()
.map(|route| route.operations.iter().copied().map(Into::into).collect())
.unwrap_or_default(),
managed_targets: spec
.managed_route
.as_ref()
.map(|route| route.targets.clone())
.map(|route| resolve_managed_models(&route.targets))
.transpose()?
.unwrap_or_default(),
managed_premium_targets: spec
.managed_route
.as_ref()
.and_then(|route| route.premium_targets.clone()),
.and_then(|route| route.premium_targets.as_deref())
.map(resolve_managed_models)
.transpose()?,
managed_selectable_targets: spec
.managed_route
.as_ref()
@@ -354,55 +302,25 @@ fn compile_prompt_spec(spec: &PromptCatalogSpec, partials: &BTreeMap<String, Str
route
.selectable_targets
.iter()
.map(|target| BuiltInManagedTargetDefinition {
id: target.id.clone(),
model_id: target.model_id.clone(),
display_name: target.display_name.clone(),
minimum_tier: target.minimum_tier,
.map(|choice| {
let target =
managed_model(&choice.target).ok_or_else(|| format!("Unknown managed model route: {}", choice.target))?;
Ok(BuiltInManagedTargetSpec {
id: choice.id.clone(),
target: target.clone(),
display_name: choice.display_name.clone(),
minimum_tier: choice.minimum_tier,
})
})
.collect()
.collect::<Result<Vec<_>, String>>()
})
.transpose()?
.unwrap_or_default(),
config: spec.config.clone().filter(|value| !value.is_null()),
messages,
})
}
fn valid_managed_route(route: &BuiltInManagedRouteSpec) -> bool {
if route.targets.is_empty()
|| route.targets.iter().any(|target| target.trim().is_empty())
|| route
.premium_targets
.as_ref()
.is_some_and(|targets| targets.is_empty() || targets.iter().any(|target| target.trim().is_empty()))
{
return false;
}
let ids = route
.selectable_targets
.iter()
.map(|target| target.id.as_str())
.collect::<BTreeSet<_>>();
let models = route
.selectable_targets
.iter()
.map(|target| target.model_id.as_str())
.collect::<BTreeSet<_>>();
if route.selectable_targets.iter().any(|target| {
target.id.trim().is_empty() || target.model_id.trim().is_empty() || target.display_name.trim().is_empty()
}) || ids.len() != route.selectable_targets.len()
|| models.len() != route.selectable_targets.len()
{
return false;
}
route.selectable_targets.is_empty()
|| route
.targets
.iter()
.chain(route.premium_targets.iter().flatten())
.all(|model| models.contains(model.as_str()))
}
fn normalize_prompt_param(spec: &PromptParamSpec) -> Value {
match spec.enum_values.as_ref() {
Some(values) if !values.is_empty() => {
@@ -575,12 +493,21 @@ mod tests {
assert!(chat_tools.iter().any(|tool| tool == "artifactSearch"));
assert!(!chat_tools.iter().any(|tool| tool == "contextSearch"));
assert!(!chat_tools.iter().any(|tool| tool == "blobRead"));
assert_eq!(chat.managed_targets, ["gpt-6-luna"]);
assert_eq!(
chat
.managed_premium_targets
.as_deref()
.map(|targets| targets.iter().map(String::as_str).collect::<Vec<_>>()),
chat.managed_targets,
[ManagedTarget {
api_shapes: vec![
llm_adapter::backend::ApiShape::Responses,
llm_adapter::backend::ApiShape::ChatCompletions
],
model_id: "gpt-6-luna".into()
}]
);
assert_eq!(
chat.managed_premium_targets.as_deref().map(|targets| targets
.iter()
.map(|target| target.model_id.as_str())
.collect::<Vec<_>>()),
Some(vec!["gpt-6-luna"])
);
let options = llm_get_built_in_route_options(chat.name.clone()).expect("chat route options");
@@ -588,21 +515,30 @@ mod tests {
assert_eq!(options.premium_default_target_id.as_deref(), Some("luna"));
assert_eq!(options.choices.len(), 4);
assert_eq!(
built_in_managed_target(&chat.name, "terra", false).map(|target| target.model_id.as_str()),
built_in_managed_target(&chat.name, "terra", false).map(|target| target.target.model_id.as_str()),
None
);
assert_eq!(
built_in_managed_target(&chat.name, "terra", true).map(|target| target.model_id.as_str()),
built_in_managed_target(&chat.name, "terra", true).map(|target| target.target.model_id.as_str()),
Some("gpt-6.1-terra")
);
let transcript = built_in_prompt("Transcript audio structured").expect("transcript prompt");
assert_eq!(transcript.managed_targets, ["gemini-3.5-flash-lite"]);
assert_eq!(
transcript
.managed_premium_targets
.as_deref()
.map(|targets| targets.iter().map(String::as_str).collect::<Vec<_>>()),
transcript.managed_targets,
[ManagedTarget {
api_shapes: vec![
llm_adapter::backend::ApiShape::GeminiApi,
llm_adapter::backend::ApiShape::GeminiVertex
],
model_id: "gemini-3.5-flash-lite".into()
}]
);
assert_eq!(
transcript.managed_premium_targets.as_deref().map(|targets| targets
.iter()
.map(|target| target.model_id.as_str())
.collect::<Vec<_>>()),
Some(vec!["gemini-3.8-flash"])
);
}
@@ -0,0 +1,148 @@
use super::{BTreeMap, BTreeSet, Deserialize, LazyLock, Serialize, built_in_prompt, napi};
static MANAGED_MODELS: LazyLock<BTreeMap<String, ManagedTarget>> = LazyLock::new(|| {
serde_json::from_str(include_str!("../assets/prompts/managed-models.json"))
.expect("built-in managed model routes must be valid JSON")
});
#[derive(Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(super) struct BuiltInManagedRouteSpec {
pub(super) targets: Vec<String>,
pub(super) operations: Vec<crate::llm::route::RouteOperation>,
#[serde(default)]
pub(super) premium_targets: Option<Vec<String>>,
#[serde(default)]
pub(super) selectable_targets: Vec<BuiltInManagedTargetSpec<String>>,
}
#[derive(Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct BuiltInManagedTargetSpec<T> {
pub(crate) id: String,
pub(crate) target: T,
pub(crate) display_name: String,
pub(crate) minimum_tier: BuiltInManagedTargetTier,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct ManagedTarget {
pub(crate) api_shapes: Vec<llm_adapter::backend::ApiShape>,
pub(crate) model_id: String,
}
#[napi(string_enum)]
#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum BuiltInManagedTargetTier {
Standard,
Premium,
}
#[napi(object)]
#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BuiltInManagedTarget {
pub id: String,
pub display_name: String,
pub minimum_tier: BuiltInManagedTargetTier,
}
#[napi(object)]
#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BuiltInRouteOptions {
pub route_id: String,
pub standard_default_target_id: Option<String>,
pub premium_default_target_id: Option<String>,
pub choices: Vec<BuiltInManagedTarget>,
}
pub(crate) fn managed_model(reference: &str) -> Option<&'static ManagedTarget> {
MANAGED_MODELS.get(reference)
}
pub(crate) fn llm_get_built_in_route_options(name: String) -> Option<BuiltInRouteOptions> {
let prompt = built_in_prompt(&name)?;
if prompt.managed_selectable_targets.is_empty() {
return None;
}
let target_id_for_model = |model: Option<&ManagedTarget>| {
model.and_then(|model| {
prompt
.managed_selectable_targets
.iter()
.find(|target| &target.target == model)
.map(|target| target.id.clone())
})
};
Some(BuiltInRouteOptions {
route_id: prompt.name.clone(),
standard_default_target_id: target_id_for_model(prompt.managed_targets.first()),
premium_default_target_id: target_id_for_model(
prompt
.managed_premium_targets
.as_ref()
.and_then(|targets| targets.first())
.or_else(|| prompt.managed_targets.first()),
),
choices: prompt
.managed_selectable_targets
.iter()
.map(|target| BuiltInManagedTarget {
id: target.id.clone(),
display_name: target.display_name.clone(),
minimum_tier: target.minimum_tier,
})
.collect(),
})
}
pub(super) fn resolve_managed_models(references: &[String]) -> Result<Vec<ManagedTarget>, String> {
references
.iter()
.map(|reference| {
managed_model(reference)
.cloned()
.ok_or_else(|| format!("Unknown managed model route: {reference}"))
})
.collect()
}
pub(super) fn valid_managed_route(route: &BuiltInManagedRouteSpec) -> bool {
if route.targets.len() != 1
|| route.operations.is_empty()
|| route
.targets
.iter()
.chain(route.premium_targets.iter().flatten())
.any(|reference| managed_model(reference).is_none())
|| route.premium_targets.as_ref().is_some_and(|targets| targets.len() != 1)
{
return false;
}
let ids = route
.selectable_targets
.iter()
.map(|choice| &choice.id)
.collect::<BTreeSet<_>>();
let models = route
.selectable_targets
.iter()
.map(|choice| &choice.target)
.collect::<BTreeSet<_>>();
if ids.len() != route.selectable_targets.len()
|| models.len() != route.selectable_targets.len()
|| route.selectable_targets.iter().any(|choice| {
choice.id.trim().is_empty() || choice.display_name.trim().is_empty() || managed_model(&choice.target).is_none()
})
{
return false;
}
route.selectable_targets.is_empty()
|| route
.targets
.iter()
.chain(route.premium_targets.iter().flatten())
.all(|target| models.contains(target))
}
@@ -2,12 +2,14 @@ use llm_adapter::capability::{
AttachmentKind, AttachmentSource, ModelFeature, ModelInput, ModelOutput, ModelRequirements,
};
pub(crate) use crate::llm::prompt_catalog::ManagedTarget;
use crate::llm::{
prompt_catalog::{built_in_managed_target, built_in_managed_targets},
prompt_catalog::{built_in_managed_catalog, built_in_managed_target, built_in_managed_targets, managed_model},
route::CopilotManagedTier,
};
#[derive(Clone, Copy, PartialEq, Eq)]
#[derive(Clone, Copy, PartialEq, Eq, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum RouteOperation {
Chat,
Structured,
@@ -17,6 +19,18 @@ pub(crate) enum RouteOperation {
Transcription,
}
impl From<RouteOperation> for llm_adapter::target::BackendOperation {
fn from(operation: RouteOperation) -> Self {
match operation {
RouteOperation::Chat => Self::Chat,
RouteOperation::Structured | RouteOperation::Transcription => Self::Structured,
RouteOperation::Embedding => Self::Embedding,
RouteOperation::Rerank => Self::Rerank,
RouteOperation::Image => Self::Image,
}
}
}
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct CatalogSlot {
pub(crate) id: &'static str,
@@ -45,28 +59,49 @@ pub(crate) fn managed_targets(
slot: &CatalogSlot,
built_in_route_id: Option<&str>,
managed_tier: CopilotManagedTier,
) -> Option<Vec<String>> {
) -> Option<Vec<ManagedTarget>> {
if let Some(route_id) = built_in_route_id {
return built_in_managed_targets(route_id, managed_tier == CopilotManagedTier::Premium).map(<[String]>::to_vec);
return built_in_managed_targets(route_id, managed_tier == CopilotManagedTier::Premium)
.map(<[ManagedTarget]>::to_vec);
}
match slot.id {
"index.embedding" => Some(vec!["gemini-embedding-001".to_string()]),
"search.rerank" => Some(vec!["gpt-4o-mini".to_string()]),
"index.embedding" => Some(vec![managed_model("embedding").unwrap().clone()]),
"search.rerank" => Some(vec![managed_model("rerank").unwrap().clone()]),
_ => None,
}
}
pub(crate) fn managed_catalog() -> Vec<(ManagedTarget, llm_adapter::target::BackendOperation)> {
let mut targets = built_in_managed_catalog();
for id in ["index.embedding", "search.rerank"] {
let system_slot = slot(id).unwrap();
targets.extend(
managed_targets(&system_slot, None, CopilotManagedTier::Standard)
.unwrap()
.into_iter()
.map(|target| (target, system_slot.operation.into())),
);
}
let mut unique = Vec::new();
for target in targets {
if !unique.contains(&target) {
unique.push(target);
}
}
unique
}
pub(crate) fn managed_selected_target(
built_in_route_id: Option<&str>,
target_id: &str,
managed_tier: CopilotManagedTier,
) -> Option<String> {
) -> Option<ManagedTarget> {
built_in_managed_target(
built_in_route_id?,
target_id,
managed_tier == CopilotManagedTier::Premium,
)
.map(|target| target.model_id.clone())
.map(|target| target.target.clone())
}
pub(crate) fn slot(id: &str) -> Option<CatalogSlot> {
@@ -245,7 +280,13 @@ mod tests {
let prompt_slot = slot("prompt.text").unwrap();
assert_eq!(
managed_targets(&prompt_slot, Some("Summary as title"), CopilotManagedTier::Standard).unwrap(),
["gpt-6-luna"]
[ManagedTarget {
api_shapes: vec![
llm_adapter::backend::ApiShape::Responses,
llm_adapter::backend::ApiShape::ChatCompletions
],
model_id: "gpt-6-luna".into()
}]
);
assert!(
managed_targets(
@@ -262,15 +303,55 @@ mod tests {
CopilotManagedTier::Premium
)
.unwrap(),
["gemini-3.8-flash"]
[ManagedTarget {
api_shapes: vec![
llm_adapter::backend::ApiShape::GeminiApi,
llm_adapter::backend::ApiShape::GeminiVertex
],
model_id: "gemini-3.8-flash".into()
}]
);
assert_eq!(
managed_targets(&slot("index.embedding").unwrap(), None, CopilotManagedTier::Standard).unwrap(),
["gemini-embedding-001"]
[ManagedTarget {
api_shapes: vec![
llm_adapter::backend::ApiShape::GeminiApi,
llm_adapter::backend::ApiShape::GeminiVertex
],
model_id: "gemini-embedding-001".into()
}]
);
assert_eq!(
managed_targets(&slot("search.rerank").unwrap(), None, CopilotManagedTier::Standard).unwrap(),
["gpt-4o-mini"]
[ManagedTarget {
api_shapes: vec![llm_adapter::backend::ApiShape::ChatCompletions],
model_id: "gpt-4o-mini".into()
}]
);
let inventory = managed_catalog();
assert!(
inventory
.iter()
.any(|(target, operation)| target.model_id == "gpt-image-2.6-flare"
&& *operation == llm_adapter::target::BackendOperation::Image)
);
assert!(
!inventory
.iter()
.any(|(target, operation)| target.model_id == "gpt-image-2.6-flare"
&& *operation == llm_adapter::target::BackendOperation::Chat)
);
assert!(
inventory
.iter()
.any(|(target, operation)| target.model_id == "gemini-3.5-flash-lite"
&& *operation == llm_adapter::target::BackendOperation::Structured)
);
assert!(
!inventory
.iter()
.any(|(target, operation)| target.model_id == "gemini-3.5-flash-lite"
&& *operation == llm_adapter::target::BackendOperation::Image)
);
assert!(matches!(
quota_policy(&prompt_slot, Some("Summary as title")),
@@ -1,3 +1,27 @@
use std::sync::Arc;
use llm_adapter::{
backend::{BackendProvider, CredentialPlan},
capability::{DeclaredModelCapability, declared_model_matches},
target::PreparedBackendTarget,
};
#[derive(Clone)]
pub(crate) struct ResolvedManagedTarget {
pub(crate) profile_id: String,
pub(crate) provider: BackendProvider,
pub(crate) target: PreparedBackendTarget,
pub(crate) capabilities: Vec<DeclaredModelCapability>,
pub(crate) credential_plan: Arc<CredentialPlan>,
pub(crate) credential_config: Arc<serde_json::Value>,
}
impl ResolvedManagedTarget {
pub(crate) fn supports(&self, slot: &super::CatalogSlot) -> bool {
declared_model_matches(&self.capabilities, &slot.requirements)
}
}
#[napi_derive::napi(string_enum)]
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum CopilotManagedTier {
+4 -3
View File
@@ -3,13 +3,14 @@ mod contract;
mod policy;
pub(crate) use catalog::{
CatalogSlot, QuotaPolicy, RouteOperation, managed_selected_target, managed_targets, quota_policy, slot,
with_request_requirements,
CatalogSlot, ManagedTarget, QuotaPolicy, RouteOperation, managed_catalog, managed_selected_target, managed_targets,
quota_policy, slot, with_request_requirements,
};
pub(crate) use contract::ResolvedManagedTarget;
pub use contract::{
CopilotAccessProjection, CopilotExecuteInput, CopilotManagedTier, CopilotRouteCheckInput, CopilotTargetOverrideInput,
};
pub(crate) use policy::{
AuthorizedProviderProfile, AuthorizedTargetRef, CredentialRef, ProfileSource, RouteDecision, RouteDecisionReason,
AuthorizedProviderProfile, AuthorizedRouteTargets, CredentialRef, ProfileSource, RouteDecision, RouteDecisionReason,
RoutePolicyInput, TargetOverride, decide,
};
+110 -113
View File
@@ -1,33 +1,33 @@
use llm_adapter::{
capability::declared_model_matches,
target::{BackendEndpoint, EgressPolicy, OpenAiDialect},
};
use llm_adapter::{capability::declared_model_matches, target::EgressPolicy};
use super::CatalogSlot;
use crate::{llm::byok::ByokModelDeclaration, runtime::Deployment};
use super::{CatalogSlot, ResolvedManagedTarget};
use crate::{
llm::byok::{ByokEndpoint, ByokModelDeclaration},
runtime::Deployment,
};
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum ProfileSource {
Server,
Local,
Managed,
}
#[derive(Clone)]
pub(crate) struct AuthorizedProviderProfile {
pub(crate) profile_id: String,
pub(crate) source: ProfileSource,
pub(crate) provider: String,
pub(crate) endpoint: BackendEndpoint,
pub(crate) openai_dialect: Option<OpenAiDialect>,
pub(crate) endpoint: ByokEndpoint,
pub(crate) egress_policy: EgressPolicy,
pub(crate) models: Vec<ByokModelDeclaration>,
pub(crate) sort_order: i32,
pub(crate) credential_ref: CredentialRef,
}
pub(crate) enum CredentialRef {
Envelope { encrypted: String, aad: Vec<u8> },
Managed { profile_id: String },
#[derive(Clone)]
pub(crate) struct CredentialRef {
pub(crate) encrypted: String,
pub(crate) aad: Vec<u8>,
}
#[derive(Clone, PartialEq, Eq)]
@@ -46,11 +46,16 @@ pub(crate) enum RouteDecisionReason {
}
pub(crate) enum RouteDecision {
Ready(Vec<AuthorizedTargetRef>),
Ready(AuthorizedRouteTargets),
Denied(RouteDecisionReason),
NoRoute(RouteDecisionReason),
}
pub(crate) enum AuthorizedRouteTargets {
Byok(Vec<AuthorizedTargetRef>),
Managed(ResolvedManagedTarget),
}
pub(crate) struct AuthorizedTargetRef {
pub(crate) profile_index: usize,
pub(crate) model_index: usize,
@@ -62,6 +67,7 @@ pub(crate) struct RoutePolicyInput<'a> {
pub(crate) byok_enabled: bool,
pub(crate) access_available: bool,
pub(crate) profiles: &'a [AuthorizedProviderProfile],
pub(crate) managed_target: Option<&'a ResolvedManagedTarget>,
pub(crate) target_override: Option<&'a TargetOverride>,
pub(crate) target_override_managed: bool,
}
@@ -78,10 +84,14 @@ pub(crate) fn decide(input: RoutePolicyInput<'_>) -> RouteDecision {
if !input.access_available {
return RouteDecision::Denied(RouteDecisionReason::AccessUnavailable);
}
return match input.managed_target.filter(|target| target.supports(input.slot)) {
Some(target) => RouteDecision::Ready(AuthorizedRouteTargets::Managed(target.clone())),
None => RouteDecision::Denied(RouteDecisionReason::ExplicitTargetUnavailable),
};
}
if let Some(target) = input.target_override {
let mut selected = compatible_targets(&input, input.target_override_managed);
let mut selected = compatible_targets(&input);
selected.retain(|candidate| {
let profile = &input.profiles[candidate.profile_index];
let model = &profile.models[candidate.model_index];
@@ -90,35 +100,28 @@ pub(crate) fn decide(input: RoutePolicyInput<'_>) -> RouteDecision {
return if selected.is_empty() {
RouteDecision::Denied(RouteDecisionReason::ExplicitTargetUnavailable)
} else {
RouteDecision::Ready(selected)
RouteDecision::Ready(AuthorizedRouteTargets::Byok(selected))
};
}
let byok = compatible_targets(&input, false);
let byok = compatible_targets(&input);
if !byok.is_empty() {
return RouteDecision::Ready(byok);
return RouteDecision::Ready(AuthorizedRouteTargets::Byok(byok));
}
if !input.access_available {
return RouteDecision::Denied(RouteDecisionReason::AccessUnavailable);
}
if input.deployment == Deployment::Cloud {
let managed = compatible_targets(&input, true);
if managed.is_empty() {
RouteDecision::NoRoute(RouteDecisionReason::ManagedPresetUnavailable)
} else {
RouteDecision::Ready(managed)
match input.managed_target.filter(|target| target.supports(input.slot)) {
Some(target) => RouteDecision::Ready(AuthorizedRouteTargets::Managed(target.clone())),
None => RouteDecision::NoRoute(RouteDecisionReason::ManagedPresetUnavailable),
}
} else {
RouteDecision::NoRoute(RouteDecisionReason::NoCompatibleTarget)
}
}
fn compatible_targets(input: &RoutePolicyInput<'_>, managed: bool) -> Vec<AuthorizedTargetRef> {
let mut profiles = input
.profiles
.iter()
.enumerate()
.filter(|(_, profile)| (profile.source == ProfileSource::Managed) == managed)
.collect::<Vec<_>>();
fn compatible_targets(input: &RoutePolicyInput<'_>) -> Vec<AuthorizedTargetRef> {
let mut profiles = input.profiles.iter().enumerate().collect::<Vec<_>>();
profiles.sort_by_key(|(_, profile)| profile.sort_order);
profiles
.into_iter()
@@ -141,18 +144,20 @@ mod tests {
use llm_adapter::capability::{DeclaredModelCapability, ModelInput, ModelOutput};
use super::*;
use crate::llm::{byok::ByokModelDeclaration, route::catalog};
use crate::llm::route::slot;
fn profile(id: &str, source: ProfileSource, model: &str, output: ModelOutput) -> AuthorizedProviderProfile {
fn profile(model: &str, output: ModelOutput) -> AuthorizedProviderProfile {
AuthorizedProviderProfile {
profile_id: id.to_string(),
source,
provider: "openai".to_string(),
endpoint: BackendEndpoint::Custom("https://example.test/v1".to_string()),
openai_dialect: Some(OpenAiDialect::Responses),
profile_id: "byok".into(),
source: ProfileSource::Server,
provider: "openai".into(),
endpoint: ByokEndpoint::OpenAiCompatible {
url: "https://example.test/v1".into(),
dialect: llm_adapter::target::OpenAiDialect::Responses,
},
egress_policy: EgressPolicy::PublicOnly,
models: vec![ByokModelDeclaration {
model_id: model.to_string(),
model_id: model.into(),
enabled: true,
capabilities: vec![DeclaredModelCapability {
input: vec![ModelInput::Text],
@@ -163,130 +168,114 @@ mod tests {
}],
}],
sort_order: 0,
credential_ref: CredentialRef::Managed {
profile_id: id.to_string(),
credential_ref: CredentialRef {
encrypted: String::new(),
aad: Vec::new(),
},
}
}
fn managed() -> ResolvedManagedTarget {
let mut config: crate::runtime::CopilotRuntimeConfig = serde_json::from_value(serde_json::json!({
"providers": {"profiles": [{"id": "managed", "type": "openai", "config": {"apiKey": "test"}}]}
}))
.unwrap();
config.initialize_managed_routes().unwrap();
config
.managed_routes
.iter()
.find(|binding| {
binding.target.model_id == "gpt-6-luna" && binding.operation == llm_adapter::target::BackendOperation::Chat
})
.unwrap()
.resolved
.clone()
}
#[test]
fn deployment_matrix_and_unsupported_only_fallback() {
let slot = catalog::slot("chat.default").unwrap();
let cases = [
(
Deployment::Cloud,
true,
vec![profile("managed", ProfileSource::Managed, "A", ModelOutput::Text)],
true,
),
(
Deployment::SelfHosted,
false,
vec![profile("managed", ProfileSource::Managed, "A", ModelOutput::Text)],
false,
),
let slot = slot("chat.default").unwrap();
let managed = managed();
for (deployment, byok_enabled, profiles, ready) in [
(Deployment::Cloud, true, vec![], true),
(Deployment::SelfHosted, false, vec![], false),
(Deployment::SelfHosted, true, vec![], false),
(
Deployment::Cloud,
true,
vec![
profile("byok", ProfileSource::Server, "B", ModelOutput::Image),
profile("managed", ProfileSource::Managed, "A", ModelOutput::Text),
],
vec![profile("opaque", ModelOutput::Image)],
true,
),
];
for (deployment, byok_enabled, profiles, ready) in cases {
] {
let decision = decide(RoutePolicyInput {
slot: &slot,
deployment,
byok_enabled,
access_available: true,
profiles: &profiles,
managed_target: Some(&managed),
target_override: None,
target_override_managed: false,
});
assert_eq!(matches!(decision, RouteDecision::Ready(_)), ready);
}
let profiles = vec![
profile("byok", ProfileSource::Server, "B", ModelOutput::Text),
profile("managed", ProfileSource::Managed, "A", ModelOutput::Text),
];
assert!(matches!(
decide(RoutePolicyInput {
let profiles = vec![profile("vendor/opaque", ModelOutput::Text)];
for (profiles, expected_byok) in [(&profiles[..], true), (&[][..], false)] {
let decision = decide(RoutePolicyInput {
slot: &slot,
deployment: Deployment::Cloud,
byok_enabled: true,
access_available: false,
profiles: &profiles,
profiles,
managed_target: Some(&managed),
target_override: None,
target_override_managed: false,
}),
RouteDecision::Ready(_)
));
assert!(matches!(
decide(RoutePolicyInput {
slot: &slot,
deployment: Deployment::Cloud,
byok_enabled: true,
access_available: false,
profiles: &profiles[1..],
target_override: None,
target_override_managed: false,
}),
RouteDecision::Denied(RouteDecisionReason::AccessUnavailable)
));
});
if expected_byok {
assert!(matches!(
decision,
RouteDecision::Ready(AuthorizedRouteTargets::Byok(_))
));
} else {
assert!(matches!(
decision,
RouteDecision::Denied(RouteDecisionReason::AccessUnavailable)
));
}
}
}
#[test]
fn override_is_complete_and_custom_model_is_not_replaced_by_managed_model() {
let slot = catalog::slot("chat.default").unwrap();
let profiles = vec![
profile("byok", ProfileSource::Server, "vendor/model:B", ModelOutput::Text),
profile("managed", ProfileSource::Managed, "model:A", ModelOutput::Text),
];
let slot = slot("chat.default").unwrap();
let managed = managed();
let profiles = vec![profile("vendor/model:B", ModelOutput::Text)];
let target = TargetOverride {
profile_id: "byok".to_string(),
model_id: "vendor/model:B".to_string(),
profile_id: "byok".into(),
model_id: "vendor/model:B".into(),
};
let RouteDecision::Ready(candidates) = decide(RoutePolicyInput {
let input = |explicit| RoutePolicyInput {
slot: &slot,
deployment: Deployment::Cloud,
byok_enabled: true,
access_available: true,
profiles: &profiles,
managed_target: Some(&managed),
target_override: Some(&target),
target_override_managed: false,
}) else {
panic!("override should resolve");
target_override_managed: explicit,
};
let RouteDecision::Ready(AuthorizedRouteTargets::Byok(candidates)) = decide(input(false)) else {
panic!("BYOK override should resolve");
};
assert_eq!(
profiles[candidates[0].profile_index].models[candidates[0].model_index].model_id,
"vendor/model:B"
);
let managed_target = TargetOverride {
profile_id: "managed".to_string(),
model_id: "model:A".to_string(),
};
let RouteDecision::Ready(candidates) = decide(RoutePolicyInput {
slot: &slot,
deployment: Deployment::Cloud,
byok_enabled: true,
access_available: true,
profiles: &profiles,
target_override: Some(&managed_target),
target_override_managed: true,
}) else {
let RouteDecision::Ready(AuthorizedRouteTargets::Managed(selected)) = decide(input(true)) else {
panic!("managed selection should resolve");
};
assert!(matches!(
profiles[candidates[0].profile_index].source,
ProfileSource::Managed
));
let mut disabled = profile("disabled", ProfileSource::Server, "model:C", ModelOutput::Text);
assert_eq!(selected.profile_id, "managed");
let mut disabled = profile("model:C", ModelOutput::Text);
disabled.models[0].enabled = false;
assert!(matches!(
decide(RoutePolicyInput {
@@ -295,10 +284,18 @@ mod tests {
byok_enabled: true,
access_available: true,
profiles: &[disabled],
managed_target: None,
target_override: None,
target_override_managed: false,
}),
RouteDecision::NoRoute(RouteDecisionReason::NoCompatibleTarget)
));
assert!(matches!(
decide(RoutePolicyInput {
managed_target: None,
..input(true)
}),
RouteDecision::Denied(RouteDecisionReason::ExplicitTargetUnavailable)
));
}
}
@@ -6,4 +6,4 @@ pub(super) use local::{LocalLeasePayload, create as create_local_lease};
pub(super) use profile::{create, delete, list, probe_draft, probe_profile, reorder, replace, rotate};
use profile::{envelope_key, require_text};
use super::{RuntimeError, RuntimeResult, backend_provider, executable_protocol, token_hash};
use super::{RuntimeError, RuntimeResult, token_hash};
@@ -1,7 +1,7 @@
use std::collections::HashSet;
use llm_adapter::{
backend::{BackendError, DefaultHttpClient},
backend::{BackendError, BackendProvider, DefaultHttpClient},
capability::{
AttachmentKind, AttachmentSource, ModelFeature, ModelInput, ModelOutput, ModelRequirements, declared_model_matches,
},
@@ -9,14 +9,12 @@ use llm_adapter::{
CoreContent, CoreMessage, CoreRequest, CoreRole, CoreToolChoice, CoreToolDefinition, EmbeddingRequest,
ImageOptions, ImageProviderOptions, ImageRequest, RerankCandidate, RerankRequest, StructuredRequest,
},
router::{ExecutablePreparedRoute, ExecutableRequest, ExecutableResponse, dispatch_prepared_route},
target::{
BackendCredential, BackendEndpoint, BackendOperation, BackendTargetInput, EgressPolicy, compile_backend_target,
},
router::{ExecutableRequest, ExecutableResponse, dispatch_prepared_route},
target::{BackendCredential, BackendOperation, EgressPolicy, prepare_backend_target},
};
use serde_json::json;
use super::{RuntimeError, RuntimeResult, backend_provider, executable_protocol};
use super::{RuntimeError, RuntimeResult};
use crate::llm::{
ByokModelProbeCheckOutput, ByokModelProbeOutput, ByokProbeCheckInput, ByokProbeResultOutput, ByokProbeStatusOutput,
byok::{ByokEndpoint, ByokPolicy, ByokProfileDefinition, SensitiveCredential, definition_fingerprint},
@@ -129,36 +127,37 @@ fn dispatch_check(
"image" => BackendOperation::Image,
_ => return not_tested(),
};
let target = compile_backend_target(BackendTargetInput {
provider: match backend_provider(provider) {
Ok(provider) => provider,
Err(_) => return failed(checked_at, "unsupported_provider"),
},
operation: operation_kind,
endpoint: match endpoint {
ByokEndpoint::ProviderDefault => BackendEndpoint::ProviderDefault,
ByokEndpoint::OpenAiCompatible { url, .. } => BackendEndpoint::Custom(url.clone()),
},
openai_dialect: match endpoint {
ByokEndpoint::ProviderDefault => None,
ByokEndpoint::OpenAiCompatible { dialect, .. } => Some(*dialect),
},
model: model_id.to_string(),
credential: BackendCredential::new(credential),
timeout_ms: Some(15_000),
egress_policy,
});
let provider = match provider.parse::<BackendProvider>() {
Ok(provider) => provider,
Err(_) => return failed(checked_at, "unsupported_provider"),
};
let dialect = match endpoint {
ByokEndpoint::ProviderDefault => None,
ByokEndpoint::OpenAiCompatible { dialect, .. } => Some(*dialect),
};
let target = (|| {
let layer = provider.default_request_layer(operation_kind, dialect)?;
let endpoint = match endpoint {
ByokEndpoint::ProviderDefault => provider.endpoint(&json!({}))?,
ByokEndpoint::OpenAiCompatible { url, .. } => url.clone(),
};
prepare_backend_target(
layer,
operation_kind,
model_id.to_string(),
&endpoint,
Some(15_000),
egress_policy,
)
})();
let target = match target {
Ok(target) => target,
Err(_) => return failed(checked_at, "unsupported_operation"),
};
let route = ExecutablePreparedRoute::new(
executable_protocol(target.protocol),
target.model,
target.config,
let route = match target.bind(
BackendCredential::new(credential),
probe_request_for_operation(operation),
);
let route = match route {
) {
Ok(route) => route,
Err(_) => return failed(checked_at, "invalid_probe_request"),
};
@@ -1,17 +1,13 @@
use llm_adapter::{
capability::provider_default_capability_upper_bound,
target::{BackendEndpoint, OpenAiDialect},
};
use sqlx::{FromRow, PgPool, Row};
use super::super::{LocalLeasePayload, RuntimeError, RuntimeResult, token_hash};
use crate::{
llm::{
CopilotAccessProjection,
byok::{ByokEndpoint, ByokPolicy, ByokProfileDefinition, local_aad, server_aad},
route::{self, AuthorizedProviderProfile, CatalogSlot, CredentialRef, ProfileSource},
byok::{ByokPolicy, ByokProfileDefinition, local_aad, server_aad},
route::{self, AuthorizedProviderProfile, CatalogSlot, CredentialRef, ProfileSource, ResolvedManagedTarget},
},
runtime::{BackendRuntimeConfig, CopilotManagedProfileConfig, CopilotRuntimeConfig},
runtime::{BackendRuntimeConfig, CopilotRuntimeConfig},
};
#[derive(FromRow)]
@@ -25,13 +21,10 @@ struct ServerProfileRow {
}
pub(super) struct ProfileLoadInput<'a> {
pub(super) slot: &'a CatalogSlot,
pub(super) built_in_route_id: Option<&'a str>,
pub(super) workspace_id: Option<&'a str>,
pub(super) user_id: Option<&'a str>,
pub(super) local_lease_id: Option<&'a str>,
pub(super) access: &'a CopilotAccessProjection,
pub(super) managed_target_id: Option<&'a str>,
}
pub(super) async fn load_profiles(
@@ -51,13 +44,6 @@ pub(super) async fn load_profiles(
{
profiles.extend(load_local_profiles(pool, workspace_id, user_id, lease_id, &policy).await?);
}
profiles.extend(load_managed_profiles(
&config.copilot,
input.slot,
input.built_in_route_id,
input.access.managed_tier,
input.managed_target_id,
)?);
Ok(profiles)
}
@@ -99,7 +85,7 @@ async fn load_server_profiles(
definition,
policy,
row.sort_order,
CredentialRef::Envelope {
CredentialRef {
encrypted: row.encrypted_api_key,
aad,
},
@@ -159,7 +145,7 @@ async fn load_local_profiles(
provider.definition,
policy,
index as i32,
CredentialRef::Envelope {
CredentialRef {
encrypted: provider.encrypted_credential,
aad,
},
@@ -169,95 +155,24 @@ async fn load_local_profiles(
)
}
fn load_managed_profiles(
pub(super) fn load_managed_target(
config: &CopilotRuntimeConfig,
slot: &CatalogSlot,
built_in_route_id: Option<&str>,
managed_tier: route::CopilotManagedTier,
managed_target_id: Option<&str>,
) -> RuntimeResult<Vec<AuthorizedProviderProfile>> {
let targets = if let Some(target_id) = managed_target_id {
vec![
route::managed_selected_target(built_in_route_id, target_id, managed_tier)
.ok_or_else(|| RuntimeError::invalid_input("managed_target_unavailable"))?,
]
} else if let Some(targets) = route::managed_targets(slot, built_in_route_id, managed_tier) {
targets
) -> RuntimeResult<Option<ResolvedManagedTarget>> {
let target = if let Some(target_id) = managed_target_id {
route::managed_selected_target(built_in_route_id, target_id, managed_tier)
.ok_or_else(|| RuntimeError::invalid_input("managed_target_unavailable"))?
} else if let Some(target) =
route::managed_targets(slot, built_in_route_id, managed_tier).and_then(|targets| targets.into_iter().next())
{
target
} else {
return Ok(Vec::new());
return Ok(None);
};
targets
.iter()
.enumerate()
.map(|(index, model_id)| {
let matches = config
.providers
.profiles
.iter()
.filter(|profile| profile.enabled && profile.models.iter().any(|model| model == model_id))
.collect::<Vec<_>>();
let Some(profile) = matches.first() else {
return Ok(None);
};
if matches.len() > 1 {
return Err(RuntimeError::invalid_state(
"built-in managed route model matches multiple profiles",
));
}
let capabilities = provider_default_capability_upper_bound(&profile.provider, model_id)
.ok_or_else(|| RuntimeError::invalid_state("built-in managed route model is incompatible with its profile"))?;
let endpoint = managed_endpoint(profile)?;
Ok(Some(AuthorizedProviderProfile {
profile_id: profile.id.clone(),
source: ProfileSource::Managed,
provider: profile.provider.clone(),
endpoint,
openai_dialect: (profile.provider == "openai").then_some(OpenAiDialect::Responses),
egress_policy: llm_adapter::target::EgressPolicy::PublicOnly,
models: vec![crate::llm::byok::ByokModelDeclaration {
model_id: model_id.clone(),
enabled: true,
capabilities,
}],
sort_order: index as i32,
credential_ref: CredentialRef::Managed {
profile_id: profile.id.clone(),
},
}))
})
.filter_map(|profile| profile.transpose())
.collect()
}
fn managed_endpoint(profile: &CopilotManagedProfileConfig) -> RuntimeResult<BackendEndpoint> {
if let Some(base_url) = profile.config.get("baseURL").and_then(serde_json::Value::as_str) {
return llm_adapter::target::canonicalize_endpoint(base_url)
.map(BackendEndpoint::Custom)
.map_err(|error| RuntimeError::invalid_state(error.to_string()));
}
let endpoint = match profile.provider.as_str() {
"geminiVertex" | "anthropicVertex" => {
let location = required_config_text(profile, "location")?;
let project = required_config_text(profile, "project")?;
let publisher = if profile.provider == "geminiVertex" {
"google"
} else {
"anthropic"
};
let host = if location == "global" {
"aiplatform.googleapis.com".to_string()
} else {
format!("{location}-aiplatform.googleapis.com")
};
format!("https://{host}/v1/projects/{project}/locations/{location}/publishers/{publisher}")
}
"cloudflareWorkersAi" => format!(
"https://api.cloudflare.com/client/v4/accounts/{}/ai",
required_config_text(profile, "accountId")?
),
_ => return Ok(BackendEndpoint::ProviderDefault),
};
Ok(BackendEndpoint::Custom(endpoint))
Ok(config.managed_binding(&target, slot.operation.into()).cloned())
}
fn authorized_byok_profile(
@@ -270,16 +185,11 @@ fn authorized_byok_profile(
credential_ref: CredentialRef,
) -> AuthorizedProviderProfile {
let egress_policy = policy.egress_policy(&definition.endpoint);
let (endpoint, openai_dialect) = match definition.endpoint {
ByokEndpoint::ProviderDefault => (BackendEndpoint::ProviderDefault, None),
ByokEndpoint::OpenAiCompatible { url, dialect } => (BackendEndpoint::Custom(url), Some(dialect)),
};
AuthorizedProviderProfile {
profile_id,
source,
provider,
endpoint,
openai_dialect,
endpoint: definition.endpoint,
egress_policy,
models: definition.models,
sort_order,
@@ -287,64 +197,239 @@ fn authorized_byok_profile(
}
}
pub(super) fn managed_profile<'a>(
config: &'a CopilotRuntimeConfig,
profile_id: &str,
) -> RuntimeResult<&'a CopilotManagedProfileConfig> {
config
.providers
.profiles
.iter()
.find(|profile| profile.id == profile_id && profile.enabled)
.ok_or_else(|| RuntimeError::invalid_state("managed copilot credential unavailable"))
}
pub(super) fn required_config_text<'a>(
profile: &'a CopilotManagedProfileConfig,
field: &'static str,
) -> RuntimeResult<&'a str> {
profile
.config
.get(field)
.and_then(serde_json::Value::as_str)
.filter(|value| !value.trim().is_empty())
.ok_or_else(|| RuntimeError::invalid_state(format!("managed copilot profile requires {field}")))
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use serde_json::json;
use zeroize::Zeroizing;
use super::{BackendEndpoint, CopilotManagedProfileConfig, managed_endpoint};
use super::*;
use crate::runtime::Deployment;
fn vertex_profile(location: &str) -> CopilotManagedProfileConfig {
CopilotManagedProfileConfig {
id: "vertex".to_string(),
provider: "geminiVertex".to_string(),
enabled: true,
models: vec!["gemini-3.7-flash".to_string()],
config: json!({ "project": "affine-us", "location": location }),
fn load(config: &CopilotRuntimeConfig, target: &str) -> ResolvedManagedTarget {
load_managed_target(
config,
&route::slot("chat.default").unwrap(),
Some("Chat With AFFiNE AI"),
route::CopilotManagedTier::Premium,
Some(target),
)
.unwrap()
.unwrap()
}
#[test]
fn managed_routes_bind_api_shapes_and_profile_preferences() {
let mut copilot: CopilotRuntimeConfig = serde_json::from_value(json!({
"enabled": true,
"providers": { "profiles": [
{ "id": "gateway", "type": "openrouter", "config": { "apiKey": "gateway-key" } },
{ "id": "openai", "type": "openai", "models": ["stale-model"], "config": { "apiKey": "old-key" } },
{ "id": "google", "type": "gemini", "models": ["unknown-future-model"], "config": { "apiKey": "test" } },
{ "id": "official", "type": "anthropic", "models": ["claude-sonnet-4.6"], "config": { "apiKey": "test" } },
{ "id": "vertex", "type": "anthropicVertex", "config": { "project": "test", "location": "global" } }
] }
}))
.unwrap();
copilot.initialize_managed_routes().unwrap();
for (id, service, model) in [
("luna", "openrouter", "openai/gpt-6-luna"),
("terra", "openrouter", "openai/gpt-6.1-terra"),
("gemini", "gemini", "gemini-3.8-flash"),
("claude", "anthropicVertex", "claude-sonnet-4-6"),
] {
let selected = load(&copilot, id);
assert_eq!(selected.provider.as_str(), service);
assert_eq!(selected.target.wire_model(), model);
}
for binding in &copilot.managed_routes {
use llm_adapter::{capability::ModelOutput, target::BackendOperation};
assert!(!binding.resolved.capabilities.is_empty());
for capability in &binding.resolved.capabilities {
assert!(capability.output.iter().all(|output| match binding.operation {
BackendOperation::Chat => matches!(output, ModelOutput::Text | ModelOutput::Object),
BackendOperation::Structured => *output == ModelOutput::Structured,
BackendOperation::Embedding => *output == ModelOutput::Embedding,
BackendOperation::Rerank => *output == ModelOutput::Rerank,
BackendOperation::Image => *output == ModelOutput::Image,
}));
if !matches!(binding.operation, BackendOperation::Chat | BackendOperation::Structured) {
assert!(capability.features.is_empty());
}
}
}
copilot.providers.profiles[1].config["openaiDialect"] = json!("chat_completions");
copilot.providers.profiles.swap(0, 1);
copilot.initialize_managed_routes().unwrap();
assert_eq!(load(&copilot, "luna").profile_id, "gateway");
copilot.providers.profiles.swap(0, 1);
copilot.providers.profiles[1]
.config
.as_object_mut()
.unwrap()
.remove("openaiDialect");
copilot.providers.profiles.push(
serde_json::from_value(json!({
"id": "vertex-alias", "type": "anthropicVertex", "models": ["claude-sonnet-4-6"],
"config": { "project": "test", "location": "global" }
}))
.unwrap(),
);
copilot.initialize_managed_routes().unwrap();
assert_eq!(load(&copilot, "claude").profile_id, "vertex-alias");
copilot.providers.profiles.push(
serde_json::from_value(json!({
"id": "pinned", "type": "openai", "models": ["gpt-6-luna"], "config": { "apiKey": "pinned-key" }
}))
.unwrap(),
);
copilot.initialize_managed_routes().unwrap();
assert_eq!(load(&copilot, "luna").profile_id, "pinned");
copilot.providers.profiles[0].models.push("gpt-6-luna".into());
copilot.initialize_managed_routes().unwrap();
assert_eq!(load(&copilot, "luna").profile_id, "gateway");
copilot.providers.profiles[0].enabled = false;
copilot.providers.profiles[1].models.push("gpt-6-luna".into());
copilot.initialize_managed_routes().unwrap();
assert_eq!(load(&copilot, "luna").profile_id, "openai");
let pinned = copilot.providers.profiles.len() - 1;
copilot.providers.profiles.swap(1, pinned);
copilot.initialize_managed_routes().unwrap();
assert_eq!(load(&copilot, "luna").profile_id, "pinned");
let frozen = load(&copilot, "luna");
copilot.providers.profiles[1].config["apiKey"] = json!("new-key");
assert_eq!(load(&copilot, "luna").credential_config["apiKey"], "pinned-key");
copilot.initialize_managed_routes().unwrap();
assert_eq!(load(&copilot, "luna").credential_config["apiKey"], "new-key");
assert_eq!(frozen.credential_config["apiKey"], "pinned-key");
copilot.providers.profiles[1].config["apiKey"] = json!("");
assert!(copilot.initialize_managed_routes().is_err());
assert_eq!(load(&copilot, "luna").credential_config["apiKey"], "new-key");
copilot.providers.profiles.retain(|profile| profile.id == "openai");
copilot.initialize_managed_routes().unwrap();
let mut config = BackendRuntimeConfig {
database_url: "postgresql://unused".into(),
auth: Default::default(),
invite_quota: Default::default(),
private_key: Arc::new(Zeroizing::new(String::new())),
deployment: Deployment::Cloud,
copilot,
search: Default::default(),
redis: Default::default(),
payment: Default::default(),
};
let options = super::super::options::route_options(&config, "Chat With AFFiNE AI").unwrap();
assert_eq!(
options
.choices
.iter()
.map(|choice| choice.id.as_str())
.collect::<Vec<_>>(),
["luna", "terra"]
);
assert_eq!(options.standard_default_target_id.as_deref(), Some("luna"));
assert_eq!(options.premium_default_target_id.as_deref(), Some("luna"));
config.copilot.providers.profiles[0].config["openaiDialect"] = json!("chat_completions");
config.copilot.initialize_managed_routes().unwrap();
assert!(
load_managed_target(
&config.copilot,
&route::slot("chat.default").unwrap(),
Some("Chat With AFFiNE AI"),
route::CopilotManagedTier::Premium,
Some("luna")
)
.unwrap()
.is_none()
);
let frozen_options = super::super::options::route_options(&config, "Chat With AFFiNE AI").unwrap();
config.copilot.providers.profiles.clear();
assert_eq!(
super::super::options::route_options(&config, "Chat With AFFiNE AI")
.unwrap()
.choices
.len(),
frozen_options.choices.len()
);
config.copilot.initialize_managed_routes().unwrap();
let options = super::super::options::route_options(&config, "Chat With AFFiNE AI").unwrap();
assert!(options.choices.is_empty());
assert!(options.standard_default_target_id.is_none());
assert!(options.premium_default_target_id.is_none());
config.deployment = Deployment::SelfHosted;
assert!(
super::super::options::route_options(&config, "Chat With AFFiNE AI")
.unwrap()
.choices
.is_empty()
);
}
#[test]
fn managed_embedding_endpoints_preserve_persisted_fingerprints() {
for (provider, config, endpoint, fingerprint) in [
(
"gemini",
json!({"apiKey": "test"}),
"https://generativelanguage.googleapis.com/v1beta",
"f8e2b5973b9bdb9430eb770bf13e08f759c9c4cbded97dbfea4c01050d32aef1",
),
(
"gemini",
json!({"apiKey": "test", "baseURL": "https://custom.example/v1"}),
"https://custom.example/v1",
"47ab6847e37db8d443a5482402964bba977301e37f7ebbf065988f765c0327b6",
),
(
"geminiVertex",
json!({"project": "affine-us", "location": "global"}),
"https://aiplatform.googleapis.com/v1/projects/affine-us/locations/global/publishers/google",
"f9a42da40743503700bd4806b2c337d1ebc5d4a3499b97db7956a2401e9c2fec",
),
(
"geminiVertex",
json!({"project": "affine-us", "location": "us-central1"}),
"https://us-central1-aiplatform.googleapis.com/v1/projects/affine-us/locations/us-central1/publishers/google",
"0945a331a8d982aca94191af38496db3048920c0a0b7294883af103c92089da5",
),
] {
let mut copilot: CopilotRuntimeConfig = serde_json::from_value(json!({
"providers": { "profiles": [{ "id": "original", "type": provider, "config": config }] }
}))
.unwrap();
for revision in 0..2 {
copilot.providers.profiles[0].id = format!("profile-{revision}");
if copilot.providers.profiles[0].config.get("apiKey").is_some() {
copilot.providers.profiles[0].config["apiKey"] = json!(format!("key-{revision}"));
}
copilot.initialize_managed_routes().unwrap();
let slot = route::slot("index.embedding").unwrap();
let selected = load_managed_target(&copilot, &slot, None, route::CopilotManagedTier::Standard, None)
.unwrap()
.unwrap();
assert_eq!(selected.target.endpoint(), endpoint);
let authorized = super::super::AuthorizedCopilotRoute {
config: Arc::new(BackendRuntimeConfig {
database_url: "postgresql://unused".into(),
auth: Default::default(),
invite_quota: Default::default(),
private_key: Arc::new(Zeroizing::new(String::new())),
deployment: Deployment::Cloud,
copilot: copilot.clone(),
search: Default::default(),
redis: Default::default(),
payment: Default::default(),
}),
slot,
profiles: vec![],
targets: route::AuthorizedRouteTargets::Managed(selected),
};
assert_eq!(
super::super::target_from_route(&authorized).unwrap().fingerprint,
fingerprint
);
}
}
}
#[test]
fn managed_vertex_endpoint_uses_global_host() {
assert_eq!(
managed_endpoint(&vertex_profile("global")).unwrap(),
BackendEndpoint::Custom(
"https://aiplatform.googleapis.com/v1/projects/affine-us/locations/global/publishers/google".to_string()
)
);
}
#[test]
fn managed_vertex_endpoint_uses_regional_host() {
assert_eq!(
managed_endpoint(&vertex_profile("us-central1")).unwrap(),
BackendEndpoint::Custom(
"https://us-central1-aiplatform.googleapis.com/v1/projects/affine-us/locations/us-central1/publishers/google"
.to_string()
)
);
}
}
@@ -5,30 +5,28 @@ use std::{
use gcp_auth::{CustomServiceAccount, TokenProvider};
use llm_adapter::{
backend::{BackendError, DefaultHttpClient},
backend::{BackendError, CredentialPlan, DefaultHttpClient},
capability::{AttachmentKind, AttachmentSource},
core::{CoreContent, ImageInput, ImageRequest},
router::{ExecutablePreparedRoute, ExecutableProtocol, ExecutableRequest, ExecutableResponse},
target::{
BackendCredential, BackendOperation, BackendProtocol, BackendProvider, BackendTargetInput, compile_backend_target,
},
router::{ExecutableRequest, ExecutableResponse},
target::{BackendCredential, prepare_backend_target},
};
use llm_runtime::{CompiledPlan, CompiledRoute, RuntimeRouteEvent, RuntimeUsage, dispatch_compiled_plan};
use serde::Serialize;
use uuid::Uuid;
use zeroize::Zeroizing;
use super::{COPILOT_REQUEST_TIMEOUT, RuntimeError, RuntimeResult, context};
use super::{COPILOT_REQUEST_TIMEOUT, RuntimeError, RuntimeResult};
use crate::{
llm::{
LlmImageRequestContract,
byok::CredentialEnvelopeKey,
byok::{ByokEndpoint, CredentialEnvelopeKey},
route::{
AuthorizedProviderProfile, AuthorizedTargetRef, CatalogSlot, CredentialRef, RouteOperation,
AuthorizedProviderProfile, AuthorizedRouteTargets, CatalogSlot, CredentialRef, RouteOperation,
with_request_requirements,
},
},
runtime::{BackendRuntimeConfig, CopilotManagedProfileConfig},
runtime::BackendRuntimeConfig,
};
#[derive(Serialize)]
@@ -105,10 +103,10 @@ pub(super) fn execute(
slot: CatalogSlot,
request: ExecutableRequest,
profiles: Vec<AuthorizedProviderProfile>,
candidates: Vec<AuthorizedTargetRef>,
managed_credentials: HashMap<String, Zeroizing<String>>,
targets: AuthorizedRouteTargets,
managed_credential: Option<Zeroizing<String>>,
) -> RuntimeResult<CopilotExecutionResult> {
let execution = compile_execution(&config, slot, request, &profiles, &candidates, &managed_credentials)?;
let execution = compile_execution(&config, slot, request, &profiles, &targets, managed_credential.as_ref())?;
let mut runtime_events = Vec::new();
let response = dispatch_compiled_plan(&DefaultHttpClient::default(), &execution.plan, |event| {
runtime_events.push(event)
@@ -129,10 +127,10 @@ pub(super) fn execute_embeddings(
slot: CatalogSlot,
request: ExecutableRequest,
profiles: Vec<AuthorizedProviderProfile>,
candidates: Vec<AuthorizedTargetRef>,
managed_credentials: HashMap<String, Zeroizing<String>>,
targets: AuthorizedRouteTargets,
managed_credential: Option<Zeroizing<String>>,
) -> RuntimeResult<Vec<Vec<f32>>> {
let output = execute(config, slot, request, profiles, candidates, managed_credentials)?;
let output = execute(config, slot, request, profiles, targets, managed_credential)?;
let response: llm_adapter::core::EmbeddingResponse = serde_json::from_value(output.result)
.map_err(|error| RuntimeError::json("decode embedding response failed", error))?;
response
@@ -152,50 +150,81 @@ pub(super) fn compile_execution(
slot: CatalogSlot,
request: ExecutableRequest,
profiles: &[AuthorizedProviderProfile],
candidates: &[AuthorizedTargetRef],
managed_credentials: &HashMap<String, Zeroizing<String>>,
targets: &AuthorizedRouteTargets,
managed_credential: Option<&Zeroizing<String>>,
) -> RuntimeResult<CompiledExecution> {
let key = CredentialEnvelopeKey::derive(config.private_key.as_bytes())
.map_err(|_| RuntimeError::invalid_state("credential_unavailable"))?;
let mut identities = HashMap::new();
let mut routes = Vec::with_capacity(candidates.len());
for candidate in candidates {
let profile = profiles
.get(candidate.profile_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid authorized route profile"))?;
let model = profile
.models
.get(candidate.model_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid authorized route model"))?;
let credential = resolve_credential(&key, profile, managed_credentials)?;
let target = compile_backend_target(BackendTargetInput {
provider: provider(&profile.provider)?,
operation: operation(slot.operation),
endpoint: profile.endpoint.clone(),
openai_dialect: profile.openai_dialect,
model: model.model_id.clone(),
credential: BackendCredential::new(credential),
timeout_ms: Some(COPILOT_REQUEST_TIMEOUT.as_millis() as u64),
egress_policy: profile.egress_policy,
})
.map_err(|error| RuntimeError::invalid_state(error.to_string()))?;
let route_id = Uuid::new_v4().to_string();
identities.insert(
route_id.clone(),
RouteIdentity {
profile_id: profile.profile_id.clone(),
source: match profile.source {
crate::llm::route::ProfileSource::Server => "server",
crate::llm::route::ProfileSource::Local => "local",
crate::llm::route::ProfileSource::Managed => "affine_cloud",
let compiled = match targets {
AuthorizedRouteTargets::Managed(managed) => {
let credential =
managed_credential.ok_or_else(|| RuntimeError::invalid_state("managed copilot credential unavailable"))?;
let route = managed
.target
.bind(BackendCredential::new(credential.as_str().to_string()), request.clone())
.map_err(|error| RuntimeError::invalid_input(error.to_string()))?;
vec![(
RouteIdentity {
profile_id: managed.profile_id.clone(),
source: "affine_cloud",
provider: managed.provider.as_str().to_string(),
model: managed.target.wire_model().to_string(),
},
provider: profile.provider.clone(),
model: target.model.clone(),
},
);
let protocol = protocol(target.protocol);
let route = ExecutablePreparedRoute::new(protocol, target.model, target.config, request.clone())
.map_err(|error| RuntimeError::invalid_input(error.to_string()))?;
route,
)]
}
AuthorizedRouteTargets::Byok(candidates) => {
let key = CredentialEnvelopeKey::derive(config.private_key.as_bytes())
.map_err(|_| RuntimeError::invalid_state("credential_unavailable"))?;
candidates
.iter()
.map(|candidate| {
let profile = profiles
.get(candidate.profile_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid authorized route profile"))?;
let model = profile
.models
.get(candidate.model_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid authorized route model"))?;
let provider = profile.provider.parse::<llm_adapter::backend::BackendProvider>()?;
let (endpoint, dialect) = match &profile.endpoint {
ByokEndpoint::ProviderDefault => (provider.endpoint(&serde_json::json!({}))?, None),
ByokEndpoint::OpenAiCompatible { url, dialect } => (url.clone(), Some(*dialect)),
};
let layer = provider.default_request_layer(slot.operation.into(), dialect)?;
let target = prepare_backend_target(
layer,
slot.operation.into(),
model.model_id.clone(),
&endpoint,
Some(COPILOT_REQUEST_TIMEOUT.as_millis() as u64),
profile.egress_policy,
)?;
let route = target
.bind(
BackendCredential::new(resolve_credential(&key, profile)?),
request.clone(),
)
.map_err(|error| RuntimeError::invalid_input(error.to_string()))?;
Ok((
RouteIdentity {
profile_id: profile.profile_id.clone(),
source: match profile.source {
crate::llm::route::ProfileSource::Server => "server",
crate::llm::route::ProfileSource::Local => "local",
},
provider: profile.provider.clone(),
model: target.wire_model().to_string(),
},
route,
))
})
.collect::<RuntimeResult<Vec<_>>>()?
}
};
let mut identities = HashMap::new();
let mut routes = Vec::with_capacity(compiled.len());
for (identity, route) in compiled {
let route_id = Uuid::new_v4().to_string();
identities.insert(route_id.clone(), identity);
routes.push(CompiledRoute::new(route_id, route));
}
Ok(CompiledExecution {
@@ -272,106 +301,67 @@ fn collect_message_attachments(
}
}
fn resolve_credential(
key: &CredentialEnvelopeKey,
profile: &AuthorizedProviderProfile,
managed_credentials: &HashMap<String, Zeroizing<String>>,
fn resolve_credential(key: &CredentialEnvelopeKey, profile: &AuthorizedProviderProfile) -> RuntimeResult<String> {
let CredentialRef { encrypted, aad } = &profile.credential_ref;
key
.decrypt(encrypted, aad)
.map_err(|_| RuntimeError::invalid_state("credential_unavailable"))
.and_then(|credential| {
String::from_utf8(credential.expose().to_vec()).map_err(|_| RuntimeError::invalid_state("credential_unavailable"))
})
}
pub(super) async fn managed_credential(
plan: &CredentialPlan,
config: &serde_json::Value,
token_provider: Option<Arc<dyn TokenProvider>>,
) -> RuntimeResult<String> {
match &profile.credential_ref {
CredentialRef::Envelope { encrypted, aad } => key
.decrypt(encrypted, aad)
.map_err(|_| RuntimeError::invalid_state("credential_unavailable"))
.and_then(|credential| {
String::from_utf8(credential.expose().to_vec())
.map_err(|_| RuntimeError::invalid_state("credential_unavailable"))
}),
CredentialRef::Managed { profile_id } => managed_credentials
.get(profile_id)
.map(|credential| credential.as_str().to_string())
match plan {
CredentialPlan::GoogleOAuth { scopes, .. } => {
let scopes = scopes.iter().map(String::as_str).collect::<Vec<_>>();
token_provider
.ok_or_else(|| RuntimeError::invalid_state("managed Google credential unavailable"))?
.token(&scopes)
.await
.map(|token| token.as_str().to_string())
.map_err(|_| RuntimeError::invalid_state("managed Google credential unavailable"))
}
CredentialPlan::Field { field } => config
.get(field)
.and_then(serde_json::Value::as_str)
.map(str::to_string)
.ok_or_else(|| RuntimeError::invalid_state("managed copilot credential unavailable")),
}
}
pub(super) async fn managed_credential(
profile: &CopilotManagedProfileConfig,
token_provider: Option<Arc<dyn TokenProvider>>,
) -> RuntimeResult<String> {
if matches!(profile.provider.as_str(), "geminiVertex" | "anthropicVertex") {
return token_provider
.ok_or_else(|| RuntimeError::invalid_state("managed Vertex credential unavailable"))?
.token(&["https://www.googleapis.com/auth/cloud-platform"])
.await
.map(|token| token.as_str().to_string())
.map_err(|_| RuntimeError::invalid_state("managed Vertex credential unavailable"));
}
let field = if profile.provider == "cloudflareWorkersAi" {
"apiToken"
} else {
"apiKey"
pub(super) async fn create_google_token_provider(plan: &CredentialPlan) -> RuntimeResult<Arc<dyn TokenProvider>> {
let CredentialPlan::GoogleOAuth {
project, credentials, ..
} = plan
else {
return Err(RuntimeError::invalid_state("managed Google credential unavailable"));
};
Ok(context::required_config_text(profile, field)?.to_string())
}
pub(super) async fn create_vertex_token_provider(
profile: &CopilotManagedProfileConfig,
) -> RuntimeResult<Arc<dyn TokenProvider>> {
if let Some(credentials) = profile.config.pointer("/googleAuthOptions/credentials") {
let project = context::required_config_text(profile, "project")?.to_string();
if let Some(credentials) = credentials {
let mut credentials = credentials.clone();
let object = credentials
.as_object_mut()
.ok_or_else(|| RuntimeError::invalid_state("managed Vertex credentials must be an object"))?;
.ok_or_else(|| RuntimeError::invalid_state("managed Google credentials must be an object"))?;
object
.entry("type")
.or_insert_with(|| serde_json::Value::String("service_account".to_string()));
object
.entry("project_id")
.or_insert_with(|| serde_json::Value::String(project));
.or_insert_with(|| serde_json::json!("service_account"));
object.entry("project_id").or_insert_with(|| serde_json::json!(project));
object
.entry("token_uri")
.or_insert_with(|| serde_json::Value::String("https://oauth2.googleapis.com/token".to_string()));
.or_insert_with(|| serde_json::json!("https://oauth2.googleapis.com/token"));
let json = serde_json::to_string(&credentials)
.map_err(|error| RuntimeError::json("serialize managed Vertex credentials failed", error))?;
.map_err(|error| RuntimeError::json("serialize managed Google credentials failed", error))?;
return CustomServiceAccount::from_json(&json)
.map(|provider| Arc::new(provider) as Arc<dyn TokenProvider>)
.map_err(|_| RuntimeError::invalid_state("managed Vertex credential unavailable"));
.map_err(|_| RuntimeError::invalid_state("managed Google credential unavailable"));
}
gcp_auth::provider()
.await
.map_err(|_| RuntimeError::invalid_state("managed Vertex credential unavailable"))
}
pub(in crate::runtime::backend_runtime) fn provider(value: &str) -> RuntimeResult<BackendProvider> {
match value {
"openai" => Ok(BackendProvider::OpenAi),
"anthropic" => Ok(BackendProvider::Anthropic),
"anthropicVertex" => Ok(BackendProvider::AnthropicVertex),
"gemini" => Ok(BackendProvider::Gemini),
"geminiVertex" => Ok(BackendProvider::GeminiVertex),
"cloudflareWorkersAi" => Ok(BackendProvider::CloudflareWorkersAi),
"fal" => Ok(BackendProvider::Fal),
_ => Err(RuntimeError::invalid_state("unsupported copilot provider")),
}
}
fn operation(value: RouteOperation) -> BackendOperation {
match value {
RouteOperation::Chat => BackendOperation::Chat,
RouteOperation::Structured | RouteOperation::Transcription => BackendOperation::Structured,
RouteOperation::Embedding => BackendOperation::Embedding,
RouteOperation::Rerank => BackendOperation::Rerank,
RouteOperation::Image => BackendOperation::Image,
}
}
pub(in crate::runtime::backend_runtime) fn protocol(value: BackendProtocol) -> ExecutableProtocol {
match value {
BackendProtocol::Chat(value) => ExecutableProtocol::Chat(value),
BackendProtocol::Structured(value) => ExecutableProtocol::Structured(value),
BackendProtocol::Embedding(value) => ExecutableProtocol::Embedding(value),
BackendProtocol::Rerank(value) => ExecutableProtocol::Rerank(value),
BackendProtocol::Image(value) => ExecutableProtocol::Image(value),
}
.map_err(|_| RuntimeError::invalid_state("managed Google credential unavailable"))
}
fn project_event(event: RuntimeRouteEvent, identities: &HashMap<String, RouteIdentity>) -> RuntimeResult<ProductEvent> {
@@ -422,15 +412,24 @@ impl From<BackendError> for RuntimeError {
}
}
impl From<llm_adapter::target::TargetCompileError> for RuntimeError {
fn from(error: llm_adapter::target::TargetCompileError) -> Self {
RuntimeError::invalid_state(error.to_string())
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use futures_util::future::BoxFuture;
use serde_json::json;
use super::*;
use crate::llm::route::slot;
#[test]
fn image_dispatch_accepts_napi_request_contract() {
#[tokio::test]
async fn dispatch_keeps_managed_and_byok_targets_and_credentials_bound() {
let (_, request) = request_and_slot(
slot("action.image.filter.pixel").unwrap(),
json!({
@@ -462,5 +461,220 @@ mod tests {
};
assert!(request.is_edit());
assert_eq!(request.images()[0].media_type(), Some("image/png"));
struct FailedClient {
attempts: std::cell::Cell<usize>,
status: u16,
expected_url: &'static str,
}
impl llm_adapter::backend::BackendHttpClient for FailedClient {
fn post_json(
&self,
request: llm_adapter::backend::HttpRequest,
) -> Result<llm_adapter::backend::HttpResponse, BackendError> {
self.attempts.set(self.attempts.get() + 1);
assert_eq!(request.url, self.expected_url);
if self.status == 0 {
Err(BackendError::Timeout {
message: "fixture".into(),
context: None,
})
} else {
Ok(llm_adapter::backend::HttpResponse {
status: self.status,
body: json!({"error":"fixture"}),
})
}
}
fn post_sse(
&self,
_: llm_adapter::backend::HttpRequest,
_: &mut dyn FnMut(&str) -> Result<(), BackendError>,
) -> Result<(), BackendError> {
unreachable!("non-streaming fixture");
}
}
let mut copilot: crate::runtime::CopilotRuntimeConfig = serde_json::from_value(json!({
"providers":{"profiles":[
{"id":"first","type":"openai","config":{"apiKey":"old-key"}},
{"id":"second","type":"openai","config":{"apiKey":"another-key"}}
]}
}))
.unwrap();
copilot.initialize_managed_routes().unwrap();
let target = copilot
.managed_routes
.iter()
.find(|binding| {
binding.target.model_id == "gpt-6-luna" && binding.operation == llm_adapter::target::BackendOperation::Chat
})
.unwrap()
.resolved
.clone();
copilot.providers.profiles.clear();
let mut config = BackendRuntimeConfig {
database_url: "postgresql://unused".into(),
auth: Default::default(),
invite_quota: Default::default(),
private_key: Arc::new(Zeroizing::new(String::new())),
deployment: crate::runtime::Deployment::Cloud,
copilot,
search: Default::default(),
redis: Default::default(),
payment: Default::default(),
};
let (slot, request) = request_and_slot(
slot("chat.default").unwrap(),
json!({
"model":"ignored", "messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]
}),
)
.unwrap();
let targets = AuthorizedRouteTargets::Managed(target);
let credential =
super::super::resolve_managed_credential(&targets, &super::super::ManagedTokenProviderCache::default())
.await
.unwrap();
assert_eq!(credential.as_ref().unwrap().as_str(), "old-key");
let execution = compile_execution(&config, slot, request, &[], &targets, credential.as_ref()).unwrap();
assert_eq!(execution.plan.route_ids().count(), 1);
struct RefreshingTokenProvider(AtomicUsize);
impl TokenProvider for RefreshingTokenProvider {
fn token<'a, 'b, 'c, 'future>(
&'a self,
scopes: &'b [&'c str],
) -> BoxFuture<'future, Result<Arc<gcp_auth::Token>, gcp_auth::Error>>
where
'a: 'future,
'b: 'future,
'c: 'future,
Self: 'future,
{
Box::pin(async move {
assert_eq!(scopes, ["https://www.googleapis.com/auth/cloud-platform"]);
let sequence = self.0.fetch_add(1, Ordering::Relaxed);
Ok(Arc::new(
serde_json::from_value(json!({
"access_token": format!("token-{sequence}"),
"expires_in": 0
}))
.unwrap(),
))
})
}
fn project_id<'a, 'future>(&'a self) -> BoxFuture<'future, Result<Arc<str>, gcp_auth::Error>>
where
'a: 'future,
Self: 'future,
{
Box::pin(async { Ok(Arc::from("fixture-project")) })
}
}
let provider: Arc<dyn TokenProvider> = Arc::new(RefreshingTokenProvider(AtomicUsize::new(0)));
let plan = CredentialPlan::GoogleOAuth {
scopes: vec!["https://www.googleapis.com/auth/cloud-platform".into()],
project: "fixture-project".into(),
credentials: None,
};
for expected in ["token-0", "token-1"] {
assert_eq!(
managed_credential(&plan, &json!({}), Some(Arc::clone(&provider)))
.await
.unwrap(),
expected
);
}
for status in [401, 429, 404, 0] {
let client = FailedClient {
attempts: std::cell::Cell::new(0),
status,
expected_url: "https://api.openai.com/v1/responses",
};
assert!(dispatch_compiled_plan(&client, &execution.plan, |_| {}).is_err());
assert_eq!(client.attempts.get(), 1);
}
use crate::llm::{
byok::{ByokModelDeclaration, SensitiveCredential},
route::{self, ProfileSource},
};
config.private_key = Arc::new(Zeroizing::new("fixture-root-key".into()));
let key = CredentialEnvelopeKey::derive(config.private_key.as_bytes()).unwrap();
let credential_ref = CredentialRef {
encrypted: key
.encrypt(&SensitiveCredential::new("byok-key".as_bytes()), b"fixture")
.unwrap(),
aad: b"fixture".to_vec(),
};
for (provider, slot_id, request, expected_url) in [
(
"openai",
"chat.default",
json!({"model":"ignored", "messages":[]}),
"https://api.openai.com/v1/responses",
),
(
"anthropic",
"chat.default",
json!({"model":"ignored", "messages":[]}),
"https://api.anthropic.com/v1/messages",
),
(
"gemini",
"chat.default",
json!({"model":"ignored", "messages":[]}),
"https://generativelanguage.googleapis.com/v1beta/models/opaque-model:generateContent",
),
(
"fal",
"image.generate",
json!({"model":"ignored", "operation":"generate", "prompt":"fixture"}),
"https://fal.run/fal-ai/opaque-model",
),
] {
let (slot, request) = request_and_slot(route::slot(slot_id).unwrap(), request).unwrap();
let profile = AuthorizedProviderProfile {
profile_id: "fixture-profile".into(),
source: ProfileSource::Server,
provider: provider.into(),
endpoint: ByokEndpoint::ProviderDefault,
egress_policy: llm_adapter::target::EgressPolicy::PublicOnly,
models: vec![ByokModelDeclaration {
model_id: "opaque-model".into(),
enabled: true,
capabilities: vec![llm_adapter::capability::DeclaredModelCapability {
input: slot.requirements.input.clone(),
output: slot.requirements.output.clone(),
features: vec![],
attachment_kinds: vec![],
attachment_sources: vec![],
}],
}],
sort_order: 0,
credential_ref: credential_ref.clone(),
};
let profiles = [profile];
let route::RouteDecision::Ready(targets) = route::decide(route::RoutePolicyInput {
slot: &slot,
deployment: config.deployment,
byok_enabled: true,
access_available: true,
profiles: &profiles,
managed_target: None,
target_override: None,
target_override_managed: false,
}) else {
panic!("default-endpoint BYOK route should resolve");
};
let execution = compile_execution(&config, slot, request, &profiles, &targets, None).unwrap();
let client = FailedClient {
attempts: std::cell::Cell::new(0),
status: 0,
expected_url,
};
assert!(dispatch_compiled_plan(&client, &execution.plan, |_| {}).is_err());
assert_eq!(client.attempts.get(), 1);
assert_eq!(execution.identities.values().next().unwrap().model, "opaque-model");
}
}
}
@@ -1,5 +1,6 @@
mod context;
mod dispatch;
mod options;
mod stream;
use std::{
@@ -8,7 +9,6 @@ use std::{
time::Duration,
};
pub(in crate::runtime::backend_runtime) use dispatch::{protocol as executable_protocol, provider as backend_provider};
use gcp_auth::TokenProvider;
use sha2::{Digest, Sha256};
use tokio::sync::OnceCell;
@@ -18,9 +18,9 @@ use super::{BackendRuntime, RuntimeError, RuntimeResult, to_napi_error};
use crate::{
llm::{
CopilotExecuteInput, CopilotRouteCheckInput,
route::{self, AuthorizedProviderProfile, AuthorizedTargetRef, CredentialRef},
route::{self, AuthorizedRouteTargets, ResolvedManagedTarget},
},
runtime::{BackendRuntimeConfig, CopilotManagedProfileConfig},
runtime::BackendRuntimeConfig,
};
pub(super) type ManagedTokenProviderCache = RwLock<HashMap<String, Arc<OnceCell<Arc<dyn TokenProvider>>>>>;
@@ -30,7 +30,7 @@ struct AuthorizedCopilotRoute {
config: std::sync::Arc<BackendRuntimeConfig>,
slot: route::CatalogSlot,
profiles: Vec<route::AuthorizedProviderProfile>,
candidates: Vec<route::AuthorizedTargetRef>,
targets: AuthorizedRouteTargets,
}
pub(super) struct EmbeddingTarget {
@@ -85,29 +85,29 @@ impl BackgroundEmbeddingProvider {
&self.pool,
&config,
context::ProfileLoadInput {
slot: &slot,
built_in_route_id: None,
workspace_id: Some(workspace_id),
user_id: None,
local_lease_id: None,
access: &access,
managed_target_id: None,
},
)
.await?;
let candidates = match route::decide(route::RoutePolicyInput {
let managed_target = context::load_managed_target(&config.copilot, &slot, None, access.managed_tier, None)?;
let targets = match route::decide(route::RoutePolicyInput {
slot: &slot,
deployment: config.deployment,
byok_enabled: config.copilot.byok.enabled,
access_available: true,
profiles: &profiles,
managed_target: managed_target.as_ref(),
target_override: None,
target_override_managed: false,
}) {
route::RouteDecision::Ready(mut candidates) => {
route::RouteDecision::Ready(AuthorizedRouteTargets::Byok(mut candidates)) => {
candidates.truncate(1);
candidates
AuthorizedRouteTargets::Byok(candidates)
}
route::RouteDecision::Ready(targets) => targets,
route::RouteDecision::Denied(reason) => return Err(RuntimeError::invalid_input(reason_name(reason))),
route::RouteDecision::NoRoute(reason) => return Err(RuntimeError::invalid_state(reason_name(reason))),
};
@@ -115,7 +115,7 @@ impl BackgroundEmbeddingProvider {
config,
slot,
profiles,
candidates,
targets,
})
}
@@ -135,13 +135,7 @@ impl BackgroundEmbeddingProvider {
if target.fingerprint != expected_fingerprint {
return Err(RuntimeError::invalid_state("embedding_space_changed"));
}
let managed_credentials = resolve_managed_credentials(
&authorized.config,
&authorized.profiles,
&authorized.candidates,
&self.managed_token_providers,
)
.await?;
let managed_credential = resolve_managed_credential(&authorized.targets, &self.managed_token_providers).await?;
let request = llm_adapter::router::ExecutableRequest::Embedding(llm_adapter::core::EmbeddingRequest {
model: target.model_id,
inputs,
@@ -151,9 +145,9 @@ impl BackgroundEmbeddingProvider {
let config = authorized.config;
let slot = authorized.slot;
let profiles = authorized.profiles;
let candidates = authorized.candidates;
let targets = authorized.targets;
tokio::task::spawn_blocking(move || {
dispatch::execute_embeddings(config, slot, request, profiles, candidates, managed_credentials)
dispatch::execute_embeddings(config, slot, request, profiles, targets, managed_credential)
})
.await
.map_err(|error| RuntimeError::invalid_state(format!("embedding execution task failed: {error}")))?
@@ -161,33 +155,50 @@ impl BackgroundEmbeddingProvider {
}
fn target_from_route(authorized: &AuthorizedCopilotRoute) -> RuntimeResult<EmbeddingTarget> {
let candidate = authorized
.candidates
.first()
.ok_or_else(|| RuntimeError::invalid_state("embedding_route_unavailable"))?;
let profile = authorized
.profiles
.get(candidate.profile_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid embedding route profile"))?;
let model = profile
.models
.get(candidate.model_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid embedding route model"))?;
let route_source = match profile.source {
route::ProfileSource::Server => "byok",
route::ProfileSource::Managed => "managed",
route::ProfileSource::Local => return Err(RuntimeError::invalid_state("embedding_route_unavailable")),
let (route_source, provider, model_id, endpoint_identity) = match &authorized.targets {
AuthorizedRouteTargets::Managed(target) => (
"managed",
target.provider.as_str().to_string(),
target.target.wire_model().to_string(),
if target.provider.uses_default_endpoint(&target.credential_config) {
"ProviderDefault".to_string()
} else {
format!("Custom({:?})", target.target.endpoint())
},
),
AuthorizedRouteTargets::Byok(candidates) => {
let candidate = candidates
.first()
.ok_or_else(|| RuntimeError::invalid_state("embedding_route_unavailable"))?;
let profile = authorized
.profiles
.get(candidate.profile_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid embedding route profile"))?;
if profile.source == route::ProfileSource::Local {
return Err(RuntimeError::invalid_state("embedding_route_unavailable"));
}
let model = profile
.models
.get(candidate.model_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid embedding route model"))?;
(
"byok",
profile.provider.clone(),
model.model_id.clone(),
match &profile.endpoint {
crate::llm::byok::ByokEndpoint::ProviderDefault => "ProviderDefault".to_string(),
crate::llm::byok::ByokEndpoint::OpenAiCompatible { url, .. } => format!("Custom({url:?})"),
},
)
}
};
let endpoint_fingerprint = hex::encode(Sha256::digest(format!("{:?}", profile.endpoint).as_bytes()));
let identity = format!(
"{route_source}|{}|{endpoint_fingerprint}|{}|1024|cosine|1",
profile.provider, model.model_id
);
let endpoint_fingerprint = hex::encode(Sha256::digest(endpoint_identity.as_bytes()));
let identity = format!("{route_source}|{provider}|{endpoint_fingerprint}|{model_id}|1024|cosine|1");
Ok(EmbeddingTarget {
fingerprint: hex::encode(Sha256::digest(identity.as_bytes())),
route_source,
provider: profile.provider.clone(),
model_id: model.model_id.clone(),
provider,
model_id,
endpoint_fingerprint,
})
}
@@ -222,9 +233,9 @@ impl BackendRuntime {
impl BackendRuntime {
async fn execute_copilot_inner(&self, input: CopilotExecuteInput) -> RuntimeResult<String> {
let (config, slot, request, profiles, candidates, managed_credentials) = self.prepare_copilot(input).await?;
let (config, slot, request, profiles, targets, managed_credential) = self.prepare_copilot(input).await?;
let output = tokio::task::spawn_blocking(move || {
dispatch::execute(config, slot, request, profiles, candidates, managed_credentials)
dispatch::execute(config, slot, request, profiles, targets, managed_credential)
})
.await
.map_err(|error| RuntimeError::invalid_state(format!("copilot execution task failed: {error}")))??;
@@ -263,28 +274,17 @@ impl BackendRuntime {
slot,
)
.await?;
let managed_credentials = self
.resolve_managed_credentials(&authorized.config, &authorized.profiles, &authorized.candidates)
.await?;
let managed_credential = resolve_managed_credential(&authorized.targets, &self.managed_token_providers).await?;
Ok((
authorized.config,
authorized.slot,
request,
authorized.profiles,
authorized.candidates,
managed_credentials,
authorized.targets,
managed_credential,
))
}
async fn resolve_managed_credentials(
&self,
config: &BackendRuntimeConfig,
profiles: &[AuthorizedProviderProfile],
candidates: &[AuthorizedTargetRef],
) -> RuntimeResult<HashMap<String, Zeroizing<String>>> {
resolve_managed_credentials(config, profiles, candidates, &self.managed_token_providers).await
}
async fn authorize_copilot_route(
&self,
input: CopilotRouteCheckInput,
@@ -298,47 +298,35 @@ impl BackendRuntime {
&self.pool().await?,
&config,
context::ProfileLoadInput {
slot: &slot,
built_in_route_id: input.built_in_route_id.as_deref(),
workspace_id: input.workspace_id.as_deref(),
user_id: input.user_id.as_deref(),
local_lease_id: input.local_lease_id.as_deref(),
access: &input.access,
managed_target_id: input.managed_target_id.as_deref(),
},
)
.await?;
if input.managed_target_id.is_some() && input.target_override.is_some() {
return Err(RuntimeError::invalid_input("multiple_target_selections"));
}
let target_override = if let Some(target_id) = input.managed_target_id.as_deref() {
let model_id =
route::managed_selected_target(input.built_in_route_id.as_deref(), target_id, input.access.managed_tier)
.ok_or_else(|| RuntimeError::invalid_input("managed_target_unavailable"))?;
let profile = profiles
.iter()
.find(|profile| {
profile.source == route::ProfileSource::Managed
&& profile.models.iter().any(|model| model.model_id == model_id)
})
.ok_or_else(|| RuntimeError::invalid_state("managed_target_unavailable"))?;
Some(route::TargetOverride {
profile_id: profile.profile_id.clone(),
model_id,
})
} else {
input.target_override.map(|target| route::TargetOverride {
profile_id: target.profile_id,
model_id: target.model_id,
})
};
let candidates = match route::decide(route::RoutePolicyInput {
let managed_target = context::load_managed_target(
&config.copilot,
&slot,
input.built_in_route_id.as_deref(),
input.access.managed_tier,
input.managed_target_id.as_deref(),
)?;
let target_override = input.target_override.map(|target| route::TargetOverride {
profile_id: target.profile_id,
model_id: target.model_id,
});
let targets = match route::decide(route::RoutePolicyInput {
slot: &slot,
deployment: config.deployment,
byok_enabled: config.copilot.byok.enabled,
access_available: input.access.route_allowed
|| route::quota_policy(&slot, input.built_in_route_id.as_deref()) != route::QuotaPolicy::Metered,
profiles: &profiles,
managed_target: managed_target.as_ref(),
target_override: target_override.as_ref(),
target_override_managed: input.managed_target_id.is_some(),
}) {
@@ -354,52 +342,40 @@ impl BackendRuntime {
config,
slot,
profiles,
candidates,
targets,
})
}
}
async fn resolve_managed_credentials(
config: &BackendRuntimeConfig,
profiles: &[AuthorizedProviderProfile],
candidates: &[AuthorizedTargetRef],
async fn resolve_managed_credential(
targets: &AuthorizedRouteTargets,
cache: &ManagedTokenProviderCache,
) -> RuntimeResult<HashMap<String, Zeroizing<String>>> {
let mut credentials = HashMap::new();
for candidate in candidates {
let profile = profiles
.get(candidate.profile_index)
.ok_or_else(|| RuntimeError::invalid_state("invalid authorized route profile"))?;
let CredentialRef::Managed { profile_id } = &profile.credential_ref else {
continue;
};
if credentials.contains_key(profile_id) {
continue;
}
let managed = context::managed_profile(&config.copilot, profile_id)?;
let token_provider = if matches!(managed.provider.as_str(), "geminiVertex" | "anthropicVertex") {
Some(managed_token_provider(managed, cache).await?)
} else {
None
};
credentials.insert(
profile_id.clone(),
Zeroizing::new(dispatch::managed_credential(managed, token_provider).await?),
);
}
Ok(credentials)
) -> RuntimeResult<Option<Zeroizing<String>>> {
let AuthorizedRouteTargets::Managed(target) = targets else {
return Ok(None);
};
let plan = target.credential_plan.as_ref();
let token_provider = if matches!(plan, llm_adapter::backend::CredentialPlan::GoogleOAuth { .. }) {
Some(managed_token_provider(target, cache).await?)
} else {
None
};
dispatch::managed_credential(plan, &target.credential_config, token_provider)
.await
.map(Zeroizing::new)
.map(Some)
}
async fn managed_token_provider(
profile: &CopilotManagedProfileConfig,
target: &ResolvedManagedTarget,
cache: &ManagedTokenProviderCache,
) -> RuntimeResult<Arc<dyn TokenProvider>> {
let config = serde_json::to_vec(&profile.config)
.map_err(|error| RuntimeError::json("serialize managed Vertex profile failed", error))?;
let config = serde_json::to_vec(&target.credential_config)
.map_err(|error| RuntimeError::json("serialize managed Google profile failed", error))?;
let cache_key = format!(
"{}:{}:{}",
profile.id,
profile.provider,
"{}:{:?}:{}",
target.profile_id,
target.provider,
hex::encode(Sha256::digest(config))
);
let cell = {
@@ -409,7 +385,7 @@ async fn managed_token_provider(
Arc::clone(providers.entry(cache_key).or_insert_with(|| Arc::new(OnceCell::new())))
};
cell
.get_or_try_init(|| dispatch::create_vertex_token_provider(profile))
.get_or_try_init(|| dispatch::create_google_token_provider(&target.credential_plan))
.await
.map(Arc::clone)
}
@@ -0,0 +1,47 @@
use super::{BackendRuntime, to_napi_error};
use crate::{
llm::{
BuiltInManagedTargetTier, BuiltInRouteOptions, built_in_managed_operations, built_in_managed_target,
llm_get_built_in_route_options,
},
runtime::{BackendRuntimeConfig, Deployment},
};
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub fn get_copilot_route_options(&self, name: String) -> napi::Result<Option<BuiltInRouteOptions>> {
let config = self.config().map_err(to_napi_error)?;
Ok(route_options(&config, &name))
}
}
pub(super) fn route_options(config: &BackendRuntimeConfig, name: &str) -> Option<BuiltInRouteOptions> {
let mut options = llm_get_built_in_route_options(name.to_string())?;
let operations = built_in_managed_operations(name)?;
options.choices.retain(|choice| {
config.copilot.enabled
&& config.deployment == Deployment::Cloud
&& built_in_managed_target(name, &choice.id, true).is_some_and(|target| {
operations
.iter()
.all(|operation| config.copilot.managed_binding(&target.target, *operation).is_some())
})
});
if !options.standard_default_target_id.as_ref().is_some_and(|id| {
options
.choices
.iter()
.any(|choice| &choice.id == id && choice.minimum_tier == BuiltInManagedTargetTier::Standard)
}) {
options.standard_default_target_id = None;
}
if !options
.premium_default_target_id
.as_ref()
.is_some_and(|id| options.choices.iter().any(|choice| &choice.id == id))
{
options.premium_default_target_id = None;
}
Some(options)
}
@@ -1,5 +1,4 @@
use std::{
collections::HashMap,
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
@@ -28,7 +27,7 @@ use super::{BackendRuntime, COPILOT_REQUEST_TIMEOUT, RuntimeError, dispatch, to_
use crate::{
llm::{
CopilotExecuteInput,
route::{AuthorizedProviderProfile, AuthorizedTargetRef, CatalogSlot},
route::{AuthorizedProviderProfile, AuthorizedRouteTargets, CatalogSlot},
},
runtime::BackendRuntimeConfig,
};
@@ -38,8 +37,8 @@ pub(super) type PreparedCopilotExecution = (
CatalogSlot,
ExecutableRequest,
Vec<AuthorizedProviderProfile>,
Vec<AuthorizedTargetRef>,
HashMap<String, Zeroizing<String>>,
AuthorizedRouteTargets,
Option<Zeroizing<String>>,
);
const STREAM_END: &str = "__AFFINE_COPILOT_STREAM_END__";
@@ -69,7 +68,7 @@ impl BackendRuntime {
callback: ThreadsafeFunction<String, ()>,
tool_callback: ThreadsafeFunction<String, PromiseRaw<'static, String>>,
) -> Result<CopilotStreamHandle> {
let (config, slot, request, profiles, candidates, managed_credentials) =
let (config, slot, request, profiles, targets, managed_credential) =
self.prepare_copilot(input).await.map_err(to_napi_error)?;
let messages = match &request {
ExecutableRequest::Chat(request) => request.messages.clone(),
@@ -80,7 +79,7 @@ impl BackendRuntime {
}
};
let mut execution =
dispatch::compile_execution(&config, slot, request, &profiles, &candidates, &managed_credentials)
dispatch::compile_execution(&config, slot, request, &profiles, &targets, managed_credential.as_ref())
.map_err(to_napi_error)?;
let aborted = Arc::new(AtomicBool::new(false));
let worker_aborted = aborted.clone();
@@ -36,7 +36,6 @@ use std::{
};
use byok::LocalLeasePayload;
use copilot::{backend_provider, executable_protocol};
use embedding::register_artifact_source;
use napi::{Result, Status, bindgen_prelude::Buffer, threadsafe_function::ThreadsafeFunction};
#[cfg(test)]
@@ -4,9 +4,9 @@ use schemars::generate::SchemaSettings;
use serde_json::{Value, from_value, json, to_value};
use super::{
AuthConfigFile, CopilotRuntimeConfig, CopilotRuntimeConfigFile, CryptoConfigFile, MANAGED_PROFILE_REQUIREMENTS,
OAuthProviderConfigFile, PaymentRuntimeConfigFile, RedisRuntimeConfigFile, RuntimeError, SUPPORTED_BYOK_PROVIDERS,
SearchRuntimeConfigFile, auth_limit, insert_flat_override, valid_search_endpoint, validate_copilot_config,
AuthConfigFile, CopilotRuntimeConfig, CopilotRuntimeConfigFile, CryptoConfigFile, OAuthProviderConfigFile,
PaymentRuntimeConfigFile, RedisRuntimeConfigFile, RuntimeError, SUPPORTED_BYOK_PROVIDERS, SearchRuntimeConfigFile,
auth_limit, insert_flat_override, valid_search_endpoint, validate_copilot_config,
};
use crate::runtime::object_storage::{
StorageBackendConfig, default_storage_provider_config, storage_provider_schema, types::StorageProviderConfig,
@@ -90,7 +90,11 @@ fn descriptors() -> Vec<AppConfigDescriptor> {
owners and admins to send provider probe requests to the private network.",
false,
),
("providers.profiles", "The profile list for copilot providers.", true),
(
"providers.profiles",
"Credentials, endpoints and optional model preferences for built-in copilot API routes.",
true,
),
]
.into_iter()
.map(|(key, description, internal)| {
@@ -107,26 +111,7 @@ fn descriptors() -> Vec<AppConfigDescriptor> {
if key == "byok.allowedProviders" {
schema["items"]["enum"] = json!(SUPPORTED_BYOK_PROVIDERS);
}
if key == "providers.profiles" {
let profile = &mut schema["definitions"]["CopilotManagedProfileConfigFile"];
let requirements = MANAGED_PROFILE_REQUIREMENTS
.iter()
.map(|(provider, fields)| {
let properties = fields
.iter()
.map(|field| ((*field).to_string(), json!({ "type": "string", "pattern": "\\S" })))
.collect::<serde_json::Map<_, _>>();
json!({
"if": { "properties": { "type": { "const": provider } }, "required": ["type"] },
"then": { "properties": { "config": { "required": fields, "properties": properties } } }
})
})
.collect::<Vec<_>>();
profile["allOf"] = json!([{
"if": { "properties": { "enabled": { "const": false } }, "required": ["enabled"] },
"else": { "allOf": requirements }
}]);
}
AppConfigDescriptor {
key: key.to_string(),
description: description.to_string(),
@@ -736,24 +721,15 @@ mod tests {
.build(&descriptors[5].schema)
.unwrap();
for (profile, accepted) in [
(json!({"id":"openai","type":"openai","config":{"apiKey":"test"}}), true),
(json!({"id":"openai","type":"openai","config":{}}), false),
(json!({"id":"openai","type":"openai","enabled":false,"config":{}}), true),
(
json!({"id":"openai","type":"openai","models":["gpt-5.6-luna"],"config":{"apiKey":"test"}}),
true,
),
(
json!({"id":"openai","type":"openai","models":["gpt-5.6-luna"],"config":{}}),
json!({"id":"cloudflare","type":"cloudflareWorkersAi","config":{"apiToken":"test"}}),
false,
),
(
json!({"id":"openai","type":"openai","enabled":false,"models":["gpt-5.6-luna"],"config":{}}),
true,
),
(
json!({"id":"cloudflare","type":"cloudflareWorkersAi","models":["@cf/baai/bge-reranker-base"],"config":{"apiToken":"test"}}),
false,
),
(
json!({"id":"vertex","type":"geminiVertex","models":["gemini-3.7-flash"],"config":{"project":"test","location":"global"}}),
json!({"id":"vertex","type":"geminiVertex","config":{"project":"test","location":"global"}}),
true,
),
] {
@@ -789,7 +765,7 @@ mod tests {
"displayName": "OpenAI",
"priority": 1,
"enabled": true,
"models": ["gpt-5.6-luna"],
"models": ["model-only-known-to-a-newer-release"],
"middleware": {
"rust": { "request": ["normalize_messages"] },
"node": { "text": ["citation_footnote"] }
@@ -1,8 +1,13 @@
use llm_adapter::capability::provider_default_capability_upper_bound;
mod routes;
use llm_adapter::{
backend::{BackendProvider, backend_providers_config_schema},
target::{BackendOperation, EgressPolicy},
};
pub(crate) use routes::ManagedRouteBinding;
use serde::Deserialize;
use serde_json::Map;
use super::{RuntimeError, RuntimeResult};
use crate::llm::route::{self, ManagedTarget, ResolvedManagedTarget};
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", default)]
@@ -10,6 +15,8 @@ pub(crate) struct CopilotRuntimeConfig {
pub(crate) enabled: bool,
pub(crate) byok: CopilotByokRuntimeConfig,
pub(crate) providers: CopilotProvidersRuntimeConfig,
#[serde(skip)]
pub(crate) managed_routes: Vec<ManagedRouteBinding>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
@@ -35,16 +42,6 @@ impl Default for CopilotByokRuntimeConfig {
pub(in crate::runtime) const SUPPORTED_BYOK_PROVIDERS: [&str; 4] = ["openai", "anthropic", "gemini", "fal"];
pub(super) const MANAGED_PROFILE_REQUIREMENTS: [(&str, &[&str]); 7] = [
("openai", &["apiKey"]),
("anthropic", &["apiKey"]),
("gemini", &["apiKey"]),
("fal", &["apiKey"]),
("cloudflareWorkersAi", &["apiToken", "accountId"]),
("geminiVertex", &["project", "location"]),
("anthropicVertex", &["project", "location"]),
];
fn default_allowed_providers() -> Vec<String> {
SUPPORTED_BYOK_PROVIDERS.into_iter().map(str::to_string).collect()
}
@@ -60,7 +57,7 @@ pub(crate) struct CopilotProvidersRuntimeConfig {
pub(crate) struct CopilotManagedProfileConfig {
pub(crate) id: String,
#[serde(rename = "type")]
pub(crate) provider: String,
pub(crate) provider: BackendProvider,
#[serde(default = "enabled_by_default")]
pub(crate) enabled: bool,
#[serde(default)]
@@ -88,127 +85,26 @@ pub(in crate::runtime) struct CopilotProvidersRuntimeConfigFile {
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase")]
#[schemars(transform = managed_profile_schema)]
pub(crate) struct CopilotManagedProfileConfigFile {
id: String,
#[serde(rename = "type")]
provider: CopilotManagedProvider,
display_name: Option<String>,
priority: Option<f64>,
provider: BackendProvider,
#[serde(default = "enabled_by_default")]
enabled: bool,
#[serde(default)]
models: Vec<String>,
middleware: Option<CopilotProviderMiddlewareConfigFile>,
config: CopilotManagedProviderSettingsFile,
config: serde_json::Map<String, serde_json::Value>,
}
#[derive(Clone, Default, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase")]
struct CopilotManagedProviderSettingsFile {
#[serde(skip_serializing_if = "Option::is_none")]
api_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
api_token: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
account_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
project: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
location: Option<String>,
#[serde(rename = "baseURL", skip_serializing_if = "Option::is_none")]
base_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
google_auth_options: Option<CopilotGoogleAuthOptionsFile>,
#[serde(flatten)]
additional: Map<String, serde_json::Value>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "camelCase")]
struct CopilotGoogleAuthOptionsFile {
#[serde(skip_serializing_if = "Option::is_none")]
credentials: Option<CopilotGoogleCredentialsFile>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
struct CopilotGoogleCredentialsFile {
client_email: String,
private_key: String,
#[serde(flatten)]
additional: Map<String, serde_json::Value>,
}
#[derive(Clone, Copy, Deserialize, serde::Serialize, schemars::JsonSchema)]
enum CopilotManagedProvider {
#[serde(rename = "anthropic")]
Anthropic,
#[serde(rename = "anthropicVertex")]
AnthropicVertex,
#[serde(rename = "cloudflareWorkersAi")]
CloudflareWorkersAi,
#[serde(rename = "fal")]
Fal,
#[serde(rename = "gemini")]
Gemini,
#[serde(rename = "geminiVertex")]
GeminiVertex,
#[serde(rename = "openai")]
OpenAi,
}
impl CopilotManagedProvider {
fn as_str(self) -> &'static str {
match self {
Self::Anthropic => "anthropic",
Self::AnthropicVertex => "anthropicVertex",
Self::CloudflareWorkersAi => "cloudflareWorkersAi",
Self::Fal => "fal",
Self::Gemini => "gemini",
Self::GeminiVertex => "geminiVertex",
Self::OpenAi => "openai",
}
}
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
struct CopilotProviderMiddlewareConfigFile {
rust: Option<CopilotRustMiddlewareConfigFile>,
node: Option<CopilotNodeMiddlewareConfigFile>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
struct CopilotRustMiddlewareConfigFile {
request: Option<Vec<CopilotRustRequestMiddleware>>,
stream: Option<Vec<CopilotRustStreamMiddleware>>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
struct CopilotNodeMiddlewareConfigFile {
text: Option<Vec<CopilotNodeTextMiddleware>>,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
enum CopilotRustRequestMiddleware {
NormalizeMessages,
ClampMaxTokens,
ToolSchemaRewrite,
OpenaiRequestCompat,
OmitToolChoice,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
enum CopilotRustStreamMiddleware {
StreamEventNormalize,
CitationIndexing,
}
#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
enum CopilotNodeTextMiddleware {
CitationFootnote,
Callout,
ThinkingFormat,
fn managed_profile_schema(schema: &mut schemars::Schema) {
schema.insert(
"allOf".into(),
serde_json::json!([{
"if": { "properties": { "enabled": { "const": false } }, "required": ["enabled"] },
"else": backend_providers_config_schema()
}]),
);
}
impl TryFrom<CopilotRuntimeConfigFile> for CopilotRuntimeConfig {
@@ -218,6 +114,7 @@ impl TryFrom<CopilotRuntimeConfigFile> for CopilotRuntimeConfig {
Ok(Self {
enabled: value.enabled,
byok: value.byok,
managed_routes: Vec::new(),
providers: CopilotProvidersRuntimeConfig {
profiles: value
.providers
@@ -246,11 +143,10 @@ impl TryFrom<CopilotManagedProfileConfigFile> for CopilotManagedProfileConfig {
}
Ok(Self {
id: value.id,
provider: value.provider.as_str().to_string(),
provider: value.provider,
enabled: value.enabled,
models: value.models,
config: serde_json::to_value(value.config)
.map_err(|error| RuntimeError::json("serialize managed copilot config failed", error))?,
config: serde_json::Value::Object(value.config),
})
}
}
@@ -265,76 +161,15 @@ pub(in crate::runtime) fn validate_copilot_config(config: &CopilotRuntimeConfig)
}
}
let mut profile_ids = std::collections::HashSet::new();
let mut managed_models = std::collections::HashMap::new();
for profile in &config.providers.profiles {
if profile.id.trim().is_empty() || !profile_ids.insert(profile.id.as_str()) {
return Err(RuntimeError::invalid_state(
"managed copilot profile ids must be non-empty and unique",
));
}
if profile.provider.trim().is_empty() {
return Err(RuntimeError::invalid_state(
"managed copilot profile provider is required",
));
}
if profile.models.is_empty() {
return Err(RuntimeError::invalid_state(
"managed copilot profile models must be non-empty",
));
}
if profile.enabled {
let required = MANAGED_PROFILE_REQUIREMENTS
.iter()
.find(|(provider, _)| *provider == profile.provider)
.map(|(_, fields)| *fields)
.ok_or_else(|| RuntimeError::invalid_state("unsupported managed copilot provider"))?;
for field in required {
if profile
.config
.get(*field)
.and_then(serde_json::Value::as_str)
.is_none_or(|value| value.trim().is_empty())
{
return Err(RuntimeError::invalid_state(format!(
"managed copilot profile requires {field}"
)));
}
}
if let Some(base_url) = profile.config.get("baseURL").and_then(serde_json::Value::as_str) {
llm_adapter::target::canonicalize_endpoint(base_url)
.map_err(|error| RuntimeError::invalid_state(error.to_string()))?;
}
if let Some(credentials) = profile.config.pointer("/googleAuthOptions/credentials") {
for field in ["client_email", "private_key"] {
if credentials
.get(field)
.and_then(serde_json::Value::as_str)
.is_none_or(|value| value.trim().is_empty())
{
return Err(RuntimeError::invalid_state(format!(
"managed Vertex credentials require {field}"
)));
}
}
}
}
let mut models = std::collections::HashSet::new();
for model in &profile.models {
if model.trim().is_empty() || !models.insert(model.as_str()) {
return Err(RuntimeError::invalid_state(
"managed copilot profile models must be non-empty and unique",
));
}
provider_default_capability_upper_bound(&profile.provider, model)
.ok_or_else(|| RuntimeError::invalid_state("managed copilot profile model is unsupported"))?;
if profile.enabled
&& let Some(existing_profile) = managed_models.insert(model.as_str(), profile.id.as_str())
{
return Err(RuntimeError::invalid_state(format!(
"managed copilot model {model} is assigned to both {existing_profile} and {}",
profile.id
)));
}
llm_adapter::backend::validate_backend_config(profile.provider, &profile.config)
.map_err(|error| RuntimeError::invalid_state(error.to_string()))?;
}
}
Ok(())
@@ -0,0 +1,105 @@
use std::sync::Arc;
use llm_adapter::{core::default_model_registry_variants, target::prepare_backend_target};
use super::{
BackendOperation, CopilotRuntimeConfig, EgressPolicy, ManagedTarget, ResolvedManagedTarget, RuntimeResult, route,
validate_copilot_config,
};
#[derive(Clone)]
pub(crate) struct ManagedRouteBinding {
pub(crate) target: ManagedTarget,
pub(crate) operation: BackendOperation,
pub(crate) resolved: ResolvedManagedTarget,
}
impl CopilotRuntimeConfig {
pub(crate) fn initialize_managed_routes(&mut self) -> RuntimeResult<()> {
validate_copilot_config(self)?;
let profiles = self
.providers
.profiles
.iter()
.filter(|profile| profile.enabled)
.map(|profile| {
Ok((
profile,
profile.provider.endpoint(&profile.config)?,
Arc::new(profile.provider.credential_plan(&profile.config)?),
Arc::new(profile.config.clone()),
))
})
.collect::<RuntimeResult<Vec<_>>>()?;
let variants = default_model_registry_variants();
let mut bindings = Vec::new();
for (target, operation) in route::managed_catalog() {
let mut selected = None;
'shapes: for shape in &target.api_shapes {
for preferred in [true, false] {
for (profile, endpoint, credential_plan, credential_config) in &profiles {
for variant in variants.iter().filter(|variant| {
profile.provider.matches_registry_backend(&variant.backend_kind)
&& variant.matches_model(&target.model_id)
}) {
for binding in variant.operation_routes(operation)? {
let layer = binding.request_layer;
if layer.api_shape() != *shape
|| !profile
.provider
.configured_layer_matches(&profile.config, layer, operation)?
{
continue;
}
let matches = profile
.models
.iter()
.any(|model| variant.matches_model(model) || model == &binding.wire_model);
if matches != preferred {
continue;
}
selected = Some(ResolvedManagedTarget {
profile_id: profile.id.clone(),
provider: profile.provider,
target: prepare_backend_target(
layer,
operation,
binding.wire_model,
endpoint,
Some(30 * 60 * 1000),
EgressPolicy::PublicOnly,
)?,
capabilities: binding.capabilities,
credential_plan: Arc::clone(credential_plan),
credential_config: Arc::clone(credential_config),
});
break 'shapes;
}
}
}
}
}
if let Some(resolved) = selected {
bindings.push(ManagedRouteBinding {
target,
operation,
resolved,
});
}
}
self.managed_routes = bindings;
Ok(())
}
pub(crate) fn managed_binding(
&self,
target: &ManagedTarget,
operation: BackendOperation,
) -> Option<&ResolvedManagedTarget> {
self
.managed_routes
.iter()
.find(|binding| &binding.target == target && binding.operation == operation)
.map(|binding| &binding.resolved)
}
}
@@ -17,10 +17,8 @@ mod store;
mod types;
pub use admin::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value};
use auth::{AuthConfigFile, auth_limit};
pub(crate) use copilot::{CopilotByokRuntimeConfig, CopilotManagedProfileConfig, CopilotRuntimeConfig};
use copilot::{
CopilotRuntimeConfigFile, MANAGED_PROFILE_REQUIREMENTS, SUPPORTED_BYOK_PROVIDERS, validate_copilot_config,
};
pub(crate) use copilot::{CopilotByokRuntimeConfig, CopilotRuntimeConfig};
use copilot::{CopilotRuntimeConfigFile, SUPPORTED_BYOK_PROVIDERS, validate_copilot_config};
pub(crate) use file::ServerConfig;
#[cfg(test)]
use file::expand_module_config_paths;
@@ -139,13 +137,13 @@ impl BackendRuntimeConfig {
.validated()
}
fn validated(self) -> RuntimeResult<Self> {
fn validated(mut self) -> RuntimeResult<Self> {
if self.copilot.enabled && self.copilot.byok.enabled && self.private_key.is_empty() {
return Err(RuntimeError::invalid_state(
"stable crypto.privateKey is required when persistent BYOK is enabled",
));
}
validate_copilot_config(&self.copilot)?;
self.copilot.initialize_managed_routes()?;
self.payment.validate()?;
if self.search.provider != "embedded" && self.search.endpoint.is_empty() {
return Err(RuntimeError::config("remote search provider requires an endpoint"));
@@ -5,7 +5,7 @@ use sqlx::{PgPool, Row};
#[cfg(test)]
use super::{AppConfigFile, deserialize_app_config};
use super::{BackendRuntimeConfig, RuntimeError, RuntimeResult, ServerConfig, merge_config_value};
use super::{BackendRuntimeConfig, Deployment, RuntimeError, RuntimeResult, ServerConfig, merge_config_value};
use crate::runtime::object_storage::{ObjectStorageService, backends_from_flat_overrides};
pub(crate) struct AppConfigChange {
@@ -115,6 +115,17 @@ pub(crate) async fn save_app_config_changes(
actor: Option<&str>,
changes: &[AppConfigChange],
) -> RuntimeResult<Vec<String>> {
// Remove after the old cloud readers have exited; profiles remain
// configurable through deployment files.
if config.deployment() == Deployment::Cloud
&& changes
.iter()
.any(|change| change.key == "copilot.providers.profiles" && change.value.is_some())
{
return Err(RuntimeError::config(
"cloud managed provider profiles must use deployment configuration until older readers have exited",
));
}
let mut transaction = pool
.begin()
.await
@@ -269,6 +280,7 @@ pub(in crate::runtime) fn insert_flat_override(
#[cfg(test)]
mod tests {
use serde_json::Value;
use sqlx::postgres::PgPoolOptions;
use super::{AppConfigChange, ServerConfig, load_app_config_overrides_from_db, save_app_config_changes};
@@ -439,5 +451,102 @@ mod tests {
.unwrap(),
0
);
let profiles_key = "copilot.providers.profiles";
for provider in ["openai", "openrouter", "perplexity"] {
for enabled in [false, true] {
let profiles = serde_json::json!([{
"id":"provider", "type":provider, "enabled":enabled, "models":[], "config":{"apiKey":"test"}
}]);
let error = save_app_config_changes(
&pool,
&config,
None,
None,
&[
node("server.name", Some(serde_json::json!("must-not-write"))),
native(profiles_key, Some(profiles.clone())),
],
)
.await
.unwrap_err();
assert!(error.to_string().contains("older readers"));
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM app_configs")
.fetch_one(&pool)
.await
.unwrap(),
0
);
sqlx::query("INSERT INTO app_configs(id,value) VALUES($1,$2)")
.bind(profiles_key)
.bind(&profiles)
.execute(&pool)
.await
.unwrap();
save_app_config_changes(
&pool,
&config,
None,
None,
&[node("server.name", Some(serde_json::json!("repair-allowed")))],
)
.await
.unwrap();
assert_eq!(
sqlx::query_scalar::<_, Value>("SELECT value FROM app_configs WHERE id=$1")
.bind(profiles_key)
.fetch_one(&pool)
.await
.unwrap(),
profiles
);
save_app_config_changes(
&pool,
&config,
None,
None,
&[native(profiles_key, None), node("server.name", None)],
)
.await
.unwrap();
for deployment in ["cloud", "selfhosted"] {
std::fs::write(
&path,
serde_json::json!({"deployment":{"type":deployment},"copilot":{"enabled":false,"providers":{"profiles":profiles}}}).to_string(),
)
.unwrap();
let mounted = ServerConfig::open(&path, None).unwrap();
save_app_config_changes(
&pool,
&mounted,
None,
None,
&[node("server.name", Some(serde_json::json!("mounted-profile")))],
)
.await
.unwrap();
if deployment == "selfhosted" {
save_app_config_changes(
&pool,
&mounted,
None,
None,
&[native(profiles_key, Some(profiles.clone()))],
)
.await
.unwrap();
}
save_app_config_changes(
&pool,
&mounted,
None,
None,
&[native(profiles_key, None), node("server.name", None)],
)
.await
.unwrap();
}
}
}
}
}
@@ -130,49 +130,51 @@ fn expands_module_config_paths_from_json_files() {
assert_eq!(copilot.providers.profiles.len(), 1);
assert_eq!(copilot.providers.profiles[0].id, "managed-openai");
let missing_models = app_config_from_flat_overrides([(
let without_models = app_config_from_flat_overrides([(
"copilot.providers.profiles",
serde_json::json!([{
"id": "managed-openai",
"type": "openai",
"config": {}
"config": { "apiKey": "test" }
}]),
)]);
assert!(missing_models.is_err());
)])
.unwrap();
let copilot: CopilotRuntimeConfig = without_models.copilot.unwrap().try_into().unwrap();
assert!(validate_copilot_config(&copilot).is_ok());
let app_config = app_config_from_flat_overrides([(
"copilot.providers.profiles",
serde_json::json!([{
"id": "managed-openai",
"type": "openai",
"models": [],
"config": {}
"models": ["model-only-known-to-a-newer-release"],
"config": { "apiKey": "test" }
}]),
)])
.unwrap();
let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap();
assert!(validate_copilot_config(&copilot).is_err());
assert!(validate_copilot_config(&copilot).is_ok());
let app_config = app_config_from_flat_overrides([(
"copilot.providers.profiles",
serde_json::json!([
{
"id": "anthropic-direct",
"type": "anthropic",
"id": "anthropic-first",
"type": "anthropicVertex",
"models": ["claude-sonnet-4-6"],
"config": {}
"config": { "project": "test", "location": "global" }
},
{
"id": "anthropic-vertex",
"type": "anthropicVertex",
"models": ["claude-sonnet-4-6"],
"config": {}
"config": { "project": "test", "location": "global" }
}
]),
)])
.unwrap();
let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap();
assert!(validate_copilot_config(&copilot).is_err());
assert!(validate_copilot_config(&copilot).is_ok());
let directory = tempfile::tempdir().unwrap();
let config_path = directory.path().join("config.json");
+3 -3
View File
@@ -16,9 +16,9 @@ pub(super) use affine_doc_loader::blob_refs::{BlobRefProjectionError, extract_bl
pub(crate) use config::AuthRuntimeConfig;
pub use config::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value};
pub(crate) use config::{
BackendRuntimeConfig, CopilotManagedProfileConfig, CopilotRuntimeConfig, Deployment, InviteQuotaConfig,
OAuthProviderRuntimeConfig, PaymentProductConfig, PaymentRuntimeConfig, RedisRuntimeConfig, RevenueCatRuntimeConfig,
SearchRuntimeConfig, StripeRuntimeConfig,
BackendRuntimeConfig, CopilotRuntimeConfig, Deployment, InviteQuotaConfig, OAuthProviderRuntimeConfig,
PaymentProductConfig, PaymentRuntimeConfig, RedisRuntimeConfig, RevenueCatRuntimeConfig, SearchRuntimeConfig,
StripeRuntimeConfig,
};
pub(crate) use error::{RuntimeError, RuntimeResult, napi_error, to_napi_error};
pub(in crate::runtime) use http::webpki_tls_config;
@@ -53,6 +53,7 @@ import {
import { ActionStreamHost } from '../../plugins/copilot/runtime/hosts/action-stream-host';
import { AttachmentAdmissionHost } from '../../plugins/copilot/runtime/hosts/attachment-admission';
import { ImageResultHost } from '../../plugins/copilot/runtime/hosts/image-result-host';
import { mapNativeSemanticError } from '../../plugins/copilot/runtime/native-errors';
import {
collectAttachmentFootnotes,
collectDocumentFootnotes,
@@ -1860,6 +1861,25 @@ test('controller projects successful streams and preparation failures to SSE eve
),
{ type: 'message', id: 'message-1', data: 'hello' }
);
for (const message of [
'managed_target_unavailable',
'[affine-runtime:invalid_input] target_unavailable',
]) {
orchestrator.streamText = async () => {
throw mapNativeSemanticError(new Error(message));
};
t.like(
await firstValueFrom(
await controller.chatStream(
{ id: 'user-1' } as never,
request,
'session-1',
{}
)
),
{ type: 'error', data: { status: 400, name: 'COPILOT_PROMPT_INVALID' } }
);
}
t.like(
await firstValueFrom(
await controller.actionStream(
@@ -1,6 +1,13 @@
import { serverConfigQuery, ServerFeature } from '@affine/graphql';
import {
getCopilotRouteOptionsQuery,
serverConfigQuery,
ServerFeature,
} from '@affine/graphql';
import { Prisma, PrismaClient } from '@prisma/client';
import { InvalidAppConfigInput } from '../../../base';
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
import { ServerService } from '../../../core/config';
import { app, e2e } from '../test';
e2e('should expose the indexer feature when it is enabled', async t => {
@@ -33,3 +40,99 @@ e2e('should enable local workspace feature by default', async t => {
JSON.stringify(serverConfig, null, 2)
);
});
e2e(
'managed model choices follow the active configuration snapshot',
async t => {
const user = await app.signup();
const service = app.get(ServerService);
const db = app.get(PrismaClient);
const previousProfiles = await app.models.appConfig.get(
'copilot.providers.profiles'
);
const previousEnabled = await app.models.appConfig.get('copilot.enabled');
t.teardown(async () => {
if (previousProfiles) {
await db.appConfig.upsert({
where: { id: 'copilot.providers.profiles' },
create: {
id: 'copilot.providers.profiles',
value: previousProfiles.value ?? Prisma.JsonNull,
},
update: { value: previousProfiles.value ?? Prisma.JsonNull },
});
} else {
await service.updateConfig(user.id, [
{ module: 'copilot', key: 'providers.profiles', clear: true },
]);
}
await service.updateConfig(user.id, [
previousEnabled
? { module: 'copilot', key: 'enabled', value: previousEnabled.value }
: { module: 'copilot', key: 'enabled', clear: true },
]);
});
for (const { profiles, choices, defaultTargetId } of [
{
profiles: [
{
id: 'official',
type: 'openai',
models: ['gpt-5.6-luna'],
config: { apiKey: 'test-key' },
},
],
choices: [
{ id: 'luna', available: true },
{ id: 'terra', available: false },
],
defaultTargetId: 'luna',
},
{
profiles: [
{
id: 'google',
type: 'gemini',
models: ['gemini-3.7-flash'],
config: { apiKey: 'test-key' },
},
],
choices: [{ id: 'gemini', available: false }],
defaultTargetId: null,
},
{ profiles: [], choices: [], defaultTargetId: null },
]) {
const error = await t.throwsAsync(
service.updateConfig(user.id, [
{ module: 'copilot', key: 'providers.profiles', value: profiles },
]),
{ instanceOf: InvalidAppConfigInput }
);
t.regex(error.message, /deployment configuration/);
// Seed a pre-existing override to verify the shared configuration remains readable.
await db.appConfig.upsert({
where: { id: 'copilot.providers.profiles' },
create: { id: 'copilot.providers.profiles', value: profiles },
update: { value: profiles },
});
await service.updateConfig(user.id, [
{ module: 'copilot', key: 'enabled', value: true },
]);
t.deepEqual(
(await app.models.appConfig.get('copilot.providers.profiles'))?.value,
profiles
);
const { currentUser } = await app.gql({
query: getCopilotRouteOptionsQuery,
variables: { promptName: 'Chat With AFFiNE AI' },
});
const options = currentUser?.copilot?.routeOptions;
t.truthy(options);
t.is(options?.defaultTargetId, defaultTargetId);
t.deepEqual(
options?.choices.map(({ id, available }) => ({ id, available })),
choices
);
}
}
);
@@ -54,6 +54,10 @@ export class BackendRuntimeCoreOperations {
return this.runtime.copilotEnabled();
}
getCopilotRouteOptions(promptName: string) {
return this.runtime.getCopilotRouteOptions(promptName);
}
nodeCryptoPrivateKey() {
return this.runtime.nodeCryptoPrivateKey();
}
@@ -18,7 +18,10 @@ import { NODE_SECRET_CONFIG_KEYS } from '../../base/config/secrets';
import { SocketIoRedis } from '../../base/redis';
import { Models } from '../../models';
import { ServerConfigHandle } from '../../native';
import { BackendRuntimeProvider } from '../backend-runtime';
import {
BackendRuntimeError,
BackendRuntimeProvider,
} from '../backend-runtime';
import { ServerFeature } from './types';
function isConfigured(value: unknown): boolean {
@@ -282,15 +285,27 @@ export class ServerService
const nativeKeys = new Set(this.serverConfig.nativeAppConfigKeys());
const keys = updates.map(update => `${update.module}.${update.key}`);
await this.runtime.saveAppConfig(
user,
updates.map((update, index) => ({
key: keys[index],
owner: nativeKeys.has(keys[index]) ? 'native' : 'node',
operation: update.clear ? 'clear' : 'set',
valueJson: update.clear ? undefined : JSON.stringify(update.value),
}))
);
try {
await this.runtime.saveAppConfig(
user,
updates.map((update, index) => ({
key: keys[index],
owner: nativeKeys.has(keys[index]) ? 'native' : 'node',
operation: update.clear ? 'clear' : 'set',
valueJson: update.clear ? undefined : JSON.stringify(update.value),
}))
);
} catch (error) {
if (
error instanceof BackendRuntimeError &&
error.code === 'config_error'
) {
throw new InvalidAppConfigInput({
message: error.message,
});
}
throw error;
}
if (!notify) {
return {};
@@ -20,7 +20,7 @@ interface Context {
models: Models;
}
const test = ava as TestFn<Context>;
const test = ava.serial as TestFn<Context>;
test.before(async t => {
t.context.module = await createTestingModule();
@@ -347,6 +347,37 @@ test('managed provider migration preserves explicit profiles and converts legacy
},
]);
const channelProfile = {
id: 'custom-openai',
type: 'openai',
models: ['model-only-known-to-a-newer-release'],
config: { apiKey: 'custom-key', baseURL: null },
};
await t.context.db.appConfig.update({
where: { id: 'copilot.providers.profiles' },
data: { value: [channelProfile] },
});
await t.context.db.appConfig.create({
data: {
id: 'copilot.providers.openai',
value: { apiKey: 'legacy-key' },
},
});
await ConvergeManagedProviderProfiles1786810000000.up(t.context.db);
const channels = await t.context.db.appConfig.findUniqueOrThrow({
where: { id: 'copilot.providers.profiles' },
});
t.deepEqual(channels.value, [
channelProfile,
{
id: 'openai-default',
type: 'openai',
priority: 7,
models: ['gpt-5.6-luna', 'gpt-5.6-terra', 'gpt-image-1', 'gpt-4o-mini'],
config: { apiKey: 'legacy-key' },
},
]);
await t.context.db.appConfig.update({
where: { id: 'copilot.providers.profiles' },
data: { value: [{ ...profiles[0], enabled: 'true' }] },
@@ -15,9 +15,7 @@ const PROVIDERS = [
] as const;
const PROVIDER_IDS = PROVIDERS.map(provider => `copilot.providers.${provider}`);
const DEFAULT_PROFILE_IDS = new Set(
PROVIDERS.map(provider => `${provider}-default`)
);
// Keep legacy model declarations until 0.27.4 readers leave the shared database.
const PROVIDER_MODELS: Record<(typeof PROVIDERS)[number], string[]> = {
openai: ['gpt-5.6-luna', 'gpt-5.6-terra', 'gpt-image-1', 'gpt-4o-mini'],
cloudflareWorkersAi: ['@cf/baai/bge-reranker-base'],
@@ -65,24 +63,6 @@ export class ConvergeManagedProviderProfiles1786810000000 {
const byId = new Map(rows.map(row => [row.id, row]));
const profileRow = byId.get(PROFILE_KEY);
const profiles = readProfiles(profileRow?.value);
const assignedModels = new Set(
profiles.flatMap(profile =>
isRecord(profile) &&
typeof profile.id === 'string' &&
!DEFAULT_PROFILE_IDS.has(profile.id) &&
profile.enabled !== false &&
Array.isArray(profile.models) &&
serverNativeModule.validateAppConfigValue(
'copilot',
'providers.profiles',
[profile]
).length === 0
? profile.models.filter(
(model): model is string => typeof model === 'string'
)
: []
)
);
let converged = false;
for (const [index, provider] of PROVIDERS.entries()) {
@@ -105,19 +85,6 @@ export class ConvergeManagedProviderProfiles1786810000000 {
existing.models.length > 0
? existing.models
: PROVIDER_MODELS[provider];
const canEnable =
!isRecord(existing) ||
existing.enabled === undefined ||
existing.enabled === true;
const modelsAreValid = configuredModels.every(
model => typeof model === 'string'
);
const availableModels =
canEnable && modelsAreValid
? configuredModels.filter(model => !assignedModels.has(model))
: configuredModels;
const hasConflict = canEnable && availableModels.length === 0;
const models = hasConflict ? configuredModels : availableModels;
const profile: Prisma.JsonObject = {
...(legacy
? {
@@ -130,19 +97,13 @@ export class ConvergeManagedProviderProfiles1786810000000 {
...(isRecord(existing) ? existing : {}),
id,
type: provider,
models,
...(hasConflict ? { enabled: false } : {}),
models: configuredModels,
};
if (profileIndex === -1) {
profiles.push(profile);
} else {
profiles[profileIndex] = profile;
}
if (canEnable && !hasConflict) {
models.forEach(model => {
if (typeof model === 'string') assignedModels.add(model);
});
}
}
if (converged) {
-7
View File
@@ -4,7 +4,6 @@ import serverNativeModule, {
type BuiltInPromptRenderContract,
type BuiltInPromptSessionContract,
type BuiltInPromptSpec,
type BuiltInRouteOptions,
type CanonicalChatRequestContract,
type CanonicalStructuredRequestContract,
type CapabilityAttachmentContract,
@@ -815,12 +814,6 @@ export function llmGetBuiltInPromptSpec(
return nativeLlmModule.llmGetBuiltInPromptSpec(name);
}
export function llmGetBuiltInRouteOptions(
name: string
): BuiltInRouteOptions | null {
return nativeLlmModule.llmGetBuiltInRouteOptions(name);
}
export const llmGetByokCatalog = nativeLlmModule.llmGetByokCatalog;
function stripLlmRequestMiddleware<
@@ -1,7 +1,6 @@
import serverNativeModule from '@affine/server-native';
import { defineNativeModuleConfig, StorageProviderConfig } from '../../base';
import { CopilotProviderType } from './providers/types';
export type ProviderSpecificConfig = Record<string, unknown>;
@@ -26,17 +25,11 @@ export type ProviderMiddlewareConfig = {
node?: { text?: NodeTextMiddleware[] };
};
type CopilotProviderProfileCommon = {
export type CopilotProviderProfile = {
id: string;
displayName?: string;
priority?: number;
type: string;
enabled?: boolean;
models?: string[];
middleware?: ProviderMiddlewareConfig;
};
export type CopilotProviderProfile = CopilotProviderProfileCommon & {
type: CopilotProviderType;
config: ProviderSpecificConfig;
};
@@ -1,7 +1,6 @@
import { AiSessionMessageRole } from '@prisma/client';
import { z } from 'zod';
import { JSONSchema } from '../../../base';
import type {
CapabilityAttachmentContract,
CapabilityModelCapability,
@@ -13,13 +12,6 @@ import {
} from '../runtime/contracts/runtime-event-contract';
import { RetrievalScopeSchema } from '../runtime/contracts/shared';
// Owner map:
// - provider/profile/config schemas in this file are backend host ingress.
// - prompt/message/attachment Zod schemas validate Node host ingress and
// persistence surfaces before values cross into native prompt DTOs.
// - model condition/capability types are native-generated facades.
// - StreamObject is app-facing projection, not runtime event truth.
// ========== provider ==========
export enum CopilotProviderType {
@@ -32,45 +24,6 @@ export enum CopilotProviderType {
OpenAI = 'openai',
}
export const CopilotProviderSchema = z.object({
type: z.nativeEnum(CopilotProviderType),
});
export const VertexSchema: JSONSchema = {
type: 'object',
description: 'The config for the google vertex provider.',
properties: {
location: {
type: 'string',
description: 'The location of the google vertex provider.',
},
project: {
type: 'string',
description: 'The project name of the google vertex provider.',
},
googleAuthOptions: {
type: 'object',
description: 'The google auth options for the google vertex provider.',
properties: {
credentials: {
type: 'object',
description: 'The credentials for the google vertex provider.',
properties: {
client_email: {
type: 'string',
description: 'The client email for the google vertex provider.',
},
private_key: {
type: 'string',
description: 'The private key for the google vertex provider.',
},
},
},
},
},
},
};
// ========== prompt ==========
export const PromptToolsSchema = z
@@ -28,10 +28,10 @@ import {
TooManyRequest,
} from '../../base';
import { CurrentUser } from '../../core/auth';
import { BackendRuntimeProvider } from '../../core/backend-runtime';
import { DocAction } from '../../core/permission';
import { UserType } from '../../core/user';
import type { ListSessionOptions, UpdateChatSession } from '../../models';
import { llmGetBuiltInRouteOptions } from '../../native';
import { CopilotAccessService, type CopilotScopeMode } from './access';
import { ByokEntitlementPolicy } from './byok';
import { CompatHistoryProjector } from './compat/history-projector';
@@ -363,7 +363,8 @@ export class CopilotResolver {
private readonly chatSession: ChatSessionService,
private readonly historyProjector: CompatHistoryProjector,
private readonly inbox: ConversationInboxService,
private readonly entitlement: ByokEntitlementPolicy
private readonly entitlement: ByokEntitlementPolicy,
private readonly backend: BackendRuntimeProvider
) {}
@ResolveField(() => CopilotRouteOptionsType, {
@@ -375,11 +376,8 @@ export class CopilotResolver {
@CurrentUser() user: CurrentUser,
@Args('promptName') promptName: string
): Promise<CopilotRouteOptionsType | null> {
const options = llmGetBuiltInRouteOptions(promptName);
const options = this.backend.getCopilotRouteOptions(promptName);
if (!options) return null;
if (env.selfhosted) {
return { routeId: options.routeId, defaultTargetId: null, choices: [] };
}
const premium = await this.entitlement.hasAiPlan(user.id);
return {
routeId: options.routeId,
@@ -1,4 +1,8 @@
import { CopilotQuotaExceeded, NetworkError } from '../../../base';
import {
CopilotPromptInvalid,
CopilotQuotaExceeded,
NetworkError,
} from '../../../base';
const LLM_TIMEOUT_ERROR_PREFIX = 'llm_timeout:';
@@ -17,7 +21,18 @@ function nativeErrorMessage(error: unknown) {
}
export function mapNativeSemanticError(error: unknown): unknown {
const message = nativeErrorMessage(error);
const message = nativeErrorMessage(error)?.replace(
/^\[affine-runtime:[a-z0-9_]+\]\s*/,
''
);
if (
message === 'managed_target_unavailable' ||
message === 'target_unavailable'
) {
return new CopilotPromptInvalid(
'The selected AI model is unavailable. Choose an available model and try again.'
);
}
if (message === 'access_unavailable') {
return new CopilotQuotaExceeded();
}
@@ -33,22 +33,25 @@ export const test = base.extend<TestUtilsFixtures>({
settings: SettingsPanelUtils,
});
},
loggedInPage: async ({ browser }, use) => {
const context = await browser.newContext();
await skipOnboarding(context);
const page = await context.newPage();
await page.goto('http://localhost:8080/', { timeout: 240 * 1000 });
const user = await createRandomAIUser();
await page.getByTestId('sidebar-user-avatar').click({
delay: 200,
timeout: 20 * 1000,
});
await loginUserDirectly(page, user);
await openHomePage(page);
await enableCloudWorkspace(page);
await use(page);
await context.close();
},
loggedInPage: [
async ({ browser }, use) => {
const context = await browser.newContext();
await skipOnboarding(context);
const page = await context.newPage();
await page.goto('http://localhost:8080/', { timeout: 240 * 1000 });
const user = await createRandomAIUser();
await page.getByTestId('sidebar-user-avatar').click({
delay: 200,
timeout: 20 * 1000,
});
await loginUserDirectly(page, user);
await openHomePage(page);
await enableCloudWorkspace(page);
await use(page);
await context.close();
},
{ timeout: 120_000 },
],
});
export type TestFixtures = typeof test;
@@ -144,8 +144,8 @@ export class ChatPanelUtils {
expect(history).toHaveLength(expected.length);
const assistantStage = {
loading: 1,
transmitting: 1,
success: 2,
transmitting: 2,
success: 3,
} as const;
history.forEach((message, index) => {