-
-
Notifications
You must be signed in to change notification settings - Fork 4.4k
Global virtual models #2818
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: dev
Are you sure you want to change the base?
Global virtual models #2818
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -158,6 +158,9 @@ type Manager struct { | |
| // It is initialized in NewManager; never Load() before first Store(). | ||
| runtimeConfig atomic.Value | ||
|
|
||
| // virtualModels stores global virtual model definitions (name -> upstream model). | ||
| virtualModels atomic.Value | ||
|
|
||
| // Optional HTTP RoundTripper provider injected by host. | ||
| rtProvider RoundTripperProvider | ||
|
|
||
|
|
@@ -1167,6 +1170,8 @@ func (m *Manager) Load(ctx context.Context) error { | |
| // Execute performs a non-streaming execution using the configured selector and executor. | ||
| // It supports multiple providers for the same model and round-robins the starting provider per model. | ||
| func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { | ||
| req = resolveVirtualModelForRequest(m, req) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Useful? React with 👍 / 👎. |
||
|
|
||
| normalized := m.normalizeProviders(providers) | ||
| if len(normalized) == 0 { | ||
| return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"} | ||
|
|
@@ -1198,6 +1203,8 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye | |
| // ExecuteCount performs a non-streaming execution using the configured selector and executor. | ||
| // It supports multiple providers for the same model and round-robins the starting provider per model. | ||
| func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { | ||
| req = resolveVirtualModelForRequest(m, req) | ||
|
|
||
| normalized := m.normalizeProviders(providers) | ||
| if len(normalized) == 0 { | ||
| return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"} | ||
|
|
@@ -1229,6 +1236,8 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip | |
| // ExecuteStream performs a streaming execution using the configured selector and executor. | ||
| // It supports multiple providers for the same model and round-robins the starting provider per model. | ||
| func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) { | ||
| req = resolveVirtualModelForRequest(m, req) | ||
|
|
||
| normalized := m.normalizeProviders(providers) | ||
| if len(normalized) == 0 { | ||
| return nil, &Error{Code: "provider_not_found", Message: "no provider supplied"} | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package auth | ||
|
|
||
| import ( | ||
| "strings" | ||
|
|
||
| cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v6/sdk/cliproxy/executor" | ||
|
|
||
| internalconfig "github.com/router-for-me/CLIProxyAPI/v6/internal/config" | ||
| ) | ||
|
|
||
| // virtualModelTable stores the mapping of virtual model name to upstream model. | ||
| type virtualModelTable struct { | ||
| // name maps virtual model name (lowercase) to upstream model. | ||
| name map[string]string | ||
| } | ||
|
|
||
| // compileVirtualModelTable builds a lookup table from virtual model definitions. | ||
| func compileVirtualModelTable(models []internalconfig.VirtualModel) *virtualModelTable { | ||
| if len(models) == 0 { | ||
| return &virtualModelTable{} | ||
| } | ||
| out := &virtualModelTable{ | ||
| name: make(map[string]string, len(models)), | ||
| } | ||
| for _, vm := range models { | ||
| vname := strings.TrimSpace(vm.Name) | ||
| model := strings.TrimSpace(vm.Model) | ||
| if vname == "" || model == "" { | ||
| continue | ||
| } | ||
| key := strings.ToLower(vname) | ||
| if _, exists := out.name[key]; exists { | ||
| // Skip duplicates (first wins due to sanitization dedup) | ||
| continue | ||
| } | ||
| out.name[key] = model | ||
| } | ||
| if len(out.name) == 0 { | ||
| out.name = nil | ||
| } | ||
| return out | ||
| } | ||
|
|
||
| // SetVirtualModels updates the virtual model table used during request execution. | ||
| // Virtual models are resolved before provider selection and work across all auth types. | ||
| func (m *Manager) SetVirtualModels(models []internalconfig.VirtualModel) { | ||
| if m == nil { | ||
| return | ||
| } | ||
| table := compileVirtualModelTable(models) | ||
| m.virtualModels.Store(table) | ||
| } | ||
|
|
||
| // resolveVirtualModelForRequest resolves the virtual model in a request and returns the | ||
| // potentially modified request. This avoids repeating the same resolve block in every Execute* method. | ||
| func resolveVirtualModelForRequest(m *Manager, req cliproxyexecutor.Request) cliproxyexecutor.Request { | ||
| if virtualModel := m.ResolveVirtualModel(req.Model); virtualModel != "" { | ||
| req.Model = virtualModel | ||
| } | ||
| return req | ||
| } | ||
|
|
||
| // ResolveVirtualModel attempts to resolve a requested model name through the virtual model table. | ||
| // If the model is a virtual model, it returns the upstream model name. | ||
| // If not virtual or resolution fails, it returns an empty string. | ||
| func (m *Manager) ResolveVirtualModel(requestedModel string) string { | ||
| if m == nil || requestedModel == "" { | ||
| return "" | ||
| } | ||
| key := strings.ToLower(strings.TrimSpace(requestedModel)) | ||
| if key == "" { | ||
| return "" | ||
| } | ||
| raw := m.virtualModels.Load() | ||
| if raw == nil { | ||
| return "" | ||
| } | ||
| table, ok := raw.(*virtualModelTable) | ||
| if !ok || table == nil || table.name == nil { | ||
| return "" | ||
| } | ||
| model, exists := table.name[key] | ||
| if !exists { | ||
| return "" | ||
| } | ||
| return model | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This adds virtual models to the Gemini list output, but model-detail lookup still only searches registry-backed models, so a virtual ID returned by listing (for example
models/fast) will 404 onGeminiGetHandler. Clients that validate or hydrate model metadata by GET-after-list will treat advertised virtual models as unavailable.Useful? React with 👍 / 👎.