From 27944011cf84c78b4cbfb0cf371519f7c546e0d3 Mon Sep 17 00:00:00 2001 From: Yiiii0 Date: Sun, 8 Mar 2026 04:29:39 -0400 Subject: [PATCH] feat: Add Forge LLM provider support ## Changes - Automated integration updates from manager loop. Files modified: evaluation/WebArenaLiteV2/core/engine.py evaluation/WebArenaLiteV2/core/mllm.py evaluation/WebArenaLiteV2/envs/web/webarena/evaluation/vab_helper_functions.py playground/core/engine.py playground/core/mllm.py playground/envs/web/ubuntu_web_setup/scripts/util.py --- evaluation/WebArenaLiteV2/core/engine.py | 41 +++++++++++--- evaluation/WebArenaLiteV2/core/mllm.py | 3 +- .../evaluation/vab_helper_functions.py | 20 +++++-- playground/core/engine.py | 54 +++++++++++++++---- playground/core/mllm.py | 3 +- .../envs/web/ubuntu_web_setup/scripts/util.py | 21 ++++++-- 6 files changed, 113 insertions(+), 29 deletions(-) diff --git a/evaluation/WebArenaLiteV2/core/engine.py b/evaluation/WebArenaLiteV2/core/engine.py index e3d015b..1284411 100644 --- a/evaluation/WebArenaLiteV2/core/engine.py +++ b/evaluation/WebArenaLiteV2/core/engine.py @@ -12,6 +12,24 @@ DefaultHttpxClient, ) +FORGE_BASE_URL = "https://api.forge.tensorblock.co/v1" +FORGE_DEFAULT_PROVIDER = "OpenAI" + + +def _resolve_openai_config(base_url=None, api_key=None, model=None): + forge_api_key = os.getenv("FORGE_API_KEY") + if forge_api_key and api_key is None: + api_key = forge_api_key + if base_url is None: + base_url = os.getenv("FORGE_API_BASE") or FORGE_BASE_URL + if model and "/" not in model: + model = f"{FORGE_DEFAULT_PROVIDER}/{model}" + if api_key is None: + api_key = os.getenv("OPENAI_API_KEY") + if base_url is None: + base_url = os.getenv("OPENAI_BASE_URL") + return base_url, api_key, model + class LMMEngine: pass @@ -23,11 +41,12 @@ def __init__( ): assert model is not None, "model must be provided" self.model = model - self.base_url = base_url or os.getenv("OPENAI_BASE_URL") - api_key = api_key or os.getenv("OPENAI_API_KEY") + self.base_url, api_key, self.model = _resolve_openai_config( + base_url=base_url, api_key=api_key, model=self.model + ) if api_key is None: raise ValueError( - "An API Key needs to be provided in either the api_key parameter or as an environment variable named OPENAI_API_KEY" + "An API Key needs to be provided in either the api_key parameter or as an environment variable named OPENAI_API_KEY or FORGE_API_KEY" ) self.api_key = api_key @@ -125,12 +144,17 @@ def __init__( self.model = "text-embedding-3-small" self.cost_per_thousand_tokens = 0.00002 - api_key = api_key or os.getenv("OPENAI_API_KEY") + base_url, api_key, model = _resolve_openai_config( + base_url=None, api_key=api_key, model=self.model + ) if api_key is None: raise ValueError( - "An API Key needs to be provided in either the api_key parameter or as an environment variable named OPENAI_API_KEY" + "An API Key needs to be provided in either the api_key parameter or as an environment variable named OPENAI_API_KEY or FORGE_API_KEY" ) self.api_key = api_key + if model: + self.model = model + self.base_url = base_url self.display_cost = display_cost self.request_interval = 0 if rate_limit == -1 else 60.0 / rate_limit @@ -143,7 +167,12 @@ def __init__( ), ) def get_embeddings(self, text: str) -> np.ndarray: - client = OpenAI(api_key=self.api_key) + base_url, api_key, model = _resolve_openai_config( + base_url=self.base_url, api_key=self.api_key, model=self.model + ) + if model: + self.model = model + client = OpenAI(api_key=api_key, base_url=base_url) response = client.embeddings.create(model=self.model, input=text) if self.display_cost: total_tokens = response.usage.total_tokens diff --git a/evaluation/WebArenaLiteV2/core/mllm.py b/evaluation/WebArenaLiteV2/core/mllm.py index 3cd025f..8545365 100644 --- a/evaluation/WebArenaLiteV2/core/mllm.py +++ b/evaluation/WebArenaLiteV2/core/mllm.py @@ -13,6 +13,7 @@ data_type_map = { "openai": {"image_url": "image_url"}, + "forge": {"image_url": "image_url"}, "anthropic": {"image_url": "image"}, } @@ -22,7 +23,7 @@ def __init__(self, engine_params=None, system_prompt=None, engine=None): if engine is None: if engine_params is not None: engine_type = engine_params.get("engine_type") - if engine_type == "openai": + if engine_type in ("openai", "forge"): self.engine = LMMEngineOpenAI(**engine_params) elif engine_type == "anthropic": self.engine = LMMEngineAnthropic(**engine_params) diff --git a/evaluation/WebArenaLiteV2/envs/web/webarena/evaluation/vab_helper_functions.py b/evaluation/WebArenaLiteV2/envs/web/webarena/evaluation/vab_helper_functions.py index d092937..c978fad 100644 --- a/evaluation/WebArenaLiteV2/envs/web/webarena/evaluation/vab_helper_functions.py +++ b/evaluation/WebArenaLiteV2/envs/web/webarena/evaluation/vab_helper_functions.py @@ -18,6 +18,9 @@ logger = logging.getLogger("logger") +FORGE_BASE_URL = "https://api.forge.tensorblock.co/v1" +FORGE_DEFAULT_PROVIDER = "OpenAI" + def generate_from_openai_chat_completion( messages: list[dict[str, str]], @@ -32,14 +35,21 @@ def generate_from_openai_chat_completion( max_attempt = 5 cur_attempt = 0 - if "OPENAI_API_KEY" not in os.environ: + forge_api_key = os.environ.get("FORGE_API_KEY") + api_key = forge_api_key or os.environ.get("OPENAI_API_KEY") + if api_key is None: raise ValueError( - "OPENAI_API_KEY environment variable must be set when using OpenAI API." + "OPENAI_API_KEY or FORGE_API_KEY environment variable must be set when using OpenAI API." ) - client = OpenAI( - api_key=os.environ["OPENAI_API_KEY"], base_url=os.environ["OPENAI_BASE_URL"] - ) + if forge_api_key: + base_url = os.environ.get("FORGE_API_BASE") or FORGE_BASE_URL + if "/" not in model: + model = f"{FORGE_DEFAULT_PROVIDER}/{model}" + else: + base_url = os.environ.get("OPENAI_BASE_URL") or "https://api.openai.com/v1" + + client = OpenAI(api_key=api_key, base_url=base_url) while cur_attempt < max_attempt: try: response = client.chat.completions.create( diff --git a/playground/core/engine.py b/playground/core/engine.py index c0c9ec2..0f5b4f2 100644 --- a/playground/core/engine.py +++ b/playground/core/engine.py @@ -14,6 +14,25 @@ import httpx +FORGE_BASE_URL = "https://api.forge.tensorblock.co/v1" +FORGE_DEFAULT_PROVIDER = "OpenAI" + + +def _resolve_openai_config(base_url=None, api_key=None, model=None): + forge_api_key = os.getenv("FORGE_API_KEY") + if forge_api_key and api_key is None: + api_key = forge_api_key + if base_url is None: + base_url = os.getenv("FORGE_API_BASE") or FORGE_BASE_URL + if model and "/" not in model: + model = f"{FORGE_DEFAULT_PROVIDER}/{model}" + if api_key is None: + api_key = os.getenv("OPENAI_API_KEY") + if base_url is None: + base_url = os.getenv("OPENAI_BASE_URL") + return base_url, api_key, model + + class LMMEngine: pass @@ -24,11 +43,12 @@ def __init__( ): assert model is not None, "model must be provided" self.model = model - self.base_url = base_url or os.getenv("OPENAI_BASE_URL") - api_key = api_key or os.getenv("OPENAI_API_KEY") + self.base_url, api_key, self.model = _resolve_openai_config( + base_url=base_url, api_key=api_key, model=self.model + ) if api_key is None: raise ValueError( - "An API Key needs to be provided in either the api_key parameter or as an environment variable named OPENAI_API_KEY" + "An API Key needs to be provided in either the api_key parameter or as an environment variable named OPENAI_API_KEY or FORGE_API_KEY" ) self.api_key = api_key @@ -126,12 +146,19 @@ def __init__( self.model = "text-embedding-3-small" self.cost_per_thousand_tokens = 0.00002 - api_key = api_key or os.getenv("OPENAI_API_KEY") + base_url, api_key, model = _resolve_openai_config( + base_url=None, api_key=api_key, model=self.model + ) if api_key is None: raise ValueError( - "An API Key needs to be provided in either the api_key parameter or as an environment variable named OPENAI_API_KEY" + "An API Key needs to be provided in either the api_key parameter or as an environment variable named OPENAI_API_KEY or FORGE_API_KEY" ) self.api_key = api_key + if model: + self.model = model + if base_url is None: + base_url = "https://api.openai.com/v1" + self.base_url = base_url self.display_cost = display_cost self.request_interval = 0 if rate_limit == -1 else 60.0 / rate_limit @@ -144,19 +171,24 @@ def __init__( ), ) def get_embeddings(self, text: str) -> np.ndarray: - self.base_url = os.getenv("OPENAI_BASE_URL") - self.api_key = os.getenv("OPENAI_API_KEY") - if "claudeshop" not in self.base_url: + base_url, api_key, model = _resolve_openai_config( + base_url=self.base_url, api_key=self.api_key, model=self.model + ) + if model: + self.model = model + if base_url is None: + base_url = "https://api.openai.com/v1" + if "claudeshop" not in base_url: client = OpenAI( - api_key=self.api_key, - base_url=self.base_url, + api_key=api_key, + base_url=base_url, http_client=DefaultHttpxClient( proxies=os.getenv("API_PROXY"), transport=httpx.HTTPTransport(local_address="0.0.0.0"), ), ) else: - client = OpenAI(api_key=self.api_key, base_url=self.base_url) + client = OpenAI(api_key=api_key, base_url=base_url) response = client.embeddings.create(model=self.model, input=text) if self.display_cost: total_tokens = response.usage.total_tokens diff --git a/playground/core/mllm.py b/playground/core/mllm.py index 2e3a364..216619a 100644 --- a/playground/core/mllm.py +++ b/playground/core/mllm.py @@ -12,6 +12,7 @@ data_type_map = { "openai": {"image_url": "image_url"}, + "forge": {"image_url": "image_url"}, "anthropic": {"image_url": "image"}, } @@ -21,7 +22,7 @@ def __init__(self, engine_params=None, system_prompt=None, engine=None): if engine is None: if engine_params is not None: engine_type = engine_params.get("engine_type") - if engine_type == "openai": + if engine_type in ("openai", "forge"): self.engine = LMMEngineOpenAI(**engine_params) elif engine_type == "anthropic": self.engine = LMMEngineAnthropic(**engine_params) diff --git a/playground/envs/web/ubuntu_web_setup/scripts/util.py b/playground/envs/web/ubuntu_web_setup/scripts/util.py index fe37e8f..e81c0d4 100644 --- a/playground/envs/web/ubuntu_web_setup/scripts/util.py +++ b/playground/envs/web/ubuntu_web_setup/scripts/util.py @@ -17,6 +17,10 @@ logger = logging.getLogger("logger") +FORGE_BASE_URL = "https://api.forge.tensorblock.co/v1" +FORGE_DEFAULT_PROVIDER = "OpenAI" + + def get_site_comb_from_filepath(file_path: str) -> list[str]: comb = os.path.basename(file_path).rsplit("_", 1)[0].split(".") return comb @@ -401,14 +405,21 @@ def generate_from_openai_chat_completion( max_attempt = 5 cur_attempt = 0 - if "OPENAI_API_KEY" not in os.environ: + forge_api_key = os.environ.get("FORGE_API_KEY") + api_key = forge_api_key or os.environ.get("OPENAI_API_KEY") + if api_key is None: raise ValueError( - "OPENAI_API_KEY environment variable must be set when using OpenAI API." + "OPENAI_API_KEY or FORGE_API_KEY environment variable must be set when using OpenAI API." ) - client = OpenAI( - api_key=os.environ["OPENAI_API_KEY"], base_url=os.environ["OPENAI_BASE_URL"] - ) + if forge_api_key: + base_url = os.environ.get("FORGE_API_BASE") or FORGE_BASE_URL + if "/" not in model: + model = f"{FORGE_DEFAULT_PROVIDER}/{model}" + else: + base_url = os.environ.get("OPENAI_BASE_URL") or "https://api.openai.com/v1" + + client = OpenAI(api_key=api_key, base_url=base_url) while cur_attempt < max_attempt: try: response = client.chat.completions.create(