220 lines
7.4 KiB
Python
220 lines
7.4 KiB
Python
from copy import deepcopy
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from app.services import llm_model_catalog as discovery
|
|
from app.services import llm_provider_catalog as catalog
|
|
|
|
REAL_CLIENT = httpx.AsyncClient
|
|
|
|
|
|
def mock_http(monkeypatch, handler):
|
|
transport = httpx.MockTransport(handler)
|
|
monkeypatch.setattr(
|
|
discovery.httpx, "AsyncClient", lambda **kw: REAL_CLIENT(transport=transport, **kw)
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"provider,path,auth",
|
|
[
|
|
("minimax", "/anthropic/v1/models", "x-api-key"),
|
|
("anthropic", "/v1/models", "x-api-key"),
|
|
("openai", "/v1/models", "authorization"),
|
|
("deepseek", "/v1/models", "authorization"),
|
|
("alibaba", "/api/v1/models", "authorization"),
|
|
("moonshotai", "/v1/models", "authorization"),
|
|
("openrouter", "/api/v1/models", "authorization"),
|
|
("opencode-go", "/zen/go/v1/models", "authorization"),
|
|
("ollama", "/api/tags", "authorization"),
|
|
],
|
|
)
|
|
async def test_official_catalog_requests(monkeypatch, provider, path, auth):
|
|
preset = catalog.get_fallback_llm_provider_preset(provider)
|
|
|
|
def handle(request):
|
|
assert request.url.path == path
|
|
assert request.url.host == httpx.URL(preset["base_url"]).host
|
|
assert request.headers[auth] == ("test-key" if auth == "x-api-key" else "Bearer test-key")
|
|
if provider == "alibaba":
|
|
assert request.url.params["capabilities"] == "TG"
|
|
return httpx.Response(
|
|
200, json={"output": {"total": 1, "models": [{"model": "latest"}]}}
|
|
)
|
|
if provider == "ollama":
|
|
return httpx.Response(200, json={"models": [{"name": "latest:7b"}]})
|
|
return httpx.Response(200, json={"data": [{"id": "latest"}]})
|
|
|
|
mock_http(monkeypatch, handle)
|
|
result = await catalog.refresh_llm_provider_preset(provider, api_key="test-key")
|
|
assert result["models"] == (["latest:7b"] if provider == "ollama" else ["latest"])
|
|
assert result["base_url"] == preset["base_url"]
|
|
assert "test-key" not in str(result)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"provider,base,api,path",
|
|
[
|
|
(
|
|
"minimax",
|
|
"https://api.minimax.io/anthropic/v1/",
|
|
"anthropic-messages",
|
|
"/anthropic/v1/models",
|
|
),
|
|
("anthropic", "https://api.anthropic.com", "anthropic-messages", "/v1/models"),
|
|
(
|
|
"alibaba",
|
|
"https://workspace.cn-beijing.maas.aliyuncs.com/compatible-mode/v1",
|
|
"openai-completions",
|
|
"/api/v1/models",
|
|
),
|
|
(
|
|
"alibaba",
|
|
"https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
|
|
"openai-completions",
|
|
"/api/v1/models",
|
|
),
|
|
("alibaba", "https://custom.test/gateway/v1", "openai-completions", "/gateway/v1/models"),
|
|
("moonshotai", "https://api.moonshot.cn/v1", "openai-completions", "/v1/models"),
|
|
("ollama", "http://localhost:11434/api", "ollama-generate", "/api/tags"),
|
|
],
|
|
)
|
|
def test_urls_preserve_region_and_gateway(provider, base, api, path):
|
|
url = httpx.URL(discovery.model_catalog_url(provider, base, api))
|
|
assert url.host == httpx.URL(base).host
|
|
assert url.path == path
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_pagination_and_release_order(monkeypatch):
|
|
seen = []
|
|
|
|
def handle(request):
|
|
seen.append(request.url.params.get("after_id"))
|
|
if len(seen) == 1:
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"data": [{"id": "old", "created_at": "2025-01-01T00:00:00Z"}],
|
|
"has_more": True,
|
|
"last_id": "old",
|
|
},
|
|
)
|
|
return httpx.Response(
|
|
200,
|
|
json={"data": [{"id": "new", "created_at": "2026-06-01T00:00:00Z"}], "has_more": False},
|
|
)
|
|
|
|
mock_http(monkeypatch, handle)
|
|
result = await catalog.refresh_llm_provider_preset("anthropic", "test-key")
|
|
assert seen == [None, "old"]
|
|
assert result["models"] == ["new", "old"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dashscope_pagination(monkeypatch):
|
|
def handle(request):
|
|
page = int(request.url.params["page_no"])
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"output": {
|
|
"total": 2,
|
|
"models": [
|
|
{"model": f"page-{page}", "published_time": f"2026-06-0{page} 00:00:00"}
|
|
],
|
|
}
|
|
},
|
|
)
|
|
|
|
mock_http(monkeypatch, handle)
|
|
result = await catalog.refresh_llm_provider_preset("alibaba", "test-key")
|
|
assert result["models"] == ["page-2", "page-1"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"payload",
|
|
[
|
|
{},
|
|
{"data": []},
|
|
{"data": [None]},
|
|
{"data": [{}]},
|
|
{"data": [{"id": "same"}], "has_more": True, "last_id": "same"},
|
|
],
|
|
)
|
|
async def test_invalid_or_incomplete_catalog_fails(monkeypatch, payload):
|
|
mock_http(monkeypatch, lambda request: httpx.Response(200, json=payload))
|
|
with pytest.raises(discovery.LLMProviderCatalogError):
|
|
await catalog.refresh_llm_provider_preset("minimax", "test-key")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_models_retained_and_defaults_unchanged(monkeypatch):
|
|
before = deepcopy(catalog.FALLBACK_LLM_PROVIDER_PRESETS)
|
|
rows = [{"id": f"model-{i}", "created": i} for i in range(140)]
|
|
mock_http(monkeypatch, lambda request: httpx.Response(200, json={"data": rows}))
|
|
result = await catalog.refresh_llm_provider_preset("openai", "test-key")
|
|
assert len(result["models"]) == 140
|
|
assert result["models"][0] == "model-139"
|
|
assert catalog.FALLBACK_LLM_PROVIDER_PRESETS == before
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_ollama_catalog_is_valid(monkeypatch):
|
|
mock_http(monkeypatch, lambda request: httpx.Response(200, json={"models": []}))
|
|
result = await catalog.refresh_llm_provider_preset("ollama")
|
|
assert result["models"] == []
|
|
assert result["model"] == ""
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_opencode_documented_protocols(monkeypatch):
|
|
models = [
|
|
"minimax-m3",
|
|
"qwen3.8-max",
|
|
"gpt-5.6-luna",
|
|
"grok-4.6",
|
|
"muse-spark-1.3-contributor",
|
|
"kimi-k3",
|
|
]
|
|
mock_http(
|
|
monkeypatch,
|
|
lambda request: httpx.Response(200, json={"data": [{"id": model} for model in models]}),
|
|
)
|
|
result = await catalog.refresh_llm_provider_preset("opencode-go")
|
|
assert list(result["model_provider_apis"].values()) == [
|
|
"anthropic-messages",
|
|
"anthropic-messages",
|
|
"openai-responses",
|
|
"openai-responses",
|
|
"openai-responses",
|
|
"openai-completions",
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_transient_error_only(monkeypatch):
|
|
calls = []
|
|
|
|
def handle(request):
|
|
calls.append(request)
|
|
return httpx.Response(503 if len(calls) == 1 else 200, json={"data": [{"id": "latest"}]})
|
|
|
|
mock_http(monkeypatch, handle)
|
|
await catalog.refresh_llm_provider_preset("minimax", "test-key")
|
|
assert len(calls) == 2
|
|
calls.clear()
|
|
|
|
def unauthorized(request):
|
|
calls.append(request)
|
|
return httpx.Response(401, text="secret-upstream-text")
|
|
|
|
mock_http(monkeypatch, unauthorized)
|
|
with pytest.raises(httpx.HTTPStatusError) as error:
|
|
await catalog.refresh_llm_provider_preset("minimax", "test-key")
|
|
assert len(calls) == 1
|
|
assert "secret-upstream-text" not in discovery.catalog_error_message(error.value)
|