From 6689f18331a9c241d7d084b33fa7c4afcc9bedc7 Mon Sep 17 00:00:00 2001 From: kenneth Date: Sun, 12 Apr 2026 11:12:53 +0000 Subject: [PATCH] feat: migrate LLM provider to Cloudflare Workers AI Replace OpenRouter SDK with direct Cloudflare Workers AI REST API calls. Same model (glm-4.7-flash), validated with arktype. Env vars: CF_ACCOUNT_ID, WORKERS_AI_API_KEY. Co-authored-by: Ona --- apps/aelis-backend/.env.example | 9 +- apps/aelis-backend/package.json | 1 - .../src/enhancement/llm-client.ts | 117 ++++++++++++------ apps/aelis-backend/src/enhancement/schema.ts | 3 +- apps/aelis-backend/src/server.ts | 25 ++-- bun.lock | 13 +- package.json | 2 +- 7 files changed, 105 insertions(+), 65 deletions(-) diff --git a/apps/aelis-backend/.env.example b/apps/aelis-backend/.env.example index ffd672e..4454808 100644 --- a/apps/aelis-backend/.env.example +++ b/apps/aelis-backend/.env.example @@ -10,10 +10,11 @@ CREDENTIAL_ENCRYPTION_KEY= # Base URL of the backend BETTER_AUTH_URL=http://localhost:3000 -# OpenRouter (LLM feed enhancement) -OPENROUTER_API_KEY= -# Optional: override the default model (default: openai/gpt-4.1-mini) -# OPENROUTER_MODEL=openai/gpt-4.1-mini +# Cloudflare Workers AI (LLM feed enhancement) +CF_ACCOUNT_ID= +WORKERS_AI_API_KEY= +# Optional: override the default model (default: @cf/zai-org/glm-4.7-flash) +# WORKERS_AI_MODEL=@cf/zai-org/glm-4.7-flash # Apple WeatherKit credentials WEATHERKIT_PRIVATE_KEY= diff --git a/apps/aelis-backend/package.json b/apps/aelis-backend/package.json index 59a6d18..e62edc6 100644 --- a/apps/aelis-backend/package.json +++ b/apps/aelis-backend/package.json @@ -21,7 +21,6 @@ "@aelis/source-location": "workspace:*", "@aelis/source-tfl": "workspace:*", "@aelis/source-weatherkit": "workspace:*", - "@openrouter/sdk": "^0.9.11", "arktype": "^2.1.29", "better-auth": "^1", "drizzle-orm": "^0.45.1", diff --git a/apps/aelis-backend/src/enhancement/llm-client.ts b/apps/aelis-backend/src/enhancement/llm-client.ts index 1716d95..592fc1e 100644 --- a/apps/aelis-backend/src/enhancement/llm-client.ts +++ b/apps/aelis-backend/src/enhancement/llm-client.ts @@ -1,13 +1,14 @@ -import { OpenRouter } from "@openrouter/sdk" +import { type } from "arktype" import type { EnhancementResult } from "./schema.ts" import { enhancementResultJsonSchema, parseEnhancementResult } from "./schema.ts" -const DEFAULT_MODEL = "z-ai/glm-4.7-flash" +const DEFAULT_MODEL = "@cf/zai-org/glm-4.7-flash" const DEFAULT_TIMEOUT_MS = 30_000 export interface LlmClientConfig { + accountId: string apiKey: string model?: string timeoutMs?: number @@ -22,52 +23,94 @@ export interface LlmClient { enhance(request: LlmClientRequest): Promise } +const CloudflareApiResponse = type({ + result: { + choices: type({ + message: { + content: "string", + "role?": "string", + }, + }).array(), + }, + success: "boolean", + "errors?": type({ message: "string" }).array(), +}) + /** - * Creates a reusable LLM client backed by OpenRouter. - * The OpenRouter SDK instance is created once and reused across calls. + * Creates a reusable LLM client backed by Cloudflare Workers AI. + * Uses the REST API with structured JSON output. */ export function createLlmClient(config: LlmClientConfig): LlmClient { - const client = new OpenRouter({ - apiKey: config.apiKey, - timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS, - }) const model = config.model ?? DEFAULT_MODEL + const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS + const baseUrl = `https://api.cloudflare.com/client/v4/accounts/${config.accountId}/ai/run/${model}` return { async enhance(request) { - const response = await client.chat.send({ - chatGenerationParams: { - model, - messages: [ - { role: "system" as const, content: request.systemPrompt }, - { role: "user" as const, content: request.userMessage }, - ], - responseFormat: { - type: "json_schema" as const, - jsonSchema: { - name: "enhancement_result", - strict: false, - schema: enhancementResultJsonSchema, - }, + try { + const res = await fetch(baseUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${config.apiKey}`, + "Content-Type": "application/json", }, - reasoning: { effort: "none" }, - stream: false, - }, - }) + body: JSON.stringify({ + messages: [ + { role: "system", content: request.systemPrompt }, + { role: "user", content: request.userMessage }, + ], + response_format: { + type: "json_schema", + json_schema: { + name: "enhancement_result", + strict: false, + schema: enhancementResultJsonSchema, + }, + }, + stream: false, + }), + // @ts-expect-error — bun-types AbortSignal conflicts with ESNext lib in tsc; works at runtime and in VSCode + signal: AbortSignal.timeout(timeoutMs), + }) - const message = response.choices?.[0]?.message - const content = message?.content ?? message?.reasoning - if (typeof content !== "string") { - console.warn("[enhancement] LLM returned no content in response") + if (!res.ok) { + const body = await res.text() + console.warn(`[enhancement] Cloudflare API error ${res.status}: ${body}`) + return null + } + + const json: unknown = await res.json() + const parsed = CloudflareApiResponse(json) + if (parsed instanceof type.errors) { + console.warn("[enhancement] Unexpected API response shape:", parsed.summary) + return null + } + + if (!parsed.success) { + console.warn("[enhancement] Cloudflare API errors:", parsed.errors) + return null + } + + const content = parsed.result.choices[0]?.message.content + if (content === undefined) { + console.warn("[enhancement] LLM returned no choices in response") + return null + } + + const result = parseEnhancementResult(content) + if (!result) { + console.warn("[enhancement] Failed to parse LLM response:", content) + } + + return result + } catch (error) { + if (error instanceof DOMException && error.name === "TimeoutError") { + console.warn("[enhancement] LLM request timed out") + } else { + console.warn("[enhancement] LLM request failed:", error) + } return null } - - const result = parseEnhancementResult(content) - if (!result) { - console.warn("[enhancement] Failed to parse LLM response:", content) - } - - return result }, } } diff --git a/apps/aelis-backend/src/enhancement/schema.ts b/apps/aelis-backend/src/enhancement/schema.ts index 10d5f61..40a6939 100644 --- a/apps/aelis-backend/src/enhancement/schema.ts +++ b/apps/aelis-backend/src/enhancement/schema.ts @@ -15,8 +15,7 @@ export type SyntheticItem = typeof SyntheticItem.infer export type EnhancementResult = typeof EnhancementResult.infer /** - * JSON Schema passed to OpenRouter's structured output. - * OpenRouter doesn't support arktype, so this is maintained separately. + * JSON Schema passed to Cloudflare Workers AI for structured output. * * ⚠️ Must stay in sync with EnhancementResult above. * If you add/remove fields, update both schemas. diff --git a/apps/aelis-backend/src/server.ts b/apps/aelis-backend/src/server.ts index 5b6de23..f1c7113 100644 --- a/apps/aelis-backend/src/server.ts +++ b/apps/aelis-backend/src/server.ts @@ -23,17 +23,22 @@ function main() { const { db, close: closeDb } = createDatabase(process.env.DATABASE_URL!) const auth = createAuth(db) - const openrouterApiKey = process.env.OPENROUTER_API_KEY - const feedEnhancer = openrouterApiKey - ? createFeedEnhancer({ - client: createLlmClient({ - apiKey: openrouterApiKey, - model: process.env.OPENROUTER_MODEL || undefined, - }), - }) - : null + const cfAccountId = process.env.CF_ACCOUNT_ID + const workersAiApiKey = process.env.WORKERS_AI_API_KEY + const feedEnhancer = + cfAccountId && workersAiApiKey + ? createFeedEnhancer({ + client: createLlmClient({ + accountId: cfAccountId, + apiKey: workersAiApiKey, + model: process.env.WORKERS_AI_MODEL || undefined, + }), + }) + : null if (!feedEnhancer) { - console.warn("[enhancement] OPENROUTER_API_KEY not set — feed enhancement disabled") + console.warn( + "[enhancement] CF_ACCOUNT_ID/WORKERS_AI_API_KEY not set — feed enhancement disabled", + ) } const credentialEncryptionKey = process.env.CREDENTIAL_ENCRYPTION_KEY diff --git a/bun.lock b/bun.lock index add849b..61f65a7 100644 --- a/bun.lock +++ b/bun.lock @@ -7,7 +7,7 @@ "devDependencies": { "@json-render/core": "^0.12.1", "@nym.sh/jrx": "^0.2.0", - "@types/bun": "latest", + "@types/bun": "^1.3.12", "oxfmt": "^0.24.0", "oxlint": "^1.39.0", }, @@ -55,7 +55,6 @@ "@aelis/source-location": "workspace:*", "@aelis/source-tfl": "workspace:*", "@aelis/source-weatherkit": "workspace:*", - "@openrouter/sdk": "^0.9.11", "arktype": "^2.1.29", "better-auth": "^1", "drizzle-orm": "^0.45.1", @@ -768,8 +767,6 @@ "@open-draft/until": ["@open-draft/until@2.1.0", "", {}, "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg=="], - "@openrouter/sdk": ["@openrouter/sdk@0.9.11", "", { "dependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-BgFu6NcIJO4a9aVjr04y3kZ8pyM71j15I+bzfVAGEvxnj+KQNIkBYQGgwrG3D+aT1QpDKLki8btcQmpaxUas6A=="], - "@oxfmt/darwin-arm64": ["@oxfmt/darwin-arm64@0.24.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-aYXuGf/yq8nsyEcHindGhiz9I+GEqLkVq8CfPbd+6VE259CpPEH+CaGHEO1j6vIOmNr8KHRq+IAjeRO2uJpb8A=="], "@oxfmt/darwin-x64": ["@oxfmt/darwin-x64@0.24.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-vs3b8Bs53hbiNvcNeBilzE/+IhDTWKjSBB3v/ztr664nZk65j0xr+5IHMBNz3CFppmX7o/aBta2PxY+t+4KoPg=="], @@ -1372,7 +1369,7 @@ "@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="], - "@types/bun": ["@types/bun@1.3.10", "", { "dependencies": { "bun-types": "1.3.10" } }, "sha512-0+rlrUrOrTSskibryHbvQkDOWRJwJZqZlxrUs1u4oOoTln8+WIXBPmAuCF35SWB2z4Zl3E84Nl/D0P7803nigQ=="], + "@types/bun": ["@types/bun@1.3.12", "", { "dependencies": { "bun-types": "1.3.12" } }, "sha512-DBv81elK+/VSwXHDlnH3Qduw+KxkTIWi7TXkAeh24zpi5l0B2kUg9Ga3tb4nJaPcOFswflgi/yAvMVBPrxMB+A=="], "@types/bunyan": ["@types/bunyan@1.8.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-758fRH7umIMk5qt5ELmRMff4mLDlN+xyYzC+dkPTdKwbSkJFvz6xwyScrytPU0QIBbRRwbiE8/BIg8bpajerNQ=="], @@ -1664,7 +1661,7 @@ "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], - "bun-types": ["bun-types@1.3.10", "", { "dependencies": { "@types/node": "*" } }, "sha512-tcpfCCl6XWo6nCVnpcVrxQ+9AYN1iqMIzgrSKYMB/fjLtV2eyAVEg7AxQJuCq/26R6HpKWykQXuSOq/21RYcbg=="], + "bun-types": ["bun-types@1.3.12", "", { "dependencies": { "@types/node": "*" } }, "sha512-HqOLj5PoFajAQciOMRiIZGNoKxDJSr6qigAttOX40vJuSp6DN/CxWp9s3C1Xwm4oH7ybueITwiaOcWXoYVoRkA=="], "bundle-name": ["bundle-name@4.1.0", "", { "dependencies": { "run-applescript": "^7.0.0" } }, "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q=="], @@ -3932,8 +3929,6 @@ "body-parser/raw-body": ["raw-body@2.5.3", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "unpipe": "~1.0.0" } }, "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA=="], - "bun-types/@types/node": ["@types/node@22.19.15", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg=="], - "c12/dotenv": ["dotenv@16.6.1", "", {}, "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow=="], "c12/jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="], @@ -4554,8 +4549,6 @@ "body-parser/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], - "bun-types/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], - "chrome-launcher/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], "chromium-edge-launcher/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], diff --git a/package.json b/package.json index c509849..c5c94ce 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "devDependencies": { "@json-render/core": "^0.12.1", "@nym.sh/jrx": "^0.2.0", - "@types/bun": "latest", + "@types/bun": "^1.3.12", "oxfmt": "^0.24.0", "oxlint": "^1.39.0" },