28 lines
1.2 KiB
JavaScript
28 lines
1.2 KiB
JavaScript
// Use the same undici package for Agent + fetch. Global fetch on Node 18 is a
|
|
// different undici build, so a custom Agent only applies when fetch comes from
|
|
// this module too. undici is pinned to v6 for the node:18 Docker image.
|
|
const { Agent, fetch: undiciFetch } = require('undici');
|
|
|
|
/**
|
|
* The dispatcher every LLM call must use.
|
|
*
|
|
* Node's global `fetch` is undici, whose `headersTimeout` defaults to 300 s.
|
|
* Ollama is called with `stream: false`, so it sends NO response headers until
|
|
* the whole answer is generated — an end-cycle report takes longer than that.
|
|
* The connection was therefore being torn down at almost exactly 301 s with a
|
|
* bare `TypeError: fetch failed`, and every such run was quietly demoted to the
|
|
* local fallback generator.
|
|
*
|
|
* The `AbortSignal.timeout(...)` on those calls was never the effective limit:
|
|
* undici gave up first, so raising the signal to 20 minutes changed nothing.
|
|
* Both numbers have to be long for either to matter.
|
|
*/
|
|
const LLM_REQUEST_TIMEOUT_MS = 1_800_000; // 30 minutes
|
|
|
|
const llmDispatcher = new Agent({
|
|
headersTimeout: LLM_REQUEST_TIMEOUT_MS,
|
|
bodyTimeout: LLM_REQUEST_TIMEOUT_MS,
|
|
});
|
|
|
|
module.exports = { llmDispatcher, llmFetch: undiciFetch, LLM_REQUEST_TIMEOUT_MS };
|