diff --git a/app.js b/app.js index a59f16e..a8eca55 100644 --- a/app.js +++ b/app.js @@ -1,10 +1,14 @@ -const express = require('express'); +const express = require('express'); +const TurndownService = require('turndown'); +const { parse: parseHtml } = require('node-html-parser'); const app = express(); -const OLLAMA_URL = process.env.OLLAMA_URL || 'https://ollama.aquantico.de/api/chat'; -const OLLAMA_MODEL = process.env.OLLAMA_MODEL || 'qwen3.6:35b-a3b-q4_K_M'; -const OLLAMA_AUTH = process.env.OLLAMA_AUTH || '324GF44-50AA-4B57-9386-K435DLJ764DFR'; -const PORT = parseInt(process.env.PORT || '11435', 10); +const OLLAMA_URL = process.env.OLLAMA_URL || 'https://ollama.aquantico.de/api/chat'; +const OLLAMA_MODEL = process.env.OLLAMA_MODEL || 'qwen3.6:35b-a3b-q4_K_M'; +const OLLAMA_AUTH = process.env.OLLAMA_AUTH || '324GF44-50AA-4B57-9386-K435DLJ764DFR'; +const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || 'AIzaSyChzsz8ZN8iHRqMUVFnxJXwyXWP_XwWy6g'; +const GOOGLE_CX = process.env.GOOGLE_CX || '2331819c76d874bcc'; +const PORT = parseInt(process.env.PORT || '11435', 10); const colors = { reset: '\x1b[0m', @@ -43,12 +47,11 @@ app.get('/', (req, res) => { table{width:100%;border-collapse:collapse;margin:.5rem 0 1rem} td,th{border:1px solid #333;padding:.4rem .8rem;text-align:left} th{background:#1e1e2e;color:#7dd3fc} - .copy-btn{cursor:pointer;background:#2d2d4e;border:1px solid #555;color:#cba6f7;padding:.2rem .6rem;border-radius:.25rem;font-size:.75rem;margin-left:.5rem}

noThinkProxy v1.0

-

Anthropic-API → Ollama-Proxy · Think-Modus deaktiviert · Modell-Substitution aktiv

+

Anthropic-API → Ollama-Proxy · Think-Modus deaktiviert · Web-Suche aktiv

Aktuelle Konfiguration

@@ -57,6 +60,7 @@ app.get('/', (req, res) => { +
Modell${OLLAMA_MODEL}
Kontext262144 Token (256k)
Thinkfalse
Web-SucheGoogle Custom Search
Proxy-URL${host}
@@ -70,7 +74,7 @@ app.get('/', (req, res) => {

API-Endpunkt

POST ${host}/v1/messages
-

Kompatibel mit dem Anthropic SDK. Alle claude-* Modellnamen werden automatisch auf ${OLLAMA_MODEL} umgeleitet.

+

Alle claude-* Modellnamen werden auf ${OLLAMA_MODEL} umgeleitet.

`); }); @@ -131,26 +135,108 @@ echo "" `); }); +// ── Web Search ──────────────────────────────────────────────────────────────── + +const WEB_SEARCH_TOOL = { + type: 'function', + function: { + name: 'web_search', + description: 'Searches the internet for current information. Use this when you need up-to-date facts, recent news, or information you are not certain about.', + parameters: { + type: 'object', + properties: { + query: { type: 'string', description: 'The search query' } + }, + required: ['query'] + } + } +}; + +const READ_URL_TOOL = { + type: 'function', + function: { + name: 'read_url', + description: 'Fetches a URL and returns the page content as Markdown. Use this to read articles, documentation, or any web page in detail.', + parameters: { + type: 'object', + properties: { + url: { type: 'string', description: 'The URL to fetch' } + }, + required: ['url'] + } + } +}; + +async function executeReadUrl(url) { + console.log(`${colors.cyan}[Read URL] ${url}${colors.reset}`); + try { + const resp = await fetch(url, { + headers: { 'User-Agent': 'Mozilla/5.0 (compatible; noThinkProxy/1.0)' }, + signal: AbortSignal.timeout(15000) + }); + if (!resp.ok) return `Error fetching ${url}: HTTP ${resp.status}`; + + const html = await resp.text(); + const root = parseHtml(html, { blockTextElements: { script: false, style: false } }); + + // Rauschen entfernen + for (const sel of ['script', 'style', 'nav', 'header', 'footer', 'aside', 'iframe', 'noscript']) { + root.querySelectorAll(sel).forEach(el => el.remove()); + } + + const title = root.querySelector('title')?.text.trim() + || root.querySelector('h1')?.text.trim() + || ''; + + // Hauptinhalt extrahieren + const mainEl = root.querySelector('main, article, [role="main"], #content, #main, .content, .post'); + const contentHtml = mainEl ? mainEl.innerHTML : (root.querySelector('body')?.innerHTML || html); + + const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' }); + let markdown = `# ${title}\n\n${td.turndown(contentHtml)}`; + + if (markdown.length > 12000) { + markdown = markdown.substring(0, 12000) + '\n\n[... content truncated ...]'; + } + + console.log(`${colors.cyan}[Read URL] ${markdown.length} Zeichen${colors.reset}`); + return markdown; + } catch (e) { + return `Error fetching ${url}: ${e.message}`; + } +} + +async function executeWebSearch(query) { + const url = `https://www.googleapis.com/customsearch/v1?key=${GOOGLE_API_KEY}&cx=${GOOGLE_CX}&q=${encodeURIComponent(query)}&num=5`; + console.log(`${colors.cyan}[Web Search] "${query}"${colors.reset}`); + try { + const resp = await fetch(url); + const data = await resp.json(); + if (data.error) return `Search error: ${data.error.message}`; + if (!data.items?.length) return `No results found for "${query}".`; + const results = data.items + .map((item, i) => `[${i + 1}] ${item.title}\n${item.link}\n${item.snippet}`) + .join('\n\n'); + console.log(`${colors.cyan}[Web Search] ${data.items.length} Ergebnisse${colors.reset}`); + return results; + } catch (e) { + return `Search error: ${e.message}`; + } +} + // ── Hilfsfunktionen ─────────────────────────────────────────────────────────── function sanitizeToolSchema(schema) { - if (!schema || typeof schema !== 'object') { - return { type: 'object', properties: {} }; - } - + if (!schema || typeof schema !== 'object') return { type: 'object', properties: {} }; const clean = JSON.parse(JSON.stringify(schema)); - if (!clean.type) clean.type = 'object'; if (!clean.properties) clean.properties = {}; - return clean; } function convertAnthropicTools(anthropicTools) { if (!anthropicTools || anthropicTools.length === 0) return []; - const validTools = []; - for (const tool of anthropicTools) { try { const ollamaTool = { @@ -161,30 +247,24 @@ function convertAnthropicTools(anthropicTools) { parameters: sanitizeToolSchema(tool.input_schema) } }; - JSON.stringify(ollamaTool); validTools.push(ollamaTool); } catch (e) { console.error(`${colors.red}[Tool Schema Error] ${e.message}${colors.reset}`); } } - return validTools; } function stringifyToolResultContent(content) { if (Array.isArray(content)) { - return content - .map(c => { - if (typeof c === 'string') return c; - if (c?.text) return c.text; - return JSON.stringify(c); - }) - .join('\n'); + return content.map(c => { + if (typeof c === 'string') return c; + if (c?.text) return c.text; + return JSON.stringify(c); + }).join('\n'); } - if (typeof content === 'string') return content; - return JSON.stringify(content); } @@ -194,10 +274,9 @@ function convertAnthropicToOllama(anthropicBody) { if (anthropicBody.system) { ollamaMessages.push({ role: 'system', - content: - typeof anthropicBody.system === 'string' - ? anthropicBody.system - : JSON.stringify(anthropicBody.system) + content: typeof anthropicBody.system === 'string' + ? anthropicBody.system + : JSON.stringify(anthropicBody.system) }); } @@ -206,35 +285,23 @@ function convertAnthropicToOllama(anthropicBody) { ollamaMessages.push({ role: msg.role, content: msg.content }); continue; } - if (!Array.isArray(msg.content)) continue; if (msg.role === 'assistant') { const textParts = []; const toolCalls = []; - for (const item of msg.content) { - if (item.type === 'text') { - textParts.push(item.text || ''); - } else if (item.type === 'tool_use') { - toolCalls.push({ - function: { - name: item.name, - arguments: item.input || {} - } - }); + if (item.type === 'text') textParts.push(item.text || ''); + else if (item.type === 'tool_use') { + toolCalls.push({ function: { name: item.name, arguments: item.input || {} } }); } } - const assistantMsg = { role: 'assistant', content: textParts.join('\n\n') }; - if (toolCalls.length > 0) { - assistantMsg.tool_calls = toolCalls; - } + if (toolCalls.length > 0) assistantMsg.tool_calls = toolCalls; ollamaMessages.push(assistantMsg); } else { const pendingText = []; - for (const item of msg.content) { if (item.type === 'text') { pendingText.push(item.text || ''); @@ -243,16 +310,12 @@ function convertAnthropicToOllama(anthropicBody) { ollamaMessages.push({ role: 'user', content: pendingText.join('\n\n') }); pendingText.length = 0; } - const resultText = stringifyToolResultContent(item.content); console.log(`${colors.blue}📥 Tool Result ${item.tool_use_id}:${colors.reset}`); - console.log(`${colors.blue}${resultText}${colors.reset}`); - console.log(''); - + console.log(`${colors.blue}${resultText}${colors.reset}\n`); ollamaMessages.push({ role: 'tool', content: resultText }); } } - if (pendingText.length > 0) { ollamaMessages.push({ role: 'user', content: pendingText.join('\n\n') }); } @@ -271,55 +334,39 @@ function convertAnthropicToOllama(anthropicBody) { } }; - if (anthropicBody.tools && anthropicBody.tools.length > 0) { - const validTools = convertAnthropicTools(anthropicBody.tools); - - if (validTools.length > 0) { - ollamaBody.tools = validTools; - } - } + // Anthropic-Tools konvertieren + interne Tools voranstellen + const convertedTools = convertAnthropicTools(anthropicBody.tools || []); + ollamaBody.tools = [WEB_SEARCH_TOOL, READ_URL_TOOL, ...convertedTools]; return ollamaBody; } function parseToolArguments(args) { if (!args) return {}; - if (typeof args === 'string') { - try { - return JSON.parse(args); - } catch (e) { - console.error(`${colors.red}[Tool Args Parse Error] ${e.message}${colors.reset}`); - return {}; - } + try { return JSON.parse(args); } catch (e) { return {}; } } - if (typeof args === 'object') return args; - return {}; } function makeToolDedupeKey(tc) { const name = tc.function?.name || ''; const args = tc.function?.arguments || {}; - const argsString = typeof args === 'string' ? args : JSON.stringify(args); - - return `${name}:${argsString}`; + return `${name}:${typeof args === 'string' ? args : JSON.stringify(args)}`; } -// ── Response-Handler ────────────────────────────────────────────────────────── +// ── Response-Handler mit Web-Search-Loop ────────────────────────────────────── -async function handleResponse(response, anthropicBody, res, requestNum) { +async function handleResponse(initialResponse, anthropicBody, res, requestNum, ollamaBody) { res.setHeader('Content-Type', 'text/event-stream'); res.setHeader('Cache-Control', 'no-cache'); res.setHeader('Connection', 'keep-alive'); - const messageId = 'msg_' + requestNum; - res.write(`event: message_start\ndata: ${JSON.stringify({ type: 'message_start', message: { - id: messageId, + id: 'msg_' + requestNum, type: 'message', role: 'assistant', content: [], @@ -330,153 +377,171 @@ async function handleResponse(response, anthropicBody, res, requestNum) { } })}\n\n`); - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - - let contentBlocks = []; let currentBlockIndex = 0; - - const seenToolCalls = new Set(); let emittedToolUse = false; - let messageFinished = false; - let buffer = ''; + const seenToolCalls = new Set(); + let currentResponse = initialResponse; - function processChunk(data) { - if (messageFinished) return; + for (let searchIteration = 0; searchIteration < 5; searchIteration++) { + const reader = currentResponse.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + let textBlockOpen = false; + let evalCount = 0; - if (data.message?.tool_calls && data.message.tool_calls.length > 0) { - for (const tc of data.message.tool_calls) { - const dedupeKey = makeToolDedupeKey(tc); + // Gesammelt pro Iteration für eventuelle Web-Search-Fortsetzung + let iterContent = ''; + let iterAllToolCalls = []; + let iterWebSearches = []; - if (seenToolCalls.has(dedupeKey)) { - console.log(`${colors.yellow}[Duplicate Tool Call skipped] ${dedupeKey}${colors.reset}`); - continue; + function processChunk(data) { + // Tool Calls + if (data.message?.tool_calls?.length) { + for (const tc of data.message.tool_calls) { + iterAllToolCalls.push(tc); + + // Proxy-eigene Tools + Claude Code Web-Tools intern abfangen + const INTERNAL = new Set(['web_search', 'read_url', 'WebSearch', 'web_fetch', 'WebFetch']); + if (INTERNAL.has(tc.function?.name)) { + iterWebSearches.push(tc); + continue; // intern verarbeiten, nicht an Client senden + } + + const key = makeToolDedupeKey(tc); + if (seenToolCalls.has(key)) { + console.log(`${colors.yellow}[Duplicate Tool Call skipped] ${key}${colors.reset}`); + continue; + } + seenToolCalls.add(key); + emittedToolUse = true; + + const toolName = tc.function?.name; + const toolInput = parseToolArguments(tc.function?.arguments); + const toolUseId = `toolu_${requestNum}_${currentBlockIndex}`; + + console.log(`${colors.magenta}[Tool Use: ${toolName}] ${JSON.stringify(toolInput)}${colors.reset}`); + + res.write(`event: content_block_start\ndata: ${JSON.stringify({ + type: 'content_block_start', index: currentBlockIndex, + content_block: { type: 'tool_use', id: toolUseId, name: toolName, input: {} } + })}\n\n`); + res.write(`event: content_block_delta\ndata: ${JSON.stringify({ + type: 'content_block_delta', index: currentBlockIndex, + delta: { type: 'input_json_delta', partial_json: JSON.stringify(toolInput) } + })}\n\n`); + res.write(`event: content_block_stop\ndata: ${JSON.stringify({ + type: 'content_block_stop', index: currentBlockIndex + })}\n\n`); + currentBlockIndex++; + } + } + + // Text + if (data.message?.content) { + const text = data.message.content; + iterContent += text; + + if (!textBlockOpen) { + res.write(`event: content_block_start\ndata: ${JSON.stringify({ + type: 'content_block_start', index: currentBlockIndex, + content_block: { type: 'text', text: '' } + })}\n\n`); + textBlockOpen = true; } - seenToolCalls.add(dedupeKey); - emittedToolUse = true; - - const toolName = tc.function?.name; - const toolInput = parseToolArguments(tc.function?.arguments); - const toolUseId = `toolu_${requestNum}_${currentBlockIndex}`; - - console.log(`${colors.yellow}[Raw Tool Call] ${JSON.stringify(tc)}${colors.reset}`); - console.log(`${colors.magenta}[Sending Tool Use: ${toolName}]${colors.reset}`); - console.log(`${colors.magenta}Input: ${JSON.stringify(toolInput)}${colors.reset}`); - - res.write(`event: content_block_start\ndata: ${JSON.stringify({ - type: 'content_block_start', - index: currentBlockIndex, - content_block: { type: 'tool_use', id: toolUseId, name: toolName, input: {} } - })}\n\n`); - + process.stdout.write(`${colors.green}${text}${colors.reset}`); res.write(`event: content_block_delta\ndata: ${JSON.stringify({ - type: 'content_block_delta', - index: currentBlockIndex, - delta: { type: 'input_json_delta', partial_json: JSON.stringify(toolInput) } + type: 'content_block_delta', index: currentBlockIndex, + delta: { type: 'text_delta', text } })}\n\n`); + } - res.write(`event: content_block_stop\ndata: ${JSON.stringify({ - type: 'content_block_stop', - index: currentBlockIndex - })}\n\n`); - - currentBlockIndex++; + // Done + if (data.done) { + evalCount = data.eval_count || 0; + if (textBlockOpen) { + res.write(`event: content_block_stop\ndata: ${JSON.stringify({ + type: 'content_block_stop', index: currentBlockIndex + })}\n\n`); + currentBlockIndex++; + textBlockOpen = false; + } } } - if (data.message?.content) { - const text = data.message.content; - - if (contentBlocks[currentBlockIndex] === undefined) { - res.write(`event: content_block_start\ndata: ${JSON.stringify({ - type: 'content_block_start', - index: currentBlockIndex, - content_block: { type: 'text', text: '' } - })}\n\n`); - - contentBlocks[currentBlockIndex] = ''; + // Stream lesen + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { processChunk(JSON.parse(trimmed)); } + catch (e) { console.error(`${colors.red}[Parse Error] ${e.message}${colors.reset}`); } } + } + if (buffer.trim()) { + try { processChunk(JSON.parse(buffer.trim())); } + catch (e) { console.error(`${colors.red}[Final Buffer Error] ${e.message}${colors.reset}`); } + } - process.stdout.write(`${colors.green}${text}${colors.reset}`); - - res.write(`event: content_block_delta\ndata: ${JSON.stringify({ - type: 'content_block_delta', - index: currentBlockIndex, - delta: { type: 'text_delta', text } + // Offenen Text-Block schließen falls Ollama keinen done:true geschickt hat + if (textBlockOpen) { + res.write(`event: content_block_stop\ndata: ${JSON.stringify({ + type: 'content_block_stop', index: currentBlockIndex })}\n\n`); - - contentBlocks[currentBlockIndex] += text; + currentBlockIndex++; + textBlockOpen = false; } - if (data.done) { - messageFinished = true; - - if (contentBlocks[currentBlockIndex] !== undefined) { - res.write(`event: content_block_stop\ndata: ${JSON.stringify({ - type: 'content_block_stop', - index: currentBlockIndex - })}\n\n`); - } - + // Keine Web-Suchen → finale Events senden + if (iterWebSearches.length === 0) { res.write(`event: message_delta\ndata: ${JSON.stringify({ type: 'message_delta', delta: { stop_reason: emittedToolUse ? 'tool_use' : 'end_turn' }, - usage: { output_tokens: data.eval_count || 0 } + usage: { output_tokens: evalCount } })}\n\n`); - res.write(`event: message_stop\ndata: ${JSON.stringify({ type: 'message_stop' })}\n\n`); - console.log(`${colors.green}✓${colors.reset}\n`); + break; } - } - while (true) { - const { done, value } = await reader.read(); - if (done) break; + // Web-Suchen ausführen und Ergebnisse in Messages einbauen + ollamaBody.messages.push({ + role: 'assistant', + content: iterContent, + tool_calls: iterAllToolCalls + }); - buffer += decoder.decode(value, { stream: true }); - - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed) continue; - - try { - processChunk(JSON.parse(trimmed)); - } catch (e) { - console.error(`${colors.red}[Stream Parse Error] ${e.message}${colors.reset}`); - console.error(`${colors.red}${line}${colors.reset}`); + for (const tc of iterWebSearches) { + const args = parseToolArguments(tc.function?.arguments); + let result; + const name = tc.function?.name; + if (name === 'web_search' || name === 'WebSearch') { + result = await executeWebSearch(args?.query || args?.q || ''); + } else if (name === 'read_url' || name === 'web_fetch' || name === 'WebFetch') { + result = await executeReadUrl(args?.url || ''); } - } - } - - if (buffer.trim()) { - try { - processChunk(JSON.parse(buffer.trim())); - } catch (e) { - console.error(`${colors.red}[Final Buffer Parse Error] ${e.message}${colors.reset}`); - console.error(buffer); - } - } - - if (!messageFinished) { - if (contentBlocks[currentBlockIndex] !== undefined) { - res.write(`event: content_block_stop\ndata: ${JSON.stringify({ - type: 'content_block_stop', - index: currentBlockIndex - })}\n\n`); + ollamaBody.messages.push({ role: 'tool', content: result ?? `Tool '${name}' returned no result.` }); } - res.write(`event: message_delta\ndata: ${JSON.stringify({ - type: 'message_delta', - delta: { stop_reason: emittedToolUse ? 'tool_use' : 'end_turn' }, - usage: { output_tokens: 0 } - })}\n\n`); + // Neuer Ollama-Request (weiter streamen) + currentResponse = await fetch(OLLAMA_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${OLLAMA_AUTH}` + }, + body: JSON.stringify(ollamaBody) + }); - res.write(`event: message_stop\ndata: ${JSON.stringify({ type: 'message_stop' })}\n\n`); + if (!currentResponse.ok) { + const errText = await currentResponse.text(); + throw new Error(`Ollama: ${currentResponse.status} ${errText}`); + } } res.end(); @@ -486,7 +551,6 @@ async function handleResponse(response, anthropicBody, res, requestNum) { app.post('/v1/messages', async (req, res) => { const requestNum = Date.now(); - console.log(`${colors.magenta}━━━ #${requestNum} ━━━${colors.reset}`); try { @@ -499,7 +563,7 @@ app.post('/v1/messages', async (req, res) => { const ollamaBody = convertAnthropicToOllama(anthropicBody); console.log( - `${colors.magenta}[msgs=${ollamaBody.messages.length}, tools=${ollamaBody.tools?.length || 0}, ctx=256k, think=false, model=${OLLAMA_MODEL}]${colors.reset}` + `${colors.magenta}[msgs=${ollamaBody.messages.length}, tools=${ollamaBody.tools.length}, ctx=256k, think=false]${colors.reset}` ); const response = await fetch(OLLAMA_URL, { @@ -513,14 +577,13 @@ app.post('/v1/messages', async (req, res) => { if (!response.ok) { const errorText = await response.text(); - console.error(`${colors.red}${errorText}${colors.reset}`); - throw new Error(`Ollama: ${response.status}`); + throw new Error(`Ollama: ${response.status} ${errorText}`); } - return handleResponse(response, anthropicBody, res, requestNum); + return handleResponse(response, anthropicBody, res, requestNum, ollamaBody); + } catch (error) { console.error(`${colors.red}${error.message}${colors.reset}`); - if (!res.headersSent) { res.status(500).json({ type: 'error', @@ -534,7 +597,8 @@ app.post('/v1/messages', async (req, res) => { app.listen(PORT, () => { console.log(`${colors.magenta}noThinkProxy: localhost:${PORT}${colors.reset}`); - console.log(`${colors.cyan} Ollama : ${OLLAMA_URL}${colors.reset}`); - console.log(`${colors.cyan} Modell : ${OLLAMA_MODEL}${colors.reset}`); - console.log(`${colors.cyan} Ctx : 256k Think: false${colors.reset}\n`); + console.log(`${colors.cyan} Ollama : ${OLLAMA_URL}${colors.reset}`); + console.log(`${colors.cyan} Modell : ${OLLAMA_MODEL}${colors.reset}`); + console.log(`${colors.cyan} Ctx : 256k Think: false${colors.reset}`); + console.log(`${colors.cyan} Search : Google Custom Search${colors.reset}\n`); }); diff --git a/docker-compose.yml b/docker-compose.yml index d9c2d3f..2f7f540 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,8 @@ services: - OLLAMA_URL=https://ollama.aquantico.de/api/chat - OLLAMA_MODEL=qwen3.6:35b-a3b-q4_K_M - OLLAMA_AUTH=324GF44-50AA-4B57-9386-K435DLJ764DFR + - GOOGLE_API_KEY=AIzaSyChzsz8ZN8iHRqMUVFnxJXwyXWP_XwWy6g + - GOOGLE_CX=2331819c76d874bcc labels: - "traefik.enable=true" - "traefik.http.routers.nothinkproxy.rule=Host(`claude-proxy.aquantico.lan`)" diff --git a/package-lock.json b/package-lock.json index f6529bc..4db47ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,9 +6,16 @@ "": { "dependencies": { "express": "^5.2.1", - "node-fetch": "^3.3.2" + "node-fetch": "^3.3.2", + "node-html-parser": "^7.1.0", + "turndown": "^7.2.4" } }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==" + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -44,6 +51,11 @@ "url": "https://opencollective.com/express" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -115,6 +127,32 @@ "node": ">=6.6.0" } }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -147,6 +185,57 @@ "node": ">= 0.8" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -173,6 +262,17 @@ "node": ">= 0.8" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -400,6 +500,14 @@ "node": ">= 0.4" } }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "bin": { + "he": "bin/he" + } + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -551,6 +659,26 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/node-html-parser": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-7.1.0.tgz", + "integrity": "sha512-iJo8b2uYGT40Y8BTyy5ufL6IVbN8rbm/1QK2xffXU/1a/v3AAa0d1YAoqBNYqaS4R/HajkWIpIfdE6KcyFh1AQ==", + "dependencies": { + "css-select": "^5.1.0", + "he": "1.2.0" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -798,6 +926,18 @@ "node": ">=0.6" } }, + "node_modules/turndown": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz", + "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + }, + "engines": { + "node": ">=18", + "npm": ">=9" + } + }, "node_modules/type-is": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", diff --git a/package.json b/package.json index d48598a..02ea22a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,8 @@ { "dependencies": { "express": "^5.2.1", - "node-fetch": "^3.3.2" + "node-fetch": "^3.3.2", + "node-html-parser": "^7.1.0", + "turndown": "^7.2.4" } }