bilal/DBBot-telegram
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reduce ingestion buffering and improve author metadata fallback.

Browse Source 
Flush context blocks faster to avoid delayed indexing and use sender_id-based labels when Telegram sender objects are unavailable in historical messages.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
bilal
2026-06-26 12:46:24 +03:00
parent 12b9992079
commit b2874ff580
3 changed files with 21 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
DBBot Unified n8n Qdrant.json
View File

@@ -76,7 +76,7 @@
},
{
"parameters": {
"jsCode": "const WINDOW_MS = 20 * 60 * 1000;\nconst MAX_MESSAGES = 12;\nconst MIN_MESSAGES_TO_FLUSH = 3;\n\nfunction aggregateBuffer(buffer) {\n const lines = buffer.messages.map((m) => {\n const repliedTo = m.replyTo ? ` -> ${m.replyTo}` : '';\n return `[${m.date}] ${m.author}${repliedTo}: ${m.text}`;\n });\n\n return {\n skip: false,\n text: [\n `Контекст диалога (${buffer.messages.length} сообщений, участники: ${buffer.participants.join(', ')})`,\n ...lines,\n ].join('\\n'),\n metadata: {\n ...buffer.lastMeta,\n context_id: buffer.contextId,\n context_message_count: buffer.messages.length,\n context_start_date: new Date(buffer.startTs).toISOString(),\n context_end_date: new Date(buffer.lastTs).toISOString(),\n context_participants: buffer.participants,\n context_mode: 'thread_time_window_20m'\n }\n };\n}\n\nconst staticData = $getWorkflowStaticData('global');\nstaticData.contextBuffers = staticData.contextBuffers || {};\n\nconst meta = $json.metadata || {};\nconst rawText = (meta.original_text || '').trim();\nif (!rawText) {\n return { json: { skip: true, reason: 'empty_original_text' } };\n}\n\nconst nowTs = meta.date ? new Date(meta.date).getTime() : Date.now();\nconst safeTs = Number.isNaN(nowTs) ? Date.now() : nowTs;\nconst chatId = String(meta.chat_id || meta.group_id || 'unknown_chat');\nconst threadId = String(meta.thread_id || meta.reply_to_message_id || meta.message_id || 'single');\nconst contextId = `${chatId}:${threadId}`;\n\nlet buffer = staticData.contextBuffers[contextId];\n\nif (buffer && safeTs - buffer.lastTs > WINDOW_MS) {\n const staleBuffer = buffer;\n buffer = null;\n\n const author = meta.author_label || meta.sender_name || 'Unknown';\n staticData.contextBuffers[contextId] = {\n contextId,\n startTs: safeTs,\n lastTs: safeTs,\n participants: [author],\n lastMeta: meta,\n messages: [\n {\n date: meta.date || new Date(safeTs).toISOString(),\n author,\n replyTo: meta.replied_to_author_label || '',\n text: rawText\n }\n ]\n };\n\n if (staleBuffer.messages.length >= MIN_MESSAGES_TO_FLUSH) {\n return { json: aggregateBuffer(staleBuffer) };\n }\n\n return { json: { skip: true, reason: 'buffer_reset_on_timeout', context_id: contextId } };\n}\n\nif (!buffer) {\n buffer = {\n contextId,\n startTs: safeTs,\n lastTs: safeTs,\n participants: [],\n lastMeta: meta,\n messages: []\n };\n}\n\nconst author = meta.author_label || meta.sender_name || 'Unknown';\nif (!buffer.participants.includes(author)) {\n buffer.participants.push(author);\n}\n\nbuffer.messages.push({\n date: meta.date || new Date(safeTs).toISOString(),\n author,\n replyTo: meta.replied_to_author_label || '',\n text: rawText\n});\nbuffer.lastMeta = meta;\nbuffer.lastTs = safeTs;\n\nconst shouldFlush =\n buffer.messages.length >= MAX_MESSAGES ||\n buffer.lastTs - buffer.startTs >= WINDOW_MS;\n\nif (!shouldFlush) {\n staticData.contextBuffers[contextId] = buffer;\n return {\n json: {\n skip: true,\n reason: 'buffering',\n context_id: contextId,\n buffered_messages: buffer.messages.length\n }\n };\n}\n\ndelete staticData.contextBuffers[contextId];\nreturn { json: aggregateBuffer(buffer) };"
"jsCode": "const WINDOW_MS = 5 * 60 * 1000;\nconst MAX_MESSAGES = 3;\nconst MIN_MESSAGES_TO_FLUSH = 1;\n\nfunction aggregateBuffer(buffer) {\n const lines = buffer.messages.map((m) => {\n const repliedTo = m.replyTo ? ` -> ${m.replyTo}` : '';\n return `[${m.date}] ${m.author}${repliedTo}: ${m.text}`;\n });\n\n return {\n skip: false,\n text: [\n `Контекст диалога (${buffer.messages.length} сообщений, участники: ${buffer.participants.join(', ')})`,\n ...lines,\n ].join('\\n'),\n metadata: {\n ...buffer.lastMeta,\n context_id: buffer.contextId,\n context_message_count: buffer.messages.length,\n context_start_date: new Date(buffer.startTs).toISOString(),\n context_end_date: new Date(buffer.lastTs).toISOString(),\n context_participants: buffer.participants,\n context_mode: 'thread_time_window_5m_fast_flush'\n }\n };\n}\n\nconst staticData = $getWorkflowStaticData('global');\nstaticData.contextBuffers = staticData.contextBuffers || {};\n\nconst meta = $json.metadata || {};\nconst rawText = (meta.original_text || '').trim();\nif (!rawText) {\n return { json: { skip: true, reason: 'empty_original_text' } };\n}\n\nconst nowTs = meta.date ? new Date(meta.date).getTime() : Date.now();\nconst safeTs = Number.isNaN(nowTs) ? Date.now() : nowTs;\nconst chatId = String(meta.chat_id || meta.group_id || 'unknown_chat');\nconst threadId = String(meta.thread_id || meta.reply_to_message_id || meta.message_id || 'single');\nconst contextId = `${chatId}:${threadId}`;\n\nlet buffer = staticData.contextBuffers[contextId];\n\nif (buffer && safeTs - buffer.lastTs > WINDOW_MS) {\n const staleBuffer = buffer;\n buffer = null;\n\n const author = meta.author_label || meta.sender_name || 'Unknown';\n staticData.contextBuffers[contextId] = {\n contextId,\n startTs: safeTs,\n lastTs: safeTs,\n participants: [author],\n lastMeta: meta,\n messages: [\n {\n date: meta.date || new Date(safeTs).toISOString(),\n author,\n replyTo: meta.replied_to_author_label || '',\n text: rawText\n }\n ]\n };\n\n if (staleBuffer.messages.length >= MIN_MESSAGES_TO_FLUSH) {\n return { json: aggregateBuffer(staleBuffer) };\n }\n\n return { json: { skip: true, reason: 'buffer_reset_on_timeout', context_id: contextId } };\n}\n\nif (!buffer) {\n buffer = {\n contextId,\n startTs: safeTs,\n lastTs: safeTs,\n participants: [],\n lastMeta: meta,\n messages: []\n };\n}\n\nconst author = meta.author_label || meta.sender_name || 'Unknown';\nif (!buffer.participants.includes(author)) {\n buffer.participants.push(author);\n}\n\nbuffer.messages.push({\n date: meta.date || new Date(safeTs).toISOString(),\n author,\n replyTo: meta.replied_to_author_label || '',\n text: rawText\n});\nbuffer.lastMeta = meta;\nbuffer.lastTs = safeTs;\n\nconst shouldFlush =\n buffer.messages.length >= MAX_MESSAGES ||\n buffer.lastTs - buffer.startTs >= WINDOW_MS;\n\nif (!shouldFlush) {\n staticData.contextBuffers[contextId] = buffer;\n return {\n json: {\n skip: true,\n reason: 'buffering',\n context_id: contextId,\n buffered_messages: buffer.messages.length\n }\n };\n}\n\ndelete staticData.contextBuffers[contextId];\nreturn { json: aggregateBuffer(buffer) };"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,