diff --git a/common/utils/business_event.py b/common/utils/business_event.py index a505a89..2d42329 100644 --- a/common/utils/business_event.py +++ b/common/utils/business_event.py @@ -559,12 +559,24 @@ class BusinessEvent: self._log_buffer = [] def _push_to_gateway(self): - # Push metrics to the gateway + # Push metrics to the gateway with grouping key to avoid overwrites across pods/processes try: + # Determine grouping labels + pod_name = current_app.config.get('POD_NAME', current_app.config.get('COMPONENT_NAME', 'dev')) + pod_namespace = current_app.config.get('POD_NAMESPACE', current_app.config.get('FLASK_ENV', 'dev')) + worker_id = str(os.getpid()) + + grouping_key = { + 'instance': pod_name, + 'namespace': pod_namespace, + 'process': worker_id, + } + push_to_gateway( current_app.config['PUSH_GATEWAY_URL'], job=current_app.config['COMPONENT_NAME'], - registry=REGISTRY + registry=REGISTRY, + grouping_key=grouping_key, ) except Exception as e: current_app.logger.error(f"Failed to push metrics to Prometheus Push Gateway: {e}") diff --git a/common/utils/execution_progress.py b/common/utils/execution_progress.py index 646f6ca..fdd1616 100644 --- a/common/utils/execution_progress.py +++ b/common/utils/execution_progress.py @@ -124,7 +124,7 @@ class ExecutionProgressTracker: yield f"data: {message['data'].decode('utf-8')}\n\n" # Check processing_type for completion - if update_data['processing_type'] in ['Task Complete', 'Task Error']: + if update_data['processing_type'] in ['Task Complete', 'Task Error', 'EveAI Specialist Complete']: break finally: try: diff --git a/common/utils/template_filters.py b/common/utils/template_filters.py index 8eff017..ca7106a 100644 --- a/common/utils/template_filters.py +++ b/common/utils/template_filters.py @@ -110,28 +110,39 @@ def get_pagination_html(pagination, endpoint, **kwargs): def asset_url(logical_path: str): """ Resolve an asset logical path to a hashed URL using Parcel manifest when available. - Fallback to the original logical path under /static/ if manifest is missing. + Return a URL that respects STATIC_URL (CDN) when configured; otherwise serve from /static/. Examples: - - asset_url('dist/chat-client.js') -> '/static/dist/chat-client.abc123.js' - - asset_url('dist/chat-client.css') -> '/static/dist/chat-client.def456.css' + - asset_url('dist/chat-client.js') -> 'https://cdn/.../dist/chat-client.abc123.js' (when STATIC_URL set) + - asset_url('dist/chat-client.css') -> '/static/dist/chat-client.def456.css' (when STATIC_URL not set) """ if not logical_path: return logical_path try: from common.utils.asset_manifest import resolve_asset - resolved = resolve_asset(logical_path) - if not resolved: - return f"/static/{logical_path.lstrip('/')}" - # If resolved is already an absolute URL starting with /static or http(s), return as is - if resolved.startswith('/static/') or resolved.startswith('http://') or resolved.startswith('https://'): + # Resolve logical to possibly hashed path + resolved = resolve_asset(logical_path) or logical_path + + # If manifest returns an absolute URL, return as-is + if resolved.startswith('http://') or resolved.startswith('https://'): return resolved - # If it starts with 'dist/', prefix /static/ - if resolved.startswith('dist/'): - return '/static/' + resolved - # Otherwise, best effort: ensure it lives under /static/ - return '/static/' + resolved.lstrip('/') + + # Normalize: strip any leading '/static/' and leading '/' + if resolved.startswith('/static/'): + rel = resolved[len('/static/'):] + else: + rel = resolved.lstrip('/') + + # Build with STATIC_URL if configured + static_base = (current_app.config.get('STATIC_URL') or '').rstrip('/') + if static_base: + return f"{static_base}/{rel}" + # Fallback to app static + return f"/static/{rel}" except Exception: - return f"/static/{logical_path.lstrip('/')}" + # Conservative fallback also respecting STATIC_URL + static_base = (current_app.config.get('STATIC_URL') or '').rstrip('/') + rel = logical_path.lstrip('/') + return f"{static_base}/{rel}" if static_base else f"/static/{rel}" def register_filters(app): diff --git a/config/config.py b/config/config.py index a384a96..f3d9737 100644 --- a/config/config.py +++ b/config/config.py @@ -439,6 +439,10 @@ class StagingConfig(Config): MINIO_SECRET_KEY = environ.get('MINIO_SECRET_KEY') MINIO_USE_HTTPS = True + # Push gateway grouping elements + pod_name = os.getenv('POD_NAME') + pod_namespace = os.getenv('POD_NAMESPACE') + class ProdConfig(Config): DEVELOPMENT = False diff --git a/config/static-manifest/manifest.json b/config/static-manifest/manifest.json index a28e9ca..3b987e8 100644 --- a/config/static-manifest/manifest.json +++ b/config/static-manifest/manifest.json @@ -1,6 +1,6 @@ { - "dist/chat-client.js": "dist/chat-client.25888758.js", - "dist/chat-client.css": "dist/chat-client.eef0ef31.css", + "dist/chat-client.js": "dist/chat-client.24c00fcd.js", + "dist/chat-client.css": "dist/chat-client.7d8832b6.css", "dist/main.js": "dist/main.f3dde0f6.js", "dist/main.css": "dist/main.c40e57ad.css" } \ No newline at end of file diff --git a/content/changelog/1.0/1.0.0.md b/content/changelog/1.0/1.0.0.md index 80c7660..ddbf4b3 100644 --- a/content/changelog/1.0/1.0.0.md +++ b/content/changelog/1.0/1.0.0.md @@ -5,8 +5,31 @@ All notable changes to EveAI will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 3.1.7-beta + +Release date: 2025-09-30 + +### Added +- Pushgateway for logging business events to Prometheus + +### Changed +- Prometheus deployment improvements and alignment with Pushgateway +- Metrics logging in Business events to support multiple pod and processes +- Maximum height for AI message in chat input also available in desktop client +- AI message rendering now allows markdown +- markdown rendering defined in a centralized utility + +### Fixed +- Bug preventing correct loading of cache busted css en js in eveai_app solved +- Fix on rare bug preventing marked component to be displayed in SideBarExplanation + +### Security +- DOM checks on markdown text to prevent XSS + ## 3.1.3-beta +Release date: 2025-09-25 + ### Added - Cache busting for static files @@ -18,11 +41,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## 3.1.2-beta +Release date: 2025-09-23 + ### Changed - Several improvements to the mobile version of the chat client ## 3.1.1-alfa +Release date: 2025-09-22 + ### Fixed - TRA-76 - Send Button color changes implemented - TRA-72 - Translation of privacy statement and T&C @@ -34,6 +61,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## 3.1.0-alfa +Release date: 2025-09-12 + ### Added - Configuration of the full k8s staging environment - k8s installation manual (cluster-install.md) @@ -54,6 +83,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## 3.0.1-beta +Release date: 2025-08-21 + ### Changed - Podman now replaces Docker for building images - Local registry now replaces Docker Hub @@ -71,6 +102,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## 3.0.0-beta +Release date: 2025-08-15 + ### Added - Mobile Support for the chat client. - Additional visual clues for chatbot and human messages in the chat client diff --git a/documentation/PUSHGATEWAY_GROUPING.md b/documentation/PUSHGATEWAY_GROUPING.md new file mode 100644 index 0000000..515f2df --- /dev/null +++ b/documentation/PUSHGATEWAY_GROUPING.md @@ -0,0 +1,79 @@ +# Pushgateway Grouping Keys (instance, namespace, process) + +Goal: prevent metrics pushed by different Pods or worker processes from overwriting each other, while keeping Prometheus/Grafana queries simple. + +Summary of decisions +- WORKER_ID source = OS process ID (PID) +- Always include namespace in grouping labels + +What this changes +- Every push to Prometheus Pushgateway now includes a grouping_key with: + - instance = POD_NAME (fallback to HOSTNAME, then "dev") + - namespace = POD_NAMESPACE (fallback to ENVIRONMENT, then "dev") + - process = WORKER_ID (fallback to current PID) +- Prometheus will expose these as exported_instance, exported_namespace, and exported_process on the scraped series. + +Code changes (already implemented) +- common/utils/business_event.py + - push_to_gateway(..., grouping_key={instance, namespace, process}) + - Safe fallbacks ensure dev/test (Podman) keeps working with no K8s-specific env vars. + +Kubernetes manifests (already implemented) +- All Deployments that push metrics set env vars via Downward API: + - POD_NAME from metadata.name + - POD_NAMESPACE from metadata.namespace +- Files updated: + - scaleway/manifests/base/applications/frontend/eveai-app/deployment.yaml + - scaleway/manifests/base/applications/frontend/eveai-api/deployment.yaml + - scaleway/manifests/base/applications/frontend/eveai-chat-client/deployment.yaml + - scaleway/manifests/base/applications/backend/eveai-workers/deployment.yaml + - scaleway/manifests/base/applications/backend/eveai-chat-workers/deployment.yaml + - scaleway/manifests/base/applications/backend/eveai-entitlements/deployment.yaml + +No changes needed to secrets +- PUSH_GATEWAY_HOST/PORT remain provided via eveai-secrets; code composes PUSH_GATEWAY_URL internally. + +How to verify +1) Pushgateway contains per-pod/process groups + - Port-forward Pushgateway (namespace monitoring): + - kubectl -n monitoring port-forward svc/monitoring-pushgateway-prometheus-pushgateway 9091:9091 + - Inspect: + - curl -s http://127.0.0.1:9091/api/v1/metrics | jq '.[].labels' + - You should see labels including job (your service), instance (pod), namespace, process (pid). + +2) Prometheus shows the labels as exported_* + - Port-forward Prometheus (namespace monitoring): + - kubectl -n monitoring port-forward svc/monitoring-prometheus 9090:9090 + - Queries: + - label_values(eveai_llm_calls_total, exported_instance) + - label_values(eveai_llm_calls_total, exported_namespace) + - label_values(eveai_llm_calls_total, exported_process) + +PromQL query patterns +- Hide per-process by aggregating away exported_process: + - sum without(exported_process) (rate(eveai_llm_calls_total[5m])) by (exported_job, exported_instance, exported_namespace) +- Service-level totals (hide instance and process): + - sum without(exported_instance, exported_process) (rate(eveai_llm_calls_total[5m])) by (exported_job, exported_namespace) +- Histogram example (p95 per service): + - histogram_quantile(0.95, sum without(exported_process) (rate(eveai_llm_duration_seconds_bucket[5m])) by (le, exported_job, exported_namespace)) + +Dev/Test (Podman) behavior +- No Kubernetes Downward API: POD_NAME/POD_NAMESPACE are not set. +- Fallbacks used by the code: + - instance = HOSTNAME if available, else "dev" + - namespace = ENVIRONMENT if available, else "dev" + - process = current PID +- This guarantees no crashes and still avoids process-level overwrites. + +Operational notes +- Cardinality: adding process creates more series (one per worker). This is required to avoid data loss when multiple workers push concurrently. Dashboards should aggregate away exported_process unless you need per-worker detail. +- Batch jobs (future): use the same grouping and consider delete_from_gateway on successful completion to remove stale groups for that job/instance/process. + +Troubleshooting +- If you still see overwriting: + - Confirm that instance, namespace, and process all appear in Pushgateway JSON labels for each group. + - Ensure that all pods set POD_NAME and POD_NAMESPACE (kubectl -n eveai-staging exec -- env | egrep "POD_NAME|POD_NAMESPACE"). + - Verify that your app processes run push_to_gateway through the shared business_event wrapper. + +Change log reference +- Implemented on 2025-09-26 by adding grouping_key in business_event push and env vars in Deployments. diff --git a/documentation/Production Setup/cluster-install.md b/documentation/Production Setup/cluster-install.md index 97974ad..7186662 100644 --- a/documentation/Production Setup/cluster-install.md +++ b/documentation/Production Setup/cluster-install.md @@ -119,7 +119,7 @@ helm search repo prometheus-community/kube-prometheus-stack #### Create Monitoring Values File -Create `scaleway/manifests/base/monitoring/prometheus-values.yaml`: +Create `scaleway/manifests/base/monitoring/values-monitoring.yaml`: #### Deploy Monitoring Stack @@ -133,7 +133,8 @@ helm install monitoring prometheus-community/kube-prometheus-stack \ # Install pushgateway helm install monitoring-pushgateway prometheus-community/prometheus-pushgateway \ -n monitoring --create-namespace \ - --set serviceMonitor.enabled=true + --set serviceMonitor.enabled=true \ + --set serviceMonitor.additionalLabels.release=monitoring # Monitor deployment progress kubectl get pods -n monitoring -w diff --git a/eveai_chat_client/static/assets/js/utils/markdownRenderer.js b/eveai_chat_client/static/assets/js/utils/markdownRenderer.js new file mode 100644 index 0000000..a927910 --- /dev/null +++ b/eveai_chat_client/static/assets/js/utils/markdownRenderer.js @@ -0,0 +1,165 @@ +// Centralized Markdown rendering utility +// Safe defaults per requirements: no inline HTML, no images, no code blocks, allow tables, links open in new tab with rel attributes + +export function renderMarkdown(text, options = {}) { + const opts = { + allowInlineHTML: false, + enableTables: true, + enableBreaks: true, + enableImages: false, + enableCodeBlocks: false, + allowInlineCode: false, // inline code currently not allowed + linkTargetBlank: true, + sidebarAccent: false, + consoleWarnOnFallback: true, + ...options + }; + + const input = typeof text === 'string' ? text : String(text ?? ''); + if (!input) return ''; + + // Optional sidebar accent [[...]] -> ... BEFORE parsing + const preprocessed = opts.sidebarAccent + ? input.replace(/\[\[(.*?)\]\]/g, '$1') + : input; + + try { + const hasMarked = typeof window !== 'undefined' && window.marked; + + if (!hasMarked) { + if (opts.consoleWarnOnFallback) { + console.warn('renderMarkdown: Marked not available; falling back to plain text'); + } + return escapeToPlainHtml(preprocessed); + } + + // Configure marked options defensively + if (typeof window.marked?.use === 'function') { + // Create a lightweight renderer with restricted features + const renderer = new window.marked.Renderer(); + + // Links: target _blank + rel attributes; block unsafe protocols + renderer.link = (href, title, text) => { + const safeHref = sanitizeUrl(href); + if (!safeHref) { + return text; // drop link, keep text + } + const t = opts.linkTargetBlank ? ' target="_blank"' : ''; + const rel = ' rel="noopener noreferrer nofollow"'; + const titleAttr = title ? ` title="${escapeHtmlAttr(title)}"` : ''; + return `${text}`; + }; + + // Images disabled: either drop or render alt text + renderer.image = (href, title, text) => { + if (!opts.enableImages) { + return text ? escapeToPlainHtml(text) : ''; + } + // If enabled in future, still sanitize + const safeHref = sanitizeUrl(href); + if (!safeHref) return text ? escapeToPlainHtml(text) : ''; + const titleAttr = title ? ` title="${escapeHtmlAttr(title)}"` : ''; + return `${escapeHtmlAttr(text || '')}`; + }; + + // Code blocks disabled + renderer.code = (code, infostring) => { + if (!opts.enableCodeBlocks) { + return `
${escapeToPlainHtml(code)}
`; // still show as plain + } + return `
${escapeToPlainHtml(code)}
`; + }; + + // Inline code disabled + renderer.codespan = (code) => { + if (!opts.allowInlineCode) { + return escapeToPlainHtml(code); + } + return `${escapeToPlainHtml(code)}`; + }; + + // Disallow raw HTML if not allowed + const mOptions = { + gfm: true, + breaks: !!opts.enableBreaks, + headerIds: false, + mangle: false, + renderer + }; + + // Table support via GFM is on by default; if disabled, override table renderers to simple paragraphs + if (!opts.enableTables) { + renderer.table = (header, body) => `${header}${body}`; + } + + const html = window.marked.parse(preprocessed, mOptions); + + // Sanitize output + const sanitized = sanitizeHtml(html); + return sanitized; + } + + // Fallback older API: window.marked is a function + const html = typeof window.marked === 'function' ? window.marked(preprocessed) : String(preprocessed); + const sanitized = sanitizeHtml(html); + return sanitized; + } catch (err) { + console.warn('renderMarkdown: error while rendering; falling back to plain text', err); + return escapeToPlainHtml(preprocessed); + } +} + +// Very small sanitizer if DOMPurify is not provided globally. Prefer window.DOMPurify if available. +function sanitizeHtml(html) { + try { + if (typeof window !== 'undefined' && window.DOMPurify && typeof window.DOMPurify.sanitize === 'function') { + return window.DOMPurify.sanitize(html, { + USE_PROFILES: { html: true }, + ADD_ATTR: ['target', 'rel', 'title'], + ALLOWED_TAGS: undefined // use default safe set + }); + } + } catch (e) { + console.warn('sanitizeHtml: DOMPurify error, using basic sanitizer', e); + } + // Basic sanitizer: strip script tags and on* attributes + return String(html) + .replace(/]*>[\s\S]*?<\/script>/gi, '') + .replace(/ on[a-z]+="[^"]*"/gi, '') + .replace(/ on[a-z]+='[^']*'/gi, ''); +} + +// Escape plain text to safe HTML +function escapeToPlainHtml(text) { + return String(text) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') + .replace(/\n/g, '
'); +} + +function sanitizeUrl(href) { + if (!href) return ''; + try { + const str = String(href).trim(); + const lower = str.toLowerCase(); + // Block javascript: and data: except maybe safe data images (we disable images anyway) + if (lower.startsWith('javascript:')) return ''; + if (lower.startsWith('data:')) return ''; + if (lower.startsWith('vbscript:')) return ''; + return str; + } catch (e) { + return ''; + } +} + +function escapeHtmlAttr(s) { + return String(s) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} diff --git a/eveai_chat_client/static/assets/vue-components/ChatMessage.vue b/eveai_chat_client/static/assets/vue-components/ChatMessage.vue index eba33d9..fe58241 100644 --- a/eveai_chat_client/static/assets/vue-components/ChatMessage.vue +++ b/eveai_chat_client/static/assets/vue-components/ChatMessage.vue @@ -95,7 +95,7 @@ -
+
@@ -126,6 +126,7 @@ import DynamicForm from './DynamicForm.vue'; import ProgressTracker from './ProgressTracker.vue'; import { useIconManager } from '../js/composables/useIconManager.js'; import { useComponentTranslations } from '../js/services/LanguageProvider.js'; +import { renderMarkdown } from '../js/utils/markdownRenderer.js'; export default { name: 'ChatMessage', @@ -236,6 +237,31 @@ export default { // Component cleanup if needed }, computed: { + renderedMessage() { + const content = this.message?.content ?? ''; + if (!content) return ''; + // Only render markdown for AI messages and text type + if (this.message.sender !== 'ai' || this.message.type !== 'text') { + // plain text fallback + return String(content) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/\"/g, '"') + .replace(/'/g, ''') + .replace(/\n/g, '
'); + } + return renderMarkdown(content, { + allowInlineHTML: false, + enableTables: true, + enableBreaks: true, + enableImages: false, + enableCodeBlocks: false, + allowInlineCode: false, + linkTargetBlank: true, + sidebarAccent: false + }); + }, isActiveContext() { // active if in input area or sticky area return !!(this.isInInputArea || this.isInStickyArea); @@ -600,12 +626,64 @@ export default { /* Zorgt dat het lettertype consistent is */ .message-text { + color: var(--ai-message-text-color); font-family: Arial, sans-serif; font-size: 14px; - white-space: pre-wrap; + white-space: normal; word-break: break-word; } +/* Markdown typography inside message text */ +.message-text :deep(h1), +.message-text :deep(h2), +.message-text :deep(h3), +.message-text :deep(h4) { + margin: 0.8rem 0 0.4rem; + color: var(--ai-message-text-color); +} +.message-text :deep(p) { + margin: 0 0 0.6rem 0; +} +.message-text :deep(ul), +.message-text :deep(ol) { + padding-left: 1.2rem; + margin: 0.4rem 0 0.6rem; +} +.message-text :deep(li) { + margin: 0.2rem 0; +} +.message-text :deep(a) { + color: var(--primary-color); + text-decoration: underline; +} +.message-text :deep(table) { + width: 100%; + border-collapse: collapse; + margin: 0.6rem 0; + overflow-x: auto; + display: block; +} +.message-text :deep(th), +.message-text :deep(td) { + border: 1px solid #e0e0e0; + padding: 6px 8px; + text-align: left; +} +.message-text :deep(th) { + background: rgba(0,0,0,0.05); + font-weight: 600; +} +.message-text :deep(blockquote) { + border-left: 3px solid var(--primary-color); + padding-left: 10px; + margin: 0.6rem 0; + color: var(--ai-message-text-color); + opacity: 0.9; +} +.message-text :deep(hr) { + border: 0; border-top: 1px solid #ddd; margin: 0.8rem 0; +} + /* Form error styling */ .form-error { color: red; @@ -706,26 +784,21 @@ export default { } } -/* Mobile bubble height constraints and inner scroll containment */ -@media (max-width: 768px) { - /* Default/history: apply to all message bubbles */ - .message .message-content { - max-height: 33vh; /* fallback */ - overflow-y: auto; - overscroll-behavior: contain; /* prevent scroll chaining to parent */ - -webkit-overflow-scrolling: touch; /* iOS smooth inertia */ - } - /* Active contexts (input area or sticky area): allow up to half viewport */ - .message.input-area .message-content, - .message.sticky-area .message-content { - max-height: 50vh; /* fallback */ - } +/* Bubble height constraints and inner scroll containment (apply on all viewports) */ +.message .message-content { + max-height: 33vh; /* fallback */ + overflow-y: auto; + overscroll-behavior: contain; /* prevent scroll chaining to parent */ + -webkit-overflow-scrolling: touch; /* iOS smooth inertia */ +} +/* Active contexts (input area or sticky area): allow up to half viewport */ +.message.input-area .message-content, +.message.sticky-area .message-content { + max-height: 50vh; /* fallback */ } @supports (max-height: 1svh) { - @media (max-width: 768px) { - .message .message-content { max-height: 33svh; } - .message.input-area .message-content, - .message.sticky-area .message-content { max-height: 50svh; } - } + .message .message-content { max-height: 33svh; } + .message.input-area .message-content, + .message.sticky-area .message-content { max-height: 50svh; } } \ No newline at end of file diff --git a/eveai_chat_client/static/assets/vue-components/SideBarExplanation.vue b/eveai_chat_client/static/assets/vue-components/SideBarExplanation.vue index 6072ff3..be8284a 100644 --- a/eveai_chat_client/static/assets/vue-components/SideBarExplanation.vue +++ b/eveai_chat_client/static/assets/vue-components/SideBarExplanation.vue @@ -18,6 +18,7 @@