pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/Bug-fixing-round-after-containerization' into develop

Browse Source
This commit is contained in:
Josako
2024-06-28 14:56:53 +02:00
parent 04dda69298 9187947f68
commit 19e57f5adf
19 changed files with 285 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
.DS_Store vendored
View File

Binary file not shown.

4
.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
docker/db/postgresql/
docker/db/redis/
docker/logs/
docker/tenant_files/

4
common/utils/view_assistants.py
View File

@@ -44,3 +44,7 @@ def form_validation_failed(request, form):
for fieldName, errorMessages in form.errors.items():
for err in errorMessages:
flash(f"Error in {fieldName}: {err}", 'danger')
def form_to_dict(form):
return {field.name: field.data for field in form if field.name != 'csrf_token' and hasattr(field, 'data')}

6
config/config.py
View File

@@ -169,7 +169,7 @@ class DevConfig(Config):
MAIL_PASSWORD = '$6xsWGbNtx$CFMQZqc*'
# file upload settings
UPLOAD_FOLDER = '/Volumes/OWC4M2_1/Development/eveAI/file_store'
UPLOAD_FOLDER = '/app/tenant_files'
# Celery settings
# eveai_app Redis Settings
@@ -192,6 +192,10 @@ class DevConfig(Config):
SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
SOCKETIO_LOGGER = True
SOCKETIO_ENGINEIO_LOGGER = True
SOCKETIO_PING_TIMEOUT = 20000
SOCKETIO_PING_INTERVAL = 25000
SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60) # Changing this value ==> change maxConnectionDuration value in
# eveai-chat-widget.js
# Google Cloud settings
GC_PROJECT_NAME = 'eveai-420711'

BIN
docker/.DS_Store vendored
View File

Binary file not shown.

10
docker/compose.yaml
View File

@@ -39,6 +39,7 @@ services:
- ../migrations:/app/migrations
- ../scripts:/app/scripts
- ./logs:/app/logs
- ./tenant_files:/app/tenant_files
depends_on:
db:
condition: service_healthy
@@ -66,6 +67,7 @@ services:
- ../config:/app/config
- ../scripts:/app/scripts
- ./logs:/app/logs
- ./tenant_files:/app/tenant_files
depends_on:
db:
condition: service_healthy
@@ -103,6 +105,7 @@ services:
interval: 10s
timeout: 5s
retries: 5
command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
eveai_chat_workers:
build:
@@ -162,9 +165,10 @@ services:
interval: 10s
timeout: 5s
retries: 5
volumes:
db-data:
redis-data:
#volumes:
# db-data:
# redis-data:
# tenant-files:
#secrets:
# db-password:
# file: ./db/password.txt

BIN
docker/db/.DS_Store vendored
View File

Binary file not shown.

29
docker/reload-nginx.sh Executable file
View File

@@ -0,0 +1,29 @@
#!/bin/bash
# Check if the service name is provided
if [ -z "$1" ]; then
echo "Usage: $0 <service_name>"
exit 1
fi
SERVICE_NAME=$1
# Get the container ID of the service
CONTAINER_ID=$(docker-compose ps -q $SERVICE_NAME)
# Check if the container ID is found
if [ -z "$CONTAINER_ID" ]; then
echo "Service $SERVICE_NAME not found or not running."
exit 1
fi
# Reload Nginx inside the container
docker exec $CONTAINER_ID nginx -s reload
# Output the result
if [ $? -eq 0 ]; then
echo "Nginx reloaded successfully in $SERVICE_NAME."
else
echo "Failed to reload Nginx in $SERVICE_NAME."
exit 1
fi

8
eveai_app/templates/document/edit_document_language.html → eveai_app/templates/document/edit_document_version.html
View File

@@ -1,9 +1,9 @@
{% extends "base.html" %}
{% from "macros.html" import render_field %}
{% block title %}Update Document Language{% endblock %}
{% block title %}Update Document Version{% endblock %}
{% block content_title %}Update Document Language{% endblock %}
{% block content_description %}Update document language for {{ doc_details }}.{% endblock %}
{% block content_title %}Update Document Version{% endblock %}
{% block content_description %}Update document version for {{ doc_details }}.{% endblock %}
{% block content %}
<form method="post">
@@ -13,6 +13,6 @@
{% for field in form %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<button type="submit" class="btn btn-primary">Update Document</button>
<button type="submit" class="btn btn-primary">Update Document Version</button>
</form>
{% endblock %}

111
eveai_app/templates/user/tenant_overview.html
View File

@@ -62,7 +62,14 @@
{{ render_included_field(field, disabled_fields=license_fields, include_fields=license_fields) }}
{% endfor %}
<!-- Register API Key Button -->
<button type="button" class="btn btn-primary" onclick="checkAndRegisterApiKey()">Register API Key</button>
<button type="button" class="btn btn-primary" onclick="generateNewApiKey()">Register API Key</button>
<!-- API Key Display Field -->
<div id="api-key-field" style="display:none;">
<label for="api-key">API Key:</label>
<input type="text" id="api-key" class="form-control" readonly>
<button type="button" id="copy-button" class="btn btn-primary">Copy to Clipboard</button>
<p id="copy-message" style="display:none;color:green;">API key copied to clipboard</p>
</div>
</div>
<!-- HTML Chunking Settings Tab -->
<div class="tab-pane fade" id="html-chunking-tab" role="tabpanel">
@@ -89,26 +96,6 @@
</div>
</div>
</form>
<!-- Modal HTML -->
<div class="modal fade" id="confirmModal" tabindex="-1" role="dialog" aria-labelledby="confirmModalLabel" aria-hidden="true">
<div class="modal-dialog" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="confirmModalLabel">Confirm New API Key</h5>
<button type="button" class="close" data-dismiss="modal" aria-label="Close">
<span aria-hidden="true">&times;</span>
</button>
</div>
<div class="modal-body" id="modal-body-content">
Are you sure you want to register a new API key? This will replace the existing key.
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-dismiss="modal">Cancel</button>
<button type="button" class="btn btn-primary" id="confirmNewKeyBtn">Confirm</button>
</div>
</div>
</div>
</div>
{% endblock %}
@@ -118,42 +105,14 @@
{% block scripts %}
<script>
function checkAndRegisterApiKey() {
// First, check if an API key already exists
$.ajax({
url: '/user/check_chat_api_key',
type: 'POST',
contentType: 'application/json',
success: function(response) {
if (response.api_key_exists) {
$('#confirmModal').modal('show');
} else {
generateNewApiKey();
}
},
error: function(error) {
alert('Error checking API key: ' + error.responseText);
}
});
}
document.getElementById('confirmNewKeyBtn').addEventListener('click', function () {
generateNewApiKey();
});
function generateNewApiKey() {
$.ajax({
url: '/user/generate_chat_api_key',
type: 'POST',
contentType: 'application/json',
success: function(response) {
$('#modal-body-content').html(`
<p>New API key generated: <span id="new-api-key">${response.api_key}</span></p>
<button class="btn btn-primary" onclick="copyToClipboard('#new-api-key')">Copy to Clipboard</button>
<p id="copy-message" style="display:none;color:green;">API key copied to clipboard</p>
`);
$('#confirmNewKeyBtn').hide();
$('.btn-secondary').text('OK');
$('#api-key').val(response.api_key);
$('#api-key-field').show();
},
error: function(error) {
alert('Error generating new API key: ' + error.responseText);
@@ -161,16 +120,54 @@
});
}
function copyToClipboard(element) {
const text = $(element).text();
function copyToClipboard(selector) {
const element = document.querySelector(selector);
if (element) {
const text = element.value;
if (navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(text).then(function() {
$('#copy-message').show().delay(2000).fadeOut();
showCopyMessage();
}).catch(function(error) {
alert('Failed to copy text: ' + error);
});
} else {
fallbackCopyToClipboard(text);
}
</script>
<script>
} else {
console.error('Element not found for selector:', selector);
}
}
function fallbackCopyToClipboard(text) {
const textArea = document.createElement('textarea');
textArea.value = text;
document.body.appendChild(textArea);
textArea.focus();
textArea.select();
try {
document.execCommand('copy');
showCopyMessage();
} catch (err) {
alert('Fallback: Oops, unable to copy', err);
}
document.body.removeChild(textArea);
}
function showCopyMessage() {
const message = document.getElementById('copy-message');
if (message) {
message.style.display = 'block';
setTimeout(function() {
message.style.display = 'none';
}, 2000);
}
}
document.getElementById('copy-button').addEventListener('click', function() {
copyToClipboard('#api-key');
});
</script>
<script>
// JavaScript to detect user's timezone
document.addEventListener('DOMContentLoaded', (event) => {
// Detect timezone
@@ -191,5 +188,5 @@
}
});
});
</script>
</script>
{% endblock %}

24
eveai_app/views/document_views.py
View File

@@ -21,7 +21,7 @@ from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditD
from common.utils.middleware import mw_before_request
from common.utils.celery_utils import current_celery
from common.utils.nginx_utils import prefixed_url_for
from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro
from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro, form_to_dict
document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@@ -60,8 +60,9 @@ def add_document():
file = form.file.data
filename = secure_filename(file.filename)
extension = filename.rsplit('.', 1)[1].lower()
form_dict = form_to_dict(form)
new_doc, new_doc_vers = create_document_stack(form, file, filename, extension)
new_doc, new_doc_vers = create_document_stack(form_dict, file, filename, extension)
task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
session['tenant']['id'],
@@ -90,6 +91,12 @@ def add_url():
current_app.logger.info(f'Adding document for tenant {session["tenant"]["id"]}')
url = form.url.data
doc_vers = DocumentVersion.query.filter_by(url=url).all()
if doc_vers:
current_app.logger.info(f'A document with url {url} already exists. No new document created.')
flash(f'A document with url {url} already exists. No new document created.', 'info')
return redirect(prefixed_url_for('document_bp.documents'))
# Only when no document with URL exists
html = fetch_html(url)
file = io.BytesIO(html)
@@ -101,8 +108,9 @@ def add_url():
if not filename.endswith('.html'):
filename += '.html'
extension = 'html'
form_dict = form_to_dict(form)
new_doc, new_doc_vers = create_document_stack(form, file, filename, extension)
new_doc, new_doc_vers = create_document_stack(form_dict, file, filename, extension)
task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
session['tenant']['id'],
@@ -373,7 +381,7 @@ def create_document_stack(form, file, filename, extension):
new_doc = create_document(form, filename)
# Create the DocumentVersion
new_doc_vers = create_version_for_document(new_doc, form.url.data, form.language.data, form.user_context.data)
new_doc_vers = create_version_for_document(new_doc, form.get('url', ''), form['language'], form['user_context'])
try:
db.session.add(new_doc)
@@ -404,13 +412,13 @@ def log_session_state(session, msg=""):
def create_document(form, filename):
new_doc = Document()
if form.name.data == '':
if form['name'] == '':
new_doc.name = filename.rsplit('.', 1)[0]
else:
new_doc.name = form.name.data
new_doc.name = form['name']
if form.valid_from.data or form.valid_from.data != '':
new_doc.valid_from = form.valid_from.data
if form['valid_from'] and form['valid_from'] != '':
new_doc.valid_from = form['valid_from']
else:
new_doc.valid_from = dt.now(tz.utc)
new_doc.tenant_id = session['tenant']['id']

2
eveai_app/views/user_views.py
View File

@@ -430,7 +430,7 @@ def generate_chat_api_key():
db.session.rollback()
current_app.logger.error(f'Unable to store api key for tenant {tenant.id}. Error: {str(e)}')
return jsonify({'new_api_key': 'API key generated successfully.', 'api_key': new_api_key}), 200
return jsonify({'api_key': new_api_key}), 200
@user_bp.route('/tenant_overview', methods=['GET'])

4
eveai_chat/__init__.py
View File

@@ -50,7 +50,9 @@ def register_extensions(app):
async_mode=app.config.get('SOCKETIO_ASYNC_MODE'),
logger=app.config.get('SOCKETIO_LOGGER'),
engineio_logger=app.config.get('SOCKETIO_ENGINEIO_LOGGER'),
path='/socket.io'
path='/socket.io',
ping_timeout=app.config.get('SOCKETIO_PING_TIMEOUT'),
ping_interval=app.config.get('SOCKETIO_PING_INTERVAL'),
)
jwt.init_app(app)
kms_client.init_app(app)

13
eveai_chat/socket_handlers/chat_handler.py
View File

@@ -4,6 +4,7 @@ from flask_jwt_extended import create_access_token, get_jwt_identity, verify_jwt
from flask_socketio import emit, disconnect
from flask import current_app, request, session
from sqlalchemy.exc import SQLAlchemyError
from datetime import datetime, timedelta
from common.extensions import socketio, kms_client, db
from common.models.user import Tenant
@@ -35,6 +36,8 @@ def handle_connect():
if 'session_id' not in session:
session['session_id'] = str(uuid.uuid4())
session['last_activity'] = datetime.now()
# Communicate connection to client
current_app.logger.debug(f'SocketIO: Connection handling sending status to client for tenant {tenant_id}')
emit('connect', {'status': 'Connected', 'tenant_id': tenant_id})
@@ -53,11 +56,21 @@ def handle_disconnect():
current_app.logger.debug('SocketIO: Client disconnected')
@socketio.on('heartbeat')
def handle_heartbeat():
current_app.logger.debug('SocketIO: Heartbeat received')
last_activity = session.get('last_activity')
if datetime.now() - last_activity > current_app.config.get('SOCKETIO_MAX_IDLE_TIME'):
current_app.logger.debug('SocketIO: Heartbeat timed out, connection closed')
disconnect()
@socketio.on('user_message')
def handle_message(data):
try:
current_app.logger.debug(f"SocketIO: Message handling received message from tenant {data['tenantId']}: "
f"{data['message']} with token {data['token']}")
session['last_activity'] = datetime.now()
current_tenant_id = validate_incoming_data(data)

BIN
nginx/.DS_Store vendored
View File

Binary file not shown.

12
nginx/public/chat_ae.html
View File

@@ -7,19 +7,19 @@
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
<script src="https://cdn.socket.io/4.0.1/socket.io.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
<script src="/js/eveai-sdk.js" defer></script>
<script src="/js/eveai-chat-widget.js" defer></script>
<link rel="stylesheet" href="/css/eveai-chat-style.css">
<script src="/static/js/eveai-sdk.js" defer></script>
<script src="/static/js/eveai-chat-widget.js" defer></script>
<link rel="stylesheet" href="/static/css/eveai-chat-style.css">
</head>
<body>
<div id="chat-container"></div>
<script>
document.addEventListener('DOMContentLoaded', function() {
const eveAI = new EveAI(
'39',
'EveAI-CHAT-6919-1265-9848-6655-9870',
'3',
'EveAI-CHAT-7170-9132-8956-1484-6954',
'http://macstudio.ask-eve-ai-local.com',
'nl'
'en'
);
eveAI.initializeChat('chat-container');
});

21
nginx/static/css/eveai-chat-style.css
View File

@@ -37,6 +37,25 @@
/* Thumb colors */
--thumb-icon-outlined: #4b4b4b;
--thumb-icon-filled: #e9e9e9;
/* Connection Status colors */
--status-connected-color: #28a745; /* Green color for connected status */
--status-disconnected-color: #ffc107; /* Orange color for disconnected status */
}
/* Connection status styles */
.connection-status-icon {
vertical-align: middle;
font-size: 24px;
margin-right: 8px;
}
.status-connected {
color: var(--status-connected-color);
}
.status-disconnected {
color: var(--status-disconnected-color);
}
/* Custom scrollbar styles */
@@ -177,7 +196,7 @@
border-top: 1px solid #ccc; /* Subtle top border */
display: flex;
align-items: center;
justify-content: center;
justify-content: flex-start;
}
/* Algorithm-specific colors for fingerprint icon */

89
nginx/static/js/eveai-chat-widget.js
View File

@@ -9,6 +9,9 @@ class EveAIChatWidget extends HTMLElement {
this.attributesSet = false; // Flag to check if all attributes are set
this.jwtToken = null; // Initialize jwtToken to null
this.userTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone; // Detect user's timezone
this.heartbeatInterval = null;
this.idleTime = 0; // in milliseconds
this.maxConnectionIdleTime = 1 * 60 * 60 * 1000; // 1 hours in milliseconds
console.log('EveAIChatWidget constructor called');
}
@@ -19,6 +22,8 @@ class EveAIChatWidget extends HTMLElement {
this.questionInput = this.querySelector('.question-area input');
this.sendButton = this.querySelector('.send-icon');
this.statusLine = this.querySelector('.status-line');
this.statusMessage = this.querySelector('.status-message');
this.connectionStatusIcon = this.querySelector('.connection-status-icon');
this.sendButton.addEventListener('click', () => this.handleSendMessage());
this.questionInput.addEventListener('keydown', (event) => {
@@ -93,7 +98,10 @@ class EveAIChatWidget extends HTMLElement {
},
auth: {
token: 'Bearer ' + this.apiKey // Ensure token is included here
}
},
reconnectionAttempts: Infinity, // Infinite reconnection attempts
reconnectionDelay: 5000, // Delay between reconnections
timeout: 20000 // Connection timeout
});
console.log(`Finished initializing socket connection to ${this.domain}`);
@@ -101,6 +109,8 @@ class EveAIChatWidget extends HTMLElement {
this.socket.on('connect', (data) => {
console.log('Socket connected OK');
this.setStatusMessage('Connected to EveAI.');
this.updateConnectionStatus(true);
this.startHeartbeat();
});
this.socket.on('authenticated', (data) => {
@@ -114,16 +124,36 @@ class EveAIChatWidget extends HTMLElement {
this.socket.on('connect_error', (err) => {
console.error('Socket connection error:', err);
this.setStatusMessage('EveAI Chat Widget needs further configuration by site administrator.');
this.updateConnectionStatus(false);
});
this.socket.on('connect_timeout', () => {
console.error('Socket connection timeout');
this.setStatusMessage('EveAI Chat Widget needs further configuration by site administrator.');
this.updateConnectionStatus(false);
});
this.socket.on('disconnect', () => {
console.log('Socket disconnected');
this.socket.on('disconnect', (reason) => {
console.log('Socket disconnected: ', reason);
if (reason === 'io server disconnect') {
// Server disconnected the socket
this.socket.connect(); // Attempt to reconnect
}
this.setStatusMessage('Disconnected from EveAI. Please refresh the page for further interaction.');
this.updateConnectionStatus(false);
this.stopHeartbeat();
});
this.socket.on('reconnect_attempt', () => {
console.log('Attempting to reconnect to the server...');
this.setStatusMessage('Attempting to reconnect...');
});
this.socket.on('reconnect', () => {
console.log('Successfully reconnected to the server');
this.setStatusMessage('Reconnected to EveAI.');
this.updateConnectionStatus(true);
this.startHeartbeat();
});
this.socket.on('bot_response', (data) => {
@@ -153,19 +183,54 @@ class EveAIChatWidget extends HTMLElement {
}
setStatusMessage(message) {
this.statusLine.textContent = message;
this.statusMessage.textContent = message;
}
updateConnectionStatus(isConnected) {
if (isConnected) {
this.connectionStatusIcon.textContent = 'link';
this.connectionStatusIcon.classList.remove('status-disconnected');
this.connectionStatusIcon.classList.add('status-connected');
} else {
this.connectionStatusIcon.textContent = 'link_off';
this.connectionStatusIcon.classList.remove('status-connected');
this.connectionStatusIcon.classList.add('status-disconnected');
}
}
startHeartbeat() {
this.stopHeartbeat(); // Clear any existing interval
this.heartbeatInterval = setInterval(() => {
if (this.socket && this.socket.connected) {
this.socket.emit('heartbeat');
this.idleTime += 30000;
if (this.idleTime >= this.maxConnectionIdleTime) {
this.socket.disconnect();
this.setStatusMessage('Disconnected due to inactivity.');
this.updateConnectionStatus(false);
this.stopHeartbeat();
}
}
}, 30000); // Send a heartbeat every 30 seconds
}
stopHeartbeat() {
if (this.heartbeatInterval) {
clearInterval(this.heartbeatInterval);
this.heartbeatInterval = null;
}
}
updateProgress() {
if (!this.statusLine.textContent) {
this.statusLine.textContent = 'Processing...';
if (!this.statusMessage.textContent) {
this.statusMessage.textContent = 'Processing...';
} else {
this.statusLine.textContent += '.'; // Append a dot
this.statusMessage.textContent += '.'; // Append a dot
}
}
clearProgress() {
this.statusLine.textContent = '';
this.statusMessage.textContent = '';
this.toggleSendButton(false); // Re-enable and revert send button to outlined version
}
@@ -182,7 +247,10 @@ class EveAIChatWidget extends HTMLElement {
<input type="text" placeholder="Type your message here..." />
<i class="material-icons send-icon outlined">send</i>
</div>
<div class="status-line"></div>
<div class="status-line">
<i class="material-icons connection-status-icon">link_off</i>
<span class="status-message"></span>
</div>
</div>
`;
}
@@ -258,6 +326,8 @@ class EveAIChatWidget extends HTMLElement {
}
toggleFeedback(thumbsUp, thumbsDown, feedback, interactionId) {
console.log('feedback called');
this.idleTime = 0; // Reset idle time
if (feedback === 'up') {
thumbsUp.textContent = 'thumb_up'; // Change to filled icon
thumbsUp.classList.remove('outlined');
@@ -280,6 +350,7 @@ toggleFeedback(thumbsUp, thumbsDown, feedback, interactionId) {
handleSendMessage() {
console.log('handleSendMessage called');
this.idleTime = 0; // Reset idle time
const message = this.questionInput.value.trim();
if (message) {
this.addUserMessage(message);

1
scripts/start_eveai_chat.sh
View File

@@ -6,6 +6,7 @@ export PYTHONPATH="$PYTHONPATH:/app/"
# Set flask environment variables
#export FLASK_ENV=development # Use 'production' as appropriate
#export FLASK_DEBUG=1 # Use 0 for production
print "Starting EveAI Chat"
# Start Flask app
gunicorn -w 4 -k geventwebsocket.gunicorn.workers.GeventWebSocketWorker -b 0.0.0.0:5002 scripts.run_eveai_chat:app