pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Adapt Sync Wordpress Component to Catalog introduction

Browse Source 
- Small bug fixes
This commit is contained in:
Josako
2024-10-17 10:31:13 +02:00
parent 7f12c8b355
commit 74cc7ae95e
12 changed files with 94 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
common/langchain/eveai_retriever.py
View File

@@ -38,7 +38,7 @@ class EveAIRetriever(BaseRetriever, BaseModel):
similarity_threshold = self.model_variables['similarity_threshold']
k = self.model_variables['k']
if self.tenant_info['rag_tuning']:
if self.model_variables['rag_tuning']:
try:
current_date = get_date_in_timezone(self.tenant_info['timezone'])
current_app.rag_tuning_logger.debug(f'Current date: {current_date}\n')
@@ -73,7 +73,7 @@ class EveAIRetriever(BaseRetriever, BaseModel):
current_app.logger.error(f'Error generating overview: {e}')
db.session.rollback()
if self.tenant_info['rag_tuning']:
if self.model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Parameters for Retrieval of documents: \n')
current_app.rag_tuning_logger.debug(f'Similarity Threshold: {similarity_threshold}\n')
current_app.rag_tuning_logger.debug(f'K: {k}\n')
@@ -106,14 +106,14 @@ class EveAIRetriever(BaseRetriever, BaseModel):
.limit(k)
)
if self.tenant_info['rag_tuning']:
if self.model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Query executed for Retrieval of documents: \n')
current_app.rag_tuning_logger.debug(f'{query_obj.statement}\n')
current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
res = query_obj.all()
if self.tenant_info['rag_tuning']:
if self.model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Retrieved {len(res)} relevant documents \n')
current_app.rag_tuning_logger.debug(f'Data retrieved: \n')
current_app.rag_tuning_logger.debug(f'{res}\n')
@@ -121,7 +121,7 @@ class EveAIRetriever(BaseRetriever, BaseModel):
result = []
for doc in res:
if self.tenant_info['rag_tuning']:
if self.model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Document ID: {doc[0].id} - Distance: {doc[1]}\n')
current_app.rag_tuning_logger.debug(f'Chunk: \n {doc[0].chunk}\n\n')
result.append(f'SOURCE: {doc[0].id}\n\n{doc[0].chunk}\n\n')

2
common/utils/document_utils.py
View File

@@ -345,6 +345,6 @@ def refresh_document(doc_id, tenant_id):
# Function triggered when a document_version is created or updated
def mark_tenant_storage_dirty(tenant_id):
tenant = db.session.query(Tenant).filter_by(id=tenant_id).first()
tenant = db.session.query(Tenant).filter_by(id=int(tenant_id)).first()
tenant.storage_dirty = True
db.session.commit()

11
eveai_api/__init__.py
View File

@@ -56,13 +56,6 @@ def create_app(config_file=None):
app.logger.debug(f'Request URL: {request.url}')
app.logger.debug(f'Request headers: {dict(request.headers)}')
# Log request arguments
app.logger.debug(f'Request args: {request.args}')
# Log form data if it's a POST request
if request.method == 'POST':
app.logger.debug(f'Form data: {request.form}')
# Log JSON data if the content type is application/json
if request.is_json:
app.logger.debug(f'JSON data: {request.json}')
@@ -95,6 +88,10 @@ def create_app(config_file=None):
# Don't raise the exception here, let the request continue
# The appropriate error handling will be done in the specific endpoints
@app.route('/api/v1')
def swagger():
return api_rest.render_doc()
return app

2
eveai_api/api/document_api.py
View File

@@ -141,7 +141,7 @@ class AddURL(Resource):
file_content, filename, extension = process_url(args['url'], tenant_id)
api_input = {
'catalog_id': args['catlog_id'],
'catalog_id': args['catalog_id'],
'url': args['url'],
'name': args.get('name') or filename,
'language': args['language'],

2
eveai_app/templates/user/tenant.html
View File

@@ -47,8 +47,10 @@
{{ render_included_field(field, disabled_fields=[], include_fields=license_fields) }}
{% endfor %}
<!-- Register API Key Button -->
<div class="form-group">
<button type="button" class="btn btn-primary" onclick="generateNewChatApiKey()">Register Chat API Key</button>
<button type="button" class="btn btn-primary" onclick="generateNewApiKey()">Register API Key</button>
</div>
<!-- API Key Display Field -->
<div id="chat-api-key-field" style="display:none;">
<label for="chat-api-key">Chat API Key:</label>

8
eveai_app/views/document_views.py
View File

@@ -135,6 +135,14 @@ def edit_catalog(catalog_id):
form = CatalogForm(obj=catalog)
tenant_id = session.get('tenant').get('id')
# Convert arrays to comma-separated strings for display
if request.method == 'GET':
form.html_tags.data = ', '.join(catalog.html_tags or '')
form.html_end_tags.data = ', '.join(catalog.html_end_tags or '')
form.html_included_elements.data = ', '.join(catalog.html_included_elements or '')
form.html_excluded_elements.data = ', '.join(catalog.html_excluded_elements or '')
form.html_excluded_classes.data = ', '.join(catalog.html_excluded_classes or '')
if request.method == 'POST' and form.validate_on_submit():
form.populate_obj(catalog)
# Handle Embedding Variables

15
eveai_chat_workers/tasks.py
View File

@@ -93,12 +93,6 @@ def ask_question(tenant_id, question, language, session_id, user_timezone, room)
current_app.logger.error(f'ask_question: Error initializing chat session in database: {e}')
raise
if tenant.rag_tuning:
current_app.rag_tuning_logger.debug(f'Received question for tenant {tenant_id}:\n{question}. Processing...')
current_app.rag_tuning_logger.debug(f'Tenant Information: \n{tenant.to_dict()}')
current_app.rag_tuning_logger.debug(f'===================================================================')
current_app.rag_tuning_logger.debug(f'===================================================================')
with current_event.create_span("RAG Answer"):
result, interaction = answer_using_tenant_rag(question, language, tenant, chat_session)
result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name']
@@ -138,8 +132,7 @@ def answer_using_tenant_rag(question, language, tenant, chat_session):
with current_event.create_span("Detail Question"):
detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
if tenant.rag_tuning:
if model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Detailed Question for tenant {tenant.id}:\n{question}.')
current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
new_interaction.detailed_question = detailed_question
@@ -153,7 +146,7 @@ def answer_using_tenant_rag(question, language, tenant, chat_session):
full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context'])
rag_prompt = ChatPromptTemplate.from_template(full_template)
setup_and_retrieval = RunnableParallel({"context": retriever, "question": RunnablePassthrough()})
if tenant.rag_tuning:
if model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Full prompt for tenant {tenant.id}:\n{full_template}.')
current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
@@ -181,7 +174,7 @@ def answer_using_tenant_rag(question, language, tenant, chat_session):
current_app.logger.debug(f'ask_question: result answer: {result['answer']}')
current_app.logger.debug(f'ask_question: result citations: {result["citations"]}')
current_app.logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
if tenant.rag_tuning:
if model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'ask_question: result answer: {result['answer']}')
current_app.rag_tuning_logger.debug(f'ask_question: result citations: {result["citations"]}')
current_app.rag_tuning_logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
@@ -197,7 +190,7 @@ def answer_using_tenant_rag(question, language, tenant, chat_session):
)
existing_embedding_ids = [emb.id for emb in embeddings]
urls = list(set(emb.document_version.url for emb in embeddings))
if tenant.rag_tuning:
if model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Referenced documents for answer for tenant {tenant.id}:\n')
current_app.rag_tuning_logger.debug(f'{urls}')
current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')

3
eveai_workers/Processors/html_processor.py
View File

@@ -15,6 +15,7 @@ class HTMLProcessor(Processor):
self.html_end_tags = model_variables['html_end_tags']
self.html_included_elements = model_variables['html_included_elements']
self.html_excluded_elements = model_variables['html_excluded_elements']
self.html_excluded_classes = model_variables['html_excluded_classes']
self.chunk_size = model_variables['processing_chunk_size'] # Adjust this based on your LLM's optimal input size
self.chunk_overlap = model_variables[
'processing_chunk_overlap'] # Adjust for context preservation between chunks
@@ -45,7 +46,7 @@ class HTMLProcessor(Processor):
self._log(f'Parsing HTML for tenant {self.tenant.id}')
soup = BeautifulSoup(html_content, 'html.parser')
extracted_html = ''
excluded_classes = self._parse_excluded_classes(self.tenant.html_excluded_classes)
excluded_classes = self._parse_excluded_classes(self.html_excluded_classes)
if self.html_included_elements:
elements_to_parse = soup.find_all(self.html_included_elements)

4
integrations/Wordpress/eveai_sync/README.md
View File

@@ -51,6 +51,10 @@ No additional configuration is needed; the plugin will automatically detect the
## Versions
### 1.1.1 - Add Reinitialisation functionality
### 1.1.0 - Add Catalog Functionality
### 1.0.x - Bugfixing Releases
### 1.0.0 - Initial Release

28
integrations/Wordpress/eveai_sync/eveai_sync.php
View File

@@ -3,7 +3,7 @@
* Plugin Name: EveAI Sync
* Plugin URI: https://askeveai.com/
* Description: Synchronizes WordPress content with EveAI API.
* Version: 1.0.16
* Version: 1.1.1
* Author: Josako, Pieter Laroy
* Author URI: https://askeveai.com/about/
* License: GPL v2 or later
@@ -17,7 +17,7 @@ if (!defined('ABSPATH')) {
}
// Define plugin constants
define('EVEAI_SYNC_VERSION', '1.0.0');
define('EVEAI_SYNC_VERSION', '1.1.1');
define('EVEAI_SYNC_PLUGIN_DIR', plugin_dir_path(__FILE__));
define('EVEAI_SYNC_PLUGIN_URL', plugin_dir_url(__FILE__));
@@ -50,6 +50,30 @@ function eveai_delete_post_meta($post_id) {
}
add_action('before_delete_post', 'eveai_delete_post_meta');
// Clean metadata from Wordpress site
function eveai_reinitialize_site() {
check_ajax_referer('eveai_reinitialize_site', 'nonce');
if (!current_user_can('manage_options')) {
wp_send_json_error('You do not have permission to perform this action.');
return;
}
global $wpdb;
// Remove all EveAI-related post meta
$wpdb->query("DELETE FROM $wpdb->postmeta WHERE meta_key LIKE '_eveai_%'");
// Remove all EveAI-related options
delete_option('eveai_last_sync_time');
delete_option('eveai_sync_status');
// Optionally, you might want to clear any custom tables if you have any
wp_send_json_success('Site reinitialized. All EveAI metadata has been removed.');
}
add_action('wp_ajax_eveai_reinitialize_site', 'eveai_reinitialize_site');
// Display sync info in post
function eveai_display_sync_info($post) {
$document_id = get_post_meta($post->ID, '_eveai_document_id', true);

33
integrations/Wordpress/eveai_sync/includes/class-eveai-admin.php
View File

@@ -16,6 +16,7 @@ class EveAI_Admin {
register_setting('eveai_settings', 'eveai_excluded_categories');
register_setting('eveai_settings', 'eveai_access_token');
register_setting('eveai_settings', 'eveai_token_expiry');
register_setting('eveai_settings', 'eveai_catalog_id');
}
public function add_admin_menu() {
@@ -50,6 +51,10 @@ class EveAI_Admin {
<th scope="row">API Key</th>
<td><input type="text" name="eveai_api_key" value="<?php echo esc_attr(get_option('eveai_api_key')); ?>" style="width: 100%;" /></td>
</tr>
<tr valign="top">
<th scope="row">Catalog ID</th>
<td><input type="text" name="eveai_catalog_id" value="<?php echo esc_attr(get_option('eveai_catalog_id')); ?>" style="width: 100%;" /></td>
</tr>
<tr valign="top">
<th scope="row">Default Language</th>
<td><input type="text" name="eveai_default_language" value="<?php echo esc_attr(get_option('eveai_default_language', 'en')); ?>" style="width: 100%;" /></td>
@@ -71,6 +76,11 @@ class EveAI_Admin {
<?php wp_nonce_field('eveai_bulk_sync', 'eveai_bulk_sync_nonce'); ?>
<input type="submit" name="eveai_bulk_sync" class="button button-primary" value="Start Bulk Sync">
</form>
<h2>Reinitialize Site</h2>
<p>Click the button below to remove all EveAI metadata from your site. This will reset the sync status for all posts and pages.</p>
<button id="eveai-reinitialize" class="button button-secondary">Reinitialize Site</button>
<div id="eveai-sync-results" style="margin-top: 20px;"></div>
</div>
<script>
@@ -105,6 +115,29 @@ class EveAI_Admin {
});
}
});
$('#eveai-reinitialize').on('click', function(e) {
e.preventDefault();
if (confirm('Are you sure you want to reinitialize? This will remove all EveAI metadata from your site.')) {
$.ajax({
url: ajaxurl,
type: 'POST',
data: {
action: 'eveai_reinitialize_site',
nonce: '<?php echo wp_create_nonce('eveai_reinitialize_site'); ?>'
},
success: function(response) {
if (response.success) {
alert(response.data);
} else {
alert('Error: ' + response.data);
}
},
error: function() {
alert('An error occurred. Please try again.');
}
});
}
});
});
</script>
<?php

2
integrations/Wordpress/eveai_sync/includes/class-eveai-api.php
View File

@@ -13,6 +13,7 @@ class EveAI_API {
$this->api_key = get_option('eveai_api_key');
$this->access_token = get_option('eveai_access_token');
$this->token_expiry = get_option('eveai_token_expiry', 0);
$this->catalog_id = get_option('eveai_catalog_id');
}
private function ensure_valid_token() {
@@ -111,6 +112,7 @@ class EveAI_API {
}
public function add_url($data) {
$data['catalog_id'] = get_option('eveai_catalog_id'); // Include catalog_id
return $this->make_request('POST', '/api/v1/documents/add_url', $data);
}