pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Refined entitlements to work with MiB for both embeddings and storage

Browse Source 
- Improved DocumentVersion storage attributes to reflect Minio settings
- Added size to DocumentVersions to easily calculate usage
- License / LicenseTier forms and views added
This commit is contained in:
Josako
2024-10-07 14:17:44 +02:00
parent f638860e90
commit 9782e31ae5
31 changed files with 1416 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.idea/sqldialects.xml generated
View File

@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="PROJECT" dialect="PostgreSQL" />
</component>
</project>

5
common/models/document.py
View File

@@ -28,9 +28,10 @@ class DocumentVersion(db.Model):
id = db.Column(db.Integer, primary_key=True)
doc_id = db.Column(db.Integer, db.ForeignKey(Document.id), nullable=False)
url = db.Column(db.String(200), nullable=True)
file_location = db.Column(db.String(255), nullable=True)
file_name = db.Column(db.String(200), nullable=True)
bucket_name = db.Column(db.String(255), nullable=True)
object_name = db.Column(db.String(200), nullable=True)
file_type = db.Column(db.String(20), nullable=True)
file_size = db.Column(db.Float, nullable=True)
language = db.Column(db.String(2), nullable=False)
user_context = db.Column(db.Text, nullable=True)
system_context = db.Column(db.Text, nullable=True)

107
common/models/entitlements.py Normal file
View File

@@ -0,0 +1,107 @@
from common.extensions import db
class BusinessEventLog(db.Model):
__bind_key__ = 'public'
__table_args__ = {'schema': 'public'}
id = db.Column(db.Integer, primary_key=True)
timestamp = db.Column(db.DateTime, nullable=False)
event_type = db.Column(db.String(50), nullable=False)
tenant_id = db.Column(db.Integer, nullable=False)
trace_id = db.Column(db.String(50), nullable=False)
span_id = db.Column(db.String(50))
span_name = db.Column(db.String(50))
parent_span_id = db.Column(db.String(50))
document_version_id = db.Column(db.Integer)
document_version_file_size = db.Column(db.Float)
chat_session_id = db.Column(db.String(50))
interaction_id = db.Column(db.Integer)
environment = db.Column(db.String(20))
llm_metrics_total_tokens = db.Column(db.Integer)
llm_metrics_prompt_tokens = db.Column(db.Integer)
llm_metrics_completion_tokens = db.Column(db.Integer)
llm_metrics_total_time = db.Column(db.Float)
llm_metrics_call_count = db.Column(db.Integer)
llm_interaction_type = db.Column(db.String(20))
message = db.Column(db.Text)
license_usage_id = db.Column(db.Integer, db.ForeignKey('public.license_usage.id'), nullable=True)
license_usage = db.relationship('LicenseUsage', backref='events')
class License(db.Model):
__bind_key__ = 'public'
__table_args__ = {'schema': 'public'}
id = db.Column(db.Integer, primary_key=True)
tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
tier_id = db.Column(db.Integer, db.ForeignKey('public.license_tier.id'),nullable=False) # 'small', 'medium', 'custom'
start_date = db.Column(db.Date, nullable=False)
end_date = db.Column(db.Date, nullable=True)
currency = db.Column(db.String(20), nullable=False)
yearly_payment = db.Column(db.Boolean, nullable=False, default=False)
basic_fee = db.Column(db.Float, nullable=False)
max_storage_mb = db.Column(db.Integer, nullable=False)
additional_storage_price = db.Column(db.Float, nullable=False)
additional_storage_bucket = db.Column(db.Integer, nullable=False)
included_embedding_mb = db.Column(db.Integer, nullable=False)
additional_embedding_price = db.Column(db.Numeric(10, 4), nullable=False)
additional_embedding_bucket = db.Column(db.Integer, nullable=False)
included_interaction_tokens = db.Column(db.Integer, nullable=False)
additional_interaction_token_price = db.Column(db.Numeric(10, 4), nullable=False)
additional_interaction_bucket = db.Column(db.Integer, nullable=False)
overage_embedding = db.Column(db.Float, nullable=False, default=0)
overage_interaction = db.Column(db.Float, nullable=False, default=0)
tenant = db.relationship('Tenant', back_populates='licenses')
license_tier = db.relationship('LicenseTier', back_populates='licenses')
usages = db.relationship('LicenseUsage', order_by='LicenseUsage.period_start_date', back_populates='license')
class LicenseTier(db.Model):
__bind_key__ = 'public'
__table_args__ = {'schema': 'public'}
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(50), nullable=False)
version = db.Column(db.String(50), nullable=False)
start_date = db.Column(db.Date, nullable=False)
end_date = db.Column(db.Date, nullable=True)
basic_fee_d = db.Column(db.Float, nullable=True)
basic_fee_e = db.Column(db.Float, nullable=True)
max_storage_mb = db.Column(db.Integer, nullable=False)
additional_storage_price_d = db.Column(db.Numeric(10, 4), nullable=False)
additional_storage_price_e = db.Column(db.Numeric(10, 4), nullable=False)
additional_storage_bucket = db.Column(db.Integer, nullable=False)
included_embedding_mb = db.Column(db.Integer, nullable=False)
additional_embedding_price_d = db.Column(db.Numeric(10, 4), nullable=False)
additional_embedding_price_e = db.Column(db.Numeric(10, 4), nullable=False)
additional_embedding_bucket = db.Column(db.Integer, nullable=False)
included_interaction_tokens = db.Column(db.Integer, nullable=False)
additional_interaction_token_price_d = db.Column(db.Numeric(10, 4), nullable=False)
additional_interaction_token_price_e = db.Column(db.Numeric(10, 4), nullable=False)
additional_interaction_bucket = db.Column(db.Integer, nullable=False)
standard_overage_embedding = db.Column(db.Float, nullable=False, default=0)
standard_overage_interaction = db.Column(db.Float, nullable=False, default=0)
licenses = db.relationship('License', back_populates='license_tier')
class LicenseUsage(db.Model):
__bind_key__ = 'public'
__table_args__ = {'schema': 'public'}
id = db.Column(db.Integer, primary_key=True)
license_id = db.Column(db.Integer, db.ForeignKey('public.license.id'), nullable=False)
tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
storage_mb_used = db.Column(db.Integer, default=0)
storage_tokens_used = db.Column(db.Integer, default=0)
embedding_mb_used = db.Column(db.Integer, default=0)
embedding_tokens_used = db.Column(db.Integer, default=0)
interaction_tokens_used = db.Column(db.Integer, default=0)
period_start_date = db.Column(db.Date, nullable=False)
period_end_date = db.Column(db.Date, nullable=False)
license = db.relationship('License', back_populates='usages')

27
common/models/monitoring.py
View File

@@ -1,27 +0,0 @@
from common.extensions import db
class BusinessEventLog(db.Model):
__bind_key__ = 'public'
__table_args__ = {'schema': 'public'}
id = db.Column(db.Integer, primary_key=True)
timestamp = db.Column(db.DateTime, nullable=False)
event_type = db.Column(db.String(50), nullable=False)
tenant_id = db.Column(db.Integer, nullable=False)
trace_id = db.Column(db.String(50), nullable=False)
span_id = db.Column(db.String(50))
span_name = db.Column(db.String(50))
parent_span_id = db.Column(db.String(50))
document_version_id = db.Column(db.Integer)
chat_session_id = db.Column(db.String(50))
interaction_id = db.Column(db.Integer)
environment = db.Column(db.String(20))
llm_metrics_total_tokens = db.Column(db.Integer)
llm_metrics_prompt_tokens = db.Column(db.Integer)
llm_metrics_completion_tokens = db.Column(db.Integer)
llm_metrics_total_time = db.Column(db.Float)
llm_metrics_call_count = db.Column(db.Integer)
llm_interaction_type = db.Column(db.String(20))
message = db.Column(db.Text)
# Add any other fields relevant for invoicing or warnings

27
common/models/user.py
View File

@@ -1,8 +1,12 @@
from datetime import date
from common.extensions import db
from flask_security import UserMixin, RoleMixin
from sqlalchemy.dialects.postgresql import ARRAY
import sqlalchemy as sa
from common.models.entitlements import License
class Tenant(db.Model):
"""Tenant model"""
@@ -50,9 +54,6 @@ class Tenant(db.Model):
fallback_algorithms = db.Column(ARRAY(sa.String(50)), nullable=True)
# Licensing Information
license_start_date = db.Column(db.Date, nullable=True)
license_end_date = db.Column(db.Date, nullable=True)
allowed_monthly_interactions = db.Column(db.Integer, nullable=True)
encrypted_chat_api_key = db.Column(db.String(500), nullable=True)
encrypted_api_key = db.Column(db.String(500), nullable=True)
@@ -60,9 +61,24 @@ class Tenant(db.Model):
embed_tuning = db.Column(db.Boolean, nullable=True, default=False)
rag_tuning = db.Column(db.Boolean, nullable=True, default=False)
# Entitlements
currency = db.Column(db.String(20), nullable=True)
usage_email = db.Column(db.String(255), nullable=True)
# Relations
users = db.relationship('User', backref='tenant')
domains = db.relationship('TenantDomain', backref='tenant')
licenses = db.relationship('License', back_populates='tenant')
license_usages = db.relationship('LicenseUsage', backref='tenant')
@property
def current_license(self):
today = date.today()
return License.query.filter(
License.tenant_id == self.id,
License.start_date <= today,
(License.end_date.is_(None) | (License.end_date >= today))
).order_by(License.start_date.desc()).first()
def __repr__(self):
return f"<Tenant {self.id}: {self.name}>"
@@ -91,11 +107,10 @@ class Tenant(db.Model):
'chat_RAG_temperature': self.chat_RAG_temperature,
'chat_no_RAG_temperature': self.chat_no_RAG_temperature,
'fallback_algorithms': self.fallback_algorithms,
'license_start_date': self.license_start_date,
'license_end_date': self.license_end_date,
'allowed_monthly_interactions': self.allowed_monthly_interactions,
'embed_tuning': self.embed_tuning,
'rag_tuning': self.rag_tuning,
'currency': self.currency,
'usage_email': self.usage_email,
}

9
common/utils/business_event.py
View File

@@ -8,7 +8,7 @@ from portkey_ai import Portkey, Config
import logging
from .business_event_context import BusinessEventContext
from common.models.monitoring import BusinessEventLog
from common.models.entitlements import BusinessEventLog
from common.extensions import db
@@ -25,6 +25,7 @@ class BusinessEvent:
self.span_name = None
self.parent_span_id = None
self.document_version_id = kwargs.get('document_version_id')
self.document_version_file_size = kwargs.get('document_version_file_size')
self.chat_session_id = kwargs.get('chat_session_id')
self.interaction_id = kwargs.get('interaction_id')
self.environment = os.environ.get("FLASK_ENV", "development")
@@ -107,6 +108,7 @@ class BusinessEvent:
'span_name': self.span_name,
'parent_span_id': self.parent_span_id,
'document_version_id': self.document_version_id,
'document_version_file_size': self.document_version_file_size,
'chat_session_id': self.chat_session_id,
'interaction_id': self.interaction_id,
'environment': self.environment,
@@ -124,6 +126,7 @@ class BusinessEvent:
span_name=self.span_name,
parent_span_id=self.parent_span_id,
document_version_id=self.document_version_id,
document_version_file_size=self.document_version_file_size,
chat_session_id=self.chat_session_id,
interaction_id=self.interaction_id,
environment=self.environment,
@@ -144,6 +147,7 @@ class BusinessEvent:
'span_name': self.span_name,
'parent_span_id': self.parent_span_id,
'document_version_id': self.document_version_id,
'document_version_file_size': self.document_version_file_size,
'chat_session_id': self.chat_session_id,
'interaction_id': self.interaction_id,
'environment': self.environment,
@@ -166,6 +170,7 @@ class BusinessEvent:
span_name=self.span_name,
parent_span_id=self.parent_span_id,
document_version_id=self.document_version_id,
document_version_file_size=self.document_version_file_size,
chat_session_id=self.chat_session_id,
interaction_id=self.interaction_id,
environment=self.environment,
@@ -190,6 +195,7 @@ class BusinessEvent:
'span_name': self.span_name,
'parent_span_id': self.parent_span_id,
'document_version_id': self.document_version_id,
'document_version_file_size': self.document_version_file_size,
'chat_session_id': self.chat_session_id,
'interaction_id': self.interaction_id,
'environment': self.environment,
@@ -213,6 +219,7 @@ class BusinessEvent:
span_name=self.span_name,
parent_span_id=self.parent_span_id,
document_version_id=self.document_version_id,
document_version_file_size=self.document_version_file_size,
chat_session_id=self.chat_session_id,
interaction_id=self.interaction_id,
environment=self.environment,

8
common/utils/document_utils.py
View File

@@ -86,14 +86,12 @@ def create_version_for_document(document, url, language, user_context, user_meta
def upload_file_for_version(doc_vers, file, extension, tenant_id):
doc_vers.file_type = extension
doc_vers.file_name = doc_vers.calc_file_name()
doc_vers.file_location = doc_vers.calc_file_location()
# Normally, the tenant bucket should exist. But let's be on the safe side if a migration took place.
minio_client.create_tenant_bucket(tenant_id)
try:
minio_client.upload_document_file(
bn, on, size = minio_client.upload_document_file(
tenant_id,
doc_vers.doc_id,
doc_vers.language,
@@ -101,6 +99,10 @@ def upload_file_for_version(doc_vers, file, extension, tenant_id):
doc_vers.file_name,
file
)
doc_vers.bucket_name = bn
doc_vers.object_name = on
doc_vers.file_size_mb = size / 1048576 # Convert bytes to MB
db.session.commit()
current_app.logger.info(f'Successfully saved document to MinIO for tenant {tenant_id} for '
f'document version {doc_vers.id} while uploading file.')

2
common/utils/minio_utils.py
View File

@@ -50,7 +50,7 @@ class MinioClient:
self.client.put_object(
bucket_name, object_name, io.BytesIO(file_data), len(file_data)
)
return True
return bucket_name, object_name, len(file_data)
except S3Error as err:
raise Exception(f"Error occurred while uploading file: {err}")

2
common/utils/view_assistants.py
View File

@@ -44,7 +44,7 @@ def form_validation_failed(request, form):
for fieldName, errorMessages in form.errors.items():
for err in errorMessages:
flash(f"Error in {fieldName}: {err}", 'danger')
current_app.logger.debug(f"Error in {fieldName}: {err}", 'danger')
current_app.logger.debug(f"Error in {fieldName}: {err}")
def form_to_dict(form):

3
config/config.py
View File

@@ -59,6 +59,9 @@ class Config(object):
# supported languages
SUPPORTED_LANGUAGES = ['en', 'fr', 'nl', 'de', 'es']
# supported currencies
SUPPORTED_CURRENCIES = ['€', '$']
# supported LLMs
SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed']
SUPPORTED_LLMS = ['openai.gpt-4o', 'anthropic.claude-3-5-sonnet', 'openai.gpt-4o-mini']

4
eveai_app/__init__.py
View File

@@ -10,7 +10,7 @@ from common.extensions import (db, migrate, bootstrap, security, mail, login_man
minio_client, simple_encryption, metrics)
from common.models.user import User, Role, Tenant, TenantDomain
import common.models.interaction
import common.models.monitoring
import common.models.entitlements
import common.models.document
from common.utils.nginx_utils import prefixed_url_for
from config.logging_config import LOGGING
@@ -134,6 +134,8 @@ def register_blueprints(app):
app.register_blueprint(security_bp)
from .views.interaction_views import interaction_bp
app.register_blueprint(interaction_bp)
from .views.entitlements_views import entitlements_bp
app.register_blueprint(entitlements_bp)
from .views.healthz_views import healthz_bp, init_healtz
app.register_blueprint(healthz_bp)
init_healtz(app)

71
eveai_app/templates/entitlements/edit_license.html Normal file
View File

@@ -0,0 +1,71 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field, render_included_field %}
{% block title %}Edit License for Current Tenant{% endblock %}
{% block content_title %}Edit License for Current Tenant{% endblock %}
{% block content_description %}Edit a License based on the selected License Tier for the current Tenant{% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set main_fields = ['start_date', 'end_date', 'currency', 'yearly_payment', 'basic_fee'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=['currency'], include_fields=main_fields) }}
{% endfor %}
<!-- Nav Tabs -->
<div class="row mt-5">
<div class="col-lg-12">
<div class="nav-wrapper position-relative end-0">
<ul class="nav nav-pills nav-fill p-1" role="tablist">
<li class="nav-item" role="presentation">
<a class="nav-link mb-0 px-0 py-1 active" data-toggle="tab" href="#storage-tab" role="tab" aria-controls="model-info" aria-selected="true">
Storage
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#embedding-tab" role="tab" aria-controls="license-info" aria-selected="false">
Embedding
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#interaction-tab" role="tab" aria-controls="chunking" aria-selected="false">
Interaction
</a>
</li>
</ul>
</div>
<div class="tab-content tab-space">
<!-- Storage Tab -->
<div class="tab-pane fade show active" id="storage-tab" role="tabpanel">
{% set storage_fields = ['max_storage_tokens', 'additional_storage_token_price', 'additional_storage_bucket'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=storage_fields) }}
{% endfor %}
</div>
<!-- Embedding Tab -->
<div class="tab-pane fade" id="embedding-tab" role="tabpanel">
{% set embedding_fields = ['included_embedding_tokens', 'additional_embedding_token_price', 'additional_embedding_bucket'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=embedding_fields) }}
{% endfor %}
</div>
<!-- Interaction Tab -->
<div class="tab-pane fade" id="interaction-tab" role="tabpanel">
{% set interaction_fields = ['included_interaction_tokens', 'additional_interaction_token_price', 'additional_interaction_bucket'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=interaction_fields) }}
{% endfor %}
</div>
</div>
</div>
</div>
<button type="submit" class="btn btn-primary">Save License</button>
</form>
{% endblock %}
{% block content_footer %}
{% endblock %}

71
eveai_app/templates/entitlements/license.html Normal file
View File

@@ -0,0 +1,71 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field, render_included_field %}
{% block title %}Create or Edit License for Current Tenant{% endblock %}
{% block content_title %}Create or Edit License for Current Tenant{% endblock %}
{% block content_description %}Create or Edit a new License based on the selected License Tier for the current Tenant{% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set main_fields = ['start_date', 'end_date', 'currency', 'yearly_payment', 'basic_fee'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=['currency'], include_fields=main_fields) }}
{% endfor %}
<!-- Nav Tabs -->
<div class="row mt-5">
<div class="col-lg-12">
<div class="nav-wrapper position-relative end-0">
<ul class="nav nav-pills nav-fill p-1" role="tablist">
<li class="nav-item" role="presentation">
<a class="nav-link mb-0 px-0 py-1 active" data-toggle="tab" href="#storage-tab" role="tab" aria-controls="model-info" aria-selected="true">
Storage
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#embedding-tab" role="tab" aria-controls="license-info" aria-selected="false">
Embedding
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#interaction-tab" role="tab" aria-controls="chunking" aria-selected="false">
Interaction
</a>
</li>
</ul>
</div>
<div class="tab-content tab-space">
<!-- Storage Tab -->
<div class="tab-pane fade show active" id="storage-tab" role="tabpanel">
{% set storage_fields = ['max_storage_mb', 'additional_storage_price', 'additional_storage_bucket'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=storage_fields) }}
{% endfor %}
</div>
<!-- Embedding Tab -->
<div class="tab-pane fade" id="embedding-tab" role="tabpanel">
{% set embedding_fields = ['included_embedding_mb', 'additional_embedding_price', 'additional_embedding_bucket', 'overage_embedding'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=embedding_fields) }}
{% endfor %}
</div>
<!-- Interaction Tab -->
<div class="tab-pane fade" id="interaction-tab" role="tabpanel">
{% set interaction_fields = ['included_interaction_tokens', 'additional_interaction_token_price', 'additional_interaction_bucket', 'overage_interaction'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=interaction_fields) }}
{% endfor %}
</div>
</div>
</div>
</div>
<button type="submit" class="btn btn-primary">Save License</button>
</form>
{% endblock %}
{% block content_footer %}
{% endblock %}

71
eveai_app/templates/entitlements/license_tier.html Normal file
View File

@@ -0,0 +1,71 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field, render_included_field %}
{% block title %}Register or Edit License Tier{% endblock %}
{% block content_title %}Register or Edit License Tier{% endblock %}
{% block content_description %}Register or Edit License Tier{% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set main_fields = ['name', 'version', 'start_date', 'end_date', 'basic_fee_d', 'basic_fee_e'] %}
{% for field in form %}
{{ render_included_field(field, ext_disabled_fields=[], include_fields=main_fields) }}
{% endfor %}
<!-- Nav Tabs -->
<div class="row mt-5">
<div class="col-lg-12">
<div class="nav-wrapper position-relative end-0">
<ul class="nav nav-pills nav-fill p-1" role="tablist">
<li class="nav-item" role="presentation">
<a class="nav-link mb-0 px-0 py-1 active" data-toggle="tab" href="#storage-tab" role="tab" aria-controls="model-info" aria-selected="true">
Storage
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#embedding-tab" role="tab" aria-controls="license-info" aria-selected="false">
Embedding
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#interaction-tab" role="tab" aria-controls="chunking" aria-selected="false">
Interaction
</a>
</li>
</ul>
</div>
<div class="tab-content tab-space">
<!-- Storage Tab -->
<div class="tab-pane fade show active" id="storage-tab" role="tabpanel">
{% set storage_fields = ['max_storage_mb', 'additional_storage_price_d', 'additional_storage_price_e', 'additional_storage_bucket'] %}
{% for field in form %}
{{ render_included_field(field, ext_disabled_fields=[], include_fields=storage_fields) }}
{% endfor %}
</div>
<!-- Embedding Tab -->
<div class="tab-pane fade" id="embedding-tab" role="tabpanel">
{% set embedding_fields = ['included_embedding_mb', 'additional_embedding_price_d', 'additional_embedding_price_e', 'additional_embedding_bucket', 'standard_overage_embedding'] %}
{% for field in form %}
{{ render_included_field(field, ext_disabled_fields=[], include_fields=embedding_fields) }}
{% endfor %}
</div>
<!-- Interaction Tab -->
<div class="tab-pane fade" id="interaction-tab" role="tabpanel">
{% set interaction_fields = ['included_interaction_tokens', 'additional_interaction_token_price_d', 'additional_interaction_token_price_e', 'additional_interaction_bucket', 'standard_overage_interaction'] %}
{% for field in form %}
{{ render_included_field(field, ext_disabled_fields=[], include_fields=interaction_fields) }}
{% endfor %}
</div>
</div>
</div>
</div>
<button type="submit" class="btn btn-primary">Save License Tier</button>
</form>
{% endblock %}
{% block content_footer %}
{% endblock %}

24
eveai_app/templates/entitlements/view_license_tiers.html Normal file
View File

@@ -0,0 +1,24 @@
{% extends 'base.html' %}
{% from "macros.html" import render_selectable_table, render_pagination, render_field %}
{% block title %}License Tier Selection{% endblock %}
{% block content_title %}Select a License Tier{% endblock %}
{% block content_description %}Select a License Tier to continue{% endblock %}
{% block content %}
<!-- License Tier Selection Form -->
<form method="POST" action="{{ url_for('entitlements_bp.handle_license_tier_selection') }}">
{{ render_selectable_table(headers=["Name", "Version", "Start Date", "End Date"], rows=rows, selectable=True, id="licenseTierTable") }}
<div class="form-group mt-3">
<button type="submit" name="action" value="edit_license_tier" class="btn btn-primary">Edit License Tier</button>
<button type="submit" name="action" value="create_license_for_tenant" class="btn btn-secondary">Create License for Current Tenant</button>
</div>
</form>
{% endblock %}
{% block content_footer %}
{{ render_pagination(pagination, 'user_bp.select_tenant') }}
{% endblock %}

6
eveai_app/templates/navbar.html
View File

@@ -94,6 +94,12 @@
{'name': 'Chat Sessions', 'url': '/interaction/chat_sessions', 'roles': ['Super User', 'Tenant Admin']},
]) }}
{% endif %}
{% if current_user.is_authenticated %}
{{ dropdown('Administration', 'settings', [
{'name': 'License Tier Registration', 'url': '/entitlements/license_tier', 'roles': ['Super User']},
{'name': 'All License Tiers', 'url': '/entitlements/view_license_tiers', 'roles': ['Super User']},
]) }}
{% endif %}
{% if current_user.is_authenticated %}
{{ dropdown(current_user.user_name, 'person', [
{'name': 'Session Defaults', 'url': '/session_defaults', 'roles': ['Super User', 'Tenant Admin']},

216
eveai_app/templates/user/tenant.html
View File

@@ -1,21 +1,219 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field %}
{% from "macros.html" import render_field, render_included_field %}
{% block title %}Tenant Registration{% endblock %}
{% block title %}Create or Edit Tenant{% endblock %}
{% block content_title %}Register Tenant{% endblock %}
{% block content_description %}Add a new tenant to EveAI{% endblock %}
{% block content_title %}Create or Edit Tenant{% endblock %}
{% block content_description %}Create or Edit Tenant{% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set disabled_fields = [] %}
{% set exclude_fields = [] %}
<!-- Main Tenant Information -->
{% set main_fields = ['name', 'website', 'default_language', 'allowed_languages', 'rag_context', 'type'] %}
{% for field in form %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{{ render_included_field(field, disabled_fields=[], include_fields=main_fields) }}
{% endfor %}
<button type="submit" class="btn btn-primary">Register Tenant</button>
<!-- Nav Tabs -->
<div class="row mt-5">
<div class="col-lg-12">
<div class="nav-wrapper position-relative end-0">
<ul class="nav nav-pills nav-fill p-1" role="tablist">
<li class="nav-item" role="presentation">
<a class="nav-link mb-0 px-0 py-1 active" data-toggle="tab" href="#model-info-tab" role="tab" aria-controls="model-info" aria-selected="true">
Model Information
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#license-info-tab" role="tab" aria-controls="license-info" aria-selected="false">
License Information
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#chunking-tab" role="tab" aria-controls="chunking" aria-selected="false">
Chunking
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#embedding-search-tab" role="tab" aria-controls="html-chunking" aria-selected="false">
Embedding Search
</a>
</li>
<li class="nav-item">
<a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#tuning-tab" role="tab" aria-controls="html-chunking" aria-selected="false">
Tuning
</a>
</li>
</ul>
</div>
<div class="tab-content tab-space">
<!-- Model Information Tab -->
<div class="tab-pane fade show active" id="model-info-tab" role="tabpanel">
{% set model_fields = ['embedding_model', 'llm_model'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=model_fields) }}
{% endfor %}
</div>
<!-- License Information Tab -->
<div class="tab-pane fade" id="license-info-tab" role="tabpanel">
{% set license_fields = ['currency', 'usage_email', ] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=license_fields) }}
{% endfor %}
<!-- Register API Key Button -->
<button type="button" class="btn btn-primary" onclick="generateNewChatApiKey()">Register Chat API Key</button>
<button type="button" class="btn btn-primary" onclick="generateNewApiKey()">Register API Key</button>
<!-- API Key Display Field -->
<div id="chat-api-key-field" style="display:none;">
<label for="chat-api-key">Chat API Key:</label>
<input type="text" id="chat-api-key" class="form-control" readonly>
<button type="button" id="copy-chat-button" class="btn btn-primary">Copy to Clipboard</button>
<p id="copy-chat-message" style="display:none;color:green;">Chat API key copied to clipboard</p>
</div>
<div id="api-key-field" style="display:none;">
<label for="api-key">API Key:</label>
<input type="text" id="api-key" class="form-control" readonly>
<button type="button" id="copy-api-button" class="btn btn-primary">Copy to Clipboard</button>
<p id="copy-message" style="display:none;color:green;">API key copied to clipboard</p>
</div>
</div>
<!-- Chunking Settings Tab -->
<div class="tab-pane fade" id="chunking-tab" role="tabpanel">
{% set html_fields = ['html_tags', 'html_end_tags', 'html_included_elements', 'html_excluded_elements', 'html_excluded_classes', 'min_chunk_size', 'max_chunk_size'] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=html_fields) }}
{% endfor %}
</div>
<!-- Embedding Search Settings Tab -->
<div class="tab-pane fade" id="embedding-search-tab" role="tabpanel">
{% set es_fields = ['es_k', 'es_similarity_threshold', ] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=es_fields) }}
{% endfor %}
</div>
<!-- Tuning Settings Tab -->
<div class="tab-pane fade" id="tuning-tab" role="tabpanel">
{% set tuning_fields = ['embed_tuning', 'rag_tuning', ] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=[], include_fields=tuning_fields) }}
{% endfor %}
</div>
</div>
</div>
</div>
<button type="submit" class="btn btn-primary">Save Tenant</button>
</form>
{% endblock %}
{% block content_footer %} {% endblock %}
{% block content_footer %}
{% endblock %}
{% block scripts %}
<script>
// Function to generate a new Chat API Key
function generateNewChatApiKey() {
generateApiKey('/admin/user/generate_chat_api_key', '#chat-api-key', '#chat-api-key-field');
}
// Function to generate a new general API Key
function generateNewApiKey() {
generateApiKey('/admin/user/generate_api_api_key', '#api-key', '#api-key-field');
}
// Reusable function to handle API key generation
function generateApiKey(url, inputSelector, fieldSelector) {
$.ajax({
url: url,
type: 'POST',
contentType: 'application/json',
success: function(response) {
$(inputSelector).val(response.api_key);
$(fieldSelector).show();
},
error: function(error) {
alert('Error generating new API key: ' + error.responseText);
}
});
}
// Function to copy text to clipboard
function copyToClipboard(selector, messageSelector) {
const element = document.querySelector(selector);
if (element) {
const text = element.value;
if (navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(text).then(function() {
showCopyMessage(messageSelector);
}).catch(function(error) {
alert('Failed to copy text: ' + error);
});
} else {
fallbackCopyToClipboard(text, messageSelector);
}
} else {
console.error('Element not found for selector:', selector);
}
}
// Fallback method for copying text to clipboard
function fallbackCopyToClipboard(text, messageSelector) {
const textArea = document.createElement('textarea');
textArea.value = text;
document.body.appendChild(textArea);
textArea.focus();
textArea.select();
try {
document.execCommand('copy');
showCopyMessage(messageSelector);
} catch (err) {
alert('Fallback: Oops, unable to copy', err);
}
document.body.removeChild(textArea);
}
// Function to show copy confirmation message
function showCopyMessage(messageSelector) {
const message = document.querySelector(messageSelector);
if (message) {
message.style.display = 'block';
setTimeout(function() {
message.style.display = 'none';
}, 2000);
}
}
// Event listeners for copy buttons
document.getElementById('copy-chat-button').addEventListener('click', function() {
copyToClipboard('#chat-api-key', '#copy-chat-message');
});
document.getElementById('copy-api-button').addEventListener('click', function() {
copyToClipboard('#api-key', '#copy-message');
});
</script>
<script>
// JavaScript to detect user's timezone
document.addEventListener('DOMContentLoaded', (event) => {
// Detect timezone
const userTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
// Send timezone to the server via a POST request
fetch('/set_user_timezone', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ timezone: userTimezone })
}).then(response => {
if (response.ok) {
console.log('Timezone sent to server successfully');
} else {
console.error('Failed to send timezone to server');
}
});
});
</script>
{% endblock %}

4
eveai_app/templates/user/tenant_overview.html
View File

@@ -16,7 +16,7 @@
{% endfor %}
<!-- Nav Tabs -->
<div class="row">
<div class="row mt-5">
<div class="col-lg-12">
<div class="nav-wrapper position-relative end-0">
<ul class="nav nav-pills nav-fill p-1" role="tablist">
@@ -57,7 +57,7 @@
</div>
<!-- License Information Tab -->
<div class="tab-pane fade" id="license-info-tab" role="tabpanel">
{% set license_fields = ['license_start_date', 'license_end_date', 'allowed_monthly_interactions', ] %}
{% set license_fields = ['currency', 'usage_email', ] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=license_fields, include_fields=license_fields) }}
{% endfor %}

76
eveai_app/views/entitlements_forms.py Normal file
View File

@@ -0,0 +1,76 @@
from flask import current_app
from flask_wtf import FlaskForm
from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField,
SelectField, SelectMultipleField, FieldList, FormField, FloatField, TextAreaField)
from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError, InputRequired
import pytz
class LicenseTierForm(FlaskForm):
name = StringField('Name', validators=[DataRequired(), Length(max=50)])
version = StringField('Version', validators=[DataRequired(), Length(max=50)])
start_date = DateField('Start Date', id='form-control datepicker', validators=[DataRequired()])
end_date = DateField('End Date', id='form-control datepicker', validators=[Optional()])
basic_fee_d = FloatField('Basic Fee ($)', validators=[InputRequired(), NumberRange(min=0)])
basic_fee_e = FloatField('Basic Fee (€)', validators=[InputRequired(), NumberRange(min=0)])
max_storage_mb = IntegerField('Max Storage (MiB)', validators=[DataRequired(), NumberRange(min=1)])
additional_storage_price_d = FloatField('Additional Storage Fee ($)',
validators=[InputRequired(), NumberRange(min=0)])
additional_storage_price_e = FloatField('Additional Storage Fee (€)',
validators=[InputRequired(), NumberRange(min=0)])
additional_storage_bucket = IntegerField('Additional Storage Bucket Size (MiB)',
validators=[DataRequired(), NumberRange(min=1)])
included_embedding_mb = IntegerField('Included Embeddings (MiB)',
validators=[DataRequired(), NumberRange(min=1)])
additional_embedding_price_d = FloatField('Additional Embedding Fee ($)',
validators=[InputRequired(), NumberRange(min=0)])
additional_embedding_price_e = FloatField('Additional Embedding Fee (€)',
validators=[InputRequired(), NumberRange(min=0)])
additional_embedding_bucket = IntegerField('Additional Embedding Bucket Size (MiB)',
validators=[DataRequired(), NumberRange(min=1)])
included_interaction_tokens = IntegerField('Included Embedding Tokens',
validators=[DataRequired(), NumberRange(min=1)])
additional_interaction_token_price_d = FloatField('Additional Interaction Token Fee ($)',
validators=[InputRequired(), NumberRange(min=0)])
additional_interaction_token_price_e = FloatField('Additional Interaction Token Fee (€)',
validators=[InputRequired(), NumberRange(min=0)])
additional_interaction_bucket = IntegerField('Additional Interaction Bucket Size',
validators=[DataRequired(), NumberRange(min=1)])
standard_overage_embedding = FloatField('Standard Overage Embedding (%)',
validators=[DataRequired(), NumberRange(min=0)],
default=0)
standard_overage_interaction = FloatField('Standard Overage Interaction (%)',
validators=[DataRequired(), NumberRange(min=0)],
default=0)
class LicenseForm(FlaskForm):
start_date = DateField('Start Date', id='form-control datepicker', validators=[DataRequired()])
end_date = DateField('End Date', id='form-control datepicker', validators=[DataRequired()])
currency = StringField('Currency', validators=[Optional(), Length(max=20)])
yearly_payment = BooleanField('Yearly Payment', validators=[DataRequired()], default=False)
basic_fee = FloatField('Basic Fee', validators=[InputRequired(), NumberRange(min=0)])
max_storage_mb = IntegerField('Max Storage (MiB)', validators=[DataRequired(), NumberRange(min=1)])
additional_storage_price = FloatField('Additional Storage Token Fee',
validators=[InputRequired(), NumberRange(min=0)])
additional_storage_bucket = IntegerField('Additional Storage Bucket Size (MiB)',
validators=[DataRequired(), NumberRange(min=1)])
included_embedding_mb = IntegerField('Included Embedding Tokens (MiB)',
validators=[DataRequired(), NumberRange(min=1)])
additional_embedding_price = FloatField('Additional Embedding Token Fee',
validators=[InputRequired(), NumberRange(min=0)])
additional_embedding_bucket = IntegerField('Additional Embedding Bucket Size (MiB)',
validators=[DataRequired(), NumberRange(min=1)])
included_interaction_tokens = IntegerField('Included Interaction Tokens',
validators=[DataRequired(), NumberRange(min=1)])
additional_interaction_token_price = FloatField('Additional Interaction Token Fee',
validators=[InputRequired(), NumberRange(min=0)])
additional_interaction_bucket = IntegerField('Additional Interaction Bucket Size',
validators=[DataRequired(), NumberRange(min=1)])
overage_embedding = FloatField('Overage Embedding (%)',
validators=[DataRequired(), NumberRange(min=0)],
default=0)
overage_interaction = FloatField('Overage Interaction (%)',
validators=[DataRequired(), NumberRange(min=0)],
default=0)

217
eveai_app/views/entitlements_views.py Normal file
View File

@@ -0,0 +1,217 @@
import uuid
from datetime import datetime as dt, timezone as tz
from flask import request, redirect, flash, render_template, Blueprint, session, current_app, jsonify
from flask_security import hash_password, roles_required, roles_accepted, current_user
from itsdangerous import URLSafeTimedSerializer
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import or_
import ast
from common.models.entitlements import License, LicenseTier, LicenseUsage, BusinessEventLog
from common.extensions import db, security, minio_client, simple_encryption
from common.utils.security_utils import send_confirmation_email, send_reset_email
from .entitlements_forms import LicenseTierForm, LicenseForm
from common.utils.database import Database
from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
from common.utils.simple_encryption import generate_api_key
from common.utils.nginx_utils import prefixed_url_for
entitlements_bp = Blueprint('entitlements_bp', __name__, url_prefix='/entitlements')
@entitlements_bp.route('/license_tier', methods=['GET', 'POST'])
@roles_accepted('Super User')
def license_tier():
form = LicenseTierForm()
if form.validate_on_submit():
current_app.logger.info("Adding License Tier")
new_license_tier = LicenseTier()
form.populate_obj(new_license_tier)
try:
db.session.add(new_license_tier)
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.error(f'Failed to add license tier to database. Error: {str(e)}')
flash(f'Failed to add license tier to database. Error: {str(e)}', 'success')
return render_template('entitlements/license_tier.html', form=form)
current_app.logger.info(f"Successfully created license tier {new_license_tier.id}")
flash(f"Successfully created tenant license tier {new_license_tier.id}")
return redirect(prefixed_url_for('entitlements_bp.view_license_tiers'))
else:
form_validation_failed(request, form)
return render_template('entitlements/license_tier.html', form=form)
@entitlements_bp.route('/view_license_tiers', methods=['GET', 'POST'])
@roles_required('Super User')
def view_license_tiers():
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
today = dt.now(tz.utc)
query = LicenseTier.query.filter(
or_(
LicenseTier.end_date == None,
LicenseTier.end_date >= today
)
).order_by(LicenseTier.start_date.desc(), LicenseTier.id)
pagination = query.paginate(page=page, per_page=per_page, error_out=False)
license_tiers = pagination.items
rows = prepare_table_for_macro(license_tiers, [('id', ''), ('name', ''), ('version', ''), ('start_date', ''),
('end_date', '')])
return render_template('entitlements/view_license_tiers.html', rows=rows, pagination=pagination)
@entitlements_bp.route('/handle_license_tier_selection', methods=['POST'])
@roles_required('Super User')
def handle_license_tier_selection():
license_tier_identification = request.form['selected_row']
license_tier_id = ast.literal_eval(license_tier_identification).get('value')
the_license_tier = LicenseTier.query.get(license_tier_id)
action = request.form['action']
match action:
case 'edit_license_tier':
return redirect(prefixed_url_for('entitlements_bp.edit_license_tier',
license_tier_id=license_tier_id))
case 'create_license_for_tenant':
return redirect(prefixed_url_for('entitlements_bp.create_license',
license_tier_id=license_tier_id))
# Add more conditions for other actions
return redirect(prefixed_url_for('entitlements_bp.view_license_tiers'))
@entitlements_bp.route('/license_tier/<int:license_tier_id>', methods=['GET', 'POST'])
@roles_accepted('Super User')
def edit_license_tier(license_tier_id):
license_tier = LicenseTier.query.get_or_404(license_tier_id) # This will return a 404 if no license tier is found
form = LicenseTierForm(obj=license_tier)
if form.validate_on_submit():
# Populate the license_tier with form data
form.populate_obj(license_tier)
try:
db.session.add(license_tier)
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.error(f'Failed to edit License Tier. Error: {str(e)}')
flash(f'Failed to edit License Tier. Error: {str(e)}', 'danger')
return render_template('entitlements/license_tier.html', form=form, license_tier_id=license_tier.id)
flash('License Tier updated successfully.', 'success')
return redirect(
prefixed_url_for('entitlements_bp.edit_license_tier', license_tier_id=license_tier_id))
else:
form_validation_failed(request, form)
return render_template('entitlements/license_tier.html', form=form, license_tier_id=license_tier.id)
@entitlements_bp.route('/create_license/<int:license_tier_id>', methods=['GET', 'POST'])
@roles_accepted('Super User')
def create_license(license_tier_id):
form = LicenseForm()
tenant_id = session.get('tenant').get('id')
currency = session.get('tenant').get('currency')
if request.method == 'GET':
# Fetch the LicenseTier
license_tier = LicenseTier.query.get_or_404(license_tier_id)
# Prefill the form with LicenseTier data
# Currency depending data
if currency == '$':
form.basic_fee.data = license_tier.basic_fee_d
form.additional_storage_price.data = license_tier.additional_storage_price_d
form.additional_embedding_price.data = license_tier.additional_embedding_price_d
form.additional_interaction_token_price.data = license_tier.additional_interaction_token_price_d
elif currency == '€':
form.basic_fee.data = license_tier.basic_fee_e
form.additional_storage_price.data = license_tier.additional_storage_price_e
form.additional_embedding_price.data = license_tier.additional_embedding_price_e
form.additional_interaction_token_price.data = license_tier.additional_interaction_token_price_e
else:
current_app.logger.error(f'Invalid currency {currency} for tenant {tenant_id} while creating license.')
flash(f"Invalid currency {currency} for tenant {tenant_id} while creating license. "
f"Check tenant's currency and try again.", 'danger')
return redirect(prefixed_url_for('user_bp.edit_tenant', tenant_id=tenant_id))
# General data
form.currency.data = currency
form.max_storage_mb.data = license_tier.max_storage_mb
form.additional_storage_bucket.data = license_tier.additional_storage_bucket
form.included_embedding_mb.data = license_tier.included_embedding_mb
form.additional_embedding_bucket.data = license_tier.additional_embedding_bucket
form.included_interaction_tokens.data = license_tier.included_interaction_tokens
form.additional_interaction_bucket.data = license_tier.additional_interaction_bucket
form.overage_embedding.data = license_tier.standard_overage_embedding
form.overage_interaction.data = license_tier.standard_overage_interaction
else: # POST
# Create a new License instance
new_license = License(
tenant_id=tenant_id,
tier_id=license_tier_id,
)
current_app.logger.debug(f"Currency data in form: {form.currency.data}")
if form.validate_on_submit():
# Update the license with form data
form.populate_obj(new_license)
# Currency is added here again, as a form doesn't include disabled fields when passing it in the request
new_license.currency = currency
try:
db.session.add(new_license)
db.session.commit()
flash('License created successfully', 'success')
return redirect(prefixed_url_for('entitlements_bp/edit_license', license_id=new_license.id))
except Exception as e:
db.session.rollback()
flash(f'Error creating license: {str(e)}', 'error')
else:
form_validation_failed(request, form)
return render_template('entitlements/license.html', form=form)
@entitlements_bp.route('/license/<int:license_id>', methods=['GET', 'POST'])
@roles_accepted('Super User')
def edit_license(license_id):
license = License.query.get_or_404(license_id) # This will return a 404 if no license tier is found
form = LicenseForm(obj=license)
disabled_fields = []
if len(license.usages) > 0: # There already are usage records linked to this license
# Define which fields should be disabled
disabled_fields = [field.name for field in form if field.name != 'end_date']
if form.validate_on_submit():
# Populate the license with form data
form.populate_obj(license)
try:
db.session.add(license)
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.error(f'Failed to edit License. Error: {str(e)}')
flash(f'Failed to edit License. Error: {str(e)}', 'danger')
return render_template('entitlements/license.html', form=form)
flash('License updated successfully.', 'success')
return redirect(
prefixed_url_for('entitlements_bp.edit_license', license_tier_id=license_id))
else:
form_validation_failed(request, form)
return render_template('entitlements/license.html', form=form, license_tier_id=license_tier.id,
ext_disabled_fields=disabled_fields)

9
eveai_app/views/user_forms.py
View File

@@ -14,6 +14,9 @@ class TenantForm(FlaskForm):
# language fields
default_language = SelectField('Default Language', choices=[], validators=[DataRequired()])
allowed_languages = SelectMultipleField('Allowed Languages', choices=[], validators=[DataRequired()])
# invoicing fields
currency = SelectField('Currency', choices=[], validators=[DataRequired()])
usage_email = EmailField('Usage Email', validators=[DataRequired(), Email()])
# Timezone
timezone = SelectField('Timezone', choices=[], validators=[DataRequired()])
# RAG context
@@ -23,10 +26,6 @@ class TenantForm(FlaskForm):
# LLM fields
embedding_model = SelectField('Embedding Model', choices=[], validators=[DataRequired()])
llm_model = SelectField('Large Language Model', choices=[], validators=[DataRequired()])
# license fields
license_start_date = DateField('License Start Date', id='form-control datepicker', validators=[Optional()])
license_end_date = DateField('License End Date', id='form-control datepicker', validators=[Optional()])
allowed_monthly_interactions = IntegerField('Allowed Monthly Interactions', validators=[NumberRange(min=0)])
# Embedding variables
html_tags = StringField('HTML Tags', validators=[DataRequired()],
default='p, h1, h2, h3, h4, h5, h6, li')
@@ -59,6 +58,8 @@ class TenantForm(FlaskForm):
# initialise language fields
self.default_language.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
self.allowed_languages.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
# initialise currency field
self.currency.choices = [(curr, curr) for curr in current_app.config['SUPPORTED_CURRENCIES']]
# initialise timezone
self.timezone.choices = [(tz, tz) for tz in pytz.all_timezones]
# initialise LLM fields

16
eveai_app/views/user_views.py
View File

@@ -47,18 +47,6 @@ def tenant():
# Handle the required attributes
new_tenant = Tenant()
form.populate_obj(new_tenant)
# new_tenant = Tenant(name=form.name.data,
# website=form.website.data,
# default_language=form.default_language.data,
# allowed_languages=form.allowed_languages.data,
# timezone=form.timezone.data,
# embedding_model=form.embedding_model.data,
# llm_model=form.llm_model.data,
# license_start_date=form.license_start_date.data,
# license_end_date=form.license_end_date.data,
# allowed_monthly_interactions=form.allowed_monthly_interactions.data,
# embed_tuning=form.embed_tuning.data,
# rag_tuning=form.rag_tuning.data)
# Handle Embedding Variables
new_tenant.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
@@ -87,7 +75,7 @@ def tenant():
db.session.commit()
except SQLAlchemyError as e:
current_app.logger.error(f'Failed to add tenant to database. Error: {str(e)}')
flash(f'Failed to add tenant to database. Error: {str(e)}')
flash(f'Failed to add tenant to database. Error: {str(e)}', 'danger')
return render_template('user/tenant.html', form=form)
current_app.logger.info(f"Successfully created tenant {new_tenant.id} in Database")
@@ -152,7 +140,7 @@ def edit_tenant(tenant_id):
current_app.logger.debug(f'Tenant update failed with errors: {form.errors}')
form_validation_failed(request, form)
return render_template('user/edit_tenant.html', form=form, tenant_id=tenant_id)
return render_template('user/tenant.html', form=form, tenant_id=tenant_id)
@user_bp.route('/user', methods=['GET', 'POST'])

13
eveai_workers/tasks.py
View File

@@ -36,8 +36,14 @@ def ping():
@current_celery.task(name='create_embeddings', queue='embeddings')
def create_embeddings(tenant_id, document_version_id):
# Retrieve document version to process
document_version = DocumentVersion.query.get(document_version_id)
if document_version is None:
raise Exception(f'Document version {document_version_id} not found')
# BusinessEvent creates a context, which is why we need to use it with a with block
with BusinessEvent('Create Embeddings', tenant_id, document_version_id=document_version_id):
with BusinessEvent('Create Embeddings', tenant_id,
document_version_id=document_version_id,
document_version_file_size=document_version.file_size):
current_app.logger.info(f'Creating embeddings for tenant {tenant_id} on document version {document_version_id}')
try:
# Retrieve Tenant for which we are processing
@@ -52,11 +58,6 @@ def create_embeddings(tenant_id, document_version_id):
model_variables = select_model_variables(tenant)
current_app.logger.debug(f'Model variables: {model_variables}')
# Retrieve document version to process
document_version = DocumentVersion.query.get(document_version_id)
if document_version is None:
raise Exception(f'Document version {document_version_id} not found')
except Exception as e:
current_app.logger.error(f'Create Embeddings request received '
f'for non existing document version {document_version_id} '

40
migrations/public/versions/48714f1baac5_add_entitlement_information_to_tenant.py Normal file
View File

@@ -0,0 +1,40 @@
"""Add entitlement information to Tenant
Revision ID: 48714f1baac5
Revises: f201bfd23152
Create Date: 2024-10-03 14:49:53.922320
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '48714f1baac5'
down_revision = 'f201bfd23152'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.add_column(sa.Column('yearly_payment', sa.Boolean(), nullable=False))
with op.batch_alter_table('tenant', schema=None) as batch_op:
batch_op.add_column(sa.Column('currency', sa.String(length=20), nullable=True))
batch_op.add_column(sa.Column('usage_email', sa.String(length=255), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('tenant', schema=None) as batch_op:
batch_op.drop_column('usage_email')
batch_op.drop_column('currency')
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.drop_column('yearly_payment')
# ### end Alembic commands ###

106
migrations/public/versions/560d08d91e5b_introducing_new_licensing_approach.py Normal file
View File

@@ -0,0 +1,106 @@
"""Introducing new licensing approach
Revision ID: 560d08d91e5b
Revises: 254932fe7fe3
Create Date: 2024-10-02 15:29:05.963865
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '560d08d91e5b'
down_revision = '254932fe7fe3'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('license_tier',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=50), nullable=False),
sa.Column('version', sa.String(length=50), nullable=False),
sa.Column('start_date', sa.Date(), nullable=False),
sa.Column('end_date', sa.Date(), nullable=True),
sa.Column('basic_fee_d', sa.Float(), nullable=True),
sa.Column('basic_fee_e', sa.Float(), nullable=True),
sa.Column('max_storage_tokens', sa.Integer(), nullable=False),
sa.Column('additional_storage_token_price_d', sa.Numeric(precision=10, scale=4), nullable=False),
sa.Column('additional_storage_token_price_e', sa.Numeric(precision=10, scale=4), nullable=False),
sa.Column('included_embedding_tokens', sa.Integer(), nullable=False),
sa.Column('additional_embedding_token_price_d', sa.Numeric(precision=10, scale=4), nullable=False),
sa.Column('additional_embedding_token_price_e', sa.Numeric(precision=10, scale=4), nullable=False),
sa.Column('additional_embedding_bucket', sa.Integer(), nullable=False),
sa.Column('included_interaction_tokens', sa.Integer(), nullable=False),
sa.Column('additional_interaction_token_price_d', sa.Numeric(precision=10, scale=4), nullable=False),
sa.Column('additional_interaction_token_price_e', sa.Numeric(precision=10, scale=4), nullable=False),
sa.Column('additional_interaction_bucket', sa.Integer(), nullable=False),
sa.Column('allow_overage', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id'),
schema='public'
)
op.create_table('license',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('tenant_id', sa.Integer(), nullable=False),
sa.Column('tier_id', sa.Integer(), nullable=False),
sa.Column('start_date', sa.Date(), nullable=False),
sa.Column('end_date', sa.Date(), nullable=True),
sa.Column('currency', sa.String(length=20), nullable=False),
sa.Column('basic_fee', sa.Float(), nullable=False),
sa.Column('max_storage_tokens', sa.Integer(), nullable=False),
sa.Column('additional_storage_token_price', sa.Float(), nullable=False),
sa.Column('included_embedding_tokens', sa.Integer(), nullable=False),
sa.Column('additional_embedding_token_price', sa.Numeric(precision=10, scale=4), nullable=False),
sa.Column('additional_embedding_bucket', sa.Integer(), nullable=False),
sa.Column('included_interaction_tokens', sa.Integer(), nullable=False),
sa.Column('additional_interaction_token_price', sa.Numeric(precision=10, scale=4), nullable=False),
sa.Column('additional_interaction_bucket', sa.Integer(), nullable=False),
sa.Column('allow_overage', sa.Boolean(), nullable=True),
sa.ForeignKeyConstraint(['tenant_id'], ['public.tenant.id'], ),
sa.ForeignKeyConstraint(['tier_id'], ['public.license_tier.id'], ),
sa.PrimaryKeyConstraint('id'),
schema='public'
)
op.create_table('license_usage',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('license_id', sa.Integer(), nullable=False),
sa.Column('tenant_id', sa.Integer(), nullable=False),
sa.Column('storage_tokens_used', sa.Integer(), nullable=True),
sa.Column('embedding_tokens_used', sa.Integer(), nullable=True),
sa.Column('interaction_tokens_used', sa.Integer(), nullable=True),
sa.Column('period_start_date', sa.Date(), nullable=False),
sa.Column('period_end_date', sa.Date(), nullable=False),
sa.ForeignKeyConstraint(['license_id'], ['public.license.id'], ),
sa.ForeignKeyConstraint(['tenant_id'], ['public.tenant.id'], ),
sa.PrimaryKeyConstraint('id'),
schema='public'
)
with op.batch_alter_table('business_event_log', schema=None) as batch_op:
batch_op.add_column(sa.Column('license_usage_id', sa.Integer(), nullable=True))
batch_op.create_foreign_key(None, 'license_usage', ['license_usage_id'], ['id'], referent_schema='public')
with op.batch_alter_table('tenant', schema=None) as batch_op:
batch_op.drop_column('license_end_date')
batch_op.drop_column('allowed_monthly_interactions')
batch_op.drop_column('license_start_date')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('tenant', schema=None) as batch_op:
batch_op.add_column(sa.Column('license_start_date', sa.DATE(), autoincrement=False, nullable=True))
batch_op.add_column(sa.Column('allowed_monthly_interactions', sa.INTEGER(), autoincrement=False, nullable=True))
batch_op.add_column(sa.Column('license_end_date', sa.DATE(), autoincrement=False, nullable=True))
with op.batch_alter_table('business_event_log', schema=None) as batch_op:
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.drop_column('license_usage_id')
op.drop_table('license_usage', schema='public')
op.drop_table('license', schema='public')
op.drop_table('license_tier', schema='public')
# ### end Alembic commands ###

38
migrations/public/versions/6a7743d08106_adapt_licenseusage_to_accept_mb_iso_.py Normal file
View File

@@ -0,0 +1,38 @@
"""Adapt LicenseUsage to accept mb iso tokoens for Storage and Embeddings
Revision ID: 6a7743d08106
Revises: 9429f244f1a5
Create Date: 2024-10-07 06:04:39.424243
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '6a7743d08106'
down_revision = '9429f244f1a5'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license_usage', schema=None) as batch_op:
batch_op.add_column(sa.Column('storage_mb_used', sa.Integer(), nullable=True))
batch_op.add_column(sa.Column('embedding_mb_used', sa.Integer(), nullable=True))
batch_op.drop_column('storage_tokens_used')
batch_op.drop_column('embedding_tokens_used')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license_usage', schema=None) as batch_op:
batch_op.add_column(sa.Column('embedding_tokens_used', sa.INTEGER(), autoincrement=False, nullable=True))
batch_op.add_column(sa.Column('storage_tokens_used', sa.INTEGER(), autoincrement=False, nullable=True))
batch_op.drop_column('embedding_mb_used')
batch_op.drop_column('storage_mb_used')
# ### end Alembic commands ###

86
migrations/public/versions/9429f244f1a5_moved_storage_and_embedding_licensing_.py Normal file
View File

@@ -0,0 +1,86 @@
"""Moved storage and embedding licensing to Mb iso tokens
Revision ID: 9429f244f1a5
Revises: 48714f1baac5
Create Date: 2024-10-04 08:07:47.976861
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '9429f244f1a5'
down_revision = '48714f1baac5'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.add_column(sa.Column('max_storage_mb', sa.Integer(), nullable=False))
batch_op.add_column(sa.Column('additional_storage_price', sa.Float(), nullable=False))
batch_op.add_column(sa.Column('included_embedding_mb', sa.Integer(), nullable=False))
batch_op.add_column(sa.Column('additional_embedding_price', sa.Numeric(precision=10, scale=4), nullable=False))
batch_op.add_column(sa.Column('overage_embedding', sa.Float(), nullable=False))
batch_op.add_column(sa.Column('overage_interaction', sa.Float(), nullable=False))
batch_op.drop_column('additional_storage_token_price')
batch_op.drop_column('additional_embedding_token_price')
batch_op.drop_column('max_storage_tokens')
batch_op.drop_column('allow_overage')
batch_op.drop_column('included_embedding_tokens')
with op.batch_alter_table('license_tier', schema=None) as batch_op:
batch_op.add_column(sa.Column('max_storage_mb', sa.Integer(), nullable=False))
batch_op.add_column(sa.Column('additional_storage_price_d', sa.Numeric(precision=10, scale=4), nullable=False))
batch_op.add_column(sa.Column('additional_storage_price_e', sa.Numeric(precision=10, scale=4), nullable=False))
batch_op.add_column(sa.Column('included_embedding_mb', sa.Integer(), nullable=False))
batch_op.add_column(sa.Column('additional_embedding_price_d', sa.Numeric(precision=10, scale=4), nullable=False))
batch_op.add_column(sa.Column('additional_embedding_price_e', sa.Numeric(precision=10, scale=4), nullable=False))
batch_op.add_column(sa.Column('standard_overage_embedding', sa.Float(), nullable=False))
batch_op.add_column(sa.Column('standard_overage_interaction', sa.Float(), nullable=False))
batch_op.drop_column('max_storage_tokens')
batch_op.drop_column('additional_storage_token_price_e')
batch_op.drop_column('allow_overage')
batch_op.drop_column('additional_embedding_token_price_d')
batch_op.drop_column('included_embedding_tokens')
batch_op.drop_column('additional_embedding_token_price_e')
batch_op.drop_column('additional_storage_token_price_d')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license_tier', schema=None) as batch_op:
batch_op.add_column(sa.Column('additional_storage_token_price_d', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
batch_op.add_column(sa.Column('additional_embedding_token_price_e', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
batch_op.add_column(sa.Column('included_embedding_tokens', sa.INTEGER(), autoincrement=False, nullable=False))
batch_op.add_column(sa.Column('additional_embedding_token_price_d', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
batch_op.add_column(sa.Column('allow_overage', sa.BOOLEAN(), autoincrement=False, nullable=True))
batch_op.add_column(sa.Column('additional_storage_token_price_e', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
batch_op.add_column(sa.Column('max_storage_tokens', sa.INTEGER(), autoincrement=False, nullable=False))
batch_op.drop_column('standard_overage_interaction')
batch_op.drop_column('standard_overage_embedding')
batch_op.drop_column('additional_embedding_price_e')
batch_op.drop_column('additional_embedding_price_d')
batch_op.drop_column('included_embedding_mb')
batch_op.drop_column('additional_storage_price_e')
batch_op.drop_column('additional_storage_price_d')
batch_op.drop_column('max_storage_mb')
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.add_column(sa.Column('included_embedding_tokens', sa.INTEGER(), autoincrement=False, nullable=False))
batch_op.add_column(sa.Column('allow_overage', sa.BOOLEAN(), autoincrement=False, nullable=True))
batch_op.add_column(sa.Column('max_storage_tokens', sa.INTEGER(), autoincrement=False, nullable=False))
batch_op.add_column(sa.Column('additional_embedding_token_price', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
batch_op.add_column(sa.Column('additional_storage_token_price', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=False))
batch_op.drop_column('overage_interaction')
batch_op.drop_column('overage_embedding')
batch_op.drop_column('additional_embedding_price')
batch_op.drop_column('included_embedding_mb')
batch_op.drop_column('additional_storage_price')
batch_op.drop_column('max_storage_mb')
# ### end Alembic commands ###

58
migrations/public/versions/d616ea937a6a_corrections_on_licensing_models.py Normal file
View File

@@ -0,0 +1,58 @@
"""Corrections on Licensing models
Revision ID: d616ea937a6a
Revises: 560d08d91e5b
Create Date: 2024-10-02 15:39:27.668741
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'd616ea937a6a'
down_revision = '560d08d91e5b'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('business_event_log', schema=None) as batch_op:
batch_op.drop_constraint('business_event_log_license_usage_id_fkey', type_='foreignkey')
batch_op.create_foreign_key(None, 'license_usage', ['license_usage_id'], ['id'], referent_schema='public')
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.drop_constraint('license_tenant_id_fkey', type_='foreignkey')
batch_op.drop_constraint('license_tier_id_fkey', type_='foreignkey')
batch_op.create_foreign_key(None, 'license_tier', ['tier_id'], ['id'], referent_schema='public')
batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
with op.batch_alter_table('license_usage', schema=None) as batch_op:
batch_op.drop_constraint('license_usage_license_id_fkey', type_='foreignkey')
batch_op.drop_constraint('license_usage_tenant_id_fkey', type_='foreignkey')
batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
batch_op.create_foreign_key(None, 'license', ['license_id'], ['id'], referent_schema='public')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license_usage', schema=None) as batch_op:
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.create_foreign_key('license_usage_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
batch_op.create_foreign_key('license_usage_license_id_fkey', 'license', ['license_id'], ['id'])
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.create_foreign_key('license_tier_id_fkey', 'license_tier', ['tier_id'], ['id'])
batch_op.create_foreign_key('license_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
with op.batch_alter_table('business_event_log', schema=None) as batch_op:
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.create_foreign_key('business_event_log_license_usage_id_fkey', 'license_usage', ['license_usage_id'], ['id'])
# ### end Alembic commands ###

66
migrations/public/versions/f201bfd23152_add_buckets_to_license_information.py Normal file
View File

@@ -0,0 +1,66 @@
"""Add buckets to License information
Revision ID: f201bfd23152
Revises: d616ea937a6a
Create Date: 2024-10-03 09:44:32.867470
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'f201bfd23152'
down_revision = 'd616ea937a6a'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('business_event_log', schema=None) as batch_op:
batch_op.drop_constraint('business_event_log_license_usage_id_fkey', type_='foreignkey')
batch_op.create_foreign_key(None, 'license_usage', ['license_usage_id'], ['id'], referent_schema='public')
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.add_column(sa.Column('additional_storage_bucket', sa.Integer(), nullable=False))
batch_op.drop_constraint('license_tier_id_fkey', type_='foreignkey')
batch_op.drop_constraint('license_tenant_id_fkey', type_='foreignkey')
batch_op.create_foreign_key(None, 'license_tier', ['tier_id'], ['id'], referent_schema='public')
batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
with op.batch_alter_table('license_tier', schema=None) as batch_op:
batch_op.add_column(sa.Column('additional_storage_bucket', sa.Integer(), nullable=False))
with op.batch_alter_table('license_usage', schema=None) as batch_op:
batch_op.drop_constraint('license_usage_tenant_id_fkey', type_='foreignkey')
batch_op.drop_constraint('license_usage_license_id_fkey', type_='foreignkey')
batch_op.create_foreign_key(None, 'license', ['license_id'], ['id'], referent_schema='public')
batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license_usage', schema=None) as batch_op:
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.create_foreign_key('license_usage_license_id_fkey', 'license', ['license_id'], ['id'])
batch_op.create_foreign_key('license_usage_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
with op.batch_alter_table('license_tier', schema=None) as batch_op:
batch_op.drop_column('additional_storage_bucket')
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.create_foreign_key('license_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
batch_op.create_foreign_key('license_tier_id_fkey', 'license_tier', ['tier_id'], ['id'])
batch_op.drop_column('additional_storage_bucket')
with op.batch_alter_table('business_event_log', schema=None) as batch_op:
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.create_foreign_key('business_event_log_license_usage_id_fkey', 'license_usage', ['license_usage_id'], ['id'])
# ### end Alembic commands ###

94
migrations/tenant/versions/322d3cf1f17b_documentversion_update_to_bucket_name_.py Normal file
View File

@@ -0,0 +1,94 @@
"""DocumentVersion update to bucket_name, object_name and file_size to better reflet Minio reality
Revision ID: 322d3cf1f17b
Revises: 711a09a77680
Create Date: 2024-10-07 07:45:19.014017
"""
from alembic import op
import sqlalchemy as sa
import pgvector
from sqlalchemy.dialects import postgresql
from sqlalchemy.orm import Session
from sqlalchemy import text
from flask import current_app
# revision identifiers, used by Alembic.
revision = '322d3cf1f17b'
down_revision = '711a09a77680'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - Add new fields ###
op.add_column('document_version', sa.Column('bucket_name', sa.String(length=255), nullable=True))
op.add_column('document_version', sa.Column('object_name', sa.String(length=200), nullable=True))
op.add_column('document_version', sa.Column('file_size', sa.Float(), nullable=True))
# ### Upgrade values for bucket_name, object_name and file_size to reflect minio reality ###
from common.models.document import DocumentVersion
from common.extensions import minio_client
from minio.error import S3Error
# Create a connection
connection = op.get_bind()
session = Session(bind=connection)
# Get the current schema name (which should be the tenant ID)
current_schema = connection.execute(text("SELECT current_schema()")).scalar()
tenant_id = int(current_schema)
doc_versions = session.query(DocumentVersion).all()
for doc_version in doc_versions:
try:
object_name = minio_client.generate_object_name(doc_version.doc_id,
doc_version.language,
doc_version.id,
doc_version.file_name)
bucket_name = minio_client.generate_bucket_name(tenant_id)
doc_version.object_name = object_name
doc_version.bucket_name = bucket_name
try:
stat = minio_client.client.stat_object(
bucket_name=bucket_name,
object_name=object_name
)
doc_version.file_size = stat.size / 1048576
current_app.logger.info(f"Processed Upgrade for DocumentVersion {doc_version.id} for Tenant {tenant_id}")
except S3Error as e:
if e.code == "NoSuchKey":
current_app.logger.warning(
f"Object {doc_version.file_location} not found in bucket {doc_version.bucket_name}. Skipping.")
continue # Move to the next item
else:
raise e # Handle other types of S3 errors
except Exception as e:
session.rollback()
current_app.logger.error(f"Couldn't process upgrade for DocumentVersion {doc_version.id} for "
f"Tenant {tenant_id}. Error: {str(e)}")
try:
session.commit()
current_app.logger.info(f"Successfully updated file sizes for tenant schema {current_schema}")
except Exception as e:
session.rollback()
current_app.logger.error(f"Error committing changes for tenant schema {current_schema}: {str(e)}")
# ### commands auto generated by Alembic - Remove old fields ###
# op.drop_column('document_version', 'file_location')
# op.drop_column('document_version', 'file_name')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('document_version', sa.Column('file_name', sa.VARCHAR(length=200), autoincrement=False, nullable=True))
op.add_column('document_version', sa.Column('file_location', sa.VARCHAR(length=255), autoincrement=False, nullable=True))
op.drop_column('document_version', 'file_size')
op.drop_column('document_version', 'object_name')
op.drop_column('document_version', 'bucket_name')
# ### end Alembic commands ###

17
scripts/repopack_eveai.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/bin/bash
# Run repopack to generate the file
repopack
# Check if repopack generated the eveai_repo.txt file
if [[ -f "eveai_repo.txt" ]]; then
# Get the current timestamp in the format YYYY-DD-MM_HH:MM:SS
timestamp=$(date +"%Y-%d-%m_%H-%M-%S")
# Rename the file with the timestamp
mv eveai_repo.txt "${timestamp}_eveai_repo.txt"
echo "File renamed to ${timestamp}_eveai_repo.txt"
else
echo "Error: eveai_repo.txt not found. repopack may have failed."
fi