- {% set license_fields = ['license_start_date', 'license_end_date', 'allowed_monthly_interactions', ] %}
+ {% set license_fields = ['currency', 'usage_email', ] %}
{% for field in form %}
{{ render_included_field(field, disabled_fields=license_fields, include_fields=license_fields) }}
{% endfor %}
diff --git a/eveai_app/views/entitlements_forms.py b/eveai_app/views/entitlements_forms.py
new file mode 100644
index 0000000..89e6124
--- /dev/null
+++ b/eveai_app/views/entitlements_forms.py
@@ -0,0 +1,76 @@
+from flask import current_app
+from flask_wtf import FlaskForm
+from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField,
+ SelectField, SelectMultipleField, FieldList, FormField, FloatField, TextAreaField)
+from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError, InputRequired
+import pytz
+
+
+class LicenseTierForm(FlaskForm):
+ name = StringField('Name', validators=[DataRequired(), Length(max=50)])
+ version = StringField('Version', validators=[DataRequired(), Length(max=50)])
+ start_date = DateField('Start Date', id='form-control datepicker', validators=[DataRequired()])
+ end_date = DateField('End Date', id='form-control datepicker', validators=[Optional()])
+ basic_fee_d = FloatField('Basic Fee ($)', validators=[InputRequired(), NumberRange(min=0)])
+ basic_fee_e = FloatField('Basic Fee (€)', validators=[InputRequired(), NumberRange(min=0)])
+ max_storage_mb = IntegerField('Max Storage (MiB)', validators=[DataRequired(), NumberRange(min=1)])
+ additional_storage_price_d = FloatField('Additional Storage Fee ($)',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_storage_price_e = FloatField('Additional Storage Fee (€)',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_storage_bucket = IntegerField('Additional Storage Bucket Size (MiB)',
+ validators=[DataRequired(), NumberRange(min=1)])
+ included_embedding_mb = IntegerField('Included Embeddings (MiB)',
+ validators=[DataRequired(), NumberRange(min=1)])
+ additional_embedding_price_d = FloatField('Additional Embedding Fee ($)',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_embedding_price_e = FloatField('Additional Embedding Fee (€)',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_embedding_bucket = IntegerField('Additional Embedding Bucket Size (MiB)',
+ validators=[DataRequired(), NumberRange(min=1)])
+ included_interaction_tokens = IntegerField('Included Embedding Tokens',
+ validators=[DataRequired(), NumberRange(min=1)])
+ additional_interaction_token_price_d = FloatField('Additional Interaction Token Fee ($)',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_interaction_token_price_e = FloatField('Additional Interaction Token Fee (€)',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_interaction_bucket = IntegerField('Additional Interaction Bucket Size',
+ validators=[DataRequired(), NumberRange(min=1)])
+ standard_overage_embedding = FloatField('Standard Overage Embedding (%)',
+ validators=[DataRequired(), NumberRange(min=0)],
+ default=0)
+ standard_overage_interaction = FloatField('Standard Overage Interaction (%)',
+ validators=[DataRequired(), NumberRange(min=0)],
+ default=0)
+
+
+class LicenseForm(FlaskForm):
+ start_date = DateField('Start Date', id='form-control datepicker', validators=[DataRequired()])
+ end_date = DateField('End Date', id='form-control datepicker', validators=[DataRequired()])
+ currency = StringField('Currency', validators=[Optional(), Length(max=20)])
+ yearly_payment = BooleanField('Yearly Payment', validators=[DataRequired()], default=False)
+ basic_fee = FloatField('Basic Fee', validators=[InputRequired(), NumberRange(min=0)])
+ max_storage_mb = IntegerField('Max Storage (MiB)', validators=[DataRequired(), NumberRange(min=1)])
+ additional_storage_price = FloatField('Additional Storage Token Fee',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_storage_bucket = IntegerField('Additional Storage Bucket Size (MiB)',
+ validators=[DataRequired(), NumberRange(min=1)])
+ included_embedding_mb = IntegerField('Included Embedding Tokens (MiB)',
+ validators=[DataRequired(), NumberRange(min=1)])
+ additional_embedding_price = FloatField('Additional Embedding Token Fee',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_embedding_bucket = IntegerField('Additional Embedding Bucket Size (MiB)',
+ validators=[DataRequired(), NumberRange(min=1)])
+ included_interaction_tokens = IntegerField('Included Interaction Tokens',
+ validators=[DataRequired(), NumberRange(min=1)])
+ additional_interaction_token_price = FloatField('Additional Interaction Token Fee',
+ validators=[InputRequired(), NumberRange(min=0)])
+ additional_interaction_bucket = IntegerField('Additional Interaction Bucket Size',
+ validators=[DataRequired(), NumberRange(min=1)])
+ overage_embedding = FloatField('Overage Embedding (%)',
+ validators=[DataRequired(), NumberRange(min=0)],
+ default=0)
+ overage_interaction = FloatField('Overage Interaction (%)',
+ validators=[DataRequired(), NumberRange(min=0)],
+ default=0)
+
diff --git a/eveai_app/views/entitlements_views.py b/eveai_app/views/entitlements_views.py
new file mode 100644
index 0000000..039cec6
--- /dev/null
+++ b/eveai_app/views/entitlements_views.py
@@ -0,0 +1,217 @@
+import uuid
+from datetime import datetime as dt, timezone as tz
+from flask import request, redirect, flash, render_template, Blueprint, session, current_app, jsonify
+from flask_security import hash_password, roles_required, roles_accepted, current_user
+from itsdangerous import URLSafeTimedSerializer
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy import or_
+import ast
+
+from common.models.entitlements import License, LicenseTier, LicenseUsage, BusinessEventLog
+from common.extensions import db, security, minio_client, simple_encryption
+from common.utils.security_utils import send_confirmation_email, send_reset_email
+from .entitlements_forms import LicenseTierForm, LicenseForm
+from common.utils.database import Database
+from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
+from common.utils.simple_encryption import generate_api_key
+from common.utils.nginx_utils import prefixed_url_for
+
+entitlements_bp = Blueprint('entitlements_bp', __name__, url_prefix='/entitlements')
+
+
+@entitlements_bp.route('/license_tier', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def license_tier():
+ form = LicenseTierForm()
+ if form.validate_on_submit():
+ current_app.logger.info("Adding License Tier")
+
+ new_license_tier = LicenseTier()
+ form.populate_obj(new_license_tier)
+
+ try:
+ db.session.add(new_license_tier)
+ db.session.commit()
+ except SQLAlchemyError as e:
+ db.session.rollback()
+ current_app.logger.error(f'Failed to add license tier to database. Error: {str(e)}')
+ flash(f'Failed to add license tier to database. Error: {str(e)}', 'success')
+ return render_template('entitlements/license_tier.html', form=form)
+
+ current_app.logger.info(f"Successfully created license tier {new_license_tier.id}")
+ flash(f"Successfully created tenant license tier {new_license_tier.id}")
+
+ return redirect(prefixed_url_for('entitlements_bp.view_license_tiers'))
+ else:
+ form_validation_failed(request, form)
+
+ return render_template('entitlements/license_tier.html', form=form)
+
+
+@entitlements_bp.route('/view_license_tiers', methods=['GET', 'POST'])
+@roles_required('Super User')
+def view_license_tiers():
+ page = request.args.get('page', 1, type=int)
+ per_page = request.args.get('per_page', 10, type=int)
+ today = dt.now(tz.utc)
+
+ query = LicenseTier.query.filter(
+ or_(
+ LicenseTier.end_date == None,
+ LicenseTier.end_date >= today
+ )
+ ).order_by(LicenseTier.start_date.desc(), LicenseTier.id)
+
+ pagination = query.paginate(page=page, per_page=per_page, error_out=False)
+ license_tiers = pagination.items
+
+ rows = prepare_table_for_macro(license_tiers, [('id', ''), ('name', ''), ('version', ''), ('start_date', ''),
+ ('end_date', '')])
+
+ return render_template('entitlements/view_license_tiers.html', rows=rows, pagination=pagination)
+
+
+@entitlements_bp.route('/handle_license_tier_selection', methods=['POST'])
+@roles_required('Super User')
+def handle_license_tier_selection():
+ license_tier_identification = request.form['selected_row']
+ license_tier_id = ast.literal_eval(license_tier_identification).get('value')
+ the_license_tier = LicenseTier.query.get(license_tier_id)
+
+ action = request.form['action']
+
+ match action:
+ case 'edit_license_tier':
+ return redirect(prefixed_url_for('entitlements_bp.edit_license_tier',
+ license_tier_id=license_tier_id))
+ case 'create_license_for_tenant':
+ return redirect(prefixed_url_for('entitlements_bp.create_license',
+ license_tier_id=license_tier_id))
+ # Add more conditions for other actions
+ return redirect(prefixed_url_for('entitlements_bp.view_license_tiers'))
+
+
+@entitlements_bp.route('/license_tier/', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def edit_license_tier(license_tier_id):
+ license_tier = LicenseTier.query.get_or_404(license_tier_id) # This will return a 404 if no license tier is found
+ form = LicenseTierForm(obj=license_tier)
+
+ if form.validate_on_submit():
+ # Populate the license_tier with form data
+ form.populate_obj(license_tier)
+
+ try:
+ db.session.add(license_tier)
+ db.session.commit()
+ except SQLAlchemyError as e:
+ db.session.rollback()
+ current_app.logger.error(f'Failed to edit License Tier. Error: {str(e)}')
+ flash(f'Failed to edit License Tier. Error: {str(e)}', 'danger')
+ return render_template('entitlements/license_tier.html', form=form, license_tier_id=license_tier.id)
+
+ flash('License Tier updated successfully.', 'success')
+ return redirect(
+ prefixed_url_for('entitlements_bp.edit_license_tier', license_tier_id=license_tier_id))
+ else:
+ form_validation_failed(request, form)
+
+ return render_template('entitlements/license_tier.html', form=form, license_tier_id=license_tier.id)
+
+
+@entitlements_bp.route('/create_license/', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def create_license(license_tier_id):
+ form = LicenseForm()
+ tenant_id = session.get('tenant').get('id')
+ currency = session.get('tenant').get('currency')
+
+ if request.method == 'GET':
+ # Fetch the LicenseTier
+ license_tier = LicenseTier.query.get_or_404(license_tier_id)
+
+ # Prefill the form with LicenseTier data
+ # Currency depending data
+ if currency == '$':
+ form.basic_fee.data = license_tier.basic_fee_d
+ form.additional_storage_price.data = license_tier.additional_storage_price_d
+ form.additional_embedding_price.data = license_tier.additional_embedding_price_d
+ form.additional_interaction_token_price.data = license_tier.additional_interaction_token_price_d
+ elif currency == '€':
+ form.basic_fee.data = license_tier.basic_fee_e
+ form.additional_storage_price.data = license_tier.additional_storage_price_e
+ form.additional_embedding_price.data = license_tier.additional_embedding_price_e
+ form.additional_interaction_token_price.data = license_tier.additional_interaction_token_price_e
+ else:
+ current_app.logger.error(f'Invalid currency {currency} for tenant {tenant_id} while creating license.')
+ flash(f"Invalid currency {currency} for tenant {tenant_id} while creating license. "
+ f"Check tenant's currency and try again.", 'danger')
+ return redirect(prefixed_url_for('user_bp.edit_tenant', tenant_id=tenant_id))
+ # General data
+ form.currency.data = currency
+ form.max_storage_mb.data = license_tier.max_storage_mb
+ form.additional_storage_bucket.data = license_tier.additional_storage_bucket
+ form.included_embedding_mb.data = license_tier.included_embedding_mb
+ form.additional_embedding_bucket.data = license_tier.additional_embedding_bucket
+ form.included_interaction_tokens.data = license_tier.included_interaction_tokens
+ form.additional_interaction_bucket.data = license_tier.additional_interaction_bucket
+ form.overage_embedding.data = license_tier.standard_overage_embedding
+ form.overage_interaction.data = license_tier.standard_overage_interaction
+ else: # POST
+ # Create a new License instance
+ new_license = License(
+ tenant_id=tenant_id,
+ tier_id=license_tier_id,
+ )
+ current_app.logger.debug(f"Currency data in form: {form.currency.data}")
+ if form.validate_on_submit():
+ # Update the license with form data
+ form.populate_obj(new_license)
+ # Currency is added here again, as a form doesn't include disabled fields when passing it in the request
+ new_license.currency = currency
+
+ try:
+ db.session.add(new_license)
+ db.session.commit()
+ flash('License created successfully', 'success')
+ return redirect(prefixed_url_for('entitlements_bp/edit_license', license_id=new_license.id))
+ except Exception as e:
+ db.session.rollback()
+ flash(f'Error creating license: {str(e)}', 'error')
+ else:
+ form_validation_failed(request, form)
+
+ return render_template('entitlements/license.html', form=form)
+
+
+@entitlements_bp.route('/license/', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def edit_license(license_id):
+ license = License.query.get_or_404(license_id) # This will return a 404 if no license tier is found
+ form = LicenseForm(obj=license)
+ disabled_fields = []
+ if len(license.usages) > 0: # There already are usage records linked to this license
+ # Define which fields should be disabled
+ disabled_fields = [field.name for field in form if field.name != 'end_date']
+
+ if form.validate_on_submit():
+ # Populate the license with form data
+ form.populate_obj(license)
+
+ try:
+ db.session.add(license)
+ db.session.commit()
+ except SQLAlchemyError as e:
+ db.session.rollback()
+ current_app.logger.error(f'Failed to edit License. Error: {str(e)}')
+ flash(f'Failed to edit License. Error: {str(e)}', 'danger')
+ return render_template('entitlements/license.html', form=form)
+
+ flash('License updated successfully.', 'success')
+ return redirect(
+ prefixed_url_for('entitlements_bp.edit_license', license_tier_id=license_id))
+ else:
+ form_validation_failed(request, form)
+
+ return render_template('entitlements/license.html', form=form, license_tier_id=license_tier.id,
+ ext_disabled_fields=disabled_fields)
diff --git a/eveai_app/views/user_forms.py b/eveai_app/views/user_forms.py
index c728443..ae05fe9 100644
--- a/eveai_app/views/user_forms.py
+++ b/eveai_app/views/user_forms.py
@@ -14,6 +14,9 @@ class TenantForm(FlaskForm):
# language fields
default_language = SelectField('Default Language', choices=[], validators=[DataRequired()])
allowed_languages = SelectMultipleField('Allowed Languages', choices=[], validators=[DataRequired()])
+ # invoicing fields
+ currency = SelectField('Currency', choices=[], validators=[DataRequired()])
+ usage_email = EmailField('Usage Email', validators=[DataRequired(), Email()])
# Timezone
timezone = SelectField('Timezone', choices=[], validators=[DataRequired()])
# RAG context
@@ -23,10 +26,6 @@ class TenantForm(FlaskForm):
# LLM fields
embedding_model = SelectField('Embedding Model', choices=[], validators=[DataRequired()])
llm_model = SelectField('Large Language Model', choices=[], validators=[DataRequired()])
- # license fields
- license_start_date = DateField('License Start Date', id='form-control datepicker', validators=[Optional()])
- license_end_date = DateField('License End Date', id='form-control datepicker', validators=[Optional()])
- allowed_monthly_interactions = IntegerField('Allowed Monthly Interactions', validators=[NumberRange(min=0)])
# Embedding variables
html_tags = StringField('HTML Tags', validators=[DataRequired()],
default='p, h1, h2, h3, h4, h5, h6, li')
@@ -59,6 +58,8 @@ class TenantForm(FlaskForm):
# initialise language fields
self.default_language.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
self.allowed_languages.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
+ # initialise currency field
+ self.currency.choices = [(curr, curr) for curr in current_app.config['SUPPORTED_CURRENCIES']]
# initialise timezone
self.timezone.choices = [(tz, tz) for tz in pytz.all_timezones]
# initialise LLM fields
diff --git a/eveai_app/views/user_views.py b/eveai_app/views/user_views.py
index d994393..384e745 100644
--- a/eveai_app/views/user_views.py
+++ b/eveai_app/views/user_views.py
@@ -47,18 +47,6 @@ def tenant():
# Handle the required attributes
new_tenant = Tenant()
form.populate_obj(new_tenant)
- # new_tenant = Tenant(name=form.name.data,
- # website=form.website.data,
- # default_language=form.default_language.data,
- # allowed_languages=form.allowed_languages.data,
- # timezone=form.timezone.data,
- # embedding_model=form.embedding_model.data,
- # llm_model=form.llm_model.data,
- # license_start_date=form.license_start_date.data,
- # license_end_date=form.license_end_date.data,
- # allowed_monthly_interactions=form.allowed_monthly_interactions.data,
- # embed_tuning=form.embed_tuning.data,
- # rag_tuning=form.rag_tuning.data)
# Handle Embedding Variables
new_tenant.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
@@ -87,7 +75,7 @@ def tenant():
db.session.commit()
except SQLAlchemyError as e:
current_app.logger.error(f'Failed to add tenant to database. Error: {str(e)}')
- flash(f'Failed to add tenant to database. Error: {str(e)}')
+ flash(f'Failed to add tenant to database. Error: {str(e)}', 'danger')
return render_template('user/tenant.html', form=form)
current_app.logger.info(f"Successfully created tenant {new_tenant.id} in Database")
@@ -152,7 +140,7 @@ def edit_tenant(tenant_id):
current_app.logger.debug(f'Tenant update failed with errors: {form.errors}')
form_validation_failed(request, form)
- return render_template('user/edit_tenant.html', form=form, tenant_id=tenant_id)
+ return render_template('user/tenant.html', form=form, tenant_id=tenant_id)
@user_bp.route('/user', methods=['GET', 'POST'])
diff --git a/eveai_workers/tasks.py b/eveai_workers/tasks.py
index bbc28e1..5266554 100644
--- a/eveai_workers/tasks.py
+++ b/eveai_workers/tasks.py
@@ -36,8 +36,14 @@ def ping():
@current_celery.task(name='create_embeddings', queue='embeddings')
def create_embeddings(tenant_id, document_version_id):
+ # Retrieve document version to process
+ document_version = DocumentVersion.query.get(document_version_id)
+ if document_version is None:
+ raise Exception(f'Document version {document_version_id} not found')
# BusinessEvent creates a context, which is why we need to use it with a with block
- with BusinessEvent('Create Embeddings', tenant_id, document_version_id=document_version_id):
+ with BusinessEvent('Create Embeddings', tenant_id,
+ document_version_id=document_version_id,
+ document_version_file_size=document_version.file_size):
current_app.logger.info(f'Creating embeddings for tenant {tenant_id} on document version {document_version_id}')
try:
# Retrieve Tenant for which we are processing
@@ -52,11 +58,6 @@ def create_embeddings(tenant_id, document_version_id):
model_variables = select_model_variables(tenant)
current_app.logger.debug(f'Model variables: {model_variables}')
- # Retrieve document version to process
- document_version = DocumentVersion.query.get(document_version_id)
- if document_version is None:
- raise Exception(f'Document version {document_version_id} not found')
-
except Exception as e:
current_app.logger.error(f'Create Embeddings request received '
f'for non existing document version {document_version_id} '
diff --git a/migrations/public/versions/48714f1baac5_add_entitlement_information_to_tenant.py b/migrations/public/versions/48714f1baac5_add_entitlement_information_to_tenant.py
new file mode 100644
index 0000000..6dce7ed
--- /dev/null
+++ b/migrations/public/versions/48714f1baac5_add_entitlement_information_to_tenant.py
@@ -0,0 +1,40 @@
+"""Add entitlement information to Tenant
+
+Revision ID: 48714f1baac5
+Revises: f201bfd23152
+Create Date: 2024-10-03 14:49:53.922320
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '48714f1baac5'
+down_revision = 'f201bfd23152'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('license', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('yearly_payment', sa.Boolean(), nullable=False))
+
+ with op.batch_alter_table('tenant', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('currency', sa.String(length=20), nullable=True))
+ batch_op.add_column(sa.Column('usage_email', sa.String(length=255), nullable=True))
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('tenant', schema=None) as batch_op:
+ batch_op.drop_column('usage_email')
+ batch_op.drop_column('currency')
+
+ with op.batch_alter_table('license', schema=None) as batch_op:
+ batch_op.drop_column('yearly_payment')
+
+ # ### end Alembic commands ###
diff --git a/migrations/public/versions/560d08d91e5b_introducing_new_licensing_approach.py b/migrations/public/versions/560d08d91e5b_introducing_new_licensing_approach.py
new file mode 100644
index 0000000..ac3b0c0
--- /dev/null
+++ b/migrations/public/versions/560d08d91e5b_introducing_new_licensing_approach.py
@@ -0,0 +1,106 @@
+"""Introducing new licensing approach
+
+Revision ID: 560d08d91e5b
+Revises: 254932fe7fe3
+Create Date: 2024-10-02 15:29:05.963865
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '560d08d91e5b'
+down_revision = '254932fe7fe3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('license_tier',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(length=50), nullable=False),
+ sa.Column('version', sa.String(length=50), nullable=False),
+ sa.Column('start_date', sa.Date(), nullable=False),
+ sa.Column('end_date', sa.Date(), nullable=True),
+ sa.Column('basic_fee_d', sa.Float(), nullable=True),
+ sa.Column('basic_fee_e', sa.Float(), nullable=True),
+ sa.Column('max_storage_tokens', sa.Integer(), nullable=False),
+ sa.Column('additional_storage_token_price_d', sa.Numeric(precision=10, scale=4), nullable=False),
+ sa.Column('additional_storage_token_price_e', sa.Numeric(precision=10, scale=4), nullable=False),
+ sa.Column('included_embedding_tokens', sa.Integer(), nullable=False),
+ sa.Column('additional_embedding_token_price_d', sa.Numeric(precision=10, scale=4), nullable=False),
+ sa.Column('additional_embedding_token_price_e', sa.Numeric(precision=10, scale=4), nullable=False),
+ sa.Column('additional_embedding_bucket', sa.Integer(), nullable=False),
+ sa.Column('included_interaction_tokens', sa.Integer(), nullable=False),
+ sa.Column('additional_interaction_token_price_d', sa.Numeric(precision=10, scale=4), nullable=False),
+ sa.Column('additional_interaction_token_price_e', sa.Numeric(precision=10, scale=4), nullable=False),
+ sa.Column('additional_interaction_bucket', sa.Integer(), nullable=False),
+ sa.Column('allow_overage', sa.Boolean(), nullable=True),
+ sa.PrimaryKeyConstraint('id'),
+ schema='public'
+ )
+ op.create_table('license',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('tenant_id', sa.Integer(), nullable=False),
+ sa.Column('tier_id', sa.Integer(), nullable=False),
+ sa.Column('start_date', sa.Date(), nullable=False),
+ sa.Column('end_date', sa.Date(), nullable=True),
+ sa.Column('currency', sa.String(length=20), nullable=False),
+ sa.Column('basic_fee', sa.Float(), nullable=False),
+ sa.Column('max_storage_tokens', sa.Integer(), nullable=False),
+ sa.Column('additional_storage_token_price', sa.Float(), nullable=False),
+ sa.Column('included_embedding_tokens', sa.Integer(), nullable=False),
+ sa.Column('additional_embedding_token_price', sa.Numeric(precision=10, scale=4), nullable=False),
+ sa.Column('additional_embedding_bucket', sa.Integer(), nullable=False),
+ sa.Column('included_interaction_tokens', sa.Integer(), nullable=False),
+ sa.Column('additional_interaction_token_price', sa.Numeric(precision=10, scale=4), nullable=False),
+ sa.Column('additional_interaction_bucket', sa.Integer(), nullable=False),
+ sa.Column('allow_overage', sa.Boolean(), nullable=True),
+ sa.ForeignKeyConstraint(['tenant_id'], ['public.tenant.id'], ),
+ sa.ForeignKeyConstraint(['tier_id'], ['public.license_tier.id'], ),
+ sa.PrimaryKeyConstraint('id'),
+ schema='public'
+ )
+ op.create_table('license_usage',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('license_id', sa.Integer(), nullable=False),
+ sa.Column('tenant_id', sa.Integer(), nullable=False),
+ sa.Column('storage_tokens_used', sa.Integer(), nullable=True),
+ sa.Column('embedding_tokens_used', sa.Integer(), nullable=True),
+ sa.Column('interaction_tokens_used', sa.Integer(), nullable=True),
+ sa.Column('period_start_date', sa.Date(), nullable=False),
+ sa.Column('period_end_date', sa.Date(), nullable=False),
+ sa.ForeignKeyConstraint(['license_id'], ['public.license.id'], ),
+ sa.ForeignKeyConstraint(['tenant_id'], ['public.tenant.id'], ),
+ sa.PrimaryKeyConstraint('id'),
+ schema='public'
+ )
+ with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('license_usage_id', sa.Integer(), nullable=True))
+ batch_op.create_foreign_key(None, 'license_usage', ['license_usage_id'], ['id'], referent_schema='public')
+
+ with op.batch_alter_table('tenant', schema=None) as batch_op:
+ batch_op.drop_column('license_end_date')
+ batch_op.drop_column('allowed_monthly_interactions')
+ batch_op.drop_column('license_start_date')
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('tenant', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('license_start_date', sa.DATE(), autoincrement=False, nullable=True))
+ batch_op.add_column(sa.Column('allowed_monthly_interactions', sa.INTEGER(), autoincrement=False, nullable=True))
+ batch_op.add_column(sa.Column('license_end_date', sa.DATE(), autoincrement=False, nullable=True))
+
+ with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.drop_column('license_usage_id')
+
+ op.drop_table('license_usage', schema='public')
+ op.drop_table('license', schema='public')
+ op.drop_table('license_tier', schema='public')
+ # ### end Alembic commands ###
diff --git a/migrations/public/versions/6a7743d08106_adapt_licenseusage_to_accept_mb_iso_.py b/migrations/public/versions/6a7743d08106_adapt_licenseusage_to_accept_mb_iso_.py
new file mode 100644
index 0000000..e58cf62
--- /dev/null
+++ b/migrations/public/versions/6a7743d08106_adapt_licenseusage_to_accept_mb_iso_.py
@@ -0,0 +1,38 @@
+"""Adapt LicenseUsage to accept mb iso tokoens for Storage and Embeddings
+
+Revision ID: 6a7743d08106
+Revises: 9429f244f1a5
+Create Date: 2024-10-07 06:04:39.424243
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6a7743d08106'
+down_revision = '9429f244f1a5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('license_usage', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('storage_mb_used', sa.Integer(), nullable=True))
+ batch_op.add_column(sa.Column('embedding_mb_used', sa.Integer(), nullable=True))
+ batch_op.drop_column('storage_tokens_used')
+ batch_op.drop_column('embedding_tokens_used')
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('license_usage', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('embedding_tokens_used', sa.INTEGER(), autoincrement=False, nullable=True))
+ batch_op.add_column(sa.Column('storage_tokens_used', sa.INTEGER(), autoincrement=False, nullable=True))
+ batch_op.drop_column('embedding_mb_used')
+ batch_op.drop_column('storage_mb_used')
+
+ # ### end Alembic commands ###
diff --git a/migrations/public/versions/9429f244f1a5_moved_storage_and_embedding_licensing_.py b/migrations/public/versions/9429f244f1a5_moved_storage_and_embedding_licensing_.py
new file mode 100644
index 0000000..d2e7032
--- /dev/null
+++ b/migrations/public/versions/9429f244f1a5_moved_storage_and_embedding_licensing_.py
@@ -0,0 +1,86 @@
+"""Moved storage and embedding licensing to Mb iso tokens
+
+Revision ID: 9429f244f1a5
+Revises: 48714f1baac5
+Create Date: 2024-10-04 08:07:47.976861
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9429f244f1a5'
+down_revision = '48714f1baac5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('license', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('max_storage_mb', sa.Integer(), nullable=False))
+ batch_op.add_column(sa.Column('additional_storage_price', sa.Float(), nullable=False))
+ batch_op.add_column(sa.Column('included_embedding_mb', sa.Integer(), nullable=False))
+ batch_op.add_column(sa.Column('additional_embedding_price', sa.Numeric(precision=10, scale=4), nullable=False))
+ batch_op.add_column(sa.Column('overage_embedding', sa.Float(), nullable=False))
+ batch_op.add_column(sa.Column('overage_interaction', sa.Float(), nullable=False))
+ batch_op.drop_column('additional_storage_token_price')
+ batch_op.drop_column('additional_embedding_token_price')
+ batch_op.drop_column('max_storage_tokens')
+ batch_op.drop_column('allow_overage')
+ batch_op.drop_column('included_embedding_tokens')
+
+ with op.batch_alter_table('license_tier', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('max_storage_mb', sa.Integer(), nullable=False))
+ batch_op.add_column(sa.Column('additional_storage_price_d', sa.Numeric(precision=10, scale=4), nullable=False))
+ batch_op.add_column(sa.Column('additional_storage_price_e', sa.Numeric(precision=10, scale=4), nullable=False))
+ batch_op.add_column(sa.Column('included_embedding_mb', sa.Integer(), nullable=False))
+ batch_op.add_column(sa.Column('additional_embedding_price_d', sa.Numeric(precision=10, scale=4), nullable=False))
+ batch_op.add_column(sa.Column('additional_embedding_price_e', sa.Numeric(precision=10, scale=4), nullable=False))
+ batch_op.add_column(sa.Column('standard_overage_embedding', sa.Float(), nullable=False))
+ batch_op.add_column(sa.Column('standard_overage_interaction', sa.Float(), nullable=False))
+ batch_op.drop_column('max_storage_tokens')
+ batch_op.drop_column('additional_storage_token_price_e')
+ batch_op.drop_column('allow_overage')
+ batch_op.drop_column('additional_embedding_token_price_d')
+ batch_op.drop_column('included_embedding_tokens')
+ batch_op.drop_column('additional_embedding_token_price_e')
+ batch_op.drop_column('additional_storage_token_price_d')
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('license_tier', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('additional_storage_token_price_d', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
+ batch_op.add_column(sa.Column('additional_embedding_token_price_e', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
+ batch_op.add_column(sa.Column('included_embedding_tokens', sa.INTEGER(), autoincrement=False, nullable=False))
+ batch_op.add_column(sa.Column('additional_embedding_token_price_d', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
+ batch_op.add_column(sa.Column('allow_overage', sa.BOOLEAN(), autoincrement=False, nullable=True))
+ batch_op.add_column(sa.Column('additional_storage_token_price_e', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
+ batch_op.add_column(sa.Column('max_storage_tokens', sa.INTEGER(), autoincrement=False, nullable=False))
+ batch_op.drop_column('standard_overage_interaction')
+ batch_op.drop_column('standard_overage_embedding')
+ batch_op.drop_column('additional_embedding_price_e')
+ batch_op.drop_column('additional_embedding_price_d')
+ batch_op.drop_column('included_embedding_mb')
+ batch_op.drop_column('additional_storage_price_e')
+ batch_op.drop_column('additional_storage_price_d')
+ batch_op.drop_column('max_storage_mb')
+
+ with op.batch_alter_table('license', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('included_embedding_tokens', sa.INTEGER(), autoincrement=False, nullable=False))
+ batch_op.add_column(sa.Column('allow_overage', sa.BOOLEAN(), autoincrement=False, nullable=True))
+ batch_op.add_column(sa.Column('max_storage_tokens', sa.INTEGER(), autoincrement=False, nullable=False))
+ batch_op.add_column(sa.Column('additional_embedding_token_price', sa.NUMERIC(precision=10, scale=4), autoincrement=False, nullable=False))
+ batch_op.add_column(sa.Column('additional_storage_token_price', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=False))
+ batch_op.drop_column('overage_interaction')
+ batch_op.drop_column('overage_embedding')
+ batch_op.drop_column('additional_embedding_price')
+ batch_op.drop_column('included_embedding_mb')
+ batch_op.drop_column('additional_storage_price')
+ batch_op.drop_column('max_storage_mb')
+
+ # ### end Alembic commands ###
diff --git a/migrations/public/versions/d616ea937a6a_corrections_on_licensing_models.py b/migrations/public/versions/d616ea937a6a_corrections_on_licensing_models.py
new file mode 100644
index 0000000..4a06307
--- /dev/null
+++ b/migrations/public/versions/d616ea937a6a_corrections_on_licensing_models.py
@@ -0,0 +1,58 @@
+"""Corrections on Licensing models
+
+Revision ID: d616ea937a6a
+Revises: 560d08d91e5b
+Create Date: 2024-10-02 15:39:27.668741
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd616ea937a6a'
+down_revision = '560d08d91e5b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+ batch_op.drop_constraint('business_event_log_license_usage_id_fkey', type_='foreignkey')
+ batch_op.create_foreign_key(None, 'license_usage', ['license_usage_id'], ['id'], referent_schema='public')
+
+ with op.batch_alter_table('license', schema=None) as batch_op:
+ batch_op.drop_constraint('license_tenant_id_fkey', type_='foreignkey')
+ batch_op.drop_constraint('license_tier_id_fkey', type_='foreignkey')
+ batch_op.create_foreign_key(None, 'license_tier', ['tier_id'], ['id'], referent_schema='public')
+ batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
+
+ with op.batch_alter_table('license_usage', schema=None) as batch_op:
+ batch_op.drop_constraint('license_usage_license_id_fkey', type_='foreignkey')
+ batch_op.drop_constraint('license_usage_tenant_id_fkey', type_='foreignkey')
+ batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
+ batch_op.create_foreign_key(None, 'license', ['license_id'], ['id'], referent_schema='public')
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('license_usage', schema=None) as batch_op:
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.create_foreign_key('license_usage_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
+ batch_op.create_foreign_key('license_usage_license_id_fkey', 'license', ['license_id'], ['id'])
+
+ with op.batch_alter_table('license', schema=None) as batch_op:
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.create_foreign_key('license_tier_id_fkey', 'license_tier', ['tier_id'], ['id'])
+ batch_op.create_foreign_key('license_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
+
+ with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.create_foreign_key('business_event_log_license_usage_id_fkey', 'license_usage', ['license_usage_id'], ['id'])
+
+ # ### end Alembic commands ###
diff --git a/migrations/public/versions/f201bfd23152_add_buckets_to_license_information.py b/migrations/public/versions/f201bfd23152_add_buckets_to_license_information.py
new file mode 100644
index 0000000..321d98c
--- /dev/null
+++ b/migrations/public/versions/f201bfd23152_add_buckets_to_license_information.py
@@ -0,0 +1,66 @@
+"""Add buckets to License information
+
+Revision ID: f201bfd23152
+Revises: d616ea937a6a
+Create Date: 2024-10-03 09:44:32.867470
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f201bfd23152'
+down_revision = 'd616ea937a6a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+ batch_op.drop_constraint('business_event_log_license_usage_id_fkey', type_='foreignkey')
+ batch_op.create_foreign_key(None, 'license_usage', ['license_usage_id'], ['id'], referent_schema='public')
+
+ with op.batch_alter_table('license', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('additional_storage_bucket', sa.Integer(), nullable=False))
+ batch_op.drop_constraint('license_tier_id_fkey', type_='foreignkey')
+ batch_op.drop_constraint('license_tenant_id_fkey', type_='foreignkey')
+ batch_op.create_foreign_key(None, 'license_tier', ['tier_id'], ['id'], referent_schema='public')
+ batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
+
+ with op.batch_alter_table('license_tier', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('additional_storage_bucket', sa.Integer(), nullable=False))
+
+ with op.batch_alter_table('license_usage', schema=None) as batch_op:
+ batch_op.drop_constraint('license_usage_tenant_id_fkey', type_='foreignkey')
+ batch_op.drop_constraint('license_usage_license_id_fkey', type_='foreignkey')
+ batch_op.create_foreign_key(None, 'license', ['license_id'], ['id'], referent_schema='public')
+ batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('license_usage', schema=None) as batch_op:
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.create_foreign_key('license_usage_license_id_fkey', 'license', ['license_id'], ['id'])
+ batch_op.create_foreign_key('license_usage_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
+
+ with op.batch_alter_table('license_tier', schema=None) as batch_op:
+ batch_op.drop_column('additional_storage_bucket')
+
+ with op.batch_alter_table('license', schema=None) as batch_op:
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.create_foreign_key('license_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
+ batch_op.create_foreign_key('license_tier_id_fkey', 'license_tier', ['tier_id'], ['id'])
+ batch_op.drop_column('additional_storage_bucket')
+
+ with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+ batch_op.drop_constraint(None, type_='foreignkey')
+ batch_op.create_foreign_key('business_event_log_license_usage_id_fkey', 'license_usage', ['license_usage_id'], ['id'])
+
+ # ### end Alembic commands ###
diff --git a/migrations/tenant/versions/322d3cf1f17b_documentversion_update_to_bucket_name_.py b/migrations/tenant/versions/322d3cf1f17b_documentversion_update_to_bucket_name_.py
new file mode 100644
index 0000000..3ce93d8
--- /dev/null
+++ b/migrations/tenant/versions/322d3cf1f17b_documentversion_update_to_bucket_name_.py
@@ -0,0 +1,94 @@
+"""DocumentVersion update to bucket_name, object_name and file_size to better reflet Minio reality
+
+Revision ID: 322d3cf1f17b
+Revises: 711a09a77680
+Create Date: 2024-10-07 07:45:19.014017
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.orm import Session
+from sqlalchemy import text
+from flask import current_app
+
+# revision identifiers, used by Alembic.
+revision = '322d3cf1f17b'
+down_revision = '711a09a77680'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - Add new fields ###
+ op.add_column('document_version', sa.Column('bucket_name', sa.String(length=255), nullable=True))
+ op.add_column('document_version', sa.Column('object_name', sa.String(length=200), nullable=True))
+ op.add_column('document_version', sa.Column('file_size', sa.Float(), nullable=True))
+
+ # ### Upgrade values for bucket_name, object_name and file_size to reflect minio reality ###
+ from common.models.document import DocumentVersion
+ from common.extensions import minio_client
+ from minio.error import S3Error
+
+ # Create a connection
+ connection = op.get_bind()
+ session = Session(bind=connection)
+
+ # Get the current schema name (which should be the tenant ID)
+ current_schema = connection.execute(text("SELECT current_schema()")).scalar()
+ tenant_id = int(current_schema)
+
+ doc_versions = session.query(DocumentVersion).all()
+ for doc_version in doc_versions:
+ try:
+ object_name = minio_client.generate_object_name(doc_version.doc_id,
+ doc_version.language,
+ doc_version.id,
+ doc_version.file_name)
+ bucket_name = minio_client.generate_bucket_name(tenant_id)
+ doc_version.object_name = object_name
+ doc_version.bucket_name = bucket_name
+
+ try:
+ stat = minio_client.client.stat_object(
+ bucket_name=bucket_name,
+ object_name=object_name
+ )
+ doc_version.file_size = stat.size / 1048576
+ current_app.logger.info(f"Processed Upgrade for DocumentVersion {doc_version.id} for Tenant {tenant_id}")
+ except S3Error as e:
+ if e.code == "NoSuchKey":
+ current_app.logger.warning(
+ f"Object {doc_version.file_location} not found in bucket {doc_version.bucket_name}. Skipping.")
+ continue # Move to the next item
+ else:
+ raise e # Handle other types of S3 errors
+ except Exception as e:
+ session.rollback()
+ current_app.logger.error(f"Couldn't process upgrade for DocumentVersion {doc_version.id} for "
+ f"Tenant {tenant_id}. Error: {str(e)}")
+
+ try:
+ session.commit()
+ current_app.logger.info(f"Successfully updated file sizes for tenant schema {current_schema}")
+ except Exception as e:
+ session.rollback()
+ current_app.logger.error(f"Error committing changes for tenant schema {current_schema}: {str(e)}")
+
+ # ### commands auto generated by Alembic - Remove old fields ###
+ # op.drop_column('document_version', 'file_location')
+ # op.drop_column('document_version', 'file_name')
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('document_version', sa.Column('file_name', sa.VARCHAR(length=200), autoincrement=False, nullable=True))
+ op.add_column('document_version', sa.Column('file_location', sa.VARCHAR(length=255), autoincrement=False, nullable=True))
+ op.drop_column('document_version', 'file_size')
+ op.drop_column('document_version', 'object_name')
+ op.drop_column('document_version', 'bucket_name')
+
+ # ### end Alembic commands ###
diff --git a/scripts/repopack_eveai.sh b/scripts/repopack_eveai.sh
new file mode 100755
index 0000000..4ee5731
--- /dev/null
+++ b/scripts/repopack_eveai.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Run repopack to generate the file
+repopack
+
+# Check if repopack generated the eveai_repo.txt file
+if [[ -f "eveai_repo.txt" ]]; then
+ # Get the current timestamp in the format YYYY-DD-MM_HH:MM:SS
+ timestamp=$(date +"%Y-%d-%m_%H-%M-%S")
+
+ # Rename the file with the timestamp
+ mv eveai_repo.txt "${timestamp}_eveai_repo.txt"
+
+ echo "File renamed to ${timestamp}_eveai_repo.txt"
+else
+ echo "Error: eveai_repo.txt not found. repopack may have failed."
+fi