pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Add functionality to add a default dictionary for configuration fields

Browse Source 
- Correct entitlement processing
- Remove get_template functionality from ModelVariables, define it directly with LLM model definition in configuration file.
This commit is contained in:
Josako
2025-05-19 14:10:09 +02:00
parent d2a9092f46
commit 28aea85b10
15 changed files with 386 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
config/config.py
View File

@@ -75,8 +75,8 @@ class Config(object):
# supported LLMs # supported LLMs
# SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed'] # SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed']
SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed'] SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed']
SUPPORTED_LLMS = ['openai.gpt-4o', 'anthropic.claude-3-5-sonnet', 'openai.gpt-4o-mini', SUPPORTED_LLMS = ['openai.gpt-4o', 'openai.gpt-4o-mini',
'mistral.mistral-large-latest', 'mistral.mistral-small-latest'] 'mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest']
ANTHROPIC_LLM_VERSIONS = {'claude-3-5-sonnet': 'claude-3-5-sonnet-20240620', } ANTHROPIC_LLM_VERSIONS = {'claude-3-5-sonnet': 'claude-3-5-sonnet-20240620', }

2
config/prompts/global/html_parse/1.0.0.yaml
View File

@@ -13,7 +13,7 @@ content: |
HTML is between triple backquotes. HTML is between triple backquotes.
```{html}``` ```{html}```
model: "mistral.mistral-small-latest" llm_model: "mistral.mistral-small-latest"
metadata: metadata:
author: "Josako" author: "Josako"
date_added: "2024-11-10" date_added: "2024-11-10"

138
eveai_app/templates/entitlements/view_active_license_usage.html Normal file
View File

@@ -0,0 +1,138 @@
{% extends 'base.html' %}
{% from "macros.html" import render_selectable_table, render_pagination %}
{% block title %}Active License Usage{% endblock %}
{% block content_title %}Active License Usage{% endblock %}
{% block content_description %}Overview of the current license period usage{% endblock %}
{% block content %}
{% if active_period and active_period.license_usage %}
<div class="row mb-4">
<div class="col-md-6">
<div class="card shadow-sm">
<div class="card-header p-3">
<h6 class="mb-0">License Information</h6>
</div>
<div class="card-body">
<p><strong>License ID:</strong> {{ active_period.license.id }}</p>
<p><strong>License Name:</strong> {{ active_period.license.name }}</p>
<p><strong>Period Nr:</strong> {{ active_period.period_number }}</p>
<p><strong>Start Date:</strong> {{ active_period.period_start }}</p>
<p><strong>End Date:</strong> {{ active_period.period_end }}</p>
<p><strong>Status:</strong> {{ active_period.status.value }}</p>
</div>
</div>
</div>
<div class="col-md-6">
<div class="card shadow-sm">
<div class="card-header p-3">
<h6 class="mb-0">License Details for Period</h6>
</div>
<div class="card-body">
<p><strong>Max Storage:</strong> {{ active_period.max_storage_mb }} MB</p>
<p><strong>Included Embedding:</strong> {{ active_period.included_embedding_mb }} MB</p>
<p><strong>Included Interaction Tokens:</strong> {{ active_period.included_interaction_tokens }}</p>
<p><strong>Overage Storage Allowed:</strong> {{ active_period.additional_storage_allowed }}</p>
<p><strong>Overage Embedding Allowed:</strong> {{ active_period.additional_embedding_allowed }}</p>
<p><strong>Overage Interaction Allowed:</strong> {{ active_period.additional_interaction_allowed }}</p>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-md-4">
<div class="card shadow-sm">
<div class="card-header p-3">
<h6 class="mb-0">Storage Usage</h6>
</div>
<div class="card-body pt-0">
<h2 class="font-weight-bold mt-3
{% if usage_data.storage_percent >= 95 %}text-danger
{% elif usage_data.storage_percent > 80 %}text-warning
{% else %}text-info{% endif %}">
{{ usage_data.storage_percent }}%
</h2>
<div class="progress mt-2">
<div class="progress-bar
{% if usage_data.storage_percent >= 95 %}bg-danger
{% elif usage_data.storage_percent > 80 %}bg-warning
{% else %}bg-info{% endif %}"
role="progressbar"
style="width: {{ usage_data.storage_percent }}%"
aria-valuenow="{{ usage_data.storage_percent }}"
aria-valuemin="0"
aria-valuemax="100">
</div>
</div>
<p class="mt-3">{{ usage_data.storage_used_rounded or 0.0 }} / {{ active_period.max_storage_mb or 0 }} MB</p>
</div>
</div>
</div>
<div class="col-md-4">
<div class="card shadow-sm">
<div class="card-header p-3">
<h6 class="mb-0">Embedding Usage</h6>
</div>
<div class="card-body pt-0">
<h2 class="font-weight-bold mt-3
{% if usage_data.embedding_percent >= 95 %}text-danger
{% elif usage_data.embedding_percent > 80 %}text-warning
{% else %}text-info{% endif %}">
{{ usage_data.embedding_percent }}%
</h2>
<div class="progress mt-2">
<div class="progress-bar
{% if usage_data.embedding_percent >= 95 %}bg-danger
{% elif usage_data.embedding_percent > 80 %}bg-warning
{% else %}bg-info{% endif %}"
role="progressbar"
style="width: {{ usage_data.embedding_percent }}%"
aria-valuenow="{{ usage_data.embedding_percent }}"
aria-valuemin="0"
aria-valuemax="100">
</div>
</div>
<p class="mt-3">{{ usage_data.embedding_used_rounded or 0.0 }} / {{ active_period.included_embedding_mb or 0 }} MB</p>
</div>
</div>
</div>
<div class="col-md-4">
<div class="card shadow-sm">
<div class="card-header p-3">
<h6 class="mb-0">Interaction Usage</h6>
</div>
<div class="card-body pt-0">
<h2 class="font-weight-bold mt-3
{% if usage_data.interaction_percent >= 95 %}text-danger
{% elif usage_data.interaction_percent > 80 %}text-warning
{% else %}text-info{% endif %}">
{{ usage_data.interaction_percent }}%
</h2>
<div class="progress mt-2">
<div class="progress-bar
{% if usage_data.interaction_percent >= 95 %}bg-danger
{% elif usage_data.interaction_percent > 80 %}bg-warning
{% else %}bg-info{% endif %}"
role="progressbar"
style="width: {{ usage_data.interaction_percent }}%"
aria-valuenow="{{ usage_data.interaction_percent }}"
aria-valuemin="0"
aria-valuemax="100">
</div>
</div>
<p class="mt-3">{{ usage_data.interaction_used_rounded or 0.0 }} / {{ active_period.included_interaction_tokens or 0 }} M tokens</p>
</div>
</div>
</div>
</div>
{% else %}
<div class="alert alert-info">
There's no active license period for this tenant, or no usage data has been recorded yet.
</div>
{% endif %}
{% endblock %}

2
eveai_app/templates/navbar.html
View File

@@ -114,7 +114,7 @@
{'name': 'License Tiers', 'url': '/entitlements/view_license_tiers', 'roles': ['Super User', 'Partner Admin']}, {'name': 'License Tiers', 'url': '/entitlements/view_license_tiers', 'roles': ['Super User', 'Partner Admin']},
{'name': 'Trigger Actions', 'url': '/administration/trigger_actions', 'roles': ['Super User']}, {'name': 'Trigger Actions', 'url': '/administration/trigger_actions', 'roles': ['Super User']},
{'name': 'Licenses', 'url': '/entitlements/view_licenses', 'roles': ['Super User', 'Tenant Admin', 'Partner Admin']}, {'name': 'Licenses', 'url': '/entitlements/view_licenses', 'roles': ['Super User', 'Tenant Admin', 'Partner Admin']},
{'name': 'Usage', 'url': '/entitlements/view_usages', 'roles': ['Super User', 'Tenant Admin', 'Partner Admin']}, {'name': 'Active Usage', 'url': '/entitlements/active_usage', 'roles': ['Super User', 'Tenant Admin', 'Partner Admin']},
]) }} ]) }}
{% endif %} {% endif %}
{% if current_user.is_authenticated %} {% if current_user.is_authenticated %}

56
eveai_app/views/entitlements_views.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime as dt, timezone as tz, timedelta from datetime import datetime as dt, timezone as tz, timedelta
from flask import request, redirect, flash, render_template, Blueprint, session, current_app from flask import request, redirect, flash, render_template, Blueprint, session, current_app, jsonify
from flask_security import roles_accepted, current_user from flask_security import roles_accepted, current_user
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import or_, desc from sqlalchemy import or_, desc
@@ -396,3 +396,57 @@ def transition_period_status(license_id, period_id):
flash(f'Error updating status: {str(e)}', 'danger') flash(f'Error updating status: {str(e)}', 'danger')
return redirect(prefixed_url_for('entitlements_bp.view_license_periods', license_id=license_id)) return redirect(prefixed_url_for('entitlements_bp.view_license_periods', license_id=license_id))
@entitlements_bp.route('/active_usage')
@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
def active_license_usage():
# Retrieve the active license period for the current tenant
tenant_id = session.get('tenant', {}).get('id')
if not tenant_id:
flash('No active or pending license period found for this tenant', 'warning')
return redirect(prefixed_url_for('user_bp.select_tenant'))
active_period = LicensePeriod.query \
.join(License) \
.filter(
License.tenant_id == tenant_id,
LicensePeriod.status.in_([PeriodStatus.ACTIVE, PeriodStatus.PENDING])
).first()
if not active_period:
flash('Geen actieve of pending licentieperiode gevonden voor deze tenant', 'warning')
return render_template('entitlements/view_active_license_usage.html', active_period=None)
# Bereken de percentages voor gebruik
usage_data = {}
if active_period.license_usage:
# Storage percentage
if active_period.max_storage_mb > 0:
storage_used = active_period.license_usage.storage_mb_used or 0.0
usage_data['storage_used_rounded'] = round(storage_used, 2)
usage_data['storage_percent'] = round(storage_used / active_period.max_storage_mb * 100, 2)
else:
usage_data['storage_percent'] = 0.0
# Embedding percentage
if active_period.included_embedding_mb > 0:
embedding_used = active_period.license_usage.embedding_mb_used or 0.0
usage_data['embedding_used_rounded'] = round(embedding_used, 2)
usage_data['embedding_percent'] = round(embedding_used / active_period.included_embedding_mb * 100, 2)
else:
usage_data['embedding_percent'] = 0.0
# Interaction tokens percentage
if active_period.included_interaction_tokens > 0:
interaction_used = active_period.license_usage.interaction_total_tokens_used / 1_000_000 or 0.0
usage_data['interaction_used_rounded'] = round(interaction_used, 2)
usage_data['interaction_percent'] = (
round(interaction_used / active_period.included_interaction_tokens * 100, 2))
else:
usage_data['interaction_percent'] = 0
return render_template('entitlements/view_active_license_usage.html',
active_period=active_period,
usage_data=usage_data)

4
eveai_chat_workers/__init__.py
View File

@@ -3,9 +3,8 @@ import logging.config
from flask import Flask from flask import Flask
import os import os
from common.langchain.templates.template_manager import TemplateManager
from common.utils.celery_utils import make_celery, init_celery from common.utils.celery_utils import make_celery, init_celery
from common.extensions import db, template_manager, cache_manager from common.extensions import db, cache_manager
from config.logging_config import LOGGING from config.logging_config import LOGGING
from config.config import get_config from config.config import get_config
@@ -44,7 +43,6 @@ def create_app(config_file=None):
def register_extensions(app): def register_extensions(app):
db.init_app(app) db.init_app(app)
cache_manager.init_app(app) cache_manager.init_app(app)
template_manager.init_app(app)
def register_cache_handlers(app): def register_cache_handlers(app):

2
eveai_entitlements/__init__.py
View File

@@ -47,6 +47,8 @@ def register_extensions(app):
def register_cache_handlers(app): def register_cache_handlers(app):
from common.utils.cache.license_cache import register_license_cache_handlers from common.utils.cache.license_cache import register_license_cache_handlers
register_license_cache_handlers(cache_manager) register_license_cache_handlers(cache_manager)
from common.utils.cache.config_cache import register_config_cache_handlers
register_config_cache_handlers(cache_manager)
app, celery = create_app() app, celery = create_app()

25
eveai_entitlements/tasks.py
View File

@@ -28,7 +28,6 @@ def persist_business_events(log_entries):
Args: Args:
log_entries: List of log event dictionaries to persist log_entries: List of log event dictionaries to persist
""" """
try:
event_logs = [] event_logs = []
for entry in log_entries: for entry in log_entries:
event_log = BusinessEventLog( event_log = BusinessEventLog(
@@ -56,10 +55,10 @@ def persist_business_events(log_entries):
llm_interaction_type=entry.pop('llm_interaction_type', None), llm_interaction_type=entry.pop('llm_interaction_type', None),
message=entry.pop('message', None) message=entry.pop('message', None)
) )
db.session.add(event_log)
event_logs.append(event_log) event_logs.append(event_log)
# Perform a bulk insert of all entries # Perform a bulk insert of all entries
db.session.bulk_save_objects(event_logs)
db.session.commit() db.session.commit()
current_app.logger.info(f"Successfully persisted {len(event_logs)} business event logs") current_app.logger.info(f"Successfully persisted {len(event_logs)} business event logs")
@@ -68,7 +67,8 @@ def persist_business_events(log_entries):
license_period = LicensePeriodServices.find_current_license_period_for_usage(tenant_id) license_period = LicensePeriodServices.find_current_license_period_for_usage(tenant_id)
except EveAIException as e: except EveAIException as e:
current_app.logger.error(f"Failed to find license period for tenant {tenant_id}: {str(e)}") current_app.logger.error(f"Failed to find license period for tenant {tenant_id}: {str(e)}")
return raise e
lic_usage = None lic_usage = None
if not license_period.license_usage: if not license_period.license_usage:
lic_usage = LicenseUsage( lic_usage = LicenseUsage(
@@ -82,16 +82,12 @@ def persist_business_events(log_entries):
except SQLAlchemyError as e: except SQLAlchemyError as e:
db.session.rollback() db.session.rollback()
current_app.logger.error(f"Error trying to create license usage for tenant {tenant_id}: {str(e)}") current_app.logger.error(f"Error trying to create license usage for tenant {tenant_id}: {str(e)}")
return raise e
else: else:
lic_usage = license_period.license_usage lic_usage = license_period.license_usage
process_logs_for_license_usage(tenant_id, lic_usage, event_logs) process_logs_for_license_usage(tenant_id, lic_usage, event_logs)
except Exception as e:
current_app.logger.error(f"Failed to persist business event logs: {e}")
db.session.rollback()
def process_logs_for_license_usage(tenant_id, license_usage, logs): def process_logs_for_license_usage(tenant_id, license_usage, logs):
# Initialize variables to accumulate usage data # Initialize variables to accumulate usage data
@@ -103,12 +99,16 @@ def process_logs_for_license_usage(tenant_id, license_usage, logs):
interaction_completion_tokens_used = 0 interaction_completion_tokens_used = 0
interaction_total_tokens_used = 0 interaction_total_tokens_used = 0
current_app.logger.info(f"Processing {len(logs)} logs for tenant {tenant_id}")
recalculate_storage = False recalculate_storage = False
# Process each log # Process each log
for log in logs: for log in logs:
# Case for 'Create Embeddings' event # Case for 'Create Embeddings' event
current_app.logger.debug(f"Processing log for tenant {tenant_id}: {log.id} - {log.event_type} - {log.message}")
if log.event_type == 'Create Embeddings': if log.event_type == 'Create Embeddings':
current_app.logger.debug(f"In Create Embeddings")
recalculate_storage = True recalculate_storage = True
if log.message == 'Starting Trace for Create Embeddings': if log.message == 'Starting Trace for Create Embeddings':
embedding_mb_used += log.document_version_file_size embedding_mb_used += log.document_version_file_size
@@ -138,6 +138,13 @@ def process_logs_for_license_usage(tenant_id, license_usage, logs):
# Mark the log as processed by setting the license_usage_id # Mark the log as processed by setting the license_usage_id
log.license_usage_id = license_usage.id log.license_usage_id = license_usage.id
db.session.add(log)
current_app.logger.debug(f"Finished processing {len(logs)} logs for tenant {tenant_id}")
current_app.logger.debug(f"Embedding MB Used: {embedding_mb_used}")
current_app.logger.debug(f"Embedding Prompt Tokens Used: {embedding_prompt_tokens_used}")
current_app.logger.debug(f"Embedding Completion Tokens Used: {embedding_completion_tokens_used}")
current_app.logger.debug(f"Embedding Total Tokens Used: {embedding_total_tokens_used}")
# Update the LicenseUsage record with the accumulated values # Update the LicenseUsage record with the accumulated values
license_usage.embedding_mb_used += embedding_mb_used license_usage.embedding_mb_used += embedding_mb_used
@@ -154,8 +161,6 @@ def process_logs_for_license_usage(tenant_id, license_usage, logs):
# Commit the updates to the LicenseUsage and log records # Commit the updates to the LicenseUsage and log records
try: try:
db.session.add(license_usage) db.session.add(license_usage)
for log in logs:
db.session.add(log)
db.session.commit() db.session.commit()
except SQLAlchemyError as e: except SQLAlchemyError as e:
db.session.rollback() db.session.rollback()

10
eveai_workers/__init__.py
View File

@@ -4,7 +4,7 @@ from flask import Flask
import os import os
from common.utils.celery_utils import make_celery, init_celery from common.utils.celery_utils import make_celery, init_celery
from common.extensions import db, minio_client, template_manager, cache_manager from common.extensions import db, minio_client, cache_manager
import config.logging_config as logging_config import config.logging_config as logging_config
from config.config import get_config from config.config import get_config
@@ -26,6 +26,8 @@ def create_app(config_file=None):
register_extensions(app) register_extensions(app)
register_cache_handlers(app)
from . import processors from . import processors
celery = make_celery(app.name, app.config) celery = make_celery(app.name, app.config)
@@ -43,7 +45,11 @@ def register_extensions(app):
db.init_app(app) db.init_app(app)
minio_client.init_app(app) minio_client.init_app(app)
cache_manager.init_app(app) cache_manager.init_app(app)
template_manager.init_app(app)
def register_cache_handlers(app):
from common.utils.cache.config_cache import register_config_cache_handlers
register_config_cache_handlers(cache_manager)
app, celery = create_app() app, celery = create_app()

5
eveai_workers/processors/html_processor.py
View File

@@ -3,7 +3,7 @@ from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import ChatPromptTemplate from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnablePassthrough from langchain_core.runnables import RunnablePassthrough
from common.extensions import db, minio_client from common.extensions import db, minio_client
from common.utils.model_utils import create_language_template, get_embedding_llm from common.utils.model_utils import create_language_template, get_embedding_llm, get_template
from .base_processor import BaseProcessor from .base_processor import BaseProcessor
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
from .processor_registry import ProcessorRegistry from .processor_registry import ProcessorRegistry
@@ -81,8 +81,7 @@ class HTMLProcessor(BaseProcessor):
def _generate_markdown_from_html(self, html_content): def _generate_markdown_from_html(self, html_content):
self._log(f'Generating markdown from HTML for tenant {self.tenant.id}') self._log(f'Generating markdown from HTML for tenant {self.tenant.id}')
llm = get_embedding_llm() template, llm = get_template("html_parse")
template = self.model_variables.get_template("html_parse")
parse_prompt = ChatPromptTemplate.from_template(template) parse_prompt = ChatPromptTemplate.from_template(template)
setup = RunnablePassthrough() setup = RunnablePassthrough()
output_parser = StrOutputParser() output_parser = StrOutputParser()

5
eveai_workers/processors/pdf_processor.py
View File

@@ -9,7 +9,7 @@ from langchain_core.runnables import RunnablePassthrough
from common.eveai_model.tracked_mistral_ocr_client import TrackedMistralOcrClient from common.eveai_model.tracked_mistral_ocr_client import TrackedMistralOcrClient
from common.extensions import minio_client from common.extensions import minio_client
from common.utils.model_utils import create_language_template, get_embedding_llm from common.utils.model_utils import create_language_template, get_embedding_llm, get_template
from .base_processor import BaseProcessor from .base_processor import BaseProcessor
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
from .processor_registry import ProcessorRegistry from .processor_registry import ProcessorRegistry
@@ -208,8 +208,7 @@ class PDFProcessor(BaseProcessor):
return text_splitter.split_text(content) return text_splitter.split_text(content)
def _process_chunks_with_llm(self, chunks): def _process_chunks_with_llm(self, chunks):
llm = get_embedding_llm() template, llm = get_template('pdf_parse')
template = self.model_variables.get_template('pdf_parse')
pdf_prompt = ChatPromptTemplate.from_template(template) pdf_prompt = ChatPromptTemplate.from_template(template)
setup = RunnablePassthrough() setup = RunnablePassthrough()
output_parser = StrOutputParser() output_parser = StrOutputParser()

5
eveai_workers/processors/transcription_processor.py
View File

@@ -4,7 +4,7 @@ from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import ChatPromptTemplate from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnablePassthrough from langchain_core.runnables import RunnablePassthrough
from common.utils.model_utils import create_language_template, get_embedding_llm from common.utils.model_utils import create_language_template, get_embedding_llm, get_template
from .base_processor import BaseProcessor from .base_processor import BaseProcessor
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
@@ -46,8 +46,7 @@ class TranscriptionBaseProcessor(BaseProcessor):
def _process_chunks(self, chunks): def _process_chunks(self, chunks):
self.log_tuning("_process_chunks", {"Nr of Chunks": len(chunks)}) self.log_tuning("_process_chunks", {"Nr of Chunks": len(chunks)})
llm = get_embedding_llm() template, llm = get_template('transcript')
template = self.model_variables.get_template('transcript')
language_template = create_language_template(template, self.document_version.language) language_template = create_language_template(template, self.document_version.language)
transcript_prompt = ChatPromptTemplate.from_template(language_template) transcript_prompt = ChatPromptTemplate.from_template(language_template)
setup = RunnablePassthrough() setup = RunnablePassthrough()

8
eveai_workers/tasks.py
View File

@@ -12,13 +12,13 @@ from langchain_core.runnables import RunnablePassthrough
from sqlalchemy import or_ from sqlalchemy import or_
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from common.extensions import db from common.extensions import db, cache_manager
from common.models.document import DocumentVersion, Embedding, Document, Processor, Catalog from common.models.document import DocumentVersion, Embedding, Document, Processor, Catalog
from common.models.user import Tenant from common.models.user import Tenant
from common.utils.celery_utils import current_celery from common.utils.celery_utils import current_celery
from common.utils.database import Database from common.utils.database import Database
from common.utils.model_utils import create_language_template, get_model_variables, get_embedding_model_and_class, \ from common.utils.model_utils import create_language_template, get_model_variables, get_embedding_model_and_class, \
get_embedding_llm get_embedding_llm, get_template
from common.utils.business_event import BusinessEvent from common.utils.business_event import BusinessEvent
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
@@ -211,8 +211,8 @@ def enrich_chunks(tenant, model_variables, document_version, title, chunks):
def summarize_chunk(tenant, model_variables, document_version, chunk): def summarize_chunk(tenant, model_variables, document_version, chunk):
current_event.log("Starting Summarizing Chunk") current_event.log("Starting Summarizing Chunk")
llm = get_embedding_llm()
template = model_variables.get_template("summary") template, llm = get_template("summary")
language_template = create_language_template(template, document_version.language) language_template = create_language_template(template, document_version.language)
summary_prompt = ChatPromptTemplate.from_template(language_template) summary_prompt = ChatPromptTemplate.from_template(language_template)
setup = RunnablePassthrough() setup = RunnablePassthrough()

71
migrations/public/versions/4eae969dcac2_license_copied_fields_in_licenseperiod_.py Normal file
View File

@@ -0,0 +1,71 @@
"""license-copied fields in LicensePeriod should be Nullable
Revision ID: 4eae969dcac2
Revises: c08c3e7c3b1a
Create Date: 2025-05-18 20:13:11.555330
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '4eae969dcac2'
down_revision = 'c08c3e7c3b1a'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license_period', schema=None) as batch_op:
batch_op.alter_column('currency',
existing_type=sa.VARCHAR(length=20),
nullable=True)
batch_op.alter_column('basic_fee',
existing_type=sa.DOUBLE_PRECISION(precision=53),
nullable=True)
batch_op.alter_column('max_storage_mb',
existing_type=sa.INTEGER(),
nullable=True)
batch_op.alter_column('additional_storage_price',
existing_type=sa.DOUBLE_PRECISION(precision=53),
nullable=True)
batch_op.alter_column('additional_storage_bucket',
existing_type=sa.INTEGER(),
nullable=True)
batch_op.alter_column('included_embedding_mb',
existing_type=sa.INTEGER(),
nullable=True)
batch_op.alter_column('additional_embedding_price',
existing_type=sa.NUMERIC(precision=10, scale=4),
nullable=True)
batch_op.alter_column('additional_embedding_bucket',
existing_type=sa.INTEGER(),
nullable=True)
batch_op.alter_column('included_interaction_tokens',
existing_type=sa.INTEGER(),
nullable=True)
batch_op.alter_column('additional_interaction_token_price',
existing_type=sa.NUMERIC(precision=10, scale=4),
nullable=True)
batch_op.alter_column('additional_interaction_bucket',
existing_type=sa.INTEGER(),
nullable=True)
batch_op.alter_column('additional_storage_allowed',
existing_type=sa.BOOLEAN(),
nullable=True)
batch_op.alter_column('additional_embedding_allowed',
existing_type=sa.BOOLEAN(),
nullable=True)
batch_op.alter_column('additional_interaction_allowed',
existing_type=sa.BOOLEAN(),
nullable=True)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###

30
migrations/public/versions/c08c3e7c3b1a_add_end_date_calculated_field_to_license.py Normal file
View File

@@ -0,0 +1,30 @@
"""Add End Date (calculated field) to License
Revision ID: c08c3e7c3b1a
Revises: 845d0428c5fe
Create Date: 2025-05-18 19:55:32.702250
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'c08c3e7c3b1a'
down_revision = '845d0428c5fe'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('license', schema=None) as batch_op:
batch_op.add_column(sa.Column('end_date', sa.Date(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###