From 9f350b5ea09c83bc3289ecc70f0f9ee407e4dd4f Mon Sep 17 00:00:00 2001 From: Josako Date: Tue, 30 Apr 2024 22:47:44 +0200 Subject: [PATCH] realise document upload - Part 1 --- config.py | 7 +++ eveai_app/controllers/document_controller.py | 3 ++ eveai_app/models/document.py | 10 ++++ eveai_app/models/user.py | 7 +++ eveai_app/utils/database.py | 1 + eveai_app/utils/middleware.py | 30 +++++++++++ eveai_app/views/document_forms.py | 15 ++++++ eveai_app/views/document_views.py | 55 ++++++++++++++++++++ eveai_app/views/user_views.py | 3 -- 9 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 eveai_app/controllers/document_controller.py create mode 100644 eveai_app/utils/middleware.py diff --git a/config.py b/config.py index b2b4fae..94e4b69 100644 --- a/config.py +++ b/config.py @@ -28,6 +28,10 @@ class Config(object): MAIL_USE_SSL = False MAIL_DEFAULT_SENDER = ('eveAI Admin', 'eveai_admin@flow-it.net') + # file upload settings + MAX_CONTENT_LENGTH = 16 * 1024 * 1024 + UPLOAD_EXTENSIONS = ['.txt', '.pdf', '.png', '.jpg', '.jpeg', '.gif'] + class DevConfig(Config): DEVELOPMENT = True @@ -41,6 +45,9 @@ class DevConfig(Config): MAIL_USERNAME = 'eveai_super@flow-it.net' MAIL_PASSWORD = '$6xsWGbNtx$CFMQZqc*' + # file upload settings + UPLOAD_FOLDER = '/Volumes/OWC4M2_1/Development/eveAI/file_store' + class ProdConfig(Config): DEVELOPMENT = False diff --git a/eveai_app/controllers/document_controller.py b/eveai_app/controllers/document_controller.py new file mode 100644 index 0000000..e6bdd73 --- /dev/null +++ b/eveai_app/controllers/document_controller.py @@ -0,0 +1,3 @@ +from ..models.document import Document, DocumentLanguage, DocumentVersion + +def add_document(file, name, language, valid_from): diff --git a/eveai_app/models/document.py b/eveai_app/models/document.py index 69356f6..557a5d6 100644 --- a/eveai_app/models/document.py +++ b/eveai_app/models/document.py @@ -76,3 +76,13 @@ class EmbeddingMistral(db.Model): # 1024 is the MISTRAL Embedding dimension. # If another embedding model is chosen, this dimension may need to be changed. embedding = db.Column(Vector(1024), nullable=False) + + +class EmbeddingSmallOpenAI(db.Model): + id = db.Column(db.Integer, primary_key=True) + doc_vers_id = db.Column(db.Integer, db.ForeignKey(DocumentVersion.id), nullable=False) + active = db.Column(db.Boolean, nullable=False, default=True) + + # 1536 is the OpenAI Small Embedding dimension. + # If another embedding model is chosen, this dimension may need to be changed. + embedding = db.Column(Vector(1536), nullable=False) diff --git a/eveai_app/models/user.py b/eveai_app/models/user.py index 573ad31..7aa3c78 100644 --- a/eveai_app/models/user.py +++ b/eveai_app/models/user.py @@ -17,6 +17,13 @@ class Tenant(db.Model): name = db.Column(db.String(80), unique=True, nullable=False) website = db.Column(db.String(255), nullable=True) + # language information + default_language = db.Column(db.String(2), nullable=True) + + # currency information + default_currency = db.Column(db.String(80), nullable=True) + + # Licensing Information license_start_date = db.Column(db.Date, nullable=True) license_end_date = db.Column(db.Date, nullable=True) diff --git a/eveai_app/utils/database.py b/eveai_app/utils/database.py index fd2b22f..081c09b 100644 --- a/eveai_app/utils/database.py +++ b/eveai_app/utils/database.py @@ -35,6 +35,7 @@ class Database: """create new database schema, mostly used on tenant creation""" try: db.session.execute(CreateSchema(self.schema)) + db.session.execute(text(f"CREATE EXTENSION IF NOT EXISTS pgvector SCHEMA {self.schema}")) db.session.commit() except InternalError: db.session.rollback() diff --git a/eveai_app/utils/middleware.py b/eveai_app/utils/middleware.py new file mode 100644 index 0000000..c1d6c63 --- /dev/null +++ b/eveai_app/utils/middleware.py @@ -0,0 +1,30 @@ +"""Middleware for the API + +for handling tenant requests +""" + +from flask_security import current_user +from flask import session + +from ..models.user import User, Tenant +from .database import Database +from ..views.document_views import document_bp + + +def mw_before_request(): + """Before request + + switch tenant schema + """ + + tenant_id = session['tenant']['id'] + if not tenant_id: + return {"message": "You are not logged into any tenant"}, 403 + + user = User.get_by_id(current_user.id) + if user.has_roles(['Super User']) or user.tenant_id == tenant_id: + Database(tenant_id).switch_schema() + else: + return {"message": "You are not a member of this tenant"}, 403 + + diff --git a/eveai_app/views/document_forms.py b/eveai_app/views/document_forms.py index e69de29..4d967fd 100644 --- a/eveai_app/views/document_forms.py +++ b/eveai_app/views/document_forms.py @@ -0,0 +1,15 @@ +from flask_wtf import FlaskForm +from wtforms import (StringField, BooleanField, SubmitField, DateField, + SelectMultipleField, FieldList, FormField) +from wtforms.validators import DataRequired, Length +from flask_wtf.file import FileField, FileAllowed, FileRequired + + +class AddDocumentForm(FlaskForm): + file = FileField('File', validators=[FileAllowed(['pdf', 'txt']), + FileRequired()]) + name = StringField('Name', validators=[Length(max=100)]) + language = StringField('Language', validators=[Length(max=2)]) + valid_from = DateField('Valid from', id='form-control datepicker') + + submit = SubmitField('Submit') diff --git a/eveai_app/views/document_views.py b/eveai_app/views/document_views.py index e69de29..4746564 100644 --- a/eveai_app/views/document_views.py +++ b/eveai_app/views/document_views.py @@ -0,0 +1,55 @@ +import os +from datetime import datetime as dt, timezone as tz +from flask import request, redirect, url_for, flash, render_template, Blueprint, session, current_app +from flask_security import hash_password, roles_required, roles_accepted, current_user +from werkzeug.utils import secure_filename + +from ..models.document import Document, DocumentLanguage, DocumentVersion +from ..extensions import db +from .document_forms import AddDocumentForm +from ..utils.database import Database +from ..utils.middleware import mw_before_request + +document_bp = Blueprint('document_bp', __name__, url_prefix='/document') + + +@document_bp.before_request +def before_request(): + mw_before_request() + + +@document_bp.route('/add_document', methods=['GET', 'POST']) +@roles_accepted('Super User', 'Tenant Admin') +def add_document(): + form = AddDocumentForm() + if request.method == 'POST' and form.validate_on_submit(): + file = form.file.data + filename = secure_filename(file.filename) + extension = filename.rsplit('.', 1)[1].lower() + + # Create the Document + new_doc = Document() + if form.name.data == '': + new_doc.name = filename.rsplit('.', 1)[0] + else: + new_doc.name = form.name.data + + timestamp = dt.now(tz.utc) + if form.valid_from.data or form.valid_from.data != '': + new_doc.valid_from = form.valid_from.data + else: + new_doc.valid_from = timestamp + + new_doc.created_at = timestamp + new_doc.updated_at = timestamp + new_doc.created_by = current_user.id + new_doc.updated_by = current_user.id + db.session.add(new_doc) + +# TODO: Continue from here on and complete add_document + # Create the DocumentLanguage + new_doc_lang = DocumentLanguage() + language = form.language.data + + file.save(os.path.join(current_app.config['UPLOAD_FOLDER'], filename)) + return redirect(url_for('document_bp.add_document')) diff --git a/eveai_app/views/user_views.py b/eveai_app/views/user_views.py index 207213e..ba99938 100644 --- a/eveai_app/views/user_views.py +++ b/eveai_app/views/user_views.py @@ -15,8 +15,6 @@ user_bp = Blueprint('user_bp', __name__, url_prefix='/user') @user_bp.route('/tenant', methods=['GET', 'POST']) @roles_required('Super User') def tenant(): - print("SESSION:") - print(session) if request.method == 'POST': # Handle the required attributes name = request.form.get('name') @@ -58,7 +56,6 @@ def tenant(): # Create schema for new tenant if error is None: - print(new_tenant.id) Database(new_tenant.id).create_tenant_schema() flash(error) if error else flash('Tenant added successfully.')