realise document upload - Part 1

config.py

@@ -28,6 +28,10 @@ class Config(object):
     MAIL_USE_SSL = False
     MAIL_DEFAULT_SENDER = ('eveAI Admin', 'eveai_admin@flow-it.net')
 
+    # file upload settings
+    MAX_CONTENT_LENGTH = 16 * 1024 * 1024
+    UPLOAD_EXTENSIONS = ['.txt', '.pdf', '.png', '.jpg', '.jpeg', '.gif']
+
 
 class DevConfig(Config):
     DEVELOPMENT = True
@@ -41,6 +45,9 @@ class DevConfig(Config):
     MAIL_USERNAME = 'eveai_super@flow-it.net'
     MAIL_PASSWORD = '$6xsWGbNtx$CFMQZqc*'
 
+    # file upload settings
+    UPLOAD_FOLDER = '/Volumes/OWC4M2_1/Development/eveAI/file_store'
+
 
 class ProdConfig(Config):
     DEVELOPMENT = False

eveai_app/controllers/document_controller.py

@@ -0,0 +1,3 @@
+from ..models.document import Document, DocumentLanguage, DocumentVersion
+
+def add_document(file, name, language, valid_from):

eveai_app/models/document.py

@@ -76,3 +76,13 @@ class EmbeddingMistral(db.Model):
     # 1024 is the MISTRAL Embedding dimension.
     # If another embedding model is chosen, this dimension may need to be changed.
     embedding = db.Column(Vector(1024), nullable=False)
+
+
+class EmbeddingSmallOpenAI(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    doc_vers_id = db.Column(db.Integer, db.ForeignKey(DocumentVersion.id), nullable=False)
+    active = db.Column(db.Boolean, nullable=False, default=True)
+
+    # 1536 is the OpenAI Small Embedding dimension.
+    # If another embedding model is chosen, this dimension may need to be changed.
+    embedding = db.Column(Vector(1536), nullable=False)

eveai_app/models/user.py

@@ -17,6 +17,13 @@ class Tenant(db.Model):
     name = db.Column(db.String(80), unique=True, nullable=False)
     website = db.Column(db.String(255), nullable=True)
 
+    # language information
+    default_language = db.Column(db.String(2), nullable=True)
+
+    # currency information
+    default_currency = db.Column(db.String(80), nullable=True)
+
+
     # Licensing Information
     license_start_date = db.Column(db.Date, nullable=True)
     license_end_date = db.Column(db.Date, nullable=True)

eveai_app/utils/database.py

@@ -35,6 +35,7 @@ class Database:
         """create new database schema, mostly used on tenant creation"""
         try:
             db.session.execute(CreateSchema(self.schema))
+            db.session.execute(text(f"CREATE EXTENSION IF NOT EXISTS pgvector SCHEMA {self.schema}"))
             db.session.commit()
         except InternalError:
             db.session.rollback()

eveai_app/utils/middleware.py

@@ -0,0 +1,30 @@
+"""Middleware for the API
+
+for handling tenant requests
+"""
+
+from flask_security import current_user
+from flask import session
+
+from ..models.user import User, Tenant
+from .database import Database
+from ..views.document_views import document_bp
+
+
+def mw_before_request():
+    """Before request
+
+    switch tenant schema
+    """
+
+    tenant_id = session['tenant']['id']
+    if not tenant_id:
+        return {"message": "You are not logged into any tenant"}, 403
+
+    user = User.get_by_id(current_user.id)
+    if user.has_roles(['Super User']) or user.tenant_id == tenant_id:
+        Database(tenant_id).switch_schema()
+    else:
+        return {"message": "You are not a member of this tenant"}, 403
+
+

eveai_app/views/document_forms.py

@@ -0,0 +1,15 @@
+from flask_wtf import FlaskForm
+from wtforms import (StringField, BooleanField, SubmitField, DateField,
+                     SelectMultipleField, FieldList, FormField)
+from wtforms.validators import DataRequired, Length
+from flask_wtf.file import FileField, FileAllowed, FileRequired
+
+
+class AddDocumentForm(FlaskForm):
+    file = FileField('File', validators=[FileAllowed(['pdf', 'txt']),
+                                         FileRequired()])
+    name = StringField('Name', validators=[Length(max=100)])
+    language = StringField('Language', validators=[Length(max=2)])
+    valid_from = DateField('Valid from', id='form-control datepicker')
+
+    submit = SubmitField('Submit')

eveai_app/views/document_views.py

@@ -0,0 +1,55 @@
+import os
+from datetime import datetime as dt, timezone as tz
+from flask import request, redirect, url_for, flash, render_template, Blueprint, session, current_app
+from flask_security import hash_password, roles_required, roles_accepted, current_user
+from werkzeug.utils import secure_filename
+
+from ..models.document import Document, DocumentLanguage, DocumentVersion
+from ..extensions import db
+from .document_forms import AddDocumentForm
+from ..utils.database import Database
+from ..utils.middleware import mw_before_request
+
+document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
+
+
+@document_bp.before_request
+def before_request():
+    mw_before_request()
+
+
+@document_bp.route('/add_document', methods=['GET', 'POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def add_document():
+    form = AddDocumentForm()
+    if request.method == 'POST' and form.validate_on_submit():
+        file = form.file.data
+        filename = secure_filename(file.filename)
+        extension = filename.rsplit('.', 1)[1].lower()
+
+        # Create the Document
+        new_doc = Document()
+        if form.name.data == '':
+            new_doc.name = filename.rsplit('.', 1)[0]
+        else:
+            new_doc.name = form.name.data
+
+        timestamp = dt.now(tz.utc)
+        if form.valid_from.data or form.valid_from.data != '':
+            new_doc.valid_from = form.valid_from.data
+        else:
+            new_doc.valid_from = timestamp
+
+        new_doc.created_at = timestamp
+        new_doc.updated_at = timestamp
+        new_doc.created_by = current_user.id
+        new_doc.updated_by = current_user.id
+        db.session.add(new_doc)
+
+# TODO: Continue from here on and complete add_document
+        # Create the DocumentLanguage
+        new_doc_lang = DocumentLanguage()
+        language = form.language.data
+
+        file.save(os.path.join(current_app.config['UPLOAD_FOLDER'], filename))
+        return redirect(url_for('document_bp.add_document'))

eveai_app/views/user_views.py

@@ -15,8 +15,6 @@ user_bp = Blueprint('user_bp', __name__, url_prefix='/user')
 @user_bp.route('/tenant', methods=['GET', 'POST'])
 @roles_required('Super User')
 def tenant():
-    print("SESSION:")
-    print(session)
     if request.method == 'POST':
         # Handle the required attributes
         name = request.form.get('name')
@@ -58,7 +56,6 @@ def tenant():
 
             # Create schema for new tenant
             if error is None:
-                print(new_tenant.id)
                 Database(new_tenant.id).create_tenant_schema()
 
         flash(error) if error else flash('Tenant added successfully.')