- Correcting mistakes in tenant schema migrations

CHANGELOG.md

@@ -0,0 +1,80 @@
+# Changelog
+
+All notable changes to EveAI will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- For new features.
+
+### Changed
+- For changes in existing functionality.
+
+### Deprecated
+- For soon-to-be removed features.
+
+### Removed
+- For now removed features.
+
+### Fixed
+- For any bug fixes.
+
+### Security
+- In case of vulnerabilities.
+
+## [1.0.5-alfa] - 2024-09-02
+
+### Added
+- Allow chatwidget to connect to multiple servers (e.g. development and production)
+- Start implementation of API
+- Add API-key functionality to tenants
+- Deduplication of API and Document view code
+- Allow URL addition to accept all types of files, not just HTML
+- Allow new file types upload: srt, mp3, ogg, mp4
+- Improve processing of different file types using Processor classes
+
+### Removed
+- Removed direct upload of Youtube URLs, due to continuous changes in Youtube website
+
+## [1.0.4-alfa] - 2024-08-27
+Skipped
+
+## [1.0.3-alfa] - 2024-08-27
+
+### Added
+- Refinement of HTML processing - allow for excluded classes and elements.
+- Allow for multiple instances of Evie on 1 website (pure + Wordpress plugin)
+
+### Changed
+- PDF Processing extracted in new PDF Processor class.
+- Allow for longer and more complex PDFs to be uploaded.
+
+## [1.0.2-alfa] - 2024-08-22
+
+### Fixed
+- Bugfix for ResetPasswordForm in config.py
+
+## [1.0.1-alfa] - 2024-08-21
+
+### Added
+- Full Document Version Overview
+
+### Changed
+- Improvements to user creation and registration, renewal of passwords, ...
+
+## [1.0.0-alfa] - 2024-08-16
+
+### Added
+- Initial release of the project.
+
+### Changed
+- None
+
+### Fixed
+- None
+
+[Unreleased]: https://github.com/username/repo/compare/v1.0.0...HEAD
+[1.0.0]: https://github.com/username/repo/releases/tag/v1.0.0

eveai_app/views/document_forms.py

@@ -27,6 +27,7 @@ class AddDocumentForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
+        self.language.data = session.get('tenant').get('default_language')
 
 
 class AddURLForm(FlaskForm):
@@ -42,6 +43,7 @@ class AddURLForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
+        self.language.data = session.get('tenant').get('default_language')
 
 
 class AddURLsForm(FlaskForm):
@@ -57,6 +59,7 @@ class AddURLsForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
+        self.language.data = session.get('tenant').get('default_language')
 
 
 class AddYoutubeForm(FlaskForm):

eveai_app/views/document_views.py

@@ -22,8 +22,7 @@ from common.utils.document_utils import validate_file_type, create_document_stac
     process_multiple_urls, prepare_youtube_document, create_version_for_document, upload_file_for_version
 from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
     EveAIDoubleURLException, EveAIYoutubeError
-from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddYoutubeForm, \
+from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm
-    AddURLsForm
 from common.utils.middleware import mw_before_request
 from common.utils.celery_utils import current_celery
 from common.utils.nginx_utils import prefixed_url_for
@@ -170,42 +169,6 @@ def add_urls():
     return render_template('document/add_urls.html', form=form)
 
 
-@document_bp.route('/add_youtube', methods=['GET', 'POST'])
-@roles_accepted('Super User', 'Tenant Admin')
-def add_youtube():
-    form = AddYoutubeForm()
-
-    if form.validate_on_submit():
-        try:
-            tenant_id = session['tenant']['id']
-            url = form.url.data
-
-            api_input = {
-                'name': form.name.data,
-                'language': form.language.data,
-                'user_context': form.user_context.data,
-                'valid_from': form.valid_from.data
-            }
-
-            new_doc, new_doc_vers = prepare_youtube_document(url, tenant_id, api_input)
-            task_id = start_embedding_task(tenant_id, new_doc_vers.id)
-
-            flash(
-                f'Processing on YouTube document {new_doc.name}, version {new_doc_vers.id} started. Task ID: {task_id}.',
-                'success')
-            return redirect(prefixed_url_for('document_bp.documents'))
-
-        except EveAIYoutubeError as e:
-            flash(str(e), 'error')
-        except (EveAIInvalidLanguageException, EveAIUnsupportedFileType) as e:
-            flash(str(e), 'error')
-        except Exception as e:
-            current_app.logger.error(f'Error adding YouTube document: {str(e)}')
-            flash('An error occurred while adding the YouTube document.', 'error')
-
-    return render_template('document/add_youtube.html', form=form)
-
-
 @document_bp.route('/documents', methods=['GET', 'POST'])
 @roles_accepted('Super User', 'Tenant Admin')
 def documents():

migrations/public/versions/110be45f6e44_include_excluded_classes_for_tenant.py

@@ -0,0 +1,68 @@
+"""Include excluded_classes for Tenant
+
+Revision ID: 110be45f6e44
+Revises: 229774547fed
+Create Date: 2024-08-22 07:37:40.591220
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '110be45f6e44'
+down_revision = '229774547fed'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('roles_users', schema=None) as batch_op:
+        batch_op.drop_constraint('roles_users_user_id_fkey', type_='foreignkey')
+        batch_op.drop_constraint('roles_users_role_id_fkey', type_='foreignkey')
+        batch_op.create_foreign_key(None, 'role', ['role_id'], ['id'], referent_schema='public', ondelete='CASCADE')
+        batch_op.create_foreign_key(None, 'user', ['user_id'], ['id'], referent_schema='public', ondelete='CASCADE')
+
+    with op.batch_alter_table('tenant', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('html_excluded_classes', postgresql.ARRAY(sa.String(length=200)), nullable=True))
+
+    with op.batch_alter_table('tenant_domain', schema=None) as batch_op:
+        batch_op.drop_constraint('tenant_domain_updated_by_fkey', type_='foreignkey')
+        batch_op.drop_constraint('tenant_domain_tenant_id_fkey', type_='foreignkey')
+        batch_op.drop_constraint('tenant_domain_created_by_fkey', type_='foreignkey')
+        batch_op.create_foreign_key(None, 'user', ['updated_by'], ['id'], referent_schema='public')
+        batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
+        batch_op.create_foreign_key(None, 'user', ['created_by'], ['id'], referent_schema='public')
+
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.drop_constraint('user_tenant_id_fkey', type_='foreignkey')
+        batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.create_foreign_key('user_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
+
+    with op.batch_alter_table('tenant_domain', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.create_foreign_key('tenant_domain_created_by_fkey', 'user', ['created_by'], ['id'])
+        batch_op.create_foreign_key('tenant_domain_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
+        batch_op.create_foreign_key('tenant_domain_updated_by_fkey', 'user', ['updated_by'], ['id'])
+
+    with op.batch_alter_table('tenant', schema=None) as batch_op:
+        batch_op.drop_column('html_excluded_classes')
+
+    with op.batch_alter_table('roles_users', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.create_foreign_key('roles_users_role_id_fkey', 'role', ['role_id'], ['id'], ondelete='CASCADE')
+        batch_op.create_foreign_key('roles_users_user_id_fkey', 'user', ['user_id'], ['id'], ondelete='CASCADE')
+
+    # ### end Alembic commands ###

migrations/public/versions/ce6f5b62bbfb_add_api_key_to_tenant_next_to_chat_api_.py

@@ -0,0 +1,68 @@
+"""Add API Key to tenant - next to Chat API key
+
+Revision ID: ce6f5b62bbfb
+Revises: 110be45f6e44
+Create Date: 2024-08-29 07:43:20.662983
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ce6f5b62bbfb'
+down_revision = '110be45f6e44'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('roles_users', schema=None) as batch_op:
+        batch_op.drop_constraint('roles_users_role_id_fkey', type_='foreignkey')
+        batch_op.drop_constraint('roles_users_user_id_fkey', type_='foreignkey')
+        batch_op.create_foreign_key(None, 'user', ['user_id'], ['id'], referent_schema='public', ondelete='CASCADE')
+        batch_op.create_foreign_key(None, 'role', ['role_id'], ['id'], referent_schema='public', ondelete='CASCADE')
+
+    with op.batch_alter_table('tenant', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('encrypted_api_key', sa.String(length=500), nullable=True))
+
+    with op.batch_alter_table('tenant_domain', schema=None) as batch_op:
+        batch_op.drop_constraint('tenant_domain_tenant_id_fkey', type_='foreignkey')
+        batch_op.drop_constraint('tenant_domain_updated_by_fkey', type_='foreignkey')
+        batch_op.drop_constraint('tenant_domain_created_by_fkey', type_='foreignkey')
+        batch_op.create_foreign_key(None, 'user', ['updated_by'], ['id'], referent_schema='public')
+        batch_op.create_foreign_key(None, 'user', ['created_by'], ['id'], referent_schema='public')
+        batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
+
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.drop_constraint('user_tenant_id_fkey', type_='foreignkey')
+        batch_op.create_foreign_key(None, 'tenant', ['tenant_id'], ['id'], referent_schema='public')
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.create_foreign_key('user_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
+
+    with op.batch_alter_table('tenant_domain', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.create_foreign_key('tenant_domain_created_by_fkey', 'user', ['created_by'], ['id'])
+        batch_op.create_foreign_key('tenant_domain_updated_by_fkey', 'user', ['updated_by'], ['id'])
+        batch_op.create_foreign_key('tenant_domain_tenant_id_fkey', 'tenant', ['tenant_id'], ['id'])
+
+    with op.batch_alter_table('tenant', schema=None) as batch_op:
+        batch_op.drop_column('encrypted_api_key')
+
+    with op.batch_alter_table('roles_users', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.create_foreign_key('roles_users_user_id_fkey', 'user', ['user_id'], ['id'], ondelete='CASCADE')
+        batch_op.create_foreign_key('roles_users_role_id_fkey', 'role', ['role_id'], ['id'], ondelete='CASCADE')
+
+    # ### end Alembic commands ###

migrations/tenant/env.py

@@ -1,3 +1,4 @@
+import inspect
 import logging
 from logging.config import fileConfig
 
@@ -5,12 +6,12 @@ from flask import current_app
 
 from alembic import context
 from sqlalchemy import NullPool, engine_from_config, text
-
+import sqlalchemy as sa
-from common.models.user import Tenant
+from sqlalchemy.sql import schema
-
 import pgvector
 from pgvector.sqlalchemy import Vector
 
+from common.models import document, interaction, user
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -46,17 +47,34 @@ def get_engine_url():
 config.set_main_option('sqlalchemy.url', get_engine_url())
 target_db = current_app.extensions['migrate'].db
 
-# other values from the config, defined by the needs of env.py,
+
-# can be acquired:
+def get_public_table_names():
-# my_important_option = config.get_main_option("my_important_option")
+    # TODO: This function should include the necessary functionality to automatically retrieve table names
-# ... etc.
+    return ['role', 'roles_users', 'tenant', 'user', 'tenant_domain']
+
+
+PUBLIC_TABLES = get_public_table_names()
+logger.info(f"Public tables: {PUBLIC_TABLES}")
+
+
+def include_object(object, name, type_, reflected, compare_to):
+    if type_ == "table" and name in PUBLIC_TABLES:
+        logger.info(f"Excluding public table: {name}")
+        return False
+    logger.info(f"Including object: {type_} {name}")
+    return True
+
 
 # List of Tenants
-tenants_ = Tenant.query.all()
+def get_tenant_ids():
-if tenants_:
+    from common.models.user import Tenant
-    tenants = [tenant.id for tenant in tenants_]
+
-else:
+    tenants_ = Tenant.query.all()
-    tenants = []
+    if tenants_:
+        tenants = [tenant.id for tenant in tenants_]
+    else:
+        tenants = []
+    return tenants
 
 
 def get_metadata():
@@ -79,7 +97,10 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=get_metadata(), literal_binds=True
+        url=url,
+        target_metadata=get_metadata(),
+        # literal_binds=True,
+        include_object=include_object,
     )
 
     with context.begin_transaction():
@@ -99,18 +120,8 @@ def run_migrations_online():
         poolclass=NullPool,
     )
 
-    def process_revision_directives(context, revision, directives):
-        if config.cmd_opts.autogenerate:
-            script = directives[0]
-            if script.upgrade_ops is not None:
-                # Add import for pgvector and set search path
-                script.upgrade_ops.ops.insert(0, sa.schema.ExecuteSQLOp(
-                    "SET search_path TO CURRENT_SCHEMA(), public; IMPORT pgvector",
-                    execution_options=None
-                ))
-
     with connectable.connect() as connection:
-        print(tenants)
+        tenants = get_tenant_ids()
         for tenant in tenants:
             logger.info(f"Migrating tenant: {tenant}")
             # set search path on the connection, which ensures that
@@ -127,7 +138,8 @@ def run_migrations_online():
             context.configure(
                 connection=connection,
                 target_metadata=get_metadata(),
-                process_revision_directives=process_revision_directives,
+                # literal_binds=True,
+                include_object=include_object,
             )
 
             with context.begin_transaction():

migrations/tenant/versions/43eac8a7a00b_ensure_logging_information_in_document_.py

@@ -0,0 +1,27 @@
+"""Ensure logging information in document domain does not require user
+
+Revision ID: 43eac8a7a00b
+Revises: 5d5437d81041
+Create Date: 2024-09-03 09:36:06.541938
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+
+
+# revision identifiers, used by Alembic.
+revision = '43eac8a7a00b'
+down_revision = '5d5437d81041'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Manual upgrade commands
+    op.execute('ALTER TABLE document ALTER COLUMN created_by DROP NOT NULL')
+
+
+def downgrade():
+    # Manual downgrade commands
+    op.execute('ALTER TABLE document ALTER COLUMN created_by SET NOT NULL')