Prepare for html document validation (added wanted tags to tenant)

common/models/user.py

@@ -29,6 +29,9 @@ class Tenant(db.Model):
     default_llm_model = db.Column(db.String(50), nullable=True)
     allowed_llm_models = db.Column(ARRAY(sa.String(50)), nullable=True)
 
+    # Embedding variables
+    html_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
+
     # Licensing Information
     license_start_date = db.Column(db.Date, nullable=True)
     license_end_date = db.Column(db.Date, nullable=True)

common/utils/database.py

@@ -4,6 +4,7 @@ from sqlalchemy import text
 from sqlalchemy.schema import CreateSchema
 from sqlalchemy.exc import InternalError
 from sqlalchemy.orm import sessionmaker, scoped_session
+from sqlalchemy.exc import SQLAlchemyError
 from flask import current_app
 
 from common.extensions import db, migrate
@@ -35,9 +36,10 @@ class Database:
         """create new database schema, mostly used on tenant creation"""
         try:
             db.session.execute(CreateSchema(self.schema))
-            db.session.execute(text(f"CREATE EXTENSION IF NOT EXISTS pgvector SCHEMA {self.schema}"))
+            # db.session.commit()
+            db.session.execute(text(f"SET search_path TO {self.schema}, public"))
             db.session.commit()
-        except InternalError as e:
+        except SQLAlchemyError as e:
             db.session.rollback()
             db.session.close()
             current_app.logger.error(f"Error creating schema {self.schema}: {e.args}")
@@ -48,7 +50,7 @@ class Database:
 
     def switch_schema(self):
         """switch between tenant/public database schema"""
-        db.session.execute(text(f'set search_path to "{self.schema}"'))
+        db.session.execute(text(f'set search_path to "{self.schema}", public'))
         db.session.commit()
 
     def migrate_tenant_schema(self):