- Introduction of dynamic Retrievers & Specialists

- Introduction of dynamic Processors - Introduction of caching system - Introduction of a better template manager - Adaptation of ModelVariables to support dynamic Processors / Retrievers / Specialists - Start adaptation of chat client
2024-11-15 10:00:53 +01:00
parent 55a8a95f79
commit 1807435339
101 changed files with 4181 additions and 1764 deletions
--- a/migrations/tenant/versions/2370a17da7cb_allow_json_specialist_input_and_results_.py
+++ b/migrations/tenant/versions/2370a17da7cb_allow_json_specialist_input_and_results_.py
@@ -0,0 +1,31 @@
+"""Allow JSON specialist input and results for interactions
+
+Revision ID: 2370a17da7cb
+Revises: 0c347651837c
+Create Date: 2024-11-07 14:33:57.678861
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '2370a17da7cb'
+down_revision = '0c347651837c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('interaction', sa.Column('specialist_arguments', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    op.add_column('interaction', sa.Column('specialist_results', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('interaction', 'specialist_results')
+    op.drop_column('interaction', 'specialist_arguments')
+    # ### end Alembic commands ###
--- a/migrations/tenant/versions/4bf121f340e5_add_sub_file_type_to_processor_and_.py
+++ b/migrations/tenant/versions/4bf121f340e5_add_sub_file_type_to_processor_and_.py
@@ -0,0 +1,31 @@
+"""Add sub_file_type to processor and document_version
+
+Revision ID: 4bf121f340e5
+Revises: e54c830a88c9
+Create Date: 2024-11-11 13:03:23.265230
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+
+
+# revision identifiers, used by Alembic.
+revision = '4bf121f340e5'
+down_revision = 'e54c830a88c9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('document_version', sa.Column('sub_file_type', sa.String(length=50), nullable=True))
+    op.add_column('processor', sa.Column('sub_file_type', sa.String(length=50), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('processor', 'sub_file_type')
+    op.drop_column('document_version', 'sub_file_type')
+    # ### end Alembic commands ###
--- a/migrations/tenant/versions/7872ffdbac6c_removed_obsolete_interaction_fields_.py
+++ b/migrations/tenant/versions/7872ffdbac6c_removed_obsolete_interaction_fields_.py
@@ -0,0 +1,41 @@
+"""Removed obsolete interaction fields, added specialist relation to interaction
+
+Revision ID: 7872ffdbac6c
+Revises: 2370a17da7cb
+Create Date: 2024-11-07 15:14:12.263011
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+
+
+# revision identifiers, used by Alembic.
+revision = '7872ffdbac6c'
+down_revision = '2370a17da7cb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('interaction', sa.Column('specialist_id', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'interaction', 'specialist', ['specialist_id'], ['id'])
+    op.drop_column('interaction', 'question')
+    op.drop_column('interaction', 'answer')
+    op.drop_column('interaction', 'algorithm_used')
+    op.drop_column('interaction', 'language')
+    op.drop_column('interaction', 'detailed_question')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('interaction', sa.Column('detailed_question', sa.TEXT(), autoincrement=False, nullable=True))
+    op.add_column('interaction', sa.Column('language', sa.VARCHAR(length=2), autoincrement=False, nullable=False))
+    op.add_column('interaction', sa.Column('algorithm_used', sa.VARCHAR(length=20), autoincrement=False, nullable=True))
+    op.add_column('interaction', sa.Column('answer', sa.TEXT(), autoincrement=False, nullable=True))
+    op.add_column('interaction', sa.Column('question', sa.TEXT(), autoincrement=False, nullable=False))
+    op.drop_constraint(None, 'interaction', type_='foreignkey')
+    op.drop_column('interaction', 'specialist_id')
+    # ### end Alembic commands ###
--- a/migrations/tenant/versions/95f82a301dff_adding_processor_model.py
+++ b/migrations/tenant/versions/95f82a301dff_adding_processor_model.py
@@ -0,0 +1,49 @@
+"""Adding Processor Model
+
+Revision ID: 95f82a301dff
+Revises: 7872ffdbac6c
+Create Date: 2024-11-11 11:26:23.103689
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '95f82a301dff'
+down_revision = '7872ffdbac6c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('processor',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('name', sa.String(length=50), nullable=False),
+                    sa.Column('description', sa.Text(), nullable=True),
+                    sa.Column('catalog_id', sa.Integer(), nullable=True),
+                    sa.Column('type', sa.String(length=50), nullable=False),
+                    sa.Column('min_chunk_size', sa.Integer(), nullable=True),
+                    sa.Column('max_chunk_size', sa.Integer(), nullable=True),
+                    sa.Column('tuning', sa.Boolean(), nullable=True),
+                    sa.Column('user_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+                    sa.Column('system_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+                    sa.Column('configuration', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+                    sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
+                    sa.Column('created_by', sa.Integer(), nullable=True),
+                    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
+                    sa.Column('updated_by', sa.Integer(), nullable=True),
+                    sa.ForeignKeyConstraint(['catalog_id'], ['catalog.id'], ),
+                    sa.ForeignKeyConstraint(['created_by'], ['public.user.id'], ),
+                    sa.ForeignKeyConstraint(['updated_by'], ['public.user.id'], ),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('processor')
+    # ### end Alembic commands ###
--- a/migrations/tenant/versions/e54c830a88c9_remove_obsolete_columns_due_to_.py
+++ b/migrations/tenant/versions/e54c830a88c9_remove_obsolete_columns_due_to_.py
@@ -0,0 +1,47 @@
+"""Remove obsolete columns due to Processor model introduction
+
+Revision ID: e54c830a88c9
+Revises: 95f82a301dff
+Create Date: 2024-11-11 11:31:33.478714
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'e54c830a88c9'
+down_revision = '95f82a301dff'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('catalog', 'chat_RAG_temperature')
+    op.drop_column('catalog', 'html_included_elements')
+    op.drop_column('catalog', 'html_tags')
+    op.drop_column('catalog', 'embed_tuning')
+    op.drop_column('catalog', 'html_end_tags')
+    op.drop_column('catalog', 'html_excluded_classes')
+    op.drop_column('catalog', 'chat_no_RAG_temperature')
+    op.drop_column('catalog', 'html_excluded_elements')
+    op.drop_column('processor', 'min_chunk_size')
+    op.drop_column('processor', 'max_chunk_size')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('processor', sa.Column('max_chunk_size', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.add_column('processor', sa.Column('min_chunk_size', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('html_excluded_elements', postgresql.ARRAY(sa.VARCHAR(length=50)), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('chat_no_RAG_temperature', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('html_excluded_classes', postgresql.ARRAY(sa.VARCHAR(length=200)), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('html_end_tags', postgresql.ARRAY(sa.VARCHAR(length=10)), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('embed_tuning', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('html_tags', postgresql.ARRAY(sa.VARCHAR(length=10)), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('html_included_elements', postgresql.ARRAY(sa.VARCHAR(length=50)), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('chat_RAG_temperature', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True))
+    # ### end Alembic commands ###