- Allowing for multiple types of Catalogs

- Introduction of retrievers
- Ensuring processing information is collected from Catalog iso Tenant
- Introduction of a generic Form class to enable dynamic fields based on a configuration
- Realisation of Retriever functionality to support dynamic fields

migrations/tenant/versions/3717364e6429_add_retriever_model.py

@@ -0,0 +1,56 @@
+"""Add Retriever Model
+
+Revision ID: 3717364e6429
+Revises: 7b7b566e667f
+Create Date: 2024-10-21 14:22:30.258679
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '3717364e6429'
+down_revision = '7b7b566e667f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('retriever',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=50), nullable=False),
+    sa.Column('description', sa.Text(), nullable=True),
+    sa.Column('catalog_id', sa.Integer(), nullable=True),
+    sa.Column('type', sa.String(length=50), nullable=False),
+    sa.Column('user_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('system_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('configuration', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
+    sa.Column('created_by', sa.Integer(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_by', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['catalog_id'], ['catalog.id'], ),
+    sa.ForeignKeyConstraint(['created_by'], ['public.user.id'], ),
+    sa.ForeignKeyConstraint(['updated_by'], ['public.user.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.drop_column('catalog', 'es_similarity_threshold')
+    op.drop_column('catalog', 'es_k')
+    op.drop_column('catalog', 'rag_tuning')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('catalog', sa.Column('rag_tuning', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('es_k', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.add_column('catalog', sa.Column('es_similarity_threshold', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True))
+    op.drop_constraint(None, 'catalog', type_='foreignkey')
+    op.drop_constraint(None, 'catalog', type_='foreignkey')
+    op.create_foreign_key('catalog_updated_by_fkey', 'catalog', 'user', ['updated_by'], ['id'])
+    op.create_foreign_key('catalog_created_by_fkey', 'catalog', 'user', ['created_by'], ['id'])
+    op.drop_table('retriever')
+    # ### end Alembic commands ###

migrations/tenant/versions/7b7b566e667f_extensions_for_more_catalog_types_in_.py

@@ -0,0 +1,46 @@
+"""Extensions for more Catalog Types in Catalog Model
+
+Revision ID: 7b7b566e667f
+Revises: 28984b05d396
+Create Date: 2024-10-21 07:39:52.260054
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '7b7b566e667f'
+down_revision = '28984b05d396'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('catalog', sa.Column('parent_id', sa.Integer(), nullable=True))
+
+    op.add_column('catalog', sa.Column('type', sa.String(length=50), nullable=False, server_default='DEFAULT'))
+    # Update all existing rows to have the 'type' set to 'DEFAULT'
+    op.execute("UPDATE catalog SET type='DEFAULT' WHERE type IS NULL")
+    # Remove the server default (optional but recommended)
+    op.alter_column('catalog', 'type', server_default=None)
+
+    op.add_column('catalog', sa.Column('user_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    op.add_column('catalog', sa.Column('system_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    op.add_column('catalog', sa.Column('configuration', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    op.create_foreign_key(None, 'catalog', 'catalog', ['parent_id'], ['id'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'catalog', type_='foreignkey')
+    op.drop_column('catalog', 'configuration')
+    op.drop_column('catalog', 'system_metadata')
+    op.drop_column('catalog', 'user_metadata')
+    op.drop_column('catalog', 'type')
+    op.drop_column('catalog', 'parent_id')
+    # ### end Alembic commands ###