diff --git a/common/models/document.py b/common/models/document.py
index 990e8d2..fc10127 100644
--- a/common/models/document.py
+++ b/common/models/document.py
@@ -94,7 +94,7 @@ class DocumentVersion(db.Model):
system_context = db.Column(db.Text, nullable=True)
user_metadata = db.Column(JSONB, nullable=True)
system_metadata = db.Column(JSONB, nullable=True)
- catalog_configuration = db.Column(JSONB, nullable=True)
+ catalog_properties = db.Column(JSONB, nullable=True)
# Versioning Information
created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
diff --git a/common/utils/document_utils.py b/common/utils/document_utils.py
index 36c96b7..f2797db 100644
--- a/common/utils/document_utils.py
+++ b/common/utils/document_utils.py
@@ -27,6 +27,7 @@ def create_document_stack(api_input, file, filename, extension, tenant_id):
api_input.get('language', 'en'),
api_input.get('user_context', ''),
api_input.get('user_metadata'),
+ api_input.get('catalog_properties')
)
db.session.add(new_doc_vers)
@@ -63,7 +64,7 @@ def create_document(form, filename, catalog_id):
return new_doc
-def create_version_for_document(document, tenant_id, url, language, user_context, user_metadata):
+def create_version_for_document(document, tenant_id, url, language, user_context, user_metadata, catalog_properties):
new_doc_vers = DocumentVersion()
if url != '':
new_doc_vers.url = url
@@ -79,6 +80,9 @@ def create_version_for_document(document, tenant_id, url, language, user_context
if user_metadata != '' and user_metadata is not None:
new_doc_vers.user_metadata = user_metadata
+ if catalog_properties != '' and catalog_properties is not None:
+ new_doc_vers.catalog_properties = catalog_properties
+
new_doc_vers.document = document
set_logging_information(new_doc_vers, dt.now(tz.utc))
@@ -273,9 +277,10 @@ def edit_document(document_id, name, valid_from, valid_to):
return None, str(e)
-def edit_document_version(version_id, user_context):
+def edit_document_version(version_id, user_context, catalog_properties):
doc_vers = DocumentVersion.query.get_or_404(version_id)
doc_vers.user_context = user_context
+ doc_vers.catalog_properties = catalog_properties
update_logging_information(doc_vers, dt.now(tz.utc))
try:
@@ -300,6 +305,7 @@ def refresh_document_with_info(doc_id, tenant_id, api_input):
api_input.get('language', old_doc_vers.language),
api_input.get('user_context', old_doc_vers.user_context),
api_input.get('user_metadata', old_doc_vers.user_metadata),
+ api_input.get('catalog_properties', old_doc_vers.catalog_properties),
)
set_logging_information(new_doc_vers, dt.now(tz.utc))
@@ -337,7 +343,8 @@ def refresh_document(doc_id, tenant_id):
api_input = {
'language': old_doc_vers.language,
'user_context': old_doc_vers.user_context,
- 'user_metadata': old_doc_vers.user_metadata
+ 'user_metadata': old_doc_vers.user_metadata,
+ 'catalog_properties': old_doc_vers.catalog_properties,
}
return refresh_document_with_info(doc_id, tenant_id, api_input)
@@ -348,3 +355,5 @@ def mark_tenant_storage_dirty(tenant_id):
tenant = db.session.query(Tenant).filter_by(id=int(tenant_id)).first()
tenant.storage_dirty = True
db.session.commit()
+
+
diff --git a/config/catalog_types.py b/config/catalog_types.py
index 8e67fcc..756cd9a 100644
--- a/config/catalog_types.py
+++ b/config/catalog_types.py
@@ -48,6 +48,6 @@ CATALOG_TYPES = {
}
}
},
- "document_version_user_metadata": ["tagging_fields"]
+ "document_version_configurations": ["tagging_fields"]
},
}
diff --git a/docker/build_and_push_eveai.sh b/docker/build_and_push_eveai.sh
index 4913172..708a52c 100755
--- a/docker/build_and_push_eveai.sh
+++ b/docker/build_and_push_eveai.sh
@@ -169,5 +169,5 @@ for SERVICE in "${SERVICES[@]}"; do
fi
done
-echo "All specified services processed."
-echo "Finished at $(date +"%d/%m/%Y %H:%M:%S")"
\ No newline at end of file
+echo -e "\033[35mAll specified services processed.\033[0m"
+echo -e "\033[35mFinished at $(date +"%d/%m/%Y %H:%M:%S")\033[0m"
diff --git a/docker/release_and_tag_eveai.sh b/docker/release_and_tag_eveai.sh
index dc83bab..c5bcbdc 100755
--- a/docker/release_and_tag_eveai.sh
+++ b/docker/release_and_tag_eveai.sh
@@ -58,5 +58,5 @@ echo "Tagging Git repository with version: $RELEASE_VERSION"
git tag -a v$RELEASE_VERSION -m "Release $RELEASE_VERSION: $RELEASE_MESSAGE"
git push origin v$RELEASE_VERSION
-echo "Release process completed for version: $RELEASE_VERSION"
-echo "Finished at $(date +"%d/%m/%Y %H:%M:%S")"
\ No newline at end of file
+echo -e "\033[35mRelease process completed for version: $RELEASE_VERSION \033[0m"
+echo -e "\033[35mFinished at $(date +"%d/%m/%Y %H:%M:%S")\033[0m"
\ No newline at end of file
diff --git a/eveai_api/api/document_api.py b/eveai_api/api/document_api.py
index 90b6ba5..01e7aab 100644
--- a/eveai_api/api/document_api.py
+++ b/eveai_api/api/document_api.py
@@ -43,6 +43,9 @@ upload_parser.add_argument('valid_from', location='form', type=validate_date, re
help='Valid from date for the document (ISO format)')
upload_parser.add_argument('user_metadata', location='form', type=validate_json, required=False,
help='User metadata for the document (JSON format)')
+upload_parser.add_argument('catalog_properties', location='form', type=validate_json, required=False,
+ help='The catalog configuration to be passed along (JSON format). Validity is against catalog requirements '
+ 'is not checked, and is the responsibility of the calling client.')
add_document_response = document_ns.model('AddDocumentResponse', {
'message': fields.String(description='Status message'),
@@ -82,6 +85,7 @@ class AddDocument(Resource):
'user_context': args.get('user_context'),
'valid_from': args.get('valid_from'),
'user_metadata': args.get('user_metadata'),
+ 'catalog_properties': args.get('catalog_properties'),
}
new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id)
@@ -111,7 +115,11 @@ add_url_model = document_ns.model('AddURL', {
'user_context': fields.String(required=False, description='User context for the document'),
'valid_from': fields.String(required=False, description='Valid from date for the document'),
'user_metadata': fields.String(required=False, description='User metadata for the document'),
- 'system_metadata': fields.String(required=False, description='System metadata for the document')
+ 'system_metadata': fields.String(required=False, description='System metadata for the document'),
+ 'catalog_properties': fields.String(required=False, description='The catalog configuration to be passed along (JSON '
+ 'format). Validity is against catalog requirements '
+ 'is not checked, and is the responsibility of the '
+ 'calling client.'),
})
add_url_response = document_ns.model('AddURLResponse', {
@@ -148,6 +156,7 @@ class AddURL(Resource):
'user_context': args.get('user_context'),
'valid_from': args.get('valid_from'),
'user_metadata': args.get('user_metadata'),
+ 'catalog_properties': args.get('catalog_properties'),
}
new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id)
@@ -227,6 +236,7 @@ class DocumentResource(Resource):
edit_document_version_model = document_ns.model('EditDocumentVersion', {
'user_context': fields.String(required=True, description='New user context for the document version'),
+ 'catalog_properties': fields.String(required=True, description='New catalog properties for the document version'),
})
@@ -239,7 +249,7 @@ class DocumentVersionResource(Resource):
def put(self, version_id):
"""Edit a document version"""
data = request.json
- updated_version, error = edit_document_version(version_id, data['user_context'])
+ updated_version, error = edit_document_version(version_id, data['user_context'], data.get('catalog_properties'))
if updated_version:
return {'message': f'Document Version {updated_version.id} updated successfully'}, 200
else:
@@ -251,7 +261,8 @@ refresh_document_model = document_ns.model('RefreshDocument', {
'name': fields.String(required=False, description='New name for the document'),
'language': fields.String(required=False, description='Language of the document'),
'user_context': fields.String(required=False, description='User context for the document'),
- 'user_metadata': fields.Raw(required=False, description='User metadata for the document')
+ 'user_metadata': fields.Raw(required=False, description='User metadata for the document'),
+ 'catalog_properties': fields.Raw(required=False, description='Catalog properties for the document'),
})
@@ -268,7 +279,7 @@ class RefreshDocument(Resource):
current_app.logger.info(f'Refreshing document {document_id} for tenant {tenant_id}')
try:
- new_version, result = refresh_document(document_id)
+ new_version, result = refresh_document(document_id, tenant_id)
if new_version:
return {
@@ -301,7 +312,7 @@ class RefreshDocumentWithInfo(Resource):
try:
api_input = request.json
- new_version, result = refresh_document_with_info(document_id, api_input)
+ new_version, result = refresh_document_with_info(document_id, tenant_id, api_input)
if new_version:
return {
diff --git a/eveai_app/templates/document/add_document.html b/eveai_app/templates/document/add_document.html
index dcaeb1c..ef997a0 100644
--- a/eveai_app/templates/document/add_document.html
+++ b/eveai_app/templates/document/add_document.html
@@ -11,9 +11,17 @@
{{ form.hidden_tag() }}
{% set disabled_fields = [] %}
{% set exclude_fields = [] %}
- {% for field in form %}
+ {% for field in form.get_static_fields() %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
+ {% for collection_name, fields in form.get_dynamic_fields().items() %}
+ {% if fields|length > 0 %}
+ {{ collection_name }}
+ {% endif %}
+ {% for field in fields %}
+ {{ render_field(field, disabled_fields, exclude_fields) }}
+ {% endfor %}
+ {% endfor %}
{% endblock %}
diff --git a/eveai_app/views/document_forms.py b/eveai_app/views/document_forms.py
index 8ad387a..8412888 100644
--- a/eveai_app/views/document_forms.py
+++ b/eveai_app/views/document_forms.py
@@ -149,7 +149,7 @@ class EditRetrieverForm(DynamicFormBase):
self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()]
-class AddDocumentForm(FlaskForm):
+class AddDocumentForm(DynamicFormBase):
file = FileField('File', validators=[FileRequired(), allowed_file])
name = StringField('Name', validators=[Length(max=100)])
language = SelectField('Language', choices=[], validators=[Optional()])
@@ -157,17 +157,15 @@ class AddDocumentForm(FlaskForm):
valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
- submit = SubmitField('Submit')
-
- def __init__(self):
- super().__init__()
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
self.language.choices = [(language, language) for language in
session.get('tenant').get('allowed_languages')]
if not self.language.data:
self.language.data = session.get('tenant').get('default_language')
-class AddURLForm(FlaskForm):
+class AddURLForm(DynamicFormBase):
url = URLField('URL', validators=[DataRequired(), URL()])
name = StringField('Name', validators=[Length(max=100)])
language = SelectField('Language', choices=[], validators=[Optional()])
@@ -175,10 +173,8 @@ class AddURLForm(FlaskForm):
valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
- submit = SubmitField('Submit')
-
- def __init__(self):
- super().__init__()
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
self.language.choices = [(language, language) for language in
session.get('tenant').get('allowed_languages')]
if not self.language.data:
@@ -210,7 +206,7 @@ class EditDocumentForm(FlaskForm):
submit = SubmitField('Submit')
-class EditDocumentVersionForm(FlaskForm):
+class EditDocumentVersionForm(DynamicFormBase):
language = StringField('Language')
user_context = TextAreaField('User Context', validators=[Optional()])
system_context = TextAreaField('System Context', validators=[Optional()])
diff --git a/eveai_app/views/document_views.py b/eveai_app/views/document_views.py
index 2996002..06de1ad 100644
--- a/eveai_app/views/document_views.py
+++ b/eveai_app/views/document_views.py
@@ -228,6 +228,8 @@ def edit_retriever(retriever_id):
configuration_config = RETRIEVER_TYPES[retriever.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, retriever.configuration)
+ if request.method == 'POST':
+ current_app.logger.debug(f'Received POST request with {request.form}')
if form.validate_on_submit():
# Update basic fields
@@ -295,19 +297,33 @@ def handle_retriever_selection():
@document_bp.route('/add_document', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def add_document():
- form = AddDocumentForm()
+ form = AddDocumentForm(request.form)
+ catalog_id = session.get('catalog_id', None)
+ if catalog_id is None:
+ flash('You need to set a Session Catalog before adding Documents or URLs')
+ return redirect(prefixed_url_for('document_bp.catalogs'))
+
+ catalog = Catalog.query.get_or_404(catalog_id)
+ if catalog.configuration and len(catalog.configuration) > 0:
+ document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+ for config in document_version_configurations:
+ form.add_dynamic_fields(config, catalog.configuration[config])
if form.validate_on_submit():
try:
+ current_app.logger.info(f'Adding Document for {catalog_id}')
tenant_id = session['tenant']['id']
- catalog_id = session['catalog_id']
file = form.file.data
filename = secure_filename(file.filename)
extension = filename.rsplit('.', 1)[1].lower()
validate_file_type(extension)
- current_app.logger.debug(f'Language on form: {form.language.data}')
+ catalog_properties = {}
+ document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+ for config in document_version_configurations:
+ catalog_properties[config] = form.get_dynamic_data(config)
+
api_input = {
'catalog_id': catalog_id,
'name': form.name.data,
@@ -315,6 +331,7 @@ def add_document():
'user_context': form.user_context.data,
'valid_from': form.valid_from.data,
'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
+ 'catalog_properties': catalog_properties,
}
current_app.logger.debug(f'Creating document stack with input {api_input}')
@@ -337,16 +354,30 @@ def add_document():
@document_bp.route('/add_url', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def add_url():
- form = AddURLForm()
+ form = AddURLForm(request.form)
+ catalog_id = session.get('catalog_id', None)
+ if catalog_id is None:
+ flash('You need to set a Session Catalog before adding Documents or URLs')
+ return redirect(prefixed_url_for('document_bp.catalogs'))
+
+ catalog = Catalog.query.get_or_404(catalog_id)
+ if catalog.configuration and len(catalog.configuration) > 0:
+ document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+ for config in document_version_configurations:
+ form.add_dynamic_fields(config, catalog.configuration[config])
if form.validate_on_submit():
try:
tenant_id = session['tenant']['id']
- catalog_id = session['catalog_id']
url = form.url.data
file_content, filename, extension = process_url(url, tenant_id)
+ catalog_properties = {}
+ document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+ for config in document_version_configurations:
+ catalog_properties[config] = form.get_dynamic_data(config)
+
api_input = {
'catalog_id': catalog_id,
'name': form.name.data or filename,
@@ -355,6 +386,7 @@ def add_url():
'user_context': form.user_context.data,
'valid_from': form.valid_from.data,
'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
+ 'catalog_properties': catalog_properties,
}
new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id)
@@ -375,43 +407,6 @@ def add_url():
return render_template('document/add_url.html', form=form)
-@document_bp.route('/add_urls', methods=['GET', 'POST'])
-@roles_accepted('Super User', 'Tenant Admin')
-def add_urls():
- form = AddURLsForm()
-
- if form.validate_on_submit():
- try:
- tenant_id = session['tenant']['id']
- urls = form.urls.data.split('\n')
- urls = [url.strip() for url in urls if url.strip()]
-
- api_input = {
- 'name': form.name.data,
- 'language': form.language.data,
- 'user_context': form.user_context.data,
- 'valid_from': form.valid_from.data
- }
-
- results = process_multiple_urls(urls, tenant_id, api_input)
-
- for result in results:
- if result['status'] == 'success':
- flash(
- f"Processed URL: {result['url']} - Document ID: {result['document_id']}, Version ID: {result['document_version_id']}",
- 'success')
- else:
- flash(f"Error processing URL: {result['url']} - {result['message']}", 'error')
-
- return redirect(prefixed_url_for('document_bp.documents'))
-
- except Exception as e:
- current_app.logger.error(f'Error adding multiple URLs: {str(e)}')
- flash('An error occurred while adding the URLs.', 'error')
-
- return render_template('document/add_urls.html', form=form)
-
-
@document_bp.route('/documents', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def documents():
@@ -494,12 +489,29 @@ def edit_document_view(document_id):
@roles_accepted('Super User', 'Tenant Admin')
def edit_document_version_view(document_version_id):
doc_vers = DocumentVersion.query.get_or_404(document_version_id)
- form = EditDocumentVersionForm(obj=doc_vers)
+ form = EditDocumentVersionForm(request.form, obj=doc_vers)
+
+ catalog_id = session.get('catalog_id', None)
+ if catalog_id is None:
+ flash('You need to set a Session Catalog before adding Documents or URLs')
+ return redirect(prefixed_url_for('document_bp.catalogs'))
+
+ catalog = Catalog.query.get_or_404(catalog_id)
+ if catalog.configuration and len(catalog.configuration) > 0:
+ document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+ for config in document_version_configurations:
+ form.add_dynamic_fields(config, catalog.configuration[config], doc_vers.catalog_properties[config])
if form.validate_on_submit():
+ catalog_properties = {}
+ document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+ for config in document_version_configurations:
+ catalog_properties[config] = form.get_dynamic_data(config)
+
updated_version, error = edit_document_version(
document_version_id,
- form.user_context.data
+ form.user_context.data,
+ catalog_properties,
)
if updated_version:
flash(f'Document Version {updated_version.id} updated successfully', 'success')
diff --git a/eveai_app/views/dynamic_form_base.py b/eveai_app/views/dynamic_form_base.py
index 28094a1..0abe4e9 100644
--- a/eveai_app/views/dynamic_form_base.py
+++ b/eveai_app/views/dynamic_form_base.py
@@ -48,10 +48,9 @@ class DynamicFormBase(FlaskForm):
current_app.logger.debug(f"{field_name}: {field_def}")
# Prefix the field name with the collection name
full_field_name = f"{collection_name}_{field_name}"
- label = field_def.get('name')
+ label = field_def.get('name', field_name)
field_type = field_def.get('type')
description = field_def.get('description', '')
- required = field_def.get('required', False)
default = field_def.get('default')
# Determine standard validators
@@ -90,8 +89,8 @@ class DynamicFormBase(FlaskForm):
except (TypeError, ValueError) as e:
current_app.logger.error(f"Error converting initial data to JSON: {e}")
field_data = "{}"
- elif field_def.get('default') is not None:
- field_data = field_def.get('default')
+ elif default is not None:
+ field_data = default
# Create render_kw with classes and any other HTML attributes
render_kw = {'class': extra_classes} if extra_classes else {}
@@ -145,12 +144,16 @@ class DynamicFormBase(FlaskForm):
def get_dynamic_data(self, collection_name):
"""Retrieve the data from dynamic fields of a specific collection."""
data = {}
+ current_app.logger.debug(f"{collection_name} in {self.dynamic_fields}?")
if collection_name not in self.dynamic_fields:
return data
prefix_length = len(collection_name) + 1 # +1 for the underscore
for full_field_name in self.dynamic_fields[collection_name]:
+ current_app.logger.debug(f"{full_field_name}: {full_field_name}")
original_field_name = full_field_name[prefix_length:]
+ current_app.logger.debug(f"{original_field_name}: {original_field_name}")
field = getattr(self, full_field_name)
+ current_app.logger.debug(f"{field}: {field}")
# Parse JSON for tagging_fields type
if isinstance(field, TextAreaField) and field.data:
try:
@@ -207,4 +210,7 @@ def validate_tagging_fields(form, field):
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except (TypeError, ValueError) as e:
- raise ValidationError(f"Invalid field definition: {str(e)}")
\ No newline at end of file
+ raise ValidationError(f"Invalid field definition: {str(e)}")
+
+
+
diff --git a/migrations/tenant/versions/6c5ca750e60c_add_catalog_properties_to_.py b/migrations/tenant/versions/6c5ca750e60c_add_catalog_properties_to_.py
new file mode 100644
index 0000000..9e0a153
--- /dev/null
+++ b/migrations/tenant/versions/6c5ca750e60c_add_catalog_properties_to_.py
@@ -0,0 +1,29 @@
+"""Add catalog_properties to DocumentVersion
+
+Revision ID: 6c5ca750e60c
+Revises: b64d5cf32c7a
+Create Date: 2024-10-29 08:33:54.663211
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '6c5ca750e60c'
+down_revision = 'b64d5cf32c7a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('document_version', sa.Column('catalog_properties', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column('document_version', 'catalog_properties')
+ # ### end Alembic commands ###