From 532073d38e12f57134e26bb1fbe0ea5d5c1d9887 Mon Sep 17 00:00:00 2001 From: Josako Date: Wed, 30 Oct 2024 13:52:18 +0100 Subject: [PATCH] - Add dynamic fields to DocumentVersion in case the Catalog requires it. --- common/models/document.py | 2 +- common/utils/document_utils.py | 15 ++- config/catalog_types.py | 2 +- docker/build_and_push_eveai.sh | 4 +- docker/release_and_tag_eveai.sh | 4 +- eveai_api/api/document_api.py | 21 +++- .../templates/document/add_document.html | 10 +- eveai_app/views/document_forms.py | 18 ++-- eveai_app/views/document_views.py | 100 ++++++++++-------- eveai_app/views/dynamic_form_base.py | 16 ++- ...6c5ca750e60c_add_catalog_properties_to_.py | 29 +++++ 11 files changed, 146 insertions(+), 75 deletions(-) create mode 100644 migrations/tenant/versions/6c5ca750e60c_add_catalog_properties_to_.py diff --git a/common/models/document.py b/common/models/document.py index 990e8d2..fc10127 100644 --- a/common/models/document.py +++ b/common/models/document.py @@ -94,7 +94,7 @@ class DocumentVersion(db.Model): system_context = db.Column(db.Text, nullable=True) user_metadata = db.Column(JSONB, nullable=True) system_metadata = db.Column(JSONB, nullable=True) - catalog_configuration = db.Column(JSONB, nullable=True) + catalog_properties = db.Column(JSONB, nullable=True) # Versioning Information created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now()) diff --git a/common/utils/document_utils.py b/common/utils/document_utils.py index 36c96b7..f2797db 100644 --- a/common/utils/document_utils.py +++ b/common/utils/document_utils.py @@ -27,6 +27,7 @@ def create_document_stack(api_input, file, filename, extension, tenant_id): api_input.get('language', 'en'), api_input.get('user_context', ''), api_input.get('user_metadata'), + api_input.get('catalog_properties') ) db.session.add(new_doc_vers) @@ -63,7 +64,7 @@ def create_document(form, filename, catalog_id): return new_doc -def create_version_for_document(document, tenant_id, url, language, user_context, user_metadata): +def create_version_for_document(document, tenant_id, url, language, user_context, user_metadata, catalog_properties): new_doc_vers = DocumentVersion() if url != '': new_doc_vers.url = url @@ -79,6 +80,9 @@ def create_version_for_document(document, tenant_id, url, language, user_context if user_metadata != '' and user_metadata is not None: new_doc_vers.user_metadata = user_metadata + if catalog_properties != '' and catalog_properties is not None: + new_doc_vers.catalog_properties = catalog_properties + new_doc_vers.document = document set_logging_information(new_doc_vers, dt.now(tz.utc)) @@ -273,9 +277,10 @@ def edit_document(document_id, name, valid_from, valid_to): return None, str(e) -def edit_document_version(version_id, user_context): +def edit_document_version(version_id, user_context, catalog_properties): doc_vers = DocumentVersion.query.get_or_404(version_id) doc_vers.user_context = user_context + doc_vers.catalog_properties = catalog_properties update_logging_information(doc_vers, dt.now(tz.utc)) try: @@ -300,6 +305,7 @@ def refresh_document_with_info(doc_id, tenant_id, api_input): api_input.get('language', old_doc_vers.language), api_input.get('user_context', old_doc_vers.user_context), api_input.get('user_metadata', old_doc_vers.user_metadata), + api_input.get('catalog_properties', old_doc_vers.catalog_properties), ) set_logging_information(new_doc_vers, dt.now(tz.utc)) @@ -337,7 +343,8 @@ def refresh_document(doc_id, tenant_id): api_input = { 'language': old_doc_vers.language, 'user_context': old_doc_vers.user_context, - 'user_metadata': old_doc_vers.user_metadata + 'user_metadata': old_doc_vers.user_metadata, + 'catalog_properties': old_doc_vers.catalog_properties, } return refresh_document_with_info(doc_id, tenant_id, api_input) @@ -348,3 +355,5 @@ def mark_tenant_storage_dirty(tenant_id): tenant = db.session.query(Tenant).filter_by(id=int(tenant_id)).first() tenant.storage_dirty = True db.session.commit() + + diff --git a/config/catalog_types.py b/config/catalog_types.py index 8e67fcc..756cd9a 100644 --- a/config/catalog_types.py +++ b/config/catalog_types.py @@ -48,6 +48,6 @@ CATALOG_TYPES = { } } }, - "document_version_user_metadata": ["tagging_fields"] + "document_version_configurations": ["tagging_fields"] }, } diff --git a/docker/build_and_push_eveai.sh b/docker/build_and_push_eveai.sh index 4913172..708a52c 100755 --- a/docker/build_and_push_eveai.sh +++ b/docker/build_and_push_eveai.sh @@ -169,5 +169,5 @@ for SERVICE in "${SERVICES[@]}"; do fi done -echo "All specified services processed." -echo "Finished at $(date +"%d/%m/%Y %H:%M:%S")" \ No newline at end of file +echo -e "\033[35mAll specified services processed.\033[0m" +echo -e "\033[35mFinished at $(date +"%d/%m/%Y %H:%M:%S")\033[0m" diff --git a/docker/release_and_tag_eveai.sh b/docker/release_and_tag_eveai.sh index dc83bab..c5bcbdc 100755 --- a/docker/release_and_tag_eveai.sh +++ b/docker/release_and_tag_eveai.sh @@ -58,5 +58,5 @@ echo "Tagging Git repository with version: $RELEASE_VERSION" git tag -a v$RELEASE_VERSION -m "Release $RELEASE_VERSION: $RELEASE_MESSAGE" git push origin v$RELEASE_VERSION -echo "Release process completed for version: $RELEASE_VERSION" -echo "Finished at $(date +"%d/%m/%Y %H:%M:%S")" \ No newline at end of file +echo -e "\033[35mRelease process completed for version: $RELEASE_VERSION \033[0m" +echo -e "\033[35mFinished at $(date +"%d/%m/%Y %H:%M:%S")\033[0m" \ No newline at end of file diff --git a/eveai_api/api/document_api.py b/eveai_api/api/document_api.py index 90b6ba5..01e7aab 100644 --- a/eveai_api/api/document_api.py +++ b/eveai_api/api/document_api.py @@ -43,6 +43,9 @@ upload_parser.add_argument('valid_from', location='form', type=validate_date, re help='Valid from date for the document (ISO format)') upload_parser.add_argument('user_metadata', location='form', type=validate_json, required=False, help='User metadata for the document (JSON format)') +upload_parser.add_argument('catalog_properties', location='form', type=validate_json, required=False, + help='The catalog configuration to be passed along (JSON format). Validity is against catalog requirements ' + 'is not checked, and is the responsibility of the calling client.') add_document_response = document_ns.model('AddDocumentResponse', { 'message': fields.String(description='Status message'), @@ -82,6 +85,7 @@ class AddDocument(Resource): 'user_context': args.get('user_context'), 'valid_from': args.get('valid_from'), 'user_metadata': args.get('user_metadata'), + 'catalog_properties': args.get('catalog_properties'), } new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id) @@ -111,7 +115,11 @@ add_url_model = document_ns.model('AddURL', { 'user_context': fields.String(required=False, description='User context for the document'), 'valid_from': fields.String(required=False, description='Valid from date for the document'), 'user_metadata': fields.String(required=False, description='User metadata for the document'), - 'system_metadata': fields.String(required=False, description='System metadata for the document') + 'system_metadata': fields.String(required=False, description='System metadata for the document'), + 'catalog_properties': fields.String(required=False, description='The catalog configuration to be passed along (JSON ' + 'format). Validity is against catalog requirements ' + 'is not checked, and is the responsibility of the ' + 'calling client.'), }) add_url_response = document_ns.model('AddURLResponse', { @@ -148,6 +156,7 @@ class AddURL(Resource): 'user_context': args.get('user_context'), 'valid_from': args.get('valid_from'), 'user_metadata': args.get('user_metadata'), + 'catalog_properties': args.get('catalog_properties'), } new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id) @@ -227,6 +236,7 @@ class DocumentResource(Resource): edit_document_version_model = document_ns.model('EditDocumentVersion', { 'user_context': fields.String(required=True, description='New user context for the document version'), + 'catalog_properties': fields.String(required=True, description='New catalog properties for the document version'), }) @@ -239,7 +249,7 @@ class DocumentVersionResource(Resource): def put(self, version_id): """Edit a document version""" data = request.json - updated_version, error = edit_document_version(version_id, data['user_context']) + updated_version, error = edit_document_version(version_id, data['user_context'], data.get('catalog_properties')) if updated_version: return {'message': f'Document Version {updated_version.id} updated successfully'}, 200 else: @@ -251,7 +261,8 @@ refresh_document_model = document_ns.model('RefreshDocument', { 'name': fields.String(required=False, description='New name for the document'), 'language': fields.String(required=False, description='Language of the document'), 'user_context': fields.String(required=False, description='User context for the document'), - 'user_metadata': fields.Raw(required=False, description='User metadata for the document') + 'user_metadata': fields.Raw(required=False, description='User metadata for the document'), + 'catalog_properties': fields.Raw(required=False, description='Catalog properties for the document'), }) @@ -268,7 +279,7 @@ class RefreshDocument(Resource): current_app.logger.info(f'Refreshing document {document_id} for tenant {tenant_id}') try: - new_version, result = refresh_document(document_id) + new_version, result = refresh_document(document_id, tenant_id) if new_version: return { @@ -301,7 +312,7 @@ class RefreshDocumentWithInfo(Resource): try: api_input = request.json - new_version, result = refresh_document_with_info(document_id, api_input) + new_version, result = refresh_document_with_info(document_id, tenant_id, api_input) if new_version: return { diff --git a/eveai_app/templates/document/add_document.html b/eveai_app/templates/document/add_document.html index dcaeb1c..ef997a0 100644 --- a/eveai_app/templates/document/add_document.html +++ b/eveai_app/templates/document/add_document.html @@ -11,9 +11,17 @@ {{ form.hidden_tag() }} {% set disabled_fields = [] %} {% set exclude_fields = [] %} - {% for field in form %} + {% for field in form.get_static_fields() %} {{ render_field(field, disabled_fields, exclude_fields) }} {% endfor %} + {% for collection_name, fields in form.get_dynamic_fields().items() %} + {% if fields|length > 0 %} +

{{ collection_name }}

+ {% endif %} + {% for field in fields %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + {% endfor %} {% endblock %} diff --git a/eveai_app/views/document_forms.py b/eveai_app/views/document_forms.py index 8ad387a..8412888 100644 --- a/eveai_app/views/document_forms.py +++ b/eveai_app/views/document_forms.py @@ -149,7 +149,7 @@ class EditRetrieverForm(DynamicFormBase): self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()] -class AddDocumentForm(FlaskForm): +class AddDocumentForm(DynamicFormBase): file = FileField('File', validators=[FileRequired(), allowed_file]) name = StringField('Name', validators=[Length(max=100)]) language = SelectField('Language', choices=[], validators=[Optional()]) @@ -157,17 +157,15 @@ class AddDocumentForm(FlaskForm): valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()]) user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) - submit = SubmitField('Submit') - - def __init__(self): - super().__init__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.language.choices = [(language, language) for language in session.get('tenant').get('allowed_languages')] if not self.language.data: self.language.data = session.get('tenant').get('default_language') -class AddURLForm(FlaskForm): +class AddURLForm(DynamicFormBase): url = URLField('URL', validators=[DataRequired(), URL()]) name = StringField('Name', validators=[Length(max=100)]) language = SelectField('Language', choices=[], validators=[Optional()]) @@ -175,10 +173,8 @@ class AddURLForm(FlaskForm): valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()]) user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) - submit = SubmitField('Submit') - - def __init__(self): - super().__init__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.language.choices = [(language, language) for language in session.get('tenant').get('allowed_languages')] if not self.language.data: @@ -210,7 +206,7 @@ class EditDocumentForm(FlaskForm): submit = SubmitField('Submit') -class EditDocumentVersionForm(FlaskForm): +class EditDocumentVersionForm(DynamicFormBase): language = StringField('Language') user_context = TextAreaField('User Context', validators=[Optional()]) system_context = TextAreaField('System Context', validators=[Optional()]) diff --git a/eveai_app/views/document_views.py b/eveai_app/views/document_views.py index 2996002..06de1ad 100644 --- a/eveai_app/views/document_views.py +++ b/eveai_app/views/document_views.py @@ -228,6 +228,8 @@ def edit_retriever(retriever_id): configuration_config = RETRIEVER_TYPES[retriever.type]["configuration"] form.add_dynamic_fields("configuration", configuration_config, retriever.configuration) + if request.method == 'POST': + current_app.logger.debug(f'Received POST request with {request.form}') if form.validate_on_submit(): # Update basic fields @@ -295,19 +297,33 @@ def handle_retriever_selection(): @document_bp.route('/add_document', methods=['GET', 'POST']) @roles_accepted('Super User', 'Tenant Admin') def add_document(): - form = AddDocumentForm() + form = AddDocumentForm(request.form) + catalog_id = session.get('catalog_id', None) + if catalog_id is None: + flash('You need to set a Session Catalog before adding Documents or URLs') + return redirect(prefixed_url_for('document_bp.catalogs')) + + catalog = Catalog.query.get_or_404(catalog_id) + if catalog.configuration and len(catalog.configuration) > 0: + document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations'] + for config in document_version_configurations: + form.add_dynamic_fields(config, catalog.configuration[config]) if form.validate_on_submit(): try: + current_app.logger.info(f'Adding Document for {catalog_id}') tenant_id = session['tenant']['id'] - catalog_id = session['catalog_id'] file = form.file.data filename = secure_filename(file.filename) extension = filename.rsplit('.', 1)[1].lower() validate_file_type(extension) - current_app.logger.debug(f'Language on form: {form.language.data}') + catalog_properties = {} + document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations'] + for config in document_version_configurations: + catalog_properties[config] = form.get_dynamic_data(config) + api_input = { 'catalog_id': catalog_id, 'name': form.name.data, @@ -315,6 +331,7 @@ def add_document(): 'user_context': form.user_context.data, 'valid_from': form.valid_from.data, 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None, + 'catalog_properties': catalog_properties, } current_app.logger.debug(f'Creating document stack with input {api_input}') @@ -337,16 +354,30 @@ def add_document(): @document_bp.route('/add_url', methods=['GET', 'POST']) @roles_accepted('Super User', 'Tenant Admin') def add_url(): - form = AddURLForm() + form = AddURLForm(request.form) + catalog_id = session.get('catalog_id', None) + if catalog_id is None: + flash('You need to set a Session Catalog before adding Documents or URLs') + return redirect(prefixed_url_for('document_bp.catalogs')) + + catalog = Catalog.query.get_or_404(catalog_id) + if catalog.configuration and len(catalog.configuration) > 0: + document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations'] + for config in document_version_configurations: + form.add_dynamic_fields(config, catalog.configuration[config]) if form.validate_on_submit(): try: tenant_id = session['tenant']['id'] - catalog_id = session['catalog_id'] url = form.url.data file_content, filename, extension = process_url(url, tenant_id) + catalog_properties = {} + document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations'] + for config in document_version_configurations: + catalog_properties[config] = form.get_dynamic_data(config) + api_input = { 'catalog_id': catalog_id, 'name': form.name.data or filename, @@ -355,6 +386,7 @@ def add_url(): 'user_context': form.user_context.data, 'valid_from': form.valid_from.data, 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None, + 'catalog_properties': catalog_properties, } new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id) @@ -375,43 +407,6 @@ def add_url(): return render_template('document/add_url.html', form=form) -@document_bp.route('/add_urls', methods=['GET', 'POST']) -@roles_accepted('Super User', 'Tenant Admin') -def add_urls(): - form = AddURLsForm() - - if form.validate_on_submit(): - try: - tenant_id = session['tenant']['id'] - urls = form.urls.data.split('\n') - urls = [url.strip() for url in urls if url.strip()] - - api_input = { - 'name': form.name.data, - 'language': form.language.data, - 'user_context': form.user_context.data, - 'valid_from': form.valid_from.data - } - - results = process_multiple_urls(urls, tenant_id, api_input) - - for result in results: - if result['status'] == 'success': - flash( - f"Processed URL: {result['url']} - Document ID: {result['document_id']}, Version ID: {result['document_version_id']}", - 'success') - else: - flash(f"Error processing URL: {result['url']} - {result['message']}", 'error') - - return redirect(prefixed_url_for('document_bp.documents')) - - except Exception as e: - current_app.logger.error(f'Error adding multiple URLs: {str(e)}') - flash('An error occurred while adding the URLs.', 'error') - - return render_template('document/add_urls.html', form=form) - - @document_bp.route('/documents', methods=['GET', 'POST']) @roles_accepted('Super User', 'Tenant Admin') def documents(): @@ -494,12 +489,29 @@ def edit_document_view(document_id): @roles_accepted('Super User', 'Tenant Admin') def edit_document_version_view(document_version_id): doc_vers = DocumentVersion.query.get_or_404(document_version_id) - form = EditDocumentVersionForm(obj=doc_vers) + form = EditDocumentVersionForm(request.form, obj=doc_vers) + + catalog_id = session.get('catalog_id', None) + if catalog_id is None: + flash('You need to set a Session Catalog before adding Documents or URLs') + return redirect(prefixed_url_for('document_bp.catalogs')) + + catalog = Catalog.query.get_or_404(catalog_id) + if catalog.configuration and len(catalog.configuration) > 0: + document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations'] + for config in document_version_configurations: + form.add_dynamic_fields(config, catalog.configuration[config], doc_vers.catalog_properties[config]) if form.validate_on_submit(): + catalog_properties = {} + document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations'] + for config in document_version_configurations: + catalog_properties[config] = form.get_dynamic_data(config) + updated_version, error = edit_document_version( document_version_id, - form.user_context.data + form.user_context.data, + catalog_properties, ) if updated_version: flash(f'Document Version {updated_version.id} updated successfully', 'success') diff --git a/eveai_app/views/dynamic_form_base.py b/eveai_app/views/dynamic_form_base.py index 28094a1..0abe4e9 100644 --- a/eveai_app/views/dynamic_form_base.py +++ b/eveai_app/views/dynamic_form_base.py @@ -48,10 +48,9 @@ class DynamicFormBase(FlaskForm): current_app.logger.debug(f"{field_name}: {field_def}") # Prefix the field name with the collection name full_field_name = f"{collection_name}_{field_name}" - label = field_def.get('name') + label = field_def.get('name', field_name) field_type = field_def.get('type') description = field_def.get('description', '') - required = field_def.get('required', False) default = field_def.get('default') # Determine standard validators @@ -90,8 +89,8 @@ class DynamicFormBase(FlaskForm): except (TypeError, ValueError) as e: current_app.logger.error(f"Error converting initial data to JSON: {e}") field_data = "{}" - elif field_def.get('default') is not None: - field_data = field_def.get('default') + elif default is not None: + field_data = default # Create render_kw with classes and any other HTML attributes render_kw = {'class': extra_classes} if extra_classes else {} @@ -145,12 +144,16 @@ class DynamicFormBase(FlaskForm): def get_dynamic_data(self, collection_name): """Retrieve the data from dynamic fields of a specific collection.""" data = {} + current_app.logger.debug(f"{collection_name} in {self.dynamic_fields}?") if collection_name not in self.dynamic_fields: return data prefix_length = len(collection_name) + 1 # +1 for the underscore for full_field_name in self.dynamic_fields[collection_name]: + current_app.logger.debug(f"{full_field_name}: {full_field_name}") original_field_name = full_field_name[prefix_length:] + current_app.logger.debug(f"{original_field_name}: {original_field_name}") field = getattr(self, full_field_name) + current_app.logger.debug(f"{field}: {field}") # Parse JSON for tagging_fields type if isinstance(field, TextAreaField) and field.data: try: @@ -207,4 +210,7 @@ def validate_tagging_fields(form, field): except json.JSONDecodeError: raise ValidationError("Invalid JSON format") except (TypeError, ValueError) as e: - raise ValidationError(f"Invalid field definition: {str(e)}") \ No newline at end of file + raise ValidationError(f"Invalid field definition: {str(e)}") + + + diff --git a/migrations/tenant/versions/6c5ca750e60c_add_catalog_properties_to_.py b/migrations/tenant/versions/6c5ca750e60c_add_catalog_properties_to_.py new file mode 100644 index 0000000..9e0a153 --- /dev/null +++ b/migrations/tenant/versions/6c5ca750e60c_add_catalog_properties_to_.py @@ -0,0 +1,29 @@ +"""Add catalog_properties to DocumentVersion + +Revision ID: 6c5ca750e60c +Revises: b64d5cf32c7a +Create Date: 2024-10-29 08:33:54.663211 + +""" +from alembic import op +import sqlalchemy as sa +import pgvector +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '6c5ca750e60c' +down_revision = 'b64d5cf32c7a' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('document_version', sa.Column('catalog_properties', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('document_version', 'catalog_properties') + # ### end Alembic commands ###