- First 'working' version of the Zapier plugin. Needs further debugging and needs additional functionalty (only add_document.js)
This commit is contained in:
Josako
@@ -1,7 +1,9 @@
|
||||
import io
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from flask import current_app, request
|
||||
from flask_restx import Namespace, Resource, fields, reqparse
|
||||
from flask_jwt_extended import jwt_required, get_jwt_identity
|
||||
@@ -9,9 +11,9 @@ from werkzeug.datastructures import FileStorage
|
||||
from werkzeug.utils import secure_filename
|
||||
from common.utils.document_utils import (
|
||||
create_document_stack, process_url, start_embedding_task,
|
||||
validate_file_type, EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType,
|
||||
EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType,
|
||||
get_documents_list, edit_document, refresh_document, edit_document_version,
|
||||
refresh_document_with_info, lookup_document
|
||||
refresh_document_with_info, lookup_document, refresh_document_with_content
|
||||
)
|
||||
from common.utils.eveai_exceptions import EveAIException
|
||||
from eveai_api.api.auth import requires_service
|
||||
@@ -74,12 +76,17 @@ class AddDocument(Resource):
|
||||
|
||||
try:
|
||||
args = upload_parser.parse_args()
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Error parsing arguments: {str(e)}")
|
||||
current_app.logger.error(f"Exception type: {type(e)}")
|
||||
raise
|
||||
|
||||
try:
|
||||
file = args['file']
|
||||
filename = secure_filename(file.filename)
|
||||
extension = filename.rsplit('.', 1)[1].lower()
|
||||
|
||||
validate_file_type(extension)
|
||||
# validate_file_type(extension)
|
||||
|
||||
api_input = {
|
||||
'catalog_id': args.get('catalog_id'),
|
||||
@@ -109,6 +116,105 @@ class AddDocument(Resource):
|
||||
document_ns.abort(500, 'Error adding document')
|
||||
|
||||
|
||||
# Models for AddDocumentThroughURL
|
||||
add_document_through_url = document_ns.model('AddDocumentThroughURL', {
|
||||
'catalog_id': fields.Integer(required=True, description='ID of the catalog the URL needs to be added to'),
|
||||
'temp_url': fields.String(required=True, description='Temporary URL of the document to add'),
|
||||
'name': fields.String(required=False, description='Name of the document'),
|
||||
'language': fields.String(required=True, description='Language of the document'),
|
||||
'user_context': fields.String(required=False, description='User context for the document'),
|
||||
'valid_from': fields.String(required=False, description='Valid from date for the document'),
|
||||
'user_metadata': fields.String(required=False, description='User metadata for the document'),
|
||||
'system_metadata': fields.String(required=False, description='System metadata for the document'),
|
||||
'catalog_properties': fields.String(required=False, description='The catalog configuration to be passed along (JSON '
|
||||
'format). Validity is against catalog requirements '
|
||||
'is not checked, and is the responsibility of the '
|
||||
'calling client.'),
|
||||
})
|
||||
|
||||
add_document_through_url_response = document_ns.model('AddDocumentThroughURLResponse', {
|
||||
'message': fields.String(description='Status message'),
|
||||
'document_id': fields.Integer(description='ID of the created document'),
|
||||
'document_version_id': fields.Integer(description='ID of the created document version'),
|
||||
'task_id': fields.String(description='ID of the embedding task')
|
||||
})
|
||||
|
||||
@document_ns.route('/add_document_through_url')
|
||||
class AddDocumentThroughURL(Resource):
|
||||
@jwt_required()
|
||||
@requires_service('DOCAPI')
|
||||
@document_ns.expect(add_document_through_url)
|
||||
@document_ns.response(201, 'Document added successfully', add_document_through_url)
|
||||
@document_ns.response(400, 'Validation Error')
|
||||
@document_ns.response(422, 'File could not be processed')
|
||||
@document_ns.response(500, 'Internal Server Error')
|
||||
def post(self):
|
||||
"""
|
||||
Add a new document using a URL. The URL can be temporary, and will not be stored.
|
||||
Mainly used for passing temporary URLs like used in e.g. Zapier
|
||||
"""
|
||||
tenant_id = get_jwt_identity()
|
||||
current_app.logger.info(f'Adding document through url for tenant {tenant_id}')
|
||||
|
||||
try:
|
||||
args = document_ns.payload
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Error parsing arguments: {str(e)}")
|
||||
current_app.logger.error(f"Exception type: {type(e)}")
|
||||
raise
|
||||
|
||||
file_url = args['temp_url']
|
||||
current_app.logger.info(f"Downloading file from URL: {file_url}")
|
||||
try:
|
||||
response = requests.get(file_url, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
# Get filename from URL or use provided name
|
||||
filename = secure_filename(args.get('name') or file_url.split('/')[-1])
|
||||
extension = filename.rsplit('.', 1)[1].lower() if '.' in filename else ''
|
||||
|
||||
# Create FileStorage object from downloaded content
|
||||
file_content = io.BytesIO(response.content)
|
||||
file = FileStorage(
|
||||
stream=file_content,
|
||||
filename=filename,
|
||||
content_type=response.headers.get('content-type', 'application/octet-stream')
|
||||
)
|
||||
|
||||
current_app.logger.info(f"Successfully downloaded file: {filename}")
|
||||
except requests.RequestException as e:
|
||||
current_app.logger.error(f"Error downloading file: {str(e)}")
|
||||
return {'message': f'Error downloading file: {str(e)}'}, 422
|
||||
|
||||
try:
|
||||
# Prepare API input
|
||||
api_input = {
|
||||
'catalog_id': args.get('catalog_id'),
|
||||
'name': args.get('name') or filename,
|
||||
'language': args.get('language'),
|
||||
'user_context': args.get('user_context'),
|
||||
'valid_from': args.get('valid_from'),
|
||||
'user_metadata': args.get('user_metadata'),
|
||||
'catalog_properties': args.get('catalog_properties'),
|
||||
}
|
||||
|
||||
new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id)
|
||||
task_id = start_embedding_task(tenant_id, new_doc_vers.id)
|
||||
|
||||
return {
|
||||
'message': f'Processing on document {new_doc.name}, version {new_doc_vers.id} started. Task ID: {task_id}.',
|
||||
'document_id': new_doc.id,
|
||||
'document_version_id': new_doc_vers.id,
|
||||
'task_id': task_id
|
||||
}, 201
|
||||
except (EveAIInvalidLanguageException, EveAIUnsupportedFileType) as e:
|
||||
current_app.logger.error(f'Error adding document: {str(e)}')
|
||||
return {'message': str(e)}, 400
|
||||
except Exception as e:
|
||||
current_app.logger.error(f'Error adding document: {str(e)}')
|
||||
return {'message': 'Error adding document'}, 500
|
||||
|
||||
|
||||
# Models for AddURL
|
||||
add_url_model = document_ns.model('AddURL', {
|
||||
'catalog_id': fields.Integer(required='True', description='ID of the catalog the URL needs to be added to'),
|
||||
|
||||
Reference in New Issue
Block a user