6ef025363d
- menu changes to allow for partners - partner views and forms now in partner_forms.py and partner_views.py - Introduction of services layer - Allow all configuration to handle partner configurations, and adaptation of caching to allow for this
504 lines
19 KiB
Python
504 lines
19 KiB
Python
from typing import Dict, Any, Optional
|
|
from pathlib import Path
|
|
import yaml
|
|
from packaging import version
|
|
import os
|
|
from flask import current_app
|
|
|
|
from common.utils.cache.base import CacheHandler, CacheKey
|
|
from config.type_defs import agent_types, task_types, tool_types, specialist_types, retriever_types, prompt_types, \
|
|
catalog_types, partner_service_types
|
|
|
|
|
|
def is_major_minor(version: str) -> bool:
|
|
parts = version.strip('.').split('.')
|
|
return len(parts) == 2 and all(part.isdigit() for part in parts)
|
|
|
|
|
|
class BaseConfigCacheHandler(CacheHandler[Dict[str, Any]]):
|
|
"""Base handler for configuration caching"""
|
|
|
|
def __init__(self, region, config_type: str):
|
|
"""
|
|
Args:
|
|
region: Cache region
|
|
config_type: Type of configuration (agents, tasks, etc.)
|
|
"""
|
|
super().__init__(region, f'config_{config_type}')
|
|
self.config_type = config_type
|
|
self._types_module = None # Set by subclasses
|
|
self._config_dir = None # Set by subclasses
|
|
self.version_tree_cache = None
|
|
self.configure_keys('type_name', 'version')
|
|
|
|
def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Convert the data to a cacheable format"""
|
|
# For configuration data, we can just return the dictionary as is
|
|
# since it's already in a serializable format
|
|
return instance
|
|
|
|
def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
|
|
"""Convert cached data back to usable format"""
|
|
# Similarly, we can return the data directly since it's already
|
|
# in the format we need
|
|
return data
|
|
|
|
def _should_cache(self, value: Dict[str, Any]) -> bool:
|
|
"""
|
|
Validate if the value should be cached
|
|
|
|
Args:
|
|
value: The value to be cached
|
|
|
|
Returns:
|
|
bool: True if the value should be cached
|
|
"""
|
|
return isinstance(value, dict) # Cache all dictionaries
|
|
|
|
def set_version_tree_cache(self, cache):
|
|
"""Set the version tree cache dependency."""
|
|
self.version_tree_cache = cache
|
|
|
|
def _load_specific_config(self, type_name: str, version_str: str) -> Dict[str, Any]:
|
|
"""
|
|
Load a specific configuration version
|
|
Automatically handles global vs partner-specific configs
|
|
"""
|
|
version_tree = self.version_tree_cache.get_versions(type_name)
|
|
versions = version_tree['versions']
|
|
|
|
if version_str == 'latest':
|
|
version_str = version_tree['latest_version']
|
|
|
|
if version_str not in versions:
|
|
raise ValueError(f"Version {version_str} not found for {type_name}")
|
|
|
|
version_info = versions[version_str]
|
|
file_path = version_info['file_path']
|
|
partner = version_info.get('partner')
|
|
|
|
try:
|
|
with open(file_path) as f:
|
|
config = yaml.safe_load(f)
|
|
# Add partner information to the config
|
|
if partner:
|
|
config['partner'] = partner
|
|
return config
|
|
except Exception as e:
|
|
raise ValueError(f"Error loading config from {file_path}: {e}")
|
|
|
|
def get_config(self, type_name: str, version: Optional[str] = None) -> Dict[str, Any]:
|
|
"""
|
|
Get configuration for a specific type and version
|
|
If version not specified, returns latest
|
|
|
|
Args:
|
|
type_name: Configuration type name
|
|
version: Optional specific version to retrieve
|
|
|
|
Returns:
|
|
Configuration data
|
|
"""
|
|
if version is None:
|
|
version_str = self.version_tree_cache.get_latest_version(type_name)
|
|
elif is_major_minor(version):
|
|
version_str = self.version_tree_cache.get_latest_patch_version(type_name, version)
|
|
else:
|
|
version_str = version
|
|
|
|
result = self.get(
|
|
lambda type_name, version: self._load_specific_config(type_name, version),
|
|
type_name=type_name,
|
|
version=version_str
|
|
)
|
|
return result
|
|
|
|
|
|
class BaseConfigVersionTreeCacheHandler(CacheHandler[Dict[str, Any]]):
|
|
"""Base handler for configuration version tree caching"""
|
|
|
|
def __init__(self, region, config_type: str):
|
|
"""
|
|
Args:
|
|
region: Cache region
|
|
config_type: Type of configuration (agents, tasks, etc.)
|
|
"""
|
|
super().__init__(region, f'config_{config_type}_version_tree')
|
|
self.config_type = config_type
|
|
self._types_module = None # Set by subclasses
|
|
self._config_dir = None # Set by subclasses
|
|
self.configure_keys('type_name')
|
|
|
|
def _load_version_tree(self, type_name: str) -> Dict[str, Any]:
|
|
"""
|
|
Load version tree for a specific type without loading full configurations
|
|
Checks both global and partner-specific directories
|
|
"""
|
|
# First check the global path
|
|
global_path = Path(self._config_dir) / "global" / type_name
|
|
|
|
# If global path doesn't exist, check if the type exists directly in the root
|
|
# (for backward compatibility)
|
|
if not global_path.exists():
|
|
global_path = Path(self._config_dir) / type_name
|
|
|
|
if not global_path.exists():
|
|
# Check if it exists in any partner subdirectories
|
|
partner_dirs = [d for d in Path(self._config_dir).iterdir()
|
|
if d.is_dir() and d.name != "global"]
|
|
|
|
for partner_dir in partner_dirs:
|
|
partner_type_path = partner_dir / type_name
|
|
if partner_type_path.exists():
|
|
# Found in partner directory
|
|
return self._load_versions_from_path(partner_type_path)
|
|
|
|
# If we get here, the type wasn't found anywhere
|
|
raise ValueError(f"No configuration found for type {type_name}")
|
|
|
|
return self._load_versions_from_path(global_path)
|
|
|
|
def _load_versions_from_path(self, path: Path) -> Dict[str, Any]:
|
|
"""Load all versions from a specific path"""
|
|
version_files = list(path.glob('*.yaml'))
|
|
if not version_files:
|
|
raise ValueError(f"No versions found in {path}")
|
|
|
|
versions = {}
|
|
latest_version = None
|
|
latest_version_obj = None
|
|
|
|
for file_path in version_files:
|
|
ver = file_path.stem # Get version from filename
|
|
try:
|
|
ver_obj = version.parse(ver)
|
|
# Only load minimal metadata for version tree
|
|
with open(file_path) as f:
|
|
yaml_data = yaml.safe_load(f)
|
|
metadata = yaml_data.get('metadata', {})
|
|
# Add partner information if available
|
|
partner = None
|
|
if "global" not in str(file_path):
|
|
# Extract partner name from path
|
|
# Path format: config_dir/partner_name/type_name/version.yaml
|
|
partner = file_path.parent.parent.name
|
|
|
|
versions[ver] = {
|
|
'metadata': metadata,
|
|
'file_path': str(file_path),
|
|
'partner': partner
|
|
}
|
|
|
|
# Track latest version
|
|
if latest_version_obj is None or ver_obj > latest_version_obj:
|
|
latest_version = ver
|
|
latest_version_obj = ver_obj
|
|
|
|
except Exception as e:
|
|
current_app.logger.error(f"Error loading version {ver}: {e}")
|
|
continue
|
|
|
|
return {
|
|
'versions': versions,
|
|
'latest_version': latest_version
|
|
}
|
|
|
|
def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Convert the data to a cacheable format"""
|
|
# For configuration data, we can just return the dictionary as is
|
|
# since it's already in a serializable format
|
|
return instance
|
|
|
|
def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
|
|
"""Convert cached data back to usable format"""
|
|
# Similarly, we can return the data directly since it's already
|
|
# in the format we need
|
|
return data
|
|
|
|
def _should_cache(self, value: Dict[str, Any]) -> bool:
|
|
"""
|
|
Validate if the value should be cached
|
|
|
|
Args:
|
|
value: The value to be cached
|
|
|
|
Returns:
|
|
bool: True if the value should be cached
|
|
"""
|
|
return isinstance(value, dict) # Cache all dictionaries
|
|
|
|
def get_versions(self, type_name: str) -> Dict[str, Any]:
|
|
"""
|
|
Get version tree for a type
|
|
|
|
Args:
|
|
type_name: Type to get versions for
|
|
|
|
Returns:
|
|
Dict with version information
|
|
"""
|
|
return self.get(
|
|
lambda type_name: self._load_version_tree(type_name),
|
|
type_name=type_name
|
|
)
|
|
|
|
def get_latest_version(self, type_name: str) -> str:
|
|
"""
|
|
Get the latest version for a given type name.
|
|
|
|
Args:
|
|
type_name: Name of the configuration type
|
|
|
|
Returns:
|
|
Latest version string
|
|
|
|
Raises:
|
|
ValueError: If type not found or no versions available
|
|
"""
|
|
version_tree = self.get_versions(type_name)
|
|
if not version_tree or 'latest_version' not in version_tree:
|
|
raise ValueError(f"No versions found for {type_name}")
|
|
|
|
return version_tree['latest_version']
|
|
|
|
def get_latest_patch_version(self, type_name: str, major_minor: str) -> str:
|
|
"""
|
|
Get the latest patch version for a given major.minor version.
|
|
|
|
Args:
|
|
type_name: Name of the configuration type
|
|
major_minor: Major.minor version (e.g. "1.0")
|
|
|
|
Returns:
|
|
Latest patch version string (e.g. "1.0.3")
|
|
|
|
Raises:
|
|
ValueError: If type not found or no matching versions
|
|
"""
|
|
version_tree = self.get_versions(type_name)
|
|
if not version_tree or 'versions' not in version_tree:
|
|
raise ValueError(f"No versions found for {type_name}")
|
|
|
|
# Filter versions that match the major.minor prefix
|
|
matching_versions = [
|
|
ver for ver in version_tree['versions'].keys()
|
|
if ver.startswith(major_minor + '.')
|
|
]
|
|
|
|
if not matching_versions:
|
|
raise ValueError(f"No versions found for {type_name} with prefix {major_minor}")
|
|
|
|
# Return highest matching version
|
|
latest_patch = max(matching_versions, key=version.parse)
|
|
return latest_patch
|
|
|
|
|
|
class BaseConfigTypesCacheHandler(CacheHandler[Dict[str, Any]]):
|
|
"""Base handler for configuration types caching"""
|
|
|
|
def __init__(self, region, config_type: str):
|
|
"""
|
|
Args:
|
|
region: Cache region
|
|
config_type: Type of configuration (agents, tasks, etc.)
|
|
"""
|
|
super().__init__(region, f'config_{config_type}_types')
|
|
self.config_type = config_type
|
|
self._types_module = None # Set by subclasses
|
|
self._config_dir = None # Set by subclasses
|
|
self.configure_keys()
|
|
|
|
def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Convert the data to a cacheable format"""
|
|
# For configuration data, we can just return the dictionary as is
|
|
# since it's already in a serializable format
|
|
return instance
|
|
|
|
def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
|
|
"""Convert cached data back to usable format"""
|
|
# Similarly, we can return the data directly since it's already
|
|
# in the format we need
|
|
return data
|
|
|
|
def _should_cache(self, value: Dict[str, Any]) -> bool:
|
|
"""
|
|
Validate if the value should be cached
|
|
|
|
Args:
|
|
value: The value to be cached
|
|
|
|
Returns:
|
|
bool: True if the value should be cached
|
|
"""
|
|
return isinstance(value, dict) # Cache all dictionaries
|
|
|
|
def _load_type_definitions(self) -> Dict[str, Dict[str, str]]:
|
|
"""Load type definitions from the corresponding type_defs module"""
|
|
if not self._types_module:
|
|
raise ValueError("_types_module must be set by subclass")
|
|
|
|
type_definitions = {
|
|
type_id: {
|
|
'name': info['name'],
|
|
'description': info['description'],
|
|
'partner': info.get('partner') # Include partner info if available
|
|
}
|
|
for type_id, info in self._types_module.items()
|
|
}
|
|
|
|
return type_definitions
|
|
|
|
def get_types(self) -> Dict[str, Dict[str, str]]:
|
|
"""Get dictionary of available types with name and description"""
|
|
result = self.get(
|
|
lambda type_name: self._load_type_definitions(),
|
|
type_name=f'{self.config_type}_types',
|
|
)
|
|
return result
|
|
|
|
|
|
def create_config_cache_handlers(config_type: str, config_dir: str, types_module: dict) -> tuple:
|
|
"""
|
|
Factory function to dynamically create the 3 cache handler classes for a given configuration type.
|
|
The following cache names are created:
|
|
- <config_type>_config_cache
|
|
- <config_type>_version_tree_cache
|
|
- <config_type>_types_cache
|
|
|
|
|
|
Args:
|
|
config_type: The configuration type (e.g., 'agents', 'tasks').
|
|
config_dir: The directory where configuration files are stored.
|
|
types_module: The types module defining the available types for this config.
|
|
|
|
Returns:
|
|
A tuple of dynamically created classes for config, version tree, and types handlers.
|
|
"""
|
|
|
|
class ConfigCacheHandler(BaseConfigCacheHandler):
|
|
handler_name = f"{config_type}_config_cache"
|
|
|
|
def __init__(self, region):
|
|
super().__init__(region, config_type)
|
|
self._types_module = types_module
|
|
self._config_dir = config_dir
|
|
|
|
class VersionTreeCacheHandler(BaseConfigVersionTreeCacheHandler):
|
|
handler_name = f"{config_type}_version_tree_cache"
|
|
|
|
def __init__(self, region):
|
|
super().__init__(region, config_type)
|
|
self._types_module = types_module
|
|
self._config_dir = config_dir
|
|
|
|
class TypesCacheHandler(BaseConfigTypesCacheHandler):
|
|
handler_name = f"{config_type}_types_cache"
|
|
|
|
def __init__(self, region):
|
|
super().__init__(region, config_type)
|
|
self._types_module = types_module
|
|
self._config_dir = config_dir
|
|
|
|
return ConfigCacheHandler, VersionTreeCacheHandler, TypesCacheHandler
|
|
|
|
|
|
AgentConfigCacheHandler, AgentConfigVersionTreeCacheHandler, AgentConfigTypesCacheHandler = (
|
|
create_config_cache_handlers(
|
|
config_type='agents',
|
|
config_dir='config/agents',
|
|
types_module=agent_types.AGENT_TYPES
|
|
))
|
|
|
|
|
|
TaskConfigCacheHandler, TaskConfigVersionTreeCacheHandler, TaskConfigTypesCacheHandler = (
|
|
create_config_cache_handlers(
|
|
config_type='tasks',
|
|
config_dir='config/tasks',
|
|
types_module=task_types.TASK_TYPES
|
|
))
|
|
|
|
|
|
ToolConfigCacheHandler, ToolConfigVersionTreeCacheHandler, ToolConfigTypesCacheHandler = (
|
|
create_config_cache_handlers(
|
|
config_type='tools',
|
|
config_dir='config/tools',
|
|
types_module=tool_types.TOOL_TYPES
|
|
))
|
|
|
|
|
|
SpecialistConfigCacheHandler, SpecialistConfigVersionTreeCacheHandler, SpecialistConfigTypesCacheHandler = (
|
|
create_config_cache_handlers(
|
|
config_type='specialists',
|
|
config_dir='config/specialists',
|
|
types_module=specialist_types.SPECIALIST_TYPES
|
|
))
|
|
|
|
|
|
RetrieverConfigCacheHandler, RetrieverConfigVersionTreeCacheHandler, RetrieverConfigTypesCacheHandler = (
|
|
create_config_cache_handlers(
|
|
config_type='retrievers',
|
|
config_dir='config/retrievers',
|
|
types_module=retriever_types.RETRIEVER_TYPES
|
|
))
|
|
|
|
|
|
PromptConfigCacheHandler, PromptConfigVersionTreeCacheHandler, PromptConfigTypesCacheHandler = (
|
|
create_config_cache_handlers(
|
|
config_type='prompts',
|
|
config_dir='config/prompts',
|
|
types_module=prompt_types.PROMPT_TYPES
|
|
))
|
|
|
|
CatalogConfigCacheHandler, CatalogConfigVersionTreeCacheHandler, CatalogConfigTypesCacheHandler = (
|
|
create_config_cache_handlers(
|
|
config_type='catalogs',
|
|
config_dir='config/catalogs',
|
|
types_module=catalog_types.CATALOG_TYPES
|
|
))
|
|
|
|
# Add to common/utils/cache/config_cache.py
|
|
PartnerServiceConfigCacheHandler, PartnerServiceConfigVersionTreeCacheHandler, PartnerServiceConfigTypesCacheHandler = (
|
|
create_config_cache_handlers(
|
|
config_type='partner_services',
|
|
config_dir='config/partner_services',
|
|
types_module=partner_service_types.PARTNER_SERVICE_TYPES
|
|
))
|
|
|
|
|
|
def register_config_cache_handlers(cache_manager) -> None:
|
|
cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(AgentConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(AgentConfigVersionTreeCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(TaskConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(TaskConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(TaskConfigVersionTreeCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(ToolConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(ToolConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(ToolConfigVersionTreeCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(SpecialistConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(SpecialistConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(SpecialistConfigVersionTreeCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(RetrieverConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(RetrieverConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(RetrieverConfigVersionTreeCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(PromptConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(PromptConfigVersionTreeCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(PromptConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(CatalogConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(CatalogConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(CatalogConfigVersionTreeCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(AgentConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(AgentConfigVersionTreeCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(PartnerServiceConfigCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(PartnerServiceConfigTypesCacheHandler, 'eveai_config')
|
|
cache_manager.register_handler(PartnerServiceConfigVersionTreeCacheHandler, 'eveai_config')
|
|
|
|
cache_manager.agents_config_cache.set_version_tree_cache(cache_manager.agents_version_tree_cache)
|
|
cache_manager.tasks_config_cache.set_version_tree_cache(cache_manager.tasks_version_tree_cache)
|
|
cache_manager.tools_config_cache.set_version_tree_cache(cache_manager.tools_version_tree_cache)
|
|
cache_manager.specialists_config_cache.set_version_tree_cache(cache_manager.specialists_version_tree_cache)
|
|
cache_manager.retrievers_config_cache.set_version_tree_cache(cache_manager.retrievers_version_tree_cache)
|
|
cache_manager.prompts_config_cache.set_version_tree_cache(cache_manager.prompts_version_tree_cache)
|
|
cache_manager.partner_services_config_cache.set_version_tree_cache(cache_manager.partner_services_version_tree_cache)
|