7bddeb0ebd
- correct startup of applications using gevent - introduce startup scripts (eveai_app) - caching manager for all configurations
307 lines
10 KiB
Python
307 lines
10 KiB
Python
from typing import Dict, Any, Optional
|
|
from pathlib import Path
|
|
import yaml
|
|
from packaging import version
|
|
import os
|
|
from flask import current_app
|
|
|
|
from common.utils.cache.base import CacheHandler
|
|
|
|
from config.type_defs import agent_types, task_types, tool_types, specialist_types
|
|
|
|
|
|
class BaseConfigCacheHandler(CacheHandler[Dict[str, Any]]):
|
|
"""Base handler for configuration caching"""
|
|
|
|
def __init__(self, region, config_type: str):
|
|
"""
|
|
Args:
|
|
region: Cache region
|
|
config_type: Type of configuration (agents, tasks, etc.)
|
|
"""
|
|
super().__init__(region, f'config_{config_type}')
|
|
self.config_type = config_type
|
|
self._types_module = None # Set by subclasses
|
|
self._config_dir = None # Set by subclasses
|
|
|
|
def configure_keys_for_operation(self, operation: str):
|
|
"""Configure required keys based on operation"""
|
|
match operation:
|
|
case 'get_types':
|
|
self.configure_keys('type_name') # Only require type_name for type definitions
|
|
case 'get_versions':
|
|
self.configure_keys('type_name') # Only type_name needed for version tree
|
|
case 'get_config':
|
|
self.configure_keys('type_name', 'version') # Both needed for specific config
|
|
case _:
|
|
raise ValueError(f"Unknown operation: {operation}")
|
|
|
|
def _load_version_tree(self, type_name: str) -> Dict[str, Any]:
|
|
"""
|
|
Load version tree for a specific type without loading full configurations
|
|
|
|
Args:
|
|
type_name: Name of configuration type
|
|
|
|
Returns:
|
|
Dict containing available versions and their metadata
|
|
"""
|
|
type_path = Path(self._config_dir) / type_name
|
|
if not type_path.exists():
|
|
raise ValueError(f"No configuration found for type {type_name}")
|
|
|
|
version_files = list(type_path.glob('*.yaml'))
|
|
if not version_files:
|
|
raise ValueError(f"No versions found for type {type_name}")
|
|
|
|
versions = {}
|
|
latest_version = None
|
|
latest_version_obj = None
|
|
|
|
for file_path in version_files:
|
|
ver = file_path.stem # Get version from filename
|
|
try:
|
|
ver_obj = version.parse(ver)
|
|
# Only load minimal metadata for version tree
|
|
with open(file_path) as f:
|
|
yaml_data = yaml.safe_load(f)
|
|
metadata = yaml_data.get('metadata', {})
|
|
versions[ver] = {
|
|
'metadata': metadata,
|
|
'file_path': str(file_path)
|
|
}
|
|
|
|
# Track latest version
|
|
if latest_version_obj is None or ver_obj > latest_version_obj:
|
|
latest_version = ver
|
|
latest_version_obj = ver_obj
|
|
|
|
except Exception as e:
|
|
current_app.logger.error(f"Error loading version {ver}: {e}")
|
|
continue
|
|
|
|
current_app.logger.debug(f"Loaded versions for {type_name}: {versions}")
|
|
current_app.logger.debug(f"Loaded versions for {type_name}: {latest_version}")
|
|
return {
|
|
'versions': versions,
|
|
'latest_version': latest_version
|
|
}
|
|
|
|
def to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Convert the data to a cacheable format"""
|
|
# For configuration data, we can just return the dictionary as is
|
|
# since it's already in a serializable format
|
|
return instance
|
|
|
|
def from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
|
|
"""Convert cached data back to usable format"""
|
|
# Similarly, we can return the data directly since it's already
|
|
# in the format we need
|
|
return data
|
|
|
|
def should_cache(self, value: Dict[str, Any]) -> bool:
|
|
"""
|
|
Validate if the value should be cached
|
|
|
|
Args:
|
|
value: The value to be cached
|
|
|
|
Returns:
|
|
bool: True if the value should be cached
|
|
"""
|
|
if not isinstance(value, dict):
|
|
return False
|
|
|
|
# For type definitions
|
|
if 'name' in value and 'description' in value:
|
|
return True
|
|
|
|
# For configurations
|
|
if 'versions' in value and 'latest_version' in value:
|
|
return True
|
|
|
|
return False
|
|
|
|
def _load_type_definitions(self) -> Dict[str, Dict[str, str]]:
|
|
"""Load type definitions from the corresponding type_defs module"""
|
|
if not self._types_module:
|
|
raise ValueError("_types_module must be set by subclass")
|
|
|
|
type_definitions = {
|
|
type_id: {
|
|
'name': info['name'],
|
|
'description': info['description']
|
|
}
|
|
for type_id, info in self._types_module.items()
|
|
}
|
|
|
|
current_app.logger.debug(f"Loaded type definitions: {type_definitions}")
|
|
|
|
return type_definitions
|
|
|
|
def _load_specific_config(self, type_name: str, version_str: str) -> Dict[str, Any]:
|
|
"""
|
|
Load a specific configuration version
|
|
|
|
Args:
|
|
type_name: Type name
|
|
version_str: Version string
|
|
|
|
Returns:
|
|
Configuration data
|
|
"""
|
|
version_tree = self.get_versions(type_name)
|
|
versions = version_tree['versions']
|
|
|
|
if version_str == 'latest':
|
|
version_str = version_tree['latest_version']
|
|
|
|
if version_str not in versions:
|
|
raise ValueError(f"Version {version_str} not found for {type_name}")
|
|
|
|
file_path = versions[version_str]['file_path']
|
|
|
|
try:
|
|
with open(file_path) as f:
|
|
config = yaml.safe_load(f)
|
|
current_app.logger.debug(f"Loaded config for {type_name}{version_str}: {config}")
|
|
return config
|
|
except Exception as e:
|
|
raise ValueError(f"Error loading config from {file_path}: {e}")
|
|
|
|
def get_versions(self, type_name: str) -> Dict[str, Any]:
|
|
"""
|
|
Get version tree for a type
|
|
|
|
Args:
|
|
type_name: Type to get versions for
|
|
|
|
Returns:
|
|
Dict with version information
|
|
"""
|
|
self.configure_keys_for_operation('get_versions')
|
|
return self.get(
|
|
lambda type_name: self._load_version_tree(type_name),
|
|
type_name=type_name
|
|
)
|
|
|
|
def get_latest_version(self, type_name: str) -> str:
|
|
"""
|
|
Get the latest version for a given type name.
|
|
|
|
Args:
|
|
type_name: Name of the configuration type
|
|
|
|
Returns:
|
|
Latest version string
|
|
|
|
Raises:
|
|
ValueError: If type not found or no versions available
|
|
"""
|
|
version_tree = self.get_versions(type_name)
|
|
if not version_tree or 'latest_version' not in version_tree:
|
|
raise ValueError(f"No versions found for {type_name}")
|
|
|
|
return version_tree['latest_version']
|
|
|
|
def get_latest_patch_version(self, type_name: str, major_minor: str) -> str:
|
|
"""
|
|
Get the latest patch version for a given major.minor version.
|
|
|
|
Args:
|
|
type_name: Name of the configuration type
|
|
major_minor: Major.minor version (e.g. "1.0")
|
|
|
|
Returns:
|
|
Latest patch version string (e.g. "1.0.3")
|
|
|
|
Raises:
|
|
ValueError: If type not found or no matching versions
|
|
"""
|
|
version_tree = self.get_versions(type_name)
|
|
if not version_tree or 'versions' not in version_tree:
|
|
raise ValueError(f"No versions found for {type_name}")
|
|
|
|
# Filter versions that match the major.minor prefix
|
|
matching_versions = [
|
|
ver for ver in version_tree['versions'].keys()
|
|
if ver.startswith(major_minor + '.')
|
|
]
|
|
|
|
if not matching_versions:
|
|
raise ValueError(f"No versions found for {type_name} with prefix {major_minor}")
|
|
|
|
# Return highest matching version
|
|
latest_patch = max(matching_versions, key=version.parse)
|
|
return latest_patch
|
|
|
|
def get_types(self) -> Dict[str, Dict[str, str]]:
|
|
"""Get dictionary of available types with name and description"""
|
|
self.configure_keys_for_operation('get_types')
|
|
result = self.get(
|
|
lambda type_name: self._load_type_definitions(),
|
|
type_name=f'{self.config_type}_types',
|
|
)
|
|
return result
|
|
|
|
def get_config(self, type_name: str, version: Optional[str] = None) -> Dict[str, Any]:
|
|
"""
|
|
Get configuration for a specific type and version
|
|
If version not specified, returns latest
|
|
|
|
Args:
|
|
type_name: Configuration type name
|
|
version: Optional specific version to retrieve
|
|
|
|
Returns:
|
|
Configuration data
|
|
"""
|
|
self.configure_keys_for_operation('get_config')
|
|
version_str = version or 'latest'
|
|
|
|
return self.get(
|
|
lambda type_name, version: self._load_specific_config(type_name, version),
|
|
type_name=type_name,
|
|
version=version_str
|
|
)
|
|
|
|
|
|
class AgentConfigCacheHandler(BaseConfigCacheHandler):
|
|
"""Handler for agent configurations"""
|
|
handler_name = 'agent_config_cache'
|
|
|
|
def __init__(self, region):
|
|
super().__init__(region, 'agents')
|
|
self._types_module = agent_types.AGENT_TYPES
|
|
self._config_dir = os.path.join('config', 'agents')
|
|
|
|
|
|
class TaskConfigCacheHandler(BaseConfigCacheHandler):
|
|
"""Handler for task configurations"""
|
|
handler_name = 'task_config_cache'
|
|
|
|
def __init__(self, region):
|
|
super().__init__(region, 'tasks')
|
|
self._types_module = task_types.TASK_TYPES
|
|
self._config_dir = os.path.join('config', 'tasks')
|
|
|
|
|
|
class ToolConfigCacheHandler(BaseConfigCacheHandler):
|
|
"""Handler for tool configurations"""
|
|
handler_name = 'tool_config_cache'
|
|
|
|
def __init__(self, region):
|
|
super().__init__(region, 'tools')
|
|
self._types_module = tool_types.TOOL_TYPES
|
|
self._config_dir = os.path.join('config', 'tools')
|
|
|
|
|
|
class SpecialistConfigCacheHandler(BaseConfigCacheHandler):
|
|
"""Handler for specialist configurations"""
|
|
handler_name = 'specialist_config_cache'
|
|
|
|
def __init__(self, region):
|
|
super().__init__(region, 'specialists')
|
|
self._types_module = specialist_types.SPECIALIST_TYPES
|
|
self._config_dir = os.path.join('config', 'specialists')
|