pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Add configuration of agents, tasks, tools, specialist in context of SPIN specialist

Browse Source 
- correct startup of applications using gevent
- introduce startup scripts (eveai_app)
- caching manager for all configurations
This commit is contained in:
Josako
2025-01-16 20:27:00 +01:00
parent f7cd58ed2a
commit 7bddeb0ebd
69 changed files with 1991 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
common/utils/cache/base.py vendored
View File

@@ -1,57 +1,78 @@
# common/utils/cache/base.py
from typing import Any, Dict, List, Optional, TypeVar, Generic, Type
from dataclasses import dataclass
from flask import Flask
from dogpile.cache import CacheRegion
T = TypeVar('T')
T = TypeVar('T') # Generic type parameter for cached data
@dataclass
class CacheKey:
"""Represents a cache key with multiple components"""
"""
Represents a composite cache key made up of multiple components.
Enables structured and consistent key generation for cache entries.
Attributes:
components (Dict[str, Any]): Dictionary of key components and their values
Example:
key = CacheKey({'tenant_id': 123, 'user_id': 456})
str(key) -> "tenant_id=123:user_id=456"
"""
components: Dict[str, Any]
def __str__(self) -> str:
"""
Converts components into a deterministic string representation.
Components are sorted alphabetically to ensure consistent key generation.
"""
return ":".join(f"{k}={v}" for k, v in sorted(self.components.items()))
class CacheInvalidationManager:
"""Manages cache invalidation subscriptions"""
def __init__(self):
self._subscribers = {}
def subscribe(self, model: str, handler: 'CacheHandler', key_fields: List[str]):
if model not in self._subscribers:
self._subscribers[model] = []
self._subscribers[model].append((handler, key_fields))
def notify_change(self, model: str, **identifiers):
if model in self._subscribers:
for handler, key_fields in self._subscribers[model]:
if all(field in identifiers for field in key_fields):
handler.invalidate_by_model(model, **identifiers)
class CacheHandler(Generic[T]):
"""Base cache handler implementation"""
"""
Base cache handler implementation providing structured caching functionality.
Uses generics to ensure type safety of cached data.
Type Parameters:
T: Type of data being cached
Attributes:
region (CacheRegion): Dogpile cache region for storage
prefix (str): Prefix for all cache keys managed by this handler
"""
def __init__(self, region: CacheRegion, prefix: str):
self.region = region
self.prefix = prefix
self._key_components = []
self._key_components = [] # List of required key components
def configure_keys(self, *components: str):
"""
Configure required components for cache key generation.
Args:
*components: Required key component names
Returns:
self for method chaining
"""
self._key_components = components
return self
def subscribe_to_model(self, model: str, key_fields: List[str]):
invalidation_manager.subscribe(model, self, key_fields)
return self
def generate_key(self, **identifiers) -> str:
"""
Generate a cache key from provided identifiers.
Args:
**identifiers: Key-value pairs for key components
Returns:
Formatted cache key string
Raises:
ValueError: If required components are missing
"""
missing = set(self._key_components) - set(identifiers.keys())
if missing:
raise ValueError(f"Missing key components: {missing}")
@@ -60,6 +81,16 @@ class CacheHandler(Generic[T]):
return f"{self.prefix}:{str(key)}"
def get(self, creator_func, **identifiers) -> T:
"""
Get or create a cached value.
Args:
creator_func: Function to create value if not cached
**identifiers: Key components for cache key
Returns:
Cached or newly created value
"""
cache_key = self.generate_key(**identifiers)
def creator():
@@ -75,15 +106,25 @@ class CacheHandler(Generic[T]):
return self.from_cache_data(cached_data, **identifiers)
def invalidate(self, **identifiers):
"""
Invalidate a specific cache entry.
Args:
**identifiers: Key components for the cache entry
"""
cache_key = self.generate_key(**identifiers)
self.region.delete(cache_key)
def invalidate_by_model(self, model: str, **identifiers):
"""
Invalidate cache entry based on model changes.
Args:
model: Changed model name
**identifiers: Model instance identifiers
"""
try:
self.invalidate(**identifiers)
except ValueError:
pass
pass # Skip if cache key can't be generated from provided identifiers
# Create global invalidation manager
invalidation_manager = CacheInvalidationManager()

306
common/utils/cache/config_cache.py vendored Normal file
View File

@@ -0,0 +1,306 @@
from typing import Dict, Any, Optional
from pathlib import Path
import yaml
from packaging import version
import os
from flask import current_app
from common.utils.cache.base import CacheHandler
from config.type_defs import agent_types, task_types, tool_types, specialist_types
class BaseConfigCacheHandler(CacheHandler[Dict[str, Any]]):
"""Base handler for configuration caching"""
def __init__(self, region, config_type: str):
"""
Args:
region: Cache region
config_type: Type of configuration (agents, tasks, etc.)
"""
super().__init__(region, f'config_{config_type}')
self.config_type = config_type
self._types_module = None # Set by subclasses
self._config_dir = None # Set by subclasses
def configure_keys_for_operation(self, operation: str):
"""Configure required keys based on operation"""
match operation:
case 'get_types':
self.configure_keys('type_name') # Only require type_name for type definitions
case 'get_versions':
self.configure_keys('type_name') # Only type_name needed for version tree
case 'get_config':
self.configure_keys('type_name', 'version') # Both needed for specific config
case _:
raise ValueError(f"Unknown operation: {operation}")
def _load_version_tree(self, type_name: str) -> Dict[str, Any]:
"""
Load version tree for a specific type without loading full configurations
Args:
type_name: Name of configuration type
Returns:
Dict containing available versions and their metadata
"""
type_path = Path(self._config_dir) / type_name
if not type_path.exists():
raise ValueError(f"No configuration found for type {type_name}")
version_files = list(type_path.glob('*.yaml'))
if not version_files:
raise ValueError(f"No versions found for type {type_name}")
versions = {}
latest_version = None
latest_version_obj = None
for file_path in version_files:
ver = file_path.stem # Get version from filename
try:
ver_obj = version.parse(ver)
# Only load minimal metadata for version tree
with open(file_path) as f:
yaml_data = yaml.safe_load(f)
metadata = yaml_data.get('metadata', {})
versions[ver] = {
'metadata': metadata,
'file_path': str(file_path)
}
# Track latest version
if latest_version_obj is None or ver_obj > latest_version_obj:
latest_version = ver
latest_version_obj = ver_obj
except Exception as e:
current_app.logger.error(f"Error loading version {ver}: {e}")
continue
current_app.logger.debug(f"Loaded versions for {type_name}: {versions}")
current_app.logger.debug(f"Loaded versions for {type_name}: {latest_version}")
return {
'versions': versions,
'latest_version': latest_version
}
def to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
"""Convert the data to a cacheable format"""
# For configuration data, we can just return the dictionary as is
# since it's already in a serializable format
return instance
def from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
"""Convert cached data back to usable format"""
# Similarly, we can return the data directly since it's already
# in the format we need
return data
def should_cache(self, value: Dict[str, Any]) -> bool:
"""
Validate if the value should be cached
Args:
value: The value to be cached
Returns:
bool: True if the value should be cached
"""
if not isinstance(value, dict):
return False
# For type definitions
if 'name' in value and 'description' in value:
return True
# For configurations
if 'versions' in value and 'latest_version' in value:
return True
return False
def _load_type_definitions(self) -> Dict[str, Dict[str, str]]:
"""Load type definitions from the corresponding type_defs module"""
if not self._types_module:
raise ValueError("_types_module must be set by subclass")
type_definitions = {
type_id: {
'name': info['name'],
'description': info['description']
}
for type_id, info in self._types_module.items()
}
current_app.logger.debug(f"Loaded type definitions: {type_definitions}")
return type_definitions
def _load_specific_config(self, type_name: str, version_str: str) -> Dict[str, Any]:
"""
Load a specific configuration version
Args:
type_name: Type name
version_str: Version string
Returns:
Configuration data
"""
version_tree = self.get_versions(type_name)
versions = version_tree['versions']
if version_str == 'latest':
version_str = version_tree['latest_version']
if version_str not in versions:
raise ValueError(f"Version {version_str} not found for {type_name}")
file_path = versions[version_str]['file_path']
try:
with open(file_path) as f:
config = yaml.safe_load(f)
current_app.logger.debug(f"Loaded config for {type_name}{version_str}: {config}")
return config
except Exception as e:
raise ValueError(f"Error loading config from {file_path}: {e}")
def get_versions(self, type_name: str) -> Dict[str, Any]:
"""
Get version tree for a type
Args:
type_name: Type to get versions for
Returns:
Dict with version information
"""
self.configure_keys_for_operation('get_versions')
return self.get(
lambda type_name: self._load_version_tree(type_name),
type_name=type_name
)
def get_latest_version(self, type_name: str) -> str:
"""
Get the latest version for a given type name.
Args:
type_name: Name of the configuration type
Returns:
Latest version string
Raises:
ValueError: If type not found or no versions available
"""
version_tree = self.get_versions(type_name)
if not version_tree or 'latest_version' not in version_tree:
raise ValueError(f"No versions found for {type_name}")
return version_tree['latest_version']
def get_latest_patch_version(self, type_name: str, major_minor: str) -> str:
"""
Get the latest patch version for a given major.minor version.
Args:
type_name: Name of the configuration type
major_minor: Major.minor version (e.g. "1.0")
Returns:
Latest patch version string (e.g. "1.0.3")
Raises:
ValueError: If type not found or no matching versions
"""
version_tree = self.get_versions(type_name)
if not version_tree or 'versions' not in version_tree:
raise ValueError(f"No versions found for {type_name}")
# Filter versions that match the major.minor prefix
matching_versions = [
ver for ver in version_tree['versions'].keys()
if ver.startswith(major_minor + '.')
]
if not matching_versions:
raise ValueError(f"No versions found for {type_name} with prefix {major_minor}")
# Return highest matching version
latest_patch = max(matching_versions, key=version.parse)
return latest_patch
def get_types(self) -> Dict[str, Dict[str, str]]:
"""Get dictionary of available types with name and description"""
self.configure_keys_for_operation('get_types')
result = self.get(
lambda type_name: self._load_type_definitions(),
type_name=f'{self.config_type}_types',
)
return result
def get_config(self, type_name: str, version: Optional[str] = None) -> Dict[str, Any]:
"""
Get configuration for a specific type and version
If version not specified, returns latest
Args:
type_name: Configuration type name
version: Optional specific version to retrieve
Returns:
Configuration data
"""
self.configure_keys_for_operation('get_config')
version_str = version or 'latest'
return self.get(
lambda type_name, version: self._load_specific_config(type_name, version),
type_name=type_name,
version=version_str
)
class AgentConfigCacheHandler(BaseConfigCacheHandler):
"""Handler for agent configurations"""
handler_name = 'agent_config_cache'
def __init__(self, region):
super().__init__(region, 'agents')
self._types_module = agent_types.AGENT_TYPES
self._config_dir = os.path.join('config', 'agents')
class TaskConfigCacheHandler(BaseConfigCacheHandler):
"""Handler for task configurations"""
handler_name = 'task_config_cache'
def __init__(self, region):
super().__init__(region, 'tasks')
self._types_module = task_types.TASK_TYPES
self._config_dir = os.path.join('config', 'tasks')
class ToolConfigCacheHandler(BaseConfigCacheHandler):
"""Handler for tool configurations"""
handler_name = 'tool_config_cache'
def __init__(self, region):
super().__init__(region, 'tools')
self._types_module = tool_types.TOOL_TYPES
self._config_dir = os.path.join('config', 'tools')
class SpecialistConfigCacheHandler(BaseConfigCacheHandler):
"""Handler for specialist configurations"""
handler_name = 'specialist_config_cache'
def __init__(self, region):
super().__init__(region, 'specialists')
self._types_module = specialist_types.SPECIALIST_TYPES
self._config_dir = os.path.join('config', 'specialists')

34
common/utils/cache/eveai_cache_manager.py vendored
View File

@@ -3,6 +3,8 @@ from typing import Type
from flask import Flask
from common.utils.cache.base import CacheHandler
from common.utils.cache.regions import create_cache_regions
from common.utils.cache.config_cache import AgentConfigCacheHandler
class EveAICacheManager:
@@ -11,29 +13,39 @@ class EveAICacheManager:
def __init__(self):
self._regions = {}
self._handlers = {}
self._handler_instances = {}
def init_app(self, app: Flask):
"""Initialize cache regions"""
from common.utils.cache.regions import create_cache_regions
self._regions = create_cache_regions(app)
# Store regions in instance
for region_name, region in self._regions.items():
setattr(self, f"{region_name}_region", region)
# Initialize all registered handlers with their regions
for handler_class, region_name in self._handlers.items():
region = self._regions[region_name]
handler_instance = handler_class(region)
handler_name = getattr(handler_class, 'handler_name', None)
if handler_name:
app.logger.debug(f"{handler_name} is registered")
setattr(self, handler_name, handler_instance)
app.logger.info('Cache regions initialized: ' + ', '.join(self._regions.keys()))
app.logger.info(f'Cache regions initialized: {self._regions.keys()}')
def register_handler(self, handler_class: Type[CacheHandler], region: str):
"""Register a cache handler class with its region"""
if not hasattr(handler_class, 'handler_name'):
raise ValueError("Cache handler must define handler_name class attribute")
self._handlers[handler_class] = region
# Create handler instance
region_instance = self._regions[region]
handler_instance = handler_class(region_instance)
self._handler_instances[handler_class.handler_name] = handler_instance
def invalidate_region(self, region_name: str):
"""Invalidate an entire cache region"""
if region_name in self._regions:
self._regions[region_name].invalidate()
else:
raise ValueError(f"Unknown cache region: {region_name}")
def __getattr__(self, name):
"""Handle dynamic access to registered handlers"""
instances = object.__getattribute__(self, '_handler_instances')
if name in instances:
return instances[name]
raise AttributeError(f"'EveAICacheManager' object has no attribute '{name}'")

13
common/utils/cache/regions.py vendored
View File

@@ -1,4 +1,5 @@
# common/utils/cache/regions.py
import time
from dogpile.cache import make_region
from urllib.parse import urlparse
@@ -36,6 +37,7 @@ def create_cache_regions(app):
"""Initialize all cache regions with app config"""
redis_config = get_redis_config(app)
regions = {}
startup_time = int(time.time())
# Region for model-related caching (ModelVariables etc)
model_region = make_region(name='eveai_model').configure(
@@ -61,5 +63,16 @@ def create_cache_regions(app):
)
regions['eveai_workers'] = eveai_workers_region
eveai_config_region = make_region(name='eveai_config').configure(
'dogpile.cache.redis',
arguments={
**redis_config,
'redis_expiration_time': None, # No expiration in Redis
'key_mangler': lambda key: f"startup_{startup_time}:{key}" # Prefix all keys
},
replace_existing_backend=True
)
regions['eveai_config'] = eveai_config_region
return regions