```
feat: 添加AI记忆模块和事件调度系统 - 新增AI记忆模块,支持本地记忆召回和自动写入功能 - 实现事件调度系统,支持批量处理待定事件和重试机制 - 集成心跳监控机制,跟踪API、调度器和工作节点状态 - 扩展仪表板数据统计,包含AI记忆条目和心跳概要 - 添加企业运营分析报告功能,提供财务、采购等多维度分析 - 更新配置设置,增加事件调度和AI记忆相关参数 - 优化任务队列,添加事件分发任务类型 - 扩展审计日志,记录AI记忆操作和事件调度行为 - 实现领域事件模型,支持事件持久化和状态管理 - 添加观察性服务,监控系统组件健康状况 ```
This commit is contained in:
@@ -9,6 +9,10 @@ class ObservabilityKey(StrEnum):
|
||||
REDIS = "redis"
|
||||
EVENTS = "events"
|
||||
WORKFLOWS = "workflows"
|
||||
AI_MEMORY = "ai_memory"
|
||||
HEARTBEATS = "heartbeats"
|
||||
SCHEDULER = "scheduler"
|
||||
WORKER = "worker"
|
||||
|
||||
|
||||
class ObservabilityStatus(StrEnum):
|
||||
@@ -20,6 +24,23 @@ class ObservabilityStatus(StrEnum):
|
||||
|
||||
class ObservabilityMetricKey(StrEnum):
|
||||
ERROR = "error"
|
||||
ITEMS = "items"
|
||||
COMPONENT = "component"
|
||||
INSTANCE_ID = "instance_id"
|
||||
PENDING = "pending"
|
||||
FAILED = "failed"
|
||||
RUNNING = "running"
|
||||
TOTAL = "total"
|
||||
ACTIVE = "active"
|
||||
STALE = "stale"
|
||||
LAST_SEEN_AT = "last_seen_at"
|
||||
|
||||
|
||||
class HeartbeatComponent(StrEnum):
|
||||
API = "api"
|
||||
SCHEDULER = "scheduler"
|
||||
WORKER = "worker"
|
||||
|
||||
|
||||
class HeartbeatStatus(StrEnum):
|
||||
OK = "ok"
|
||||
|
||||
23
app/modules/observability/models.py
Normal file
23
app/modules/observability/models.py
Normal file
@@ -0,0 +1,23 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.core.db_base import Base
|
||||
from app.core.time import utc_now
|
||||
|
||||
|
||||
class SystemHeartbeat(Base):
|
||||
__tablename__ = "system_heartbeats"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
component: Mapped[str] = mapped_column(String(128), index=True)
|
||||
instance_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), index=True)
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime, default=utc_now, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=utc_now, index=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime,
|
||||
default=utc_now,
|
||||
onupdate=utc_now,
|
||||
)
|
||||
@@ -1,16 +1,30 @@
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.constants import ActorValue
|
||||
from app.core.time import utc_now
|
||||
from app.modules.ai_memory.service import AIMemoryService
|
||||
from app.modules.audit.constants import (
|
||||
AuditAction,
|
||||
AuditRiskLevel,
|
||||
AuditSource,
|
||||
AuditTargetType,
|
||||
)
|
||||
from app.modules.audit.schemas import AuditLogCreate
|
||||
from app.modules.audit.service import AuditService
|
||||
from app.modules.events.constants import EventStatus
|
||||
from app.modules.events.service import EventService
|
||||
from app.modules.observability.constants import (
|
||||
HeartbeatStatus,
|
||||
ObservabilityKey,
|
||||
ObservabilityMetricKey,
|
||||
ObservabilityStatus,
|
||||
)
|
||||
from app.modules.observability.models import SystemHeartbeat
|
||||
from app.modules.workflows.constants import WorkflowStatus
|
||||
from app.modules.workflows.service import WorkflowService
|
||||
|
||||
@@ -30,6 +44,7 @@ class ObservabilityService:
|
||||
ObservabilityKey.REDIS: self._redis_check(),
|
||||
ObservabilityKey.EVENTS: self._events_check(),
|
||||
ObservabilityKey.WORKFLOWS: self._workflows_check(),
|
||||
ObservabilityKey.HEARTBEATS: self._heartbeats_check(),
|
||||
}
|
||||
degraded = any(
|
||||
item[ObservabilityKey.STATUS]
|
||||
@@ -48,9 +63,75 @@ class ObservabilityService:
|
||||
ObservabilityKey.METRICS: {
|
||||
ObservabilityKey.EVENTS: EventService(self.db).count_by_status(),
|
||||
ObservabilityKey.WORKFLOWS: WorkflowService(self.db).count_by_status(),
|
||||
ObservabilityKey.AI_MEMORY: AIMemoryService(self.db).count_by_status(),
|
||||
ObservabilityKey.HEARTBEATS: self.heartbeat_summary(),
|
||||
}
|
||||
}
|
||||
|
||||
def record_heartbeat(
|
||||
self,
|
||||
component: str,
|
||||
instance_id: str,
|
||||
status_value: str = HeartbeatStatus.OK,
|
||||
actor: str = ActorValue.SYSTEM,
|
||||
) -> dict[str, Any]:
|
||||
now = utc_now()
|
||||
record = self.db.execute(
|
||||
select(SystemHeartbeat).where(
|
||||
SystemHeartbeat.component == component,
|
||||
SystemHeartbeat.instance_id == instance_id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if record is None:
|
||||
record = SystemHeartbeat(
|
||||
component=component,
|
||||
instance_id=instance_id,
|
||||
status=status_value,
|
||||
last_seen_at=now,
|
||||
)
|
||||
self.db.add(record)
|
||||
else:
|
||||
record.status = status_value
|
||||
record.last_seen_at = now
|
||||
record.updated_at = now
|
||||
self.db.commit()
|
||||
self.db.refresh(record)
|
||||
AuditService(self.db).log(
|
||||
AuditLogCreate(
|
||||
actor=actor,
|
||||
source=AuditSource.OBSERVABILITY,
|
||||
action=AuditAction.HEARTBEAT,
|
||||
target_type=AuditTargetType.HEARTBEAT,
|
||||
target_id=f"{component}:{instance_id}",
|
||||
risk_level=AuditRiskLevel.LOW,
|
||||
response_payload={
|
||||
ObservabilityMetricKey.COMPONENT: component,
|
||||
ObservabilityMetricKey.INSTANCE_ID: instance_id,
|
||||
ObservabilityKey.STATUS: status_value,
|
||||
ObservabilityMetricKey.LAST_SEEN_AT: record.last_seen_at.isoformat(),
|
||||
},
|
||||
)
|
||||
)
|
||||
return self._serialize_heartbeat(record)
|
||||
|
||||
def heartbeat_summary(self) -> dict[str, Any]:
|
||||
records = list(self.db.execute(select(SystemHeartbeat)).scalars())
|
||||
threshold = self._heartbeat_stale_threshold()
|
||||
stale = [item for item in records if item.last_seen_at < threshold]
|
||||
active = len(records) - len(stale)
|
||||
last_seen_at = max((item.last_seen_at for item in records), default=None)
|
||||
return {
|
||||
ObservabilityMetricKey.TOTAL: len(records),
|
||||
ObservabilityMetricKey.ACTIVE: active,
|
||||
ObservabilityMetricKey.STALE: len(stale),
|
||||
ObservabilityMetricKey.LAST_SEEN_AT: (
|
||||
last_seen_at.isoformat() if last_seen_at else None
|
||||
),
|
||||
ObservabilityMetricKey.ITEMS: [
|
||||
self._serialize_heartbeat(item) for item in records
|
||||
],
|
||||
}
|
||||
|
||||
def _database_check(self) -> dict[str, Any]:
|
||||
try:
|
||||
self.db.execute(text("select 1")).scalar()
|
||||
@@ -97,3 +178,34 @@ class ObservabilityService:
|
||||
ObservabilityMetricKey.RUNNING: counts.get(WorkflowStatus.RUNNING, 0),
|
||||
ObservabilityMetricKey.FAILED: failed,
|
||||
}
|
||||
|
||||
def _heartbeats_check(self) -> dict[str, Any]:
|
||||
summary = self.heartbeat_summary()
|
||||
total = summary[ObservabilityMetricKey.TOTAL]
|
||||
stale = summary[ObservabilityMetricKey.STALE]
|
||||
if total == 0:
|
||||
return {ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED}
|
||||
return {
|
||||
ObservabilityKey.STATUS: (
|
||||
ObservabilityStatus.DEGRADED if stale else ObservabilityStatus.OK
|
||||
),
|
||||
ObservabilityMetricKey.TOTAL: total,
|
||||
ObservabilityMetricKey.STALE: stale,
|
||||
ObservabilityMetricKey.LAST_SEEN_AT: summary[
|
||||
ObservabilityMetricKey.LAST_SEEN_AT
|
||||
],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _serialize_heartbeat(record: SystemHeartbeat) -> dict[str, Any]:
|
||||
return {
|
||||
ObservabilityMetricKey.COMPONENT: record.component,
|
||||
ObservabilityMetricKey.INSTANCE_ID: record.instance_id,
|
||||
ObservabilityKey.STATUS: record.status,
|
||||
ObservabilityMetricKey.LAST_SEEN_AT: record.last_seen_at.isoformat(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _heartbeat_stale_threshold() -> Any:
|
||||
settings = get_settings()
|
||||
return utc_now() - timedelta(seconds=settings.heartbeat_interval_seconds * 3)
|
||||
|
||||
Reference in New Issue
Block a user