```

feat(feishu): 添加飞书入站事件inbox和混合数据库协调功能

- 实现飞书入站事件持久化inbox机制，支持状态管理、租约锁定和重试退避
- 添加混合数据库基线协调工具，确保平台PostgreSQL结构安全对齐
- 增加运行组件心跳检测和readiness就绪检查机制
- 实现app_ticket事件的安全轮换和验证处理
- 添加生产环境运行编排和fail-closed安全机制
- 支持webhook快速确认和长连接独立进程处理
- 完善个人数据擦除时的待处理事件清理功能
```

app/modules/observability/constants.py

@@ -6,13 +6,16 @@ class ObservabilityKey(StrEnum):
     CHECKS = "checks"
     METRICS = "metrics"
     DATABASE = "database"
+    SCHEMA = "schema"
     REDIS = "redis"
     EVENTS = "events"
     WORKFLOWS = "workflows"
     AI_MEMORY = "ai_memory"
     HEARTBEATS = "heartbeats"
+    API = "api"
     SCHEDULER = "scheduler"
     WORKER = "worker"
+    FEISHU_EVENTS = "feishu_events"
 
 
 class ObservabilityStatus(StrEnum):
@@ -40,7 +43,9 @@ class HeartbeatComponent(StrEnum):
     API = "api"
     SCHEDULER = "scheduler"
     WORKER = "worker"
+    FEISHU_EVENTS = "feishu-events"
 
 
 class HeartbeatStatus(StrEnum):
     OK = "ok"
+    DEGRADED = "degraded"

app/modules/observability/runtime.py

@@ -0,0 +1,73 @@
+import logging
+from os import getpid
+from socket import gethostname
+from threading import Event, Thread
+
+from fastapi import FastAPI
+
+from app.core.config import get_settings
+from app.core.constants import ActorValue
+from app.core.database import SessionLocal
+from app.modules.observability.constants import HeartbeatComponent
+from app.modules.observability.service import ObservabilityService
+
+logger = logging.getLogger(__name__)
+
+
+def attach_api_heartbeat(app: FastAPI) -> None:
+    """Record API process liveness without coupling it to request traffic."""
+
+    interval_seconds = max(1, get_settings().heartbeat_interval_seconds)
+    instance_id = f"{gethostname()}:{getpid()}"
+
+    @app.on_event("startup")
+    def start_api_heartbeat() -> None:
+        current_thread = getattr(app.state, "api_heartbeat_thread", None)
+        if current_thread is not None and current_thread.is_alive():
+            return
+
+        stop_event = Event()
+        _record_api_heartbeat(instance_id)
+        thread = Thread(
+            target=_api_heartbeat_loop,
+            args=(stop_event, instance_id, interval_seconds),
+            name="api-heartbeat",
+            daemon=True,
+        )
+        app.state.api_heartbeat_stop_event = stop_event
+        app.state.api_heartbeat_thread = thread
+        app.state.api_heartbeat_instance_id = instance_id
+        thread.start()
+
+    @app.on_event("shutdown")
+    def stop_api_heartbeat() -> None:
+        stop_event = getattr(app.state, "api_heartbeat_stop_event", None)
+        thread = getattr(app.state, "api_heartbeat_thread", None)
+        if stop_event is not None:
+            stop_event.set()
+        if thread is not None:
+            thread.join(timeout=1)
+
+
+def _api_heartbeat_loop(
+    stop_event: Event,
+    instance_id: str,
+    interval_seconds: int,
+) -> None:
+    while not stop_event.wait(max(1, interval_seconds)):
+        _record_api_heartbeat(instance_id)
+
+
+def _record_api_heartbeat(instance_id: str) -> None:
+    db = SessionLocal()
+    try:
+        ObservabilityService(db).record_heartbeat(
+            component=HeartbeatComponent.API,
+            instance_id=instance_id,
+            actor=ActorValue.API,
+        )
+    except Exception:
+        db.rollback()
+        logger.exception("Failed to record API process heartbeat")
+    finally:
+        db.close()

app/modules/observability/service.py

@@ -9,6 +9,10 @@ from sqlalchemy.orm import Session
 
 from app.core.config import get_settings
 from app.core.constants import ActorValue
+from app.core.database.migrations import (
+    current_alembic_revisions,
+    expected_alembic_heads,
+)
 from app.core.utils.time import utc_now
 from app.modules.ai_memory.service import AIMemoryService
 from app.modules.audit.constants import (
@@ -20,12 +24,18 @@ from app.modules.audit.constants import (
 from app.modules.audit.schemas import AuditLogCreate
 from app.modules.audit.service import AuditService
 from app.modules.events.constants import EventStatus
+from app.modules.events.models import DomainEvent
 from app.modules.events.services import EventService
 from app.modules.feishu.app_tickets import FeishuAppTicketService
 from app.modules.feishu.constants import FeishuAppType
-from app.modules.feishu_users.constants import FeishuUserStatus
+from app.modules.feishu.services import FeishuInboundService
+from app.modules.feishu_users.constants import (
+    FeishuUserStatus,
+    parse_admin_identities,
+)
 from app.modules.feishu_users.models import FeishuUser
 from app.modules.observability.constants import (
+    HeartbeatComponent,
     HeartbeatStatus,
     ObservabilityKey,
     ObservabilityMetricKey,
@@ -53,10 +63,35 @@ class ObservabilityService:
     def ready(self) -> dict[str, Any]:
         checks = {
             ObservabilityKey.DATABASE: self._safe_call(self._database_check),
+            ObservabilityKey.SCHEMA: self._safe_call(self._schema_check),
             ObservabilityKey.REDIS: self._safe_call(self._redis_check),
             ObservabilityKey.EVENTS: self._safe_call(self._events_check),
             ObservabilityKey.WORKFLOWS: self._safe_call(self._workflows_check),
             ObservabilityKey.HEARTBEATS: self._safe_call(self._heartbeats_check),
+            ObservabilityKey.API: self._safe_call(
+                lambda: self._component_heartbeat_check(
+                    HeartbeatComponent.API,
+                    required=self._api_required(),
+                )
+            ),
+            ObservabilityKey.SCHEDULER: self._safe_call(
+                lambda: self._component_heartbeat_check(
+                    HeartbeatComponent.SCHEDULER,
+                    required=self._scheduler_required(),
+                )
+            ),
+            ObservabilityKey.WORKER: self._safe_call(
+                lambda: self._component_heartbeat_check(
+                    HeartbeatComponent.WORKER,
+                    required=get_settings().task_queue_enabled,
+                )
+            ),
+            ObservabilityKey.FEISHU_EVENTS: self._safe_call(
+                lambda: self._component_heartbeat_check(
+                    HeartbeatComponent.FEISHU_EVENTS,
+                    required=self._feishu_events_required(),
+                )
+            ),
             "feishu_subscriptions": self._safe_call(
                 self._feishu_subscriptions_check
             ),
@@ -87,6 +122,7 @@ class ObservabilityService:
                     self.heartbeat_summary
                 ),
                 "feishu_users": self._safe_call(self._feishu_user_metrics),
+                "feishu_inbound": self._safe_call(self._feishu_inbound_metrics),
                 "subscriptions": self._safe_call(self._subscription_metrics),
             }
         }
@@ -158,6 +194,29 @@ class ObservabilityService:
         self.db.execute(text("select 1")).scalar()
         return {ObservabilityKey.STATUS: ObservabilityStatus.OK}
 
+    def _schema_check(self) -> dict[str, Any]:
+        settings = get_settings()
+        required = (
+            settings.app_env.lower() in {"prod", "production"}
+            or settings.scheduler_enabled
+            or settings.feishu_user_features_enabled
+            or settings.feishu_event_transport != "disabled"
+        )
+        if not required:
+            return {ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED}
+
+        expected = expected_alembic_heads()
+        current = current_alembic_revisions(self.db.connection())
+        matches = current == expected and len(expected) == 1
+        return {
+            ObservabilityKey.STATUS: (
+                ObservabilityStatus.OK if matches else ObservabilityStatus.DEGRADED
+            ),
+            "current": [] if current is None else list(current),
+            "expected": list(expected),
+            "reason": None if matches else "schema_revision_mismatch",
+        }
+
     def _redis_check(self) -> dict[str, Any]:
         settings = get_settings()
         if not settings.task_queue_enabled:
@@ -215,22 +274,58 @@ class ObservabilityService:
 
     def _events_check(self) -> dict[str, Any]:
         counts = EventService(self.db).count_by_status()
+        current = utc_now()
         failed = counts.get(EventStatus.FAILED, 0)
+        processable = int(
+            self.db.scalar(
+                select(func.count())
+                .select_from(DomainEvent)
+                .where(
+                    DomainEvent.status == EventStatus.PENDING,
+                    DomainEvent.next_attempt_at.is_not(None),
+                    DomainEvent.next_attempt_at <= current,
+                    or_(
+                        DomainEvent.locked_until.is_(None),
+                        DomainEvent.locked_until <= current,
+                    ),
+                )
+            )
+            or 0
+        )
+        expired_leases = int(
+            self.db.scalar(
+                select(func.count())
+                .select_from(DomainEvent)
+                .where(
+                    DomainEvent.status == EventStatus.PENDING,
+                    DomainEvent.locked_by.is_not(None),
+                    DomainEvent.locked_until.is_not(None),
+                    DomainEvent.locked_until <= current,
+                )
+            )
+            or 0
+        )
+        reasons: list[str] = []
+        if processable and not get_settings().event_dispatch_enabled:
+            reasons.append("dispatch_disabled")
+        if expired_leases:
+            reasons.append("expired_leases")
         return {
             ObservabilityKey.STATUS: (
-                ObservabilityStatus.DEGRADED if failed else ObservabilityStatus.OK
+                ObservabilityStatus.DEGRADED if reasons else ObservabilityStatus.OK
             ),
             ObservabilityMetricKey.PENDING: counts.get(EventStatus.PENDING, 0),
             ObservabilityMetricKey.FAILED: failed,
+            "processable": processable,
+            "expired_leases": expired_leases,
+            "reasons": reasons,
         }
 
     def _workflows_check(self) -> dict[str, Any]:
         counts = WorkflowService(self.db).count_by_status()
         failed = counts.get(WorkflowStatus.FAILED, 0)
         return {
-            ObservabilityKey.STATUS: (
-                ObservabilityStatus.DEGRADED if failed else ObservabilityStatus.OK
-            ),
+            ObservabilityKey.STATUS: ObservabilityStatus.OK,
             ObservabilityMetricKey.RUNNING: counts.get(WorkflowStatus.RUNNING, 0),
             ObservabilityMetricKey.FAILED: failed,
         }
@@ -238,20 +333,86 @@ class ObservabilityService:
     def _heartbeats_check(self) -> dict[str, Any]:
         summary = self.heartbeat_summary()
         total = summary[ObservabilityMetricKey.TOTAL]
+        active = summary[ObservabilityMetricKey.ACTIVE]
         stale = summary[ObservabilityMetricKey.STALE]
         if total == 0:
             return {ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED}
         return {
             ObservabilityKey.STATUS: (
-                ObservabilityStatus.DEGRADED if stale else ObservabilityStatus.OK
+                ObservabilityStatus.OK if active else ObservabilityStatus.DEGRADED
             ),
             ObservabilityMetricKey.TOTAL: total,
+            ObservabilityMetricKey.ACTIVE: active,
             ObservabilityMetricKey.STALE: stale,
             ObservabilityMetricKey.LAST_SEEN_AT: summary[
                 ObservabilityMetricKey.LAST_SEEN_AT
             ],
         }
 
+    def _component_heartbeat_check(
+        self,
+        component: str,
+        *,
+        required: bool,
+    ) -> dict[str, Any]:
+        if not required:
+            return {
+                ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED,
+                ObservabilityMetricKey.COMPONENT: component,
+            }
+        records = list(
+            self.db.execute(
+                select(SystemHeartbeat.status, SystemHeartbeat.last_seen_at).where(
+                    SystemHeartbeat.component == component
+                )
+            ).all()
+        )
+        if not records:
+            return {
+                ObservabilityKey.STATUS: ObservabilityStatus.DEGRADED,
+                ObservabilityMetricKey.COMPONENT: component,
+                "reason": "heartbeat_missing",
+            }
+        last_seen_at = max(item.last_seen_at for item in records)
+        fresh_records = [
+            item
+            for item in records
+            if item.last_seen_at >= self._heartbeat_stale_threshold()
+        ]
+        if not fresh_records:
+            return {
+                ObservabilityKey.STATUS: ObservabilityStatus.DEGRADED,
+                ObservabilityMetricKey.COMPONENT: component,
+                ObservabilityMetricKey.LAST_SEEN_AT: last_seen_at.isoformat(),
+                "reason": "heartbeat_stale",
+            }
+        healthy = any(item.status == HeartbeatStatus.OK for item in fresh_records)
+        return {
+            ObservabilityKey.STATUS: (
+                ObservabilityStatus.OK if healthy else ObservabilityStatus.DEGRADED
+            ),
+            ObservabilityMetricKey.COMPONENT: component,
+            ObservabilityMetricKey.LAST_SEEN_AT: last_seen_at.isoformat(),
+            "reason": None if healthy else "heartbeat_degraded",
+        }
+
+    @staticmethod
+    def _api_required() -> bool:
+        return get_settings().app_env.lower() in {"prod", "production"}
+
+    @staticmethod
+    def _scheduler_required() -> bool:
+        settings = get_settings()
+        return (
+            settings.feishu_user_features_enabled
+            or settings.app_env.lower() in {"prod", "production"}
+        )
+
+    @staticmethod
+    def _feishu_events_required() -> bool:
+        settings = get_settings()
+        return settings.feishu_event_transport == "long_connection"
+
     def _feishu_subscriptions_check(self) -> dict[str, Any]:
         active = int(
             self.db.scalar(
@@ -284,13 +445,17 @@ class ObservabilityService:
             )
             or 0
         )
-        if not active and not processable_deliveries:
+        settings = get_settings()
+        if (
+            not settings.feishu_user_features_enabled
+            and not active
+            and not processable_deliveries
+        ):
             return {
                 ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED,
                 "active": 0,
                 "processable_deliveries": 0,
             }
-        settings = get_settings()
         app_id = str(settings.feishu_app_id or "").strip()
         credentials_configured = bool(app_id and settings.feishu_app_secret)
         active_tenant_count = int(
@@ -316,6 +481,31 @@ class ObservabilityService:
             str(settings.feishu_default_tenant_key or "").strip()
         )
         reasons: list[str] = []
+        if (
+            not settings.feishu_user_features_enabled
+            and (active or processable_deliveries)
+        ):
+            reasons.append("features_disabled")
+        if (
+            settings.app_env.lower() in {"prod", "production"}
+            and settings.feishu_user_features_enabled
+        ):
+            try:
+                configured_admins = parse_admin_identities(
+                    settings.feishu_admin_identities
+                )
+            except ValueError:
+                reasons.append("admin_identities_invalid")
+            else:
+                if not configured_admins:
+                    reasons.append("admin_identities_missing")
+        if settings.feishu_event_transport == "disabled":
+            reasons.append("event_transport_disabled")
+        elif (
+            settings.feishu_event_transport == "webhook"
+            and not settings.feishu_verification_token
+        ):
+            reasons.append("verification_token_missing")
         if not credentials_configured:
             reasons.append("credentials_missing")
         if settings.feishu_app_type == FeishuAppType.STORE:
@@ -341,6 +531,8 @@ class ObservabilityService:
             ),
             "active": active,
             "processable_deliveries": processable_deliveries,
+            "features_enabled": settings.feishu_user_features_enabled,
+            "event_transport": settings.feishu_event_transport,
             "app_type": settings.feishu_app_type,
             "credentials_configured": credentials_configured,
             "ticket_configured": ticket_configured,
@@ -360,6 +552,9 @@ class ObservabilityService:
         )
         return {"active": active}
 
+    def _feishu_inbound_metrics(self) -> dict[str, int]:
+        return FeishuInboundService(self.db).status_counts()
+
     def _subscription_metrics(self) -> dict[str, int]:
         active = int(
             self.db.scalar(