JiuContinent/company-ai-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

```

Browse Source 
feat(feishu): 添加飞书入站事件inbox和混合数据库协调功能

- 实现飞书入站事件持久化inbox机制，支持状态管理、租约锁定和重试退避
- 添加混合数据库基线协调工具，确保平台PostgreSQL结构安全对齐
- 增加运行组件心跳检测和readiness就绪检查机制
- 实现app_ticket事件的安全轮换和验证处理
- 添加生产环境运行编排和fail-closed安全机制
- 支持webhook快速确认和长连接独立进程处理
- 完善个人数据擦除时的待处理事件清理功能
```
This commit is contained in:
JiuContinent
2026-07-27 17:14:37 +08:00
parent d7db84571d
commit eb8267ed18
61 changed files with 8703 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
app/application/feishu/commands.py
View File

@@ -65,19 +65,44 @@ FINANCE_COMMAND_PREFIXES = ("资金需求", "未来30天资金需求", "项目
def _parse_content_text(content: Any) -> str:
"""Extract plain command text from a Feishu message content payload."""
if isinstance(content, dict):
return str(
content.get(FeishuPayloadKey.TEXT) or content.get(FeishuPayloadKey.CONTENT) or ""
)
if not isinstance(content, str):
return ""
return _structured_content_text(content)
try:
data = json.loads(content)
except json.JSONDecodeError:
return content
if isinstance(data, dict):
return str(data.get(FeishuPayloadKey.TEXT) or data.get(FeishuPayloadKey.CONTENT) or "")
return content
return _structured_content_text(data)
def _structured_content_text(value: Any) -> str:
"""Flatten Feishu text and rich-post content without stringifying metadata."""
if isinstance(value, str):
return value
if isinstance(value, list):
separator = "\n" if any(isinstance(item, list) for item in value) else ""
return separator.join(
text
for item in value
if (text := _structured_content_text(item))
)
if not isinstance(value, dict):
return ""
tag = str(value.get("tag") or "").strip().lower()
if tag == "br":
return "\n"
if tag == "at":
# Authorization uses the event's structured mentions, not display text.
return " "
text_value = value.get(FeishuPayloadKey.TEXT)
if isinstance(text_value, str):
return text_value
if FeishuPayloadKey.CONTENT in value:
return _structured_content_text(value.get(FeishuPayloadKey.CONTENT))
title = value.get("title")
return title if isinstance(title, str) else ""
def _clean_command_text(text: str) -> str:

411
app/application/feishu/events.py
View File

@@ -1,8 +1,10 @@
import json
from dataclasses import replace
from dataclasses import dataclass
from hashlib import sha256
from typing import Any
from fastapi import HTTPException, status
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.application.feishu.commands import FeishuCommandService
@@ -19,12 +21,19 @@ from app.modules.feishu.constants import (
FeishuCommandKey,
FeishuEventReceiptKey,
FeishuEventSource,
FeishuInboundStatus,
FeishuPayloadKey,
FeishuResponseKey,
)
from app.modules.feishu.models import FeishuEventReceipt
from app.modules.feishu.service import FeishuService
from app.modules.feishu.services import (
FeishuInboundAcceptance,
FeishuInboundProcessResult,
FeishuInboundService,
bind_inbound_event,
)
from app.modules.feishu_users.principal import FeishuMention, FeishuPrincipal
from app.modules.feishu_users.identifiers import feishu_audit_identity_hash
from app.modules.feishu_users.services import FeishuIdentityService
FEISHU_EVENT_ACTIONS = {
@@ -33,6 +42,16 @@ FEISHU_EVENT_ACTIONS = {
}
@dataclass(frozen=True, slots=True)
class FeishuEventAcceptance:
"""Transport-safe acknowledgement for one verified event."""
response: dict[str, Any]
event_key: str | None = None
created: bool = False
should_dispatch: bool = False
class FeishuEventService:
"""Handle Feishu message events from webhook or long connection."""
@@ -40,6 +59,7 @@ class FeishuEventService:
self.db = db
self.feishu = FeishuService(db)
self.commands = FeishuCommandService(db)
self.inbox = FeishuInboundService(db)
def handle_event(
self,
@@ -56,14 +76,134 @@ class FeishuEventService:
source: str | FeishuEventSource,
auto_reply: bool = True,
) -> dict[str, Any]:
"""Handle an event after an HTTP verifier or the Feishu SDK accepted it."""
"""Synchronously accept and process a verified event for local callers."""
acceptance = self.accept_verified_event(payload, source, auto_reply=auto_reply)
if acceptance.event_key is None:
return acceptance.response
if not acceptance.should_dispatch:
return acceptance.response
outcome = self.process_inbound_event(
acceptance.event_key,
worker_id=f"{source}:inline",
)
if outcome.handler_result is not None:
return outcome.handler_result
return _inbound_response(outcome.record, duplicate=not acceptance.created)
def accept_verified_event(
self,
payload: dict[str, Any],
source: str | FeishuEventSource,
*,
auto_reply: bool = True,
) -> FeishuEventAcceptance:
"""Durably accept a verified event without executing its command."""
challenge = payload.get(FeishuPayloadKey.CHALLENGE)
if challenge:
return {FeishuResponseKey.CHALLENGE: challenge}
return FeishuEventAcceptance(
response={FeishuResponseKey.CHALLENGE: challenge}
)
source_value = _normalize_source(source)
if _event_type(payload) == APP_TICKET_EVENT_TYPE:
return self._handle_app_ticket_event(payload, source_value)
event_type = _event_type(payload)
is_app_ticket = event_type == APP_TICKET_EVENT_TYPE
if is_app_ticket:
self._validate_app_ticket_event(payload)
event_identity = _event_identity(payload, source_value)
acceptance = self.inbox.accept(
payload,
event_identity,
event_type=event_type,
auto_reply=auto_reply,
persist_payload=not is_app_ticket,
)
if acceptance.created:
self._audit_accepted_event(payload, source_value, acceptance)
if is_app_ticket:
if acceptance.record.status == FeishuInboundStatus.SUCCEEDED:
return FeishuEventAcceptance(
response=_inbound_response(acceptance.record, duplicate=True),
event_key=acceptance.record.event_key,
created=False,
should_dispatch=False,
)
outcome = self.inbox.process(
acceptance.record.event_key,
handler_factory=lambda db: (
lambda event_payload, _source, _auto_reply, _event_key: (
FeishuEventService(db)._execute_app_ticket_event(
event_payload
)
)
),
payload_override=payload,
worker_id=f"{source_value}:app-ticket",
)
if outcome.record.status != FeishuInboundStatus.SUCCEEDED:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=(
"Failed to persist verified Feishu app ticket: "
f"{outcome.record.last_error or 'unknown_error'}"
),
)
return FeishuEventAcceptance(
response=outcome.handler_result
or _inbound_response(outcome.record, duplicate=not acceptance.created),
event_key=outcome.record.event_key,
created=acceptance.created,
should_dispatch=False,
)
return FeishuEventAcceptance(
response=_inbound_response(
acceptance.record,
duplicate=not acceptance.created,
),
event_key=acceptance.record.event_key,
created=acceptance.created,
should_dispatch=acceptance.should_dispatch,
)
def process_inbound_event(
self,
event_key: str,
*,
worker_id: str = "feishu-inbound",
) -> FeishuInboundProcessResult:
"""Claim and process one accepted event."""
return self.inbox.process(
event_key,
handler_factory=lambda db: FeishuEventService(
db
)._execute_inbound_event,
worker_id=worker_id,
)
def process_due_inbound_events(
self,
*,
limit: int,
worker_id: str = "feishu-inbound",
) -> list[FeishuInboundProcessResult]:
"""Recover due retries and expired leases from persistent state."""
return self.inbox.process_due(
handler_factory=lambda db: FeishuEventService(
db
)._execute_inbound_event,
limit=limit,
worker_id=worker_id,
)
def _execute_inbound_event(
self,
payload: dict[str, Any],
_source: str,
auto_reply: bool,
event_key: str,
) -> dict[str, Any]:
user_features_enabled = get_settings().feishu_user_features_enabled
command = (
self.commands.extract_event_command(payload)
@@ -75,62 +215,37 @@ class FeishuEventService:
if user_features_enabled and command
else None
)
event_identity = _event_identity(payload, source)
if event_identity and not self._register_event(event_identity):
return {
FeishuResponseKey.OK: True,
FeishuResponseKey.HANDLED: False,
FeishuResponseKey.DUPLICATE: True,
}
self.feishu.audit.log(
AuditLogCreate(
actor=principal.user_code if principal else ActorValue.FEISHU,
source=AuditSource.FEISHU,
action=FEISHU_EVENT_ACTIONS[source_value],
target_type=source_value,
target_id=(
event_identity.get(FeishuEventReceiptKey.EVENT_KEY)
if event_identity
else None
),
request_payload=_audit_event_metadata(
payload,
include_open_id=not user_features_enabled,
include_identity_context=user_features_enabled,
),
response_payload={FeishuResponseKey.ACCEPTED: True},
)
)
if command is None:
command = self.commands.extract_event_command(payload)
if not command:
return {FeishuResponseKey.OK: True, FeishuResponseKey.HANDLED: False}
result = self.commands.handle_text(
command[FeishuCommandKey.TEXT],
chat_id=command[FeishuCommandKey.CHAT_ID],
actor=(
principal.user_code
if principal
else (
ActorValue.FEISHU
if user_features_enabled
else command[FeishuCommandKey.ACTOR]
)
),
auto_reply=auto_reply,
principal=principal,
)
self.commands.feishu.set_message_uuid(_reply_uuid(event_key))
with bind_inbound_event(event_key):
result = self.commands.handle_text(
command[FeishuCommandKey.TEXT],
chat_id=command[FeishuCommandKey.CHAT_ID],
actor=(
principal.user_code
if principal
else (
ActorValue.FEISHU
if user_features_enabled
else command[FeishuCommandKey.ACTOR]
)
),
auto_reply=auto_reply,
principal=principal,
)
return {
FeishuResponseKey.OK: True,
FeishuResponseKey.HANDLED: True,
FeishuResponseKey.RESULT: result,
}
def _handle_app_ticket_event(
def _validate_app_ticket_event(
self,
payload: dict[str, Any],
source: FeishuEventSource,
) -> dict[str, Any]:
) -> tuple[str, str]:
settings = get_settings()
configured_app_id = str(settings.feishu_app_id or "").strip()
if not configured_app_id:
@@ -150,37 +265,47 @@ class FeishuEventService:
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Feishu app ticket app_id does not match configured application",
)
return app_id, ticket
event_identity = _event_identity(payload, source)
if event_identity is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Feishu app ticket event identity is required",
)
if not self._register_event(event_identity):
return {
FeishuResponseKey.OK: True,
FeishuResponseKey.HANDLED: False,
FeishuResponseKey.DUPLICATE: True,
}
def _execute_app_ticket_event(
self,
payload: dict[str, Any],
) -> dict[str, Any]:
app_id, ticket = self._validate_app_ticket_event(payload)
FeishuAppTicketService(self.db).store_verified(app_id, ticket)
self.feishu.audit.log(
AuditLogCreate(
actor=ActorValue.FEISHU,
source=AuditSource.FEISHU,
action=FEISHU_EVENT_ACTIONS[source],
target_type=source,
target_id=str(event_identity[FeishuEventReceiptKey.EVENT_KEY]),
request_payload=_audit_event_metadata(payload),
response_payload={FeishuResponseKey.ACCEPTED: True},
)
)
return {
FeishuResponseKey.OK: True,
FeishuResponseKey.HANDLED: True,
}
def _audit_accepted_event(
self,
payload: dict[str, Any],
source: FeishuEventSource,
acceptance: FeishuInboundAcceptance,
) -> None:
user_features_enabled = get_settings().feishu_user_features_enabled
audit_actor = (
_audit_identity_actor(payload)
if user_features_enabled
else None
)
self.feishu.audit.log(
AuditLogCreate(
actor=audit_actor or ActorValue.FEISHU,
source=AuditSource.FEISHU,
action=FEISHU_EVENT_ACTIONS[source],
target_type=source,
target_id=acceptance.record.event_key,
request_payload=_audit_event_metadata(
payload,
include_open_id=not user_features_enabled,
include_identity_context=user_features_enabled,
),
response_payload={FeishuResponseKey.ACCEPTED: True},
)
)
def _resolve_principal(
self,
payload: dict[str, Any],
@@ -217,22 +342,6 @@ class FeishuEventService:
mentions=mentions,
)
def _register_event(self, event_identity: dict[str, str | None]) -> bool:
receipt = FeishuEventReceipt(
event_key=str(event_identity[FeishuEventReceiptKey.EVENT_KEY]),
source=str(event_identity[FeishuEventReceiptKey.SOURCE]),
event_id=event_identity.get(FeishuEventReceiptKey.EVENT_ID),
message_id=event_identity.get(FeishuEventReceiptKey.MESSAGE_ID),
)
try:
with self.db.begin_nested():
self.db.add(receipt)
self.db.flush()
except IntegrityError:
return False
return True
def _audit_event_metadata(
payload: dict[str, Any],
*,
@@ -248,20 +357,35 @@ def _audit_event_metadata(
sender_id = sender.get(FeishuPayloadKey.SENDER_ID) or {}
metadata = {
"schema": payload.get("schema"),
FeishuPayloadKey.EVENT_ID: header.get(FeishuPayloadKey.EVENT_ID),
FeishuPayloadKey.EVENT_ID: _identifier_digest(
header.get(FeishuPayloadKey.EVENT_ID),
"event-id",
),
FeishuPayloadKey.EVENT_TYPE: _event_type(payload),
FeishuPayloadKey.MESSAGE_ID: message.get(FeishuPayloadKey.MESSAGE_ID),
FeishuCommandKey.CHAT_ID: message.get(FeishuCommandKey.CHAT_ID),
FeishuPayloadKey.MESSAGE_ID: _identifier_digest(
message.get(FeishuPayloadKey.MESSAGE_ID),
"message-id",
),
FeishuCommandKey.CHAT_ID: _identifier_digest(
message.get(FeishuCommandKey.CHAT_ID),
"chat-id",
),
FeishuPayloadKey.MESSAGE_TYPE: message.get(FeishuPayloadKey.MESSAGE_TYPE),
}
app_id, _ = _app_ticket_fields(payload)
if app_id:
metadata[FeishuPayloadKey.APP_ID] = app_id
metadata[FeishuPayloadKey.APP_ID] = _identifier_digest(app_id, "app-id")
if include_identity_context:
metadata[FeishuPayloadKey.TENANT_KEY] = header.get(FeishuPayloadKey.TENANT_KEY)
metadata[FeishuPayloadKey.TENANT_KEY] = _identifier_digest(
header.get(FeishuPayloadKey.TENANT_KEY),
"tenant-key",
)
metadata[FeishuCommandKey.CHAT_TYPE] = message.get(FeishuPayloadKey.CHAT_TYPE)
if include_open_id:
metadata[FeishuPayloadKey.OPEN_ID] = sender_id.get(FeishuPayloadKey.OPEN_ID)
metadata[FeishuPayloadKey.OPEN_ID] = _identifier_digest(
sender_id.get(FeishuPayloadKey.OPEN_ID),
"open-id",
)
return metadata
@@ -269,10 +393,22 @@ def _normalize_source(source: str | FeishuEventSource) -> FeishuEventSource:
return FeishuEventSource(source)
def _audit_identity_actor(payload: dict[str, Any]) -> str | None:
header = payload.get(FeishuPayloadKey.HEADER) or {}
event = payload.get(FeishuPayloadKey.EVENT) or {}
sender = event.get(FeishuPayloadKey.SENDER) or {}
sender_id = sender.get(FeishuPayloadKey.SENDER_ID) or {}
tenant_key = str(header.get(FeishuPayloadKey.TENANT_KEY) or "").strip()
open_id = str(sender_id.get(FeishuPayloadKey.OPEN_ID) or "").strip()
if not tenant_key or not open_id:
return None
return feishu_audit_identity_hash(tenant_key, open_id)
def _event_identity(
payload: dict[str, Any],
source: str | FeishuEventSource,
) -> dict[str, str | None] | None:
) -> dict[str, str | None]:
source_value = _normalize_source(source)
header = payload.get(FeishuPayloadKey.HEADER) or {}
event = payload.get(FeishuPayloadKey.EVENT) or {}
@@ -284,23 +420,92 @@ def _event_identity(
or event.get(FeishuPayloadKey.UUID)
)
message_id = message.get(FeishuPayloadKey.MESSAGE_ID)
stable_id = event_id or message_id
if not stable_id:
return None
stable_id = event_id or message_id or _payload_fingerprint(payload)
event_type = _event_type(payload)
app_id, _ = _app_ticket_fields(payload)
tenant_key = header.get(FeishuPayloadKey.TENANT_KEY) or app_id or "unknown-tenant"
event_key = ":".join(
raw_event_key = ":".join(
str(part) for part in (tenant_key, event_type or FeishuPayloadKey.EVENT, stable_id)
)
event_key = _identifier_digest(raw_event_key, "event-key")
return {
FeishuEventReceiptKey.EVENT_KEY: event_key,
FeishuEventReceiptKey.SOURCE: source_value,
FeishuEventReceiptKey.EVENT_ID: str(event_id) if event_id else None,
FeishuEventReceiptKey.MESSAGE_ID: str(message_id) if message_id else None,
FeishuEventReceiptKey.EVENT_ID: _identifier_digest(event_id, "event-id"),
FeishuEventReceiptKey.MESSAGE_ID: _identifier_digest(
message_id,
"message-id",
),
}
def _payload_fingerprint(payload: dict[str, Any]) -> str:
scrubbed = _scrub_transport_secrets(payload)
serialized = json.dumps(
scrubbed,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
default=str,
)
return f"sha256-{sha256(serialized.encode('utf-8')).hexdigest()}"
def _scrub_transport_secrets(value: Any) -> Any:
if isinstance(value, dict):
return {
str(key): _scrub_transport_secrets(item)
for key, item in value.items()
if str(key).casefold()
not in {
"access_token",
"app_access_token",
"app_secret",
"app_ticket",
"authorization",
"encrypt",
"refresh_token",
"secret",
"tenant_access_token",
"token",
}
}
if isinstance(value, list):
return [_scrub_transport_secrets(item) for item in value]
return value
def _reply_uuid(event_key: str) -> str:
digest = sha256(event_key.encode("utf-8")).hexdigest()[:32]
return f"inbound-{digest}"
def _identifier_digest(value: Any, domain: str) -> str | None:
text = str(value or "").strip()
if not text:
return None
digest = sha256(
f"company-ai-platform:feishu:{domain}:v1\0{text}".encode("utf-8")
).hexdigest()
return f"sha256-{digest}"
def _inbound_response(
record: Any,
*,
duplicate: bool,
) -> dict[str, Any]:
response = {
FeishuResponseKey.OK: True,
FeishuResponseKey.ACCEPTED: True,
FeishuResponseKey.HANDLED: record.status == FeishuInboundStatus.SUCCEEDED,
FeishuResponseKey.STATUS: record.status,
}
if duplicate:
response[FeishuResponseKey.DUPLICATE] = True
return response
def _event_type(payload: dict[str, Any]) -> str:
header = payload.get(FeishuPayloadKey.HEADER) or {}
return str(

52
app/application/feishu/inbound.py Normal file
View File

@@ -0,0 +1,52 @@
from socket import gethostname
from typing import Any
from app.application.feishu.events import FeishuEventService
from app.core.constants import ActorValue
from app.core.database import SessionLocal
from app.modules.feishu.constants import FEISHU_INBOUND_BATCH_SIZE
def process_feishu_inbound_event(
event_key: str,
*,
actor: str = ActorValue.WORKER,
) -> dict[str, Any]:
"""Process one durable Feishu inbox row in an isolated session."""
db = SessionLocal()
try:
outcome = FeishuEventService(db).process_inbound_event(
event_key,
worker_id=f"{actor}:{gethostname()}",
)
return _serialize_outcome(outcome)
finally:
db.close()
def process_due_feishu_inbound_events(
*,
limit: int = FEISHU_INBOUND_BATCH_SIZE,
actor: str = ActorValue.WORKER,
) -> list[dict[str, Any]]:
"""Recover due retries and expired Feishu inbox leases."""
db = SessionLocal()
try:
outcomes = FeishuEventService(db).process_due_inbound_events(
limit=limit,
worker_id=f"{actor}:{gethostname()}",
)
return [_serialize_outcome(outcome) for outcome in outcomes]
finally:
db.close()
def _serialize_outcome(outcome: Any) -> dict[str, Any]:
return {
"event_key": outcome.record.event_key,
"status": outcome.record.status,
"attempt_count": outcome.record.attempt_count,
"handled": outcome.handler_result is not None,
}

26
app/application/feishu/personal_data.py
View File

@@ -16,7 +16,12 @@ from app.modules.feishu_users.constants import (
FeishuUserStatus,
parse_admin_identities,
)
from app.modules.feishu.services import (
FeishuInboundService,
current_inbound_event_key,
)
from app.modules.feishu_users.bootstrap import admin_bootstrap_identity_hash
from app.modules.feishu_users.identifiers import feishu_audit_identity_hash
from app.modules.feishu_users.models import (
FeishuAdminBootstrapTombstone,
FeishuUser,
@@ -187,6 +192,10 @@ class FeishuPersonalDataService:
user.open_id,
user.union_id,
user.user_id,
feishu_audit_identity_hash(
user.tenant_key,
user.open_id,
),
)
if value
},
@@ -230,6 +239,21 @@ class FeishuPersonalDataService:
"subscriptions": subscriptions,
}
def clear_pending_inbound_events(
db: Session,
_owner_id: int,
_anonymous_id: str,
) -> dict[str, int]:
# The caller already holds the FeishuUser row. Inbound handlers
# acquire that same identity fence before their own receipt row, so
# erasure can safely fence related receipts in identity -> inbox order.
cleared = FeishuInboundService(db).erase_identity_payloads(
tenant_key=user.tenant_key,
open_id=user.open_id,
exclude_event_key=current_inbound_event_key(),
)
return {"inbound_events": cleared}
def finalize_identity(
db: Session,
_owner_id: int,
@@ -281,7 +305,7 @@ class FeishuPersonalDataService:
return self.erasure.confirm_and_erase(
user.id,
confirmation_code,
before_hooks=(delete_subscriptions,),
before_hooks=(delete_subscriptions, clear_pending_inbound_events),
extra_hooks=(*self.extra_hooks, finalize_identity),
)

30
app/application/scheduling/scheduler.py
View File

@@ -1,5 +1,5 @@
from collections.abc import Callable
from datetime import date
from datetime import UTC, date, datetime
from socket import gethostname
from typing import Any
@@ -40,6 +40,7 @@ def create_scheduler(app: FastAPI | None = None) -> Any:
enqueue_attendance_summary_push,
enqueue_daily_brief_push,
enqueue_event_dispatch,
enqueue_feishu_inbound_cycle,
enqueue_legacy_project_sync,
enqueue_legacy_task_sync,
enqueue_lifecycle_report,
@@ -47,6 +48,7 @@ def create_scheduler(app: FastAPI | None = None) -> Any:
enqueue_project_weekly_push,
enqueue_risk_progress_push,
enqueue_subscription_cycle,
enqueue_worker_heartbeat,
enqueue_work_daily_push,
enqueue_work_weekly_push,
)
@@ -186,6 +188,10 @@ def create_scheduler(app: FastAPI | None = None) -> Any:
)
_set_state(app, "last_event_dispatch", dispatch)
def run_feishu_inbound_cycle() -> None:
dispatch = enqueue_feishu_inbound_cycle(actor=ActorValue.SCHEDULER)
_set_state(app, "last_feishu_inbound_cycle", dispatch)
def record_scheduler_heartbeat() -> None:
db = SessionLocal()
try:
@@ -202,6 +208,10 @@ def create_scheduler(app: FastAPI | None = None) -> Any:
dispatch = enqueue_subscription_cycle(actor=ActorValue.SCHEDULER)
_set_state(app, "last_subscription_cycle", dispatch)
def run_worker_heartbeat() -> None:
dispatch = enqueue_worker_heartbeat(actor=ActorValue.SCHEDULER)
_set_state(app, "last_worker_heartbeat", dispatch)
def run_personalization_retention_cleanup() -> None:
db = SessionLocal()
try:
@@ -285,6 +295,7 @@ def create_scheduler(app: FastAPI | None = None) -> Any:
trigger="cron",
minute=settings.event_dispatch_cron_minute,
id="event_dispatch",
next_run_time=datetime.now(UTC),
replace_existing=True,
)
if settings.market_analysis_enabled:
@@ -320,6 +331,23 @@ def create_scheduler(app: FastAPI | None = None) -> Any:
trigger="interval",
seconds=settings.heartbeat_interval_seconds,
id="scheduler_heartbeat",
next_run_time=datetime.now(UTC),
replace_existing=True,
)
if settings.task_queue_enabled:
scheduler.add_job(
run_worker_heartbeat,
trigger="interval",
seconds=settings.heartbeat_interval_seconds,
id="worker_heartbeat",
next_run_time=datetime.now(UTC),
replace_existing=True,
)
scheduler.add_job(
run_feishu_inbound_cycle,
trigger="interval",
minutes=1,
id="feishu_inbound_event_cycle",
replace_existing=True,
)
if settings.feishu_user_features_enabled:

14
app/core/background/task_queue/__init__.py
View File

@@ -1,6 +1,8 @@
from app.tasks.constants import (
TASK_DISPATCH_PENDING_EVENTS,
TASK_GENERATE_RISK_EVENTS,
TASK_PROCESS_DUE_FEISHU_INBOUND,
TASK_PROCESS_FEISHU_INBOUND,
TASK_PUSH_ATTENDANCE_SUMMARY,
TASK_PUSH_DAILY_BRIEF,
TASK_PUSH_PROJECT_WEEKLY,
@@ -13,15 +15,21 @@ from app.tasks.constants import (
TASK_RUN_MARKET_CLOSE,
TASK_RUN_MARKET_REPORT,
TASK_RUN_SUBSCRIPTION_CYCLE,
TASK_RECORD_WORKER_HEARTBEAT,
)
from app.core.background.task_queue.dispatcher import dispatch_task
from app.core.background.task_queue.events import enqueue_event_dispatch
from app.core.background.task_queue.feishu import (
enqueue_feishu_inbound_cycle,
enqueue_feishu_inbound_event,
)
from app.core.background.task_queue.legacy import (
enqueue_legacy_project_sync,
enqueue_legacy_task_sync,
)
from app.core.background.task_queue.lifecycle import enqueue_lifecycle_report
from app.core.background.task_queue.market import enqueue_market_close, enqueue_market_report
from app.core.background.task_queue.observability import enqueue_worker_heartbeat
from app.core.background.task_queue.reports import (
enqueue_attendance_summary_push,
enqueue_daily_brief_push,
@@ -37,6 +45,8 @@ from app.core.background.task_queue.subscriptions import enqueue_subscription_cy
__all__ = [
"TASK_DISPATCH_PENDING_EVENTS",
"TASK_GENERATE_RISK_EVENTS",
"TASK_PROCESS_DUE_FEISHU_INBOUND",
"TASK_PROCESS_FEISHU_INBOUND",
"TASK_PUSH_ATTENDANCE_SUMMARY",
"TASK_PUSH_DAILY_BRIEF",
"TASK_PUSH_PROJECT_WEEKLY",
@@ -49,15 +59,19 @@ __all__ = [
"TASK_RUN_MARKET_CLOSE",
"TASK_RUN_MARKET_REPORT",
"TASK_RUN_SUBSCRIPTION_CYCLE",
"TASK_RECORD_WORKER_HEARTBEAT",
"dispatch_task",
"enqueue_attendance_summary_push",
"enqueue_daily_brief_push",
"enqueue_event_dispatch",
"enqueue_feishu_inbound_cycle",
"enqueue_feishu_inbound_event",
"enqueue_legacy_project_sync",
"enqueue_legacy_task_sync",
"enqueue_lifecycle_report",
"enqueue_market_close",
"enqueue_market_report",
"enqueue_worker_heartbeat",
"enqueue_project_weekly_push",
"enqueue_risk_progress_push",
"enqueue_risk_event_generation",

37
app/core/background/task_queue/feishu.py Normal file
View File

@@ -0,0 +1,37 @@
from typing import Any
from app.application.feishu.inbound import (
process_due_feishu_inbound_events,
process_feishu_inbound_event,
)
from app.core.background.task_queue.dispatcher import dispatch_task
from app.core.constants import ActorValue
from app.modules.feishu.constants import FEISHU_INBOUND_BATCH_SIZE
from app.tasks.constants import (
TASK_PROCESS_DUE_FEISHU_INBOUND,
TASK_PROCESS_FEISHU_INBOUND,
)
def enqueue_feishu_inbound_event(
event_key: str,
*,
actor: str = ActorValue.WORKER,
) -> dict[str, Any]:
return dispatch_task(
TASK_PROCESS_FEISHU_INBOUND,
{"event_key": event_key, "actor": actor},
lambda: process_feishu_inbound_event(event_key, actor=actor),
)
def enqueue_feishu_inbound_cycle(
*,
limit: int = FEISHU_INBOUND_BATCH_SIZE,
actor: str = ActorValue.SCHEDULER,
) -> dict[str, Any]:
return dispatch_task(
TASK_PROCESS_DUE_FEISHU_INBOUND,
{"limit": limit, "actor": actor},
lambda: process_due_feishu_inbound_events(limit=limit, actor=actor),
)

32
app/core/background/task_queue/observability.py Normal file
View File

@@ -0,0 +1,32 @@
from socket import gethostname
from app.core.background.task_queue.dispatcher import dispatch_task
from app.core.constants import ActorValue
from app.core.database import SessionLocal
from app.modules.observability.constants import HeartbeatComponent
from app.modules.observability.service import ObservabilityService
from app.tasks.constants import TASK_RECORD_WORKER_HEARTBEAT
def enqueue_worker_heartbeat(
actor: str = ActorValue.SCHEDULER,
) -> dict:
"""Ask a Celery worker to prove it can consume tasks."""
return dispatch_task(
TASK_RECORD_WORKER_HEARTBEAT,
{"actor": ActorValue.WORKER},
lambda: _record_inline_heartbeat(actor),
)
def _record_inline_heartbeat(actor: str) -> dict:
db = SessionLocal()
try:
return ObservabilityService(db).record_heartbeat(
component=HeartbeatComponent.WORKER,
instance_id=f"inline:{gethostname()}",
actor=actor,
)
finally:
db.close()

120
app/core/config/settings.py
View File

@@ -3,7 +3,7 @@ import os
from functools import lru_cache
from typing import Annotated, Any, Literal
from pydantic import Field, field_validator, model_validator
from pydantic import Field, ValidationInfo, field_validator, model_validator
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
from app.core.constants import (
@@ -12,6 +12,7 @@ from app.core.constants import (
DEFAULT_MODEL_PROVIDER,
DEFAULT_OPENCLAW_ACTION_JSON,
)
from app.core.database.safety import database_password, database_target
_DOTENV_DISABLED = os.getenv("COMPANY_AI_DISABLE_DOTENV", "").strip().lower() in {
"1",
@@ -19,15 +20,43 @@ _DOTENV_DISABLED = os.getenv("COMPANY_AI_DISABLE_DOTENV", "").strip().lower() in
"yes",
"on",
}
_DOTENV_FILE = None if _DOTENV_DISABLED else ".env"
_PRODUCTION_SECRET_MIN_LENGTH = 24
_DATABASE_PASSWORD_MIN_LENGTH = 10
_PLACEHOLDER_SECRET_PARTS = (
"change-me",
"changeme",
"example",
"placeholder",
"replace-with",
"your-",
)
def _is_unsafe_production_secret(value: str) -> bool:
normalized = value.strip().casefold()
return (
len(normalized) < _PRODUCTION_SECRET_MIN_LENGTH
or any(part in normalized for part in _PLACEHOLDER_SECRET_PARTS)
)
def _is_unsafe_database_password(value: str) -> bool:
normalized = value.strip().casefold()
return (
len(normalized) < _DATABASE_PASSWORD_MIN_LENGTH
or any(part in normalized for part in _PLACEHOLDER_SECRET_PARTS)
)
class Settings(BaseSettings):
"""Runtime settings loaded from environment variables and `.env`."""
"""Runtime settings loaded from the environment and optional `.env` file."""
model_config = SettingsConfigDict(
env_file=None if _DOTENV_DISABLED else ".env",
env_file=_DOTENV_FILE,
env_file_encoding="utf-8",
extra="ignore",
hide_input_in_errors=True,
)
app_name: str = "Company AI Management Platform"
@@ -63,6 +92,11 @@ class Settings(BaseSettings):
feishu_encrypt_key: str | None = None
feishu_default_chat_id: str | None = None
feishu_default_tenant_key: str | None = None
feishu_event_transport: Literal[
"disabled",
"webhook",
"long_connection",
] = "disabled"
feishu_user_features_enabled: bool = False
feishu_admin_identities: Annotated[list[str], NoDecode] = Field(default_factory=list)
model_provider: str = DEFAULT_MODEL_PROVIDER
@@ -189,11 +223,13 @@ class Settings(BaseSettings):
@field_validator(
"feishu_app_type",
"feishu_event_transport",
mode="before",
)
@classmethod
def normalize_feishu_app_type(cls, value: Any) -> str:
return str(value or "self").strip().lower()
def normalize_feishu_choice(cls, value: Any, info: ValidationInfo) -> str:
default = "self" if info.field_name == "feishu_app_type" else "disabled"
return str(value or default).strip().lower()
@field_validator(
"openclaw_allowed_tools",
@@ -291,12 +327,34 @@ class Settings(BaseSettings):
errors: list[str] = []
api_key_values = _enabled_keys(self.api_key, self.api_keys)
audit_key_values = _enabled_keys(self.audit_api_key, self.audit_api_keys)
if self.database_url.startswith("sqlite"):
platform_target = database_target(self.database_url)
legacy_target = database_target(self.legacy_database_url)
if platform_target is None or platform_target[0] != "postgresql":
errors.append("DATABASE_URL must use PostgreSQL in production")
platform_password = database_password(self.database_url)
if platform_password and _is_unsafe_database_password(platform_password):
errors.append(
"DATABASE_URL password must use a non-placeholder value of at least "
f"{_DATABASE_PASSWORD_MIN_LENGTH} characters in production"
)
if platform_target is not None and platform_target == legacy_target:
errors.append(
"DATABASE_URL and LEGACY_DATABASE_URL must target different databases"
)
if not api_key_values:
errors.append("API_KEY or API_KEYS is required in production")
elif any(_is_unsafe_production_secret(value) for value in api_key_values):
errors.append(
"API_KEY/API_KEYS must use non-placeholder values of at least "
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
)
if not audit_key_values:
errors.append("AUDIT_API_KEY or AUDIT_API_KEYS is required in production")
elif any(_is_unsafe_production_secret(value) for value in audit_key_values):
errors.append(
"AUDIT_API_KEY/AUDIT_API_KEYS must use non-placeholder values of at least "
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
)
if api_key_values & audit_key_values:
errors.append(
"API_KEY/API_KEYS and AUDIT_API_KEY/AUDIT_API_KEYS cannot overlap in production"
@@ -309,10 +367,52 @@ class Settings(BaseSettings):
errors.append("MASK_SENSITIVE_RESPONSES must be true in production")
if not self.read_only_mode:
errors.append("READ_ONLY_MODE must be true in production")
if self.feishu_user_features_enabled and not self.feishu_admin_identities:
errors.append(
"FEISHU_ADMIN_IDENTITIES is required when Feishu user features are enabled"
)
if self.feishu_event_transport != "disabled":
if not self.feishu_app_id or not self.feishu_app_secret:
errors.append(
"FEISHU_APP_ID and FEISHU_APP_SECRET are required when "
"Feishu event transport is enabled"
)
elif _is_unsafe_production_secret(self.feishu_app_secret):
errors.append(
"FEISHU_APP_SECRET must use a non-placeholder value of at least "
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
)
if (
self.feishu_event_transport == "webhook"
and not self.feishu_verification_token
):
errors.append(
"FEISHU_VERIFICATION_TOKEN is required for Feishu webhook transport"
)
elif (
self.feishu_event_transport == "webhook"
and self.feishu_verification_token
and _is_unsafe_production_secret(self.feishu_verification_token)
):
errors.append(
"FEISHU_VERIFICATION_TOKEN must use a non-placeholder value of at "
f"least {_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
)
if self.feishu_user_features_enabled:
if not self.feishu_admin_identities:
errors.append(
"FEISHU_ADMIN_IDENTITIES is required when Feishu user features are enabled"
)
else:
try:
from app.modules.feishu_users.constants import (
parse_admin_identities,
)
parse_admin_identities(self.feishu_admin_identities)
except ValueError as exc:
errors.append(str(exc))
if self.feishu_event_transport == "disabled":
errors.append(
"FEISHU_EVENT_TRANSPORT must be webhook or long_connection "
"when Feishu user features are enabled"
)
if errors:
raise ValueError("; ".join(errors))
return self

26
app/core/database/migrations.py Normal file
View File

@@ -0,0 +1,26 @@
from pathlib import Path
from alembic.config import Config
from alembic.script import ScriptDirectory
from sqlalchemy import inspect, text
from sqlalchemy.engine import Connection
def expected_alembic_heads(project_root: Path | None = None) -> tuple[str, ...]:
"""Return the repository's configured Alembic heads."""
root = project_root or Path(__file__).resolve().parents[3]
config = Config(str(root / "alembic.ini"))
config.set_main_option("script_location", str(root / "alembic"))
return tuple(sorted(ScriptDirectory.from_config(config).get_heads()))
def current_alembic_revisions(connection: Connection) -> tuple[str, ...] | None:
"""Return database revisions, or ``None`` when it was never versioned."""
if "alembic_version" not in inspect(connection).get_table_names():
return None
revisions = connection.execute(
text("SELECT version_num FROM alembic_version ORDER BY version_num")
).scalars()
return tuple(str(revision) for revision in revisions)

62
app/core/database/safety.py Normal file
View File

@@ -0,0 +1,62 @@
from sqlalchemy.engine import URL, make_url
from sqlalchemy.exc import ArgumentError
_DEFAULT_DATABASE_PORTS = {
"mysql": 3306,
"postgresql": 5432,
}
_PLATFORM_MIGRATION_BACKENDS = frozenset({"postgresql", "sqlite"})
DatabaseTarget = tuple[str, str, int | None, str]
def database_target(value: str | URL | None) -> DatabaseTarget | None:
"""Return a credential-free physical database identity."""
if not value:
return None
try:
url = make_url(value)
except (ArgumentError, TypeError, ValueError):
return None
backend = url.get_backend_name().lower()
return (
backend,
str(url.host or "").casefold(),
url.port or _DEFAULT_DATABASE_PORTS.get(backend),
str(url.database or ""),
)
def database_password(value: str | URL | None) -> str | None:
"""Return a configured password without including it in diagnostics."""
if not value:
return None
try:
password = make_url(value).password
except (ArgumentError, TypeError, ValueError):
return None
return str(password) if password is not None else None
def validate_platform_migration_target(
database_url: str | URL,
legacy_database_url: str | None,
) -> DatabaseTarget:
"""Fail closed before Alembic can connect to an unsafe database target."""
platform = database_target(database_url)
if platform is None:
raise RuntimeError("DATABASE_URL is not a valid platform database URL")
if platform[0] not in _PLATFORM_MIGRATION_BACKENDS:
raise RuntimeError(
"Platform migrations are supported only for PostgreSQL or local SQLite"
)
legacy = database_target(legacy_database_url)
if legacy is not None and platform == legacy:
raise RuntimeError(
"Refusing to migrate DATABASE_URL because it matches LEGACY_DATABASE_URL"
)
return platform

2
app/main.py
View File

@@ -6,6 +6,7 @@ from app.core.config import get_settings
from app.core.http.middleware import request_id_middleware
from app.core.http.responses import MaskedJSONResponse
from app.application.scheduling import attach_scheduler
from app.modules.observability.runtime import attach_api_heartbeat
def _allow_cors_credentials(cors_origins: list[str]) -> bool:
@@ -37,6 +38,7 @@ def create_app() -> FastAPI:
)
app.include_router(api_router, prefix=settings.api_prefix)
attach_api_heartbeat(app)
attach_scheduler(app)
return app

1
app/modules/ai_memory/models.py
View File

@@ -22,6 +22,7 @@ class AIMemoryEntry(Base):
"owner_id",
"fingerprint",
name="uq_ai_memory_owner_fingerprint",
postgresql_nulls_not_distinct=True,
),
)

7
app/modules/business/models/market.py
View File

@@ -91,7 +91,12 @@ class MarketAnnouncement(Base, TimestampMixin):
class MarketWatchlist(Base, TimestampMixin):
__tablename__ = "market_watchlists"
__table_args__ = (
UniqueConstraint("owner_id", "symbol", name="uq_market_watchlist_owner_symbol"),
UniqueConstraint(
"owner_id",
"symbol",
name="uq_market_watchlist_owner_symbol",
postgresql_nulls_not_distinct=True,
),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
owner_id: Mapped[int | None] = mapped_column(

18
app/modules/feishu/constants.py
View File

@@ -21,6 +21,20 @@ class FeishuEventSource(StrEnum):
LONG_CONNECTION = "long_connection"
class FeishuInboundStatus(StrEnum):
PENDING = "pending"
PROCESSING = "processing"
SUCCEEDED = "succeeded"
RETRY = "retry"
FAILED = "failed"
class FeishuEventTransport(StrEnum):
DISABLED = "disabled"
WEBHOOK = "webhook"
LONG_CONNECTION = "long_connection"
class FeishuPayloadKey(StrEnum):
APP_ACCESS_TOKEN = "app_access_token"
APP_ID = "app_id"
@@ -180,6 +194,10 @@ FEISHU_RECEIVE_ID_MISSING = "receive_id or FEISHU_DEFAULT_CHAT_ID is required"
FEISHU_SUCCESS_CODE = 0
FEISHU_DEFAULT_TOKEN_EXPIRE_SECONDS = 7200
FEISHU_TOKEN_EXPIRE_SAFETY_SECONDS = 300
FEISHU_INBOUND_BATCH_SIZE = 100
FEISHU_INBOUND_LEASE_SECONDS = 300
FEISHU_INBOUND_MAX_ATTEMPTS = 4
FEISHU_INBOUND_RETRY_DELAYS_SECONDS = (60, 300, 900)
FEISHU_AI_REPLY_TITLE = "AI 回复"
FEISHU_EMPTY_CARD_TEXT = "暂无数据"
FEISHU_MENTION_PATTERN = r"@\S+"

155
app/modules/feishu/long_connection.py
View File

@@ -1,12 +1,24 @@
import json
import logging
from socket import gethostname
from threading import Event, Thread
from time import monotonic
from typing import Any
from urllib.parse import urlsplit
from app.core.background.task_queue import enqueue_feishu_inbound_event
from app.core.config import get_settings
from app.core.constants import ActorValue
from app.core.database import SessionLocal
from app.modules.feishu.constants import FEISHU_DEFAULT_OPEN_API_DOMAIN, FeishuEventSource
from app.application.feishu import FeishuEventService
from app.modules.feishu.constants import (
FEISHU_DEFAULT_OPEN_API_DOMAIN,
FeishuEventSource,
FeishuEventTransport,
FeishuResponseKey,
)
from app.modules.observability.constants import HeartbeatComponent, HeartbeatStatus
from app.modules.observability.service import ObservabilityService
logger = logging.getLogger(__name__)
@@ -18,6 +30,118 @@ def _sdk_domain(base_url: str) -> str:
return f"{parsed.scheme}://{parsed.netloc}"
def _sdk_connection_is_open(client: Any) -> bool:
"""Return whether the SDK exposes a currently open WebSocket connection."""
try:
connection = getattr(client, "_conn", None)
except Exception:
return False
if connection is None:
return False
try:
state = getattr(connection, "state", None)
except Exception:
return False
if state is not None:
state_name = getattr(state, "name", None)
state_text = str(state_name or state).strip().lower()
if state_text == "open" or state_text.endswith(".open"):
return True
if any(
marker in state_text
for marker in ("connecting", "closing", "closed")
):
return False
try:
closed = getattr(connection, "closed", None)
except Exception:
return False
if isinstance(closed, bool):
return not closed
try:
opened = getattr(connection, "open", None)
except Exception:
return False
if isinstance(opened, bool):
return opened
try:
if getattr(connection, "close_code", None) is not None:
return False
transport = getattr(connection, "transport", None)
is_closing = getattr(transport, "is_closing", None)
if callable(is_closing) and is_closing():
return False
except Exception:
return False
# A non-null SDK-private connection object is not enough evidence that the
# WebSocket handshake completed or that the transport remains usable.
return False
def _connection_heartbeat_status(client: Any) -> str:
if _sdk_connection_is_open(client):
return HeartbeatStatus.OK
return HeartbeatStatus.DEGRADED
def _record_runtime_heartbeat(instance_id: str, status_value: str) -> None:
db = SessionLocal()
try:
ObservabilityService(db).record_heartbeat(
component=HeartbeatComponent.FEISHU_EVENTS,
instance_id=instance_id,
status_value=status_value,
actor=ActorValue.FEISHU,
)
except Exception:
db.rollback()
logger.exception("Failed to record Feishu event process heartbeat")
finally:
db.close()
def _heartbeat_loop(
stop_event: Event,
instance_id: str,
interval_seconds: int,
client: Any,
) -> None:
last_status: str | None = None
next_heartbeat_at = 0.0
while not stop_event.is_set():
status_value = _connection_heartbeat_status(client)
current_time = monotonic()
if status_value != last_status or current_time >= next_heartbeat_at:
_record_runtime_heartbeat(instance_id, status_value)
last_status = status_value
next_heartbeat_at = current_time + max(1, interval_seconds)
stop_event.wait(1)
def _start_heartbeat_loop(client: Any) -> tuple[Event, Thread]:
settings = get_settings()
stop_event = Event()
thread = Thread(
target=_heartbeat_loop,
args=(
stop_event,
gethostname(),
settings.heartbeat_interval_seconds,
client,
),
name="feishu-events-heartbeat",
daemon=True,
)
thread.start()
return stop_event, thread
def _sdk_event_to_payload(event: Any) -> dict[str, Any]:
try:
from lark_oapi.core.json import JSON
@@ -42,20 +166,38 @@ def _handle_verified_sdk_event(event: Any) -> None:
payload = _sdk_event_to_payload(event)
db = SessionLocal()
try:
result = FeishuEventService(db)._handle_verified_event(
acceptance = FeishuEventService(db).accept_verified_event(
payload,
source=FeishuEventSource.LONG_CONNECTION,
auto_reply=True,
)
logger.info("Handled Feishu long connection event: %s", result)
finally:
db.close()
if (
acceptance.event_key is not None
and acceptance.should_dispatch
and get_settings().task_queue_enabled
):
enqueue_feishu_inbound_event(
acceptance.event_key,
actor=ActorValue.FEISHU,
)
logger.info(
"Accepted Feishu long connection event key=%s status=%s duplicate=%s",
acceptance.event_key,
acceptance.response.get(FeishuResponseKey.STATUS),
bool(acceptance.response.get(FeishuResponseKey.DUPLICATE)),
)
def run_long_connection() -> None:
"""Start the Feishu long connection client and block forever."""
settings = get_settings()
if settings.feishu_event_transport != FeishuEventTransport.LONG_CONNECTION:
raise RuntimeError(
"FEISHU_EVENT_TRANSPORT must be long_connection for this process"
)
if not settings.feishu_app_id or not settings.feishu_app_secret:
raise RuntimeError("FEISHU_APP_ID and FEISHU_APP_SECRET are required")
@@ -81,7 +223,12 @@ def run_long_connection() -> None:
domain=_sdk_domain(settings.feishu_base_url),
)
logger.info("Starting Feishu long connection client")
client.start()
stop_event, heartbeat_thread = _start_heartbeat_loop(client)
try:
client.start()
finally:
stop_event.set()
heartbeat_thread.join(timeout=1)
if __name__ == "__main__":

78
app/modules/feishu/models.py
View File

@@ -1,10 +1,14 @@
from datetime import datetime
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy import JSON, Boolean, DateTime, Integer, String, Text, true
from sqlalchemy.orm import Mapped, mapped_column
from app.core.database import Base
from app.core.utils.time import utc_now
from app.modules.feishu.constants import (
FEISHU_INBOUND_MAX_ATTEMPTS,
FeishuInboundStatus,
)
class FeishuEventReceipt(Base):
@@ -15,7 +19,79 @@ class FeishuEventReceipt(Base):
source: Mapped[str] = mapped_column(String(64), index=True)
event_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
message_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
event_type: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
payload: Mapped[dict | None] = mapped_column(JSON, nullable=True)
auto_reply: Mapped[bool] = mapped_column(
Boolean,
default=True,
server_default=true(),
)
status: Mapped[str] = mapped_column(
String(32),
default=FeishuInboundStatus.PENDING,
server_default=FeishuInboundStatus.PENDING,
index=True,
)
attempt_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
max_attempts: Mapped[int] = mapped_column(
Integer,
default=FEISHU_INBOUND_MAX_ATTEMPTS,
server_default=str(FEISHU_INBOUND_MAX_ATTEMPTS),
)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
next_attempt_at: Mapped[datetime | None] = mapped_column(
DateTime,
nullable=True,
index=True,
)
locked_until: Mapped[datetime | None] = mapped_column(
DateTime,
nullable=True,
index=True,
)
locked_by: Mapped[str | None] = mapped_column(
String(128),
nullable=True,
index=True,
)
received_at: Mapped[datetime] = mapped_column(DateTime, default=utc_now, index=True)
processed_at: Mapped[datetime | None] = mapped_column(
DateTime,
nullable=True,
index=True,
)
reply_payload: Mapped[dict | None] = mapped_column(JSON, nullable=True)
reply_status: Mapped[str | None] = mapped_column(
String(32),
nullable=True,
index=True,
)
reply_attempt_count: Mapped[int] = mapped_column(
Integer,
default=0,
server_default="0",
)
reply_last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
reply_next_attempt_at: Mapped[datetime | None] = mapped_column(
DateTime,
nullable=True,
index=True,
)
reply_locked_until: Mapped[datetime | None] = mapped_column(
DateTime,
nullable=True,
index=True,
)
reply_locked_by: Mapped[str | None] = mapped_column(
String(128),
nullable=True,
index=True,
)
reply_sent_at: Mapped[datetime | None] = mapped_column(
DateTime,
nullable=True,
index=True,
)
class FeishuAppTicket(Base):

32
app/modules/feishu/routes.py
View File

@@ -1,10 +1,18 @@
from fastapi import APIRouter, Depends, Request
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, status
from sqlalchemy.orm import Session
from app.core.background.task_queue import enqueue_feishu_inbound_event
from app.core.config import get_settings
from app.core.constants import ActorValue
from app.core.database import get_db
from app.core.security import ApiPrincipal, require_api_key
from app.application.feishu import FeishuCommandService, FeishuEventService
from app.modules.feishu.constants import FeishuEventSource, FeishuPayloadKey, FeishuResponseKey
from app.modules.feishu.constants import (
FeishuEventSource,
FeishuEventTransport,
FeishuPayloadKey,
FeishuResponseKey,
)
from app.modules.feishu.event_verification import FeishuWebhookVerifier
from app.modules.feishu.schemas import (
FeishuCardMessage,
@@ -19,15 +27,31 @@ router = APIRouter()
@router.post("/webhook")
async def feishu_webhook(request: Request, db: Session = Depends(get_db)) -> dict:
async def feishu_webhook(
request: Request,
background_tasks: BackgroundTasks,
db: Session = Depends(get_db),
) -> dict:
"""Handle Feishu webhook challenge and text command events."""
if get_settings().feishu_event_transport != FeishuEventTransport.WEBHOOK:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Feishu webhook transport is disabled",
)
payload = FeishuWebhookVerifier().verify(await request.body(), request.headers)
return FeishuEventService(db)._handle_verified_event(
acceptance = FeishuEventService(db).accept_verified_event(
payload,
source=FeishuEventSource.WEBHOOK,
auto_reply=True,
)
if acceptance.event_key is not None and acceptance.should_dispatch:
background_tasks.add_task(
enqueue_feishu_inbound_event,
acceptance.event_key,
actor=ActorValue.FEISHU,
)
return acceptance.response
@router.post("/send-text", response_model=FeishuSendResult)

56
app/modules/feishu/service.py
View File

@@ -18,6 +18,7 @@ from app.modules.feishu.constants import (
FeishuPayloadKey,
FeishuReceiveIdType,
)
from app.modules.feishu.services.reply_outbox import current_reply_outbox
class FeishuService:
@@ -30,12 +31,18 @@ class FeishuService:
self.tenant_key = _optional_text(tenant_key) or _optional_text(
get_settings().feishu_default_tenant_key
)
self.message_uuid: str | None = None
def set_tenant_key(self, tenant_key: str | None) -> None:
"""Set the default tenant used by subsequent outbound operations."""
self.tenant_key = _optional_text(tenant_key)
def set_message_uuid(self, message_uuid: str | None) -> None:
"""Set the idempotency UUID used by replies in the current command."""
self.message_uuid = _optional_text(message_uuid)
def verify_event(self, payload: dict[str, Any]) -> None:
settings = get_settings()
expected = settings.feishu_verification_token
@@ -62,12 +69,26 @@ class FeishuService:
tenant_key: str | None = None,
record_audit: bool = True,
) -> dict[str, Any]:
resolved_uuid = uuid or self.message_uuid
resolved_tenant_key = self._resolve_tenant_key(tenant_key)
reply_outbox = current_reply_outbox()
if reply_outbox is not None:
return reply_outbox.capture_text(
text=text,
receive_id=receive_id,
default_receive_id=get_settings().feishu_default_chat_id,
receive_id_type=receive_id_type,
actor=actor,
message_uuid=resolved_uuid,
tenant_key=resolved_tenant_key,
record_audit=record_audit,
)
result = self.client.send_text(
text,
receive_id,
receive_id_type,
uuid,
tenant_key=self._resolve_tenant_key(tenant_key),
resolved_uuid,
tenant_key=resolved_tenant_key,
)
if record_audit:
self.audit.log(
@@ -79,7 +100,7 @@ class FeishuService:
"receive_target_hash": _target_fingerprint(receive_id),
FeishuPayloadKey.RECEIVE_ID_TYPE: receive_id_type,
"content_length": len(text),
FeishuPayloadKey.UUID: uuid,
FeishuPayloadKey.UUID: resolved_uuid,
},
response_payload=result,
)
@@ -95,12 +116,25 @@ class FeishuService:
uuid: str | None = None,
tenant_key: str | None = None,
) -> dict[str, Any]:
resolved_uuid = uuid or self.message_uuid
resolved_tenant_key = self._resolve_tenant_key(tenant_key)
reply_outbox = current_reply_outbox()
if reply_outbox is not None:
return reply_outbox.capture_card(
card=card,
receive_id=receive_id,
default_receive_id=get_settings().feishu_default_chat_id,
receive_id_type=receive_id_type,
actor=actor,
message_uuid=resolved_uuid,
tenant_key=resolved_tenant_key,
)
result = self.client.send_card(
card,
receive_id,
receive_id_type,
uuid,
tenant_key=self._resolve_tenant_key(tenant_key),
resolved_uuid,
tenant_key=resolved_tenant_key,
)
self.audit.log(
AuditLogCreate(
@@ -111,7 +145,7 @@ class FeishuService:
"receive_target_hash": _target_fingerprint(receive_id),
FeishuPayloadKey.RECEIVE_ID_TYPE: receive_id_type,
"card_element_count": len(card.get(FeishuPayloadKey.ELEMENTS) or []),
FeishuPayloadKey.UUID: uuid,
FeishuPayloadKey.UUID: resolved_uuid,
},
response_payload=result,
)
@@ -124,9 +158,17 @@ class FeishuService:
actor: str = ActorValue.SYSTEM,
tenant_key: str | None = None,
) -> dict[str, Any]:
resolved_tenant_key = self._resolve_tenant_key(tenant_key)
reply_outbox = current_reply_outbox()
if reply_outbox is not None:
return reply_outbox.capture_image(
image=image,
actor=actor,
tenant_key=resolved_tenant_key,
)
result = self.client.upload_image(
image,
tenant_key=self._resolve_tenant_key(tenant_key),
tenant_key=resolved_tenant_key,
)
self.audit.log(
AuditLogCreate(

17
app/modules/feishu/services/__init__.py Normal file
View File

@@ -0,0 +1,17 @@
from app.modules.feishu.services.inbox import (
FeishuInboundAcceptance,
FeishuInboundProcessResult,
FeishuInboundService,
)
from app.modules.feishu.services.context import (
bind_inbound_event,
current_inbound_event_key,
)
__all__ = [
"FeishuInboundAcceptance",
"FeishuInboundProcessResult",
"FeishuInboundService",
"bind_inbound_event",
"current_inbound_event_key",
]

23
app/modules/feishu/services/context.py Normal file
View File

@@ -0,0 +1,23 @@
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
_CURRENT_INBOUND_EVENT_KEY: ContextVar[str | None] = ContextVar(
"current_feishu_inbound_event_key",
default=None,
)
@contextmanager
def bind_inbound_event(event_key: str) -> Iterator[None]:
"""Expose the current inbox key to privacy-cleanup hooks."""
token = _CURRENT_INBOUND_EVENT_KEY.set(event_key)
try:
yield
finally:
_CURRENT_INBOUND_EVENT_KEY.reset(token)
def current_inbound_event_key() -> str | None:
return _CURRENT_INBOUND_EVENT_KEY.get()

1305
app/modules/feishu/services/inbox.py Normal file
View File

File diff suppressed because it is too large Load Diff

247
app/modules/feishu/services/reply_outbox.py Normal file
View File

@@ -0,0 +1,247 @@
import base64
from contextlib import contextmanager
from contextvars import ContextVar
from copy import deepcopy
from dataclasses import dataclass, field
from typing import Any, Iterator
from fastapi import HTTPException, status
from app.modules.feishu.constants import FEISHU_RECEIVE_ID_MISSING
_IMAGE_PLACEHOLDER_PREFIX = "__feishu_reply_image__:"
_reply_collector: ContextVar["FeishuReplyCollector | None"] = ContextVar(
"feishu_reply_collector",
default=None,
)
@dataclass(slots=True)
class FeishuReplyCollector:
"""Capture outbound Feishu operations before an inbound transaction commits."""
operations: list[dict[str, Any]] = field(default_factory=list)
def capture_text(
self,
*,
text: str,
receive_id: str | None,
default_receive_id: str | None,
receive_id_type: str,
actor: str,
message_uuid: str | None,
tenant_key: str | None,
record_audit: bool,
) -> dict[str, Any]:
target = _required_receive_id(receive_id, default_receive_id)
self.operations.append(
{
"kind": "text",
"text": text,
"receive_id": target,
"receive_id_type": str(receive_id_type),
"actor": str(actor),
"message_uuid": message_uuid,
"tenant_key": tenant_key,
"record_audit": bool(record_audit),
}
)
return {"code": 0, "queued": True}
def capture_card(
self,
*,
card: dict[str, Any],
receive_id: str | None,
default_receive_id: str | None,
receive_id_type: str,
actor: str,
message_uuid: str | None,
tenant_key: str | None,
) -> dict[str, Any]:
target = _required_receive_id(receive_id, default_receive_id)
self.operations.append(
{
"kind": "card",
"card": deepcopy(card),
"receive_id": target,
"receive_id_type": str(receive_id_type),
"actor": str(actor),
"message_uuid": message_uuid,
"tenant_key": tenant_key,
"record_audit": True,
}
)
return {"code": 0, "queued": True}
def capture_image(
self,
*,
image: bytes,
actor: str,
tenant_key: str | None,
) -> dict[str, Any]:
placeholder = f"{_IMAGE_PLACEHOLDER_PREFIX}{len(self.operations)}"
self.operations.append(
{
"kind": "image",
"placeholder": placeholder,
"image_base64": base64.b64encode(image).decode("ascii"),
"actor": str(actor),
"tenant_key": tenant_key,
}
)
return {
"code": 0,
"queued": True,
"data": {"image_key": placeholder},
}
def as_payload(
self,
*,
identity: tuple[str, str] | None,
identity_fence_required: bool,
) -> dict[str, Any] | None:
if not self.operations:
return None
return {
"version": 1,
"identity": (
{"tenant_key": identity[0], "open_id": identity[1]}
if identity is not None
else None
),
"identity_fence_required": identity_fence_required,
"operations": deepcopy(self.operations),
}
@contextmanager
def bind_reply_outbox() -> Iterator[FeishuReplyCollector]:
"""Capture Feishu side effects for the current inbound command."""
collector = FeishuReplyCollector()
token = _reply_collector.set(collector)
try:
yield collector
finally:
_reply_collector.reset(token)
def current_reply_outbox() -> FeishuReplyCollector | None:
return _reply_collector.get()
def decode_image(operation: dict[str, Any]) -> bytes:
encoded = operation.get("image_base64")
if not isinstance(encoded, str) or not encoded:
raise ValueError("Feishu reply image payload is unavailable")
try:
return base64.b64decode(encoded, validate=True)
except (ValueError, TypeError) as exc:
raise ValueError("Feishu reply image payload is invalid") from exc
def resolved_message_operation(payload: dict[str, Any]) -> dict[str, Any]:
operations = _operations(payload)
messages = [
operation
for operation in operations
if operation.get("kind") in {"text", "card"}
]
if len(messages) != 1:
raise ValueError("Feishu reply outbox requires exactly one message")
image_keys = {
str(operation.get("placeholder")): str(operation.get("image_key"))
for operation in operations
if operation.get("kind") == "image"
and operation.get("placeholder")
and operation.get("image_key")
}
message = deepcopy(messages[0])
if message.get("kind") == "card":
message["card"] = _replace_image_placeholders(
message.get("card"),
image_keys,
)
return message
def pending_image_indexes(payload: dict[str, Any]) -> list[int]:
return [
index
for index, operation in enumerate(_operations(payload))
if operation.get("kind") == "image" and not operation.get("image_key")
]
def operations_copy(payload: dict[str, Any]) -> list[dict[str, Any]]:
return deepcopy(_operations(payload))
def payload_identity(payload: Any) -> tuple[str, str] | None:
if not isinstance(payload, dict):
return None
identity = payload.get("identity")
if not isinstance(identity, dict):
return None
tenant_key = str(identity.get("tenant_key") or "").strip()
open_id = str(identity.get("open_id") or "").strip()
if not tenant_key or not open_id:
return None
return tenant_key, open_id
def identity_fence_required(payload: Any) -> bool:
return isinstance(payload, dict) and bool(payload.get("identity_fence_required"))
def _operations(payload: dict[str, Any]) -> list[dict[str, Any]]:
if payload.get("version") != 1:
raise ValueError("Unsupported Feishu reply outbox payload version")
operations = payload.get("operations")
if not isinstance(operations, list) or not all(
isinstance(operation, dict) for operation in operations
):
raise ValueError("Feishu reply outbox operations are invalid")
return operations
def _replace_image_placeholders(
value: Any,
image_keys: dict[str, str],
) -> Any:
if isinstance(value, dict):
return {
key: _replace_image_placeholders(item, image_keys)
for key, item in value.items()
}
if isinstance(value, list):
return [
_replace_image_placeholders(item, image_keys)
for item in value
]
if (
isinstance(value, str)
and value.startswith(_IMAGE_PLACEHOLDER_PREFIX)
):
image_key = image_keys.get(value)
if not image_key:
raise ValueError("Feishu reply image was not prepared")
return image_key
return value
def _required_receive_id(
receive_id: str | None,
default_receive_id: str | None,
) -> str:
target = str(receive_id or default_receive_id or "").strip()
if not target:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=FEISHU_RECEIVE_ID_MISSING,
)
return target

10
app/modules/feishu_users/constants.py
View File

@@ -80,7 +80,15 @@ def parse_admin_identities(
tenant_key, separator, open_id = text.partition(":")
tenant_key = tenant_key.strip()
open_id = open_id.strip()
if not separator or not tenant_key or not open_id:
if (
not separator
or not tenant_key
or not open_id
or ":" in open_id
or len(tenant_key) > 128
or len(open_id) > 128
or any(character in text for character in "\r\n")
):
raise ValueError(INVALID_ADMIN_IDENTITY)
identities.add((tenant_key, open_id))
return frozenset(identities)

20
app/modules/feishu_users/identifiers.py Normal file
View File

@@ -0,0 +1,20 @@
from hashlib import sha256
_AUDIT_IDENTITY_DOMAIN = b"company-ai-platform:feishu-audit-identity:v1\0"
def feishu_audit_identity_hash(tenant_key: str, open_id: str) -> str:
"""Return the erasable pseudonymous subject used by pre-registration audits."""
tenant_bytes = tenant_key.encode("utf-8")
open_id_bytes = open_id.encode("utf-8")
identity = b"".join(
(
len(tenant_bytes).to_bytes(4, "big"),
tenant_bytes,
len(open_id_bytes).to_bytes(4, "big"),
open_id_bytes,
)
)
digest = sha256(_AUDIT_IDENTITY_DOMAIN + identity).hexdigest()
return f"feishu-identity-sha256-{digest}"

5
app/modules/observability/constants.py
View File

@@ -6,13 +6,16 @@ class ObservabilityKey(StrEnum):
CHECKS = "checks"
METRICS = "metrics"
DATABASE = "database"
SCHEMA = "schema"
REDIS = "redis"
EVENTS = "events"
WORKFLOWS = "workflows"
AI_MEMORY = "ai_memory"
HEARTBEATS = "heartbeats"
API = "api"
SCHEDULER = "scheduler"
WORKER = "worker"
FEISHU_EVENTS = "feishu_events"
class ObservabilityStatus(StrEnum):
@@ -40,7 +43,9 @@ class HeartbeatComponent(StrEnum):
API = "api"
SCHEDULER = "scheduler"
WORKER = "worker"
FEISHU_EVENTS = "feishu-events"
class HeartbeatStatus(StrEnum):
OK = "ok"
DEGRADED = "degraded"

73
app/modules/observability/runtime.py Normal file
View File

@@ -0,0 +1,73 @@
import logging
from os import getpid
from socket import gethostname
from threading import Event, Thread
from fastapi import FastAPI
from app.core.config import get_settings
from app.core.constants import ActorValue
from app.core.database import SessionLocal
from app.modules.observability.constants import HeartbeatComponent
from app.modules.observability.service import ObservabilityService
logger = logging.getLogger(__name__)
def attach_api_heartbeat(app: FastAPI) -> None:
"""Record API process liveness without coupling it to request traffic."""
interval_seconds = max(1, get_settings().heartbeat_interval_seconds)
instance_id = f"{gethostname()}:{getpid()}"
@app.on_event("startup")
def start_api_heartbeat() -> None:
current_thread = getattr(app.state, "api_heartbeat_thread", None)
if current_thread is not None and current_thread.is_alive():
return
stop_event = Event()
_record_api_heartbeat(instance_id)
thread = Thread(
target=_api_heartbeat_loop,
args=(stop_event, instance_id, interval_seconds),
name="api-heartbeat",
daemon=True,
)
app.state.api_heartbeat_stop_event = stop_event
app.state.api_heartbeat_thread = thread
app.state.api_heartbeat_instance_id = instance_id
thread.start()
@app.on_event("shutdown")
def stop_api_heartbeat() -> None:
stop_event = getattr(app.state, "api_heartbeat_stop_event", None)
thread = getattr(app.state, "api_heartbeat_thread", None)
if stop_event is not None:
stop_event.set()
if thread is not None:
thread.join(timeout=1)
def _api_heartbeat_loop(
stop_event: Event,
instance_id: str,
interval_seconds: int,
) -> None:
while not stop_event.wait(max(1, interval_seconds)):
_record_api_heartbeat(instance_id)
def _record_api_heartbeat(instance_id: str) -> None:
db = SessionLocal()
try:
ObservabilityService(db).record_heartbeat(
component=HeartbeatComponent.API,
instance_id=instance_id,
actor=ActorValue.API,
)
except Exception:
db.rollback()
logger.exception("Failed to record API process heartbeat")
finally:
db.close()

211
app/modules/observability/service.py
View File

@@ -9,6 +9,10 @@ from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.constants import ActorValue
from app.core.database.migrations import (
current_alembic_revisions,
expected_alembic_heads,
)
from app.core.utils.time import utc_now
from app.modules.ai_memory.service import AIMemoryService
from app.modules.audit.constants import (
@@ -20,12 +24,18 @@ from app.modules.audit.constants import (
from app.modules.audit.schemas import AuditLogCreate
from app.modules.audit.service import AuditService
from app.modules.events.constants import EventStatus
from app.modules.events.models import DomainEvent
from app.modules.events.services import EventService
from app.modules.feishu.app_tickets import FeishuAppTicketService
from app.modules.feishu.constants import FeishuAppType
from app.modules.feishu_users.constants import FeishuUserStatus
from app.modules.feishu.services import FeishuInboundService
from app.modules.feishu_users.constants import (
FeishuUserStatus,
parse_admin_identities,
)
from app.modules.feishu_users.models import FeishuUser
from app.modules.observability.constants import (
HeartbeatComponent,
HeartbeatStatus,
ObservabilityKey,
ObservabilityMetricKey,
@@ -53,10 +63,35 @@ class ObservabilityService:
def ready(self) -> dict[str, Any]:
checks = {
ObservabilityKey.DATABASE: self._safe_call(self._database_check),
ObservabilityKey.SCHEMA: self._safe_call(self._schema_check),
ObservabilityKey.REDIS: self._safe_call(self._redis_check),
ObservabilityKey.EVENTS: self._safe_call(self._events_check),
ObservabilityKey.WORKFLOWS: self._safe_call(self._workflows_check),
ObservabilityKey.HEARTBEATS: self._safe_call(self._heartbeats_check),
ObservabilityKey.API: self._safe_call(
lambda: self._component_heartbeat_check(
HeartbeatComponent.API,
required=self._api_required(),
)
),
ObservabilityKey.SCHEDULER: self._safe_call(
lambda: self._component_heartbeat_check(
HeartbeatComponent.SCHEDULER,
required=self._scheduler_required(),
)
),
ObservabilityKey.WORKER: self._safe_call(
lambda: self._component_heartbeat_check(
HeartbeatComponent.WORKER,
required=get_settings().task_queue_enabled,
)
),
ObservabilityKey.FEISHU_EVENTS: self._safe_call(
lambda: self._component_heartbeat_check(
HeartbeatComponent.FEISHU_EVENTS,
required=self._feishu_events_required(),
)
),
"feishu_subscriptions": self._safe_call(
self._feishu_subscriptions_check
),
@@ -87,6 +122,7 @@ class ObservabilityService:
self.heartbeat_summary
),
"feishu_users": self._safe_call(self._feishu_user_metrics),
"feishu_inbound": self._safe_call(self._feishu_inbound_metrics),
"subscriptions": self._safe_call(self._subscription_metrics),
}
}
@@ -158,6 +194,29 @@ class ObservabilityService:
self.db.execute(text("select 1")).scalar()
return {ObservabilityKey.STATUS: ObservabilityStatus.OK}
def _schema_check(self) -> dict[str, Any]:
settings = get_settings()
required = (
settings.app_env.lower() in {"prod", "production"}
or settings.scheduler_enabled
or settings.feishu_user_features_enabled
or settings.feishu_event_transport != "disabled"
)
if not required:
return {ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED}
expected = expected_alembic_heads()
current = current_alembic_revisions(self.db.connection())
matches = current == expected and len(expected) == 1
return {
ObservabilityKey.STATUS: (
ObservabilityStatus.OK if matches else ObservabilityStatus.DEGRADED
),
"current": [] if current is None else list(current),
"expected": list(expected),
"reason": None if matches else "schema_revision_mismatch",
}
def _redis_check(self) -> dict[str, Any]:
settings = get_settings()
if not settings.task_queue_enabled:
@@ -215,22 +274,58 @@ class ObservabilityService:
def _events_check(self) -> dict[str, Any]:
counts = EventService(self.db).count_by_status()
current = utc_now()
failed = counts.get(EventStatus.FAILED, 0)
processable = int(
self.db.scalar(
select(func.count())
.select_from(DomainEvent)
.where(
DomainEvent.status == EventStatus.PENDING,
DomainEvent.next_attempt_at.is_not(None),
DomainEvent.next_attempt_at <= current,
or_(
DomainEvent.locked_until.is_(None),
DomainEvent.locked_until <= current,
),
)
)
or 0
)
expired_leases = int(
self.db.scalar(
select(func.count())
.select_from(DomainEvent)
.where(
DomainEvent.status == EventStatus.PENDING,
DomainEvent.locked_by.is_not(None),
DomainEvent.locked_until.is_not(None),
DomainEvent.locked_until <= current,
)
)
or 0
)
reasons: list[str] = []
if processable and not get_settings().event_dispatch_enabled:
reasons.append("dispatch_disabled")
if expired_leases:
reasons.append("expired_leases")
return {
ObservabilityKey.STATUS: (
ObservabilityStatus.DEGRADED if failed else ObservabilityStatus.OK
ObservabilityStatus.DEGRADED if reasons else ObservabilityStatus.OK
),
ObservabilityMetricKey.PENDING: counts.get(EventStatus.PENDING, 0),
ObservabilityMetricKey.FAILED: failed,
"processable": processable,
"expired_leases": expired_leases,
"reasons": reasons,
}
def _workflows_check(self) -> dict[str, Any]:
counts = WorkflowService(self.db).count_by_status()
failed = counts.get(WorkflowStatus.FAILED, 0)
return {
ObservabilityKey.STATUS: (
ObservabilityStatus.DEGRADED if failed else ObservabilityStatus.OK
),
ObservabilityKey.STATUS: ObservabilityStatus.OK,
ObservabilityMetricKey.RUNNING: counts.get(WorkflowStatus.RUNNING, 0),
ObservabilityMetricKey.FAILED: failed,
}
@@ -238,20 +333,86 @@ class ObservabilityService:
def _heartbeats_check(self) -> dict[str, Any]:
summary = self.heartbeat_summary()
total = summary[ObservabilityMetricKey.TOTAL]
active = summary[ObservabilityMetricKey.ACTIVE]
stale = summary[ObservabilityMetricKey.STALE]
if total == 0:
return {ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED}
return {
ObservabilityKey.STATUS: (
ObservabilityStatus.DEGRADED if stale else ObservabilityStatus.OK
ObservabilityStatus.OK if active else ObservabilityStatus.DEGRADED
),
ObservabilityMetricKey.TOTAL: total,
ObservabilityMetricKey.ACTIVE: active,
ObservabilityMetricKey.STALE: stale,
ObservabilityMetricKey.LAST_SEEN_AT: summary[
ObservabilityMetricKey.LAST_SEEN_AT
],
}
def _component_heartbeat_check(
self,
component: str,
*,
required: bool,
) -> dict[str, Any]:
if not required:
return {
ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED,
ObservabilityMetricKey.COMPONENT: component,
}
records = list(
self.db.execute(
select(SystemHeartbeat.status, SystemHeartbeat.last_seen_at).where(
SystemHeartbeat.component == component
)
).all()
)
if not records:
return {
ObservabilityKey.STATUS: ObservabilityStatus.DEGRADED,
ObservabilityMetricKey.COMPONENT: component,
"reason": "heartbeat_missing",
}
last_seen_at = max(item.last_seen_at for item in records)
fresh_records = [
item
for item in records
if item.last_seen_at >= self._heartbeat_stale_threshold()
]
if not fresh_records:
return {
ObservabilityKey.STATUS: ObservabilityStatus.DEGRADED,
ObservabilityMetricKey.COMPONENT: component,
ObservabilityMetricKey.LAST_SEEN_AT: last_seen_at.isoformat(),
"reason": "heartbeat_stale",
}
healthy = any(item.status == HeartbeatStatus.OK for item in fresh_records)
return {
ObservabilityKey.STATUS: (
ObservabilityStatus.OK if healthy else ObservabilityStatus.DEGRADED
),
ObservabilityMetricKey.COMPONENT: component,
ObservabilityMetricKey.LAST_SEEN_AT: last_seen_at.isoformat(),
"reason": None if healthy else "heartbeat_degraded",
}
@staticmethod
def _api_required() -> bool:
return get_settings().app_env.lower() in {"prod", "production"}
@staticmethod
def _scheduler_required() -> bool:
settings = get_settings()
return (
settings.feishu_user_features_enabled
or settings.app_env.lower() in {"prod", "production"}
)
@staticmethod
def _feishu_events_required() -> bool:
settings = get_settings()
return settings.feishu_event_transport == "long_connection"
def _feishu_subscriptions_check(self) -> dict[str, Any]:
active = int(
self.db.scalar(
@@ -284,13 +445,17 @@ class ObservabilityService:
)
or 0
)
if not active and not processable_deliveries:
settings = get_settings()
if (
not settings.feishu_user_features_enabled
and not active
and not processable_deliveries
):
return {
ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED,
"active": 0,
"processable_deliveries": 0,
}
settings = get_settings()
app_id = str(settings.feishu_app_id or "").strip()
credentials_configured = bool(app_id and settings.feishu_app_secret)
active_tenant_count = int(
@@ -316,6 +481,31 @@ class ObservabilityService:
str(settings.feishu_default_tenant_key or "").strip()
)
reasons: list[str] = []
if (
not settings.feishu_user_features_enabled
and (active or processable_deliveries)
):
reasons.append("features_disabled")
if (
settings.app_env.lower() in {"prod", "production"}
and settings.feishu_user_features_enabled
):
try:
configured_admins = parse_admin_identities(
settings.feishu_admin_identities
)
except ValueError:
reasons.append("admin_identities_invalid")
else:
if not configured_admins:
reasons.append("admin_identities_missing")
if settings.feishu_event_transport == "disabled":
reasons.append("event_transport_disabled")
elif (
settings.feishu_event_transport == "webhook"
and not settings.feishu_verification_token
):
reasons.append("verification_token_missing")
if not credentials_configured:
reasons.append("credentials_missing")
if settings.feishu_app_type == FeishuAppType.STORE:
@@ -341,6 +531,8 @@ class ObservabilityService:
),
"active": active,
"processable_deliveries": processable_deliveries,
"features_enabled": settings.feishu_user_features_enabled,
"event_transport": settings.feishu_event_transport,
"app_type": settings.feishu_app_type,
"credentials_configured": credentials_configured,
"ticket_configured": ticket_configured,
@@ -360,6 +552,9 @@ class ObservabilityService:
)
return {"active": active}
def _feishu_inbound_metrics(self) -> dict[str, int]:
return FeishuInboundService(self.db).status_counts()
def _subscription_metrics(self) -> dict[str, int]:
active = int(
self.db.scalar(

2
app/tasks/__init__.py
View File

@@ -1,8 +1,10 @@
from app.tasks.app import celery_app
from app.tasks import events as _events # noqa: F401
from app.tasks import feishu as _feishu # noqa: F401
from app.tasks import legacy as _legacy # noqa: F401
from app.tasks import lifecycle as _lifecycle # noqa: F401
from app.tasks import market as _market # noqa: F401
from app.tasks import observability as _observability # noqa: F401
from app.tasks import reports as _reports # noqa: F401
from app.tasks import risk as _risk # noqa: F401
from app.tasks import subscriptions as _subscriptions # noqa: F401

3
app/tasks/constants.py
View File

@@ -12,3 +12,6 @@ TASK_RUN_LIFECYCLE = "reports.run_lifecycle"
TASK_RUN_MARKET_REPORT = "market.report.run"
TASK_RUN_MARKET_CLOSE = TASK_RUN_MARKET_REPORT
TASK_RUN_SUBSCRIPTION_CYCLE = "subscriptions.run_cycle"
TASK_PROCESS_FEISHU_INBOUND = "feishu.process_inbound"
TASK_PROCESS_DUE_FEISHU_INBOUND = "feishu.process_inbound_due"
TASK_RECORD_WORKER_HEARTBEAT = "observability.record_worker_heartbeat"

29
app/tasks/feishu.py Normal file
View File

@@ -0,0 +1,29 @@
from typing import Any
from app.application.feishu.inbound import (
process_due_feishu_inbound_events,
process_feishu_inbound_event,
)
from app.core.constants import ActorValue
from app.modules.feishu.constants import FEISHU_INBOUND_BATCH_SIZE
from app.tasks.app import celery_app
from app.tasks.constants import (
TASK_PROCESS_DUE_FEISHU_INBOUND,
TASK_PROCESS_FEISHU_INBOUND,
)
@celery_app.task(name=TASK_PROCESS_FEISHU_INBOUND)
def process_feishu_inbound_event_task(
event_key: str,
actor: str = ActorValue.WORKER,
) -> dict[str, Any]:
return process_feishu_inbound_event(event_key, actor=actor)
@celery_app.task(name=TASK_PROCESS_DUE_FEISHU_INBOUND)
def process_due_feishu_inbound_events_task(
limit: int = FEISHU_INBOUND_BATCH_SIZE,
actor: str = ActorValue.WORKER,
) -> list[dict[str, Any]]:
return process_due_feishu_inbound_events(limit=limit, actor=actor)

25
app/tasks/observability.py Normal file
View File

@@ -0,0 +1,25 @@
from socket import gethostname
from app.core.constants import ActorValue
from app.core.database import SessionLocal
from app.modules.observability.constants import HeartbeatComponent
from app.modules.observability.service import ObservabilityService
from app.tasks.app import celery_app
from app.tasks.constants import TASK_RECORD_WORKER_HEARTBEAT
@celery_app.task(name=TASK_RECORD_WORKER_HEARTBEAT)
def record_worker_heartbeat_task(
actor: str = ActorValue.WORKER,
) -> dict:
"""Record liveness from the Celery process that actually executes the task."""
db = SessionLocal()
try:
return ObservabilityService(db).record_heartbeat(
component=HeartbeatComponent.WORKER,
instance_id=gethostname(),
actor=actor,
)
finally:
db.close()

924
app/tools/reconcile_platform_schema.py Normal file
View File

@@ -0,0 +1,924 @@
"""Audit and atomically baseline the known unversioned platform schema.
The command is read-only unless ``--apply`` is supplied together with the
schema fingerprint printed by a preceding dry run. It never accepts a database
URL on the command line, so credentials are not exposed through process
arguments.
"""
import argparse
from collections.abc import Mapping
from dataclasses import dataclass
import hashlib
import json
from pathlib import Path
import re
from typing import Any
from alembic import command
from alembic.autogenerate import compare_metadata
from alembic.config import Config
from alembic.migration import MigrationContext
from alembic.script import ScriptDirectory
from sqlalchemy import MetaData, Table, create_engine, inspect, select, text
from sqlalchemy.engine import Connection, Engine
from sqlalchemy.pool import NullPool
from app.core.config import get_settings
from app.core.database import Base
from app.core.database.safety import validate_platform_migration_target
from app.modules.ai_memory import models as ai_memory_models
from app.modules.audit import models as audit_models
from app.modules.business import models as business_models
from app.modules.events import models as event_models
from app.modules.feishu import models as feishu_models
from app.modules.feishu_users import models as feishu_user_models
from app.modules.observability import models as observability_models
from app.modules.personalization import models as personalization_models
from app.modules.subscriptions import models as subscription_models
from app.modules.workflows import models as workflow_models
PREVIOUS_REVISION = "202607260005"
TARGET_REVISION = "202607270002"
APPROVAL_TABLE = "approval_requests"
SCHEMA_FINGERPRINT_PATTERN = re.compile(r"^[0-9a-f]{64}$")
ADVISORY_LOCK_KEY = int.from_bytes(b"CAIPSCHE", byteorder="big", signed=True)
MINIMUM_POSTGRESQL_VERSION_NUM = 150000
# Keep imports referenced so every model is registered with Base.metadata.
_REGISTERED_MODEL_MODULES = (
ai_memory_models,
audit_models,
business_models,
event_models,
feishu_models,
feishu_user_models,
observability_models,
personalization_models,
subscription_models,
workflow_models,
)
_ADD_COLUMNS = {
"ai_memory_entries": {"kind", "owner_id"},
"attendance_records": {
"attendance_scope",
"is_active",
"last_seen_at",
"source_location_status",
"source_status",
"source_updated_at",
},
"audit_logs": {"request_id"},
"feishu_event_receipts": {
"attempt_count",
"auto_reply",
"event_type",
"last_error",
"locked_by",
"locked_until",
"max_attempts",
"next_attempt_at",
"payload",
"processed_at",
"reply_attempt_count",
"reply_last_error",
"reply_locked_by",
"reply_locked_until",
"reply_next_attempt_at",
"reply_payload",
"reply_sent_at",
"reply_status",
"status",
},
"market_watchlists": {"owner_id"},
"projects": {
"department_code",
"department_name",
"display_code",
"is_active",
"last_seen_at",
"owner_employee_code",
"source_archived",
"source_contract_amount",
"source_created_at",
"source_project_investment_amount",
"source_stage",
"source_stage_label",
"source_updated_at",
},
"risk_events": {
"assigned_to",
"closed_at",
"closed_reason",
"resolved_at",
"review_summary",
},
"work_reports": {
"employee_code",
"external_id",
"is_active",
"is_draft",
"is_late",
"last_seen_at",
"source_updated_at",
},
"work_tasks": {
"employee_code",
"external_id",
"is_active",
"last_seen_at",
"source_created_at",
"source_system",
"source_updated_at",
},
}
_ADD_INDEXES = {
"ai_memory_entries": {
"ix_ai_memory_entries_fingerprint",
"ix_ai_memory_entries_kind",
"ix_ai_memory_entries_owner_id",
},
"attendance_records": {
"ix_attendance_records_attendance_scope",
"ix_attendance_records_is_active",
"ix_attendance_records_last_seen_at",
"ix_attendance_records_source_status",
"ix_attendance_records_source_updated_at",
},
"audit_logs": {"ix_audit_logs_request_id"},
"feishu_admin_bootstrap_tombstones": {
"ix_feishu_admin_bootstrap_tombstones_created_at",
"ix_feishu_admin_bootstrap_tombstones_identity_hash",
},
"feishu_event_receipts": {
"ix_feishu_event_receipts_event_type",
"ix_feishu_event_receipts_locked_by",
"ix_feishu_event_receipts_locked_until",
"ix_feishu_event_receipts_next_attempt_at",
"ix_feishu_event_receipts_processed_at",
"ix_feishu_event_receipts_reply_locked_by",
"ix_feishu_event_receipts_reply_locked_until",
"ix_feishu_event_receipts_reply_next_attempt_at",
"ix_feishu_event_receipts_reply_sent_at",
"ix_feishu_event_receipts_reply_status",
"ix_feishu_event_receipts_status",
},
"feishu_app_tickets": {
"ix_feishu_app_tickets_app_id",
"ix_feishu_app_tickets_received_at",
},
"market_watchlists": {"ix_market_watchlists_owner_id"},
"projects": {
"ix_projects_department_code",
"ix_projects_department_name",
"ix_projects_display_code",
"ix_projects_is_active",
"ix_projects_last_seen_at",
"ix_projects_owner_employee_code",
"ix_projects_source_archived",
"ix_projects_source_stage",
"ix_projects_source_updated_at",
},
"risk_events": {"ix_risk_events_assigned_to"},
"work_reports": {
"ix_work_reports_employee_code",
"ix_work_reports_external_id",
"ix_work_reports_is_active",
"ix_work_reports_is_draft",
"ix_work_reports_is_late",
"ix_work_reports_last_seen_at",
"ix_work_reports_source_updated_at",
},
"work_tasks": {
"ix_work_tasks_employee_code",
"ix_work_tasks_external_id",
"ix_work_tasks_is_active",
"ix_work_tasks_last_seen_at",
"ix_work_tasks_source_updated_at",
},
}
_REMOVE_INDEXES = {
"ai_memory_entries": {"ix_ai_memory_entries_fingerprint"},
APPROVAL_TABLE: {
"ix_approval_requests_action",
"ix_approval_requests_applicant",
"ix_approval_requests_approver",
"ix_approval_requests_created_at",
"ix_approval_requests_domain",
"ix_approval_requests_record_id",
"ix_approval_requests_status",
"ix_approval_requests_ticket_id",
},
}
_MODIFY_DEFAULTS = {
"attendance_records": {"attendance_scope", "is_active"},
"projects": {"is_active", "source_archived"},
"work_reports": {"is_active", "is_draft", "is_late"},
"work_tasks": {"is_active"},
}
_IMPACTED_TABLES = frozenset(
{
*_ADD_COLUMNS,
*_ADD_INDEXES,
*_MODIFY_DEFAULTS,
*_REMOVE_INDEXES,
APPROVAL_TABLE,
"feishu_users",
}
)
_CONTENT_FINGERPRINT_TABLES = _IMPACTED_TABLES - {APPROVAL_TABLE}
class ReconciliationError(RuntimeError):
"""Raised when a baseline precondition is not satisfied."""
@dataclass(frozen=True, order=True)
class DriftKey:
operation: str
table: str
object_name: str
def render(self) -> str:
suffix = f":{self.object_name}" if self.object_name else ""
return f"{self.operation}:{self.table}{suffix}"
@dataclass(frozen=True)
class BaselineAudit:
dialect: str
fingerprint: str
impacted_table_row_counts: tuple[tuple[str, int], ...]
impacted_table_content_digests: tuple[tuple[str, str], ...]
postgresql_server_version_num: int | None
postgresql_version_supported: bool | None
revision_rows: tuple[str, ...] | None
observed_drift: frozenset[DriftKey]
unexpected_drift: frozenset[DriftKey]
approval_rows: int | None
approval_inbound_foreign_keys: int
schema_privileges_ok: bool | None
table_ownership_ok: bool | None
@property
def eligible(self) -> bool:
return (
self.revision_rows == ()
and not self.unexpected_drift
and (self.approval_rows is None or self.approval_rows == 0)
and self.approval_inbound_foreign_keys == 0
and self.schema_privileges_ok is not False
and self.table_ownership_ok is not False
and self.postgresql_version_supported is not False
)
def public_dict(self) -> dict[str, Any]:
return {
"eligible": self.eligible,
"dialect": self.dialect,
"schema_fingerprint_sha256": self.fingerprint,
"impacted_table_row_counts": dict(self.impacted_table_row_counts),
"postgresql_server_version_num": self.postgresql_server_version_num,
"postgresql_version_supported": self.postgresql_version_supported,
"alembic_version_state": (
"missing"
if self.revision_rows is None
else "empty"
if not self.revision_rows
else "versioned"
),
"alembic_revision_count": (
None if self.revision_rows is None else len(self.revision_rows)
),
"observed_drift_count": len(self.observed_drift),
"unexpected_drift": [
item.render() for item in sorted(self.unexpected_drift)
],
"approval_rows": self.approval_rows,
"approval_inbound_foreign_keys": self.approval_inbound_foreign_keys,
"schema_privileges_ok": self.schema_privileges_ok,
"table_ownership_ok": self.table_ownership_ok,
}
def _allowed_drift() -> frozenset[DriftKey]:
items = {
DriftKey("add_table", "feishu_app_tickets", ""),
DriftKey("add_table", "feishu_admin_bootstrap_tombstones", ""),
DriftKey("remove_table", APPROVAL_TABLE, ""),
DriftKey(
"add_constraint",
"ai_memory_entries",
"uq_ai_memory_owner_fingerprint",
),
DriftKey(
"add_constraint",
"market_watchlists",
"uq_market_watchlist_owner_symbol",
),
DriftKey(
"remove_constraint",
"ai_memory_entries",
"uq_ai_memory_owner_fingerprint",
),
DriftKey(
"remove_constraint",
"market_watchlists",
"uq_market_watchlist_owner_symbol",
),
DriftKey(
"remove_constraint",
"market_watchlists",
"uq_market_watchlist_actor_symbol",
),
DriftKey(
"add_fk",
"ai_memory_entries",
"owner_id->feishu_users.id",
),
DriftKey(
"add_fk",
"market_watchlists",
"owner_id->feishu_users.id",
),
DriftKey("modify_nullable", "work_tasks", "source_system"),
}
for table_name, columns in _ADD_COLUMNS.items():
items.update(
DriftKey("add_column", table_name, column_name)
for column_name in columns
)
for table_name, indexes in _ADD_INDEXES.items():
items.update(
DriftKey("add_index", table_name, index_name)
for index_name in indexes
)
for table_name, indexes in _REMOVE_INDEXES.items():
items.update(
DriftKey("remove_index", table_name, index_name)
for index_name in indexes
)
for table_name, columns in _MODIFY_DEFAULTS.items():
items.update(
DriftKey("modify_default", table_name, column_name)
for column_name in columns
)
return frozenset(items)
ALLOWED_DRIFT = _allowed_drift()
def _foreign_key_name(constraint: Any) -> str:
local_columns = ",".join(column.name for column in constraint.columns)
remote_columns = ",".join(
element.target_fullname for element in constraint.elements
)
return f"{local_columns}->{remote_columns}"
def _normalize_diff(diff: tuple[Any, ...]) -> DriftKey:
operation = str(diff[0])
if operation in {"add_table", "remove_table"}:
return DriftKey(operation, str(diff[1].name), "")
if operation in {"add_column", "remove_column"}:
return DriftKey(operation, str(diff[2]), str(diff[3].name))
if operation in {"add_index", "remove_index"}:
index = diff[1]
return DriftKey(operation, str(index.table.name), str(index.name))
if operation in {"add_constraint", "remove_constraint"}:
constraint = diff[1]
return DriftKey(
operation,
str(constraint.table.name),
str(constraint.name),
)
if operation in {"add_fk", "remove_fk"}:
constraint = diff[1]
return DriftKey(
operation,
str(constraint.table.name),
_foreign_key_name(constraint),
)
if operation.startswith("modify_"):
return DriftKey(operation, str(diff[2]), str(diff[3]))
return DriftKey(operation, "<unknown>", "<unknown>")
def _metadata_drift(connection: Connection) -> frozenset[DriftKey]:
context = MigrationContext.configure(
connection,
opts={
"compare_type": True,
"compare_server_default": True,
},
)
raw_diffs = compare_metadata(context, Base.metadata)
flattened: list[tuple[Any, ...]] = []
for item in raw_diffs:
if isinstance(item, list):
flattened.extend(item)
else:
flattened.append(item)
return frozenset(_normalize_diff(diff) for diff in flattened)
def _canonical_reflection_value(value: Any) -> Any:
"""Convert SQLAlchemy reflection values into stable JSON-compatible data."""
if value is None or isinstance(value, (bool, int, float, str)):
return value
if isinstance(value, bytes):
return {"bytes_hex": value.hex()}
if isinstance(value, Mapping):
return {
str(key): _canonical_reflection_value(value[key])
for key in sorted(value, key=lambda item: str(item))
}
if isinstance(value, (list, tuple)):
return [_canonical_reflection_value(item) for item in value]
if isinstance(value, (set, frozenset)):
items = [_canonical_reflection_value(item) for item in value]
return sorted(items, key=_stable_json)
return str(value)
def _stable_json(value: Any) -> str:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
)
def _sorted_reflection_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
normalized = [_canonical_reflection_value(item) for item in items]
return sorted(normalized, key=_stable_json)
def _schema_snapshot(connection: Connection) -> list[dict[str, Any]]:
inspector = inspect(connection)
snapshot: list[dict[str, Any]] = []
for table_name in sorted(inspector.get_table_names()):
snapshot.append(
{
"table": table_name,
"columns": [
_canonical_reflection_value(column)
for column in inspector.get_columns(table_name)
],
"pk": _canonical_reflection_value(
inspector.get_pk_constraint(table_name)
),
"uniques": _sorted_reflection_items(
inspector.get_unique_constraints(table_name)
),
"checks": _sorted_reflection_items(
inspector.get_check_constraints(table_name)
),
"fks": _sorted_reflection_items(
inspector.get_foreign_keys(table_name)
),
"indexes": _sorted_reflection_items(
inspector.get_indexes(table_name)
),
}
)
return snapshot
def _impacted_table_row_counts(
connection: Connection,
) -> tuple[tuple[str, int], ...]:
existing_tables = set(inspect(connection).get_table_names())
preparer = connection.dialect.identifier_preparer
rows: list[tuple[str, int]] = []
for table_name in sorted(_IMPACTED_TABLES & existing_tables):
quoted_table = preparer.quote_identifier(table_name)
count = connection.execute(
text(f"SELECT COUNT(*) FROM {quoted_table}")
).scalar_one()
rows.append((table_name, int(count)))
return tuple(rows)
def _row_content_digest(row: Mapping[str, Any]) -> str:
canonical_row = {
str(column_name): _canonical_reflection_value(value)
for column_name, value in row.items()
}
return hashlib.sha256(_stable_json(canonical_row).encode()).hexdigest()
def _table_content_digest(
connection: Connection,
table_name: str,
primary_key_columns: tuple[str, ...],
) -> str:
table = Table(
table_name,
MetaData(),
autoload_with=connection,
resolve_fks=False,
)
statement = select(table)
has_stable_primary_key = bool(primary_key_columns) and all(
column_name in table.c for column_name in primary_key_columns
)
if has_stable_primary_key:
statement = statement.order_by(
*(table.c[column_name].asc() for column_name in primary_key_columns)
)
row_digests = (
_row_content_digest(row)
for row in connection.execute(statement).mappings()
)
if not has_stable_primary_key:
row_digests = iter(sorted(row_digests))
table_digest = hashlib.sha256()
for row_digest in row_digests:
table_digest.update(row_digest.encode("ascii"))
table_digest.update(b"\n")
return table_digest.hexdigest()
def _impacted_table_content_digests(
connection: Connection,
) -> tuple[tuple[str, str], ...]:
inspector = inspect(connection)
existing_tables = set(inspector.get_table_names())
digests: list[tuple[str, str]] = []
for table_name in sorted(_CONTENT_FINGERPRINT_TABLES & existing_tables):
primary_key = inspector.get_pk_constraint(table_name)
primary_key_columns = tuple(
str(column_name)
for column_name in primary_key.get("constrained_columns") or ()
)
digests.append(
(
table_name,
_table_content_digest(
connection,
table_name,
primary_key_columns,
),
)
)
return tuple(digests)
def _schema_fingerprint(
connection: Connection,
impacted_table_row_counts: tuple[tuple[str, int], ...] | None = None,
impacted_table_content_digests: tuple[tuple[str, str], ...] | None = None,
) -> str:
row_counts = (
impacted_table_row_counts
if impacted_table_row_counts is not None
else _impacted_table_row_counts(connection)
)
content_digests = (
impacted_table_content_digests
if impacted_table_content_digests is not None
else _impacted_table_content_digests(connection)
)
payload = _stable_json(
{
"schema": _schema_snapshot(connection),
"impacted_table_row_counts": [
{"table": table_name, "row_count": row_count}
for table_name, row_count in row_counts
],
"impacted_table_content_digests": [
{"table": table_name, "sha256": digest}
for table_name, digest in content_digests
],
}
).encode()
return hashlib.sha256(payload).hexdigest()
def _revision_rows(connection: Connection) -> tuple[str, ...] | None:
if "alembic_version" not in inspect(connection).get_table_names():
return None
rows = connection.execute(
text("SELECT version_num FROM alembic_version ORDER BY version_num")
).scalars()
return tuple(str(row) for row in rows)
def _approval_state(connection: Connection) -> tuple[int | None, int]:
inspector = inspect(connection)
tables = inspector.get_table_names()
if APPROVAL_TABLE not in tables:
return None, 0
rows = int(
connection.execute(
text("SELECT COUNT(*) FROM approval_requests")
).scalar_one()
)
inbound = 0
for table_name in tables:
inbound += sum(
1
for foreign_key in inspector.get_foreign_keys(table_name)
if foreign_key.get("referred_table") == APPROVAL_TABLE
)
return rows, inbound
def _postgres_privileges(
connection: Connection,
) -> tuple[bool | None, bool | None]:
if connection.dialect.name != "postgresql":
return None, None
schema_ok = bool(
connection.execute(
text(
"""
SELECT
has_schema_privilege(current_schema(), 'USAGE')
AND has_schema_privilege(current_schema(), 'CREATE')
"""
)
).scalar_one()
)
ownership_ok = bool(
connection.execute(
text(
"""
SELECT COALESCE(bool_and(pg_has_role(c.relowner, 'USAGE')), true)
FROM pg_class AS c
JOIN pg_namespace AS n ON n.oid = c.relnamespace
WHERE n.nspname = current_schema()
AND c.relname = ANY(:table_names)
"""
),
{"table_names": sorted(_IMPACTED_TABLES)},
).scalar_one()
)
return schema_ok, ownership_ok
def _postgres_version_state(
connection: Connection,
) -> tuple[int | None, bool | None]:
if connection.dialect.name != "postgresql":
return None, None
version_num = int(
connection.execute(
text("SELECT current_setting('server_version_num')")
).scalar_one()
)
return version_num, version_num >= MINIMUM_POSTGRESQL_VERSION_NUM
def audit_connection(connection: Connection) -> BaselineAudit:
"""Return a non-secret, read-only assessment of baseline eligibility."""
observed = _metadata_drift(connection)
approval_rows, approval_inbound = _approval_state(connection)
schema_ok, ownership_ok = _postgres_privileges(connection)
postgres_version_num, postgres_version_supported = _postgres_version_state(
connection
)
impacted_table_row_counts = _impacted_table_row_counts(connection)
impacted_table_content_digests = _impacted_table_content_digests(connection)
return BaselineAudit(
dialect=connection.dialect.name,
fingerprint=_schema_fingerprint(
connection,
impacted_table_row_counts,
impacted_table_content_digests,
),
impacted_table_row_counts=impacted_table_row_counts,
impacted_table_content_digests=impacted_table_content_digests,
postgresql_server_version_num=postgres_version_num,
postgresql_version_supported=postgres_version_supported,
revision_rows=_revision_rows(connection),
observed_drift=observed,
unexpected_drift=observed - ALLOWED_DRIFT,
approval_rows=approval_rows,
approval_inbound_foreign_keys=approval_inbound,
schema_privileges_ok=schema_ok,
table_ownership_ok=ownership_ok,
)
def audit_engine(engine: Engine) -> BaselineAudit:
"""Run the default read-only audit."""
with engine.connect() as connection:
transaction = connection.begin()
try:
if connection.dialect.name == "postgresql":
connection.exec_driver_sql("SET TRANSACTION READ ONLY")
return audit_connection(connection)
finally:
transaction.rollback()
def _alembic_config(connection: Connection) -> Config:
project_root = Path(__file__).resolve().parents[2]
config = Config(str(project_root / "alembic.ini"))
config.set_main_option("script_location", str(project_root / "alembic"))
config.attributes["connection"] = connection
return config
def _validate_expected_head(config: Config) -> None:
heads = tuple(ScriptDirectory.from_config(config).get_heads())
if heads != (TARGET_REVISION,):
raise ReconciliationError("Alembic head changed after this baseline was prepared")
def _acquire_postgres_lock(connection: Connection) -> None:
connection.exec_driver_sql("SET LOCAL lock_timeout = '5s'")
connection.exec_driver_sql("SET LOCAL statement_timeout = '5min'")
connection.execute(
text("SELECT pg_advisory_xact_lock(:lock_key)"),
{"lock_key": ADVISORY_LOCK_KEY},
)
def _lock_impacted_postgres_tables(connection: Connection) -> None:
if connection.dialect.name != "postgresql":
return
existing_tables = set(inspect(connection).get_table_names())
table_names = sorted(_IMPACTED_TABLES & existing_tables)
if not table_names:
return
preparer = connection.dialect.identifier_preparer
quoted_tables = ", ".join(
preparer.quote_identifier(table_name) for table_name in table_names
)
connection.exec_driver_sql(
f"LOCK TABLE {quoted_tables} IN ACCESS EXCLUSIVE MODE"
)
def _validate_apply_preconditions(
audit: BaselineAudit,
expected_fingerprint: str,
) -> None:
if audit.postgresql_version_supported is False:
raise ReconciliationError(
"PostgreSQL 15 or newer is required for NULLS NOT DISTINCT constraints"
)
if audit.fingerprint != expected_fingerprint:
raise ReconciliationError(
"Schema fingerprint changed after dry run; run the audit again"
)
if audit.revision_rows is None:
raise ReconciliationError("alembic_version table is missing")
if audit.revision_rows:
raise ReconciliationError("Database already has an Alembic revision")
if audit.unexpected_drift:
raise ReconciliationError("Schema contains drift outside the approved allowlist")
if audit.approval_rows not in {None, 0}:
raise ReconciliationError("approval_requests is not empty")
if audit.approval_inbound_foreign_keys:
raise ReconciliationError("approval_requests has dependent foreign keys")
if audit.schema_privileges_ok is False or audit.table_ownership_ok is False:
raise ReconciliationError("Database role lacks required schema ownership privileges")
def apply_baseline(
engine: Engine,
expected_fingerprint: str,
*,
require_postgresql: bool = True,
) -> BaselineAudit:
"""Atomically stamp, reconcile, upgrade, and verify the approved schema."""
if not SCHEMA_FINGERPRINT_PATTERN.fullmatch(expected_fingerprint):
raise ReconciliationError("Expected fingerprint must be a lowercase SHA-256 value")
if require_postgresql and engine.dialect.name != "postgresql":
raise ReconciliationError("Baseline apply is supported only on PostgreSQL")
with engine.begin() as connection:
if connection.dialect.name == "postgresql":
_acquire_postgres_lock(connection)
_lock_impacted_postgres_tables(connection)
before = audit_connection(connection)
_validate_apply_preconditions(before, expected_fingerprint)
config = _alembic_config(connection)
_validate_expected_head(config)
command.stamp(config, PREVIOUS_REVISION)
command.upgrade(config, "head")
after = audit_connection(connection)
if after.revision_rows != (TARGET_REVISION,):
raise ReconciliationError("Alembic revision was not advanced atomically")
if after.observed_drift:
raise ReconciliationError("Schema still differs from SQLAlchemy metadata")
if after.approval_rows is not None:
raise ReconciliationError("Obsolete approval_requests table still exists")
return after
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=(
"Audit the platform schema, or atomically apply the approved one-time baseline"
)
)
parser.add_argument(
"--apply",
action="store_true",
help="Apply the baseline; without this flag the command is read-only",
)
parser.add_argument(
"--expected-fingerprint",
help="Exact SHA-256 fingerprint printed by the immediately preceding dry run",
)
return parser
def _validated_cli_database_url() -> str:
settings = get_settings()
try:
target = validate_platform_migration_target(
settings.database_url,
settings.legacy_database_url,
)
except RuntimeError as exc:
raise ReconciliationError(str(exc)) from None
if target[0] != "postgresql":
raise ReconciliationError(
"Schema reconciliation CLI requires a PostgreSQL platform database"
)
return settings.database_url
def main() -> None:
parser = _build_parser()
arguments = parser.parse_args()
if arguments.apply and not arguments.expected_fingerprint:
parser.error("--apply requires --expected-fingerprint")
if not arguments.apply and arguments.expected_fingerprint:
parser.error("--expected-fingerprint is only valid with --apply")
engine: Engine | None = None
try:
engine = create_engine(
_validated_cli_database_url(),
poolclass=NullPool,
)
if arguments.apply:
report = apply_baseline(
engine,
str(arguments.expected_fingerprint),
)
payload = {
"applied": True,
"target_revision": TARGET_REVISION,
**report.public_dict(),
}
else:
report = audit_engine(engine)
payload = {"applied": False, **report.public_dict()}
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
if not arguments.apply and not report.eligible:
raise SystemExit(2)
except ReconciliationError as exc:
print(
json.dumps(
{
"applied": False,
"error": str(exc),
},
ensure_ascii=False,
sort_keys=True,
)
)
raise SystemExit(2) from None
except Exception as exc:
print(
json.dumps(
{
"applied": False,
"error": f"Unexpected {type(exc).__name__}",
},
ensure_ascii=False,
sort_keys=True,
)
)
raise SystemExit(1) from None
finally:
if engine is not None:
engine.dispose()
if __name__ == "__main__":
main()

3
app/tools/run_scheduler.py
View File

@@ -1,9 +1,12 @@
from time import sleep
from app.application.scheduling import create_scheduler
from app.core.config import get_settings
def main() -> None:
if not get_settings().scheduler_enabled:
raise RuntimeError("SCHEDULER_ENABLED must be true for the scheduler process")
scheduler = create_scheduler()
scheduler.start()
try:

90
app/tools/runtime_preflight.py Normal file
View File

@@ -0,0 +1,90 @@
"""Fail-closed checks required before starting managed runtime processes."""
from dataclasses import dataclass
from sqlalchemy import create_engine
from sqlalchemy.pool import NullPool
from app.core.config import Settings, get_settings
from app.core.database.migrations import (
current_alembic_revisions,
expected_alembic_heads,
)
from app.core.database.safety import validate_platform_migration_target
from app.modules.feishu_users.constants import parse_admin_identities
class RuntimePreflightError(RuntimeError):
"""Raised when the configured runtime is not safe to start."""
@dataclass(frozen=True, slots=True)
class RuntimePreflightResult:
revisions: tuple[str, ...]
transport: str
def run_preflight(settings: Settings | None = None) -> RuntimePreflightResult:
"""Validate the platform target, migration head, and enabled transport."""
runtime_settings = settings or get_settings()
validate_platform_migration_target(
runtime_settings.database_url,
runtime_settings.legacy_database_url,
)
if runtime_settings.feishu_event_transport == "long_connection" and (
not runtime_settings.feishu_app_id or not runtime_settings.feishu_app_secret
):
raise RuntimePreflightError(
"FEISHU_APP_ID and FEISHU_APP_SECRET are required for long_connection"
)
if runtime_settings.feishu_admin_identities:
try:
parse_admin_identities(runtime_settings.feishu_admin_identities)
except ValueError as exc:
raise RuntimePreflightError(str(exc)) from exc
expected = expected_alembic_heads()
if len(expected) != 1:
raise RuntimePreflightError("Runtime requires exactly one Alembic head")
engine = None
try:
engine = create_engine(runtime_settings.database_url, poolclass=NullPool)
with engine.connect() as connection:
current = current_alembic_revisions(connection)
except Exception as exc:
raise RuntimePreflightError(
"Platform database schema could not be inspected"
) from exc
finally:
if engine is not None:
engine.dispose()
if current is None:
raise RuntimePreflightError(
"Platform database is not Alembic-versioned; run the approved migration first"
)
if current != expected:
raise RuntimePreflightError(
"Platform database is not at the current Alembic head"
)
return RuntimePreflightResult(
revisions=current,
transport=runtime_settings.feishu_event_transport,
)
def main() -> None:
try:
result = run_preflight()
except (RuntimeError, ValueError) as exc:
raise SystemExit(f"Runtime preflight failed: {exc}") from exc
print(
"Runtime preflight passed: "
f"schema={result.revisions[0]}, transport={result.transport}"
)
if __name__ == "__main__":
main()