feat(feishu): 添加飞书入站事件inbox和混合数据库协调功能

- 实现飞书入站事件持久化inbox机制,支持状态管理、租约锁定和重试退避
- 添加混合数据库基线协调工具,确保平台PostgreSQL结构安全对齐
- 增加运行组件心跳检测和readiness就绪检查机制
- 实现app_ticket事件的安全轮换和验证处理
- 添加生产环境运行编排和fail-closed安全机制
- 支持webhook快速确认和长连接独立进程处理
- 完善个人数据擦除时的待处理事件清理功能
```
This commit is contained in:
2026-07-27 17:14:37 +08:00
parent d7db84571d
commit eb8267ed18
61 changed files with 8703 additions and 183 deletions

View File

@@ -1,6 +1,8 @@
from app.tasks.constants import (
TASK_DISPATCH_PENDING_EVENTS,
TASK_GENERATE_RISK_EVENTS,
TASK_PROCESS_DUE_FEISHU_INBOUND,
TASK_PROCESS_FEISHU_INBOUND,
TASK_PUSH_ATTENDANCE_SUMMARY,
TASK_PUSH_DAILY_BRIEF,
TASK_PUSH_PROJECT_WEEKLY,
@@ -13,15 +15,21 @@ from app.tasks.constants import (
TASK_RUN_MARKET_CLOSE,
TASK_RUN_MARKET_REPORT,
TASK_RUN_SUBSCRIPTION_CYCLE,
TASK_RECORD_WORKER_HEARTBEAT,
)
from app.core.background.task_queue.dispatcher import dispatch_task
from app.core.background.task_queue.events import enqueue_event_dispatch
from app.core.background.task_queue.feishu import (
enqueue_feishu_inbound_cycle,
enqueue_feishu_inbound_event,
)
from app.core.background.task_queue.legacy import (
enqueue_legacy_project_sync,
enqueue_legacy_task_sync,
)
from app.core.background.task_queue.lifecycle import enqueue_lifecycle_report
from app.core.background.task_queue.market import enqueue_market_close, enqueue_market_report
from app.core.background.task_queue.observability import enqueue_worker_heartbeat
from app.core.background.task_queue.reports import (
enqueue_attendance_summary_push,
enqueue_daily_brief_push,
@@ -37,6 +45,8 @@ from app.core.background.task_queue.subscriptions import enqueue_subscription_cy
__all__ = [
"TASK_DISPATCH_PENDING_EVENTS",
"TASK_GENERATE_RISK_EVENTS",
"TASK_PROCESS_DUE_FEISHU_INBOUND",
"TASK_PROCESS_FEISHU_INBOUND",
"TASK_PUSH_ATTENDANCE_SUMMARY",
"TASK_PUSH_DAILY_BRIEF",
"TASK_PUSH_PROJECT_WEEKLY",
@@ -49,15 +59,19 @@ __all__ = [
"TASK_RUN_MARKET_CLOSE",
"TASK_RUN_MARKET_REPORT",
"TASK_RUN_SUBSCRIPTION_CYCLE",
"TASK_RECORD_WORKER_HEARTBEAT",
"dispatch_task",
"enqueue_attendance_summary_push",
"enqueue_daily_brief_push",
"enqueue_event_dispatch",
"enqueue_feishu_inbound_cycle",
"enqueue_feishu_inbound_event",
"enqueue_legacy_project_sync",
"enqueue_legacy_task_sync",
"enqueue_lifecycle_report",
"enqueue_market_close",
"enqueue_market_report",
"enqueue_worker_heartbeat",
"enqueue_project_weekly_push",
"enqueue_risk_progress_push",
"enqueue_risk_event_generation",

View File

@@ -0,0 +1,37 @@
from typing import Any
from app.application.feishu.inbound import (
process_due_feishu_inbound_events,
process_feishu_inbound_event,
)
from app.core.background.task_queue.dispatcher import dispatch_task
from app.core.constants import ActorValue
from app.modules.feishu.constants import FEISHU_INBOUND_BATCH_SIZE
from app.tasks.constants import (
TASK_PROCESS_DUE_FEISHU_INBOUND,
TASK_PROCESS_FEISHU_INBOUND,
)
def enqueue_feishu_inbound_event(
event_key: str,
*,
actor: str = ActorValue.WORKER,
) -> dict[str, Any]:
return dispatch_task(
TASK_PROCESS_FEISHU_INBOUND,
{"event_key": event_key, "actor": actor},
lambda: process_feishu_inbound_event(event_key, actor=actor),
)
def enqueue_feishu_inbound_cycle(
*,
limit: int = FEISHU_INBOUND_BATCH_SIZE,
actor: str = ActorValue.SCHEDULER,
) -> dict[str, Any]:
return dispatch_task(
TASK_PROCESS_DUE_FEISHU_INBOUND,
{"limit": limit, "actor": actor},
lambda: process_due_feishu_inbound_events(limit=limit, actor=actor),
)

View File

@@ -0,0 +1,32 @@
from socket import gethostname
from app.core.background.task_queue.dispatcher import dispatch_task
from app.core.constants import ActorValue
from app.core.database import SessionLocal
from app.modules.observability.constants import HeartbeatComponent
from app.modules.observability.service import ObservabilityService
from app.tasks.constants import TASK_RECORD_WORKER_HEARTBEAT
def enqueue_worker_heartbeat(
actor: str = ActorValue.SCHEDULER,
) -> dict:
"""Ask a Celery worker to prove it can consume tasks."""
return dispatch_task(
TASK_RECORD_WORKER_HEARTBEAT,
{"actor": ActorValue.WORKER},
lambda: _record_inline_heartbeat(actor),
)
def _record_inline_heartbeat(actor: str) -> dict:
db = SessionLocal()
try:
return ObservabilityService(db).record_heartbeat(
component=HeartbeatComponent.WORKER,
instance_id=f"inline:{gethostname()}",
actor=actor,
)
finally:
db.close()

View File

@@ -3,7 +3,7 @@ import os
from functools import lru_cache
from typing import Annotated, Any, Literal
from pydantic import Field, field_validator, model_validator
from pydantic import Field, ValidationInfo, field_validator, model_validator
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
from app.core.constants import (
@@ -12,6 +12,7 @@ from app.core.constants import (
DEFAULT_MODEL_PROVIDER,
DEFAULT_OPENCLAW_ACTION_JSON,
)
from app.core.database.safety import database_password, database_target
_DOTENV_DISABLED = os.getenv("COMPANY_AI_DISABLE_DOTENV", "").strip().lower() in {
"1",
@@ -19,15 +20,43 @@ _DOTENV_DISABLED = os.getenv("COMPANY_AI_DISABLE_DOTENV", "").strip().lower() in
"yes",
"on",
}
_DOTENV_FILE = None if _DOTENV_DISABLED else ".env"
_PRODUCTION_SECRET_MIN_LENGTH = 24
_DATABASE_PASSWORD_MIN_LENGTH = 10
_PLACEHOLDER_SECRET_PARTS = (
"change-me",
"changeme",
"example",
"placeholder",
"replace-with",
"your-",
)
def _is_unsafe_production_secret(value: str) -> bool:
normalized = value.strip().casefold()
return (
len(normalized) < _PRODUCTION_SECRET_MIN_LENGTH
or any(part in normalized for part in _PLACEHOLDER_SECRET_PARTS)
)
def _is_unsafe_database_password(value: str) -> bool:
normalized = value.strip().casefold()
return (
len(normalized) < _DATABASE_PASSWORD_MIN_LENGTH
or any(part in normalized for part in _PLACEHOLDER_SECRET_PARTS)
)
class Settings(BaseSettings):
"""Runtime settings loaded from environment variables and `.env`."""
"""Runtime settings loaded from the environment and optional `.env` file."""
model_config = SettingsConfigDict(
env_file=None if _DOTENV_DISABLED else ".env",
env_file=_DOTENV_FILE,
env_file_encoding="utf-8",
extra="ignore",
hide_input_in_errors=True,
)
app_name: str = "Company AI Management Platform"
@@ -63,6 +92,11 @@ class Settings(BaseSettings):
feishu_encrypt_key: str | None = None
feishu_default_chat_id: str | None = None
feishu_default_tenant_key: str | None = None
feishu_event_transport: Literal[
"disabled",
"webhook",
"long_connection",
] = "disabled"
feishu_user_features_enabled: bool = False
feishu_admin_identities: Annotated[list[str], NoDecode] = Field(default_factory=list)
model_provider: str = DEFAULT_MODEL_PROVIDER
@@ -189,11 +223,13 @@ class Settings(BaseSettings):
@field_validator(
"feishu_app_type",
"feishu_event_transport",
mode="before",
)
@classmethod
def normalize_feishu_app_type(cls, value: Any) -> str:
return str(value or "self").strip().lower()
def normalize_feishu_choice(cls, value: Any, info: ValidationInfo) -> str:
default = "self" if info.field_name == "feishu_app_type" else "disabled"
return str(value or default).strip().lower()
@field_validator(
"openclaw_allowed_tools",
@@ -291,12 +327,34 @@ class Settings(BaseSettings):
errors: list[str] = []
api_key_values = _enabled_keys(self.api_key, self.api_keys)
audit_key_values = _enabled_keys(self.audit_api_key, self.audit_api_keys)
if self.database_url.startswith("sqlite"):
platform_target = database_target(self.database_url)
legacy_target = database_target(self.legacy_database_url)
if platform_target is None or platform_target[0] != "postgresql":
errors.append("DATABASE_URL must use PostgreSQL in production")
platform_password = database_password(self.database_url)
if platform_password and _is_unsafe_database_password(platform_password):
errors.append(
"DATABASE_URL password must use a non-placeholder value of at least "
f"{_DATABASE_PASSWORD_MIN_LENGTH} characters in production"
)
if platform_target is not None and platform_target == legacy_target:
errors.append(
"DATABASE_URL and LEGACY_DATABASE_URL must target different databases"
)
if not api_key_values:
errors.append("API_KEY or API_KEYS is required in production")
elif any(_is_unsafe_production_secret(value) for value in api_key_values):
errors.append(
"API_KEY/API_KEYS must use non-placeholder values of at least "
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
)
if not audit_key_values:
errors.append("AUDIT_API_KEY or AUDIT_API_KEYS is required in production")
elif any(_is_unsafe_production_secret(value) for value in audit_key_values):
errors.append(
"AUDIT_API_KEY/AUDIT_API_KEYS must use non-placeholder values of at least "
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
)
if api_key_values & audit_key_values:
errors.append(
"API_KEY/API_KEYS and AUDIT_API_KEY/AUDIT_API_KEYS cannot overlap in production"
@@ -309,10 +367,52 @@ class Settings(BaseSettings):
errors.append("MASK_SENSITIVE_RESPONSES must be true in production")
if not self.read_only_mode:
errors.append("READ_ONLY_MODE must be true in production")
if self.feishu_user_features_enabled and not self.feishu_admin_identities:
errors.append(
"FEISHU_ADMIN_IDENTITIES is required when Feishu user features are enabled"
)
if self.feishu_event_transport != "disabled":
if not self.feishu_app_id or not self.feishu_app_secret:
errors.append(
"FEISHU_APP_ID and FEISHU_APP_SECRET are required when "
"Feishu event transport is enabled"
)
elif _is_unsafe_production_secret(self.feishu_app_secret):
errors.append(
"FEISHU_APP_SECRET must use a non-placeholder value of at least "
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
)
if (
self.feishu_event_transport == "webhook"
and not self.feishu_verification_token
):
errors.append(
"FEISHU_VERIFICATION_TOKEN is required for Feishu webhook transport"
)
elif (
self.feishu_event_transport == "webhook"
and self.feishu_verification_token
and _is_unsafe_production_secret(self.feishu_verification_token)
):
errors.append(
"FEISHU_VERIFICATION_TOKEN must use a non-placeholder value of at "
f"least {_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
)
if self.feishu_user_features_enabled:
if not self.feishu_admin_identities:
errors.append(
"FEISHU_ADMIN_IDENTITIES is required when Feishu user features are enabled"
)
else:
try:
from app.modules.feishu_users.constants import (
parse_admin_identities,
)
parse_admin_identities(self.feishu_admin_identities)
except ValueError as exc:
errors.append(str(exc))
if self.feishu_event_transport == "disabled":
errors.append(
"FEISHU_EVENT_TRANSPORT must be webhook or long_connection "
"when Feishu user features are enabled"
)
if errors:
raise ValueError("; ".join(errors))
return self

View File

@@ -0,0 +1,26 @@
from pathlib import Path
from alembic.config import Config
from alembic.script import ScriptDirectory
from sqlalchemy import inspect, text
from sqlalchemy.engine import Connection
def expected_alembic_heads(project_root: Path | None = None) -> tuple[str, ...]:
"""Return the repository's configured Alembic heads."""
root = project_root or Path(__file__).resolve().parents[3]
config = Config(str(root / "alembic.ini"))
config.set_main_option("script_location", str(root / "alembic"))
return tuple(sorted(ScriptDirectory.from_config(config).get_heads()))
def current_alembic_revisions(connection: Connection) -> tuple[str, ...] | None:
"""Return database revisions, or ``None`` when it was never versioned."""
if "alembic_version" not in inspect(connection).get_table_names():
return None
revisions = connection.execute(
text("SELECT version_num FROM alembic_version ORDER BY version_num")
).scalars()
return tuple(str(revision) for revision in revisions)

View File

@@ -0,0 +1,62 @@
from sqlalchemy.engine import URL, make_url
from sqlalchemy.exc import ArgumentError
_DEFAULT_DATABASE_PORTS = {
"mysql": 3306,
"postgresql": 5432,
}
_PLATFORM_MIGRATION_BACKENDS = frozenset({"postgresql", "sqlite"})
DatabaseTarget = tuple[str, str, int | None, str]
def database_target(value: str | URL | None) -> DatabaseTarget | None:
"""Return a credential-free physical database identity."""
if not value:
return None
try:
url = make_url(value)
except (ArgumentError, TypeError, ValueError):
return None
backend = url.get_backend_name().lower()
return (
backend,
str(url.host or "").casefold(),
url.port or _DEFAULT_DATABASE_PORTS.get(backend),
str(url.database or ""),
)
def database_password(value: str | URL | None) -> str | None:
"""Return a configured password without including it in diagnostics."""
if not value:
return None
try:
password = make_url(value).password
except (ArgumentError, TypeError, ValueError):
return None
return str(password) if password is not None else None
def validate_platform_migration_target(
database_url: str | URL,
legacy_database_url: str | None,
) -> DatabaseTarget:
"""Fail closed before Alembic can connect to an unsafe database target."""
platform = database_target(database_url)
if platform is None:
raise RuntimeError("DATABASE_URL is not a valid platform database URL")
if platform[0] not in _PLATFORM_MIGRATION_BACKENDS:
raise RuntimeError(
"Platform migrations are supported only for PostgreSQL or local SQLite"
)
legacy = database_target(legacy_database_url)
if legacy is not None and platform == legacy:
raise RuntimeError(
"Refusing to migrate DATABASE_URL because it matches LEGACY_DATABASE_URL"
)
return platform