```
feat(feishu): 添加飞书入站事件inbox和混合数据库协调功能 - 实现飞书入站事件持久化inbox机制,支持状态管理、租约锁定和重试退避 - 添加混合数据库基线协调工具,确保平台PostgreSQL结构安全对齐 - 增加运行组件心跳检测和readiness就绪检查机制 - 实现app_ticket事件的安全轮换和验证处理 - 添加生产环境运行编排和fail-closed安全机制 - 支持webhook快速确认和长连接独立进程处理 - 完善个人数据擦除时的待处理事件清理功能 ```
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
from app.tasks.constants import (
|
||||
TASK_DISPATCH_PENDING_EVENTS,
|
||||
TASK_GENERATE_RISK_EVENTS,
|
||||
TASK_PROCESS_DUE_FEISHU_INBOUND,
|
||||
TASK_PROCESS_FEISHU_INBOUND,
|
||||
TASK_PUSH_ATTENDANCE_SUMMARY,
|
||||
TASK_PUSH_DAILY_BRIEF,
|
||||
TASK_PUSH_PROJECT_WEEKLY,
|
||||
@@ -13,15 +15,21 @@ from app.tasks.constants import (
|
||||
TASK_RUN_MARKET_CLOSE,
|
||||
TASK_RUN_MARKET_REPORT,
|
||||
TASK_RUN_SUBSCRIPTION_CYCLE,
|
||||
TASK_RECORD_WORKER_HEARTBEAT,
|
||||
)
|
||||
from app.core.background.task_queue.dispatcher import dispatch_task
|
||||
from app.core.background.task_queue.events import enqueue_event_dispatch
|
||||
from app.core.background.task_queue.feishu import (
|
||||
enqueue_feishu_inbound_cycle,
|
||||
enqueue_feishu_inbound_event,
|
||||
)
|
||||
from app.core.background.task_queue.legacy import (
|
||||
enqueue_legacy_project_sync,
|
||||
enqueue_legacy_task_sync,
|
||||
)
|
||||
from app.core.background.task_queue.lifecycle import enqueue_lifecycle_report
|
||||
from app.core.background.task_queue.market import enqueue_market_close, enqueue_market_report
|
||||
from app.core.background.task_queue.observability import enqueue_worker_heartbeat
|
||||
from app.core.background.task_queue.reports import (
|
||||
enqueue_attendance_summary_push,
|
||||
enqueue_daily_brief_push,
|
||||
@@ -37,6 +45,8 @@ from app.core.background.task_queue.subscriptions import enqueue_subscription_cy
|
||||
__all__ = [
|
||||
"TASK_DISPATCH_PENDING_EVENTS",
|
||||
"TASK_GENERATE_RISK_EVENTS",
|
||||
"TASK_PROCESS_DUE_FEISHU_INBOUND",
|
||||
"TASK_PROCESS_FEISHU_INBOUND",
|
||||
"TASK_PUSH_ATTENDANCE_SUMMARY",
|
||||
"TASK_PUSH_DAILY_BRIEF",
|
||||
"TASK_PUSH_PROJECT_WEEKLY",
|
||||
@@ -49,15 +59,19 @@ __all__ = [
|
||||
"TASK_RUN_MARKET_CLOSE",
|
||||
"TASK_RUN_MARKET_REPORT",
|
||||
"TASK_RUN_SUBSCRIPTION_CYCLE",
|
||||
"TASK_RECORD_WORKER_HEARTBEAT",
|
||||
"dispatch_task",
|
||||
"enqueue_attendance_summary_push",
|
||||
"enqueue_daily_brief_push",
|
||||
"enqueue_event_dispatch",
|
||||
"enqueue_feishu_inbound_cycle",
|
||||
"enqueue_feishu_inbound_event",
|
||||
"enqueue_legacy_project_sync",
|
||||
"enqueue_legacy_task_sync",
|
||||
"enqueue_lifecycle_report",
|
||||
"enqueue_market_close",
|
||||
"enqueue_market_report",
|
||||
"enqueue_worker_heartbeat",
|
||||
"enqueue_project_weekly_push",
|
||||
"enqueue_risk_progress_push",
|
||||
"enqueue_risk_event_generation",
|
||||
|
||||
37
app/core/background/task_queue/feishu.py
Normal file
37
app/core/background/task_queue/feishu.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from typing import Any
|
||||
|
||||
from app.application.feishu.inbound import (
|
||||
process_due_feishu_inbound_events,
|
||||
process_feishu_inbound_event,
|
||||
)
|
||||
from app.core.background.task_queue.dispatcher import dispatch_task
|
||||
from app.core.constants import ActorValue
|
||||
from app.modules.feishu.constants import FEISHU_INBOUND_BATCH_SIZE
|
||||
from app.tasks.constants import (
|
||||
TASK_PROCESS_DUE_FEISHU_INBOUND,
|
||||
TASK_PROCESS_FEISHU_INBOUND,
|
||||
)
|
||||
|
||||
|
||||
def enqueue_feishu_inbound_event(
|
||||
event_key: str,
|
||||
*,
|
||||
actor: str = ActorValue.WORKER,
|
||||
) -> dict[str, Any]:
|
||||
return dispatch_task(
|
||||
TASK_PROCESS_FEISHU_INBOUND,
|
||||
{"event_key": event_key, "actor": actor},
|
||||
lambda: process_feishu_inbound_event(event_key, actor=actor),
|
||||
)
|
||||
|
||||
|
||||
def enqueue_feishu_inbound_cycle(
|
||||
*,
|
||||
limit: int = FEISHU_INBOUND_BATCH_SIZE,
|
||||
actor: str = ActorValue.SCHEDULER,
|
||||
) -> dict[str, Any]:
|
||||
return dispatch_task(
|
||||
TASK_PROCESS_DUE_FEISHU_INBOUND,
|
||||
{"limit": limit, "actor": actor},
|
||||
lambda: process_due_feishu_inbound_events(limit=limit, actor=actor),
|
||||
)
|
||||
32
app/core/background/task_queue/observability.py
Normal file
32
app/core/background/task_queue/observability.py
Normal file
@@ -0,0 +1,32 @@
|
||||
from socket import gethostname
|
||||
|
||||
from app.core.background.task_queue.dispatcher import dispatch_task
|
||||
from app.core.constants import ActorValue
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.observability.constants import HeartbeatComponent
|
||||
from app.modules.observability.service import ObservabilityService
|
||||
from app.tasks.constants import TASK_RECORD_WORKER_HEARTBEAT
|
||||
|
||||
|
||||
def enqueue_worker_heartbeat(
|
||||
actor: str = ActorValue.SCHEDULER,
|
||||
) -> dict:
|
||||
"""Ask a Celery worker to prove it can consume tasks."""
|
||||
|
||||
return dispatch_task(
|
||||
TASK_RECORD_WORKER_HEARTBEAT,
|
||||
{"actor": ActorValue.WORKER},
|
||||
lambda: _record_inline_heartbeat(actor),
|
||||
)
|
||||
|
||||
|
||||
def _record_inline_heartbeat(actor: str) -> dict:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
return ObservabilityService(db).record_heartbeat(
|
||||
component=HeartbeatComponent.WORKER,
|
||||
instance_id=f"inline:{gethostname()}",
|
||||
actor=actor,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
from functools import lru_cache
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
from pydantic import Field, ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
||||
|
||||
from app.core.constants import (
|
||||
@@ -12,6 +12,7 @@ from app.core.constants import (
|
||||
DEFAULT_MODEL_PROVIDER,
|
||||
DEFAULT_OPENCLAW_ACTION_JSON,
|
||||
)
|
||||
from app.core.database.safety import database_password, database_target
|
||||
|
||||
_DOTENV_DISABLED = os.getenv("COMPANY_AI_DISABLE_DOTENV", "").strip().lower() in {
|
||||
"1",
|
||||
@@ -19,15 +20,43 @@ _DOTENV_DISABLED = os.getenv("COMPANY_AI_DISABLE_DOTENV", "").strip().lower() in
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
_DOTENV_FILE = None if _DOTENV_DISABLED else ".env"
|
||||
_PRODUCTION_SECRET_MIN_LENGTH = 24
|
||||
_DATABASE_PASSWORD_MIN_LENGTH = 10
|
||||
_PLACEHOLDER_SECRET_PARTS = (
|
||||
"change-me",
|
||||
"changeme",
|
||||
"example",
|
||||
"placeholder",
|
||||
"replace-with",
|
||||
"your-",
|
||||
)
|
||||
|
||||
|
||||
def _is_unsafe_production_secret(value: str) -> bool:
|
||||
normalized = value.strip().casefold()
|
||||
return (
|
||||
len(normalized) < _PRODUCTION_SECRET_MIN_LENGTH
|
||||
or any(part in normalized for part in _PLACEHOLDER_SECRET_PARTS)
|
||||
)
|
||||
|
||||
|
||||
def _is_unsafe_database_password(value: str) -> bool:
|
||||
normalized = value.strip().casefold()
|
||||
return (
|
||||
len(normalized) < _DATABASE_PASSWORD_MIN_LENGTH
|
||||
or any(part in normalized for part in _PLACEHOLDER_SECRET_PARTS)
|
||||
)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Runtime settings loaded from environment variables and `.env`."""
|
||||
"""Runtime settings loaded from the environment and optional `.env` file."""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=None if _DOTENV_DISABLED else ".env",
|
||||
env_file=_DOTENV_FILE,
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
hide_input_in_errors=True,
|
||||
)
|
||||
|
||||
app_name: str = "Company AI Management Platform"
|
||||
@@ -63,6 +92,11 @@ class Settings(BaseSettings):
|
||||
feishu_encrypt_key: str | None = None
|
||||
feishu_default_chat_id: str | None = None
|
||||
feishu_default_tenant_key: str | None = None
|
||||
feishu_event_transport: Literal[
|
||||
"disabled",
|
||||
"webhook",
|
||||
"long_connection",
|
||||
] = "disabled"
|
||||
feishu_user_features_enabled: bool = False
|
||||
feishu_admin_identities: Annotated[list[str], NoDecode] = Field(default_factory=list)
|
||||
model_provider: str = DEFAULT_MODEL_PROVIDER
|
||||
@@ -189,11 +223,13 @@ class Settings(BaseSettings):
|
||||
|
||||
@field_validator(
|
||||
"feishu_app_type",
|
||||
"feishu_event_transport",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
def normalize_feishu_app_type(cls, value: Any) -> str:
|
||||
return str(value or "self").strip().lower()
|
||||
def normalize_feishu_choice(cls, value: Any, info: ValidationInfo) -> str:
|
||||
default = "self" if info.field_name == "feishu_app_type" else "disabled"
|
||||
return str(value or default).strip().lower()
|
||||
|
||||
@field_validator(
|
||||
"openclaw_allowed_tools",
|
||||
@@ -291,12 +327,34 @@ class Settings(BaseSettings):
|
||||
errors: list[str] = []
|
||||
api_key_values = _enabled_keys(self.api_key, self.api_keys)
|
||||
audit_key_values = _enabled_keys(self.audit_api_key, self.audit_api_keys)
|
||||
if self.database_url.startswith("sqlite"):
|
||||
platform_target = database_target(self.database_url)
|
||||
legacy_target = database_target(self.legacy_database_url)
|
||||
if platform_target is None or platform_target[0] != "postgresql":
|
||||
errors.append("DATABASE_URL must use PostgreSQL in production")
|
||||
platform_password = database_password(self.database_url)
|
||||
if platform_password and _is_unsafe_database_password(platform_password):
|
||||
errors.append(
|
||||
"DATABASE_URL password must use a non-placeholder value of at least "
|
||||
f"{_DATABASE_PASSWORD_MIN_LENGTH} characters in production"
|
||||
)
|
||||
if platform_target is not None and platform_target == legacy_target:
|
||||
errors.append(
|
||||
"DATABASE_URL and LEGACY_DATABASE_URL must target different databases"
|
||||
)
|
||||
if not api_key_values:
|
||||
errors.append("API_KEY or API_KEYS is required in production")
|
||||
elif any(_is_unsafe_production_secret(value) for value in api_key_values):
|
||||
errors.append(
|
||||
"API_KEY/API_KEYS must use non-placeholder values of at least "
|
||||
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
|
||||
)
|
||||
if not audit_key_values:
|
||||
errors.append("AUDIT_API_KEY or AUDIT_API_KEYS is required in production")
|
||||
elif any(_is_unsafe_production_secret(value) for value in audit_key_values):
|
||||
errors.append(
|
||||
"AUDIT_API_KEY/AUDIT_API_KEYS must use non-placeholder values of at least "
|
||||
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
|
||||
)
|
||||
if api_key_values & audit_key_values:
|
||||
errors.append(
|
||||
"API_KEY/API_KEYS and AUDIT_API_KEY/AUDIT_API_KEYS cannot overlap in production"
|
||||
@@ -309,10 +367,52 @@ class Settings(BaseSettings):
|
||||
errors.append("MASK_SENSITIVE_RESPONSES must be true in production")
|
||||
if not self.read_only_mode:
|
||||
errors.append("READ_ONLY_MODE must be true in production")
|
||||
if self.feishu_user_features_enabled and not self.feishu_admin_identities:
|
||||
errors.append(
|
||||
"FEISHU_ADMIN_IDENTITIES is required when Feishu user features are enabled"
|
||||
)
|
||||
if self.feishu_event_transport != "disabled":
|
||||
if not self.feishu_app_id or not self.feishu_app_secret:
|
||||
errors.append(
|
||||
"FEISHU_APP_ID and FEISHU_APP_SECRET are required when "
|
||||
"Feishu event transport is enabled"
|
||||
)
|
||||
elif _is_unsafe_production_secret(self.feishu_app_secret):
|
||||
errors.append(
|
||||
"FEISHU_APP_SECRET must use a non-placeholder value of at least "
|
||||
f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
|
||||
)
|
||||
if (
|
||||
self.feishu_event_transport == "webhook"
|
||||
and not self.feishu_verification_token
|
||||
):
|
||||
errors.append(
|
||||
"FEISHU_VERIFICATION_TOKEN is required for Feishu webhook transport"
|
||||
)
|
||||
elif (
|
||||
self.feishu_event_transport == "webhook"
|
||||
and self.feishu_verification_token
|
||||
and _is_unsafe_production_secret(self.feishu_verification_token)
|
||||
):
|
||||
errors.append(
|
||||
"FEISHU_VERIFICATION_TOKEN must use a non-placeholder value of at "
|
||||
f"least {_PRODUCTION_SECRET_MIN_LENGTH} characters in production"
|
||||
)
|
||||
if self.feishu_user_features_enabled:
|
||||
if not self.feishu_admin_identities:
|
||||
errors.append(
|
||||
"FEISHU_ADMIN_IDENTITIES is required when Feishu user features are enabled"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
from app.modules.feishu_users.constants import (
|
||||
parse_admin_identities,
|
||||
)
|
||||
|
||||
parse_admin_identities(self.feishu_admin_identities)
|
||||
except ValueError as exc:
|
||||
errors.append(str(exc))
|
||||
if self.feishu_event_transport == "disabled":
|
||||
errors.append(
|
||||
"FEISHU_EVENT_TRANSPORT must be webhook or long_connection "
|
||||
"when Feishu user features are enabled"
|
||||
)
|
||||
if errors:
|
||||
raise ValueError("; ".join(errors))
|
||||
return self
|
||||
|
||||
26
app/core/database/migrations.py
Normal file
26
app/core/database/migrations.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from pathlib import Path
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.script import ScriptDirectory
|
||||
from sqlalchemy import inspect, text
|
||||
from sqlalchemy.engine import Connection
|
||||
|
||||
|
||||
def expected_alembic_heads(project_root: Path | None = None) -> tuple[str, ...]:
|
||||
"""Return the repository's configured Alembic heads."""
|
||||
|
||||
root = project_root or Path(__file__).resolve().parents[3]
|
||||
config = Config(str(root / "alembic.ini"))
|
||||
config.set_main_option("script_location", str(root / "alembic"))
|
||||
return tuple(sorted(ScriptDirectory.from_config(config).get_heads()))
|
||||
|
||||
|
||||
def current_alembic_revisions(connection: Connection) -> tuple[str, ...] | None:
|
||||
"""Return database revisions, or ``None`` when it was never versioned."""
|
||||
|
||||
if "alembic_version" not in inspect(connection).get_table_names():
|
||||
return None
|
||||
revisions = connection.execute(
|
||||
text("SELECT version_num FROM alembic_version ORDER BY version_num")
|
||||
).scalars()
|
||||
return tuple(str(revision) for revision in revisions)
|
||||
62
app/core/database/safety.py
Normal file
62
app/core/database/safety.py
Normal file
@@ -0,0 +1,62 @@
|
||||
from sqlalchemy.engine import URL, make_url
|
||||
from sqlalchemy.exc import ArgumentError
|
||||
|
||||
|
||||
_DEFAULT_DATABASE_PORTS = {
|
||||
"mysql": 3306,
|
||||
"postgresql": 5432,
|
||||
}
|
||||
_PLATFORM_MIGRATION_BACKENDS = frozenset({"postgresql", "sqlite"})
|
||||
|
||||
DatabaseTarget = tuple[str, str, int | None, str]
|
||||
|
||||
|
||||
def database_target(value: str | URL | None) -> DatabaseTarget | None:
|
||||
"""Return a credential-free physical database identity."""
|
||||
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
url = make_url(value)
|
||||
except (ArgumentError, TypeError, ValueError):
|
||||
return None
|
||||
backend = url.get_backend_name().lower()
|
||||
return (
|
||||
backend,
|
||||
str(url.host or "").casefold(),
|
||||
url.port or _DEFAULT_DATABASE_PORTS.get(backend),
|
||||
str(url.database or ""),
|
||||
)
|
||||
|
||||
|
||||
def database_password(value: str | URL | None) -> str | None:
|
||||
"""Return a configured password without including it in diagnostics."""
|
||||
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
password = make_url(value).password
|
||||
except (ArgumentError, TypeError, ValueError):
|
||||
return None
|
||||
return str(password) if password is not None else None
|
||||
|
||||
|
||||
def validate_platform_migration_target(
|
||||
database_url: str | URL,
|
||||
legacy_database_url: str | None,
|
||||
) -> DatabaseTarget:
|
||||
"""Fail closed before Alembic can connect to an unsafe database target."""
|
||||
|
||||
platform = database_target(database_url)
|
||||
if platform is None:
|
||||
raise RuntimeError("DATABASE_URL is not a valid platform database URL")
|
||||
if platform[0] not in _PLATFORM_MIGRATION_BACKENDS:
|
||||
raise RuntimeError(
|
||||
"Platform migrations are supported only for PostgreSQL or local SQLite"
|
||||
)
|
||||
legacy = database_target(legacy_database_url)
|
||||
if legacy is not None and platform == legacy:
|
||||
raise RuntimeError(
|
||||
"Refusing to migrate DATABASE_URL because it matches LEGACY_DATABASE_URL"
|
||||
)
|
||||
return platform
|
||||
Reference in New Issue
Block a user