From eb8267ed18f51d4ecfb1acfcdfc13d5e051f71ee Mon Sep 17 00:00:00 2001 From: JiuContinent Date: Mon, 27 Jul 2026 17:14:37 +0800 Subject: [PATCH] =?UTF-8?q?```=20feat(feishu):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E9=A3=9E=E4=B9=A6=E5=85=A5=E7=AB=99=E4=BA=8B=E4=BB=B6inbox?= =?UTF-8?q?=E5=92=8C=E6=B7=B7=E5=90=88=E6=95=B0=E6=8D=AE=E5=BA=93=E5=8D=8F?= =?UTF-8?q?=E8=B0=83=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 实现飞书入站事件持久化inbox机制，支持状态管理、租约锁定和重试退避 - 添加混合数据库基线协调工具，确保平台PostgreSQL结构安全对齐 - 增加运行组件心跳检测和readiness就绪检查机制 - 实现app_ticket事件的安全轮换和验证处理 - 添加生产环境运行编排和fail-closed安全机制 - 支持webhook快速确认和长连接独立进程处理 - 完善个人数据擦除时的待处理事件清理功能 ``` --- .../feishu-account-personalization/design.md | 66 + .../requirements.md | 54 + .../feishu-account-personalization/tasks.md | 27 +- .dockerignore | 1 + .gitignore | 1 + alembic/env.py | 19 + ...1_reconcile_unversioned_platform_schema.py | 663 +++++++++ ...202607270002_feishu_inbound_event_inbox.py | 251 ++++ app/application/feishu/commands.py | 41 +- app/application/feishu/events.py | 411 ++++-- app/application/feishu/inbound.py | 52 + app/application/feishu/personal_data.py | 26 +- app/application/scheduling/scheduler.py | 30 +- app/core/background/task_queue/__init__.py | 14 + app/core/background/task_queue/feishu.py | 37 + .../background/task_queue/observability.py | 32 + app/core/config/settings.py | 120 +- app/core/database/migrations.py | 26 + app/core/database/safety.py | 62 + app/main.py | 2 + app/modules/ai_memory/models.py | 1 + app/modules/business/models/market.py | 7 +- app/modules/feishu/constants.py | 18 + app/modules/feishu/long_connection.py | 155 +- app/modules/feishu/models.py | 78 +- app/modules/feishu/routes.py | 32 +- app/modules/feishu/service.py | 56 +- app/modules/feishu/services/__init__.py | 17 + app/modules/feishu/services/context.py | 23 + app/modules/feishu/services/inbox.py | 1305 +++++++++++++++++ app/modules/feishu/services/reply_outbox.py | 247 ++++ app/modules/feishu_users/constants.py | 10 +- app/modules/feishu_users/identifiers.py | 20 + app/modules/observability/constants.py | 5 + app/modules/observability/runtime.py | 73 + app/modules/observability/service.py | 211 ++- app/tasks/__init__.py | 2 + app/tasks/constants.py | 3 + app/tasks/feishu.py | 29 + app/tasks/observability.py | 25 + app/tools/reconcile_platform_schema.py | 924 ++++++++++++ app/tools/run_scheduler.py | 3 + app/tools/runtime_preflight.py | 90 ++ docker-compose.external-db.yml | 53 + docker-compose.yml | 62 +- scripts/start_runtime.ps1 | 178 +++ scripts/stop_runtime.ps1 | 44 + tests/conftest.py | 1 + tests/test_feishu_app_ticket.py | 75 +- tests/test_feishu_inbound_reliability.py | 1220 +++++++++++++++ tests/test_feishu_long_connection_health.py | 226 +++ .../test_feishu_personalization_migration.py | 2 +- tests/test_feishu_subscription_commands.py | 131 +- tests/test_feishu_subscription_readiness.py | 9 +- tests/test_personal_data_erasure.py | 38 + tests/test_runtime_activation.py | 608 ++++++++ tests/test_schema_reconciliation.py | 626 ++++++++ tests/test_schema_reconciliation_postgres.py | 281 ++++ tests/test_smoke.py | 49 +- tests/test_subscription_runtime_wiring.py | 1 + tests/test_v2_hardening.py | 13 +- 61 files changed, 8703 insertions(+), 183 deletions(-) create mode 100644 alembic/versions/202607270001_reconcile_unversioned_platform_schema.py create mode 100644 alembic/versions/202607270002_feishu_inbound_event_inbox.py create mode 100644 app/application/feishu/inbound.py create mode 100644 app/core/background/task_queue/feishu.py create mode 100644 app/core/background/task_queue/observability.py create mode 100644 app/core/database/migrations.py create mode 100644 app/core/database/safety.py create mode 100644 app/modules/feishu/services/__init__.py create mode 100644 app/modules/feishu/services/context.py create mode 100644 app/modules/feishu/services/inbox.py create mode 100644 app/modules/feishu/services/reply_outbox.py create mode 100644 app/modules/feishu_users/identifiers.py create mode 100644 app/modules/observability/runtime.py create mode 100644 app/tasks/feishu.py create mode 100644 app/tasks/observability.py create mode 100644 app/tools/reconcile_platform_schema.py create mode 100644 app/tools/runtime_preflight.py create mode 100644 docker-compose.external-db.yml create mode 100644 scripts/start_runtime.ps1 create mode 100644 scripts/stop_runtime.ps1 create mode 100644 tests/test_feishu_inbound_reliability.py create mode 100644 tests/test_feishu_long_connection_health.py create mode 100644 tests/test_runtime_activation.py create mode 100644 tests/test_schema_reconciliation.py create mode 100644 tests/test_schema_reconciliation_postgres.py diff --git a/.claude/specs/feishu-account-personalization/design.md b/.claude/specs/feishu-account-personalization/design.md index 95f9b50..9e8f4e2 100644 --- a/.claude/specs/feishu-account-personalization/design.md +++ b/.claude/specs/feishu-account-personalization/design.md @@ -103,6 +103,34 @@ sequenceDiagram 使用 `FEISHU_DEFAULT_TENANT_KEY`，不得复用另一租户的令牌。 - AI adapter context 增加每用户 session id；`noop` 明确返回不可用且不写会话、偏好或记忆。 +### 混合数据库协调与运行编排 + +- `app/tools/reconcile_platform_schema.py` 只面向平台 PostgreSQL。它先生成结构指纹和允许列表 + dry-run，获取 PostgreSQL advisory transaction lock 后执行一次性基线协调；实际结构、 + 行数、依赖、权限或版本任一不符即中止。 +- 协调 revision 只补齐当前 metadata 缺失的表、列、索引、外键和唯一语义；除已审核为空且 + 无依赖的遗留表外不删除远端对象。完成后在同一事务内验证零漂移并登记 Alembic revision。 +- `FEISHU_EVENT_TRANSPORT` 明确选择 `disabled|webhook|long_connection`。生产环境开启用户 + 功能时必须具备相应凭据；长连接使用独立受管进程，不嵌入 API 工作线程。 +- API、scheduler、worker 和长连接进程使用 heartbeat 表报告存活。readiness 同时校验 + Alembic head、期望组件 heartbeat、队列和飞书依赖；缺失或陈旧时返回 HTTP 503。 +- Compose 提供内部 PostgreSQL默认编排和不覆盖 `DATABASE_URL` 的外部数据库覆盖方式， + 并为持续进程配置重启策略。运行配置只使用本地且不提交的 `.env`。 + +### 飞书入站 inbox + +- 已验证的消息事件先写入 `FeishuEventReceipt` inbox，再由持久化扫描任务执行。 +- inbox 保存处理所需的短期规范化载荷、状态、尝试次数、下次尝试时间、租约和最小错误摘要； + 成功或最终失败后清除原始载荷，避免长期保存个人消息。 +- 命令事务内通过 reply outbox 捕获唯一的文本、卡片或图片回复意图、目标租户和稳定 UUID； + 只有命令写入、receipt 成功状态与回复意图一起提交后才允许访问飞书网络。 +- 回复使用独立状态、租约和退避时间。发送失败或发送成功后进程在状态提交前退出时，只以 + 同一内容和 UUID 重试回复，不重新执行用户命令；终态后清除回复载荷。 +- webhook 在验真、挑战处理和 inbox 入库后立即确认；长连接 SDK 回调复用同一入库路径。 +- 执行器通过唯一事件键、数据库锁、租约 token 和状态条件更新防止并发重复执行。失败进入 + 有界重试，进程崩溃后由过期租约重新领取；成功事件的后续重复投递只返回已有状态。 +- `app_ticket` 仍在已验证边界内同步轮换，不把票据写入普通消息 inbox。 + ## 数据模型 ### `FeishuUser` @@ -158,6 +186,15 @@ sequenceDiagram - `id`, `app_id`, `app_ticket`, `received_at`, `updated_at` - `app_id` 全局唯一；只保留当前有效票据，不保留票据历史。 +### `FeishuEventReceipt` inbox 扩展 + +- `event_key`, `source`, `event_id`, `message_id`, `received_at` +- `status`, `payload`, `attempt_count`, `next_attempt_at` +- `locked_until`, `lock_token`, `last_error`, `processed_at`, `updated_at` +- `reply_payload`, `reply_status`, `reply_attempt_count`, `reply_next_attempt_at` +- `reply_locked_until`, `reply_locked_by`, `reply_last_error`, `reply_sent_at` +- 旧 receipt 迁为已成功状态；新消息使用唯一 `event_key` 保证幂等。 + ## 业务流程 ### 身份与权限 @@ -219,6 +256,28 @@ flowchart TD `pending/retry` 状态。唯一投递键由 `subscription_id + scheduled_for` 派生；同一键重复任务 返回已有结果，不再次调用飞书。 +### 入站事件处理 + +```mermaid +flowchart TD + F["飞书已验证事件"] --> C{"挑战或 app_ticket？"} + C -- 是 --> S["立即安全处理并确认"] + C -- 否 --> I["幂等写入 inbox pending"] + I --> A["立即 ACK"] + W["持久化扫描器"] --> L["领取租约 processing"] + L --> E["执行身份、权限和命令"] + E --> O{"结果"} + O -- 成功 --> X["原子提交 succeeded 与 reply outbox"] + O -- 可重试 --> R["retry + 退避时间"] + O -- 超限 --> Z["failed + 最小错误并清空 payload"] + X --> Q["独立领取并发送 reply"] + Q --> Y{"发送结果"} + Y -- 成功 --> C["reply succeeded 并清空载荷"] + Y -- 可重试 --> QR["仅重试同一内容与 UUID"] + QR --> W + R --> W +``` + ### 忘记我 首次命令仅保存哈希确认码与短期过期时间。确认后在一个事务内删除个人规则/记忆、偏好、 @@ -237,6 +296,10 @@ flowchart TD - 商店应用缺少可用 `app_ticket` 或默认租户时 readiness 返回 degraded；自建应用若启用订阅 涉及多个租户时 readiness 返回 degraded，避免把单租户令牌错误用于其他租户。 - 数据库竞争：依赖唯一约束兜底；冲突后回滚到保存点并读取已存在投递。 +- 混合数据库基线不匹配：协调工具中止并输出不含凭据/数据的结构差异，不自动猜测或 stamp。 +- 期望运行组件无 heartbeat、Alembic 非 head 或事件 transport 不可用：readiness 返回 503。 +- 入站命令失败或进程丢失租约：保留短期 inbox payload 并按退避重试；成功或最终失败后清除。 +- 入站回复失败或进程在发送后退出：保留短期 reply payload 并以稳定 UUID 重试，绝不重跑命令。 ## 测试策略 @@ -249,3 +312,6 @@ flowchart TD 图片/订阅均使用目标租户、固定群任务使用默认租户。 - 迁移测试：Alembic head 与元数据一致，旧规则/记忆/自选按既定策略迁移。 - 回归测试：现有固定群报表调度与现有内部接口继续工作。 +- 基线测试：用远端结构的脱敏快照验证 dry-run 指纹、允许列表、事务回滚和零漂移。 +- 运行测试：Compose 契约、transport 配置、Alembic head、组件 heartbeat 与 fail-closed。 +- 入站测试：快速 ACK、失败重试、租约回收、并发重复只执行一次、成功后不重复。 diff --git a/.claude/specs/feishu-account-personalization/requirements.md b/.claude/specs/feishu-account-personalization/requirements.md index b517cac..083a2c2 100644 --- a/.claude/specs/feishu-account-personalization/requirements.md +++ b/.claude/specs/feishu-account-personalization/requirements.md @@ -211,3 +211,57 @@ 飞书非零业务码和数据库迁移一致性。 6. WHEN 完整验证运行 THEN Ruff、Python 编译检查、Alembic 元数据一致性测试和全部 pytest 测试 SHALL 通过。 + +### 需求 11：混合数据库安全基线 + +**用户故事：** 作为维护人员，我希望现有未登记 Alembic 版本的混合平台数据库可以安全 +协调到当前模型，而不丢失已有平台数据或误操作业务数据库。 + +#### 验收条件 + +1. WHEN 协调工具连接数据库 THEN 系统 SHALL 只接受平台 PostgreSQL，且不得连接或修改 + `LEGACY_DATABASE_URL`。 +2. WHEN 数据库版本为空且结构符合已审核的混合基线 THEN 系统 SHALL 在事务锁内仅执行 + 允许列表中的结构补齐、旧约束替换和已确认空表清理。 +3. IF 实际结构指纹、数据行数、依赖关系、权限或 Alembic 版本不符合预检 THEN 系统 SHALL + 中止且不得留下部分 DDL 或版本记录。 +4. WHEN 协调完成 THEN 系统 SHALL 验证 SQLAlchemy metadata 无漂移，再原子登记 Alembic + 版本；IF 任一步失败 THEN 全部变更 SHALL 回滚。 +5. WHEN 常规数据库已经位于受支持 revision THEN 系统 SHALL 继续使用标准 Alembic 升级， + 不得重复执行一次性基线协调。 + +### 需求 12：生产运行与就绪判定 + +**用户故事：** 作为运维人员，我希望 API、飞书事件、调度器和任务执行进程可以持续运行， +且系统只在依赖和迁移真正就绪时接收流量。 + +#### 验收条件 + +1. WHEN 启用飞书用户功能 THEN 系统 SHALL 明确选择 `webhook` 或 `long_connection` 事件入口； + IF 对应凭据或进程未就绪 THEN production 配置或 readiness SHALL 失败关闭。 +2. WHEN 使用长连接部署 THEN 运行编排 SHALL 启动独立飞书事件进程，并为 API、事件进程、 + scheduler 和 worker 配置重启策略。 +3. WHEN scheduler、worker 或长连接属于当前配置期望组件 THEN 系统 SHALL 要求其存在新鲜 + heartbeat；缺失或过期 SHALL 使 readiness 返回 HTTP 503。 +4. WHEN readiness 检查平台数据库 THEN 系统 SHALL 验证当前 Alembic revision 与 head 一致， + 而不只执行 `SELECT 1`。 +5. WHEN 生成部署配置样例 THEN 系统 SHALL 只包含占位符和安全默认值，不得提交真实密钥； + 外部 PostgreSQL 部署 SHALL 能避免被 Compose 内部数据库 URL 强制覆盖。 + +### 需求 13：可靠的飞书入站事件 + +**用户故事：** 作为飞书用户，我希望机器人在临时故障或进程重启后仍能处理我的消息， +同时不会因飞书重复投递而重复执行命令。 + +#### 验收条件 + +1. WHEN webhook 或长连接收到已经验证的事件 THEN 系统 SHALL 先以唯一事件键持久化 inbox + 状态，再快速确认接收，不得在 webhook 请求内同步等待 AI 或飞书回复。 +2. IF 多个进程并发接收相同事件 THEN 系统 SHALL 只创建一个 inbox 记录，并只允许一个 + 有效租约执行命令。 +3. IF 命令执行发生可重试失败或执行进程在完成前退出 THEN 事件 SHALL 回到持久化重试状态， + 不得因为 receipt 已登记而永久丢失。 +4. WHEN 命令执行成功 THEN 系统 SHALL 原子标记成功；后续重复事件 SHALL 返回已接收且不得 + 再次执行命令。 +5. WHEN 重试超过上限 THEN 系统 SHALL 标记最终失败、保存最小错误摘要并提供运行指标， + 不得记录密钥或完整敏感消息内容。 diff --git a/.claude/specs/feishu-account-personalization/tasks.md b/.claude/specs/feishu-account-personalization/tasks.md index 4734dad..b87b121 100644 --- a/.claude/specs/feishu-account-personalization/tasks.md +++ b/.claude/specs/feishu-account-personalization/tasks.md @@ -47,12 +47,37 @@ - _Requirements: 6.2, 9, 10.2, 10.4_ - [x] 8. 完成配置、Alembic 迁移与全量验证 - - 增加安全关闭的功能开关、管理员身份配置并更新 `.env.example`。 + - 增加安全关闭的功能开关、管理员身份配置并更新 `.env` 配置。 - 创建迁移并处理旧公司规则、旧自动记忆和旧自选延迟认领。 - 补齐身份、隔离、权限、计划、并发、投递、删除和迁移回归测试。 - 运行 Ruff、compileall、完整 pytest 和迁移一致性检查。 - _Requirements: 3.7, 10_ +- [x] 9. 安全协调现有混合平台数据库 + - 新增严格允许列表的 reconciliation revision 和一次性 baseline runner。 + - 实现结构指纹、advisory transaction lock、权限/行数/依赖预检和原子版本登记。 + - 用脱敏结构快照验证成功、漂移拒绝和失败回滚；不得连接或修改业务 MySQL。 + - _Requirements: 10, 11_ + +- [x] 10. 补齐生产运行编排和 fail-closed readiness + - 增加明确事件 transport、生产配置校验、`.env` 配置和长连接服务编排。 + - 为 API、scheduler、worker、飞书事件进程增加重启/heartbeat，并校验 Alembic head。 + - 支持外部 PostgreSQL 部署且不被 Compose 内部数据库 URL 意外覆盖。 + - _Requirements: 12_ + +- [x] 11. 实现可靠飞书入站 inbox + - 扩展事件 receipt 为持久化 inbox，支持状态、租约、退避重试和终态载荷清理。 + - webhook 快速 ACK，长连接共用入库路径；并发重复和进程重启不得重复或丢失命令。 + - 命令事务原子保存 reply outbox，回复失败只以稳定 UUID 重试，不重新执行命令。 + - 接入 scheduler/Celery/inline 执行适配并增加运行指标。 + - _Requirements: 1, 9, 13_ + +- [ ] 12. 应用平台迁移并完成真实运行验收 + - 停止相关进程，执行 dry-run、备份/事务预检、协调迁移和零漂移验证。 + - 启动 API、事件入口和 scheduler，验证 health/readiness、heartbeat 与无待处理错误。 + - 通过真实已验证事件绑定首位管理员，并完成私聊订阅和群订阅 smoke。 + - _Requirements: 11, 12, 13_ + ```mermaid flowchart LR T1["1 身份权限"] --> T2["2 事件主体"] diff --git a/.dockerignore b/.dockerignore index 28a1b76..6d410ca 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,7 @@ __pycache__/ .ruff_cache/ logs/ +.runtime/ docs/ migration/ README.md diff --git a/.gitignore b/.gitignore index c9b5504..925070a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ # Runtime logs /logs/ +/.runtime/ *.log # Local docs and agent instructions diff --git a/alembic/env.py b/alembic/env.py index ad710c2..6e3a7a5 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -5,6 +5,7 @@ from sqlalchemy import engine_from_config, pool from app.core.config import get_settings from app.core.database import Base +from app.core.database.safety import validate_platform_migration_target from app.modules.ai_memory import models as ai_memory_models from app.modules.audit import models as audit_models from app.modules.business import models as business_models @@ -23,6 +24,10 @@ if config.config_file_name is not None: target_metadata = Base.metadata settings = get_settings() +validate_platform_migration_target( + settings.database_url, + settings.legacy_database_url, +) # Keep imports referenced so SQLAlchemy model classes register with Base.metadata. _REGISTERED_MODEL_MODULES = ( @@ -52,6 +57,20 @@ def run_migrations_offline() -> None: def run_migrations_online() -> None: + supplied_connection = config.attributes.get("connection") + if supplied_connection is not None: + validate_platform_migration_target( + supplied_connection.engine.url, + settings.legacy_database_url, + ) + context.configure( + connection=supplied_connection, + target_metadata=target_metadata, + ) + with context.begin_transaction(): + context.run_migrations() + return + configuration = config.get_section(config.config_ini_section, {}) configuration["sqlalchemy.url"] = settings.database_url connectable = engine_from_config( diff --git a/alembic/versions/202607270001_reconcile_unversioned_platform_schema.py b/alembic/versions/202607270001_reconcile_unversioned_platform_schema.py new file mode 100644 index 0000000..d08624d --- /dev/null +++ b/alembic/versions/202607270001_reconcile_unversioned_platform_schema.py @@ -0,0 +1,663 @@ +"""Reconcile the known unversioned PostgreSQL schema. + +Revision ID: 202607270001 +Revises: 202607260005 +Create Date: 2026-07-27 + +This revision is intentionally idempotent. A normally versioned database at +202607260005 only receives the PostgreSQL NULLS NOT DISTINCT hardening. The +one-time baseline runner may also use it, in the same transaction as an atomic +stamp, to repair the explicitly allowlisted mixed schema. +""" + +from collections.abc import Iterable + +from alembic import op +import sqlalchemy as sa + + +revision = "202607270001" +down_revision = "202607260005" +branch_labels = None +depends_on = None + +AI_MEMORY_TABLE = "ai_memory_entries" +MARKET_WATCHLIST_TABLE = "market_watchlists" +APPROVAL_TABLE = "approval_requests" + +AI_MEMORY_OWNER_UNIQUE = "uq_ai_memory_owner_fingerprint" +MARKET_WATCHLIST_OWNER_UNIQUE = "uq_market_watchlist_owner_symbol" +MARKET_WATCHLIST_LEGACY_UNIQUE = "uq_market_watchlist_actor_symbol" + + +def _inspector() -> sa.Inspector: + return sa.inspect(op.get_bind()) + + +def _table_exists(table_name: str) -> bool: + return table_name in _inspector().get_table_names() + + +def _column_names(table_name: str) -> set[str]: + return { + str(column["name"]) + for column in _inspector().get_columns(table_name) + } + + +def _add_missing_columns( + table_name: str, + columns: Iterable[sa.Column], +) -> None: + existing = _column_names(table_name) + for column in columns: + if column.name not in existing: + op.add_column(table_name, column) + + +def _index_by_name(table_name: str, index_name: str) -> dict | None: + return next( + ( + index + for index in _inspector().get_indexes(table_name) + if index.get("name") == index_name + ), + None, + ) + + +def _ensure_index( + table_name: str, + index_name: str, + columns: tuple[str, ...], + *, + unique: bool = False, +) -> None: + existing = _index_by_name(table_name, index_name) + if existing is not None: + existing_columns = tuple(existing.get("column_names") or ()) + if existing_columns != columns or bool(existing.get("unique")) != unique: + raise RuntimeError( + f"Existing index {index_name} does not match the reconciliation definition" + ) + return + op.create_index(index_name, table_name, list(columns), unique=unique) + + +def _drop_index_if_present(table_name: str, index_name: str) -> None: + if _index_by_name(table_name, index_name) is not None: + op.drop_index(index_name, table_name=table_name) + + +def _unique_by_name(table_name: str, constraint_name: str) -> dict | None: + return next( + ( + constraint + for constraint in _inspector().get_unique_constraints(table_name) + if constraint.get("name") == constraint_name + ), + None, + ) + + +def _drop_unique_if_present(table_name: str, constraint_name: str) -> None: + if _unique_by_name(table_name, constraint_name) is None: + return + with op.batch_alter_table(table_name) as batch_op: + batch_op.drop_constraint(constraint_name, type_="unique") + + +def _uses_nulls_not_distinct(constraint: dict) -> bool: + options = constraint.get("dialect_options") or {} + return bool(options.get("postgresql_nulls_not_distinct")) + + +def _ensure_owner_unique( + table_name: str, + constraint_name: str, + columns: tuple[str, ...], +) -> None: + existing = _unique_by_name(table_name, constraint_name) + dialect_name = op.get_bind().dialect.name + if existing is not None: + if tuple(existing.get("column_names") or ()) != columns: + raise RuntimeError( + f"Existing constraint {constraint_name} does not match the reconciliation definition" + ) + if dialect_name != "postgresql" or _uses_nulls_not_distinct(existing): + return + with op.batch_alter_table(table_name) as batch_op: + batch_op.drop_constraint(constraint_name, type_="unique") + + with op.batch_alter_table(table_name) as batch_op: + batch_op.create_unique_constraint( + constraint_name, + list(columns), + postgresql_nulls_not_distinct=True, + ) + + +def _ensure_foreign_key( + table_name: str, + constraint_name: str, + local_columns: tuple[str, ...], + remote_table: str, + remote_columns: tuple[str, ...], +) -> None: + for foreign_key in _inspector().get_foreign_keys(table_name): + if ( + tuple(foreign_key.get("constrained_columns") or ()) == local_columns + and foreign_key.get("referred_table") == remote_table + and tuple(foreign_key.get("referred_columns") or ()) == remote_columns + ): + ondelete = str((foreign_key.get("options") or {}).get("ondelete") or "") + if ondelete.upper() != "CASCADE": + raise RuntimeError( + f"Existing foreign key on {table_name} does not use ON DELETE CASCADE" + ) + return + with op.batch_alter_table(table_name) as batch_op: + batch_op.create_foreign_key( + constraint_name, + remote_table, + list(local_columns), + list(remote_columns), + ondelete="CASCADE", + ) + + +def _alter_without_server_default( + table_name: str, + columns: tuple[tuple[str, sa.types.TypeEngine], ...], +) -> None: + with op.batch_alter_table(table_name) as batch_op: + for column_name, column_type in columns: + batch_op.alter_column( + column_name, + existing_type=column_type, + nullable=False, + server_default=None, + ) + + +def _add_business_columns() -> None: + _add_missing_columns( + "attendance_records", + ( + sa.Column( + "attendance_scope", + sa.String(length=32), + nullable=False, + server_default="company", + ), + sa.Column("source_status", sa.String(length=64), nullable=True), + sa.Column("source_location_status", sa.String(length=64), nullable=True), + sa.Column("source_updated_at", sa.DateTime(), nullable=True), + sa.Column("last_seen_at", sa.DateTime(), nullable=True), + sa.Column( + "is_active", + sa.Boolean(), + nullable=False, + server_default=sa.true(), + ), + ), + ) + _add_missing_columns( + "audit_logs", + (sa.Column("request_id", sa.String(length=64), nullable=True),), + ) + _add_missing_columns( + "projects", + ( + sa.Column("display_code", sa.String(length=64), nullable=True), + sa.Column("department_code", sa.String(length=64), nullable=True), + sa.Column("department_name", sa.String(length=128), nullable=True), + sa.Column("owner_employee_code", sa.String(length=64), nullable=True), + sa.Column("source_stage", sa.String(length=32), nullable=True), + sa.Column("source_stage_label", sa.String(length=128), nullable=True), + sa.Column( + "source_archived", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column("source_created_at", sa.DateTime(), nullable=True), + sa.Column("source_updated_at", sa.DateTime(), nullable=True), + sa.Column("last_seen_at", sa.DateTime(), nullable=True), + sa.Column( + "is_active", + sa.Boolean(), + nullable=False, + server_default=sa.true(), + ), + sa.Column( + "source_contract_amount", + sa.Numeric(precision=16, scale=2), + nullable=True, + ), + sa.Column( + "source_project_investment_amount", + sa.Numeric(precision=16, scale=2), + nullable=True, + ), + ), + ) + _add_missing_columns( + "risk_events", + ( + sa.Column("assigned_to", sa.String(length=128), nullable=True), + sa.Column("resolved_at", sa.DateTime(), nullable=True), + sa.Column("closed_at", sa.DateTime(), nullable=True), + sa.Column("closed_reason", sa.Text(), nullable=True), + sa.Column("review_summary", sa.Text(), nullable=True), + ), + ) + _add_missing_columns( + "work_reports", + ( + sa.Column("employee_code", sa.String(length=64), nullable=True), + sa.Column("external_id", sa.String(length=128), nullable=True), + sa.Column( + "is_late", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column( + "is_draft", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column("source_updated_at", sa.DateTime(), nullable=True), + sa.Column("last_seen_at", sa.DateTime(), nullable=True), + sa.Column( + "is_active", + sa.Boolean(), + nullable=False, + server_default=sa.true(), + ), + ), + ) + _add_missing_columns( + "work_tasks", + ( + sa.Column( + "source_system", + sa.String(length=64), + nullable=False, + server_default="internal", + ), + sa.Column("external_id", sa.String(length=128), nullable=True), + sa.Column("employee_code", sa.String(length=64), nullable=True), + sa.Column("source_created_at", sa.DateTime(), nullable=True), + sa.Column("source_updated_at", sa.DateTime(), nullable=True), + sa.Column("last_seen_at", sa.DateTime(), nullable=True), + sa.Column( + "is_active", + sa.Boolean(), + nullable=False, + server_default=sa.true(), + ), + ), + ) + + op.execute( + sa.text( + """ + UPDATE attendance_records + SET attendance_scope = 'company' + WHERE attendance_scope IS NULL + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE attendance_records + SET is_active = true + WHERE is_active IS NULL + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE projects + SET source_archived = false + WHERE source_archived IS NULL + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE projects + SET is_active = true + WHERE is_active IS NULL + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE work_reports + SET is_late = false + WHERE is_late IS NULL + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE work_reports + SET is_draft = false + WHERE is_draft IS NULL + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE work_reports + SET is_active = true + WHERE is_active IS NULL + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE work_tasks + SET source_system = 'internal' + WHERE source_system IS NULL + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE work_tasks + SET is_active = true + WHERE is_active IS NULL + """ + ) + ) + + _alter_without_server_default( + "attendance_records", + ( + ("attendance_scope", sa.String(length=32)), + ("is_active", sa.Boolean()), + ), + ) + _alter_without_server_default( + "projects", + ( + ("source_archived", sa.Boolean()), + ("is_active", sa.Boolean()), + ), + ) + _alter_without_server_default( + "work_reports", + ( + ("is_late", sa.Boolean()), + ("is_draft", sa.Boolean()), + ("is_active", sa.Boolean()), + ), + ) + _alter_without_server_default( + "work_tasks", + ( + ("source_system", sa.String(length=64)), + ("is_active", sa.Boolean()), + ), + ) + + +def _ensure_business_indexes() -> None: + definitions = { + "attendance_records": ( + "attendance_scope", + "is_active", + "last_seen_at", + "source_status", + "source_updated_at", + ), + "audit_logs": ("request_id",), + "projects": ( + "department_code", + "department_name", + "display_code", + "is_active", + "last_seen_at", + "owner_employee_code", + "source_archived", + "source_stage", + "source_updated_at", + ), + "risk_events": ("assigned_to",), + "work_reports": ( + "employee_code", + "external_id", + "is_active", + "is_draft", + "is_late", + "last_seen_at", + "source_updated_at", + ), + "work_tasks": ( + "employee_code", + "external_id", + "is_active", + "last_seen_at", + "source_updated_at", + ), + } + for table_name, columns in definitions.items(): + for column_name in columns: + _ensure_index( + table_name, + f"ix_{table_name}_{column_name}", + (column_name,), + ) + + +def _reconcile_ai_memory() -> None: + _add_missing_columns( + AI_MEMORY_TABLE, + ( + sa.Column("owner_id", sa.Integer(), nullable=True), + sa.Column( + "kind", + sa.String(length=32), + nullable=False, + server_default="memory", + ), + ), + ) + op.execute( + sa.text( + """ + UPDATE ai_memory_entries + SET + owner_id = NULL, + kind = 'company_rule', + source = 'legacy_company', + status = 'active' + WHERE source = 'user_rule' + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE ai_memory_entries + SET kind = 'memory', status = 'archived' + WHERE owner_id IS NULL AND source IN ('auto', 'hermes') + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE ai_memory_entries + SET kind = 'memory' + WHERE kind IS NULL + """ + ) + ) + _alter_without_server_default( + AI_MEMORY_TABLE, + (("kind", sa.String(length=32)),), + ) + + fingerprint_index = _index_by_name( + AI_MEMORY_TABLE, + "ix_ai_memory_entries_fingerprint", + ) + if fingerprint_index is not None and bool(fingerprint_index.get("unique")): + op.drop_index( + "ix_ai_memory_entries_fingerprint", + table_name=AI_MEMORY_TABLE, + ) + _ensure_index( + AI_MEMORY_TABLE, + "ix_ai_memory_entries_fingerprint", + ("fingerprint",), + ) + _ensure_index( + AI_MEMORY_TABLE, + "ix_ai_memory_entries_kind", + ("kind",), + ) + _ensure_index( + AI_MEMORY_TABLE, + "ix_ai_memory_entries_owner_id", + ("owner_id",), + ) + _ensure_foreign_key( + AI_MEMORY_TABLE, + "fk_ai_memory_entries_owner_id", + ("owner_id",), + "feishu_users", + ("id",), + ) + _ensure_owner_unique( + AI_MEMORY_TABLE, + AI_MEMORY_OWNER_UNIQUE, + ("owner_id", "fingerprint"), + ) + + +def _reconcile_market_watchlists() -> None: + _add_missing_columns( + MARKET_WATCHLIST_TABLE, + (sa.Column("owner_id", sa.Integer(), nullable=True),), + ) + _drop_unique_if_present( + MARKET_WATCHLIST_TABLE, + MARKET_WATCHLIST_LEGACY_UNIQUE, + ) + _ensure_index( + MARKET_WATCHLIST_TABLE, + "ix_market_watchlists_owner_id", + ("owner_id",), + ) + _ensure_foreign_key( + MARKET_WATCHLIST_TABLE, + "fk_market_watchlists_owner_id", + ("owner_id",), + "feishu_users", + ("id",), + ) + _ensure_owner_unique( + MARKET_WATCHLIST_TABLE, + MARKET_WATCHLIST_OWNER_UNIQUE, + ("owner_id", "symbol"), + ) + + +def _create_feishu_app_tickets_if_missing() -> None: + if not _table_exists("feishu_app_tickets"): + op.create_table( + "feishu_app_tickets", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("app_id", sa.String(length=128), nullable=False), + sa.Column("app_ticket", sa.Text(), nullable=False), + sa.Column("received_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + _ensure_index( + "feishu_app_tickets", + "ix_feishu_app_tickets_app_id", + ("app_id",), + unique=True, + ) + _ensure_index( + "feishu_app_tickets", + "ix_feishu_app_tickets_received_at", + ("received_at",), + ) + + +def _create_admin_tombstones_if_missing() -> None: + if not _table_exists("feishu_admin_bootstrap_tombstones"): + op.create_table( + "feishu_admin_bootstrap_tombstones", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("identity_hash", sa.String(length=64), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + _ensure_index( + "feishu_admin_bootstrap_tombstones", + "ix_feishu_admin_bootstrap_tombstones_created_at", + ("created_at",), + ) + _ensure_index( + "feishu_admin_bootstrap_tombstones", + "ix_feishu_admin_bootstrap_tombstones_identity_hash", + ("identity_hash",), + unique=True, + ) + + +def _drop_empty_approval_table() -> None: + if not _table_exists(APPROVAL_TABLE): + return + count = int( + op.get_bind().execute( + sa.text("SELECT COUNT(*) FROM approval_requests") + ).scalar_one() + ) + if count: + raise RuntimeError( + "Refusing to remove approval_requests because it contains rows" + ) + for table_name in _inspector().get_table_names(): + for foreign_key in _inspector().get_foreign_keys(table_name): + if foreign_key.get("referred_table") == APPROVAL_TABLE: + raise RuntimeError( + "Refusing to remove approval_requests because a foreign key depends on it" + ) + op.drop_table(APPROVAL_TABLE) + + +def upgrade() -> None: + _add_business_columns() + _ensure_business_indexes() + _reconcile_ai_memory() + _reconcile_market_watchlists() + _create_feishu_app_tickets_if_missing() + _create_admin_tombstones_if_missing() + _drop_empty_approval_table() + + +def downgrade() -> None: + raise RuntimeError( + "Revision 202607270001 is irreversible: it classifies legacy data, " + "adds ownership relationships, and removes the obsolete approval table. " + "Restore a verified database backup instead of downgrading." + ) diff --git a/alembic/versions/202607270002_feishu_inbound_event_inbox.py b/alembic/versions/202607270002_feishu_inbound_event_inbox.py new file mode 100644 index 0000000..25685e2 --- /dev/null +++ b/alembic/versions/202607270002_feishu_inbound_event_inbox.py @@ -0,0 +1,251 @@ +"""Upgrade Feishu event receipts to a durable inbound inbox. + +Revision ID: 202607270002 +Revises: 202607270001 +Create Date: 2026-07-27 +""" + +import re +from hashlib import sha256 +from typing import Any + +from alembic import op +import sqlalchemy as sa + + +revision = "202607270002" +down_revision = "202607270001" +branch_labels = None +depends_on = None + +TABLE_NAME = "feishu_event_receipts" +INDEX_COLUMNS = ( + "event_type", + "status", + "next_attempt_at", + "locked_until", + "locked_by", + "processed_at", + "reply_status", + "reply_next_attempt_at", + "reply_locked_until", + "reply_locked_by", + "reply_sent_at", +) +IDENTIFIER_DIGEST_PATTERN = re.compile(r"sha256-[0-9a-f]{64}\Z") +IDENTIFIER_DIGEST_PREFIX = "company-ai-platform:feishu" + + +def _identifier_digest(value: Any, domain: str) -> str | None: + text = str(value or "").strip() + if not text: + return None + if IDENTIFIER_DIGEST_PATTERN.fullmatch(text): + return text + digest = sha256( + f"{IDENTIFIER_DIGEST_PREFIX}:{domain}:v1\0{text}".encode("utf-8") + ).hexdigest() + return f"sha256-{digest}" + + +def _migrate_historical_identifiers() -> None: + """Replace legacy raw receipt identifiers without hashing them twice. + + A strict current-format digest is the only durable marker available in the + legacy schema, so matching values are treated as already migrated. A raw + identifier that happens to match that format is therefore intentionally + indistinguishable and remains unchanged. + """ + + bind = op.get_bind() + receipts = sa.table( + TABLE_NAME, + sa.column("id", sa.Integer()), + sa.column("event_key", sa.String()), + sa.column("event_id", sa.String()), + sa.column("message_id", sa.String()), + ) + rows = bind.execute( + sa.select( + receipts.c.id, + receipts.c.event_key, + receipts.c.event_id, + receipts.c.message_id, + ).order_by(receipts.c.id) + ).mappings() + updates: list[tuple[int, dict[str, str | None]]] = [] + target_event_keys: dict[str, int] = {} + for row in rows: + row_id = int(row["id"]) + values = { + "event_key": _identifier_digest(row["event_key"], "event-key"), + "event_id": _identifier_digest(row["event_id"], "event-id"), + "message_id": _identifier_digest(row["message_id"], "message-id"), + } + target_event_key = values["event_key"] + if target_event_key is None: + raise RuntimeError( + f"Cannot migrate blank Feishu receipt event_key at row {row_id}" + ) + conflicting_row_id = target_event_keys.setdefault(target_event_key, row_id) + if conflicting_row_id != row_id: + raise RuntimeError( + "Cannot migrate colliding Feishu receipt event_key values at " + f"rows {conflicting_row_id} and {row_id}" + ) + if any(values[column] != row[column] for column in values): + updates.append((row_id, values)) + + for row_id, values in updates: + bind.execute( + sa.update(receipts).where(receipts.c.id == row_id).values(**values) + ) + + +def upgrade() -> None: + _migrate_historical_identifiers() + + op.add_column( + TABLE_NAME, + sa.Column("event_type", sa.String(length=128), nullable=True), + ) + op.add_column(TABLE_NAME, sa.Column("payload", sa.JSON(), nullable=True)) + op.add_column( + TABLE_NAME, + sa.Column( + "auto_reply", + sa.Boolean(), + server_default=sa.true(), + nullable=False, + ), + ) + op.add_column( + TABLE_NAME, + sa.Column( + "status", + sa.String(length=32), + server_default="pending", + nullable=False, + ), + ) + op.add_column( + TABLE_NAME, + sa.Column( + "attempt_count", + sa.Integer(), + server_default="0", + nullable=False, + ), + ) + op.add_column( + TABLE_NAME, + sa.Column( + "max_attempts", + sa.Integer(), + server_default="4", + nullable=False, + ), + ) + op.add_column(TABLE_NAME, sa.Column("last_error", sa.Text(), nullable=True)) + op.add_column( + TABLE_NAME, + sa.Column("next_attempt_at", sa.DateTime(), nullable=True), + ) + op.add_column( + TABLE_NAME, + sa.Column("locked_until", sa.DateTime(), nullable=True), + ) + op.add_column( + TABLE_NAME, + sa.Column("locked_by", sa.String(length=128), nullable=True), + ) + op.add_column( + TABLE_NAME, + sa.Column("processed_at", sa.DateTime(), nullable=True), + ) + op.add_column(TABLE_NAME, sa.Column("reply_payload", sa.JSON(), nullable=True)) + op.add_column( + TABLE_NAME, + sa.Column("reply_status", sa.String(length=32), nullable=True), + ) + op.add_column( + TABLE_NAME, + sa.Column( + "reply_attempt_count", + sa.Integer(), + server_default="0", + nullable=False, + ), + ) + op.add_column( + TABLE_NAME, + sa.Column("reply_last_error", sa.Text(), nullable=True), + ) + op.add_column( + TABLE_NAME, + sa.Column("reply_next_attempt_at", sa.DateTime(), nullable=True), + ) + op.add_column( + TABLE_NAME, + sa.Column("reply_locked_until", sa.DateTime(), nullable=True), + ) + op.add_column( + TABLE_NAME, + sa.Column("reply_locked_by", sa.String(length=128), nullable=True), + ) + op.add_column( + TABLE_NAME, + sa.Column("reply_sent_at", sa.DateTime(), nullable=True), + ) + + # Historical receipt-only rows have no replayable payload. Treat them as + # completed so deploying the inbox cannot execute old commands. + op.execute( + sa.text( + f""" + UPDATE {TABLE_NAME} + SET status = 'succeeded', + attempt_count = 1, + processed_at = received_at + """ + ) + ) + + for column in INDEX_COLUMNS: + op.create_index( + op.f(f"ix_{TABLE_NAME}_{column}"), + TABLE_NAME, + [column], + unique=False, + ) + + +def downgrade() -> None: + # Identifier digests are intentionally irreversible and remain in place. + for column in reversed(INDEX_COLUMNS): + op.drop_index( + op.f(f"ix_{TABLE_NAME}_{column}"), + table_name=TABLE_NAME, + ) + for column in ( + "processed_at", + "locked_by", + "locked_until", + "next_attempt_at", + "last_error", + "max_attempts", + "attempt_count", + "status", + "auto_reply", + "payload", + "event_type", + "reply_sent_at", + "reply_locked_by", + "reply_locked_until", + "reply_next_attempt_at", + "reply_last_error", + "reply_attempt_count", + "reply_status", + "reply_payload", + ): + op.drop_column(TABLE_NAME, column) diff --git a/app/application/feishu/commands.py b/app/application/feishu/commands.py index 4a25760..b39ad64 100644 --- a/app/application/feishu/commands.py +++ b/app/application/feishu/commands.py @@ -65,19 +65,44 @@ FINANCE_COMMAND_PREFIXES = ("资金需求", "未来30天资金需求", "项目 def _parse_content_text(content: Any) -> str: """Extract plain command text from a Feishu message content payload.""" - if isinstance(content, dict): - return str( - content.get(FeishuPayloadKey.TEXT) or content.get(FeishuPayloadKey.CONTENT) or "" - ) if not isinstance(content, str): - return "" + return _structured_content_text(content) try: data = json.loads(content) except json.JSONDecodeError: return content - if isinstance(data, dict): - return str(data.get(FeishuPayloadKey.TEXT) or data.get(FeishuPayloadKey.CONTENT) or "") - return content + return _structured_content_text(data) + + +def _structured_content_text(value: Any) -> str: + """Flatten Feishu text and rich-post content without stringifying metadata.""" + + if isinstance(value, str): + return value + if isinstance(value, list): + separator = "\n" if any(isinstance(item, list) for item in value) else "" + return separator.join( + text + for item in value + if (text := _structured_content_text(item)) + ) + if not isinstance(value, dict): + return "" + + tag = str(value.get("tag") or "").strip().lower() + if tag == "br": + return "\n" + if tag == "at": + # Authorization uses the event's structured mentions, not display text. + return " " + + text_value = value.get(FeishuPayloadKey.TEXT) + if isinstance(text_value, str): + return text_value + if FeishuPayloadKey.CONTENT in value: + return _structured_content_text(value.get(FeishuPayloadKey.CONTENT)) + title = value.get("title") + return title if isinstance(title, str) else "" def _clean_command_text(text: str) -> str: diff --git a/app/application/feishu/events.py b/app/application/feishu/events.py index 79a6494..147a10c 100644 --- a/app/application/feishu/events.py +++ b/app/application/feishu/events.py @@ -1,8 +1,10 @@ +import json from dataclasses import replace +from dataclasses import dataclass +from hashlib import sha256 from typing import Any from fastapi import HTTPException, status -from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from app.application.feishu.commands import FeishuCommandService @@ -19,12 +21,19 @@ from app.modules.feishu.constants import ( FeishuCommandKey, FeishuEventReceiptKey, FeishuEventSource, + FeishuInboundStatus, FeishuPayloadKey, FeishuResponseKey, ) -from app.modules.feishu.models import FeishuEventReceipt from app.modules.feishu.service import FeishuService +from app.modules.feishu.services import ( + FeishuInboundAcceptance, + FeishuInboundProcessResult, + FeishuInboundService, + bind_inbound_event, +) from app.modules.feishu_users.principal import FeishuMention, FeishuPrincipal +from app.modules.feishu_users.identifiers import feishu_audit_identity_hash from app.modules.feishu_users.services import FeishuIdentityService FEISHU_EVENT_ACTIONS = { @@ -33,6 +42,16 @@ FEISHU_EVENT_ACTIONS = { } +@dataclass(frozen=True, slots=True) +class FeishuEventAcceptance: + """Transport-safe acknowledgement for one verified event.""" + + response: dict[str, Any] + event_key: str | None = None + created: bool = False + should_dispatch: bool = False + + class FeishuEventService: """Handle Feishu message events from webhook or long connection.""" @@ -40,6 +59,7 @@ class FeishuEventService: self.db = db self.feishu = FeishuService(db) self.commands = FeishuCommandService(db) + self.inbox = FeishuInboundService(db) def handle_event( self, @@ -56,14 +76,134 @@ class FeishuEventService: source: str | FeishuEventSource, auto_reply: bool = True, ) -> dict[str, Any]: - """Handle an event after an HTTP verifier or the Feishu SDK accepted it.""" + """Synchronously accept and process a verified event for local callers.""" + + acceptance = self.accept_verified_event(payload, source, auto_reply=auto_reply) + if acceptance.event_key is None: + return acceptance.response + if not acceptance.should_dispatch: + return acceptance.response + outcome = self.process_inbound_event( + acceptance.event_key, + worker_id=f"{source}:inline", + ) + if outcome.handler_result is not None: + return outcome.handler_result + return _inbound_response(outcome.record, duplicate=not acceptance.created) + + def accept_verified_event( + self, + payload: dict[str, Any], + source: str | FeishuEventSource, + *, + auto_reply: bool = True, + ) -> FeishuEventAcceptance: + """Durably accept a verified event without executing its command.""" challenge = payload.get(FeishuPayloadKey.CHALLENGE) if challenge: - return {FeishuResponseKey.CHALLENGE: challenge} + return FeishuEventAcceptance( + response={FeishuResponseKey.CHALLENGE: challenge} + ) source_value = _normalize_source(source) - if _event_type(payload) == APP_TICKET_EVENT_TYPE: - return self._handle_app_ticket_event(payload, source_value) + event_type = _event_type(payload) + is_app_ticket = event_type == APP_TICKET_EVENT_TYPE + if is_app_ticket: + self._validate_app_ticket_event(payload) + event_identity = _event_identity(payload, source_value) + acceptance = self.inbox.accept( + payload, + event_identity, + event_type=event_type, + auto_reply=auto_reply, + persist_payload=not is_app_ticket, + ) + if acceptance.created: + self._audit_accepted_event(payload, source_value, acceptance) + if is_app_ticket: + if acceptance.record.status == FeishuInboundStatus.SUCCEEDED: + return FeishuEventAcceptance( + response=_inbound_response(acceptance.record, duplicate=True), + event_key=acceptance.record.event_key, + created=False, + should_dispatch=False, + ) + outcome = self.inbox.process( + acceptance.record.event_key, + handler_factory=lambda db: ( + lambda event_payload, _source, _auto_reply, _event_key: ( + FeishuEventService(db)._execute_app_ticket_event( + event_payload + ) + ) + ), + payload_override=payload, + worker_id=f"{source_value}:app-ticket", + ) + if outcome.record.status != FeishuInboundStatus.SUCCEEDED: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=( + "Failed to persist verified Feishu app ticket: " + f"{outcome.record.last_error or 'unknown_error'}" + ), + ) + return FeishuEventAcceptance( + response=outcome.handler_result + or _inbound_response(outcome.record, duplicate=not acceptance.created), + event_key=outcome.record.event_key, + created=acceptance.created, + should_dispatch=False, + ) + return FeishuEventAcceptance( + response=_inbound_response( + acceptance.record, + duplicate=not acceptance.created, + ), + event_key=acceptance.record.event_key, + created=acceptance.created, + should_dispatch=acceptance.should_dispatch, + ) + + def process_inbound_event( + self, + event_key: str, + *, + worker_id: str = "feishu-inbound", + ) -> FeishuInboundProcessResult: + """Claim and process one accepted event.""" + + return self.inbox.process( + event_key, + handler_factory=lambda db: FeishuEventService( + db + )._execute_inbound_event, + worker_id=worker_id, + ) + + def process_due_inbound_events( + self, + *, + limit: int, + worker_id: str = "feishu-inbound", + ) -> list[FeishuInboundProcessResult]: + """Recover due retries and expired leases from persistent state.""" + + return self.inbox.process_due( + handler_factory=lambda db: FeishuEventService( + db + )._execute_inbound_event, + limit=limit, + worker_id=worker_id, + ) + + def _execute_inbound_event( + self, + payload: dict[str, Any], + _source: str, + auto_reply: bool, + event_key: str, + ) -> dict[str, Any]: user_features_enabled = get_settings().feishu_user_features_enabled command = ( self.commands.extract_event_command(payload) @@ -75,62 +215,37 @@ class FeishuEventService: if user_features_enabled and command else None ) - event_identity = _event_identity(payload, source) - if event_identity and not self._register_event(event_identity): - return { - FeishuResponseKey.OK: True, - FeishuResponseKey.HANDLED: False, - FeishuResponseKey.DUPLICATE: True, - } - self.feishu.audit.log( - AuditLogCreate( - actor=principal.user_code if principal else ActorValue.FEISHU, - source=AuditSource.FEISHU, - action=FEISHU_EVENT_ACTIONS[source_value], - target_type=source_value, - target_id=( - event_identity.get(FeishuEventReceiptKey.EVENT_KEY) - if event_identity - else None - ), - request_payload=_audit_event_metadata( - payload, - include_open_id=not user_features_enabled, - include_identity_context=user_features_enabled, - ), - response_payload={FeishuResponseKey.ACCEPTED: True}, - ) - ) if command is None: command = self.commands.extract_event_command(payload) if not command: return {FeishuResponseKey.OK: True, FeishuResponseKey.HANDLED: False} - result = self.commands.handle_text( - command[FeishuCommandKey.TEXT], - chat_id=command[FeishuCommandKey.CHAT_ID], - actor=( - principal.user_code - if principal - else ( - ActorValue.FEISHU - if user_features_enabled - else command[FeishuCommandKey.ACTOR] - ) - ), - auto_reply=auto_reply, - principal=principal, - ) + self.commands.feishu.set_message_uuid(_reply_uuid(event_key)) + with bind_inbound_event(event_key): + result = self.commands.handle_text( + command[FeishuCommandKey.TEXT], + chat_id=command[FeishuCommandKey.CHAT_ID], + actor=( + principal.user_code + if principal + else ( + ActorValue.FEISHU + if user_features_enabled + else command[FeishuCommandKey.ACTOR] + ) + ), + auto_reply=auto_reply, + principal=principal, + ) return { FeishuResponseKey.OK: True, FeishuResponseKey.HANDLED: True, FeishuResponseKey.RESULT: result, } - def _handle_app_ticket_event( + def _validate_app_ticket_event( self, payload: dict[str, Any], - source: FeishuEventSource, - ) -> dict[str, Any]: + ) -> tuple[str, str]: settings = get_settings() configured_app_id = str(settings.feishu_app_id or "").strip() if not configured_app_id: @@ -150,37 +265,47 @@ class FeishuEventService: status_code=status.HTTP_401_UNAUTHORIZED, detail="Feishu app ticket app_id does not match configured application", ) + return app_id, ticket - event_identity = _event_identity(payload, source) - if event_identity is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Feishu app ticket event identity is required", - ) - if not self._register_event(event_identity): - return { - FeishuResponseKey.OK: True, - FeishuResponseKey.HANDLED: False, - FeishuResponseKey.DUPLICATE: True, - } - + def _execute_app_ticket_event( + self, + payload: dict[str, Any], + ) -> dict[str, Any]: + app_id, ticket = self._validate_app_ticket_event(payload) FeishuAppTicketService(self.db).store_verified(app_id, ticket) - self.feishu.audit.log( - AuditLogCreate( - actor=ActorValue.FEISHU, - source=AuditSource.FEISHU, - action=FEISHU_EVENT_ACTIONS[source], - target_type=source, - target_id=str(event_identity[FeishuEventReceiptKey.EVENT_KEY]), - request_payload=_audit_event_metadata(payload), - response_payload={FeishuResponseKey.ACCEPTED: True}, - ) - ) return { FeishuResponseKey.OK: True, FeishuResponseKey.HANDLED: True, } + def _audit_accepted_event( + self, + payload: dict[str, Any], + source: FeishuEventSource, + acceptance: FeishuInboundAcceptance, + ) -> None: + user_features_enabled = get_settings().feishu_user_features_enabled + audit_actor = ( + _audit_identity_actor(payload) + if user_features_enabled + else None + ) + self.feishu.audit.log( + AuditLogCreate( + actor=audit_actor or ActorValue.FEISHU, + source=AuditSource.FEISHU, + action=FEISHU_EVENT_ACTIONS[source], + target_type=source, + target_id=acceptance.record.event_key, + request_payload=_audit_event_metadata( + payload, + include_open_id=not user_features_enabled, + include_identity_context=user_features_enabled, + ), + response_payload={FeishuResponseKey.ACCEPTED: True}, + ) + ) + def _resolve_principal( self, payload: dict[str, Any], @@ -217,22 +342,6 @@ class FeishuEventService: mentions=mentions, ) - def _register_event(self, event_identity: dict[str, str | None]) -> bool: - receipt = FeishuEventReceipt( - event_key=str(event_identity[FeishuEventReceiptKey.EVENT_KEY]), - source=str(event_identity[FeishuEventReceiptKey.SOURCE]), - event_id=event_identity.get(FeishuEventReceiptKey.EVENT_ID), - message_id=event_identity.get(FeishuEventReceiptKey.MESSAGE_ID), - ) - try: - with self.db.begin_nested(): - self.db.add(receipt) - self.db.flush() - except IntegrityError: - return False - return True - - def _audit_event_metadata( payload: dict[str, Any], *, @@ -248,20 +357,35 @@ def _audit_event_metadata( sender_id = sender.get(FeishuPayloadKey.SENDER_ID) or {} metadata = { "schema": payload.get("schema"), - FeishuPayloadKey.EVENT_ID: header.get(FeishuPayloadKey.EVENT_ID), + FeishuPayloadKey.EVENT_ID: _identifier_digest( + header.get(FeishuPayloadKey.EVENT_ID), + "event-id", + ), FeishuPayloadKey.EVENT_TYPE: _event_type(payload), - FeishuPayloadKey.MESSAGE_ID: message.get(FeishuPayloadKey.MESSAGE_ID), - FeishuCommandKey.CHAT_ID: message.get(FeishuCommandKey.CHAT_ID), + FeishuPayloadKey.MESSAGE_ID: _identifier_digest( + message.get(FeishuPayloadKey.MESSAGE_ID), + "message-id", + ), + FeishuCommandKey.CHAT_ID: _identifier_digest( + message.get(FeishuCommandKey.CHAT_ID), + "chat-id", + ), FeishuPayloadKey.MESSAGE_TYPE: message.get(FeishuPayloadKey.MESSAGE_TYPE), } app_id, _ = _app_ticket_fields(payload) if app_id: - metadata[FeishuPayloadKey.APP_ID] = app_id + metadata[FeishuPayloadKey.APP_ID] = _identifier_digest(app_id, "app-id") if include_identity_context: - metadata[FeishuPayloadKey.TENANT_KEY] = header.get(FeishuPayloadKey.TENANT_KEY) + metadata[FeishuPayloadKey.TENANT_KEY] = _identifier_digest( + header.get(FeishuPayloadKey.TENANT_KEY), + "tenant-key", + ) metadata[FeishuCommandKey.CHAT_TYPE] = message.get(FeishuPayloadKey.CHAT_TYPE) if include_open_id: - metadata[FeishuPayloadKey.OPEN_ID] = sender_id.get(FeishuPayloadKey.OPEN_ID) + metadata[FeishuPayloadKey.OPEN_ID] = _identifier_digest( + sender_id.get(FeishuPayloadKey.OPEN_ID), + "open-id", + ) return metadata @@ -269,10 +393,22 @@ def _normalize_source(source: str | FeishuEventSource) -> FeishuEventSource: return FeishuEventSource(source) +def _audit_identity_actor(payload: dict[str, Any]) -> str | None: + header = payload.get(FeishuPayloadKey.HEADER) or {} + event = payload.get(FeishuPayloadKey.EVENT) or {} + sender = event.get(FeishuPayloadKey.SENDER) or {} + sender_id = sender.get(FeishuPayloadKey.SENDER_ID) or {} + tenant_key = str(header.get(FeishuPayloadKey.TENANT_KEY) or "").strip() + open_id = str(sender_id.get(FeishuPayloadKey.OPEN_ID) or "").strip() + if not tenant_key or not open_id: + return None + return feishu_audit_identity_hash(tenant_key, open_id) + + def _event_identity( payload: dict[str, Any], source: str | FeishuEventSource, -) -> dict[str, str | None] | None: +) -> dict[str, str | None]: source_value = _normalize_source(source) header = payload.get(FeishuPayloadKey.HEADER) or {} event = payload.get(FeishuPayloadKey.EVENT) or {} @@ -284,23 +420,92 @@ def _event_identity( or event.get(FeishuPayloadKey.UUID) ) message_id = message.get(FeishuPayloadKey.MESSAGE_ID) - stable_id = event_id or message_id - if not stable_id: - return None + stable_id = event_id or message_id or _payload_fingerprint(payload) event_type = _event_type(payload) app_id, _ = _app_ticket_fields(payload) tenant_key = header.get(FeishuPayloadKey.TENANT_KEY) or app_id or "unknown-tenant" - event_key = ":".join( + raw_event_key = ":".join( str(part) for part in (tenant_key, event_type or FeishuPayloadKey.EVENT, stable_id) ) + event_key = _identifier_digest(raw_event_key, "event-key") return { FeishuEventReceiptKey.EVENT_KEY: event_key, FeishuEventReceiptKey.SOURCE: source_value, - FeishuEventReceiptKey.EVENT_ID: str(event_id) if event_id else None, - FeishuEventReceiptKey.MESSAGE_ID: str(message_id) if message_id else None, + FeishuEventReceiptKey.EVENT_ID: _identifier_digest(event_id, "event-id"), + FeishuEventReceiptKey.MESSAGE_ID: _identifier_digest( + message_id, + "message-id", + ), } +def _payload_fingerprint(payload: dict[str, Any]) -> str: + scrubbed = _scrub_transport_secrets(payload) + serialized = json.dumps( + scrubbed, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + return f"sha256-{sha256(serialized.encode('utf-8')).hexdigest()}" + + +def _scrub_transport_secrets(value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): _scrub_transport_secrets(item) + for key, item in value.items() + if str(key).casefold() + not in { + "access_token", + "app_access_token", + "app_secret", + "app_ticket", + "authorization", + "encrypt", + "refresh_token", + "secret", + "tenant_access_token", + "token", + } + } + if isinstance(value, list): + return [_scrub_transport_secrets(item) for item in value] + return value + + +def _reply_uuid(event_key: str) -> str: + digest = sha256(event_key.encode("utf-8")).hexdigest()[:32] + return f"inbound-{digest}" + + +def _identifier_digest(value: Any, domain: str) -> str | None: + text = str(value or "").strip() + if not text: + return None + digest = sha256( + f"company-ai-platform:feishu:{domain}:v1\0{text}".encode("utf-8") + ).hexdigest() + return f"sha256-{digest}" + + +def _inbound_response( + record: Any, + *, + duplicate: bool, +) -> dict[str, Any]: + response = { + FeishuResponseKey.OK: True, + FeishuResponseKey.ACCEPTED: True, + FeishuResponseKey.HANDLED: record.status == FeishuInboundStatus.SUCCEEDED, + FeishuResponseKey.STATUS: record.status, + } + if duplicate: + response[FeishuResponseKey.DUPLICATE] = True + return response + + def _event_type(payload: dict[str, Any]) -> str: header = payload.get(FeishuPayloadKey.HEADER) or {} return str( diff --git a/app/application/feishu/inbound.py b/app/application/feishu/inbound.py new file mode 100644 index 0000000..49f0bfe --- /dev/null +++ b/app/application/feishu/inbound.py @@ -0,0 +1,52 @@ +from socket import gethostname +from typing import Any + +from app.application.feishu.events import FeishuEventService +from app.core.constants import ActorValue +from app.core.database import SessionLocal +from app.modules.feishu.constants import FEISHU_INBOUND_BATCH_SIZE + + +def process_feishu_inbound_event( + event_key: str, + *, + actor: str = ActorValue.WORKER, +) -> dict[str, Any]: + """Process one durable Feishu inbox row in an isolated session.""" + + db = SessionLocal() + try: + outcome = FeishuEventService(db).process_inbound_event( + event_key, + worker_id=f"{actor}:{gethostname()}", + ) + return _serialize_outcome(outcome) + finally: + db.close() + + +def process_due_feishu_inbound_events( + *, + limit: int = FEISHU_INBOUND_BATCH_SIZE, + actor: str = ActorValue.WORKER, +) -> list[dict[str, Any]]: + """Recover due retries and expired Feishu inbox leases.""" + + db = SessionLocal() + try: + outcomes = FeishuEventService(db).process_due_inbound_events( + limit=limit, + worker_id=f"{actor}:{gethostname()}", + ) + return [_serialize_outcome(outcome) for outcome in outcomes] + finally: + db.close() + + +def _serialize_outcome(outcome: Any) -> dict[str, Any]: + return { + "event_key": outcome.record.event_key, + "status": outcome.record.status, + "attempt_count": outcome.record.attempt_count, + "handled": outcome.handler_result is not None, + } diff --git a/app/application/feishu/personal_data.py b/app/application/feishu/personal_data.py index 937bddf..f8f2e77 100644 --- a/app/application/feishu/personal_data.py +++ b/app/application/feishu/personal_data.py @@ -16,7 +16,12 @@ from app.modules.feishu_users.constants import ( FeishuUserStatus, parse_admin_identities, ) +from app.modules.feishu.services import ( + FeishuInboundService, + current_inbound_event_key, +) from app.modules.feishu_users.bootstrap import admin_bootstrap_identity_hash +from app.modules.feishu_users.identifiers import feishu_audit_identity_hash from app.modules.feishu_users.models import ( FeishuAdminBootstrapTombstone, FeishuUser, @@ -187,6 +192,10 @@ class FeishuPersonalDataService: user.open_id, user.union_id, user.user_id, + feishu_audit_identity_hash( + user.tenant_key, + user.open_id, + ), ) if value }, @@ -230,6 +239,21 @@ class FeishuPersonalDataService: "subscriptions": subscriptions, } + def clear_pending_inbound_events( + db: Session, + _owner_id: int, + _anonymous_id: str, + ) -> dict[str, int]: + # The caller already holds the FeishuUser row. Inbound handlers + # acquire that same identity fence before their own receipt row, so + # erasure can safely fence related receipts in identity -> inbox order. + cleared = FeishuInboundService(db).erase_identity_payloads( + tenant_key=user.tenant_key, + open_id=user.open_id, + exclude_event_key=current_inbound_event_key(), + ) + return {"inbound_events": cleared} + def finalize_identity( db: Session, _owner_id: int, @@ -281,7 +305,7 @@ class FeishuPersonalDataService: return self.erasure.confirm_and_erase( user.id, confirmation_code, - before_hooks=(delete_subscriptions,), + before_hooks=(delete_subscriptions, clear_pending_inbound_events), extra_hooks=(*self.extra_hooks, finalize_identity), ) diff --git a/app/application/scheduling/scheduler.py b/app/application/scheduling/scheduler.py index 645a723..bce0251 100644 --- a/app/application/scheduling/scheduler.py +++ b/app/application/scheduling/scheduler.py @@ -1,5 +1,5 @@ from collections.abc import Callable -from datetime import date +from datetime import UTC, date, datetime from socket import gethostname from typing import Any @@ -40,6 +40,7 @@ def create_scheduler(app: FastAPI | None = None) -> Any: enqueue_attendance_summary_push, enqueue_daily_brief_push, enqueue_event_dispatch, + enqueue_feishu_inbound_cycle, enqueue_legacy_project_sync, enqueue_legacy_task_sync, enqueue_lifecycle_report, @@ -47,6 +48,7 @@ def create_scheduler(app: FastAPI | None = None) -> Any: enqueue_project_weekly_push, enqueue_risk_progress_push, enqueue_subscription_cycle, + enqueue_worker_heartbeat, enqueue_work_daily_push, enqueue_work_weekly_push, ) @@ -186,6 +188,10 @@ def create_scheduler(app: FastAPI | None = None) -> Any: ) _set_state(app, "last_event_dispatch", dispatch) + def run_feishu_inbound_cycle() -> None: + dispatch = enqueue_feishu_inbound_cycle(actor=ActorValue.SCHEDULER) + _set_state(app, "last_feishu_inbound_cycle", dispatch) + def record_scheduler_heartbeat() -> None: db = SessionLocal() try: @@ -202,6 +208,10 @@ def create_scheduler(app: FastAPI | None = None) -> Any: dispatch = enqueue_subscription_cycle(actor=ActorValue.SCHEDULER) _set_state(app, "last_subscription_cycle", dispatch) + def run_worker_heartbeat() -> None: + dispatch = enqueue_worker_heartbeat(actor=ActorValue.SCHEDULER) + _set_state(app, "last_worker_heartbeat", dispatch) + def run_personalization_retention_cleanup() -> None: db = SessionLocal() try: @@ -285,6 +295,7 @@ def create_scheduler(app: FastAPI | None = None) -> Any: trigger="cron", minute=settings.event_dispatch_cron_minute, id="event_dispatch", + next_run_time=datetime.now(UTC), replace_existing=True, ) if settings.market_analysis_enabled: @@ -320,6 +331,23 @@ def create_scheduler(app: FastAPI | None = None) -> Any: trigger="interval", seconds=settings.heartbeat_interval_seconds, id="scheduler_heartbeat", + next_run_time=datetime.now(UTC), + replace_existing=True, + ) + if settings.task_queue_enabled: + scheduler.add_job( + run_worker_heartbeat, + trigger="interval", + seconds=settings.heartbeat_interval_seconds, + id="worker_heartbeat", + next_run_time=datetime.now(UTC), + replace_existing=True, + ) + scheduler.add_job( + run_feishu_inbound_cycle, + trigger="interval", + minutes=1, + id="feishu_inbound_event_cycle", replace_existing=True, ) if settings.feishu_user_features_enabled: diff --git a/app/core/background/task_queue/__init__.py b/app/core/background/task_queue/__init__.py index 46306de..552628a 100644 --- a/app/core/background/task_queue/__init__.py +++ b/app/core/background/task_queue/__init__.py @@ -1,6 +1,8 @@ from app.tasks.constants import ( TASK_DISPATCH_PENDING_EVENTS, TASK_GENERATE_RISK_EVENTS, + TASK_PROCESS_DUE_FEISHU_INBOUND, + TASK_PROCESS_FEISHU_INBOUND, TASK_PUSH_ATTENDANCE_SUMMARY, TASK_PUSH_DAILY_BRIEF, TASK_PUSH_PROJECT_WEEKLY, @@ -13,15 +15,21 @@ from app.tasks.constants import ( TASK_RUN_MARKET_CLOSE, TASK_RUN_MARKET_REPORT, TASK_RUN_SUBSCRIPTION_CYCLE, + TASK_RECORD_WORKER_HEARTBEAT, ) from app.core.background.task_queue.dispatcher import dispatch_task from app.core.background.task_queue.events import enqueue_event_dispatch +from app.core.background.task_queue.feishu import ( + enqueue_feishu_inbound_cycle, + enqueue_feishu_inbound_event, +) from app.core.background.task_queue.legacy import ( enqueue_legacy_project_sync, enqueue_legacy_task_sync, ) from app.core.background.task_queue.lifecycle import enqueue_lifecycle_report from app.core.background.task_queue.market import enqueue_market_close, enqueue_market_report +from app.core.background.task_queue.observability import enqueue_worker_heartbeat from app.core.background.task_queue.reports import ( enqueue_attendance_summary_push, enqueue_daily_brief_push, @@ -37,6 +45,8 @@ from app.core.background.task_queue.subscriptions import enqueue_subscription_cy __all__ = [ "TASK_DISPATCH_PENDING_EVENTS", "TASK_GENERATE_RISK_EVENTS", + "TASK_PROCESS_DUE_FEISHU_INBOUND", + "TASK_PROCESS_FEISHU_INBOUND", "TASK_PUSH_ATTENDANCE_SUMMARY", "TASK_PUSH_DAILY_BRIEF", "TASK_PUSH_PROJECT_WEEKLY", @@ -49,15 +59,19 @@ __all__ = [ "TASK_RUN_MARKET_CLOSE", "TASK_RUN_MARKET_REPORT", "TASK_RUN_SUBSCRIPTION_CYCLE", + "TASK_RECORD_WORKER_HEARTBEAT", "dispatch_task", "enqueue_attendance_summary_push", "enqueue_daily_brief_push", "enqueue_event_dispatch", + "enqueue_feishu_inbound_cycle", + "enqueue_feishu_inbound_event", "enqueue_legacy_project_sync", "enqueue_legacy_task_sync", "enqueue_lifecycle_report", "enqueue_market_close", "enqueue_market_report", + "enqueue_worker_heartbeat", "enqueue_project_weekly_push", "enqueue_risk_progress_push", "enqueue_risk_event_generation", diff --git a/app/core/background/task_queue/feishu.py b/app/core/background/task_queue/feishu.py new file mode 100644 index 0000000..d165dc0 --- /dev/null +++ b/app/core/background/task_queue/feishu.py @@ -0,0 +1,37 @@ +from typing import Any + +from app.application.feishu.inbound import ( + process_due_feishu_inbound_events, + process_feishu_inbound_event, +) +from app.core.background.task_queue.dispatcher import dispatch_task +from app.core.constants import ActorValue +from app.modules.feishu.constants import FEISHU_INBOUND_BATCH_SIZE +from app.tasks.constants import ( + TASK_PROCESS_DUE_FEISHU_INBOUND, + TASK_PROCESS_FEISHU_INBOUND, +) + + +def enqueue_feishu_inbound_event( + event_key: str, + *, + actor: str = ActorValue.WORKER, +) -> dict[str, Any]: + return dispatch_task( + TASK_PROCESS_FEISHU_INBOUND, + {"event_key": event_key, "actor": actor}, + lambda: process_feishu_inbound_event(event_key, actor=actor), + ) + + +def enqueue_feishu_inbound_cycle( + *, + limit: int = FEISHU_INBOUND_BATCH_SIZE, + actor: str = ActorValue.SCHEDULER, +) -> dict[str, Any]: + return dispatch_task( + TASK_PROCESS_DUE_FEISHU_INBOUND, + {"limit": limit, "actor": actor}, + lambda: process_due_feishu_inbound_events(limit=limit, actor=actor), + ) diff --git a/app/core/background/task_queue/observability.py b/app/core/background/task_queue/observability.py new file mode 100644 index 0000000..737ad0e --- /dev/null +++ b/app/core/background/task_queue/observability.py @@ -0,0 +1,32 @@ +from socket import gethostname + +from app.core.background.task_queue.dispatcher import dispatch_task +from app.core.constants import ActorValue +from app.core.database import SessionLocal +from app.modules.observability.constants import HeartbeatComponent +from app.modules.observability.service import ObservabilityService +from app.tasks.constants import TASK_RECORD_WORKER_HEARTBEAT + + +def enqueue_worker_heartbeat( + actor: str = ActorValue.SCHEDULER, +) -> dict: + """Ask a Celery worker to prove it can consume tasks.""" + + return dispatch_task( + TASK_RECORD_WORKER_HEARTBEAT, + {"actor": ActorValue.WORKER}, + lambda: _record_inline_heartbeat(actor), + ) + + +def _record_inline_heartbeat(actor: str) -> dict: + db = SessionLocal() + try: + return ObservabilityService(db).record_heartbeat( + component=HeartbeatComponent.WORKER, + instance_id=f"inline:{gethostname()}", + actor=actor, + ) + finally: + db.close() diff --git a/app/core/config/settings.py b/app/core/config/settings.py index 28f14ad..dc4bed0 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -3,7 +3,7 @@ import os from functools import lru_cache from typing import Annotated, Any, Literal -from pydantic import Field, field_validator, model_validator +from pydantic import Field, ValidationInfo, field_validator, model_validator from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict from app.core.constants import ( @@ -12,6 +12,7 @@ from app.core.constants import ( DEFAULT_MODEL_PROVIDER, DEFAULT_OPENCLAW_ACTION_JSON, ) +from app.core.database.safety import database_password, database_target _DOTENV_DISABLED = os.getenv("COMPANY_AI_DISABLE_DOTENV", "").strip().lower() in { "1", @@ -19,15 +20,43 @@ _DOTENV_DISABLED = os.getenv("COMPANY_AI_DISABLE_DOTENV", "").strip().lower() in "yes", "on", } +_DOTENV_FILE = None if _DOTENV_DISABLED else ".env" +_PRODUCTION_SECRET_MIN_LENGTH = 24 +_DATABASE_PASSWORD_MIN_LENGTH = 10 +_PLACEHOLDER_SECRET_PARTS = ( + "change-me", + "changeme", + "example", + "placeholder", + "replace-with", + "your-", +) + + +def _is_unsafe_production_secret(value: str) -> bool: + normalized = value.strip().casefold() + return ( + len(normalized) < _PRODUCTION_SECRET_MIN_LENGTH + or any(part in normalized for part in _PLACEHOLDER_SECRET_PARTS) + ) + + +def _is_unsafe_database_password(value: str) -> bool: + normalized = value.strip().casefold() + return ( + len(normalized) < _DATABASE_PASSWORD_MIN_LENGTH + or any(part in normalized for part in _PLACEHOLDER_SECRET_PARTS) + ) class Settings(BaseSettings): - """Runtime settings loaded from environment variables and `.env`.""" + """Runtime settings loaded from the environment and optional `.env` file.""" model_config = SettingsConfigDict( - env_file=None if _DOTENV_DISABLED else ".env", + env_file=_DOTENV_FILE, env_file_encoding="utf-8", extra="ignore", + hide_input_in_errors=True, ) app_name: str = "Company AI Management Platform" @@ -63,6 +92,11 @@ class Settings(BaseSettings): feishu_encrypt_key: str | None = None feishu_default_chat_id: str | None = None feishu_default_tenant_key: str | None = None + feishu_event_transport: Literal[ + "disabled", + "webhook", + "long_connection", + ] = "disabled" feishu_user_features_enabled: bool = False feishu_admin_identities: Annotated[list[str], NoDecode] = Field(default_factory=list) model_provider: str = DEFAULT_MODEL_PROVIDER @@ -189,11 +223,13 @@ class Settings(BaseSettings): @field_validator( "feishu_app_type", + "feishu_event_transport", mode="before", ) @classmethod - def normalize_feishu_app_type(cls, value: Any) -> str: - return str(value or "self").strip().lower() + def normalize_feishu_choice(cls, value: Any, info: ValidationInfo) -> str: + default = "self" if info.field_name == "feishu_app_type" else "disabled" + return str(value or default).strip().lower() @field_validator( "openclaw_allowed_tools", @@ -291,12 +327,34 @@ class Settings(BaseSettings): errors: list[str] = [] api_key_values = _enabled_keys(self.api_key, self.api_keys) audit_key_values = _enabled_keys(self.audit_api_key, self.audit_api_keys) - if self.database_url.startswith("sqlite"): + platform_target = database_target(self.database_url) + legacy_target = database_target(self.legacy_database_url) + if platform_target is None or platform_target[0] != "postgresql": errors.append("DATABASE_URL must use PostgreSQL in production") + platform_password = database_password(self.database_url) + if platform_password and _is_unsafe_database_password(platform_password): + errors.append( + "DATABASE_URL password must use a non-placeholder value of at least " + f"{_DATABASE_PASSWORD_MIN_LENGTH} characters in production" + ) + if platform_target is not None and platform_target == legacy_target: + errors.append( + "DATABASE_URL and LEGACY_DATABASE_URL must target different databases" + ) if not api_key_values: errors.append("API_KEY or API_KEYS is required in production") + elif any(_is_unsafe_production_secret(value) for value in api_key_values): + errors.append( + "API_KEY/API_KEYS must use non-placeholder values of at least " + f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production" + ) if not audit_key_values: errors.append("AUDIT_API_KEY or AUDIT_API_KEYS is required in production") + elif any(_is_unsafe_production_secret(value) for value in audit_key_values): + errors.append( + "AUDIT_API_KEY/AUDIT_API_KEYS must use non-placeholder values of at least " + f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production" + ) if api_key_values & audit_key_values: errors.append( "API_KEY/API_KEYS and AUDIT_API_KEY/AUDIT_API_KEYS cannot overlap in production" @@ -309,10 +367,52 @@ class Settings(BaseSettings): errors.append("MASK_SENSITIVE_RESPONSES must be true in production") if not self.read_only_mode: errors.append("READ_ONLY_MODE must be true in production") - if self.feishu_user_features_enabled and not self.feishu_admin_identities: - errors.append( - "FEISHU_ADMIN_IDENTITIES is required when Feishu user features are enabled" - ) + if self.feishu_event_transport != "disabled": + if not self.feishu_app_id or not self.feishu_app_secret: + errors.append( + "FEISHU_APP_ID and FEISHU_APP_SECRET are required when " + "Feishu event transport is enabled" + ) + elif _is_unsafe_production_secret(self.feishu_app_secret): + errors.append( + "FEISHU_APP_SECRET must use a non-placeholder value of at least " + f"{_PRODUCTION_SECRET_MIN_LENGTH} characters in production" + ) + if ( + self.feishu_event_transport == "webhook" + and not self.feishu_verification_token + ): + errors.append( + "FEISHU_VERIFICATION_TOKEN is required for Feishu webhook transport" + ) + elif ( + self.feishu_event_transport == "webhook" + and self.feishu_verification_token + and _is_unsafe_production_secret(self.feishu_verification_token) + ): + errors.append( + "FEISHU_VERIFICATION_TOKEN must use a non-placeholder value of at " + f"least {_PRODUCTION_SECRET_MIN_LENGTH} characters in production" + ) + if self.feishu_user_features_enabled: + if not self.feishu_admin_identities: + errors.append( + "FEISHU_ADMIN_IDENTITIES is required when Feishu user features are enabled" + ) + else: + try: + from app.modules.feishu_users.constants import ( + parse_admin_identities, + ) + + parse_admin_identities(self.feishu_admin_identities) + except ValueError as exc: + errors.append(str(exc)) + if self.feishu_event_transport == "disabled": + errors.append( + "FEISHU_EVENT_TRANSPORT must be webhook or long_connection " + "when Feishu user features are enabled" + ) if errors: raise ValueError("; ".join(errors)) return self diff --git a/app/core/database/migrations.py b/app/core/database/migrations.py new file mode 100644 index 0000000..b9b2264 --- /dev/null +++ b/app/core/database/migrations.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from alembic.config import Config +from alembic.script import ScriptDirectory +from sqlalchemy import inspect, text +from sqlalchemy.engine import Connection + + +def expected_alembic_heads(project_root: Path | None = None) -> tuple[str, ...]: + """Return the repository's configured Alembic heads.""" + + root = project_root or Path(__file__).resolve().parents[3] + config = Config(str(root / "alembic.ini")) + config.set_main_option("script_location", str(root / "alembic")) + return tuple(sorted(ScriptDirectory.from_config(config).get_heads())) + + +def current_alembic_revisions(connection: Connection) -> tuple[str, ...] | None: + """Return database revisions, or ``None`` when it was never versioned.""" + + if "alembic_version" not in inspect(connection).get_table_names(): + return None + revisions = connection.execute( + text("SELECT version_num FROM alembic_version ORDER BY version_num") + ).scalars() + return tuple(str(revision) for revision in revisions) diff --git a/app/core/database/safety.py b/app/core/database/safety.py new file mode 100644 index 0000000..af1ef3b --- /dev/null +++ b/app/core/database/safety.py @@ -0,0 +1,62 @@ +from sqlalchemy.engine import URL, make_url +from sqlalchemy.exc import ArgumentError + + +_DEFAULT_DATABASE_PORTS = { + "mysql": 3306, + "postgresql": 5432, +} +_PLATFORM_MIGRATION_BACKENDS = frozenset({"postgresql", "sqlite"}) + +DatabaseTarget = tuple[str, str, int | None, str] + + +def database_target(value: str | URL | None) -> DatabaseTarget | None: + """Return a credential-free physical database identity.""" + + if not value: + return None + try: + url = make_url(value) + except (ArgumentError, TypeError, ValueError): + return None + backend = url.get_backend_name().lower() + return ( + backend, + str(url.host or "").casefold(), + url.port or _DEFAULT_DATABASE_PORTS.get(backend), + str(url.database or ""), + ) + + +def database_password(value: str | URL | None) -> str | None: + """Return a configured password without including it in diagnostics.""" + + if not value: + return None + try: + password = make_url(value).password + except (ArgumentError, TypeError, ValueError): + return None + return str(password) if password is not None else None + + +def validate_platform_migration_target( + database_url: str | URL, + legacy_database_url: str | None, +) -> DatabaseTarget: + """Fail closed before Alembic can connect to an unsafe database target.""" + + platform = database_target(database_url) + if platform is None: + raise RuntimeError("DATABASE_URL is not a valid platform database URL") + if platform[0] not in _PLATFORM_MIGRATION_BACKENDS: + raise RuntimeError( + "Platform migrations are supported only for PostgreSQL or local SQLite" + ) + legacy = database_target(legacy_database_url) + if legacy is not None and platform == legacy: + raise RuntimeError( + "Refusing to migrate DATABASE_URL because it matches LEGACY_DATABASE_URL" + ) + return platform diff --git a/app/main.py b/app/main.py index 6ecbd91..e57c783 100644 --- a/app/main.py +++ b/app/main.py @@ -6,6 +6,7 @@ from app.core.config import get_settings from app.core.http.middleware import request_id_middleware from app.core.http.responses import MaskedJSONResponse from app.application.scheduling import attach_scheduler +from app.modules.observability.runtime import attach_api_heartbeat def _allow_cors_credentials(cors_origins: list[str]) -> bool: @@ -37,6 +38,7 @@ def create_app() -> FastAPI: ) app.include_router(api_router, prefix=settings.api_prefix) + attach_api_heartbeat(app) attach_scheduler(app) return app diff --git a/app/modules/ai_memory/models.py b/app/modules/ai_memory/models.py index 52c4543..9e633eb 100644 --- a/app/modules/ai_memory/models.py +++ b/app/modules/ai_memory/models.py @@ -22,6 +22,7 @@ class AIMemoryEntry(Base): "owner_id", "fingerprint", name="uq_ai_memory_owner_fingerprint", + postgresql_nulls_not_distinct=True, ), ) diff --git a/app/modules/business/models/market.py b/app/modules/business/models/market.py index ea0a7d0..e4244a1 100644 --- a/app/modules/business/models/market.py +++ b/app/modules/business/models/market.py @@ -91,7 +91,12 @@ class MarketAnnouncement(Base, TimestampMixin): class MarketWatchlist(Base, TimestampMixin): __tablename__ = "market_watchlists" __table_args__ = ( - UniqueConstraint("owner_id", "symbol", name="uq_market_watchlist_owner_symbol"), + UniqueConstraint( + "owner_id", + "symbol", + name="uq_market_watchlist_owner_symbol", + postgresql_nulls_not_distinct=True, + ), ) id: Mapped[int] = mapped_column(Integer, primary_key=True) owner_id: Mapped[int | None] = mapped_column( diff --git a/app/modules/feishu/constants.py b/app/modules/feishu/constants.py index f83a8bf..f94a184 100644 --- a/app/modules/feishu/constants.py +++ b/app/modules/feishu/constants.py @@ -21,6 +21,20 @@ class FeishuEventSource(StrEnum): LONG_CONNECTION = "long_connection" +class FeishuInboundStatus(StrEnum): + PENDING = "pending" + PROCESSING = "processing" + SUCCEEDED = "succeeded" + RETRY = "retry" + FAILED = "failed" + + +class FeishuEventTransport(StrEnum): + DISABLED = "disabled" + WEBHOOK = "webhook" + LONG_CONNECTION = "long_connection" + + class FeishuPayloadKey(StrEnum): APP_ACCESS_TOKEN = "app_access_token" APP_ID = "app_id" @@ -180,6 +194,10 @@ FEISHU_RECEIVE_ID_MISSING = "receive_id or FEISHU_DEFAULT_CHAT_ID is required" FEISHU_SUCCESS_CODE = 0 FEISHU_DEFAULT_TOKEN_EXPIRE_SECONDS = 7200 FEISHU_TOKEN_EXPIRE_SAFETY_SECONDS = 300 +FEISHU_INBOUND_BATCH_SIZE = 100 +FEISHU_INBOUND_LEASE_SECONDS = 300 +FEISHU_INBOUND_MAX_ATTEMPTS = 4 +FEISHU_INBOUND_RETRY_DELAYS_SECONDS = (60, 300, 900) FEISHU_AI_REPLY_TITLE = "AI 回复" FEISHU_EMPTY_CARD_TEXT = "暂无数据" FEISHU_MENTION_PATTERN = r"@\S+" diff --git a/app/modules/feishu/long_connection.py b/app/modules/feishu/long_connection.py index bdd389e..0435825 100644 --- a/app/modules/feishu/long_connection.py +++ b/app/modules/feishu/long_connection.py @@ -1,12 +1,24 @@ import json import logging +from socket import gethostname +from threading import Event, Thread +from time import monotonic from typing import Any from urllib.parse import urlsplit +from app.core.background.task_queue import enqueue_feishu_inbound_event from app.core.config import get_settings +from app.core.constants import ActorValue from app.core.database import SessionLocal -from app.modules.feishu.constants import FEISHU_DEFAULT_OPEN_API_DOMAIN, FeishuEventSource from app.application.feishu import FeishuEventService +from app.modules.feishu.constants import ( + FEISHU_DEFAULT_OPEN_API_DOMAIN, + FeishuEventSource, + FeishuEventTransport, + FeishuResponseKey, +) +from app.modules.observability.constants import HeartbeatComponent, HeartbeatStatus +from app.modules.observability.service import ObservabilityService logger = logging.getLogger(__name__) @@ -18,6 +30,118 @@ def _sdk_domain(base_url: str) -> str: return f"{parsed.scheme}://{parsed.netloc}" +def _sdk_connection_is_open(client: Any) -> bool: + """Return whether the SDK exposes a currently open WebSocket connection.""" + + try: + connection = getattr(client, "_conn", None) + except Exception: + return False + if connection is None: + return False + + try: + state = getattr(connection, "state", None) + except Exception: + return False + if state is not None: + state_name = getattr(state, "name", None) + state_text = str(state_name or state).strip().lower() + if state_text == "open" or state_text.endswith(".open"): + return True + if any( + marker in state_text + for marker in ("connecting", "closing", "closed") + ): + return False + + try: + closed = getattr(connection, "closed", None) + except Exception: + return False + if isinstance(closed, bool): + return not closed + + try: + opened = getattr(connection, "open", None) + except Exception: + return False + if isinstance(opened, bool): + return opened + + try: + if getattr(connection, "close_code", None) is not None: + return False + transport = getattr(connection, "transport", None) + is_closing = getattr(transport, "is_closing", None) + if callable(is_closing) and is_closing(): + return False + except Exception: + return False + + # A non-null SDK-private connection object is not enough evidence that the + # WebSocket handshake completed or that the transport remains usable. + return False + + +def _connection_heartbeat_status(client: Any) -> str: + if _sdk_connection_is_open(client): + return HeartbeatStatus.OK + return HeartbeatStatus.DEGRADED + + +def _record_runtime_heartbeat(instance_id: str, status_value: str) -> None: + db = SessionLocal() + try: + ObservabilityService(db).record_heartbeat( + component=HeartbeatComponent.FEISHU_EVENTS, + instance_id=instance_id, + status_value=status_value, + actor=ActorValue.FEISHU, + ) + except Exception: + db.rollback() + logger.exception("Failed to record Feishu event process heartbeat") + finally: + db.close() + + +def _heartbeat_loop( + stop_event: Event, + instance_id: str, + interval_seconds: int, + client: Any, +) -> None: + last_status: str | None = None + next_heartbeat_at = 0.0 + while not stop_event.is_set(): + status_value = _connection_heartbeat_status(client) + current_time = monotonic() + if status_value != last_status or current_time >= next_heartbeat_at: + _record_runtime_heartbeat(instance_id, status_value) + last_status = status_value + next_heartbeat_at = current_time + max(1, interval_seconds) + stop_event.wait(1) + + +def _start_heartbeat_loop(client: Any) -> tuple[Event, Thread]: + settings = get_settings() + stop_event = Event() + thread = Thread( + target=_heartbeat_loop, + args=( + stop_event, + gethostname(), + settings.heartbeat_interval_seconds, + client, + ), + name="feishu-events-heartbeat", + daemon=True, + ) + thread.start() + return stop_event, thread + + def _sdk_event_to_payload(event: Any) -> dict[str, Any]: try: from lark_oapi.core.json import JSON @@ -42,20 +166,38 @@ def _handle_verified_sdk_event(event: Any) -> None: payload = _sdk_event_to_payload(event) db = SessionLocal() try: - result = FeishuEventService(db)._handle_verified_event( + acceptance = FeishuEventService(db).accept_verified_event( payload, source=FeishuEventSource.LONG_CONNECTION, auto_reply=True, ) - logger.info("Handled Feishu long connection event: %s", result) finally: db.close() + if ( + acceptance.event_key is not None + and acceptance.should_dispatch + and get_settings().task_queue_enabled + ): + enqueue_feishu_inbound_event( + acceptance.event_key, + actor=ActorValue.FEISHU, + ) + logger.info( + "Accepted Feishu long connection event key=%s status=%s duplicate=%s", + acceptance.event_key, + acceptance.response.get(FeishuResponseKey.STATUS), + bool(acceptance.response.get(FeishuResponseKey.DUPLICATE)), + ) def run_long_connection() -> None: """Start the Feishu long connection client and block forever.""" settings = get_settings() + if settings.feishu_event_transport != FeishuEventTransport.LONG_CONNECTION: + raise RuntimeError( + "FEISHU_EVENT_TRANSPORT must be long_connection for this process" + ) if not settings.feishu_app_id or not settings.feishu_app_secret: raise RuntimeError("FEISHU_APP_ID and FEISHU_APP_SECRET are required") @@ -81,7 +223,12 @@ def run_long_connection() -> None: domain=_sdk_domain(settings.feishu_base_url), ) logger.info("Starting Feishu long connection client") - client.start() + stop_event, heartbeat_thread = _start_heartbeat_loop(client) + try: + client.start() + finally: + stop_event.set() + heartbeat_thread.join(timeout=1) if __name__ == "__main__": diff --git a/app/modules/feishu/models.py b/app/modules/feishu/models.py index bb691f6..fde4352 100644 --- a/app/modules/feishu/models.py +++ b/app/modules/feishu/models.py @@ -1,10 +1,14 @@ from datetime import datetime -from sqlalchemy import DateTime, Integer, String, Text +from sqlalchemy import JSON, Boolean, DateTime, Integer, String, Text, true from sqlalchemy.orm import Mapped, mapped_column from app.core.database import Base from app.core.utils.time import utc_now +from app.modules.feishu.constants import ( + FEISHU_INBOUND_MAX_ATTEMPTS, + FeishuInboundStatus, +) class FeishuEventReceipt(Base): @@ -15,7 +19,79 @@ class FeishuEventReceipt(Base): source: Mapped[str] = mapped_column(String(64), index=True) event_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) message_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + event_type: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + payload: Mapped[dict | None] = mapped_column(JSON, nullable=True) + auto_reply: Mapped[bool] = mapped_column( + Boolean, + default=True, + server_default=true(), + ) + status: Mapped[str] = mapped_column( + String(32), + default=FeishuInboundStatus.PENDING, + server_default=FeishuInboundStatus.PENDING, + index=True, + ) + attempt_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + max_attempts: Mapped[int] = mapped_column( + Integer, + default=FEISHU_INBOUND_MAX_ATTEMPTS, + server_default=str(FEISHU_INBOUND_MAX_ATTEMPTS), + ) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + next_attempt_at: Mapped[datetime | None] = mapped_column( + DateTime, + nullable=True, + index=True, + ) + locked_until: Mapped[datetime | None] = mapped_column( + DateTime, + nullable=True, + index=True, + ) + locked_by: Mapped[str | None] = mapped_column( + String(128), + nullable=True, + index=True, + ) received_at: Mapped[datetime] = mapped_column(DateTime, default=utc_now, index=True) + processed_at: Mapped[datetime | None] = mapped_column( + DateTime, + nullable=True, + index=True, + ) + reply_payload: Mapped[dict | None] = mapped_column(JSON, nullable=True) + reply_status: Mapped[str | None] = mapped_column( + String(32), + nullable=True, + index=True, + ) + reply_attempt_count: Mapped[int] = mapped_column( + Integer, + default=0, + server_default="0", + ) + reply_last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + reply_next_attempt_at: Mapped[datetime | None] = mapped_column( + DateTime, + nullable=True, + index=True, + ) + reply_locked_until: Mapped[datetime | None] = mapped_column( + DateTime, + nullable=True, + index=True, + ) + reply_locked_by: Mapped[str | None] = mapped_column( + String(128), + nullable=True, + index=True, + ) + reply_sent_at: Mapped[datetime | None] = mapped_column( + DateTime, + nullable=True, + index=True, + ) class FeishuAppTicket(Base): diff --git a/app/modules/feishu/routes.py b/app/modules/feishu/routes.py index 6aeecb9..159f89f 100644 --- a/app/modules/feishu/routes.py +++ b/app/modules/feishu/routes.py @@ -1,10 +1,18 @@ -from fastapi import APIRouter, Depends, Request +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, status from sqlalchemy.orm import Session +from app.core.background.task_queue import enqueue_feishu_inbound_event +from app.core.config import get_settings +from app.core.constants import ActorValue from app.core.database import get_db from app.core.security import ApiPrincipal, require_api_key from app.application.feishu import FeishuCommandService, FeishuEventService -from app.modules.feishu.constants import FeishuEventSource, FeishuPayloadKey, FeishuResponseKey +from app.modules.feishu.constants import ( + FeishuEventSource, + FeishuEventTransport, + FeishuPayloadKey, + FeishuResponseKey, +) from app.modules.feishu.event_verification import FeishuWebhookVerifier from app.modules.feishu.schemas import ( FeishuCardMessage, @@ -19,15 +27,31 @@ router = APIRouter() @router.post("/webhook") -async def feishu_webhook(request: Request, db: Session = Depends(get_db)) -> dict: +async def feishu_webhook( + request: Request, + background_tasks: BackgroundTasks, + db: Session = Depends(get_db), +) -> dict: """Handle Feishu webhook challenge and text command events.""" + if get_settings().feishu_event_transport != FeishuEventTransport.WEBHOOK: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Feishu webhook transport is disabled", + ) payload = FeishuWebhookVerifier().verify(await request.body(), request.headers) - return FeishuEventService(db)._handle_verified_event( + acceptance = FeishuEventService(db).accept_verified_event( payload, source=FeishuEventSource.WEBHOOK, auto_reply=True, ) + if acceptance.event_key is not None and acceptance.should_dispatch: + background_tasks.add_task( + enqueue_feishu_inbound_event, + acceptance.event_key, + actor=ActorValue.FEISHU, + ) + return acceptance.response @router.post("/send-text", response_model=FeishuSendResult) diff --git a/app/modules/feishu/service.py b/app/modules/feishu/service.py index 625bd64..6881bed 100644 --- a/app/modules/feishu/service.py +++ b/app/modules/feishu/service.py @@ -18,6 +18,7 @@ from app.modules.feishu.constants import ( FeishuPayloadKey, FeishuReceiveIdType, ) +from app.modules.feishu.services.reply_outbox import current_reply_outbox class FeishuService: @@ -30,12 +31,18 @@ class FeishuService: self.tenant_key = _optional_text(tenant_key) or _optional_text( get_settings().feishu_default_tenant_key ) + self.message_uuid: str | None = None def set_tenant_key(self, tenant_key: str | None) -> None: """Set the default tenant used by subsequent outbound operations.""" self.tenant_key = _optional_text(tenant_key) + def set_message_uuid(self, message_uuid: str | None) -> None: + """Set the idempotency UUID used by replies in the current command.""" + + self.message_uuid = _optional_text(message_uuid) + def verify_event(self, payload: dict[str, Any]) -> None: settings = get_settings() expected = settings.feishu_verification_token @@ -62,12 +69,26 @@ class FeishuService: tenant_key: str | None = None, record_audit: bool = True, ) -> dict[str, Any]: + resolved_uuid = uuid or self.message_uuid + resolved_tenant_key = self._resolve_tenant_key(tenant_key) + reply_outbox = current_reply_outbox() + if reply_outbox is not None: + return reply_outbox.capture_text( + text=text, + receive_id=receive_id, + default_receive_id=get_settings().feishu_default_chat_id, + receive_id_type=receive_id_type, + actor=actor, + message_uuid=resolved_uuid, + tenant_key=resolved_tenant_key, + record_audit=record_audit, + ) result = self.client.send_text( text, receive_id, receive_id_type, - uuid, - tenant_key=self._resolve_tenant_key(tenant_key), + resolved_uuid, + tenant_key=resolved_tenant_key, ) if record_audit: self.audit.log( @@ -79,7 +100,7 @@ class FeishuService: "receive_target_hash": _target_fingerprint(receive_id), FeishuPayloadKey.RECEIVE_ID_TYPE: receive_id_type, "content_length": len(text), - FeishuPayloadKey.UUID: uuid, + FeishuPayloadKey.UUID: resolved_uuid, }, response_payload=result, ) @@ -95,12 +116,25 @@ class FeishuService: uuid: str | None = None, tenant_key: str | None = None, ) -> dict[str, Any]: + resolved_uuid = uuid or self.message_uuid + resolved_tenant_key = self._resolve_tenant_key(tenant_key) + reply_outbox = current_reply_outbox() + if reply_outbox is not None: + return reply_outbox.capture_card( + card=card, + receive_id=receive_id, + default_receive_id=get_settings().feishu_default_chat_id, + receive_id_type=receive_id_type, + actor=actor, + message_uuid=resolved_uuid, + tenant_key=resolved_tenant_key, + ) result = self.client.send_card( card, receive_id, receive_id_type, - uuid, - tenant_key=self._resolve_tenant_key(tenant_key), + resolved_uuid, + tenant_key=resolved_tenant_key, ) self.audit.log( AuditLogCreate( @@ -111,7 +145,7 @@ class FeishuService: "receive_target_hash": _target_fingerprint(receive_id), FeishuPayloadKey.RECEIVE_ID_TYPE: receive_id_type, "card_element_count": len(card.get(FeishuPayloadKey.ELEMENTS) or []), - FeishuPayloadKey.UUID: uuid, + FeishuPayloadKey.UUID: resolved_uuid, }, response_payload=result, ) @@ -124,9 +158,17 @@ class FeishuService: actor: str = ActorValue.SYSTEM, tenant_key: str | None = None, ) -> dict[str, Any]: + resolved_tenant_key = self._resolve_tenant_key(tenant_key) + reply_outbox = current_reply_outbox() + if reply_outbox is not None: + return reply_outbox.capture_image( + image=image, + actor=actor, + tenant_key=resolved_tenant_key, + ) result = self.client.upload_image( image, - tenant_key=self._resolve_tenant_key(tenant_key), + tenant_key=resolved_tenant_key, ) self.audit.log( AuditLogCreate( diff --git a/app/modules/feishu/services/__init__.py b/app/modules/feishu/services/__init__.py new file mode 100644 index 0000000..73ba95f --- /dev/null +++ b/app/modules/feishu/services/__init__.py @@ -0,0 +1,17 @@ +from app.modules.feishu.services.inbox import ( + FeishuInboundAcceptance, + FeishuInboundProcessResult, + FeishuInboundService, +) +from app.modules.feishu.services.context import ( + bind_inbound_event, + current_inbound_event_key, +) + +__all__ = [ + "FeishuInboundAcceptance", + "FeishuInboundProcessResult", + "FeishuInboundService", + "bind_inbound_event", + "current_inbound_event_key", +] diff --git a/app/modules/feishu/services/context.py b/app/modules/feishu/services/context.py new file mode 100644 index 0000000..2ed9aa8 --- /dev/null +++ b/app/modules/feishu/services/context.py @@ -0,0 +1,23 @@ +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar + +_CURRENT_INBOUND_EVENT_KEY: ContextVar[str | None] = ContextVar( + "current_feishu_inbound_event_key", + default=None, +) + + +@contextmanager +def bind_inbound_event(event_key: str) -> Iterator[None]: + """Expose the current inbox key to privacy-cleanup hooks.""" + + token = _CURRENT_INBOUND_EVENT_KEY.set(event_key) + try: + yield + finally: + _CURRENT_INBOUND_EVENT_KEY.reset(token) + + +def current_inbound_event_key() -> str | None: + return _CURRENT_INBOUND_EVENT_KEY.get() diff --git a/app/modules/feishu/services/inbox.py b/app/modules/feishu/services/inbox.py new file mode 100644 index 0000000..bb9e518 --- /dev/null +++ b/app/modules/feishu/services/inbox.py @@ -0,0 +1,1305 @@ +from collections.abc import Callable +from copy import deepcopy +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any +from uuid import uuid4 + +from fastapi import HTTPException, status +from sqlalchemy import and_, func, or_, select, update +from sqlalchemy.engine import Connection, Engine +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.core.http.pagination import bounded_limit +from app.core.utils.time import utc_now +from app.modules.feishu.constants import ( + FEISHU_INBOUND_BATCH_SIZE, + FEISHU_INBOUND_LEASE_SECONDS, + FEISHU_INBOUND_MAX_ATTEMPTS, + FEISHU_INBOUND_RETRY_DELAYS_SECONDS, + FeishuCommandResultKey, + FeishuEventReceiptKey, + FeishuInboundStatus, + FeishuResponseKey, +) +from app.modules.feishu.errors import FeishuAPIError +from app.modules.feishu.models import FeishuEventReceipt +from app.modules.feishu.services.reply_outbox import ( + bind_reply_outbox, + decode_image, + identity_fence_required, + operations_copy, + payload_identity as reply_payload_identity, + pending_image_indexes, + resolved_message_operation, +) + +InboundHandler = Callable[[dict[str, Any], str, bool, str], dict[str, Any]] +InboundHandlerFactory = Callable[[Session], InboundHandler] +_TERMINAL_STATUSES = frozenset( + { + FeishuInboundStatus.SUCCEEDED, + FeishuInboundStatus.FAILED, + } +) +_SENSITIVE_TRANSPORT_KEYS = frozenset( + { + "access_token", + "app_access_token", + "app_secret", + "app_ticket", + "authorization", + "encrypt", + "refresh_token", + "secret", + "tenant_access_token", + "token", + } +) + + +@dataclass(frozen=True, slots=True) +class FeishuInboundAcceptance: + """Result of durably accepting one verified Feishu event.""" + + record: FeishuEventReceipt + created: bool + + @property + def should_dispatch(self) -> bool: + return self.record.status not in _TERMINAL_STATUSES + + +@dataclass(frozen=True, slots=True) +class FeishuInboundProcessResult: + """Persistent state and the transient command result from one attempt.""" + + record: FeishuEventReceipt + handler_result: dict[str, Any] | None = None + + +class FeishuInboundService: + """Persist, claim, retry, and finalize verified Feishu inbound events.""" + + def __init__( + self, + db: Session, + *, + lease_seconds: int = FEISHU_INBOUND_LEASE_SECONDS, + ) -> None: + self.db = db + self.lease_seconds = lease_seconds + + def accept( + self, + payload: dict[str, Any], + event_identity: dict[str, str | None], + *, + event_type: str | None, + auto_reply: bool, + persist_payload: bool = True, + now: datetime | None = None, + ) -> FeishuInboundAcceptance: + """Commit an inbox row before transport acknowledgement.""" + + current = now or utc_now() + event_key = str(event_identity[FeishuEventReceiptKey.EVENT_KEY]) + receipt = FeishuEventReceipt( + event_key=event_key, + source=str(event_identity[FeishuEventReceiptKey.SOURCE]), + event_id=event_identity.get(FeishuEventReceiptKey.EVENT_ID), + message_id=event_identity.get(FeishuEventReceiptKey.MESSAGE_ID), + event_type=event_type or None, + payload=_sanitize_payload(payload) if persist_payload else None, + auto_reply=auto_reply, + status=FeishuInboundStatus.PENDING, + attempt_count=0, + max_attempts=FEISHU_INBOUND_MAX_ATTEMPTS, + next_attempt_at=current, + received_at=current, + ) + try: + with self.db.begin_nested(): + self.db.add(receipt) + self.db.flush() + except IntegrityError: + self.db.rollback() + existing = self._get(event_key) + return FeishuInboundAcceptance(record=existing, created=False) + self.db.commit() + self.db.refresh(receipt) + return FeishuInboundAcceptance(record=receipt, created=True) + + def process( + self, + event_key: str, + *, + handler_factory: InboundHandlerFactory, + payload_override: dict[str, Any] | None = None, + now: datetime | None = None, + worker_id: str = "feishu-inbound", + ) -> FeishuInboundProcessResult: + """Claim and execute one event, fencing concurrent duplicate workers.""" + + current = now or utc_now() + existing = self._get(event_key) + if existing.status in _TERMINAL_STATUSES: + return FeishuInboundProcessResult(record=existing) + if existing.attempt_count >= existing.max_attempts: + return FeishuInboundProcessResult( + record=self._fail_exhausted(existing.id, current) + ) + + lock_owner = f"{worker_id}:{uuid4().hex}" + if not self._claim( + existing.id, + lock_owner, + current, + max_attempts=existing.max_attempts, + expected_attempt_count=existing.attempt_count, + allow_pending_immediate=( + payload_override is not None and existing.payload is None + ), + ): + self.db.rollback() + return FeishuInboundProcessResult(record=self._get(event_key)) + record = self._get(event_key) + effective_payload = ( + payload_override + if payload_override is not None + else record.payload + ) + if not isinstance(effective_payload, dict): + return FeishuInboundProcessResult( + record=self._finish_failure( + event_key, + lock_owner, + current, + ValueError("Feishu inbound payload is unavailable"), + retryable=False, + ) + ) + + source = str(record.source) + auto_reply = bool(record.auto_reply) + self.db.rollback() + try: + result = self._execute_atomically( + event_key=event_key, + lock_owner=lock_owner, + current=current, + payload=dict(effective_payload), + source=source, + auto_reply=auto_reply, + handler_factory=handler_factory, + ) + except Exception as exc: + self.db.rollback() + return FeishuInboundProcessResult( + record=self._finish_failure( + event_key, + lock_owner, + current, + exc, + retryable=_is_retryable(exc), + ) + ) + reply_result = self.dispatch_reply( + event_key, + now=current, + worker_id=f"{worker_id}:reply", + ) + _attach_provider_response(result, reply_result) + self.db.expire_all() + return FeishuInboundProcessResult( + record=self._get(event_key), + handler_result=result, + ) + + def process_due( + self, + *, + handler_factory: InboundHandlerFactory, + now: datetime | None = None, + limit: int = FEISHU_INBOUND_BATCH_SIZE, + worker_id: str = "feishu-inbound", + ) -> list[FeishuInboundProcessResult]: + """Process due pending/retry rows and reclaim expired processing leases.""" + + current = now or utc_now() + stmt = ( + select(FeishuEventReceipt.event_key) + .where( + _claimable(current), + FeishuEventReceipt.payload.is_not(None), + ) + .order_by( + FeishuEventReceipt.next_attempt_at.asc(), + FeishuEventReceipt.id.asc(), + ) + .limit(bounded_limit(limit)) + ) + event_keys = list(self.db.execute(stmt).scalars()) + outcomes = [ + self.process( + event_key, + handler_factory=handler_factory, + now=current, + worker_id=worker_id, + ) + for event_key in event_keys + ] + self.process_due_replies( + now=current, + limit=limit, + worker_id=f"{worker_id}:reply", + ) + return outcomes + + def process_due_replies( + self, + *, + now: datetime | None = None, + limit: int = FEISHU_INBOUND_BATCH_SIZE, + worker_id: str = "feishu-inbound-reply", + ) -> int: + """Retry committed reply outboxes without re-running their commands.""" + + current = now or utc_now() + event_keys = list( + self.db.execute( + select(FeishuEventReceipt.event_key) + .where( + FeishuEventReceipt.status == FeishuInboundStatus.SUCCEEDED, + FeishuEventReceipt.reply_payload.is_not(None), + _reply_claimable(current), + ) + .order_by( + FeishuEventReceipt.reply_next_attempt_at.asc(), + FeishuEventReceipt.id.asc(), + ) + .limit(bounded_limit(limit)) + ).scalars() + ) + for event_key in event_keys: + self.dispatch_reply( + event_key, + now=current, + worker_id=worker_id, + ) + return len(event_keys) + + def dispatch_reply( + self, + event_key: str, + *, + now: datetime | None = None, + worker_id: str = "feishu-inbound-reply", + ) -> dict[str, Any] | None: + """Send one committed reply intent with a stable provider UUID.""" + + current = now or utc_now() + record = self._get(event_key) + if record.reply_status in _TERMINAL_STATUSES or record.reply_payload is None: + return None + if record.reply_attempt_count >= record.max_attempts: + self._fail_reply_exhausted(record.id, current) + return None + + lock_owner = f"{worker_id}:{uuid4().hex}" + if not self._claim_reply(record.id, lock_owner, current): + self.db.rollback() + return None + try: + self._prepare_reply_images(event_key, lock_owner) + return self._send_prepared_reply( + event_key, + lock_owner, + current, + ) + except Exception as exc: + self.db.rollback() + self._finish_reply_failure( + event_key, + lock_owner, + current, + exc, + retryable=_is_retryable(exc), + ) + return None + + def erase_identity_payloads( + self, + *, + tenant_key: str, + open_id: str, + exclude_event_key: str | None = None, + now: datetime | None = None, + ) -> int: + """Clear replayable events for an identity being permanently erased.""" + + current = now or utc_now() + candidates = list( + self.db.execute( + select(FeishuEventReceipt).where( + FeishuEventReceipt.payload.is_not(None), + FeishuEventReceipt.status.in_( + [ + FeishuInboundStatus.PENDING, + FeishuInboundStatus.PROCESSING, + FeishuInboundStatus.RETRY, + ] + ), + ) + ).scalars() + ) + matching_ids = [ + record.id + for record in candidates + if record.event_key != exclude_event_key + and _matches_identity(record.payload, tenant_key, open_id) + ] + if matching_ids: + self.db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.id.in_(matching_ids), + FeishuEventReceipt.status.in_( + [ + FeishuInboundStatus.PENDING, + FeishuInboundStatus.PROCESSING, + FeishuInboundStatus.RETRY, + ] + ), + ) + .values( + status=FeishuInboundStatus.FAILED, + payload=None, + last_error="personal_data_erased", + next_attempt_at=None, + locked_by=None, + locked_until=None, + processed_at=current, + ) + .execution_options(synchronize_session=False) + ) + + reply_candidates = list( + self.db.execute( + select(FeishuEventReceipt).where( + FeishuEventReceipt.reply_payload.is_not(None), + FeishuEventReceipt.reply_status.in_( + [ + FeishuInboundStatus.PENDING, + FeishuInboundStatus.PROCESSING, + FeishuInboundStatus.RETRY, + ] + ), + ) + ).scalars() + ) + reply_matching_ids = [ + record.id + for record in reply_candidates + if record.event_key != exclude_event_key + and reply_payload_identity(record.reply_payload) + == (tenant_key, open_id) + ] + if reply_matching_ids: + self.db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.id.in_(reply_matching_ids), + FeishuEventReceipt.reply_status.in_( + [ + FeishuInboundStatus.PENDING, + FeishuInboundStatus.PROCESSING, + FeishuInboundStatus.RETRY, + ] + ), + ) + .values( + reply_status=FeishuInboundStatus.FAILED, + reply_payload=None, + reply_last_error="personal_data_erased", + reply_next_attempt_at=None, + reply_locked_by=None, + reply_locked_until=None, + ) + .execution_options(synchronize_session=False) + ) + self.db.flush() + self.db.expire_all() + return len(set(matching_ids) | set(reply_matching_ids)) + + def status_counts(self) -> dict[str, int]: + """Return operational inbox counts without loading event payloads.""" + + tracked = ( + FeishuInboundStatus.PENDING, + FeishuInboundStatus.RETRY, + FeishuInboundStatus.PROCESSING, + FeishuInboundStatus.FAILED, + ) + rows = self.db.execute( + select( + FeishuEventReceipt.status, + func.count(FeishuEventReceipt.id), + ) + .where(FeishuEventReceipt.status.in_(tracked)) + .group_by(FeishuEventReceipt.status) + ) + counts = {str(status_value): 0 for status_value in tracked} + counts.update( + { + str(status_value): int(count) + for status_value, count in rows + } + ) + reply_rows = self.db.execute( + select( + FeishuEventReceipt.reply_status, + func.count(FeishuEventReceipt.id), + ) + .where(FeishuEventReceipt.reply_status.in_(tracked)) + .group_by(FeishuEventReceipt.reply_status) + ) + counts.update( + { + f"reply_{status_value}": 0 + for status_value in tracked + } + ) + counts.update( + { + f"reply_{status_value}": int(count) + for status_value, count in reply_rows + } + ) + return counts + + def _execute_atomically( + self, + *, + event_key: str, + lock_owner: str, + current: datetime, + payload: dict[str, Any], + source: str, + auto_reply: bool, + handler_factory: InboundHandlerFactory, + ) -> dict[str, Any]: + """Commit command writes and inbox success as one database transaction.""" + + bind = self.db.get_bind() + engine = bind.engine if isinstance(bind, Connection) else bind + if not isinstance(engine, Engine): + raise RuntimeError("Feishu inbound processing requires a SQLAlchemy engine") + + with engine.connect() as connection: + transaction = connection.begin() + try: + if connection.dialect.name == "sqlite": + # SQLite has no SELECT FOR UPDATE. BEGIN IMMEDIATE provides + # the single-process test substitute while also ensuring a + # released handler SAVEPOINT cannot escape the outer rollback. + connection.exec_driver_sql("BEGIN IMMEDIATE") + with Session( + bind=connection, + expire_on_commit=False, + join_transaction_mode="create_savepoint", + ) as atomic_db: + _lock_identity_fence(atomic_db, payload) + receipt = atomic_db.scalar( + select(FeishuEventReceipt) + .where(FeishuEventReceipt.event_key == event_key) + .with_for_update() + ) + if ( + receipt is None + or receipt.status != FeishuInboundStatus.PROCESSING + or receipt.locked_by != lock_owner + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Feishu inbound event lease was lost", + ) + with bind_reply_outbox() as reply_outbox: + result = handler_factory(atomic_db)( + payload, + source, + auto_reply, + event_key, + ) + identity = _payload_identity(payload) + reply_payload = reply_outbox.as_payload( + identity=identity, + identity_fence_required=_identity_exists( + atomic_db, + identity, + ), + ) + if reply_payload is not None: + receipt.reply_payload = reply_payload + receipt.reply_status = FeishuInboundStatus.PENDING + receipt.reply_attempt_count = 0 + receipt.reply_last_error = None + receipt.reply_next_attempt_at = current + receipt.reply_locked_by = None + receipt.reply_locked_until = None + receipt.reply_sent_at = None + self._mark_success( + atomic_db, + event_key, + lock_owner, + current, + ) + atomic_db.commit() + transaction.commit() + return result + except Exception: + if transaction.is_active: + transaction.rollback() + raise + + def _claim_reply( + self, + receipt_id: int, + lock_owner: str, + current: datetime, + ) -> bool: + result = self.db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.id == receipt_id, + FeishuEventReceipt.status == FeishuInboundStatus.SUCCEEDED, + FeishuEventReceipt.reply_payload.is_not(None), + _reply_claimable(current), + FeishuEventReceipt.reply_attempt_count + < FeishuEventReceipt.max_attempts, + ) + .values( + reply_status=FeishuInboundStatus.PROCESSING, + reply_attempt_count=( + FeishuEventReceipt.reply_attempt_count + 1 + ), + reply_locked_by=lock_owner, + reply_locked_until=current + timedelta(seconds=self.lease_seconds), + ) + .execution_options(synchronize_session=False) + ) + self.db.commit() + self.db.expire_all() + return result.rowcount == 1 + + def _prepare_reply_images( + self, + event_key: str, + lock_owner: str, + ) -> None: + while True: + self.db.expire_all() + record = self._get(event_key) + payload = record.reply_payload + if not isinstance(payload, dict): + raise ValueError("Feishu reply outbox payload is unavailable") + indexes = pending_image_indexes(payload) + self.db.rollback() + if not indexes: + return + self._prepare_reply_image( + event_key, + lock_owner, + indexes[0], + ) + + def _prepare_reply_image( + self, + event_key: str, + lock_owner: str, + operation_index: int, + ) -> None: + bind = self.db.get_bind() + engine = bind.engine if isinstance(bind, Connection) else bind + if not isinstance(engine, Engine): + raise RuntimeError("Feishu reply processing requires a SQLAlchemy engine") + + with engine.connect() as connection: + transaction = connection.begin() + try: + if connection.dialect.name == "sqlite": + connection.exec_driver_sql("BEGIN IMMEDIATE") + with Session( + bind=connection, + expire_on_commit=False, + join_transaction_mode="create_savepoint", + ) as atomic_db: + receipt = self._locked_reply( + atomic_db, + event_key, + lock_owner, + ) + payload = receipt.reply_payload + if not isinstance(payload, dict): + raise ValueError( + "Feishu reply outbox payload is unavailable" + ) + _lock_reply_identity(atomic_db, payload) + operations = operations_copy(payload) + if operation_index >= len(operations): + raise ValueError( + "Feishu reply image operation is unavailable" + ) + operation = operations[operation_index] + if operation.get("kind") != "image": + raise ValueError( + "Feishu reply image operation is invalid" + ) + if operation.get("image_key"): + atomic_db.commit() + transaction.commit() + return + + from app.modules.feishu.service import FeishuService + + result = FeishuService(atomic_db).upload_image( + decode_image(operation), + actor=str(operation.get("actor") or "feishu"), + tenant_key=_optional_text( + operation.get("tenant_key") + ), + ) + image_key = str( + (result.get("data") or {}).get("image_key") or "" + ).strip() + if not image_key: + raise ValueError( + "Feishu image upload did not return image_key" + ) + operation["image_key"] = image_key + operation.pop("image_base64", None) + updated_payload = deepcopy(payload) + updated_payload["operations"] = operations + receipt.reply_payload = updated_payload + atomic_db.commit() + transaction.commit() + except Exception: + if transaction.is_active: + transaction.rollback() + raise + + def _send_prepared_reply( + self, + event_key: str, + lock_owner: str, + current: datetime, + ) -> dict[str, Any]: + bind = self.db.get_bind() + engine = bind.engine if isinstance(bind, Connection) else bind + if not isinstance(engine, Engine): + raise RuntimeError("Feishu reply processing requires a SQLAlchemy engine") + + with engine.connect() as connection: + transaction = connection.begin() + try: + if connection.dialect.name == "sqlite": + connection.exec_driver_sql("BEGIN IMMEDIATE") + with Session( + bind=connection, + expire_on_commit=False, + join_transaction_mode="create_savepoint", + ) as atomic_db: + receipt = self._locked_reply( + atomic_db, + event_key, + lock_owner, + ) + payload = receipt.reply_payload + if not isinstance(payload, dict): + raise ValueError( + "Feishu reply outbox payload is unavailable" + ) + _lock_reply_identity(atomic_db, payload) + operation = resolved_message_operation(payload) + + from app.modules.feishu.service import FeishuService + + feishu = FeishuService( + atomic_db, + tenant_key=_optional_text( + operation.get("tenant_key") + ), + ) + receive_id = _required_text( + operation.get("receive_id"), + "Feishu reply receive_id is unavailable", + ) + receive_id_type = _required_text( + operation.get("receive_id_type"), + "Feishu reply receive_id_type is unavailable", + ) + actor = str(operation.get("actor") or "feishu") + message_uuid = _optional_text( + operation.get("message_uuid") + ) + if operation.get("kind") == "text": + response = feishu.send_text( + str(operation.get("text") or ""), + receive_id=receive_id, + receive_id_type=receive_id_type, + actor=actor, + uuid=message_uuid, + tenant_key=_optional_text( + operation.get("tenant_key") + ), + record_audit=bool( + operation.get("record_audit", True) + ), + ) + else: + card = operation.get("card") + if not isinstance(card, dict): + raise ValueError( + "Feishu reply card payload is invalid" + ) + response = feishu.send_card( + card, + receive_id=receive_id, + receive_id_type=receive_id_type, + actor=actor, + uuid=message_uuid, + tenant_key=_optional_text( + operation.get("tenant_key") + ), + ) + self._mark_reply_success( + atomic_db, + event_key, + lock_owner, + current, + ) + atomic_db.commit() + transaction.commit() + return response + except Exception: + if transaction.is_active: + transaction.rollback() + raise + + @staticmethod + def _locked_reply( + db: Session, + event_key: str, + lock_owner: str, + ) -> FeishuEventReceipt: + receipt = db.scalar( + select(FeishuEventReceipt) + .where(FeishuEventReceipt.event_key == event_key) + .with_for_update() + ) + if ( + receipt is None + or receipt.status != FeishuInboundStatus.SUCCEEDED + or receipt.reply_status != FeishuInboundStatus.PROCESSING + or receipt.reply_locked_by != lock_owner + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Feishu inbound reply lease was lost", + ) + return receipt + + @staticmethod + def _mark_reply_success( + db: Session, + event_key: str, + lock_owner: str, + current: datetime, + ) -> None: + result = db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.event_key == event_key, + FeishuEventReceipt.reply_status + == FeishuInboundStatus.PROCESSING, + FeishuEventReceipt.reply_locked_by == lock_owner, + ) + .values( + reply_status=FeishuInboundStatus.SUCCEEDED, + reply_payload=None, + reply_last_error=None, + reply_next_attempt_at=None, + reply_locked_by=None, + reply_locked_until=None, + reply_sent_at=current, + ) + .execution_options(synchronize_session=False) + ) + if result.rowcount != 1: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Feishu inbound reply lease was lost", + ) + + def _finish_reply_failure( + self, + event_key: str, + lock_owner: str, + current: datetime, + exc: Exception, + *, + retryable: bool, + ) -> None: + self.db.expire_all() + record = self._get(event_key) + retry_index = record.reply_attempt_count - 1 + will_retry = ( + retryable + and 0 <= retry_index < len(FEISHU_INBOUND_RETRY_DELAYS_SECONDS) + and record.reply_attempt_count < record.max_attempts + ) + next_attempt_at = ( + current + + timedelta(seconds=FEISHU_INBOUND_RETRY_DELAYS_SECONDS[retry_index]) + if will_retry + else None + ) + self.db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.event_key == event_key, + FeishuEventReceipt.reply_status + == FeishuInboundStatus.PROCESSING, + FeishuEventReceipt.reply_locked_by == lock_owner, + ) + .values( + reply_status=( + FeishuInboundStatus.RETRY + if will_retry + else FeishuInboundStatus.FAILED + ), + reply_payload=record.reply_payload if will_retry else None, + reply_last_error=_error_summary(exc), + reply_next_attempt_at=next_attempt_at, + reply_locked_by=None, + reply_locked_until=None, + ) + .execution_options(synchronize_session=False) + ) + self.db.commit() + self.db.expire_all() + + def _fail_reply_exhausted( + self, + receipt_id: int, + current: datetime, + ) -> None: + self.db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.id == receipt_id, + FeishuEventReceipt.reply_status.not_in(_TERMINAL_STATUSES), + or_( + FeishuEventReceipt.reply_locked_until.is_(None), + FeishuEventReceipt.reply_locked_until <= current, + ), + ) + .values( + reply_status=FeishuInboundStatus.FAILED, + reply_payload=None, + reply_last_error="attempts_exhausted", + reply_next_attempt_at=None, + reply_locked_by=None, + reply_locked_until=None, + ) + .execution_options(synchronize_session=False) + ) + self.db.commit() + self.db.expire_all() + + def _claim( + self, + receipt_id: int, + lock_owner: str, + current: datetime, + *, + max_attempts: int, + expected_attempt_count: int, + allow_pending_immediate: bool = False, + ) -> bool: + if allow_pending_immediate: + record = self.db.execute( + select(FeishuEventReceipt) + .where(FeishuEventReceipt.id == receipt_id) + .with_for_update(skip_locked=True) + ).scalar_one_or_none() + if ( + record is None + or record.attempt_count != expected_attempt_count + or record.attempt_count >= max_attempts + or record.locked_by is not None + ): + self.db.commit() + return False + record.status = FeishuInboundStatus.PROCESSING + record.attempt_count += 1 + record.locked_by = lock_owner + record.locked_until = current + timedelta(seconds=self.lease_seconds) + self.db.commit() + self.db.expire_all() + return True + result = self.db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.id == receipt_id, + _claimable(current), + FeishuEventReceipt.attempt_count < max_attempts, + ) + .values( + status=FeishuInboundStatus.PROCESSING, + attempt_count=FeishuEventReceipt.attempt_count + 1, + locked_by=lock_owner, + locked_until=current + timedelta(seconds=self.lease_seconds), + ) + .execution_options(synchronize_session=False) + ) + self.db.commit() + self.db.expire_all() + return result.rowcount == 1 + + @staticmethod + def _mark_success( + db: Session, + event_key: str, + lock_owner: str, + current: datetime, + ) -> None: + result = db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.event_key == event_key, + FeishuEventReceipt.status == FeishuInboundStatus.PROCESSING, + FeishuEventReceipt.locked_by == lock_owner, + ) + .values( + status=FeishuInboundStatus.SUCCEEDED, + payload=None, + last_error=None, + next_attempt_at=None, + locked_by=None, + locked_until=None, + processed_at=current, + ) + .execution_options(synchronize_session=False) + ) + if result.rowcount != 1: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Feishu inbound event lease was lost", + ) + + def _finish_failure( + self, + event_key: str, + lock_owner: str, + current: datetime, + exc: Exception, + *, + retryable: bool, + ) -> FeishuEventReceipt: + self.db.expire_all() + record = self._get(event_key) + retry_index = record.attempt_count - 1 + will_retry = ( + retryable + and 0 <= retry_index < len(FEISHU_INBOUND_RETRY_DELAYS_SECONDS) + and record.attempt_count < record.max_attempts + ) + next_attempt_at = ( + current + + timedelta(seconds=FEISHU_INBOUND_RETRY_DELAYS_SECONDS[retry_index]) + if will_retry + else None + ) + result = self.db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.event_key == event_key, + FeishuEventReceipt.status == FeishuInboundStatus.PROCESSING, + FeishuEventReceipt.locked_by == lock_owner, + ) + .values( + status=( + FeishuInboundStatus.RETRY + if will_retry + else FeishuInboundStatus.FAILED + ), + payload=record.payload if will_retry else None, + last_error=_error_summary(exc), + next_attempt_at=next_attempt_at, + locked_by=None, + locked_until=None, + processed_at=None if will_retry else current, + ) + .execution_options(synchronize_session=False) + ) + if result.rowcount != 1: + self.db.rollback() + return self._get(event_key) + self.db.commit() + self.db.expire_all() + return self._get(event_key) + + def _fail_exhausted( + self, + receipt_id: int, + current: datetime, + ) -> FeishuEventReceipt: + self.db.execute( + update(FeishuEventReceipt) + .where( + FeishuEventReceipt.id == receipt_id, + FeishuEventReceipt.status.not_in(_TERMINAL_STATUSES), + or_( + FeishuEventReceipt.locked_until.is_(None), + FeishuEventReceipt.locked_until <= current, + ), + ) + .values( + status=FeishuInboundStatus.FAILED, + payload=None, + last_error="attempts_exhausted", + next_attempt_at=None, + locked_by=None, + locked_until=None, + processed_at=current, + ) + .execution_options(synchronize_session=False) + ) + self.db.commit() + self.db.expire_all() + record = self.db.get(FeishuEventReceipt, receipt_id) + if record is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Feishu inbound event was not found", + ) + return record + + def _get(self, event_key: str) -> FeishuEventReceipt: + record = self.db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == event_key + ) + ) + if record is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Feishu inbound event was not found", + ) + return record + + +def _claimable(current: datetime) -> Any: + return or_( + and_( + FeishuEventReceipt.status.in_( + [ + FeishuInboundStatus.PENDING, + FeishuInboundStatus.RETRY, + ] + ), + FeishuEventReceipt.next_attempt_at.is_not(None), + FeishuEventReceipt.next_attempt_at <= current, + or_( + FeishuEventReceipt.locked_until.is_(None), + FeishuEventReceipt.locked_until <= current, + ), + ), + and_( + FeishuEventReceipt.status == FeishuInboundStatus.PROCESSING, + FeishuEventReceipt.locked_until.is_not(None), + FeishuEventReceipt.locked_until <= current, + ), + ) + + +def _reply_claimable(current: datetime) -> Any: + return or_( + and_( + FeishuEventReceipt.reply_status.in_( + [ + FeishuInboundStatus.PENDING, + FeishuInboundStatus.RETRY, + ] + ), + FeishuEventReceipt.reply_next_attempt_at.is_not(None), + FeishuEventReceipt.reply_next_attempt_at <= current, + or_( + FeishuEventReceipt.reply_locked_until.is_(None), + FeishuEventReceipt.reply_locked_until <= current, + ), + ), + and_( + FeishuEventReceipt.reply_status + == FeishuInboundStatus.PROCESSING, + FeishuEventReceipt.reply_locked_until.is_not(None), + FeishuEventReceipt.reply_locked_until <= current, + ), + ) + + +def _sanitize_payload(value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): _sanitize_payload(item) + for key, item in value.items() + if str(key).casefold() not in _SENSITIVE_TRANSPORT_KEYS + } + if isinstance(value, list): + return [_sanitize_payload(item) for item in value] + if isinstance(value, tuple): + return [_sanitize_payload(item) for item in value] + return value + + +def _matches_identity( + payload: Any, + tenant_key: str, + open_id: str, +) -> bool: + return _payload_identity(payload) == (tenant_key, open_id) + + +def _payload_identity(payload: Any) -> tuple[str, str] | None: + if not isinstance(payload, dict): + return None + header = payload.get("header") or {} + event = payload.get("event") or {} + sender = event.get("sender") or {} + sender_id = sender.get("sender_id") or {} + if not isinstance(header, dict) or not isinstance(sender_id, dict): + return None + tenant_key = str(header.get("tenant_key") or "").strip() + open_id = str(sender_id.get("open_id") or "").strip() + if not tenant_key or not open_id: + return None + return tenant_key, open_id + + +def _lock_identity_fence(db: Session, payload: dict[str, Any]) -> None: + """Serialize an identity's handler with erasure before locking its inbox row.""" + + identity = _payload_identity(payload) + if identity is None: + return + + # Imported locally to keep the durable inbox usable for app-ticket events + # without introducing an import cycle at module load time. + from app.modules.feishu_users.models import FeishuUser + + tenant_key, open_id = identity + db.scalar( + select(FeishuUser.id) + .where( + FeishuUser.tenant_key == tenant_key, + FeishuUser.open_id == open_id, + ) + .with_for_update() + ) + + +def _identity_exists( + db: Session, + identity: tuple[str, str] | None, +) -> bool: + if identity is None: + return False + + from app.modules.feishu_users.models import FeishuUser + + tenant_key, open_id = identity + return ( + db.scalar( + select(FeishuUser.id).where( + FeishuUser.tenant_key == tenant_key, + FeishuUser.open_id == open_id, + ) + ) + is not None + ) + + +def _lock_reply_identity(db: Session, payload: dict[str, Any]) -> None: + if not identity_fence_required(payload): + return + identity = reply_payload_identity(payload) + if identity is None: + raise ValueError("Feishu reply identity fence is unavailable") + + from app.modules.feishu_users.models import FeishuUser + + tenant_key, open_id = identity + owner_id = db.scalar( + select(FeishuUser.id) + .where( + FeishuUser.tenant_key == tenant_key, + FeishuUser.open_id == open_id, + ) + .with_for_update() + ) + if owner_id is None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Feishu reply identity no longer exists", + ) + + +def _is_retryable(exc: Exception) -> bool: + if isinstance(exc, FeishuAPIError): + return exc.retryable + if isinstance(exc, HTTPException): + return ( + exc.status_code == status.HTTP_429_TOO_MANY_REQUESTS + or exc.status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR + ) + if isinstance(exc, (TypeError, ValueError)): + return False + return True + + +def _error_summary(exc: Exception) -> str: + if isinstance(exc, FeishuAPIError): + return ( + "FeishuAPIError:" + f"http_status={exc.http_status}:provider_code={exc.provider_code}" + )[:2000] + if isinstance(exc, HTTPException): + return f"HTTPException:status_code={exc.status_code}" + return type(exc).__name__[:2000] + + +def _attach_provider_response( + result: dict[str, Any], + provider_response: dict[str, Any] | None, +) -> None: + if provider_response is None: + return + command_result = result.get(FeishuResponseKey.RESULT) + if not isinstance(command_result, dict): + return + if FeishuCommandResultKey.PROVIDER_RESPONSE in command_result: + command_result[FeishuCommandResultKey.PROVIDER_RESPONSE] = provider_response + + +def _optional_text(value: Any) -> str | None: + text = str(value or "").strip() + return text or None + + +def _required_text(value: Any, message: str) -> str: + text = _optional_text(value) + if text is None: + raise ValueError(message) + return text diff --git a/app/modules/feishu/services/reply_outbox.py b/app/modules/feishu/services/reply_outbox.py new file mode 100644 index 0000000..a151522 --- /dev/null +++ b/app/modules/feishu/services/reply_outbox.py @@ -0,0 +1,247 @@ +import base64 +from contextlib import contextmanager +from contextvars import ContextVar +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Any, Iterator + +from fastapi import HTTPException, status + +from app.modules.feishu.constants import FEISHU_RECEIVE_ID_MISSING + +_IMAGE_PLACEHOLDER_PREFIX = "__feishu_reply_image__:" +_reply_collector: ContextVar["FeishuReplyCollector | None"] = ContextVar( + "feishu_reply_collector", + default=None, +) + + +@dataclass(slots=True) +class FeishuReplyCollector: + """Capture outbound Feishu operations before an inbound transaction commits.""" + + operations: list[dict[str, Any]] = field(default_factory=list) + + def capture_text( + self, + *, + text: str, + receive_id: str | None, + default_receive_id: str | None, + receive_id_type: str, + actor: str, + message_uuid: str | None, + tenant_key: str | None, + record_audit: bool, + ) -> dict[str, Any]: + target = _required_receive_id(receive_id, default_receive_id) + self.operations.append( + { + "kind": "text", + "text": text, + "receive_id": target, + "receive_id_type": str(receive_id_type), + "actor": str(actor), + "message_uuid": message_uuid, + "tenant_key": tenant_key, + "record_audit": bool(record_audit), + } + ) + return {"code": 0, "queued": True} + + def capture_card( + self, + *, + card: dict[str, Any], + receive_id: str | None, + default_receive_id: str | None, + receive_id_type: str, + actor: str, + message_uuid: str | None, + tenant_key: str | None, + ) -> dict[str, Any]: + target = _required_receive_id(receive_id, default_receive_id) + self.operations.append( + { + "kind": "card", + "card": deepcopy(card), + "receive_id": target, + "receive_id_type": str(receive_id_type), + "actor": str(actor), + "message_uuid": message_uuid, + "tenant_key": tenant_key, + "record_audit": True, + } + ) + return {"code": 0, "queued": True} + + def capture_image( + self, + *, + image: bytes, + actor: str, + tenant_key: str | None, + ) -> dict[str, Any]: + placeholder = f"{_IMAGE_PLACEHOLDER_PREFIX}{len(self.operations)}" + self.operations.append( + { + "kind": "image", + "placeholder": placeholder, + "image_base64": base64.b64encode(image).decode("ascii"), + "actor": str(actor), + "tenant_key": tenant_key, + } + ) + return { + "code": 0, + "queued": True, + "data": {"image_key": placeholder}, + } + + def as_payload( + self, + *, + identity: tuple[str, str] | None, + identity_fence_required: bool, + ) -> dict[str, Any] | None: + if not self.operations: + return None + return { + "version": 1, + "identity": ( + {"tenant_key": identity[0], "open_id": identity[1]} + if identity is not None + else None + ), + "identity_fence_required": identity_fence_required, + "operations": deepcopy(self.operations), + } + + +@contextmanager +def bind_reply_outbox() -> Iterator[FeishuReplyCollector]: + """Capture Feishu side effects for the current inbound command.""" + + collector = FeishuReplyCollector() + token = _reply_collector.set(collector) + try: + yield collector + finally: + _reply_collector.reset(token) + + +def current_reply_outbox() -> FeishuReplyCollector | None: + return _reply_collector.get() + + +def decode_image(operation: dict[str, Any]) -> bytes: + encoded = operation.get("image_base64") + if not isinstance(encoded, str) or not encoded: + raise ValueError("Feishu reply image payload is unavailable") + try: + return base64.b64decode(encoded, validate=True) + except (ValueError, TypeError) as exc: + raise ValueError("Feishu reply image payload is invalid") from exc + + +def resolved_message_operation(payload: dict[str, Any]) -> dict[str, Any]: + operations = _operations(payload) + messages = [ + operation + for operation in operations + if operation.get("kind") in {"text", "card"} + ] + if len(messages) != 1: + raise ValueError("Feishu reply outbox requires exactly one message") + image_keys = { + str(operation.get("placeholder")): str(operation.get("image_key")) + for operation in operations + if operation.get("kind") == "image" + and operation.get("placeholder") + and operation.get("image_key") + } + message = deepcopy(messages[0]) + if message.get("kind") == "card": + message["card"] = _replace_image_placeholders( + message.get("card"), + image_keys, + ) + return message + + +def pending_image_indexes(payload: dict[str, Any]) -> list[int]: + return [ + index + for index, operation in enumerate(_operations(payload)) + if operation.get("kind") == "image" and not operation.get("image_key") + ] + + +def operations_copy(payload: dict[str, Any]) -> list[dict[str, Any]]: + return deepcopy(_operations(payload)) + + +def payload_identity(payload: Any) -> tuple[str, str] | None: + if not isinstance(payload, dict): + return None + identity = payload.get("identity") + if not isinstance(identity, dict): + return None + tenant_key = str(identity.get("tenant_key") or "").strip() + open_id = str(identity.get("open_id") or "").strip() + if not tenant_key or not open_id: + return None + return tenant_key, open_id + + +def identity_fence_required(payload: Any) -> bool: + return isinstance(payload, dict) and bool(payload.get("identity_fence_required")) + + +def _operations(payload: dict[str, Any]) -> list[dict[str, Any]]: + if payload.get("version") != 1: + raise ValueError("Unsupported Feishu reply outbox payload version") + operations = payload.get("operations") + if not isinstance(operations, list) or not all( + isinstance(operation, dict) for operation in operations + ): + raise ValueError("Feishu reply outbox operations are invalid") + return operations + + +def _replace_image_placeholders( + value: Any, + image_keys: dict[str, str], +) -> Any: + if isinstance(value, dict): + return { + key: _replace_image_placeholders(item, image_keys) + for key, item in value.items() + } + if isinstance(value, list): + return [ + _replace_image_placeholders(item, image_keys) + for item in value + ] + if ( + isinstance(value, str) + and value.startswith(_IMAGE_PLACEHOLDER_PREFIX) + ): + image_key = image_keys.get(value) + if not image_key: + raise ValueError("Feishu reply image was not prepared") + return image_key + return value + + +def _required_receive_id( + receive_id: str | None, + default_receive_id: str | None, +) -> str: + target = str(receive_id or default_receive_id or "").strip() + if not target: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=FEISHU_RECEIVE_ID_MISSING, + ) + return target diff --git a/app/modules/feishu_users/constants.py b/app/modules/feishu_users/constants.py index 8368dbb..7dcf345 100644 --- a/app/modules/feishu_users/constants.py +++ b/app/modules/feishu_users/constants.py @@ -80,7 +80,15 @@ def parse_admin_identities( tenant_key, separator, open_id = text.partition(":") tenant_key = tenant_key.strip() open_id = open_id.strip() - if not separator or not tenant_key or not open_id: + if ( + not separator + or not tenant_key + or not open_id + or ":" in open_id + or len(tenant_key) > 128 + or len(open_id) > 128 + or any(character in text for character in "\r\n") + ): raise ValueError(INVALID_ADMIN_IDENTITY) identities.add((tenant_key, open_id)) return frozenset(identities) diff --git a/app/modules/feishu_users/identifiers.py b/app/modules/feishu_users/identifiers.py new file mode 100644 index 0000000..f04a42c --- /dev/null +++ b/app/modules/feishu_users/identifiers.py @@ -0,0 +1,20 @@ +from hashlib import sha256 + +_AUDIT_IDENTITY_DOMAIN = b"company-ai-platform:feishu-audit-identity:v1\0" + + +def feishu_audit_identity_hash(tenant_key: str, open_id: str) -> str: + """Return the erasable pseudonymous subject used by pre-registration audits.""" + + tenant_bytes = tenant_key.encode("utf-8") + open_id_bytes = open_id.encode("utf-8") + identity = b"".join( + ( + len(tenant_bytes).to_bytes(4, "big"), + tenant_bytes, + len(open_id_bytes).to_bytes(4, "big"), + open_id_bytes, + ) + ) + digest = sha256(_AUDIT_IDENTITY_DOMAIN + identity).hexdigest() + return f"feishu-identity-sha256-{digest}" diff --git a/app/modules/observability/constants.py b/app/modules/observability/constants.py index 16876ef..71a2a53 100644 --- a/app/modules/observability/constants.py +++ b/app/modules/observability/constants.py @@ -6,13 +6,16 @@ class ObservabilityKey(StrEnum): CHECKS = "checks" METRICS = "metrics" DATABASE = "database" + SCHEMA = "schema" REDIS = "redis" EVENTS = "events" WORKFLOWS = "workflows" AI_MEMORY = "ai_memory" HEARTBEATS = "heartbeats" + API = "api" SCHEDULER = "scheduler" WORKER = "worker" + FEISHU_EVENTS = "feishu_events" class ObservabilityStatus(StrEnum): @@ -40,7 +43,9 @@ class HeartbeatComponent(StrEnum): API = "api" SCHEDULER = "scheduler" WORKER = "worker" + FEISHU_EVENTS = "feishu-events" class HeartbeatStatus(StrEnum): OK = "ok" + DEGRADED = "degraded" diff --git a/app/modules/observability/runtime.py b/app/modules/observability/runtime.py new file mode 100644 index 0000000..cb19ff5 --- /dev/null +++ b/app/modules/observability/runtime.py @@ -0,0 +1,73 @@ +import logging +from os import getpid +from socket import gethostname +from threading import Event, Thread + +from fastapi import FastAPI + +from app.core.config import get_settings +from app.core.constants import ActorValue +from app.core.database import SessionLocal +from app.modules.observability.constants import HeartbeatComponent +from app.modules.observability.service import ObservabilityService + +logger = logging.getLogger(__name__) + + +def attach_api_heartbeat(app: FastAPI) -> None: + """Record API process liveness without coupling it to request traffic.""" + + interval_seconds = max(1, get_settings().heartbeat_interval_seconds) + instance_id = f"{gethostname()}:{getpid()}" + + @app.on_event("startup") + def start_api_heartbeat() -> None: + current_thread = getattr(app.state, "api_heartbeat_thread", None) + if current_thread is not None and current_thread.is_alive(): + return + + stop_event = Event() + _record_api_heartbeat(instance_id) + thread = Thread( + target=_api_heartbeat_loop, + args=(stop_event, instance_id, interval_seconds), + name="api-heartbeat", + daemon=True, + ) + app.state.api_heartbeat_stop_event = stop_event + app.state.api_heartbeat_thread = thread + app.state.api_heartbeat_instance_id = instance_id + thread.start() + + @app.on_event("shutdown") + def stop_api_heartbeat() -> None: + stop_event = getattr(app.state, "api_heartbeat_stop_event", None) + thread = getattr(app.state, "api_heartbeat_thread", None) + if stop_event is not None: + stop_event.set() + if thread is not None: + thread.join(timeout=1) + + +def _api_heartbeat_loop( + stop_event: Event, + instance_id: str, + interval_seconds: int, +) -> None: + while not stop_event.wait(max(1, interval_seconds)): + _record_api_heartbeat(instance_id) + + +def _record_api_heartbeat(instance_id: str) -> None: + db = SessionLocal() + try: + ObservabilityService(db).record_heartbeat( + component=HeartbeatComponent.API, + instance_id=instance_id, + actor=ActorValue.API, + ) + except Exception: + db.rollback() + logger.exception("Failed to record API process heartbeat") + finally: + db.close() diff --git a/app/modules/observability/service.py b/app/modules/observability/service.py index f5b63db..9f4f9bd 100644 --- a/app/modules/observability/service.py +++ b/app/modules/observability/service.py @@ -9,6 +9,10 @@ from sqlalchemy.orm import Session from app.core.config import get_settings from app.core.constants import ActorValue +from app.core.database.migrations import ( + current_alembic_revisions, + expected_alembic_heads, +) from app.core.utils.time import utc_now from app.modules.ai_memory.service import AIMemoryService from app.modules.audit.constants import ( @@ -20,12 +24,18 @@ from app.modules.audit.constants import ( from app.modules.audit.schemas import AuditLogCreate from app.modules.audit.service import AuditService from app.modules.events.constants import EventStatus +from app.modules.events.models import DomainEvent from app.modules.events.services import EventService from app.modules.feishu.app_tickets import FeishuAppTicketService from app.modules.feishu.constants import FeishuAppType -from app.modules.feishu_users.constants import FeishuUserStatus +from app.modules.feishu.services import FeishuInboundService +from app.modules.feishu_users.constants import ( + FeishuUserStatus, + parse_admin_identities, +) from app.modules.feishu_users.models import FeishuUser from app.modules.observability.constants import ( + HeartbeatComponent, HeartbeatStatus, ObservabilityKey, ObservabilityMetricKey, @@ -53,10 +63,35 @@ class ObservabilityService: def ready(self) -> dict[str, Any]: checks = { ObservabilityKey.DATABASE: self._safe_call(self._database_check), + ObservabilityKey.SCHEMA: self._safe_call(self._schema_check), ObservabilityKey.REDIS: self._safe_call(self._redis_check), ObservabilityKey.EVENTS: self._safe_call(self._events_check), ObservabilityKey.WORKFLOWS: self._safe_call(self._workflows_check), ObservabilityKey.HEARTBEATS: self._safe_call(self._heartbeats_check), + ObservabilityKey.API: self._safe_call( + lambda: self._component_heartbeat_check( + HeartbeatComponent.API, + required=self._api_required(), + ) + ), + ObservabilityKey.SCHEDULER: self._safe_call( + lambda: self._component_heartbeat_check( + HeartbeatComponent.SCHEDULER, + required=self._scheduler_required(), + ) + ), + ObservabilityKey.WORKER: self._safe_call( + lambda: self._component_heartbeat_check( + HeartbeatComponent.WORKER, + required=get_settings().task_queue_enabled, + ) + ), + ObservabilityKey.FEISHU_EVENTS: self._safe_call( + lambda: self._component_heartbeat_check( + HeartbeatComponent.FEISHU_EVENTS, + required=self._feishu_events_required(), + ) + ), "feishu_subscriptions": self._safe_call( self._feishu_subscriptions_check ), @@ -87,6 +122,7 @@ class ObservabilityService: self.heartbeat_summary ), "feishu_users": self._safe_call(self._feishu_user_metrics), + "feishu_inbound": self._safe_call(self._feishu_inbound_metrics), "subscriptions": self._safe_call(self._subscription_metrics), } } @@ -158,6 +194,29 @@ class ObservabilityService: self.db.execute(text("select 1")).scalar() return {ObservabilityKey.STATUS: ObservabilityStatus.OK} + def _schema_check(self) -> dict[str, Any]: + settings = get_settings() + required = ( + settings.app_env.lower() in {"prod", "production"} + or settings.scheduler_enabled + or settings.feishu_user_features_enabled + or settings.feishu_event_transport != "disabled" + ) + if not required: + return {ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED} + + expected = expected_alembic_heads() + current = current_alembic_revisions(self.db.connection()) + matches = current == expected and len(expected) == 1 + return { + ObservabilityKey.STATUS: ( + ObservabilityStatus.OK if matches else ObservabilityStatus.DEGRADED + ), + "current": [] if current is None else list(current), + "expected": list(expected), + "reason": None if matches else "schema_revision_mismatch", + } + def _redis_check(self) -> dict[str, Any]: settings = get_settings() if not settings.task_queue_enabled: @@ -215,22 +274,58 @@ class ObservabilityService: def _events_check(self) -> dict[str, Any]: counts = EventService(self.db).count_by_status() + current = utc_now() failed = counts.get(EventStatus.FAILED, 0) + processable = int( + self.db.scalar( + select(func.count()) + .select_from(DomainEvent) + .where( + DomainEvent.status == EventStatus.PENDING, + DomainEvent.next_attempt_at.is_not(None), + DomainEvent.next_attempt_at <= current, + or_( + DomainEvent.locked_until.is_(None), + DomainEvent.locked_until <= current, + ), + ) + ) + or 0 + ) + expired_leases = int( + self.db.scalar( + select(func.count()) + .select_from(DomainEvent) + .where( + DomainEvent.status == EventStatus.PENDING, + DomainEvent.locked_by.is_not(None), + DomainEvent.locked_until.is_not(None), + DomainEvent.locked_until <= current, + ) + ) + or 0 + ) + reasons: list[str] = [] + if processable and not get_settings().event_dispatch_enabled: + reasons.append("dispatch_disabled") + if expired_leases: + reasons.append("expired_leases") return { ObservabilityKey.STATUS: ( - ObservabilityStatus.DEGRADED if failed else ObservabilityStatus.OK + ObservabilityStatus.DEGRADED if reasons else ObservabilityStatus.OK ), ObservabilityMetricKey.PENDING: counts.get(EventStatus.PENDING, 0), ObservabilityMetricKey.FAILED: failed, + "processable": processable, + "expired_leases": expired_leases, + "reasons": reasons, } def _workflows_check(self) -> dict[str, Any]: counts = WorkflowService(self.db).count_by_status() failed = counts.get(WorkflowStatus.FAILED, 0) return { - ObservabilityKey.STATUS: ( - ObservabilityStatus.DEGRADED if failed else ObservabilityStatus.OK - ), + ObservabilityKey.STATUS: ObservabilityStatus.OK, ObservabilityMetricKey.RUNNING: counts.get(WorkflowStatus.RUNNING, 0), ObservabilityMetricKey.FAILED: failed, } @@ -238,20 +333,86 @@ class ObservabilityService: def _heartbeats_check(self) -> dict[str, Any]: summary = self.heartbeat_summary() total = summary[ObservabilityMetricKey.TOTAL] + active = summary[ObservabilityMetricKey.ACTIVE] stale = summary[ObservabilityMetricKey.STALE] if total == 0: return {ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED} return { ObservabilityKey.STATUS: ( - ObservabilityStatus.DEGRADED if stale else ObservabilityStatus.OK + ObservabilityStatus.OK if active else ObservabilityStatus.DEGRADED ), ObservabilityMetricKey.TOTAL: total, + ObservabilityMetricKey.ACTIVE: active, ObservabilityMetricKey.STALE: stale, ObservabilityMetricKey.LAST_SEEN_AT: summary[ ObservabilityMetricKey.LAST_SEEN_AT ], } + def _component_heartbeat_check( + self, + component: str, + *, + required: bool, + ) -> dict[str, Any]: + if not required: + return { + ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED, + ObservabilityMetricKey.COMPONENT: component, + } + records = list( + self.db.execute( + select(SystemHeartbeat.status, SystemHeartbeat.last_seen_at).where( + SystemHeartbeat.component == component + ) + ).all() + ) + if not records: + return { + ObservabilityKey.STATUS: ObservabilityStatus.DEGRADED, + ObservabilityMetricKey.COMPONENT: component, + "reason": "heartbeat_missing", + } + last_seen_at = max(item.last_seen_at for item in records) + fresh_records = [ + item + for item in records + if item.last_seen_at >= self._heartbeat_stale_threshold() + ] + if not fresh_records: + return { + ObservabilityKey.STATUS: ObservabilityStatus.DEGRADED, + ObservabilityMetricKey.COMPONENT: component, + ObservabilityMetricKey.LAST_SEEN_AT: last_seen_at.isoformat(), + "reason": "heartbeat_stale", + } + healthy = any(item.status == HeartbeatStatus.OK for item in fresh_records) + return { + ObservabilityKey.STATUS: ( + ObservabilityStatus.OK if healthy else ObservabilityStatus.DEGRADED + ), + ObservabilityMetricKey.COMPONENT: component, + ObservabilityMetricKey.LAST_SEEN_AT: last_seen_at.isoformat(), + "reason": None if healthy else "heartbeat_degraded", + } + + @staticmethod + def _api_required() -> bool: + return get_settings().app_env.lower() in {"prod", "production"} + + @staticmethod + def _scheduler_required() -> bool: + settings = get_settings() + return ( + settings.feishu_user_features_enabled + or settings.app_env.lower() in {"prod", "production"} + ) + + @staticmethod + def _feishu_events_required() -> bool: + settings = get_settings() + return settings.feishu_event_transport == "long_connection" + def _feishu_subscriptions_check(self) -> dict[str, Any]: active = int( self.db.scalar( @@ -284,13 +445,17 @@ class ObservabilityService: ) or 0 ) - if not active and not processable_deliveries: + settings = get_settings() + if ( + not settings.feishu_user_features_enabled + and not active + and not processable_deliveries + ): return { ObservabilityKey.STATUS: ObservabilityStatus.SKIPPED, "active": 0, "processable_deliveries": 0, } - settings = get_settings() app_id = str(settings.feishu_app_id or "").strip() credentials_configured = bool(app_id and settings.feishu_app_secret) active_tenant_count = int( @@ -316,6 +481,31 @@ class ObservabilityService: str(settings.feishu_default_tenant_key or "").strip() ) reasons: list[str] = [] + if ( + not settings.feishu_user_features_enabled + and (active or processable_deliveries) + ): + reasons.append("features_disabled") + if ( + settings.app_env.lower() in {"prod", "production"} + and settings.feishu_user_features_enabled + ): + try: + configured_admins = parse_admin_identities( + settings.feishu_admin_identities + ) + except ValueError: + reasons.append("admin_identities_invalid") + else: + if not configured_admins: + reasons.append("admin_identities_missing") + if settings.feishu_event_transport == "disabled": + reasons.append("event_transport_disabled") + elif ( + settings.feishu_event_transport == "webhook" + and not settings.feishu_verification_token + ): + reasons.append("verification_token_missing") if not credentials_configured: reasons.append("credentials_missing") if settings.feishu_app_type == FeishuAppType.STORE: @@ -341,6 +531,8 @@ class ObservabilityService: ), "active": active, "processable_deliveries": processable_deliveries, + "features_enabled": settings.feishu_user_features_enabled, + "event_transport": settings.feishu_event_transport, "app_type": settings.feishu_app_type, "credentials_configured": credentials_configured, "ticket_configured": ticket_configured, @@ -360,6 +552,9 @@ class ObservabilityService: ) return {"active": active} + def _feishu_inbound_metrics(self) -> dict[str, int]: + return FeishuInboundService(self.db).status_counts() + def _subscription_metrics(self) -> dict[str, int]: active = int( self.db.scalar( diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py index 13cc82d..4362883 100644 --- a/app/tasks/__init__.py +++ b/app/tasks/__init__.py @@ -1,8 +1,10 @@ from app.tasks.app import celery_app from app.tasks import events as _events # noqa: F401 +from app.tasks import feishu as _feishu # noqa: F401 from app.tasks import legacy as _legacy # noqa: F401 from app.tasks import lifecycle as _lifecycle # noqa: F401 from app.tasks import market as _market # noqa: F401 +from app.tasks import observability as _observability # noqa: F401 from app.tasks import reports as _reports # noqa: F401 from app.tasks import risk as _risk # noqa: F401 from app.tasks import subscriptions as _subscriptions # noqa: F401 diff --git a/app/tasks/constants.py b/app/tasks/constants.py index e64237e..654c1de 100644 --- a/app/tasks/constants.py +++ b/app/tasks/constants.py @@ -12,3 +12,6 @@ TASK_RUN_LIFECYCLE = "reports.run_lifecycle" TASK_RUN_MARKET_REPORT = "market.report.run" TASK_RUN_MARKET_CLOSE = TASK_RUN_MARKET_REPORT TASK_RUN_SUBSCRIPTION_CYCLE = "subscriptions.run_cycle" +TASK_PROCESS_FEISHU_INBOUND = "feishu.process_inbound" +TASK_PROCESS_DUE_FEISHU_INBOUND = "feishu.process_inbound_due" +TASK_RECORD_WORKER_HEARTBEAT = "observability.record_worker_heartbeat" diff --git a/app/tasks/feishu.py b/app/tasks/feishu.py new file mode 100644 index 0000000..3b56dc7 --- /dev/null +++ b/app/tasks/feishu.py @@ -0,0 +1,29 @@ +from typing import Any + +from app.application.feishu.inbound import ( + process_due_feishu_inbound_events, + process_feishu_inbound_event, +) +from app.core.constants import ActorValue +from app.modules.feishu.constants import FEISHU_INBOUND_BATCH_SIZE +from app.tasks.app import celery_app +from app.tasks.constants import ( + TASK_PROCESS_DUE_FEISHU_INBOUND, + TASK_PROCESS_FEISHU_INBOUND, +) + + +@celery_app.task(name=TASK_PROCESS_FEISHU_INBOUND) +def process_feishu_inbound_event_task( + event_key: str, + actor: str = ActorValue.WORKER, +) -> dict[str, Any]: + return process_feishu_inbound_event(event_key, actor=actor) + + +@celery_app.task(name=TASK_PROCESS_DUE_FEISHU_INBOUND) +def process_due_feishu_inbound_events_task( + limit: int = FEISHU_INBOUND_BATCH_SIZE, + actor: str = ActorValue.WORKER, +) -> list[dict[str, Any]]: + return process_due_feishu_inbound_events(limit=limit, actor=actor) diff --git a/app/tasks/observability.py b/app/tasks/observability.py new file mode 100644 index 0000000..3719ff1 --- /dev/null +++ b/app/tasks/observability.py @@ -0,0 +1,25 @@ +from socket import gethostname + +from app.core.constants import ActorValue +from app.core.database import SessionLocal +from app.modules.observability.constants import HeartbeatComponent +from app.modules.observability.service import ObservabilityService +from app.tasks.app import celery_app +from app.tasks.constants import TASK_RECORD_WORKER_HEARTBEAT + + +@celery_app.task(name=TASK_RECORD_WORKER_HEARTBEAT) +def record_worker_heartbeat_task( + actor: str = ActorValue.WORKER, +) -> dict: + """Record liveness from the Celery process that actually executes the task.""" + + db = SessionLocal() + try: + return ObservabilityService(db).record_heartbeat( + component=HeartbeatComponent.WORKER, + instance_id=gethostname(), + actor=actor, + ) + finally: + db.close() diff --git a/app/tools/reconcile_platform_schema.py b/app/tools/reconcile_platform_schema.py new file mode 100644 index 0000000..830a9ee --- /dev/null +++ b/app/tools/reconcile_platform_schema.py @@ -0,0 +1,924 @@ +"""Audit and atomically baseline the known unversioned platform schema. + +The command is read-only unless ``--apply`` is supplied together with the +schema fingerprint printed by a preceding dry run. It never accepts a database +URL on the command line, so credentials are not exposed through process +arguments. +""" + +import argparse +from collections.abc import Mapping +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path +import re +from typing import Any + +from alembic import command +from alembic.autogenerate import compare_metadata +from alembic.config import Config +from alembic.migration import MigrationContext +from alembic.script import ScriptDirectory +from sqlalchemy import MetaData, Table, create_engine, inspect, select, text +from sqlalchemy.engine import Connection, Engine +from sqlalchemy.pool import NullPool + +from app.core.config import get_settings +from app.core.database import Base +from app.core.database.safety import validate_platform_migration_target +from app.modules.ai_memory import models as ai_memory_models +from app.modules.audit import models as audit_models +from app.modules.business import models as business_models +from app.modules.events import models as event_models +from app.modules.feishu import models as feishu_models +from app.modules.feishu_users import models as feishu_user_models +from app.modules.observability import models as observability_models +from app.modules.personalization import models as personalization_models +from app.modules.subscriptions import models as subscription_models +from app.modules.workflows import models as workflow_models + + +PREVIOUS_REVISION = "202607260005" +TARGET_REVISION = "202607270002" +APPROVAL_TABLE = "approval_requests" +SCHEMA_FINGERPRINT_PATTERN = re.compile(r"^[0-9a-f]{64}$") +ADVISORY_LOCK_KEY = int.from_bytes(b"CAIPSCHE", byteorder="big", signed=True) +MINIMUM_POSTGRESQL_VERSION_NUM = 150000 + +# Keep imports referenced so every model is registered with Base.metadata. +_REGISTERED_MODEL_MODULES = ( + ai_memory_models, + audit_models, + business_models, + event_models, + feishu_models, + feishu_user_models, + observability_models, + personalization_models, + subscription_models, + workflow_models, +) + +_ADD_COLUMNS = { + "ai_memory_entries": {"kind", "owner_id"}, + "attendance_records": { + "attendance_scope", + "is_active", + "last_seen_at", + "source_location_status", + "source_status", + "source_updated_at", + }, + "audit_logs": {"request_id"}, + "feishu_event_receipts": { + "attempt_count", + "auto_reply", + "event_type", + "last_error", + "locked_by", + "locked_until", + "max_attempts", + "next_attempt_at", + "payload", + "processed_at", + "reply_attempt_count", + "reply_last_error", + "reply_locked_by", + "reply_locked_until", + "reply_next_attempt_at", + "reply_payload", + "reply_sent_at", + "reply_status", + "status", + }, + "market_watchlists": {"owner_id"}, + "projects": { + "department_code", + "department_name", + "display_code", + "is_active", + "last_seen_at", + "owner_employee_code", + "source_archived", + "source_contract_amount", + "source_created_at", + "source_project_investment_amount", + "source_stage", + "source_stage_label", + "source_updated_at", + }, + "risk_events": { + "assigned_to", + "closed_at", + "closed_reason", + "resolved_at", + "review_summary", + }, + "work_reports": { + "employee_code", + "external_id", + "is_active", + "is_draft", + "is_late", + "last_seen_at", + "source_updated_at", + }, + "work_tasks": { + "employee_code", + "external_id", + "is_active", + "last_seen_at", + "source_created_at", + "source_system", + "source_updated_at", + }, +} + +_ADD_INDEXES = { + "ai_memory_entries": { + "ix_ai_memory_entries_fingerprint", + "ix_ai_memory_entries_kind", + "ix_ai_memory_entries_owner_id", + }, + "attendance_records": { + "ix_attendance_records_attendance_scope", + "ix_attendance_records_is_active", + "ix_attendance_records_last_seen_at", + "ix_attendance_records_source_status", + "ix_attendance_records_source_updated_at", + }, + "audit_logs": {"ix_audit_logs_request_id"}, + "feishu_admin_bootstrap_tombstones": { + "ix_feishu_admin_bootstrap_tombstones_created_at", + "ix_feishu_admin_bootstrap_tombstones_identity_hash", + }, + "feishu_event_receipts": { + "ix_feishu_event_receipts_event_type", + "ix_feishu_event_receipts_locked_by", + "ix_feishu_event_receipts_locked_until", + "ix_feishu_event_receipts_next_attempt_at", + "ix_feishu_event_receipts_processed_at", + "ix_feishu_event_receipts_reply_locked_by", + "ix_feishu_event_receipts_reply_locked_until", + "ix_feishu_event_receipts_reply_next_attempt_at", + "ix_feishu_event_receipts_reply_sent_at", + "ix_feishu_event_receipts_reply_status", + "ix_feishu_event_receipts_status", + }, + "feishu_app_tickets": { + "ix_feishu_app_tickets_app_id", + "ix_feishu_app_tickets_received_at", + }, + "market_watchlists": {"ix_market_watchlists_owner_id"}, + "projects": { + "ix_projects_department_code", + "ix_projects_department_name", + "ix_projects_display_code", + "ix_projects_is_active", + "ix_projects_last_seen_at", + "ix_projects_owner_employee_code", + "ix_projects_source_archived", + "ix_projects_source_stage", + "ix_projects_source_updated_at", + }, + "risk_events": {"ix_risk_events_assigned_to"}, + "work_reports": { + "ix_work_reports_employee_code", + "ix_work_reports_external_id", + "ix_work_reports_is_active", + "ix_work_reports_is_draft", + "ix_work_reports_is_late", + "ix_work_reports_last_seen_at", + "ix_work_reports_source_updated_at", + }, + "work_tasks": { + "ix_work_tasks_employee_code", + "ix_work_tasks_external_id", + "ix_work_tasks_is_active", + "ix_work_tasks_last_seen_at", + "ix_work_tasks_source_updated_at", + }, +} + +_REMOVE_INDEXES = { + "ai_memory_entries": {"ix_ai_memory_entries_fingerprint"}, + APPROVAL_TABLE: { + "ix_approval_requests_action", + "ix_approval_requests_applicant", + "ix_approval_requests_approver", + "ix_approval_requests_created_at", + "ix_approval_requests_domain", + "ix_approval_requests_record_id", + "ix_approval_requests_status", + "ix_approval_requests_ticket_id", + }, +} + +_MODIFY_DEFAULTS = { + "attendance_records": {"attendance_scope", "is_active"}, + "projects": {"is_active", "source_archived"}, + "work_reports": {"is_active", "is_draft", "is_late"}, + "work_tasks": {"is_active"}, +} + +_IMPACTED_TABLES = frozenset( + { + *_ADD_COLUMNS, + *_ADD_INDEXES, + *_MODIFY_DEFAULTS, + *_REMOVE_INDEXES, + APPROVAL_TABLE, + "feishu_users", + } +) +_CONTENT_FINGERPRINT_TABLES = _IMPACTED_TABLES - {APPROVAL_TABLE} + + +class ReconciliationError(RuntimeError): + """Raised when a baseline precondition is not satisfied.""" + + +@dataclass(frozen=True, order=True) +class DriftKey: + operation: str + table: str + object_name: str + + def render(self) -> str: + suffix = f":{self.object_name}" if self.object_name else "" + return f"{self.operation}:{self.table}{suffix}" + + +@dataclass(frozen=True) +class BaselineAudit: + dialect: str + fingerprint: str + impacted_table_row_counts: tuple[tuple[str, int], ...] + impacted_table_content_digests: tuple[tuple[str, str], ...] + postgresql_server_version_num: int | None + postgresql_version_supported: bool | None + revision_rows: tuple[str, ...] | None + observed_drift: frozenset[DriftKey] + unexpected_drift: frozenset[DriftKey] + approval_rows: int | None + approval_inbound_foreign_keys: int + schema_privileges_ok: bool | None + table_ownership_ok: bool | None + + @property + def eligible(self) -> bool: + return ( + self.revision_rows == () + and not self.unexpected_drift + and (self.approval_rows is None or self.approval_rows == 0) + and self.approval_inbound_foreign_keys == 0 + and self.schema_privileges_ok is not False + and self.table_ownership_ok is not False + and self.postgresql_version_supported is not False + ) + + def public_dict(self) -> dict[str, Any]: + return { + "eligible": self.eligible, + "dialect": self.dialect, + "schema_fingerprint_sha256": self.fingerprint, + "impacted_table_row_counts": dict(self.impacted_table_row_counts), + "postgresql_server_version_num": self.postgresql_server_version_num, + "postgresql_version_supported": self.postgresql_version_supported, + "alembic_version_state": ( + "missing" + if self.revision_rows is None + else "empty" + if not self.revision_rows + else "versioned" + ), + "alembic_revision_count": ( + None if self.revision_rows is None else len(self.revision_rows) + ), + "observed_drift_count": len(self.observed_drift), + "unexpected_drift": [ + item.render() for item in sorted(self.unexpected_drift) + ], + "approval_rows": self.approval_rows, + "approval_inbound_foreign_keys": self.approval_inbound_foreign_keys, + "schema_privileges_ok": self.schema_privileges_ok, + "table_ownership_ok": self.table_ownership_ok, + } + + +def _allowed_drift() -> frozenset[DriftKey]: + items = { + DriftKey("add_table", "feishu_app_tickets", ""), + DriftKey("add_table", "feishu_admin_bootstrap_tombstones", ""), + DriftKey("remove_table", APPROVAL_TABLE, ""), + DriftKey( + "add_constraint", + "ai_memory_entries", + "uq_ai_memory_owner_fingerprint", + ), + DriftKey( + "add_constraint", + "market_watchlists", + "uq_market_watchlist_owner_symbol", + ), + DriftKey( + "remove_constraint", + "ai_memory_entries", + "uq_ai_memory_owner_fingerprint", + ), + DriftKey( + "remove_constraint", + "market_watchlists", + "uq_market_watchlist_owner_symbol", + ), + DriftKey( + "remove_constraint", + "market_watchlists", + "uq_market_watchlist_actor_symbol", + ), + DriftKey( + "add_fk", + "ai_memory_entries", + "owner_id->feishu_users.id", + ), + DriftKey( + "add_fk", + "market_watchlists", + "owner_id->feishu_users.id", + ), + DriftKey("modify_nullable", "work_tasks", "source_system"), + } + for table_name, columns in _ADD_COLUMNS.items(): + items.update( + DriftKey("add_column", table_name, column_name) + for column_name in columns + ) + for table_name, indexes in _ADD_INDEXES.items(): + items.update( + DriftKey("add_index", table_name, index_name) + for index_name in indexes + ) + for table_name, indexes in _REMOVE_INDEXES.items(): + items.update( + DriftKey("remove_index", table_name, index_name) + for index_name in indexes + ) + for table_name, columns in _MODIFY_DEFAULTS.items(): + items.update( + DriftKey("modify_default", table_name, column_name) + for column_name in columns + ) + return frozenset(items) + + +ALLOWED_DRIFT = _allowed_drift() + + +def _foreign_key_name(constraint: Any) -> str: + local_columns = ",".join(column.name for column in constraint.columns) + remote_columns = ",".join( + element.target_fullname for element in constraint.elements + ) + return f"{local_columns}->{remote_columns}" + + +def _normalize_diff(diff: tuple[Any, ...]) -> DriftKey: + operation = str(diff[0]) + if operation in {"add_table", "remove_table"}: + return DriftKey(operation, str(diff[1].name), "") + if operation in {"add_column", "remove_column"}: + return DriftKey(operation, str(diff[2]), str(diff[3].name)) + if operation in {"add_index", "remove_index"}: + index = diff[1] + return DriftKey(operation, str(index.table.name), str(index.name)) + if operation in {"add_constraint", "remove_constraint"}: + constraint = diff[1] + return DriftKey( + operation, + str(constraint.table.name), + str(constraint.name), + ) + if operation in {"add_fk", "remove_fk"}: + constraint = diff[1] + return DriftKey( + operation, + str(constraint.table.name), + _foreign_key_name(constraint), + ) + if operation.startswith("modify_"): + return DriftKey(operation, str(diff[2]), str(diff[3])) + return DriftKey(operation, "", "") + + +def _metadata_drift(connection: Connection) -> frozenset[DriftKey]: + context = MigrationContext.configure( + connection, + opts={ + "compare_type": True, + "compare_server_default": True, + }, + ) + raw_diffs = compare_metadata(context, Base.metadata) + flattened: list[tuple[Any, ...]] = [] + for item in raw_diffs: + if isinstance(item, list): + flattened.extend(item) + else: + flattened.append(item) + return frozenset(_normalize_diff(diff) for diff in flattened) + + +def _canonical_reflection_value(value: Any) -> Any: + """Convert SQLAlchemy reflection values into stable JSON-compatible data.""" + + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, bytes): + return {"bytes_hex": value.hex()} + if isinstance(value, Mapping): + return { + str(key): _canonical_reflection_value(value[key]) + for key in sorted(value, key=lambda item: str(item)) + } + if isinstance(value, (list, tuple)): + return [_canonical_reflection_value(item) for item in value] + if isinstance(value, (set, frozenset)): + items = [_canonical_reflection_value(item) for item in value] + return sorted(items, key=_stable_json) + return str(value) + + +def _stable_json(value: Any) -> str: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ) + + +def _sorted_reflection_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [_canonical_reflection_value(item) for item in items] + return sorted(normalized, key=_stable_json) + + +def _schema_snapshot(connection: Connection) -> list[dict[str, Any]]: + inspector = inspect(connection) + snapshot: list[dict[str, Any]] = [] + for table_name in sorted(inspector.get_table_names()): + snapshot.append( + { + "table": table_name, + "columns": [ + _canonical_reflection_value(column) + for column in inspector.get_columns(table_name) + ], + "pk": _canonical_reflection_value( + inspector.get_pk_constraint(table_name) + ), + "uniques": _sorted_reflection_items( + inspector.get_unique_constraints(table_name) + ), + "checks": _sorted_reflection_items( + inspector.get_check_constraints(table_name) + ), + "fks": _sorted_reflection_items( + inspector.get_foreign_keys(table_name) + ), + "indexes": _sorted_reflection_items( + inspector.get_indexes(table_name) + ), + } + ) + return snapshot + + +def _impacted_table_row_counts( + connection: Connection, +) -> tuple[tuple[str, int], ...]: + existing_tables = set(inspect(connection).get_table_names()) + preparer = connection.dialect.identifier_preparer + rows: list[tuple[str, int]] = [] + for table_name in sorted(_IMPACTED_TABLES & existing_tables): + quoted_table = preparer.quote_identifier(table_name) + count = connection.execute( + text(f"SELECT COUNT(*) FROM {quoted_table}") + ).scalar_one() + rows.append((table_name, int(count))) + return tuple(rows) + + +def _row_content_digest(row: Mapping[str, Any]) -> str: + canonical_row = { + str(column_name): _canonical_reflection_value(value) + for column_name, value in row.items() + } + return hashlib.sha256(_stable_json(canonical_row).encode()).hexdigest() + + +def _table_content_digest( + connection: Connection, + table_name: str, + primary_key_columns: tuple[str, ...], +) -> str: + table = Table( + table_name, + MetaData(), + autoload_with=connection, + resolve_fks=False, + ) + statement = select(table) + has_stable_primary_key = bool(primary_key_columns) and all( + column_name in table.c for column_name in primary_key_columns + ) + if has_stable_primary_key: + statement = statement.order_by( + *(table.c[column_name].asc() for column_name in primary_key_columns) + ) + + row_digests = ( + _row_content_digest(row) + for row in connection.execute(statement).mappings() + ) + if not has_stable_primary_key: + row_digests = iter(sorted(row_digests)) + + table_digest = hashlib.sha256() + for row_digest in row_digests: + table_digest.update(row_digest.encode("ascii")) + table_digest.update(b"\n") + return table_digest.hexdigest() + + +def _impacted_table_content_digests( + connection: Connection, +) -> tuple[tuple[str, str], ...]: + inspector = inspect(connection) + existing_tables = set(inspector.get_table_names()) + digests: list[tuple[str, str]] = [] + for table_name in sorted(_CONTENT_FINGERPRINT_TABLES & existing_tables): + primary_key = inspector.get_pk_constraint(table_name) + primary_key_columns = tuple( + str(column_name) + for column_name in primary_key.get("constrained_columns") or () + ) + digests.append( + ( + table_name, + _table_content_digest( + connection, + table_name, + primary_key_columns, + ), + ) + ) + return tuple(digests) + + +def _schema_fingerprint( + connection: Connection, + impacted_table_row_counts: tuple[tuple[str, int], ...] | None = None, + impacted_table_content_digests: tuple[tuple[str, str], ...] | None = None, +) -> str: + row_counts = ( + impacted_table_row_counts + if impacted_table_row_counts is not None + else _impacted_table_row_counts(connection) + ) + content_digests = ( + impacted_table_content_digests + if impacted_table_content_digests is not None + else _impacted_table_content_digests(connection) + ) + payload = _stable_json( + { + "schema": _schema_snapshot(connection), + "impacted_table_row_counts": [ + {"table": table_name, "row_count": row_count} + for table_name, row_count in row_counts + ], + "impacted_table_content_digests": [ + {"table": table_name, "sha256": digest} + for table_name, digest in content_digests + ], + } + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def _revision_rows(connection: Connection) -> tuple[str, ...] | None: + if "alembic_version" not in inspect(connection).get_table_names(): + return None + rows = connection.execute( + text("SELECT version_num FROM alembic_version ORDER BY version_num") + ).scalars() + return tuple(str(row) for row in rows) + + +def _approval_state(connection: Connection) -> tuple[int | None, int]: + inspector = inspect(connection) + tables = inspector.get_table_names() + if APPROVAL_TABLE not in tables: + return None, 0 + rows = int( + connection.execute( + text("SELECT COUNT(*) FROM approval_requests") + ).scalar_one() + ) + inbound = 0 + for table_name in tables: + inbound += sum( + 1 + for foreign_key in inspector.get_foreign_keys(table_name) + if foreign_key.get("referred_table") == APPROVAL_TABLE + ) + return rows, inbound + + +def _postgres_privileges( + connection: Connection, +) -> tuple[bool | None, bool | None]: + if connection.dialect.name != "postgresql": + return None, None + schema_ok = bool( + connection.execute( + text( + """ + SELECT + has_schema_privilege(current_schema(), 'USAGE') + AND has_schema_privilege(current_schema(), 'CREATE') + """ + ) + ).scalar_one() + ) + ownership_ok = bool( + connection.execute( + text( + """ + SELECT COALESCE(bool_and(pg_has_role(c.relowner, 'USAGE')), true) + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() + AND c.relname = ANY(:table_names) + """ + ), + {"table_names": sorted(_IMPACTED_TABLES)}, + ).scalar_one() + ) + return schema_ok, ownership_ok + + +def _postgres_version_state( + connection: Connection, +) -> tuple[int | None, bool | None]: + if connection.dialect.name != "postgresql": + return None, None + version_num = int( + connection.execute( + text("SELECT current_setting('server_version_num')") + ).scalar_one() + ) + return version_num, version_num >= MINIMUM_POSTGRESQL_VERSION_NUM + + +def audit_connection(connection: Connection) -> BaselineAudit: + """Return a non-secret, read-only assessment of baseline eligibility.""" + + observed = _metadata_drift(connection) + approval_rows, approval_inbound = _approval_state(connection) + schema_ok, ownership_ok = _postgres_privileges(connection) + postgres_version_num, postgres_version_supported = _postgres_version_state( + connection + ) + impacted_table_row_counts = _impacted_table_row_counts(connection) + impacted_table_content_digests = _impacted_table_content_digests(connection) + return BaselineAudit( + dialect=connection.dialect.name, + fingerprint=_schema_fingerprint( + connection, + impacted_table_row_counts, + impacted_table_content_digests, + ), + impacted_table_row_counts=impacted_table_row_counts, + impacted_table_content_digests=impacted_table_content_digests, + postgresql_server_version_num=postgres_version_num, + postgresql_version_supported=postgres_version_supported, + revision_rows=_revision_rows(connection), + observed_drift=observed, + unexpected_drift=observed - ALLOWED_DRIFT, + approval_rows=approval_rows, + approval_inbound_foreign_keys=approval_inbound, + schema_privileges_ok=schema_ok, + table_ownership_ok=ownership_ok, + ) + + +def audit_engine(engine: Engine) -> BaselineAudit: + """Run the default read-only audit.""" + + with engine.connect() as connection: + transaction = connection.begin() + try: + if connection.dialect.name == "postgresql": + connection.exec_driver_sql("SET TRANSACTION READ ONLY") + return audit_connection(connection) + finally: + transaction.rollback() + + +def _alembic_config(connection: Connection) -> Config: + project_root = Path(__file__).resolve().parents[2] + config = Config(str(project_root / "alembic.ini")) + config.set_main_option("script_location", str(project_root / "alembic")) + config.attributes["connection"] = connection + return config + + +def _validate_expected_head(config: Config) -> None: + heads = tuple(ScriptDirectory.from_config(config).get_heads()) + if heads != (TARGET_REVISION,): + raise ReconciliationError("Alembic head changed after this baseline was prepared") + + +def _acquire_postgres_lock(connection: Connection) -> None: + connection.exec_driver_sql("SET LOCAL lock_timeout = '5s'") + connection.exec_driver_sql("SET LOCAL statement_timeout = '5min'") + connection.execute( + text("SELECT pg_advisory_xact_lock(:lock_key)"), + {"lock_key": ADVISORY_LOCK_KEY}, + ) + + +def _lock_impacted_postgres_tables(connection: Connection) -> None: + if connection.dialect.name != "postgresql": + return + existing_tables = set(inspect(connection).get_table_names()) + table_names = sorted(_IMPACTED_TABLES & existing_tables) + if not table_names: + return + preparer = connection.dialect.identifier_preparer + quoted_tables = ", ".join( + preparer.quote_identifier(table_name) for table_name in table_names + ) + connection.exec_driver_sql( + f"LOCK TABLE {quoted_tables} IN ACCESS EXCLUSIVE MODE" + ) + + +def _validate_apply_preconditions( + audit: BaselineAudit, + expected_fingerprint: str, +) -> None: + if audit.postgresql_version_supported is False: + raise ReconciliationError( + "PostgreSQL 15 or newer is required for NULLS NOT DISTINCT constraints" + ) + if audit.fingerprint != expected_fingerprint: + raise ReconciliationError( + "Schema fingerprint changed after dry run; run the audit again" + ) + if audit.revision_rows is None: + raise ReconciliationError("alembic_version table is missing") + if audit.revision_rows: + raise ReconciliationError("Database already has an Alembic revision") + if audit.unexpected_drift: + raise ReconciliationError("Schema contains drift outside the approved allowlist") + if audit.approval_rows not in {None, 0}: + raise ReconciliationError("approval_requests is not empty") + if audit.approval_inbound_foreign_keys: + raise ReconciliationError("approval_requests has dependent foreign keys") + if audit.schema_privileges_ok is False or audit.table_ownership_ok is False: + raise ReconciliationError("Database role lacks required schema ownership privileges") + + +def apply_baseline( + engine: Engine, + expected_fingerprint: str, + *, + require_postgresql: bool = True, +) -> BaselineAudit: + """Atomically stamp, reconcile, upgrade, and verify the approved schema.""" + + if not SCHEMA_FINGERPRINT_PATTERN.fullmatch(expected_fingerprint): + raise ReconciliationError("Expected fingerprint must be a lowercase SHA-256 value") + if require_postgresql and engine.dialect.name != "postgresql": + raise ReconciliationError("Baseline apply is supported only on PostgreSQL") + + with engine.begin() as connection: + if connection.dialect.name == "postgresql": + _acquire_postgres_lock(connection) + _lock_impacted_postgres_tables(connection) + before = audit_connection(connection) + _validate_apply_preconditions(before, expected_fingerprint) + + config = _alembic_config(connection) + _validate_expected_head(config) + command.stamp(config, PREVIOUS_REVISION) + command.upgrade(config, "head") + + after = audit_connection(connection) + if after.revision_rows != (TARGET_REVISION,): + raise ReconciliationError("Alembic revision was not advanced atomically") + if after.observed_drift: + raise ReconciliationError("Schema still differs from SQLAlchemy metadata") + if after.approval_rows is not None: + raise ReconciliationError("Obsolete approval_requests table still exists") + return after + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Audit the platform schema, or atomically apply the approved one-time baseline" + ) + ) + parser.add_argument( + "--apply", + action="store_true", + help="Apply the baseline; without this flag the command is read-only", + ) + parser.add_argument( + "--expected-fingerprint", + help="Exact SHA-256 fingerprint printed by the immediately preceding dry run", + ) + return parser + + +def _validated_cli_database_url() -> str: + settings = get_settings() + try: + target = validate_platform_migration_target( + settings.database_url, + settings.legacy_database_url, + ) + except RuntimeError as exc: + raise ReconciliationError(str(exc)) from None + if target[0] != "postgresql": + raise ReconciliationError( + "Schema reconciliation CLI requires a PostgreSQL platform database" + ) + return settings.database_url + + +def main() -> None: + parser = _build_parser() + arguments = parser.parse_args() + if arguments.apply and not arguments.expected_fingerprint: + parser.error("--apply requires --expected-fingerprint") + if not arguments.apply and arguments.expected_fingerprint: + parser.error("--expected-fingerprint is only valid with --apply") + + engine: Engine | None = None + try: + engine = create_engine( + _validated_cli_database_url(), + poolclass=NullPool, + ) + if arguments.apply: + report = apply_baseline( + engine, + str(arguments.expected_fingerprint), + ) + payload = { + "applied": True, + "target_revision": TARGET_REVISION, + **report.public_dict(), + } + else: + report = audit_engine(engine) + payload = {"applied": False, **report.public_dict()} + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + if not arguments.apply and not report.eligible: + raise SystemExit(2) + except ReconciliationError as exc: + print( + json.dumps( + { + "applied": False, + "error": str(exc), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + raise SystemExit(2) from None + except Exception as exc: + print( + json.dumps( + { + "applied": False, + "error": f"Unexpected {type(exc).__name__}", + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + raise SystemExit(1) from None + finally: + if engine is not None: + engine.dispose() + + +if __name__ == "__main__": + main() diff --git a/app/tools/run_scheduler.py b/app/tools/run_scheduler.py index 8107cbf..d3bb386 100644 --- a/app/tools/run_scheduler.py +++ b/app/tools/run_scheduler.py @@ -1,9 +1,12 @@ from time import sleep from app.application.scheduling import create_scheduler +from app.core.config import get_settings def main() -> None: + if not get_settings().scheduler_enabled: + raise RuntimeError("SCHEDULER_ENABLED must be true for the scheduler process") scheduler = create_scheduler() scheduler.start() try: diff --git a/app/tools/runtime_preflight.py b/app/tools/runtime_preflight.py new file mode 100644 index 0000000..d22cad6 --- /dev/null +++ b/app/tools/runtime_preflight.py @@ -0,0 +1,90 @@ +"""Fail-closed checks required before starting managed runtime processes.""" + +from dataclasses import dataclass + +from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool + +from app.core.config import Settings, get_settings +from app.core.database.migrations import ( + current_alembic_revisions, + expected_alembic_heads, +) +from app.core.database.safety import validate_platform_migration_target +from app.modules.feishu_users.constants import parse_admin_identities + + +class RuntimePreflightError(RuntimeError): + """Raised when the configured runtime is not safe to start.""" + + +@dataclass(frozen=True, slots=True) +class RuntimePreflightResult: + revisions: tuple[str, ...] + transport: str + + +def run_preflight(settings: Settings | None = None) -> RuntimePreflightResult: + """Validate the platform target, migration head, and enabled transport.""" + + runtime_settings = settings or get_settings() + validate_platform_migration_target( + runtime_settings.database_url, + runtime_settings.legacy_database_url, + ) + if runtime_settings.feishu_event_transport == "long_connection" and ( + not runtime_settings.feishu_app_id or not runtime_settings.feishu_app_secret + ): + raise RuntimePreflightError( + "FEISHU_APP_ID and FEISHU_APP_SECRET are required for long_connection" + ) + if runtime_settings.feishu_admin_identities: + try: + parse_admin_identities(runtime_settings.feishu_admin_identities) + except ValueError as exc: + raise RuntimePreflightError(str(exc)) from exc + + expected = expected_alembic_heads() + if len(expected) != 1: + raise RuntimePreflightError("Runtime requires exactly one Alembic head") + + engine = None + try: + engine = create_engine(runtime_settings.database_url, poolclass=NullPool) + with engine.connect() as connection: + current = current_alembic_revisions(connection) + except Exception as exc: + raise RuntimePreflightError( + "Platform database schema could not be inspected" + ) from exc + finally: + if engine is not None: + engine.dispose() + + if current is None: + raise RuntimePreflightError( + "Platform database is not Alembic-versioned; run the approved migration first" + ) + if current != expected: + raise RuntimePreflightError( + "Platform database is not at the current Alembic head" + ) + return RuntimePreflightResult( + revisions=current, + transport=runtime_settings.feishu_event_transport, + ) + + +def main() -> None: + try: + result = run_preflight() + except (RuntimeError, ValueError) as exc: + raise SystemExit(f"Runtime preflight failed: {exc}") from exc + print( + "Runtime preflight passed: " + f"schema={result.revisions[0]}, transport={result.transport}" + ) + + +if __name__ == "__main__": + main() diff --git a/docker-compose.external-db.yml b/docker-compose.external-db.yml new file mode 100644 index 0000000..79d718e --- /dev/null +++ b/docker-compose.external-db.yml @@ -0,0 +1,53 @@ +# Use with docker-compose.yml when PostgreSQL is managed outside Compose: +# docker compose -f docker-compose.yml -f docker-compose.external-db.yml up +# COMPOSE_DATABASE_URL is required. POSTGRES_PASSWORD and the internal db are unused. + +services: + db: + profiles: ["internal-database"] + + migrate: + environment: + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:?COMPOSE_DATABASE_URL is required for external PostgreSQL mode}" + depends_on: !override {} + + api: + environment: + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:?COMPOSE_DATABASE_URL is required for external PostgreSQL mode}" + depends_on: !override + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + + worker: + environment: + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:?COMPOSE_DATABASE_URL is required for external PostgreSQL mode}" + depends_on: !override + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + + scheduler: + environment: + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:?COMPOSE_DATABASE_URL is required for external PostgreSQL mode}" + depends_on: !override + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + + feishu-events: + environment: + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:?COMPOSE_DATABASE_URL is required for external PostgreSQL mode}" + depends_on: !override + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully diff --git a/docker-compose.yml b/docker-compose.yml index 49681d6..c5849d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,13 @@ +x-app-env-files: &app-env-files + - path: .env + required: true + services: db: image: postgres:16-alpine environment: POSTGRES_USER: ${POSTGRES_USER:-company_ai} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-} POSTGRES_DB: ${POSTGRES_DB:-company_ai} volumes: - postgres_data:/var/lib/postgresql/data @@ -12,6 +16,7 @@ services: interval: 10s timeout: 5s retries: 5 + restart: unless-stopped redis: image: redis:7-alpine @@ -22,13 +27,14 @@ services: interval: 10s timeout: 5s retries: 5 + restart: unless-stopped migrate: build: . - env_file: - - .env + env_file: *app-env-files environment: - DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@db:5432/${POSTGRES_DB:-company_ai} + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:-}@db:5432/${POSTGRES_DB:-company_ai}}" REDIS_URL: redis://redis:6379/0 command: ["alembic", "upgrade", "head"] depends_on: @@ -38,12 +44,13 @@ services: api: build: . - env_file: - - .env + env_file: *app-env-files environment: - DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@db:5432/${POSTGRES_DB:-company_ai} + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:-}@db:5432/${POSTGRES_DB:-company_ai}}" REDIS_URL: redis://redis:6379/0 SCHEDULER_ENABLED: "false" + TASK_QUEUE_ENABLED: "true" ports: - "8010:8010" depends_on: @@ -59,22 +66,24 @@ services: "CMD", "python", "-c", - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8010/api/v1/health/live', timeout=3).read()", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8010/api/v1/health/ready', timeout=3).read()", ] interval: 10s timeout: 5s retries: 5 + start_period: 30s + restart: unless-stopped worker: build: . - env_file: - - .env + env_file: *app-env-files environment: - DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@db:5432/${POSTGRES_DB:-company_ai} + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:-}@db:5432/${POSTGRES_DB:-company_ai}}" REDIS_URL: redis://redis:6379/0 TASK_QUEUE_ENABLED: "true" SCHEDULER_ENABLED: "false" - command: ["celery", "-A", "app.tasks.celery_app", "worker", "--loglevel=info"] + command: ["celery", "-A", "app.tasks:celery_app", "worker", "--loglevel=info"] depends_on: db: condition: service_healthy @@ -82,13 +91,14 @@ services: condition: service_healthy migrate: condition: service_completed_successfully + restart: unless-stopped scheduler: build: . - env_file: - - .env + env_file: *app-env-files environment: - DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@db:5432/${POSTGRES_DB:-company_ai} + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:-}@db:5432/${POSTGRES_DB:-company_ai}}" REDIS_URL: redis://redis:6379/0 SCHEDULER_ENABLED: "true" TASK_QUEUE_ENABLED: "true" @@ -100,6 +110,28 @@ services: condition: service_healthy migrate: condition: service_completed_successfully + restart: unless-stopped + + feishu-events: + build: . + profiles: ["long-connection"] + env_file: *app-env-files + environment: + APP_ENV: production + DATABASE_URL: "${COMPOSE_DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-company_ai}:${POSTGRES_PASSWORD:-}@db:5432/${POSTGRES_DB:-company_ai}}" + REDIS_URL: redis://redis:6379/0 + FEISHU_EVENT_TRANSPORT: long_connection + SCHEDULER_ENABLED: "false" + TASK_QUEUE_ENABLED: "true" + command: ["python", "-m", "app.modules.feishu.long_connection"] + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + restart: unless-stopped volumes: postgres_data: diff --git a/scripts/start_runtime.ps1 b/scripts/start_runtime.ps1 new file mode 100644 index 0000000..011952f --- /dev/null +++ b/scripts/start_runtime.ps1 @@ -0,0 +1,178 @@ +param( + [int]$Port = 8010, + [int]$ReadyTimeoutSeconds = 60 +) + +$ErrorActionPreference = "Stop" +$projectRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$runtimeRoot = Join-Path $projectRoot ".runtime" +$logRoot = Join-Path $projectRoot "logs\runtime" +New-Item -ItemType Directory -Force -Path $runtimeRoot, $logRoot | Out-Null + +function Get-OwnedRuntimeProcess { + param([string]$PidFile) + + if (-not (Test-Path -LiteralPath $PidFile)) { + return $null + } + $parts = (Get-Content -LiteralPath $PidFile -Raw).Trim().Split("|") + if ($parts.Count -ne 2) { + return $null + } + $processId = 0 + $startTicks = 0L + if ( + -not [int]::TryParse($parts[0], [ref]$processId) -or + -not [long]::TryParse($parts[1], [ref]$startTicks) + ) { + return $null + } + $process = Get-Process -Id $processId -ErrorAction SilentlyContinue + if ( + $null -eq $process -or + $process.StartTime.ToUniversalTime().Ticks -ne $startTicks + ) { + return $null + } + return $process +} + +$pidFiles = @{ + Api = Join-Path $runtimeRoot "api.pid" + FeishuEvents = Join-Path $runtimeRoot "feishu-events.pid" +} +foreach ($entry in $pidFiles.GetEnumerator()) { + if ($null -ne (Get-OwnedRuntimeProcess -PidFile $entry.Value)) { + throw "$($entry.Key) is already running. Stop it before starting another instance." + } + if (Test-Path -LiteralPath $entry.Value) { + Remove-Item -LiteralPath $entry.Value -Force + } +} + +$pythonOutput = & conda run -n company-ai-platform python -c "import sys; print(sys.executable)" +if ($LASTEXITCODE -ne 0) { + throw "Unable to locate Python in the company-ai-platform Conda environment." +} +$pythonExe = ( + $pythonOutput | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | + Select-Object -Last 1 +) +if (-not [string]::IsNullOrWhiteSpace($pythonExe)) { + $pythonExe = $pythonExe.Trim() +} +if ( + [string]::IsNullOrWhiteSpace($pythonExe) -or + -not (Test-Path -LiteralPath $pythonExe) +) { + $condaEnvironmentOutput = & conda env list --json + if ($LASTEXITCODE -ne 0) { + throw "Unable to list Conda environments." + } + $condaEnvironmentList = ( + ($condaEnvironmentOutput -join [Environment]::NewLine) | + ConvertFrom-Json + ) + $matchingEnvironmentRoots = @( + $condaEnvironmentList.envs | + Where-Object { + (Split-Path -Leaf $_) -eq "company-ai-platform" + } + ) + if ($matchingEnvironmentRoots.Count -ne 1) { + throw "Unable to resolve a unique company-ai-platform Conda environment." + } + $pythonExe = Join-Path $matchingEnvironmentRoots[0] "python.exe" +} +if (-not (Test-Path -LiteralPath $pythonExe -PathType Leaf)) { + throw "The company-ai-platform Python executable was not found." +} + +# Process environment variables take precedence over .env. +$env:SCHEDULER_ENABLED = "true" +$env:TASK_QUEUE_ENABLED = "false" +$env:FEISHU_EVENT_TRANSPORT = "long_connection" + +$preflightOutput = & $pythonExe -m app.tools.runtime_preflight +if ($LASTEXITCODE -ne 0) { + throw "Runtime preflight failed. Review the configuration and migrate the platform database." +} +Write-Host ($preflightOutput | Select-Object -Last 1) + +$timestamp = Get-Date -Format "yyyyMMdd-HHmmss" +function Start-HiddenRuntimeProcess { + param( + [string]$Name, + [string[]]$Arguments, + [string]$PidFile + ) + + $stdoutPath = Join-Path $logRoot "$Name-$timestamp.stdout.log" + $stderrPath = Join-Path $logRoot "$Name-$timestamp.stderr.log" + $process = Start-Process ` + -FilePath $pythonExe ` + -ArgumentList $Arguments ` + -WorkingDirectory $projectRoot ` + -WindowStyle Hidden ` + -RedirectStandardOutput $stdoutPath ` + -RedirectStandardError $stderrPath ` + -PassThru + $identity = "$($process.Id)|$($process.StartTime.ToUniversalTime().Ticks)" + Set-Content -LiteralPath $PidFile -Value $identity -NoNewline + return @{ + Process = $process + Stdout = $stdoutPath + Stderr = $stderrPath + } +} + +$started = @() +try { + $api = Start-HiddenRuntimeProcess ` + -Name "api" ` + -Arguments @( + "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", + "--port", "$Port" + ) ` + -PidFile $pidFiles.Api + $started += $api + + $feishuEvents = Start-HiddenRuntimeProcess ` + -Name "feishu-events" ` + -Arguments @("-m", "app.modules.feishu.long_connection") ` + -PidFile $pidFiles.FeishuEvents + $started += $feishuEvents + + $deadline = (Get-Date).AddSeconds($ReadyTimeoutSeconds) + $readyUrl = "http://127.0.0.1:$Port/api/v1/health/ready" + while ((Get-Date) -lt $deadline) { + foreach ($item in $started) { + if ($item.Process.HasExited) { + throw "A runtime process exited before readiness succeeded." + } + } + try { + $response = Invoke-WebRequest -Uri $readyUrl -TimeoutSec 3 -UseBasicParsing + if ($response.StatusCode -eq 200) { + Write-Host "Runtime is ready at http://127.0.0.1:$Port" + Write-Host "Logs: $logRoot" + return + } + } catch { + # Readiness can be unavailable briefly while both processes initialize. + } + Start-Sleep -Seconds 1 + } + throw "Runtime did not become ready within $ReadyTimeoutSeconds seconds." +} catch { + foreach ($item in $started) { + if (-not $item.Process.HasExited) { + Stop-Process -Id $item.Process.Id -Force -ErrorAction SilentlyContinue + } + } + Remove-Item -LiteralPath $pidFiles.Api, $pidFiles.FeishuEvents ` + -Force -ErrorAction SilentlyContinue + throw +} diff --git a/scripts/stop_runtime.ps1 b/scripts/stop_runtime.ps1 new file mode 100644 index 0000000..fc95445 --- /dev/null +++ b/scripts/stop_runtime.ps1 @@ -0,0 +1,44 @@ +$ErrorActionPreference = "Stop" +$projectRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$runtimeRoot = Join-Path $projectRoot ".runtime" + +function Stop-OwnedRuntimeProcess { + param( + [string]$Name, + [string]$PidFile + ) + + if (-not (Test-Path -LiteralPath $PidFile)) { + Write-Host "$Name is not running." + return + } + $parts = (Get-Content -LiteralPath $PidFile -Raw).Trim().Split("|") + $processId = 0 + $startTicks = 0L + $validIdentity = ( + $parts.Count -eq 2 -and + [int]::TryParse($parts[0], [ref]$processId) -and + [long]::TryParse($parts[1], [ref]$startTicks) + ) + $process = $null + if ($validIdentity) { + $process = Get-Process -Id $processId -ErrorAction SilentlyContinue + } + if ( + $null -ne $process -and + $process.StartTime.ToUniversalTime().Ticks -eq $startTicks + ) { + Stop-Process -Id $processId -Force + Write-Host "Stopped $Name (PID $processId)." + } else { + Write-Host "$Name had no matching live process; removed its stale PID file." + } + Remove-Item -LiteralPath $PidFile -Force +} + +Stop-OwnedRuntimeProcess ` + -Name "Feishu events" ` + -PidFile (Join-Path $runtimeRoot "feishu-events.pid") +Stop-OwnedRuntimeProcess ` + -Name "API and embedded scheduler" ` + -PidFile (Join-Path $runtimeRoot "api.pid") diff --git a/tests/conftest.py b/tests/conftest.py index da24f26..bbee405 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,6 +38,7 @@ os.environ.update( "FEISHU_ENCRYPT_KEY": "", "FEISHU_DEFAULT_CHAT_ID": "", "FEISHU_VERIFICATION_TOKEN": "test-feishu-token", + "FEISHU_EVENT_TRANSPORT": "webhook", "FEISHU_ADMIN_IDENTITIES": "", "FEISHU_USER_FEATURES_ENABLED": "false", "MARKET_DATA_TOKEN": "", diff --git a/tests/test_feishu_app_ticket.py b/tests/test_feishu_app_ticket.py index 8c142a3..9ea7998 100644 --- a/tests/test_feishu_app_ticket.py +++ b/tests/test_feishu_app_ticket.py @@ -10,7 +10,7 @@ from sqlalchemy import create_engine, func, select from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import StaticPool -from app.application.feishu.events import FeishuEventService +from app.application.feishu.events import FeishuEventService, _identifier_digest from app.core.config import get_settings from app.core.database import Base from app.modules.audit.models import AuditLog @@ -18,6 +18,7 @@ from app.modules.feishu import long_connection from app.modules.feishu.app_tickets import FeishuAppTicketService from app.modules.feishu.constants import FeishuEventSource from app.modules.feishu.models import FeishuAppTicket, FeishuEventReceipt +from app.modules.feishu.services import FeishuInboundService @pytest.fixture @@ -26,6 +27,7 @@ def session_factory( ) -> Iterator[sessionmaker[Session]]: monkeypatch.setenv("FEISHU_APP_ID", "cli-ticket-app") monkeypatch.setenv("FEISHU_APP_SECRET", "ticket-app-secret") + monkeypatch.setenv("FEISHU_EVENT_TRANSPORT", "long_connection") monkeypatch.setenv("FEISHU_VERIFICATION_TOKEN", "ticket-token") monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "false") get_settings.cache_clear() @@ -135,6 +137,9 @@ def test_verified_ticket_is_deduplicated_rotated_and_never_leaked( assert current.received_at >= first_received_at assert db.scalar(select(func.count()).select_from(FeishuAppTicket)) == 1 assert db.scalar(select(func.count()).select_from(FeishuEventReceipt)) == 2 + receipts = list(db.execute(select(FeishuEventReceipt)).scalars()) + assert all(item.status == "succeeded" for item in receipts) + assert all(item.payload is None for item in receipts) assert rotated_ticket not in json.dumps(rotated, ensure_ascii=False) audits = list(db.execute(select(AuditLog)).scalars()) @@ -196,6 +201,59 @@ def test_only_verified_matching_app_ticket_events_can_write( assert db.scalar(select(FeishuAppTicket)) is None +def test_app_ticket_write_and_receipt_success_are_atomic( + session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = _v2_ticket_event("atomic-app-ticket", "atomic-ticket-secret") + original_mark_success = FeishuInboundService._mark_success + fail_before_commit = True + + def mark_success( + atomic_db: Session, + event_key: str, + lock_owner: str, + current: Any, + ) -> None: + nonlocal fail_before_commit + if fail_before_commit: + fail_before_commit = False + raise RuntimeError("simulated crash before atomic app-ticket commit") + original_mark_success(atomic_db, event_key, lock_owner, current) + + monkeypatch.setattr( + FeishuInboundService, + "_mark_success", + staticmethod(mark_success), + ) + with session_factory() as db: + service = FeishuEventService(db) + with pytest.raises(HTTPException) as first_error: + service._handle_verified_event( + payload, + source=FeishuEventSource.WEBHOOK, + ) + + assert first_error.value.status_code == 503 + assert db.scalar(select(FeishuAppTicket)) is None + receipt = db.scalar(select(FeishuEventReceipt)) + assert receipt is not None + assert receipt.status == "retry" + + result = service._handle_verified_event( + payload, + source=FeishuEventSource.WEBHOOK, + ) + + assert result == {"ok": True, "handled": True} + assert FeishuAppTicketService(db).get_ticket("cli-ticket-app") == ( + "atomic-ticket-secret" + ) + db.refresh(receipt) + assert receipt.status == "succeeded" + assert receipt.attempt_count == 2 + + def test_v1_app_ticket_payload_uses_uuid_receipt( session_factory: sessionmaker[Session], ) -> None: @@ -209,8 +267,18 @@ def test_v1_app_ticket_payload_uses_uuid_receipt( receipt = db.scalar(select(FeishuEventReceipt)) assert result == {"ok": True, "handled": True} assert receipt is not None - assert receipt.event_id == "v1-ticket-uuid" - assert receipt.event_key == "cli-ticket-app:app_ticket:v1-ticket-uuid" + assert receipt.event_id == _identifier_digest( + "v1-ticket-uuid", + "event-id", + ) + assert receipt.event_key == _identifier_digest( + "cli-ticket-app:app_ticket:v1-ticket-uuid", + "event-key", + ) + assert "v1-ticket-uuid" not in receipt.event_id + assert "cli-ticket-app" not in receipt.event_key + assert receipt.status == "succeeded" + assert receipt.payload is None assert FeishuAppTicketService(db).get_ticket("cli-ticket-app") == ticket assert ticket not in json.dumps(result, ensure_ascii=False) @@ -257,6 +325,7 @@ def test_long_connection_registers_custom_app_ticket_handler( monkeypatch.setitem(sys.modules, "lark_oapi", fake_lark) monkeypatch.setenv("FEISHU_APP_ID", "cli-ticket-app") monkeypatch.setenv("FEISHU_APP_SECRET", "ticket-app-secret") + monkeypatch.setenv("FEISHU_EVENT_TRANSPORT", "long_connection") monkeypatch.setenv("FEISHU_VERIFICATION_TOKEN", "ticket-token") get_settings.cache_clear() try: diff --git a/tests/test_feishu_inbound_reliability.py b/tests/test_feishu_inbound_reliability.py new file mode 100644 index 0000000..03a2cfb --- /dev/null +++ b/tests/test_feishu_inbound_reliability.py @@ -0,0 +1,1220 @@ +import json +import tempfile +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta +from hashlib import sha256 +from pathlib import Path +from threading import Event, Lock +from typing import Any + +import pytest +from alembic import command +from alembic.config import Config +from fastapi import HTTPException +from sqlalchemy import create_engine, func, select, text +from sqlalchemy.orm import Session, sessionmaker + +from app.application.feishu.commands import FeishuCommandService +from app.application.feishu.events import ( + FeishuEventService, + _audit_event_metadata, + _identifier_digest, +) +from app.application.feishu.personal_data import FeishuPersonalDataService +from app.application.feishu.results import command_result +from app.core.config import get_settings +from app.core.database import Base +from app.core.utils.time import utc_now +from app.modules.audit.models import AuditLog +from app.modules.feishu.client import FeishuClient +from app.modules.feishu import long_connection +from app.modules.feishu.constants import ( + FeishuCommandName, + FeishuEventSource, + FeishuInboundStatus, + FeishuReplyType, +) +from app.modules.feishu.models import FeishuEventReceipt +from app.modules.feishu.services import FeishuInboundService +from app.modules.feishu_users.models import FeishuUser +from app.modules.feishu_users.identifiers import feishu_audit_identity_hash +from app.modules.feishu_users.principal import FeishuPrincipal +from app.modules.feishu_users.services import FeishuIdentityService + + +@pytest.fixture +def session_factory( + monkeypatch: pytest.MonkeyPatch, +) -> sessionmaker[Session]: + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "false") + monkeypatch.setenv("FEISHU_VERIFICATION_TOKEN", "verified-token") + get_settings.cache_clear() + database_file = tempfile.NamedTemporaryFile( + prefix="feishu-inbound-", + suffix=".db", + delete=False, + ) + database_file.close() + database_path = Path(database_file.name) + engine = create_engine( + f"sqlite:///{database_path.as_posix()}", + connect_args={"check_same_thread": False, "timeout": 5}, + ) + FeishuEventReceipt.__table__.create(engine) + AuditLog.__table__.create(engine) + FeishuUser.__table__.create(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + try: + yield factory + finally: + FeishuUser.__table__.drop(engine) + AuditLog.__table__.drop(engine) + FeishuEventReceipt.__table__.drop(engine) + engine.dispose() + database_path.unlink(missing_ok=True) + get_settings.cache_clear() + + +@pytest.fixture +def full_session_factory( + monkeypatch: pytest.MonkeyPatch, +) -> sessionmaker[Session]: + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "true") + monkeypatch.setenv("FEISHU_VERIFICATION_TOKEN", "verified-token") + get_settings.cache_clear() + database_file = tempfile.NamedTemporaryFile( + prefix="feishu-inbound-erasure-", + suffix=".db", + delete=False, + ) + database_file.close() + database_path = Path(database_file.name) + engine = create_engine( + f"sqlite:///{database_path.as_posix()}", + connect_args={"check_same_thread": False, "timeout": 5}, + ) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + try: + yield factory + finally: + engine.dispose() + database_path.unlink(missing_ok=True) + get_settings.cache_clear() + + +def _message_event( + event_id: str, + *, + text: str = "hello", + tenant_key: str = "tenant-a", + open_id: str = "ou-user", +) -> dict[str, Any]: + return { + "schema": "2.0", + "header": { + "event_id": event_id, + "event_type": "im.message.receive_v1", + "tenant_key": tenant_key, + "token": "verified-token", + }, + "event": { + "sender": {"sender_id": {"open_id": open_id}}, + "message": { + "chat_id": "oc-chat", + "chat_type": "p2p", + "message_id": f"om-{event_id}", + "message_type": "text", + "content": json.dumps({"text": text}), + }, + }, + } + + +def _accept( + db: Session, + payload: dict[str, Any], +) -> str: + accepted = FeishuEventService(db).accept_verified_event( + payload, + source=FeishuEventSource.WEBHOOK, + auto_reply=False, + ) + assert accepted.event_key is not None + return accepted.event_key + + +def _expected_identifier_digest(value: str, domain: str) -> str: + digest = sha256( + f"company-ai-platform:feishu:{domain}:v1\0{value}".encode("utf-8") + ).hexdigest() + return f"sha256-{digest}" + + +def _create_erasure_confirmation( + db: Session, + *, + tenant_key: str = "tenant-a", + open_id: str = "ou-user", +) -> tuple[FeishuPrincipal, str]: + user = FeishuUser( + code=f"FSU-ERASURE-{open_id}", + tenant_key=tenant_key, + open_id=open_id, + ) + db.add(user) + db.commit() + db.refresh(user) + principal = FeishuPrincipal.from_user( + user, + chat_id="oc-chat", + chat_type="p2p", + ) + confirmation = FeishuPersonalDataService(db).request_confirmation( + principal + ) + return principal, confirmation.confirmation_code + + +def test_identifier_metadata_uses_fixed_domain_separated_digests( + session_factory: sessionmaker[Session], +) -> None: + raw_identifier = "shared-raw-identifier" + payload = _message_event( + raw_identifier, + tenant_key=raw_identifier, + open_id=raw_identifier, + ) + payload["header"]["app_id"] = raw_identifier + payload["event"]["message"]["chat_id"] = raw_identifier + payload["event"]["message"]["message_id"] = raw_identifier + + with session_factory() as db: + _accept(db, payload) + receipt = db.scalar(select(FeishuEventReceipt)) + + assert receipt is not None + metadata = _audit_event_metadata( + payload, + include_open_id=True, + include_identity_context=True, + ) + digests = { + receipt.event_key, + metadata["event_id"], + metadata["message_id"], + metadata["chat_id"], + metadata["open_id"], + metadata["tenant_key"], + metadata["app_id"], + } + assert len(digests) == 7 + assert all( + isinstance(value, str) + and len(value) == 71 + and value.startswith("sha256-") + for value in digests + ) + assert raw_identifier not in json.dumps(metadata) + + +def test_verified_acceptance_audit_uses_erasable_identity_subject( + full_session_factory: sessionmaker[Session], +) -> None: + with full_session_factory() as db: + _accept( + db, + _message_event( + "erasable-audit-subject", + tenant_key="tenant-audit", + open_id="ou-audit", + ), + ) + record = db.execute( + select(AuditLog).where(AuditLog.action == "webhook_event") + ).scalar_one() + assert record.actor == feishu_audit_identity_hash( + "tenant-audit", + "ou-audit", + ) + assert "tenant-audit" not in str(record.request_payload) + assert "ou-audit" not in str(record.request_payload) + + +def test_inbox_migration_digests_legacy_identifiers_once( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + database_path = tmp_path / "legacy-feishu-receipts.db" + database_url = f"sqlite:///{database_path.as_posix()}" + monkeypatch.setenv("DATABASE_URL", database_url) + get_settings.cache_clear() + config = Config("alembic.ini") + engine = create_engine(database_url) + received_at = datetime(2026, 7, 27, 1, 0) + raw_event_key = "tenant-a:im.message.receive_v1:legacy-event" + existing_event_key = _expected_identifier_digest( + "tenant-a:im.message.receive_v1:already-private", + "event-key", + ) + existing_event_id = _expected_identifier_digest( + "already-private", + "event-id", + ) + existing_message_id = _expected_identifier_digest( + "om-already-private", + "message-id", + ) + + try: + command.upgrade(config, "202607270001") + with engine.begin() as connection: + connection.execute( + text( + """ + INSERT INTO feishu_event_receipts ( + event_key, + source, + event_id, + message_id, + received_at + ) + VALUES ( + :event_key, + 'webhook', + :event_id, + :message_id, + :received_at + ) + """ + ), + [ + { + "event_key": raw_event_key, + "event_id": "legacy-event", + "message_id": "om-legacy-event", + "received_at": received_at, + }, + { + "event_key": existing_event_key, + "event_id": existing_event_id, + "message_id": existing_message_id, + "received_at": received_at, + }, + ], + ) + + command.upgrade(config, "head") + + with engine.connect() as connection: + rows = list( + connection.execute( + text( + """ + SELECT event_key, event_id, message_id, status, payload + FROM feishu_event_receipts + ORDER BY id + """ + ) + ).mappings() + ) + + assert rows[0]["event_key"] == _expected_identifier_digest( + raw_event_key, + "event-key", + ) + assert rows[0]["event_id"] == _expected_identifier_digest( + "legacy-event", + "event-id", + ) + assert rows[0]["message_id"] == _expected_identifier_digest( + "om-legacy-event", + "message-id", + ) + assert rows[1]["event_key"] == existing_event_key + assert rows[1]["event_id"] == existing_event_id + assert rows[1]["message_id"] == existing_message_id + assert all(row["status"] == FeishuInboundStatus.SUCCEEDED for row in rows) + assert all(row["payload"] is None for row in rows) + assert "legacy-event" not in json.dumps(rows, default=str) + finally: + engine.dispose() + get_settings.cache_clear() + + +def test_verified_intake_is_durable_sanitized_and_does_not_execute_command( + session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + FeishuCommandService, + "handle_text", + lambda *_args, **_kwargs: pytest.fail("intake must not execute commands"), + ) + payload = _message_event("durable-intake", text="private question") + payload["event"]["token"] = "nested-secret" + + with session_factory() as db: + accepted = FeishuEventService(db).accept_verified_event( + payload, + source=FeishuEventSource.WEBHOOK, + auto_reply=False, + ) + record = db.scalar(select(FeishuEventReceipt)) + + assert accepted.response["accepted"] is True + assert accepted.response["handled"] is False + assert accepted.should_dispatch is True + assert record is not None + assert record.status == FeishuInboundStatus.PENDING + assert record.next_attempt_at is not None + serialized = json.dumps(record.payload) + assert "verified-token" not in serialized + assert "nested-secret" not in serialized + assert "private question" in serialized + + +def test_concurrent_duplicate_workers_execute_only_once( + session_factory: sessionmaker[Session], +) -> None: + with session_factory() as db: + event_key = _accept(db, _message_event("concurrent-once")) + + started = Event() + release = Event() + counter_lock = Lock() + execution_count = 0 + + def handler( + _payload: dict[str, Any], + _source: str, + _auto_reply: bool, + _event_key: str, + ) -> dict[str, Any]: + nonlocal execution_count + with counter_lock: + execution_count += 1 + started.set() + assert release.wait(timeout=5) + return {"ok": True} + + def process(worker_id: str) -> str: + with session_factory() as db: + return FeishuInboundService(db).process( + event_key, + handler_factory=lambda _db: handler, + worker_id=worker_id, + ).record.status + + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(process, "worker-one") + assert started.wait(timeout=5) + second = executor.submit(process, "worker-two") + release.set() + assert first.result(timeout=5) == FeishuInboundStatus.SUCCEEDED + assert second.result(timeout=5) == FeishuInboundStatus.SUCCEEDED + + assert execution_count == 1 + with session_factory() as db: + record = db.scalar(select(FeishuEventReceipt)) + assert record is not None + assert record.attempt_count == 1 + assert record.payload is None + + +def test_handler_exceeding_lease_is_not_executed_concurrently( + session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + with session_factory() as db: + event_key = _accept(db, _message_event("slow-handler-lease")) + claimed_at = utc_now() + + handler_started = Event() + release_handler = Event() + replacement_claim_started = Event() + counter_lock = Lock() + execution_count = 0 + original_claim = FeishuInboundService._claim + + def observed_claim( + self: FeishuInboundService, + receipt_id: int, + lock_owner: str, + current: datetime, + **kwargs: Any, + ) -> bool: + if lock_owner.startswith("replacement:"): + replacement_claim_started.set() + return original_claim( + self, + receipt_id, + lock_owner, + current, + **kwargs, + ) + + def handler( + _payload: dict[str, Any], + _source: str, + _auto_reply: bool, + _event_key: str, + ) -> dict[str, bool]: + nonlocal execution_count + with counter_lock: + execution_count += 1 + handler_started.set() + assert release_handler.wait(timeout=5) + return {"ok": True} + + def process(worker_id: str, current: datetime) -> str: + with session_factory() as db: + return FeishuInboundService(db, lease_seconds=1).process( + event_key, + handler_factory=lambda _db: handler, + now=current, + worker_id=worker_id, + ).record.status + + monkeypatch.setattr(FeishuInboundService, "_claim", observed_claim) + with ThreadPoolExecutor(max_workers=2) as executor: + original = executor.submit(process, "original", claimed_at) + assert handler_started.wait(timeout=5) + replacement = executor.submit( + process, + "replacement", + claimed_at + timedelta(seconds=2), + ) + assert replacement_claim_started.wait(timeout=5) + assert execution_count == 1 + release_handler.set() + + assert original.result(timeout=5) == FeishuInboundStatus.SUCCEEDED + assert replacement.result(timeout=5) == FeishuInboundStatus.SUCCEEDED + + assert execution_count == 1 + with session_factory() as db: + record = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == event_key + ) + ) + assert record is not None + assert record.attempt_count == 1 + + +def test_failure_retries_with_same_reply_uuid_and_redacted_error( + session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen_uuids: list[str | None] = [] + attempt_count = 0 + + def flaky_handle( + self: FeishuCommandService, + *_args: Any, + **_kwargs: Any, + ) -> dict[str, Any]: + nonlocal attempt_count + attempt_count += 1 + seen_uuids.append(self.feishu.message_uuid) + if attempt_count == 1: + raise RuntimeError("private prompt must never enter last_error") + return {"command": "fallback_ai"} + + monkeypatch.setattr(FeishuCommandService, "handle_text", flaky_handle) + with session_factory() as db: + service = FeishuEventService(db) + event_key = _accept(db, _message_event("retry-stable-uuid")) + first = service.process_inbound_event(event_key, worker_id="first") + assert first.record.status == FeishuInboundStatus.RETRY + assert first.record.last_error == "RuntimeError" + assert "private prompt" not in str(first.record.last_error) + first.record.next_attempt_at = utc_now() + db.commit() + + second = service.process_inbound_event(event_key, worker_id="second") + assert second.record.status == FeishuInboundStatus.SUCCEEDED + assert second.record.payload is None + + duplicate = service.process_inbound_event(event_key, worker_id="duplicate") + assert duplicate.record.status == FeishuInboundStatus.SUCCEEDED + + assert attempt_count == 2 + assert len(seen_uuids) == 2 + assert seen_uuids[0] == seen_uuids[1] + assert seen_uuids[0] is not None + assert seen_uuids[0].startswith("inbound-") + + +def test_command_writes_and_receipt_success_commit_atomically( + session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + with session_factory() as db: + event_key = _accept(db, _message_event("atomic-command")) + service = FeishuInboundService(db) + original_mark_success = FeishuInboundService._mark_success + fail_before_commit = True + + def mark_success( + atomic_db: Session, + key: str, + lock_owner: str, + current: datetime, + ) -> None: + nonlocal fail_before_commit + if fail_before_commit: + fail_before_commit = False + raise RuntimeError("simulated crash before atomic commit") + original_mark_success(atomic_db, key, lock_owner, current) + + def handler_factory(atomic_db: Session) -> Any: + def handler(*_args: Any) -> dict[str, bool]: + atomic_db.add( + AuditLog( + actor="atomic-test", + source="test", + action="atomic-side-effect", + ) + ) + atomic_db.commit() + return {"ok": True} + + return handler + + monkeypatch.setattr( + FeishuInboundService, + "_mark_success", + staticmethod(mark_success), + ) + first = service.process( + event_key, + handler_factory=handler_factory, + worker_id="first", + ) + assert first.record.status == FeishuInboundStatus.RETRY + assert ( + db.scalar( + select(func.count()) + .select_from(AuditLog) + .where(AuditLog.action == "atomic-side-effect") + ) + == 0 + ) + + first.record.next_attempt_at = utc_now() + db.commit() + second = service.process( + event_key, + handler_factory=handler_factory, + worker_id="second", + ) + assert second.record.status == FeishuInboundStatus.SUCCEEDED + assert ( + db.scalar( + select(func.count()) + .select_from(AuditLog) + .where(AuditLog.action == "atomic-side-effect") + ) + == 1 + ) + + +def test_reply_outbox_does_not_rerun_command_after_post_send_crash( + session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("FEISHU_APP_ID", "cli_test_app") + monkeypatch.setenv("FEISHU_APP_SECRET", "test-secret-value") + get_settings.cache_clear() + command_runs = 0 + send_attempts: list[tuple[str, str | None]] = [] + + def handle_text( + self: FeishuCommandService, + _text: str, + chat_id: str | None = None, + actor: str = "feishu", + auto_reply: bool = True, + **_kwargs: Any, + ) -> dict[str, Any]: + nonlocal command_runs + command_runs += 1 + content = f"SUB-{command_runs}" + self.db.add( + AuditLog( + actor="reply-outbox-test", + source="test", + action="reply-outbox-command", + ) + ) + self.db.commit() + response = ( + self.feishu.send_text( + content, + receive_id=chat_id, + actor=actor, + ) + if auto_reply + else None + ) + return command_result( + FeishuCommandName.SUBSCRIPTION_CREATE, + FeishuReplyType.TEXT, + "订阅", + content, + response, + ) + + def send_text( + _self: FeishuClient, + text_value: str, + _receive_id: str | None = None, + _receive_id_type: str = "chat_id", + uuid: str | None = None, + tenant_key: str | None = None, + ) -> dict[str, Any]: + del tenant_key + send_attempts.append((text_value, uuid)) + return {"code": 0, "data": {"message_id": "om-reply"}} + + original_mark_reply_success = FeishuInboundService._mark_reply_success + crash_after_send = True + + def mark_reply_success( + atomic_db: Session, + key: str, + lock_owner: str, + current: datetime, + ) -> None: + nonlocal crash_after_send + if crash_after_send: + crash_after_send = False + raise RuntimeError("simulated crash after provider accepted reply") + original_mark_reply_success( + atomic_db, + key, + lock_owner, + current, + ) + + monkeypatch.setattr(FeishuCommandService, "handle_text", handle_text) + monkeypatch.setattr(FeishuClient, "send_text", send_text) + monkeypatch.setattr( + FeishuInboundService, + "_mark_reply_success", + staticmethod(mark_reply_success), + ) + + with session_factory() as db: + accepted = FeishuEventService(db).accept_verified_event( + _message_event("reply-post-send-crash", text="订阅 每天 09:00：提醒我"), + source=FeishuEventSource.WEBHOOK, + auto_reply=True, + ) + assert accepted.event_key is not None + event_key = accepted.event_key + + first = FeishuEventService(db).process_inbound_event( + event_key, + worker_id="first", + ) + assert first.record.status == FeishuInboundStatus.SUCCEEDED + assert first.record.reply_status == FeishuInboundStatus.RETRY + assert first.record.reply_payload is not None + assert command_runs == 1 + assert len(send_attempts) == 1 + assert send_attempts[0][0] == "SUB-1" + + first.record.reply_next_attempt_at = utc_now() + db.commit() + FeishuInboundService(db).dispatch_reply( + event_key, + worker_id="reply-retry", + ) + db.expire_all() + completed = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == event_key + ) + ) + assert completed is not None + assert completed.reply_status == FeishuInboundStatus.SUCCEEDED + assert completed.reply_payload is None + assert command_runs == 1 + assert len(send_attempts) == 2 + assert send_attempts[0] == send_attempts[1] + assert send_attempts[0][1] is not None + assert send_attempts[0][1].startswith("inbound-") + assert ( + db.scalar( + select(func.count()) + .select_from(AuditLog) + .where(AuditLog.action == "reply-outbox-command") + ) + == 1 + ) + + +def test_reply_outbox_defers_image_and_card_until_command_commit( + session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("FEISHU_APP_ID", "cli_test_app") + monkeypatch.setenv("FEISHU_APP_SECRET", "test-secret-value") + get_settings.cache_clear() + image_uploads: list[bytes] = [] + sent_cards: list[tuple[dict[str, Any], str | None]] = [] + command_runs = 0 + + def handle_text( + self: FeishuCommandService, + _text: str, + chat_id: str | None = None, + actor: str = "feishu", + auto_reply: bool = True, + **_kwargs: Any, + ) -> dict[str, Any]: + nonlocal command_runs + command_runs += 1 + response = None + if auto_reply: + image = self.feishu.upload_image(b"stable-chart", actor=actor) + image_key = (image.get("data") or {}).get("image_key") + card = self.feishu.build_basic_card( + "可靠卡片", + ["同一事务提交后发送"], + image_key=image_key, + ) + response = self.feishu.send_card( + card, + receive_id=chat_id, + actor=actor, + ) + return command_result( + FeishuCommandName.MARKET_OVERVIEW, + FeishuReplyType.CARD, + "可靠卡片", + "同一事务提交后发送", + response, + ["同一事务提交后发送"], + ) + + def upload_image( + _self: FeishuClient, + image: bytes, + _filename: str = "lifecycle-report.png", + tenant_key: str | None = None, + ) -> dict[str, Any]: + del tenant_key + image_uploads.append(image) + return {"code": 0, "data": {"image_key": "img-stable"}} + + def send_card( + _self: FeishuClient, + card: dict[str, Any], + _receive_id: str | None = None, + _receive_id_type: str = "chat_id", + uuid: str | None = None, + tenant_key: str | None = None, + ) -> dict[str, Any]: + del tenant_key + sent_cards.append((card, uuid)) + return {"code": 0, "data": {"message_id": "om-card"}} + + original_mark_success = FeishuInboundService._mark_success + fail_before_commit = True + + def mark_success( + atomic_db: Session, + key: str, + lock_owner: str, + current: datetime, + ) -> None: + nonlocal fail_before_commit + if fail_before_commit: + fail_before_commit = False + raise RuntimeError("simulated crash before command commit") + original_mark_success(atomic_db, key, lock_owner, current) + + monkeypatch.setattr(FeishuCommandService, "handle_text", handle_text) + monkeypatch.setattr(FeishuClient, "upload_image", upload_image) + monkeypatch.setattr(FeishuClient, "send_card", send_card) + monkeypatch.setattr( + FeishuInboundService, + "_mark_success", + staticmethod(mark_success), + ) + + with session_factory() as db: + accepted = FeishuEventService(db).accept_verified_event( + _message_event("reply-card-commit"), + source=FeishuEventSource.WEBHOOK, + auto_reply=True, + ) + assert accepted.event_key is not None + event_key = accepted.event_key + first = FeishuEventService(db).process_inbound_event( + event_key, + worker_id="first", + ) + assert first.record.status == FeishuInboundStatus.RETRY + assert image_uploads == [] + assert sent_cards == [] + + first.record.next_attempt_at = utc_now() + db.commit() + second = FeishuEventService(db).process_inbound_event( + event_key, + worker_id="second", + ) + assert second.record.status == FeishuInboundStatus.SUCCEEDED + assert second.record.reply_status == FeishuInboundStatus.SUCCEEDED + assert command_runs == 2 + assert image_uploads == [b"stable-chart"] + assert len(sent_cards) == 1 + assert sent_cards[0][1] is not None + assert sent_cards[0][0]["elements"][0]["img_key"] == "img-stable" + + +def test_expired_lease_is_reclaimed_and_terminal_failure_clears_payload( + session_factory: sessionmaker[Session], +) -> None: + current = datetime(2026, 7, 27, 2, 0) + with session_factory() as db: + event_key = _accept(db, _message_event("expired-lease")) + record = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == event_key + ) + ) + assert record is not None + record.status = FeishuInboundStatus.PROCESSING + record.attempt_count = 1 + record.locked_by = "dead-worker" + record.locked_until = current - timedelta(seconds=1) + db.commit() + + recovered = FeishuInboundService(db).process( + event_key, + handler_factory=lambda _db: ( + lambda *_args: {"ok": True} + ), + now=current, + worker_id="replacement", + ) + assert recovered.record.status == FeishuInboundStatus.SUCCEEDED + assert recovered.record.attempt_count == 2 + + failed_key = _accept(db, _message_event("terminal-failure")) + service = FeishuInboundService(db) + pending_failure = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == failed_key + ) + ) + assert pending_failure is not None + assert pending_failure.next_attempt_at is not None + next_time = pending_failure.next_attempt_at + timedelta(microseconds=1) + for _ in range(4): + failed = service.process( + failed_key, + handler_factory=lambda _db: ( + lambda *_args: (_ for _ in ()).throw( + RuntimeError("secret") + ) + ), + now=next_time, + ) + next_time = ( + failed.record.next_attempt_at + timedelta(microseconds=1) + if failed.record.next_attempt_at is not None + else next_time + ) + assert failed.record.status == FeishuInboundStatus.FAILED + assert failed.record.attempt_count == 4 + assert failed.record.payload is None + assert failed.record.last_error == "RuntimeError" + + +def test_erasure_clears_other_pending_identity_payloads( + session_factory: sessionmaker[Session], +) -> None: + with session_factory() as db: + current_key = _accept(db, _message_event("erase-current")) + other_key = _accept(db, _message_event("erase-other")) + reply_key = _accept(db, _message_event("erase-pending-reply")) + untouched_key = _accept( + db, + _message_event( + "erase-unrelated", + tenant_key="tenant-b", + open_id="ou-other", + ), + ) + reply_record = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == reply_key + ) + ) + assert reply_record is not None + reply_record.status = FeishuInboundStatus.SUCCEEDED + reply_record.payload = None + reply_record.reply_status = FeishuInboundStatus.PENDING + reply_record.reply_next_attempt_at = utc_now() + reply_record.reply_payload = { + "version": 1, + "identity": { + "tenant_key": "tenant-a", + "open_id": "ou-user", + }, + "identity_fence_required": False, + "operations": [ + { + "kind": "text", + "text": "must not be sent after erasure", + "receive_id": "oc-chat", + "receive_id_type": "chat_id", + } + ], + } + db.commit() + cleared = FeishuInboundService(db).erase_identity_payloads( + tenant_key="tenant-a", + open_id="ou-user", + exclude_event_key=current_key, + ) + db.commit() + + records = { + item.event_key: item + for item in db.execute(select(FeishuEventReceipt)).scalars() + } + assert cleared == 2 + assert records[current_key].payload is not None + assert records[other_key].status == FeishuInboundStatus.FAILED + assert records[other_key].payload is None + assert records[reply_key].reply_status == FeishuInboundStatus.FAILED + assert records[reply_key].reply_payload is None + assert records[untouched_key].payload is not None + + +def test_erasure_fences_already_claimed_event_before_identity_deletion( + full_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + lock_owner = "old-worker:claimed" + with full_session_factory() as db: + _principal, confirmation_code = _create_erasure_confirmation(db) + old_payload = _message_event("old-processing-event") + old_key = _accept(db, old_payload) + deletion_key = _accept( + db, + _message_event( + "delete-identity-event", + text=f"确认忘记我 {confirmation_code}", + ), + ) + old_record = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == old_key + ) + ) + assert old_record is not None + old_record.status = FeishuInboundStatus.PROCESSING + old_record.attempt_count = 1 + old_record.locked_by = lock_owner + old_record.locked_until = utc_now() + timedelta(minutes=5) + db.commit() + + erasure_has_identity_fence = Event() + allow_erasure = Event() + old_worker_attempting = Event() + old_handler_calls = 0 + original_erase = FeishuInboundService.erase_identity_payloads + + def blocking_erase( + self: FeishuInboundService, + **kwargs: Any, + ) -> int: + erasure_has_identity_fence.set() + assert allow_erasure.wait(timeout=5) + return original_erase(self, **kwargs) + + def process_deletion() -> str: + with full_session_factory() as db: + return FeishuEventService(db).process_inbound_event( + deletion_key, + worker_id="deletion", + ).record.status + + def resume_claimed_event() -> int: + nonlocal old_handler_calls + old_worker_attempting.set() + + def handler_factory(atomic_db: Session) -> Any: + def handler(*_args: Any) -> dict[str, bool]: + nonlocal old_handler_calls + old_handler_calls += 1 + FeishuIdentityService(atomic_db).resolve_or_register( + tenant_key="tenant-a", + open_id="ou-user", + ) + return {"ok": True} + + return handler + + with full_session_factory() as db: + service = FeishuInboundService(db) + try: + service._execute_atomically( + event_key=old_key, + lock_owner=lock_owner, + current=utc_now(), + payload=old_payload, + source=FeishuEventSource.WEBHOOK, + auto_reply=False, + handler_factory=handler_factory, + ) + except HTTPException as exc: + return exc.status_code + return 200 + + monkeypatch.setattr( + FeishuInboundService, + "erase_identity_payloads", + blocking_erase, + ) + with ThreadPoolExecutor(max_workers=2) as executor: + deletion = executor.submit(process_deletion) + assert erasure_has_identity_fence.wait(timeout=5) + old_worker = executor.submit(resume_claimed_event) + assert old_worker_attempting.wait(timeout=5) + assert not old_worker.done() + + allow_erasure.set() + + assert deletion.result(timeout=5) == FeishuInboundStatus.SUCCEEDED + assert old_worker.result(timeout=5) == 409 + + assert old_handler_calls == 0 + with full_session_factory() as db: + assert db.scalar(select(FeishuUser)) is None + old_record = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == old_key + ) + ) + deletion_record = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_key == deletion_key + ) + ) + assert old_record is not None + assert old_record.status == FeishuInboundStatus.FAILED + assert old_record.payload is None + assert deletion_record is not None + assert deletion_record.status == FeishuInboundStatus.SUCCEEDED + + monkeypatch.setattr( + FeishuIdentityService, + "resolve_or_register", + lambda *_args, **_kwargs: pytest.fail( + "a terminal erased event must not recreate its identity" + ), + ) + retry = FeishuEventService(db).process_inbound_event( + old_key, + worker_id="old-retry", + ) + assert retry.record.status == FeishuInboundStatus.FAILED + assert db.scalar(select(FeishuUser)) is None + + +def test_completed_erasure_event_duplicate_does_not_recreate_identity( + full_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + with full_session_factory() as db: + _principal, confirmation_code = _create_erasure_confirmation(db) + payload = _message_event( + "completed-delete-event", + text=f"确认忘记我 {confirmation_code}", + ) + event_key = _accept(db, payload) + first = FeishuEventService(db).process_inbound_event( + event_key, + worker_id="first-delete", + ) + + assert first.record.status == FeishuInboundStatus.SUCCEEDED + assert db.scalar(select(FeishuUser)) is None + + monkeypatch.setattr( + FeishuIdentityService, + "resolve_or_register", + lambda *_args, **_kwargs: pytest.fail( + "a completed deletion duplicate must not recreate its identity" + ), + ) + duplicate = FeishuEventService(db).process_inbound_event( + event_key, + worker_id="duplicate-delete", + ) + accepted_duplicate = FeishuEventService(db).accept_verified_event( + payload, + source=FeishuEventSource.WEBHOOK, + auto_reply=False, + ) + + assert duplicate.record.status == FeishuInboundStatus.SUCCEEDED + assert accepted_duplicate.should_dispatch is False + assert db.scalar(select(FeishuUser)) is None + + +def test_long_connection_dispatches_only_when_durable_queue_is_enabled( + session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = _message_event("long-connection-inbox") + queued: list[str] = [] + monkeypatch.setattr(long_connection, "_sdk_event_to_payload", lambda _event: payload) + monkeypatch.setattr(long_connection, "SessionLocal", session_factory) + monkeypatch.setattr( + long_connection, + "enqueue_feishu_inbound_event", + lambda event_key, **_kwargs: queued.append(event_key), + ) + + long_connection._handle_verified_sdk_event(object()) + + with session_factory() as db: + record = db.scalar(select(FeishuEventReceipt)) + assert record is not None + assert queued == [] + assert record.source == FeishuEventSource.LONG_CONNECTION + assert record.status == FeishuInboundStatus.PENDING + + queued_payload = _message_event("long-connection-queued") + monkeypatch.setattr( + long_connection, + "_sdk_event_to_payload", + lambda _event: queued_payload, + ) + monkeypatch.setenv("TASK_QUEUE_ENABLED", "true") + get_settings.cache_clear() + long_connection._handle_verified_sdk_event(object()) + + with session_factory() as db: + queued_record = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_id + == _identifier_digest( + "long-connection-queued", + "event-id", + ) + ) + ) + assert queued_record is not None + assert queued == [queued_record.event_key] diff --git a/tests/test_feishu_long_connection_health.py b/tests/test_feishu_long_connection_health.py new file mode 100644 index 0000000..fab0aa0 --- /dev/null +++ b/tests/test_feishu_long_connection_health.py @@ -0,0 +1,226 @@ +from types import SimpleNamespace +from typing import Any + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, text +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import StaticPool + +from app.core.config import get_settings +from app.core.database import Base, get_db +from app.core.database.migrations import expected_alembic_heads +from app.modules.feishu import long_connection +from app.modules.observability.constants import ( + HeartbeatComponent, + HeartbeatStatus, + ObservabilityKey, + ObservabilityStatus, +) +from app.modules.observability.routes import router as observability_router +from app.modules.observability.service import ObservabilityService + + +def test_connection_heartbeat_tracks_disconnect_and_recovery( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = SimpleNamespace(_conn=None) + statuses: list[str] = [] + + class FakeStopEvent: + def __init__(self) -> None: + self.wait_count = 0 + + def is_set(self) -> bool: + return self.wait_count >= 3 + + def wait(self, timeout: float) -> None: + assert timeout == 1 + self.wait_count += 1 + if self.wait_count == 1: + client._conn = SimpleNamespace( + state=SimpleNamespace(name="OPEN"), + ) + elif self.wait_count == 2: + client._conn = SimpleNamespace( + state=SimpleNamespace(name="CLOSED"), + ) + + monkeypatch.setattr( + long_connection, + "_record_runtime_heartbeat", + lambda _instance_id, status_value: statuses.append(status_value), + ) + + long_connection._heartbeat_loop( + FakeStopEvent(), # type: ignore[arg-type] + "feishu-events-test", + 30, + client, + ) + + assert statuses == [ + HeartbeatStatus.DEGRADED, + HeartbeatStatus.OK, + HeartbeatStatus.DEGRADED, + ] + + +def test_connection_adapter_supports_sdk_state_variants() -> None: + assert long_connection._sdk_connection_is_open(SimpleNamespace(_conn=None)) is False + unknown_client = SimpleNamespace(_conn=SimpleNamespace()) + assert long_connection._sdk_connection_is_open(unknown_client) is False + assert ( + long_connection._connection_heartbeat_status(unknown_client) + == HeartbeatStatus.DEGRADED + ) + assert ( + long_connection._sdk_connection_is_open( + SimpleNamespace(_conn=SimpleNamespace(closed=False)) + ) + is True + ) + assert ( + long_connection._sdk_connection_is_open( + SimpleNamespace(_conn=SimpleNamespace(open=False)) + ) + is False + ) + assert ( + long_connection._sdk_connection_is_open( + SimpleNamespace( + _conn=SimpleNamespace(state=SimpleNamespace(name="OPEN")) + ) + ) + is True + ) + assert ( + long_connection._sdk_connection_is_open( + SimpleNamespace( + _conn=SimpleNamespace(state=SimpleNamespace(name="CLOSING")) + ) + ) + is False + ) + + +def test_readiness_returns_503_while_disconnected_and_recovers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("APP_ENV", "test") + monkeypatch.setenv("FEISHU_EVENT_TRANSPORT", "long_connection") + monkeypatch.setenv("FEISHU_APP_ID", "app-id") + monkeypatch.setenv("FEISHU_APP_SECRET", "app-secret") + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "false") + monkeypatch.setenv("SCHEDULER_ENABLED", "false") + monkeypatch.setenv("TASK_QUEUE_ENABLED", "false") + get_settings.cache_clear() + + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + with engine.begin() as connection: + connection.execute( + text("create table alembic_version (version_num varchar(32))") + ) + connection.execute( + text("insert into alembic_version (version_num) values (:revision)"), + {"revision": expected_alembic_heads()[0]}, + ) + session_factory = sessionmaker(bind=engine, expire_on_commit=False) + app = FastAPI() + app.include_router(observability_router, prefix="/api/v1") + + def override_get_db() -> Any: + db = session_factory() + try: + yield db + finally: + db.close() + + app.dependency_overrides[get_db] = override_get_db + client = TestClient(app) + try: + with Session(engine) as db: + ObservabilityService(db).record_heartbeat( + HeartbeatComponent.FEISHU_EVENTS, + "feishu-events-test", + status_value=HeartbeatStatus.DEGRADED, + ) + + disconnected = client.get("/api/v1/health/ready") + + assert disconnected.status_code == 503 + disconnected_check = disconnected.json()[ObservabilityKey.CHECKS][ + ObservabilityKey.FEISHU_EVENTS + ] + assert ( + disconnected_check[ObservabilityKey.STATUS] + == ObservabilityStatus.DEGRADED + ) + assert disconnected_check["reason"] == "heartbeat_degraded" + + with Session(engine) as db: + ObservabilityService(db).record_heartbeat( + HeartbeatComponent.FEISHU_EVENTS, + "feishu-events-test", + status_value=HeartbeatStatus.OK, + ) + + recovered = client.get("/api/v1/health/ready") + + assert recovered.status_code == 200 + recovered_check = recovered.json()[ObservabilityKey.CHECKS][ + ObservabilityKey.FEISHU_EVENTS + ] + assert recovered_check[ObservabilityKey.STATUS] == ObservabilityStatus.OK + assert recovered_check["reason"] is None + finally: + client.close() + engine.dispose() + get_settings.cache_clear() + + +@pytest.mark.parametrize( + ("configured_identities", "expected_reason"), + [ + ([""], "admin_identities_missing"), + (["not-an-identity"], "admin_identities_invalid"), + ], +) +def test_production_readiness_parses_admin_identities_without_exposing_them( + monkeypatch: pytest.MonkeyPatch, + configured_identities: list[str], + expected_reason: str, +) -> None: + sensitive_value = configured_identities[0] + settings = SimpleNamespace( + app_env="production", + feishu_user_features_enabled=True, + feishu_admin_identities=configured_identities, + feishu_event_transport="long_connection", + feishu_app_id="app-id", + feishu_app_secret="app-secret", + feishu_app_type="self", + feishu_default_tenant_key=None, + ) + monkeypatch.setattr( + "app.modules.observability.service.get_settings", + lambda: settings, + ) + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + try: + with Session(engine) as db: + result = ObservabilityService(db)._feishu_subscriptions_check() + + assert result[ObservabilityKey.STATUS] == ObservabilityStatus.DEGRADED + assert expected_reason in result["reasons"] + if sensitive_value: + assert sensitive_value not in str(result) + finally: + engine.dispose() diff --git a/tests/test_feishu_personalization_migration.py b/tests/test_feishu_personalization_migration.py index 00f018b..0184634 100644 --- a/tests/test_feishu_personalization_migration.py +++ b/tests/test_feishu_personalization_migration.py @@ -107,7 +107,7 @@ def test_personalization_migration_preserves_and_classifies_legacy_data( {"created_at": timestamp, "updated_at": timestamp}, ) - command.upgrade(config, "202607260005") + command.upgrade(config, "head") inspector = inspect(engine) expected_tables = { diff --git a/tests/test_feishu_subscription_commands.py b/tests/test_feishu_subscription_commands.py index 7dc9325..a439028 100644 --- a/tests/test_feishu_subscription_commands.py +++ b/tests/test_feishu_subscription_commands.py @@ -1,3 +1,4 @@ +import json from datetime import datetime from uuid import uuid4 @@ -5,9 +6,12 @@ import pytest from sqlalchemy import create_engine, func, select from sqlalchemy.orm import Session +from app.application.feishu.commands import FeishuCommandService from app.application.feishu.handlers.subscriptions import handle_subscription_command +from app.core.config import get_settings from app.core.database import Base -from app.modules.feishu.constants import FeishuCommandName +from app.modules.ai_agent.service import AIService +from app.modules.feishu.constants import FeishuCommandKey, FeishuCommandName from app.modules.feishu.service import FeishuService from app.modules.feishu_users.constants import FeishuUserRole from app.modules.feishu_users.models import FeishuUser @@ -91,6 +95,131 @@ def test_create_private_subscription_replies_with_normalized_plan() -> None: engine.dispose() +def test_rich_text_event_routes_subscription_instead_of_fallback_ai( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "true") + get_settings.cache_clear() + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + try: + with Session(engine) as db: + user = _user(db, suffix="rich-text-command") + principal = FeishuPrincipal.from_user( + user, + chat_id="private-chat", + chat_type="p2p", + ) + text = "订阅 每隔 15 分钟：给我一句简短的工作提醒" + payload = { + "header": {"tenant_key": user.tenant_key}, + "event": { + "message": { + "chat_id": "private-chat", + "chat_type": "p2p", + "content": json.dumps( + { + "content": [ + [ + { + "tag": "text", + "text": text, + "style": [], + } + ] + ] + }, + ensure_ascii=False, + ), + }, + "sender": {"sender_id": {"open_id": user.open_id}}, + }, + } + + commands = FeishuCommandService(db) + extracted = commands.extract_event_command(payload) + + assert extracted is not None + assert extracted[FeishuCommandKey.TEXT] == text + result = commands.handle_text( + extracted[FeishuCommandKey.TEXT], + chat_id="private-chat", + principal=principal, + auto_reply=False, + ) + subscription = db.scalar(select(PushSubscription)) + assert result["command"] == FeishuCommandName.SUBSCRIPTION_CREATE + assert "计划：每隔 15 分钟" in result["content"] + assert subscription is not None + assert subscription.prompt == "给我一句简短的工作提醒" + finally: + get_settings.cache_clear() + engine.dispose() + + +def test_rich_text_event_routes_help_instead_of_fallback_ai( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "true") + monkeypatch.setattr( + AIService, + "ask_personalized", + lambda *args, **kwargs: pytest.fail("Rich-text help command reached AI fallback"), + ) + get_settings.cache_clear() + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + try: + with Session(engine) as db: + user = _user(db, suffix="rich-text-help") + principal = FeishuPrincipal.from_user( + user, + chat_id="private-chat", + chat_type="p2p", + ) + payload = { + "header": {"tenant_key": user.tenant_key}, + "event": { + "message": { + "chat_id": "private-chat", + "chat_type": "p2p", + "content": json.dumps( + { + "content": [ + [ + { + "tag": "text", + "text": "帮助", + "style": [], + } + ] + ] + }, + ensure_ascii=False, + ), + }, + "sender": {"sender_id": {"open_id": user.open_id}}, + }, + } + + commands = FeishuCommandService(db) + extracted = commands.extract_event_command(payload) + + assert extracted is not None + assert extracted[FeishuCommandKey.TEXT] == "帮助" + result = commands.handle_text( + extracted[FeishuCommandKey.TEXT], + chat_id="private-chat", + principal=principal, + auto_reply=False, + ) + assert result["command"] == FeishuCommandName.HELP + assert "你可以使用：" in result["content"] + finally: + get_settings.cache_clear() + engine.dispose() + + def test_group_subscription_uses_current_verified_chat_and_requires_admin() -> None: engine = create_engine("sqlite://") Base.metadata.create_all(engine) diff --git a/tests/test_feishu_subscription_readiness.py b/tests/test_feishu_subscription_readiness.py index 2d803b2..d172c43 100644 --- a/tests/test_feishu_subscription_readiness.py +++ b/tests/test_feishu_subscription_readiness.py @@ -25,7 +25,8 @@ from app.modules.subscriptions.models import PushDelivery, PushSubscription @pytest.fixture(autouse=True) -def _reset_settings() -> None: +def _reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "true") get_settings.cache_clear() yield get_settings.cache_clear() @@ -248,7 +249,11 @@ def test_readiness_checks_credentials_for_processable_deliveries_without_active_ engine.dispose() -def test_readiness_ignores_terminal_deliveries_without_active_plan() -> None: +def test_readiness_ignores_terminal_deliveries_without_active_plan( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "false") + get_settings.cache_clear() engine = create_engine("sqlite://") Base.metadata.create_all(engine) try: diff --git a/tests/test_personal_data_erasure.py b/tests/test_personal_data_erasure.py index a34640e..e629cde 100644 --- a/tests/test_personal_data_erasure.py +++ b/tests/test_personal_data_erasure.py @@ -21,6 +21,7 @@ from app.modules.ai_memory.models import AIMemoryEntry from app.modules.audit.models import AuditLog from app.modules.business.models import MarketWatchlist from app.modules.feishu_users.constants import FeishuUserRole +from app.modules.feishu_users.identifiers import feishu_audit_identity_hash from app.modules.feishu_users.models import ( FeishuAdminBootstrapTombstone, FeishuUser, @@ -175,6 +176,26 @@ def _seed_personal_data(db: Session, user: FeishuUser) -> None: request_id=f"request-{user.code}", ) ) + db.add( + AuditLog( + actor=feishu_audit_identity_hash( + user.tenant_key, + user.open_id, + ), + source="feishu", + action="webhook_event", + target_type="webhook", + target_id="sha256-event-evidence", + request_payload=json.dumps( + { + "tenant_key": "sha256-tenant-evidence", + "chat_id": "sha256-chat-evidence", + } + ), + response_payload=json.dumps({"accepted": True}), + request_id=f"accepted-{user.code}", + ) + ) db.commit() @@ -224,6 +245,10 @@ def test_confirm_erases_all_personal_data_and_anonymizes_audit( tenant_key = user.tenant_key open_id = user.open_id identifiers = (user.code, user.open_id, user.union_id, user.user_id) + accepted_actor = feishu_audit_identity_hash( + user.tenant_key, + user.open_id, + ) principal = FeishuPrincipal.from_user(user) service = FeishuPersonalDataService(db) @@ -270,6 +295,7 @@ def test_confirm_erases_all_personal_data_and_anonymizes_audit( for log in logs ) assert all(identifier not in serialized_logs for identifier in identifiers) + assert accepted_actor not in serialized_logs final_log = db.execute( select(AuditLog).where(AuditLog.action == PERSONAL_DATA_ERASURE_ACTION) ).scalar_one() @@ -294,6 +320,18 @@ def test_confirm_erases_all_personal_data_and_anonymizes_audit( assert anonymized_history.request_id is None assert anonymized_history.status == "success" + anonymized_acceptance = db.execute( + select(AuditLog).where( + AuditLog.action == "webhook_event", + AuditLog.actor == result.anonymous_id, + ) + ).scalar_one() + assert anonymized_acceptance.target_type is None + assert anonymized_acceptance.target_id is None + assert anonymized_acceptance.request_payload is None + assert anonymized_acceptance.response_payload is None + assert anonymized_acceptance.request_id is None + recreated = FeishuIdentityService(db).resolve_or_register( tenant_key=tenant_key, open_id=open_id, diff --git a/tests/test_runtime_activation.py b/tests/test_runtime_activation.py new file mode 100644 index 0000000..b04a221 --- /dev/null +++ b/tests/test_runtime_activation.py @@ -0,0 +1,608 @@ +import shutil +import subprocess +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import StaticPool + +from app.application.scheduling import create_scheduler +from app.core.config import Settings, get_settings +from app.core.database import Base +from app.core.database.safety import validate_platform_migration_target +from app.core.utils.time import utc_now +from app.modules.events.constants import EventStatus +from app.modules.events.models import DomainEvent +from app.modules.feishu.constants import FeishuInboundStatus +from app.modules.feishu.models import FeishuEventReceipt +from app.modules.observability.constants import ( + HeartbeatComponent, + ObservabilityKey, + ObservabilityStatus, +) +from app.modules.observability.models import SystemHeartbeat +from app.modules.observability import runtime as observability_runtime +from app.modules.observability.service import ObservabilityService +from app.modules.workflows.constants import WorkflowStatus +from app.modules.workflows.models import WorkflowInstance +from app.tools import run_scheduler, runtime_preflight + + +def _production_settings(**overrides: object) -> Settings: + values: dict[str, object] = { + "app_env": "production", + "database_url": ( + "postgresql+psycopg://app:runtime-database-password-2026@db/app" + ), + "api_key": "runtime-service-key-2026-primary", + "audit_api_key": "runtime-audit-key-2026-independent", + "cors_origins": ["https://internal.example.com"], + "debug": False, + "mask_sensitive_responses": True, + "read_only_mode": True, + "feishu_app_id": None, + "feishu_app_secret": None, + "feishu_event_transport": "disabled", + "feishu_verification_token": None, + "feishu_user_features_enabled": False, + } + values.update(overrides) + return Settings(_env_file=None, **values) + + +def _stamp_alembic_head(engine: Engine) -> None: + expected = runtime_preflight.expected_alembic_heads() + assert len(expected) == 1 + with engine.begin() as connection: + connection.execute( + text("create table alembic_version (version_num varchar(32))") + ) + connection.execute( + text("insert into alembic_version (version_num) values (:revision)"), + {"revision": expected[0]}, + ) + + +def test_production_feishu_transport_fails_closed() -> None: + with pytest.raises(ValueError, match="FEISHU_EVENT_TRANSPORT"): + _production_settings( + feishu_user_features_enabled=True, + feishu_admin_identities=["tenant:open-id"], + ) + + with pytest.raises(ValueError, match="FEISHU_VERIFICATION_TOKEN"): + _production_settings( + feishu_event_transport="webhook", + feishu_app_id="app-id", + feishu_app_secret="feishu-app-secret-2026-secure-value", + ) + + with pytest.raises(ValueError, match="FEISHU_APP_ID"): + _production_settings(feishu_event_transport="long_connection") + + settings = _production_settings( + feishu_event_transport="long_connection", + feishu_app_id="app-id", + feishu_app_secret="feishu-app-secret-2026-secure-value", + feishu_user_features_enabled=True, + feishu_admin_identities=["tenant:open-id"], + ) + assert settings.feishu_event_transport == "long_connection" + + +def test_production_database_target_fails_closed() -> None: + with pytest.raises(ValueError, match="must use PostgreSQL"): + _production_settings( + database_url=( + "mysql+pymysql://app:runtime-database-password-2026@legacy/business" + ), + ) + + with pytest.raises(ValueError, match="must target different databases"): + _production_settings( + database_url=( + "postgresql+psycopg://platform:platform-password-2026-secure@db/platform" + ), + legacy_database_url=( + "postgresql://readonly:readonly-password-2026-secure@db:5432/platform" + ), + ) + + settings = _production_settings( + database_url=( + "postgresql+psycopg://platform:platform-password-2026-secure@db/platform" + ), + legacy_database_url=( + "mysql+pymysql://readonly:readonly-password-2026-secure@legacy/business" + ), + ) + assert settings.database_url.startswith("postgresql") + + +def test_production_placeholders_and_invalid_admin_identities_fail_closed() -> None: + placeholder_key = "replace-with-a-random-service-key" + with pytest.raises(ValueError, match="API_KEY/API_KEYS") as error: + _production_settings(api_key=placeholder_key) + assert placeholder_key not in str(error.value) + with pytest.raises(ValueError, match="DATABASE_URL password"): + _production_settings( + database_url="postgresql+psycopg://app:change-me@db/app", + ) + with pytest.raises(ValueError, match="tenant_key:open_id"): + _production_settings( + feishu_event_transport="long_connection", + feishu_app_id="app-id", + feishu_app_secret="feishu-app-secret-2026-secure-value", + feishu_user_features_enabled=True, + feishu_admin_identities=["not-a-valid-identity"], + ) + + +def test_platform_migration_target_never_accepts_mysql_or_legacy_database() -> None: + with pytest.raises(RuntimeError, match="PostgreSQL or local SQLite"): + validate_platform_migration_target( + "mysql+pymysql://platform:secret@legacy/business", + None, + ) + with pytest.raises(RuntimeError, match="matches LEGACY_DATABASE_URL"): + validate_platform_migration_target( + "postgresql+psycopg://platform:one@db/platform", + "postgresql://readonly:two@db:5432/platform", + ) + + +def test_environment_overrides_dotenv( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + base = tmp_path / ".env" + base.write_text("SCHEDULER_ENABLED=true\n", encoding="utf-8") + monkeypatch.delenv("SCHEDULER_ENABLED", raising=False) + + assert Settings(_env_file=base).scheduler_enabled is True + + monkeypatch.setenv("SCHEDULER_ENABLED", "false") + assert Settings(_env_file=base).scheduler_enabled is False + + +def test_scheduler_process_refuses_disabled_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + run_scheduler, + "get_settings", + lambda: SimpleNamespace(scheduler_enabled=False), + ) + monkeypatch.setattr( + run_scheduler, + "create_scheduler", + lambda: pytest.fail("disabled scheduler must not be created"), + ) + + with pytest.raises(RuntimeError, match="SCHEDULER_ENABLED"): + run_scheduler.main() + + +def test_runtime_preflight_requires_exact_alembic_head(tmp_path: Path) -> None: + database_path = tmp_path / "runtime-preflight.db" + database_url = f"sqlite:///{database_path.as_posix()}" + settings = Settings( + _env_file=None, + database_url=database_url, + feishu_event_transport="disabled", + ) + expected = runtime_preflight.expected_alembic_heads() + assert len(expected) == 1 + engine = create_engine(database_url) + try: + with engine.begin() as connection: + connection.execute( + text("create table alembic_version (version_num varchar(32))") + ) + connection.execute( + text("insert into alembic_version (version_num) values ('old')") + ) + with pytest.raises( + runtime_preflight.RuntimePreflightError, + match="current Alembic head", + ): + runtime_preflight.run_preflight(settings) + + with engine.begin() as connection: + connection.execute(text("delete from alembic_version")) + connection.execute( + text( + "insert into alembic_version (version_num) values (:revision)" + ), + {"revision": expected[0]}, + ) + result = runtime_preflight.run_preflight(settings) + assert result.revisions == expected + finally: + engine.dispose() + + +def test_scheduler_registers_immediate_runtime_heartbeats( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("TASK_QUEUE_ENABLED", "true") + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "false") + get_settings.cache_clear() + try: + scheduler = create_scheduler() + + assert scheduler.get_job("scheduler_heartbeat").next_run_time is not None + assert scheduler.get_job("worker_heartbeat").next_run_time is not None + assert scheduler.get_job("event_dispatch").next_run_time is not None + assert scheduler.get_job("feishu_inbound_event_cycle") is not None + finally: + get_settings.cache_clear() + + +def test_required_component_heartbeats_control_readiness( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "true") + monkeypatch.setenv("FEISHU_ADMIN_IDENTITIES", "") + monkeypatch.setenv("FEISHU_EVENT_TRANSPORT", "long_connection") + monkeypatch.setenv("FEISHU_APP_ID", "app-id") + monkeypatch.setenv("FEISHU_APP_SECRET", "app-secret") + monkeypatch.setenv("TASK_QUEUE_ENABLED", "false") + get_settings.cache_clear() + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + _stamp_alembic_head(engine) + try: + with Session(engine) as db: + service = ObservabilityService(db) + missing = service.ready() + assert missing[ObservabilityKey.STATUS] == ObservabilityStatus.DEGRADED + assert missing[ObservabilityKey.CHECKS][ObservabilityKey.SCHEDULER][ + ObservabilityKey.STATUS + ] == ObservabilityStatus.DEGRADED + assert missing[ObservabilityKey.CHECKS][ObservabilityKey.FEISHU_EVENTS][ + ObservabilityKey.STATUS + ] == ObservabilityStatus.DEGRADED + + service.record_heartbeat( + HeartbeatComponent.SCHEDULER, + "scheduler-test", + ) + service.record_heartbeat( + HeartbeatComponent.FEISHU_EVENTS, + "feishu-events-test", + ) + ready = service.ready() + assert ready[ObservabilityKey.STATUS] == ObservabilityStatus.OK + + heartbeat = db.query(SystemHeartbeat).filter_by( + component=HeartbeatComponent.SCHEDULER, + ).one() + heartbeat.last_seen_at = utc_now() - timedelta(minutes=10) + db.commit() + stale = service.ready() + assert stale[ObservabilityKey.CHECKS][ObservabilityKey.SCHEDULER][ + ObservabilityKey.STATUS + ] == ObservabilityStatus.DEGRADED + finally: + get_settings.cache_clear() + engine.dispose() + + +def test_api_lifecycle_records_immediate_heartbeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + monkeypatch.setattr(observability_runtime, "SessionLocal", factory) + monkeypatch.setattr( + observability_runtime, + "get_settings", + lambda: SimpleNamespace(heartbeat_interval_seconds=3600), + ) + app = FastAPI() + observability_runtime.attach_api_heartbeat(app) + + try: + with TestClient(app): + thread = app.state.api_heartbeat_thread + assert thread.is_alive() + with Session(engine) as db: + heartbeat = db.query(SystemHeartbeat).filter_by( + component=HeartbeatComponent.API, + instance_id=app.state.api_heartbeat_instance_id, + ).one() + assert heartbeat.status == ObservabilityStatus.OK + + assert not thread.is_alive() + finally: + engine.dispose() + + +def test_production_readiness_requires_api_heartbeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = SimpleNamespace( + app_env="production", + scheduler_enabled=False, + task_queue_enabled=False, + feishu_user_features_enabled=False, + feishu_event_transport="disabled", + event_dispatch_enabled=True, + heartbeat_interval_seconds=60, + heartbeat_retention_seconds=86400, + ) + monkeypatch.setattr( + "app.modules.observability.service.get_settings", + lambda: settings, + ) + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + _stamp_alembic_head(engine) + try: + with Session(engine) as db: + service = ObservabilityService(db) + service.record_heartbeat( + HeartbeatComponent.SCHEDULER, + "scheduler-test", + ) + + missing = service.ready() + assert missing[ObservabilityKey.CHECKS][ObservabilityKey.API][ + ObservabilityKey.STATUS + ] == ObservabilityStatus.DEGRADED + assert missing[ObservabilityKey.CHECKS][ObservabilityKey.API][ + "reason" + ] == "heartbeat_missing" + + service.record_heartbeat( + HeartbeatComponent.API, + "api-test", + ) + ready = service.ready() + assert ready[ObservabilityKey.STATUS] == ObservabilityStatus.OK + assert ready[ObservabilityKey.CHECKS][ObservabilityKey.API][ + ObservabilityKey.STATUS + ] == ObservabilityStatus.OK + finally: + engine.dispose() + + +def test_production_schema_readiness_requires_alembic_head( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "app.modules.observability.service.get_settings", + lambda: SimpleNamespace(app_env="production"), + ) + engine = create_engine("sqlite://") + try: + with Session(engine) as db: + db.execute(text("create table alembic_version (version_num varchar(32))")) + db.execute( + text("insert into alembic_version (version_num) values ('old-revision')") + ) + db.commit() + service = ObservabilityService(db) + mismatch = service._schema_check() + assert mismatch[ObservabilityKey.STATUS] == ObservabilityStatus.DEGRADED + expected = mismatch["expected"] + assert len(expected) == 1 + + db.execute(text("delete from alembic_version")) + db.execute( + text("insert into alembic_version (version_num) values (:revision)"), + {"revision": expected[0]}, + ) + db.commit() + assert service._schema_check()[ObservabilityKey.STATUS] == ObservabilityStatus.OK + finally: + engine.dispose() + + +def test_historical_terminal_failures_do_not_block_readiness() -> None: + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + try: + with Session(engine) as db: + db.add( + DomainEvent( + event_id="terminal-event", + event_type="test.terminal", + aggregate_type="test", + status=EventStatus.FAILED, + max_attempts=1, + ) + ) + db.add( + WorkflowInstance( + code="WF-TERMINAL", + workflow_type="test", + aggregate_type="test", + status=WorkflowStatus.FAILED, + ) + ) + db.commit() + + service = ObservabilityService(db) + events = service._events_check() + workflows = service._workflows_check() + assert events[ObservabilityKey.STATUS] == ObservabilityStatus.OK + assert events["failed"] == 1 + assert workflows[ObservabilityKey.STATUS] == ObservabilityStatus.OK + assert workflows["failed"] == 1 + finally: + engine.dispose() + + +def test_feishu_inbound_metrics_expose_counts_without_payloads() -> None: + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + try: + with Session(engine) as db: + for index, status_value in enumerate( + ( + FeishuInboundStatus.PENDING, + FeishuInboundStatus.RETRY, + FeishuInboundStatus.PROCESSING, + FeishuInboundStatus.FAILED, + ) + ): + db.add( + FeishuEventReceipt( + event_key=f"metric-event-{index}", + source="webhook", + status=status_value, + payload={"text": "must not appear in metrics"}, + ) + ) + db.commit() + + metrics = ObservabilityService(db)._feishu_inbound_metrics() + assert metrics == { + "pending": 1, + "retry": 1, + "processing": 1, + "failed": 1, + "reply_pending": 0, + "reply_retry": 0, + "reply_processing": 0, + "reply_failed": 0, + } + assert "payload" not in metrics + assert "must not appear" not in str(metrics) + finally: + engine.dispose() + + +def test_runtime_deployment_uses_single_dotenv_file() -> None: + compose = Path("docker-compose.yml").read_text(encoding="utf-8") + external_compose = Path("docker-compose.external-db.yml").read_text( + encoding="utf-8" + ) + gitignore = Path(".gitignore").read_text(encoding="utf-8") + start_script = Path("scripts/start_runtime.ps1").read_text(encoding="utf-8") + + assert "feishu-events:" in compose + assert 'profiles: ["long-connection"]' in compose + assert "/api/v1/health/ready" in compose + assert "/api/v1/health/live" not in compose + assert "${COMPOSE_DATABASE_URL:-postgresql+psycopg://" in compose + assert "@db:5432/" in compose + assert "${DATABASE_URL:" not in compose + assert compose.count("APP_ENV: production") == 5 + assert 'profiles: ["internal-database"]' in external_compose + assert external_compose.count("depends_on: !override") == 5 + assert external_compose.count( + "${COMPOSE_DATABASE_URL:?COMPOSE_DATABASE_URL is required " + "for external PostgreSQL mode}" + ) == 5 + assert "${POSTGRES_PASSWORD" not in external_compose + assert compose.count("restart: unless-stopped") >= 6 + assert compose.count("path: .env") == 1 + assert compose.count("required: true") == 1 + assert ".env.runtime" not in compose + assert ".env" in gitignore.splitlines() + assert ".env.runtime" not in gitignore.splitlines() + assert "-WindowStyle Hidden" in start_script + assert "TASK_QUEUE_ENABLED = \"false\"" in start_script + assert "app.tools.runtime_preflight" in start_script + assert "/api/v1/health/ready" in start_script + + +def test_default_and_external_compose_configs_validate_without_project_dotenv( + tmp_path: Path, +) -> None: + docker = shutil.which("docker") + if docker is None: + pytest.skip("Docker CLI is not installed") + + base_compose = tmp_path / "docker-compose.yml" + external_compose = tmp_path / "docker-compose.external-db.yml" + shutil.copyfile("docker-compose.yml", base_compose) + shutil.copyfile("docker-compose.external-db.yml", external_compose) + (tmp_path / ".env").write_text( + "\n".join( + ( + "API_KEY=config-test-service-key", + "AUDIT_API_KEY=config-test-audit-key", + "CORS_ORIGINS=[]", + ) + ) + + "\n", + encoding="utf-8", + ) + + default_interpolation = tmp_path / "default.compose.env" + default_interpolation.write_text( + "POSTGRES_PASSWORD=config-test-database-password\n", + encoding="utf-8", + ) + default_command = [ + docker, + "compose", + "--project-directory", + str(tmp_path), + "--env-file", + str(default_interpolation), + "-f", + str(base_compose), + ] + _run_compose_config([*default_command, "config", "--quiet"], tmp_path) + default_services = _run_compose_config( + [*default_command, "config", "--services"], + tmp_path, + ) + assert "db" in default_services.splitlines() + + external_interpolation = tmp_path / "external.compose.env" + external_interpolation.write_text( + ( + "COMPOSE_DATABASE_URL=" + "postgresql+psycopg://config-user:config-password@" + "external.invalid:5432/platform\n" + ), + encoding="utf-8", + ) + external_command = [ + docker, + "compose", + "--project-directory", + str(tmp_path), + "--env-file", + str(external_interpolation), + "-f", + str(base_compose), + "-f", + str(external_compose), + ] + _run_compose_config([*external_command, "config", "--quiet"], tmp_path) + external_services = _run_compose_config( + [*external_command, "config", "--services"], + tmp_path, + ) + assert "db" not in external_services.splitlines() + + +def _run_compose_config(command: list[str], cwd: Path) -> str: + result = subprocess.run( + command, + cwd=cwd, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + return result.stdout diff --git a/tests/test_schema_reconciliation.py b/tests/test_schema_reconciliation.py new file mode 100644 index 0000000..7b3e759 --- /dev/null +++ b/tests/test_schema_reconciliation.py @@ -0,0 +1,626 @@ +import json +from pathlib import Path +from types import SimpleNamespace + +from alembic import command +from alembic.config import Config +import pytest +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.dialects import postgresql, sqlite +from sqlalchemy.schema import CreateTable + +from app.core.config import get_settings +from app.modules.ai_memory.models import AIMemoryEntry +from app.modules.business.models import MarketWatchlist +from app.tools import reconcile_platform_schema as schema_tool + + +def _database( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + name: str, + revision: str, +): + database_path = tmp_path / f"{name}.db" + database_url = f"sqlite:///{database_path.as_posix()}" + monkeypatch.setenv("DATABASE_URL", database_url) + get_settings.cache_clear() + config = Config("alembic.ini") + command.upgrade(config, revision) + return create_engine(database_url), config + + +def _remove_revision(engine) -> None: + with engine.begin() as connection: + connection.execute(text("DELETE FROM alembic_version")) + + +def test_remote_drift_allowlist_snapshot_is_explicit() -> None: + operation_counts: dict[str, int] = {} + for item in schema_tool.ALLOWED_DRIFT: + operation_counts[item.operation] = operation_counts.get(item.operation, 0) + 1 + + assert operation_counts == { + "add_column": 61, + "add_constraint": 2, + "add_fk": 2, + "add_index": 47, + "add_table": 2, + "modify_default": 8, + "modify_nullable": 1, + "remove_constraint": 3, + "remove_index": 9, + "remove_table": 1, + } + + clean_005_only = { + schema_tool.DriftKey( + "remove_constraint", + "ai_memory_entries", + "uq_ai_memory_owner_fingerprint", + ), + schema_tool.DriftKey( + "remove_constraint", + "market_watchlists", + "uq_market_watchlist_owner_symbol", + ), + schema_tool.DriftKey( + "modify_nullable", + "work_tasks", + "source_system", + ), + *{ + schema_tool.DriftKey("modify_default", table_name, column_name) + for table_name, columns in schema_tool._MODIFY_DEFAULTS.items() + for column_name in columns + }, + } + assert len(schema_tool.ALLOWED_DRIFT - clean_005_only) == 125 + + +@pytest.mark.parametrize( + ("table", "constraint_name"), + [ + (AIMemoryEntry.__table__, "uq_ai_memory_owner_fingerprint"), + (MarketWatchlist.__table__, "uq_market_watchlist_owner_symbol"), + ], +) +def test_owner_unique_constraints_use_postgres_nulls_not_distinct( + table, + constraint_name: str, +) -> None: + constraint = next( + item for item in table.constraints if item.name == constraint_name + ) + + assert ( + constraint.dialect_options["postgresql"]["nulls_not_distinct"] is True + ) + postgres_ddl = str(CreateTable(table).compile(dialect=postgresql.dialect())) + sqlite_ddl = str(CreateTable(table).compile(dialect=sqlite.dialect())) + assert "UNIQUE NULLS NOT DISTINCT" in postgres_ddl + assert "NULLS NOT DISTINCT" not in sqlite_ddl + + +def test_clean_005_upgrades_to_head_with_zero_metadata_drift( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, config = _database( + tmp_path, + monkeypatch, + "clean-upgrade", + schema_tool.PREVIOUS_REVISION, + ) + try: + command.upgrade(config, "head") + with engine.connect() as connection: + audit = schema_tool.audit_connection(connection) + + assert audit.revision_rows == (schema_tool.TARGET_REVISION,) + assert audit.observed_drift == frozenset() + assert "approval_requests" not in inspect(engine).get_table_names() + command.check(config) + finally: + engine.dispose() + get_settings.cache_clear() + + +def test_baseline_stamps_and_upgrades_in_one_external_transaction( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, _ = _database( + tmp_path, + monkeypatch, + "baseline-success", + schema_tool.PREVIOUS_REVISION, + ) + try: + with engine.begin() as connection: + connection.execute( + text("CREATE TABLE approval_requests (id INTEGER PRIMARY KEY)") + ) + _remove_revision(engine) + before = schema_tool.audit_engine(engine) + + assert before.eligible is True + assert schema_tool.DriftKey( + "remove_table", + "approval_requests", + "", + ) in before.observed_drift + + after = schema_tool.apply_baseline( + engine, + before.fingerprint, + require_postgresql=False, + ) + + assert after.revision_rows == (schema_tool.TARGET_REVISION,) + assert after.observed_drift == frozenset() + assert "approval_requests" not in inspect(engine).get_table_names() + finally: + engine.dispose() + get_settings.cache_clear() + + +def test_baseline_failure_rolls_back_atomic_stamp( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, _ = _database( + tmp_path, + monkeypatch, + "baseline-rollback", + schema_tool.PREVIOUS_REVISION, + ) + try: + _remove_revision(engine) + before = schema_tool.audit_engine(engine) + + def fail_upgrade(*_args, **_kwargs) -> None: + raise RuntimeError("injected migration failure") + + monkeypatch.setattr(schema_tool.command, "upgrade", fail_upgrade) + with pytest.raises(RuntimeError, match="injected migration failure"): + schema_tool.apply_baseline( + engine, + before.fingerprint, + require_postgresql=False, + ) + + with engine.connect() as connection: + revisions = tuple( + connection.execute( + text("SELECT version_num FROM alembic_version") + ).scalars() + ) + assert revisions == () + finally: + engine.dispose() + get_settings.cache_clear() + + +def test_baseline_rejects_changed_fingerprint_before_stamp( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, _ = _database( + tmp_path, + monkeypatch, + "baseline-fingerprint", + schema_tool.PREVIOUS_REVISION, + ) + try: + _remove_revision(engine) + with pytest.raises(schema_tool.ReconciliationError, match="fingerprint changed"): + schema_tool.apply_baseline( + engine, + "0" * 64, + require_postgresql=False, + ) + with engine.connect() as connection: + assert schema_tool._revision_rows(connection) == () + finally: + engine.dispose() + get_settings.cache_clear() + + +def test_preflight_fingerprint_includes_full_server_default_expression() -> None: + engine = create_engine("sqlite://") + try: + with engine.begin() as connection: + connection.execute( + text( + """ + CREATE TABLE projects ( + id INTEGER PRIMARY KEY, + lifecycle_state TEXT DEFAULT 'draft' + ) + """ + ) + ) + with engine.connect() as connection: + first = schema_tool._schema_fingerprint(connection) + + with engine.begin() as connection: + connection.execute(text("DROP TABLE projects")) + connection.execute( + text( + """ + CREATE TABLE projects ( + id INTEGER PRIMARY KEY, + lifecycle_state TEXT DEFAULT 'active' + ) + """ + ) + ) + with engine.connect() as connection: + second = schema_tool._schema_fingerprint(connection) + + assert first != second + finally: + engine.dispose() + + +def test_preflight_fingerprint_binds_impacted_row_counts_without_data() -> None: + engine = create_engine("sqlite://") + private_value = "must-not-appear-in-reconciliation-output" + try: + with engine.begin() as connection: + connection.execute( + text( + """ + CREATE TABLE approval_requests ( + id INTEGER PRIMARY KEY, + private_payload TEXT + ) + """ + ) + ) + before = schema_tool.audit_engine(engine) + + with engine.begin() as connection: + connection.execute( + text( + """ + INSERT INTO approval_requests (id, private_payload) + VALUES (1, :private_payload) + """ + ), + {"private_payload": private_value}, + ) + after = schema_tool.audit_engine(engine) + + assert before.fingerprint != after.fingerprint + assert dict(before.impacted_table_row_counts) == {"approval_requests": 0} + assert dict(after.impacted_table_row_counts) == {"approval_requests": 1} + public_payload = after.public_dict() + assert public_payload["impacted_table_row_counts"] == { + "approval_requests": 1 + } + assert "impacted_table_content_digests" not in public_payload + assert private_value not in json.dumps(public_payload) + finally: + engine.dispose() + + +def test_content_fingerprint_is_order_stable_and_detects_same_count_change() -> None: + engine = create_engine("sqlite://") + try: + with engine.begin() as connection: + connection.execute( + text( + """ + CREATE TABLE projects ( + project_code TEXT, + private_payload TEXT + ) + """ + ) + ) + connection.execute( + text( + """ + INSERT INTO projects (project_code, private_payload) + VALUES ('P-1', 'first'), ('P-2', 'second') + """ + ) + ) + with engine.connect() as connection: + first = schema_tool._schema_fingerprint(connection) + + with engine.begin() as connection: + connection.execute(text("DELETE FROM projects")) + connection.execute( + text( + """ + INSERT INTO projects (project_code, private_payload) + VALUES ('P-2', 'second'), ('P-1', 'first') + """ + ) + ) + with engine.connect() as connection: + reordered = schema_tool._schema_fingerprint(connection) + + with engine.begin() as connection: + connection.execute( + text( + """ + UPDATE projects + SET private_payload = 'changed' + WHERE project_code = 'P-1' + """ + ) + ) + with engine.connect() as connection: + changed = schema_tool._schema_fingerprint(connection) + + assert reordered == first + assert changed != first + finally: + engine.dispose() + + +def test_apply_rejects_same_row_count_content_change_after_dry_run( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, _ = _database( + tmp_path, + monkeypatch, + "baseline-content-fingerprint", + schema_tool.PREVIOUS_REVISION, + ) + try: + with engine.begin() as connection: + connection.execute( + text( + """ + INSERT INTO feishu_event_receipts ( + event_key, + source, + event_id, + message_id, + received_at + ) + VALUES ( + 'legacy-content-key', + 'webhook', + 'event-before', + 'message-stable', + '2026-07-27 00:00:00' + ) + """ + ) + ) + _remove_revision(engine) + before = schema_tool.audit_engine(engine) + + with engine.begin() as connection: + connection.execute( + text( + """ + UPDATE feishu_event_receipts + SET event_id = 'event-after' + WHERE event_key = 'legacy-content-key' + """ + ) + ) + changed = schema_tool.audit_engine(engine) + + assert changed.impacted_table_row_counts == before.impacted_table_row_counts + assert ( + changed.impacted_table_content_digests + != before.impacted_table_content_digests + ) + assert changed.fingerprint != before.fingerprint + with pytest.raises(schema_tool.ReconciliationError, match="fingerprint changed"): + schema_tool.apply_baseline( + engine, + before.fingerprint, + require_postgresql=False, + ) + with engine.connect() as connection: + assert schema_tool._revision_rows(connection) == () + finally: + engine.dispose() + get_settings.cache_clear() + + +@pytest.mark.parametrize( + ("database_url", "legacy_database_url"), + [ + ("sqlite:///local-platform.db", None), + ( + "postgresql://platform@database/platform", + "postgresql+psycopg://legacy@database:5432/platform", + ), + ], +) +def test_cli_dry_run_rejects_unsafe_target_before_engine_creation( + database_url: str, + legacy_database_url: str | None, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + engine_creation_count = 0 + + def record_engine_creation(*_args, **_kwargs): + nonlocal engine_creation_count + engine_creation_count += 1 + raise AssertionError("unsafe target reached create_engine") + + monkeypatch.setattr( + schema_tool, + "get_settings", + lambda: SimpleNamespace( + database_url=database_url, + legacy_database_url=legacy_database_url, + ), + ) + monkeypatch.setattr(schema_tool, "create_engine", record_engine_creation) + monkeypatch.setattr("sys.argv", ["reconcile_platform_schema"]) + + with pytest.raises(SystemExit) as exit_info: + schema_tool.main() + + assert exit_info.value.code == 2 + assert engine_creation_count == 0 + output = json.loads(capsys.readouterr().out) + assert output["applied"] is False + assert "error" in output + assert database_url not in output["error"] + assert legacy_database_url is None or legacy_database_url not in output["error"] + + +def test_postgres_below_15_is_ineligible_and_apply_precondition_rejects() -> None: + audit = schema_tool.BaselineAudit( + dialect="postgresql", + fingerprint="a" * 64, + impacted_table_row_counts=(), + impacted_table_content_digests=(), + postgresql_server_version_num=140012, + postgresql_version_supported=False, + revision_rows=(), + observed_drift=frozenset(), + unexpected_drift=frozenset(), + approval_rows=None, + approval_inbound_foreign_keys=0, + schema_privileges_ok=True, + table_ownership_ok=True, + ) + + assert audit.eligible is False + assert audit.public_dict()["postgresql_server_version_num"] == 140012 + assert audit.public_dict()["postgresql_version_supported"] is False + with pytest.raises(schema_tool.ReconciliationError, match="PostgreSQL 15"): + schema_tool._validate_apply_preconditions(audit, audit.fingerprint) + + +def test_nonempty_approval_table_is_never_removed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, _ = _database( + tmp_path, + monkeypatch, + "baseline-nonempty-approval", + schema_tool.PREVIOUS_REVISION, + ) + try: + with engine.begin() as connection: + connection.execute( + text("CREATE TABLE approval_requests (id INTEGER PRIMARY KEY)") + ) + connection.execute(text("INSERT INTO approval_requests (id) VALUES (1)")) + _remove_revision(engine) + before = schema_tool.audit_engine(engine) + + assert before.eligible is False + assert before.approval_rows == 1 + with pytest.raises(schema_tool.ReconciliationError, match="not empty"): + schema_tool.apply_baseline( + engine, + before.fingerprint, + require_postgresql=False, + ) + with engine.connect() as connection: + assert connection.scalar( + text("SELECT COUNT(*) FROM approval_requests") + ) == 1 + assert schema_tool._revision_rows(connection) == () + finally: + engine.dispose() + get_settings.cache_clear() + + +def test_reconciliation_downgrade_is_fail_closed_and_irreversible( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, config = _database( + tmp_path, + monkeypatch, + "irreversible-downgrade", + "202607270001", + ) + try: + with engine.connect() as connection: + before = schema_tool._schema_fingerprint(connection) + + with pytest.raises(RuntimeError, match="irreversible"): + command.downgrade(config, schema_tool.PREVIOUS_REVISION) + + with engine.connect() as connection: + after = schema_tool._schema_fingerprint(connection) + revisions = schema_tool._revision_rows(connection) + assert after == before + assert revisions == ("202607270001",) + finally: + engine.dispose() + get_settings.cache_clear() + + +def test_reconciliation_backfills_required_legacy_values( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine, config = _database( + tmp_path, + monkeypatch, + "required-backfill", + schema_tool.PREVIOUS_REVISION, + ) + try: + with engine.begin() as connection: + connection.execute( + text("ALTER TABLE work_tasks DROP COLUMN source_system") + ) + connection.execute( + text( + """ + INSERT INTO work_tasks ( + code, + title, + status, + priority, + created_at, + updated_at + ) + VALUES ( + 'legacy-task', + 'Legacy task', + 'todo', + 'P2', + '2026-07-27 00:00:00', + '2026-07-27 00:00:00' + ) + """ + ) + ) + + command.upgrade(config, "head") + + with engine.connect() as connection: + source_system = connection.scalar( + text( + """ + SELECT source_system + FROM work_tasks + WHERE code = 'legacy-task' + """ + ) + ) + columns = { + column["name"]: column + for column in inspect(engine).get_columns("work_tasks") + } + assert source_system == "internal" + assert columns["source_system"]["nullable"] is False + assert columns["source_system"]["default"] is None + assert columns["is_active"]["default"] is None + finally: + engine.dispose() + get_settings.cache_clear() diff --git a/tests/test_schema_reconciliation_postgres.py b/tests/test_schema_reconciliation_postgres.py new file mode 100644 index 0000000..1a319fb --- /dev/null +++ b/tests/test_schema_reconciliation_postgres.py @@ -0,0 +1,281 @@ +import os +import re +from collections.abc import Iterator +from uuid import uuid4 + +from alembic import command +import pytest +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.engine import Engine, URL, make_url +from sqlalchemy.exc import ArgumentError, DBAPIError +from sqlalchemy.pool import NullPool + +from app.core.config import get_settings +from app.tools import reconcile_platform_schema as schema_tool + + +TEST_DATABASE_ENV = "TEST_POSTGRES_DATABASE_URL" +TEST_SCHEMA_PATTERN = re.compile(r"caip_test_[0-9a-f]{32}\Z") + +pytestmark = pytest.mark.skipif( + not os.getenv(TEST_DATABASE_ENV, "").strip(), + reason=f"{TEST_DATABASE_ENV} is not configured", +) + + +def _configured_postgres_url() -> URL: + raw_url = os.getenv(TEST_DATABASE_ENV, "").strip() + if not raw_url: + pytest.skip(f"{TEST_DATABASE_ENV} is not configured") + try: + url = make_url(raw_url) + except (ArgumentError, TypeError, ValueError): + pytest.fail(f"{TEST_DATABASE_ENV} is not a valid SQLAlchemy URL") + if url.get_backend_name() != "postgresql": + pytest.fail(f"{TEST_DATABASE_ENV} must use PostgreSQL") + return url + + +def _quoted_schema(engine: Engine, schema_name: str) -> str: + if not TEST_SCHEMA_PATTERN.fullmatch(schema_name): + raise RuntimeError("Refusing unsafe PostgreSQL test schema name") + return engine.dialect.identifier_preparer.quote_identifier(schema_name) + + +@pytest.fixture +def postgres_schema_engine( + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[Engine]: + base_url = _configured_postgres_url() + schema_name = f"caip_test_{uuid4().hex}" + admin_engine = create_engine(base_url, poolclass=NullPool) + isolated_engine: Engine | None = None + schema_created = False + try: + quoted_schema = _quoted_schema(admin_engine, schema_name) + with admin_engine.begin() as connection: + connection.exec_driver_sql(f"CREATE SCHEMA {quoted_schema}") + schema_created = True + + isolated_url = base_url.update_query_dict( + {"options": f"-csearch_path={schema_name}"} + ) + isolated_engine = create_engine(isolated_url, poolclass=NullPool) + monkeypatch.setenv( + "DATABASE_URL", + isolated_url.render_as_string(hide_password=False), + ) + get_settings.cache_clear() + with isolated_engine.connect() as connection: + assert connection.scalar(text("SELECT current_schema()")) == schema_name + yield isolated_engine + finally: + if isolated_engine is not None: + isolated_engine.dispose() + get_settings.cache_clear() + if schema_created: + quoted_schema = _quoted_schema(admin_engine, schema_name) + with admin_engine.begin() as connection: + connection.exec_driver_sql( + f"DROP SCHEMA IF EXISTS {quoted_schema} CASCADE" + ) + admin_engine.dispose() + + +def _upgrade_to_previous(engine: Engine) -> None: + with engine.begin() as connection: + command.upgrade( + schema_tool._alembic_config(connection), + schema_tool.PREVIOUS_REVISION, + ) + + +def _prepare_mixed_unversioned_baseline(engine: Engine) -> None: + _upgrade_to_previous(engine) + with engine.begin() as connection: + connection.execute( + text("ALTER TABLE projects DROP COLUMN department_code") + ) + connection.execute( + text("CREATE TABLE approval_requests (id INTEGER PRIMARY KEY)") + ) + connection.execute(text("DELETE FROM alembic_version")) + + +def test_postgres_mixed_baseline_applies_to_head_without_metadata_drift( + postgres_schema_engine: Engine, +) -> None: + _prepare_mixed_unversioned_baseline(postgres_schema_engine) + before = schema_tool.audit_engine(postgres_schema_engine) + + assert before.eligible is True + assert ( + before.postgresql_server_version_num + and before.postgresql_server_version_num + >= schema_tool.MINIMUM_POSTGRESQL_VERSION_NUM + ) + assert before.postgresql_version_supported is True + assert before.revision_rows == () + assert schema_tool.DriftKey( + "add_column", + "projects", + "department_code", + ) in before.observed_drift + assert schema_tool.DriftKey( + "remove_table", + schema_tool.APPROVAL_TABLE, + "", + ) in before.observed_drift + + after = schema_tool.apply_baseline( + postgres_schema_engine, + before.fingerprint, + ) + + assert after.revision_rows == (schema_tool.TARGET_REVISION,) + assert after.observed_drift == frozenset() + assert after.unexpected_drift == frozenset() + assert schema_tool.APPROVAL_TABLE not in inspect( + postgres_schema_engine + ).get_table_names() + + +def test_postgres_upgrade_failure_rolls_back_ddl_and_stamp( + postgres_schema_engine: Engine, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _upgrade_to_previous(postgres_schema_engine) + with postgres_schema_engine.begin() as connection: + connection.execute(text("DELETE FROM alembic_version")) + before = schema_tool.audit_engine(postgres_schema_engine) + + def fail_upgrade(config, _revision: str) -> None: + connection = config.attributes["connection"] + connection.execute( + text("CREATE TABLE injected_upgrade_artifact (id INTEGER)") + ) + connection.execute( + text("ALTER TABLE projects ADD COLUMN injected_upgrade_marker TEXT") + ) + raise RuntimeError("injected PostgreSQL migration failure") + + monkeypatch.setattr(schema_tool.command, "upgrade", fail_upgrade) + with pytest.raises( + RuntimeError, + match="injected PostgreSQL migration failure", + ): + schema_tool.apply_baseline( + postgres_schema_engine, + before.fingerprint, + ) + + with postgres_schema_engine.connect() as connection: + after = schema_tool.audit_connection(connection) + table_names = inspect(connection).get_table_names() + project_columns = { + column["name"] + for column in inspect(connection).get_columns("projects") + } + assert after.fingerprint == before.fingerprint + assert after.revision_rows == () + assert "injected_upgrade_artifact" not in table_names + assert "injected_upgrade_marker" not in project_columns + + +def test_postgres_apply_rejects_same_count_content_change( + postgres_schema_engine: Engine, +) -> None: + _upgrade_to_previous(postgres_schema_engine) + with postgres_schema_engine.begin() as connection: + connection.execute( + text( + """ + INSERT INTO feishu_event_receipts ( + event_key, + source, + event_id, + message_id, + received_at + ) + VALUES ( + 'postgres-content-key', + 'webhook', + 'event-before', + 'message-stable', + '2026-07-27 00:00:00' + ) + """ + ) + ) + connection.execute(text("DELETE FROM alembic_version")) + before = schema_tool.audit_engine(postgres_schema_engine) + + with postgres_schema_engine.begin() as connection: + connection.execute( + text( + """ + UPDATE feishu_event_receipts + SET event_id = 'event-after' + WHERE event_key = 'postgres-content-key' + """ + ) + ) + changed = schema_tool.audit_engine(postgres_schema_engine) + + assert changed.impacted_table_row_counts == before.impacted_table_row_counts + assert ( + changed.impacted_table_content_digests + != before.impacted_table_content_digests + ) + assert changed.fingerprint != before.fingerprint + with pytest.raises(schema_tool.ReconciliationError, match="fingerprint changed"): + schema_tool.apply_baseline( + postgres_schema_engine, + before.fingerprint, + ) + with postgres_schema_engine.connect() as connection: + assert schema_tool._revision_rows(connection) == () + + +def test_postgres_advisory_transaction_lock_excludes_second_connection( + postgres_schema_engine: Engine, +) -> None: + with postgres_schema_engine.begin() as connection: + connection.execute(text("CREATE TABLE projects (id INTEGER PRIMARY KEY)")) + + with ( + postgres_schema_engine.connect() as first, + postgres_schema_engine.connect() as second, + ): + first_transaction = first.begin() + second_transaction = second.begin() + try: + assert first.scalar(text("SELECT pg_backend_pid()")) != second.scalar( + text("SELECT pg_backend_pid()") + ) + schema_tool._acquire_postgres_lock(first) + schema_tool._lock_impacted_postgres_tables(first) + second_acquired = second.scalar( + text("SELECT pg_try_advisory_xact_lock(:lock_key)"), + {"lock_key": schema_tool.ADVISORY_LOCK_KEY}, + ) + assert second_acquired is False + second.exec_driver_sql("SET LOCAL lock_timeout = '250ms'") + with pytest.raises(DBAPIError): + second.exec_driver_sql( + "LOCK TABLE projects IN ROW EXCLUSIVE MODE NOWAIT" + ) + second_transaction.rollback() + + first_transaction.commit() + second_transaction = second.begin() + second_acquired_after_release = second.scalar( + text("SELECT pg_try_advisory_xact_lock(:lock_key)"), + {"lock_key": schema_tool.ADVISORY_LOCK_KEY}, + ) + assert second_acquired_after_release is True + finally: + if first_transaction.is_active: + first_transaction.rollback() + if second_transaction.is_active: + second_transaction.rollback() diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 62b898a..a976bf3 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -3,7 +3,7 @@ from datetime import date, datetime, timedelta import pytest from fastapi import HTTPException -from sqlalchemy import select +from sqlalchemy import select, text from app.modules.ai_agent.constants import AIProviderName, AIResponseKey from app.modules.business.constants import BusinessResponseKey, StatusValue @@ -13,6 +13,7 @@ from fastapi.testclient import TestClient from app.core.config import Settings, get_settings from app.application.scheduling import create_scheduler from app.core.database import Base, SessionLocal, engine +from app.core.database.migrations import expected_alembic_heads from app.core.http.pagination import bounded_limit, bounded_offset from app.core.http.responses import MaskedJSONResponse from app.core.security import require_api_key, require_audit_api_key @@ -89,9 +90,14 @@ from app.application.delivery import ReportDeliveryService from app.modules.reports.chart import render_lifecycle_chart from app.modules.reports.services import ReportService from app.modules.feishu.service import FeishuService +from app.modules.feishu.models import FeishuEventReceipt from app.application.feishu import FeishuCommandService -from app.application.feishu.events import FeishuEventService, _audit_event_metadata -from app.modules.feishu.constants import FeishuEventSource +from app.application.feishu.events import ( + FeishuEventService, + _audit_event_metadata, + _identifier_digest, +) +from app.modules.feishu.constants import FeishuEventSource, FeishuInboundStatus from app.modules.risk.constants import RiskEventActionValue from app.modules.market.chart import render_market_chart from app.modules.market.service import MarketService, TushareClient, normalize_symbol @@ -101,6 +107,18 @@ from app.modules.workflows.models import WorkflowInstance Base.metadata.create_all(bind=engine) +with engine.begin() as connection: + connection.execute( + text( + "CREATE TABLE IF NOT EXISTS alembic_version " + "(version_num VARCHAR(32) NOT NULL)" + ) + ) + connection.execute(text("DELETE FROM alembic_version")) + connection.execute( + text("INSERT INTO alembic_version (version_num) VALUES (:revision)"), + {"revision": expected_alembic_heads()[0]}, + ) client = TestClient(app) headers = {"X-API-Key": "test-key"} audit_headers = {"X-API-Key": "test-key", "X-Audit-API-Key": "audit-key"} @@ -183,8 +201,22 @@ def test_feishu_webhook_routes_message_event() -> None: response = client.post("/api/v1/integrations/feishu/webhook", json=payload) assert response.status_code == 200 data = response.json() - assert data["handled"] is True - assert data["result"]["command"] == "risk_summary" + assert data["accepted"] is True + assert data["handled"] is False + assert data["status"] == FeishuInboundStatus.PENDING + with SessionLocal() as db: + receipt = db.scalar( + select(FeishuEventReceipt).where( + FeishuEventReceipt.event_id + == _identifier_digest( + "evt-smoke-risk-001", + "event-id", + ) + ) + ) + assert receipt is not None + assert receipt.status == FeishuInboundStatus.SUCCEEDED + assert receipt.payload is None audit_metadata = _audit_event_metadata(payload) assert "content" not in json.dumps(audit_metadata) assert "test-feishu-token" not in json.dumps(audit_metadata) @@ -192,6 +224,7 @@ def test_feishu_webhook_routes_message_event() -> None: duplicate_response = client.post("/api/v1/integrations/feishu/webhook", json=payload) assert duplicate_response.status_code == 200 assert duplicate_response.json()["duplicate"] is True + assert duplicate_response.json()["status"] == FeishuInboundStatus.SUCCEEDED blocked_logs_response = client.get("/api/v1/audit/logs", headers=headers) assert blocked_logs_response.status_code == 401 @@ -200,7 +233,11 @@ def test_feishu_webhook_routes_message_event() -> None: assert logs_response.status_code == 200 audit_payload = json.dumps(logs_response.json(), ensure_ascii=False) assert "test-feishu-token" not in audit_payload - assert "evt-smoke-risk-001" in audit_payload + assert "evt-smoke-risk-001" not in audit_payload + assert ( + _identifier_digest("evt-smoke-risk-001", "event-id") + in audit_payload + ) def test_feishu_rule_commands_create_list_disable_and_enable(monkeypatch) -> None: diff --git a/tests/test_subscription_runtime_wiring.py b/tests/test_subscription_runtime_wiring.py index c59f424..55437a7 100644 --- a/tests/test_subscription_runtime_wiring.py +++ b/tests/test_subscription_runtime_wiring.py @@ -330,6 +330,7 @@ def test_ready_route_returns_503_for_processable_delivery_without_credentials( monkeypatch.setenv("FEISHU_APP_TYPE", "self") monkeypatch.setenv("FEISHU_APP_ID", "") monkeypatch.setenv("FEISHU_APP_SECRET", "") + monkeypatch.setenv("FEISHU_USER_FEATURES_ENABLED", "true") get_settings.cache_clear() with session_factory() as db: subscription = _seed_subscription( diff --git a/tests/test_v2_hardening.py b/tests/test_v2_hardening.py index 6544f10..f786237 100644 --- a/tests/test_v2_hardening.py +++ b/tests/test_v2_hardening.py @@ -28,13 +28,16 @@ from app.tools import init_db def _production_settings(**overrides: object) -> Settings: values: dict[str, object] = { "app_env": "production", - "database_url": "postgresql+psycopg://app:secret@db/app", - "api_key": "service-key", - "audit_api_key": "audit-key", + "database_url": ( + "postgresql+psycopg://app:runtime-database-password-2026@db/app" + ), + "api_key": "runtime-service-key-2026-primary", + "audit_api_key": "runtime-audit-key-2026-independent", "cors_origins": ["https://internal.example.com"], "debug": False, "mask_sensitive_responses": True, "read_only_mode": True, + "feishu_event_transport": "disabled", } values.update(overrides) return Settings(_env_file=None, **values) @@ -42,7 +45,7 @@ def _production_settings(**overrides: object) -> Settings: def test_production_keys_must_be_enabled_isolated_and_safe() -> None: settings = _production_settings() - assert settings.api_key == "service-key" + assert settings.api_key == "runtime-service-key-2026-primary" with pytest.raises(ValueError, match="API_KEY or API_KEYS"): _production_settings( @@ -51,7 +54,7 @@ def test_production_keys_must_be_enabled_isolated_and_safe() -> None: ) with pytest.raises(ValueError, match="cannot overlap"): - _production_settings(audit_api_key="service-key") + _production_settings(audit_api_key="runtime-service-key-2026-primary") with pytest.raises(ValueError, match="MASK_SENSITIVE_RESPONSES"): _production_settings(mask_sensitive_responses=False)