JiuContinent
eb8267ed18
feat(feishu): 添加飞书入站事件inbox和混合数据库协调功能 - 实现飞书入站事件持久化inbox机制，支持状态管理、租约锁定和重试退避 - 添加混合数据库基线协调工具，确保平台PostgreSQL结构安全对齐 - 增加运行组件心跳检测和readiness就绪检查机制 - 实现app_ticket事件的安全轮换和验证处理 - 添加生产环境运行编排和fail-closed安全机制 - 支持webhook快速确认和长连接独立进程处理 - 完善个人数据擦除时的待处理事件清理功能 ```
664 lines
19 KiB
Python
664 lines
19 KiB
Python
"""Reconcile the known unversioned PostgreSQL schema.
|
|
|
|
Revision ID: 202607270001
|
|
Revises: 202607260005
|
|
Create Date: 2026-07-27
|
|
|
|
This revision is intentionally idempotent. A normally versioned database at
|
|
202607260005 only receives the PostgreSQL NULLS NOT DISTINCT hardening. The
|
|
one-time baseline runner may also use it, in the same transaction as an atomic
|
|
stamp, to repair the explicitly allowlisted mixed schema.
|
|
"""
|
|
|
|
from collections.abc import Iterable
|
|
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
|
|
|
|
revision = "202607270001"
|
|
down_revision = "202607260005"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
AI_MEMORY_TABLE = "ai_memory_entries"
|
|
MARKET_WATCHLIST_TABLE = "market_watchlists"
|
|
APPROVAL_TABLE = "approval_requests"
|
|
|
|
AI_MEMORY_OWNER_UNIQUE = "uq_ai_memory_owner_fingerprint"
|
|
MARKET_WATCHLIST_OWNER_UNIQUE = "uq_market_watchlist_owner_symbol"
|
|
MARKET_WATCHLIST_LEGACY_UNIQUE = "uq_market_watchlist_actor_symbol"
|
|
|
|
|
|
def _inspector() -> sa.Inspector:
|
|
return sa.inspect(op.get_bind())
|
|
|
|
|
|
def _table_exists(table_name: str) -> bool:
|
|
return table_name in _inspector().get_table_names()
|
|
|
|
|
|
def _column_names(table_name: str) -> set[str]:
|
|
return {
|
|
str(column["name"])
|
|
for column in _inspector().get_columns(table_name)
|
|
}
|
|
|
|
|
|
def _add_missing_columns(
|
|
table_name: str,
|
|
columns: Iterable[sa.Column],
|
|
) -> None:
|
|
existing = _column_names(table_name)
|
|
for column in columns:
|
|
if column.name not in existing:
|
|
op.add_column(table_name, column)
|
|
|
|
|
|
def _index_by_name(table_name: str, index_name: str) -> dict | None:
|
|
return next(
|
|
(
|
|
index
|
|
for index in _inspector().get_indexes(table_name)
|
|
if index.get("name") == index_name
|
|
),
|
|
None,
|
|
)
|
|
|
|
|
|
def _ensure_index(
|
|
table_name: str,
|
|
index_name: str,
|
|
columns: tuple[str, ...],
|
|
*,
|
|
unique: bool = False,
|
|
) -> None:
|
|
existing = _index_by_name(table_name, index_name)
|
|
if existing is not None:
|
|
existing_columns = tuple(existing.get("column_names") or ())
|
|
if existing_columns != columns or bool(existing.get("unique")) != unique:
|
|
raise RuntimeError(
|
|
f"Existing index {index_name} does not match the reconciliation definition"
|
|
)
|
|
return
|
|
op.create_index(index_name, table_name, list(columns), unique=unique)
|
|
|
|
|
|
def _drop_index_if_present(table_name: str, index_name: str) -> None:
|
|
if _index_by_name(table_name, index_name) is not None:
|
|
op.drop_index(index_name, table_name=table_name)
|
|
|
|
|
|
def _unique_by_name(table_name: str, constraint_name: str) -> dict | None:
|
|
return next(
|
|
(
|
|
constraint
|
|
for constraint in _inspector().get_unique_constraints(table_name)
|
|
if constraint.get("name") == constraint_name
|
|
),
|
|
None,
|
|
)
|
|
|
|
|
|
def _drop_unique_if_present(table_name: str, constraint_name: str) -> None:
|
|
if _unique_by_name(table_name, constraint_name) is None:
|
|
return
|
|
with op.batch_alter_table(table_name) as batch_op:
|
|
batch_op.drop_constraint(constraint_name, type_="unique")
|
|
|
|
|
|
def _uses_nulls_not_distinct(constraint: dict) -> bool:
|
|
options = constraint.get("dialect_options") or {}
|
|
return bool(options.get("postgresql_nulls_not_distinct"))
|
|
|
|
|
|
def _ensure_owner_unique(
|
|
table_name: str,
|
|
constraint_name: str,
|
|
columns: tuple[str, ...],
|
|
) -> None:
|
|
existing = _unique_by_name(table_name, constraint_name)
|
|
dialect_name = op.get_bind().dialect.name
|
|
if existing is not None:
|
|
if tuple(existing.get("column_names") or ()) != columns:
|
|
raise RuntimeError(
|
|
f"Existing constraint {constraint_name} does not match the reconciliation definition"
|
|
)
|
|
if dialect_name != "postgresql" or _uses_nulls_not_distinct(existing):
|
|
return
|
|
with op.batch_alter_table(table_name) as batch_op:
|
|
batch_op.drop_constraint(constraint_name, type_="unique")
|
|
|
|
with op.batch_alter_table(table_name) as batch_op:
|
|
batch_op.create_unique_constraint(
|
|
constraint_name,
|
|
list(columns),
|
|
postgresql_nulls_not_distinct=True,
|
|
)
|
|
|
|
|
|
def _ensure_foreign_key(
|
|
table_name: str,
|
|
constraint_name: str,
|
|
local_columns: tuple[str, ...],
|
|
remote_table: str,
|
|
remote_columns: tuple[str, ...],
|
|
) -> None:
|
|
for foreign_key in _inspector().get_foreign_keys(table_name):
|
|
if (
|
|
tuple(foreign_key.get("constrained_columns") or ()) == local_columns
|
|
and foreign_key.get("referred_table") == remote_table
|
|
and tuple(foreign_key.get("referred_columns") or ()) == remote_columns
|
|
):
|
|
ondelete = str((foreign_key.get("options") or {}).get("ondelete") or "")
|
|
if ondelete.upper() != "CASCADE":
|
|
raise RuntimeError(
|
|
f"Existing foreign key on {table_name} does not use ON DELETE CASCADE"
|
|
)
|
|
return
|
|
with op.batch_alter_table(table_name) as batch_op:
|
|
batch_op.create_foreign_key(
|
|
constraint_name,
|
|
remote_table,
|
|
list(local_columns),
|
|
list(remote_columns),
|
|
ondelete="CASCADE",
|
|
)
|
|
|
|
|
|
def _alter_without_server_default(
|
|
table_name: str,
|
|
columns: tuple[tuple[str, sa.types.TypeEngine], ...],
|
|
) -> None:
|
|
with op.batch_alter_table(table_name) as batch_op:
|
|
for column_name, column_type in columns:
|
|
batch_op.alter_column(
|
|
column_name,
|
|
existing_type=column_type,
|
|
nullable=False,
|
|
server_default=None,
|
|
)
|
|
|
|
|
|
def _add_business_columns() -> None:
|
|
_add_missing_columns(
|
|
"attendance_records",
|
|
(
|
|
sa.Column(
|
|
"attendance_scope",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="company",
|
|
),
|
|
sa.Column("source_status", sa.String(length=64), nullable=True),
|
|
sa.Column("source_location_status", sa.String(length=64), nullable=True),
|
|
sa.Column("source_updated_at", sa.DateTime(), nullable=True),
|
|
sa.Column("last_seen_at", sa.DateTime(), nullable=True),
|
|
sa.Column(
|
|
"is_active",
|
|
sa.Boolean(),
|
|
nullable=False,
|
|
server_default=sa.true(),
|
|
),
|
|
),
|
|
)
|
|
_add_missing_columns(
|
|
"audit_logs",
|
|
(sa.Column("request_id", sa.String(length=64), nullable=True),),
|
|
)
|
|
_add_missing_columns(
|
|
"projects",
|
|
(
|
|
sa.Column("display_code", sa.String(length=64), nullable=True),
|
|
sa.Column("department_code", sa.String(length=64), nullable=True),
|
|
sa.Column("department_name", sa.String(length=128), nullable=True),
|
|
sa.Column("owner_employee_code", sa.String(length=64), nullable=True),
|
|
sa.Column("source_stage", sa.String(length=32), nullable=True),
|
|
sa.Column("source_stage_label", sa.String(length=128), nullable=True),
|
|
sa.Column(
|
|
"source_archived",
|
|
sa.Boolean(),
|
|
nullable=False,
|
|
server_default=sa.false(),
|
|
),
|
|
sa.Column("source_created_at", sa.DateTime(), nullable=True),
|
|
sa.Column("source_updated_at", sa.DateTime(), nullable=True),
|
|
sa.Column("last_seen_at", sa.DateTime(), nullable=True),
|
|
sa.Column(
|
|
"is_active",
|
|
sa.Boolean(),
|
|
nullable=False,
|
|
server_default=sa.true(),
|
|
),
|
|
sa.Column(
|
|
"source_contract_amount",
|
|
sa.Numeric(precision=16, scale=2),
|
|
nullable=True,
|
|
),
|
|
sa.Column(
|
|
"source_project_investment_amount",
|
|
sa.Numeric(precision=16, scale=2),
|
|
nullable=True,
|
|
),
|
|
),
|
|
)
|
|
_add_missing_columns(
|
|
"risk_events",
|
|
(
|
|
sa.Column("assigned_to", sa.String(length=128), nullable=True),
|
|
sa.Column("resolved_at", sa.DateTime(), nullable=True),
|
|
sa.Column("closed_at", sa.DateTime(), nullable=True),
|
|
sa.Column("closed_reason", sa.Text(), nullable=True),
|
|
sa.Column("review_summary", sa.Text(), nullable=True),
|
|
),
|
|
)
|
|
_add_missing_columns(
|
|
"work_reports",
|
|
(
|
|
sa.Column("employee_code", sa.String(length=64), nullable=True),
|
|
sa.Column("external_id", sa.String(length=128), nullable=True),
|
|
sa.Column(
|
|
"is_late",
|
|
sa.Boolean(),
|
|
nullable=False,
|
|
server_default=sa.false(),
|
|
),
|
|
sa.Column(
|
|
"is_draft",
|
|
sa.Boolean(),
|
|
nullable=False,
|
|
server_default=sa.false(),
|
|
),
|
|
sa.Column("source_updated_at", sa.DateTime(), nullable=True),
|
|
sa.Column("last_seen_at", sa.DateTime(), nullable=True),
|
|
sa.Column(
|
|
"is_active",
|
|
sa.Boolean(),
|
|
nullable=False,
|
|
server_default=sa.true(),
|
|
),
|
|
),
|
|
)
|
|
_add_missing_columns(
|
|
"work_tasks",
|
|
(
|
|
sa.Column(
|
|
"source_system",
|
|
sa.String(length=64),
|
|
nullable=False,
|
|
server_default="internal",
|
|
),
|
|
sa.Column("external_id", sa.String(length=128), nullable=True),
|
|
sa.Column("employee_code", sa.String(length=64), nullable=True),
|
|
sa.Column("source_created_at", sa.DateTime(), nullable=True),
|
|
sa.Column("source_updated_at", sa.DateTime(), nullable=True),
|
|
sa.Column("last_seen_at", sa.DateTime(), nullable=True),
|
|
sa.Column(
|
|
"is_active",
|
|
sa.Boolean(),
|
|
nullable=False,
|
|
server_default=sa.true(),
|
|
),
|
|
),
|
|
)
|
|
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE attendance_records
|
|
SET attendance_scope = 'company'
|
|
WHERE attendance_scope IS NULL
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE attendance_records
|
|
SET is_active = true
|
|
WHERE is_active IS NULL
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE projects
|
|
SET source_archived = false
|
|
WHERE source_archived IS NULL
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE projects
|
|
SET is_active = true
|
|
WHERE is_active IS NULL
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE work_reports
|
|
SET is_late = false
|
|
WHERE is_late IS NULL
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE work_reports
|
|
SET is_draft = false
|
|
WHERE is_draft IS NULL
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE work_reports
|
|
SET is_active = true
|
|
WHERE is_active IS NULL
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE work_tasks
|
|
SET source_system = 'internal'
|
|
WHERE source_system IS NULL
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE work_tasks
|
|
SET is_active = true
|
|
WHERE is_active IS NULL
|
|
"""
|
|
)
|
|
)
|
|
|
|
_alter_without_server_default(
|
|
"attendance_records",
|
|
(
|
|
("attendance_scope", sa.String(length=32)),
|
|
("is_active", sa.Boolean()),
|
|
),
|
|
)
|
|
_alter_without_server_default(
|
|
"projects",
|
|
(
|
|
("source_archived", sa.Boolean()),
|
|
("is_active", sa.Boolean()),
|
|
),
|
|
)
|
|
_alter_without_server_default(
|
|
"work_reports",
|
|
(
|
|
("is_late", sa.Boolean()),
|
|
("is_draft", sa.Boolean()),
|
|
("is_active", sa.Boolean()),
|
|
),
|
|
)
|
|
_alter_without_server_default(
|
|
"work_tasks",
|
|
(
|
|
("source_system", sa.String(length=64)),
|
|
("is_active", sa.Boolean()),
|
|
),
|
|
)
|
|
|
|
|
|
def _ensure_business_indexes() -> None:
|
|
definitions = {
|
|
"attendance_records": (
|
|
"attendance_scope",
|
|
"is_active",
|
|
"last_seen_at",
|
|
"source_status",
|
|
"source_updated_at",
|
|
),
|
|
"audit_logs": ("request_id",),
|
|
"projects": (
|
|
"department_code",
|
|
"department_name",
|
|
"display_code",
|
|
"is_active",
|
|
"last_seen_at",
|
|
"owner_employee_code",
|
|
"source_archived",
|
|
"source_stage",
|
|
"source_updated_at",
|
|
),
|
|
"risk_events": ("assigned_to",),
|
|
"work_reports": (
|
|
"employee_code",
|
|
"external_id",
|
|
"is_active",
|
|
"is_draft",
|
|
"is_late",
|
|
"last_seen_at",
|
|
"source_updated_at",
|
|
),
|
|
"work_tasks": (
|
|
"employee_code",
|
|
"external_id",
|
|
"is_active",
|
|
"last_seen_at",
|
|
"source_updated_at",
|
|
),
|
|
}
|
|
for table_name, columns in definitions.items():
|
|
for column_name in columns:
|
|
_ensure_index(
|
|
table_name,
|
|
f"ix_{table_name}_{column_name}",
|
|
(column_name,),
|
|
)
|
|
|
|
|
|
def _reconcile_ai_memory() -> None:
|
|
_add_missing_columns(
|
|
AI_MEMORY_TABLE,
|
|
(
|
|
sa.Column("owner_id", sa.Integer(), nullable=True),
|
|
sa.Column(
|
|
"kind",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="memory",
|
|
),
|
|
),
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE ai_memory_entries
|
|
SET
|
|
owner_id = NULL,
|
|
kind = 'company_rule',
|
|
source = 'legacy_company',
|
|
status = 'active'
|
|
WHERE source = 'user_rule'
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE ai_memory_entries
|
|
SET kind = 'memory', status = 'archived'
|
|
WHERE owner_id IS NULL AND source IN ('auto', 'hermes')
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE ai_memory_entries
|
|
SET kind = 'memory'
|
|
WHERE kind IS NULL
|
|
"""
|
|
)
|
|
)
|
|
_alter_without_server_default(
|
|
AI_MEMORY_TABLE,
|
|
(("kind", sa.String(length=32)),),
|
|
)
|
|
|
|
fingerprint_index = _index_by_name(
|
|
AI_MEMORY_TABLE,
|
|
"ix_ai_memory_entries_fingerprint",
|
|
)
|
|
if fingerprint_index is not None and bool(fingerprint_index.get("unique")):
|
|
op.drop_index(
|
|
"ix_ai_memory_entries_fingerprint",
|
|
table_name=AI_MEMORY_TABLE,
|
|
)
|
|
_ensure_index(
|
|
AI_MEMORY_TABLE,
|
|
"ix_ai_memory_entries_fingerprint",
|
|
("fingerprint",),
|
|
)
|
|
_ensure_index(
|
|
AI_MEMORY_TABLE,
|
|
"ix_ai_memory_entries_kind",
|
|
("kind",),
|
|
)
|
|
_ensure_index(
|
|
AI_MEMORY_TABLE,
|
|
"ix_ai_memory_entries_owner_id",
|
|
("owner_id",),
|
|
)
|
|
_ensure_foreign_key(
|
|
AI_MEMORY_TABLE,
|
|
"fk_ai_memory_entries_owner_id",
|
|
("owner_id",),
|
|
"feishu_users",
|
|
("id",),
|
|
)
|
|
_ensure_owner_unique(
|
|
AI_MEMORY_TABLE,
|
|
AI_MEMORY_OWNER_UNIQUE,
|
|
("owner_id", "fingerprint"),
|
|
)
|
|
|
|
|
|
def _reconcile_market_watchlists() -> None:
|
|
_add_missing_columns(
|
|
MARKET_WATCHLIST_TABLE,
|
|
(sa.Column("owner_id", sa.Integer(), nullable=True),),
|
|
)
|
|
_drop_unique_if_present(
|
|
MARKET_WATCHLIST_TABLE,
|
|
MARKET_WATCHLIST_LEGACY_UNIQUE,
|
|
)
|
|
_ensure_index(
|
|
MARKET_WATCHLIST_TABLE,
|
|
"ix_market_watchlists_owner_id",
|
|
("owner_id",),
|
|
)
|
|
_ensure_foreign_key(
|
|
MARKET_WATCHLIST_TABLE,
|
|
"fk_market_watchlists_owner_id",
|
|
("owner_id",),
|
|
"feishu_users",
|
|
("id",),
|
|
)
|
|
_ensure_owner_unique(
|
|
MARKET_WATCHLIST_TABLE,
|
|
MARKET_WATCHLIST_OWNER_UNIQUE,
|
|
("owner_id", "symbol"),
|
|
)
|
|
|
|
|
|
def _create_feishu_app_tickets_if_missing() -> None:
|
|
if not _table_exists("feishu_app_tickets"):
|
|
op.create_table(
|
|
"feishu_app_tickets",
|
|
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
|
|
sa.Column("app_id", sa.String(length=128), nullable=False),
|
|
sa.Column("app_ticket", sa.Text(), nullable=False),
|
|
sa.Column("received_at", sa.DateTime(), nullable=False),
|
|
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
|
sa.PrimaryKeyConstraint("id"),
|
|
)
|
|
_ensure_index(
|
|
"feishu_app_tickets",
|
|
"ix_feishu_app_tickets_app_id",
|
|
("app_id",),
|
|
unique=True,
|
|
)
|
|
_ensure_index(
|
|
"feishu_app_tickets",
|
|
"ix_feishu_app_tickets_received_at",
|
|
("received_at",),
|
|
)
|
|
|
|
|
|
def _create_admin_tombstones_if_missing() -> None:
|
|
if not _table_exists("feishu_admin_bootstrap_tombstones"):
|
|
op.create_table(
|
|
"feishu_admin_bootstrap_tombstones",
|
|
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
|
|
sa.Column("identity_hash", sa.String(length=64), nullable=False),
|
|
sa.Column("created_at", sa.DateTime(), nullable=False),
|
|
sa.PrimaryKeyConstraint("id"),
|
|
)
|
|
_ensure_index(
|
|
"feishu_admin_bootstrap_tombstones",
|
|
"ix_feishu_admin_bootstrap_tombstones_created_at",
|
|
("created_at",),
|
|
)
|
|
_ensure_index(
|
|
"feishu_admin_bootstrap_tombstones",
|
|
"ix_feishu_admin_bootstrap_tombstones_identity_hash",
|
|
("identity_hash",),
|
|
unique=True,
|
|
)
|
|
|
|
|
|
def _drop_empty_approval_table() -> None:
|
|
if not _table_exists(APPROVAL_TABLE):
|
|
return
|
|
count = int(
|
|
op.get_bind().execute(
|
|
sa.text("SELECT COUNT(*) FROM approval_requests")
|
|
).scalar_one()
|
|
)
|
|
if count:
|
|
raise RuntimeError(
|
|
"Refusing to remove approval_requests because it contains rows"
|
|
)
|
|
for table_name in _inspector().get_table_names():
|
|
for foreign_key in _inspector().get_foreign_keys(table_name):
|
|
if foreign_key.get("referred_table") == APPROVAL_TABLE:
|
|
raise RuntimeError(
|
|
"Refusing to remove approval_requests because a foreign key depends on it"
|
|
)
|
|
op.drop_table(APPROVAL_TABLE)
|
|
|
|
|
|
def upgrade() -> None:
|
|
_add_business_columns()
|
|
_ensure_business_indexes()
|
|
_reconcile_ai_memory()
|
|
_reconcile_market_watchlists()
|
|
_create_feishu_app_tickets_if_missing()
|
|
_create_admin_tombstones_if_missing()
|
|
_drop_empty_approval_table()
|
|
|
|
|
|
def downgrade() -> None:
|
|
raise RuntimeError(
|
|
"Revision 202607270001 is irreversible: it classifies legacy data, "
|
|
"adds ownership relationships, and removes the obsolete approval table. "
|
|
"Restore a verified database backup instead of downgrading."
|
|
)
|