JiuContinent
fbd0aaa9e4
refactor(api): 使用常量替代硬编码字符串 - 在health_check接口中使用ApiResponseKey.STATUS和ApiStatus.OK常量 - 替换硬编码的状态返回值为枚举常量 refactor(core): 配置模块错误信息统一使用常量 - 从constants模块导入ConfigErrorDetail并替换CORS_ORIGINS和LEGACY_ALLOWED_QUERIES的验证错误信息 - 配置类中的默认值使用constants中定义的常量 feat(constants): 添加API响应、安全错误和配置错误常量类 - 新增ApiResponseKey用于API状态键名 - 新增ApiStatus用于API状态值 - 新增SecurityErrorDetail用于安全认证错误详情 - 新增ConfigErrorDetail用于配置验证错误详情 - 添加DEFAULT_MODEL_PROVIDER和DEFAULT_OPENCLAW_ACTION_JSON常量 refactor(security): 安全认证模块使用错误常量 - 将硬编码的安全错误信息替换为SecurityErrorDetail常量 - 包括API密钥、审批密钥和审计密钥的相关错误信息 refactor(ai-agent): AI代理适配器改进错误处理 - 将HTTP状态码替换为FastAPI状态常量 - 添加OpenClaw工具和操作的错误常量 - 修复健康检查和工具调用中的状态码比较逻辑 - 添加AIToolAuditKey用于工具审计键名 feat(ai-agent): 扩展AI代理常量定义 - 新增AIToolAuditKey用于工具审计字段 - 添加OpenClaw相关的错误常量如OPENCLAW_CHAT_PROVIDER_REQUIRED等 - 添加UNSUPPORTED_AI_SKILL_TEMPLATE模板字符串 refactor(approvals): 审批模块常量化重构 - 新增ApprovalPayloadKey用于审批载荷字段 - 添加approval_action函数和APPROVAL_ACTION_SEPARATOR分隔符 - 使用常量替换字面量值 feat(audit): 审计模块新增飞书事件动作类型 - 添加FEISHU_WEBHOOK_EVENT和FEISHU_LONG_CONNECTION_EVENT审计动作 refactor(business): 业务模块全面常量化 - 新增BusinessDomain枚举包含所有业务域 - 添加BusinessResponseKey、BusinessPayloadKey等常量类 - 重构DOMAIN_MODELS为frozenset以提高性能 - 添加normalize_domain等辅助函数用于域标准化 - 使用常量替换路由和业务服务中的硬编码字符串 - 添加业务错误常量和字段验证模板 refactor(feishu): 飞书客户端错误处理优化 - 将HTTP状态码替换为FastAPI标准状态常量 - 改进错误处理的一致性 refactor(approvals): 审批服务使用新常量结构 - 使用ApprovalPayloadKey常量重构载荷字段 - 使用approval_action函数统一动作命名格式 - 优化高风险域判断逻辑 ```
412 lines
15 KiB
Python
412 lines
15 KiB
Python
from datetime import date, datetime
|
|
from decimal import Decimal
|
|
from typing import Any
|
|
|
|
from fastapi import HTTPException, status
|
|
from sqlalchemy import select, text
|
|
from sqlalchemy.engine import Engine, RowMapping
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.constants import ActorValue, ApiStatus
|
|
from app.core.config import get_settings
|
|
from app.core.database import legacy_engine
|
|
from app.core.pagination import bounded_limit
|
|
from app.core.time import utc_now
|
|
from app.modules.audit.constants import AuditAction, AuditRiskLevel, AuditSource, AuditStatus
|
|
from app.modules.audit.schemas import AuditLogCreate
|
|
from app.modules.audit.service import AuditService
|
|
from app.modules.business.constants import BusinessDomain, SourceSystem, StatusValue
|
|
from app.modules.business.models import LegacySyncRun, Project
|
|
from app.modules.business.service import serialize_model
|
|
from app.modules.legacy_mysql.constants import (
|
|
LEGACY_PROJECT_QUERY_SOURCE,
|
|
LEGACY_PROJECT_SYNC_NOTE,
|
|
LEGACY_SYNC_MISSING_ID_REASON,
|
|
LEGACY_SYNC_RUN_CODE_PREFIX,
|
|
LEGACY_HEALTH_SQL,
|
|
LEGACY_LIMIT_CLAUSE,
|
|
LEGACY_LIMIT_MARKER,
|
|
LEGACY_MYSQL_CONNECTION_FAILED_TEMPLATE,
|
|
LEGACY_PROJECT_CODE_TEMPLATE,
|
|
LEGACY_SELECT_PREFIX,
|
|
LEGACY_SQL_TRAILING_TERMINATOR,
|
|
LEGACY_UNNAMED_PROJECT,
|
|
LegacyProjectField,
|
|
LegacyQueryError,
|
|
LegacyQueryName,
|
|
LegacyResponseKey,
|
|
LegacySyncAction,
|
|
)
|
|
|
|
FORBIDDEN_SQL_TOKENS = {
|
|
"insert",
|
|
"update",
|
|
"delete",
|
|
"drop",
|
|
"alter",
|
|
"truncate",
|
|
"create",
|
|
"replace",
|
|
"grant",
|
|
"revoke",
|
|
}
|
|
|
|
|
|
def _jsonable(value: Any) -> Any:
|
|
"""Convert database scalar values into JSON-friendly values."""
|
|
|
|
if isinstance(value, (datetime, date)):
|
|
return value.isoformat()
|
|
if isinstance(value, Decimal):
|
|
return float(value)
|
|
return value
|
|
|
|
|
|
def _row_to_dict(row: RowMapping) -> dict[str, Any]:
|
|
"""Convert a SQLAlchemy row mapping to a serializable dictionary."""
|
|
|
|
return {key: _jsonable(value) for key, value in row.items()}
|
|
|
|
|
|
def _normalize_sql(sql: str) -> str:
|
|
return " ".join(sql.strip().rstrip(LEGACY_SQL_TRAILING_TERMINATOR).split()).lower()
|
|
|
|
|
|
def _query_name_text(query_name: str | LegacyQueryName | None) -> str:
|
|
if query_name is None:
|
|
return LegacyQueryName.PROJECTS.value
|
|
if isinstance(query_name, LegacyQueryName):
|
|
return query_name.value
|
|
return str(query_name)
|
|
|
|
|
|
class LegacyMySQLService:
|
|
"""Read legacy MySQL data and sync projects into the internal ledger."""
|
|
|
|
def __init__(self, db: Session | None):
|
|
self.db = db
|
|
|
|
@staticmethod
|
|
def _ensure_engine() -> Engine:
|
|
if legacy_engine is None:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail=LegacyQueryError.DATABASE_NOT_CONFIGURED,
|
|
)
|
|
return legacy_engine
|
|
|
|
@staticmethod
|
|
def _ensure_readonly(sql: str) -> None:
|
|
stripped = sql.strip().lower()
|
|
if not stripped.startswith(LEGACY_SELECT_PREFIX):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=LegacyQueryError.ONLY_SELECT_ALLOWED,
|
|
)
|
|
tokens = {token.strip(" ;,\n\t") for token in stripped.replace("(", " ").split()}
|
|
if tokens & FORBIDDEN_SQL_TOKENS:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=LegacyQueryError.FORBIDDEN_SQL_TOKEN,
|
|
)
|
|
|
|
@staticmethod
|
|
def _allowed_queries() -> dict[str, str]:
|
|
settings = get_settings()
|
|
queries = {
|
|
_query_name_text(name): sql
|
|
for name, sql in settings.legacy_allowed_queries.items()
|
|
}
|
|
if settings.legacy_project_query:
|
|
queries.setdefault(LegacyQueryName.PROJECTS.value, settings.legacy_project_query)
|
|
return queries
|
|
|
|
def health(self) -> dict[str, str]:
|
|
engine = self._ensure_engine()
|
|
try:
|
|
with engine.connect() as conn:
|
|
conn.execute(text(LEGACY_HEALTH_SQL))
|
|
except SQLAlchemyError as exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail=LEGACY_MYSQL_CONNECTION_FAILED_TEMPLATE.format(error=exc),
|
|
) from exc
|
|
return {LegacyResponseKey.STATUS: ApiStatus.OK}
|
|
|
|
def execute_readonly(
|
|
self,
|
|
sql: str,
|
|
params: dict[str, Any] | None = None,
|
|
limit: int = 100,
|
|
) -> dict[str, Any]:
|
|
"""Execute a readonly SQL statement only when it matches the allowlist."""
|
|
|
|
normalized_sql = _normalize_sql(sql)
|
|
for allowed_sql in self._allowed_queries().values():
|
|
if _normalize_sql(allowed_sql) == normalized_sql:
|
|
return self._execute_readonly_sql(allowed_sql, params, limit)
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail=LegacyQueryError.QUERY_NOT_ALLOWED,
|
|
)
|
|
|
|
def execute_allowed_query(
|
|
self,
|
|
query_name: str | None,
|
|
params: dict[str, Any] | None = None,
|
|
limit: int = 100,
|
|
) -> dict[str, Any]:
|
|
queries = self._allowed_queries()
|
|
normalized_name = _query_name_text(query_name)
|
|
sql = queries.get(normalized_name)
|
|
if not sql:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail=LegacyQueryError.QUERY_NOT_ALLOWED,
|
|
)
|
|
return self._execute_readonly_sql(sql, params, limit)
|
|
|
|
def _execute_readonly_sql(
|
|
self,
|
|
sql: str,
|
|
params: dict[str, Any] | None = None,
|
|
limit: int = 100,
|
|
) -> dict[str, Any]:
|
|
self._ensure_readonly(sql)
|
|
engine = self._ensure_engine()
|
|
params = dict(params or {})
|
|
try:
|
|
params[LegacyResponseKey.LIMIT] = bounded_limit(
|
|
params.get(LegacyResponseKey.LIMIT, limit)
|
|
)
|
|
except (TypeError, ValueError) as exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail=LegacyQueryError.INVALID_LIMIT,
|
|
) from exc
|
|
limited_sql = sql
|
|
if LEGACY_LIMIT_MARKER not in sql.lower():
|
|
limited_sql = f"{sql.rstrip(LEGACY_SQL_TRAILING_TERMINATOR)}{LEGACY_LIMIT_CLAUSE}"
|
|
with engine.connect() as conn:
|
|
result = conn.execute(text(limited_sql), params)
|
|
rows = [_row_to_dict(row) for row in result.mappings().all()]
|
|
columns = list(rows[0].keys()) if rows else []
|
|
return {
|
|
LegacyResponseKey.COLUMNS: columns,
|
|
LegacyResponseKey.ROWS: rows,
|
|
LegacyResponseKey.ROW_COUNT: len(rows),
|
|
}
|
|
|
|
def fetch_default_projects(self, limit: int = 100) -> dict[str, Any]:
|
|
settings = get_settings()
|
|
if not settings.legacy_project_query:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=LegacyQueryError.PROJECT_QUERY_NOT_CONFIGURED,
|
|
)
|
|
return self.execute_allowed_query(
|
|
LegacyQueryName.PROJECTS,
|
|
{LegacyResponseKey.LIMIT: limit},
|
|
limit=limit,
|
|
)
|
|
|
|
@staticmethod
|
|
def _value(
|
|
row: dict[str, Any],
|
|
field_map: dict[str, str],
|
|
internal_name: str,
|
|
fallback: Any = None,
|
|
) -> Any:
|
|
source_name = field_map.get(internal_name, internal_name)
|
|
if source_name in row:
|
|
return row[source_name]
|
|
return fallback
|
|
|
|
def _project_payload(self, row: dict[str, Any], field_map: dict[str, str]) -> dict[str, Any]:
|
|
settings = get_settings()
|
|
external_id = self._value(
|
|
row,
|
|
field_map,
|
|
LegacyProjectField.EXTERNAL_ID,
|
|
row.get(LegacyProjectField.ID),
|
|
)
|
|
raw_code = self._value(row, field_map, LegacyProjectField.CODE, None)
|
|
code = None
|
|
if raw_code:
|
|
code = str(raw_code)
|
|
elif external_id is not None:
|
|
code = LEGACY_PROJECT_CODE_TEMPLATE.format(
|
|
prefix=settings.legacy_project_code_prefix,
|
|
external_id=external_id,
|
|
)
|
|
return {
|
|
LegacyProjectField.CODE: code,
|
|
LegacyProjectField.EXTERNAL_ID: str(external_id) if external_id is not None else code,
|
|
LegacyProjectField.SOURCE_SYSTEM: SourceSystem.LEGACY_MYSQL,
|
|
LegacyProjectField.NAME: self._value(
|
|
row,
|
|
field_map,
|
|
LegacyProjectField.NAME,
|
|
LEGACY_UNNAMED_PROJECT,
|
|
),
|
|
LegacyProjectField.OWNER: self._value(row, field_map, LegacyProjectField.OWNER, None),
|
|
LegacyProjectField.STATUS: self._value(row, field_map, LegacyProjectField.STATUS, StatusValue.UNKNOWN),
|
|
LegacyProjectField.PROGRESS_PERCENT: int(
|
|
self._value(
|
|
row,
|
|
field_map,
|
|
LegacyProjectField.PROGRESS_PERCENT,
|
|
row.get(LegacyProjectField.PROGRESS) or 0,
|
|
)
|
|
or 0
|
|
),
|
|
LegacyProjectField.START_DATE: self._value(row, field_map, LegacyProjectField.START_DATE, None),
|
|
LegacyProjectField.DUE_DATE: self._value(row, field_map, LegacyProjectField.DUE_DATE, None),
|
|
LegacyProjectField.BUDGET_AMOUNT: (
|
|
self._value(row, field_map, LegacyProjectField.BUDGET_AMOUNT, row.get(LegacyProjectField.BUDGET) or 0) or 0
|
|
),
|
|
LegacyProjectField.ACTUAL_AMOUNT: (
|
|
self._value(row, field_map, LegacyProjectField.ACTUAL_AMOUNT, row.get(LegacyProjectField.ACTUAL_COST) or 0) or 0
|
|
),
|
|
LegacyProjectField.DESCRIPTION: self._value(row, field_map, LegacyProjectField.DESCRIPTION, None),
|
|
}
|
|
|
|
def sync_projects(
|
|
self,
|
|
source_query: str | None = None,
|
|
source_query_name: str | None = None,
|
|
field_map: dict[str, str] | None = None,
|
|
limit: int = 100,
|
|
dry_run: bool = True,
|
|
actor: str = ActorValue.API,
|
|
) -> dict[str, Any]:
|
|
if self.db is None:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail=LegacyQueryError.APP_DB_UNAVAILABLE,
|
|
)
|
|
|
|
query_name = source_query_name or LegacyQueryName.PROJECTS
|
|
if source_query:
|
|
rows = self.execute_readonly(
|
|
source_query,
|
|
{LegacyResponseKey.LIMIT: limit},
|
|
limit=limit,
|
|
)[LegacyResponseKey.ROWS]
|
|
query_ref = LegacySyncAction.ALLOWLISTED_INLINE_SQL
|
|
else:
|
|
rows = self.execute_allowed_query(
|
|
query_name,
|
|
{LegacyResponseKey.LIMIT: limit},
|
|
limit=limit,
|
|
)[LegacyResponseKey.ROWS]
|
|
query_ref = _query_name_text(query_name)
|
|
field_map = field_map or {}
|
|
created = 0
|
|
updated = 0
|
|
skipped = 0
|
|
items: list[dict[str, Any]] = []
|
|
|
|
for row in rows:
|
|
payload = self._project_payload(row, field_map)
|
|
if not payload[LegacyProjectField.EXTERNAL_ID] and not payload[LegacyProjectField.CODE]:
|
|
skipped += 1
|
|
items.append(
|
|
{
|
|
LegacyResponseKey.ACTION: LegacySyncAction.SKIPPED,
|
|
LegacyResponseKey.REASON: LEGACY_SYNC_MISSING_ID_REASON,
|
|
LegacyResponseKey.SOURCE: row,
|
|
}
|
|
)
|
|
continue
|
|
|
|
stmt = select(Project).where(
|
|
Project.source_system == SourceSystem.LEGACY_MYSQL,
|
|
Project.external_id == payload[LegacyProjectField.EXTERNAL_ID],
|
|
)
|
|
record = self.db.execute(stmt).scalar_one_or_none()
|
|
if record is None:
|
|
record = self.db.execute(
|
|
select(Project).where(Project.code == payload[LegacyProjectField.CODE])
|
|
).scalar_one_or_none()
|
|
|
|
if record is None:
|
|
created += 1
|
|
action = LegacySyncAction.CREATE
|
|
result = payload
|
|
if not dry_run:
|
|
record = Project(**payload)
|
|
self.db.add(record)
|
|
self.db.flush()
|
|
result = serialize_model(record)
|
|
else:
|
|
updated += 1
|
|
action = LegacySyncAction.UPDATE
|
|
if not dry_run:
|
|
for key, value in payload.items():
|
|
setattr(record, key, value)
|
|
self.db.flush()
|
|
result = serialize_model(record)
|
|
else:
|
|
result = payload
|
|
items.append(
|
|
{
|
|
LegacyResponseKey.ACTION: action,
|
|
LegacyResponseKey.PROJECT: result,
|
|
LegacyResponseKey.SOURCE: row,
|
|
}
|
|
)
|
|
|
|
if not dry_run:
|
|
self.db.commit()
|
|
|
|
result = {
|
|
LegacyResponseKey.DRY_RUN: dry_run,
|
|
LegacyResponseKey.CREATED: created,
|
|
LegacyResponseKey.UPDATED: updated,
|
|
LegacyResponseKey.SKIPPED: skipped,
|
|
LegacyResponseKey.ITEMS: items,
|
|
}
|
|
sync_run = LegacySyncRun(
|
|
code=f"{LEGACY_SYNC_RUN_CODE_PREFIX}-{utc_now():%Y%m%d%H%M%S%f}",
|
|
domain=BusinessDomain.PROJECTS,
|
|
source_table=LEGACY_PROJECT_QUERY_SOURCE,
|
|
status=StatusValue.DRY_RUN if dry_run else AuditStatus.SUCCESS,
|
|
finished_at=utc_now(),
|
|
created_count=created,
|
|
updated_count=updated,
|
|
skipped_count=skipped,
|
|
note=LEGACY_PROJECT_SYNC_NOTE,
|
|
)
|
|
self.db.add(sync_run)
|
|
self.db.commit()
|
|
self.db.refresh(sync_run)
|
|
result[LegacyResponseKey.SYNC_RUN_CODE] = sync_run.code
|
|
|
|
AuditService(self.db).log(
|
|
AuditLogCreate(
|
|
actor=actor,
|
|
source=AuditSource.LEGACY_MYSQL,
|
|
action=AuditAction.LEGACY_SYNC_PROJECTS,
|
|
target_type=BusinessDomain.PROJECTS,
|
|
risk_level=AuditRiskLevel.MEDIUM,
|
|
request_payload={
|
|
LegacyResponseKey.SOURCE_QUERY: query_ref,
|
|
LegacyResponseKey.FIELD_MAP: field_map,
|
|
LegacyResponseKey.LIMIT: limit,
|
|
LegacyResponseKey.DRY_RUN: dry_run,
|
|
},
|
|
response_payload={
|
|
key: result[key]
|
|
for key in [
|
|
LegacyResponseKey.DRY_RUN,
|
|
LegacyResponseKey.CREATED,
|
|
LegacyResponseKey.UPDATED,
|
|
LegacyResponseKey.SKIPPED,
|
|
]
|
|
},
|
|
)
|
|
)
|
|
return result
|