from datetime import date, datetime from decimal import Decimal from typing import Any from fastapi import HTTPException, status from sqlalchemy import select, text from sqlalchemy.engine import Engine, RowMapping from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session from app.core.constants import ActorValue, ApiStatus from app.core.config import get_settings from app.core.database import legacy_engine from app.core.http.pagination import bounded_limit from app.core.utils.time import utc_now from app.modules.audit.constants import AuditAction, AuditRiskLevel, AuditSource, AuditStatus from app.modules.audit.schemas import AuditLogCreate from app.modules.audit.service import AuditService from app.modules.business.constants import BusinessDomain, SourceSystem, StatusValue from app.modules.business.models import LegacySyncRun, Project, WorkTask from app.modules.business.service import serialize_model from app.modules.events.constants import ( EventAggregateType, EventPayloadKey, EventSource, EventType, ) from app.modules.events.service import EventService from app.modules.legacy_mysql.constants import ( LEGACY_PROJECT_QUERY_SOURCE, LEGACY_PROJECT_SYNC_NOTE, LEGACY_SYNC_MISSING_ID_REASON, LEGACY_SYNC_RUN_CODE_PREFIX, LEGACY_TASK_CODE_TEMPLATE, LEGACY_TASK_QUERY_SOURCE, LEGACY_TASK_SYNC_NOTE, LEGACY_TASK_SYNC_RUN_CODE_PREFIX, LEGACY_HEALTH_SQL, LEGACY_LIMIT_CLAUSE, LEGACY_LIMIT_MARKER, LEGACY_MYSQL_CONNECTION_FAILED_TEMPLATE, LEGACY_PROJECT_CODE_TEMPLATE, LEGACY_SELECT_PREFIX, LEGACY_SQL_TRAILING_TERMINATOR, LEGACY_UNNAMED_TASK, LEGACY_UNNAMED_PROJECT, LegacyProjectField, LegacyQueryError, LegacyQueryName, LegacyResponseKey, LegacySyncAction, LegacyTaskField, ) FORBIDDEN_SQL_TOKENS = { "insert", "update", "delete", "drop", "alter", "truncate", "create", "replace", "grant", "revoke", } def _jsonable(value: Any) -> Any: """Convert database scalar values into JSON-friendly values.""" if isinstance(value, (datetime, date)): return value.isoformat() if isinstance(value, Decimal): return float(value) return value def _row_to_dict(row: RowMapping) -> dict[str, Any]: """Convert a SQLAlchemy row mapping to a serializable dictionary.""" return {key: _jsonable(value) for key, value in row.items()} def _normalize_sql(sql: str) -> str: return " ".join(sql.strip().rstrip(LEGACY_SQL_TRAILING_TERMINATOR).split()).lower() def _query_name_text(query_name: str | LegacyQueryName | None) -> str: if query_name is None: return LegacyQueryName.PROJECTS.value if isinstance(query_name, LegacyQueryName): return query_name.value return str(query_name) class LegacyMySQLService: """Read legacy MySQL data and sync projects into the internal ledger.""" def __init__(self, db: Session | None): self.db = db @staticmethod def _ensure_engine() -> Engine: if legacy_engine is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=LegacyQueryError.DATABASE_NOT_CONFIGURED, ) return legacy_engine @staticmethod def _ensure_readonly(sql: str) -> None: stripped = sql.strip().lower() if not stripped.startswith(LEGACY_SELECT_PREFIX): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=LegacyQueryError.ONLY_SELECT_ALLOWED, ) tokens = {token.strip(" ;,\n\t") for token in stripped.replace("(", " ").split()} if tokens & FORBIDDEN_SQL_TOKENS: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=LegacyQueryError.FORBIDDEN_SQL_TOKEN, ) @staticmethod def _allowed_queries() -> dict[str, str]: settings = get_settings() queries = { _query_name_text(name): sql for name, sql in settings.legacy_allowed_queries.items() } if settings.legacy_project_query: queries.setdefault(LegacyQueryName.PROJECTS.value, settings.legacy_project_query) if settings.legacy_task_query: queries.setdefault(LegacyQueryName.TASKS.value, settings.legacy_task_query) return queries def health(self) -> dict[str, str]: engine = self._ensure_engine() try: with engine.connect() as conn: conn.execute(text(LEGACY_HEALTH_SQL)) except SQLAlchemyError as exc: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=LEGACY_MYSQL_CONNECTION_FAILED_TEMPLATE.format(error=exc), ) from exc return {LegacyResponseKey.STATUS: ApiStatus.OK} def execute_readonly( self, sql: str, params: dict[str, Any] | None = None, limit: int = 100, ) -> dict[str, Any]: """Execute a readonly SQL statement only when it matches the allowlist.""" normalized_sql = _normalize_sql(sql) for allowed_sql in self._allowed_queries().values(): if _normalize_sql(allowed_sql) == normalized_sql: return self._execute_readonly_sql(allowed_sql, params, limit) raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=LegacyQueryError.QUERY_NOT_ALLOWED, ) def execute_allowed_query( self, query_name: str | None, params: dict[str, Any] | None = None, limit: int = 100, ) -> dict[str, Any]: queries = self._allowed_queries() normalized_name = _query_name_text(query_name) sql = queries.get(normalized_name) if not sql: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=LegacyQueryError.QUERY_NOT_ALLOWED, ) return self._execute_readonly_sql(sql, params, limit) def _execute_readonly_sql( self, sql: str, params: dict[str, Any] | None = None, limit: int = 100, ) -> dict[str, Any]: self._ensure_readonly(sql) engine = self._ensure_engine() params = dict(params or {}) try: params[LegacyResponseKey.LIMIT] = bounded_limit( params.get(LegacyResponseKey.LIMIT, limit) ) except (TypeError, ValueError) as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=LegacyQueryError.INVALID_LIMIT, ) from exc limited_sql = sql if LEGACY_LIMIT_MARKER not in sql.lower(): limited_sql = f"{sql.rstrip(LEGACY_SQL_TRAILING_TERMINATOR)}{LEGACY_LIMIT_CLAUSE}" with engine.connect() as conn: result = conn.execute(text(limited_sql), params) rows = [_row_to_dict(row) for row in result.mappings().all()] columns = list(rows[0].keys()) if rows else [] return { LegacyResponseKey.COLUMNS: columns, LegacyResponseKey.ROWS: rows, LegacyResponseKey.ROW_COUNT: len(rows), } def fetch_default_projects(self, limit: int = 100) -> dict[str, Any]: settings = get_settings() if not settings.legacy_project_query: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=LegacyQueryError.PROJECT_QUERY_NOT_CONFIGURED, ) return self.execute_allowed_query( LegacyQueryName.PROJECTS, {LegacyResponseKey.LIMIT: limit}, limit=limit, ) def fetch_default_tasks(self, limit: int = 100) -> dict[str, Any]: settings = get_settings() if not settings.legacy_task_query: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=LegacyQueryError.TASK_QUERY_NOT_CONFIGURED, ) return self.execute_allowed_query( LegacyQueryName.TASKS, {LegacyResponseKey.LIMIT: limit}, limit=limit, ) @staticmethod def _value( row: dict[str, Any], field_map: dict[str, str], internal_name: str, fallback: Any = None, ) -> Any: source_name = field_map.get(internal_name, internal_name) if source_name in row: return row[source_name] return fallback def _project_payload(self, row: dict[str, Any], field_map: dict[str, str]) -> dict[str, Any]: settings = get_settings() external_id = self._value( row, field_map, LegacyProjectField.EXTERNAL_ID, row.get(LegacyProjectField.ID), ) raw_code = self._value(row, field_map, LegacyProjectField.CODE, None) code = None if raw_code: code = str(raw_code) elif external_id is not None: code = LEGACY_PROJECT_CODE_TEMPLATE.format( prefix=settings.legacy_project_code_prefix, external_id=external_id, ) return { LegacyProjectField.CODE: code, LegacyProjectField.EXTERNAL_ID: str(external_id) if external_id is not None else code, LegacyProjectField.SOURCE_SYSTEM: SourceSystem.LEGACY_MYSQL, LegacyProjectField.NAME: self._value( row, field_map, LegacyProjectField.NAME, LEGACY_UNNAMED_PROJECT, ), LegacyProjectField.OWNER: self._value(row, field_map, LegacyProjectField.OWNER, None), LegacyProjectField.STATUS: self._value(row, field_map, LegacyProjectField.STATUS, StatusValue.UNKNOWN), LegacyProjectField.PROGRESS_PERCENT: int( self._value( row, field_map, LegacyProjectField.PROGRESS_PERCENT, row.get(LegacyProjectField.PROGRESS) or 0, ) or 0 ), LegacyProjectField.START_DATE: self._value(row, field_map, LegacyProjectField.START_DATE, None), LegacyProjectField.DUE_DATE: self._value(row, field_map, LegacyProjectField.DUE_DATE, None), LegacyProjectField.BUDGET_AMOUNT: ( self._value(row, field_map, LegacyProjectField.BUDGET_AMOUNT, row.get(LegacyProjectField.BUDGET) or 0) or 0 ), LegacyProjectField.ACTUAL_AMOUNT: ( self._value(row, field_map, LegacyProjectField.ACTUAL_AMOUNT, row.get(LegacyProjectField.ACTUAL_COST) or 0) or 0 ), LegacyProjectField.DESCRIPTION: self._value(row, field_map, LegacyProjectField.DESCRIPTION, None), } def _task_payload(self, row: dict[str, Any], field_map: dict[str, str]) -> dict[str, Any]: settings = get_settings() external_id = self._value( row, field_map, LegacyTaskField.EXTERNAL_ID, row.get(LegacyTaskField.ID), ) raw_code = self._value(row, field_map, LegacyTaskField.CODE, None) code = None if raw_code: code = str(raw_code) elif external_id is not None: code = LEGACY_TASK_CODE_TEMPLATE.format( prefix=settings.legacy_task_code_prefix, external_id=external_id, ) return { LegacyTaskField.CODE: code, LegacyTaskField.EXTERNAL_ID: str(external_id) if external_id is not None else code, LegacyTaskField.SOURCE_SYSTEM: SourceSystem.LEGACY_MYSQL, LegacyTaskField.TITLE: self._value( row, field_map, LegacyTaskField.TITLE, LEGACY_UNNAMED_TASK, ), LegacyTaskField.PROJECT_CODE: self._value( row, field_map, LegacyTaskField.PROJECT_CODE, None, ), LegacyTaskField.OWNER: self._value(row, field_map, LegacyTaskField.OWNER, None), LegacyTaskField.STATUS: self._value( row, field_map, LegacyTaskField.STATUS, StatusValue.TODO, ), LegacyTaskField.PRIORITY: self._value( row, field_map, LegacyTaskField.PRIORITY, "P2", ), LegacyTaskField.DUE_DATE: self._value(row, field_map, LegacyTaskField.DUE_DATE, None), LegacyTaskField.COMPLETED_AT: self._value( row, field_map, LegacyTaskField.COMPLETED_AT, None, ), LegacyTaskField.BLOCKER: self._value(row, field_map, LegacyTaskField.BLOCKER, None), LegacyTaskField.DESCRIPTION: self._value( row, field_map, LegacyTaskField.DESCRIPTION, None, ), } def sync_projects( self, source_query: str | None = None, source_query_name: str | None = None, field_map: dict[str, str] | None = None, limit: int = 100, dry_run: bool = True, actor: str = ActorValue.API, ) -> dict[str, Any]: if self.db is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=LegacyQueryError.APP_DB_UNAVAILABLE, ) query_name = source_query_name or LegacyQueryName.PROJECTS if source_query: rows = self.execute_readonly( source_query, {LegacyResponseKey.LIMIT: limit}, limit=limit, )[LegacyResponseKey.ROWS] query_ref = LegacySyncAction.ALLOWLISTED_INLINE_SQL else: rows = self.execute_allowed_query( query_name, {LegacyResponseKey.LIMIT: limit}, limit=limit, )[LegacyResponseKey.ROWS] query_ref = _query_name_text(query_name) field_map = field_map or {} created = 0 updated = 0 skipped = 0 items: list[dict[str, Any]] = [] for row in rows: payload = self._project_payload(row, field_map) if not payload[LegacyProjectField.EXTERNAL_ID] and not payload[LegacyProjectField.CODE]: skipped += 1 items.append( { LegacyResponseKey.ACTION: LegacySyncAction.SKIPPED, LegacyResponseKey.REASON: LEGACY_SYNC_MISSING_ID_REASON, LegacyResponseKey.SOURCE: row, } ) continue stmt = select(Project).where( Project.source_system == SourceSystem.LEGACY_MYSQL, Project.external_id == payload[LegacyProjectField.EXTERNAL_ID], ) record = self.db.execute(stmt).scalar_one_or_none() if record is None: record = self.db.execute( select(Project).where(Project.code == payload[LegacyProjectField.CODE]) ).scalar_one_or_none() if record is None: created += 1 action = LegacySyncAction.CREATE result = payload if not dry_run: record = Project(**payload) self.db.add(record) self.db.flush() result = serialize_model(record) else: updated += 1 action = LegacySyncAction.UPDATE if not dry_run: for key, value in payload.items(): setattr(record, key, value) self.db.flush() result = serialize_model(record) else: result = payload items.append( { LegacyResponseKey.ACTION: action, LegacyResponseKey.PROJECT: result, LegacyResponseKey.SOURCE: row, } ) if not dry_run: self.db.commit() result = { LegacyResponseKey.DRY_RUN: dry_run, LegacyResponseKey.CREATED: created, LegacyResponseKey.UPDATED: updated, LegacyResponseKey.SKIPPED: skipped, LegacyResponseKey.ITEMS: items, } sync_run = LegacySyncRun( code=f"{LEGACY_SYNC_RUN_CODE_PREFIX}-{utc_now():%Y%m%d%H%M%S%f}", domain=BusinessDomain.PROJECTS, source_table=LEGACY_PROJECT_QUERY_SOURCE, status=StatusValue.DRY_RUN if dry_run else AuditStatus.SUCCESS, finished_at=utc_now(), created_count=created, updated_count=updated, skipped_count=skipped, note=LEGACY_PROJECT_SYNC_NOTE, ) self.db.add(sync_run) self.db.commit() self.db.refresh(sync_run) result[LegacyResponseKey.SYNC_RUN_CODE] = sync_run.code AuditService(self.db).log( AuditLogCreate( actor=actor, source=AuditSource.LEGACY_MYSQL, action=AuditAction.LEGACY_SYNC_PROJECTS, target_type=BusinessDomain.PROJECTS, risk_level=AuditRiskLevel.MEDIUM, request_payload={ LegacyResponseKey.SOURCE_QUERY: query_ref, LegacyResponseKey.FIELD_MAP: field_map, LegacyResponseKey.LIMIT: limit, LegacyResponseKey.DRY_RUN: dry_run, }, response_payload={ key: result[key] for key in [ LegacyResponseKey.DRY_RUN, LegacyResponseKey.CREATED, LegacyResponseKey.UPDATED, LegacyResponseKey.SKIPPED, ] }, ) ) EventService(self.db).emit( event_type=EventType.LEGACY_SYNC_COMPLETED, source=EventSource.LEGACY_MYSQL, aggregate_type=EventAggregateType.LEGACY_SYNC_RUN, aggregate_id=sync_run.code, actor=actor, payload={ EventPayloadKey.CODE: sync_run.code, EventPayloadKey.DOMAIN: BusinessDomain.PROJECTS, EventPayloadKey.STATUS: sync_run.status, EventPayloadKey.CREATED: created, EventPayloadKey.UPDATED: updated, EventPayloadKey.SKIPPED: skipped, }, idempotency_key=f"legacy-sync:{sync_run.code}", ) return result def sync_tasks( self, source_query: str | None = None, source_query_name: str | None = None, field_map: dict[str, str] | None = None, limit: int = 100, dry_run: bool = True, actor: str = ActorValue.API, ) -> dict[str, Any]: if self.db is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=LegacyQueryError.APP_DB_UNAVAILABLE, ) query_name = source_query_name or LegacyQueryName.TASKS if source_query: rows = self.execute_readonly( source_query, {LegacyResponseKey.LIMIT: limit}, limit=limit, )[LegacyResponseKey.ROWS] query_ref = LegacySyncAction.ALLOWLISTED_INLINE_SQL else: rows = self.execute_allowed_query( query_name, {LegacyResponseKey.LIMIT: limit}, limit=limit, )[LegacyResponseKey.ROWS] query_ref = _query_name_text(query_name) field_map = field_map or {} created = 0 updated = 0 skipped = 0 items: list[dict[str, Any]] = [] for row in rows: payload = self._task_payload(row, field_map) if not payload[LegacyTaskField.EXTERNAL_ID] and not payload[LegacyTaskField.CODE]: skipped += 1 items.append( { LegacyResponseKey.ACTION: LegacySyncAction.SKIPPED, LegacyResponseKey.REASON: LEGACY_SYNC_MISSING_ID_REASON, LegacyResponseKey.SOURCE: row, } ) continue stmt = select(WorkTask).where( WorkTask.source_system == SourceSystem.LEGACY_MYSQL, WorkTask.external_id == payload[LegacyTaskField.EXTERNAL_ID], ) record = self.db.execute(stmt).scalar_one_or_none() if record is None: record = self.db.execute( select(WorkTask).where(WorkTask.code == payload[LegacyTaskField.CODE]) ).scalar_one_or_none() if record is None: created += 1 action = LegacySyncAction.CREATE result = payload if not dry_run: record = WorkTask(**payload) self.db.add(record) self.db.flush() result = serialize_model(record) else: updated += 1 action = LegacySyncAction.UPDATE if not dry_run: for key, value in payload.items(): setattr(record, key, value) self.db.flush() result = serialize_model(record) else: result = payload items.append( { LegacyResponseKey.ACTION: action, LegacyResponseKey.TASK: result, LegacyResponseKey.SOURCE: row, } ) if not dry_run: self.db.commit() result = { LegacyResponseKey.DRY_RUN: dry_run, LegacyResponseKey.CREATED: created, LegacyResponseKey.UPDATED: updated, LegacyResponseKey.SKIPPED: skipped, LegacyResponseKey.ITEMS: items, } sync_run = LegacySyncRun( code=f"{LEGACY_TASK_SYNC_RUN_CODE_PREFIX}-{utc_now():%Y%m%d%H%M%S%f}", domain=BusinessDomain.TASKS, source_table=LEGACY_TASK_QUERY_SOURCE, status=StatusValue.DRY_RUN if dry_run else AuditStatus.SUCCESS, finished_at=utc_now(), created_count=created, updated_count=updated, skipped_count=skipped, note=LEGACY_TASK_SYNC_NOTE, ) self.db.add(sync_run) self.db.commit() self.db.refresh(sync_run) result[LegacyResponseKey.SYNC_RUN_CODE] = sync_run.code AuditService(self.db).log( AuditLogCreate( actor=actor, source=AuditSource.LEGACY_MYSQL, action=AuditAction.LEGACY_SYNC_TASKS, target_type=BusinessDomain.TASKS, risk_level=AuditRiskLevel.MEDIUM, request_payload={ LegacyResponseKey.SOURCE_QUERY: query_ref, LegacyResponseKey.FIELD_MAP: field_map, LegacyResponseKey.LIMIT: limit, LegacyResponseKey.DRY_RUN: dry_run, }, response_payload={ key: result[key] for key in [ LegacyResponseKey.DRY_RUN, LegacyResponseKey.CREATED, LegacyResponseKey.UPDATED, LegacyResponseKey.SKIPPED, ] }, ) ) EventService(self.db).emit( event_type=EventType.LEGACY_SYNC_COMPLETED, source=EventSource.LEGACY_MYSQL, aggregate_type=EventAggregateType.LEGACY_SYNC_RUN, aggregate_id=sync_run.code, actor=actor, payload={ EventPayloadKey.CODE: sync_run.code, EventPayloadKey.DOMAIN: BusinessDomain.TASKS, EventPayloadKey.STATUS: sync_run.status, EventPayloadKey.CREATED: created, EventPayloadKey.UPDATED: updated, EventPayloadKey.SKIPPED: skipped, }, idempotency_key=f"legacy-sync:{sync_run.code}", ) return result