```
refactor(core,ai): 调整模块导入路径并移除废弃文件 - 修复 scheduler.py 中的导入路径错误,将 reports.service 改为 reports.services - 移除废弃的 app/core/background/task_queue.py 文件 - 移除废弃的 app/modules/ai_agent/adapters.py 文件 - 修复 ai_memory/service.py 中的导入路径错误,将 events.service 改为 events.services ```
This commit is contained in:
4
app/modules/legacy_mysql/services/__init__.py
Normal file
4
app/modules/legacy_mysql/services/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
from app.modules.legacy_mysql.services.service import LegacyMySQLService
|
||||
|
||||
|
||||
__all__ = ["LegacyMySQLService"]
|
||||
48
app/modules/legacy_mysql/services/common.py
Normal file
48
app/modules/legacy_mysql/services/common.py
Normal file
@@ -0,0 +1,48 @@
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.engine import RowMapping
|
||||
|
||||
from app.modules.legacy_mysql.constants import LEGACY_SQL_TRAILING_TERMINATOR, LegacyQueryName
|
||||
|
||||
FORBIDDEN_SQL_TOKENS = {
|
||||
"insert",
|
||||
"update",
|
||||
"delete",
|
||||
"drop",
|
||||
"alter",
|
||||
"truncate",
|
||||
"create",
|
||||
"replace",
|
||||
"grant",
|
||||
"revoke",
|
||||
}
|
||||
|
||||
|
||||
def _jsonable(value: Any) -> Any:
|
||||
"""Convert database scalar values into JSON-friendly values."""
|
||||
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, Decimal):
|
||||
return float(value)
|
||||
return value
|
||||
|
||||
|
||||
def _row_to_dict(row: RowMapping) -> dict[str, Any]:
|
||||
"""Convert a SQLAlchemy row mapping to a serializable dictionary."""
|
||||
|
||||
return {key: _jsonable(value) for key, value in row.items()}
|
||||
|
||||
|
||||
def _normalize_sql(sql: str) -> str:
|
||||
return " ".join(sql.strip().rstrip(LEGACY_SQL_TRAILING_TERMINATOR).split()).lower()
|
||||
|
||||
|
||||
def _query_name_text(query_name: str | LegacyQueryName | None) -> str:
|
||||
if query_name is None:
|
||||
return LegacyQueryName.PROJECTS.value
|
||||
if isinstance(query_name, LegacyQueryName):
|
||||
return query_name.value
|
||||
return str(query_name)
|
||||
138
app/modules/legacy_mysql/services/mappers.py
Normal file
138
app/modules/legacy_mysql/services/mappers.py
Normal file
@@ -0,0 +1,138 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.modules.business.constants import SourceSystem, StatusValue
|
||||
from app.modules.legacy_mysql.constants import (
|
||||
LEGACY_PROJECT_CODE_TEMPLATE,
|
||||
LEGACY_TASK_CODE_TEMPLATE,
|
||||
LEGACY_UNNAMED_PROJECT,
|
||||
LEGACY_UNNAMED_TASK,
|
||||
LegacyProjectField,
|
||||
LegacyTaskField,
|
||||
)
|
||||
|
||||
|
||||
class LegacyMapperMixin:
|
||||
@staticmethod
|
||||
def _value(
|
||||
row: dict[str, Any],
|
||||
field_map: dict[str, str],
|
||||
internal_name: str,
|
||||
fallback: Any = None,
|
||||
) -> Any:
|
||||
source_name = field_map.get(internal_name, internal_name)
|
||||
if source_name in row:
|
||||
return row[source_name]
|
||||
return fallback
|
||||
|
||||
def _project_payload(self, row: dict[str, Any], field_map: dict[str, str]) -> dict[str, Any]:
|
||||
settings = get_settings()
|
||||
external_id = self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyProjectField.EXTERNAL_ID,
|
||||
row.get(LegacyProjectField.ID),
|
||||
)
|
||||
raw_code = self._value(row, field_map, LegacyProjectField.CODE, None)
|
||||
code = None
|
||||
if raw_code:
|
||||
code = str(raw_code)
|
||||
elif external_id is not None:
|
||||
code = LEGACY_PROJECT_CODE_TEMPLATE.format(
|
||||
prefix=settings.legacy_project_code_prefix,
|
||||
external_id=external_id,
|
||||
)
|
||||
return {
|
||||
LegacyProjectField.CODE: code,
|
||||
LegacyProjectField.EXTERNAL_ID: str(external_id) if external_id is not None else code,
|
||||
LegacyProjectField.SOURCE_SYSTEM: SourceSystem.LEGACY_MYSQL,
|
||||
LegacyProjectField.NAME: self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyProjectField.NAME,
|
||||
LEGACY_UNNAMED_PROJECT,
|
||||
),
|
||||
LegacyProjectField.OWNER: self._value(row, field_map, LegacyProjectField.OWNER, None),
|
||||
LegacyProjectField.STATUS: self._value(row, field_map, LegacyProjectField.STATUS, StatusValue.UNKNOWN),
|
||||
LegacyProjectField.PROGRESS_PERCENT: int(
|
||||
self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyProjectField.PROGRESS_PERCENT,
|
||||
row.get(LegacyProjectField.PROGRESS) or 0,
|
||||
)
|
||||
or 0
|
||||
),
|
||||
LegacyProjectField.START_DATE: self._value(row, field_map, LegacyProjectField.START_DATE, None),
|
||||
LegacyProjectField.DUE_DATE: self._value(row, field_map, LegacyProjectField.DUE_DATE, None),
|
||||
LegacyProjectField.BUDGET_AMOUNT: (
|
||||
self._value(row, field_map, LegacyProjectField.BUDGET_AMOUNT, row.get(LegacyProjectField.BUDGET) or 0) or 0
|
||||
),
|
||||
LegacyProjectField.ACTUAL_AMOUNT: (
|
||||
self._value(row, field_map, LegacyProjectField.ACTUAL_AMOUNT, row.get(LegacyProjectField.ACTUAL_COST) or 0) or 0
|
||||
),
|
||||
LegacyProjectField.DESCRIPTION: self._value(row, field_map, LegacyProjectField.DESCRIPTION, None),
|
||||
}
|
||||
|
||||
def _task_payload(self, row: dict[str, Any], field_map: dict[str, str]) -> dict[str, Any]:
|
||||
settings = get_settings()
|
||||
external_id = self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyTaskField.EXTERNAL_ID,
|
||||
row.get(LegacyTaskField.ID),
|
||||
)
|
||||
raw_code = self._value(row, field_map, LegacyTaskField.CODE, None)
|
||||
code = None
|
||||
if raw_code:
|
||||
code = str(raw_code)
|
||||
elif external_id is not None:
|
||||
code = LEGACY_TASK_CODE_TEMPLATE.format(
|
||||
prefix=settings.legacy_task_code_prefix,
|
||||
external_id=external_id,
|
||||
)
|
||||
return {
|
||||
LegacyTaskField.CODE: code,
|
||||
LegacyTaskField.EXTERNAL_ID: str(external_id) if external_id is not None else code,
|
||||
LegacyTaskField.SOURCE_SYSTEM: SourceSystem.LEGACY_MYSQL,
|
||||
LegacyTaskField.TITLE: self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyTaskField.TITLE,
|
||||
LEGACY_UNNAMED_TASK,
|
||||
),
|
||||
LegacyTaskField.PROJECT_CODE: self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyTaskField.PROJECT_CODE,
|
||||
None,
|
||||
),
|
||||
LegacyTaskField.OWNER: self._value(row, field_map, LegacyTaskField.OWNER, None),
|
||||
LegacyTaskField.STATUS: self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyTaskField.STATUS,
|
||||
StatusValue.TODO,
|
||||
),
|
||||
LegacyTaskField.PRIORITY: self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyTaskField.PRIORITY,
|
||||
"P2",
|
||||
),
|
||||
LegacyTaskField.DUE_DATE: self._value(row, field_map, LegacyTaskField.DUE_DATE, None),
|
||||
LegacyTaskField.COMPLETED_AT: self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyTaskField.COMPLETED_AT,
|
||||
None,
|
||||
),
|
||||
LegacyTaskField.BLOCKER: self._value(row, field_map, LegacyTaskField.BLOCKER, None),
|
||||
LegacyTaskField.DESCRIPTION: self._value(
|
||||
row,
|
||||
field_map,
|
||||
LegacyTaskField.DESCRIPTION,
|
||||
None,
|
||||
),
|
||||
}
|
||||
189
app/modules/legacy_mysql/services/project_sync.py
Normal file
189
app/modules/legacy_mysql/services/project_sync.py
Normal file
@@ -0,0 +1,189 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.constants import ActorValue
|
||||
from app.core.utils.time import utc_now
|
||||
from app.modules.audit.constants import AuditAction, AuditRiskLevel, AuditSource, AuditStatus
|
||||
from app.modules.audit.schemas import AuditLogCreate
|
||||
from app.modules.audit.service import AuditService
|
||||
from app.modules.business.constants import BusinessDomain, SourceSystem, StatusValue
|
||||
from app.modules.business.models import LegacySyncRun, Project
|
||||
from app.modules.business.service import serialize_model
|
||||
from app.modules.events.constants import (
|
||||
EventAggregateType,
|
||||
EventPayloadKey,
|
||||
EventSource,
|
||||
EventType,
|
||||
)
|
||||
from app.modules.events.services import EventService
|
||||
from app.modules.legacy_mysql.constants import (
|
||||
LEGACY_PROJECT_QUERY_SOURCE,
|
||||
LEGACY_PROJECT_SYNC_NOTE,
|
||||
LEGACY_SYNC_MISSING_ID_REASON,
|
||||
LEGACY_SYNC_RUN_CODE_PREFIX,
|
||||
LegacyProjectField,
|
||||
LegacyQueryError,
|
||||
LegacyQueryName,
|
||||
LegacyResponseKey,
|
||||
LegacySyncAction,
|
||||
)
|
||||
|
||||
from app.modules.legacy_mysql.services.common import _query_name_text
|
||||
|
||||
|
||||
class LegacyProjectSyncMixin:
|
||||
def sync_projects(
|
||||
self,
|
||||
source_query: str | None = None,
|
||||
source_query_name: str | None = None,
|
||||
field_map: dict[str, str] | None = None,
|
||||
limit: int = 100,
|
||||
dry_run: bool = True,
|
||||
actor: str = ActorValue.API,
|
||||
) -> dict[str, Any]:
|
||||
if self.db is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=LegacyQueryError.APP_DB_UNAVAILABLE,
|
||||
)
|
||||
|
||||
query_name = source_query_name or LegacyQueryName.PROJECTS
|
||||
if source_query:
|
||||
rows = self.execute_readonly(
|
||||
source_query,
|
||||
{LegacyResponseKey.LIMIT: limit},
|
||||
limit=limit,
|
||||
)[LegacyResponseKey.ROWS]
|
||||
query_ref = LegacySyncAction.ALLOWLISTED_INLINE_SQL
|
||||
else:
|
||||
rows = self.execute_allowed_query(
|
||||
query_name,
|
||||
{LegacyResponseKey.LIMIT: limit},
|
||||
limit=limit,
|
||||
)[LegacyResponseKey.ROWS]
|
||||
query_ref = _query_name_text(query_name)
|
||||
field_map = field_map or {}
|
||||
created = 0
|
||||
updated = 0
|
||||
skipped = 0
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
for row in rows:
|
||||
payload = self._project_payload(row, field_map)
|
||||
if not payload[LegacyProjectField.EXTERNAL_ID] and not payload[LegacyProjectField.CODE]:
|
||||
skipped += 1
|
||||
items.append(
|
||||
{
|
||||
LegacyResponseKey.ACTION: LegacySyncAction.SKIPPED,
|
||||
LegacyResponseKey.REASON: LEGACY_SYNC_MISSING_ID_REASON,
|
||||
LegacyResponseKey.SOURCE: row,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
stmt = select(Project).where(
|
||||
Project.source_system == SourceSystem.LEGACY_MYSQL,
|
||||
Project.external_id == payload[LegacyProjectField.EXTERNAL_ID],
|
||||
)
|
||||
record = self.db.execute(stmt).scalar_one_or_none()
|
||||
if record is None:
|
||||
record = self.db.execute(
|
||||
select(Project).where(Project.code == payload[LegacyProjectField.CODE])
|
||||
).scalar_one_or_none()
|
||||
|
||||
if record is None:
|
||||
created += 1
|
||||
action = LegacySyncAction.CREATE
|
||||
result = payload
|
||||
if not dry_run:
|
||||
record = Project(**payload)
|
||||
self.db.add(record)
|
||||
self.db.flush()
|
||||
result = serialize_model(record)
|
||||
else:
|
||||
updated += 1
|
||||
action = LegacySyncAction.UPDATE
|
||||
if not dry_run:
|
||||
for key, value in payload.items():
|
||||
setattr(record, key, value)
|
||||
self.db.flush()
|
||||
result = serialize_model(record)
|
||||
else:
|
||||
result = payload
|
||||
items.append(
|
||||
{
|
||||
LegacyResponseKey.ACTION: action,
|
||||
LegacyResponseKey.PROJECT: result,
|
||||
LegacyResponseKey.SOURCE: row,
|
||||
}
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
self.db.commit()
|
||||
|
||||
result = {
|
||||
LegacyResponseKey.DRY_RUN: dry_run,
|
||||
LegacyResponseKey.CREATED: created,
|
||||
LegacyResponseKey.UPDATED: updated,
|
||||
LegacyResponseKey.SKIPPED: skipped,
|
||||
LegacyResponseKey.ITEMS: items,
|
||||
}
|
||||
sync_run = LegacySyncRun(
|
||||
code=f"{LEGACY_SYNC_RUN_CODE_PREFIX}-{utc_now():%Y%m%d%H%M%S%f}",
|
||||
domain=BusinessDomain.PROJECTS,
|
||||
source_table=LEGACY_PROJECT_QUERY_SOURCE,
|
||||
status=StatusValue.DRY_RUN if dry_run else AuditStatus.SUCCESS,
|
||||
finished_at=utc_now(),
|
||||
created_count=created,
|
||||
updated_count=updated,
|
||||
skipped_count=skipped,
|
||||
note=LEGACY_PROJECT_SYNC_NOTE,
|
||||
)
|
||||
self.db.add(sync_run)
|
||||
self.db.commit()
|
||||
self.db.refresh(sync_run)
|
||||
result[LegacyResponseKey.SYNC_RUN_CODE] = sync_run.code
|
||||
|
||||
AuditService(self.db).log(
|
||||
AuditLogCreate(
|
||||
actor=actor,
|
||||
source=AuditSource.LEGACY_MYSQL,
|
||||
action=AuditAction.LEGACY_SYNC_PROJECTS,
|
||||
target_type=BusinessDomain.PROJECTS,
|
||||
risk_level=AuditRiskLevel.MEDIUM,
|
||||
request_payload={
|
||||
LegacyResponseKey.SOURCE_QUERY: query_ref,
|
||||
LegacyResponseKey.FIELD_MAP: field_map,
|
||||
LegacyResponseKey.LIMIT: limit,
|
||||
LegacyResponseKey.DRY_RUN: dry_run,
|
||||
},
|
||||
response_payload={
|
||||
key: result[key]
|
||||
for key in [
|
||||
LegacyResponseKey.DRY_RUN,
|
||||
LegacyResponseKey.CREATED,
|
||||
LegacyResponseKey.UPDATED,
|
||||
LegacyResponseKey.SKIPPED,
|
||||
]
|
||||
},
|
||||
)
|
||||
)
|
||||
EventService(self.db).emit(
|
||||
event_type=EventType.LEGACY_SYNC_COMPLETED,
|
||||
source=EventSource.LEGACY_MYSQL,
|
||||
aggregate_type=EventAggregateType.LEGACY_SYNC_RUN,
|
||||
aggregate_id=sync_run.code,
|
||||
actor=actor,
|
||||
payload={
|
||||
EventPayloadKey.CODE: sync_run.code,
|
||||
EventPayloadKey.DOMAIN: BusinessDomain.PROJECTS,
|
||||
EventPayloadKey.STATUS: sync_run.status,
|
||||
EventPayloadKey.CREATED: created,
|
||||
EventPayloadKey.UPDATED: updated,
|
||||
EventPayloadKey.SKIPPED: skipped,
|
||||
},
|
||||
idempotency_key=f"legacy-sync:{sync_run.code}",
|
||||
)
|
||||
return result
|
||||
165
app/modules/legacy_mysql/services/query.py
Normal file
165
app/modules/legacy_mysql/services/query.py
Normal file
@@ -0,0 +1,165 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.constants import ApiStatus
|
||||
from app.core.database import legacy_engine
|
||||
from app.core.http.pagination import bounded_limit
|
||||
from app.modules.legacy_mysql.constants import (
|
||||
LEGACY_HEALTH_SQL,
|
||||
LEGACY_LIMIT_CLAUSE,
|
||||
LEGACY_LIMIT_MARKER,
|
||||
LEGACY_MYSQL_CONNECTION_FAILED_TEMPLATE,
|
||||
LEGACY_SELECT_PREFIX,
|
||||
LEGACY_SQL_TRAILING_TERMINATOR,
|
||||
LegacyQueryError,
|
||||
LegacyQueryName,
|
||||
LegacyResponseKey,
|
||||
)
|
||||
|
||||
from app.modules.legacy_mysql.services.common import FORBIDDEN_SQL_TOKENS, _normalize_sql, _query_name_text, _row_to_dict
|
||||
|
||||
|
||||
class LegacyQueryMixin:
|
||||
@staticmethod
|
||||
def _ensure_engine() -> Engine:
|
||||
if legacy_engine is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=LegacyQueryError.DATABASE_NOT_CONFIGURED,
|
||||
)
|
||||
return legacy_engine
|
||||
|
||||
@staticmethod
|
||||
def _ensure_readonly(sql: str) -> None:
|
||||
stripped = sql.strip().lower()
|
||||
if not stripped.startswith(LEGACY_SELECT_PREFIX):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=LegacyQueryError.ONLY_SELECT_ALLOWED,
|
||||
)
|
||||
tokens = {token.strip(" ;,\n\t") for token in stripped.replace("(", " ").split()}
|
||||
if tokens & FORBIDDEN_SQL_TOKENS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=LegacyQueryError.FORBIDDEN_SQL_TOKEN,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _allowed_queries() -> dict[str, str]:
|
||||
settings = get_settings()
|
||||
queries = {
|
||||
_query_name_text(name): sql
|
||||
for name, sql in settings.legacy_allowed_queries.items()
|
||||
}
|
||||
if settings.legacy_project_query:
|
||||
queries.setdefault(LegacyQueryName.PROJECTS.value, settings.legacy_project_query)
|
||||
if settings.legacy_task_query:
|
||||
queries.setdefault(LegacyQueryName.TASKS.value, settings.legacy_task_query)
|
||||
return queries
|
||||
|
||||
def health(self) -> dict[str, str]:
|
||||
engine = self._ensure_engine()
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text(LEGACY_HEALTH_SQL))
|
||||
except SQLAlchemyError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=LEGACY_MYSQL_CONNECTION_FAILED_TEMPLATE.format(error=exc),
|
||||
) from exc
|
||||
return {LegacyResponseKey.STATUS: ApiStatus.OK}
|
||||
|
||||
def execute_readonly(
|
||||
self,
|
||||
sql: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
limit: int = 100,
|
||||
) -> dict[str, Any]:
|
||||
"""Execute a readonly SQL statement only when it matches the allowlist."""
|
||||
|
||||
normalized_sql = _normalize_sql(sql)
|
||||
for allowed_sql in self._allowed_queries().values():
|
||||
if _normalize_sql(allowed_sql) == normalized_sql:
|
||||
return self._execute_readonly_sql(allowed_sql, params, limit)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=LegacyQueryError.QUERY_NOT_ALLOWED,
|
||||
)
|
||||
|
||||
def execute_allowed_query(
|
||||
self,
|
||||
query_name: str | None,
|
||||
params: dict[str, Any] | None = None,
|
||||
limit: int = 100,
|
||||
) -> dict[str, Any]:
|
||||
queries = self._allowed_queries()
|
||||
normalized_name = _query_name_text(query_name)
|
||||
sql = queries.get(normalized_name)
|
||||
if not sql:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=LegacyQueryError.QUERY_NOT_ALLOWED,
|
||||
)
|
||||
return self._execute_readonly_sql(sql, params, limit)
|
||||
|
||||
def _execute_readonly_sql(
|
||||
self,
|
||||
sql: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
limit: int = 100,
|
||||
) -> dict[str, Any]:
|
||||
self._ensure_readonly(sql)
|
||||
engine = self._ensure_engine()
|
||||
params = dict(params or {})
|
||||
try:
|
||||
params[LegacyResponseKey.LIMIT] = bounded_limit(
|
||||
params.get(LegacyResponseKey.LIMIT, limit)
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=LegacyQueryError.INVALID_LIMIT,
|
||||
) from exc
|
||||
limited_sql = sql
|
||||
if LEGACY_LIMIT_MARKER not in sql.lower():
|
||||
limited_sql = f"{sql.rstrip(LEGACY_SQL_TRAILING_TERMINATOR)}{LEGACY_LIMIT_CLAUSE}"
|
||||
with engine.connect() as conn:
|
||||
result = conn.execute(text(limited_sql), params)
|
||||
rows = [_row_to_dict(row) for row in result.mappings().all()]
|
||||
columns = list(rows[0].keys()) if rows else []
|
||||
return {
|
||||
LegacyResponseKey.COLUMNS: columns,
|
||||
LegacyResponseKey.ROWS: rows,
|
||||
LegacyResponseKey.ROW_COUNT: len(rows),
|
||||
}
|
||||
|
||||
def fetch_default_projects(self, limit: int = 100) -> dict[str, Any]:
|
||||
settings = get_settings()
|
||||
if not settings.legacy_project_query:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=LegacyQueryError.PROJECT_QUERY_NOT_CONFIGURED,
|
||||
)
|
||||
return self.execute_allowed_query(
|
||||
LegacyQueryName.PROJECTS,
|
||||
{LegacyResponseKey.LIMIT: limit},
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def fetch_default_tasks(self, limit: int = 100) -> dict[str, Any]:
|
||||
settings = get_settings()
|
||||
if not settings.legacy_task_query:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=LegacyQueryError.TASK_QUERY_NOT_CONFIGURED,
|
||||
)
|
||||
return self.execute_allowed_query(
|
||||
LegacyQueryName.TASKS,
|
||||
{LegacyResponseKey.LIMIT: limit},
|
||||
limit=limit,
|
||||
)
|
||||
18
app/modules/legacy_mysql/services/service.py
Normal file
18
app/modules/legacy_mysql/services/service.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.modules.legacy_mysql.services.mappers import LegacyMapperMixin
|
||||
from app.modules.legacy_mysql.services.project_sync import LegacyProjectSyncMixin
|
||||
from app.modules.legacy_mysql.services.query import LegacyQueryMixin
|
||||
from app.modules.legacy_mysql.services.task_sync import LegacyTaskSyncMixin
|
||||
|
||||
|
||||
class LegacyMySQLService(
|
||||
LegacyTaskSyncMixin,
|
||||
LegacyProjectSyncMixin,
|
||||
LegacyMapperMixin,
|
||||
LegacyQueryMixin,
|
||||
):
|
||||
"""Read legacy MySQL data and sync projects into the internal ledger."""
|
||||
|
||||
def __init__(self, db: Session | None):
|
||||
self.db = db
|
||||
189
app/modules/legacy_mysql/services/task_sync.py
Normal file
189
app/modules/legacy_mysql/services/task_sync.py
Normal file
@@ -0,0 +1,189 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.constants import ActorValue
|
||||
from app.core.utils.time import utc_now
|
||||
from app.modules.audit.constants import AuditAction, AuditRiskLevel, AuditSource, AuditStatus
|
||||
from app.modules.audit.schemas import AuditLogCreate
|
||||
from app.modules.audit.service import AuditService
|
||||
from app.modules.business.constants import BusinessDomain, SourceSystem, StatusValue
|
||||
from app.modules.business.models import LegacySyncRun, WorkTask
|
||||
from app.modules.business.service import serialize_model
|
||||
from app.modules.events.constants import (
|
||||
EventAggregateType,
|
||||
EventPayloadKey,
|
||||
EventSource,
|
||||
EventType,
|
||||
)
|
||||
from app.modules.events.services import EventService
|
||||
from app.modules.legacy_mysql.constants import (
|
||||
LEGACY_SYNC_MISSING_ID_REASON,
|
||||
LEGACY_TASK_QUERY_SOURCE,
|
||||
LEGACY_TASK_SYNC_NOTE,
|
||||
LEGACY_TASK_SYNC_RUN_CODE_PREFIX,
|
||||
LegacyQueryError,
|
||||
LegacyQueryName,
|
||||
LegacyResponseKey,
|
||||
LegacySyncAction,
|
||||
LegacyTaskField,
|
||||
)
|
||||
|
||||
from app.modules.legacy_mysql.services.common import _query_name_text
|
||||
|
||||
|
||||
class LegacyTaskSyncMixin:
|
||||
def sync_tasks(
|
||||
self,
|
||||
source_query: str | None = None,
|
||||
source_query_name: str | None = None,
|
||||
field_map: dict[str, str] | None = None,
|
||||
limit: int = 100,
|
||||
dry_run: bool = True,
|
||||
actor: str = ActorValue.API,
|
||||
) -> dict[str, Any]:
|
||||
if self.db is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=LegacyQueryError.APP_DB_UNAVAILABLE,
|
||||
)
|
||||
|
||||
query_name = source_query_name or LegacyQueryName.TASKS
|
||||
if source_query:
|
||||
rows = self.execute_readonly(
|
||||
source_query,
|
||||
{LegacyResponseKey.LIMIT: limit},
|
||||
limit=limit,
|
||||
)[LegacyResponseKey.ROWS]
|
||||
query_ref = LegacySyncAction.ALLOWLISTED_INLINE_SQL
|
||||
else:
|
||||
rows = self.execute_allowed_query(
|
||||
query_name,
|
||||
{LegacyResponseKey.LIMIT: limit},
|
||||
limit=limit,
|
||||
)[LegacyResponseKey.ROWS]
|
||||
query_ref = _query_name_text(query_name)
|
||||
field_map = field_map or {}
|
||||
created = 0
|
||||
updated = 0
|
||||
skipped = 0
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
for row in rows:
|
||||
payload = self._task_payload(row, field_map)
|
||||
if not payload[LegacyTaskField.EXTERNAL_ID] and not payload[LegacyTaskField.CODE]:
|
||||
skipped += 1
|
||||
items.append(
|
||||
{
|
||||
LegacyResponseKey.ACTION: LegacySyncAction.SKIPPED,
|
||||
LegacyResponseKey.REASON: LEGACY_SYNC_MISSING_ID_REASON,
|
||||
LegacyResponseKey.SOURCE: row,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
stmt = select(WorkTask).where(
|
||||
WorkTask.source_system == SourceSystem.LEGACY_MYSQL,
|
||||
WorkTask.external_id == payload[LegacyTaskField.EXTERNAL_ID],
|
||||
)
|
||||
record = self.db.execute(stmt).scalar_one_or_none()
|
||||
if record is None:
|
||||
record = self.db.execute(
|
||||
select(WorkTask).where(WorkTask.code == payload[LegacyTaskField.CODE])
|
||||
).scalar_one_or_none()
|
||||
|
||||
if record is None:
|
||||
created += 1
|
||||
action = LegacySyncAction.CREATE
|
||||
result = payload
|
||||
if not dry_run:
|
||||
record = WorkTask(**payload)
|
||||
self.db.add(record)
|
||||
self.db.flush()
|
||||
result = serialize_model(record)
|
||||
else:
|
||||
updated += 1
|
||||
action = LegacySyncAction.UPDATE
|
||||
if not dry_run:
|
||||
for key, value in payload.items():
|
||||
setattr(record, key, value)
|
||||
self.db.flush()
|
||||
result = serialize_model(record)
|
||||
else:
|
||||
result = payload
|
||||
items.append(
|
||||
{
|
||||
LegacyResponseKey.ACTION: action,
|
||||
LegacyResponseKey.TASK: result,
|
||||
LegacyResponseKey.SOURCE: row,
|
||||
}
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
self.db.commit()
|
||||
|
||||
result = {
|
||||
LegacyResponseKey.DRY_RUN: dry_run,
|
||||
LegacyResponseKey.CREATED: created,
|
||||
LegacyResponseKey.UPDATED: updated,
|
||||
LegacyResponseKey.SKIPPED: skipped,
|
||||
LegacyResponseKey.ITEMS: items,
|
||||
}
|
||||
sync_run = LegacySyncRun(
|
||||
code=f"{LEGACY_TASK_SYNC_RUN_CODE_PREFIX}-{utc_now():%Y%m%d%H%M%S%f}",
|
||||
domain=BusinessDomain.TASKS,
|
||||
source_table=LEGACY_TASK_QUERY_SOURCE,
|
||||
status=StatusValue.DRY_RUN if dry_run else AuditStatus.SUCCESS,
|
||||
finished_at=utc_now(),
|
||||
created_count=created,
|
||||
updated_count=updated,
|
||||
skipped_count=skipped,
|
||||
note=LEGACY_TASK_SYNC_NOTE,
|
||||
)
|
||||
self.db.add(sync_run)
|
||||
self.db.commit()
|
||||
self.db.refresh(sync_run)
|
||||
result[LegacyResponseKey.SYNC_RUN_CODE] = sync_run.code
|
||||
|
||||
AuditService(self.db).log(
|
||||
AuditLogCreate(
|
||||
actor=actor,
|
||||
source=AuditSource.LEGACY_MYSQL,
|
||||
action=AuditAction.LEGACY_SYNC_TASKS,
|
||||
target_type=BusinessDomain.TASKS,
|
||||
risk_level=AuditRiskLevel.MEDIUM,
|
||||
request_payload={
|
||||
LegacyResponseKey.SOURCE_QUERY: query_ref,
|
||||
LegacyResponseKey.FIELD_MAP: field_map,
|
||||
LegacyResponseKey.LIMIT: limit,
|
||||
LegacyResponseKey.DRY_RUN: dry_run,
|
||||
},
|
||||
response_payload={
|
||||
key: result[key]
|
||||
for key in [
|
||||
LegacyResponseKey.DRY_RUN,
|
||||
LegacyResponseKey.CREATED,
|
||||
LegacyResponseKey.UPDATED,
|
||||
LegacyResponseKey.SKIPPED,
|
||||
]
|
||||
},
|
||||
)
|
||||
)
|
||||
EventService(self.db).emit(
|
||||
event_type=EventType.LEGACY_SYNC_COMPLETED,
|
||||
source=EventSource.LEGACY_MYSQL,
|
||||
aggregate_type=EventAggregateType.LEGACY_SYNC_RUN,
|
||||
aggregate_id=sync_run.code,
|
||||
actor=actor,
|
||||
payload={
|
||||
EventPayloadKey.CODE: sync_run.code,
|
||||
EventPayloadKey.DOMAIN: BusinessDomain.TASKS,
|
||||
EventPayloadKey.STATUS: sync_run.status,
|
||||
EventPayloadKey.CREATED: created,
|
||||
EventPayloadKey.UPDATED: updated,
|
||||
EventPayloadKey.SKIPPED: skipped,
|
||||
},
|
||||
idempotency_key=f"legacy-sync:{sync_run.code}",
|
||||
)
|
||||
return result
|
||||
Reference in New Issue
Block a user