```
refactor(core): 重构核心模块结构并更新导入路径 - 将配置相关的设置从 app.core.config 移除 - 将常量定义从 app.core.constants 移除 - 将数据库相关功能从 app.core.database 移除 - 将基础数据库模型从 app.core.db_base 移除 - 将敏感信息掩码功能从 app.core.masking 移除 - 将中间件定义从 app.core.middleware 移除 - 将操作保护功能从 app.core.operation_guard 移除 - 将分页工具从 app.core.pagination 移除 - 将请求上下文管理从 app.core.request_context 移除 - 将调度器功能从 app.core.scheduler 移除 - 将安全认证逻辑从 app.core.security 移除 - 将任务队列相关功能从 app.core.task_queue 移除 - 将时间工具从 app.core.time 移除 - 更新 alembic 配置中的 Base 模型导入路径 - 更新各模块中对重构后组件的引用路径 ```
This commit is contained in:
1
app/core/background/__init__.py
Normal file
1
app/core/background/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Background scheduler and queue helpers."""
|
||||
185
app/core/background/scheduler.py
Normal file
185
app/core/background/scheduler.py
Normal file
@@ -0,0 +1,185 @@
|
||||
from socket import gethostname
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.constants import ActorValue
|
||||
from app.modules.feishu.constants import FeishuReceiveIdType
|
||||
from app.modules.observability.constants import HeartbeatComponent
|
||||
|
||||
|
||||
def attach_scheduler(app: FastAPI) -> None:
|
||||
"""Attach optional APScheduler jobs to the FastAPI application."""
|
||||
|
||||
settings = get_settings()
|
||||
if not settings.scheduler_enabled:
|
||||
return
|
||||
|
||||
scheduler = create_scheduler(app)
|
||||
|
||||
@app.on_event("startup")
|
||||
def start_scheduler() -> None:
|
||||
scheduler.start()
|
||||
|
||||
@app.on_event("shutdown")
|
||||
def stop_scheduler() -> None:
|
||||
scheduler.shutdown(wait=False)
|
||||
|
||||
|
||||
def create_scheduler(app: FastAPI | None = None) -> Any:
|
||||
"""Create the V2/V3 scheduler without requiring a FastAPI process."""
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from app.core.database import SessionLocal
|
||||
from app.core.background.task_queue import (
|
||||
enqueue_daily_brief_push,
|
||||
enqueue_event_dispatch,
|
||||
enqueue_legacy_project_sync,
|
||||
enqueue_legacy_task_sync,
|
||||
enqueue_project_weekly_push,
|
||||
)
|
||||
from app.modules.observability.service import ObservabilityService
|
||||
from app.modules.reports.service import ReportService
|
||||
|
||||
settings = get_settings()
|
||||
scheduler = BackgroundScheduler(timezone="Asia/Shanghai")
|
||||
|
||||
def run_daily_brief() -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
report = ReportService(db).daily_brief()
|
||||
_set_state(app, "last_daily_brief", report)
|
||||
if (
|
||||
settings.feishu_app_id
|
||||
and settings.feishu_app_secret
|
||||
and settings.feishu_default_chat_id
|
||||
):
|
||||
if settings.task_queue_enabled:
|
||||
dispatch = enqueue_daily_brief_push(
|
||||
receive_id=settings.feishu_default_chat_id,
|
||||
receive_id_type=FeishuReceiveIdType.CHAT_ID,
|
||||
actor=ActorValue.SCHEDULER,
|
||||
)
|
||||
_set_state(app, "last_daily_brief_dispatch", dispatch)
|
||||
return
|
||||
ReportService(db).push_report(
|
||||
report,
|
||||
receive_id=settings.feishu_default_chat_id,
|
||||
receive_id_type=FeishuReceiveIdType.CHAT_ID,
|
||||
actor=ActorValue.SCHEDULER,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def run_project_weekly() -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
report = ReportService(db).project_weekly()
|
||||
_set_state(app, "last_project_weekly", report)
|
||||
if (
|
||||
settings.feishu_app_id
|
||||
and settings.feishu_app_secret
|
||||
and settings.feishu_default_chat_id
|
||||
):
|
||||
if settings.task_queue_enabled:
|
||||
dispatch = enqueue_project_weekly_push(
|
||||
receive_id=settings.feishu_default_chat_id,
|
||||
receive_id_type=FeishuReceiveIdType.CHAT_ID,
|
||||
actor=ActorValue.SCHEDULER,
|
||||
)
|
||||
_set_state(app, "last_project_weekly_dispatch", dispatch)
|
||||
return
|
||||
ReportService(db).push_report(
|
||||
report,
|
||||
receive_id=settings.feishu_default_chat_id,
|
||||
receive_id_type=FeishuReceiveIdType.CHAT_ID,
|
||||
actor=ActorValue.SCHEDULER,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def run_legacy_project_sync() -> None:
|
||||
dispatch = enqueue_legacy_project_sync(actor=ActorValue.SCHEDULER)
|
||||
_set_state(app, "last_legacy_project_sync_dispatch", dispatch)
|
||||
|
||||
def run_legacy_task_sync() -> None:
|
||||
dispatch = enqueue_legacy_task_sync(actor=ActorValue.SCHEDULER)
|
||||
_set_state(app, "last_legacy_task_sync_dispatch", dispatch)
|
||||
|
||||
def run_event_dispatch() -> None:
|
||||
dispatch = enqueue_event_dispatch(
|
||||
limit=settings.event_dispatch_batch_size,
|
||||
actor=ActorValue.SCHEDULER,
|
||||
)
|
||||
_set_state(app, "last_event_dispatch", dispatch)
|
||||
|
||||
def record_scheduler_heartbeat() -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
heartbeat = ObservabilityService(db).record_heartbeat(
|
||||
component=HeartbeatComponent.SCHEDULER,
|
||||
instance_id=gethostname(),
|
||||
actor=ActorValue.SCHEDULER,
|
||||
)
|
||||
_set_state(app, "last_scheduler_heartbeat", heartbeat)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
scheduler.add_job(
|
||||
run_daily_brief,
|
||||
trigger="cron",
|
||||
hour=settings.daily_brief_cron_hour,
|
||||
minute=settings.daily_brief_cron_minute,
|
||||
id="daily_brief_push",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
run_project_weekly,
|
||||
trigger="cron",
|
||||
day_of_week=settings.weekly_project_report_day_of_week,
|
||||
hour=settings.weekly_project_report_cron_hour,
|
||||
minute=settings.weekly_project_report_cron_minute,
|
||||
id="project_weekly_push",
|
||||
replace_existing=True,
|
||||
)
|
||||
if settings.event_dispatch_enabled:
|
||||
scheduler.add_job(
|
||||
run_event_dispatch,
|
||||
trigger="cron",
|
||||
minute=settings.event_dispatch_cron_minute,
|
||||
id="event_dispatch",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
record_scheduler_heartbeat,
|
||||
trigger="interval",
|
||||
seconds=settings.heartbeat_interval_seconds,
|
||||
id="scheduler_heartbeat",
|
||||
replace_existing=True,
|
||||
)
|
||||
if settings.legacy_sync_enabled and settings.legacy_project_query:
|
||||
scheduler.add_job(
|
||||
run_legacy_project_sync,
|
||||
trigger="cron",
|
||||
hour=settings.legacy_project_sync_cron_hour,
|
||||
minute=settings.legacy_project_sync_cron_minute,
|
||||
id="legacy_project_sync",
|
||||
replace_existing=True,
|
||||
)
|
||||
if settings.legacy_sync_enabled and settings.legacy_task_query:
|
||||
scheduler.add_job(
|
||||
run_legacy_task_sync,
|
||||
trigger="cron",
|
||||
hour=settings.legacy_task_sync_cron_hour,
|
||||
minute=settings.legacy_task_sync_cron_minute,
|
||||
id="legacy_task_sync",
|
||||
replace_existing=True,
|
||||
)
|
||||
return scheduler
|
||||
|
||||
|
||||
def _set_state(app: FastAPI | None, key: str, value: Any) -> None:
|
||||
if app is not None:
|
||||
setattr(app.state, key, value)
|
||||
295
app/core/background/task_queue.py
Normal file
295
app/core/background/task_queue.py
Normal file
@@ -0,0 +1,295 @@
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.modules.feishu.constants import FeishuReceiveIdType
|
||||
|
||||
TASK_PUSH_DAILY_BRIEF = "reports.push_daily_brief"
|
||||
TASK_PUSH_PROJECT_WEEKLY = "reports.push_project_weekly"
|
||||
TASK_GENERATE_RISK_EVENTS = "risks.generate_events"
|
||||
TASK_SYNC_LEGACY_PROJECTS = "legacy.sync_projects"
|
||||
TASK_SYNC_LEGACY_TASKS = "legacy.sync_tasks"
|
||||
TASK_DISPATCH_PENDING_EVENTS = "events.dispatch_pending"
|
||||
|
||||
|
||||
def dispatch_task(
|
||||
task_name: str,
|
||||
kwargs: dict[str, Any],
|
||||
inline: Callable[[], Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Dispatch a task through Celery when enabled, otherwise run inline."""
|
||||
|
||||
settings = get_settings()
|
||||
if settings.task_queue_enabled:
|
||||
from app.tasks import celery_app
|
||||
|
||||
async_result = celery_app.signature(task_name, kwargs=kwargs).apply_async()
|
||||
return {
|
||||
"queued": True,
|
||||
"mode": "celery",
|
||||
"task_name": task_name,
|
||||
"task_id": async_result.id,
|
||||
}
|
||||
return {
|
||||
"queued": False,
|
||||
"mode": "inline",
|
||||
"task_name": task_name,
|
||||
"result": inline(),
|
||||
}
|
||||
|
||||
|
||||
def enqueue_daily_brief_push(
|
||||
receive_id: str | None = None,
|
||||
receive_id_type: str = FeishuReceiveIdType.CHAT_ID,
|
||||
actor: str = "scheduler",
|
||||
) -> dict[str, Any]:
|
||||
settings = get_settings()
|
||||
if settings.task_queue_enabled:
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.reports.constants import ReportPushStatus, ReportTitle, ReportType
|
||||
from app.modules.reports.service import ReportService
|
||||
from app.tasks import celery_app
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
push_run = ReportService(db).create_push_run(
|
||||
report_type=ReportType.DAILY,
|
||||
title=ReportTitle.DAILY_BRIEF,
|
||||
receive_id=receive_id,
|
||||
receive_id_type=receive_id_type,
|
||||
actor=actor,
|
||||
status=ReportPushStatus.QUEUED,
|
||||
)
|
||||
async_result = celery_app.signature(
|
||||
TASK_PUSH_DAILY_BRIEF,
|
||||
kwargs={
|
||||
"receive_id": receive_id,
|
||||
"receive_id_type": receive_id_type,
|
||||
"actor": actor,
|
||||
"push_run_code": push_run.code,
|
||||
},
|
||||
).apply_async()
|
||||
ReportService(db).update_push_run(
|
||||
push_run.code,
|
||||
ReportPushStatus.QUEUED,
|
||||
task_id=async_result.id,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
return {
|
||||
"queued": True,
|
||||
"mode": "celery",
|
||||
"task_name": TASK_PUSH_DAILY_BRIEF,
|
||||
"task_id": async_result.id,
|
||||
"push_run_code": push_run.code,
|
||||
}
|
||||
|
||||
def inline() -> Any:
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.reports.service import ReportService
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
report = ReportService(db).daily_brief()
|
||||
return ReportService(db).push_report(report, receive_id, receive_id_type, actor)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return dispatch_task(
|
||||
TASK_PUSH_DAILY_BRIEF,
|
||||
{
|
||||
"receive_id": receive_id,
|
||||
"receive_id_type": receive_id_type,
|
||||
"actor": actor,
|
||||
},
|
||||
inline,
|
||||
)
|
||||
|
||||
|
||||
def enqueue_project_weekly_push(
|
||||
receive_id: str | None = None,
|
||||
receive_id_type: str = FeishuReceiveIdType.CHAT_ID,
|
||||
actor: str = "scheduler",
|
||||
) -> dict[str, Any]:
|
||||
settings = get_settings()
|
||||
if settings.task_queue_enabled:
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.reports.constants import ReportPushStatus, ReportTitle, ReportType
|
||||
from app.modules.reports.service import ReportService
|
||||
from app.tasks import celery_app
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
push_run = ReportService(db).create_push_run(
|
||||
report_type=ReportType.WEEKLY,
|
||||
title=ReportTitle.PROJECT_WEEKLY,
|
||||
receive_id=receive_id,
|
||||
receive_id_type=receive_id_type,
|
||||
actor=actor,
|
||||
status=ReportPushStatus.QUEUED,
|
||||
)
|
||||
async_result = celery_app.signature(
|
||||
TASK_PUSH_PROJECT_WEEKLY,
|
||||
kwargs={
|
||||
"receive_id": receive_id,
|
||||
"receive_id_type": receive_id_type,
|
||||
"actor": actor,
|
||||
"push_run_code": push_run.code,
|
||||
},
|
||||
).apply_async()
|
||||
ReportService(db).update_push_run(
|
||||
push_run.code,
|
||||
ReportPushStatus.QUEUED,
|
||||
task_id=async_result.id,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
return {
|
||||
"queued": True,
|
||||
"mode": "celery",
|
||||
"task_name": TASK_PUSH_PROJECT_WEEKLY,
|
||||
"task_id": async_result.id,
|
||||
"push_run_code": push_run.code,
|
||||
}
|
||||
|
||||
def inline() -> Any:
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.reports.service import ReportService
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
report = ReportService(db).project_weekly()
|
||||
return ReportService(db).push_report(report, receive_id, receive_id_type, actor)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return dispatch_task(
|
||||
TASK_PUSH_PROJECT_WEEKLY,
|
||||
{
|
||||
"receive_id": receive_id,
|
||||
"receive_id_type": receive_id_type,
|
||||
"actor": actor,
|
||||
},
|
||||
inline,
|
||||
)
|
||||
|
||||
|
||||
def enqueue_risk_event_generation(actor: str = "scheduler") -> dict[str, Any]:
|
||||
def inline() -> Any:
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.risk.service import RiskService
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
return RiskService(db).generate_events(actor=actor)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return dispatch_task(
|
||||
TASK_GENERATE_RISK_EVENTS,
|
||||
{"actor": actor},
|
||||
inline,
|
||||
)
|
||||
|
||||
|
||||
def enqueue_event_dispatch(
|
||||
limit: int | None = None,
|
||||
actor: str = "scheduler",
|
||||
) -> dict[str, Any]:
|
||||
def inline() -> Any:
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.events.service import EventService
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
return EventService(db).dispatch_pending(
|
||||
limit=limit or get_settings().event_dispatch_batch_size,
|
||||
worker_id=actor,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return dispatch_task(
|
||||
TASK_DISPATCH_PENDING_EVENTS,
|
||||
{"limit": limit, "actor": actor},
|
||||
inline,
|
||||
)
|
||||
|
||||
|
||||
def enqueue_legacy_project_sync(
|
||||
source_query: str | None = None,
|
||||
source_query_name: str | None = None,
|
||||
field_map: dict[str, str] | None = None,
|
||||
limit: int = 100,
|
||||
dry_run: bool = False,
|
||||
actor: str = "scheduler",
|
||||
) -> dict[str, Any]:
|
||||
def inline() -> Any:
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.legacy_mysql.service import LegacyMySQLService
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
return LegacyMySQLService(db).sync_projects(
|
||||
source_query=source_query,
|
||||
source_query_name=source_query_name,
|
||||
field_map=field_map or {},
|
||||
limit=limit,
|
||||
dry_run=dry_run,
|
||||
actor=actor,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return dispatch_task(
|
||||
TASK_SYNC_LEGACY_PROJECTS,
|
||||
{
|
||||
"source_query": source_query,
|
||||
"source_query_name": source_query_name,
|
||||
"field_map": field_map or {},
|
||||
"limit": limit,
|
||||
"dry_run": dry_run,
|
||||
"actor": actor,
|
||||
},
|
||||
inline,
|
||||
)
|
||||
|
||||
|
||||
def enqueue_legacy_task_sync(
|
||||
source_query: str | None = None,
|
||||
source_query_name: str | None = None,
|
||||
field_map: dict[str, str] | None = None,
|
||||
limit: int = 100,
|
||||
dry_run: bool = False,
|
||||
actor: str = "scheduler",
|
||||
) -> dict[str, Any]:
|
||||
def inline() -> Any:
|
||||
from app.core.database import SessionLocal
|
||||
from app.modules.legacy_mysql.service import LegacyMySQLService
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
return LegacyMySQLService(db).sync_tasks(
|
||||
source_query=source_query,
|
||||
source_query_name=source_query_name,
|
||||
field_map=field_map or {},
|
||||
limit=limit,
|
||||
dry_run=dry_run,
|
||||
actor=actor,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return dispatch_task(
|
||||
TASK_SYNC_LEGACY_TASKS,
|
||||
{
|
||||
"source_query": source_query,
|
||||
"source_query_name": source_query_name,
|
||||
"field_map": field_map or {},
|
||||
"limit": limit,
|
||||
"dry_run": dry_run,
|
||||
"actor": actor,
|
||||
},
|
||||
inline,
|
||||
)
|
||||
Reference in New Issue
Block a user