From 0dbe5c1c2d3003163038295996bf6b0c646824f1 Mon Sep 17 00:00:00 2001 From: JiuContinent Date: Mon, 6 Jul 2026 11:03:29 +0800 Subject: [PATCH] =?UTF-8?q?```=20feat(database):=20=E4=BD=BF=E7=94=A8SQLit?= =?UTF-8?q?e=E6=9B=BF=E6=8D=A2MySQL=E4=BD=9C=E4=B8=BA=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将默认数据库从MySQL切换到SQLite以简化本地开发环境配置 BREAKING CHANGE: 数据库连接字符串已从MySQL更改为SQLite格式 --- refactor(audit): 实现敏感数据脱敏功能 添加敏感键名常量定义，并实现递归脱敏函数， 确保审计日志中不会泄露敏感信息如API密钥、密码等 --- fix(legacy-mysql): 修复查询参数限制验证和注入漏洞 增强只读查询参数处理逻辑，添加类型验证并防止SQL注入 同时修复参数限制数值越界问题 --- test(smoke): 增加审计脱敏和审批流程测试用例 添加审计日志脱敏验证测试和审批决策流程测试， 确保敏感数据不会被记录到审计日志中 --- chore(config): 添加Ruff缓存目录到忽略列表 更新.gitignore和.dockerignore文件， 添加.ruff_cache/目录到忽略列表以避免提交临时文件 --- build(deps): 添加Ruff依赖项到环境配置 在environment.yml中添加ruff==0.8.4依赖项， 并在pyproject.toml中配置相关忽略规则 --- refactor(approval): 移除审批创建中的冗余字段 从ApprovalDecision模型中移除不必要的approver字段， 简化审批决策接口设计 --- refactor(database): 明确数据库模块导出接口 为app/core/database.py添加__all__列表， 明确指定模块对外暴露的公共接口 ``` --- .dockerignore | 1 + .gitignore | 1 + app/core/config.py | 2 +- app/core/database.py | 10 +++++ app/modules/approvals/schemas.py | 1 - app/modules/audit/constants.py | 20 ++++++++++ app/modules/audit/service.py | 20 +++++++++- app/modules/legacy_mysql/service.py | 8 +++- environment.yml | 1 + pyproject.toml | 4 ++ tests/test_smoke.py | 61 ++++++++++++++++++++++++++++- 11 files changed, 124 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index 3ed92d2..28a1b76 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ __pycache__/ *.py[cod] .pytest_cache/ +.ruff_cache/ logs/ docs/ diff --git a/.gitignore b/.gitignore index ecf95cb..c9b5504 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__/ *.py[cod] .pytest_cache/ +.ruff_cache/ # Runtime logs /logs/ diff --git a/app/core/config.py b/app/core/config.py index 1dd7e6f..348d24c 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -25,7 +25,7 @@ class Settings(BaseSettings): approval_api_actor: str = ActorValue.APPROVER cors_origins: list[str] = Field(default_factory=lambda: ["*"]) - database_url: str = "mysql+pymysql://root:password@127.0.0.1:3306/company_ai?charset=utf8mb4" + database_url: str = "sqlite:///./company_ai.db" legacy_database_url: str | None = None legacy_project_query: str | None = None legacy_allowed_queries: dict[str, str] = Field(default_factory=dict) diff --git a/app/core/database.py b/app/core/database.py index 2e41d9b..e559571 100644 --- a/app/core/database.py +++ b/app/core/database.py @@ -23,6 +23,16 @@ LegacySessionLocal = ( else None ) +__all__ = [ + "Base", + "LegacySessionLocal", + "SessionLocal", + "engine", + "get_db", + "get_legacy_db", + "legacy_engine", +] + def get_db() -> Generator[Session, None, None]: """Yield an application database session for FastAPI dependencies.""" diff --git a/app/modules/approvals/schemas.py b/app/modules/approvals/schemas.py index f1f8988..b8b427b 100644 --- a/app/modules/approvals/schemas.py +++ b/app/modules/approvals/schemas.py @@ -16,7 +16,6 @@ class ApprovalCreate(BaseModel): class ApprovalDecision(BaseModel): - approver: str comment: str | None = None diff --git a/app/modules/audit/constants.py b/app/modules/audit/constants.py index c9a8e96..da5840f 100644 --- a/app/modules/audit/constants.py +++ b/app/modules/audit/constants.py @@ -39,3 +39,23 @@ class AuditTargetType(StrEnum): class AuditStatus(StrEnum): SUCCESS = "success" + + +AUDIT_REDACTED_VALUE = "[REDACTED]" +AUDIT_SENSITIVE_KEYS = frozenset( + { + "authorization", + "api_key", + "apikey", + "access_token", + "tenant_access_token", + "token", + "secret", + "password", + "openclaw_gateway_token", + "hermes_api_key", + "direct_llm_api_key", + "feishu_app_secret", + "feishu_verification_token", + } +) diff --git a/app/modules/audit/service.py b/app/modules/audit/service.py index 81a5383..ded98b3 100644 --- a/app/modules/audit/service.py +++ b/app/modules/audit/service.py @@ -5,10 +5,28 @@ from sqlalchemy import select from sqlalchemy.orm import Session from app.core.pagination import bounded_limit +from app.modules.audit.constants import AUDIT_REDACTED_VALUE, AUDIT_SENSITIVE_KEYS from app.modules.audit.models import AuditLog from app.modules.audit.schemas import AuditLogCreate +def _redact(value: Any) -> Any: + if isinstance(value, dict): + safe: dict[str, Any] = {} + for key, item in value.items(): + key_text = str(key) + if key_text.lower() in AUDIT_SENSITIVE_KEYS: + safe[key_text] = AUDIT_REDACTED_VALUE + else: + safe[key_text] = _redact(item) + return safe + if isinstance(value, list): + return [_redact(item) for item in value] + if isinstance(value, tuple): + return [_redact(item) for item in value] + return value + + def _dump(value: Any | None) -> str | None: """Serialize audit payloads while preserving existing strings.""" @@ -16,7 +34,7 @@ def _dump(value: Any | None) -> str | None: return None if isinstance(value, str): return value - return json.dumps(value, ensure_ascii=False, default=str) + return json.dumps(_redact(value), ensure_ascii=False, default=str) class AuditService: diff --git a/app/modules/legacy_mysql/service.py b/app/modules/legacy_mysql/service.py index 0f9e0fe..75a9446 100644 --- a/app/modules/legacy_mysql/service.py +++ b/app/modules/legacy_mysql/service.py @@ -175,7 +175,13 @@ class LegacyMySQLService: self._ensure_readonly(sql) engine = self._ensure_engine() params = dict(params or {}) - params.setdefault("limit", bounded_limit(limit)) + try: + params["limit"] = bounded_limit(params.get("limit", limit)) + except (TypeError, ValueError) as exc: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Invalid readonly query limit", + ) from exc limited_sql = sql if " limit " not in sql.lower(): limited_sql = f"{sql.rstrip(';')} LIMIT :limit" diff --git a/environment.yml b/environment.yml index 303f7a4..34f1b59 100644 --- a/environment.yml +++ b/environment.yml @@ -21,3 +21,4 @@ dependencies: - cryptography==44.0.0 - pandas==2.2.3 - pytest==8.3.4 + - ruff==0.8.4 diff --git a/pyproject.toml b/pyproject.toml index 815ee71..d107b9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,10 @@ requires-python = ">=3.11" line-length = 100 target-version = "py311" +[tool.ruff.lint.per-file-ignores] +"scripts/verify_smoke.py" = ["E402"] +"tests/test_smoke.py" = ["E402"] + [tool.pytest.ini_options] pythonpath = ["."] testpaths = ["tests"] diff --git a/tests/test_smoke.py b/tests/test_smoke.py index c0efac5..f614d9d 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -33,6 +33,7 @@ from app.core.database import Base, engine from app.core.pagination import bounded_limit, bounded_offset from app.core.security import require_api_key, require_approval_api_key from app.main import _allow_cors_credentials, app +from app.modules.audit.constants import AUDIT_REDACTED_VALUE from app.modules.legacy_mysql.service import LegacyMySQLService from app.modules.reports.constants import ( LifecycleAttentionKey, @@ -108,6 +109,12 @@ def test_feishu_webhook_routes_message_event() -> None: assert data["handled"] is True assert data["result"]["command"] == "risk_summary" + logs_response = client.get("/api/v1/audit/logs", headers=headers) + assert logs_response.status_code == 200 + audit_payload = json.dumps(logs_response.json(), ensure_ascii=False) + assert "test-feishu-token" not in audit_payload + assert AUDIT_REDACTED_VALUE in audit_payload + def test_api_key_and_feishu_webhook_fail_closed(monkeypatch) -> None: monkeypatch.setenv("API_KEY", "") @@ -195,7 +202,7 @@ def test_approval_gate_for_high_risk_update() -> None: approve_create_response = client.post( f"/api/v1/approvals/{create_ticket_id}/approve", headers=approval_headers, - json={"approver": "spoofed-manager", "comment": "ok"}, + json={"comment": "ok"}, ) assert approve_create_response.status_code == 200 assert approve_create_response.json()["approver"] == "approval-manager" @@ -606,3 +613,55 @@ def test_legacy_readonly_query_requires_allowlist(monkeypatch) -> None: finally: monkeypatch.delenv("LEGACY_PROJECT_QUERY", raising=False) get_settings.cache_clear() + + +def test_legacy_readonly_query_clamps_param_limit(monkeypatch) -> None: + captured: dict[str, dict] = {} + + class FakeResult: + def mappings(self) -> "FakeResult": + return self + + def all(self) -> list: + return [] + + class FakeConnection: + def __enter__(self) -> "FakeConnection": + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + return None + + def execute(self, statement, params): + captured["params"] = params + return FakeResult() + + class FakeEngine: + def connect(self) -> FakeConnection: + return FakeConnection() + + monkeypatch.setenv("LEGACY_PROJECT_QUERY", "SELECT id FROM projects LIMIT :limit") + get_settings.cache_clear() + monkeypatch.setattr( + LegacyMySQLService, + "_ensure_engine", + staticmethod(lambda: FakeEngine()), + ) + try: + result = LegacyMySQLService(None).execute_readonly( + "SELECT id FROM projects LIMIT :limit", + {"limit": 9999}, + limit=9999, + ) + assert result["row_count"] == 0 + assert captured["params"]["limit"] == 500 + + with pytest.raises(HTTPException) as exc_info: + LegacyMySQLService(None).execute_readonly( + "SELECT id FROM projects LIMIT :limit", + {"limit": "invalid"}, + ) + assert exc_info.value.status_code == 422 + finally: + monkeypatch.delenv("LEGACY_PROJECT_QUERY", raising=False) + get_settings.cache_clear()