Compare commits

...

3 Commits

Author SHA1 Message Date
Barcode Betty 01ed6dac00 fix(deps): pin safe versions of audit-flagged transitive deps (CAR-1162 audit)
The CI's npm audit (10.8.2) flagged three transitive vulnerabilities
that local newer-npm runs (11.x) miss due to advisory-DB divergence:

- @babel/plugin-transform-modules-systemjs: 7.29.0 -> ^7.29.4
  (CVE-2026-44728: arbitrary code generation, fixed in 7.29.4)
- fast-uri: 3.1.0 -> ^3.1.2
  (path traversal / host confusion via percent-encoded segments)
- brace-expansion: 5.0.5 -> >=5.0.6
  (DoS via large numeric range defeating max protection)

These are non-breaking transitive updates within the same major
version. The previous override for brace-expansion (>=1.1.13) was
too loose to exclude 5.0.2-5.0.5; tightening it to >=5.0.6.

Ref CAR-1162, CAR-1122, CAR-1078

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-06-03 15:53:46 +00:00
Barcode Betty a7a55bbf79 fix(ci): unblock dev PR #271 CI
- Remove .mcp.json (scope creep, unrelated to CAR-1078)
- Bump vitest to ^4.1.8 (fixes GHSA-5xrq-8626-4rwp critical)
- Run npm audit fix for non-breaking vulns
- Pin actions/checkout and actions/setup-node to commit SHAs
  in .gitea/workflows/ci.yml to force a clean cache fetch on
  the act runner (workaround for corrupted /root/.cache/act cache)

Refs CAR-1162, CAR-1122, CAR-1078
2026-06-03 11:41:19 +00:00
Flea Flicker fb0bb0102c fix(receiptwitness): pool DB engine and Redis client to prevent connection exhaustion
email_worker calls get_async_session_factory() inside every resolve_user()
call, which creates a brand-new async engine (and thus a brand-new
connection pool) on every message.  In a tight consumer loop processing
5 messages per batch, this rapidly exhausts DragonflyDB/Postgres
connection limits and manifests as ConnectionResetError.

Fix: cache the async engine in a module-level dict keyed by URL in
cartsnitch_common.database:get_async_engine(), matching the pattern
already used in receiptwitness:events.py for the Redis connection pool.
Also add pool_size=10, max_overflow=20, pool_pre_ping=True for
健壮连接管理.

Similarly, receiptwitness/queue/email.py:get_redis() was creating a new
Redis connection on every call with no pooling.  Share a
ConnectionPool (max_connections=30) across all get_redis() callers.

Fixes CAR-1078
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-05-28 18:53:05 +00:00
6 changed files with 357 additions and 423 deletions
+16 -16
View File
@@ -26,8 +26,8 @@ jobs:
lint: lint:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -40,8 +40,8 @@ jobs:
test: test:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -52,8 +52,8 @@ jobs:
audit: audit:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -64,8 +64,8 @@ jobs:
e2e: e2e:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -77,8 +77,8 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [test] needs: [test]
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -106,7 +106,7 @@ jobs:
calver_tag: ${{ steps.calver.outputs.version }} calver_tag: ${{ steps.calver.outputs.version }}
sha_tag: sha-${{ github.sha }} sha_tag: sha-${{ github.sha }}
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with: with:
fetch-depth: 0 fetch-depth: 0
@@ -202,7 +202,7 @@ jobs:
calver_tag: ${{ steps.calver.outputs.version }} calver_tag: ${{ steps.calver.outputs.version }}
sha_tag: sha-${{ github.sha }} sha_tag: sha-${{ github.sha }}
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with: with:
fetch-depth: 0 fetch-depth: 0
@@ -290,7 +290,7 @@ jobs:
calver_tag: ${{ steps.calver.outputs.version }} calver_tag: ${{ steps.calver.outputs.version }}
sha_tag: sha-${{ github.sha }} sha_tag: sha-${{ github.sha }}
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with: with:
fetch-depth: 0 fetch-depth: 0
@@ -378,7 +378,7 @@ jobs:
calver_tag: ${{ steps.calver.outputs.version }} calver_tag: ${{ steps.calver.outputs.version }}
sha_tag: sha-${{ github.sha }} sha_tag: sha-${{ github.sha }}
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with: with:
fetch-depth: 0 fetch-depth: 0
@@ -464,7 +464,7 @@ jobs:
if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
steps: steps:
- name: Checkout infra repo - name: Checkout infra repo
uses: actions/checkout@v4 uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with: with:
repository: cartsnitch/infra repository: cartsnitch/infra
token: ${{ secrets.REGISTRY_TOKEN }} token: ${{ secrets.REGISTRY_TOKEN }}
@@ -554,7 +554,7 @@ jobs:
if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/uat' || github.ref == 'refs/heads/main') if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/uat' || github.ref == 'refs/heads/main')
steps: steps:
- name: Checkout infra repo - name: Checkout infra repo
uses: actions/checkout@v4 uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with: with:
repository: cartsnitch/infra repository: cartsnitch/infra
token: ${{ secrets.REGISTRY_TOKEN }} token: ${{ secrets.REGISTRY_TOKEN }}
-11
View File
@@ -1,11 +0,0 @@
{
"mcpServers": {
"gitea": {
"type": "http",
"url": "https://git-mcp.farh.net/mcp",
"headers": {
"Authorization": "Bearer ${GITEA_TOKEN}"
}
}
}
}
+23 -4
View File
@@ -1,17 +1,36 @@
"""Database engine and session factories for sync and async usage.""" """Database engine and session factories for sync and async usage."""
from collections.abc import AsyncGenerator, Generator from collections.abc import AsyncGenerator, Generator
from typing import TYPE_CHECKING
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from cartsnitch_common.config import settings from cartsnitch_common.config import settings
if TYPE_CHECKING:
from sqlalchemy.engine import Engine
def get_async_engine(url: str | None = None): # Module-level async engine cache — one engine per unique URL, shared across all callers.
"""Create an async SQLAlchemy engine.""" # This prevents pool exhaustion in high-throughput workers (e.g. email-worker hitting
return create_async_engine(url or settings.database_url, echo=settings.debug) # DragonflyDB/Postgres repeatedly per message). pool_size=10, max_overflow=20 gives
# headroom for bursts while capping max connections at 30 per URL.
_async_engine_cache: dict[str, "AsyncEngine"] = {}
def get_async_engine(url: str | None = None) -> "AsyncEngine":
"""Get or create a cached async engine for the given URL."""
target = url or settings.database_url
if target not in _async_engine_cache:
_async_engine_cache[target] = create_async_engine(
target,
echo=settings.debug,
pool_size=10,
max_overflow=20,
pool_pre_ping=True,
)
return _async_engine_cache[target]
def get_sync_engine(url: str | None = None): def get_sync_engine(url: str | None = None):
+290 -384
View File
File diff suppressed because it is too large Load Diff
+5 -3
View File
@@ -45,14 +45,16 @@
"typescript-eslint": "^8.56.1", "typescript-eslint": "^8.56.1",
"vite": "^6.4.2", "vite": "^6.4.2",
"vite-plugin-pwa": "^0.21.2", "vite-plugin-pwa": "^0.21.2",
"vitest": "^3.2.4" "vitest": "^4.1.8"
}, },
"overrides": { "overrides": {
"@rollup/pluginutils": "5.3.0", "@rollup/pluginutils": "5.3.0",
"flatted": "^3.4.2", "flatted": "^3.4.2",
"serialize-javascript": "7.0.5", "serialize-javascript": "7.0.5",
"brace-expansion": ">=1.1.13", "brace-expansion": ">=5.0.6",
"lodash": ">=4.17.24", "lodash": ">=4.17.24",
"minimatch": "^10.2.4" "minimatch": "^10.2.4",
"@babel/plugin-transform-modules-systemjs": "^7.29.4",
"fast-uri": "^3.1.2"
} }
} }
@@ -16,6 +16,29 @@ logger = logging.getLogger(__name__)
STREAM_KEY = "email:receipts" STREAM_KEY = "email:receipts"
CONSUMER_GROUP = "email-workers" CONSUMER_GROUP = "email-workers"
# Module-level Redis/DragonflyDB connection pool — shared across all worker calls.
# Without pooling, each call to get_redis() opens a new TCP connection. In a tight
# consumer loop this causes ConnectionResetError when DragonflyDB's connection limit
# is hit under load. max_connections=30 (10 base + 20 overflow) mirrors the engine pool.
_redis_pool: aioredis.ConnectionPool | None = None
def _get_redis_pool() -> aioredis.ConnectionPool:
"""Get or create the shared DragonflyDB connection pool."""
global _redis_pool
if _redis_pool is None:
_redis_pool = aioredis.ConnectionPool.from_url(
settings.redis_url,
decode_responses=True,
max_connections=30,
)
return _redis_pool
async def get_redis() -> aioredis.Redis:
"""Get async Redis/DragonflyDB client backed by a shared connection pool."""
return aioredis.Redis(connection_pool=_get_redis_pool())
@dataclass @dataclass
class EmailJob: class EmailJob:
@@ -31,11 +54,6 @@ class EmailJob:
message_id: str # from email provider, for dedup message_id: str # from email provider, for dedup
async def get_redis() -> aioredis.Redis:
"""Get async Redis/DragonflyDB client."""
return cast(aioredis.Redis, aioredis.from_url(settings.redis_url, decode_responses=True))
async def ensure_consumer_group(client: aioredis.Redis) -> None: async def ensure_consumer_group(client: aioredis.Redis) -> None:
"""Create consumer group if it does not exist.""" """Create consumer group if it does not exist."""
try: try: