Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e807ca4c65 |
+48
-44
@@ -16,7 +16,7 @@ permissions:
|
||||
security-events: write
|
||||
|
||||
env:
|
||||
REGISTRY: git.farh.net
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: cartsnitch/cartsnitch
|
||||
RECEIPTWITNESS_IMAGE_NAME: cartsnitch/receiptwitness
|
||||
API_IMAGE_NAME: cartsnitch/api
|
||||
@@ -26,7 +26,11 @@ jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: ESLint
|
||||
run: npx eslint .
|
||||
@@ -36,8 +40,8 @@ jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
@@ -48,8 +52,8 @@ jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
@@ -60,8 +64,8 @@ jobs:
|
||||
e2e:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
@@ -73,8 +77,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test]
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
@@ -102,7 +106,7 @@ jobs:
|
||||
calver_tag: ${{ steps.calver.outputs.version }}
|
||||
sha_tag: sha-${{ github.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -130,13 +134,13 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Gitea registry
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
@@ -171,7 +175,7 @@ jobs:
|
||||
only-fixed: "true"
|
||||
output-format: sarif
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Push Docker image
|
||||
if: github.event_name == 'push'
|
||||
@@ -198,7 +202,7 @@ jobs:
|
||||
calver_tag: ${{ steps.calver.outputs.version }}
|
||||
sha_tag: sha-${{ github.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -220,13 +224,13 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Gitea registry
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
@@ -263,7 +267,7 @@ jobs:
|
||||
only-fixed: "true"
|
||||
output-format: sarif
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Push Docker image
|
||||
if: github.event_name == 'push'
|
||||
@@ -286,7 +290,7 @@ jobs:
|
||||
calver_tag: ${{ steps.calver.outputs.version }}
|
||||
sha_tag: sha-${{ github.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -308,13 +312,13 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Gitea registry
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Extract metadata (API)
|
||||
id: meta
|
||||
@@ -351,7 +355,7 @@ jobs:
|
||||
only-fixed: "true"
|
||||
output-format: sarif
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Push Docker image
|
||||
if: github.event_name == 'push'
|
||||
@@ -374,7 +378,7 @@ jobs:
|
||||
calver_tag: ${{ steps.calver.outputs.version }}
|
||||
sha_tag: sha-${{ github.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -396,13 +400,13 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Gitea registry
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Extract metadata (auth)
|
||||
id: meta
|
||||
@@ -439,7 +443,7 @@ jobs:
|
||||
only-fixed: "true"
|
||||
output-format: sarif
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Push Docker image
|
||||
if: github.event_name == 'push'
|
||||
@@ -460,10 +464,10 @@ jobs:
|
||||
if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
|
||||
steps:
|
||||
- name: Checkout infra repo
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: cartsnitch/infra
|
||||
token: ${{ secrets.REGISTRY_TOKEN }}
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
ref: main
|
||||
path: infra
|
||||
|
||||
@@ -486,7 +490,7 @@ jobs:
|
||||
if: needs.build-and-push.result == 'success'
|
||||
run: |
|
||||
cd infra/apps/overlays/dev
|
||||
kustomize edit set image ghcr.io/cartsnitch/cartsnitch=git.farh.net/cartsnitch/cartsnitch:${{ steps.frontend_tag.outputs.tag }}
|
||||
kustomize edit set image ghcr.io/cartsnitch/cartsnitch:${{ steps.frontend_tag.outputs.tag }}
|
||||
|
||||
- name: Determine image tag for receiptwitness
|
||||
id: receiptwitness_tag
|
||||
@@ -501,7 +505,7 @@ jobs:
|
||||
if: needs.build-and-push-receiptwitness.result == 'success'
|
||||
run: |
|
||||
cd infra/apps/overlays/dev
|
||||
kustomize edit set image ghcr.io/cartsnitch/receiptwitness=git.farh.net/cartsnitch/receiptwitness:${{ steps.receiptwitness_tag.outputs.tag }}
|
||||
kustomize edit set image ghcr.io/cartsnitch/receiptwitness:${{ steps.receiptwitness_tag.outputs.tag }}
|
||||
|
||||
- name: Determine image tag for api
|
||||
id: api_tag
|
||||
@@ -516,7 +520,7 @@ jobs:
|
||||
if: needs.build-and-push-api.result == 'success'
|
||||
run: |
|
||||
cd infra/apps/overlays/dev
|
||||
kustomize edit set image ghcr.io/cartsnitch/api=git.farh.net/cartsnitch/api:${{ steps.api_tag.outputs.tag }}
|
||||
kustomize edit set image ghcr.io/cartsnitch/api:${{ steps.api_tag.outputs.tag }}
|
||||
|
||||
- name: Determine image tag for auth
|
||||
id: auth_tag
|
||||
@@ -531,13 +535,13 @@ jobs:
|
||||
if: needs.build-and-push-auth.result == 'success'
|
||||
run: |
|
||||
cd infra/apps/overlays/dev
|
||||
kustomize edit set image ghcr.io/cartsnitch/auth=git.farh.net/cartsnitch/auth:${{ steps.auth_tag.outputs.tag }}
|
||||
kustomize edit set image ghcr.io/cartsnitch/auth:${{ steps.auth_tag.outputs.tag }}
|
||||
|
||||
- name: Commit and push to infra
|
||||
run: |
|
||||
cd infra
|
||||
git config user.name "cartsnitch-ci[bot]"
|
||||
git config user.email "cartsnitch-ci[bot]@users.noreply.git.farh.net"
|
||||
git config user.email "cartsnitch-ci[bot]@users.noreply.github.com"
|
||||
git add apps/overlays/dev/kustomization.yaml
|
||||
git diff --cached --quiet && echo "No image changes to deploy" && exit 0
|
||||
git commit -m "ci(dev): update cartsnitch, receiptwitness, api, and auth images"
|
||||
@@ -550,10 +554,10 @@ jobs:
|
||||
if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/uat' || github.ref == 'refs/heads/main')
|
||||
steps:
|
||||
- name: Checkout infra repo
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: cartsnitch/infra
|
||||
token: ${{ secrets.REGISTRY_TOKEN }}
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
ref: main
|
||||
path: infra
|
||||
|
||||
@@ -576,7 +580,7 @@ jobs:
|
||||
if: needs.build-and-push.result == 'success'
|
||||
run: |
|
||||
cd infra/apps/overlays/uat
|
||||
kustomize edit set image ghcr.io/cartsnitch/cartsnitch=git.farh.net/cartsnitch/cartsnitch:${{ steps.frontend_tag.outputs.tag }}
|
||||
kustomize edit set image ghcr.io/cartsnitch/cartsnitch:${{ steps.frontend_tag.outputs.tag }}
|
||||
|
||||
- name: Determine image tag for receiptwitness
|
||||
id: receiptwitness_tag
|
||||
@@ -591,7 +595,7 @@ jobs:
|
||||
if: needs.build-and-push-receiptwitness.result == 'success'
|
||||
run: |
|
||||
cd infra/apps/overlays/uat
|
||||
kustomize edit set image ghcr.io/cartsnitch/receiptwitness=git.farh.net/cartsnitch/receiptwitness:${{ steps.receiptwitness_tag.outputs.tag }}
|
||||
kustomize edit set image ghcr.io/cartsnitch/receiptwitness:${{ steps.receiptwitness_tag.outputs.tag }}
|
||||
|
||||
- name: Determine image tag for api
|
||||
id: api_tag
|
||||
@@ -606,7 +610,7 @@ jobs:
|
||||
if: needs.build-and-push-api.result == 'success'
|
||||
run: |
|
||||
cd infra/apps/overlays/uat
|
||||
kustomize edit set image ghcr.io/cartsnitch/api=git.farh.net/cartsnitch/api:${{ steps.api_tag.outputs.tag }}
|
||||
kustomize edit set image ghcr.io/cartsnitch/api:${{ steps.api_tag.outputs.tag }}
|
||||
|
||||
- name: Determine image tag for auth
|
||||
id: auth_tag
|
||||
@@ -621,13 +625,13 @@ jobs:
|
||||
if: needs.build-and-push-auth.result == 'success'
|
||||
run: |
|
||||
cd infra/apps/overlays/uat
|
||||
kustomize edit set image ghcr.io/cartsnitch/auth=git.farh.net/cartsnitch/auth:${{ steps.auth_tag.outputs.tag }}
|
||||
kustomize edit set image ghcr.io/cartsnitch/auth:${{ steps.auth_tag.outputs.tag }}
|
||||
|
||||
- name: Commit and push to infra
|
||||
run: |
|
||||
cd infra
|
||||
git config user.name "cartsnitch-ci[bot]"
|
||||
git config user.email "cartsnitch-ci[bot]@users.noreply.git.farh.net"
|
||||
git config user.email "cartsnitch-ci[bot]@users.noreply.github.com"
|
||||
git add apps/overlays/uat/kustomization.yaml
|
||||
git diff --cached --quiet && echo "No image changes to deploy" && exit 0
|
||||
git commit -m "ci(uat): update cartsnitch, receiptwitness, api, and auth images"
|
||||
|
||||
@@ -15,7 +15,7 @@ permissions:
|
||||
packages: write
|
||||
|
||||
env:
|
||||
REGISTRY: git.farh.net
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: cartsnitch/api
|
||||
|
||||
jobs:
|
||||
@@ -130,13 +130,13 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Gitea registry
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
|
||||
@@ -1,36 +1,17 @@
|
||||
"""Database engine and session factories for sync and async usage."""
|
||||
|
||||
from collections.abc import AsyncGenerator, Generator
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from cartsnitch_common.config import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
# Module-level async engine cache — one engine per unique URL, shared across all callers.
|
||||
# This prevents pool exhaustion in high-throughput workers (e.g. email-worker hitting
|
||||
# DragonflyDB/Postgres repeatedly per message). pool_size=10, max_overflow=20 gives
|
||||
# headroom for bursts while capping max connections at 30 per URL.
|
||||
_async_engine_cache: dict[str, "AsyncEngine"] = {}
|
||||
|
||||
|
||||
def get_async_engine(url: str | None = None) -> "AsyncEngine":
|
||||
"""Get or create a cached async engine for the given URL."""
|
||||
target = url or settings.database_url
|
||||
if target not in _async_engine_cache:
|
||||
_async_engine_cache[target] = create_async_engine(
|
||||
target,
|
||||
echo=settings.debug,
|
||||
pool_size=10,
|
||||
max_overflow=20,
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
return _async_engine_cache[target]
|
||||
def get_async_engine(url: str | None = None):
|
||||
"""Create an async SQLAlchemy engine."""
|
||||
return create_async_engine(url or settings.database_url, echo=settings.debug)
|
||||
|
||||
|
||||
def get_sync_engine(url: str | None = None):
|
||||
|
||||
Generated
+383
-289
File diff suppressed because it is too large
Load Diff
+3
-5
@@ -45,16 +45,14 @@
|
||||
"typescript-eslint": "^8.56.1",
|
||||
"vite": "^6.4.2",
|
||||
"vite-plugin-pwa": "^0.21.2",
|
||||
"vitest": "^4.1.8"
|
||||
"vitest": "^3.2.4"
|
||||
},
|
||||
"overrides": {
|
||||
"@rollup/pluginutils": "5.3.0",
|
||||
"flatted": "^3.4.2",
|
||||
"serialize-javascript": "7.0.5",
|
||||
"brace-expansion": ">=5.0.6",
|
||||
"brace-expansion": ">=1.1.13",
|
||||
"lodash": ">=4.17.24",
|
||||
"minimatch": "^10.2.4",
|
||||
"@babel/plugin-transform-modules-systemjs": "^7.29.4",
|
||||
"fast-uri": "^3.1.2"
|
||||
"minimatch": "^10.2.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,29 +16,6 @@ logger = logging.getLogger(__name__)
|
||||
STREAM_KEY = "email:receipts"
|
||||
CONSUMER_GROUP = "email-workers"
|
||||
|
||||
# Module-level Redis/DragonflyDB connection pool — shared across all worker calls.
|
||||
# Without pooling, each call to get_redis() opens a new TCP connection. In a tight
|
||||
# consumer loop this causes ConnectionResetError when DragonflyDB's connection limit
|
||||
# is hit under load. max_connections=30 (10 base + 20 overflow) mirrors the engine pool.
|
||||
_redis_pool: aioredis.ConnectionPool | None = None
|
||||
|
||||
|
||||
def _get_redis_pool() -> aioredis.ConnectionPool:
|
||||
"""Get or create the shared DragonflyDB connection pool."""
|
||||
global _redis_pool
|
||||
if _redis_pool is None:
|
||||
_redis_pool = aioredis.ConnectionPool.from_url(
|
||||
settings.redis_url,
|
||||
decode_responses=True,
|
||||
max_connections=30,
|
||||
)
|
||||
return _redis_pool
|
||||
|
||||
|
||||
async def get_redis() -> aioredis.Redis:
|
||||
"""Get async Redis/DragonflyDB client backed by a shared connection pool."""
|
||||
return aioredis.Redis(connection_pool=_get_redis_pool())
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmailJob:
|
||||
@@ -54,6 +31,11 @@ class EmailJob:
|
||||
message_id: str # from email provider, for dedup
|
||||
|
||||
|
||||
async def get_redis() -> aioredis.Redis:
|
||||
"""Get async Redis/DragonflyDB client."""
|
||||
return cast(aioredis.Redis, aioredis.from_url(settings.redis_url, decode_responses=True))
|
||||
|
||||
|
||||
async def ensure_consumer_group(client: aioredis.Redis) -> None:
|
||||
"""Create consumer group if it does not exist."""
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user