Compare commits

..

1 Commits

Author SHA1 Message Date
Flea Flicker 1674e759f6 ci: push Docker images to Gitea registry (git.farh.net)
CTO decision: use Gitea built-in OCI registry per CAR-963.

- Replace GHCR login with Gitea Container Registry login
- REGISTRY: ghcr.io -> git.farh.net
- Use secrets.GITEA_TOKEN for authentication
- Tag convention unchanged: CalVer, latest, sha-<hash>

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-05-23 15:33:24 +00:00
9 changed files with 484 additions and 517 deletions
@@ -20,14 +20,13 @@ env:
IMAGE_NAME: cartsnitch/cartsnitch IMAGE_NAME: cartsnitch/cartsnitch
RECEIPTWITNESS_IMAGE_NAME: cartsnitch/receiptwitness RECEIPTWITNESS_IMAGE_NAME: cartsnitch/receiptwitness
API_IMAGE_NAME: cartsnitch/api API_IMAGE_NAME: cartsnitch/api
AUTH_IMAGE_NAME: cartsnitch/auth
jobs: jobs:
lint: lint:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
steps: steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/checkout@v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 - uses: actions/setup-node@v4
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -38,10 +37,10 @@ jobs:
run: npx tsc --noEmit run: npx tsc --noEmit
test: test:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
steps: steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/checkout@v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 - uses: actions/setup-node@v4
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -50,10 +49,10 @@ jobs:
run: npx vitest run run: npx vitest run
audit: audit:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
steps: steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/checkout@v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 - uses: actions/setup-node@v4
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -62,10 +61,10 @@ jobs:
run: npm audit --audit-level=high run: npm audit --audit-level=high
e2e: e2e:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
steps: steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/checkout@v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 - uses: actions/setup-node@v4
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -74,11 +73,11 @@ jobs:
- run: npx playwright test - run: npx playwright test
lighthouse: lighthouse:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
needs: [test] needs: [test]
steps: steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/checkout@v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 - uses: actions/setup-node@v4
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -99,14 +98,14 @@ jobs:
CHROME_PATH="$CHROME_PATH" lhci autorun --chrome-flags="--headless=new --no-sandbox --disable-gpu --disable-dev-shm-usage" CHROME_PATH="$CHROME_PATH" lhci autorun --chrome-flags="--headless=new --no-sandbox --disable-gpu --disable-dev-shm-usage"
build-and-push: build-and-push:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
if: github.event_name == 'push' if: github.event_name == 'push'
needs: [lint, test, e2e] needs: [lint, test, e2e]
outputs: outputs:
calver_tag: ${{ steps.calver.outputs.version }} calver_tag: ${{ steps.calver.outputs.version }}
sha_tag: sha-${{ github.sha }} sha_tag: sha-${{ github.sha }}
steps: steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/checkout@v4
with: with:
fetch-depth: 0 fetch-depth: 0
@@ -134,13 +133,13 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }} username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }} password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to Gitea registry - name: Log in to Gitea Container Registry
if: github.event_name == 'push' if: github.event_name == 'push'
uses: docker/login-action@v3 uses: docker/login-action@v3
with: with:
registry: ${{ env.REGISTRY }} registry: git.farh.net
username: ${{ github.actor }} username: cartsnitch
password: ${{ secrets.REGISTRY_TOKEN }} password: ${{ secrets.GITEA_TOKEN }}
- name: Extract metadata - name: Extract metadata
id: meta id: meta
@@ -175,7 +174,11 @@ jobs:
only-fixed: "true" only-fixed: "true"
output-format: sarif output-format: sarif
- name: Upload frontend scan results to GitHub Security
uses: github/codeql-action/upload-sarif@v3
if: always()
with:
sarif_file: ${{ steps.scan.outputs.sarif }}
- name: Push Docker image - name: Push Docker image
if: github.event_name == 'push' if: github.event_name == 'push'
@@ -195,14 +198,14 @@ jobs:
git push origin "v${{ steps.calver.outputs.version }}" git push origin "v${{ steps.calver.outputs.version }}"
build-and-push-receiptwitness: build-and-push-receiptwitness:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
if: github.event_name == 'push' if: github.event_name == 'push'
needs: [lint, test] needs: [lint, test]
outputs: outputs:
calver_tag: ${{ steps.calver.outputs.version }} calver_tag: ${{ steps.calver.outputs.version }}
sha_tag: sha-${{ github.sha }} sha_tag: sha-${{ github.sha }}
steps: steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/checkout@v4
with: with:
fetch-depth: 0 fetch-depth: 0
@@ -224,13 +227,13 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }} username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }} password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to Gitea registry - name: Log in to Gitea Container Registry
if: github.event_name == 'push' if: github.event_name == 'push'
uses: docker/login-action@v3 uses: docker/login-action@v3
with: with:
registry: ${{ env.REGISTRY }} registry: git.farh.net
username: ${{ github.actor }} username: cartsnitch
password: ${{ secrets.REGISTRY_TOKEN }} password: ${{ secrets.GITEA_TOKEN }}
- name: Extract metadata - name: Extract metadata
id: meta id: meta
@@ -267,7 +270,11 @@ jobs:
only-fixed: "true" only-fixed: "true"
output-format: sarif output-format: sarif
- name: Upload receiptwitness scan results to GitHub Security
uses: github/codeql-action/upload-sarif@v3
if: always()
with:
sarif_file: ${{ steps.scan.outputs.sarif }}
- name: Push Docker image - name: Push Docker image
if: github.event_name == 'push' if: github.event_name == 'push'
@@ -283,14 +290,14 @@ jobs:
cache-from: type=gha cache-from: type=gha
build-and-push-api: build-and-push-api:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
if: github.event_name == 'push' if: github.event_name == 'push'
needs: [lint, test] needs: [lint, test]
outputs: outputs:
calver_tag: ${{ steps.calver.outputs.version }} calver_tag: ${{ steps.calver.outputs.version }}
sha_tag: sha-${{ github.sha }} sha_tag: sha-${{ github.sha }}
steps: steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/checkout@v4
with: with:
fetch-depth: 0 fetch-depth: 0
@@ -312,13 +319,13 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }} username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }} password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to Gitea registry - name: Log in to Gitea Container Registry
if: github.event_name == 'push' if: github.event_name == 'push'
uses: docker/login-action@v3 uses: docker/login-action@v3
with: with:
registry: ${{ env.REGISTRY }} registry: git.farh.net
username: ${{ github.actor }} username: cartsnitch
password: ${{ secrets.REGISTRY_TOKEN }} password: ${{ secrets.GITEA_TOKEN }}
- name: Extract metadata (API) - name: Extract metadata (API)
id: meta id: meta
@@ -355,7 +362,11 @@ jobs:
only-fixed: "true" only-fixed: "true"
output-format: sarif output-format: sarif
- name: Upload api scan results to GitHub Security
uses: github/codeql-action/upload-sarif@v3
if: always()
with:
sarif_file: ${{ steps.scan.outputs.sarif }}
- name: Push Docker image - name: Push Docker image
if: github.event_name == 'push' if: github.event_name == 'push'
@@ -370,104 +381,25 @@ jobs:
APT_CACHE_BUST=${{ github.run_id }} APT_CACHE_BUST=${{ github.run_id }}
cache-from: type=gha cache-from: type=gha
build-and-push-auth:
runs-on: ubuntu-latest
if: github.event_name == 'push'
needs: [lint, test]
outputs:
calver_tag: ${{ steps.calver.outputs.version }}
sha_tag: sha-${{ github.sha }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
fetch-depth: 0
- name: Generate CalVer tag
id: calver
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
run: |
DATE_TAG=$(date -u +%Y.%m.%d)
EXISTING=$(git tag -l "v${DATE_TAG}*" | sort -V | tail -1)
if [ -z "$EXISTING" ]; then VERSION="$DATE_TAG"
elif [ "$EXISTING" = "v${DATE_TAG}" ]; then VERSION="${DATE_TAG}.2"
else BUILD_NUM=$(echo "$EXISTING" | sed "s/v${DATE_TAG}\.//"); VERSION="${DATE_TAG}.$((BUILD_NUM + 1))"; fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
- name: Log in to Docker Hub
if: github.event_name == 'push'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to Gitea registry
if: github.event_name == 'push'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Extract metadata (auth)
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.AUTH_IMAGE_NAME }}
tags: |
type=sha,prefix=sha-,format=long
type=raw,value=${{ steps.calver.outputs.version }},enable=${{ github.ref == 'refs/heads/main' }}
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
- name: Build Docker image
uses: docker/build-push-action@v6
with:
context: ./auth
file: ./auth/Dockerfile
load: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
APT_CACHE_BUST=${{ github.run_id }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Scan auth image for vulnerabilities
uses: anchore/scan-action@v5
id: scan
env:
GRYPE_CONFIG: .grype.yaml
with:
image: "${{ env.REGISTRY }}/${{ env.AUTH_IMAGE_NAME }}:sha-${{ github.sha }}"
fail-build: true
severity-cutoff: high
only-fixed: "true"
output-format: sarif
- name: Push Docker image
if: github.event_name == 'push'
uses: docker/build-push-action@v6
with:
context: ./auth
file: ./auth/Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
APT_CACHE_BUST=${{ github.run_id }}
cache-from: type=gha
deploy-dev: deploy-dev:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
needs: [build-and-push, build-and-push-receiptwitness, build-and-push-api, build-and-push-auth] needs: [build-and-push, build-and-push-receiptwitness, build-and-push-api]
if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
steps: steps:
- name: Generate GitHub App token
id: app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.CARTSNITCH_APP_ID }}
private-key: ${{ secrets.CARTSNITCH_APP_PRIVATE_KEY }}
owner: ${{ github.repository_owner }}
repositories: infra
- name: Checkout infra repo - name: Checkout infra repo
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 uses: actions/checkout@v4
with: with:
repository: cartsnitch/infra repository: cartsnitch/infra
token: ${{ secrets.REGISTRY_TOKEN }} token: ${{ steps.app-token.outputs.token }}
ref: main ref: main
path: infra path: infra
@@ -490,7 +422,7 @@ jobs:
if: needs.build-and-push.result == 'success' if: needs.build-and-push.result == 'success'
run: | run: |
cd infra/apps/overlays/dev cd infra/apps/overlays/dev
kustomize edit set image ghcr.io/cartsnitch/cartsnitch=git.farh.net/cartsnitch/cartsnitch:${{ steps.frontend_tag.outputs.tag }} kustomize edit set image ghcr.io/cartsnitch/cartsnitch:${{ steps.frontend_tag.outputs.tag }}
- name: Determine image tag for receiptwitness - name: Determine image tag for receiptwitness
id: receiptwitness_tag id: receiptwitness_tag
@@ -505,7 +437,7 @@ jobs:
if: needs.build-and-push-receiptwitness.result == 'success' if: needs.build-and-push-receiptwitness.result == 'success'
run: | run: |
cd infra/apps/overlays/dev cd infra/apps/overlays/dev
kustomize edit set image ghcr.io/cartsnitch/receiptwitness=git.farh.net/cartsnitch/receiptwitness:${{ steps.receiptwitness_tag.outputs.tag }} kustomize edit set image ghcr.io/cartsnitch/receiptwitness:${{ steps.receiptwitness_tag.outputs.tag }}
- name: Determine image tag for api - name: Determine image tag for api
id: api_tag id: api_tag
@@ -520,44 +452,38 @@ jobs:
if: needs.build-and-push-api.result == 'success' if: needs.build-and-push-api.result == 'success'
run: | run: |
cd infra/apps/overlays/dev cd infra/apps/overlays/dev
kustomize edit set image ghcr.io/cartsnitch/api=git.farh.net/cartsnitch/api:${{ steps.api_tag.outputs.tag }} kustomize edit set image ghcr.io/cartsnitch/api:${{ steps.api_tag.outputs.tag }}
- name: Determine image tag for auth
id: auth_tag
run: |
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
echo "tag=${{ needs.build-and-push-auth.outputs.calver_tag }}" >> "$GITHUB_OUTPUT"
else
echo "tag=${{ needs.build-and-push-auth.outputs.sha_tag }}" >> "$GITHUB_OUTPUT"
fi
- name: Update auth image tag
if: needs.build-and-push-auth.result == 'success'
run: |
cd infra/apps/overlays/dev
kustomize edit set image ghcr.io/cartsnitch/auth=git.farh.net/cartsnitch/auth:${{ steps.auth_tag.outputs.tag }}
- name: Commit and push to infra - name: Commit and push to infra
run: | run: |
cd infra cd infra
git config user.name "cartsnitch-ci[bot]" git config user.name "cartsnitch-ci[bot]"
git config user.email "cartsnitch-ci[bot]@users.noreply.git.farh.net" git config user.email "cartsnitch-ci[bot]@users.noreply.github.com"
git add apps/overlays/dev/kustomization.yaml git add apps/overlays/dev/kustomization.yaml
git diff --cached --quiet && echo "No image changes to deploy" && exit 0 git diff --cached --quiet && echo "No image changes to deploy" && exit 0
git commit -m "ci(dev): update cartsnitch, receiptwitness, api, and auth images" git commit -m "ci(dev): update cartsnitch, receiptwitness, and api images"
git pull --rebase origin main git pull --rebase origin main
git push origin main git push origin main
deploy-uat: deploy-uat:
runs-on: ubuntu-latest runs-on: runners-cartsnitch
needs: [build-and-push, build-and-push-receiptwitness, build-and-push-api, build-and-push-auth] needs: [build-and-push, build-and-push-receiptwitness, build-and-push-api]
if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/uat' || github.ref == 'refs/heads/main') if: always() && !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/uat' || github.ref == 'refs/heads/main')
steps: steps:
- name: Generate GitHub App token
id: app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.CARTSNITCH_APP_ID }}
private-key: ${{ secrets.CARTSNITCH_APP_PRIVATE_KEY }}
owner: ${{ github.repository_owner }}
repositories: infra
- name: Checkout infra repo - name: Checkout infra repo
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 uses: actions/checkout@v4
with: with:
repository: cartsnitch/infra repository: cartsnitch/infra
token: ${{ secrets.REGISTRY_TOKEN }} token: ${{ steps.app-token.outputs.token }}
ref: main ref: main
path: infra path: infra
@@ -580,7 +506,7 @@ jobs:
if: needs.build-and-push.result == 'success' if: needs.build-and-push.result == 'success'
run: | run: |
cd infra/apps/overlays/uat cd infra/apps/overlays/uat
kustomize edit set image ghcr.io/cartsnitch/cartsnitch=git.farh.net/cartsnitch/cartsnitch:${{ steps.frontend_tag.outputs.tag }} kustomize edit set image ghcr.io/cartsnitch/cartsnitch:${{ steps.frontend_tag.outputs.tag }}
- name: Determine image tag for receiptwitness - name: Determine image tag for receiptwitness
id: receiptwitness_tag id: receiptwitness_tag
@@ -595,7 +521,7 @@ jobs:
if: needs.build-and-push-receiptwitness.result == 'success' if: needs.build-and-push-receiptwitness.result == 'success'
run: | run: |
cd infra/apps/overlays/uat cd infra/apps/overlays/uat
kustomize edit set image ghcr.io/cartsnitch/receiptwitness=git.farh.net/cartsnitch/receiptwitness:${{ steps.receiptwitness_tag.outputs.tag }} kustomize edit set image ghcr.io/cartsnitch/receiptwitness:${{ steps.receiptwitness_tag.outputs.tag }}
- name: Determine image tag for api - name: Determine image tag for api
id: api_tag id: api_tag
@@ -610,30 +536,15 @@ jobs:
if: needs.build-and-push-api.result == 'success' if: needs.build-and-push-api.result == 'success'
run: | run: |
cd infra/apps/overlays/uat cd infra/apps/overlays/uat
kustomize edit set image ghcr.io/cartsnitch/api=git.farh.net/cartsnitch/api:${{ steps.api_tag.outputs.tag }} kustomize edit set image ghcr.io/cartsnitch/api:${{ steps.api_tag.outputs.tag }}
- name: Determine image tag for auth
id: auth_tag
run: |
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
echo "tag=${{ needs.build-and-push-auth.outputs.calver_tag }}" >> "$GITHUB_OUTPUT"
else
echo "tag=${{ needs.build-and-push-auth.outputs.sha_tag }}" >> "$GITHUB_OUTPUT"
fi
- name: Update auth image tag
if: needs.build-and-push-auth.result == 'success'
run: |
cd infra/apps/overlays/uat
kustomize edit set image ghcr.io/cartsnitch/auth=git.farh.net/cartsnitch/auth:${{ steps.auth_tag.outputs.tag }}
- name: Commit and push to infra - name: Commit and push to infra
run: | run: |
cd infra cd infra
git config user.name "cartsnitch-ci[bot]" git config user.name "cartsnitch-ci[bot]"
git config user.email "cartsnitch-ci[bot]@users.noreply.git.farh.net" git config user.email "cartsnitch-ci[bot]@users.noreply.github.com"
git add apps/overlays/uat/kustomization.yaml git add apps/overlays/uat/kustomization.yaml
git diff --cached --quiet && echo "No image changes to deploy" && exit 0 git diff --cached --quiet && echo "No image changes to deploy" && exit 0
git commit -m "ci(uat): update cartsnitch, receiptwitness, api, and auth images" git commit -m "ci(uat): update cartsnitch, receiptwitness, and api images"
git pull --rebase origin main git pull --rebase origin main
git push origin main git push origin main
@@ -15,7 +15,7 @@ permissions:
packages: write packages: write
env: env:
REGISTRY: git.farh.net REGISTRY: ghcr.io
IMAGE_NAME: cartsnitch/api IMAGE_NAME: cartsnitch/api
jobs: jobs:
@@ -130,13 +130,13 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }} username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }} password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to Gitea registry - name: Log in to GHCR
if: github.event_name == 'push' && github.ref == 'refs/heads/main' if: github.event_name == 'push' && github.ref == 'refs/heads/main'
uses: docker/login-action@v3 uses: docker/login-action@v3
with: with:
registry: ${{ env.REGISTRY }} registry: ${{ env.REGISTRY }}
username: ${{ github.actor }} username: ${{ github.actor }}
password: ${{ secrets.REGISTRY_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata - name: Extract metadata
id: meta id: meta
+1
View File
@@ -6,6 +6,7 @@ from fastapi import APIRouter, FastAPI
from cartsnitch_api.auth.routes import router as auth_router from cartsnitch_api.auth.routes import router as auth_router
from cartsnitch_api.cache import cache_client from cartsnitch_api.cache import cache_client
from cartsnitch_api.database import dispose_engine
from cartsnitch_api.middleware.cors import add_cors_middleware from cartsnitch_api.middleware.cors import add_cors_middleware
from cartsnitch_api.middleware.error_handler import add_error_handlers, add_error_monitor_middleware from cartsnitch_api.middleware.error_handler import add_error_handlers, add_error_monitor_middleware
from cartsnitch_api.middleware.rate_limit import add_rate_limit_middleware from cartsnitch_api.middleware.rate_limit import add_rate_limit_middleware
+1 -1
View File
@@ -12,5 +12,5 @@
"resolveJsonModule": true "resolveJsonModule": true
}, },
"include": ["src"], "include": ["src"],
"exclude": ["node_modules", "dist", "src/__tests__"] "exclude": ["node_modules", "dist"]
} }
+4 -23
View File
@@ -1,36 +1,17 @@
"""Database engine and session factories for sync and async usage.""" """Database engine and session factories for sync and async usage."""
from collections.abc import AsyncGenerator, Generator from collections.abc import AsyncGenerator, Generator
from typing import TYPE_CHECKING
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from cartsnitch_common.config import settings from cartsnitch_common.config import settings
if TYPE_CHECKING:
from sqlalchemy.engine import Engine
# Module-level async engine cache — one engine per unique URL, shared across all callers. def get_async_engine(url: str | None = None):
# This prevents pool exhaustion in high-throughput workers (e.g. email-worker hitting """Create an async SQLAlchemy engine."""
# DragonflyDB/Postgres repeatedly per message). pool_size=10, max_overflow=20 gives return create_async_engine(url or settings.database_url, echo=settings.debug)
# headroom for bursts while capping max connections at 30 per URL.
_async_engine_cache: dict[str, "AsyncEngine"] = {}
def get_async_engine(url: str | None = None) -> "AsyncEngine":
"""Get or create a cached async engine for the given URL."""
target = url or settings.database_url
if target not in _async_engine_cache:
_async_engine_cache[target] = create_async_engine(
target,
echo=settings.debug,
pool_size=10,
max_overflow=20,
pool_pre_ping=True,
)
return _async_engine_cache[target]
def get_sync_engine(url: str | None = None): def get_sync_engine(url: str | None = None):
+383 -289
View File
File diff suppressed because it is too large Load Diff
+3 -5
View File
@@ -45,16 +45,14 @@
"typescript-eslint": "^8.56.1", "typescript-eslint": "^8.56.1",
"vite": "^6.4.2", "vite": "^6.4.2",
"vite-plugin-pwa": "^0.21.2", "vite-plugin-pwa": "^0.21.2",
"vitest": "^4.1.8" "vitest": "^3.2.4"
}, },
"overrides": { "overrides": {
"@rollup/pluginutils": "5.3.0", "@rollup/pluginutils": "5.3.0",
"flatted": "^3.4.2", "flatted": "^3.4.2",
"serialize-javascript": "7.0.5", "serialize-javascript": "7.0.5",
"brace-expansion": ">=5.0.6", "brace-expansion": ">=1.1.13",
"lodash": ">=4.17.24", "lodash": ">=4.17.24",
"minimatch": "^10.2.4", "minimatch": "^10.2.4"
"@babel/plugin-transform-modules-systemjs": "^7.29.4",
"fast-uri": "^3.1.2"
} }
} }
@@ -16,29 +16,6 @@ logger = logging.getLogger(__name__)
STREAM_KEY = "email:receipts" STREAM_KEY = "email:receipts"
CONSUMER_GROUP = "email-workers" CONSUMER_GROUP = "email-workers"
# Module-level Redis/DragonflyDB connection pool — shared across all worker calls.
# Without pooling, each call to get_redis() opens a new TCP connection. In a tight
# consumer loop this causes ConnectionResetError when DragonflyDB's connection limit
# is hit under load. max_connections=30 (10 base + 20 overflow) mirrors the engine pool.
_redis_pool: aioredis.ConnectionPool | None = None
def _get_redis_pool() -> aioredis.ConnectionPool:
"""Get or create the shared DragonflyDB connection pool."""
global _redis_pool
if _redis_pool is None:
_redis_pool = aioredis.ConnectionPool.from_url(
settings.redis_url,
decode_responses=True,
max_connections=30,
)
return _redis_pool
async def get_redis() -> aioredis.Redis:
"""Get async Redis/DragonflyDB client backed by a shared connection pool."""
return aioredis.Redis(connection_pool=_get_redis_pool())
@dataclass @dataclass
class EmailJob: class EmailJob:
@@ -54,6 +31,11 @@ class EmailJob:
message_id: str # from email provider, for dedup message_id: str # from email provider, for dedup
async def get_redis() -> aioredis.Redis:
"""Get async Redis/DragonflyDB client."""
return cast(aioredis.Redis, aioredis.from_url(settings.redis_url, decode_responses=True))
async def ensure_consumer_group(client: aioredis.Redis) -> None: async def ensure_consumer_group(client: aioredis.Redis) -> None:
"""Create consumer group if it does not exist.""" """Create consumer group if it does not exist."""
try: try: