Compare commits

..

2 Commits

Author SHA1 Message Date
Paperclip 1aff898545 fix: update vite to 6.4.2 to patch audit vulnerabilities
Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-04-14 14:31:02 +00:00
CartSnitch Engineer Bot 24f0dd0e67 fix: replace N+1 UPC query with SQL containment in normalization
- Add PostgreSQL JSONB containment (@>) query for match_by_upc
- Add SQLite LIKE fallback for test compatibility
- Update upc_variants column to JSONB with variant for cross-db support
- Add GIN index migration for upc_variants

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-04-14 11:59:28 +00:00
13 changed files with 115 additions and 411 deletions
@@ -0,0 +1,38 @@
"""Add GIN index on upc_variants and alter column to JSONB.
Revision ID: 009_add_gin_index_upc_variants
Revises: 008_create_domain_tables
Create Date: 2026-04-14
"""
import sqlalchemy as sa
from alembic import op
revision = "009_add_gin_index_upc_variants"
down_revision = "008_create_domain_tables"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.alter_column(
"normalized_products",
"upc_variants",
type_=sa.dialects.postgresql.JSONB(),
postgresql_using="upc_variants::jsonb",
)
op.create_index(
"ix_normalized_products_upc_variants_gin",
"normalized_products",
["upc_variants"],
postgresql_using="gin",
)
def downgrade() -> None:
op.drop_index("ix_normalized_products_upc_variants_gin", table_name="normalized_products")
op.alter_column(
"normalized_products",
"upc_variants",
type_=sa.JSON(),
)
+8 -33
View File
@@ -1,51 +1,26 @@
"""Redis/DragonflyDB caching helpers.""" """Redis/DragonflyDB caching helpers."""
import redis.asyncio as redis
from cartsnitch_api.config import settings from cartsnitch_api.config import settings
class CacheClient: class CacheClient:
"""Redis/DragonflyDB caching with connection pooling. """Stub for Redis/DragonflyDB caching.
Will be used for expensive queries: price trends, product comparisons. Will be used for expensive queries: price trends, product comparisons.
Cache invalidation via Redis pub/sub events from other services. Cache invalidation via Redis pub/sub events from other services.
""" """
def __init__(self) -> None: def __init__(self) -> None:
self._pool: redis.ConnectionPool | None = None self.url = settings.redis_url
self._client: redis.Redis | None = None
async def initialize(self) -> None:
"""Initialize the Redis connection pool."""
self._pool = redis.ConnectionPool.from_url(
settings.redis_url,
max_connections=20,
decode_responses=True,
)
self._client = redis.Redis(connection_pool=self._pool)
async def close(self) -> None:
"""Close the Redis connection pool."""
if self._client:
await self._client.aclose()
if self._pool:
await self._pool.aclose()
async def get(self, key: str) -> str | None: async def get(self, key: str) -> str | None:
if not self._client: # TODO: implement with redis-py async
return None return None
return await self._client.get(key)
async def set(self, key: str, value: str, ttl_seconds: int = 300) -> None: async def set(self, key: str, value: str, ttl_seconds: int = 300) -> None:
if not self._client: # TODO: implement with redis-py async
return pass
await self._client.set(key, value, ex=ttl_seconds)
async def delete(self, key: str) -> None: async def delete(self, key: str) -> None:
if not self._client: # TODO: implement with redis-py async
return pass
await self._client.delete(key)
cache_client = CacheClient()
+1 -6
View File
@@ -32,9 +32,6 @@ class Settings(BaseSettings):
rate_limit_requests: int = 60 rate_limit_requests: int = 60
rate_limit_window_seconds: int = 60 rate_limit_window_seconds: int = 60
rate_limit_auth_requests: int = 5
rate_limit_auth_window_seconds: int = 60
rate_limit_redis_enabled: bool = True
rate_limit_enabled: bool = True rate_limit_enabled: bool = True
_PLACEHOLDER_VALUES = {"change-me-in-production"} _PLACEHOLDER_VALUES = {"change-me-in-production"}
@@ -75,9 +72,7 @@ class Settings(BaseSettings):
def normalize_database_url(self): def normalize_database_url(self):
"""Normalize postgresql:// → postgresql+asyncpg:// for the asyncpg driver.""" """Normalize postgresql:// → postgresql+asyncpg:// for the asyncpg driver."""
if self.database_url.startswith("postgresql://"): if self.database_url.startswith("postgresql://"):
self.database_url = self.database_url.replace( self.database_url = self.database_url.replace("postgresql://", "postgresql+asyncpg://", 1)
"postgresql://", "postgresql+asyncpg://", 1
)
return self return self
+1 -13
View File
@@ -6,14 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn
from cartsnitch_api.config import settings from cartsnitch_api.config import settings
engine = create_async_engine( engine = create_async_engine(settings.database_url, echo=False)
settings.database_url,
echo=False,
pool_size=10,
max_overflow=20,
pool_pre_ping=True,
pool_recycle=3600,
)
async_session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) async_session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
@@ -21,8 +14,3 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]:
"""FastAPI dependency that yields an async DB session.""" """FastAPI dependency that yields an async DB session."""
async with async_session_factory() as session: async with async_session_factory() as session:
yield session yield session
async def dispose_engine() -> None:
"""Dispose the database engine, closing all pooled connections."""
await engine.dispose()
+2 -5
View File
@@ -5,8 +5,6 @@ from contextlib import asynccontextmanager
from fastapi import APIRouter, FastAPI from fastapi import APIRouter, FastAPI
from cartsnitch_api.auth.routes import router as auth_router from cartsnitch_api.auth.routes import router as auth_router
from cartsnitch_api.cache import cache_client
from cartsnitch_api.database import dispose_engine
from cartsnitch_api.middleware.cors import add_cors_middleware from cartsnitch_api.middleware.cors import add_cors_middleware
from cartsnitch_api.middleware.error_handler import add_error_handlers, add_error_monitor_middleware from cartsnitch_api.middleware.error_handler import add_error_handlers, add_error_monitor_middleware
from cartsnitch_api.middleware.rate_limit import add_rate_limit_middleware from cartsnitch_api.middleware.rate_limit import add_rate_limit_middleware
@@ -25,10 +23,9 @@ from cartsnitch_api.routes.user import router as user_router
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
await cache_client.initialize() # TODO: initialize DB session pool, Redis connection, service clients
yield yield
await cache_client.close() # TODO: cleanup connections
await dispose_engine()
def create_app() -> FastAPI: def create_app() -> FastAPI:
+19 -101
View File
@@ -4,32 +4,18 @@ Uses in-memory sliding window as fallback, Redis/DragonflyDB when available.
Per-IP limiting on public endpoints, per-token limiting on authenticated endpoints. Per-IP limiting on public endpoints, per-token limiting on authenticated endpoints.
""" """
import hashlib
import logging
import time import time
import uuid
from collections import defaultdict from collections import defaultdict
from threading import Lock from threading import Lock
from typing import Protocol
from fastapi import FastAPI, Request, status from fastapi import FastAPI, Request, status
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from redis.asyncio import Redis, RedisError
from starlette.middleware.base import BaseHTTPMiddleware from starlette.middleware.base import BaseHTTPMiddleware
from cartsnitch_api.config import settings from cartsnitch_api.config import settings
logger = logging.getLogger(__name__)
class _SlidingWindowCounter:
class RateLimitBackend(Protocol):
"""Protocol for rate limit backends."""
async def is_allowed(self, key: str) -> tuple[bool, int, int]:
"""Check if request is allowed. Returns (allowed, remaining, retry_after)."""
class InMemorySlidingWindow:
"""Thread-safe in-memory sliding window rate limiter.""" """Thread-safe in-memory sliding window rate limiter."""
def __init__(self, max_requests: int, window_seconds: int) -> None: def __init__(self, max_requests: int, window_seconds: int) -> None:
@@ -38,12 +24,13 @@ class InMemorySlidingWindow:
self._hits: dict[str, list[float]] = defaultdict(list) self._hits: dict[str, list[float]] = defaultdict(list)
self._lock = Lock() self._lock = Lock()
async def is_allowed(self, key: str) -> tuple[bool, int, int]: def is_allowed(self, key: str) -> tuple[bool, int, int]:
"""Check if request is allowed. Returns (allowed, remaining, retry_after).""" """Check if request is allowed. Returns (allowed, remaining, retry_after)."""
now = time.monotonic() now = time.monotonic()
cutoff = now - self.window_seconds cutoff = now - self.window_seconds
with self._lock: with self._lock:
# Prune expired entries
self._hits[key] = [t for t in self._hits[key] if t > cutoff] self._hits[key] = [t for t in self._hits[key] if t > cutoff]
current_count = len(self._hits[key]) current_count = len(self._hits[key])
@@ -56,84 +43,15 @@ class InMemorySlidingWindow:
return True, remaining, 0 return True, remaining, 0
class RedisSlidingWindow: # Module-level counters — one for public (per-IP), one for auth (per-token)
"""Redis-backed sliding window rate limiter using sorted sets.""" _public_limiter = _SlidingWindowCounter(
max_requests=settings.rate_limit_requests,
def __init__(self, redis: Redis, max_requests: int, window_seconds: int) -> None: window_seconds=settings.rate_limit_window_seconds,
self.redis = redis )
self.max_requests = max_requests _auth_limiter = _SlidingWindowCounter(
self.window_seconds = window_seconds max_requests=settings.rate_limit_requests * 5, # 300/min for authenticated users
window_seconds=settings.rate_limit_window_seconds,
async def is_allowed(self, key: str) -> tuple[bool, int, int]: )
"""Check if request is allowed. Returns (allowed, remaining, retry_after)."""
try:
now = time.monotonic()
cutoff = now - self.window_seconds
now_ms = int(now * 1000)
cutoff_ms = int(cutoff * 1000)
pipe = self.redis.pipeline()
pipe.zremrangebyscore(key, 0, cutoff_ms)
pipe.zcard(key)
results = await pipe.execute()
current_count = results[1]
if current_count >= self.max_requests:
oldest = await self.redis.zrange(key, 0, 0, withscores=True)
if oldest:
retry_after = int((oldest[0][1] - cutoff) / 1000) + 1
else:
retry_after = self.window_seconds
return False, 0, retry_after
member = f"{now_ms}:{uuid.uuid4().hex[:8]}"
pipe = self.redis.pipeline()
pipe.zadd(key, {member: now_ms})
pipe.expire(key, self.window_seconds)
await pipe.execute()
remaining = self.max_requests - current_count - 1
return True, remaining, 0
except RedisError as e:
logger.warning("Redis rate limit error, falling back to in-memory: %s", e)
in_memory = InMemorySlidingWindow(self.max_requests, self.window_seconds)
return await in_memory.is_allowed(key)
_redis_client: Redis | None = None
_use_redis = False
if settings.rate_limit_redis_enabled:
try:
_redis_client = Redis.from_url(settings.redis_url)
_use_redis = True
logger.info("Rate limiting will use Redis at %s", settings.redis_url)
except Exception as e:
logger.warning("Failed to connect to Redis for rate limiting, using in-memory: %s", e)
_use_redis = False
if _use_redis and _redis_client:
_public_limiter = RedisSlidingWindow(
_redis_client, settings.rate_limit_requests, settings.rate_limit_window_seconds
)
_auth_limiter = RedisSlidingWindow(
_redis_client, settings.rate_limit_requests * 5, settings.rate_limit_window_seconds
)
_auth_strict_limiter = RedisSlidingWindow(
_redis_client, settings.rate_limit_auth_requests, settings.rate_limit_auth_window_seconds
)
else:
_public_limiter = InMemorySlidingWindow(
settings.rate_limit_requests, settings.rate_limit_window_seconds
)
_auth_limiter = InMemorySlidingWindow(
settings.rate_limit_requests * 5, settings.rate_limit_window_seconds
)
_auth_strict_limiter = InMemorySlidingWindow(
settings.rate_limit_auth_requests, settings.rate_limit_auth_window_seconds
)
def _get_client_ip(request: Request) -> str: def _get_client_ip(request: Request) -> str:
@@ -144,30 +62,30 @@ def _get_client_ip(request: Request) -> str:
return request.client.host if request.client else "unknown" return request.client.host if request.client else "unknown"
def _get_rate_limit_key(request: Request) -> tuple[str, RateLimitBackend]: def _get_rate_limit_key(request: Request) -> tuple[str, _SlidingWindowCounter]:
"""Determine rate limit key and which limiter to use.""" """Determine rate limit key and which limiter to use."""
if request.url.path.startswith("/public"): if request.url.path.startswith("/public"):
return f"ip:{_get_client_ip(request)}", _public_limiter return f"ip:{_get_client_ip(request)}", _public_limiter
if request.url.path.startswith("/auth/") and request.method == "POST": # For authenticated endpoints, use Bearer token as key if present
return f"ip:{_get_client_ip(request)}", _auth_strict_limiter
auth_header = request.headers.get("authorization", "") auth_header = request.headers.get("authorization", "")
if auth_header.startswith("Bearer "): if auth_header.startswith("Bearer "):
token = auth_header[7:] token = auth_header[7:]
token_hash = hashlib.sha256(token.encode()).hexdigest() # Use last 16 chars of token as key to avoid storing full tokens
return f"token:{token_hash}", _auth_limiter return f"token:{token[-16:]}", _auth_limiter
# Fallback to IP for unauthenticated non-public endpoints
return f"ip:{_get_client_ip(request)}", _public_limiter return f"ip:{_get_client_ip(request)}", _public_limiter
class RateLimitMiddleware(BaseHTTPMiddleware): class RateLimitMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next): async def dispatch(self, request: Request, call_next):
# Skip rate limiting when disabled (e.g. in tests) or for health checks
if not settings.rate_limit_enabled or request.url.path == "/health": if not settings.rate_limit_enabled or request.url.path == "/health":
return await call_next(request) return await call_next(request)
key, limiter = _get_rate_limit_key(request) key, limiter = _get_rate_limit_key(request)
allowed, remaining, retry_after = await limiter.is_allowed(key) allowed, remaining, retry_after = limiter.is_allowed(key)
if not allowed: if not allowed:
return JSONResponse( return JSONResponse(
+18 -154
View File
@@ -1,184 +1,47 @@
"""Tests for rate limiting middleware.""" """Tests for rate limiting middleware."""
import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest import pytest
from cartsnitch_api.config import settings from cartsnitch_api.middleware.rate_limit import _SlidingWindowCounter
from cartsnitch_api.middleware.rate_limit import (
InMemorySlidingWindow,
RedisSlidingWindow,
_get_client_ip,
_get_rate_limit_key,
)
class TestInMemorySlidingWindow: class TestSlidingWindowCounter:
def test_allows_within_limit(self): def test_allows_within_limit(self):
limiter = InMemorySlidingWindow(max_requests=5, window_seconds=60) counter = _SlidingWindowCounter(max_requests=5, window_seconds=60)
for i in range(5): for i in range(5):
allowed, remaining, retry = limiter.is_allowed("test-key") allowed, remaining, retry = counter.is_allowed("test-key")
assert allowed is True assert allowed is True
assert remaining == 4 - i assert remaining == 4 - i
def test_blocks_over_limit(self): def test_blocks_over_limit(self):
limiter = InMemorySlidingWindow(max_requests=3, window_seconds=60) counter = _SlidingWindowCounter(max_requests=3, window_seconds=60)
for _ in range(3): for _ in range(3):
limiter.is_allowed("test-key") counter.is_allowed("test-key")
allowed, remaining, retry = limiter.is_allowed("test-key") allowed, remaining, retry = counter.is_allowed("test-key")
assert allowed is False assert allowed is False
assert remaining == 0 assert remaining == 0
assert retry > 0 assert retry > 0
def test_separate_keys(self): def test_separate_keys(self):
limiter = InMemorySlidingWindow(max_requests=2, window_seconds=60) counter = _SlidingWindowCounter(max_requests=2, window_seconds=60)
limiter.is_allowed("key-a") # Fill key-a
limiter.is_allowed("key-a") counter.is_allowed("key-a")
allowed_a, _, _ = limiter.is_allowed("key-a") counter.is_allowed("key-a")
allowed_a, _, _ = counter.is_allowed("key-a")
assert allowed_a is False assert allowed_a is False
allowed_b, remaining, _ = limiter.is_allowed("key-b") # key-b should still be allowed
allowed_b, remaining, _ = counter.is_allowed("key-b")
assert allowed_b is True assert allowed_b is True
assert remaining == 1 assert remaining == 1
def test_resets_after_window_expires(self):
limiter = InMemorySlidingWindow(max_requests=2, window_seconds=1)
for _ in range(2):
limiter.is_allowed("test-key")
allowed, remaining, _ = limiter.is_allowed("test-key")
assert allowed is False
time.sleep(1.1)
allowed, remaining, _ = limiter.is_allowed("test-key")
assert allowed is True
assert remaining == 1
class TestGetClientIp:
def test_x_forwarded_for_single(self):
req = MagicMock()
req.headers = {"x-forwarded-for": "192.168.1.1"}
req.client = None
assert _get_client_ip(req) == "192.168.1.1"
def test_x_forwarded_for_multiple(self):
req = MagicMock()
req.headers = {"x-forwarded-for": "192.168.1.1, 10.0.0.1, 172.16.0.1"}
req.client = None
assert _get_client_ip(req) == "192.168.1.1"
def test_x_forwarded_for_with_port(self):
req = MagicMock()
req.headers = {"x-forwarded-for": "192.168.1.1:8080"}
req.client = None
assert _get_client_ip(req) == "192.168.1.1"
def test_no_forwarded_header(self):
req = MagicMock()
req.headers = {}
req.client.host = "127.0.0.1"
assert _get_client_ip(req) == "127.0.0.1"
def test_no_client(self):
req = MagicMock()
req.headers = {}
req.client = None
assert _get_client_ip(req) == "unknown"
class TestGetRateLimitKey:
def _make_request(
self,
path: str = "/purchases",
method: str = "GET",
auth_header: str = "",
headers: dict | None = None,
) -> MagicMock:
req = MagicMock()
req.url.path = path
req.method = method
req.headers = dict(headers) if headers else {}
if auth_header:
req.headers["authorization"] = auth_header
return req
def test_public_path_uses_public_limiter(self):
req = self._make_request("/public/inflation")
key, limiter = _get_rate_limit_key(req)
assert key.startswith("ip:")
assert limiter.max_requests == settings.rate_limit_requests
def test_auth_post_path_uses_strict_limiter(self):
req = self._make_request("/auth/login", method="POST")
key, limiter = _get_rate_limit_key(req)
assert key.startswith("ip:")
assert limiter.max_requests == settings.rate_limit_auth_requests
assert limiter.window_seconds == settings.rate_limit_auth_window_seconds
def test_auth_get_path_uses_auth_limiter(self):
req = self._make_request("/auth/me", method="GET")
key, limiter = _get_rate_limit_key(req)
assert key.startswith("ip:")
assert limiter.max_requests == settings.rate_limit_requests * 5
def test_authenticated_token_uses_auth_limiter(self):
req = self._make_request("/purchases", auth_header="Bearer token123")
key, limiter = _get_rate_limit_key(req)
assert key.startswith("token:")
assert limiter.max_requests == settings.rate_limit_requests * 5
def test_distinct_tokens_produce_distinct_keys(self):
req1 = self._make_request("/purchases", auth_header="Bearer token_alpha_12345")
req2 = self._make_request("/purchases", auth_header="Bearer token_beta_67890")
key1, _ = _get_rate_limit_key(req1)
key2, _ = _get_rate_limit_key(req2)
assert key1 != key2
def test_same_token_produces_same_key(self):
req1 = self._make_request("/purchases", auth_header="Bearer same_token_value_abc")
req2 = self._make_request("/purchases", auth_header="Bearer same_token_value_abc")
key1, _ = _get_rate_limit_key(req1)
key2, _ = _get_rate_limit_key(req2)
assert key1 == key2
def test_key_does_not_contain_raw_token_suffix(self):
raw_token = "my_secret_jwt_token_xyz"
req = self._make_request("/purchases", auth_header=f"Bearer {raw_token}")
key, _ = _get_rate_limit_key(req)
assert raw_token[-16:] not in key
assert raw_token not in key
class TestRedisSlidingWindowFallback:
@pytest.mark.asyncio
async def test_fallback_on_redis_connection_error(self):
mock_redis = AsyncMock()
mock_redis.pipeline.return_value = AsyncMock()
pipe_mock = AsyncMock()
pipe_mock.execute.side_effect = Exception("Connection refused")
mock_redis.pipeline.return_value = pipe_mock
limiter = RedisSlidingWindow(mock_redis, max_requests=5, window_seconds=60)
allowed, remaining, retry = await limiter.is_allowed("test-key")
assert allowed is True
assert remaining == 4
@pytest.mark.asyncio
async def test_fallback_on_redis_error_during_pipeline(self):
mock_redis = AsyncMock()
pipe_mock = AsyncMock()
pipe_mock.execute.side_effect = Exception("Redis error")
mock_redis.pipeline.return_value = pipe_mock
limiter = RedisSlidingWindow(mock_redis, max_requests=3, window_seconds=60)
allowed, remaining, retry = await limiter.is_allowed("test-key")
assert allowed is True
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_rate_limit_returns_429(client): async def test_rate_limit_returns_429(client):
"""Public endpoint should return 429 after limit exceeded."""
# The default limit is 60/min — we won't hit it in normal tests,
# but we verify the middleware adds rate limit headers.
resp = await client.get("/public/inflation") resp = await client.get("/public/inflation")
assert "x-ratelimit-limit" in resp.headers assert "x-ratelimit-limit" in resp.headers
assert "x-ratelimit-remaining" in resp.headers assert "x-ratelimit-remaining" in resp.headers
@@ -186,6 +49,7 @@ async def test_rate_limit_returns_429(client):
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_health_skips_rate_limit(client): async def test_health_skips_rate_limit(client):
"""Health endpoint should not have rate limit headers."""
resp = await client.get("/health") resp = await client.get("/health")
assert resp.status_code == 200 assert resp.status_code == 200
assert "x-ratelimit-limit" not in resp.headers assert "x-ratelimit-limit" not in resp.headers
@@ -3,6 +3,7 @@
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from sqlalchemy import JSON, String from sqlalchemy import JSON, String
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm import Mapped, mapped_column, relationship
from cartsnitch_common.constants import ProductCategory, SizeUnit from cartsnitch_common.constants import ProductCategory, SizeUnit
@@ -26,7 +27,9 @@ class NormalizedProduct(UUIDPrimaryKeyMixin, TimestampMixin, Base):
brand: Mapped[str | None] = mapped_column(String(200)) brand: Mapped[str | None] = mapped_column(String(200))
size: Mapped[str | None] = mapped_column(String(50)) size: Mapped[str | None] = mapped_column(String(50))
size_unit: Mapped[SizeUnit | None] = mapped_column(String(10)) size_unit: Mapped[SizeUnit | None] = mapped_column(String(10))
upc_variants: Mapped[list[str] | None] = mapped_column(JSON, default=list) upc_variants: Mapped[list[str] | None] = mapped_column(
JSON().with_variant(JSONB(), "postgresql"), default=list
)
# Relationships # Relationships
purchase_items: Mapped[list["PurchaseItem"]] = relationship(back_populates="normalized_product") purchase_items: Mapped[list["PurchaseItem"]] = relationship(back_populates="normalized_product")
+3 -3
View File
@@ -9805,9 +9805,9 @@
} }
}, },
"node_modules/vite": { "node_modules/vite": {
"version": "6.4.1", "version": "6.4.2",
"resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz",
"integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==",
"devOptional": true, "devOptional": true,
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
+1 -34
View File
@@ -1,12 +1,8 @@
"""Service-specific configuration for ReceiptWitness.""" """Service-specific configuration for ReceiptWitness."""
from pydantic import model_validator
from pydantic_settings import BaseSettings from pydantic_settings import BaseSettings
_PLACEHOLDER_VALUES = {"change-me-in-production"}
class ReceiptWitnessSettings(BaseSettings): class ReceiptWitnessSettings(BaseSettings):
model_config = {"env_prefix": "RW_"} model_config = {"env_prefix": "RW_"}
@@ -34,34 +30,5 @@ class ReceiptWitnessSettings(BaseSettings):
# Mailgun inbound email webhook # Mailgun inbound email webhook
mailgun_webhook_signing_key: str = "" mailgun_webhook_signing_key: str = ""
@model_validator(mode="after")
def validate_required_vars(self):
errors = []
if not self.session_encryption_key or self.session_encryption_key in _PLACEHOLDER_VALUES:
errors.append(
"RW_SESSION_ENCRYPTION_KEY must be set to a secure value. "
'Generate one with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"'
)
if self.notifications_enabled and not self.resend_api_key:
errors.append(
"RW_RESEND_API_KEY must be set when RW_NOTIFICATIONS_ENABLED=true. "
"Get an API key from https://resend.com/api-keys"
)
if errors:
raise ValueError(
"ReceiptWitness startup failed — missing required config:\n"
+ "\n".join(f" - {e}" for e in errors)
)
return self
settings = ReceiptWitnessSettings()
class _LazySettings:
_instance: ReceiptWitnessSettings | None = None
def __getattr__(self, name: str):
if _LazySettings._instance is None:
_LazySettings._instance = ReceiptWitnessSettings()
return getattr(_LazySettings._instance, name)
settings = _LazySettings()
@@ -5,12 +5,14 @@ Matches products across retailers by:
2. Fuzzy name matching via token-based Jaccard similarity (lower confidence) 2. Fuzzy name matching via token-based Jaccard similarity (lower confidence)
""" """
import json
import re import re
from dataclasses import dataclass from dataclasses import dataclass
from enum import StrEnum from enum import StrEnum
from cartsnitch_common.models.product import NormalizedProduct from cartsnitch_common.models.product import NormalizedProduct
from sqlalchemy import select from sqlalchemy import cast, func, select, String
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -96,17 +98,24 @@ def jaccard_similarity(a: str, b: str) -> float:
def match_by_upc(session: Session, upc: str) -> MatchResult | None: def match_by_upc(session: Session, upc: str) -> MatchResult | None:
"""Find a normalized product by exact UPC match. """Find a normalized product by exact UPC match.
Loads products with upc_variants and checks membership in Python Uses PostgreSQL JSONB containment (@>) for production efficiency.
for cross-database compatibility (works on both PostgreSQL and SQLite). Falls back to LIKE on SQLite for test compatibility.
""" """
# TODO: Use PostgreSQL JSON containment query (@>) for production. dialect_name = session.bind.dialect.name if session.bind else "default"
# Current approach loads all products into memory — acceptable for tests if dialect_name == "postgresql":
# and small datasets, but will not scale. stmt = select(NormalizedProduct).where(
stmt = select(NormalizedProduct).where(NormalizedProduct.upc_variants.is_not(None)) cast(NormalizedProduct.upc_variants, JSONB).op("@>")(
products = session.execute(stmt).scalars().all() func.cast(json.dumps([upc]), JSONB)
for product in products: )
if product.upc_variants and upc in product.upc_variants: )
return MatchResult(product=product, confidence=1.0, method=MatchMethod.UPC) else:
stmt = select(NormalizedProduct).where(
NormalizedProduct.upc_variants.is_not(None),
cast(NormalizedProduct.upc_variants, String).contains(upc),
)
product = session.execute(stmt).scalars().first()
if product:
return MatchResult(product=product, confidence=1.0, method=MatchMethod.UPC)
return None return None
-4
View File
@@ -1,16 +1,12 @@
"""Shared test fixtures.""" """Shared test fixtures."""
import json import json
import os
from pathlib import Path from pathlib import Path
import pytest import pytest
FIXTURES_DIR = Path(__file__).parent / "fixtures" FIXTURES_DIR = Path(__file__).parent / "fixtures"
os.environ.setdefault("RW_SESSION_ENCRYPTION_KEY", "test-secret-key-for-unit-tests-only-32bytes!")
os.environ.setdefault("RW_MAILGUN_WEBHOOK_SIGNING_KEY", "test-mailgun-signing-key")
@pytest.fixture @pytest.fixture
def meijer_receipt_data() -> dict: def meijer_receipt_data() -> dict:
-46
View File
@@ -1,46 +0,0 @@
import pytest
from receiptwitness.config import ReceiptWitnessSettings
def test_valid_config():
s = ReceiptWitnessSettings(
session_encryption_key="7reF42nmTwbdN21PBoubGp7h_FU8qSimstmlaMLoRK8="
)
assert s.session_encryption_key
def test_missing_session_encryption_key_raises():
with pytest.raises(ValueError, match="RW_SESSION_ENCRYPTION_KEY"):
ReceiptWitnessSettings(session_encryption_key="")
def test_placeholder_session_encryption_key_raises():
with pytest.raises(ValueError, match="RW_SESSION_ENCRYPTION_KEY"):
ReceiptWitnessSettings(session_encryption_key="change-me-in-production")
def test_notifications_enabled_without_resend_key_raises():
with pytest.raises(ValueError, match="RW_RESEND_API_KEY"):
ReceiptWitnessSettings(
session_encryption_key="7reF42nmTwbdN21PBoubGp7h_FU8qSimstmlaMLoRK8=",
notifications_enabled=True,
resend_api_key="",
)
def test_notifications_disabled_without_resend_key_ok():
s = ReceiptWitnessSettings(
session_encryption_key="7reF42nmTwbdN21PBoubGp7h_FU8qSimstmlaMLoRK8=",
notifications_enabled=False,
resend_api_key="",
)
assert s.notifications_enabled is False
def test_notifications_enabled_with_resend_key_ok():
s = ReceiptWitnessSettings(
session_encryption_key="7reF42nmTwbdN21PBoubGp7h_FU8qSimstmlaMLoRK8=",
notifications_enabled=True,
resend_api_key="re_test_1234567890",
)
assert s.resend_api_key == "re_test_1234567890"