feat: add monitoring and instrumentation

.gitignore

@@ -1,5 +1,6 @@
 todo.md
 .env
+.env.prod
 
 /Language*Learning*API/
-__pycache__/
+__pycache__/

Makefile

@@ -1,16 +1,32 @@
-.PHONY: down build up logs shell lock migrate migration import-dictionary
+.PHONY: down build up logs shell lock migrate migration import-dictionary run-prod-locally
 
 build:
 	docker compose build --no-cache
 
-up:
+build-dev:
-	docker compose up -d
+	docker compose -f docker-compose-dev.yml --env-file .env build --no-cache
+	
+up-dev:
+	docker compose -f docker-compose-dev.yml --env-file .env up -d
+
+up-prod:
+	docker compose -f docker-compose-prod.yml --env-file .env.prod up -d
+
+run-prod-locally:
+	docker compose -f docker-compose-prod.yml -f docker-compose-local-override.yml --env-file .env.prod build && \
+	docker compose -f docker-compose-prod.yml -f docker-compose-local-override.yml --env-file .env.prod up
 
 down:
 	docker compose down
 
-logs:
+down-dev:
-	docker compose logs -f api
+	docker compose -f docker-compose-dev.yml down
+
+logs-dev:
+	docker compose -f docker-compose-dev.yml logs -f
+
+logs-prod:
+	docker compose -f docker-compose-prod.yml logs -f
 
 shell:
 	docker compose exec api bash

README.md

@@ -12,6 +12,8 @@ Language Learning App is a set of software packages that deliver a language lear
 
 ## Description of product
 
+This is an app designed to help people learn a second(+) language. Initially from English. The app will start with French, Spanish, Italian, and German as the target languages. With English as the only source language.
+
 Although spaced repetition is an effective mechanism to better remember words, showing words in context remains an important "before" step. This app adds value by providing the user with realistic-looking written and audio content in the language(s) they are learning at an appropriate level. From there, the user can identify vocabulary that they are unfamiliar with, and would like to commit to memory.
 
 Additionally, Language Learning App treats the text-audio pair as important. Language learners don't just want to be able to read and write a language, they need to know how words sound
@@ -20,15 +22,11 @@ At present, the app doesn't have a solution to recognising speech, another impor
 
 ## Technical Specifics
 
-This is an app designed to help people learn a second(+) language. Initially from English. The app will start with French, Spanish, Italian, and German as the target languages. With English as the only source language.
-
 The application has a back-end written in python (fastapi), because of the Python ecosystem around data and machine learning.
 
-In the future, the API will generate XML endpoints for postcast-playing app integration (as it's an audio-first medium).
-
 The application has a web-based front end written in Svelte Kit. It will adopt progressive web app standards, to allow offline use. Due to technical complexity, and limitations, there are no plans for native app development.
 
-The app relies on containerisation and docker to orchestrate moving parts. In production, there will be a need to consider Content Delivery Networks (CDNs) as high bandwidth is expected.
+The app relies on containerisation and docker to orchestrate moving parts.
 
 Content generation relies heavily on asynchronous jobs.
 
@@ -36,6 +34,200 @@ The app should rely on self-hostable infrastructure as much as possible. Vendor-
 
 Communication between the two is through HTTP, authenticated with JWT tokens.
 
+## Running Locally with Docker Compose
+
+This project supports two local runtime modes:
+
+- Development mode: live-reload API and local MinIO storage.
+- Production mode (run locally): production compose stack and Bunny-backed storage/CDN config.
+
+### Make targets you will use most
+
+- `make build-dev`: build images for dev stack (`docker-compose-dev.yml`).
+- `make up-dev`: start dev stack in detached mode.
+- `make logs-dev`: stream logs for the dev stack.
+- `make up-prod`: start production stack in detached mode (`docker-compose-prod.yml`).
+- `make logs-prod`: stream logs for the production stack.
+- `make run-prod-locally`: build and run production stack locally with `docker-compose-local-override.yml`.
+- `make down`: stop containers.
+- `make migrate`: rebuild API image and run `alembic upgrade head` in the running API container.
+- `make migrate-no-build`: run pending migrations without rebuilding.
+
+### 1) Development mode (MinIO)
+
+Create `.env` in repo root with at least:
+
+```dotenv
+POSTGRES_PASSWORD=replace_me
+JWT_SECRET=replace_me
+ANTHROPIC_API_KEY=replace_me
+DEEPL_API_KEY=replace_me
+DEEPGRAM_API_KEY=replace_me
+GEMINI_API_KEY=replace_me
+STORAGE_SECRET_KEY=replace_me
+
+# Optional (have defaults in compose)
+# POSTGRES_USER=langlearn
+# POSTGRES_DB=langlearn
+# STORAGE_ACCESS_KEY=langlearn
+# STORAGE_BUCKET=langlearn
+# API_PORT=8000
+# FRONTEND_PORT=3000
+# API_BASE_URL=http://localhost:8000
+# ORIGIN=http://localhost:3000
+# PUBLIC_API_BASE_URL=http://api:8000
+```
+
+Run:
+
+```sh
+make build-dev
+make up-dev
+make migrate-no-build
+make logs-dev
+```
+
+Services in dev mode:
+
+- API: `http://localhost:8000`
+- Frontend: `http://localhost:3000`
+- MinIO API: `http://localhost:9000`
+- MinIO Console: `http://localhost:9001`
+
+### 2) Production mode, locally
+
+Create `.env.prod` in repo root. This mode uses Bunny for storage/CDN and production-like API startup.
+
+```dotenv
+POSTGRES_PASSWORD=replace_me
+JWT_SECRET=replace_me
+ADMIN_USER_EMAILS=admin@example.com
+API_BASE_URL=http://localhost:8000
+PUBLIC_API_BASE_URL=http://localhost:8000
+ORIGIN=http://localhost:3000
+
+ANTHROPIC_API_KEY=replace_me
+DEEPL_API_KEY=replace_me
+DEEPGRAM_API_KEY=replace_me
+GEMINI_API_KEY=replace_me
+
+TRANSACTIONAL_EMAIL_PROVIDER=stub
+
+BUNNY_ZONE=replace_me
+BUNNY_API_KEY=replace_me
+BUNNY_CDN_BASE_URL=https://your-zone.b-cdn.net
+BUNNY_TOKEN_AUTH_KEY=replace_me
+BUNNY_STORAGE_ENDPOINT=https://storage.bunnycdn.com
+
+# Optional (have defaults in compose)
+# POSTGRES_USER=langlearn
+# POSTGRES_DB=langlearn
+# API_PORT=8000
+# FRONTEND_PORT=3000
+```
+
+Run:
+
+```sh
+make run-prod-locally
+```
+
+This command uses:
+
+- `docker-compose-prod.yml`
+- `docker-compose-local-override.yml` (binds Postgres to `127.0.0.1` for local tooling)
+
+## Database Access
+
+To connect a local SQL viewer to the Postgres instance without exposing the port, use an SSH tunnel.
+
+**Local machine** (running `docker-compose-prod.yml` locally):
+
+Use the local override, which binds Postgres to `127.0.0.1` only:
+
+```sh
+docker compose -f docker-compose-prod.yml -f docker-compose-local-override.yml up
+```
+
+Then point your SQL viewer at `localhost:5432` directly.
+
+**Remote machine** (`lla.thomaswilson.xyz`):
+
+```sh
+ssh -L 5432:localhost:5432 wilson@lla.thomaswilson.xyz -N
+```
+
+Then point your SQL viewer at `localhost:5432`. Run in a separate terminal and kill it when done.
+
+## Deploying to a Self-Hosted Server
+
+This stack is designed for VPS/self-host deployments driven by Docker Compose.
+
+### High-level deployment flow
+
+1. Install Docker Engine and Docker Compose plugin on the server.
+2. Clone this repository on the server.
+3. Create `.env.prod` in the repo root.
+4. Start services with:
+
+```sh
+docker compose -f docker-compose-prod.yml --env-file .env.prod up -d --build
+```
+
+5. Check status and logs:
+
+```sh
+docker compose -f docker-compose-prod.yml --env-file .env.prod ps
+docker compose -f docker-compose-prod.yml --env-file .env.prod logs -f
+```
+
+### Production environment variables
+
+Configure these in `.env.prod`.
+
+Core application and auth:
+
+- `JWT_SECRET`: JWT signing secret used by API and frontend private auth logic.
+- `ADMIN_USER_EMAILS`: comma-separated admin emails.
+- `API_BASE_URL`: public base URL of API (used in generated links).
+- `ORIGIN`: frontend origin URL.
+- `PUBLIC_API_BASE_URL`: API URL exposed to frontend runtime/build.
+
+Database:
+
+- `POSTGRES_USER`: database username (default `langlearn` if omitted).
+- `POSTGRES_PASSWORD`: database password.
+- `POSTGRES_DB`: database name (default `langlearn` if omitted).
+
+AI/translation providers:
+
+- `ANTHROPIC_API_KEY`
+- `DEEPL_API_KEY`
+- `DEEPGRAM_API_KEY`
+- `GEMINI_API_KEY`
+
+Object storage and CDN (Bunny in production):
+
+- `BUNNY_ZONE`: Bunny storage zone name.
+- `BUNNY_API_KEY`: Bunny storage API key.
+- `BUNNY_CDN_BASE_URL`: CDN base URL (for example `https://your-zone.b-cdn.net`).
+- `BUNNY_TOKEN_AUTH_KEY`: token auth key used to sign expiring URLs.
+- `BUNNY_STORAGE_ENDPOINT`: Bunny storage endpoint base URL (for example `https://storage.bunnycdn.com`).
+
+Email delivery:
+
+- `TRANSACTIONAL_EMAIL_PROVIDER`: `stub` or `scaleway`.
+- If using Scaleway TEM, also set:
+  - `SCALEWAY_TEM_SECRET_KEY`
+  - `SCALEWAY_TEM_PROJECT_ID`
+  - `SCALEWAY_TEM_FROM_ADDRESS`
+  - `SCALEWAY_TEM_REGION` (defaults to `fr-par`)
+
+Optional port overrides:
+
+- `API_PORT`: host port for API container (default `8000`).
+- `FRONTEND_PORT`: host port for frontend container (default `3000`).
+
 ## Technical Design
 
 This application must remain self-hostable. It should not rely on proprietary infrastructure (e.g. AWS Lambda functions) to run. It should use Docker Compose and Makefiles to build projects and deploy them onto a local server or a VPS.
@@ -43,5 +235,7 @@ This application must remain self-hostable. It should not rely on proprietary in
 The main components so far are:
 
 - Backend server (fastapi)
-- Front end (SvelteKit), yet to be built
+- Front end (SvelteKit)
-- Object storage (Ceph), yet to be built
+- Object storage strategy:
+  - MinIO for local development
+  - Bunny Storage + Bunny CDN for deployed environments

api/app/config.py

@@ -24,7 +24,7 @@ class Settings(BaseSettings):
     bunny_api_key: str = ""
     bunny_cdn_base_url: str = ""
     bunny_token_auth_key: str = ""
-    bunny_storage_endpoint: str = "https://storage.bunnycdn.com"
+    bunny_storage_endpoint: str = ""
     stub_generation: bool = False
 
     model_config = {"env_file": ".env"}

api/app/main.py

@@ -1,6 +1,6 @@
 import asyncio
 from contextlib import asynccontextmanager
-
+from prometheus_fastapi_instrumentator import Instrumentator
 from .routers.api import generation, pos
 from fastapi import FastAPI
 
@@ -9,12 +9,14 @@ from .routers import media as media_router
 from .routers.api.main import api_router
 from .routers.bff.main import bff_router
 from .outbound.storage_factory import init_storage
+from .observability import setup_observability
 from . import worker
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     init_storage()
+    setup_observability(app)
     worker_task = asyncio.create_task(worker.worker_loop())
     yield
     worker_task.cancel()
@@ -29,7 +31,7 @@ app = FastAPI(title="Language Learning API", lifespan=lifespan)
 app.include_router(api_router)
 app.include_router(bff_router)
 app.include_router(media_router.router)
-
+Instrumentator().instrument(app).expose(app, should_gzip=True)
 
 @app.get("/health")
 async def health() -> dict:

api/app/observability.py

@@ -0,0 +1,49 @@
+import os
+
+from fastapi import FastAPI
+from opentelemetry import metrics, trace
+from opentelemetry.exporter.prometheus import PrometheusMetricReader
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    BatchSpanProcessor,
+    ConsoleSpanExporter,
+)
+from prometheus_client import start_http_server
+
+
+_observability_initialized = False
+
+def setup_observability(app: FastAPI) -> None:
+    global _observability_initialized
+    if _observability_initialized:
+        return
+
+    
+    
+    service_name = os.getenv("OTEL_SERVICE_NAME", "language-learning-api")
+    metrics_host = os.getenv("OTEL_EXPORTER_PROMETHEUS_HOST", "0.0.0.0")
+    metrics_port = int(os.getenv("OTEL_EXPORTER_PROMETHEUS_PORT", "9464"))
+
+    resource = Resource.create({SERVICE_NAME: service_name})
+
+    tracer_provider = TracerProvider(resource=resource)
+    trace.set_tracer_provider(tracer_provider)
+
+    tracer_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+
+    metric_reader = PrometheusMetricReader()
+    meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
+    metrics.set_meter_provider(meter_provider)
+
+    LoggingInstrumentor().instrument(set_logging_format=True)
+
+    # Expose OTel metrics for Prometheus scraping on the standard endpoint.
+    start_http_server(port=metrics_port, addr=metrics_host)
+
+    
+    
+    _observability_initialized = True

api/app/outbound/bunny/bunny_client.py

@@ -1,5 +1,6 @@
 import base64
 import hashlib
+import json
 import time
 import urllib.error
 import urllib.request
@@ -14,7 +15,7 @@ class BunnyClient:
         api_key: str,
         cdn_base_url: str,
         token_auth_key: str,
-        storage_endpoint: str = "https://storage.bunnycdn.com",
+        storage_endpoint: str,
     ) -> None:
         self._zone = zone
         self._api_key = api_key
@@ -24,6 +25,27 @@ class BunnyClient:
 
     def _storage_url(self, path: str) -> str:
         return f"{self._storage_endpoint}/{self._zone}/{path.lstrip('/')}"
+    
+    def list_directory(self, path: str) -> list[str]:
+        print(f"Listing directories in: {self._storage_url(path)}")
+        req = urllib.request.Request(
+            self._storage_url(path),
+            method="GET",
+            headers={"AccessKey": self._api_key},
+        )
+        try:
+            with urllib.request.urlopen(req) as resp:
+                if resp.status == 200:
+                    print(f"Successfully listed directory '{path}' with Bunny storage client.")
+                    response_content = resp.read().decode()
+                    print(f"Response content: {response_content}")
+                    return json.loads(response_content)
+                else:
+                    print(f"Unexpected response status {resp.status} when listing directory: {resp.read().decode()}")
+                    return []
+        except urllib.error.HTTPError as e:
+            print(f"HTTPError when listing directory: {e}")
+            return []
 
     def upload(self, path: str, data: bytes) -> bool:
         req = urllib.request.Request(
@@ -37,8 +59,14 @@ class BunnyClient:
         )
         try:
             with urllib.request.urlopen(req) as resp:
-                return resp.status == 201
+                if resp.status == 201:
-        except urllib.error.HTTPError:
+                    print(f"Successfully uploaded '{path}' with Bunny storage client.")
+                    return True
+                else:
+                    print(f"Unexpected response status {resp.status} when uploading '{path}': {resp.read().decode()}")
+                    return False
+        except urllib.error.HTTPError as e:
+            print(f"HTTPError when uploading '{path}': {e}")
             return False
 
     def get_url(self, path: str) -> str:

api/app/outbound/storage_factory.py

@@ -1,19 +1,24 @@
 from ..config import settings
-from .storage_client import StorageClient, _set_storage_client
-from .minio.minio_client import MinioClient
 from .bunny.bunny_client import BunnyClient
+from .minio.minio_client import MinioClient
+from .storage_client import StorageClient, _set_storage_client
 
 
 def init_storage() -> None:
     client: StorageClient
     if settings.storage_provider == "bunny":
+        print("Initialising bunny storage client...")
         client = BunnyClient(
             zone=settings.bunny_zone,
             api_key=settings.bunny_api_key,
             cdn_base_url=settings.bunny_cdn_base_url,
             token_auth_key=settings.bunny_token_auth_key,
             storage_endpoint=settings.bunny_storage_endpoint,
-        )
+       )
+        print("Testing bunny storage client connection...")
+        client.list_directory("")  # Test connection
+        print("...successfully connected to bunny storage client.")
+        
     else:
         minio = MinioClient(
             endpoint_url=settings.storage_endpoint_url,

api/docs/object-storage.md

@@ -0,0 +1,92 @@
+# Object Storage
+
+This document explains how object storage works today, and how to control it across environments.
+
+## TL;DR
+
+- The app has one storage interface (`StorageClient`) and two implementations:
+  - `MinioClient` for local/dev (S3-compatible MinIO)
+  - `BunnyClient` for deployed environments (Bunny Storage + Bunny CDN)
+- Provider selection is controlled by `STORAGE_PROVIDER`:
+  - `local` -> MinIO
+  - `bunny` -> Bunny
+- The client is initialised once at API startup and stored as a process-level singleton.
+
+## Runtime Lifecycle
+
+1. API startup runs `init_storage()` from `app.outbound.storage_factory`.
+2. `init_storage()` reads config from `app.config.settings`.
+3. It creates either `MinioClient` or `BunnyClient`.
+4. The client instance is set via `_set_storage_client(...)`.
+5. App code calls `get_storage_client()` anywhere it needs object URLs or file operations.
+
+If storage is used before startup initialisation, `get_storage_client()` raises an assertion error.
+
+## Interface Contract
+
+`StorageClient` currently exposes:
+
+- `upload(path, data) -> bool`
+- `get_url(path) -> str`
+- `get_public_url(path) -> str`
+- `delete(path) -> bool`
+- `download(path) -> (bytes, content_type)`
+
+Important behavior differences:
+
+- MinIO supports `download(...)` for API media proxy routes.
+- Bunny does not support direct download in this adapter and raises `NotImplementedError`; callers should use signed CDN URLs from `get_url(...)`.
+
+## URL Behavior
+
+### Local/MinIO mode (`STORAGE_PROVIDER=local`)
+
+- `get_url(path)` returns API-proxied URLs under `/media/...`.
+- Browser requests go through the API media router.
+- Media router validates DB ownership/existence, then streams bytes from storage.
+
+### Bunny mode (`STORAGE_PROVIDER=bunny`)
+
+- `get_url(path)` returns a signed Bunny CDN URL.
+- Signature uses token auth key + path + expiry (currently 1 hour).
+- Browser requests go directly to Bunny CDN (no API proxy hop).
+
+## Configuration
+
+### Local/MinIO settings
+
+- `STORAGE_PROVIDER=local`
+- `STORAGE_ENDPOINT_URL` (for Docker dev: `http://storage:9000`)
+- `STORAGE_ACCESS_KEY`
+- `STORAGE_SECRET_KEY`
+- `STORAGE_BUCKET`
+- `API_BASE_URL` (used to build `/media/...` URLs)
+
+On startup, `MinioClient.ensure_bucket_exists()` is called.
+
+### Bunny settings
+
+- `STORAGE_PROVIDER=bunny`
+- `BUNNY_ZONE`
+- `BUNNY_API_KEY`
+- `BUNNY_CDN_BASE_URL`
+- `BUNNY_TOKEN_AUTH_KEY`
+- `BUNNY_STORAGE_ENDPOINT`
+
+On startup, Bunny client runs `list_directory("")` as a connection test.
+
+## Where Storage Is Used
+
+- BFF routers call `get_storage_client().get_url(...)` to expose audio URLs.
+- Media router calls `get_storage_client().download(...)` to stream files for `/media/...` routes.
+
+Practically:
+
+- In local mode, `/media/...` endpoints are expected and functional.
+- In Bunny mode, clients should consume returned CDN URLs directly.
+
+## Operational Notes
+
+- Upload content type is currently fixed to `audio/wav` in both adapters.
+- Bunny signed URL expiry is `_SIGNED_URL_EXPIRY_SECONDS = 3600`.
+- The storage client is per-process; each API process initialises its own instance at boot.

api/pyproject.toml

@@ -17,7 +17,14 @@ dependencies = [
     "google-genai>=1.0.0",
     "boto3>=1.35.0",
     "httpx>=0.28.1",
-    "deepgram-sdk>=6.1.0"
+    "deepgram-sdk>=6.1.0",
+    "opentelemetry-instrumentation-logging>=0.63b1",
+    "opentelemetry-instrumentation-fastapi>=0.63b1",
+    "opentelemetry-api>=1.42.1",
+    "opentelemetry-sdk>=1.42.1",
+    "opentelemetry-exporter-prometheus>=0.63b1",
+    "prometheus-client>=0.25.0",
+    "prometheus-fastapi-instrumentator>=7.1.0",
 ]
 
 [build-system]

docker-compose-dev.yml

@@ -27,14 +27,18 @@ services:
     volumes:
       - storagedata:/data
     healthcheck:
-      test: ["CMD-SHELL", "curl -sf http://localhost:9000/minio/health/live || exit 1"]
+      test:
+        [
+          "CMD-SHELL",
+          "curl -sf http://localhost:9000/minio/health/live || exit 1",
+        ]
       interval: 5s
       timeout: 5s
       retries: 10
 
   api:
     build: ./api
-    volumes: 
+    volumes:
       - ./api:/app:z
     ports:
       - "${API_PORT:-8000}:8000"
@@ -55,6 +59,9 @@ services:
       STORAGE_SECRET_KEY: ${STORAGE_SECRET_KEY}
       STORAGE_BUCKET: ${STORAGE_BUCKET:-langlearn}
       TRANSACTIONAL_EMAIL_PROVIDER: ${TRANSACTIONAL_EMAIL_PROVIDER:-stub}
+      OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-language-learning-api}
+      OTEL_EXPORTER_PROMETHEUS_HOST: 0.0.0.0
+      OTEL_EXPORTER_PROMETHEUS_PORT: ${OTEL_EXPORTER_PROMETHEUS_PORT:-9464}
     depends_on:
       db:
         condition: service_healthy
@@ -66,15 +73,84 @@ services:
     build:
       context: ./frontend
       args:
-        PUBLIC_API_BASE_URL: ${PUBLIC_API_BASE_URL:-http://localhost:8000}
+        PUBLIC_API_BASE_URL: ${PUBLIC_API_BASE_URL:-http://api:8000}
     ports:
-      - "${FRONTEND_PORT:-3000}:3000"
+      - "${FRONTEND_PORT:-3001}:3001"
     environment:
-      ORIGIN: ${ORIGIN:-http://localhost:3000}
+      ORIGIN: ${ORIGIN:-http://localhost:3001}
+      PUBLIC_API_BASE_URL: ${PUBLIC_API_BASE_URL:-http://api:8000}
+      PRIVATE_JWT_SECRET: ${JWT_SECRET}
+      PRIVATE_DEEPL_API_KEY: ${DEEPL_API_KEY}
+
     depends_on:
       - api
     restart: unless-stopped
 
+  prometheus:
+    image: prom/prometheus:v2.54.1
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro,z
+      - prometheusdata:/prometheus
+    ports:
+      - "9090:9090"
+    depends_on:
+      - api
+    restart: unless-stopped
+
+  loki:
+    image: grafana/loki:3.1.1
+    command: -config.file=/etc/loki/loki-config.yml
+    volumes:
+      - ./monitoring/loki/loki-config.yml:/etc/loki/loki-config.yml:ro,z
+      - lokidata:/loki
+    ports:
+      - "3100:3100"
+    restart: unless-stopped
+
+  alloy:
+    image: grafana/alloy:v1.7.1
+    user: "0:0"
+    security_opt:
+      - label=disable
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
+    volumes:
+      - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro,z
+      - /var/run/docker.sock:/var/run/docker.sock:ro,z
+      - alloydata:/var/lib/alloy/data
+    ports:
+      - "12345:12345"
+    depends_on:
+      - loki
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:11.2.0
+    volumes:
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro,z
+      - grafanadata:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_AUTH_ANONYMOUS_ENABLED: "false"
+    depends_on:
+      - prometheus
+      - loki
+    restart: unless-stopped
+
 volumes:
   pgdata:
   storagedata:
+  prometheusdata:
+  grafanadata:
+  lokidata:
+  alloydata:

docker-compose-local-override.yml

@@ -0,0 +1,4 @@
+services:
+  db:
+    ports:
+      - "127.0.0.1:5432:5432"

docker-compose-prod.yml

@@ -16,14 +16,14 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '1'
+          cpus: "1"
           memory: 1G
 
   api:
     build: ./api
     ports:
       - "${API_PORT:-8000}:8000"
-    command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers 2
+    command: sh -c "alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers 2"
     environment:
       DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-langlearn}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-langlearn}
       ADMIN_USER_EMAILS: ${ADMIN_USER_EMAILS}
@@ -39,13 +39,22 @@ services:
       BUNNY_API_KEY: ${BUNNY_API_KEY}
       BUNNY_CDN_BASE_URL: ${BUNNY_CDN_BASE_URL}
       BUNNY_TOKEN_AUTH_KEY: ${BUNNY_TOKEN_AUTH_KEY}
+      BUNNY_STORAGE_ENDPOINT: ${BUNNY_STORAGE_ENDPOINT}
       TRANSACTIONAL_EMAIL_PROVIDER: ${TRANSACTIONAL_EMAIL_PROVIDER}
+      OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-language-learning-api}
+      OTEL_EXPORTER_PROMETHEUS_HOST: 0.0.0.0
+      OTEL_EXPORTER_PROMETHEUS_PORT: ${OTEL_EXPORTER_PROMETHEUS_PORT:-9464}
     healthcheck:
-      test: ["CMD-SHELL", "curl -sf http://localhost:8000/health || exit 1"]
+      test:
+        [
+          "CMD-SHELL",
+          'python -c "import urllib.request; urllib.request.urlopen(''http://localhost:8000/health'')" || exit 1',
+        ]
       interval: 10s
       timeout: 5s
       retries: 10
-      start_period: 20s
+      start_period: 30s
+
     depends_on:
       db:
         condition: service_healthy
@@ -53,7 +62,7 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '1'
+          cpus: "1"
           memory: 1G
 
   frontend:
@@ -62,9 +71,13 @@ services:
       args:
         PUBLIC_API_BASE_URL: ${PUBLIC_API_BASE_URL}
     ports:
-      - "${FRONTEND_PORT:-3000}:3000"
+      - "${FRONTEND_PORT:-3001}:3000"
     environment:
       ORIGIN: ${ORIGIN}
+      PUBLIC_API_BASE_URL: ${PUBLIC_API_BASE_URL}
+      PRIVATE_JWT_SECRET: ${JWT_SECRET}
+      PRIVATE_DEEPL_API_KEY: ${DEEPL_API_KEY}
+
     depends_on:
       api:
         condition: service_healthy
@@ -72,11 +85,97 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '0.5'
+          cpus: "0.5"
           memory: 256M
 
+  prometheus:
+    image: prom/prometheus:v2.54.1
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheusdata:/prometheus
+    ports:
+      - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
+    depends_on:
+      api:
+        condition: service_healthy
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          cpus: "0.5"
+          memory: 512M
+
+  loki:
+    image: grafana/loki:3.1.1
+    command: -config.file=/etc/loki/loki-config.yml
+    volumes:
+      - ./monitoring/loki/loki-config.yml:/etc/loki/loki-config.yml:ro
+      - lokidata:/loki
+    ports:
+      - "127.0.0.1:${LOKI_PORT:-3100}:3100"
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          cpus: "0.5"
+          memory: 512M
+
+  alloy:
+    image: grafana/alloy:v1.7.1
+    user: "0:0"
+    security_opt:
+      - label=disable
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
+    volumes:
+      - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - alloydata:/var/lib/alloy/data
+    ports:
+      - "127.0.0.1:${ALLOY_PORT:-12345}:12345"
+    depends_on:
+      - loki
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          cpus: "0.5"
+          memory: 256M
+
+  grafana:
+    image: grafana/grafana:11.2.0
+    volumes:
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - grafanadata:/var/lib/grafana
+    ports:
+      - "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_AUTH_ANONYMOUS_ENABLED: "false"
+    depends_on:
+      - prometheus
+      - loki
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          cpus: "0.5"
+          memory: 512M
+
 volumes:
   pgdata:
+  prometheusdata:
+  grafanadata:
+  lokidata:
+  alloydata:
 
 networks:
   default:

docker-compose.yml

@@ -54,6 +54,9 @@ services:
       STORAGE_SECRET_KEY: ${STORAGE_SECRET_KEY}
       STORAGE_BUCKET: ${STORAGE_BUCKET:-langlearn}
       TRANSACTIONAL_EMAIL_PROVIDER: ${TRANSACTIONAL_EMAIL_PROVIDER:-stub}
+      OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-language-learning-api}
+      OTEL_EXPORTER_PROMETHEUS_HOST: 0.0.0.0
+      OTEL_EXPORTER_PROMETHEUS_PORT: ${OTEL_EXPORTER_PROMETHEUS_PORT:-9464}
     depends_on:
       db:
         condition: service_healthy
@@ -67,13 +70,78 @@ services:
       args:
         PUBLIC_API_BASE_URL: ${PUBLIC_API_BASE_URL:-http://localhost:8000}
     ports:
-      - "${FRONTEND_PORT:-3000}:3000"
+      - "${FRONTEND_PORT:-3001}:3000"
     environment:
-      ORIGIN: ${ORIGIN:-http://localhost:3000}
+      ORIGIN: ${ORIGIN:-http://localhost:3001}
     depends_on:
       - api
     restart: unless-stopped
 
+  prometheus:
+    image: prom/prometheus:v2.54.1
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheusdata:/prometheus
+    ports:
+      - "9090:9090"
+    depends_on:
+      - api
+    restart: unless-stopped
+
+  loki:
+    image: grafana/loki:3.1.1
+    command: -config.file=/etc/loki/loki-config.yml
+    volumes:
+      - ./monitoring/loki/loki-config.yml:/etc/loki/loki-config.yml:ro
+      - lokidata:/loki
+    ports:
+      - "3100:3100"
+    restart: unless-stopped
+
+  alloy:
+    image: grafana/alloy:v1.7.1
+    user: "0:0"
+    security_opt:
+      - label=disable
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
+    volumes:
+      - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - alloydata:/var/lib/alloy/data
+    ports:
+      - "12345:12345"
+    depends_on:
+      - loki
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:11.2.0
+    volumes:
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - grafanadata:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_AUTH_ANONYMOUS_ENABLED: "false"
+    depends_on:
+      - prometheus
+      - loki
+    restart: unless-stopped
+
 volumes:
   pgdata:
   storagedata:
+  prometheusdata:
+  grafanadata:
+  lokidata:
+  alloydata:

frontend/Dockerfile

@@ -4,7 +4,9 @@ RUN corepack enable && corepack prepare pnpm@latest --activate
 
 WORKDIR /app
 
-COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc ./
+ENV CI=true
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc svelte.config.js ./
+RUN pnpm approve-builds esbuild@0.27.4
 RUN pnpm install --frozen-lockfile
 
 COPY . .
@@ -23,9 +25,9 @@ COPY --from=builder /app/build ./build
 COPY --from=builder /app/node_modules ./node_modules
 COPY --from=builder /app/package.json ./package.json
 
-EXPOSE 3000
+EXPOSE 3001
 
-ENV PORT=3000
+ENV PORT=3001
 ENV HOST=0.0.0.0
 
 CMD ["node", "build/index.js"]

frontend/pnpm-workspace.yaml

@@ -1,2 +1,5 @@
+allowBuilds:
+  esbuild: true
+
 onlyBuiltDependencies:
   - esbuild

monitoring/alloy/config.alloy

@@ -0,0 +1,35 @@
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+    target_label  = "compose_project"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+}
+
+loki.source.docker "containers" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.relabel.containers.output
+  forward_to = [loki.write.default.receiver]
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}

monitoring/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,15 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false

monitoring/loki/loki-config.yml

@@ -0,0 +1,40 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  filesystem:
+    directory: /loki/chunks
+
+limits_config:
+  volume_enabled: true
+
+compactor:
+  working_directory: /loki/compactor
+
+ruler:
+  alertmanager_url: http://localhost:9093

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["prometheus:9090"]
+
+  - job_name: api
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["api:9464"]