From 43ae10d7dd1115987277ee3352025c98c27b8451 Mon Sep 17 00:00:00 2001 From: Ayobami Date: Thu, 31 Jul 2025 21:38:18 +0100 Subject: [PATCH] feat: add project documentation --- .dockerignore | 19 +++ .gitignore | 1 + Dockerfile | 37 +++++ README.md | 126 ++++++++++++++--- design.md | 329 +++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 72 +++++++++- prometheus.yml | 19 +++ 7 files changed, 586 insertions(+), 17 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 design.md create mode 100644 prometheus.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b053ede --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +node_modules +npm-debug.log +.git +.gitignore +README.md +.env +.nyc_output +coverage +.DS_Store +*.log +logs/* +tickets/* +.vscode +.idea +*.md +!design.md +Dockerfile +docker-compose.yml +prometheus.yml diff --git a/.gitignore b/.gitignore index 33e55ef..fa0ff74 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ /dist /package-lock.json /tickets +/logs .env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5dafe97 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Use official Node.js runtime as base image +FROM node:18-alpine + +# Set working directory in container +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm ci --only=production + +# Copy application code +COPY . . + +# Create necessary directories +RUN mkdir -p logs tickets + +# Create non-root user for security +RUN addgroup -g 1001 -S nodejs +RUN adduser -S nodejs -u 1001 + +# Change ownership of app directory +RUN chown -R nodejs:nodejs /app + +# Switch to non-root user +USER nodejs + +# Expose port +EXPOSE 3049 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD node -e "require('http').get('http://localhost:3049/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => process.exit(1))" + +# Start the application +CMD ["npm", "start"] diff --git a/README.md b/README.md index fea9327..92ed2b3 100644 --- a/README.md +++ b/README.md @@ -42,34 +42,128 @@ Your task is to extract the high-throughput ticket purchasing component and exte ### Prerequisites -- Node.js (v14+ recommended) -- npm -- Redis (installed locally or via Docker, as per the provided docker-compose configuration) +- Node.js (v18+ recommended) +- npm or yarn +- Docker and Docker Compose +- Git -### Setup +### Quick Start -1. Clone the repository. -2. Install dependencies: +1. **Clone the repository** + ```bash + git clone + cd module4_backend_project + ``` + +2. **Install dependencies** + ```bash npm install -3. (Optional) Copy the environment variable template: + ``` + +3. **Set up environment** + ```bash cp .env.example .env -4. Seed the Redis store with tickets for multiple events. You might modify the seeding script to handle multiple event keys (e.g., `event:1:tickets`, `event:2:tickets`, etc.). -5. Start the application: - npm start + # Edit .env file if needed + ``` + +4. **Start with Docker (Recommended)** + ```bash + # Start core services (Redis + App) + docker-compose up -d + + # Or start with monitoring (Prometheus + Grafana) + docker-compose --profile monitoring up -d + ``` + +5. **Seed the database** + ```bash + # Seed 5 events with 10,000 tickets each + npm run seed + + # Custom seeding: 3 events with 5,000 tickets each + npm run seed 3 5000 + ``` + +### Manual Setup (Development) + +If you prefer to run components separately: + +1. **Start Redis** + ```bash + docker-compose up -d redis + ``` + +2. **Start the application** + ```bash + npm run dev # Development with auto-reload + # or + npm start # Production mode + ``` + +### API Endpoints + +Once running, the following endpoints are available: + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/health` | System health check | +| GET | `/events` | List all events with statistics | +| GET | `/events/:eventId` | Get specific event details | +| POST | `/buy/:eventId` | Purchase a ticket for an event | +| GET | `/tickets/:purchaseId` | Download ticket PDF | +| GET | `/metrics` | Real-time system metrics | +| GET | `/admin/pdf-stats` | PDF management statistics | +| POST | `/admin/cleanup-tickets` | Cleanup old ticket files | ### Load Testing -Simulate high load using a tool like [autocannon](https://github.com/mcollina/autocannon) or [wrk](https://github.com/wg/wrk). For example, to simulate 5000 concurrent connections on event 1: +The system includes a comprehensive load testing framework: -npx autocannon -c 5000 -d 30 http://localhost:3049/buy/1 +```bash +# Run full test suite (5000+ concurrent connections) +npm run test:load -- --full -### Metrics +# Test specific event +npm run test:load -- --event 1 --connections 5000 --duration 30 -Access real-time service metrics at: +# Multi-event concurrent testing +npm run test:load -- --multi --events 1,2,3 --connections 6000 -http://localhost:3049/metrics +# Custom load test +node tests/load-test.js --event 2 --connections 1000 --duration 10 +``` -These metrics should include data on tickets sold, remaining tickets per event, and any instances where the fallback mechanism was activated. +### Monitoring & Metrics + +#### Application Metrics +Access real-time service metrics at: http://localhost:3049/metrics + +#### Prometheus (if enabled) +Prometheus dashboard: http://localhost:9090 + +#### Grafana (if enabled) +Grafana dashboard: http://localhost:3000 +- Username: `admin` +- Password: `admin` + +### Docker Commands + +```bash +# Start core services +docker-compose up -d + +# Start with monitoring +docker-compose --profile monitoring up -d + +# View logs +docker-compose logs -f app + +# Stop services +docker-compose down + +# Rebuild and restart +docker-compose up -d --build +``` ## Evaluation Criteria diff --git a/design.md b/design.md new file mode 100644 index 0000000..8013461 --- /dev/null +++ b/design.md @@ -0,0 +1,329 @@ +# Ticket Scaling Microservice - Design Document + +## Table of Contents +1. [Architecture Overview](#architecture-overview) +2. [System Components](#system-components) +3. [Scalability Strategies](#scalability-strategies) +4. [Atomic Operations](#atomic-operations) +5. [Fallback Mechanisms](#fallback-mechanisms) +6. [Performance Optimizations](#performance-optimizations) +7. [Monitoring & Observability](#monitoring--observability) +8. [Security Considerations](#security-considerations) +9. [Deployment Strategy](#deployment-strategy) +10. [Future Enhancements](#future-enhancements) + +## Architecture Overview + +### High-Level Architecture +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Load Balancer │ │ Prometheus │ │ Grafana │ +│ (Optional) │ │ Monitoring │ │ Dashboard │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + │ │ │ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ │ │ │ │ │ +│ Ticket Service │◄───┤ Redis │ │ In-Memory │ +│ (Node.js/ │ │ Primary Store │ │ Fallback Store │ +│ Express) │ │ │ │ │ +│ │ │ │ │ │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ + │ +┌─────────────────┐ +│ PDF Generator │ +│ (PDFKit) │ +└─────────────────┘ +``` + +### Design Principles +1. **High Availability**: Fallback mechanisms ensure service continuity +2. **Atomic Operations**: Redis Lua scripts prevent race conditions +3. **Horizontal Scalability**: Stateless design enables easy scaling +4. **Observability**: Comprehensive logging and metrics +5. **Performance**: Optimized for high-throughput scenarios + +## System Components + +### 1. Core Application (server.js) +- **Technology**: Node.js with Express framework +- **Responsibilities**: + - HTTP request handling + - Business logic orchestration + - Error handling and logging + - PDF generation coordination + +### 2. Redis Client (redis-client.js) +- **Technology**: Redis with Lua scripting +- **Responsibilities**: + - Atomic ticket operations + - Event metadata management + - Connection health monitoring + - Script execution + +### 3. Fallback Store (fallback-store.js) +- **Technology**: In-memory JavaScript Map +- **Responsibilities**: + - Emergency ticket storage + - Temporary operation continuity + - Graceful degradation + +### 4. PDF Generator (pdf-generator.js) +- **Technology**: PDFKit library +- **Responsibilities**: + - Professional ticket generation + - File management + - Cleanup operations + +### 5. Logging System (logger.js) +- **Technology**: Winston logging framework +- **Responsibilities**: + - Structured logging + - Request tracking + - Error reporting + - Performance metrics + +## Scalability Strategies + +### Horizontal Scaling +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Instance 1 │ │ Instance 2 │ │ Instance N │ +│ Port: 3049 │ │ Port: 3050 │ │ Port: 305X │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + └───────────────────────┼───────────────────────┘ + │ + ┌─────────────────┐ + │ Shared Redis │ + │ Cluster │ + └─────────────────┘ +``` + +**Key Features**: +- Stateless application design +- Shared Redis backend +- Load balancer distribution +- Independent scaling + +### Vertical Scaling +- **CPU**: Multi-core utilization through Node.js cluster mode +- **Memory**: Configurable heap sizes for high-throughput +- **I/O**: Async operations prevent blocking + +### Database Scaling +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Redis Master │ │ Redis Replica │ │ Redis Replica │ +│ (Read/Write) │───▶│ (Read Only) │ │ (Read Only) │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +**Strategies**: +- Redis clustering for horizontal scaling +- Read replicas for metrics/stats queries +- Sharding by event ID for massive scale + +## Atomic Operations + +### Lua Script Design +Our core purchase operation uses a Redis Lua script to ensure atomicity: + +```lua +-- Atomic ticket purchase script +local ticketKey = KEYS[1] -- event:X:tickets +local metaKey = KEYS[2] -- event:X:meta +local globalKey = KEYS[3] -- global:stats + +-- Atomic operations: +1. Check event exists +2. Pop ticket from list +3. Update sold count +4. Update global stats +5. Store purchase record +``` + +**Benefits**: +- **Race Condition Prevention**: All operations execute atomically +- **Consistency**: No partial state updates +- **Performance**: Single round-trip to Redis +- **Reliability**: All-or-nothing execution + +### Concurrency Handling +- **Optimistic Locking**: Lua scripts handle concurrent access +- **Queue Management**: Redis lists provide FIFO ticket distribution +- **Connection Pooling**: Efficient Redis connection reuse + +## Fallback Mechanisms + +### Activation Triggers +1. **Redis Connection Failure**: Network issues or Redis downtime +2. **Script Execution Errors**: Lua script failures +3. **Timeout Scenarios**: Slow Redis responses + +### Fallback Architecture +``` +┌─────────────────┐ +│ Request Comes │ +└─────────────────┘ + │ + ▼ +┌─────────────────┐ ┌─────────────────┐ +│ Try Redis │───▶│ Redis Success │ +│ Operation │ │ Return Result │ +└─────────────────┘ └─────────────────┘ + │ + ▼ (On Failure) +┌─────────────────┐ ┌─────────────────┐ +│ Activate │───▶│ In-Memory │ +│ Fallback Store │ │ Operation │ +└─────────────────┘ └─────────────────┘ +``` + +### Fallback Limitations +- **Non-Persistent**: Data lost on restart +- **Single Instance**: No cross-instance synchronization +- **Capacity Limited**: Memory constraints +- **Warning Logs**: Clear indication of degraded mode + +## Performance Optimizations + +### Application Level +1. **Async Operations**: Non-blocking I/O throughout +2. **Connection Pooling**: Reuse Redis connections +3. **Batch Operations**: Bulk ticket seeding +4. **Caching**: Event metadata caching + +### Redis Optimizations +1. **Lua Scripts**: Reduced network round-trips +2. **Pipeline Operations**: Batch commands +3. **Memory Management**: Efficient data structures +4. **Persistence**: AOF for durability + +### PDF Generation +1. **Async Generation**: Non-blocking PDF creation +2. **Stream Processing**: Memory-efficient file handling +3. **Cleanup Jobs**: Automatic old file removal +4. **Error Isolation**: PDF failures don't affect purchases + +## Monitoring & Observability + +### Metrics Collection +```json +{ + "global": { + "totalEvents": 5, + "totalTickets": 50000, + "totalSold": 1250 + }, + "events": [ + { + "eventId": "1", + "soldTickets": 250, + "remainingTickets": 9750 + } + ], + "system": { + "usingFallback": false, + "redisConnected": true, + "uptime": 3600, + "memoryUsage": {...} + }, + "pdf": { + "totalTickets": 1250, + "totalSizeMB": "15.6" + } +} +``` + +### Logging Strategy +- **Structured Logging**: JSON format for parsing +- **Request Tracking**: Unique IDs for tracing +- **Performance Metrics**: Response times and throughput +- **Error Categorization**: Different log levels + +### Health Checks +- **Application Health**: `/health` endpoint +- **Redis Connectivity**: Connection status +- **Fallback Status**: Degraded mode indication +- **Resource Usage**: Memory and CPU monitoring + +## Security Considerations + +### Input Validation +- **Event ID Validation**: Numeric constraints +- **Request Rate Limiting**: DDoS protection +- **Parameter Sanitization**: Injection prevention + +### Container Security +- **Non-Root User**: Principle of least privilege +- **Minimal Base Image**: Alpine Linux for smaller attack surface +- **Health Checks**: Container monitoring + +### Data Protection +- **No Sensitive Data**: Tickets are identifiers only +- **Audit Logging**: Purchase tracking +- **Secure Defaults**: Production-ready configuration + +## Deployment Strategy + +### Development Environment +```bash +# Local development +npm install +npm run docker:up # Start Redis +npm run seed # Seed events +npm run dev # Start with nodemon +``` + +### Production Environment +```bash +# Docker deployment +docker-compose up -d # Core services +docker-compose --profile monitoring up # With monitoring +``` + +### Container Orchestration +- **Docker Compose**: Local and small deployments +- **Kubernetes**: Large-scale deployments +- **Health Checks**: Automatic restart on failure +- **Resource Limits**: CPU and memory constraints + +## Future Enhancements + +### Performance Improvements +1. **Redis Clustering**: Horizontal database scaling +2. **CDN Integration**: PDF delivery optimization +3. **Caching Layer**: Application-level caching +4. **Connection Optimization**: Advanced pooling + +### Feature Additions +1. **QR Code Generation**: Enhanced ticket security +2. **Email Integration**: Automatic ticket delivery +3. **Payment Processing**: Complete purchase flow +4. **Event Management**: Dynamic event creation + +### Monitoring Enhancements +1. **Distributed Tracing**: Request flow tracking +2. **Custom Dashboards**: Business metrics visualization +3. **Alerting**: Proactive issue detection +4. **Performance Profiling**: Bottleneck identification + +### Security Hardening +1. **Authentication**: API key management +2. **Rate Limiting**: Advanced throttling +3. **Encryption**: Data in transit protection +4. **Audit Trails**: Comprehensive logging + +## Conclusion + +This design provides a robust, scalable foundation for high-volume ticket sales with the following key strengths: + +- **Atomic Operations**: Guaranteed consistency under load +- **High Availability**: Graceful degradation capabilities +- **Observability**: Comprehensive monitoring and logging +- **Scalability**: Horizontal and vertical scaling support +- **Performance**: Optimized for high-throughput scenarios + +The architecture successfully handles the challenge requirements of processing thousands of concurrent requests while maintaining data integrity and system reliability. diff --git a/docker-compose.yml b/docker-compose.yml index 7fdc896..e435d91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,80 @@ -version: "3" +version: "3.8" services: redis: image: redis:alpine + container_name: ticket-redis ports: - "6379:6379" volumes: - redis_data:/data + command: redis-server --appendonly yes + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 3 + networks: + - ticket-network + + app: + build: . + container_name: ticket-microservice + ports: + - "3049:3049" + environment: + - NODE_ENV=production + - REDIS_URL=redis://redis:6379 + - PORT=3049 + - LOG_LEVEL=info + depends_on: + redis: + condition: service_healthy + volumes: + - ./logs:/app/logs + - ./tickets:/app/tickets + networks: + - ticket-network + restart: unless-stopped + + prometheus: + image: prom/prometheus:latest + container_name: ticket-prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention.time=200h' + - '--web.enable-lifecycle' + networks: + - ticket-network + profiles: + - monitoring + + grafana: + image: grafana/grafana:latest + container_name: ticket-grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - grafana_data:/var/lib/grafana + networks: + - ticket-network + profiles: + - monitoring + volumes: redis_data: + prometheus_data: + grafana_data: + +networks: + ticket-network: + driver: bridge diff --git a/prometheus.yml b/prometheus.yml new file mode 100644 index 0000000..d022b01 --- /dev/null +++ b/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'ticket-microservice' + static_configs: + - targets: ['app:3049'] + metrics_path: '/metrics' + scrape_interval: 5s + scrape_timeout: 3s