feat: add project documentation
This commit is contained in:
Ayobami
@@ -0,0 +1,19 @@
|
||||
node_modules
|
||||
npm-debug.log
|
||||
.git
|
||||
.gitignore
|
||||
README.md
|
||||
.env
|
||||
.nyc_output
|
||||
coverage
|
||||
.DS_Store
|
||||
*.log
|
||||
logs/*
|
||||
tickets/*
|
||||
.vscode
|
||||
.idea
|
||||
*.md
|
||||
!design.md
|
||||
Dockerfile
|
||||
docker-compose.yml
|
||||
prometheus.yml
|
||||
@@ -2,4 +2,5 @@
|
||||
/dist
|
||||
/package-lock.json
|
||||
/tickets
|
||||
/logs
|
||||
.env
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
# Use official Node.js runtime as base image
|
||||
FROM node:18-alpine
|
||||
|
||||
# Set working directory in container
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files
|
||||
COPY package*.json ./
|
||||
|
||||
# Install dependencies
|
||||
RUN npm ci --only=production
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p logs tickets
|
||||
|
||||
# Create non-root user for security
|
||||
RUN addgroup -g 1001 -S nodejs
|
||||
RUN adduser -S nodejs -u 1001
|
||||
|
||||
# Change ownership of app directory
|
||||
RUN chown -R nodejs:nodejs /app
|
||||
|
||||
# Switch to non-root user
|
||||
USER nodejs
|
||||
|
||||
# Expose port
|
||||
EXPOSE 3049
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD node -e "require('http').get('http://localhost:3049/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => process.exit(1))"
|
||||
|
||||
# Start the application
|
||||
CMD ["npm", "start"]
|
||||
@@ -42,34 +42,128 @@ Your task is to extract the high-throughput ticket purchasing component and exte
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Node.js (v14+ recommended)
|
||||
- npm
|
||||
- Redis (installed locally or via Docker, as per the provided docker-compose configuration)
|
||||
- Node.js (v18+ recommended)
|
||||
- npm or yarn
|
||||
- Docker and Docker Compose
|
||||
- Git
|
||||
|
||||
### Setup
|
||||
### Quick Start
|
||||
|
||||
1. Clone the repository.
|
||||
2. Install dependencies:
|
||||
1. **Clone the repository**
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd module4_backend_project
|
||||
```
|
||||
|
||||
2. **Install dependencies**
|
||||
```bash
|
||||
npm install
|
||||
3. (Optional) Copy the environment variable template:
|
||||
```
|
||||
|
||||
3. **Set up environment**
|
||||
```bash
|
||||
cp .env.example .env
|
||||
4. Seed the Redis store with tickets for multiple events. You might modify the seeding script to handle multiple event keys (e.g., `event:1:tickets`, `event:2:tickets`, etc.).
|
||||
5. Start the application:
|
||||
npm start
|
||||
# Edit .env file if needed
|
||||
```
|
||||
|
||||
4. **Start with Docker (Recommended)**
|
||||
```bash
|
||||
# Start core services (Redis + App)
|
||||
docker-compose up -d
|
||||
|
||||
# Or start with monitoring (Prometheus + Grafana)
|
||||
docker-compose --profile monitoring up -d
|
||||
```
|
||||
|
||||
5. **Seed the database**
|
||||
```bash
|
||||
# Seed 5 events with 10,000 tickets each
|
||||
npm run seed
|
||||
|
||||
# Custom seeding: 3 events with 5,000 tickets each
|
||||
npm run seed 3 5000
|
||||
```
|
||||
|
||||
### Manual Setup (Development)
|
||||
|
||||
If you prefer to run components separately:
|
||||
|
||||
1. **Start Redis**
|
||||
```bash
|
||||
docker-compose up -d redis
|
||||
```
|
||||
|
||||
2. **Start the application**
|
||||
```bash
|
||||
npm run dev # Development with auto-reload
|
||||
# or
|
||||
npm start # Production mode
|
||||
```
|
||||
|
||||
### API Endpoints
|
||||
|
||||
Once running, the following endpoints are available:
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/health` | System health check |
|
||||
| GET | `/events` | List all events with statistics |
|
||||
| GET | `/events/:eventId` | Get specific event details |
|
||||
| POST | `/buy/:eventId` | Purchase a ticket for an event |
|
||||
| GET | `/tickets/:purchaseId` | Download ticket PDF |
|
||||
| GET | `/metrics` | Real-time system metrics |
|
||||
| GET | `/admin/pdf-stats` | PDF management statistics |
|
||||
| POST | `/admin/cleanup-tickets` | Cleanup old ticket files |
|
||||
|
||||
### Load Testing
|
||||
|
||||
Simulate high load using a tool like [autocannon](https://github.com/mcollina/autocannon) or [wrk](https://github.com/wg/wrk). For example, to simulate 5000 concurrent connections on event 1:
|
||||
The system includes a comprehensive load testing framework:
|
||||
|
||||
npx autocannon -c 5000 -d 30 http://localhost:3049/buy/1
|
||||
```bash
|
||||
# Run full test suite (5000+ concurrent connections)
|
||||
npm run test:load -- --full
|
||||
|
||||
### Metrics
|
||||
# Test specific event
|
||||
npm run test:load -- --event 1 --connections 5000 --duration 30
|
||||
|
||||
Access real-time service metrics at:
|
||||
# Multi-event concurrent testing
|
||||
npm run test:load -- --multi --events 1,2,3 --connections 6000
|
||||
|
||||
http://localhost:3049/metrics
|
||||
# Custom load test
|
||||
node tests/load-test.js --event 2 --connections 1000 --duration 10
|
||||
```
|
||||
|
||||
These metrics should include data on tickets sold, remaining tickets per event, and any instances where the fallback mechanism was activated.
|
||||
### Monitoring & Metrics
|
||||
|
||||
#### Application Metrics
|
||||
Access real-time service metrics at: http://localhost:3049/metrics
|
||||
|
||||
#### Prometheus (if enabled)
|
||||
Prometheus dashboard: http://localhost:9090
|
||||
|
||||
#### Grafana (if enabled)
|
||||
Grafana dashboard: http://localhost:3000
|
||||
- Username: `admin`
|
||||
- Password: `admin`
|
||||
|
||||
### Docker Commands
|
||||
|
||||
```bash
|
||||
# Start core services
|
||||
docker-compose up -d
|
||||
|
||||
# Start with monitoring
|
||||
docker-compose --profile monitoring up -d
|
||||
|
||||
# View logs
|
||||
docker-compose logs -f app
|
||||
|
||||
# Stop services
|
||||
docker-compose down
|
||||
|
||||
# Rebuild and restart
|
||||
docker-compose up -d --build
|
||||
```
|
||||
|
||||
## Evaluation Criteria
|
||||
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
# Ticket Scaling Microservice - Design Document
|
||||
|
||||
## Table of Contents
|
||||
1. [Architecture Overview](#architecture-overview)
|
||||
2. [System Components](#system-components)
|
||||
3. [Scalability Strategies](#scalability-strategies)
|
||||
4. [Atomic Operations](#atomic-operations)
|
||||
5. [Fallback Mechanisms](#fallback-mechanisms)
|
||||
6. [Performance Optimizations](#performance-optimizations)
|
||||
7. [Monitoring & Observability](#monitoring--observability)
|
||||
8. [Security Considerations](#security-considerations)
|
||||
9. [Deployment Strategy](#deployment-strategy)
|
||||
10. [Future Enhancements](#future-enhancements)
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
### High-Level Architecture
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Load Balancer │ │ Prometheus │ │ Grafana │
|
||||
│ (Optional) │ │ Monitoring │ │ Dashboard │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
│ │ │
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ │ │ │ │ │
|
||||
│ Ticket Service │◄───┤ Redis │ │ In-Memory │
|
||||
│ (Node.js/ │ │ Primary Store │ │ Fallback Store │
|
||||
│ Express) │ │ │ │ │
|
||||
│ │ │ │ │ │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│
|
||||
│
|
||||
┌─────────────────┐
|
||||
│ PDF Generator │
|
||||
│ (PDFKit) │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
### Design Principles
|
||||
1. **High Availability**: Fallback mechanisms ensure service continuity
|
||||
2. **Atomic Operations**: Redis Lua scripts prevent race conditions
|
||||
3. **Horizontal Scalability**: Stateless design enables easy scaling
|
||||
4. **Observability**: Comprehensive logging and metrics
|
||||
5. **Performance**: Optimized for high-throughput scenarios
|
||||
|
||||
## System Components
|
||||
|
||||
### 1. Core Application (server.js)
|
||||
- **Technology**: Node.js with Express framework
|
||||
- **Responsibilities**:
|
||||
- HTTP request handling
|
||||
- Business logic orchestration
|
||||
- Error handling and logging
|
||||
- PDF generation coordination
|
||||
|
||||
### 2. Redis Client (redis-client.js)
|
||||
- **Technology**: Redis with Lua scripting
|
||||
- **Responsibilities**:
|
||||
- Atomic ticket operations
|
||||
- Event metadata management
|
||||
- Connection health monitoring
|
||||
- Script execution
|
||||
|
||||
### 3. Fallback Store (fallback-store.js)
|
||||
- **Technology**: In-memory JavaScript Map
|
||||
- **Responsibilities**:
|
||||
- Emergency ticket storage
|
||||
- Temporary operation continuity
|
||||
- Graceful degradation
|
||||
|
||||
### 4. PDF Generator (pdf-generator.js)
|
||||
- **Technology**: PDFKit library
|
||||
- **Responsibilities**:
|
||||
- Professional ticket generation
|
||||
- File management
|
||||
- Cleanup operations
|
||||
|
||||
### 5. Logging System (logger.js)
|
||||
- **Technology**: Winston logging framework
|
||||
- **Responsibilities**:
|
||||
- Structured logging
|
||||
- Request tracking
|
||||
- Error reporting
|
||||
- Performance metrics
|
||||
|
||||
## Scalability Strategies
|
||||
|
||||
### Horizontal Scaling
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Instance 1 │ │ Instance 2 │ │ Instance N │
|
||||
│ Port: 3049 │ │ Port: 3050 │ │ Port: 305X │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
└───────────────────────┼───────────────────────┘
|
||||
│
|
||||
┌─────────────────┐
|
||||
│ Shared Redis │
|
||||
│ Cluster │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
**Key Features**:
|
||||
- Stateless application design
|
||||
- Shared Redis backend
|
||||
- Load balancer distribution
|
||||
- Independent scaling
|
||||
|
||||
### Vertical Scaling
|
||||
- **CPU**: Multi-core utilization through Node.js cluster mode
|
||||
- **Memory**: Configurable heap sizes for high-throughput
|
||||
- **I/O**: Async operations prevent blocking
|
||||
|
||||
### Database Scaling
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Redis Master │ │ Redis Replica │ │ Redis Replica │
|
||||
│ (Read/Write) │───▶│ (Read Only) │ │ (Read Only) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
**Strategies**:
|
||||
- Redis clustering for horizontal scaling
|
||||
- Read replicas for metrics/stats queries
|
||||
- Sharding by event ID for massive scale
|
||||
|
||||
## Atomic Operations
|
||||
|
||||
### Lua Script Design
|
||||
Our core purchase operation uses a Redis Lua script to ensure atomicity:
|
||||
|
||||
```lua
|
||||
-- Atomic ticket purchase script
|
||||
local ticketKey = KEYS[1] -- event:X:tickets
|
||||
local metaKey = KEYS[2] -- event:X:meta
|
||||
local globalKey = KEYS[3] -- global:stats
|
||||
|
||||
-- Atomic operations:
|
||||
1. Check event exists
|
||||
2. Pop ticket from list
|
||||
3. Update sold count
|
||||
4. Update global stats
|
||||
5. Store purchase record
|
||||
```
|
||||
|
||||
**Benefits**:
|
||||
- **Race Condition Prevention**: All operations execute atomically
|
||||
- **Consistency**: No partial state updates
|
||||
- **Performance**: Single round-trip to Redis
|
||||
- **Reliability**: All-or-nothing execution
|
||||
|
||||
### Concurrency Handling
|
||||
- **Optimistic Locking**: Lua scripts handle concurrent access
|
||||
- **Queue Management**: Redis lists provide FIFO ticket distribution
|
||||
- **Connection Pooling**: Efficient Redis connection reuse
|
||||
|
||||
## Fallback Mechanisms
|
||||
|
||||
### Activation Triggers
|
||||
1. **Redis Connection Failure**: Network issues or Redis downtime
|
||||
2. **Script Execution Errors**: Lua script failures
|
||||
3. **Timeout Scenarios**: Slow Redis responses
|
||||
|
||||
### Fallback Architecture
|
||||
```
|
||||
┌─────────────────┐
|
||||
│ Request Comes │
|
||||
└─────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐ ┌─────────────────┐
|
||||
│ Try Redis │───▶│ Redis Success │
|
||||
│ Operation │ │ Return Result │
|
||||
└─────────────────┘ └─────────────────┘
|
||||
│
|
||||
▼ (On Failure)
|
||||
┌─────────────────┐ ┌─────────────────┐
|
||||
│ Activate │───▶│ In-Memory │
|
||||
│ Fallback Store │ │ Operation │
|
||||
└─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
### Fallback Limitations
|
||||
- **Non-Persistent**: Data lost on restart
|
||||
- **Single Instance**: No cross-instance synchronization
|
||||
- **Capacity Limited**: Memory constraints
|
||||
- **Warning Logs**: Clear indication of degraded mode
|
||||
|
||||
## Performance Optimizations
|
||||
|
||||
### Application Level
|
||||
1. **Async Operations**: Non-blocking I/O throughout
|
||||
2. **Connection Pooling**: Reuse Redis connections
|
||||
3. **Batch Operations**: Bulk ticket seeding
|
||||
4. **Caching**: Event metadata caching
|
||||
|
||||
### Redis Optimizations
|
||||
1. **Lua Scripts**: Reduced network round-trips
|
||||
2. **Pipeline Operations**: Batch commands
|
||||
3. **Memory Management**: Efficient data structures
|
||||
4. **Persistence**: AOF for durability
|
||||
|
||||
### PDF Generation
|
||||
1. **Async Generation**: Non-blocking PDF creation
|
||||
2. **Stream Processing**: Memory-efficient file handling
|
||||
3. **Cleanup Jobs**: Automatic old file removal
|
||||
4. **Error Isolation**: PDF failures don't affect purchases
|
||||
|
||||
## Monitoring & Observability
|
||||
|
||||
### Metrics Collection
|
||||
```json
|
||||
{
|
||||
"global": {
|
||||
"totalEvents": 5,
|
||||
"totalTickets": 50000,
|
||||
"totalSold": 1250
|
||||
},
|
||||
"events": [
|
||||
{
|
||||
"eventId": "1",
|
||||
"soldTickets": 250,
|
||||
"remainingTickets": 9750
|
||||
}
|
||||
],
|
||||
"system": {
|
||||
"usingFallback": false,
|
||||
"redisConnected": true,
|
||||
"uptime": 3600,
|
||||
"memoryUsage": {...}
|
||||
},
|
||||
"pdf": {
|
||||
"totalTickets": 1250,
|
||||
"totalSizeMB": "15.6"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Logging Strategy
|
||||
- **Structured Logging**: JSON format for parsing
|
||||
- **Request Tracking**: Unique IDs for tracing
|
||||
- **Performance Metrics**: Response times and throughput
|
||||
- **Error Categorization**: Different log levels
|
||||
|
||||
### Health Checks
|
||||
- **Application Health**: `/health` endpoint
|
||||
- **Redis Connectivity**: Connection status
|
||||
- **Fallback Status**: Degraded mode indication
|
||||
- **Resource Usage**: Memory and CPU monitoring
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Input Validation
|
||||
- **Event ID Validation**: Numeric constraints
|
||||
- **Request Rate Limiting**: DDoS protection
|
||||
- **Parameter Sanitization**: Injection prevention
|
||||
|
||||
### Container Security
|
||||
- **Non-Root User**: Principle of least privilege
|
||||
- **Minimal Base Image**: Alpine Linux for smaller attack surface
|
||||
- **Health Checks**: Container monitoring
|
||||
|
||||
### Data Protection
|
||||
- **No Sensitive Data**: Tickets are identifiers only
|
||||
- **Audit Logging**: Purchase tracking
|
||||
- **Secure Defaults**: Production-ready configuration
|
||||
|
||||
## Deployment Strategy
|
||||
|
||||
### Development Environment
|
||||
```bash
|
||||
# Local development
|
||||
npm install
|
||||
npm run docker:up # Start Redis
|
||||
npm run seed # Seed events
|
||||
npm run dev # Start with nodemon
|
||||
```
|
||||
|
||||
### Production Environment
|
||||
```bash
|
||||
# Docker deployment
|
||||
docker-compose up -d # Core services
|
||||
docker-compose --profile monitoring up # With monitoring
|
||||
```
|
||||
|
||||
### Container Orchestration
|
||||
- **Docker Compose**: Local and small deployments
|
||||
- **Kubernetes**: Large-scale deployments
|
||||
- **Health Checks**: Automatic restart on failure
|
||||
- **Resource Limits**: CPU and memory constraints
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### Performance Improvements
|
||||
1. **Redis Clustering**: Horizontal database scaling
|
||||
2. **CDN Integration**: PDF delivery optimization
|
||||
3. **Caching Layer**: Application-level caching
|
||||
4. **Connection Optimization**: Advanced pooling
|
||||
|
||||
### Feature Additions
|
||||
1. **QR Code Generation**: Enhanced ticket security
|
||||
2. **Email Integration**: Automatic ticket delivery
|
||||
3. **Payment Processing**: Complete purchase flow
|
||||
4. **Event Management**: Dynamic event creation
|
||||
|
||||
### Monitoring Enhancements
|
||||
1. **Distributed Tracing**: Request flow tracking
|
||||
2. **Custom Dashboards**: Business metrics visualization
|
||||
3. **Alerting**: Proactive issue detection
|
||||
4. **Performance Profiling**: Bottleneck identification
|
||||
|
||||
### Security Hardening
|
||||
1. **Authentication**: API key management
|
||||
2. **Rate Limiting**: Advanced throttling
|
||||
3. **Encryption**: Data in transit protection
|
||||
4. **Audit Trails**: Comprehensive logging
|
||||
|
||||
## Conclusion
|
||||
|
||||
This design provides a robust, scalable foundation for high-volume ticket sales with the following key strengths:
|
||||
|
||||
- **Atomic Operations**: Guaranteed consistency under load
|
||||
- **High Availability**: Graceful degradation capabilities
|
||||
- **Observability**: Comprehensive monitoring and logging
|
||||
- **Scalability**: Horizontal and vertical scaling support
|
||||
- **Performance**: Optimized for high-throughput scenarios
|
||||
|
||||
The architecture successfully handles the challenge requirements of processing thousands of concurrent requests while maintaining data integrity and system reliability.
|
||||
+71
-1
@@ -1,10 +1,80 @@
|
||||
version: "3"
|
||||
version: "3.8"
|
||||
services:
|
||||
redis:
|
||||
image: redis:alpine
|
||||
container_name: ticket-redis
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
command: redis-server --appendonly yes
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
networks:
|
||||
- ticket-network
|
||||
|
||||
app:
|
||||
build: .
|
||||
container_name: ticket-microservice
|
||||
ports:
|
||||
- "3049:3049"
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- PORT=3049
|
||||
- LOG_LEVEL=info
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./logs:/app/logs
|
||||
- ./tickets:/app/tickets
|
||||
networks:
|
||||
- ticket-network
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
container_name: ticket-prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
- prometheus_data:/prometheus
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--web.console.libraries=/etc/prometheus/console_libraries'
|
||||
- '--web.console.templates=/etc/prometheus/consoles'
|
||||
- '--storage.tsdb.retention.time=200h'
|
||||
- '--web.enable-lifecycle'
|
||||
networks:
|
||||
- ticket-network
|
||||
profiles:
|
||||
- monitoring
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: ticket-grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
networks:
|
||||
- ticket-network
|
||||
profiles:
|
||||
- monitoring
|
||||
|
||||
volumes:
|
||||
redis_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
|
||||
networks:
|
||||
ticket-network:
|
||||
driver: bridge
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
# - "first_rules.yml"
|
||||
# - "second_rules.yml"
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
- job_name: 'ticket-microservice'
|
||||
static_configs:
|
||||
- targets: ['app:3049']
|
||||
metrics_path: '/metrics'
|
||||
scrape_interval: 5s
|
||||
scrape_timeout: 3s
|
||||
Reference in New Issue
Block a user