docker nuke and docs update
This commit is contained in:
parent
639041abc5
commit
e701652589
80
.github/workflows/ci.yml
vendored
80
.github/workflows/ci.yml
vendored
@ -117,70 +117,60 @@ jobs:
|
||||
with:
|
||||
sarif_file: gosec.sarif
|
||||
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, lint]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: Login to Container Registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.prod
|
||||
push: true
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}:latest
|
||||
ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build-docker]
|
||||
needs: [test, lint]
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 6379:6379
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Setup test environment
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y sqlite3 bc curl jq
|
||||
sudo apt-get install -y sqlite3 bc curl jq ffmpeg
|
||||
go mod download
|
||||
|
||||
- name: Start services
|
||||
- name: Build application
|
||||
run: |
|
||||
docker-compose -f docker-compose.test.yml up -d
|
||||
go build -o bin/gateway cmd/gateway/main.go
|
||||
chmod +x bin/gateway
|
||||
|
||||
- name: Start application
|
||||
run: |
|
||||
mkdir -p data/{blobs,chunks}
|
||||
./bin/gateway &
|
||||
GATEWAY_PID=$!
|
||||
echo "GATEWAY_PID=$GATEWAY_PID" >> $GITHUB_ENV
|
||||
|
||||
- name: Wait for services
|
||||
run: |
|
||||
timeout 60 bash -c 'until curl -sf http://localhost:9876/api/health; do sleep 1; done'
|
||||
timeout 60 bash -c 'until curl -sf http://localhost:9877/api/health; do sleep 1; done'
|
||||
|
||||
- name: Run E2E tests
|
||||
run: |
|
||||
chmod +x test/e2e/*.sh
|
||||
./test/e2e/run_all_tests.sh
|
||||
if [ -d test/e2e ]; then
|
||||
chmod +x test/e2e/*.sh
|
||||
./test/e2e/run_all_tests.sh
|
||||
else
|
||||
echo "E2E tests directory not found, skipping"
|
||||
fi
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
docker-compose -f docker-compose.test.yml logs
|
||||
|
||||
- name: Stop services
|
||||
- name: Stop application
|
||||
if: always()
|
||||
run: |
|
||||
docker-compose -f docker-compose.test.yml down -v
|
||||
if [ -n "$GATEWAY_PID" ]; then
|
||||
kill $GATEWAY_PID || true
|
||||
fi
|
||||
3
.github/workflows/deploy.yml
vendored
3
.github/workflows/deploy.yml
vendored
@ -98,8 +98,9 @@ jobs:
|
||||
mkdir -p deploy
|
||||
cp bin/gateway deploy/
|
||||
cp -r configs deploy/
|
||||
cp docker-compose.prod.yml deploy/
|
||||
cp -r scripts deploy/
|
||||
cp INSTALL.md deploy/
|
||||
cp README.md deploy/
|
||||
tar -czf torrent-gateway-${{ github.ref_name }}.tar.gz -C deploy .
|
||||
|
||||
- name: Deploy to production
|
||||
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@ -73,8 +73,6 @@ dist/
|
||||
build/
|
||||
bin/
|
||||
|
||||
# Docker
|
||||
.dockerignore
|
||||
|
||||
# Backup files
|
||||
*.bak
|
||||
@ -201,9 +199,6 @@ metadata.db*
|
||||
*.ffmpeg.tmp
|
||||
ffmpeg2pass-*
|
||||
|
||||
# Docker compose overrides
|
||||
docker-compose.override.yml
|
||||
docker-compose.local.yml
|
||||
|
||||
# SSL/TLS certificates and keys
|
||||
ssl/
|
||||
|
||||
153
README.md
153
README.md
@ -99,150 +99,51 @@ go build -o gateway ./cmd/gateway
|
||||
|
||||
The web interface will be available at http://localhost:9877
|
||||
|
||||
### Production Deployment (No Docker)
|
||||
### Production Deployment
|
||||
|
||||
For production deployment without Docker, use the native installation script:
|
||||
For production deployment, use the automated installation script:
|
||||
|
||||
```bash
|
||||
# Make installation script executable
|
||||
chmod +x scripts/install_native.sh
|
||||
|
||||
# Install as system service
|
||||
# Basic installation (HTTP only)
|
||||
sudo ./scripts/install_native.sh
|
||||
|
||||
# Start the service
|
||||
sudo systemctl start torrent-gateway
|
||||
sudo systemctl enable torrent-gateway
|
||||
# Full installation with SSL/HTTPS
|
||||
sudo ./scripts/install_native.sh --domain gateway.example.com --email admin@example.com
|
||||
|
||||
# Check status
|
||||
sudo systemctl status torrent-gateway
|
||||
# With monitoring stack
|
||||
sudo ./scripts/install_native.sh --domain gateway.example.com --email admin@example.com --with-monitoring
|
||||
```
|
||||
|
||||
### Nginx Reverse Proxy Configuration
|
||||
This will automatically:
|
||||
- Install all dependencies (Go, nginx, Redis, FFmpeg, etc.)
|
||||
- Build and deploy the application
|
||||
- Configure systemd service
|
||||
- Set up nginx reverse proxy
|
||||
- Configure SSL certificates (if domain provided)
|
||||
- Start all services
|
||||
|
||||
For production deployments, use Nginx as a reverse proxy:
|
||||
See `INSTALL.md` for detailed installation instructions and domain setup.
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name your-domain.com;
|
||||
### Manual Installation (Development)
|
||||
|
||||
# Redirect HTTP to HTTPS
|
||||
return 301 https://$server_name$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name your-domain.com;
|
||||
|
||||
# SSL Configuration
|
||||
ssl_certificate /path/to/your/certificate.pem;
|
||||
ssl_certificate_key /path/to/your/private-key.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384;
|
||||
ssl_prefer_server_ciphers off;
|
||||
|
||||
# Security Headers
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header Referrer-Policy "no-referrer-when-downgrade" always;
|
||||
add_header Content-Security-Policy "default-src 'self' http: https: data: blob: 'unsafe-inline'" always;
|
||||
|
||||
# Gateway API and Web Interface
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:9877;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket support for real-time features
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
# Increase timeouts for large file uploads
|
||||
proxy_connect_timeout 60s;
|
||||
proxy_send_timeout 60s;
|
||||
proxy_read_timeout 60s;
|
||||
}
|
||||
|
||||
# Blossom Server (optional, if running separately)
|
||||
location /blossom/ {
|
||||
proxy_pass http://127.0.0.1:8082/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# Increase client max body size for file uploads
|
||||
client_max_body_size 10G;
|
||||
|
||||
# Enable gzip compression
|
||||
gzip on;
|
||||
gzip_vary on;
|
||||
gzip_min_length 1024;
|
||||
gzip_proxied any;
|
||||
gzip_comp_level 6;
|
||||
gzip_types text/plain text/css text/xml text/javascript application/json application/javascript application/xml+rss application/atom+xml image/svg+xml;
|
||||
}
|
||||
```
|
||||
|
||||
### SystemD Service Configuration
|
||||
|
||||
The native installer creates a systemd service. Manual configuration:
|
||||
For development or manual setup, you can run the application directly:
|
||||
|
||||
```bash
|
||||
# Create service file
|
||||
sudo tee /etc/systemd/system/torrent-gateway.service > /dev/null <<EOF
|
||||
[Unit]
|
||||
Description=Torrent Gateway Service
|
||||
After=network.target
|
||||
# Build the application
|
||||
go build -o bin/gateway cmd/gateway/main.go
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=torrent-gateway
|
||||
Group=torrent-gateway
|
||||
WorkingDirectory=/opt/torrent-gateway
|
||||
ExecStart=/opt/torrent-gateway/gateway -config /etc/torrent-gateway/config.yaml
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=65536
|
||||
# Create basic directories
|
||||
mkdir -p data/{blobs,chunks}
|
||||
|
||||
# Security settings
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/opt/torrent-gateway/data
|
||||
ReadWritePaths=/var/log/torrent-gateway
|
||||
# Run with default config
|
||||
./bin/gateway
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
# Create dedicated user
|
||||
sudo useradd --system --home /opt/torrent-gateway --shell /bin/false torrent-gateway
|
||||
|
||||
# Create directories and set permissions
|
||||
sudo mkdir -p /opt/torrent-gateway/{data/{blobs,chunks},logs}
|
||||
sudo mkdir -p /etc/torrent-gateway
|
||||
sudo mkdir -p /var/log/torrent-gateway
|
||||
sudo chown -R torrent-gateway:torrent-gateway /opt/torrent-gateway
|
||||
sudo chown -R torrent-gateway:torrent-gateway /var/log/torrent-gateway
|
||||
|
||||
# Copy binary and config
|
||||
sudo cp gateway /opt/torrent-gateway/
|
||||
sudo cp configs/config.yaml /etc/torrent-gateway/
|
||||
sudo chmod +x /opt/torrent-gateway/gateway
|
||||
|
||||
# Enable and start service
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable torrent-gateway
|
||||
sudo systemctl start torrent-gateway
|
||||
# Or with custom config
|
||||
./bin/gateway -config configs/config.yaml
|
||||
```
|
||||
|
||||
For production deployments, use the automated installation script instead.
|
||||
|
||||
### Configuration
|
||||
|
||||
The default configuration is in `configs/config.yaml`. Customize settings there:
|
||||
|
||||
@ -1,51 +0,0 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
gateway:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.dev
|
||||
ports:
|
||||
- "9876:9876" # Gateway API
|
||||
- "8081:8081" # Blossom server
|
||||
- "6882:6882/udp" # DHT node
|
||||
volumes:
|
||||
- .:/app
|
||||
- ./data:/app/data
|
||||
- ./configs:/app/configs
|
||||
environment:
|
||||
- GO_ENV=development
|
||||
- CGO_ENABLED=1
|
||||
restart: unless-stopped
|
||||
command: ["air", "-c", ".air.toml"] # Hot reload with air
|
||||
depends_on:
|
||||
- redis
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
command: redis-server --appendonly yes
|
||||
restart: unless-stopped
|
||||
|
||||
# Development database browser
|
||||
sqlite-web:
|
||||
image: coleifer/sqlite-web
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
environment:
|
||||
- SQLITE_DATABASE=/data/metadata.db
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- gateway
|
||||
|
||||
volumes:
|
||||
redis_data:
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: torrent-gateway-dev
|
||||
@ -1,151 +0,0 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
gateway:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.prod
|
||||
ports:
|
||||
- "9876:9876" # Gateway API
|
||||
- "8081:8081" # Blossom server
|
||||
- "6882:6882/udp" # DHT node
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./configs:/app/configs:ro
|
||||
- ./logs:/app/logs
|
||||
environment:
|
||||
- GO_ENV=production
|
||||
- CGO_ENABLED=1
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
depends_on:
|
||||
- redis
|
||||
- prometheus
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9876/api/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
- ./configs/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
command: redis-server /usr/local/etc/redis/redis.conf
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# Monitoring Stack
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./configs/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./configs/alert_rules.yml:/etc/prometheus/alert_rules.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--web.console.libraries=/etc/prometheus/console_libraries'
|
||||
- '--web.console.templates=/etc/prometheus/consoles'
|
||||
- '--storage.tsdb.retention.time=200h'
|
||||
- '--web.enable-lifecycle'
|
||||
- '--web.enable-admin-api'
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./configs/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./configs/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin123
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_INSTALL_PLUGINS=grafana-piechart-panel
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
loki:
|
||||
image: grafana/loki:latest
|
||||
ports:
|
||||
- "3100:3100"
|
||||
volumes:
|
||||
- ./configs/loki.yml:/etc/loki/local-config.yaml:ro
|
||||
- loki_data:/tmp/loki
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
restart: unless-stopped
|
||||
|
||||
promtail:
|
||||
image: grafana/promtail:latest
|
||||
volumes:
|
||||
- ./logs:/var/log/gateway:ro
|
||||
- ./configs/promtail.yml:/etc/promtail/config.yml:ro
|
||||
- /var/log:/var/log:ro
|
||||
command: -config.file=/etc/promtail/config.yml
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- loki
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:latest
|
||||
ports:
|
||||
- "9093:9093"
|
||||
volumes:
|
||||
- ./configs/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
- alertmanager_data:/alertmanager
|
||||
command:
|
||||
- '--config.file=/etc/alertmanager/alertmanager.yml'
|
||||
- '--storage.path=/alertmanager'
|
||||
- '--web.external-url=http://localhost:9093'
|
||||
restart: unless-stopped
|
||||
|
||||
# Reverse proxy with SSL termination
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
volumes:
|
||||
- ./configs/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./configs/nginx/ssl:/etc/nginx/ssl:ro
|
||||
- ./logs/nginx:/var/log/nginx
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- gateway
|
||||
|
||||
# Database backup service
|
||||
backup:
|
||||
image: alpine:latest
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./backups:/app/backups
|
||||
- ./scripts/backup.sh:/app/backup.sh:ro
|
||||
command: ["sh", "-c", "while true; do sh /app/backup.sh; sleep 3600; done"]
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
redis_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
loki_data:
|
||||
alertmanager_data:
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: torrent-gateway-prod
|
||||
@ -1,75 +0,0 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
gateway-test:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.test
|
||||
environment:
|
||||
- GO_ENV=test
|
||||
- CGO_ENABLED=1
|
||||
- TEST_DATABASE_URL=sqlite3:///tmp/test.db
|
||||
volumes:
|
||||
- .:/app
|
||||
- /tmp:/tmp
|
||||
command: ["go", "test", "-v", "./test/...", "-timeout", "10m"]
|
||||
depends_on:
|
||||
- redis-test
|
||||
- mock-nostr-relay
|
||||
|
||||
redis-test:
|
||||
image: redis:7-alpine
|
||||
command: redis-server --port 6380
|
||||
ports:
|
||||
- "6380:6380"
|
||||
|
||||
# Mock Nostr relay for testing
|
||||
mock-nostr-relay:
|
||||
image: scsibug/nostr-rs-relay:latest
|
||||
ports:
|
||||
- "7777:8080"
|
||||
environment:
|
||||
- RUST_LOG=warn
|
||||
volumes:
|
||||
- test_relay_data:/usr/src/app/db
|
||||
|
||||
# Test database
|
||||
test-db:
|
||||
image: sqlite:latest
|
||||
volumes:
|
||||
- test_db_data:/data
|
||||
environment:
|
||||
- SQLITE_DATABASE=/data/test.db
|
||||
|
||||
# Integration test runner
|
||||
integration-tests:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.test
|
||||
environment:
|
||||
- BASE_URL=http://gateway-test:9876
|
||||
- TEST_TIMEOUT=300
|
||||
volumes:
|
||||
- ./test:/app/test
|
||||
command: ["go", "test", "-v", "./test", "-tags=integration", "-timeout", "15m"]
|
||||
depends_on:
|
||||
- gateway-test
|
||||
|
||||
# E2E test runner
|
||||
e2e-tests:
|
||||
image: curlimages/curl:latest
|
||||
volumes:
|
||||
- ./test/e2e:/tests
|
||||
environment:
|
||||
- BASE_URL=http://gateway-test:9876
|
||||
command: ["sh", "/tests/run_all_tests.sh"]
|
||||
depends_on:
|
||||
- gateway-test
|
||||
|
||||
volumes:
|
||||
test_relay_data:
|
||||
test_db_data:
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: torrent-gateway-test
|
||||
@ -1,47 +1,55 @@
|
||||
# Deployment Guide
|
||||
# Production Deployment Guide
|
||||
|
||||
## Overview
|
||||
|
||||
This guide covers deploying the Torrent Gateway in production using Docker Compose with comprehensive monitoring.
|
||||
This guide covers production deployment of the Torrent Gateway using the native installation script.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker and Docker Compose installed
|
||||
- SQLite3 for database operations
|
||||
- FFmpeg for video transcoding (optional but recommended)
|
||||
- 4GB+ RAM recommended (8GB+ for transcoding)
|
||||
- 50GB+ disk space for storage
|
||||
- Ubuntu 20.04+ or Debian 11+ server
|
||||
- Root access (sudo)
|
||||
- 4GB+ RAM (8GB recommended for high traffic)
|
||||
- 50GB+ disk space
|
||||
- Domain name (optional, for SSL)
|
||||
|
||||
## Quick Deployment
|
||||
|
||||
1. **Build and start services:**
|
||||
```bash
|
||||
./scripts/deploy.sh production v1.0.0
|
||||
```
|
||||
### Basic Production Setup
|
||||
```bash
|
||||
# Clone repository
|
||||
git clone https://git.sovbit.dev/enki/torrentGateway.git
|
||||
cd torrentGateway
|
||||
|
||||
2. **Verify deployment:**
|
||||
```bash
|
||||
./scripts/health_check.sh
|
||||
```
|
||||
# Run production installation
|
||||
sudo ./scripts/install_native.sh --domain gateway.example.com --email admin@example.com --with-monitoring
|
||||
```
|
||||
|
||||
This single command will:
|
||||
- Install all dependencies
|
||||
- Configure nginx reverse proxy
|
||||
- Set up SSL certificates
|
||||
- Install monitoring stack
|
||||
- Start all services
|
||||
|
||||
## Manual Deployment Steps
|
||||
|
||||
### 1. Environment Setup
|
||||
### 1. System Preparation
|
||||
|
||||
```bash
|
||||
# Set environment variables
|
||||
export DEPLOY_ENV=production
|
||||
export VERSION=v1.0.0
|
||||
# Update system
|
||||
sudo apt update && sudo apt upgrade -y
|
||||
|
||||
# Create required directories
|
||||
mkdir -p data/{blobs,chunks} logs backups
|
||||
# Install git if needed
|
||||
sudo apt install -y git
|
||||
|
||||
# Clone repository
|
||||
git clone https://git.sovbit.dev/enki/torrentGateway.git
|
||||
cd torrentGateway
|
||||
```
|
||||
|
||||
### 2. Database Initialization
|
||||
|
||||
```bash
|
||||
# Start services to initialize database
|
||||
docker-compose -f docker-compose.prod.yml up -d gateway redis
|
||||
sudo systemctl start torrent-gateway redis-server nginx
|
||||
|
||||
# Wait for gateway to initialize database
|
||||
./scripts/health_check.sh
|
||||
@ -53,138 +61,238 @@ Review and update configurations:
|
||||
- `configs/prometheus.yml` - Metrics collection
|
||||
- `configs/grafana/` - Dashboard settings
|
||||
- `configs/loki.yml` - Log aggregation
|
||||
- `docker-compose.prod.yml` - Service configuration
|
||||
- `/opt/torrent-gateway/configs/config.yaml` - Main configuration
|
||||
|
||||
### 4. Start Full Stack
|
||||
|
||||
```bash
|
||||
# Start all services including monitoring
|
||||
docker-compose -f docker-compose.prod.yml up -d
|
||||
sudo systemctl start torrent-gateway redis-server nginx prometheus grafana-server
|
||||
|
||||
# Wait for all services to be healthy
|
||||
timeout 120 bash -c 'until curl -sf http://localhost:9876/api/health; do sleep 5; done'
|
||||
timeout 120 bash -c 'until curl -sf http://localhost/api/health; do sleep 5; done'
|
||||
```
|
||||
|
||||
### 5. Verify Deployment
|
||||
|
||||
```bash
|
||||
# Run comprehensive health checks
|
||||
./scripts/health_check.sh
|
||||
sudo /opt/torrent-gateway/scripts/health_check.sh
|
||||
|
||||
# Check service logs
|
||||
docker-compose -f docker-compose.prod.yml logs
|
||||
sudo journalctl -u torrent-gateway -n 50
|
||||
```
|
||||
|
||||
## Service URLs
|
||||
|
||||
- **Gateway API:** http://localhost:9876
|
||||
- **Admin Panel:** http://localhost:9876/admin
|
||||
- **Gateway API:** https://gateway.example.com/api/
|
||||
- **Admin Panel:** https://gateway.example.com/admin
|
||||
- **Prometheus:** http://localhost:9090
|
||||
- **Grafana:** http://localhost:3000 (admin/admin)
|
||||
- **AlertManager:** http://localhost:9093
|
||||
- **Web Interface:** https://gateway.example.com
|
||||
|
||||
## Production Checklist
|
||||
|
||||
- [ ] SSL/TLS certificates configured
|
||||
- [ ] Firewall rules configured
|
||||
- [ ] Backup strategy tested
|
||||
- [ ] Monitoring alerts configured
|
||||
- [ ] SSL certificates configured
|
||||
- [ ] Firewall rules applied
|
||||
- [ ] Monitoring enabled
|
||||
- [ ] Backups scheduled
|
||||
- [ ] Log rotation configured
|
||||
- [ ] Storage limits set
|
||||
- [ ] Resource limits configured
|
||||
- [ ] Security headers enabled
|
||||
|
||||
## Scaling
|
||||
|
||||
### Horizontal Scaling
|
||||
|
||||
For high traffic, you can run multiple instances behind a load balancer:
|
||||
|
||||
```bash
|
||||
# Scale gateway instances
|
||||
docker-compose -f docker-compose.prod.yml up -d --scale gateway=3
|
||||
# Edit systemd service to run on different ports
|
||||
sudo systemctl edit torrent-gateway
|
||||
```
|
||||
|
||||
### Resource Limits
|
||||
|
||||
Update `docker-compose.prod.yml`:
|
||||
```yaml
|
||||
services:
|
||||
gateway:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2G
|
||||
cpus: '1.0'
|
||||
The systemd service includes resource limits:
|
||||
- Memory: 2G max
|
||||
- File descriptors: 65536
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Daily Tasks
|
||||
```bash
|
||||
# Health check
|
||||
sudo /opt/torrent-gateway/scripts/health_check.sh
|
||||
|
||||
# Check disk usage
|
||||
df -h /opt/torrent-gateway/
|
||||
|
||||
# Review logs
|
||||
sudo journalctl -u torrent-gateway --since "1 day ago"
|
||||
```
|
||||
|
||||
## SSL/TLS Setup
|
||||
### Weekly Tasks
|
||||
```bash
|
||||
# Update system
|
||||
sudo apt update && sudo apt upgrade -y
|
||||
|
||||
1. **Obtain certificates:**
|
||||
```bash
|
||||
# Using Let's Encrypt
|
||||
certbot certonly --standalone -d yourdomain.com
|
||||
```
|
||||
# Run cleanup
|
||||
sudo /opt/torrent-gateway/scripts/cleanup.sh
|
||||
|
||||
2. **Update compose file:**
|
||||
```yaml
|
||||
gateway:
|
||||
volumes:
|
||||
- /etc/letsencrypt/live/yourdomain.com:/certs:ro
|
||||
```
|
||||
|
||||
3. **Configure reverse proxy:**
|
||||
Add nginx or traefik for SSL termination.
|
||||
|
||||
## Backup Strategy
|
||||
|
||||
- **Automated backups:** Cron job runs `./scripts/backup.sh` daily
|
||||
- **Manual backup:** `./scripts/backup.sh`
|
||||
- **Retention:** Keep 30 daily, 12 monthly backups
|
||||
- **Storage:** Offsite backup recommended
|
||||
|
||||
## Monitoring Setup
|
||||
|
||||
### Grafana Dashboards
|
||||
|
||||
1. Login to Grafana (admin/admin)
|
||||
2. Change default password
|
||||
3. Import provided dashboards from `configs/grafana/dashboards/`
|
||||
|
||||
### Alert Configuration
|
||||
|
||||
1. Review `configs/alertmanager.yml`
|
||||
2. Configure notification channels (Slack, email, etc.)
|
||||
3. Test alert routing
|
||||
# Check SSL certificate
|
||||
sudo certbot certificates
|
||||
```
|
||||
|
||||
## Security Hardening
|
||||
|
||||
1. **Change default passwords**
|
||||
2. **Enable firewall:**
|
||||
```bash
|
||||
ufw allow 9876/tcp # Gateway API
|
||||
ufw allow 22/tcp # SSH
|
||||
ufw enable
|
||||
```
|
||||
3. **Regular updates:**
|
||||
```bash
|
||||
# Update system packages
|
||||
apt update && apt upgrade -y
|
||||
### 1. Firewall Configuration
|
||||
```bash
|
||||
# Enable UFW
|
||||
sudo ufw enable
|
||||
|
||||
# Update Docker images
|
||||
docker-compose -f docker-compose.prod.yml pull
|
||||
```
|
||||
# Basic rules (already configured by installer)
|
||||
sudo ufw allow ssh
|
||||
sudo ufw allow 80/tcp
|
||||
sudo ufw allow 443/tcp
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
### 2. SSL/TLS Configuration
|
||||
SSL is automatically configured by the installer. To manually renew:
|
||||
```bash
|
||||
sudo certbot renew
|
||||
```
|
||||
|
||||
### Gateway Won't Start
|
||||
- Check disk space: `df -h`
|
||||
- Check database permissions: `ls -la data/`
|
||||
- Review logs: `docker-compose logs gateway`
|
||||
### 3. Service Security
|
||||
The installer configures systemd security features:
|
||||
- Non-root user execution
|
||||
- Read-only system protection
|
||||
- Private temporary directories
|
||||
- Resource limits
|
||||
|
||||
### Database Corruption
|
||||
- Run integrity check: `sqlite3 data/metadata.db "PRAGMA integrity_check;"`
|
||||
- Restore from backup: `./scripts/restore.sh <timestamp>`
|
||||
## Monitoring
|
||||
|
||||
### High Memory Usage
|
||||
- Check for memory leaks in logs
|
||||
- Restart services: `docker-compose restart`
|
||||
- Scale down if necessary
|
||||
### Prometheus Metrics
|
||||
- Service health status
|
||||
- Request rates and latencies
|
||||
- Storage usage
|
||||
- System resources
|
||||
|
||||
### Grafana Dashboards
|
||||
Pre-configured dashboards for:
|
||||
- Service overview
|
||||
- Performance metrics
|
||||
- Error rates
|
||||
- Storage statistics
|
||||
|
||||
### Alerting
|
||||
Configure alerting rules in `configs/alert_rules.yml`
|
||||
|
||||
## Backup & Recovery
|
||||
|
||||
### Automated Backups
|
||||
Daily backups are automatically configured:
|
||||
```bash
|
||||
# Manual backup
|
||||
sudo /opt/torrent-gateway/scripts/backup.sh
|
||||
|
||||
# List backups
|
||||
ls -la /opt/torrent-gateway/backups/
|
||||
|
||||
# Restore from backup
|
||||
sudo /opt/torrent-gateway/scripts/restore.sh 20240827_120000
|
||||
```
|
||||
|
||||
### Database Backups
|
||||
```bash
|
||||
# SQLite backup
|
||||
sqlite3 /opt/torrent-gateway/data/metadata.db ".backup /opt/torrent-gateway/backups/manual_db_backup.db"
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Service Won't Start
|
||||
```bash
|
||||
# Check service status
|
||||
sudo systemctl status torrent-gateway
|
||||
|
||||
# View detailed logs
|
||||
sudo journalctl -u torrent-gateway -f
|
||||
|
||||
# Check configuration
|
||||
sudo /opt/torrent-gateway/bin/gateway -config /opt/torrent-gateway/configs/config.yaml -check
|
||||
```
|
||||
|
||||
### SSL Certificate Issues
|
||||
```bash
|
||||
# Check certificate status
|
||||
sudo certbot certificates
|
||||
|
||||
# Test renewal
|
||||
sudo certbot renew --dry-run
|
||||
|
||||
# Manual certificate request
|
||||
sudo certbot --nginx -d gateway.example.com
|
||||
```
|
||||
|
||||
### Performance Issues
|
||||
```bash
|
||||
# Check resource usage
|
||||
htop
|
||||
sudo systemctl status torrent-gateway
|
||||
|
||||
# Review performance metrics
|
||||
curl http://localhost/metrics
|
||||
```
|
||||
|
||||
## Updates
|
||||
|
||||
### Application Updates
|
||||
```bash
|
||||
cd /path/to/torrentGateway
|
||||
git pull
|
||||
sudo ./scripts/install_native.sh --skip-build=false
|
||||
```
|
||||
|
||||
### System Updates
|
||||
```bash
|
||||
sudo apt update && sudo apt upgrade -y
|
||||
sudo systemctl restart torrent-gateway
|
||||
```
|
||||
|
||||
## Load Balancing
|
||||
|
||||
For very high traffic, use nginx load balancing:
|
||||
|
||||
```nginx
|
||||
upstream torrent_gateway {
|
||||
server 127.0.0.1:9877;
|
||||
server 127.0.0.1:9878;
|
||||
server 127.0.0.1:9879;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name gateway.example.com;
|
||||
|
||||
location / {
|
||||
proxy_pass http://torrent_gateway;
|
||||
# ... other proxy settings
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Migration from Development
|
||||
|
||||
### Export Development Data
|
||||
```bash
|
||||
# On development machine
|
||||
sudo /opt/torrent-gateway/scripts/backup.sh
|
||||
scp /opt/torrent-gateway/backups/gateway_backup_*.tar.gz user@production-server:~/
|
||||
```
|
||||
|
||||
### Import to Production
|
||||
```bash
|
||||
# On production server
|
||||
sudo /opt/torrent-gateway/scripts/restore.sh ~/gateway_backup_*.tar.gz
|
||||
sudo systemctl restart torrent-gateway
|
||||
```
|
||||
@ -116,7 +116,7 @@ server {
|
||||
|
||||
Configure rate limits based on usage patterns:
|
||||
```yaml
|
||||
# In docker-compose.prod.yml
|
||||
# In sudo systemctl
|
||||
environment:
|
||||
- RATE_LIMIT_UPLOAD=10/minute
|
||||
- RATE_LIMIT_DOWNLOAD=100/minute
|
||||
@ -256,7 +256,7 @@ services:
|
||||
**Multiple Gateway Instances:**
|
||||
```bash
|
||||
# Scale to 3 instances
|
||||
docker-compose -f docker-compose.prod.yml up -d --scale gateway=3
|
||||
sudo systemctl
|
||||
```
|
||||
|
||||
**Load Balancer Configuration:**
|
||||
|
||||
@ -430,7 +430,7 @@ sudo chmod 600 /opt/torrent-gateway/configs/*.yml
|
||||
**Export from Docker deployment:**
|
||||
```bash
|
||||
# Create backup from Docker deployment
|
||||
docker-compose -f docker-compose.prod.yml exec gateway /scripts/backup.sh
|
||||
sudo systemctl
|
||||
|
||||
# Copy backup out of container
|
||||
docker cp container_name:/app/backups/gateway_backup_*.tar.gz ./
|
||||
|
||||
@ -1,44 +1,44 @@
|
||||
# Troubleshooting Guide
|
||||
|
||||
## Common Issues and Solutions
|
||||
This guide covers common issues and their solutions for the Torrent Gateway native installation.
|
||||
|
||||
### Service Startup Issues
|
||||
## Service Issues
|
||||
|
||||
#### Gateway Won't Start
|
||||
|
||||
**Symptoms:** Container exits immediately or health checks fail
|
||||
**Symptoms:** Service exits immediately or health checks fail
|
||||
|
||||
**Diagnostic Steps:**
|
||||
```bash
|
||||
# Check container logs
|
||||
docker-compose -f docker-compose.prod.yml logs gateway
|
||||
# Check service status
|
||||
sudo systemctl status torrent-gateway
|
||||
|
||||
# Check detailed logs
|
||||
sudo journalctl -u torrent-gateway -f
|
||||
|
||||
# Check database file
|
||||
ls -la data/metadata.db
|
||||
ls -la /opt/torrent-gateway/data/metadata.db
|
||||
|
||||
# Test database connection
|
||||
sqlite3 data/metadata.db "SELECT COUNT(*) FROM files;"
|
||||
sqlite3 /opt/torrent-gateway/data/metadata.db "SELECT COUNT(*) FROM files;"
|
||||
```
|
||||
|
||||
**Common Causes & Solutions:**
|
||||
|
||||
1. **Database permissions:**
|
||||
**Solutions:**
|
||||
1. **Fix permissions:**
|
||||
```bash
|
||||
sudo chown -R $USER:$USER data/
|
||||
chmod -R 755 data/
|
||||
sudo chown -R torrent-gateway:torrent-gateway /opt/torrent-gateway/data
|
||||
```
|
||||
|
||||
2. **Port conflicts:**
|
||||
2. **Check disk space:**
|
||||
```bash
|
||||
# Check what's using port 9876
|
||||
sudo netstat -tulpn | grep 9876
|
||||
# Kill conflicting process or change port
|
||||
df -h /opt/torrent-gateway/
|
||||
```
|
||||
|
||||
3. **Insufficient disk space:**
|
||||
3. **Reset database (last resort):**
|
||||
```bash
|
||||
df -h
|
||||
# Free up space or add storage
|
||||
sudo systemctl stop torrent-gateway
|
||||
sudo mv /opt/torrent-gateway/data/metadata.db /opt/torrent-gateway/data/metadata.db.backup
|
||||
sudo systemctl start torrent-gateway
|
||||
```
|
||||
|
||||
#### Redis Connection Issues
|
||||
@ -47,14 +47,14 @@ sqlite3 data/metadata.db "SELECT COUNT(*) FROM files;"
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Check Redis container
|
||||
docker-compose -f docker-compose.prod.yml logs redis
|
||||
# Check Redis service
|
||||
sudo systemctl status redis-server
|
||||
|
||||
# Test Redis connection
|
||||
docker exec -it torrentgateway_redis_1 redis-cli ping
|
||||
redis-cli ping
|
||||
|
||||
# Restart Redis
|
||||
docker-compose -f docker-compose.prod.yml restart redis
|
||||
sudo systemctl restart redis-server
|
||||
```
|
||||
|
||||
### Performance Issues
|
||||
@ -63,125 +63,113 @@ docker-compose -f docker-compose.prod.yml restart redis
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Check container resource usage
|
||||
docker stats
|
||||
# Check service resources
|
||||
systemctl status torrent-gateway
|
||||
htop
|
||||
|
||||
# Check system resources
|
||||
top
|
||||
htop
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
1. **Scale gateway instances:**
|
||||
1. **Check for resource leaks:**
|
||||
```bash
|
||||
docker-compose -f docker-compose.prod.yml up -d --scale gateway=2
|
||||
sudo journalctl -u torrent-gateway | grep -i "memory\|leak"
|
||||
```
|
||||
|
||||
2. **Optimize database:**
|
||||
```bash
|
||||
./scripts/migrate.sh # Runs VACUUM and ANALYZE
|
||||
/opt/torrent-gateway/scripts/migrate.sh # Runs VACUUM and ANALYZE
|
||||
```
|
||||
|
||||
3. **Add resource limits:**
|
||||
```yaml
|
||||
services:
|
||||
gateway:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1.0'
|
||||
memory: 1G
|
||||
```
|
||||
Edit systemd service limits in `/etc/systemd/system/torrent-gateway.service`
|
||||
|
||||
#### High Memory Usage
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Check memory usage by container
|
||||
docker stats --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}"
|
||||
# Check memory usage
|
||||
free -h
|
||||
ps aux | grep gateway
|
||||
|
||||
# Check for memory leaks in logs
|
||||
docker-compose logs gateway | grep -i "memory\|leak\|oom"
|
||||
sudo journalctl -u torrent-gateway | grep -i "memory\|leak\|oom"
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
1. **Restart affected containers:**
|
||||
1. **Restart service:**
|
||||
```bash
|
||||
docker-compose -f docker-compose.prod.yml restart gateway
|
||||
sudo systemctl restart torrent-gateway
|
||||
```
|
||||
|
||||
2. **Implement memory limits:**
|
||||
```yaml
|
||||
services:
|
||||
gateway:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2G
|
||||
2. **Implement stricter memory limits:**
|
||||
```ini
|
||||
# In /etc/systemd/system/torrent-gateway.service
|
||||
[Service]
|
||||
MemoryMax=1G
|
||||
```
|
||||
|
||||
#### Slow Response Times
|
||||
#### Slow File Operations
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Test API response time
|
||||
curl -w "@curl-format.txt" -o /dev/null -s http://localhost:9876/api/health
|
||||
|
||||
# Check database performance
|
||||
sqlite3 data/metadata.db "EXPLAIN QUERY PLAN SELECT * FROM files LIMIT 10;"
|
||||
```
|
||||
**Common Causes:**
|
||||
- Disk I/O bottleneck
|
||||
- Database fragmentation
|
||||
- Too many concurrent operations
|
||||
|
||||
**Solutions:**
|
||||
1. **Add database indexes:**
|
||||
```bash
|
||||
./scripts/migrate.sh # Applies performance indexes
|
||||
```
|
||||
|
||||
2. **Optimize storage:**
|
||||
```bash
|
||||
# Check storage I/O
|
||||
iostat -x 1 5
|
||||
```
|
||||
|
||||
### Database Issues
|
||||
|
||||
#### Database Corruption
|
||||
|
||||
**Symptoms:** SQLite errors, integrity check failures
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Check database integrity
|
||||
sqlite3 data/metadata.db "PRAGMA integrity_check;"
|
||||
# Check disk I/O
|
||||
iostat -x 1
|
||||
|
||||
# Check database size and structure
|
||||
sqlite3 data/metadata.db ".schema"
|
||||
ls -lh data/metadata.db
|
||||
# Optimize database
|
||||
sqlite3 /opt/torrent-gateway/data/metadata.db "VACUUM; ANALYZE;"
|
||||
|
||||
# Check file system
|
||||
df -h
|
||||
find /opt/torrent-gateway/data -name "*.tmp" -delete
|
||||
```
|
||||
|
||||
**Recovery:**
|
||||
```bash
|
||||
# Attempt repair
|
||||
sqlite3 data/metadata.db "VACUUM;"
|
||||
## Database Issues
|
||||
|
||||
# If repair fails, restore from backup
|
||||
./scripts/restore.sh $(ls backups/ | grep gateway_backup | tail -1 | sed 's/gateway_backup_\(.*\).tar.gz/\1/')
|
||||
```
|
||||
|
||||
#### Database Lock Issues
|
||||
#### Database Locked Errors
|
||||
|
||||
**Symptoms:** "database is locked" errors
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Find processes using database
|
||||
lsof data/metadata.db
|
||||
lsof /opt/torrent-gateway/data/metadata.db
|
||||
|
||||
# Force unlock (dangerous - stop gateway first)
|
||||
docker-compose -f docker-compose.prod.yml stop gateway
|
||||
rm -f data/metadata.db-wal data/metadata.db-shm
|
||||
# Stop service and clean up (if safe)
|
||||
sudo systemctl stop torrent-gateway
|
||||
rm -f /opt/torrent-gateway/data/metadata.db-wal /opt/torrent-gateway/data/metadata.db-shm
|
||||
sudo systemctl start torrent-gateway
|
||||
```
|
||||
|
||||
#### Database Corruption
|
||||
|
||||
**Symptoms:** SQL errors, integrity check failures
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Check database integrity
|
||||
sqlite3 /opt/torrent-gateway/data/metadata.db "PRAGMA integrity_check;"
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
1. **Try repair:**
|
||||
```bash
|
||||
sudo systemctl stop torrent-gateway
|
||||
sqlite3 /opt/torrent-gateway/data/metadata.db ".recover" > /tmp/recovered.sql
|
||||
sqlite3 /opt/torrent-gateway/data/metadata_new.db < /tmp/recovered.sql
|
||||
```
|
||||
|
||||
2. **Restore from backup:**
|
||||
```bash
|
||||
sudo /opt/torrent-gateway/scripts/restore.sh BACKUP_TIMESTAMP
|
||||
```
|
||||
|
||||
### Storage Issues
|
||||
|
||||
#### Disk Space Full
|
||||
@ -189,61 +177,66 @@ rm -f data/metadata.db-wal data/metadata.db-shm
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Check disk usage
|
||||
df -h
|
||||
du -sh data/*
|
||||
|
||||
# Find large files
|
||||
find data/ -type f -size +100M -exec ls -lh {} \;
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
1. **Clean up old files:**
|
||||
```bash
|
||||
# Remove files older than 30 days
|
||||
find data/blobs/ -type f -mtime +30 -delete
|
||||
find data/chunks/ -type f -mtime +30 -delete
|
||||
```
|
||||
|
||||
2. **Cleanup orphaned data:**
|
||||
```bash
|
||||
./scripts/migrate.sh # Removes orphaned chunks
|
||||
```
|
||||
|
||||
#### Storage Corruption
|
||||
|
||||
**Symptoms:** File integrity check failures
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Run E2E tests to verify storage
|
||||
./test/e2e/run_all_tests.sh
|
||||
|
||||
# Check file system
|
||||
fsck /dev/disk/by-label/data
|
||||
```
|
||||
|
||||
### Network Issues
|
||||
|
||||
#### API Timeouts
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Test network connectivity
|
||||
curl -v http://localhost:9876/api/health
|
||||
|
||||
# Check Docker network
|
||||
docker network ls
|
||||
docker network inspect torrentgateway_default
|
||||
df -h /opt/torrent-gateway/
|
||||
du -sh /opt/torrent-gateway/data/*
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Restart networking
|
||||
docker-compose -f docker-compose.prod.yml down
|
||||
docker-compose -f docker-compose.prod.yml up -d
|
||||
# Run cleanup script
|
||||
sudo /opt/torrent-gateway/scripts/cleanup.sh
|
||||
|
||||
# Increase timeouts in client
|
||||
curl --connect-timeout 30 --max-time 60 http://localhost:9876/api/health
|
||||
# Manual cleanup
|
||||
find /opt/torrent-gateway/data/chunks -type f -mtime +30 -delete
|
||||
find /opt/torrent-gateway/logs -name "*.log" -mtime +7 -delete
|
||||
```
|
||||
|
||||
#### File Upload Failures
|
||||
|
||||
**Common Issues:**
|
||||
- Nginx client_max_body_size too small
|
||||
- Disk space insufficient
|
||||
- Permissions problems
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Check nginx configuration
|
||||
sudo nginx -t
|
||||
sudo systemctl reload nginx
|
||||
|
||||
# Check upload directory permissions
|
||||
ls -la /opt/torrent-gateway/data/
|
||||
|
||||
# Increase nginx limits
|
||||
sudo vim /etc/nginx/sites-available/torrent-gateway
|
||||
# Update: client_max_body_size 50G;
|
||||
```
|
||||
|
||||
## Network Issues
|
||||
|
||||
#### Connection Timeouts
|
||||
|
||||
**Symptoms:** API requests timeout, health checks fail
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Test local connectivity
|
||||
curl -v http://localhost:9877/api/health
|
||||
|
||||
# Check network configuration
|
||||
ss -tulpn | grep 9877
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Restart nginx
|
||||
sudo systemctl restart nginx
|
||||
|
||||
# Check firewall
|
||||
sudo ufw status
|
||||
|
||||
# Increase timeouts
|
||||
curl --connect-timeout 30 --max-time 60 http://localhost:9877/api/health
|
||||
```
|
||||
|
||||
#### Port Binding Issues
|
||||
@ -252,17 +245,47 @@ curl --connect-timeout 30 --max-time 60 http://localhost:9876/api/health
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Check port usage
|
||||
sudo netstat -tulpn | grep :9876
|
||||
sudo lsof -i :9876
|
||||
# Check what's using the port
|
||||
sudo lsof -i :9877
|
||||
sudo netstat -tulpn | grep 9877
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Kill conflicting process
|
||||
sudo kill $(sudo lsof -t -i:9876)
|
||||
sudo kill $(sudo lsof -t -i:9877)
|
||||
|
||||
# Or change port in docker-compose.yml
|
||||
# Or change port in configuration
|
||||
sudo vim /opt/torrent-gateway/configs/config.yaml
|
||||
```
|
||||
|
||||
### SSL/HTTPS Issues
|
||||
|
||||
#### Certificate Errors
|
||||
|
||||
**Symptoms:** SSL warnings, certificate expired
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Check certificate status
|
||||
sudo certbot certificates
|
||||
|
||||
# Renew certificates
|
||||
sudo certbot renew
|
||||
|
||||
# Test renewal
|
||||
sudo certbot renew --dry-run
|
||||
```
|
||||
|
||||
#### Mixed Content Warnings
|
||||
|
||||
**Causes:** HTTP resources loaded on HTTPS page
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Check nginx configuration
|
||||
sudo vim /etc/nginx/sites-available/torrent-gateway
|
||||
# Ensure all proxy_set_header X-Forwarded-Proto $scheme;
|
||||
```
|
||||
|
||||
### Monitoring Issues
|
||||
@ -271,20 +294,20 @@ sudo kill $(sudo lsof -t -i:9876)
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Check Prometheus targets
|
||||
curl -s http://localhost:9090/api/v1/targets
|
||||
# Check Prometheus service
|
||||
sudo systemctl status prometheus
|
||||
|
||||
# Check metrics endpoint
|
||||
curl -s http://localhost:9876/metrics
|
||||
curl -s http://localhost:9877/metrics
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Restart Prometheus
|
||||
docker-compose -f docker-compose.prod.yml restart prometheus
|
||||
sudo systemctl restart prometheus
|
||||
|
||||
# Check configuration
|
||||
docker-compose -f docker-compose.prod.yml exec prometheus cat /etc/prometheus/prometheus.yml
|
||||
sudo vim /opt/prometheus/prometheus.yml
|
||||
```
|
||||
|
||||
#### Grafana Dashboard Issues
|
||||
@ -292,35 +315,59 @@ docker-compose -f docker-compose.prod.yml exec prometheus cat /etc/prometheus/pr
|
||||
**Common Problems:**
|
||||
1. **No data in dashboards:**
|
||||
- Check Prometheus data source configuration
|
||||
- Verify metrics are being collected
|
||||
- Verify metrics are being collected: `curl http://localhost:9877/metrics`
|
||||
|
||||
2. **Dashboard import failures:**
|
||||
- Check JSON syntax
|
||||
2. **Dashboard import errors:**
|
||||
- Verify dashboard version compatibility
|
||||
|
||||
### Log Analysis
|
||||
## Video Transcoding Issues
|
||||
|
||||
#### FFmpeg Errors
|
||||
|
||||
**Symptoms:** Transcoding fails, codec errors
|
||||
|
||||
**Diagnostic:**
|
||||
```bash
|
||||
# Check FFmpeg installation
|
||||
ffmpeg -version
|
||||
|
||||
# Check logs for transcoding errors
|
||||
sudo journalctl -u torrent-gateway | grep -i "transcode\|ffmpeg"
|
||||
```
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Install/update FFmpeg
|
||||
sudo apt update
|
||||
sudo apt install -y ffmpeg
|
||||
|
||||
# Check disk space for temp files
|
||||
df -h /opt/torrent-gateway/data/transcoded/
|
||||
```
|
||||
|
||||
## Log Analysis
|
||||
|
||||
#### Finding Specific Errors
|
||||
|
||||
```bash
|
||||
# Gateway application logs
|
||||
docker-compose -f docker-compose.prod.yml logs gateway | grep -i error
|
||||
sudo journalctl -u torrent-gateway --since "1 hour ago"
|
||||
|
||||
# System logs with timestamps
|
||||
docker-compose -f docker-compose.prod.yml logs --timestamps
|
||||
sudo journalctl --since "1 hour ago" | grep gateway
|
||||
|
||||
# Follow logs in real-time
|
||||
docker-compose -f docker-compose.prod.yml logs -f gateway
|
||||
sudo journalctl -u torrent-gateway -f
|
||||
```
|
||||
|
||||
#### Log Rotation Issues
|
||||
|
||||
```bash
|
||||
# Check log sizes
|
||||
docker-compose -f docker-compose.prod.yml exec gateway ls -lh /app/logs/
|
||||
sudo du -sh /opt/torrent-gateway/logs/*
|
||||
|
||||
# Manually rotate logs
|
||||
docker-compose -f docker-compose.prod.yml exec gateway logrotate /etc/logrotate.conf
|
||||
sudo logrotate -f /etc/logrotate.d/torrent-gateway
|
||||
```
|
||||
|
||||
## Emergency Procedures
|
||||
@ -329,7 +376,7 @@ docker-compose -f docker-compose.prod.yml exec gateway logrotate /etc/logrotate.
|
||||
|
||||
1. **Stop all services:**
|
||||
```bash
|
||||
docker-compose -f docker-compose.prod.yml down
|
||||
sudo systemctl stop torrent-gateway nginx redis-server
|
||||
```
|
||||
|
||||
2. **Check system resources:**
|
||||
@ -341,27 +388,25 @@ docker-compose -f docker-compose.prod.yml exec gateway logrotate /etc/logrotate.
|
||||
|
||||
3. **Restore from backup:**
|
||||
```bash
|
||||
./scripts/restore.sh <timestamp>
|
||||
sudo /opt/torrent-gateway/scripts/restore.sh LATEST_BACKUP
|
||||
```
|
||||
|
||||
4. **Restart services:**
|
||||
```bash
|
||||
sudo systemctl start redis-server nginx torrent-gateway
|
||||
```
|
||||
|
||||
### Data Recovery
|
||||
|
||||
1. **Create immediate backup:**
|
||||
```bash
|
||||
./scripts/backup.sh emergency
|
||||
```
|
||||
```bash
|
||||
# Create emergency backup
|
||||
sudo tar -czf /tmp/gateway_emergency_$(date +%Y%m%d_%H%M%S).tar.gz /opt/torrent-gateway/data/
|
||||
|
||||
2. **Assess data integrity:**
|
||||
```bash
|
||||
sqlite3 data/metadata.db "PRAGMA integrity_check;"
|
||||
```
|
||||
# Check database
|
||||
sqlite3 /opt/torrent-gateway/data/metadata.db ".schema"
|
||||
```
|
||||
|
||||
3. **Restore if necessary:**
|
||||
```bash
|
||||
./scripts/restore.sh <last_good_backup>
|
||||
```
|
||||
|
||||
## Getting Help
|
||||
## Diagnostic Information Collection
|
||||
|
||||
### Log Collection
|
||||
|
||||
@ -370,9 +415,9 @@ Before reporting issues, collect relevant logs:
|
||||
```bash
|
||||
# Create diagnostics package
|
||||
mkdir -p diagnostics
|
||||
docker-compose -f docker-compose.prod.yml logs > diagnostics/service_logs.txt
|
||||
./scripts/health_check.sh > diagnostics/health_check.txt 2>&1
|
||||
cp data/metadata.db diagnostics/ 2>/dev/null || echo "Database not accessible"
|
||||
sudo journalctl -u torrent-gateway --since "1 day ago" > diagnostics/service_logs.txt
|
||||
sudo /opt/torrent-gateway/scripts/health_check.sh > diagnostics/health_check.txt 2>&1
|
||||
cp /opt/torrent-gateway/data/metadata.db diagnostics/ 2>/dev/null || echo "Database not accessible"
|
||||
tar -czf diagnostics_$(date +%Y%m%d_%H%M%S).tar.gz diagnostics/
|
||||
```
|
||||
|
||||
@ -380,94 +425,49 @@ tar -czf diagnostics_$(date +%Y%m%d_%H%M%S).tar.gz diagnostics/
|
||||
|
||||
Always include health check results:
|
||||
```bash
|
||||
./scripts/health_check.sh | tee health_status.txt
|
||||
sudo /opt/torrent-gateway/scripts/health_check.sh | tee health_status.txt
|
||||
```
|
||||
|
||||
### System Information
|
||||
|
||||
```bash
|
||||
# Collect system info
|
||||
echo "Docker version: $(docker --version)" > system_info.txt
|
||||
echo "Docker Compose version: $(docker-compose --version)" >> system_info.txt
|
||||
echo "System: $(uname -a)" >> system_info.txt
|
||||
echo "System: $(uname -a)" > system_info.txt
|
||||
echo "Memory: $(free -h)" >> system_info.txt
|
||||
echo "Disk: $(df -h)" >> system_info.txt
|
||||
echo "FFmpeg: $(ffmpeg -version 2>/dev/null | head -1 || echo 'Not installed')" >> system_info.txt
|
||||
echo "Go version: $(go version 2>/dev/null || echo 'Not installed')" >> system_info.txt
|
||||
```
|
||||
|
||||
## Video Transcoding Issues
|
||||
## Getting Help
|
||||
|
||||
### FFmpeg Not Found
|
||||
### Information to Include
|
||||
|
||||
**Symptoms:** Transcoding fails with "ffmpeg not found" errors
|
||||
When reporting issues, always include:
|
||||
1. System information (OS, version, architecture)
|
||||
2. Installation method and version
|
||||
3. Error messages and logs
|
||||
4. Steps to reproduce
|
||||
5. Expected vs actual behavior
|
||||
|
||||
### Log Locations
|
||||
|
||||
- **Application logs:** `sudo journalctl -u torrent-gateway`
|
||||
- **Nginx logs:** `/var/log/nginx/error.log`
|
||||
- **System logs:** `/var/log/syslog`
|
||||
- **Health check logs:** `/var/log/torrent-gateway-health.log`
|
||||
|
||||
### Useful Commands
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Install FFmpeg
|
||||
sudo apt install ffmpeg # Ubuntu/Debian
|
||||
sudo yum install ffmpeg # CentOS/RHEL
|
||||
brew install ffmpeg # macOS
|
||||
# Full system status
|
||||
sudo /opt/torrent-gateway/scripts/health_check.sh
|
||||
|
||||
# Verify installation
|
||||
ffmpeg -version
|
||||
# Service overview
|
||||
sudo systemctl status torrent-gateway nginx redis-server
|
||||
|
||||
# Resource usage
|
||||
htop
|
||||
df -h
|
||||
free -h
|
||||
```
|
||||
|
||||
### Transcoding Jobs Stuck
|
||||
|
||||
**Symptoms:** Videos remain in "queued" or "processing" status
|
||||
|
||||
**Diagnostic Steps:**
|
||||
```bash
|
||||
# Check transcoding status
|
||||
curl -H "Authorization: Bearer $TOKEN" \
|
||||
http://localhost:9877/api/users/me/files/$HASH/transcoding-status
|
||||
|
||||
# Check process resources
|
||||
ps aux | grep ffmpeg
|
||||
top -p $(pgrep ffmpeg)
|
||||
```
|
||||
|
||||
**Common Causes:**
|
||||
- Insufficient disk space in work directory
|
||||
- Memory limits exceeded
|
||||
- Invalid video format
|
||||
- Corrupted source file
|
||||
|
||||
### High Resource Usage
|
||||
|
||||
**Symptoms:** System slow during transcoding, high CPU/memory usage
|
||||
|
||||
**Solutions:**
|
||||
```yaml
|
||||
# Reduce concurrent jobs
|
||||
transcoding:
|
||||
concurrent_jobs: 2 # Lower from 4
|
||||
|
||||
# Limit CPU usage
|
||||
transcoding:
|
||||
max_cpu_percent: 50 # Reduce from 80
|
||||
nice_level: 15 # Increase from 10
|
||||
|
||||
# Increase minimum file size threshold
|
||||
transcoding:
|
||||
min_file_size: 200MB # Skip more small files
|
||||
```
|
||||
|
||||
### Failed Transcoding Jobs
|
||||
|
||||
**Symptoms:** Jobs marked as "failed" in status API
|
||||
|
||||
**Diagnostic Steps:**
|
||||
```bash
|
||||
# Check transcoding logs
|
||||
grep "transcoding" /var/log/torrent-gateway.log
|
||||
|
||||
# Check FFmpeg error output
|
||||
journalctl -u torrent-gateway | grep ffmpeg
|
||||
```
|
||||
|
||||
**Common Solutions:**
|
||||
- Verify source file is not corrupted
|
||||
- Check available disk space
|
||||
- Ensure FFmpeg supports input format
|
||||
- Review resource limits
|
||||
@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Systemd Setup Script
|
||||
# Sets up Torrent Gateway as a systemd service without Docker
|
||||
# Sets up Torrent Gateway as a systemd service
|
||||
|
||||
set -e
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user