From 9a35774df168e7f55bd248538ac1c887b79039d7 Mon Sep 17 00:00:00 2001 From: Jonas Rosland Date: Fri, 4 Apr 2025 00:54:20 -0400 Subject: [PATCH] Moved logging to volume instead of mount, and added better rotation Signed-off-by: Jonas Rosland --- .env.example | 19 ++++--- .gitignore | 11 +++- Dockerfile | 20 +++---- README.md | 53 ++++++++++++++++++- config.json | 9 ---- config.json.example | 24 +++++++++ data/config/CTCaer_hekate_admin_hekate.json | 11 ---- data/config/SabreTools_MPF_admin_MPF2.json | 12 ----- .../config/glanceapp_glance_admin_glance.json | 16 ------ .../config/microsoft_vscode_admin_vscode.json | 9 ---- ...erver_switch-guide_admin_switch-guide.json | 16 ------ ...ill_TegraExplorer_admin_TegraExplorer.json | 14 ----- docker-compose.yml | 9 ++-- gitmirror/mirror.py | 4 +- gitmirror/utils/logging.py | 27 +++++----- gitmirror/web.py | 2 +- start.sh | 8 --- 17 files changed, 133 insertions(+), 131 deletions(-) delete mode 100644 config.json create mode 100644 config.json.example delete mode 100755 data/config/CTCaer_hekate_admin_hekate.json delete mode 100755 data/config/SabreTools_MPF_admin_MPF2.json delete mode 100755 data/config/glanceapp_glance_admin_glance.json delete mode 100755 data/config/microsoft_vscode_admin_vscode.json delete mode 100755 data/config/nh-server_switch-guide_admin_switch-guide.json delete mode 100755 data/config/suchmememanyskill_TegraExplorer_admin_TegraExplorer.json diff --git a/.env.example b/.env.example index c0595da..8e1e48d 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,20 @@ # GitHub Personal Access Token (create one at https://github.com/settings/tokens) -# Required scopes: repo -GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# Required scopes: repo (for private repositories) +# For public repositories, this is optional but recommended +GITHUB_TOKEN=your_github_token # Gitea Access Token (create one in your Gitea instance under Settings > Applications) -# Required permissions: write:repository, write:issue -GITEA_TOKEN=gta_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# Required permissions: read:user, write:repository, write:issue +GITEA_TOKEN=your_gitea_token # Your Gitea instance URL (no trailing slash) -GITEA_URL=https://gitea.example.com +GITEA_URL=https://your-gitea-instance.com -# Secret key for the web UI (optional) +# Secret key for the web UI (OPTIONAL) # This key is used to secure Flask sessions and flash messages # If not provided, a random key will be automatically generated at container start -# SECRET_KEY=your_secret_key \ No newline at end of file +# SECRET_KEY=your_secret_key + +# Log retention days (OPTIONAL, defaults to 30 days) +# This setting controls how long log files are kept before being automatically deleted +LOG_RETENTION_DAYS=30 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 50045e0..2c72bfa 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ .env.* !.env.example +# Configuration files +config.json + # Logs logs/ *.log @@ -53,4 +56,10 @@ venv.bak/ .DS_Store # Docker artifacts -.dockerignore \ No newline at end of file +.dockerignore + +# Data directory +data/ + +# Docker +.docker/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b9555a3..ae13fd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,20 +36,22 @@ COPY . . # Install the package in development mode RUN pip install -e . -# Create logs and config directories -RUN mkdir -p logs -RUN mkdir -p /app/data/config +# Create a non-root user for security +RUN groupadd -g ${PGID:-1000} appgroup && \ + useradd -m -u ${PUID:-1000} -g appgroup appuser && \ + chown -R appuser:appgroup /app && \ + chmod +x start.sh + +# Create config directory +RUN mkdir -p /app/data/config && \ + chown -R appuser:appgroup /app/data/config + +USER appuser # Set environment variables ENV PYTHONUNBUFFERED=1 ENV GITMIRROR_CONFIG_DIR=/app/data/config -# Create a non-root user for security -RUN useradd -m appuser && \ - chown -R appuser:appuser /app && \ - chmod +x start.sh -USER appuser - # Expose port for web UI EXPOSE 5000 diff --git a/README.md b/README.md index 96f9e4e..bc050f7 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,9 @@ GITEA_URL=https://your-gitea-instance.com # This key is used to secure Flask sessions and flash messages # If not provided, a random key will be automatically generated at container start # SECRET_KEY=your_secret_key + +# Log retention days (OPTIONAL, defaults to 30 days) +LOG_RETENTION_DAYS=30 ``` ### Authentication for Private Repositories @@ -124,6 +127,44 @@ docker-compose logs -f docker-compose down ``` +### Using Bind Mounts for Logs + +By default, logs are stored in a Docker volume for better permission handling. If you prefer to use bind mounts instead (to access logs directly on the host filesystem), you can modify the `docker-compose.yml` file: + +1. Change the volume configuration from: +```yaml +volumes: + - gitmirror_logs:/app/logs +``` + +to: +```yaml +volumes: + - ./logs:/app/logs +``` + +2. Create the logs directory with the correct permissions: +```bash +mkdir -p logs +chmod 755 logs +``` + +3. Update the container user to match your host user's UID/GID: +```yaml +environment: + - PUID=$(id -u) + - PGID=$(id -g) +user: "${PUID}:${PGID}" +``` + +4. Remove the `volumes` section at the bottom of the file that defines `gitmirror_logs` + +This setup will: +- Store logs directly in the `logs` directory on your host +- Allow you to access logs without using Docker commands +- Maintain proper permissions for the container to write logs +- Keep the same log rotation and retention settings + ### Using Docker Directly To run the application with Docker directly: @@ -293,7 +334,17 @@ When an error occurs during mirroring, you can click on the error message to vie ## Logs -Logs are stored in the `logs` directory with a date-based naming convention. The web UI provides a convenient way to view these logs, with direct links from error messages. +Logs are stored in the `logs` directory and are: +- Rotated daily at midnight +- Retained for a configurable number of days (default: 30) +- Separated by service (web and mirror) +- Viewable through the web UI + +Log files follow this naming convention: +- Current log file: `web.log` or `mirror.log` +- Rotated log files: `web.log.2024-03-20`, `web.log.2024-03-19`, etc. + +The log retention period can be configured using the `LOG_RETENTION_DAYS` environment variable in your `.env` file. ## Development and Testing diff --git a/config.json b/config.json deleted file mode 100644 index 64bee20..0000000 --- a/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "scheduler_enabled": false, - "mirror_interval": 8, - "last_run": null, - "next_run": 1741959205, - "log_level": "INFO", - "mirror_interval_hours": 8, - "last_mirror_run": 1741806267.980561 -} \ No newline at end of file diff --git a/config.json.example b/config.json.example new file mode 100644 index 0000000..bc7fd08 --- /dev/null +++ b/config.json.example @@ -0,0 +1,24 @@ +{ + "scheduler_enabled": false, + "mirror_interval": 8, + "last_run": null, + "next_run": null, + "log_level": "INFO", + "repositories": { + "owner/repo": { + "gitea_owner": "gitea_owner", + "gitea_repo": "gitea_repo", + "mirror_issues": true, + "mirror_pull_requests": true, + "mirror_labels": true, + "mirror_milestones": true, + "mirror_wiki": false, + "mirror_releases": false, + "last_mirror_timestamp": null, + "last_mirror_date": null, + "last_mirror_status": null, + "last_mirror_messages": [], + "last_mirror_log": null + } + } +} \ No newline at end of file diff --git a/data/config/CTCaer_hekate_admin_hekate.json b/data/config/CTCaer_hekate_admin_hekate.json deleted file mode 100755 index a07ae46..0000000 --- a/data/config/CTCaer_hekate_admin_hekate.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "mirror_metadata": true, - "mirror_issues": true, - "mirror_pull_requests": true, - "mirror_labels": true, - "mirror_milestones": true, - "mirror_wiki": true, - "last_mirror_timestamp": 1741891663, - "last_mirror_date": "2025-03-13 18:47:43", - "last_mirror_log": "mirror-2025-03-13.log" -} \ No newline at end of file diff --git a/data/config/SabreTools_MPF_admin_MPF2.json b/data/config/SabreTools_MPF_admin_MPF2.json deleted file mode 100755 index 3ce9aea..0000000 --- a/data/config/SabreTools_MPF_admin_MPF2.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "mirror_metadata": true, - "mirror_issues": false, - "mirror_pull_requests": false, - "mirror_labels": false, - "mirror_milestones": false, - "mirror_wiki": false, - "mirror_releases": false, - "last_mirror_timestamp": 1741891512, - "last_mirror_date": "2025-03-13 18:45:12", - "last_mirror_log": "mirror-2025-03-13.log" -} \ No newline at end of file diff --git a/data/config/glanceapp_glance_admin_glance.json b/data/config/glanceapp_glance_admin_glance.json deleted file mode 100755 index c5ac5e0..0000000 --- a/data/config/glanceapp_glance_admin_glance.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "mirror_metadata": true, - "mirror_issues": true, - "mirror_pull_requests": true, - "mirror_labels": true, - "mirror_milestones": true, - "mirror_wiki": true, - "mirror_releases": true, - "last_mirror_timestamp": 1741934040, - "last_mirror_date": "2025-03-14 06:34:00", - "last_mirror_status": "warning", - "last_mirror_messages": [ - "Failed to mirror wiki: Wiki mirroring failed" - ], - "last_mirror_log": "mirror-2025-03-14.log" -} \ No newline at end of file diff --git a/data/config/microsoft_vscode_admin_vscode.json b/data/config/microsoft_vscode_admin_vscode.json deleted file mode 100755 index a172ce4..0000000 --- a/data/config/microsoft_vscode_admin_vscode.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "mirror_metadata": false, - "mirror_issues": false, - "mirror_pull_requests": false, - "mirror_labels": false, - "mirror_milestones": false, - "mirror_wiki": false, - "mirror_releases": true -} \ No newline at end of file diff --git a/data/config/nh-server_switch-guide_admin_switch-guide.json b/data/config/nh-server_switch-guide_admin_switch-guide.json deleted file mode 100755 index ae97792..0000000 --- a/data/config/nh-server_switch-guide_admin_switch-guide.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "mirror_metadata": true, - "mirror_issues": true, - "mirror_pull_requests": true, - "mirror_labels": true, - "mirror_milestones": true, - "mirror_wiki": true, - "mirror_releases": true, - "last_mirror_timestamp": 1741933088, - "last_mirror_date": "2025-03-14 06:18:08", - "last_mirror_status": "warning", - "last_mirror_messages": [ - "Failed to mirror wiki: Wiki mirroring failed" - ], - "last_mirror_log": "mirror-2025-03-14.log" -} \ No newline at end of file diff --git a/data/config/suchmememanyskill_TegraExplorer_admin_TegraExplorer.json b/data/config/suchmememanyskill_TegraExplorer_admin_TegraExplorer.json deleted file mode 100755 index c247f10..0000000 --- a/data/config/suchmememanyskill_TegraExplorer_admin_TegraExplorer.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "mirror_metadata": true, - "mirror_issues": false, - "mirror_pull_requests": false, - "mirror_labels": false, - "mirror_milestones": false, - "mirror_wiki": true, - "mirror_releases": true, - "last_mirror_timestamp": 1741935405, - "last_mirror_date": "2025-03-14 06:56:45", - "last_mirror_status": "success", - "last_mirror_messages": [], - "last_mirror_log": "mirror-2025-03-14.log" -} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1974f79..223c280 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: ports: - "5000:5000" volumes: - - ./logs:/app/logs + - gitmirror_logs:/app/logs - ./config.json:/app/config.json - ./data/config:/app/data/config env_file: @@ -21,10 +21,13 @@ services: mirror: build: . volumes: - - ./logs:/app/logs + - gitmirror_logs:/app/logs - ./config.json:/app/config.json - ./data/config:/app/data/config env_file: - .env command: mirror - restart: "no" \ No newline at end of file + restart: "no" + +volumes: + gitmirror_logs: \ No newline at end of file diff --git a/gitmirror/mirror.py b/gitmirror/mirror.py index f688b9f..a0ef1ba 100644 --- a/gitmirror/mirror.py +++ b/gitmirror/mirror.py @@ -11,9 +11,9 @@ from .gitea.repository import ( from .gitea.release import create_gitea_release from .gitea.metadata import mirror_github_metadata from .utils.config import get_repo_config, save_repo_config -from .utils.logging import get_current_log_filename +from .utils.logging import setup_logging, get_current_log_filename -logger = logging.getLogger('github-gitea-mirror') +logger = setup_logging(service_name='mirror') def mirror_repository(github_token, gitea_token, gitea_url, github_repo, gitea_owner, gitea_repo, skip_repo_config=False, mirror_metadata=None, mirror_releases=None, repo_config=None, force_recreate=False): """Set up a repository as a pull mirror from GitHub to Gitea and sync releases""" diff --git a/gitmirror/utils/logging.py b/gitmirror/utils/logging.py index b70fb79..7b6a7d5 100644 --- a/gitmirror/utils/logging.py +++ b/gitmirror/utils/logging.py @@ -1,13 +1,14 @@ import os import logging -from datetime import datetime -from logging.handlers import RotatingFileHandler +from datetime import datetime, timedelta +from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler -def setup_logging(log_level='INFO'): +def setup_logging(log_level='INFO', service_name='mirror'): """Set up logging configuration with log rotation Args: log_level (str): The logging level to use (DEBUG, INFO, WARNING, ERROR, CRITICAL) + service_name (str): The name of the service (web or mirror) Returns: logging.Logger: The configured logger @@ -15,8 +16,8 @@ def setup_logging(log_level='INFO'): log_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'logs') os.makedirs(log_dir, exist_ok=True) - timestamp = datetime.now().strftime('%Y-%m-%d') - log_file = os.path.join(log_dir, f'mirror-{timestamp}.log') + # Get log retention period from environment variable (default to 30 days) + retention_days = int(os.getenv('LOG_RETENTION_DAYS', '30')) # Convert string log level to logging constant numeric_level = getattr(logging, log_level.upper(), None) @@ -29,11 +30,13 @@ def setup_logging(log_level='INFO'): level=numeric_level, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ - # Use RotatingFileHandler instead of FileHandler for log rotation - RotatingFileHandler( - log_file, - maxBytes=10 * 1024 * 1024, # 10 MB - backupCount=5, # Keep 5 backup files + # Use TimedRotatingFileHandler for daily rotation and retention + TimedRotatingFileHandler( + os.path.join(log_dir, f'{service_name}.log'), + when='midnight', + interval=1, # daily + backupCount=retention_days, # keep logs for specified number of days + encoding='utf-8' ), logging.StreamHandler() ] @@ -43,7 +46,7 @@ def setup_logging(log_level='INFO'): logging.getLogger('requests').setLevel(logging.WARNING) logging.getLogger('urllib3').setLevel(logging.WARNING) - return logging.getLogger('github-gitea-mirror') + return logging.getLogger(f'github-gitea-mirror-{service_name}') def get_current_log_filename(logger): """Get the current log file name from the logger handlers @@ -55,7 +58,7 @@ def get_current_log_filename(logger): str: The basename of the log file, or a fallback name if not found """ try: - # Check for both RotatingFileHandler and regular FileHandler + # Check for both RotatingFileHandler and TimedRotatingFileHandler for handler in logger.handlers: if hasattr(handler, 'baseFilename'): return os.path.basename(handler.baseFilename) diff --git a/gitmirror/web.py b/gitmirror/web.py index 803813b..b472bf1 100644 --- a/gitmirror/web.py +++ b/gitmirror/web.py @@ -26,7 +26,7 @@ from .gitea.metadata import mirror_github_metadata, delete_all_issues_and_prs from .mirror import mirror_repository, process_all_repositories # Set up logging -logger = logging.getLogger('github-gitea-mirror') +logger = setup_logging(service_name='web') # Create Flask app app = Flask(__name__, template_folder=os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'templates')) diff --git a/start.sh b/start.sh index 726e2bc..8aba008 100644 --- a/start.sh +++ b/start.sh @@ -1,14 +1,6 @@ #!/bin/bash set -e -# Create necessary directories -mkdir -p /app/logs -mkdir -p /app/data/config - -# Set default permissions -chmod -R 755 /app/logs -chmod -R 755 /app/data/config - # Generate a random SECRET_KEY if not provided if [ -z "$SECRET_KEY" ]; then echo "No SECRET_KEY found in environment, generating a random one..."