Moved logging to volume instead of mount, and added better rotation

Signed-off-by: Jonas Rosland <jonas.rosland@gmail.com>

.env.example

@@ -1,15 +1,20 @@
 # GitHub Personal Access Token (create one at https://github.com/settings/tokens)
-# Required scopes: repo
-GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+# Required scopes: repo (for private repositories)
+# For public repositories, this is optional but recommended
+GITHUB_TOKEN=your_github_token
 
 # Gitea Access Token (create one in your Gitea instance under Settings > Applications)
-# Required permissions: write:repository, write:issue
-GITEA_TOKEN=gta_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+# Required permissions: read:user, write:repository, write:issue
+GITEA_TOKEN=your_gitea_token
 
 # Your Gitea instance URL (no trailing slash)
-GITEA_URL=https://gitea.example.com
+GITEA_URL=https://your-gitea-instance.com
 
-# Secret key for the web UI (optional)
+# Secret key for the web UI (OPTIONAL)
 # This key is used to secure Flask sessions and flash messages
 # If not provided, a random key will be automatically generated at container start
-# SECRET_KEY=your_secret_key 
+# SECRET_KEY=your_secret_key
+
+# Log retention days (OPTIONAL, defaults to 30 days)
+# This setting controls how long log files are kept before being automatically deleted
+LOG_RETENTION_DAYS=30 

.gitignore

@@ -4,6 +4,9 @@
 .env.*
 !.env.example
 
+# Configuration files
+config.json
+
 # Logs
 logs/
 *.log
@@ -53,4 +56,10 @@ venv.bak/
 .DS_Store
 
 # Docker artifacts
-.dockerignore 
+.dockerignore
+
+# Data directory
+data/
+
+# Docker
+.docker/ 

Dockerfile

@@ -36,20 +36,22 @@ COPY . .
 # Install the package in development mode
 RUN pip install -e .
 
-# Create logs and config directories
-RUN mkdir -p logs
-RUN mkdir -p /app/data/config
+# Create a non-root user for security
+RUN groupadd -g ${PGID:-1000} appgroup && \
+    useradd -m -u ${PUID:-1000} -g appgroup appuser && \
+    chown -R appuser:appgroup /app && \
+    chmod +x start.sh
+
+# Create config directory
+RUN mkdir -p /app/data/config && \
+    chown -R appuser:appgroup /app/data/config
+
+USER appuser
 
 # Set environment variables
 ENV PYTHONUNBUFFERED=1
 ENV GITMIRROR_CONFIG_DIR=/app/data/config
 
-# Create a non-root user for security
-RUN useradd -m appuser && \
-    chown -R appuser:appuser /app && \
-    chmod +x start.sh
-USER appuser
-
 # Expose port for web UI
 EXPOSE 5000
 

README.md

@@ -73,6 +73,9 @@ GITEA_URL=https://your-gitea-instance.com
 # This key is used to secure Flask sessions and flash messages
 # If not provided, a random key will be automatically generated at container start
 # SECRET_KEY=your_secret_key
+
+# Log retention days (OPTIONAL, defaults to 30 days)
+LOG_RETENTION_DAYS=30
 ```
 
 ### Authentication for Private Repositories
@@ -124,6 +127,44 @@ docker-compose logs -f
 docker-compose down
 ```
 
+### Using Bind Mounts for Logs
+
+By default, logs are stored in a Docker volume for better permission handling. If you prefer to use bind mounts instead (to access logs directly on the host filesystem), you can modify the `docker-compose.yml` file:
+
+1. Change the volume configuration from:
+```yaml
+volumes:
+  - gitmirror_logs:/app/logs
+```
+
+to:
+```yaml
+volumes:
+  - ./logs:/app/logs
+```
+
+2. Create the logs directory with the correct permissions:
+```bash
+mkdir -p logs
+chmod 755 logs
+```
+
+3. Update the container user to match your host user's UID/GID:
+```yaml
+environment:
+  - PUID=$(id -u)
+  - PGID=$(id -g)
+user: "${PUID}:${PGID}"
+```
+
+4. Remove the `volumes` section at the bottom of the file that defines `gitmirror_logs`
+
+This setup will:
+- Store logs directly in the `logs` directory on your host
+- Allow you to access logs without using Docker commands
+- Maintain proper permissions for the container to write logs
+- Keep the same log rotation and retention settings
+
 ### Using Docker Directly
 
 To run the application with Docker directly:
@@ -293,7 +334,17 @@ When an error occurs during mirroring, you can click on the error message to vie
 
 ## Logs
 
-Logs are stored in the `logs` directory with a date-based naming convention. The web UI provides a convenient way to view these logs, with direct links from error messages.
+Logs are stored in the `logs` directory and are:
+- Rotated daily at midnight
+- Retained for a configurable number of days (default: 30)
+- Separated by service (web and mirror)
+- Viewable through the web UI
+
+Log files follow this naming convention:
+- Current log file: `web.log` or `mirror.log`
+- Rotated log files: `web.log.2024-03-20`, `web.log.2024-03-19`, etc.
+
+The log retention period can be configured using the `LOG_RETENTION_DAYS` environment variable in your `.env` file.
 
 ## Development and Testing
 

config.json

@@ -1,9 +0,0 @@
-{
-    "scheduler_enabled": false,
-    "mirror_interval": 8,
-    "last_run": null,
-    "next_run": 1741959205,
-    "log_level": "INFO",
-    "mirror_interval_hours": 8,
-    "last_mirror_run": 1741806267.980561
-}

config.json.example

@@ -0,0 +1,24 @@
+{
+    "scheduler_enabled": false,
+    "mirror_interval": 8,
+    "last_run": null,
+    "next_run": null,
+    "log_level": "INFO",
+    "repositories": {
+        "owner/repo": {
+            "gitea_owner": "gitea_owner",
+            "gitea_repo": "gitea_repo",
+            "mirror_issues": true,
+            "mirror_pull_requests": true,
+            "mirror_labels": true,
+            "mirror_milestones": true,
+            "mirror_wiki": false,
+            "mirror_releases": false,
+            "last_mirror_timestamp": null,
+            "last_mirror_date": null,
+            "last_mirror_status": null,
+            "last_mirror_messages": [],
+            "last_mirror_log": null
+        }
+    }
+} 

data/config/CTCaer_hekate_admin_hekate.json

@@ -1,11 +0,0 @@
-{
-  "mirror_metadata": true,
-  "mirror_issues": true,
-  "mirror_pull_requests": true,
-  "mirror_labels": true,
-  "mirror_milestones": true,
-  "mirror_wiki": true,
-  "last_mirror_timestamp": 1741891663,
-  "last_mirror_date": "2025-03-13 18:47:43",
-  "last_mirror_log": "mirror-2025-03-13.log"
-}

data/config/SabreTools_MPF_admin_MPF2.json

@@ -1,12 +0,0 @@
-{
-  "mirror_metadata": true,
-  "mirror_issues": false,
-  "mirror_pull_requests": false,
-  "mirror_labels": false,
-  "mirror_milestones": false,
-  "mirror_wiki": false,
-  "mirror_releases": false,
-  "last_mirror_timestamp": 1741891512,
-  "last_mirror_date": "2025-03-13 18:45:12",
-  "last_mirror_log": "mirror-2025-03-13.log"
-}

data/config/glanceapp_glance_admin_glance.json

@@ -1,16 +0,0 @@
-{
-  "mirror_metadata": true,
-  "mirror_issues": true,
-  "mirror_pull_requests": true,
-  "mirror_labels": true,
-  "mirror_milestones": true,
-  "mirror_wiki": true,
-  "mirror_releases": true,
-  "last_mirror_timestamp": 1741934040,
-  "last_mirror_date": "2025-03-14 06:34:00",
-  "last_mirror_status": "warning",
-  "last_mirror_messages": [
-    "Failed to mirror wiki: Wiki mirroring failed"
-  ],
-  "last_mirror_log": "mirror-2025-03-14.log"
-}

data/config/microsoft_vscode_admin_vscode.json

@@ -1,9 +0,0 @@
-{
-  "mirror_metadata": false,
-  "mirror_issues": false,
-  "mirror_pull_requests": false,
-  "mirror_labels": false,
-  "mirror_milestones": false,
-  "mirror_wiki": false,
-  "mirror_releases": true
-}

data/config/nh-server_switch-guide_admin_switch-guide.json

@@ -1,16 +0,0 @@
-{
-  "mirror_metadata": true,
-  "mirror_issues": true,
-  "mirror_pull_requests": true,
-  "mirror_labels": true,
-  "mirror_milestones": true,
-  "mirror_wiki": true,
-  "mirror_releases": true,
-  "last_mirror_timestamp": 1741933088,
-  "last_mirror_date": "2025-03-14 06:18:08",
-  "last_mirror_status": "warning",
-  "last_mirror_messages": [
-    "Failed to mirror wiki: Wiki mirroring failed"
-  ],
-  "last_mirror_log": "mirror-2025-03-14.log"
-}

data/config/suchmememanyskill_TegraExplorer_admin_TegraExplorer.json

@@ -1,14 +0,0 @@
-{
-  "mirror_metadata": true,
-  "mirror_issues": false,
-  "mirror_pull_requests": false,
-  "mirror_labels": false,
-  "mirror_milestones": false,
-  "mirror_wiki": true,
-  "mirror_releases": true,
-  "last_mirror_timestamp": 1741935405,
-  "last_mirror_date": "2025-03-14 06:56:45",
-  "last_mirror_status": "success",
-  "last_mirror_messages": [],
-  "last_mirror_log": "mirror-2025-03-14.log"
-}

docker-compose.yml

@@ -4,7 +4,7 @@ services:
     ports:
       - "5000:5000"
     volumes:
-      - ./logs:/app/logs
+      - gitmirror_logs:/app/logs
       - ./config.json:/app/config.json
       - ./data/config:/app/data/config
     env_file:
@@ -21,10 +21,13 @@ services:
   mirror:
     build: .
     volumes:
-      - ./logs:/app/logs
+      - gitmirror_logs:/app/logs
       - ./config.json:/app/config.json
       - ./data/config:/app/data/config
     env_file:
       - .env
     command: mirror
-    restart: "no" 
+    restart: "no"
+
+volumes:
+  gitmirror_logs: 

gitmirror/mirror.py

@@ -11,9 +11,9 @@ from .gitea.repository import (
 from .gitea.release import create_gitea_release
 from .gitea.metadata import mirror_github_metadata
 from .utils.config import get_repo_config, save_repo_config
-from .utils.logging import get_current_log_filename
+from .utils.logging import setup_logging, get_current_log_filename
 
-logger = logging.getLogger('github-gitea-mirror')
+logger = setup_logging(service_name='mirror')
 
 def mirror_repository(github_token, gitea_token, gitea_url, github_repo, gitea_owner, gitea_repo, skip_repo_config=False, mirror_metadata=None, mirror_releases=None, repo_config=None, force_recreate=False):
     """Set up a repository as a pull mirror from GitHub to Gitea and sync releases"""

gitmirror/utils/logging.py

@@ -1,13 +1,14 @@
 import os
 import logging
-from datetime import datetime
-from logging.handlers import RotatingFileHandler
+from datetime import datetime, timedelta
+from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
 
-def setup_logging(log_level='INFO'):
+def setup_logging(log_level='INFO', service_name='mirror'):
     """Set up logging configuration with log rotation
     
     Args:
         log_level (str): The logging level to use (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        service_name (str): The name of the service (web or mirror)
         
     Returns:
         logging.Logger: The configured logger
@@ -15,8 +16,8 @@ def setup_logging(log_level='INFO'):
     log_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'logs')
     os.makedirs(log_dir, exist_ok=True)
     
-    timestamp = datetime.now().strftime('%Y-%m-%d')
-    log_file = os.path.join(log_dir, f'mirror-{timestamp}.log')
+    # Get log retention period from environment variable (default to 30 days)
+    retention_days = int(os.getenv('LOG_RETENTION_DAYS', '30'))
     
     # Convert string log level to logging constant
     numeric_level = getattr(logging, log_level.upper(), None)
@@ -29,11 +30,13 @@ def setup_logging(log_level='INFO'):
         level=numeric_level,
         format='%(asctime)s - %(levelname)s - %(message)s',
         handlers=[
-            # Use RotatingFileHandler instead of FileHandler for log rotation
-            RotatingFileHandler(
-                log_file,
-                maxBytes=10 * 1024 * 1024,  # 10 MB
-                backupCount=5,  # Keep 5 backup files
+            # Use TimedRotatingFileHandler for daily rotation and retention
+            TimedRotatingFileHandler(
+                os.path.join(log_dir, f'{service_name}.log'),
+                when='midnight',
+                interval=1,  # daily
+                backupCount=retention_days,  # keep logs for specified number of days
+                encoding='utf-8'
             ),
             logging.StreamHandler()
         ]
@@ -43,7 +46,7 @@ def setup_logging(log_level='INFO'):
     logging.getLogger('requests').setLevel(logging.WARNING)
     logging.getLogger('urllib3').setLevel(logging.WARNING)
     
-    return logging.getLogger('github-gitea-mirror')
+    return logging.getLogger(f'github-gitea-mirror-{service_name}')
 
 def get_current_log_filename(logger):
     """Get the current log file name from the logger handlers
@@ -55,7 +58,7 @@ def get_current_log_filename(logger):
         str: The basename of the log file, or a fallback name if not found
     """
     try:
-        # Check for both RotatingFileHandler and regular FileHandler
+        # Check for both RotatingFileHandler and TimedRotatingFileHandler
         for handler in logger.handlers:
             if hasattr(handler, 'baseFilename'):
                 return os.path.basename(handler.baseFilename)

gitmirror/web.py

@@ -26,7 +26,7 @@ from .gitea.metadata import mirror_github_metadata, delete_all_issues_and_prs
 from .mirror import mirror_repository, process_all_repositories
 
 # Set up logging
-logger = logging.getLogger('github-gitea-mirror')
+logger = setup_logging(service_name='web')
 
 # Create Flask app
 app = Flask(__name__, template_folder=os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'templates'))

start.sh

@@ -1,14 +1,6 @@
 #!/bin/bash
 set -e
 
-# Create necessary directories
-mkdir -p /app/logs
-mkdir -p /app/data/config
-
-# Set default permissions
-chmod -R 755 /app/logs
-chmod -R 755 /app/data/config
-
 # Generate a random SECRET_KEY if not provided
 if [ -z "$SECRET_KEY" ]; then
     echo "No SECRET_KEY found in environment, generating a random one..."