Fix problem with multiple loggers

TODO.md

@@ -0,0 +1,55 @@
+
+# TODO for 1.0
+
+Upgrade restic to 15.x
+
+* Make backup types generic with some standard protocol
+  - New backup types can be registered
+  - When a backup is started we invoke methods in the specific backend
+  - The backend should have access to all information about containers
+  - The backend should be able to run the command in its own container or the target container
+* Don't fetch all containers for all commands. Some commands are just alerts and restic only related
+* More detailed cron setup separating backup time, purge time etc
+* Support mariadb
+* Support influxdb
+* Support backup priority (restic-compose-backup.before-backup.priority=1)
+* Look at bug fixes in forks
+* Use shorter label names. `rcb.priority` instead of `restic-compose-backup.before-backup.priority`
+* Support simple commands in labels
+
+
+## Other misc
+
+* restic unlock needed in some cases?
+* Each snapshot in restic could be tagged with the service name
+
+
+* Is there some elegant way to support a restore?
+* Possibly back up volumes in different snapshots?
+
+
+Use generators in some way to chain actions?
+Action -> Some command
+Use global logger
+
+
+## Dockerfile
+
+Testing
+
+    docker run -it --entrypoint sh --rm restic/restic:0.15.1
+
+Will install python 3.10
+
+    apk add --no-cache python3 py3-pip dcron
+
+## Changelog
+
+* Upgrade restic to 0.15.1
+* Upgraded to python docker 6.0.x
+
+## Misc
+
+* Run rcb command
+* (Optional) Collect docker info
+* (Optional) Issue restic command

docker-compose.yaml

@@ -4,13 +4,13 @@ services:
     build: ./src
     env_file:
       - restic_compose_backup.env
-      - alerts.env
+      # - alerts.env
     labels:
       restic-compose-backup.volumes: true
       restic-compose-backup.volumes.include: 'src'
-    networks:
-      - default
-      - global
+    # networks:
+    #   - default
+    #   - global
     volumes:
       # Map in docker socket
       - /var/run/docker.sock:/tmp/docker.sock:ro
@@ -67,11 +67,16 @@ services:
     volumes:
       - pgdata:/var/lib/postgresql/data
 
+  # InfluxDB OSS
+  # influxdb:
+  #   image: influxdb:1.8
+
 volumes:
   mysqldata:
   mariadbdata:
   pgdata:
+  # influx_data:
 
-networks:
-  global:
-    external: true
+# networks:
+#   global:
+#     external: true

restic_compose_backup.env

@@ -4,7 +4,7 @@
 # DOCKER_TLS_VERIFY=1
 # DOCKER_CERT_PATH=''
 
-SWARM_MODE=true
+SWARM_MODE=false
 INCLUDE_PROJECT_NAME=false
 EXCLUDE_BIND_MOUNTS=false
 

src/Dockerfile

@@ -1,6 +1,8 @@
-FROM restic/restic:0.9.6
+FROM restic/restic:0.15.1
 
-RUN apk update && apk add python3 dcron mariadb-client postgresql-client
+RUN apk update && apk add python3 py3-pip dcron
+# Should be removed
+RUN apk update && apk add mariadb-client postgresql-client
 
 ADD . /restic-compose-backup
 WORKDIR /restic-compose-backup

src/restic_compose_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.6.0'
+__version__ = '1.0.0'

src/restic_compose_backup/backup/base.py

@@ -0,0 +1,37 @@
+
+
+class BackupBase:
+    """
+    Base class for specific backup types such as various databases.
+
+    A backup type is responsible for processing all actions defined
+    on a service. This includes pre-run and post-run actions.
+
+    All backup objects are instantiated before all the backup
+    execution begins to sanity check the configuration and
+    report the current parsed configuration to the user.
+    """
+    def __init__(self, *args, **kwargs):
+        # Possibly pass in the service object here?
+        # Grab labels from service.
+        pass
+
+    def pre_run(self):
+        """
+        Pre-run raw command.
+        Pre-run execution in a container.
+        """
+        pass
+
+    def run(self):
+        """
+        Run the backup
+        """
+        raise NotImplementedError
+
+    def post_run(self):
+        """
+        Post-run raw command.
+        Post-run execution in a container.
+        """
+        pass

src/restic_compose_backup/backup/volumes.py

@@ -0,0 +1,9 @@
+
+#  Zepla 
+#     listens
+#         to 
+#             Asmongolds
+#             advice
+#         on
+#     Online 
+# Harassment 

src/restic_compose_backup/cli.py

@@ -1,310 +1,22 @@
 import argparse
-import os
 import logging
+from typing import List
 
-from restic_compose_backup import (
-    alerts,
-    backup_runner,
-    log,
-    restic,
-)
-from restic_compose_backup.config import Config
-from restic_compose_backup.containers import RunningContainers
-from restic_compose_backup import cron, utils
-
-logger = logging.getLogger(__name__)
+from restic_compose_backup import commands, log
 
 
 def main():
-    """CLI entrypoint"""
-    args = parse_args()
-    config = Config()
-    log.setup(level=args.log_level or config.log_level)
-    containers = RunningContainers()
-
-    # Ensure log level is propagated to parent container if overridden
-    if args.log_level:
-        containers.this_container.set_config_env('LOG_LEVEL', args.log_level)
-
-    if args.action == 'status':
-        status(config, containers)
-
-    elif args.action == 'snapshots':
-        snapshots(config, containers)
-
-    elif args.action == 'backup':
-        backup(config, containers)
-
-    elif args.action == 'start-backup-process':
-        start_backup_process(config, containers)
-
-    elif args.action == 'cleanup':
-        cleanup(config, containers)
-
-    elif args.action == 'alert':
-        alert(config, containers)
-
-    elif args.action == 'version':
-        import restic_compose_backup
-        print(restic_compose_backup.__version__)
-
-    elif args.action == "crontab":
-        crontab(config)
-
-    # Random test stuff here
-    elif args.action == "test":
-        nodes = utils.get_swarm_nodes()
-        print("Swarm nodes:")
-        for node in nodes:
-            addr = node.attrs['Status']['Addr']
-            state = node.attrs['Status']['State']
-            print(' - {} {} {}'.format(node.id, addr, state))
+    """Main entry point for the application"""
+    args = parse_args(sorted(commands.COMMANDS.keys()))
+    command = commands.COMMANDS[args.action](args)
+    command.run()
 
 
-def status(config, containers):
-    """Outputs the backup config for the compose setup"""
-    logger.info("Status for compose project '%s'", containers.project_name)
-    logger.info("Repository: '%s'", config.repository)
-    logger.info("Backup currently running?: %s", containers.backup_process_running)
-    logger.info("Include project name in backup path?: %s", utils.is_true(config.include_project_name))
-    logger.debug("Exclude bind mounts from backups?: %s", utils.is_true(config.exclude_bind_mounts))
-    logger.info("Checking docker availability")
-
-    utils.list_containers()
-
-    if containers.stale_backup_process_containers:
-        utils.remove_containers(containers.stale_backup_process_containers)
-
-    # Check if repository is initialized with restic snapshots
-    if not restic.is_initialized(config.repository):
-        logger.info("Could not get repository info. Attempting to initialize it.")
-        result = restic.init_repo(config.repository)
-        if result == 0:
-            logger.info("Successfully initialized repository: %s", config.repository)
-        else:
-            logger.error("Failed to initialize repository")
-
-    logger.info("%s Detected Config %s", "-" * 25, "-" * 25)
-
-    # Start making snapshots
-    backup_containers = containers.containers_for_backup()
-    for container in backup_containers:
-        logger.info('service: %s', container.service_name)
-
-        if container.volume_backup_enabled:
-            for mount in container.filter_mounts():
-                logger.info(
-                    ' - volume: %s -> %s',
-                    mount.source,
-                    container.get_volume_backup_destination(mount, '/volumes'),
-                )
-
-        if container.database_backup_enabled:
-            instance = container.instance
-            ping = instance.ping()
-            logger.info(
-                ' - %s (is_ready=%s) -> %s',
-                instance.container_type,
-                ping == 0,
-                instance.backup_destination_path(),
-            )
-            if ping != 0:
-                logger.error("Database '%s' in service %s cannot be reached",
-                             instance.container_type, container.service_name)
-
-    if len(backup_containers) == 0:
-        logger.info("No containers in the project has 'restic-compose-backup.*' label")
-
-    logger.info("-" * 67)
-
-
-def backup(config, containers):
-    """Request a backup to start"""
-    # Make sure we don't spawn multiple backup processes
-    if containers.backup_process_running:
-        alerts.send(
-            subject="Backup process container already running",
-            body=(
-                "A backup process container is already running. \n"
-                f"Id: {containers.backup_process_container.id}\n"
-                f"Name: {containers.backup_process_container.name}\n"
-            ),
-            alert_type='ERROR',
-        )
-        raise RuntimeError("Backup process already running")
-
-    # Map all volumes from the backup container into the backup process container
-    volumes = containers.this_container.volumes
-
-    # Map volumes from other containers we are backing up
-    mounts = containers.generate_backup_mounts('/volumes')
-    volumes.update(mounts)
-
-    logger.debug('Starting backup container with image %s', containers.this_container.image)
-    try:
-        result = backup_runner.run(
-            image=containers.this_container.image,
-            command='restic-compose-backup start-backup-process',
-            volumes=volumes,
-            environment=containers.this_container.environment,
-            source_container_id=containers.this_container.id,
-            labels={
-                containers.backup_process_label: 'True',
-                "com.docker.compose.project": containers.project_name,
-            },
-        )
-    except Exception as ex:
-        logger.exception(ex)
-        alerts.send(
-            subject="Exception during backup",
-            body=str(ex),
-            alert_type='ERROR',
-        )
-        return
-
-    logger.info('Backup container exit code: %s', result)
-
-    # Alert the user if something went wrong
-    if result != 0:
-        alerts.send(
-            subject="Backup process exited with non-zero code",
-            body=open('backup.log').read(),
-            alert_type='ERROR',
-        )
-
-
-def start_backup_process(config, containers):
-    """The actual backup process running inside the spawned container"""
-    if not utils.is_true(os.environ.get('BACKUP_PROCESS_CONTAINER')):
-        logger.error(
-            "Cannot run backup process in this container. Use backup command instead. "
-            "This will spawn a new container with the necessary mounts."
-        )
-        alerts.send(
-            subject="Cannot run backup process in this container",
-            body=(
-                "Cannot run backup process in this container. Use backup command instead. "
-                "This will spawn a new container with the necessary mounts."
-            )
-        )
-        exit(1)
-
-    status(config, containers)
-    errors = False
-
-    # Did we actually get any volumes mounted?
-    try:
-        has_volumes = os.stat('/volumes') is not None
-    except FileNotFoundError:
-        logger.warning("Found no volumes to back up")
-        has_volumes = False
-
-    # Warn if there is nothing to do
-    if len(containers.containers_for_backup()) == 0 and not has_volumes:
-        logger.error("No containers for backup found")
-        exit(1)
-
-    if has_volumes:
-        try:
-            logger.info('Backing up volumes')
-            vol_result = restic.backup_files(config.repository, source='/volumes')
-            logger.debug('Volume backup exit code: %s', vol_result)
-            if vol_result != 0:
-                logger.error('Volume backup exited with non-zero code: %s', vol_result)
-                errors = True
-        except Exception as ex:
-            logger.error('Exception raised during volume backup')
-            logger.exception(ex)
-            errors = True
-
-    # back up databases
-    logger.info('Backing up databases')
-    for container in containers.containers_for_backup():
-        if container.database_backup_enabled:
-            try:
-                instance = container.instance
-                logger.info('Backing up %s in service %s', instance.container_type, instance.service_name)
-                result = instance.backup()
-                logger.debug('Exit code: %s', result)
-                if result != 0:
-                    logger.error('Backup command exited with non-zero code: %s', result)
-                    errors = True
-            except Exception as ex:
-                logger.exception(ex)
-                errors = True
-
-    if errors:
-        logger.error('Exit code: %s', errors)
-        exit(1)
-
-    # Only run cleanup if backup was successful
-    result = cleanup(config, container)
-    logger.debug('cleanup exit code: %s', result)
-    if result != 0:
-        logger.error('cleanup exit code: %s', result)
-        exit(1)
-
-    # Test the repository for errors
-    logger.info("Checking the repository for errors")
-    result = restic.check(config.repository)
-    if result != 0:
-        logger.error('Check exit code: %s', result)
-        exit(1)
-
-    logger.info('Backup completed')
-
-
-def cleanup(config, containers):
-    """Run forget / prune to minimize storage space"""
-    logger.info('Forget outdated snapshots')
-    forget_result = restic.forget(
-        config.repository,
-        config.keep_daily,
-        config.keep_weekly,
-        config.keep_monthly,
-        config.keep_yearly,
-    )
-    logger.info('Prune stale data freeing storage space')
-    prune_result = restic.prune(config.repository)
-    return forget_result and prune_result
-
-
-def snapshots(config, containers):
-    """Display restic snapshots"""
-    stdout, stderr = restic.snapshots(config.repository, last=True)
-    for line in stdout.decode().split('\n'):
-        print(line)
-
-
-def alert(config, containers):
-    """Test alerts"""
-    logger.info("Testing alerts")
-    alerts.send(
-        subject="{}: Test Alert".format(containers.project_name),
-        body="Test message",
-    )
-
-
-def crontab(config):
-    """Generate the crontab"""
-    print(cron.generate_crontab(config))
-
-
-def parse_args():
+def parse_args(choices: List[str]):
     parser = argparse.ArgumentParser(prog='restic_compose_backup')
     parser.add_argument(
         'action',
-        choices=[
-            'status',
-            'snapshots',
-            'backup',
-            'start-backup-process',
-            'alert',
-            'cleanup',
-            'version',
-            'crontab',
-            'test',
-        ],
+        choices=choices,
     )
     parser.add_argument(
         '--log-level',

src/restic_compose_backup/commands.py

@@ -1,91 +0,0 @@
-import logging
-from typing import List, Tuple
-from subprocess import Popen, PIPE
-
-logger = logging.getLogger(__name__)
-
-
-def test():
-    return run(['ls', '/volumes'])
-
-
-def ping_mysql(host, port, username) -> int:
-    """Check if the mysql is up and can be reached"""
-    return run([
-        'mysqladmin',
-        'ping',
-        '--host',
-        host,
-        '--port',
-        port,
-        '--user',
-        username,
-    ])
-
-
-def ping_mariadb(host, port, username) -> int:
-    """Check if the mariadb is up and can be reached"""
-    return run([
-        'mysqladmin',
-        'ping',
-        '--host',
-        host,
-        '--port',
-        port,
-        '--user',
-        username,
-    ])
-
-
-def ping_postgres(host, port, username, password) -> int:
-    """Check if postgres can be reached"""
-    return run([
-        "pg_isready",
-        f"--host={host}",
-        f"--port={port}",
-        f"--username={username}",
-    ])
-
-
-def run(cmd: List[str]) -> int:
-    """Run a command with parameters"""
-    logger.debug('cmd: %s', ' '.join(cmd))
-    child = Popen(cmd, stdout=PIPE, stderr=PIPE)
-    stdoutdata, stderrdata = child.communicate()
-
-    if stdoutdata.strip():
-        log_std('stdout', stdoutdata.decode(),
-                logging.DEBUG if child.returncode == 0 else logging.ERROR)
-
-    if stderrdata.strip():
-        log_std('stderr', stderrdata.decode(), logging.ERROR)
-
-    logger.debug("returncode %s", child.returncode)
-    return child.returncode
-
-
-def run_capture_std(cmd: List[str]) -> Tuple[str, str]:
-    """Run a command with parameters and return stdout, stderr"""
-    logger.debug('cmd: %s', ' '.join(cmd))
-    child = Popen(cmd, stdout=PIPE, stderr=PIPE)
-    return child.communicate()
-
-
-def log_std(source: str, data: str, level: int):
-    if isinstance(data, bytes):
-        data = data.decode()
-
-    if not data.strip():
-        return
-
-    log_func = logger.debug if level == logging.DEBUG else logger.error
-    log_func('%s %s %s', '-' * 10, source, '-' * 10)
-
-    lines = data.split('\n')
-    if lines[-1] == '':
-        lines.pop()
-
-    for line in lines:
-        log_func(line)
-
-    log_func('-' * 28)

src/restic_compose_backup/commands/__init__.py

@@ -0,0 +1,19 @@
+import sys
+from importlib import import_module
+from pkgutil import iter_modules
+from typing import Dict
+from .base import BaseCommand
+
+
+def get_commands() -> Dict[str, BaseCommand]:
+    """Return the list of available command classes"""
+    _commands = {}
+    current_module = sys.modules[__name__]
+    for module_info in iter_modules(current_module.__path__):
+        module = import_module(f'restic_compose_backup.commands.{module_info.name}')
+        if hasattr(module, 'Command'):
+            _commands[module_info.name] = module.Command
+    return _commands
+
+
+COMMANDS = get_commands()

src/restic_compose_backup/commands/alert.py

@@ -0,0 +1,15 @@
+from .base import BaseCommand
+from restic_compose_backup import alerts
+
+class Command(BaseCommand):
+    """Send an alert"""
+    name = "alert"
+
+    def run(self):
+        """Test alerts"""
+        self.logger.info("Testing alerts")
+        containers = self.get_containers()
+        alerts.send(
+            subject="{}: Test Alert".format(containers.project_name),
+            body="Test message",
+        )

src/restic_compose_backup/commands/backup.py

@@ -0,0 +1,62 @@
+from .base import BaseCommand
+from restic_compose_backup import backup_runner, alerts
+
+
+class Command(BaseCommand):
+    """Backup a directory"""
+    name = "backup"
+
+    def run(self):
+        """Run the backup command"""
+        containers = self.get_containers()
+
+        if containers.backup_process_running:
+            alerts.send(
+                subject="Backup process container already running",
+                body=(
+                    "A backup process container is already running. \n"
+                    f"Id: {containers.backup_process_container.id}\n"
+                    f"Name: {containers.backup_process_container.name}\n"
+                ),
+                alert_type='ERROR',
+            )
+            raise RuntimeError("Backup process already running")
+
+        # Map all volumes from the backup container into the backup process container
+        volumes = containers.this_container.volumes
+
+        # Map volumes from other containers we are backing up
+        mounts = containers.generate_backup_mounts('/volumes')
+        volumes.update(mounts)
+
+        self.logger.debug('Starting backup container with image %s', containers.this_container.image)
+        try:
+            result = backup_runner.run(
+                image=containers.this_container.image,
+                command='restic-compose-backup start_backup_process',
+                volumes=volumes,
+                environment=containers.this_container.environment,
+                source_container_id=containers.this_container.id,
+                labels={
+                    containers.backup_process_label: 'True',
+                    "com.docker.compose.project": containers.project_name,
+                },
+            )
+        except Exception as ex:
+            self.logger.exception(ex)
+            alerts.send(
+                subject="Exception during backup",
+                body=str(ex),
+                alert_type='ERROR',
+            )
+            return
+
+        self.logger.info('Backup container exit code: %s', result)
+
+        # Alert the user if something went wrong
+        if result != 0:
+            alerts.send(
+                subject="Backup process exited with non-zero code",
+                body=open('backup.log').read(),
+                alert_type='ERROR',
+            )

src/restic_compose_backup/commands/base.py

@@ -0,0 +1,32 @@
+import logging
+from restic_compose_backup.config import Config
+from restic_compose_backup.containers import RunningContainers
+from restic_compose_backup import log
+
+
+class BaseCommand:
+    """Base class for all commands"""
+    name = "base"
+
+    def __init__(self, cli_args, *args, **kwargs):
+        self.cli_args = cli_args
+        self.log_level = cli_args.log_level
+        self.config = Config()
+        log.setup(level=self.log_level or self.config.log_level)
+        self.logger = log.logger
+
+    def get_containers(self):
+        """Get running containers"""
+        containers = RunningContainers()
+        containers.this_container.set_config_env('LOG_LEVEL', self.log_level)
+        return containers
+
+    def run(self):
+        """Run the command"""
+        raise NotImplementedError
+
+    def run_command(self, command: str):
+        """Run a command by name and return the result"""
+        from . import COMMANDS
+        command = COMMANDS[command]
+        command(self.cli_args).run()

src/restic_compose_backup/commands/cleanup.py

@@ -0,0 +1,21 @@
+from .base import BaseCommand
+from restic_compose_backup import restic
+
+
+class Command(BaseCommand):
+    """Cleanup old snapshots"""
+    name = "cleanup"
+
+    def run(self):
+        """Run forget / prune to minimize storage space"""
+        self.logger.info('Forget outdated snapshots')
+        forget_result = restic.forget(
+            self.config.repository,
+            self.config.keep_daily,
+            self.config.keep_weekly,
+            self.config.keep_monthly,
+            self.config.keep_yearly,
+        )
+        self.logger.info('Prune stale data freeing storage space')
+        prune_result = restic.prune(self.config.repository)
+        return forget_result and prune_result

src/restic_compose_backup/commands/crontab.py

@@ -0,0 +1,10 @@
+from .base import BaseCommand
+from restic_compose_backup import cron
+
+class Command(BaseCommand):
+    """Manage crontab"""
+    name = "crontab"
+
+    def run(self):
+        """Generate the crontab"""
+        print(cron.generate_crontab(self.config))

src/restic_compose_backup/commands/snapshots.py

@@ -0,0 +1,13 @@
+from .base import BaseCommand
+from restic_compose_backup import restic
+
+
+class Command(BaseCommand):
+    """List snapshots"""
+    name = "snapshots"
+
+    def run(self):
+        """Display restic snapshots"""
+        stdout, stderr = restic.snapshots(self.config.repository, last=True)
+        for line in stdout.decode().split('\n'):
+            print(line)

src/restic_compose_backup/commands/start_backup_process.py

@@ -0,0 +1,89 @@
+import os
+from . import BaseCommand
+from restic_compose_backup import restic, alerts, utils
+
+class Command(BaseCommand):
+    name = "start_backup_process"
+
+    def run(self):
+        """The actual backup process running inside the spawned container"""
+        if not utils.is_true(os.environ.get('BACKUP_PROCESS_CONTAINER')):
+            self.logger.error(
+                "Cannot run backup process in this container. Use backup command instead. "
+                "This will spawn a new container with the necessary mounts."
+            )
+            alerts.send(
+                subject="Cannot run backup process in this container",
+                body=(
+                    "Cannot run backup process in this container. Use backup command instead. "
+                    "This will spawn a new container with the necessary mounts."
+                )
+            )
+            exit(1)
+
+        self.run_command("status")  # status(config, containers)
+        errors = False
+        containers = self.get_containers()
+
+        # Did we actually get any volumes mounted?
+        try:
+            has_volumes = os.stat('/volumes') is not None
+        except FileNotFoundError:
+            self.logger.warning("Found no volumes to back up")
+            has_volumes = False
+
+        # Warn if there is nothing to do
+        if len(containers.containers_for_backup()) == 0 and not has_volumes:
+            self.logger.error("No containers for backup found")
+            exit(1)
+
+        if has_volumes:
+            try:
+                self.logger.info('Backing up volumes')
+                vol_result = restic.backup_files(self.config.repository, source='/volumes')
+                self.logger.debug('Volume backup exit code: %s', vol_result)
+                if vol_result != 0:
+                    self.logger.error('Volume backup exited with non-zero code: %s', vol_result)
+                    errors = True
+            except Exception as ex:
+                self.logger.error('Exception raised during volume backup')
+                self.logger.exception(ex)
+                errors = True
+
+        # back up databases
+        self.logger.info('Backing up databases')
+        for container in containers.containers_for_backup():
+            if container.database_backup_enabled:
+                try:
+                    instance = container.instance
+                    self.logger.info('Backing up %s in service %s', instance.container_type, instance.service_name)
+                    result = instance.backup()
+                    self.logger.debug('Exit code: %s', result)
+                    if result != 0:
+                        self.logger.error('Backup command exited with non-zero code: %s', result)
+                        errors = True
+                except Exception as ex:
+                    self.logger.exception(ex)
+                    errors = True
+
+        if errors:
+            self.logger.error('Exit code: %s', errors)
+            exit(1)
+
+        # Only run cleanup if backup was successful
+        #result = cleanup(config, container)
+        self.run_command("cleanup")
+
+        self.logger.debug('cleanup exit code: %s', result)
+        if result != 0:
+            self.logger.error('cleanup exit code: %s', result)
+            exit(1)
+
+        # Test the repository for errors
+        self.logger.info("Checking the repository for errors")
+        result = restic.check(self.config.repository)
+        if result != 0:
+            self.logger.error('Check exit code: %s', result)
+            exit(1)
+
+        self.logger.info('Backup completed')

src/restic_compose_backup/commands/status.py

@@ -0,0 +1,66 @@
+from .base import BaseCommand
+from restic_compose_backup import utils, restic
+
+
+class Command(BaseCommand):
+    """Show status"""
+    name = "status"
+
+    def run(self):
+        """Outputs the backup config for the compose setup"""
+        containers = self.get_containers()
+
+        self.logger.info("Status for compose project '%s'", containers.project_name)
+        self.logger.info("Repository: '%s'", self.config.repository)
+        self.logger.info("Backup currently running?: %s", containers.backup_process_running)
+        self.logger.info("Include project name in backup path?: %s", utils.is_true(self.config.include_project_name))
+        self.logger.debug("Exclude bind mounts from backups?: %s", utils.is_true(self.config.exclude_bind_mounts))
+        self.logger.info("Checking docker availability")
+
+        utils.list_containers()
+
+        if containers.stale_backup_process_containers:
+            utils.remove_containers(containers.stale_backup_process_containers)
+
+        # Check if repository is initialized with restic snapshots
+        if not restic.is_initialized(self.config.repository):
+            self.logger.info("Could not get repository info. Attempting to initialize it.")
+            result = restic.init_repo(self.config.repository)
+            if result == 0:
+                self.logger.info("Successfully initialized repository: %s", self.config.repository)
+            else:
+                self.logger.error("Failed to initialize repository")
+
+        self.logger.info("%s Detected Config %s", "-" * 25, "-" * 25)
+
+        # Start making snapshots
+        backup_containers = containers.containers_for_backup()
+        for container in backup_containers:
+            self.logger.info('service: %s', container.service_name)
+
+            if container.volume_backup_enabled:
+                for mount in container.filter_mounts():
+                    self.logger.info(
+                        ' - volume: %s -> %s',
+                        mount.source,
+                        container.get_volume_backup_destination(mount, '/volumes'),
+                    )
+
+            if container.database_backup_enabled:
+                instance = container.instance
+                # ping = instance.ping()
+                ping = 0
+                self.logger.info(
+                    ' - %s (is_ready=%s) -> %s',
+                    instance.container_type,
+                    ping == 0,
+                    instance.backup_destination_path(),
+                )
+                if ping != 0:
+                    self.logger.error("Database '%s' in service %s cannot be reached",
+                                instance.container_type, container.service_name)
+
+        if len(backup_containers) == 0:
+            self.logger.info("No containers in the project has 'restic-compose-backup.*' label")
+
+        self.logger.info("-" * 67)

src/restic_compose_backup/commands/test.py

@@ -0,0 +1,15 @@
+from .base import BaseCommand
+from restic_compose_backup import utils
+
+class Command(BaseCommand):
+    """Test a command"""
+    name = "test"
+
+    def run(self):
+        """Random test command"""
+        nodes = utils.get_swarm_nodes()
+        print("Swarm nodes:")
+        for node in nodes:
+            addr = node.attrs['Status']['Addr']
+            state = node.attrs['Status']['State']
+            print(' - {} {} {}'.format(node.id, addr, state))

src/restic_compose_backup/commands/version.py

@@ -0,0 +1,10 @@
+from .base import BaseCommand
+
+
+class Command(BaseCommand):
+    """Show version"""
+    name = "version"
+
+    def run(self):
+        import restic_compose_backup
+        print(restic_compose_backup.__version__)

src/restic_compose_backup/log.py

@@ -25,4 +25,7 @@ def setup(level: str = 'warning'):
     # ch.setFormatter(logging.Formatter('%(asctime)s - {HOSTNAME} - %(name)s - %(levelname)s - %(message)s'))
     # ch.setFormatter(logging.Formatter('%(asctime)s - {HOSTNAME} - %(levelname)s - %(message)s'))
     ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s: %(message)s'))
-    logger.addHandler(ch)
+
+    # Prevent duplicate handlers
+    if not logger.handlers:
+        logger.addHandler(ch)

src/restic_compose_backup/restic.py

@@ -4,7 +4,7 @@ Restic commands
 import logging
 from typing import List, Tuple
 from subprocess import Popen, PIPE
-from restic_compose_backup import commands
+from restic_compose_backup import utils
 
 logger = logging.getLogger(__name__)
 
@@ -14,13 +14,13 @@ def init_repo(repository: str):
     Attempt to initialize the repository.
     Doing this after the repository is initialized
     """
-    return commands.run(restic(repository, [
+    return utils.run(restic(repository, [
         "init",
     ]))
 
 
 def backup_files(repository: str, source='/volumes'):
-    return commands.run(restic(repository, [
+    return utils.run(restic(repository, [
         "--verbose",
         "backup",
         source,
@@ -49,10 +49,10 @@ def backup_from_stdin(repository: str, filename: str, source_command: List[str])
     exit_code = 0 if (source_exit == 0 and dest_exit == 0) else 1
 
     if stdout:
-        commands.log_std('stdout', stdout, logging.DEBUG if exit_code == 0 else logging.ERROR)
+        utils.log_std('stdout', stdout, logging.DEBUG if exit_code == 0 else logging.ERROR)
 
     if stderr:
-        commands.log_std('stderr', stderr, logging.ERROR)
+        utils.log_std('stderr', stderr, logging.ERROR)
 
     return exit_code
 
@@ -62,21 +62,21 @@ def snapshots(repository: str, last=True) -> Tuple[str, str]:
     args = ["snapshots"]
     if last:
         args.append('--last')
-    return commands.run_capture_std(restic(repository, args))
+    return utils.run_capture_std(restic(repository, args))
 
 
 def is_initialized(repository: str) -> bool:
     """
     Checks if a repository is initialized using snapshots command.
-    Note that this cannot separate between uninitalized repo
-    and other errors, but this method is reccomended by the restic
+    Note that this cannot separate between uninitialized repo
+    and other errors, but this method is recommended by the restic
     community.
     """
-    return commands.run(restic(repository, ["snapshots", '--last'])) == 0
+    return utils.run(restic(repository, ["snapshots", '--last'])) == 0
 
 
 def forget(repository: str, daily: str, weekly: str, monthly: str, yearly: str):
-    return commands.run(restic(repository, [
+    return utils.run(restic(repository, [
         'forget',
         '--group-by',
         'paths',
@@ -92,13 +92,13 @@ def forget(repository: str, daily: str, weekly: str, monthly: str, yearly: str):
 
 
 def prune(repository: str):
-    return commands.run(restic(repository, [
+    return utils.run(restic(repository, [
         'prune',
     ]))
 
 
 def check(repository: str):
-    return commands.run(restic(repository, [
+    return utils.run(restic(repository, [
         "check",
         # "--with-cache",
     ]))

src/restic_compose_backup/utils.py

@@ -1,6 +1,7 @@
 import os
 import logging
-from typing import List, TYPE_CHECKING
+from typing import List, Tuple, TYPE_CHECKING
+from subprocess import Popen, PIPE
 from contextlib import contextmanager
 import docker
 
@@ -8,7 +9,6 @@ if TYPE_CHECKING:
     from restic_compose_backup.containers import Container
 
 logger = logging.getLogger(__name__)
-
 TRUE_VALUES = ['1', 'true', 'True', True, 1]
 
 
@@ -64,9 +64,9 @@ def remove_containers(containers: List['Container']):
             logger.exception(ex)
 
 
-def is_true(value):
+def is_true(value) -> True:
     """
-    Evaluates the truthfullness of a bool value in container labels
+    Evaluates the truthfulness of a bool value in container labels
     """
     return value in TRUE_VALUES
 
@@ -85,7 +85,7 @@ def strip_root(path):
 
 @contextmanager
 def environment(name, value):
-    """Tempset env var"""
+    """Temp-set environment variables"""
     old_val = os.environ.get(name)
     os.environ[name] = value
     try:
@@ -95,3 +95,89 @@ def environment(name, value):
             del os.environ[name]
         else:
             os.environ[name] = old_val
+
+
+def test():
+    return run(['ls', '/volumes'])
+
+
+def ping_mysql(host, port, username) -> int:
+    """Check if the mysql is up and can be reached"""
+    return run([
+        'mysqladmin',
+        'ping',
+        '--host',
+        host,
+        '--port',
+        port,
+        '--user',
+        username,
+    ])
+
+
+def ping_mariadb(host, port, username) -> int:
+    """Check if the mariadb is up and can be reached"""
+    return run([
+        'mysqladmin',
+        'ping',
+        '--host',
+        host,
+        '--port',
+        port,
+        '--user',
+        username,
+    ])
+
+
+def ping_postgres(host, port, username, password) -> int:
+    """Check if postgres can be reached"""
+    return run([
+        "pg_isready",
+        f"--host={host}",
+        f"--port={port}",
+        f"--username={username}",
+    ])
+
+
+def run(cmd: List[str]) -> int:
+    """Run a command with parameters"""
+    logger.debug('cmd: %s', ' '.join(cmd))
+    child = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    stdoutdata, stderrdata = child.communicate()
+
+    if stdoutdata.strip():
+        log_std('stdout', stdoutdata.decode(),
+                logging.DEBUG if child.returncode == 0 else logging.ERROR)
+
+    if stderrdata.strip():
+        log_std('stderr', stderrdata.decode(), logging.ERROR)
+
+    logger.debug("returncode %s", child.returncode)
+    return child.returncode
+
+
+def run_capture_std(cmd: List[str]) -> Tuple[str, str]:
+    """Run a command with parameters and return stdout, stderr"""
+    logger.debug('cmd: %s', ' '.join(cmd))
+    child = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    return child.communicate()
+
+
+def log_std(source: str, data: str, level: int):
+    if isinstance(data, bytes):
+        data = data.decode()
+
+    if not data.strip():
+        return
+
+    log_func = logger.debug if level == logging.DEBUG else logger.error
+    log_func('%s %s %s', '-' * 10, source, '-' * 10)
+
+    lines = data.split('\n')
+    if lines[-1] == '':
+        lines.pop()
+
+    for line in lines:
+        log_func(line)
+
+    log_func('-' * 28)

src/setup.py

@@ -3,12 +3,13 @@ from setuptools import setup, find_namespace_packages
 setup(
     name="restic-compose-backup",
     url="https://github.com/ZettaIO/restic-compose-backup",
-    version="0.6.0",
+    version="1.0.0",
     author="Einar Forselv",
     author_email="eforselv@gmail.com",
     packages=find_namespace_packages(include=['restic_compose_backup']),
     install_requires=[
-        'docker==4.1.*',
+        # 'docker==4.1.*',
+        'docker~=6.0.0',
     ],
     entry_points={'console_scripts': [
         'restic-compose-backup = restic_compose_backup.cli:main',