.readthedocs.yaml

@@ -1,24 +0,0 @@
-# Read the Docs configuration file for Sphinx projects
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
-version: 2
-
-# Set the OS, Python version and other tools you might need
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.10"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
-
-# Optionally build your docs in additional formats such as PDF and ePub
-# formats:
-#   - pdf
-#   - epub
-
-python:
-  install:
-    - requirements: docs/requirements.txt

docker-compose.yaml

@@ -4,7 +4,7 @@ services:
     build: ./src
     env_file:
       - restic_compose_backup.env
-      # - alerts.env
+      - alerts.env
     labels:
       restic-compose-backup.volumes: true
       restic-compose-backup.volumes.include: 'src'
@@ -32,7 +32,7 @@ services:
       - SOME_VALUE=test
       - ANOTHER_VALUE=1
 
-  mysql5:
+  mysql:
     image: mysql:5
     labels:
       restic-compose-backup.mysql: true
@@ -42,19 +42,7 @@ services:
       - MYSQL_USER=myuser
       - MYSQL_PASSWORD=mypassword
     volumes:
-      - mysqldata5:/var/lib/mysql
-
-  mysql8:
-    image: mysql:8
-    labels:
-      restic-compose-backup.mysql: true
-    environment:
-      - MYSQL_ROOT_PASSWORD=my-secret-pw
-      - MYSQL_DATABASE=mydb
-      - MYSQL_USER=myuser
-      - MYSQL_PASSWORD=mypassword
-    volumes:
-      - mysqldata8:/var/lib/mysql
+      - mysqldata:/var/lib/mysql
 
   mariadb:
     image: mariadb:10
@@ -80,8 +68,7 @@ services:
       - pgdata:/var/lib/postgresql/data
 
 volumes:
-  mysqldata5:
-  mysqldata8:
+  mysqldata:
   mariadbdata:
   pgdata:
 

src/Dockerfile

@@ -1,13 +1,6 @@
-FROM restic/restic:0.17.3
+FROM restic/restic:0.9.6
 
-ENV PIP_BREAK_SYSTEM_PACKAGES 1
-
-RUN apk update && apk add python3 \ 
-    py3-pip \
-    dcron \
-    mariadb-client \
-    postgresql-client \
-    mariadb-connector-c-dev 
+RUN apk update && apk add python3 dcron mariadb-client postgresql-client
 
 ADD . /restic-compose-backup
 WORKDIR /restic-compose-backup

src/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # Dump all env vars so we can source them in cron jobs
-rcb dump-env > /env.sh
+printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
 
 # Write crontab
 rcb crontab > crontab

src/restic_compose_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.7.1'
+__version__ = '0.6.0'

src/restic_compose_backup/alerts/smtp.py

@@ -15,7 +15,7 @@ class SMTPAlert(BaseAlert):
         self.host = host
         self.port = port
         self.user = user
-        self.password = password or ""
+        self.password = password
         self.to = to
 
     @classmethod
@@ -34,7 +34,7 @@ class SMTPAlert(BaseAlert):
 
     @property
     def properly_configured(self) -> bool:
-        return self.host and self.port and self.user and len(self.to) > 0
+        return self.host and self.port and self.user and self.password and len(self.to) > 0
 
     def send(self, subject: str = None, body: str = None, alert_type: str = 'INFO'):
         # send_mail("Hello world!")

src/restic_compose_backup/cli.py

@@ -51,9 +51,6 @@ def main():
     elif args.action == "crontab":
         crontab(config)
 
-    elif args.action == "dump-env":
-        dump_env()
-
     # Random test stuff here
     elif args.action == "test":
         nodes = utils.get_swarm_nodes()
@@ -108,10 +105,10 @@ def status(config, containers):
             logger.info(
                 ' - %s (is_ready=%s) -> %s',
                 instance.container_type,
-                ping,
+                ping == 0,
                 instance.backup_destination_path(),
             )
-            if not ping:
+            if ping != 0:
                 logger.error("Database '%s' in service %s cannot be reached",
                              instance.container_type, container.service_name)
 
@@ -293,14 +290,6 @@ def crontab(config):
     print(cron.generate_crontab(config))
 
 
-def dump_env():
-    """Dump all environment variables to a script that can be sourced from cron"""
-    print("#!/bin/bash")
-    print("# This file was generated by restic-compose-backup")
-    for key, value in os.environ.items():
-        print("export {}='{}'".format(key, value))
-
-
 def parse_args():
     parser = argparse.ArgumentParser(prog='restic_compose_backup')
     parser.add_argument(
@@ -314,7 +303,6 @@ def parse_args():
             'cleanup',
             'version',
             'crontab',
-            'dump-env',
             'test',
         ],
     )

src/restic_compose_backup/commands.py

@@ -1,9 +1,7 @@
 import logging
-from typing import List, Tuple, Union
+from typing import List, Tuple
 from subprocess import Popen, PIPE
 
-from restic_compose_backup import utils
-
 logger = logging.getLogger(__name__)
 
 
@@ -11,54 +9,43 @@ def test():
     return run(['ls', '/volumes'])
 
 
-def ping_mysql(container_id, host, port, username, password) -> int:
+def ping_mysql(host, port, username) -> int:
     """Check if the mysql is up and can be reached"""
-    return docker_exec(container_id, [
+    return run([
         'mysqladmin',
         'ping',
+        '--host',
+        host,
+        '--port',
+        port,
         '--user',
         username,
-    ], environment={
-        'MYSQL_PWD': password
-    })
+    ])
 
 
-def ping_mariadb(container_id, host, port, username, password) -> int:
+def ping_mariadb(host, port, username) -> int:
     """Check if the mariadb is up and can be reached"""
-    return docker_exec(container_id, [
+    return run([
         'mysqladmin',
         'ping',
+        '--host',
+        host,
+        '--port',
+        port,
         '--user',
         username,
-    ], environment={
-        'MYSQL_PWD': password
-    })
+    ])
 
 
-def ping_postgres(container_id, host, port, username, password) -> int:
+def ping_postgres(host, port, username, password) -> int:
     """Check if postgres can be reached"""
-    return docker_exec(container_id, [
+    return run([
         "pg_isready",
         f"--host={host}",
         f"--port={port}",
         f"--username={username}",
     ])
 
-def docker_exec(container_id: str, cmd: List[str], environment: Union[dict, list] = []) -> int:
-    """Execute a command within the given container"""
-    client = utils.docker_client()
-    logger.debug('docker exec inside %s: %s', container_id, ' '.join(cmd))
-    exit_code, (stdout, stderr) = client.containers.get(container_id).exec_run(cmd, demux=True, environment=environment)
-
-    if stdout:
-        log_std('stdout', stdout.decode(),
-                logging.DEBUG if exit_code == 0 else logging.ERROR)
-
-    if stderr:
-        log_std('stderr', stderr.decode(), logging.ERROR)
-
-    return exit_code
-
 
 def run(cmd: List[str]) -> int:
     """Run a command with parameters"""

src/restic_compose_backup/containers_db.py

@@ -25,37 +25,34 @@ class MariadbContainer(Container):
         """Check the availability of the service"""
         creds = self.get_credentials()
 
-        return commands.ping_mariadb(
-            self.id,
-            creds['host'],
-            creds['port'],
-            creds['username'],
-            creds['password']
-        ) == 0
+        with utils.environment('MYSQL_PWD', creds['password']):
+            return commands.ping_mariadb(
+                creds['host'],
+                creds['port'],
+                creds['username'],
+            )
 
     def dump_command(self) -> list:
         """list: create a dump command restic and use to send data through stdin"""
         creds = self.get_credentials()
         return [
             "mysqldump",
+            f"--host={creds['host']}",
+            f"--port={creds['port']}",
             f"--user={creds['username']}",
             "--all-databases",
-            "--no-tablespaces",
         ]
 
     def backup(self):
         config = Config()
         creds = self.get_credentials()
 
-        return restic.backup_from_stdin(
-            config.repository,
-            self.backup_destination_path(),
-            self.id,
-            self.dump_command(),
-            environment={
-                'MYSQL_PWD': creds['password']
-            }
-        )
+        with utils.environment('MYSQL_PWD', creds['password']):
+            return restic.backup_from_stdin(
+                config.repository,
+                self.backup_destination_path(),
+                self.dump_command(),
+            )
 
     def backup_destination_path(self) -> str:
         destination = Path("/databases")
@@ -87,37 +84,34 @@ class MysqlContainer(Container):
         """Check the availability of the service"""
         creds = self.get_credentials()
 
-        return commands.ping_mysql(
-            self.id,
-            creds['host'],
-            creds['port'],
-            creds['username'],
-            creds['password']
-        ) == 0
+        with utils.environment('MYSQL_PWD', creds['password']):
+            return commands.ping_mysql(
+                creds['host'],
+                creds['port'],
+                creds['username'],
+            )
 
     def dump_command(self) -> list:
         """list: create a dump command restic and use to send data through stdin"""
         creds = self.get_credentials()
         return [
             "mysqldump",
+            f"--host={creds['host']}",
+            f"--port={creds['port']}",
             f"--user={creds['username']}",
             "--all-databases",
-            "--no-tablespaces",
         ]
 
     def backup(self):
         config = Config()
         creds = self.get_credentials()
 
-        return restic.backup_from_stdin(
-            config.repository,
-            self.backup_destination_path(),
-            self.id,
-            self.dump_command(),
-            environment={
-                "MYSQL_PWD": creds['password']
-            }
-        )
+        with utils.environment('MYSQL_PWD', creds['password']):
+            return restic.backup_from_stdin(
+                config.repository,
+                self.backup_destination_path(),
+                self.dump_command(),
+            )
 
     def backup_destination_path(self) -> str:
         destination = Path("/databases")
@@ -150,12 +144,11 @@ class PostgresContainer(Container):
         """Check the availability of the service"""
         creds = self.get_credentials()
         return commands.ping_postgres(
-            self.id,
             creds['host'],
             creds['port'],
             creds['username'],
             creds['password'],
-        ) == 0
+        )
 
     def dump_command(self) -> list:
         """list: create a dump command restic and use to send data through stdin"""
@@ -163,19 +156,22 @@ class PostgresContainer(Container):
         creds = self.get_credentials()
         return [
             "pg_dump",
+            f"--host={creds['host']}",
+            f"--port={creds['port']}",
             f"--username={creds['username']}",
             creds['database'],
         ]
 
     def backup(self):
         config = Config()
+        creds = self.get_credentials()
 
-        return restic.backup_from_stdin(
-            config.repository,
-            self.backup_destination_path(),
-            self.id,
-            self.dump_command(),
-        )
+        with utils.environment('PGPASSWORD', creds['password']):
+            return restic.backup_from_stdin(
+                config.repository,
+                self.backup_destination_path(),
+                self.dump_command(),
+            )
 
     def backup_destination_path(self) -> str:
         destination = Path("/databases")

src/restic_compose_backup/restic.py

@@ -2,10 +2,9 @@
 Restic commands
 """
 import logging
-from typing import List, Tuple, Union
+from typing import List, Tuple
 from subprocess import Popen, PIPE
 from restic_compose_backup import commands
-from restic_compose_backup import utils
 
 logger = logging.getLogger(__name__)
 
@@ -28,10 +27,9 @@ def backup_files(repository: str, source='/volumes'):
     ]))
 
 
-def backup_from_stdin(repository: str, filename: str, container_id: str,
-                      source_command: List[str], environment: Union[dict, list] = None):
+def backup_from_stdin(repository: str, filename: str, source_command: List[str]):
     """
-    Backs up from stdin running the source_command passed in within the given container.
+    Backs up from stdin running the source_command passed in.
     It will appear in restic with the filename (including path) passed in.
     """
     dest_command = restic(repository, [
@@ -41,43 +39,20 @@ def backup_from_stdin(repository: str, filename: str, container_id: str,
         filename,
     ])
 
-    client = utils.docker_client()
-
-    logger.debug('docker exec inside %s: %s', container_id, ' '.join(source_command))
-
-    # Create and start source command inside the given container
-    handle = client.api.exec_create(container_id, source_command, environment=environment)
-    exec_id = handle["Id"]
-    stream = client.api.exec_start(exec_id, stream=True, demux=True)
-    source_stderr = ""
-
-    # Create the restic process to receive the output of the source command
-    dest_process = Popen(dest_command, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=65536)
-
-    # Send the ouptut of the source command over to restic in the chunks received
-    for stdout_chunk, stderr_chunk in stream:
-        if stdout_chunk:
-            dest_process.stdin.write(stdout_chunk)
-
-        if stderr_chunk:
-            source_stderr += stderr_chunk.decode()
-
-    # Wait for restic to finish
+    # pipe source command into dest command
+    source_process = Popen(source_command, stdout=PIPE, bufsize=65536)
+    dest_process = Popen(dest_command, stdin=source_process.stdout, stdout=PIPE, stderr=PIPE, bufsize=65536)
     stdout, stderr = dest_process.communicate()
 
     # Ensure both processes exited with code 0
-    source_exit = client.api.exec_inspect(exec_id).get("ExitCode")
-    dest_exit = dest_process.poll()
-    exit_code = source_exit or dest_exit
+    source_exit, dest_exit = source_process.poll(), dest_process.poll()
+    exit_code = 0 if (source_exit == 0 and dest_exit == 0) else 1
 
     if stdout:
         commands.log_std('stdout', stdout, logging.DEBUG if exit_code == 0 else logging.ERROR)
 
-    if source_stderr:
-        commands.log_std(f'stderr ({source_command[0]})', source_stderr, logging.ERROR)
-
     if stderr:
-        commands.log_std('stderr (restic)', stderr, logging.ERROR)
+        commands.log_std('stderr', stderr, logging.ERROR)
 
     return exit_code
 

src/restic_compose_backup/utils.py

@@ -3,7 +3,6 @@ import logging
 from typing import List, TYPE_CHECKING
 from contextlib import contextmanager
 import docker
-from docker import DockerClient
 
 if TYPE_CHECKING:
     from restic_compose_backup.containers import Container
@@ -13,7 +12,7 @@ logger = logging.getLogger(__name__)
 TRUE_VALUES = ['1', 'true', 'True', True, 1]
 
 
-def docker_client() -> DockerClient:
+def docker_client():
     """
     Create a docker client from the following environment variables::
 

src/setup.py

@@ -3,15 +3,12 @@ from setuptools import setup, find_namespace_packages
 setup(
     name="restic-compose-backup",
     url="https://github.com/ZettaIO/restic-compose-backup",
-    version="0.7.1",
+    version="0.6.0",
     author="Einar Forselv",
     author_email="eforselv@gmail.com",
-    packages=find_namespace_packages(include=[
-        'restic_compose_backup',
-        'restic_compose_backup.*',
-    ]),
+    packages=find_namespace_packages(include=['restic_compose_backup']),
     install_requires=[
-        'docker~=6.1.3',
+        'docker==4.1.*',
     ],
     entry_points={'console_scripts': [
         'restic-compose-backup = restic_compose_backup.cli:main',