diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 590e892fb68911c86f10176a03d4c44a4c4c0e50..7016c5e5b6d33ad2dcc4714b91de6b598938a5e1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -34,6 +34,6 @@ "ms-azuretools.vscode-docker", "mhutchie.git-graph", "timonwong.shellcheck", - "ms-python.python" + "ms-python.vscode-pylance" ] } \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 24cd1f4416e162f1e1d6fdbe7f232d2ed68b95b3..f75aedf456bea17498a891c0cc62654a2d300c2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ test-dev-environment: docker-compose up -d --build --remove-orphans dind storage sleep 5 docker-compose run -T orchestrator /bin/bash -c "./init-backup.sh" - docker-compose run -T orchestrator /bin/bash -c "python3 -m unittest discover -s ./dobas/test" + docker-compose run -T orchestrator /bin/bash -c "python3 -m unittest discover -s ./dobas/test -p test*.py" echo "check that source-mount into environment is working in expected path" cd "$CI_PROJECT_DIR" extends: @@ -86,6 +86,6 @@ test-orchestrator-build: docker-compose build orchestrator sleep 5 docker-compose run -T orchestrator /bin/bash -c "./init-backup.sh" - docker-compose run -T orchestrator /bin/bash -c "python3 -m unittest discover -s ./dobas/test" + docker-compose run -T orchestrator /bin/bash -c "python3 -m unittest discover -s ./dobas/test -p test*.py" extends: - .test-rules diff --git a/.vscode/launch.json b/.vscode/launch.json index 17e15f27ec2bb23f50627f046a7e8d1b225be974..896bb6efaa1ae614318dd53a068194d367789d2d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,6 +9,11 @@ "type": "python", "request": "launch", "program": "${file}", + "args": [ + "--orchestrator.mode.restore", + "--orchestrator.docker.host=${env:DOCKER_HOST}", + "--verbose" + ], "console": "integratedTerminal" } ] diff --git a/dobas/__init__.py b/dobas/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..266b1f110c475622dadf2aea35923c69bdcfbd07 100644 --- a/dobas/__init__.py +++ b/dobas/__init__.py @@ -0,0 +1,36 @@ +import logging +from logging import log +import sys + +__version__ = "0.1.0" + +logger = logging.getLogger(__name__) + +# create console handler and set level to debug +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) + +# create formatter +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s", + "%Y-%m-%d %H:%M:%S") + +# add formatter to ch +ch.setFormatter(formatter) + +# add ch to logger +logger.addHandler(ch) + + +# add filehandler +# fh = logging.FileHandler("./dobas.log") +# fh.setLevel(logging.DEBUG) +# fh.setFormatter(formatter) +# logger.addHandler(fh) + +def configure_module_logger(): + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.DEBUG) + logger.addHandler(handler) + + +configure_module_logger() diff --git a/dobas/dobas.py b/dobas/dobas.py deleted file mode 100644 index f1e72e69a3b01072118114ae9d89d1a47629013b..0000000000000000000000000000000000000000 --- a/dobas/dobas.py +++ /dev/null @@ -1,125 +0,0 @@ - -"""Borg backup -Usage: - dobas.py backup [options] - dobas.py restore [options] - dobas.py (-h | --help) - dobas.py --version - -Options: - -c , --config Borg configuration file [default: x] - -d, --dry Make dry run. Don't change anything. - --host Docker host [default: unix://var/run/docker.sock] - -v, --verbose Increase verbosity - -h, --help Show this screen. - --version Show version. - -- find subjects to backup -- for each subject: - do_subject specific backup - - -- subject types: - - pure docker volume -""" -from dataclasses import dataclass -import logging -import sys -import docker -import subprocess -from pathlib import PurePath - - -@dataclass -class DobasCfg: - dry_run: bool = False - verbosity: int = logging.WARNING - borg_prune_rules: str = "--keep-daily 14 --keep-monthly 10" - - -class DobasStrategy(list): - @staticmethod - def find_enabled_volumes() -> [docker.models.volumes.Volume]: - """ get volumes to be handled - - list docker volumes labled with "orchestrator.enable: true" - - return list of volumes - """ - client = docker.from_env() - - print(f"docker client: {client}") - - volumes = [volume for volume in client.volumes.list() if bool( - volume.attrs.get('Labels').get('orchestrator.enable'))] - - return volumes - - @staticmethod - def find_enabled_container_volumes() -> [docker.models.volumes.Volume]: - return [] - - @classmethod - def discover(cls): - s = DobasStrategy() - for rule in [DobasStrategy.find_enabled_volumes, DobasStrategy.find_enabled_container_volumes]: - s += rule() - return s - - -class DobasRunner(object): - def __init__(self, client: docker.DockerClient, cfg: DobasCfg = DobasCfg()): - self.cfg = cfg - self.client = client - - def run(self, strategy: DobasStrategy = None): - for element in strategy: - if type(element) is docker.models.volumes.Volume: - self.volume_backup_handler(volume=element) - - def prune(self): - ans = subprocess.run(["borg", "prune", "-v", "--list"] + - self.cfg.borg_prune_rules.split(" ") + ["::"]) - - if ans.returncode == 0: - pass - else: - print("oje") - - def volume_backup_handler(self, volume: docker.models.volumes.Volume = None): - if volume: - # risk of name duplicates here - mounted_volume = {f"{volume.id}": { - 'bind': f"/mnt/{volume.id}", 'mode': 'ro'}} - - self.client.containers.run("alpine", command="ls -la /mnt", - volumes=mounted_volume) - - ans = subprocess.run(["borg", "create", "--stats", - "::{now}"] + volume.get('bind')) - if ans.returncode > 0: - print(f"NO BACKUP CREATED. BORG Collective failed to assimilate biological " - "entity. (Exitcode: {creation.returncode})", file=sys.stderr) - else: - print("borg exec done") - - def simple_folder_handler(self, paths: [PurePath] = None): - if len(paths): - pass - - -module_cfg = DobasCfg() - - -def do_backup(): - strategy = DobasStrategy.discover() - runner = DobasRunner(client=docker.from_env(), cfg=module_cfg) - runner.run(strategy) - - -def do_restore(): - print(__name__) - - -if __name__ == "__main__": - do_backup() diff --git a/dobas/helper.py b/dobas/helper.py new file mode 100644 index 0000000000000000000000000000000000000000..0a2b7bb1fee8091d311a285109f796c69f11c057 --- /dev/null +++ b/dobas/helper.py @@ -0,0 +1,5 @@ +def removeprefix(string: str, prefix: str) -> str: + if string.startswith(prefix): + return string[len(prefix):] + else: + return string[:] diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4f01097497bc00791a2099bb4d5fd5890f7ad28e 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -0,0 +1,155 @@ +from typing import Dict, List +from .worker import Worker +import docker +import os +import dataclasses + +from . import logger +from .subject import Subject +from .orchestrator_config import OrchestratorConfig +from .worker_config import WorkerConfig +from .helper import removeprefix + + +class Orchestrator: + """ + """ + + def __init__(self, config: OrchestratorConfig): + self.__config: OrchestratorConfig = config + logger.debug("implement typechecked cfg") + logger.info(f"create orchestrator {config}") + + if(self.__config.docker_host): + self.__docker_client = docker.DockerClient( + base_url=self.__config.docker_host) + else: + self.__docker_client = docker.DockerClient.from_env() + + self._default_worker_configs = self._discoverWorkerConfigsFromEnv() + self._subjects = self._discover() + logger.info(f"created subjects {self._subjects}") + + def _discoverWorkerConfigsFromEnv(self) -> Dict[str, WorkerConfig]: + return self._dockerWorkerConfigsFromDict({removeprefix(string=key, prefix="dobas.worker."): value for key, + value in os.environ.items() if "dobas.worker." in key.lower()}) + + # expects already dobas.worker prefix stripped keys with dots as split symbols and worker_name as prefix + def _dockerWorkerConfigsFromDict(self, inputDict: Dict[str, str]) -> Dict[str, WorkerConfig]: + # collect all different worker names as keys + worker_configs = {key.split(".")[0]: None + for key, value in inputDict.items()} + + # collect worker config based on worker names and create WorkerConfig dataclasses + for worker_id, _ in worker_configs.items(): + # find alls envs for current worker_id and replace all . with _ + configs_for_worker = { + removeprefix(string=str(env), prefix=worker_id+".").replace(".", "_"): value + for env, value in inputDict.items() if (env.startswith(worker_id) and "docker.env." not in env) + } + + docker_envs = {} + # mapping docker.env.=value is managed by WorkerConfig docker_env dict and should not be part of configs_for_worker + for docker_env, value in inputDict.items(): + if str(worker_id)+".docker.env." in docker_env: + docker_env_name = removeprefix( + string=docker_env, prefix=str(worker_id)+".docker.env.").split(".")[0] + docker_envs[docker_env_name] = value + + worker_configs[worker_id] = WorkerConfig( + identifier=worker_id, docker_env=docker_envs, **configs_for_worker) + + return worker_configs + + def _discover(self) -> List[Subject]: + logger.debug("discover subjects") + + # TODO: discover and handle also mounts without volumes + volume_label_filter = [] + if not self.__config.docker_volumes_enabledByDefault: + volume_label_filter.append("dobas.enable=True") + volume_filter = {"label": volume_label_filter} + + volumes = self.__docker_client.volumes.list(filters=volume_filter) + + # remove every explicit disabled volume with "dobas.disable" + for volume in volumes: + if ("dobas.disable" in volume.attrs.get("Labels")): + if (volume.attrs.get("Labels")["dobas.disable"] == 'True'): + volumes.remove(volume) + + # TODO: bundle services and volumes together + # TODO: create individual worker config per subject + subjects = [] + for volume in volumes: + # gather all dobas labels from the volume + volume_dobas_labels = {removeprefix(string=label, prefix="dobas.worker."): + volume.attrs.get("Labels")[label] for label in volume.attrs.get("Labels") + if label.startswith("dobas.worker.")} + + # determine which workers are active on this volume + # collect all different worker names as keys + volume_worker_configs = {key.split(".")[0]: WorkerConfig() + for key, _ in volume_dobas_labels.items()} + + # init with default WorkerConfig if present for the worker name + for volume_worker_name, _ in volume_worker_configs.items(): + if volume_worker_name in self._default_worker_configs.keys(): + volume_worker_configs[volume_worker_name] = WorkerConfig( + **dataclasses.asdict(self._default_worker_configs[volume_worker_name])) + + # worker config with worker name specific volume labels + for volume_worker_name, volume_worker_config in volume_worker_configs.items(): + # init docker.env from default config + default_worker_docker_envs = volume_worker_configs[volume_worker_name].docker_env + volume_worker_docker_envs = default_worker_docker_envs if default_worker_docker_envs else {} + # gather docker.env from volume labels + volume_worker_docker_env_label_prefix = "dobas.worker." + \ + str(volume_worker_name)+".docker.env." + for docker_env_label in [label for label in volume.attrs.get("Labels") if label.startswith(volume_worker_docker_env_label_prefix)]: + volume_worker_docker_envs[removeprefix(string=docker_env_label, prefix=volume_worker_docker_env_label_prefix)] = volume.attrs.get( + "Labels")[docker_env_label] + + worker_config_updates = dataclasses.asdict( + volume_worker_config) + worker_config_updates.update( + {removeprefix(string=key, prefix=str(volume_worker_name)+".").replace(".", "_"): value for key, + value in volume_dobas_labels.items() if key.split(".")[0] == volume_worker_name and "docker.env." not in key} + ) + if("docker_env" in worker_config_updates): + del worker_config_updates["docker_env"] + volume_worker_configs[volume_worker_name] = WorkerConfig( + **worker_config_updates, + docker_env=volume_worker_docker_envs, + ) + + # create one subjects per volume referencing all workers from labels + subjects.append(Subject(worker_configs=[config for _, config in volume_worker_configs.items()], + volume=volume, services=None)) + + return subjects + + def __processSubjects(self): + logger.debug("run process subjects") + for subject in self._subjects: + for worker_config in subject.worker_configs: + worker = Worker(worker_config=worker_config, + client=self.__docker_client, + volume=subject.volume) + worker.process() + + def run(self): + logger.debug("run orchestrator") + if(self.__config.daemon): + logger.error("daemon not implement yet.") + # self.__start_daemon() + else: + self.__processSubjects() + + # def do_backup(self, strategy: Strategy, *args, **kwargs): + # logger.info(__name__) + # # worker = Worker(client=docker.from_env(), cfg=dobas.cfg) + # # worker.process(strategy) + + # def do_restore(self): + # print(__name__) diff --git a/dobas/orchestrator_config.py b/dobas/orchestrator_config.py new file mode 100644 index 0000000000000000000000000000000000000000..7e1e4281c4f2ff2ddf463a79a0887c22acb7398b --- /dev/null +++ b/dobas/orchestrator_config.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +import os +from .worker_config import WorkerConfig + + +@dataclass(frozen=True) +class OrchestratorConfig: + """ static configuration generated at startup + + There are some szenarios, where the orchestrator is not able to query its own config from the references docker-host. This is the case for the test setup, where the orchestrator lives besides the dind, where the orchestrators docker-client is connected to. So the docker-client is not able to see it's own orchestrator container. + """ + docker: bool = True + docker_host: str = os.environ['DOCKER_HOST'] + docker_volumes_enabledByDefault: bool = False + docker_services_enabledByDefault: bool = False + + daemon: bool = True + mode_backup: bool = False + mode_restore: bool = False diff --git a/dobas/subject.py b/dobas/subject.py new file mode 100644 index 0000000000000000000000000000000000000000..9054e4abaa0d68f8483581ef258aeadbe767fde6 --- /dev/null +++ b/dobas/subject.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass +from typing import List +from .worker_config import WorkerConfig +import docker + + +@dataclass +class Subject: + worker_configs: List[WorkerConfig] = None + volume: docker.models.volumes.Volume = None + # TODO: implement services / docker-compose folder support in Subject + services: None = None diff --git a/dobas/test/docker-compose-disable-named-volume-disabled-label.yml b/dobas/test/docker-compose-disable-named-volume-disabled-label.yml new file mode 100644 index 0000000000000000000000000000000000000000..499a890809f3ceddcd0d7b9e23dc3afa7a3550d4 --- /dev/null +++ b/dobas/test/docker-compose-disable-named-volume-disabled-label.yml @@ -0,0 +1,6 @@ +version: '3' + +volumes: + named-volume-disabled: + labels: + dobas.disable: true diff --git a/dobas/test/docker-compose-enable-named-volume-bind-label.yml b/dobas/test/docker-compose-enable-named-volume-bind-label.yml new file mode 100644 index 0000000000000000000000000000000000000000..600a689c3aa43ac65b890f985b86234a7b117436 --- /dev/null +++ b/dobas/test/docker-compose-enable-named-volume-bind-label.yml @@ -0,0 +1,6 @@ +version: '3' + +volumes: + named-volume-bind: + labels: + dobas.enable: true diff --git a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml new file mode 100644 index 0000000000000000000000000000000000000000..14c5ab1ec7d3924ce04f671b4f0fc952940e682e --- /dev/null +++ b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml @@ -0,0 +1,12 @@ +version: '3' + +volumes: + named-volume: + labels: + dobas.enable: true + dobas.worker.TestWorker1.docker.env.EnvLabelKey: EnvLabelValue + dobas.worker.TestWorker1.docker.image: alpine:latest + dobas.worker.TestWorker1.docker.command: /bin/sh -c "env | grep EnvLabelKey" + dobas.worker.TestWorker2.docker.env.EnvLabelKeyOverridden: EnvLabelValueOverridden + dobas.worker.TestWorker2.docker.image: alpine:edge-overridden + dobas.worker.TestWorker2.docker.command: /bin/sh -c "env | grep EnvLabelKeyOverridden" \ No newline at end of file diff --git a/dobas/test/docker-compose-enable-named-volume-label.yml b/dobas/test/docker-compose-enable-named-volume-label.yml new file mode 100644 index 0000000000000000000000000000000000000000..970e0c7fb3d99c2f0610bab0011b28c0d6b9e75e --- /dev/null +++ b/dobas/test/docker-compose-enable-named-volume-label.yml @@ -0,0 +1,6 @@ +version: '3' + +volumes: + named-volume: + labels: + dobas.enable: true diff --git a/dobas/test/docker-compose-enable-service-label.yml b/dobas/test/docker-compose-enable-service-label.yml new file mode 100644 index 0000000000000000000000000000000000000000..143f778e04151fbf7c05c208f79a34e390e02061 --- /dev/null +++ b/dobas/test/docker-compose-enable-service-label.yml @@ -0,0 +1,6 @@ +version: '3' + +services: + service: + labels: + dobas.enable: true diff --git a/dobas/test/docker-compose-monolythic.yml b/dobas/test/docker-compose-monolythic.yml new file mode 100644 index 0000000000000000000000000000000000000000..b2140b4f359cd8dfa42027bd83dac4d7256e96a9 --- /dev/null +++ b/dobas/test/docker-compose-monolythic.yml @@ -0,0 +1,26 @@ +version: '3' + +services: + service: + image: alpine + labels: + dobas.enable: true + volumes: + - ./:/tmp/src + - named-volume:/tmp/named-volume + - named-volume-bind:/tmp/named-volume-bind + +volumes: + named-volume-disabled: + labels: + dobas.disable: true + named-volume: + labels: + dobas.enable: true + named-volume-bind: + labels: + dobas.enable: true + driver_opts: + type: none + device: / + o: bind diff --git a/dobas/test/docker-compose.yml b/dobas/test/docker-compose-no-labels.yml similarity index 89% rename from dobas/test/docker-compose.yml rename to dobas/test/docker-compose-no-labels.yml index 6eb60264cd29d20470e807cddf19ec63fa7a0626..666dba3a0431acb2763c3fcd4064675eac777f72 100644 --- a/dobas/test/docker-compose.yml +++ b/dobas/test/docker-compose-no-labels.yml @@ -1,7 +1,7 @@ version: '3' services: - nginx: + service: image: alpine volumes: - ./:/tmp/src @@ -9,6 +9,7 @@ services: - named-volume-bind:/tmp/named-volume-bind volumes: + named-volume-disabled: named-volume: named-volume-bind: driver_opts: diff --git a/dobas/test/test_content_provider.py b/dobas/test/test_content_provider.py index d1e008bc583c186198a06d49922cd07fcbb846a4..e214e6cb2c6e8b0a18f8c2f767e6b0a4f59deaa3 100644 --- a/dobas/test/test_content_provider.py +++ b/dobas/test/test_content_provider.py @@ -1,32 +1,23 @@ -import docker import os from unittest import TestCase -import subprocess +import docker + +from test_helper import compose_up, compose_down class TestContentProvider(TestCase): + def setUp(self): - self.docker_client = docker.from_env() - # create example set of service volume combinations - process = subprocess.Popen(args=['docker-compose', '-f', 'dobas/test/docker-compose.yml', 'up', '-d'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - self.assertEqual(process.returncode, 0, - msg="process failed with stdout {} and stderr {}".format(stdout, stderr)) + compose_up(["docker-compose-monolythic.yml"]) def tearDown(self): - process = subprocess.Popen(args=['docker-compose', '-f', 'dobas/test/docker-compose.yml', 'down', '--volumes'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - self.assertEqual(process.returncode, 0, - msg="process failed with stdout {} and stderr {}".format(stdout, stderr)) + compose_down(["docker-compose-monolythic.yml"]) def test_startContainerWithExternalVolumesToBackup(self): + docker_client = docker.from_env() # find volumes to backup volumes_to_backup = [] - for container in self.docker_client.containers.list(all=True): + for container in docker_client.containers.list(all=True): volumes_to_backup.extend( [mount for mount in container.attrs.get('Mounts')]) @@ -51,7 +42,7 @@ class TestContentProvider(TestCase): # The key is either the host path or a volume name, and the value is a dictionary with the keys: # - bind The path to mount the volume inside the container # - mode Either rw to mount the volume read/write, or ro to mount it read-only. - backup_folder_listing = self.docker_client.containers.run( + backup_folder_listing = docker_client.containers.run( image='alpine', command='bin/sh -c "ls -1 /backup/"', remove=True, diff --git a/dobas/test/test_dummy.py b/dobas/test/test_dummy.py deleted file mode 100644 index 63cca4eb25e6c5cbf234eeea4d35521678cbb2e2..0000000000000000000000000000000000000000 --- a/dobas/test/test_dummy.py +++ /dev/null @@ -1,20 +0,0 @@ -import unittest - - -class TestStringMethods(unittest.TestCase): - @classmethod - def setUpClass(cls): - pass - - @classmethod - def tearDownClass(cls): - pass - - def setUp(self): - self.setUp_passed = True - - def tearDown(self): - self.setUp_passed = True - - def test_dummy(self): - self.assertTrue(True) diff --git a/dobas/test/test_helper.py b/dobas/test/test_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..8445031a5e8a16b6d5f2bc8d609e9c3b06430951 --- /dev/null +++ b/dobas/test/test_helper.py @@ -0,0 +1,75 @@ +import subprocess +import os +from typing import List + +dobas_env_worker_config_dict_testworker1 = { + "dobas.worker.TestWorker1.scheduling": "0 3 * * *1", + "dobas.worker.TestWorker1.storage.destination": "/destination1", + "dobas.worker.TestWorker1.storage.ssh.knownHost": "knownHost1", + "dobas.worker.TestWorker1.storage.ssh.username": "username1", + "dobas.worker.TestWorker1.storage.ssh.sshKeypairPath": "sshKeyPairPath1", + "dobas.worker.TestWorker1.docker.image": "alpine:latest1", + "dobas.worker.TestWorker1.docker.entrypoint": "entrypoint1", + "dobas.worker.TestWorker1.docker.command": "/bin/sh -c \"echo hello world\"1", + "dobas.worker.TestWorker1.docker.workdir": "/root1", + "dobas.worker.TestWorker1.docker.sourceMountBasePath": "/backup1", + "dobas.worker.TestWorker1.docker.env.EnvLabelKey1": "EnvLabelValue11", + "dobas.worker.TestWorker1.docker.env.EnvLabelKey2": "EnvLabelValue21", + "dobas.worker.TestWorker1.docker.env.EnvLabelKey3": "EnvLabelValue31", +} + +dobas_env_worker_config_dict_testworker2 = { + "dobas.worker.TestWorker2.scheduling": "0 3 * * *2", + "dobas.worker.TestWorker2.storage.destination": "/destination2", + "dobas.worker.TestWorker2.storage.ssh.knownHost": "knownHost2", + "dobas.worker.TestWorker2.storage.ssh.username": "username2", + "dobas.worker.TestWorker2.storage.ssh.sshKeypairPath": "sshKeyPairPath2", + "dobas.worker.TestWorker2.docker.image": "alpine:latest2", + "dobas.worker.TestWorker2.docker.entrypoint": "entrypoint2", + "dobas.worker.TestWorker2.docker.command": "/bin/sh -c \"echo hello world\"2", + "dobas.worker.TestWorker2.docker.workdir": "/root2", + "dobas.worker.TestWorker2.docker.sourceMountBasePath": "/backup2", + "dobas.worker.TestWorker2.docker.env.EnvLabelKey1": "EnvLabelValue12", + "dobas.worker.TestWorker2.docker.env.EnvLabelKey2": "EnvLabelValue22", + "dobas.worker.TestWorker2.docker.env.EnvLabelKey3": "EnvLabelValue32", +} + + +def _run_cli_command(args: List[str]): + print(args) + process = subprocess.Popen(args=args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + process.communicate() + # assert(process.returncode == 0) + + +def compose_up(docker_compose_file_names: List[str]): + file_list_packed = [['-f', os.path.join('dobas/test/', name)] + for name in docker_compose_file_names] + file_args = [] + for file in file_list_packed: + for parameter in file: + file_args.append(parameter) + + _run_cli_command(['docker-compose', + *file_args, + 'up', '-d']) + + +def compose_down(docker_compose_file_names: List[str]): + _run_cli_command(['docker-compose', '-f', + *[os.path.join('dobas/test/', name) + for name in docker_compose_file_names], + 'down', '--volumes']) + + +def docker_container_and_volume_prune(): + _run_cli_command(['/bin/bash', '-c', 'docker rm -f $(docker ps -a -q)']) + _run_cli_command( + ['/bin/bash', '-c', 'docker volume rm $(docker volume ls -q)']) + + +def docker_system_prune(): + _run_cli_command(['docker', 'system', 'prune', + '--all', '--volumes', '--force']) diff --git a/dobas/test/test_integration.py b/dobas/test/test_integration.py new file mode 100644 index 0000000000000000000000000000000000000000..23654ae63d4752704e7b1ea1cadd264c9b9e4979 --- /dev/null +++ b/dobas/test/test_integration.py @@ -0,0 +1,31 @@ + +from unittest import TestCase + +from dobas.orchestrator import Orchestrator +from dobas.orchestrator_config import OrchestratorConfig +from dobas.worker import Worker +from dobas.worker_config import WorkerConfig +from test_helper import compose_up, docker_container_and_volume_prune + + +# class TestOrchestratorWorkerIntegration(TestCase): + +# def tearDown(self): +# docker_container_and_volume_prune() + +# def test_docker_env(self): +# compose_up(["docker-compose-no-labels.yml", +# "docker-compose-enable-named-volume-label-docker-env.yml"]) + +# def __testSubjects(orchestrator_config): +# orchestrator = Orchestrator(orchestrator_config) +# orchestrator.run() + +# __testSubjects(OrchestratorConfig( +# docker_volumes_enabledByDefault=False, +# daemon=False, +# )) +# __testSubjects(OrchestratorConfig( +# docker_volumes_enabledByDefault=True, +# daemon=False, +# )) diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py new file mode 100644 index 0000000000000000000000000000000000000000..4e0e739e9b18640d3f3bdd50a206bd0d97ed2c7d --- /dev/null +++ b/dobas/test/test_orchestrator.py @@ -0,0 +1,211 @@ +import os +from unittest import TestCase, mock + +from test_helper import compose_up, docker_container_and_volume_prune, dobas_env_worker_config_dict_testworker1, dobas_env_worker_config_dict_testworker2 +from dobas.orchestrator import Orchestrator +from dobas.orchestrator_config import OrchestratorConfig +from dobas.worker_config import WorkerConfig + + +class TestOrchestrator(TestCase): + + def tearDown(self): + docker_container_and_volume_prune() + + def test_discovery_default_behavior_without_labels(self): + compose_up(["docker-compose-no-labels.yml"]) + self.assertEqual(len(Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=False, + ))._discover()), 0) + + subjects = Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=True, + ))._discover() + self.assertEqual(len(subjects), 3) + + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume-disabled", volume_ids) + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) + + def test_discovery_with_labels(self): + compose_up(["docker-compose-no-labels.yml", + "docker-compose-disable-named-volume-disabled-label.yml"]) + self.assertEqual(len(Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=False, + ))._discover()), 0) + + subjects = Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=True, + ))._discover() + self.assertEqual(len(subjects), 2) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) + docker_container_and_volume_prune() + + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label.yml", + "docker-compose-disable-named-volume-disabled-label.yml"]) + subjects = Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=False, + ))._discover() + self.assertEqual(len(subjects), 1) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + + subjects = Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=True, + ))._discover() + self.assertEqual(len(subjects), 2) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) + docker_container_and_volume_prune() + + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label.yml", + "docker-compose-enable-named-volume-bind-label.yml", + "docker-compose-disable-named-volume-disabled-label.yml"]) + subjects = Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=False, + ))._discover() + self.assertEqual(len(subjects), 2) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) + + subjects = Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=True, + ))._discover() + self.assertEqual(len(subjects), 2) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) + docker_container_and_volume_prune() + + @mock.patch.dict(os.environ, {**dobas_env_worker_config_dict_testworker1, **dobas_env_worker_config_dict_testworker2}) + def test_env_named_worker_defaults(self): + + print(os.environ) + orchestrator = Orchestrator(OrchestratorConfig()) + self.assertDictEqual({ + "TestWorker1": WorkerConfig( + identifier="TestWorker1", + scheduling="0 3 * * *1", + storage_destination="/destination1", + storage_ssh_knownHost="knownHost1", + storage_ssh_username="username1", + storage_ssh_sshKeypairPath="sshKeyPairPath1", + docker_image="alpine:latest1", + docker_entrypoint="entrypoint1", + docker_command=r'/bin/sh -c "echo hello world"1', + docker_workdir="/root1", + docker_sourceMountBasePath="/backup1", + docker_env={ + "EnvLabelKey1": "EnvLabelValue11", + "EnvLabelKey2": "EnvLabelValue21", + "EnvLabelKey3": "EnvLabelValue31", + } + ), + "TestWorker2": WorkerConfig( + identifier="TestWorker2", + scheduling="0 3 * * *2", + storage_destination="/destination2", + storage_ssh_knownHost="knownHost2", + storage_ssh_username="username2", + storage_ssh_sshKeypairPath="sshKeyPairPath2", + docker_image="alpine:latest2", + docker_entrypoint="entrypoint2", + docker_command=r'/bin/sh -c "echo hello world"2', + docker_workdir="/root2", + docker_sourceMountBasePath="/backup2", + docker_env={ + "EnvLabelKey1": "EnvLabelValue12", + "EnvLabelKey2": "EnvLabelValue22", + "EnvLabelKey3": "EnvLabelValue32", + } + ) + }, orchestrator._default_worker_configs) + + @ mock.patch.dict(os.environ, {**dobas_env_worker_config_dict_testworker1, **dobas_env_worker_config_dict_testworker2}) + def test_orchestrator_defaults_partial_overrides_by_volume_labels(self): + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label-docker-env.yml"]) + orchestrator = Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=False, + )) + subjects = orchestrator._subjects + self.assertEqual(len(subjects), 1) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + workerConfigs = {} + for subject in subjects: + for workerconfig in subject.worker_configs: + workerConfigs[str(workerconfig.identifier)] = workerconfig + + self.assertEqual(WorkerConfig( + identifier="TestWorker1", + scheduling="0 3 * * *1", + storage_destination="/destination1", + storage_ssh_knownHost="knownHost1", + storage_ssh_username="username1", + storage_ssh_sshKeypairPath="sshKeyPairPath1", + docker_image="alpine:latest", + docker_entrypoint="entrypoint1", + docker_command=r'/bin/sh -c "env | grep EnvLabelKey"', + docker_workdir="/root1", + docker_sourceMountBasePath="/backup1", + docker_env={ + "EnvLabelKey1": "EnvLabelValue11", + "EnvLabelKey2": "EnvLabelValue21", + "EnvLabelKey3": "EnvLabelValue31", + "EnvLabelKey": "EnvLabelValue" + } + ), workerConfigs["TestWorker1"]) + + self.assertEqual(WorkerConfig( + identifier="TestWorker2", + scheduling="0 3 * * *2", + storage_destination="/destination2", + storage_ssh_knownHost="knownHost2", + storage_ssh_username="username2", + storage_ssh_sshKeypairPath="sshKeyPairPath2", + docker_image="alpine:edge-overridden", + docker_entrypoint="entrypoint2", + docker_command=r'/bin/sh -c "env | grep EnvLabelKeyOverridden"', + docker_workdir="/root2", + docker_sourceMountBasePath="/backup2", + docker_env={ + "EnvLabelKey1": "EnvLabelValue12", + "EnvLabelKey2": "EnvLabelValue22", + "EnvLabelKey3": "EnvLabelValue32", + "EnvLabelKeyOverridden": "EnvLabelValueOverridden" + } + ), workerConfigs["TestWorker2"]) + + def test_discovery_label_docker_envs(self): + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label-docker-env.yml"]) + + def __testSubjects(orchestrator_config): + subjects = Orchestrator(orchestrator_config)._subjects + + envs_linearized_dict = {} + for subject in subjects: + for worker_config in subject.worker_configs: + if worker_config.docker_env: + envs_linearized_dict.update(worker_config.docker_env) + + self.assertEqual(len(envs_linearized_dict.items()), 2) + self.assertEqual( + envs_linearized_dict["EnvLabelKey"], "EnvLabelValue") + self.assertEqual( + envs_linearized_dict["EnvLabelKeyOverridden"], "EnvLabelValueOverridden") + + __testSubjects(OrchestratorConfig( + docker_volumes_enabledByDefault=False, + )) + __testSubjects(OrchestratorConfig( + docker_volumes_enabledByDefault=True, + )) diff --git a/dobas/test/test_worker.py b/dobas/test/test_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..a57978b34f3f9d9e9ead60ea7b05de4e00f4c58b --- /dev/null +++ b/dobas/test/test_worker.py @@ -0,0 +1,24 @@ +from unittest import TestCase + +from dobas.worker import Worker +from dobas.worker_config import WorkerConfig +from test_helper import docker_container_and_volume_prune +import docker + + +class TestWorker(TestCase): + + def tearDown(self): + docker_container_and_volume_prune() + + def test_docker_env(self): + worker = Worker(worker_config=WorkerConfig( + docker_image="alpine", + docker_command="/bin/sh -c \"env | grep EnvLabelKey\"", + docker_env={"EnvLabelKey": "EnvLabelValue"} + ), + client=docker.from_env(), + volume=None) + + output = worker.process() + self.assertIn("EnvLabelKey=EnvLabelValue", output.decode("ascii")) diff --git a/dobas/worker.py b/dobas/worker.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..6a95e8d71b05f188b95622e9edec1eac4cba5dc1 100644 --- a/dobas/worker.py +++ b/dobas/worker.py @@ -0,0 +1,57 @@ +import os +from typing import List +import docker +import sys +import subprocess + +from . import logger +from .worker_config import WorkerConfig + + +class Worker(object): + def __init__(self, + worker_config: WorkerConfig, + client: docker.DockerClient, + volume: docker.models.volumes.Volume): + self.__docker_client = client + self.__worker_config = worker_config + self.__volume = volume + logger.debug( + f"new worker {self.__worker_config.identifier} using client {self.__docker_client} with config {self.__worker_config} for volume {self.__volume}") + + def process(self): + remapped_volume = {} + if type(self.__volume) is docker.models.volumes.Volume: + remapped_volume[self.__volume.name] = { + "bind": os.path.join(self.__worker_config.docker_sourceMountBasePath.strip('"\''), + str(self.__volume.id)), + "mode": "ro" + } + # self.volume_backup_handler(volume=volume) + else: + logger.warning( + f"no handler for volume {self.__volume}, skipping ...") + # TODO: add functionality to capture subject completely + + # volumes: + # A dictionary to configure volumes mounted inside the container. + # The key is either the host path or a volume name, and the value is a dictionary with the keys: + # - bind The path to mount the volume inside the container + # - mode Either rw to mount the volume read/write, or ro to mount it read-only. + logger.debug( + f"start {self.__worker_config.docker_image} with command {self.__worker_config.docker_command}") + docker_run_output = self.__docker_client.containers.run( + image=self.__worker_config.docker_image, + entrypoint=self.__worker_config.docker_entrypoint, + command=self.__worker_config.docker_command, + working_dir=self.__worker_config.docker_workdir.strip("'\""), + volumes=remapped_volume, + detach=False, + remove=True, + environment=self.__worker_config.docker_env, + ) + + logger.debug( + f"worker container run {docker_run_output.decode('ascii')}") + + return docker_run_output diff --git a/dobas/worker_config.py b/dobas/worker_config.py new file mode 100644 index 0000000000000000000000000000000000000000..3b130acdadfce9f27bef857fdf7eacfe5ef5f68e --- /dev/null +++ b/dobas/worker_config.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass +import os +import docker +from typing import Dict + + +@dataclass +class WorkerConfig: + identifier: str = "" + scheduling: str = "" + + # configured by labels + storage_destination: str = "" + storage_ssh_knownHost: str = "" + storage_ssh_username: str = "" + storage_ssh_sshKeypairPath: str = "" + + docker_image: str = "" + docker_entrypoint: str = "" + docker_command: str = "" + docker_workdir: str = "" + docker_sourceMountBasePath: str = "" + docker_env: Dict[str, str] = None diff --git a/main.py b/main.py new file mode 100755 index 0000000000000000000000000000000000000000..ea48ebaf9936f50a396fe81f5fcf469ac9af6b31 --- /dev/null +++ b/main.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Docker Backup Service +Usage: + main.py [-v ...] [options] + main.py (-h | --help) + main.py --version + +Options: + -c , --config Borg configuration file [default: x] + -d, --dry Make dry run. Don't change anything. + -v, --verbose Increase verbosity + -h, --help Show this screen. + --version Show version. + + --orchestrator.docker= [default: True] + --orchestrator.docker.host= URL to docker daemon hosting the services and volumes to backup. + If orchestrator.docker.host is empty, we use the content of DOCKER_HOST env variable. [default: ] + --orchestrator.docker.volumes.enabledByDefault= [default: False] + --orchestrator.docker.services.enabledByDefault= [default: False] + --orchestrator.daemon= [default: True] + --orchestrator.mode.restore= [default: False] + --orchestrator.mode.backup= [default: True] +""" + +from docopt import docopt + +from dobas.orchestrator import Orchestrator +from dobas import logger +from logging import DEBUG, INFO, WARN +from dobas.orchestrator_config import OrchestratorConfig +from dobas.worker_config import WorkerConfig +from dobas.helper import removeprefix + + +def __dict_string_value_to_bool(dict_to_update): + for key, value in dict_to_update.items(): + if value == "True" or value == "true": + dict_to_update[key] = True + elif value == "False" or value == "false": + dict_to_update[key] = False + return dict_to_update + + +if __name__ == '__main__': + arguments = docopt(__doc__) + + # default_worker_config = WorkerConfig(**__dict_string_value_to_bool({ + # removeprefix(string=key, prefix="--worker.").replace(".", "_"): value for key, value in arguments.items() if + # key.startswith("--worker.") + # })) + + orchestrator_config = OrchestratorConfig( + **__dict_string_value_to_bool({ + removeprefix(string=key, prefix="--orchestrator.").replace(".", "_"): value for key, value in arguments.items() if + key.startswith("--orchestrator.") + })) + + if arguments['--verbose'] == 0: + logger.setLevel(WARN) + elif arguments['--verbose'] == 1: + logger.setLevel(INFO) + elif arguments['--verbose'] >= 2: + logger.setLevel(DEBUG) + + logger.debug(arguments) + logger.info(orchestrator_config) + + # --> we need to discover the orchestrator config without usage of docker here, + # since we do not garantie to run the orchestrator daemon on the same dockerhost, where the global docker client ist connected to + # (where the volumes and services to backup are living). This is already the case in the tests, since the orchestrator container runs outside of the dind, + # where the services and volumes to backup are living. + # orchestrator behaviour, NOT container specific i.e.: + # - dobas.enable.default = (true | false) + # - dobas.worker.scheduling = "cron string" + # ... + # - dobas.worker.storage.host = xxx + # - default.values (pre.backup.action, post.backup.action) + + orchestrator = Orchestrator(config=orchestrator_config) + orchestrator.run() # TODO: check python http server (must be stoppable)