From 515bf01a057f0b40d89c6b7b247eb4e2fc19d1b7 Mon Sep 17 00:00:00 2001 From: David Timber Date: Fri, 13 May 2022 14:45:59 +0800 Subject: Impl ... - launch.json: change debug cwd to the project root dir - Add subcmd "mods" - Docs - Tidy up sample and debug config files - Change core exec - 'dnf-list-instaled' -> 'rpm-list-installed' as dnf does not work on ro fs - Accept the exit code 1 from tar(allow live fs change) - Add the generic sample config - Fix 'run' subcmd not accepting empty task-id - Change module loading: modules are not required to have the 'backup_backends' var - Reduce required Python version by removing the use of match ... case - Fix 'exec-append' not taking 'env' into account - Remove use of exceptions from irrelevant packages - Fix unimpl methods of NullBackupBackend - Tidy up instantiation of raised exceptions - Change "include" behaviour - Relative config paths are now resolved like #include C preprocessor - Fix bug where "include" circular ref checked is not done with absolute paths of config files - Add own exception hierachy - aws-s3: change storage class only when "rot-storage-class" is different from "sink-storage-class" --- src/conf/py-debug/aws.jsonc | 160 +++++++++++++++++++++++++++++++++ src/conf/py-debug/aws.sample.jsonc | 160 --------------------------------- src/conf/py-debug/conf.d | 1 + src/conf/py-debug/conf.d/core.jsonc | 44 --------- src/conf/py-debug/localfs.jsonc | 157 ++++++++++++++++++++++++++++++++ src/conf/py-debug/localfs.sample.jsonc | 157 -------------------------------- src/conf/py-debug/null.jsonc | 141 +++++++++++++++++++++++++++++ src/conf/py-debug/null.sample.jsonc | 140 ----------------------------- src/conf/py-debug/palhm.jsonc | 1 + src/conf/py-sample/conf.d/core.json | 41 +++++++++ src/conf/py-sample/sample.jsonc | 127 ++++++++++++++++++++++++++ src/palhm.py | 74 ++++++++++++--- src/palhm/__init__.py | 120 +++++++++++++++---------- src/palhm/exceptions.py | 2 + src/palhm/mod/aws.py | 17 ++-- 15 files changed, 778 insertions(+), 564 deletions(-) create mode 100644 src/conf/py-debug/aws.jsonc delete mode 100644 src/conf/py-debug/aws.sample.jsonc create mode 120000 src/conf/py-debug/conf.d delete mode 100644 src/conf/py-debug/conf.d/core.jsonc create mode 100644 src/conf/py-debug/localfs.jsonc delete mode 100644 src/conf/py-debug/localfs.sample.jsonc create mode 100644 src/conf/py-debug/null.jsonc delete mode 100644 src/conf/py-debug/null.sample.jsonc create mode 120000 src/conf/py-debug/palhm.jsonc create mode 100644 src/conf/py-sample/conf.d/core.json create mode 100644 src/conf/py-sample/sample.jsonc create mode 100644 src/palhm/exceptions.py (limited to 'src') diff --git a/src/conf/py-debug/aws.jsonc b/src/conf/py-debug/aws.jsonc new file mode 100644 index 0000000..df9a63a --- /dev/null +++ b/src/conf/py-debug/aws.jsonc @@ -0,0 +1,160 @@ +// PALHM Instance Config +{ + "include": [ "conf.d/core.json" ], + "modules": [ "aws" ], + "nb-workers": 0, // assumed $(nproc) - default + // "nb-workers": 1, // to disable concurrent task despatch + // To unlimit the number of workers. + // Does not fail on resource alloc failure. + // "nb-workers": -1, + "vl": 4, + "tasks": [ + { + "id": "backup", + "type": "backup", + "backend": "aws-s3", + "backend-param": { + // "profile": "default", + "bucket": "palhm.test", + "root": "/palhm/backup", + "prefix": { + "type": "default" + // "type": "iso8601", + // "timespec": "seconds", + // "tz": "utc" + }, + // "sink-storage-class": "STANDARD_IA", + // "rot-storage-class": "ONEZONE_IA", + "nb-copy-limit": 2, // or Infinity assumed(not in JSON spec) + "root-size-limit": "Infinity" // or Infinity assumed + }, + "object-groups": [ + { "id": "pre-start" }, + { + "id": "data-dump", + "depends": [ "pre-start" ] + }, + { + "id": "tar-media-0", + "depends": [ "data-dump" ] + }, + { + "id": "tar-media-1", + "depends": [ "data-dump" ] + } + ], + "objects": [ + { + "path": "pm-list.gz", + "group": "pre-start", + "pipeline": [ + { "type": "exec", "exec-id": "rpm-list-installed" }, + { "type": "exec", "exec-id": "filter-gzip-plain" } + ] + }, + { + "path": "lsblk.json.gz", + "group": "pre-start", + "pipeline": [ + { + "type": "exec-append", + "exec-id": "lsblk-all-json", + "argv": [ "-a" ] + }, + { "type": "exec", "exec-id": "filter-gzip-plain" } + ] + }, + { + "path": "random-dump.sql.xz", + "group": "data-dump", + "pipeline": [ + { + "type": "exec-inline", + "argv": [ + "/bin/dd", + "if=/dev/urandom", + "bs=4096", + "count=512", + "status=none" + ] + }, + { "type": "exec", "exec-id": "filter-xz-parallel" } + ] + }, + { + "path": "random-dump.0.xz", + "group": "tar-media-0", + "pipeline": [ + { + "type": "exec-inline", + "argv": [ + "/bin/dd", + "if=/dev/zero", + "bs=4096", + "count=512", + "status=none" + ] + }, + { "type": "exec", "exec-id": "filter-xz-parallel" } + ] + }, + { + "path": "random-dump.1.xz", + "group": "tar-media-1", + "pipeline": [ + { + "type": "exec-inline", + "argv": [ + "/bin/dd", + "if=/dev/zero", + "bs=4096", + "count=512", + "status=none" + ] + }, + { "type": "exec", "exec-id": "filter-xz-parallel" } + ] + } + ] + }, + { + "id": "update", + "type": "routine", + "routine": [ + { + "type": "exec-inline", + "argv": [ "/bin/echo", "0" ] + }, + { + "type": "exec-inline", + "argv": [ "/bin/sleep", "1" ] + }, + { + "type": "exec-inline", + "argv": [ "/bin/echo", "1" ] + } + ] + }, + { + "id": "default", + "type": "routine", + "routine": [ + { "type": "task", "task-id": "backup" }, + { "type": "task", "task-id": "update" }, + { + // Block SIGTERM from systemd/init.d so the program is not + // affected by the reboot command. + "type": "builtin", + "builtin-id": "sigmask", + "param": [ + { "action": "block", "sig": [ "TERM" ] } + ] + }, + { + "type": "exec-inline", + "argv": [ "/bin/true" ] + } + ] + } + ] +} diff --git a/src/conf/py-debug/aws.sample.jsonc b/src/conf/py-debug/aws.sample.jsonc deleted file mode 100644 index 46ad562..0000000 --- a/src/conf/py-debug/aws.sample.jsonc +++ /dev/null @@ -1,160 +0,0 @@ -// PALHM Instance Config -{ - "include": [ "conf/py-debug/conf.d/core.jsonc" ], - "modules": [ "aws" ], - "nb-workers": 0, // assumed $(nproc) - default - // "nb-workers": 1, // to disable concurrent task despatch - // To unlimit the number of workers. - // Does not fail on resource alloc failure. - // "nb-workers": -1, - "vl": 4, - "tasks": [ - { - "id": "backup", - "type": "backup", - "backend": "aws-s3", - "backend-param": { - // "profile": "default", - "bucket": "palhm.test", - "root": "/palhm/backup", - "prefix": { - "type": "default" - // "type": "iso8601", - // "timespec": "seconds", - // "tz": "utc" - }, - // "sink-storage-class": "STANDARD_IA", - // "rot-storage-class": "ONEZONE_IA", - "nb-copy-limit": 2, // or Infinity assumed(not in JSON spec) - "root-size-limit": "Infinity" // or Infinity assumed - }, - "object-groups": [ - { "id": "pre-start" }, - { - "id": "data-dump", - "depends": [ "pre-start" ] - }, - { - "id": "tar-media-0", - "depends": [ "data-dump" ] - }, - { - "id": "tar-media-1", - "depends": [ "data-dump" ] - } - ], - "objects": [ - { - "path": "pm-list.gz", - "group": "pre-start", - "pipeline": [ - { "type": "exec", "exec-id": "dnf-list-installed" }, - { "type": "exec", "exec-id": "filter-gzip-plain" } - ] - }, - { - "path": "lsblk.json.gz", - "group": "pre-start", - "pipeline": [ - { - "type": "exec-append", - "exec-id": "lsblk-all-json", - "argv": [ "-a" ] - }, - { "type": "exec", "exec-id": "filter-gzip-plain" } - ] - }, - { - "path": "random-dump.sql.xz", - "group": "data-dump", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/urandom", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - }, - { - "path": "random-dump.0.xz", - "group": "tar-media-0", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/zero", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - }, - { - "path": "random-dump.1.xz", - "group": "tar-media-1", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/zero", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - } - ] - }, - { - "id": "update", - "type": "routine", - "routine": [ - { - "type": "exec-inline", - "argv": [ "/bin/echo", "0" ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/sleep", "1" ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/echo", "1" ] - } - ] - }, - { - "id": "default", - "type": "routine", - "routine": [ - { "type": "task", "task-id": "backup" }, - { "type": "task", "task-id": "update" }, - { - // Block SIGTERM from systemd/init.d so the program is not - // affected by the reboot command. - "type": "builtin", - "builtin-id": "sigmask", - "param": [ - { "action": "block", "sig": [ "TERM" ] } - ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/true" ] - } - ] - } - ] -} diff --git a/src/conf/py-debug/conf.d b/src/conf/py-debug/conf.d new file mode 120000 index 0000000..a32163d --- /dev/null +++ b/src/conf/py-debug/conf.d @@ -0,0 +1 @@ +../py-sample/conf.d \ No newline at end of file diff --git a/src/conf/py-debug/conf.d/core.jsonc b/src/conf/py-debug/conf.d/core.jsonc deleted file mode 100644 index 4afe7f5..0000000 --- a/src/conf/py-debug/conf.d/core.jsonc +++ /dev/null @@ -1,44 +0,0 @@ -// PALHM Core Config -{ - "execs": [ - // { - // "id": "Exec ID", - // "argv": [ "cmd", "--option1=opt1_val", "-o", "opt2_val" ], - // "env": { "NAME": "VAL" }, - // "ec": "0", // this is assumed - // "ec": "0-127", // inclusive range (not terminated by a signal) - // "ec": "<1", // range (only 0) - // "ec": "<=1", // range (0 and 1) - // "ec": ">0", // range (always fail) - // "ec": ">=0", // range (only 0) - // "vl-stderr": 1 // verbosity level of stderr produced by this process - // verbosity level of stderr produced by this process. Ignored if used - // as part of pipeline - // "vl-stdout": 2 - // }, - { - "id": "tar", - "argv": [ "/usr/bin/tar", "--xattrs", "--selinux" ] - }, - { - "id": "filter-xz-parallel", - "argv": [ "/usr/bin/xz", "-T0" ] - }, - { - "id": "filter-gzip-plain", - "argv": [ "/usr/bin/gzip" ] - }, - { - "id": "filter-zstd-plain", - "argv": [ "/usr/bin/zstd" ] - }, - { - "id": "dnf-list-installed", - "argv": [ "/usr/bin/dnf", "-yq", "list", "installed" ] - }, - { - "id": "lsblk-all-json", - "argv": [ "/usr/bin/lsblk", "-JbOa" ] - } - ] -} diff --git a/src/conf/py-debug/localfs.jsonc b/src/conf/py-debug/localfs.jsonc new file mode 100644 index 0000000..a33060d --- /dev/null +++ b/src/conf/py-debug/localfs.jsonc @@ -0,0 +1,157 @@ +// PALHM Instance Config +{ + "include": [ "conf.d/core.json" ], + "nb-workers": 0, // assumed $(nproc) - default + // "nb-workers": 1, // to disable concurrent task despatch + // To unlimit the number of workers. + // Does not fail on resource alloc failure. + // "nb-workers": -1, + "vl": 3, + "tasks": [ + { + "id": "backup", + "type": "backup", + "backend": "localfs", + "backend-param": { + "root": "/var/tmp/palhm-backup-root", + "prefix": { + "type": "default" + // "type": "iso8601", + // "timespec": "seconds", + // "tz": "utc" + }, + // "dmode": "755", + // "fmode": "644", + "nb-copy-limit": 2, + "root-size-limit": "Infinity" + }, + "object-groups": [ + { "id": "pre-start" }, + { + "id": "data-dump", + "depends": [ "pre-start" ] + }, + { + "id": "tar-media-0", + "depends": [ "data-dump" ] + }, + { + "id": "tar-media-1", + "depends": [ "data-dump" ] + } + ], + "objects": [ + { + "path": "pm-list.gz", + "group": "pre-start", + "pipeline": [ + { "type": "exec", "exec-id": "rpm-list-installed" }, + { "type": "exec", "exec-id": "filter-gzip-plain" } + ] + }, + { + "path": "lsblk.json.gz", + "group": "pre-start", + "pipeline": [ + { + "type": "exec-append", + "exec-id": "lsblk-all-json", + "argv": [ "-a" ] + }, + { "type": "exec", "exec-id": "filter-gzip-plain" } + ] + }, + { + "path": "random-dump.sql.xz", + "group": "data-dump", + "pipeline": [ + { + "type": "exec-inline", + "argv": [ + "/bin/dd", + "if=/dev/urandom", + "bs=4096", + "count=512", + "status=none" + ] + }, + { "type": "exec", "exec-id": "filter-xz-parallel" } + ] + }, + { + "path": "random-dump.0.xz", + "group": "tar-media-0", + "pipeline": [ + { + "type": "exec-inline", + "argv": [ + "/bin/dd", + "if=/dev/zero", + "bs=4096", + "count=512", + "status=none" + ] + }, + { "type": "exec", "exec-id": "filter-xz-parallel" } + ] + }, + { + "path": "random-dump.1.xz", + "group": "tar-media-1", + "pipeline": [ + { + "type": "exec-inline", + "argv": [ + "/bin/dd", + "if=/dev/zero", + "bs=4096", + "count=512", + "status=none" + ] + }, + { "type": "exec", "exec-id": "filter-xz-parallel" } + ] + } + ] + }, + { + "id": "update", + "type": "routine", + "routine": [ + { + "type": "exec-inline", + "argv": [ "/bin/echo", "0" ] + }, + { + "type": "exec-inline", + "argv": [ "/bin/sleep", "1" ] + }, + { + "type": "exec-inline", + "argv": [ "/bin/echo", "1" ] + } + ] + }, + { + "id": "default", + "type": "routine", + "routine": [ + { "type": "task", "task-id": "backup" }, + { "type": "task", "task-id": "update" }, + { + // Block SIGTERM from systemd/init.d so the program is not + // affected by the reboot command. + "type": "builtin", + "builtin-id": "sigmask", + "param": [ + { "action": "block", "sig": [ "TERM" ] } + ] + }, + { + "type": "exec-inline", + "argv": [ "/bin/true" ] + } + ] + } + ] +} diff --git a/src/conf/py-debug/localfs.sample.jsonc b/src/conf/py-debug/localfs.sample.jsonc deleted file mode 100644 index ec12808..0000000 --- a/src/conf/py-debug/localfs.sample.jsonc +++ /dev/null @@ -1,157 +0,0 @@ -// PALHM Instance Config -{ - "include": [ "conf/py-debug/conf.d/core.jsonc" ], - "nb-workers": 0, // assumed $(nproc) - default - // "nb-workers": 1, // to disable concurrent task despatch - // To unlimit the number of workers. - // Does not fail on resource alloc failure. - // "nb-workers": -1, - "vl": 4, - "tasks": [ - { - "id": "backup", - "type": "backup", - "backend": "localfs", - "backend-param": { - "root": "/var/tmp/palhm-backup-root", - "prefix": { - "type": "default" - // "type": "iso8601", - // "timespec": "seconds", - // "tz": "utc" - }, - // "dmode": "755", - // "fmode": "644", - "nb-copy-limit": 2, - "root-size-limit": "Infinity" - }, - "object-groups": [ - { "id": "pre-start" }, - { - "id": "data-dump", - "depends": [ "pre-start" ] - }, - { - "id": "tar-media-0", - "depends": [ "data-dump" ] - }, - { - "id": "tar-media-1", - "depends": [ "data-dump" ] - } - ], - "objects": [ - { - "path": "pm-list.gz", - "group": "pre-start", - "pipeline": [ - { "type": "exec", "exec-id": "dnf-list-installed" }, - { "type": "exec", "exec-id": "filter-gzip-plain" } - ] - }, - { - "path": "lsblk.json.gz", - "group": "pre-start", - "pipeline": [ - { - "type": "exec-append", - "exec-id": "lsblk-all-json", - "argv": [ "-a" ] - }, - { "type": "exec", "exec-id": "filter-gzip-plain" } - ] - }, - { - "path": "random-dump.sql.xz", - "group": "data-dump", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/urandom", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - }, - { - "path": "random-dump.0.xz", - "group": "tar-media-0", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/zero", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - }, - { - "path": "random-dump.1.xz", - "group": "tar-media-1", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/zero", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - } - ] - }, - { - "id": "update", - "type": "routine", - "routine": [ - { - "type": "exec-inline", - "argv": [ "/bin/echo", "0" ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/sleep", "1" ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/echo", "1" ] - } - ] - }, - { - "id": "default", - "type": "routine", - "routine": [ - { "type": "task", "task-id": "backup" }, - { "type": "task", "task-id": "update" }, - { - // Block SIGTERM from systemd/init.d so the program is not - // affected by the reboot command. - "type": "builtin", - "builtin-id": "sigmask", - "param": [ - { "action": "block", "sig": [ "TERM" ] } - ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/true" ] - } - ] - } - ] -} diff --git a/src/conf/py-debug/null.jsonc b/src/conf/py-debug/null.jsonc new file mode 100644 index 0000000..b5ce9f8 --- /dev/null +++ b/src/conf/py-debug/null.jsonc @@ -0,0 +1,141 @@ +{ + "include": [ "conf.d/core.json" ], + "nb-workers": 0, // assumed $(nproc) + // "nb-workers": 1, // to disable concurrent task despatch + // "nb-workers": -1, // to unlimit the number of workers. + "vl": 3, + "tasks": [ + { + "id": "backup", + "type": "backup", + "backend": "null", + "object-groups": [ + { "id": "pre-start" }, + { + "id": "data-dump", + "depends": [ "pre-start" ] + }, + { + "id": "tar-0", + "depends": [ "data-dump" ] + }, + { + "id": "tar-1", + "depends": [ "data-dump" ] + } + ], + "objects": [ + { + "path": "pm-list.zstd", + "group": "pre-start", + "pipeline": [ + { "type": "exec", "exec-id": "rpm-list-installed" }, + { "type": "exec", "exec-id": "filter-zstd-plain" } + ] + }, + { + "path": "lsblk.json.zstd", + "group": "pre-start", + "pipeline": [ + { "type": "exec", "exec-id": "lsblk-all-json" }, + { "type": "exec", "exec-id": "filter-zstd-plain" } + ] + }, + { + "path": "db.sql.zstd", + "group": "data-dump", + "pipeline": [ + { + "type": "exec-inline", + "argv": [ + "/bin/mysqldump", + "-uroot", + "--all-databases" + ] + }, + { "type": "exec", "exec-id": "filter-zstd-parallel" } + ] + }, + { + "path": "root.tar.zstd", + "group": "tar-0", + "pipeline": [ + { + "type": "exec-append", + "exec-id": "tar", + "argv": [ + "-C", + "/", + "/etc", + "/home", + "/root", + "/var" + ] + }, + { "type": "exec", "exec-id": "filter-zstd-parallel" } + ] + }, + { + "path": "srv.tar.zstd", + "group": "tar-1", + "pipeline": [ + { + "type": "exec-append", + "exec-id": "tar", + "argv": [ + "-C", + "/", + "/srv" + ] + }, + { "type": "exec", "exec-id": "filter-zstd-parallel" } + ] + } + ] + }, + { + "id": "update", + "type": "routine", + "routine": [ + { + "type": "exec-inline", + "argv": [ "/bin/dnf", "--refresh", "-yq", "update" ] + }, + { + "type": "exec-inline", + "argv": [ "/bin/sa-update" ] + } + ] + }, + { + "id": "reboot", + "type": "routine", + "routine": [ + { +/* + * Block SIGTERM from systemd/init.d so PALHM can exit gracefully after issuing + * reboot. + */ + "type": "builtin", + "builtin-id": "sigmask", + "param": [ + { "action": "block", "sig": [ "TERM" ] } + ] + }, + { + "type": "exec-inline", + "argv": [ "/sbin/reboot" ] + } + ] + }, + { + "id": "default", + "type": "routine", + "routine": [ + { "type": "task", "task-id": "backup" }, + { "type": "task", "task-id": "update" }, + { "type": "task", "task-id": "reboot" } + ] + } + ] +} diff --git a/src/conf/py-debug/null.sample.jsonc b/src/conf/py-debug/null.sample.jsonc deleted file mode 100644 index a83de95..0000000 --- a/src/conf/py-debug/null.sample.jsonc +++ /dev/null @@ -1,140 +0,0 @@ -// PALHM Instance Config -{ - "include": [ "conf/py-debug/conf.d/core.jsonc" ], - "nb-workers": 1, - "vl": 3, - "tasks": [ - { - "id": "backup", - "type": "backup", - "backend": "null", - "object-groups": [ - { "id": "pre-start" }, - { - "id": "data-dump", - "depends": [ "pre-start" ] - }, - { - "id": "tar-media-0", - "depends": [ "data-dump" ] - }, - { - "id": "tar-media-1", - "depends": [ "data-dump" ] - } - ], - "objects": [ - { - "path": "pm-list.gz", - "group": "pre-start", - "pipeline": [ - { "type": "exec", "exec-id": "dnf-list-installed" }, - { "type": "exec", "exec-id": "filter-gzip-plain" } - ] - }, - { - "path": "lsblk.json.gz", - "group": "pre-start", - "pipeline": [ - { - "type": "exec-append", - "exec-id": "lsblk-all-json", - "argv": [ "-a" ] - }, - { "type": "exec", "exec-id": "filter-gzip-plain" } - ] - }, - { - "path": "random-dump.sql.xz", - "group": "data-dump", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/urandom", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - }, - { - "path": "random-dump.0.xz", - "group": "tar-media-0", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/zero", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - }, - { - "path": "random-dump.1.xz", - "group": "tar-media-1", - "pipeline": [ - { - "type": "exec-inline", - "argv": [ - "/bin/dd", - "if=/dev/zero", - "bs=4096", - "count=512", - "status=none" - ] - }, - { "type": "exec", "exec-id": "filter-xz-parallel" } - ] - } - ] - }, - { - "id": "update", - "type": "routine", - "routine": [ - { - "type": "exec-inline", - "argv": [ "/bin/echo", "0" ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/sleep", "1" ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/echo", "1" ] - } - ] - }, - { - "id": "default", - "type": "routine", - "routine": [ - { "type": "task", "task-id": "backup" }, - { "type": "task", "task-id": "update" }, - { - // Block SIGTERM from systemd/init.d so the program is not - // affected by the reboot command. - "type": "builtin", - "builtin-id": "sigmask", - "param": [ - { "action": "block", "sig": [ "TERM" ] } - ] - }, - { - "type": "exec-inline", - "argv": [ "/bin/true" ] - } - ] - } - ] -} diff --git a/src/conf/py-debug/palhm.jsonc b/src/conf/py-debug/palhm.jsonc new file mode 120000 index 0000000..fb68baf --- /dev/null +++ b/src/conf/py-debug/palhm.jsonc @@ -0,0 +1 @@ +aws.jsonc \ No newline at end of file diff --git a/src/conf/py-sample/conf.d/core.json b/src/conf/py-sample/conf.d/core.json new file mode 100644 index 0000000..46d3feb --- /dev/null +++ b/src/conf/py-sample/conf.d/core.json @@ -0,0 +1,41 @@ +{ + "execs": [ + { + "id": "tar", + "argv": [ "/bin/tar", "--xattrs", "--selinux", "--warning=none", "-cf", "-" ], + "ec": "<2" + }, + { + "id": "filter-xz-parallel", + "argv": [ "/bin/xz", "-T0" ] + }, + { + "id": "filter-gzip-plain", + "argv": [ "/bin/gzip" ] + }, + { + "id": "filter-zstd-plain", + "argv": [ "/bin/zstd" ] + }, + { + "id": "filter-zstd-parallel", + "argv": [ "/bin/zstd", "-T0" ] + }, + { + "id": "rpm-list-installed", + "argv": [ "/bin/rpm", "-qa" ] + }, + { + "id": "dpkg-list-installed", + "argv": [ "/bin/dpkg-query", "-l" ] + }, + { + "id": "lsblk-all-json", + "argv": [ "/bin/lsblk", "-JbOa" ] + }, + { + "id": "os-release", + "argv": [ "/bin/cat", "/etc/os-release" ] + } + ] +} diff --git a/src/conf/py-sample/sample.jsonc b/src/conf/py-sample/sample.jsonc new file mode 100644 index 0000000..f1c4501 --- /dev/null +++ b/src/conf/py-sample/sample.jsonc @@ -0,0 +1,127 @@ +{ + "include": [ "/etc/palhm/conf.d/core.json" ], + // "modules": [ "aws" ], + "nb-workers": 0, + // "vl": 4, + "tasks": [ + { + "id": "backup", + "type": "backup", + "backend": "null", + "backend-param": {}, + "object-groups": [ + { "id": "meta-run" }, + { + "id": "data-dump", + "depends": [ "meta-run" ] + }, + { + "id": "tar-root", + "depends": [ "data-dump" ] + } + ], + "objects": [ + { + "path": "os-release", + "group": "meta-run", + "pipeline": [ { "type": "exec" , "exec-id": "os-release" } ] + }, + { + "path": "pm-list.zstd", + "group": "meta-run", + "pipeline": [ + { "type": "exec", "exec-id": "rpm-list-installed" }, + { "type": "exec", "exec-id": "filter-zstd-plain" } + ] + }, + { + "path": "lsblk.json.zstd", + "group": "meta-run", + "pipeline": [ + { "type": "exec", "exec-id": "lsblk-all-json" }, + { "type": "exec", "exec-id": "filter-zstd-plain" } + ] + }, + // { + // "path": "db.sql.zstd", + // "group": "data-dump", + // "pipeline": [ + // { + // "type": "exec-inline", + // "argv": [ + // "/bin/mysqldump", + // "-uroot", + // "--all-databases" + // ] + // // "ec": "<=2" // don't fail when the DB is offline + // }, + // { "type": "exec", "exec-id": "filter-zstd-parallel" } + // ] + // }, + { + "path": "root.tar.zstd", + "group": "tar-root", + "pipeline": [ + { + "type": "exec-append", + "exec-id": "tar", + "argv": [ + "-C", + "/", + "etc", + "home", + "root", + "var" + ] + }, + { "type": "exec", "exec-id": "filter-zstd-parallel" } + ] + } + ] + }, + { + "id": "update", + "type": "routine", + "routine": [ + { + "type": "exec-inline", + "argv": [ "/bin/dnf", "--refresh", "-yq", "update" ] + } + // { + // "type": "exec-inline", + // "argv": [ "/bin/sa-update" ] + // } + ] + }, + { + "id": "reboot", + "type": "routine", + "routine": [ + { +/* + * Block SIGTERM from systemd/init.d so PALHM can exit gracefully after issuing + * reboot. + */ + "type": "builtin", + "builtin-id": "sigmask", + "param": [ + { "action": "block", "sig": [ "TERM" ] } + ] + }, + { + "type": "exec-inline", + "argv": [ "/sbin/reboot" ] + } + ] + }, + { + "id": "default", + "type": "routine", + "routine": [ + { "type": "task", "task-id": "backup" }, + { "type": "task", "task-id": "update" }, + { "type": "task", "task-id": "reboot" } + ] + } + ] +} diff --git a/src/palhm.py b/src/palhm.py index f3f412b..722664e 100755 --- a/src/palhm.py +++ b/src/palhm.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import importlib import logging +import os import sys from abc import ABC, abstractmethod from getopt import getopt @@ -49,7 +51,7 @@ class RunCmd (Cmd): def do_cmd (self): ProgConf.alloc_ctx() - if self.args: + if self.args and self.args[0]: # empty string as "default" task = self.args[0] else: task = palhm.DEFAULT.RUN_TASK.value @@ -64,6 +66,53 @@ class RunCmd (Cmd): Run a task in config. Run the "''' + palhm.DEFAULT.RUN_TASK.value + '''" task if [TASK] is not specified.''') +class ModsCmd (Cmd): + def __init__ (self, *args, **kwargs): + pass + + def _walk_mods (self, path: str): + def is_mod_dir (path: str) -> bool: + try: + for i in os.scandir(path): + if i.name.startswith("__init__.py"): + return True + except NotADirectoryError: + pass + return False + + def is_mod_file (path: str) -> str: + if not os.path.isfile(path): + return None + + try: + pos = path.rindex(".") + if path[pos + 1:].startswith("py"): + return os.path.basename(path[:pos]) + except ValueError: + pass + + for i in os.scandir(path): + if i.name.startswith("_"): + continue + elif is_mod_dir(i.path): + print(i.name) + self._walk_mods(i.path) + else: + name = is_mod_file(i.path) + if name: + print(name) + + def do_cmd (self): + for i in importlib.util.find_spec("palhm.mod").submodule_search_locations: + self._walk_mods(i) + + return 0 + + def print_help (): + print( +"Usage: " + sys.argv[0] + " mods" + ''' +Prints the available modules to stdout.''') + class HelpCmd (Cmd): def __init__ (self, optlist, args): self.optlist = optlist @@ -84,7 +133,7 @@ class HelpCmd (Cmd): print( "Usage: " + sys.argv[0] + " [options] CMD [command options ...]" + ''' Options: - -q Set the verbosity level to 0(FATAL error only). Overrides config + -q Set the verbosity level to 0(CRITIAL). Overrides config -v Increase the verbosity level by 1. Overrides config -f FILE Load config from FILE instead of the hard-coded default Config: ''' + ProgConf.conf + ''' @@ -92,14 +141,16 @@ Commands: run run a task config load config and print the contents help [CMD] print this message and exit normally if [CMD] is not specified. - Print usage of [CMD] otherwise''') + Print usage of [CMD] otherwise + mods list available modules''') return 0 CmdMap = { "config": ConfigCmd, "run": RunCmd, - "help": HelpCmd + "help": HelpCmd, + "mods": ModsCmd } optlist, args = getopt(sys.argv[1:], "qvf:") @@ -115,14 +166,13 @@ if not args or not args[0] in CmdMap: err_unknown_cmd() for p in optlist: - match p[0]: - case "-q": ProgConf.override_vl = logging.ERROR - case "-v": - if ProgConf.override_vl is None: - ProgConf.override_vl = palhm.DEFAULT.VL.value - 10 - else: - ProgConf.override_vl -= 10 - case "-f": ProgConf.conf = p[1] + if p[0] == "-q": ProgConf.override_vl = logging.ERROR + elif p[0] == "-v": + if ProgConf.override_vl is None: + ProgConf.override_vl = palhm.DEFAULT.VL.value - 10 + else: + ProgConf.override_vl -= 10 + elif p[0] == "-f": ProgConf.conf = p[1] logging.basicConfig(format = "%(name)s %(message)s") diff --git a/src/palhm/__init__.py b/src/palhm/__init__.py index 8c44ace..7e5afb4 100644 --- a/src/palhm/__init__.py +++ b/src/palhm/__init__.py @@ -1,3 +1,4 @@ +from .exceptions import InvalidConfigError import io import json import logging @@ -15,8 +16,6 @@ from datetime import datetime, timezone from decimal import Decimal from enum import Enum from importlib import import_module -from mailbox import FormatError -from multiprocessing import ProcessError from typing import Iterable @@ -72,10 +71,16 @@ class GlobalContext: for m in jobj.get("modules", iter(())): loaded = self.modules[m] = import_module("." + m, "palhm.mod") - intersect = set(self.backup_backends.keys()).intersection(loaded.backup_backends.keys()) - if intersect: - raise RuntimeError("Backup Backend conflict detected. ID(s): " + intersect) - self.backup_backends |= loaded.backup_backends + + if hasattr(loaded, "backup_backends"): + intersect = ( + set(self.backup_backends.keys()) + .intersection(loaded.backup_backends.keys())) + if intersect: + raise InvalidConfigError( + "Backup Backend conflict detected.", + intersect) + self.backup_backends |= loaded.backup_backends def get_vl (self) -> int: return self.vl @@ -123,36 +128,33 @@ class Exec (Runnable, ExecvHolder): b = int(m[2]) ret = range(a, b + 1) if len(ret) == 0: - raise ValueError("Invalid range: " + ec) + raise ValueError("Invalid range", ec) return ret m = re.match(Exec.RE.EC_RANGE.value, x) if m: op = str(m[1]) if m[1] else "==" n = int(m[2]) - match op: - case "==": return range(n, n + 1) - case "<": return range(0, n) - case "<=": return range(0, n + 1) - case ">": return range(n + 1, 256) - case ">=": return range(n, 256) - case _: raise RuntimeError("FIXME") + if op == "==": return range(n, n + 1) + elif op == "<": return range(0, n) + elif op == "<=": return range(0, n + 1) + elif op == ">": return range(n + 1, 256) + elif op == ">=": return range(n, 256) + else: raise RuntimeError("FIXME") - raise ValueError("Invalid value: " + ec) + raise ValueError("Invalid value", ec) def from_conf (ctx: GlobalContext, jobj: dict): - match jobj["type"]: - case "exec": - exec_id = jobj["exec-id"] - exec = ctx.exec_map[exec_id] - ret = exec - case "exec-append": - exec_id = jobj["exec-id"] - exec = ctx.exec_map[exec_id] - ret = exec.mkappend(jobj["argv"]) - case "exec-inline": - ret = Exec(jobj) - # case _: - # raise RuntimeError("FIXME") + if jobj["type"] == "exec": + exec_id = jobj["exec-id"] + exec = ctx.exec_map[exec_id] + ret = exec + elif jobj["type"] == "exec-append": + exec_id = jobj["exec-id"] + exec = ctx.exec_map[exec_id] + ret = exec.mkappend(jobj["argv"], jobj.get("env", {})) + elif jobj["type"] == "exec-inline": + ret = Exec(jobj) + else: raise RuntimeError("FIXME") ret.vl_stderr = jobj.get("vl-stderr", ret.vl_stderr) ret.vl_stdout = jobj.get("vl-stdout", ret.vl_stdout) @@ -173,9 +175,10 @@ class Exec (Runnable, ExecvHolder): self.vl_stderr = jobj.get("vl-stderr", Exec.DEFAULT.VL_STDERR.value) self.vl_stdout = jobj.get("vl-stdout", Exec.DEFAULT.VL_STDOUT.value) - def mkappend (self, extra_argv: Iterable): + def mkappend (self, extra_argv: Iterable, extra_env: dict = {}): ny = deepcopy(self) ny.argv.extend(extra_argv) + ny.env |= extra_env return ny def run (self, ctx: GlobalContext): @@ -201,8 +204,10 @@ class Exec (Runnable, ExecvHolder): def raise_oob_ec (self, ec: int): if not self.test_ec(ec): - raise ProcessError( - str(self) + " returned " + str(ec) + " not in " + str(self.ec)) + raise ChildProcessError( + str(self) + ": exit code test fail", + ec, + self.ec) def __str__ (self) -> str: return str().join( @@ -308,6 +313,18 @@ class NullBackupBackend (BackupBackend): def rotate (self, ctx: GlobalContext): pass + def _fs_usage_info (self, ctx: GlobalContext) -> Iterable[tuple[str, int]]: + return iter(()) + + def _excl_fs_copies (self, ctx: GlobalContext) -> set[str]: + return set[str]() + + def _rm_fs_recursive (self, ctx: GlobalContext, pl: Iterable[str]): + pass + + def _fs_quota_target (self, ctx: GlobalContext) -> tuple[Decimal, Decimal]: + return (Decimal('inf'), Decimal('inf')) + def __str__ (self): return "null" @@ -469,7 +486,7 @@ class RoutineTask (Task): def run (self, ctx: GlobalContext): for r in self.routines: - self.l.debug("run: " + str(r)) + self.l.info("run: " + str(r)) p = r.run(ctx) return self @@ -536,7 +553,7 @@ class DepResolv: def build (og_map: dict): def dive (og: BackupObjectGroup, obj_set: set, recurse_path: set): if og in recurse_path: - raise RecursionError("Circular reference detected.") + raise RecursionError("Circular reference detected whilst building dependency tree") recurse_path.add(og) obj_set.update(og.objects) @@ -611,7 +628,7 @@ class BackupTask (Task): for og in jobj_ogrps: ogid = og["id"] if ogid in og_map: - raise KeyError("Duplicate object group: " + ogid) + raise KeyError("Duplicate object group", ogid) og_map[ogid] = BackupObjectGroup() # load depends @@ -620,7 +637,8 @@ class BackupTask (Task): for depend in og.get("depends", iter(())): if ogid == depend: raise ReferenceError( - "An object group dependent on itself: " + ogid) + "An object group dependent on itself", + ogid) og_map[ogid].depends.add(og_map[depend]) # implicit default @@ -633,7 +651,7 @@ class BackupTask (Task): gid = jo.get("group", DEFAULT.OBJ_GRP.value) if path in obj_path_set: - raise KeyError("Duplicate path: " + path) + raise KeyError("Duplicate path", path) obj_path_set.add(path) og_map[gid].objects.append(BackupObject(jo, ctx)) @@ -651,6 +669,7 @@ class BackupTask (Task): for bo in self.dep_tree.avail_q: bo.bbctx = bbctx + self.l.info("make: " + bo.path) self.l.debug("despatch: " + str(bo)) fs.add(th_pool.submit(bo.run, ctx)) self.dep_tree.avail_q.clear() @@ -686,11 +705,11 @@ def merge_conf (a: dict, b: dict) -> dict: # exec conflicts c = chk_dup_id("execs", a, b) if c: - raise KeyError("Dup execs: " + c) + raise KeyError("Dup execs", c) # task conflicts c = chk_dup_id("tasks", a, b) if c: - raise KeyError("Dup tasks: " + c) + raise KeyError("Dup tasks", c) return a | b @@ -701,27 +720,38 @@ def load_jsonc (path: str) -> dict: stdin = in_file, capture_output = True) if p.returncode != 0: - raise FormatError(path) + raise ChildProcessError(p, path) return json.load(io.BytesIO(p.stdout)) def load_conf (path: str, inc_set: set = set()) -> dict: - if path in inc_set: - raise ReferenceError("Config included multiple times: " + path) - inc_set.add(path) + JSONC_EXT = ".jsonc" - if path.endswith(".jsonc"): - jobj = load_jsonc(path) + rpath = os.path.realpath(path, strict = True) + if rpath in inc_set: + raise RecursionError("Config already included", rpath) + inc_set.add(rpath) + + if rpath[-len(JSONC_EXT):].lower() == JSONC_EXT: + jobj = load_jsonc(rpath) else: - with open(path) as file: + with open(rpath) as file: jobj = json.load(file) # TODO: do schema validation + # pushd + saved_cwd = os.getcwd() + dn = os.path.dirname(rpath) + os.chdir(dn) + for i in jobj.get("include", iter(())): inc_conf = load_conf(i, inc_set) jobj = merge_conf(jobj, inc_conf) + # popd + os.chdir(saved_cwd) + return jobj def setup_conf (jobj: dict) -> GlobalContext: diff --git a/src/palhm/exceptions.py b/src/palhm/exceptions.py new file mode 100644 index 0000000..f63f2f9 --- /dev/null +++ b/src/palhm/exceptions.py @@ -0,0 +1,2 @@ +class InvalidConfigError (Exception): ... +class APIFailError (Exception): ... diff --git a/src/palhm/mod/aws.py b/src/palhm/mod/aws.py index fcb16f1..01fb8bc 100644 --- a/src/palhm/mod/aws.py +++ b/src/palhm/mod/aws.py @@ -7,6 +7,7 @@ from typing import Callable, Iterable import boto3 import botocore from palhm import BackupBackend, Exec, GlobalContext +from palhm.exceptions import APIFailError class CONST (Enum): @@ -50,10 +51,13 @@ class S3BackupBackend (BackupBackend): Key = self.cur_backup_key) sleep(1) # Make sure we don't proceed - raise FileExistsError(self.cur_backup_uri) + raise FileExistsError( + "Failed to set up a backup dir. Check the prefix function", + self.cur_backup_uri) except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] != "404": # expected status code - raise + c = e.response["Error"]["Code"] + if c != "404": # expected status code + raise APIFailError("Unexpected status code", c) return super().open(ctx) @@ -125,8 +129,9 @@ class S3BackupBackend (BackupBackend): o_key = i["Key"] o_size = i.get("Size", 0) if not o_key.startswith(self.root_key): - raise RuntimeError("The endpoint returned an object " + - "irrelevant to the request: " + o_key) + raise APIFailError( + "The endpoint returned an object irrelevant to the request", + o_key) l = o_key.find("/", len(prefix)) if l >= 0: @@ -200,7 +205,7 @@ class S3BackupBackend (BackupBackend): def rotate (self, ctx: GlobalContext): ret = super()._do_fs_rotate(ctx) - if self.sc_rot: + if self.sc_rot and self.sc_rot != self.sc_sink: def chsc (k): self.client.copy_object( Bucket = self.bucket, -- cgit v1.2.3-70-g09d2