diff options
| -rw-r--r-- | .vscode/launch.json | 15 | ||||
| -rw-r--r-- | README.md | 10 | ||||
| -rw-r--r-- | src/conf/crontab | 6 | ||||
| -rw-r--r-- | src/conf/palhm-boot-report.service | 4 | ||||
| l--------- | src/conf/py-debug/palhm.jsonc | 2 | ||||
| -rw-r--r-- | src/conf/py-sample/boot-report.jsonc | 7 | ||||
| -rw-r--r-- | src/conf/py-sample/conf.d/core.json | 12 | ||||
| -rw-r--r-- | src/conf/py-sample/sample.jsonc | 37 | ||||
| -rwxr-xr-x | src/palhm-dnssec-check.sh | 8 | ||||
| -rw-r--r-- | src/palhm/__init__.py | 67 |
10 files changed, 122 insertions, 46 deletions
diff --git a/.vscode/launch.json b/.vscode/launch.json index 5ed76e2..1e96b2f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -43,6 +43,21 @@ "args": [ "-f", "src/conf/py-debug/palhm.jsonc", "run" ], "console": "integratedTerminal", "justMyCode": true + }, + { + "name": "palhm run check-dnssec", + "type": "python", + "request": "launch", + "cwd": "${workspaceFolder}", + "program": "src/palhm.py", + "args": [ + "-f", + "src/conf/py-debug/palhm.jsonc", + "run", + "check-dnssec" + ], + "console": "integratedTerminal", + "justMyCode": true } ] } @@ -242,5 +242,15 @@ Also, you can always do a dry run of your backup task by setting the backend to ## TODO * JSON schema validation +### AWS S3 Replication Daemon +To prepare for very unlikely events of +[disasters](https://docs.aws.amazon.com/whitepapers/latest/disaster-recovery-workloads-on-aws/disaster-recovery-options-in-the-cloud.html) +affecting an entire AWS region, you may wish to implement cross-region +replication of S3 objects. The replication the S3 provides does not work on very +large objects. So replication of large objects across AWS regions has to be done +manually by a client - another implementation is required. + +Cross-region data transfer is costly, so this idea came to a halt. + ## Footnotes [^1]: Even with SSDs, disrupting sequential reads decreases overall performance diff --git a/src/conf/crontab b/src/conf/crontab new file mode 100644 index 0000000..35f52ce --- /dev/null +++ b/src/conf/crontab @@ -0,0 +1,6 @@ +# PALHM will produce stderr on exception. Mail the output to root +MAILTO="root" +# Run default task every Sunday at midnight +0 0 * * sun root /var/lib/PALHM/src/palhm.py -q run +# Check dnssec validity every hour +# 0 * * * * root systemd-run -qP -p Nice=15 -p ProtectSystem=strict -p ReadOnlyPaths=/ -p PrivateDevices=true --wait /var/lib/PALHM/src/palhm.py -q run check-dnssec diff --git a/src/conf/palhm-boot-report.service b/src/conf/palhm-boot-report.service index 288aabd..37a4e61 100644 --- a/src/conf/palhm-boot-report.service +++ b/src/conf/palhm-boot-report.service @@ -6,9 +6,7 @@ After=postfix.service sendmail.service exim.service Type=oneshot ExecStart=/var/lib/PALHM/src/palhm.py -q boot-report Nice=10 -ProtectSystem=strict -ReadOnlyPaths=/ -PrivateDevices=true +User=palhm [Install] WantedBy=multi-user.target diff --git a/src/conf/py-debug/palhm.jsonc b/src/conf/py-debug/palhm.jsonc index fb68baf..c40e201 120000 --- a/src/conf/py-debug/palhm.jsonc +++ b/src/conf/py-debug/palhm.jsonc @@ -1 +1 @@ -aws.jsonc
\ No newline at end of file +../py-sample/sample.jsonc
\ No newline at end of file diff --git a/src/conf/py-sample/boot-report.jsonc b/src/conf/py-sample/boot-report.jsonc deleted file mode 100644 index dd9d606..0000000 --- a/src/conf/py-sample/boot-report.jsonc +++ /dev/null @@ -1,7 +0,0 @@ -{ - "boot-report": { - // "mua": "stdout", - "mua": "mailx", - "mail-to": [ "root" ] - } -} diff --git a/src/conf/py-sample/conf.d/core.json b/src/conf/py-sample/conf.d/core.json index 46d3feb..7145eae 100644 --- a/src/conf/py-sample/conf.d/core.json +++ b/src/conf/py-sample/conf.d/core.json @@ -36,6 +36,18 @@ { "id": "os-release", "argv": [ "/bin/cat", "/etc/os-release" ] + }, + { + "id": "dig-dnssec", + "argv": [ "/bin/dig", "+short", "+dnssec", "+notcp" ] + }, + { + "id": "grep-any", + "argv": [ "/bin/grep", "." ] + }, + { + "id": "null-stdout-sink", + "argv": [ "/bin/cp", "/dev/stdin", "/dev/null" ] } ] } diff --git a/src/conf/py-sample/sample.jsonc b/src/conf/py-sample/sample.jsonc index f1c4501..0da72a6 100644 --- a/src/conf/py-sample/sample.jsonc +++ b/src/conf/py-sample/sample.jsonc @@ -3,6 +3,16 @@ // "modules": [ "aws" ], "nb-workers": 0, // "vl": 4, + "boot-report": { + // "mua": "stdout", + "mua": "mailx", + "mail-to": [ "root" ] + // "subject": "Custom Boot Report Subject from {hostname}", + // "header": "Custom header content with {hostname} substitution." + // "uptime-since": true, + // "uptime": true, + // "bootid": true + }, "tasks": [ { "id": "backup", @@ -115,6 +125,33 @@ ] }, { + "id": "check-dnssec", + "type": "backup", + "backend": "null", + "objects": [ + { + "path": "example.com", // Placeholder + "pipeline": [ + /* + * Check if dig can query the record with the DNSSEC + * validation flag. Empty stdout with zero return code + * means SERVFAIL. + */ + { + "type": "exec-append", + "exec-id": "dig-dnssec", + "argv": [ "ANY", "example.com" ] + }, + /* + * Trap for empty dig output grep will return non-zero if + * dig have not produced any output + */ + { "type": "exec", "exec-id": "grep-any" } + ] + } + ] + }, + { "id": "default", "type": "routine", "routine": [ diff --git a/src/palhm-dnssec-check.sh b/src/palhm-dnssec-check.sh index 4601d8e..122e51d 100755 --- a/src/palhm-dnssec-check.sh +++ b/src/palhm-dnssec-check.sh @@ -1,4 +1,8 @@ #!/bin/bash + +# This script is a legacy. The same functionality can be implemented by setting +# up a back up task. See [conf/py-sample/sample.jsonc] + do_query () { # dig returns 0 upon successful reception and parse of the response message. # All the other exit codes other than 0 will cause the script to terminate @@ -6,7 +10,7 @@ do_query () { # We assume that a status code has returned when dig produces no output with # the option. Caution must be taken in this approach as zones with no # record will also return nothing with the status code zero. - dig +short +dnssec ANY "$TARGET" > "$tmpf" + dig +short +dnssec +notcp ANY "$TARGET" > "$tmpf" if [ ! -s "$tmpf" ]; then echo "The nameserver returned no RR! DNSSEC verification probably failed." >&2 @@ -33,7 +37,7 @@ declare TARGET="$1" declare tmpf="$(mktemp --tmpdir "palhm-dnssec.XXXXXXXXXX")" do_query & set +e -wait -f "$!" +wait ec="$?" rm "$tmpf" diff --git a/src/palhm/__init__.py b/src/palhm/__init__.py index f9ab9b3..9f2ff4f 100644 --- a/src/palhm/__init__.py +++ b/src/palhm/__init__.py @@ -1,4 +1,3 @@ -from imp import load_module import platform import sys import time @@ -78,6 +77,8 @@ class GlobalContext: elif self.nb_workers < 0: self.nb_workers = None + self.child_io_size = 4096 + for m in jobj.get("modules", iter(())): loaded = self.modules[m] = import_module("." + m, "palhm.mod") @@ -152,7 +153,6 @@ class BootReport: self.uptime_since = jobj.get("uptime-since", True) self.uptime = jobj.get("uptime", True) self.bootid = jobj.get("boot-id", True) - self.systemd_analyze = jobj.get("systemd-analyze", True) def get_subject (self) -> str: return BootReport._do_format(self.subject) @@ -168,36 +168,31 @@ class BootReport: body["tz"] = list(time.tzname) + [time.timezone] if self.uptime_since: - p = subprocess.run( + with subprocess.Popen( [ "/bin/uptime", "--since" ], stdin = subprocess.DEVNULL, - capture_output = True) - if p.returncode != 0: - raise ChildProcessError("uptime-since", p.returncode) - body["uptime-since"] = p.stdout.decode().strip() + stdout = subprocess.PIPE) as p: + body["uptime-since"] = str( + p.stdout.read(ctx.child_io_size)).strip() + + if p.wait() != 0: + raise ChildProcessError(p) if self.uptime: - p = subprocess.run( + with subprocess.Popen( [ "/bin/uptime", "-p" ], stdin = subprocess.DEVNULL, - capture_output = True) - if p.returncode != 0: - raise ChildProcessError("uptime", p.returncode) - body["uptime"] = p.stdout.decode().strip() + stdout = subprocess.PIPE) as p: + body["uptime"] = str( + p.stdout.read(ctx.child_io_size)).strip() + + if p.wait() != 0: + raise ChildProcessError(p) if self.bootid: with open("/proc/sys/kernel/random/boot_id") as f: body["bood-id"] = f.readline(36) - if self.systemd_analyze: - p = subprocess.run( - [ "/bin/systemd-analyze" ], - stdin = subprocess.DEVNULL, - capture_output = True) - if p.returncode != 0: - raise ChildProcessError("systemd-analyze", p.returncode) - body["systemd-analyze"] = p.stdout.decode().strip() - yield self.yaml.dump(root_doc) def do_send (self, ctx: GlobalContext) -> int: @@ -742,8 +737,8 @@ class DepResolv: class BackupTask (Task): def __init__ (self, ctx: GlobalContext, jobj: dict): og_map = {} - jobj_ogrps = jobj["object-groups"] - jobj_list = jobj["objects"] + jobj_ogrps = jobj.get("object-groups", []) + jobj_list = jobj.get("objects", []) obj_path_set = set() self.l = ctx.l.getChild("BackupTask@" + jobj.get("id", hex(id(self)))) @@ -835,27 +830,33 @@ def merge_conf (a: dict, b: dict) -> dict: c = chk_dup_id("tasks", a, b) if c: raise KeyError("Dup tasks", c) - # "boot-report" conflict - if "boot-report" in a and "boot-report" in b: - raise InvalidConfigError( - "'boot-report' already defined in the previous config") ret = a | b ret["execs"] = a.get("execs", []) + b.get("execs", []) ret["tasks"] = a.get("tasks", []) + b.get("tasks", []) + # "boot-report" conflict and merge + if "boot-report" in a and "boot-report" in b: + if "mua" in a["boot-report"] and "mua" in b["boot-report"]: + raise InvalidConfigError("Overriding 'mua' in 'boot-report'") + ret["boot-report"]["mail-to"] = ( + a["boot-report"]["mail-to"] + + b["boot-report"]["mail-to"]) + return ret def load_jsonc (path: str) -> dict: - with open(path) as in_file: - p = subprocess.run( + with (open(path) as in_file, + subprocess.Popen( [ "/bin/json_reformat" ], stdin = in_file, - capture_output = True) - if p.returncode != 0: - raise ChildProcessError(p, path) + stdout = subprocess.PIPE) as p): + ret = json.load(p.stdout) - return json.load(io.BytesIO(p.stdout)) + if p.wait() != 0: + raise ChildProcessError(p, path) + + return ret def load_conf (path: str, inc_set: set = set()) -> dict: JSONC_EXT = ".jsonc" |