Impl ...

- launch.json: change debug cwd to the project root dir
- Add subcmd "mods"
- Docs
- Tidy up sample and debug config files
- Change core exec
  - 'dnf-list-instaled' -> 'rpm-list-installed' as dnf does not work
    on ro fs
  - Accept the exit code 1 from tar(allow live fs change)
- Add the generic sample config
- Fix 'run' subcmd not accepting empty task-id
- Change module loading: modules are not required to have
  the 'backup_backends' var
- Reduce required Python version by removing the use of match ... case
- Fix 'exec-append' not taking 'env' into account
- Remove use of exceptions from irrelevant packages
- Fix unimpl methods of NullBackupBackend
- Tidy up instantiation of raised exceptions
- Change "include" behaviour
  - Relative config paths are now resolved like #include C preprocessor
- Fix bug where "include" circular ref checked is not done with absolute
  paths of config files
- Add own exception hierachy
- aws-s3: change storage class only when "rot-storage-class" is
  different from "sink-storage-class"

13 files changed, 466 insertions, 252 deletions

diff --git a/src/conf/py-debug/aws.sample.jsonc b/src/conf/py-debug/aws.jsonc
index 46ad562..df9a63a 100644
--- a/src/conf/py-debug/aws.sample.jsonc
+++ b/src/conf/py-debug/aws.jsonc
@@ -1,6 +1,6 @@
 // PALHM Instance Config
 {
-	"include": [ "conf/py-debug/conf.d/core.jsonc" ],
+	"include": [ "conf.d/core.json" ],
 	"modules": [ "aws" ],
 	"nb-workers": 0, // assumed $(nproc) - default
 	// "nb-workers": 1, // to disable concurrent task despatch
@@ -48,7 +48,7 @@
 					"path": "pm-list.gz",
 					"group": "pre-start",
 					"pipeline": [
-						{ "type": "exec", "exec-id": "dnf-list-installed" },
+						{ "type": "exec", "exec-id": "rpm-list-installed" },
 						{ "type": "exec", "exec-id": "filter-gzip-plain" }
 					]
 				},
diff --git a/src/conf/py-debug/conf.d b/src/conf/py-debug/conf.d
new file mode 120000
index 0000000..a32163d
--- /dev/null
+++ b/src/conf/py-debug/conf.d
@@ -0,0 +1 @@
+../py-sample/conf.d
\ No newline at end of file
diff --git a/src/conf/py-debug/conf.d/core.jsonc b/src/conf/py-debug/conf.d/core.jsonc
deleted file mode 100644
index 4afe7f5..0000000
--- a/src/conf/py-debug/conf.d/core.jsonc
+++ /dev/null
@@ -1,44 +0,0 @@
-// PALHM Core Config
-{
-	"execs": [
-		// {
-		// 	"id": "Exec ID",
-		// 	"argv": [ "cmd", "--option1=opt1_val", "-o", "opt2_val" ],
-		// 	"env": { "NAME": "VAL" },
-		// 	"ec": "0", // this is assumed
-		// 	"ec": "0-127", // inclusive range (not terminated by a signal)
-		// 	"ec": "<1", // range (only 0)
-		// 	"ec": "<=1", // range (0 and 1)
-		// 	"ec": ">0", // range (always fail)
-		// 	"ec": ">=0", // range (only 0)
-		// 	"vl-stderr": 1 //  verbosity level of stderr produced by this process
-		//  verbosity level of stderr produced by this process. Ignored if used
-		//  as part of pipeline
-		// 	"vl-stdout": 2
-		// },
-		{
-			"id": "tar",
-			"argv": [ "/usr/bin/tar", "--xattrs", "--selinux" ]
-		},
-		{
-			"id": "filter-xz-parallel",
-			"argv": [ "/usr/bin/xz", "-T0" ]
-		},
-		{
-			"id": "filter-gzip-plain",
-			"argv": [ "/usr/bin/gzip" ]
-		},
-		{
-			"id": "filter-zstd-plain",
-			"argv": [ "/usr/bin/zstd" ]
-		},
-		{
-			"id": "dnf-list-installed",
-			"argv": [ "/usr/bin/dnf", "-yq", "list", "installed" ]
-		},
-		{
-			"id": "lsblk-all-json",
-			"argv": [ "/usr/bin/lsblk", "-JbOa" ]
-		}
-	]
-}
diff --git a/src/conf/py-debug/localfs.sample.jsonc b/src/conf/py-debug/localfs.jsonc
index ec12808..a33060d 100644
--- a/src/conf/py-debug/localfs.sample.jsonc
+++ b/src/conf/py-debug/localfs.jsonc
@@ -1,12 +1,12 @@
 // PALHM Instance Config
 {
-	"include": [ "conf/py-debug/conf.d/core.jsonc" ],
+	"include": [ "conf.d/core.json" ],
 	"nb-workers": 0, // assumed $(nproc) - default
 	// "nb-workers": 1, // to disable concurrent task despatch
 	// To unlimit the number of workers.
 	// Does not fail on resource alloc failure.
 	// "nb-workers": -1,
-	"vl": 4,
+	"vl": 3,
 	"tasks": [
 		{
 			"id": "backup",
@@ -45,7 +45,7 @@
 					"path": "pm-list.gz",
 					"group": "pre-start",
 					"pipeline": [
-						{ "type": "exec", "exec-id": "dnf-list-installed" },
+						{ "type": "exec", "exec-id": "rpm-list-installed" },
 						{ "type": "exec", "exec-id": "filter-gzip-plain" }
 					]
 				},
diff --git a/src/conf/py-debug/null.jsonc b/src/conf/py-debug/null.jsonc
new file mode 100644
index 0000000..b5ce9f8
--- /dev/null
+++ b/src/conf/py-debug/null.jsonc
@@ -0,0 +1,141 @@
+{
+	"include": [ "conf.d/core.json" ],
+	"nb-workers": 0, // assumed $(nproc)
+	// "nb-workers": 1, // to disable concurrent task despatch
+	// "nb-workers": -1, // to unlimit the number of workers.
+	"vl": 3,
+	"tasks": [
+		{
+			"id": "backup",
+			"type": "backup",
+			"backend": "null",
+			"object-groups": [
+				{ "id": "pre-start" },
+				{
+					"id": "data-dump",
+					"depends": [ "pre-start" ]
+				},
+				{
+					"id": "tar-0",
+					"depends": [ "data-dump" ]
+				},
+				{
+					"id": "tar-1",
+					"depends": [ "data-dump" ]
+				}
+			],
+			"objects": [
+				{
+					"path": "pm-list.zstd",
+					"group": "pre-start",
+					"pipeline": [
+						{ "type": "exec", "exec-id": "rpm-list-installed" },
+						{ "type": "exec", "exec-id": "filter-zstd-plain" }
+					]
+				},
+				{
+					"path": "lsblk.json.zstd",
+					"group": "pre-start",
+					"pipeline": [
+						{ "type": "exec", "exec-id": "lsblk-all-json" },
+						{ "type": "exec", "exec-id": "filter-zstd-plain" }
+					]
+				},
+				{
+					"path": "db.sql.zstd",
+					"group": "data-dump",
+					"pipeline": [
+						{
+							"type": "exec-inline",
+							"argv": [
+								"/bin/mysqldump",
+								"-uroot",
+								"--all-databases"
+							]
+						},
+						{ "type": "exec", "exec-id": "filter-zstd-parallel" }
+					]
+				},
+				{
+					"path": "root.tar.zstd",
+					"group": "tar-0",
+					"pipeline": [
+						{
+							"type": "exec-append",
+							"exec-id": "tar",
+							"argv": [
+								"-C",
+								"/",
+								"/etc",
+								"/home",
+								"/root",
+								"/var"
+							]
+						},
+						{ "type": "exec", "exec-id": "filter-zstd-parallel" }
+					]
+				},
+				{
+					"path": "srv.tar.zstd",
+					"group": "tar-1",
+					"pipeline": [
+						{
+							"type": "exec-append",
+							"exec-id": "tar",
+							"argv": [
+								"-C",
+								"/",
+								"/srv"
+							]
+						},
+						{ "type": "exec", "exec-id": "filter-zstd-parallel" }
+					]
+				}
+			]
+		},
+		{
+			"id": "update",
+			"type": "routine",
+			"routine": [
+				{
+					"type": "exec-inline",
+					"argv": [ "/bin/dnf", "--refresh", "-yq", "update" ]
+				},
+				{
+					"type": "exec-inline",
+					"argv": [ "/bin/sa-update" ]
+				}
+			]
+		},
+		{
+			"id": "reboot",
+			"type": "routine",
+			"routine": [
+				{
+/*
+ * Block SIGTERM from systemd/init.d so PALHM can exit gracefully after issuing
+ * reboot.
+ */
+					"type": "builtin",
+					"builtin-id": "sigmask",
+					"param": [
+						{ "action": "block", "sig": [ "TERM" ] }
+					]
+				},
+				{
+					"type": "exec-inline",
+					"argv": [ "/sbin/reboot" ]
+				}
+			]
+		},
+		{
+			"id": "default",
+			"type": "routine",
+			"routine": [
+				{ "type": "task", "task-id": "backup" },
+				{ "type": "task", "task-id": "update" },
+				{ "type": "task", "task-id": "reboot" }
+			]
+		}
+	]
+}
diff --git a/src/conf/py-debug/null.sample.jsonc b/src/conf/py-debug/null.sample.jsonc
deleted file mode 100644
index a83de95..0000000
--- a/src/conf/py-debug/null.sample.jsonc
+++ /dev/null
@@ -1,140 +0,0 @@
-// PALHM Instance Config
-{
-	"include": [ "conf/py-debug/conf.d/core.jsonc" ],
-	"nb-workers": 1,
-	"vl": 3,
-	"tasks": [
-		{
-			"id": "backup",
-			"type": "backup",
-			"backend": "null",
-			"object-groups": [
-				{ "id": "pre-start" },
-				{
-					"id": "data-dump",
-					"depends": [ "pre-start" ]
-				},
-				{
-					"id": "tar-media-0",
-					"depends": [ "data-dump" ]
-				},
-				{
-					"id": "tar-media-1",
-					"depends": [ "data-dump" ]
-				}
-			],
-			"objects": [
-				{
-					"path": "pm-list.gz",
-					"group": "pre-start",
-					"pipeline": [
-						{ "type": "exec", "exec-id": "dnf-list-installed" },
-						{ "type": "exec", "exec-id": "filter-gzip-plain" }
-					]
-				},
-				{
-					"path": "lsblk.json.gz",
-					"group": "pre-start",
-					"pipeline": [
-						{
-							"type": "exec-append",
-							"exec-id": "lsblk-all-json",
-							"argv": [ "-a" ]
-						},
-						{ "type": "exec", "exec-id": "filter-gzip-plain" }
-					]
-				},
-				{
-					"path": "random-dump.sql.xz",
-					"group": "data-dump",
-					"pipeline": [
-						{
-							"type": "exec-inline",
-							"argv": [
-								"/bin/dd",
-								"if=/dev/urandom",
-								"bs=4096",
-								"count=512",
-								"status=none"
-							]
-						},
-						{ "type": "exec", "exec-id": "filter-xz-parallel" }
-					]
-				},
-				{
-					"path": "random-dump.0.xz",
-					"group": "tar-media-0",
-					"pipeline": [
-						{
-							"type": "exec-inline",
-							"argv": [
-								"/bin/dd",
-								"if=/dev/zero",
-								"bs=4096",
-								"count=512",
-								"status=none"
-							]
-						},
-						{ "type": "exec", "exec-id": "filter-xz-parallel" }
-					]
-				},
-				{
-					"path": "random-dump.1.xz",
-					"group": "tar-media-1",
-					"pipeline": [
-						{
-							"type": "exec-inline",
-							"argv": [
-								"/bin/dd",
-								"if=/dev/zero",
-								"bs=4096",
-								"count=512",
-								"status=none"
-							]
-						},
-						{ "type": "exec", "exec-id": "filter-xz-parallel" }
-					]
-				}
-			]
-		},
-		{
-			"id": "update",
-			"type": "routine",
-			"routine": [
-				{
-					"type": "exec-inline",
-					"argv": [ "/bin/echo", "0" ]
-				},
-				{
-					"type": "exec-inline",
-					"argv": [ "/bin/sleep", "1" ]
-				},
-				{
-					"type": "exec-inline",
-					"argv": [ "/bin/echo", "1" ]
-				}
-			]
-		},
-		{
-			"id": "default",
-			"type": "routine",
-			"routine": [
-				{ "type": "task", "task-id": "backup" },
-				{ "type": "task", "task-id": "update" },
-				{
-					// Block SIGTERM from systemd/init.d so the program is not
-					// affected by the reboot command.
-					"type": "builtin",
-					"builtin-id": "sigmask",
-					"param": [
-						{ "action": "block", "sig": [ "TERM" ] }
-					]
-				},
-				{
-					"type": "exec-inline",
-					"argv": [ "/bin/true" ]
-				}
-			]
-		}
-	]
-}
diff --git a/src/conf/py-debug/palhm.jsonc b/src/conf/py-debug/palhm.jsonc
new file mode 120000
index 0000000..fb68baf
--- /dev/null
+++ b/src/conf/py-debug/palhm.jsonc
@@ -0,0 +1 @@
+aws.jsonc
\ No newline at end of file
diff --git a/src/conf/py-sample/conf.d/core.json b/src/conf/py-sample/conf.d/core.json
new file mode 100644
index 0000000..46d3feb
--- /dev/null
+++ b/src/conf/py-sample/conf.d/core.json
@@ -0,0 +1,41 @@
+{
+	"execs": [
+		{
+			"id": "tar",
+			"argv": [ "/bin/tar", "--xattrs", "--selinux", "--warning=none", "-cf", "-" ],
+			"ec": "<2"
+		},
+		{
+			"id": "filter-xz-parallel",
+			"argv": [ "/bin/xz", "-T0" ]
+		},
+		{
+			"id": "filter-gzip-plain",
+			"argv": [ "/bin/gzip" ]
+		},
+		{
+			"id": "filter-zstd-plain",
+			"argv": [ "/bin/zstd" ]
+		},
+		{
+			"id": "filter-zstd-parallel",
+			"argv": [ "/bin/zstd", "-T0" ]
+		},
+		{
+			"id": "rpm-list-installed",
+			"argv": [ "/bin/rpm", "-qa" ]
+		},
+		{
+			"id": "dpkg-list-installed",
+			"argv": [ "/bin/dpkg-query", "-l" ]
+		},
+		{
+			"id": "lsblk-all-json",
+			"argv": [ "/bin/lsblk", "-JbOa" ]
+		},
+		{
+			"id": "os-release",
+			"argv": [ "/bin/cat", "/etc/os-release" ]
+		}
+	]
+}
diff --git a/src/conf/py-sample/sample.jsonc b/src/conf/py-sample/sample.jsonc
new file mode 100644
index 0000000..f1c4501
--- /dev/null
+++ b/src/conf/py-sample/sample.jsonc
@@ -0,0 +1,127 @@
+{
+	"include": [ "/etc/palhm/conf.d/core.json" ],
+	// "modules": [ "aws" ],
+	"nb-workers": 0,
+	// "vl": 4,
+	"tasks": [
+		{
+			"id": "backup",
+			"type": "backup",
+			"backend": "null",
+			"backend-param": {},
+			"object-groups": [
+				{ "id": "meta-run" },
+				{
+					"id": "data-dump",
+					"depends": [ "meta-run" ]
+				},
+				{
+					"id": "tar-root",
+					"depends": [ "data-dump" ]
+				}
+			],
+			"objects": [
+				{
+					"path": "os-release",
+					"group": "meta-run",
+					"pipeline": [ { "type": "exec" , "exec-id": "os-release" } ]
+				},
+				{
+					"path": "pm-list.zstd",
+					"group": "meta-run",
+					"pipeline": [
+						{ "type": "exec", "exec-id": "rpm-list-installed" },
+						{ "type": "exec", "exec-id": "filter-zstd-plain" }
+					]
+				},
+				{
+					"path": "lsblk.json.zstd",
+					"group": "meta-run",
+					"pipeline": [
+						{ "type": "exec", "exec-id": "lsblk-all-json" },
+						{ "type": "exec", "exec-id": "filter-zstd-plain" }
+					]
+				},
+				// {
+				// 	"path": "db.sql.zstd",
+				// 	"group": "data-dump",
+				// 	"pipeline": [
+				// 		{
+				// 			"type": "exec-inline",
+				// 			"argv": [
+				// 				"/bin/mysqldump",
+				// 				"-uroot",
+				// 				"--all-databases"
+				// 			]
+				// 			// "ec": "<=2" // don't fail when the DB is offline
+				// 		},
+				// 		{ "type": "exec", "exec-id": "filter-zstd-parallel" }
+				// 	]
+				// },
+				{
+					"path": "root.tar.zstd",
+					"group": "tar-root",
+					"pipeline": [
+						{
+							"type": "exec-append",
+							"exec-id": "tar",
+							"argv": [
+								"-C",
+								"/",
+								"etc",
+								"home",
+								"root",
+								"var"
+							]
+						},
+						{ "type": "exec", "exec-id": "filter-zstd-parallel" }
+					]
+				}
+			]
+		},
+		{
+			"id": "update",
+			"type": "routine",
+			"routine": [
+				{
+					"type": "exec-inline",
+					"argv": [ "/bin/dnf", "--refresh", "-yq", "update" ]
+				}
+				// {
+				// 	"type": "exec-inline",
+				// 	"argv": [ "/bin/sa-update" ]
+				// }
+			]
+		},
+		{
+			"id": "reboot",
+			"type": "routine",
+			"routine": [
+				{
+/*
+ * Block SIGTERM from systemd/init.d so PALHM can exit gracefully after issuing
+ * reboot.
+ */
+					"type": "builtin",
+					"builtin-id": "sigmask",
+					"param": [
+						{ "action": "block", "sig": [ "TERM" ] }
+					]
+				},
+				{
+					"type": "exec-inline",
+					"argv": [ "/sbin/reboot" ]
+				}
+			]
+		},
+		{
+			"id": "default",
+			"type": "routine",
+			"routine": [
+				{ "type": "task", "task-id": "backup" },
+				{ "type": "task", "task-id": "update" },
+				{ "type": "task", "task-id": "reboot" }
+			]
+		}
+	]
+}
diff --git a/src/palhm.py b/src/palhm.py
index f3f412b..722664e 100755
--- a/src/palhm.py
+++ b/src/palhm.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
+import importlib
 import logging
+import os
 import sys
 from abc import ABC, abstractmethod
 from getopt import getopt
@@ -49,7 +51,7 @@ class RunCmd (Cmd):
 	def do_cmd (self):
 		ProgConf.alloc_ctx()
 
-		if self.args:
+		if self.args and self.args[0]: # empty string as "default"
 			task = self.args[0]
 		else:
 			task = palhm.DEFAULT.RUN_TASK.value
@@ -64,6 +66,53 @@ class RunCmd (Cmd):
 Run a task in config. Run the "''' + palhm.DEFAULT.RUN_TASK.value +
 '''" task if [TASK] is not specified.''')
 
+class ModsCmd (Cmd):
+	def __init__ (self, *args, **kwargs):
+		pass
+
+	def _walk_mods (self, path: str):
+		def is_mod_dir (path: str) -> bool:
+			try:
+				for i in os.scandir(path):
+					if i.name.startswith("__init__.py"):
+						return True
+			except NotADirectoryError:
+				pass
+			return False
+
+		def is_mod_file (path: str) -> str:
+			if not os.path.isfile(path):
+				return None
+
+			try:
+				pos = path.rindex(".")
+				if path[pos + 1:].startswith("py"):
+					return os.path.basename(path[:pos])
+			except ValueError:
+				pass
+
+		for i in os.scandir(path):
+			if i.name.startswith("_"):
+				continue
+			elif is_mod_dir(i.path):
+				print(i.name)
+				self._walk_mods(i.path)
+			else:
+				name = is_mod_file(i.path)
+				if name:
+					print(name)
+
+	def do_cmd (self):
+		for i in importlib.util.find_spec("palhm.mod").submodule_search_locations:
+			self._walk_mods(i)
+
+		return 0
+
+	def print_help ():
+		print(
+"Usage: " + sys.argv[0] + " mods" + '''
+Prints the available modules to stdout.''')
+
 class HelpCmd (Cmd):
 	def __init__ (self, optlist, args):
 		self.optlist = optlist
@@ -84,7 +133,7 @@ class HelpCmd (Cmd):
 		print(
 "Usage: " + sys.argv[0] + " [options] CMD [command options ...]" + '''
 Options:
-  -q       Set the verbosity level to 0(FATAL error only). Overrides config
+  -q       Set the verbosity level to 0(CRITIAL). Overrides config
   -v       Increase the verbosity level by 1. Overrides config
   -f FILE  Load config from FILE instead of the hard-coded default
 Config: ''' + ProgConf.conf + '''
@@ -92,14 +141,16 @@ Commands:
   run         run a task
   config      load config and print the contents
   help [CMD]  print this message and exit normally if [CMD] is not specified.
-              Print usage of [CMD] otherwise''')
+              Print usage of [CMD] otherwise
+  mods        list available modules''')
 
 		return 0
 
 CmdMap = {
 	"config": ConfigCmd,
 	"run": RunCmd,
-	"help": HelpCmd
+	"help": HelpCmd,
+	"mods": ModsCmd
 }
 
 optlist, args = getopt(sys.argv[1:], "qvf:")
@@ -115,14 +166,13 @@ if not args or not args[0] in CmdMap:
 	err_unknown_cmd()
 
 for p in optlist:
-	match p[0]:
-		case "-q": ProgConf.override_vl = logging.ERROR
-		case "-v":
-			if ProgConf.override_vl is None:
-				ProgConf.override_vl = palhm.DEFAULT.VL.value - 10
-			else:
-				ProgConf.override_vl -= 10
-		case "-f": ProgConf.conf = p[1]
+	if p[0] == "-q": ProgConf.override_vl = logging.ERROR
+	elif p[0] == "-v":
+		if ProgConf.override_vl is None:
+			ProgConf.override_vl = palhm.DEFAULT.VL.value - 10
+		else:
+			ProgConf.override_vl -= 10
+	elif p[0] == "-f": ProgConf.conf = p[1]
 
 logging.basicConfig(format = "%(name)s %(message)s")
 
diff --git a/src/palhm/__init__.py b/src/palhm/__init__.py
index 8c44ace..7e5afb4 100644
--- a/src/palhm/__init__.py
+++ b/src/palhm/__init__.py
@@ -1,3 +1,4 @@
+from .exceptions import InvalidConfigError
 import io
 import json
 import logging
@@ -15,8 +16,6 @@ from datetime import datetime, timezone
 from decimal import Decimal
 from enum import Enum
 from importlib import import_module
-from mailbox import FormatError
-from multiprocessing import ProcessError
 from typing import Iterable
 
 
@@ -72,10 +71,16 @@ class GlobalContext:
 
 		for m in jobj.get("modules", iter(())):
 			loaded = self.modules[m] = import_module("." + m, "palhm.mod")
-			intersect = set(self.backup_backends.keys()).intersection(loaded.backup_backends.keys())
-			if intersect:
-				raise RuntimeError("Backup Backend conflict detected. ID(s): " + intersect)
-			self.backup_backends |= loaded.backup_backends
+
+			if hasattr(loaded, "backup_backends"):
+				intersect = (
+					set(self.backup_backends.keys())
+					.intersection(loaded.backup_backends.keys()))
+				if intersect:
+					raise InvalidConfigError(
+						"Backup Backend conflict detected.",
+						intersect)
+				self.backup_backends |= loaded.backup_backends
 
 	def get_vl (self) -> int:
 		return self.vl
@@ -123,36 +128,33 @@ class Exec (Runnable, ExecvHolder):
 			b = int(m[2])
 			ret = range(a, b + 1)
 			if len(ret) == 0:
-				raise ValueError("Invalid range: " + ec)
+				raise ValueError("Invalid range", ec)
 			return ret
 		m = re.match(Exec.RE.EC_RANGE.value, x)
 		if m:
 			op = str(m[1]) if m[1] else "=="
 			n = int(m[2])
-			match op:
-				case "==": return range(n, n + 1)
-				case "<": return range(0, n)
-				case "<=": return range(0, n + 1)
-				case ">": return range(n + 1, 256)
-				case ">=": return range(n, 256)
-				case _: raise RuntimeError("FIXME")
+			if op == "==": return range(n, n + 1)
+			elif op == "<": return range(0, n)
+			elif op == "<=": return range(0, n + 1)
+			elif op == ">": return range(n + 1, 256)
+			elif op == ">=": return range(n, 256)
+			else: raise RuntimeError("FIXME")
 
-		raise ValueError("Invalid value: " + ec)
+		raise ValueError("Invalid value", ec)
 
 	def from_conf (ctx: GlobalContext, jobj: dict):
-		match jobj["type"]:
-			case "exec":
-				exec_id = jobj["exec-id"]
-				exec = ctx.exec_map[exec_id]
-				ret = exec
-			case "exec-append":
-				exec_id = jobj["exec-id"]
-				exec = ctx.exec_map[exec_id]
-				ret = exec.mkappend(jobj["argv"])
-			case "exec-inline":
-				ret = Exec(jobj)
-			# case _:
-			# 	raise RuntimeError("FIXME")
+		if jobj["type"] == "exec":
+			exec_id = jobj["exec-id"]
+			exec = ctx.exec_map[exec_id]
+			ret = exec
+		elif jobj["type"] == "exec-append":
+			exec_id = jobj["exec-id"]
+			exec = ctx.exec_map[exec_id]
+			ret = exec.mkappend(jobj["argv"], jobj.get("env", {}))
+		elif jobj["type"] == "exec-inline":
+			ret = Exec(jobj)
+		else: raise RuntimeError("FIXME")
 
 		ret.vl_stderr = jobj.get("vl-stderr", ret.vl_stderr)
 		ret.vl_stdout = jobj.get("vl-stdout", ret.vl_stdout)
@@ -173,9 +175,10 @@ class Exec (Runnable, ExecvHolder):
 			self.vl_stderr = jobj.get("vl-stderr", Exec.DEFAULT.VL_STDERR.value)
 			self.vl_stdout = jobj.get("vl-stdout", Exec.DEFAULT.VL_STDOUT.value)
 
-	def mkappend (self, extra_argv: Iterable):
+	def mkappend (self, extra_argv: Iterable, extra_env: dict = {}):
 		ny = deepcopy(self)
 		ny.argv.extend(extra_argv)
+		ny.env |= extra_env
 		return ny
 
 	def run (self, ctx: GlobalContext):
@@ -201,8 +204,10 @@ class Exec (Runnable, ExecvHolder):
 
 	def raise_oob_ec (self, ec: int):
 		if not self.test_ec(ec):
-			raise ProcessError(
-				str(self) + " returned " + str(ec) + " not in " + str(self.ec))
+			raise ChildProcessError(
+				str(self) + ": exit code test fail",
+				ec,
+				self.ec)
 
 	def __str__ (self) -> str:
 		return str().join(
@@ -308,6 +313,18 @@ class NullBackupBackend (BackupBackend):
 	def rotate (self, ctx: GlobalContext):
 		pass
 
+	def _fs_usage_info (self, ctx: GlobalContext) -> Iterable[tuple[str, int]]:
+		return iter(())
+
+	def _excl_fs_copies (self, ctx: GlobalContext) -> set[str]:
+		return set[str]()
+
+	def _rm_fs_recursive (self, ctx: GlobalContext, pl: Iterable[str]):
+		pass
+
+	def _fs_quota_target (self, ctx: GlobalContext) -> tuple[Decimal, Decimal]:
+		return (Decimal('inf'), Decimal('inf'))
+
 	def __str__ (self):
 		return "null"
 
@@ -469,7 +486,7 @@ class RoutineTask (Task):
 
 	def run (self, ctx: GlobalContext):
 		for r in self.routines:
-			self.l.debug("run: " + str(r))
+			self.l.info("run: " + str(r))
 			p = r.run(ctx)
 		return self
 
@@ -536,7 +553,7 @@ class DepResolv:
 	def build (og_map: dict):
 		def dive (og: BackupObjectGroup, obj_set: set, recurse_path: set):
 			if og in recurse_path:
-				raise RecursionError("Circular reference detected.")
+				raise RecursionError("Circular reference detected whilst building dependency tree")
 			recurse_path.add(og)
 
 			obj_set.update(og.objects)
@@ -611,7 +628,7 @@ class BackupTask (Task):
 		for og in jobj_ogrps:
 			ogid = og["id"]
 			if ogid in og_map:
-				raise KeyError("Duplicate object group: " + ogid)
+				raise KeyError("Duplicate object group", ogid)
 			og_map[ogid] = BackupObjectGroup()
 
 		# load depends
@@ -620,7 +637,8 @@ class BackupTask (Task):
 			for depend in og.get("depends", iter(())):
 				if ogid == depend:
 					raise ReferenceError(
-						"An object group dependent on itself: " + ogid)
+						"An object group dependent on itself",
+						ogid)
 				og_map[ogid].depends.add(og_map[depend])
 
 		# implicit default
@@ -633,7 +651,7 @@ class BackupTask (Task):
 			gid = jo.get("group", DEFAULT.OBJ_GRP.value)
 
 			if path in obj_path_set:
-				raise KeyError("Duplicate path: " + path)
+				raise KeyError("Duplicate path", path)
 			obj_path_set.add(path)
 			og_map[gid].objects.append(BackupObject(jo, ctx))
 
@@ -651,6 +669,7 @@ class BackupTask (Task):
 
 				for bo in self.dep_tree.avail_q:
 					bo.bbctx = bbctx
+					self.l.info("make: " + bo.path)
 					self.l.debug("despatch: " + str(bo))
 					fs.add(th_pool.submit(bo.run, ctx))
 				self.dep_tree.avail_q.clear()
@@ -686,11 +705,11 @@ def merge_conf (a: dict, b: dict) -> dict:
 	# exec conflicts
 	c = chk_dup_id("execs", a, b)
 	if c:
-		raise KeyError("Dup execs: " + c)
+		raise KeyError("Dup execs", c)
 	# task conflicts
 	c = chk_dup_id("tasks", a, b)
 	if c:
-		raise KeyError("Dup tasks: " + c)
+		raise KeyError("Dup tasks", c)
 
 	return a | b
 
@@ -701,27 +720,38 @@ def load_jsonc (path: str) -> dict:
 			stdin = in_file,
 			capture_output = True)
 	if p.returncode != 0:
-		raise FormatError(path)
+		raise ChildProcessError(p, path)
 
 	return json.load(io.BytesIO(p.stdout))
 
 def load_conf (path: str, inc_set: set = set()) -> dict:
-	if path in inc_set:
-		raise ReferenceError("Config included multiple times: " + path)
-	inc_set.add(path)
+	JSONC_EXT = ".jsonc"
 
-	if path.endswith(".jsonc"):
-		jobj = load_jsonc(path)
+	rpath = os.path.realpath(path, strict = True)
+	if rpath in inc_set:
+		raise RecursionError("Config already included", rpath)
+	inc_set.add(rpath)
+
+	if rpath[-len(JSONC_EXT):].lower() == JSONC_EXT:
+		jobj = load_jsonc(rpath)
 	else:
-		with open(path) as file:
+		with open(rpath) as file:
 			jobj = json.load(file)
 
 	# TODO: do schema validation
 
+	# pushd
+	saved_cwd = os.getcwd()
+	dn = os.path.dirname(rpath)
+	os.chdir(dn)
+
 	for i in jobj.get("include", iter(())):
 		inc_conf = load_conf(i, inc_set)
 		jobj = merge_conf(jobj, inc_conf)
 
+	# popd
+	os.chdir(saved_cwd)
+
 	return jobj
 
 def setup_conf (jobj: dict) -> GlobalContext:
diff --git a/src/palhm/exceptions.py b/src/palhm/exceptions.py
new file mode 100644
index 0000000..f63f2f9
--- /dev/null
+++ b/src/palhm/exceptions.py
@@ -0,0 +1,2 @@
+class InvalidConfigError (Exception): ...
+class APIFailError (Exception): ...
diff --git a/src/palhm/mod/aws.py b/src/palhm/mod/aws.py
index fcb16f1..01fb8bc 100644
--- a/src/palhm/mod/aws.py
+++ b/src/palhm/mod/aws.py
@@ -7,6 +7,7 @@ from typing import Callable, Iterable
 import boto3
 import botocore
 from palhm import BackupBackend, Exec, GlobalContext
+from palhm.exceptions import APIFailError
 
 
 class CONST (Enum):
@@ -50,10 +51,13 @@ class S3BackupBackend (BackupBackend):
 					Key = self.cur_backup_key)
 				sleep(1)
 			# Make sure we don't proceed
-			raise FileExistsError(self.cur_backup_uri)
+			raise FileExistsError(
+				"Failed to set up a backup dir. Check the prefix function",
+				self.cur_backup_uri)
 		except botocore.exceptions.ClientError as e:
-			if e.response["Error"]["Code"] != "404": # expected status code
-				raise
+			c = e.response["Error"]["Code"]
+			if c != "404": # expected status code
+				raise APIFailError("Unexpected status code", c)
 
 		return super().open(ctx)
 
@@ -125,8 +129,9 @@ class S3BackupBackend (BackupBackend):
 			o_key = i["Key"]
 			o_size = i.get("Size", 0)
 			if not o_key.startswith(self.root_key):
-				raise RuntimeError("The endpoint returned an object " +
-					"irrelevant to the request: " + o_key)
+				raise APIFailError(
+					"The endpoint returned an object irrelevant to the request",
+					o_key)
 
 			l = o_key.find("/", len(prefix))
 			if l >= 0:
@@ -200,7 +205,7 @@ class S3BackupBackend (BackupBackend):
 	def rotate (self, ctx: GlobalContext):
 		ret = super()._do_fs_rotate(ctx)
 
-		if self.sc_rot:
+		if self.sc_rot and self.sc_rot != self.sc_sink:
 			def chsc (k):
 				self.client.copy_object(
 					Bucket = self.bucket,