Initial commit

15 files changed, 256 insertions, 0 deletions

diff --git a/.github/workflows/snapshot-automation.yml b/.github/workflows/snapshot-automation.yml
new file mode 100644
index 00000000..b6279cd2
--- /dev/null
+++ b/.github/workflows/snapshot-automation.yml
@@ -0,0 +1,40 @@
+name: Snapshot
+run-name: Snapshot
+on:
+  schedule:
+    - cron: "00 * * * *"
+  workflow_dispatch:
+permissions:
+  contents: write
+jobs:
+  Do-snapshot:
+    name: Do data dump snapshot
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run make
+        run: |
+          make clean
+          make
+      - name: Do commit
+        run: |
+          git config --global user.name 'Snapshot Automation Bot'
+          git config --global user.email '_@users.noreply.github.com'
+          git add --all
+          git commit -sm 'Data snapshot commit'
+          git push
+      - name: Install SSH secrets
+        env:
+          SSH_KEY: ${{ secrets.MIRROR_SSH_KEY }}
+          SSH_KNOWN_HOSTS: ${{ secrets.MIRROR_SSH_KNOWN_HOSTS }}
+        run: |
+          pushd ~
+          mkdir -p .ssh .ssh_keys
+          echo "$SSH_KNOWN_HOSTS" >> .ssh/known_hosts
+          echo "$SSH_KEY" > .ssh_keys/sync-key
+          chmod 700 .ssh .ssh_keys
+          chmod 600 .ssh/known_hosts .ssh_keys/sync-key
+          popd
+      - name: Sync mirrors
+        run: |
+          ssh -T -i ~/.ssh_keys/sync-key cdci@embla.dev.snart.me
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..8b03f042
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+/tmp
+
+*.tmp
+__pycache__
diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..23bc556b
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,36 @@
+all: README.md
+
+.PHONY: clean
+
+clean:
+	rm -rf data/regions data/latency README.md tmp
+
+data/regions: get-regions
+	./get-regions > data/regions.tmp
+	mv data/regions.tmp data/regions
+
+data/latency: get-latency data/regions
+	./get-latency < data/regions > data/latency.tmp
+	mv data/latency.tmp data/latency
+
+data/latency-avg: make-avg data/latency
+	./make-avg data/latency-avg data/latency > data/latency-avg.tmp
+	mv data/latency-avg.tmp data/latency-avg
+
+
+tmp/firstplace: data/latency
+	mkdir -p tmp
+	awk '{print $$2 "\t" $$1}' < data/latency | sort -g | head -n1 | awk '{print $$2}' > tmp/firstplace
+
+data/place: make-place data/latency tmp/firstplace
+	./make-place tmp/firstplace < data/place > data/place.tmp
+	mv data/place.tmp data/place
+
+tmp/table.md: make-table data/latency-avg data/place
+	mkdir -p tmp
+	./make-table data/latency-avg data/place > tmp/table.md.tmp
+	mv tmp/table.md.tmp tmp/table.md
+
+README.md: make-readme README.skel.md tmp/table.md
+	./make-readme tmp/table.md < README.skel.md > README.md.tmp
+	mv README.md.tmp README.md
diff --git a/README.skel.md b/README.skel.md
new file mode 100644
index 00000000..5902d2e6
--- /dev/null
+++ b/README.skel.md
@@ -0,0 +1,35 @@
+# Github Actions Metrics
+Information on Github hosted runners like the Azure region they run on is
+necessary info when optimising CD/CI pipelines(especially network latencies and
+route path bandwidth). Github does not disclose it so I did it myself.
+
+Using this info, place the resources(DB, object storage, other instances) near
+the runners are usually run.
+
+A few pieces of info I could gather online:
+
+- Azure doesn't provide a list of VM service endpoints like AWS
+- Github-hosted Actions runners are actually Azure VMs (surprisingly, not in a
+  container)
+- Github is hosted in the data centre somewhere in the US, probably in the same
+  data centre where Azure is present
+
+Microsoft definitely has more points of presence than any other cloud service
+providers, but there's no official list of data center endpoints to ping. If you
+look at the map,
+
+<a href="https://aws.amazon.com/about-aws/global-infrastructure/regions_az/">
+<img src="image.png" style="width: 500px;">
+</a>
+<a href="https://datacenters.microsoft.com/globe/explore">
+<img src="image-1.png" style="width: 500px;">
+</a>
+
+they're close enough. For most devs, all that matters is probably how close
+their S3 buckets are to the Github Actions runners. Some AWS and Azure regions
+are under the same roof, but then again, no official data.
+
+## DATA
+Updated: %%UPDATED_TS%%
+
+%%TABLE%%
diff --git a/common/__init__.py b/common/__init__.py
new file mode 100644
index 00000000..5770ec96
--- /dev/null
+++ b/common/__init__.py
@@ -0,0 +1,27 @@
+import math
+from typing import Any, Callable
+
+def load_data_from_file (path: str, mask_noent = True) -> dict[str, float]:
+	try:
+		with open(path) as f:
+			return load_data(f.readline)
+	except FileNotFoundError as e:
+		if not mask_noent:
+			raise e
+
+	return dict[str, float]()
+
+def load_data (read_f: Callable) -> dict[str, float]:
+	ret = dict[str, float]()
+
+	while True:
+		l = read_f()
+		if not l:
+			break
+
+		tpl = l.strip().split()
+		v = float(tpl[1])
+		if math.isfinite(v):
+			ret[tpl[0]] = v
+
+	return ret
diff --git a/data/place b/data/place
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/data/place
diff --git a/get-latency b/get-latency
new file mode 100755
index 00000000..53da95cf
--- /dev/null
+++ b/get-latency
@@ -0,0 +1,6 @@
+#!/bin/bash
+while read l
+do
+	ping=$(./ping-curl "https://ec2.$l.api.aws/")
+	printf "%-15s\t%s\n" "$l" "$ping"
+done
diff --git a/get-regions b/get-regions
new file mode 100755
index 00000000..bb3fd744
--- /dev/null
+++ b/get-regions
@@ -0,0 +1,18 @@
+#!/bin/bash
+filter_output () {
+	# drop any lines that doesn't contain a dash
+	# (to filter out "GLOBAL")
+	while read l
+	do
+		if [[ "$l" == *"-"* ]]; then
+			echo $l
+		fi
+	done
+}
+
+set -o pipefail
+curl -sSL 'https://ip-ranges.amazonaws.com/ip-ranges.json' |
+	jq -r '.prefixes[].region, .ipv6_prefixes[].region' |
+	sort |
+	uniq |
+	filter_output
diff --git a/image-1.png b/image-1.png
new file mode 100644
index 00000000..cd21b210
--- /dev/null
+++ b/image-1.png
diff --git a/image.png b/image.png
new file mode 100644
index 00000000..0a43b3ff
--- /dev/null
+++ b/image.png
diff --git a/make-avg b/make-avg
new file mode 100755
index 00000000..3ed5c115
--- /dev/null
+++ b/make-avg
@@ -0,0 +1,22 @@
+#!/bin/env python3
+import sys
+import common
+
+a = common.load_data_from_file(sys.argv[1])
+b = common.load_data_from_file(sys.argv[2])
+
+c = set(a.keys())
+c.update(b.keys())
+keys = list(c)
+keys.sort()
+for k in keys:
+	v_a = a.get(k)
+	v_b = b.get(k)
+	if v_a is None:
+		v = v_b
+	elif v_b is None:
+		v = v_a
+	else:
+		v = (v_a + v_b) / 2.0
+
+	print('%s\t%0.3lf' % (k, v))
diff --git a/make-place b/make-place
new file mode 100755
index 00000000..70f2bfac
--- /dev/null
+++ b/make-place
@@ -0,0 +1,13 @@
+#!/bin/env python3
+import sys
+import common
+
+target = open(sys.argv[1]).read().strip()
+data = common.load_data(sys.stdin.readline)
+
+data[target] = data.get(target, 0.0) + 1.0
+
+keys = list(data.keys())
+keys.sort()
+for k in keys:
+	print('%-15s\t%.0lf' % (k, data[k]))
diff --git a/make-readme b/make-readme
new file mode 100755
index 00000000..90b95b6f
--- /dev/null
+++ b/make-readme
@@ -0,0 +1,17 @@
+#!/bin/env python3
+import datetime
+import sys
+
+table = open(sys.argv[1]).read()
+doc = sys.stdin.read()
+
+try:
+	now = datetime.datetime.now(datetime.UTC)
+except AttributeError:
+	now = datetime.datetime.utcnow()
+
+doc = (doc
+		.replace('%%UPDATED_TS%%', now.isoformat())
+		.replace('%%TABLE%%', table))
+
+sys.stdout.write(doc)
diff --git a/make-table b/make-table
new file mode 100755
index 00000000..ca08748b
--- /dev/null
+++ b/make-table
@@ -0,0 +1,17 @@
+#!/bin/env python
+import sys
+import common
+
+data_lavg = common.load_data_from_file(sys.argv[1])
+data_place = common.load_data_from_file(sys.argv[2])
+
+print('''| AWS Region | Avg Latency | Least |''')
+print('''| - | - | - |''')
+for k, v in data_lavg.items():
+	place = data_place.get(k, 0.0)
+	if place:
+		place = '%.0lf' % place
+	else:
+		place = ""
+
+	print('| %s | %.3lf | %s |' % (k, v, place))
diff --git a/ping-curl b/ping-curl
new file mode 100755
index 00000000..ecde7217
--- /dev/null
+++ b/ping-curl
@@ -0,0 +1,21 @@
+#!/bin/bash
+export LC_ALL="C"
+set -o pipefail
+
+get_readtime () {
+	grep -Ei '^real' | grep -Eio '[0-9]+\.[0-9]+'
+}
+
+do_ping () {
+	time curl --connect-timeout 2 -sS "$1" -o /dev/null
+}
+
+for (( i = 0; i < 4; i += 1 ))
+do
+	ping=$(do_ping "$1" 2>&1)
+	if [ $? -ne 0 ]; then
+		echo "inf"
+		exit 1
+	fi
+	echo "$ping" | get_readtime
+done | uniq | sort | head -n1