From ca08936a767d443571cec1d973a197d2cd692ead Mon Sep 17 00:00:00 2001 From: David Timber Date: Mon, 28 Oct 2024 13:36:35 +0100 Subject: Fix race condition, fix doc ... - Remove use of named pipe which was a bad design in the first place - Fix rare case where netcat's error slipping processing the last AS - Doc: fix factual errors, add some more info --- Makefile | 29 +++++++++++++++++------------ README.skel.md | 10 +++++++--- get-kisa-as-list | 41 ----------------------------------------- get-kisa-asmap | 41 +++++++++++++++++++++++++++++++++++++++++ query-prefixes | 14 +++----------- query-rpki-prefixes | 2 ++ 6 files changed, 70 insertions(+), 67 deletions(-) delete mode 100755 get-kisa-as-list create mode 100755 get-kisa-asmap diff --git a/Makefile b/Makefile index 7f4427c..d854e5a 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ README.md: table.md table6.md clean: rm -f \ *.tmp \ + asmap \ aslist \ routelist \ route6list \ @@ -20,16 +21,20 @@ clean: result6.csv \ README.md -aslist: +asmap: curl -sS 'https://krnic.kisa.or.kr/jsp/business/management/asList.jsp' | \ - ./get-kisa-as-list > aslist.tmp + ./get-kisa-asmap > asmap.tmp + mv asmap.tmp asmap + +aslist: asmap + grep -Eoi 'AS[0-9]+' asmap > aslist.tmp mv aslist.tmp aslist routelist: aslist - ./query-prefixes < aslist > routelist.tmp + ./query-prefixes < aslist 3< aslist > routelist.tmp mv routelist.tmp routelist route6list: aslist - ./query-prefixes -6 < aslist > route6list.tmp + ./query-prefixes -6 < aslist 3< aslist > route6list.tmp mv route6list.tmp route6list rpki-enabled: routelist @@ -39,16 +44,16 @@ rpki6-enabled: route6list ./query-rpki-prefixes < route6list > rpki6-enabled.tmp mv rpki6-enabled.tmp rpki6-enabled -table.md: aslist routelist rpki-enabled - ./produce-table aslist routelist rpki-enabled > table.md.tmp +table.md: asmap routelist rpki-enabled + ./produce-table asmap routelist rpki-enabled > table.md.tmp mv table.md.tmp table.md -table6.md: aslist route6list rpki6-enabled - ./produce-table aslist route6list rpki6-enabled > table6.md.tmp +table6.md: asmap route6list rpki6-enabled + ./produce-table asmap route6list rpki6-enabled > table6.md.tmp mv table6.md.tmp table6.md -result.csv: aslist routelist rpki-enabled - ./produce-table -c aslist routelist rpki-enabled > result.csv.tmp +result.csv: asmap routelist rpki-enabled + ./produce-table -c asmap routelist rpki-enabled > result.csv.tmp mv result.csv.tmp result.csv -result6.csv: aslist route6list rpki6-enabled - ./produce-table -c aslist route6list rpki6-enabled > result6.csv.tmp +result6.csv: asmap route6list rpki6-enabled + ./produce-table -c asmap route6list rpki6-enabled > result6.csv.tmp mv result6.csv.tmp result6.csv diff --git a/README.skel.md b/README.skel.md index 9bfd86a..69813ab 100644 --- a/README.skel.md +++ b/README.skel.md @@ -19,7 +19,11 @@ https://stat.ripe.net/ui2013/widget/rpki-by-country#w.resource=kr%2Cus%2Cjp%2Cau RPKI가 BGP 설정 오류로 인한 장애 예방에 좋으면 빨리 적용해야겠죠? 각 사업자의 RPKI 적용률을 여기서 확인하세요. -UTC+9 기준 매주 월요일 자정에 업데이트 자동 업데이트 중(Github Actions). +UTC 기준 매주 월요일 자정에 업데이트 자동 업데이트 중([Github +Actions](https://github.com/dxdxdt/rpki-tracker-kr/actions)). 데이터는 +`master`에 자동으로 커밋됨. 개발 편의를 위해 `dev`와 `stable` 브랜치에서는 +데이터 파일이 트래킹되지 않음. 소스코드 수정은 해당 브랜치에서 진행, CD/CI에 +적용 필요 시 `master`와 머지. ## 스크립트 실행 절차 [Makefile 레시피](Makefile) 설명. @@ -70,14 +74,14 @@ validation을 할 필요가 없기 때문에, 보급률이 100%가 될 필요는 ## Usage ### INSTALL ```sh -dnf install make python3 +dnf install curl netcat make python3 +git clone https://github.com/dxdxdt/rpki-tracker-kr ``` ### Generate 실행 시간 5분 내외. 메모리 사용량(RSS): 약 50MB ```sh -git clone https://github.com/dxdxdt/rpki-tracker-kr make clean make ``` diff --git a/get-kisa-as-list b/get-kisa-as-list deleted file mode 100755 index b4b1c10..0000000 --- a/get-kisa-as-list +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/env python3 -import html -import html.parser -import sys - -def sanitize_as_name (s: str): - return ' '.join(s.split()) - -class KISAASListExtractor (html.parser.HTMLParser): - _table_started = False - _td = list[str]() - _tag_stack = list[str]() - - def handle_starttag (self, tag, attrs): - self._tag_stack.append(tag) # FIXME: don't push void elements - - if self._table_started: - if tag == 'tr': - self._td.clear() - else: - if tag == 'table': - attr_map = dict[str, str](attrs) - - if attr_map.get('class') == 'datatable': - self._table_started = True - - def handle_endtag (self, tag): - if self._table_started and tag == 'table': - self._table_started = False - if tag == 'tr' and self._td: - print("%-12s\t%s" % (self._td[1], sanitize_as_name(self._td[0]))) - - self._tag_stack.pop() - - def handle_data (self, data): - if self._table_started and self._tag_stack[-1] == 'td': - self._td.append(data) - - -doc_parser = KISAASListExtractor() -doc_parser.feed(sys.stdin.read()) diff --git a/get-kisa-asmap b/get-kisa-asmap new file mode 100755 index 0000000..b4b1c10 --- /dev/null +++ b/get-kisa-asmap @@ -0,0 +1,41 @@ +#!/bin/env python3 +import html +import html.parser +import sys + +def sanitize_as_name (s: str): + return ' '.join(s.split()) + +class KISAASListExtractor (html.parser.HTMLParser): + _table_started = False + _td = list[str]() + _tag_stack = list[str]() + + def handle_starttag (self, tag, attrs): + self._tag_stack.append(tag) # FIXME: don't push void elements + + if self._table_started: + if tag == 'tr': + self._td.clear() + else: + if tag == 'table': + attr_map = dict[str, str](attrs) + + if attr_map.get('class') == 'datatable': + self._table_started = True + + def handle_endtag (self, tag): + if self._table_started and tag == 'table': + self._table_started = False + if tag == 'tr' and self._td: + print("%-12s\t%s" % (self._td[1], sanitize_as_name(self._td[0]))) + + self._tag_stack.pop() + + def handle_data (self, data): + if self._table_started and self._tag_stack[-1] == 'td': + self._td.append(data) + + +doc_parser = KISAASListExtractor() +doc_parser.feed(sys.stdin.read()) diff --git a/query-prefixes b/query-prefixes index dc1bff6..746dc01 100755 --- a/query-prefixes +++ b/query-prefixes @@ -1,4 +1,5 @@ #!/bin/bash +set -e . common.sh PIPE_NAME="$$.fifo.tmp" @@ -14,7 +15,6 @@ read_aslist () { while read asn do - echo ${asn} >&3 echo ${Q_PREFIX}${asn} done } @@ -30,13 +30,5 @@ output_filter () { } -mkfifo "$PIPE_NAME" || exit 1 -grep -Eoi 'AS[0-9]+' | - read_aslist 3> "$PIPE_NAME" | - open_db | - extract_answer | - output_filter \ - 3< "$PIPE_NAME" & -rm "$PIPE_NAME" - -wait +set -o pipefail +read_aslist | open_db | extract_answer | output_filter diff --git a/query-rpki-prefixes b/query-rpki-prefixes index 34c365a..5ed7815 100755 --- a/query-rpki-prefixes +++ b/query-rpki-prefixes @@ -1,4 +1,5 @@ #!/bin/bash +set -e . common.sh read_prefixes () { @@ -25,4 +26,5 @@ process_result () { } +set -o pipefail read_prefixes | open_db | extract_answer | process_result -- cgit