#!/usr/bin/env bash # Real demos for P065 "How the Internet Works" (Networking for Beginners, episode 9). # A tiny internet in Docker on this machine only: four separate networks (autonomous systems) running real BGP # with FRRouting, using the AS numbers reserved for documentation (RFC 5398: 64496-64511) and documentation # address blocks (RFC 5737). Nothing leaves this machine: every default route to Docker's gateway is removed. # # AS64500 Ana's internet provider router "isp" customers 203.0.113.0/24 (Ana's laptop .20) # AS64501 a big carrier (transit) router "carrier1" # AS64502 another carrier (transit) router "carrier2" # AS64503 the video site router "video" servers 198.51.100.0/24 (web server .10) # An exchange point (IXP): one shared switch, 192.0.2.0/24, where "isp" and "video" connect directly. # # isp --10.0.1.0/24-- carrier1 --10.0.2.0/24-- carrier2 --10.0.3.0/24-- video # isp --------------------- 192.0.2.0/24 (exchange point) --------------- video # # The story: the video site is running, but Ana can't reach it until it announces its addresses with BGP. # The announcement spreads network by network; Ana's provider hears two paths and picks the shorter one # (through the exchange point). The exchange cable is unplugged: after BGP's hold time the provider moves to # the carriers' path. Finally the video site withdraws its announcement (as Facebook's routers did on # 4 October 2021) and it vanishes for everyone. Plus two real numbers from RIPE NCC's routing data (RIS). set -euo pipefail cd "$(dirname "$0")" P=ce-p065 ROUTERS="isp carrier1 carrier2 video" HOSTS_="ana web" cleanup() { for b in $ROUTERS $HOSTS_; do docker rm -f $P-$b >/dev/null 2>&1 || true; done for n in home vid l1 l2 l3 ixp; do docker network rm $P-$n >/dev/null 2>&1 || true; done } cleanup trap cleanup EXIT W=/tmp/ce-p065 rm -rf $W && mkdir -p $W/img $W/cfg cat > $W/img/Dockerfile <<'DOCKER' FROM ubuntu:24.04 RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \ iproute2 iputils-ping traceroute curl python3 procps >/dev/null \ && rm -rf /var/lib/apt/lists/* \ && userdel -r ubuntu 2>/dev/null; useradd -m -s /bin/bash ana DOCKER docker build -q -t $P-host $W/img >/dev/null FRR=quay.io/frrouting/frr:10.4.1 # ------------------------------------------------------------ BGP settings for each network's router # Timers: keepalive 3 s, hold time 9 s (shortened for the demo; FRR's traditional default is 60/180 s). common() { # common ASN ROUTER_ID cat < $W/cfg/isp.conf { common 64501 10.255.0.2 carrier1 echo " neighbor 10.0.1.1 remote-as 64500"; echo " neighbor 10.0.2.2 remote-as 64502" } > $W/cfg/carrier1.conf { common 64502 10.255.0.3 carrier2 echo " neighbor 10.0.2.1 remote-as 64501"; echo " neighbor 10.0.3.2 remote-as 64503" } > $W/cfg/carrier2.conf { common 64503 10.255.0.4 video echo " neighbor 10.0.3.1 remote-as 64502"; echo " neighbor 192.0.2.1 remote-as 64500" echo " address-family ipv4 unicast" echo " neighbor 10.0.3.1 route-map ONLY-OURS out"; echo " neighbor 192.0.2.1 route-map ONLY-OURS out" echo " exit-address-family"; ours 198.51.100.0/24 } > $W/cfg/video.conf : > $W/cfg/vtysh.conf sed 's/^bgpd=no/bgpd=yes/' > $W/cfg/daemons < <(docker run --rm --entrypoint cat $FRR /etc/frr/daemons) # The video site's web page. cat > $W/site.py <<'PY' import http.server class H(http.server.BaseHTTPRequestHandler): server_version, sys_version = "video-site/1.0", "" def log_message(self, *a): pass def do_GET(self): b = b"video.example: now playing 'How tea is made'\n" self.send_response(200); self.send_header("Content-Length", str(len(b))); self.end_headers(); self.wfile.write(b) http.server.ThreadingHTTPServer(("0.0.0.0", 80), H).serve_forever() PY net() { docker network create --subnet $2 --gateway $3 $P-$1 >/dev/null; } net home 203.0.113.0/24 203.0.113.254 net vid 198.51.100.0/24 198.51.100.254 net l1 10.0.1.0/24 10.0.1.254 net l2 10.0.2.0/24 10.0.2.254 net l3 10.0.3.0/24 10.0.3.254 net ixp 192.0.2.0/24 192.0.2.254 router() { # router NAME FIRST_NET IP docker run -d --name $P-$1 --hostname $1 --privileged --sysctl net.ipv4.ip_forward=1 \ --sysctl net.ipv6.conf.all.disable_ipv6=1 --network $P-$2 --ip $3 \ -v $W/cfg/daemons:/etc/frr/daemons:ro -v $W/cfg/vtysh.conf:/etc/frr/vtysh.conf:ro \ --sysctl net.ipv4.conf.all.rp_filter=0 --sysctl net.ipv4.conf.default.rp_filter=0 -v $W/cfg/$1.conf:/etc/frr/frr.conf $FRR >/dev/null } conn() { docker network connect --ip $3 $P-$2 $P-$1; } router isp l1 10.0.1.1; conn isp home 203.0.113.1; conn isp ixp 192.0.2.1 router carrier1 l1 10.0.1.2; conn carrier1 l2 10.0.2.1 router carrier2 l2 10.0.2.2; conn carrier2 l3 10.0.3.1 router video l3 10.0.3.2; conn video vid 198.51.100.1; conn video ixp 192.0.2.3 docker run -d --name $P-ana --hostname laptop --cap-add NET_ADMIN --cap-add NET_RAW \ --sysctl net.ipv6.conf.all.disable_ipv6=1 --network $P-home --ip 203.0.113.20 $P-host sleep infinity >/dev/null docker run -d --name $P-web --hostname web --cap-add NET_ADMIN \ --sysctl net.ipv6.conf.all.disable_ipv6=1 --network $P-vid --ip 198.51.100.10 $P-host sleep infinity >/dev/null ex() { docker exec $P-$1 sh -c "$2"; } for r in $ROUTERS; do ex $r "ip route del default 2>/dev/null || true"; done ex ana "ip route replace default via 203.0.113.1" ex web "ip route replace default via 198.51.100.1" ex ana "echo '198.51.100.10 video.example' >> /etc/hosts" docker cp $W/site.py $P-web:/srv/site.py docker exec -d $P-web sh -c "exec python3 /srv/site.py" # Name each router's network cards after what they connect to, so traceroute and the tables read clearly. for r in $ROUTERS; do docker exec $P-$r sh -c 'for d in $(ls /sys/class/net | grep ^eth); do a=$(ip -4 -o addr show $d | awk "{print \$4}"); case $a in 10.0.1.*) n=to-carrier1;; 10.0.2.*) n=to-carrier2;; 10.0.3.*) n=to-video;; 192.0.2.*) n=exchange;; 203.0.113.*) n=customers;; 198.51.100.*) n=servers;; esac; ip link set $d down; ip link set $d name $n; ip link set $n up; done' 2>/dev/null || true done # carrier1 faces isp on 10.0.1, carrier2 faces carrier1 on 10.0.2, video faces carrier2 on 10.0.3: fix the names. ex carrier1 "ip link set to-carrier1 down; ip link set to-carrier1 name to-isp; ip link set to-isp up" ex carrier2 "ip link set to-carrier2 down; ip link set to-carrier2 name to-carrier1; ip link set to-carrier1 up" ex video "ip link set to-video down; ip link set to-video name to-carrier2; ip link set to-carrier2 up" for r in $ROUTERS; do ex $r "ip route del default 2>/dev/null || true"; done sleep 15 # BGP sessions come up (Established) # Real numbers from the real internet: RIPE NCC's Routing Information Service (public data, read-only). curl -s -m 30 "https://stat.ripe.net/data/ris-asns/data.json?list_asns=false" > $W/ris-asns.json curl -s -m 30 "https://stat.ripe.net/data/ris-prefixes/data.json?resource=AS32934&list_prefixes=false" > $W/ris-meta.json curl -s -m 30 "https://stat.ripe.net/data/as-overview/data.json?resource=AS32934" > $W/as-meta.json python3 - "$P" "$W" <<'PYDRIVER' import fcntl, json, os, pty, re, select, struct, subprocess, sys, termios, time P, W = sys.argv[1], sys.argv[2] steps, faults = {}, [] READY = re.compile(rb"__READY__:(.*?):__ ") def sh(cmd, check=True): r = subprocess.run(cmd, shell=True, capture_output=True, text=True) if check and r.returncode: raise SystemExit(f"FAILED: {cmd}\n{r.stdout}\n{r.stderr}") return r.stdout def fault(box, what, cmd): """Change something from outside the terminals (a cable pulled), and record exactly what was done.""" sh(f"docker exec {P}-{box} sh -c {json.dumps(cmd)}") faults.append({"box": box, "what": what, "cmd": cmd, "t": time.time()}) print(f"## {box}: {what} ({cmd})", flush=True) def clean(text): text = re.sub(r"\x1b\[[0-9;?]*[A-Za-z]|\x1b[()][A-Z0-9]", "", text) lines = [l.rstrip("\r").split("\r")[-1] for l in text.split("\n")] return "\n".join(l.rstrip() for l in lines).strip("\n") class Shell: """One interactive bash in a real terminal (80 columns) on one machine, like a terminal window.""" def __init__(self, box, user, host): self.host, self.user, self.where = host, user, "~" home = "/root" if user == "root" else f"/home/{user}" pid, fd = pty.fork() if pid == 0: fcntl.ioctl(0, termios.TIOCSWINSZ, struct.pack("HHHH", 40, 80, 0, 0)) os.execvp("docker", ["docker", "exec", "-it", "-u", user, "-w", home, "-e", f"HOME={home}", "-e", "TERM=xterm", "-e", "LANG=C.UTF-8", f"{P}-{box}", "bash", "-i"]) self.fd = fd os.write(fd, b"stty -echo; PS1='__REA''DY__:\\w:__ '\n") self.read() def read(self, timeout=180): buf, end = b"", time.time() + timeout while time.time() < end: if select.select([self.fd], [], [], 0.3)[0]: buf += os.read(self.fd, 65536) m = None for m in READY.finditer(buf): pass if m: self.where = m.group(1).decode() return clean(buf[:m.start()].decode("utf-8", "replace")) raise SystemExit(f"{self.host}: timed out; last output: {buf[-400:]!r}") def shown(self, cmd): sign = "#" if self.user == "root" else "$" return f"{self.user}@{self.host}:{self.where}{sign} " + cmd def run(self, key, cmd): shown, t0 = self.shown(cmd), time.time() os.write(self.fd, cmd.encode() + b"\n") o = self.read() self.save(key, shown, o, t0) def start(self, key, cmd): self.pending = (key, self.shown(cmd), time.time()) os.write(self.fd, cmd.encode() + b"\n") def finish(self): key, shown, t0 = self.pending self.save(key, shown, self.read(), t0) def save(self, key, shown, o, t0): assert key not in steps, key steps[key] = {"cmd": shown, "out": o, "host": self.host, "after_fault": len(faults), "seconds": round(time.time() - t0, 1), "t0": t0} print(f"== {key}\n{shown}\n{o}\n", flush=True) def wait_bgp(box, prefix, want_path, timeout=60): """Off-screen: wait until a router's best path for prefix is want_path ('' = no route).""" end = time.time() + timeout while time.time() < end: o = sh(f"docker exec {P}-{box} vtysh -c 'show ip bgp {prefix} json'", check=False) try: j = json.loads(o or "{}") except ValueError: j = {} best = [p for p in j.get("paths", []) if p.get("bestpath", {}).get("overall")] got = best[0]["aspath"]["string"] if best else "" if got == want_path: return round(time.time(), 2) time.sleep(0.25) raise SystemExit(f"{box}: best path for {prefix} never became {want_path!r}") # Every BGP session must be up before the story starts. for r in ("isp", "carrier1", "carrier2", "video"): o = sh(f"docker exec {P}-{r} vtysh -c 'show bgp summary json'") peers = json.loads(o)["ipv4Unicast"]["peers"] assert all(p["state"] == "Established" for p in peers.values()), (r, peers) ana = Shell("ana", "ana", "laptop") isp = Shell("isp", "root", "isp") c1 = Shell("carrier1", "root", "carrier1") c2 = Shell("carrier2", "root", "carrier2") video = Shell("video", "root", "video") V = "vtysh -c" # ---------------------------------------------------------------- the hook: a site nobody can find ana.run("hook_curl", "curl -m 5 http://video.example/") ana.run("hook_ping", "ping -n -c 2 video.example") isp.run("isp_before", f"{V} 'show ip bgp'") # ---------------------------------------------------------------- the video site announces its addresses video.run("announce", f"{V} 'configure' -c 'router bgp 64503' -c 'network 198.51.100.0/24'") t_ann = time.time() t_isp = wait_bgp("isp", "198.51.100.0/24", "64503") t_c1 = wait_bgp("carrier1", "198.51.100.0/24", "64502 64503") steps["spread"] = {"cmd": "timing", "out": json.dumps({"isp_seconds": round(t_isp - t_ann, 2), "carrier1_seconds": round(t_c1 - t_ann, 2)})} time.sleep(4) c2.run("carrier2_table", f"{V} 'show ip bgp'") c1.run("carrier1_table", f"{V} 'show ip bgp'") isp.run("isp_table", f"{V} 'show ip bgp'") wait_bgp("isp", "198.51.100.0/24", "64503") isp.run("isp_detail", f"{V} 'show ip bgp 198.51.100.0/24'") ana.run("curl_ok", "curl -m 5 http://video.example/") ana.run("trace_ixp", "traceroute -n -q 1 video.example") # ---------------------------------------------------------------- a cable breaks at the exchange point isp.run("hold", f"{V} 'show ip bgp neighbors 192.0.2.3' | grep -E 'Hold time is|BGP state'") ana.start("ping_cut", "ping -n -i 0.2 -c 100 -q video.example") time.sleep(4) fault("video", "the video site's cable at the exchange point is unplugged", "ip link set exchange down") t_cut = time.time() t_moved = wait_bgp("isp", "198.51.100.0/24", "64501 64502 64503") ana.finish() steps["cut_timing"] = {"cmd": "timing", "out": json.dumps({"reroute_seconds": round(t_moved - t_cut, 2)})} isp.run("isp_after_cut", f"{V} 'show ip bgp'") ana.run("trace_carriers", "traceroute -n -q 1 video.example") ana.run("curl_after_cut", "curl -m 5 http://video.example/") # ---------------------------------------------------------------- the video site withdraws (like Facebook) video.run("withdraw", f"{V} 'configure' -c 'router bgp 64503' -c 'no network 198.51.100.0/24'") t_w = time.time() wait_bgp("isp", "198.51.100.0/24", "") wait_bgp("carrier1", "198.51.100.0/24", "") steps["withdraw_timing"] = {"cmd": "timing", "out": json.dumps({"gone_seconds": round(time.time() - t_w, 2)})} c1.run("carrier1_gone", f"{V} 'show ip bgp 198.51.100.0/24'") ana.run("curl_gone", "curl -m 5 http://video.example/") ana.run("ping_gone", "ping -n -c 2 video.example") video.run("web_still_up", "wget -qO- http://198.51.100.10/") # ---------------------------------------------------------------- real routing data from the real internet asns = json.load(open(f"{W}/ris-asns.json"))["data"] meta = json.load(open(f"{W}/ris-meta.json"))["data"] ov = json.load(open(f"{W}/as-meta.json"))["data"] steps["ris"] = {"cmd": "RIPE NCC RIS (stat.ripe.net)", "out": json.dumps({ "asns_total": asns["counts"]["total"], "query_time": asns["query_time"], "meta_prefixes": meta.get("counts"), "meta_holder": ov.get("holder"), "meta_time": meta.get("query_time")})} steps["version"] = {"cmd": "versions", "out": sh( f"docker exec {P}-isp sh -c 'vtysh -c \"show version\" | head -1'; " f"docker exec {P}-ana sh -c '. /etc/os-release; echo $PRETTY_NAME'; uname -r; date -u +%Y-%m-%dT%H:%MZ").strip()} for f in faults: f.pop("t", None) for s in steps.values(): s.pop("t0", None) steps["faults"] = {"cmd": "faults", "out": json.dumps(faults)} json.dump(steps, open("demo_output.json", "w"), indent=1, ensure_ascii=False) print("saved demo_output.json") PYDRIVER