HyperShreds: premium Solana shreds for $250 a month

Which of two feeds delivers each shred first? Python 3, standard library only.

#!/usr/bin/env python3# first_shred.py: which of two feeds delivers each shred first?# Point one destination at each port below, then run this on that one machine: python3 first_shred.py [seconds]import selectors, socket, struct, sys, time
FEEDS = {"current": 20001, "hypershreds": 20002}   # feed name -> UDP port on this machineSECONDS = int(sys.argv[1]) if len(sys.argv) > 1 else 60HEADER = 83   # common shred header: signature 64, variant 1, slot 8, index 4, version 2, fec set 4
sel = selectors.DefaultSelector()for name, port in FEEDS.items():    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)    sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8 << 20)    sock.bind(("0.0.0.0", port))    sock.setblocking(False)    sel.register(sock, selectors.EVENT_READ, name)
arrived = {}                          # (slot, index, variant) -> {feed: first arrival in ns}packets = dict.fromkeys(FEEDS, 0)end = time.monotonic() + SECONDSwhile time.monotonic() < end:    for key, _ in sel.select(timeout=1):        while True:            try:                pkt = key.fileobj.recv(2048)            except BlockingIOError:                break            now = time.perf_counter_ns()            if len(pkt) < HEADER or pkt.startswith(b"SHREDCAST-VERIFY"):                continue            slot, index = struct.unpack_from("<QI", pkt, 65)            arrived.setdefault((slot, index, pkt[64]), {}).setdefault(key.data, now)            packets[key.data] += 1
a, b = FEEDSboth = [t for t in arrived.values() if a in t and b in t]print("packets received:", packets)print("shreds seen on both feeds:", len(both))for first, second in ((a, b), (b, a)):    lead = sorted((t[second] - t[first]) / 1e6 for t in both if t[first] < t[second])    if lead:        p95 = lead[min(len(lead) - 1, int(len(lead) * 0.95))]        print(f"{first} first on {len(lead) / len(both):.0%} of them: median {lead[len(lead) // 2]:.2f} ms ahead, 95th percentile {p95:.2f} ms")

How many FEC sets does each feed deliver in full? Python 3, standard library only.

#!/usr/bin/env python3# fec_check.py: how many FEC sets does each feed deliver in full?# Point one destination at each port below, then run this on that one machine: python3 fec_check.py [seconds]import selectors, socket, struct, sys, timefrom collections import Counter
FEEDS = {"current": 20001, "hypershreds": 20002}   # feed name -> UDP port on this machineSECONDS = int(sys.argv[1]) if len(sys.argv) > 1 else 60HEADER = 83   # common shred header; a coding shred adds num_data, num_coding and position (u16 each)
def kind(variant):   # "data", "code", or None for a variant this script does not know    if variant == 0xA5 or variant & 0xF0 in (0x80, 0x90, 0xB0):        return "data"    if variant == 0x5A or variant & 0xF0 in (0x40, 0x60, 0x70):        return "code"
sel = selectors.DefaultSelector()for name, port in FEEDS.items():    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)    sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8 << 20)    sock.bind(("0.0.0.0", port))    sock.setblocking(False)    sel.register(sock, selectors.EVENT_READ, name)
new = lambda: {"data": set(), "code": set(), "need": None}   # data indexes, coding positions, data shreds in the setsets = {name: {} for name in FEEDS}                          # feed -> (slot, fec set) -> new()end = time.monotonic() + SECONDSwhile time.monotonic() < end:    for key, _ in sel.select(timeout=1):        while True:            try:                pkt = key.fileobj.recv(2048)            except BlockingIOError:                break            if len(pkt) < HEADER or pkt.startswith(b"SHREDCAST-VERIFY") or not kind(pkt[64]):                continue            slot, index = struct.unpack_from("<QI", pkt, 65)            (fec_set,) = struct.unpack_from("<I", pkt, 79)            fec = sets[key.data].setdefault((slot, fec_set), new())            if kind(pkt[64]) == "data":                fec["data"].add(index)            elif len(pkt) >= HEADER + 6:                need, _, position = struct.unpack_from("<HHH", pkt, HEADER)                fec["code"].add(position)                fec["need"] = need
def verdict(fec):    if fec["need"] is None:        return "unknown"       # no coding shred seen, so the size of the set is not known    if len(fec["data"]) >= fec["need"]:        return "complete"      # every data shred arrived    if len(fec["data"]) + len(fec["code"]) >= fec["need"]:        return "recoverable"   # some are missing, but enough shreds arrived to rebuild them    return "incomplete"
both = {}                      # what you would hold if you ran both feeds and combined themfor view in sets.values():    for key, fec in view.items():        merged = both.setdefault(key, new())        merged["data"] |= fec["data"]        merged["code"] |= fec["code"]        merged["need"] = merged["need"] or fec["need"]
slots = {key[0] for key in both}partial = {min(slots), max(slots)} if slots else set()   # the first and last slots are only partly capturedprint(f"{'':13}{'sets':>6}{'complete':>10}{'recoverable':>13}{'incomplete':>12}{'unknown':>9}")for name, view in {**sets, "both feeds": both}.items():    kept = [fec for key, fec in view.items() if key[0] not in partial]    count = Counter(verdict(fec) for fec in kept)    share = lambda v: f"{count[v] / len(kept):.0%}" if kept else "-"    print(f"{name:13}{len(kept):6}{share('complete'):>10}{share('recoverable'):>13}{share('incomplete'):>12}{share('unknown'):>9}")