MiniABS 1.1: HTTPS support

- Bundle current root certificates (Let's Encrypt, GTS, Sectigo, DigiCert G2,
  Amazon, ...) missing from Android 2.x's store (Roots.java, tools/gen_roots.py)
- Ask once for unknown certificates, showing the SHA-256 fingerprint, and
  remember the answer; replaces "accept any certificate" (Trust*, Settings)
- Local streaming proxy so Android 2.x's MediaPlayer can play https:// servers,
  with seeking and a fresh token per request (StreamProxy.java)
- Tests over real TLS with good, expired, wrong-host, forged and self-signed
  certificates (test/make_pki.sh, TrustTest.java)
- README: server setup for a legacy TLS 1.0 port

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/com/miniabs/ApiTest.java

@@ -48,7 +48,7 @@ public class ApiTest {
         check(cut.getString("d").equals("0123456789…"), "long strings truncated with an ellipsis");
 
         System.out.println("Sign-in");
-        p.setServer(LoginActivity_normalize(base + "/login/"), false);
+        p.setServer(LoginActivity_normalize(base + "/login/"));
         check(p.server().equals(base), "address normalised (" + p.server() + ")");
         check(Api.status(c, p.server()).equals("2.30.0"), "status check finds the server");
         try {
@@ -193,7 +193,7 @@ public class ApiTest {
         }
 
         System.out.println("Sub-path servers");
-        p.setServer("http://nas.local:8080/abs", false);
+        p.setServer("http://nas.local:8080/abs");
         check(Api.absolute(c, "/abs/api/items/x/file/1").equals("http://nas.local:8080/abs/api/items/x/file/1"),
                 "content URL under a sub-path not doubled");
         check(Api.absolute(c, "/api/items/x/file/1").equals("http://nas.local:8080/abs/api/items/x/file/1"),

test/com/miniabs/TrustTest.java

@@ -0,0 +1,172 @@
+package com.miniabs;
+
+import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.MessageDigest;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import android.content.Context;
+
+/** HTTPS: certificate trust, pinning, attacks, raw PATCH and the streaming proxy, over real TLS. */
+public class TrustTest {
+    static int pass, fail;
+    static Context c = new Context();
+    static Prefs p = Prefs.get(c);
+
+    static void check(boolean ok, String what) {
+        if (ok) pass++;
+        else fail++;
+        System.out.println((ok ? "  PASS  " : "  FAIL  ") + what);
+    }
+
+    static X509Certificate cert(String path) throws Exception {
+        return (X509Certificate) CertificateFactory.getInstance("X.509").generateCertificate(new FileInputStream(path));
+    }
+
+    /** Tries /status on a server; returns the exception, or null on success. */
+    static Exception tryStatus(int port) {
+        Trust.lastRejected = null;
+        try {
+            Api.status(c, "https://localhost:" + port);
+            return null;
+        } catch (Exception e) {
+            return e;
+        }
+    }
+
+    static byte[] get(String url, String range, int[] code, String[] contentRange) throws Exception {
+        HttpURLConnection h = (HttpURLConnection) new URL(url).openConnection();
+        if (range != null) h.setRequestProperty("Range", range);
+        code[0] = h.getResponseCode();
+        if (contentRange != null) contentRange[0] = h.getHeaderField("Content-Range");
+        if (code[0] >= 400) return new byte[0];
+        InputStream in = h.getInputStream();
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        byte[] b = new byte[8192];
+        int n;
+        while ((n = in.read(b)) > 0) out.write(b, 0, n);
+        in.close();
+        return out.toByteArray();
+    }
+
+    static byte[] audio(int from, int to) {
+        byte[] b = new byte[to - from + 1];
+        for (int i = from; i <= to; i++) b[i - from] = (byte) (i % 251);
+        return b;
+    }
+
+    public static void main(String[] a) throws Exception {
+        String pki = a[0];
+        int good = Integer.parseInt(a[1]), wrongHost = Integer.parseInt(a[2]), forged = Integer.parseInt(a[3]),
+                expired = Integer.parseInt(a[4]), self = Integer.parseInt(a[5]);
+
+        System.out.println("Bundled roots");
+        List<X509Certificate> roots = new ArrayList<X509Certificate>();
+        for (String r : Roots.DER_BASE64) roots.add(Trust.parse(android.util.Base64.decode(r, 0)));
+        check(roots.size() == 12, "12 modern root certificates bundled");
+        String isrg = Trust.fingerprint(roots.get(0)).replace("\n", ":");
+        check(isrg.equals("96:BC:EC:06:26:49:76:F3:74:60:77:9A:CF:28:C5:A7:CF:E8:A3:C0:AA:E1:1A:8F:FC:EE:05:C0:BD:DF:08:C6"),
+                "Let's Encrypt ISRG Root X1 matches its published fingerprint");
+        X509Certificate root = cert(pki + "/root.crt"), inter = cert(pki + "/int.crt"), leaf = cert(pki + "/leaf.crt");
+        check(Trust.verify(new X509Certificate[]{leaf, inter}, Arrays.asList(root)) == null,
+                "leaf + intermediate validate up to a bundled-style root");
+        check("unknown issuer".equals(Trust.verify(new X509Certificate[]{leaf}, Arrays.asList(root))),
+                "missing intermediate is rejected");
+
+        System.out.println("Unknown certificate, then trusting it");
+        p.setServer("https://localhost:" + good);
+        Exception e = tryStatus(good);
+        Trust.Rejected r = Trust.recentRejection();
+        check(e instanceof javax.net.ssl.SSLException && r != null, "unrecognised certificate rejected and remembered");
+        check(Util.describe(e).equals("This phone doesn't recognise the server's certificate."),
+                "error message: \"" + Util.describe(e) + "\"");
+        String text = Trust.describe(r, "localhost");
+        byte[] d = MessageDigest.getInstance("SHA-256").digest(leaf.getEncoded());
+        String fp = String.format("%02X:%02X:%02X", d[0], d[1], d[2]);
+        check(text.contains(fp) && text.contains("Issued by: Test Intermediate R1")
+                        && text.contains("trust certificates issued by \"Test Intermediate R1\""),
+                "prompt shows the server's fingerprint and what will be trusted");
+        X509Certificate pinned = Trust.pin(c, r.chain);
+        check(pinned.equals(inter), "trusting pins the issuing CA, so renewals keep working");
+        check(tryStatus(good) == null, "server reachable after trusting");
+
+        System.out.println("Signed in over HTTPS");
+        Api.login(c, p.server(), "demo", "pw");
+        check(Api.libraries(c).length() == 2, "API calls work");
+        Api.setProgress(c, "pod1", "ep7", 100, 1800, Boolean.TRUE);
+        JSONArray log = (JSONArray) Api.call(c, "GET", "/__log", null);
+        boolean patched = false;
+        for (int i = 0; i < log.length(); i++) {
+            JSONObject l = log.getJSONObject(i);
+            if (l.getString("m").equals("PATCH") && l.getString("p").equals("/api/me/progress/pod1/ep7")) patched = true;
+        }
+        check(patched, "mark-as-played PATCH works over TLS (raw socket path)");
+
+        System.out.println("Streaming proxy");
+        check(StreamProxy.needed(c), "https server streams through the proxy");
+        String local = StreamProxy.get(c).register("/api/items/pod1/file/ino7");
+        check(local.startsWith("http://127.0.0.1:"), "player gets a local http:// URL: " + local.replaceAll("/[0-9a-f]{24}", "/<id>"));
+        int[] code = new int[1];
+        String[] cr = new String[1];
+        byte[] whole = get(local, null, code, null);
+        check(code[0] == 200 && Arrays.equals(whole, audio(0, 199999)), "whole file streamed correctly (200 KB)");
+        byte[] part = get(local, "bytes=150000-150999", code, cr);
+        check(code[0] == 206 && "bytes 150000-150999/200000".equals(cr[0]) && Arrays.equals(part, audio(150000, 150999)),
+                "seek: Range request forwarded, 206 + Content-Range passed back");
+        byte[] tail = get(local, "bytes=199000-", code, cr);
+        check(code[0] == 206 && Arrays.equals(tail, audio(199000, 199999)), "open-ended range works");
+        int before = ((JSONObject) Api.call(c, "GET", "/__refreshes", null)).getInt("n");
+        Api.call(c, "GET", "/__expire", null);
+        byte[] again = get(local, "bytes=0-99", code, cr);
+        int after = ((JSONObject) Api.call(c, "GET", "/__refreshes", null)).getInt("n");
+        check(code[0] == 206 && Arrays.equals(again, audio(0, 99)) && after == before + 1,
+                "token expired mid-episode: proxy refreshes it and the stream continues");
+        get(local.replaceAll("/[0-9a-f]{24}", "/000000000000000000000000"), null, code, null);
+        check(code[0] == 404, "unknown stream id refused (other apps can't use the proxy)");
+
+        System.out.println("Attacks and broken certificates");
+        Exception wh = tryStatus(wrongHost);
+        check(wh != null, "certificate for another host rejected, even from a trusted CA");
+        Exception fe = tryStatus(forged);
+        Trust.Rejected fr = Trust.recentRejection();
+        check(fe != null && fr != null && fr.reason.contains("isn't a CA"),
+                "forged certificate signed by a normal site certificate rejected (" + (fr == null ? "-" : fr.reason) + ")");
+        Trust.pin(c, fr.chain);
+        check(tryStatus(forged) != null, "...and still rejected even if the user taps Trust");
+        Exception ee = tryStatus(expired);
+        check(ee != null && Trust.recentRejection() != null && Trust.recentRejection().expired()
+                        && Util.describe(ee).contains("expired"),
+                "expired certificate rejected: \"" + Util.describe(ee) + "\"");
+
+        System.out.println("Self-signed server");
+        Exception se = tryStatus(self);
+        Trust.Rejected sr = Trust.recentRejection();
+        check(se != null && sr != null, "self-signed certificate needs confirming");
+        X509Certificate selfPin = Trust.pin(c, sr.chain);
+        check(selfPin.equals(cert(pki + "/self.crt")) && tryStatus(self) == null, "after trusting, it connects");
+
+        System.out.println("Forgetting");
+        check(Trust.pinned(c).size() == 2, "two certificates trusted (" + Trust.pinned(c).size() + ")");
+        Trust.forgetPins(c);
+        check(tryStatus(good) != null && tryStatus(self) != null, "forgetting them brings the prompt back");
+
+        System.out.println("Host names");
+        check(Trust.hostMatches(leaf, "localhost") && Trust.hostMatches(leaf, "127.0.0.1")
+                && !Trust.hostMatches(leaf, "evil.com"), "SAN DNS and IP matching");
+
+        StreamProxy.shutdown();
+        System.out.println();
+        System.out.println(pass + " passed, " + fail + " failed");
+        System.exit(fail == 0 ? 0 : 1);
+    }
+}

test/make_pki.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+# Test certificates: a root CA, an intermediate CA, and server certificates, good and bad.
+set -e
+D=${1:-build/pki}
+rm -rf "$D" && mkdir -p "$D" && cd "$D"
+q() { openssl "$@" > /dev/null 2>err.log || { cat err.log; exit 1; }; }
+
+cat > ca.ext <<'EOF'
+basicConstraints=critical,CA:TRUE
+keyUsage=critical,keyCertSign,cRLSign
+EOF
+cat > leaf.ext <<'EOF'
+basicConstraints=CA:FALSE
+keyUsage=critical,digitalSignature,keyEncipherment
+extendedKeyUsage=serverAuth
+subjectAltName=DNS:localhost,IP:127.0.0.1
+EOF
+cat > other.ext <<'EOF'
+basicConstraints=CA:FALSE
+keyUsage=critical,digitalSignature,keyEncipherment
+extendedKeyUsage=serverAuth
+subjectAltName=DNS:other.example
+EOF
+
+q req -x509 -newkey rsa:2048 -nodes -keyout root.key -out root.crt -days 3650 -subj "/CN=Test Root CA" -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign"
+q req -newkey rsa:2048 -nodes -keyout int.key -out int.csr -subj "/CN=Test Intermediate R1"
+q x509 -req -in int.csr -CA root.crt -CAkey root.key -CAcreateserial -out int.crt -days 1825 -extfile ca.ext
+
+# good server certificate
+q req -newkey rsa:2048 -nodes -keyout leaf.key -out leaf.csr -subj "/CN=localhost"
+q x509 -req -in leaf.csr -CA int.crt -CAkey int.key -CAcreateserial -out leaf.crt -days 90 -extfile leaf.ext
+cat leaf.crt int.crt > good.chain
+
+# certificate for a different host, same CA
+q req -newkey rsa:2048 -nodes -keyout other.key -out other.csr -subj "/CN=other.example"
+q x509 -req -in other.csr -CA int.crt -CAkey int.key -CAcreateserial -out other.crt -days 90 -extfile other.ext
+cat other.crt int.crt > wronghost.chain
+
+# forgery: an attacker uses their own valid (non-CA) certificate to sign one for localhost
+q req -newkey rsa:2048 -nodes -keyout forged.key -out forged.csr -subj "/CN=localhost"
+q x509 -req -in forged.csr -CA other.crt -CAkey other.key -CAcreateserial -out forged.crt -days 90 -extfile leaf.ext
+cat forged.crt other.crt int.crt > forged.chain
+cp forged.key forged.chain.key
+
+# expired certificate
+q req -newkey rsa:2048 -nodes -keyout old.key -out old.csr -subj "/CN=localhost"
+python3 - <<'PY'
+import datetime
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+ca = x509.load_pem_x509_certificate(open("int.crt", "rb").read())
+ca_key = serialization.load_pem_private_key(open("int.key", "rb").read(), None)
+csr = x509.load_pem_x509_csr(open("old.csr", "rb").read())
+cert = (x509.CertificateBuilder().subject_name(csr.subject).issuer_name(ca.subject)
+        .public_key(csr.public_key()).serial_number(x509.random_serial_number())
+        .not_valid_before(datetime.datetime(2020, 1, 1)).not_valid_after(datetime.datetime(2021, 1, 1))
+        .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=False)
+        .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False)
+        .sign(ca_key, hashes.SHA256()))
+open("old.crt", "wb").write(cert.public_bytes(serialization.Encoding.PEM))
+PY
+cat old.crt int.crt > expired.chain
+
+# self-signed server
+q req -x509 -newkey rsa:2048 -nodes -keyout self.key -out self.crt -days 365 -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
+cp self.crt self.chain
+
+cp leaf.key good.chain.key; cp other.key wronghost.chain.key; cp old.key expired.chain.key; cp self.key self.chain.key
+ls *.chain

test/mock_abs.py

@@ -14,6 +14,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 PORT = int(sys.argv[1])
 LEGACY = len(sys.argv) > 2 and sys.argv[2] == "legacy"
+CHAIN = sys.argv[3] if len(sys.argv) > 3 else None  # serve HTTPS with this certificate chain
+AUDIO = bytes(i % 251 for i in range(200000))
 lock = threading.Lock()
 state = {"access": None, "refresh": None, "n": 0, "log": [], "progress": {}, "refreshes": 0}
 
@@ -159,6 +161,26 @@ class H(BaseHTTPRequestHandler):
                                    "currentTime": 600.0, "duration": 1807.5, "playMethod": 0,
                                    "audioTracks": [{"index": 1, "contentUrl": "/api/items/pod1/file/ino7",
                                                     "mimeType": "audio/mpeg", "duration": 1807.5}]})
+        if path == "/api/items/pod1/file/ino7":
+            rng = self.headers.get("Range")
+            if rng and rng.startswith("bytes="):
+                a, _, b = rng[6:].partition("-")
+                a = int(a)
+                b = int(b) if b else len(AUDIO) - 1
+                part = AUDIO[a:b + 1]
+                self.send_response(206)
+                self.send_header("Content-Range", "bytes %d-%d/%d" % (a, a + len(part) - 1, len(AUDIO)))
+            else:
+                part = AUDIO
+                self.send_response(200)
+            self.send_header("Content-Type", "audio/mpeg")
+            self.send_header("Accept-Ranges", "bytes")
+            self.send_header("Content-Length", str(len(part)))
+            self.send_header("Connection", "close")
+            self.end_headers()
+            if method != "HEAD":
+                self.wfile.write(part)
+            return
         if path.startswith("/api/session/") and method == "POST":
             return self.send(200, "OK", "text/plain")
         if path.startswith("/api/me/progress/") and method == "PATCH":
@@ -172,8 +194,19 @@ class H(BaseHTTPRequestHandler):
     def do_POST(self):
         self.handle_any("POST")
 
+    def do_HEAD(self):
+        self.handle_any("HEAD")
+
     def do_PATCH(self):
         self.handle_any("PATCH")
 
 
-ThreadingHTTPServer(("127.0.0.1", PORT), H).serve_forever()
+srv = ThreadingHTTPServer(("127.0.0.1", PORT), H)
+if CHAIN:
+    import ssl
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    ctx.load_cert_chain(CHAIN, CHAIN + ".key")
+    srv.socket = ctx.wrap_socket(srv.socket, server_side=True)
+    # a TLS handshake the client rejects shows up as an error here; that's expected in tests
+    srv.handle_error = lambda request, client_address: None
+srv.serve_forever()

test/run_tests.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Runs MiniABS's API/JSON/progress code on the desktop JVM against mock_abs.py.
+# Runs MiniABS's API/JSON/progress/HTTPS code on the desktop JVM against mock_abs.py.
 set -e
 cd "$(dirname "$0")/.."
 OUT=build/test
@@ -7,14 +7,30 @@ JSON_JAR=/usr/share/java/com.android.json.jar
 rm -rf "$OUT" && mkdir -p "$OUT"
 S=src/com/miniabs
 javac -nowarn -encoding UTF-8 -cp "$JSON_JAR" -d "$OUT" $(find test/shim -name '*.java') \
-    $S/Json.java $S/Prefs.java $S/Episode.java $S/Util.java $S/Api.java $S/App.java test/com/miniabs/ApiTest.java
+    $S/Json.java $S/Prefs.java $S/Episode.java $S/Util.java $S/Api.java $S/App.java \
+    $S/Trust.java $S/Roots.java $S/StreamProxy.java \
+    test/com/miniabs/ApiTest.java test/com/miniabs/TrustTest.java
+CP="$OUT:$JSON_JAR"
+PIDS=""
+cleanup() { [ -n "$PIDS" ] && kill $PIDS 2>/dev/null || true; }
+trap cleanup EXIT
+port() { echo $((20000 + $(od -An -N2 -tu2 /dev/urandom | tr -d ' ') % 20000)); }
 
 for mode in current legacy; do
-    PORT=$((18000 + $(od -An -N2 -tu2 /dev/urandom | tr -d ' ') % 1000))
-    python3 test/mock_abs.py $PORT $mode &
-    PID=$!
+    P=$(port)
+    python3 test/mock_abs.py $P $mode & PIDS="$PIDS $!"
     sleep 1
-    echo "=== $mode server"
-    java -cp "$OUT:$JSON_JAR" com.miniabs.ApiTest $PORT $mode || { kill $PID; exit 1; }
-    kill $PID
+    echo "=== $mode server (http)"
+    java -cp "$CP" com.miniabs.ApiTest $P $mode
 done
+
+echo "=== https"
+test/make_pki.sh build/pki > /dev/null
+ARGS="build/pki"
+for chain in good wronghost forged expired self; do
+    P=$(port)
+    python3 test/mock_abs.py $P current build/pki/$chain.chain & PIDS="$PIDS $!"
+    ARGS="$ARGS $P"
+done
+sleep 1
+java -cp "$CP" com.miniabs.TrustTest $ARGS

test/shim/android/util/Base64.java

@@ -1,5 +1,8 @@
 package android.util;
 public class Base64 {
-    public static final int URL_SAFE = 8;
-    public static byte[] decode(String s, int flags) { return java.util.Base64.getUrlDecoder().decode(s); }
+    public static final int DEFAULT = 0, NO_WRAP = 2, URL_SAFE = 8;
+    public static byte[] decode(String s, int flags) {
+        return (flags & URL_SAFE) != 0 ? java.util.Base64.getUrlDecoder().decode(s) : java.util.Base64.getMimeDecoder().decode(s);
+    }
+    public static String encodeToString(byte[] b, int flags) { return java.util.Base64.getEncoder().encodeToString(b); }
 }