fix: only send ia:// credentials over https

tests/test_ia.py

@@ -12,6 +12,7 @@
 
 import warc2zip
 from warc2zip import (
+    IA_DOWNLOAD_URL,
     InternetArchiveFileSystem,
     cli,
     default_output_path,
@@ -227,6 +228,19 @@ def test_authorization_does_not_leave_the_domain(ia_server, warc_path):
     assert "Authorization" not in data_hop.headers
 
 
+def test_authorization_is_not_sent_over_plain_http(ia_server, warc_path, monkeypatch):
+    """With an https download URL, an http hop inside `auth_domain` gets no Authorization header:
+    the keys would cross the network in clear text."""
+    ia_server.add("public", warc_path)
+    monkeypatch.setattr(InternetArchiveFileSystem, "download_url", IA_DOWNLOAD_URL)
+    url = f"http://localhost:{ia_server.node_port}/items/public/{warc_path.name}"
+
+    fs = InternetArchiveFileSystem(access_key="k", secret_key="s", skip_instance_cache=True)
+    assert fs.cat_file(url, start=0, end=2) == b"\x1f\x8b"  # gzip magic
+    assert ia_server.hits
+    assert not any("Authorization" in hit.headers for hit in ia_server.hits)
+
+
 def test_cli_explains_a_refused_ia_item(ia_server, warc_path, tmp_path, monkeypatch, capsys):
     uri = ia_server.add("restricted", warc_path)
     ia_server.require_auth = "LOW nope:nope"

warc2zip.py

@@ -123,18 +123,22 @@ def in_domain(host, domain):
     return host == domain or host.endswith("." + domain)
 
 
-def authorized_request_class(authorization, domain, base=aiohttp.ClientRequest):
+def authorized_request_class(authorization, domain, base=aiohttp.ClientRequest, https_only=True):
     """A ``ClientRequest`` that sends `authorization` to every host in `domain`.
 
     aiohttp drops ``Authorization`` when a redirect changes origin, and every archive.org download
     is such a redirect (to a data node). It builds a new request from the session's
     ``request_class`` for each hop, so setting the header here puts it back on the data-node hop;
-    testing the host keeps it from leaving `domain` should a data node redirect elsewhere.
+    testing the host keeps it from leaving `domain` should a data node redirect elsewhere. With
+    `https_only`, a plain http hop gets no header either, as aiohttp would also have dropped it on
+    an https -> http redirect.
     """
 
     class AuthorizedRequest(base):
         def update_headers(self, headers):
             super().update_headers(headers)
+            if https_only and self.url.scheme != "https":
+                return
             if in_domain(self.url.host, domain):
                 self.headers[hdrs.AUTHORIZATION] = authorization
 
@@ -158,7 +162,7 @@ class InternetArchiveFileSystem(HTTPFileSystem):
         IA S3 keys, sent as ``Authorization: LOW <access>:<secret>`` to every archive.org host,
         including the data node the download URL redirects to (aiohttp drops the header on a
         cross-origin redirect; the session's request class puts it back, and only for
-        ``.archive.org`` hosts). Both or neither; ``IA_ACCESS_KEY_ID`` / ``IA_SECRET_ACCESS_KEY``
+        ``.archive.org`` hosts over https). Both or neither; ``IA_ACCESS_KEY_ID`` / ``IA_SECRET_ACCESS_KEY``
         in the environment override ``ia.ini``. Pass empty strings to stay anonymous despite an
         ``ia.ini``.
     config_file: str, optional
@@ -186,6 +190,7 @@ def __init__(self, access_key=None, secret_key=None, config_file=None, **kwargs)
                 self.authorization,
                 self.auth_domain,
                 client_kwargs.get("request_class", aiohttp.ClientRequest),
+                https_only=self.download_url.startswith("https://"),
             )
             kwargs["client_kwargs"] = client_kwargs
         super().__init__(**kwargs)