diff --git a/tests/test_ia.py b/tests/test_ia.py index 77ff8f5..45b0470 100644 --- a/tests/test_ia.py +++ b/tests/test_ia.py @@ -12,6 +12,7 @@ import warc2zip from warc2zip import ( + IA_DOWNLOAD_URL, InternetArchiveFileSystem, cli, default_output_path, @@ -227,6 +228,19 @@ def test_authorization_does_not_leave_the_domain(ia_server, warc_path): assert "Authorization" not in data_hop.headers +def test_authorization_is_not_sent_over_plain_http(ia_server, warc_path, monkeypatch): + """With an https download URL, an http hop inside `auth_domain` gets no Authorization header: + the keys would cross the network in clear text.""" + ia_server.add("public", warc_path) + monkeypatch.setattr(InternetArchiveFileSystem, "download_url", IA_DOWNLOAD_URL) + url = f"http://localhost:{ia_server.node_port}/items/public/{warc_path.name}" + + fs = InternetArchiveFileSystem(access_key="k", secret_key="s", skip_instance_cache=True) + assert fs.cat_file(url, start=0, end=2) == b"\x1f\x8b" # gzip magic + assert ia_server.hits + assert not any("Authorization" in hit.headers for hit in ia_server.hits) + + def test_cli_explains_a_refused_ia_item(ia_server, warc_path, tmp_path, monkeypatch, capsys): uri = ia_server.add("restricted", warc_path) ia_server.require_auth = "LOW nope:nope" diff --git a/warc2zip.py b/warc2zip.py index 3192aa4..d337672 100644 --- a/warc2zip.py +++ b/warc2zip.py @@ -123,18 +123,22 @@ def in_domain(host, domain): return host == domain or host.endswith("." + domain) -def authorized_request_class(authorization, domain, base=aiohttp.ClientRequest): +def authorized_request_class(authorization, domain, base=aiohttp.ClientRequest, https_only=True): """A ``ClientRequest`` that sends `authorization` to every host in `domain`. aiohttp drops ``Authorization`` when a redirect changes origin, and every archive.org download is such a redirect (to a data node). It builds a new request from the session's ``request_class`` for each hop, so setting the header here puts it back on the data-node hop; - testing the host keeps it from leaving `domain` should a data node redirect elsewhere. + testing the host keeps it from leaving `domain` should a data node redirect elsewhere. With + `https_only`, a plain http hop gets no header either, as aiohttp would also have dropped it on + an https -> http redirect. """ class AuthorizedRequest(base): def update_headers(self, headers): super().update_headers(headers) + if https_only and self.url.scheme != "https": + return if in_domain(self.url.host, domain): self.headers[hdrs.AUTHORIZATION] = authorization @@ -158,7 +162,7 @@ class InternetArchiveFileSystem(HTTPFileSystem): IA S3 keys, sent as ``Authorization: LOW :`` to every archive.org host, including the data node the download URL redirects to (aiohttp drops the header on a cross-origin redirect; the session's request class puts it back, and only for - ``.archive.org`` hosts). Both or neither; ``IA_ACCESS_KEY_ID`` / ``IA_SECRET_ACCESS_KEY`` + ``.archive.org`` hosts over https). Both or neither; ``IA_ACCESS_KEY_ID`` / ``IA_SECRET_ACCESS_KEY`` in the environment override ``ia.ini``. Pass empty strings to stay anonymous despite an ``ia.ini``. config_file: str, optional @@ -186,6 +190,7 @@ def __init__(self, access_key=None, secret_key=None, config_file=None, **kwargs) self.authorization, self.auth_domain, client_kwargs.get("request_class", aiohttp.ClientRequest), + https_only=self.download_url.startswith("https://"), ) kwargs["client_kwargs"] = client_kwargs super().__init__(**kwargs)