From ba85b22c5d0b1bd74c71267528597f402e28c04f Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 18 Oct 2026 11:33:12 +0200 Subject: [PATCH 01/11] Add a browser login step before the crawl A site behind a login form or a CAPTCHA cannot be mirrored today, because the engine has no way to pass the gate. The user now passes it in a WebView inside the app, and the crawl reuses that browser's session. "Log in with a browser" on the address pane opens the first address in a WebView. On "Use this login" the screen queries every host the user reached and writes a Netscape cookies.txt into private storage. That path goes to the existing CookiesFile field (-%K), and the WebView's exact User-Agent to UserID (-F). It also ticks Accept cookies, because cookie_load() never reads the file otherwise. It drops the connection count to 2 and appends -*logout*, so the crawl does not end its own session. The window opens https only. The app declares no cleartext permission, so a WebView will not load http at all, and a password does not belong on cleartext either. Verified against the pinned engine 67d3a174 rather than its documentation. A file in this format makes the engine send the cookie on every request. The same crawl without -%K sends none, which is the control. The #HttpOnly_ form that browser extensions export is skipped, so this writer never emits it. Verified on an emulator (API 36, WebView 133). CookieManager.getCookie returns an HttpOnly cookie where document.cookie does not. The written file is mode 600. The login survives a rotation and a background recreation. Signed-off-by: Xavier Roche Co-Authored-By: Claude Opus 5 (1M context) --- app/src/main/AndroidManifest.xml | 5 + .../httrack/android/BrowserLoginActivity.java | 288 ++++++++++++++++++ .../httrack/android/BrowserLoginCookies.java | 176 +++++++++++ .../httrack/android/BrowserLoginPolicy.java | 146 +++++++++ .../com/httrack/android/HTTrackActivity.java | 67 ++++ .../res/layout/activity_browser_login.xml | 67 ++++ .../main/res/layout/activity_proj_setup.xml | 9 + app/src/main/res/menu/browser_login.xml | 8 + app/src/main/res/values/strings.xml | 12 + .../android/BrowserLoginCookiesTest.java | 174 +++++++++++ .../android/BrowserLoginPolicyTest.java | 101 ++++++ .../android/BrowserLoginWiringTest.java | 118 +++++++ 12 files changed, 1171 insertions(+) create mode 100644 app/src/main/java/com/httrack/android/BrowserLoginActivity.java create mode 100644 app/src/main/java/com/httrack/android/BrowserLoginCookies.java create mode 100644 app/src/main/java/com/httrack/android/BrowserLoginPolicy.java create mode 100644 app/src/main/res/layout/activity_browser_login.xml create mode 100644 app/src/main/res/menu/browser_login.xml create mode 100644 app/src/test/java/com/httrack/android/BrowserLoginCookiesTest.java create mode 100644 app/src/test/java/com/httrack/android/BrowserLoginPolicyTest.java create mode 100644 app/src/test/java/com/httrack/android/BrowserLoginWiringTest.java diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 96ffad86..cf236c5e 100755 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -58,6 +58,11 @@ android:name="com.httrack.android.FileChooserActivity" android:label="@string/title_activity_file_chooser" > + + hosts = new LinkedHashSet(); + + private static final String STATE_HOSTS = "hosts"; + + @Override + protected void onCreate(final Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + EdgeToEdge.fitSystemWindows(this); + setContentView(R.layout.activity_browser_login); + + final Bundle extras = getIntent().getExtras(); + final String url = extras != null ? extras.getString(EXTRA_URL) : null; + cookiesFile = extras != null ? File.class.cast(extras.get(EXTRA_COOKIES_FILE)) : null; + if (url == null || cookiesFile == null) { + Log.w(getClass().getSimpleName(), "no address to log in to"); + finish(); + return; + } + + web = WebView.class.cast(findViewById(R.id.webLogin)); + final WebSettings settings = web.getSettings(); + // A login form usually needs scripts, and often its storage too. + settings.setJavaScriptEnabled(true); + settings.setDomStorageEnabled(true); + // A page here has no business reading our storage, and the mirrors are in it. + settings.setAllowFileAccess(false); + settings.setAllowContentAccess(false); + // A no-op from API 26 on, but it still keeps a password out of the form store below that. + settings.setSaveFormData(false); + final CookieManager cookies = CookieManager.getInstance(); + cookies.setAcceptCookie(true); + // Single sign-on sets its cookies from a third-party frame, so the login needs them. + cookies.setAcceptThirdPartyCookies(web, true); + web.setWebViewClient(new LoginClient()); + getOnBackPressedDispatcher().addCallback(this, backCallback); + if (savedInstanceState == null) { + web.loadUrl(url); + return; + } + // Recreated: reloading the first address would throw away a login half way through. + web.restoreState(savedInstanceState); + final List seen = savedInstanceState.getStringArrayList(STATE_HOSTS); + if (seen != null) { + hosts.addAll(seen); + } + } + + @Override + protected void onSaveInstanceState(final Bundle out) { + super.onSaveInstanceState(out); + if (web != null) { + web.saveState(out); + } + out.putStringArrayList(STATE_HOSTS, new ArrayList(hosts)); + } + + @Override + protected void onDestroy() { + if (web != null) { + web.destroy(); + web = null; + } + super.onDestroy(); + } + + /** Back walks the login's own history first, so a wrong turn does not leave the screen. */ + private final OnBackPressedCallback backCallback = new OnBackPressedCallback(true) { + @Override + public void handleOnBackPressed() { + if (web != null && web.canGoBack()) { + web.goBack(); + } else { + setResult(RESULT_CANCELED); + finish(); + } + } + }; + + /** + * Records where the user went and keeps the window on https. There is deliberately no + * onReceivedSslError, because its default refuses a certificate we could not verify. + */ + private final class LoginClient extends WebViewClient { + @Override + public void onPageStarted(final WebView view, final String url, final Bitmap favicon) { + final String host = BrowserLoginPolicy.hostOf(url); + if (host != null) { + hosts.add(host); + } + // The user is about to type a password, so name the site they are typing it into. + final ActionBar bar = getActionBar(); + if (bar != null) { + bar.setSubtitle(host); + } + } + + @Override + public boolean shouldOverrideUrlLoading(final WebView view, final WebResourceRequest request) { + final String url = request.getUrl().toString(); + if (BrowserLoginPolicy.hostOf(url) != null) { + return false; + } + // Say why nothing happened: the app declares no cleartext permission, so http stays blank. + Toast.makeText(BrowserLoginActivity.this, + getString(R.string.browser_login_https_only), Toast.LENGTH_LONG).show(); + return true; + } + } + + @Override + public boolean onCreateOptionsMenu(final Menu menu) { + getMenuInflater().inflate(R.menu.browser_login, menu); + return true; + } + + @Override + public boolean onOptionsItemSelected(final MenuItem item) { + if (item.getItemId() == R.id.action_login_forget) { + forget(); + return true; + } + return super.onOptionsItemSelected(item); + } + + /** Drop the saved session: the browser's cookies and the file the engine would have read. */ + private void forget() { + CookieManager.getInstance().removeAllCookies(null); + CookieManager.getInstance().flush(); + hosts.clear(); + if (cookiesFile.exists() && !cookiesFile.delete()) { + Log.w(getClass().getSimpleName(), "could not delete " + cookiesFile); + } + if (web != null) { + web.clearCache(true); + web.reload(); + } + Toast.makeText(this, getString(R.string.browser_login_forgotten), Toast.LENGTH_SHORT).show(); + } + + /** "Cancel" */ + public void onClickLoginCancel(final View view) { + setResult(RESULT_CANCELED); + finish(); + } + + /** "Use this login" */ + public void onClickLoginDone(final View view) { + CookieManager.getInstance().flush(); + final Map byHost = new LinkedHashMap(); + for (final String host : hosts) { + final String header = CookieManager.getInstance() + .getCookie(BrowserLoginPolicy.SCHEME + "://" + host + "/"); + if (header != null && header.length() != 0) { + byHost.put(host, header); + } + } + final BrowserLoginCookies.Result result = BrowserLoginCookies.file(byHost); + if (result.cookies == 0) { + Toast.makeText(this, getString(R.string.browser_login_no_cookies), Toast.LENGTH_LONG).show(); + return; + } + if (!writeOwnerOnly(cookiesFile, result.text)) { + Toast.makeText(this, getString(R.string.browser_login_write_failed), Toast.LENGTH_LONG) + .show(); + return; + } + final Intent data = new Intent(); + data.putExtra(EXTRA_USER_AGENT, web.getSettings().getUserAgentString()); + data.putExtra(EXTRA_COOKIE_COUNT, result.cookies); + data.putExtra(EXTRA_HOST_COUNT, result.hosts); + setResult(RESULT_OK, data); + finish(); + } + + /** + * Write TEXT to FILE, readable by this app alone. + * + * The file holds a live session, so it goes through a neighbour and a rename: a write that + * fails halfway would otherwise leave the previous login truncated and the crawl with nothing. + * + * @return true when FILE now holds TEXT + */ + static boolean writeOwnerOnly(final File file, final String text) { + final File dir = file.getParentFile(); + if (dir != null && !dir.isDirectory() && !dir.mkdirs()) { + return false; + } + final File staged = new File(file.getPath() + ".new"); + try (final OutputStream out = new FileOutputStream(staged)) { + out.write(text.getBytes(UTF8)); + } catch (final IOException e) { + Log.w(BrowserLoginActivity.class.getSimpleName(), "cannot write " + staged, e); + staged.delete(); + return false; + } + // Before the rename, so the name the engine reads is never briefly world-readable. + staged.setReadable(false, false); + staged.setWritable(false, false); + staged.setReadable(true, true); + staged.setWritable(true, true); + if (!staged.renameTo(file)) { + Log.w(BrowserLoginActivity.class.getSimpleName(), "cannot rename onto " + file); + staged.delete(); + return false; + } + return true; + } +} diff --git a/app/src/main/java/com/httrack/android/BrowserLoginCookies.java b/app/src/main/java/com/httrack/android/BrowserLoginCookies.java new file mode 100644 index 00000000..ff4298a1 --- /dev/null +++ b/app/src/main/java/com/httrack/android/BrowserLoginCookies.java @@ -0,0 +1,176 @@ +/* +HTTrack Android Java Interface. + +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 3 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + + +package com.httrack.android; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * The Netscape cookies.txt the in-app browser login hands to the engine through -%K. + * + * WebView gives us a "name=value; name2=value2" header string per host and no attributes, so a + * line states the host it came from, every subdomain of it, path "/" and a session expiry. + * Every length here is in bytes, because the engine's parse measures with strlen(). + */ +public final class BrowserLoginCookies { + private BrowserLoginCookies() { + } + + /** What cookie_load() reads a record's fields into, so a longer one is skipped there. */ + static final int MAX_DOMAIN = 255; + static final int MAX_NAME = 1023; + static final int MAX_VALUE = 8191; + + /** cookie_load() drops a whole line at or above its line_max of 8000. */ + static final int MAX_LINE = 7999; + + /** + * Bytes we may write. The engine's jar holds 30000 and loads the mirror's own cookies.txt + * first, so this leaves that file room and the jar room for its longer expiry field. + */ + static final int MAX_BYTES = 20000; + + private static final Charset UTF8 = Charset.forName("UTF-8"); + + private static final String HEADER = "# Netscape HTTP Cookie File\n" + + "# Saved by HTTrack Android from the in-app browser login.\n"; + + /** What {@link #file} produced, for the message that tells the user what was saved. */ + public static final class Result { + /** The file text, header included. */ + public final String text; + + public final int cookies; + + /** Hosts that gave at least one cookie. */ + public final int hosts; + + /** Cookies left out, because a field was too long or the budget ran out. */ + public final int dropped; + + Result(final String text, final int cookies, final int hosts, final int dropped) { + this.text = text; + this.cookies = cookies; + this.hosts = hosts; + this.dropped = dropped; + } + } + + static int bytes(final String s) { + return s.getBytes(UTF8).length; + } + + /** + * Whether FIELD can go in a record. A tab or a newline would split the line somewhere the + * engine's parse does not expect, so such a value never reaches the file. + */ + static boolean isClean(final String field) { + if (field == null) { + return false; + } + for (int i = 0; i < field.length(); i++) { + final char c = field.charAt(i); + if (c == '\t' || c == '\n' || c == '\r' || c < ' ' || c == 0x7f) { + return false; + } + } + return true; + } + + /** + * One record for cookie NAME on HOST. A cookie may hold an empty value, but not an empty name. + * + * @return the line, newline included, or null when the engine would skip it + */ + static String line(final String host, final String name, final String value) { + if (!isClean(host) || !isClean(name) || !isClean(value)) { + return null; + } + // cookie_load() skips any line starting with '#', and the host leads every record. + if (host.length() == 0 || name.length() == 0 || host.charAt(0) == '#') { + return null; + } + if (bytes(host) > MAX_DOMAIN || bytes(name) > MAX_NAME || bytes(value) > MAX_VALUE) { + return null; + } + final String line = host + "\tTRUE\t/\tFALSE\t0\t" + name + "\t" + value; + return bytes(line) <= MAX_LINE ? line + "\n" : null; + } + + /** + * Split a {@code CookieManager.getCookie()} header into name and value pairs. + * + * @return two entries per cookie, the name then the value, in the header's order + */ + static List split(final String header) { + final List pairs = new ArrayList(); + if (header == null) { + return pairs; + } + for (final String item : header.split(";")) { + final String entry = item.trim(); + final int equals = entry.indexOf('='); + // A value may hold '=', so only the first one separates. A nameless entry is not a cookie. + if (equals > 0) { + pairs.add(entry.substring(0, equals).trim()); + pairs.add(entry.substring(equals + 1).trim()); + } + } + return pairs; + } + + /** + * Build the file from what each host's cookie header held. + * + * @param cookiesByHost + * host to the header {@code CookieManager.getCookie()} returned for it, in visit order + */ + public static Result file(final Map cookiesByHost) { + final StringBuilder text = new StringBuilder(HEADER); + int size = bytes(HEADER); + int cookies = 0; + int hosts = 0; + int dropped = 0; + for (final Map.Entry entry : cookiesByHost.entrySet()) { + final List pairs = split(entry.getValue()); + boolean used = false; + for (int i = 0; i + 1 < pairs.size(); i += 2) { + final String line = line(entry.getKey(), pairs.get(i), pairs.get(i + 1)); + if (line == null || size + bytes(line) > MAX_BYTES) { + dropped++; + continue; + } + text.append(line); + size += bytes(line); + cookies++; + used = true; + } + if (used) { + hosts++; + } + } + return new Result(text.toString(), cookies, hosts, dropped); + } +} diff --git a/app/src/main/java/com/httrack/android/BrowserLoginPolicy.java b/app/src/main/java/com/httrack/android/BrowserLoginPolicy.java new file mode 100644 index 00000000..efd1bd59 --- /dev/null +++ b/app/src/main/java/com/httrack/android/BrowserLoginPolicy.java @@ -0,0 +1,146 @@ +/* +HTTrack Android Java Interface. + +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 3 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + + +package com.httrack.android; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Locale; + +/** + * What the browser login opens, and what it changes about the crawl. + * + * The engine reads a -%K file only when it accepts cookies, and a logged-in crawl needs to stay + * quiet and keep away from the links that end the session, so the hand-off settles those too. + */ +public final class BrowserLoginPolicy { + private BrowserLoginPolicy() { + } + + /** Connections a logged-in crawl may open: a live session plus many sockets reads as abuse. */ + static final int MAX_SOCKETS = 2; + + /** Keeps the crawl off the links that end the session. The engine's last match wins. */ + static final String LOGOUT_FILTER = "-*logout*"; + + /** + * The login window only opens this scheme. The app declares no cleartext permission, so a + * WebView will not load http at all, and a password does not belong on cleartext either. + */ + static final String SCHEME = "https"; + + /** A leading "scheme:", which is what tells an address apart from a bare host name. */ + private static final String HAS_SCHEME = "^[a-zA-Z][a-zA-Z0-9+.-]*:.*"; + + /** + * The address to open for the login, taken from the project's address field. An http address + * is raised to https, which leaves the addresses the crawl itself uses untouched. + * + * @param urlField + * the field's text, one or more whitespace-separated addresses + * @return the first address we may open, or null when the field holds none + */ + static String startUrl(final String urlField) { + if (urlField == null) { + return null; + } + for (final String token : urlField.trim().split("\\s+")) { + if (token.length() == 0 || CommandlineTokens.isOptionLike(token)) { + continue; + } + final String url = secureUrl(token); + if (hostOf(url) != null) { + return url; + } + } + return null; + } + + /** TOKEN as an https address: a bare host gains the scheme, and http is raised to it. */ + private static String secureUrl(final String token) { + if (!token.matches(HAS_SCHEME)) { + return SCHEME + "://" + token; + } + if (token.regionMatches(true, 0, "http://", 0, "http://".length())) { + return SCHEME + "://" + token.substring("http://".length()); + } + return token; + } + + /** + * The host URL names, lowercased the way the engine compares a cookie domain. + * + * @return the host, or null unless URL is an https address naming one + */ + static String hostOf(final String url) { + if (url == null) { + return null; + } + final URI uri; + try { + uri = new URI(url); + } catch (final URISyntaxException e) { + return null; + } + final String host = uri.getHost(); + // Only https: a WebView given a file: or a content: address would read our own storage. + if (host == null || host.length() == 0 || uri.getScheme() == null + || !uri.getScheme().equalsIgnoreCase(SCHEME)) { + return null; + } + return host.toLowerCase(Locale.ROOT); + } + + /** + * The connection count to use, which only ever goes down. + * + * @param current + * the Sockets value the project holds, possibly empty or absent + */ + static String sockets(final String current) { + if (current != null) { + try { + final int value = Integer.parseInt(current.trim()); + if (value > 0 && value <= MAX_SOCKETS) { + return current.trim(); + } + } catch (final NumberFormatException e) { + // An empty or unreadable field leaves the engine on its own default, which is higher. + } + } + return String.valueOf(MAX_SOCKETS); + } + + /** + * The scan rules to use, with the logout filter appended when nothing there mentions logout. + * + * @param current + * the WildCardFilters value the project holds, possibly empty or absent + */ + static String rules(final String current) { + final String rules = current == null ? "" : current.trim(); + if (rules.toLowerCase(Locale.ROOT).indexOf("logout") != -1) { + return rules; + } + return rules.length() == 0 ? LOGOUT_FILTER : rules + " " + LOGOUT_FILTER; + } +} diff --git a/app/src/main/java/com/httrack/android/HTTrackActivity.java b/app/src/main/java/com/httrack/android/HTTrackActivity.java index d9b24e7b..faf88a1d 100755 --- a/app/src/main/java/com/httrack/android/HTTrackActivity.java +++ b/app/src/main/java/com/httrack/android/HTTrackActivity.java @@ -195,6 +195,7 @@ protected static class VERSION_CODES { protected static final int ACTIVITY_PROJECT_NAME_CHOOSER = 2; protected static final int ACTIVITY_CLEANUP = 3; protected static final int ACTIVITY_IMPORT_TREE = 4; + protected static final int ACTIVITY_BROWSER_LOGIN = 5; // Constant on purpose: a per-process tag travels through saved state and through notification // extras, so a cold launch would adopt a dead process's tag. @@ -2885,6 +2886,67 @@ public void onClickMoreProjectName(final View view) { } } + /** + * Where this project's browser login is kept. Private storage, because the file holds a live + * session and a mirror folder is readable by every app on the device. + * + * @return the file, or null until the project has a usable name + */ + private File getLoginCookiesFile() { + final String name = mapper.getProjectName(); + if (name == null || name.length() == 0 || !StoragePaths.isValidProjectName(name)) { + return null; + } + return new File(new File(getFilesDir(), "logins"), name + ".txt"); + } + + /** + * "Log in with a browser" + */ + public void onClickBrowserLogin(final View view) { + // The address field is this pane's, and the login opens what it holds right now. + savePaneFields(); + final File cookies = getLoginCookiesFile(); + if (cookies == null) { + Toast.makeText(this, R.string.browser_login_no_project, Toast.LENGTH_LONG).show(); + return; + } + final String url = BrowserLoginPolicy.startUrl(getMap(R.id.fieldWebsiteURLs)); + if (url == null) { + Toast.makeText(this, R.string.browser_login_no_address, Toast.LENGTH_LONG).show(); + return; + } + final Intent intent = new Intent(this, BrowserLoginActivity.class); + intent.putExtra(BrowserLoginActivity.EXTRA_URL, url); + intent.putExtra(BrowserLoginActivity.EXTRA_COOKIES_FILE, cookies); + startActivityForResult(intent, ACTIVITY_BROWSER_LOGIN); + } + + /** + * Point the crawl at the login the browser window saved, and tell the user what that changed. + */ + private void applyBrowserLogin(final Intent data) { + final File cookies = getLoginCookiesFile(); + final String agent = data.getStringExtra(BrowserLoginActivity.EXTRA_USER_AGENT); + if (cookies == null || agent == null || agent.length() == 0) { + return; + } + setMap(R.id.editCookiesFile, cookies.getAbsolutePath()); + // The exact string, "wv" included: a clearance bound to the browser's agent breaks on any other. + setMap(R.id.editBrowserIdentity, agent); + // cookie_load() runs only when the engine accepts cookies, so an unticked box drops the file. + setMap(R.id.checkAcceptCookies, "1"); + setMap(R.id.editNumberOfConnections, + BrowserLoginPolicy.sockets(getMap(R.id.editNumberOfConnections))); + setMap(R.id.editRules, BrowserLoginPolicy.rules(getMap(R.id.editRules))); + new AlertDialog.Builder(this) + .setMessage(getString(R.string.browser_login_saved, + data.getIntExtra(BrowserLoginActivity.EXTRA_COOKIE_COUNT, 0), + data.getIntExtra(BrowserLoginActivity.EXTRA_HOST_COUNT, 0), + BrowserLoginPolicy.MAX_SOCKETS)) + .setPositiveButton(android.R.string.ok, null).show(); + } + /** Restore a previously saved map context. **/ private void loadParcelable(final Parcelable data) { // Load modified map @@ -2919,6 +2981,11 @@ protected void onActivityResult(final int requestCode, final int resultCode, importMirrorsFrom(data.getData()); } break; + case ACTIVITY_BROWSER_LOGIN: + if (resultCode == Activity.RESULT_OK && data != null) { + applyBrowserLogin(data); + } + break; case ACTIVITY_PROJECT_NAME_CHOOSER: if (resultCode == Activity.RESULT_OK) { final String projectName = data diff --git a/app/src/main/res/layout/activity_browser_login.xml b/app/src/main/res/layout/activity_browser_login.xml new file mode 100644 index 00000000..3c5c9f08 --- /dev/null +++ b/app/src/main/res/layout/activity_browser_login.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + +