diff --git a/app/src/main/java/com/httrack/android/BrowserLoginActivity.java b/app/src/main/java/com/httrack/android/BrowserLoginActivity.java index bfb26c4..ad0a00f 100644 --- a/app/src/main/java/com/httrack/android/BrowserLoginActivity.java +++ b/app/src/main/java/com/httrack/android/BrowserLoginActivity.java @@ -69,6 +69,9 @@ public class BrowserLoginActivity extends FragmentActivity { /** Results: the browser's own User-Agent, and what the file ended up holding. */ static final String EXTRA_USER_AGENT = "com.httrack.android.userAgent"; + + /** Where the login ended, which is the address the user opened the browser to reach. */ + static final String EXTRA_LANDED_URL = "com.httrack.android.landedUrl"; static final String EXTRA_COOKIE_COUNT = "com.httrack.android.cookieCount"; static final String EXTRA_HOST_COUNT = "com.httrack.android.hostCount"; static final String EXTRA_DROPPED_COUNT = "com.httrack.android.droppedCount"; @@ -255,6 +258,8 @@ public void onClickLoginDone(final View view) { } final Intent data = new Intent(); data.putExtra(EXTRA_USER_AGENT, web.getSettings().getUserAgentString()); + // The WebView's own current address, so a redirect chain reports where it stopped. + data.putExtra(EXTRA_LANDED_URL, web.getUrl()); data.putExtra(EXTRA_COOKIE_COUNT, result.cookies); data.putExtra(EXTRA_HOST_COUNT, result.hosts); data.putExtra(EXTRA_DROPPED_COUNT, result.dropped); diff --git a/app/src/main/java/com/httrack/android/BrowserLoginPolicy.java b/app/src/main/java/com/httrack/android/BrowserLoginPolicy.java index ecd65f8..725c7be 100644 --- a/app/src/main/java/com/httrack/android/BrowserLoginPolicy.java +++ b/app/src/main/java/com/httrack/android/BrowserLoginPolicy.java @@ -78,6 +78,97 @@ static String hostOf(final String url) { return host.toLowerCase(Locale.ROOT); } + /** + * The address field with LANDED added on a line of its own. Where the user ended up is what + * they opened the browser to reach, and a login usually ends deeper than it began. The + * addresses already there are left alone, so the user trims what they do not want. + * + * @param urlField + * the field's text, one or more whitespace-separated addresses + * @param landed + * the address the login ended on, which is ignored unless it is an https address + * naming a host + * @return the field's new text + */ + static String withLandedUrl(final String urlField, final String landed) { + final String current = urlField == null ? "" : urlField.trim(); + final String address = crawlRoot(landed); + if (address == null || !namesTheSameSite(current, hostOf(address))) { + return current; + } + for (final String token : CommandlineTokens.urlTokens(current)) { + if (secureUrl(token).equals(address)) { + return current; + } + } + return current.length() == 0 ? address : current + "\n" + address; + } + + /** + * LANDED as an address the crawl may start from, or null when it is none. The address field + * is saved into the project's profile and handed to the engine, so a secret must not reach + * it: userinfo carries a password, and a query or fragment carries an OAuth code or token. + * + * @param landed + * the address the login ended on + * @return the address without its query or fragment, or null + */ + private static String crawlRoot(final String landed) { + if (hostOf(landed) == null) { + return null; + } + final URI uri; + try { + uri = new URI(landed); + } catch (final URISyntaxException e) { + return null; + } + // Dropped rather than stripped: the engine would authenticate with it, which we never meant. + if (uri.getUserInfo() != null) { + return null; + } + /* Cutting the query costs a crawl root that needed one, which is affordable here and not + everywhere: onClickBrowserLogin refuses to open without an address, so the field already + names a start and this address is an addition to it. WinHTTrack keeps the query for that + reason, because its field is usually empty and this would be the only root. */ + final int query = landed.indexOf('?'); + final int fragment = landed.indexOf('#'); + int cut = landed.length(); + if (query != -1) { + cut = query; + } + if (fragment != -1 && fragment < cut) { + cut = fragment; + } + return landed.substring(0, cut); + } + + /** + * Whether HOST belongs to a site the address field already names. A login can end on an + * identity provider, and adding that would set the crawl copying someone else's site with a + * live session. Either direction counts, because a login moves between a site and its + * subdomains in both. + * + * @param urlField + * the field's text + * @param host + * the landed address's host + * @return true when the field names that host, a parent of it, or a child of it + */ + private static boolean namesTheSameSite(final String urlField, final String host) { + if (host == null) { + return false; + } + for (final String token : CommandlineTokens.urlTokens(urlField)) { + final String known = hostOf(secureUrl(token)); + if (known != null && (known.equals(host) || known.endsWith("." + host) + || host.endsWith("." + known))) { + return true; + } + } + return false; + } + /** * The connection count to use, which only ever goes down. * diff --git a/app/src/main/java/com/httrack/android/HTTrackActivity.java b/app/src/main/java/com/httrack/android/HTTrackActivity.java index 8fbc999..d412f7e 100755 --- a/app/src/main/java/com/httrack/android/HTTrackActivity.java +++ b/app/src/main/java/com/httrack/android/HTTrackActivity.java @@ -2927,6 +2927,15 @@ private void applyBrowserLogin(final Intent data) { if (cookies == null || agent == null || agent.length() == 0) { return; } + final String addresses = BrowserLoginPolicy.withLandedUrl(getMap(R.id.fieldWebsiteURLs), + data.getStringExtra(BrowserLoginActivity.EXTRA_LANDED_URL)); + setMap(R.id.fieldWebsiteURLs, addresses); + /* The widget too, because this field is on the pane the user is looking at. setMap alone + leaves the old text on screen, and the next savePaneFields copies it back over the map. + Guarded on the pane, because setFieldText throws when the view is not inflated. */ + if (pane_id == LAYOUT_PROJECT_SETUP) { + setFieldText(R.id.fieldWebsiteURLs, addresses); + } setMap(R.id.editCookiesFile, cookies.getAbsolutePath()); // The exact string, "wv" included: a clearance bound to the browser's agent breaks on any other. setMap(R.id.editBrowserIdentity, agent); diff --git a/app/src/test/java/com/httrack/android/BrowserLoginPolicyTest.java b/app/src/test/java/com/httrack/android/BrowserLoginPolicyTest.java index 46f8d84..0cecb6f 100644 --- a/app/src/test/java/com/httrack/android/BrowserLoginPolicyTest.java +++ b/app/src/test/java/com/httrack/android/BrowserLoginPolicyTest.java @@ -107,4 +107,87 @@ public void anExistingLogoutRuleIsLeftAsTheUserWroteIt() { // Only the word itself counts, so a rule about another name still gets the filter. assertEquals("-*signout* -*logout*", BrowserLoginPolicy.rules("-*signout*")); } + + /** A login ends deeper than it began, and that page is the one the user came to copy. */ + @Test + public void theLandedAddressJoinsTheField() { + assertEquals("https://a.example\nhttps://a.example/members/", + BrowserLoginPolicy.withLandedUrl("https://a.example", "https://a.example/members/")); + // Either direction, because a login moves between a site and its subdomains both ways. + assertEquals("https://a.example\nhttps://login.a.example/home", + BrowserLoginPolicy.withLandedUrl("https://a.example", "https://login.a.example/home")); + assertEquals("https://www.a.example\nhttps://a.example/members/", + BrowserLoginPolicy.withLandedUrl("https://www.a.example", "https://a.example/members/")); + // Nothing in the field names a site, so there is nothing the landed address belongs to. + assertEquals("", BrowserLoginPolicy.withLandedUrl("", "https://a.example/members/")); + assertEquals("", BrowserLoginPolicy.withLandedUrl(null, "https://a.example/members/")); + } + + /** The field is the user's own text, so a refusal must not disturb what is already in it. */ + @Test + public void aFieldOfSeveralAddressesSurvivesEitherAnswer() { + final String many = "https://a.example/one\n-foo\nhttps://b.example/two"; + // Accepted: appended last, the rest in order, including a token urlTokens would reject. + assertEquals(many + "\nhttps://b.example/deep", + BrowserLoginPolicy.withLandedUrl(many, "https://b.example/deep")); + // Refused: byte for byte what went in. + assertEquals(many, BrowserLoginPolicy.withLandedUrl(many, "https://other.example/x")); + assertEquals(many, BrowserLoginPolicy.withLandedUrl(many, "https://u:p@a.example/x")); + } + + /** A login often ends on an identity provider, which is not the site the user asked for. */ + @Test + public void anAddressOnAnotherSiteIsRefused() { + assertEquals("https://a.example", BrowserLoginPolicy.withLandedUrl("https://a.example", + "https://accounts.idp.example/signin/oauth")); + // A suffix that is not a label boundary is another site, whatever it looks like. + assertEquals("https://a.example", + BrowserLoginPolicy.withLandedUrl("https://a.example", "https://nota.example/x")); + } + + /** The field is saved into the project profile, so a secret must never reach it. */ + @Test + public void noSecretRidesInOnTheLandedAddress() { + final String field = "https://a.example"; + // Userinfo is dropped rather than stripped: the engine would authenticate with it. + assertEquals(field, + BrowserLoginPolicy.withLandedUrl(field, "https://user:s3cr3t@a.example/home")); + assertEquals(field, BrowserLoginPolicy.withLandedUrl(field, "https://:pw@a.example/home")); + // An OAuth callback carries the code in the query and the token in the fragment. + assertEquals(field + "\nhttps://a.example/cb", + BrowserLoginPolicy.withLandedUrl(field, "https://a.example/cb?code=AQAB&state=xyz")); + assertEquals(field + "\nhttps://a.example/cb", + BrowserLoginPolicy.withLandedUrl(field, "https://a.example/cb#access_token=eyJ.P.S")); + // Both at once, and the EARLIEST separator wins whichever comes first in the string. + assertEquals(field + "\nhttps://a.example/cb", + BrowserLoginPolicy.withLandedUrl(field, "https://a.example/cb?a=1#b=2")); + assertEquals(field + "\nhttps://a.example/cb", + BrowserLoginPolicy.withLandedUrl(field, "https://a.example/cb#b=2?a=1")); + } + + @Test + public void anAddressAlreadyInTheFieldIsNotRepeated() { + final String landed = "https://a.example/members/"; + assertEquals(landed, BrowserLoginPolicy.withLandedUrl(landed, landed)); + /* The field holds what the user typed, and startUrl raised it to https before opening it, + so the landed address is compared against the same raising. */ + assertEquals("a.example/members/", + BrowserLoginPolicy.withLandedUrl("a.example/members/", landed)); + assertEquals("http://a.example/members/", + BrowserLoginPolicy.withLandedUrl("http://a.example/members/", landed)); + } + + /** The field is the crawl's own input, so nothing the login reports may be put in unchecked. */ + @Test + public void onlyAnHttpsAddressReachesTheField() { + assertEquals("https://a.example", BrowserLoginPolicy.withLandedUrl("https://a.example", null)); + // A WebView can end on these, and none of them names a host the crawl may start from. + assertEquals("https://a.example", + BrowserLoginPolicy.withLandedUrl("https://a.example", "about:blank")); + assertEquals("https://a.example", + BrowserLoginPolicy.withLandedUrl("https://a.example", "file:///etc/passwd")); + assertEquals("https://a.example", + BrowserLoginPolicy.withLandedUrl("https://a.example", "http://a.example/")); + assertEquals("https://a.example", BrowserLoginPolicy.withLandedUrl("https://a.example", "")); + } } diff --git a/app/src/test/java/com/httrack/android/BrowserLoginWiringTest.java b/app/src/test/java/com/httrack/android/BrowserLoginWiringTest.java index 4894866..733e5ab 100644 --- a/app/src/test/java/com/httrack/android/BrowserLoginWiringTest.java +++ b/app/src/test/java/com/httrack/android/BrowserLoginWiringTest.java @@ -156,6 +156,26 @@ public void theStagedWriteBuildsItsNameFromTheSharedHelper() throws IOException assertTrue(body, body.contains("staged.renameTo(file)")); } + /** Without the landed address the user has to retype where the login took them. */ + @Test + public void theLandedAddressIsReturnedAndShown() throws IOException { + final String source = TestSources + .withoutCommentsAndStrings(TestSources.javaSource("BrowserLoginActivity")); + final String done = TestSources.balancedBlock(source, + TestSources.indexOf(source, "public void onClickLoginDone(")); + // The WebView's own address, not the one the activity was started with. + assertTrue(done, done.contains("EXTRA_LANDED_URL, web.getUrl()")); + final String caller = TestSources + .withoutCommentsAndStrings(TestSources.javaSource("HTTrackActivity")); + final String apply = TestSources.balancedBlock(caller, + TestSources.indexOf(caller, "private void applyBrowserLogin(")); + assertTrue(apply, apply.contains("BrowserLoginPolicy.withLandedUrl(")); + assertTrue(apply, apply.contains("setMap(R.id.fieldWebsiteURLs, addresses)")); + /* The widget as well as the map. setMap alone left the old text on screen and the next + savePaneFields copied it back, so the whole feature was invisible. */ + assertTrue(apply, apply.contains("setFieldText(R.id.fieldWebsiteURLs, addresses)")); + } + /** -%K is read only when the engine accepts cookies, so the hand-off has to tick the box. */ @Test public void theHandOverTurnsCookiesOn() throws IOException {