From 7632553070930b5452b079c5da6a008a1afa432e Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 13:48:04 +0200 Subject: [PATCH 01/10] Read a login bundle named by a start URL A start URL ending in #httrack-bundle= crawls with the user agent, headers and cookies of .hts. The file is read from the project folder, or from the folder given to --bundle-dir (-@b), and the engine never writes it. The suffix is read on argv, -%L and doit.log, and cut before the URL reaches any buffer. The bundle's UA and headers replace -F and -%X only for hosts in its scope: the registrable domains of its URLs and cookie domains, or its scope lines. A cookie outside the scope is dropped with a warning. Overlapping bundles must agree on UA, headers and cookie values. Bundle cookies load first and cookies.txt or -%K loads over them. The per-bundle table is a registry in htsbundle.c keyed by opt, so no installed struct changes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- html/httrack.man.html | 80 +++- man/httrack.1 | 52 ++- man/httrack.tmpl | 43 +++ src/Makefile.am | 5 +- src/htsalias.c | 3 + src/htsback.c | 5 + src/htsbauth.c | 370 +++++++++++-------- src/htsbauth.h | 21 ++ src/htsbundle.c | 690 +++++++++++++++++++++++++++++++++++ src/htsbundle.h | 97 +++++ src/htsbundle_selftest.c | 129 +++++++ src/htscore.c | 114 +++++- src/htscoremain.c | 56 ++- src/htshelp.c | 4 + src/htslib.c | 15 + src/htslib.h | 5 + src/htsselftest.c | 2 +- src/htsselftest_int.h | 1 + src/libhttrack.vcxproj | 2 + tests/655_engine-bundle.test | 151 ++++++++ tests/656_local-bundles.test | 112 ++++++ tests/bundle-server.py | 73 ++++ 22 files changed, 1848 insertions(+), 182 deletions(-) create mode 100644 src/htsbundle.c create mode 100644 src/htsbundle.h create mode 100644 src/htsbundle_selftest.c create mode 100644 tests/655_engine-bundle.test create mode 100644 tests/656_local-bundles.test create mode 100644 tests/bundle-server.py diff --git a/html/httrack.man.html b/html/httrack.man.html index c42c7f739..33cd33d87 100644 --- a/html/httrack.man.html +++ b/html/httrack.man.html @@ -101,15 +101,16 @@

SYNOPSIS --generate-errors[=N] ] [ -X, --purge-old[=0] ] [ -%p, --preserve ] [ -%T, --utf8-conversion[=0] ] [ -bN, --cookies[=N] ] [ -%K, --cookies-file -] [ -%Y, --why ] [ -u, --check-type[=N] ] [ --j, --parse-java[=N] ] [ -sN, --robots[=N] ] [ --%h, --http-10[=0] ] [ -%k, --keep-alive[=0] ] -[ -%z, --disable-compression[=0] ] [ -%B, ---tolerant[=0] ] [ -%s, --updatehack[=0] ] [ --%u, --urlhack[=0] ] [ -%A, --assume ] [ --@iN, --protocol[=N] ] [ -@mN, --mptcp[=0] ] [ --%w, --disable-module ] [ -F, --user-agent ] [ --%R, --referer ] [ -%E, --from ] [ -%F, +] [ -@b, --bundle-dir ] [ -%Y, --why ] [ +-u, --check-type[=N] ] [ -j, --parse-java[=N] +] [ -sN, --robots[=N] ] [ -%h, --http-10[=0] ] +[ -%k, --keep-alive[=0] ] [ -%z, +--disable-compression[=0] ] [ -%B, --tolerant[=0] +] [ -%s, --updatehack[=0] ] [ -%u, +--urlhack[=0] ] [ -%A, --assume ] [ -@iN, +--protocol[=N] ] [ -@mN, --mptcp[=0] ] [ -%w, +--disable-module ] [ -F, --user-agent ] [ -%R, +--referer ] [ -%E, --from ] [ -%F, --footer ] [ -%l, --language ] [ -%a, --accept ] [ -%X, --headers ] [ -%Q, --browser-headers[=0] ] [ -C, --cache[=N] ] [ @@ -886,13 +887,36 @@

Spider options: -

-%Y

+

-@b

-

explain which filter rule accepts or rejects a URL, then -exit (--why <param>)

+

folder holding the login bundles that start URLs name +(--bundle-dir <param>)

+ + + +

<URL>#httrack-bundle=<name> +crawls <URL> with the user agent,
+headers and cookies of <name>.hts, by default in the +project folder

+ + + + + + + - - - - -
+ + +

-%Y

 + + +

explain which filter rule +accepts or rejects a URL, then exit (--why +<param>)

@@ -2465,6 +2489,38 @@

FILES the Cookie header from DevTools "Copy as cURL", pasted here.

+

<name>.hts

+ +

A login bundle, named by a +start URL that ends with +#httrack-bundle=<name>, such as +https://example.com/members/#httrack-bundle=login1. +The name holds letters, digits, ., _ and +- only. HTTrack reads the file in the project folder, +or in the folder given to --bundle-dir, and never +writes it. HTTrack reads the suffix on the command line, in +a -%L list and in doit.log, never on a crawled link. +A missing file, or one that does not parse, stops the run +before any request.

+ +

The first line +is # HTTrack bundle 1. Then come url lines +(more start URLs), at most one ua line, and +header and scope lines. A cookies line +ends them, and a Netscape cookie jar follows it.

+ +

The +bundle’s user agent and headers replace -F and +-%X for each host in its scope, and only there. By +default the scope is the registrable domains of the start +URL naming the bundle, of its url lines and of its cookies. +Scope lines replace that default. HTTrack refuses a cookie +outside the scope, and a Host, Cookie, Proxy-* or framing +header. Its cookies load before cookies.txt, so a cookie the +server refreshed wins. Two bundles whose scopes overlap must +give the same user agent, headers and cookie values, or the +run stops.

+

A running mirror also watches its own output directory for these. Creating one is how a script drives an engine it has no other handle diff --git a/man/httrack.1 b/man/httrack.1 index ed2adc446..b4f0ff7f2 100644 --- a/man/httrack.1 +++ b/man/httrack.1 @@ -3,7 +3,7 @@ .\" .\" This file is generated by man/makeman.sh; do not edit by hand. .\" SPDX-License-Identifier: GPL-3.0-or-later -.TH httrack 1 "19 October 2026" "httrack website copier" +.TH httrack 1 "21 October 2026" "httrack website copier" .SH NAME httrack \- offline browser : copy websites to a local directory .SH SYNOPSIS @@ -56,6 +56,7 @@ httrack \- offline browser : copy websites to a local directory [ \fB\-%T, \-\-utf8\-conversion[=0]\fR ] [ \fB\-bN, \-\-cookies[=N]\fR ] [ \fB\-%K, \-\-cookies\-file\fR ] +[ \fB\-@b, \-\-bundle\-dir\fR ] [ \fB\-%Y, \-\-why\fR ] [ \fB\-u, \-\-check\-type[=N]\fR ] [ \fB\-j, \-\-parse\-java[=N]\fR ] @@ -242,6 +243,12 @@ links conversion to UTF\-8 (\-\-utf8\-conversion[=0]) accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N]) .IP \-%K use a Netscape cookies.txt as the jar, rewritten at the end (\-\-cookies\-file ) +.IP \-@b +folder holding the login bundles that start URLs name (\-\-bundle\-dir ) +.br +#httrack\-bundle= crawls with the user agent, +.br +headers and cookies of .hts, by default in the project folder .IP \-%Y explain which filter rule accepts or rejects a URL, then exit (\-\-why ) .IP \-u @@ -582,6 +589,49 @@ start URL and its subdomains, or to that host alone when it is an address or a public suffix. They are Secure when that URL is https. HTTrack never rewrites such a file. HttpOnly cookies are not in it. A cookies.txt export has them, and so does the Cookie header from DevTools "Copy as cURL", pasted here. +.TP +.I .hts +A login bundle, named by a start URL that ends with +.IR #httrack\-bundle= , +such as +.IR https://example.com/members/#httrack\-bundle=login1 . +The name holds letters, digits, +.IR . , +.I _ +and +.I \- +only. HTTrack reads the file in the project folder, or in the folder given to +.IR \-\-bundle\-dir , +and never writes it. HTTrack reads the suffix on the command line, in a +.I \-%L +list and in doit.log, never on a crawled link. A missing file, or one that +does not parse, stops the run before any request. +.IP +The first line is +.IR "# HTTrack bundle 1" . +Then come +.I url +lines (more start URLs), at most one +.I ua +line, and +.I header +and +.I scope +lines. A +.I cookies +line ends them, and a Netscape cookie jar follows it. +.IP +The bundle's user agent and headers replace +.I \-F +and +.I \-%X +for each host in its scope, and only there. By default the scope is the +registrable domains of the start URL naming the bundle, of its url lines and +of its cookies. Scope lines replace that default. HTTrack refuses a cookie +outside the scope, and a Host, Cookie, Proxy\-* or framing header. Its cookies +load before cookies.txt, so a cookie the server refreshed wins. Two bundles +whose scopes overlap must give the same user agent, headers and cookie values, +or the run stops. .PP A running mirror also watches its own output directory for these. Creating one is how a script drives an engine it has no other handle on. On Windows, which diff --git a/man/httrack.tmpl b/man/httrack.tmpl index 1e3b818a1..6c42d01c0 100644 --- a/man/httrack.tmpl +++ b/man/httrack.tmpl @@ -69,6 +69,49 @@ start URL and its subdomains, or to that host alone when it is an address or a public suffix. They are Secure when that URL is https. HTTrack never rewrites such a file. HttpOnly cookies are not in it. A cookies.txt export has them, and so does the Cookie header from DevTools "Copy as cURL", pasted here. +.TP +.I .hts +A login bundle, named by a start URL that ends with +.IR #httrack\-bundle= , +such as +.IR https://example.com/members/#httrack\-bundle=login1 . +The name holds letters, digits, +.IR . , +.I _ +and +.I \- +only. HTTrack reads the file in the project folder, or in the folder given to +.IR \-\-bundle\-dir , +and never writes it. HTTrack reads the suffix on the command line, in a +.I \-%L +list and in doit.log, never on a crawled link. A missing file, or one that +does not parse, stops the run before any request. +.IP +The first line is +.IR "# HTTrack bundle 1" . +Then come +.I url +lines (more start URLs), at most one +.I ua +line, and +.I header +and +.I scope +lines. A +.I cookies +line ends them, and a Netscape cookie jar follows it. +.IP +The bundle's user agent and headers replace +.I \-F +and +.I \-%X +for each host in its scope, and only there. By default the scope is the +registrable domains of the start URL naming the bundle, of its url lines and +of its cookies. Scope lines replace that default. HTTrack refuses a cookie +outside the scope, and a Host, Cookie, Proxy\-* or framing header. Its cookies +load before cookies.txt, so a cookie the server refreshed wins. Two bundles +whose scopes overlap must give the same user agent, headers and cookie values, +or the run stops. .PP A running mirror also watches its own output directory for these. Creating one is how a script drives an engine it has no other handle on. On Windows, which diff --git a/src/Makefile.am b/src/Makefile.am index 4107093ee..f03f3b885 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -86,6 +86,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \ htsback_selftest.c htscache_selftest.c htscharset_selftest.c \ htscmdline_selftest.c \ htscookie_selftest.c htsdns_selftest.c htsfetch_selftest.c htspsl_selftest.c \ + htsbundle_selftest.c \ htsfilters_selftest.c \ htsheader_selftest.c htsio_selftest.c htslib_selftest.c \ htslines_selftest.c \ @@ -96,14 +97,14 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \ htscatchurl.c htsfetch.c htspsl.c htspsl_data.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \ htscmdline.c htshelp.c htslib.c htslines.c htsurlport.c htsdate.c htscoremain.c \ htsname.c htsrobots.c htstools.c htsrandom.c htswizard.c \ - htsalias.c htsthread.c htsindex.c htsbauth.c \ + htsalias.c htsthread.c htsindex.c htsbauth.c htsbundle.c \ htscrashtest.c \ htsmd5.c htsnet.c htscodec.c htswarc.c htschanges.c htssinglefile.c htssitemap.c htsproxy.c htszlib.c htswrap.c htsconcat.c \ htsmodules.c htscharset.c punycode.c htsencoding.c htsescape.c htssniff.c \ md5.c \ minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \ hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h htsdate.h \ - htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htscookie_selftest.h htsdns_selftest.h htsparse_selftest.h \ + htsbasenet.h htsbauth.h htsbundle.h htscache.h htscache_selftest.h htscookie_selftest.h htsdns_selftest.h htsparse_selftest.h \ htsselftest.h htsselftest_int.h htsselftest_util.h htscatchurl.h \ htscmdline.h htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \ htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \ diff --git a/src/htsalias.c b/src/htsalias.c index aa9b6a7b4..99061caff 100644 --- a/src/htsalias.c +++ b/src/htsalias.c @@ -165,6 +165,8 @@ const char *hts_optalias[][4] = { "longest Retry-After delay obeyed, in seconds"}, {"mptcp", "-@m", "onoff", "carry connections over Multipath TCP where the system offers it"}, + {"bundle-dir", "-@b", "param1", + "folder where #httrack-bundle= names are read, not the project folder"}, {"generate-errors", "-o", "level", ""}, {"do-not-generate-errors", "-o0", "single", ""}, {"purge-old", "-X", "onoff", ""}, @@ -347,6 +349,7 @@ static const char *const dashvalue_opt[] = { "-%L", /* list */ "-%S", /* urllist */ "-%rf", /* warc-file */ + "-@b", /* bundle-dir */ "-F", /* user-agent */ NULL }; diff --git a/src/htsback.c b/src/htsback.c index 0c2c139e2..f45300b92 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -36,6 +36,7 @@ Please visit our Website: http://www.httrack.com /* specific definitions */ #include "htsnet.h" +#include "htsbundle.h" #include "htscore.h" #include "htsio.h" #include "htswarc.h" @@ -3068,6 +3069,10 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back[p].r.req.accept = StringBuff(opt->accept); back[p].r.req.headers = StringBuff(opt->headers); back[p].r.req.user_agent_send = opt->user_agent_send; + /* a login bundle's site gets the bundle's UA and headers */ + if (hts_bundle_identity(opt, adr, &back[p].r.req.user_agent, + &back[p].r.req.headers)) + back[p].r.req.user_agent_send = 1; // et http11 back[p].r.req.http11 = back[p].http11; back[p].r.req.nocompression = opt->nocompression; diff --git a/src/htsbauth.c b/src/htsbauth.c index caac90c55..f64d26fd8 100644 --- a/src/htsbauth.c +++ b/src/htsbauth.c @@ -652,10 +652,9 @@ static hts_boolean cookie_is_flat(FILE *fp, int *quote) { return flat; } -/* Copy the host of url, with or without a scheme, into dst as cookie_host - does, and set *https. */ -static hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size, - hts_boolean *https) { +/* See htsbauth.h. */ +hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size, + hts_boolean *https) { const char *const rest = jump_protocol_const(url); const size_t len = strcspn(rest, "/?#"); char adr[HTS_URLMAXSIZE]; @@ -818,6 +817,160 @@ static int cookie_load_flat(httrackp *opt, t_cookie *cookie, FILE *fp, return COOKIE_LOAD_KEEP; } +/* Read the Netscape records from fp's position to its end into cookie, and + name the file as given in warnings. A UTF-8 BOM is skipped on the first line + when first_line is HTS_TRUE. Returns what cookie_load_from does. */ +static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, + const char *given, hts_boolean first_line) { + char buffer[8192]; + int refused = 0; + hts_boolean keep = HTS_FALSE; + char BIGSTK line[8192]; + const size_t line_max = 8000; + // this load's records alone, so a line repeating one is caught + t_cookie *seen = (t_cookie *) calloct(1, sizeof(t_cookie)); + + if (seen != NULL) { + seen->max_len = sizeof(seen->data); + } else { + hts_log_print(opt, LOG_WARNING, + "Cookies: out of memory, so %s is left unchanged", given); + keep = HTS_TRUE; + } + // ferror ends the loop too, because a directory never reaches EOF + while (!feof(fp) && !ferror(fp) && strlen(cookie->data) < cookie->max_len) { + size_t len; + + // len counts what the buffer cut too, so this catches that line + cookie_readline(fp, line, sizeof(line), &len); + if (first_line && len >= sizeof(cookie_bom) - 1 && + strncmp(line, cookie_bom, sizeof(cookie_bom) - 1) == 0) { + len -= sizeof(cookie_bom) - 1; + memmove(line, line + sizeof(cookie_bom) - 1, + strlen(line) + 1 - (sizeof(cookie_bom) - 1)); + } + first_line = HTS_FALSE; + if (len >= line_max) { + hts_log_print(opt, LOG_WARNING, + "Cookies: ignoring an over-long line " + "(%d bytes, maximum %d): %s", + (int) len, (int) line_max - 1, given); + refused++; + } else if (memchr(line, '\0', len) != NULL || + memchr(line, '\r', len) != NULL) { + hts_log_print(opt, LOG_WARNING, + "Cookies: ignoring a line holding a NUL or a CR: %s", + given); + refused++; + } else if (line[strspn(line, " \t")] != '\0') { + /* curl and browser exports mark an HttpOnly cookie this way */ + static const char httponly[] = COOKIE_HTTPONLY_PREFIX; + const hts_boolean is_httponly = + strncmp(line, httponly, sizeof(httponly) - 1) == 0; + const char *const rec = is_httponly ? line + sizeof(httponly) - 1 : line; + + if (rec[0] == '#') { + if (!cookie_known_comment(rec)) { + hts_log_print(opt, LOG_WARNING, + "Cookies: ignoring an unknown comment in %s: %.64s", + given, rec); + refused++; + } + } else { + char domain[256]; // .netscape.com + char path[256]; // / + char BIGSTK cook_name[COOKIE_MAX_PAIR + 1]; // MYCOOK + char BIGSTK cook_value[8192]; // ID=toto,S=1234 + const char *p; + int tabs = 0; + +#if DEBUG_COOK + printf("%s\n", rec); +#endif + for (p = strchr(rec, '\t'); p != NULL; p = strchr(p + 1, '\t')) + tabs++; + if (tabs < 6 || !strnotempty(cookie_get(buffer, rec, 5))) { + hts_log_print(opt, LOG_WARNING, + "Cookies: ignoring a line with fewer than 7 " + "fields or no name: %s", + given); + refused++; + } else if (tabs > 7 || + (tabs == 7 && strcmp(cookie_get(buffer, rec, 7), + COOKIE_HTTPONLY_FIELD) != 0)) { + // a TAB in the value, because only HTTrack writes an eighth field + hts_log_print(opt, LOG_WARNING, + "Cookies: ignoring a line with more than 7 " + "fields: %s", + given); + refused++; + } else if (!cookie_is_expiry(cookie_get(buffer, rec, 4))) { + hts_log_print(opt, LOG_WARNING, + "Cookies: ignoring a line whose expiry is not a " + "whole number: %s", + given); + refused++; + } else if (cookie_load_field(opt, domain, sizeof(domain), buffer, rec, + 0, "domain", given) && + cookie_load_field(opt, path, sizeof(path), buffer, rec, 2, + "path", given) && + cookie_load_field(opt, cook_name, sizeof(cook_name), buffer, + rec, 5, "name", given) && + cookie_load_field(opt, cook_value, sizeof(cook_value), + buffer, rec, 6, "value", given)) { + cookie_attr attr; + char host[256]; + + attr.host_only = cookie_is_host_only(rec); + attr.secure = strcmpnocase(cookie_get(buffer, rec, 3), "TRUE") == 0; + attr.http_only = is_httponly || tabs == 7; + attr.expires = cookie_record_expiry(rec); + // an expired one is skipped, because a browser drops it too + if (attr.expires == 0 || attr.expires > (int64_t) time(NULL)) { + const hts_boolean repeated = + seen != NULL && cookie_host(domain, host, sizeof(host)) && + cookie_find_exact(seen->data, cook_name, host, path) != NULL; + + // cookie_set stores it host-only + if (!attr.host_only && cookie_domain_is_suffix(domain)) + hts_log_print(opt, LOG_WARNING, + "Cookies: '%.64s' for the public suffix %s is " + "now host-only: %s", + cook_name, domain, given); + // the last one wins, as curl reads it, but a rewrite loses one + if (repeated) { + hts_log_print(opt, LOG_WARNING, + "Cookies: '%.64s' appears twice in %s", cook_name, + given); + refused++; + } + if (cookie_set(cookie, cook_name, cook_value, domain, path, + &attr) != 0) { + // the jar caps a field tighter still, and a full one refuses + hts_log_print(opt, LOG_WARNING, + "Cookies: the jar did not store '%.64s' from " + "%s", + cook_name, given); + refused++; + } else if (seen != NULL && !repeated) { + (void) cookie_set(seen, cook_name, "", domain, path, NULL); + } + } + } else { + refused++; + } + } + } + } + if (!feof(fp)) { + hts_log_print(opt, LOG_WARNING, + "Cookies: stopped reading before the end of %s", given); + refused++; + } + freet(seen); + return keep ? COOKIE_LOAD_KEEP : refused; +} + /* See htsbauth.h. */ int cookie_load(httrackp *opt, t_cookie *cookie, const char *fpath, const char *name) { @@ -828,7 +981,6 @@ int cookie_load(httrackp *opt, t_cookie *cookie, const char *fpath, int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, const char *name, const char *start_url) { char catbuff[CATBUFF_SIZE]; - char buffer[8192]; // Merge any IE cookies first #ifdef _WIN32 @@ -914,8 +1066,6 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, const char *const given = fconcat(catbuff, sizeof(catbuff), fpath, name); char file[CATBUFF_SIZE]; char bak[CATBUFF_SIZE]; - int refused = 0; - hts_boolean keep = HTS_FALSE; int quote = 0; FILE *fp; @@ -938,156 +1088,10 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, return ret; } if (fp) { - char BIGSTK line[8192]; - const size_t line_max = 8000; - hts_boolean first_line = HTS_TRUE; - // this load's records alone, so a line repeating one is caught - t_cookie *seen = (t_cookie *) calloct(1, sizeof(t_cookie)); - - if (seen != NULL) { - seen->max_len = sizeof(seen->data); - } else { - hts_log_print(opt, LOG_WARNING, - "Cookies: out of memory, so %s is left unchanged", given); - keep = HTS_TRUE; - } - // ferror ends the loop too, because a directory never reaches EOF - while (!feof(fp) && !ferror(fp) && - strlen(cookie->data) < cookie->max_len) { - size_t len; - - // len counts what the buffer cut too, so this catches that line - cookie_readline(fp, line, sizeof(line), &len); - if (first_line && len >= sizeof(cookie_bom) - 1 && - strncmp(line, cookie_bom, sizeof(cookie_bom) - 1) == 0) { - len -= sizeof(cookie_bom) - 1; - memmove(line, line + sizeof(cookie_bom) - 1, - strlen(line) + 1 - (sizeof(cookie_bom) - 1)); - } - first_line = HTS_FALSE; - if (len >= line_max) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring an over-long line " - "(%d bytes, maximum %d): %s", - (int) len, (int) line_max - 1, given); - refused++; - } else if (memchr(line, '\0', len) != NULL || - memchr(line, '\r', len) != NULL) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a line holding a NUL or a CR: %s", - given); - refused++; - } else if (line[strspn(line, " \t")] != '\0') { - /* curl and browser exports mark an HttpOnly cookie this way */ - static const char httponly[] = COOKIE_HTTPONLY_PREFIX; - const hts_boolean is_httponly = - strncmp(line, httponly, sizeof(httponly) - 1) == 0; - const char *const rec = - is_httponly ? line + sizeof(httponly) - 1 : line; - - if (rec[0] == '#') { - if (!cookie_known_comment(rec)) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring an unknown comment in %s: %.64s", - given, rec); - refused++; - } - } else { - char domain[256]; // .netscape.com - char path[256]; // / - char BIGSTK cook_name[COOKIE_MAX_PAIR + 1]; // MYCOOK - char BIGSTK cook_value[8192]; // ID=toto,S=1234 - const char *p; - int tabs = 0; + const int ret = cookie_load_records(opt, cookie, fp, given, HTS_TRUE); -#if DEBUG_COOK - printf("%s\n", rec); -#endif - for (p = strchr(rec, '\t'); p != NULL; p = strchr(p + 1, '\t')) - tabs++; - if (tabs < 6 || !strnotempty(cookie_get(buffer, rec, 5))) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a line with fewer than 7 " - "fields or no name: %s", - given); - refused++; - } else if (tabs > 7 || - (tabs == 7 && strcmp(cookie_get(buffer, rec, 7), - COOKIE_HTTPONLY_FIELD) != 0)) { - // a TAB in the value, because only HTTrack writes an eighth field - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a line with more than 7 " - "fields: %s", - given); - refused++; - } else if (!cookie_is_expiry(cookie_get(buffer, rec, 4))) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a line whose expiry is not a " - "whole number: %s", - given); - refused++; - } else if (cookie_load_field(opt, domain, sizeof(domain), buffer, - rec, 0, "domain", given) && - cookie_load_field(opt, path, sizeof(path), buffer, rec, - 2, "path", given) && - cookie_load_field(opt, cook_name, sizeof(cook_name), - buffer, rec, 5, "name", given) && - cookie_load_field(opt, cook_value, sizeof(cook_value), - buffer, rec, 6, "value", given)) { - cookie_attr attr; - char host[256]; - - attr.host_only = cookie_is_host_only(rec); - attr.secure = - strcmpnocase(cookie_get(buffer, rec, 3), "TRUE") == 0; - attr.http_only = is_httponly || tabs == 7; - attr.expires = cookie_record_expiry(rec); - // an expired one is skipped, because a browser drops it too - if (attr.expires == 0 || attr.expires > (int64_t) time(NULL)) { - const hts_boolean repeated = - seen != NULL && cookie_host(domain, host, sizeof(host)) && - cookie_find_exact(seen->data, cook_name, host, path) != - NULL; - - // cookie_set stores it host-only - if (!attr.host_only && cookie_domain_is_suffix(domain)) - hts_log_print(opt, LOG_WARNING, - "Cookies: '%.64s' for the public suffix %s is " - "now host-only: %s", - cook_name, domain, given); - // the last one wins, as curl reads it, but a rewrite loses one - if (repeated) { - hts_log_print(opt, LOG_WARNING, - "Cookies: '%.64s' appears twice in %s", - cook_name, given); - refused++; - } - if (cookie_set(cookie, cook_name, cook_value, domain, path, - &attr) != 0) { - // the jar caps a field tighter still, and a full one refuses - hts_log_print(opt, LOG_WARNING, - "Cookies: the jar did not store '%.64s' from " - "%s", - cook_name, given); - refused++; - } else if (seen != NULL && !repeated) { - (void) cookie_set(seen, cook_name, "", domain, path, NULL); - } - } - } else { - refused++; - } - } - } - } - if (!feof(fp)) { - hts_log_print(opt, LOG_WARNING, - "Cookies: stopped reading before the end of %s", given); - refused++; - } fclose(fp); - freet(seen); - return keep ? COOKIE_LOAD_KEEP : refused; + return ret; } else if (errno != ENOENT) { // the file exists, so a rewrite would replace what was never read hts_log_print(opt, LOG_WARNING | LOG_ERRNO, @@ -1099,6 +1103,62 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, return -1; } +/* See htsbauth.h. */ +int cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, + const char *name) { + return cookie_load_records(opt, cookie, fp, name, HTS_FALSE); +} + +/* See htsbauth.h. */ +int cookie_merge(t_cookie *dst, t_cookie *src) { + char BIGSTK domain[8192]; + char BIGSTK path[8192]; + char BIGSTK name[8192]; + char BIGSTK value[8192]; + char buffer[8192]; + int refused = 0; + char *a; + + for (a = src->data; *a != '\0'; a = cookie_nextfield(a)) { + cookie_attr attr; + + attr.host_only = cookie_is_host_only(a); + attr.secure = strcmpnocase(cookie_get(buffer, a, 3), "TRUE") == 0; + attr.http_only = + strcmp(cookie_get(buffer, a, 7), COOKIE_HTTPONLY_FIELD) == 0; + attr.expires = cookie_record_expiry(a); + if (cookie_set(dst, cookie_get(name, a, 5), cookie_get(value, a, 6), + cookie_get(domain, a, 0), cookie_get(path, a, 2), + &attr) != 0) + refused++; + } + return refused; +} + +/* See htsbauth.h. */ +hts_boolean cookie_jars_conflict(t_cookie *a, t_cookie *b, char *name, + size_t name_size) { + char BIGSTK domain[8192]; + char BIGSTK path[8192]; + char BIGSTK cook_name[8192]; + char buffer[8192]; + char *rec; + + for (rec = a->data; *rec != '\0'; rec = cookie_nextfield(rec)) { + const char *const other = + cookie_find_exact(b->data, cookie_get(cook_name, rec, 5), + cookie_get(domain, rec, 0), cookie_get(path, rec, 2)); + + if (other != NULL && + strcmp(cookie_get(buffer, rec, 6), cookie_get(domain, other, 6)) != 0) { + name[0] = '\0'; + strlncatbuff(name, cook_name, name_size, name_size - 1); + return HTS_TRUE; + } + } + return HTS_FALSE; +} + /* Open name for writing, creating it if needed, and return NULL on failure. The file is owner-only on Unix because it holds session cookies. */ static FILE *cookie_create(const char *name) { diff --git a/src/htsbauth.h b/src/htsbauth.h index c925774bf..da304f2b4 100644 --- a/src/htsbauth.h +++ b/src/htsbauth.h @@ -73,6 +73,7 @@ struct t_cookie { /* Library internal definitions */ #ifdef HTS_INTERNAL_BYTECODE +#include #include #ifndef HTS_DEF_FWSTRUCT_httrackp @@ -89,6 +90,11 @@ typedef struct httrackp httrackp; HTS_FALSE means the caller sends no cookie. */ hts_boolean cookie_host(const char *adr, char *dst, size_t dst_size); +/** Copy the host of url, with or without a scheme, into dst as cookie_host + does, and set *https. */ +hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size, + hts_boolean *https); + /** A jar record holds this many bytes of name plus value. */ #define COOKIE_MAX_PAIR 4096 @@ -148,6 +154,21 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *path, const char *name, const char *start_url); #define COOKIE_LOAD_KEEP (-2) +/** Load Netscape jar records from fp's position to its end, as cookie_load + reads a file, naming name in warnings. fp stays open. Returns the count of + lines skipped, or COOKIE_LOAD_KEEP when memory ran out. */ +int cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, + const char *name); + +/** Store every record of src into dst, with its flags and expiry, replacing + a record of the same name, domain and path. Returns how many dst refused. */ +int cookie_merge(t_cookie *dst, t_cookie *src); + +/** Does b hold a record with the name, domain and path of one in a, but + another value? If so, copy that name into name (name_size bytes). */ +hts_boolean cookie_jars_conflict(t_cookie *a, t_cookie *b, char *name, + size_t name_size); + /** Write the jar's cookies to name in Netscape format, owner-only on Unix. The new jar goes to a temporary file first, which then replaces name in one step, so name holds the old or the new jar whatever fails. A symbolic link diff --git a/src/htsbundle.c b/src/htsbundle.c new file mode 100644 index 000000000..f613e9a07 --- /dev/null +++ b/src/htsbundle.c @@ -0,0 +1,690 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) 2026 Xavier Roche and other contributors + +SPDX-License-Identifier: GPL-3.0-or-later + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Ethical use: we kindly ask that you NOT use this software to harvest email +addresses or to collect any other private information about people. Doing so +would dishonor our work and waste the many hours we have spent on it. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: htsbundle.c subroutines: */ +/* login bundles named by a start URL */ +/* ------------------------------------------------------------ */ + +#define HTS_INTERNAL_BYTECODE + +#include "htsbundle.h" + +#include "htscore.h" +#include "htslib.h" +#include "htspsl.h" +#include "htssafe.h" +#include "htsthread.h" + +#include + +/* A bundle line, the jar aside, holds fewer bytes than this. */ +#define BUNDLE_LINE_MAX 8192 + +/* The jar room the crawl's own jar gets (see httpmirror). */ +#define BUNDLE_JAR_MAX 30000 + +typedef struct bundle_entry { + char name[HTS_BUNDLE_NAME_MAX]; + char file[HTS_URLMAXSIZE * 2]; + char **named; /* the start URLs naming it */ + size_t nnamed; + char **urls; /* its url lines */ + size_t nurls; + char **scope; /* lowercase domains, each covering its subdomains */ + size_t nscope; + char *ua; /* NULL when it gives none */ + String headers; + t_cookie *jar; /* NULL until loaded */ +} bundle_entry; + +/* The bundles of one opt, kept out of httrackp so its layout is unchanged. */ +typedef struct bundle_set { + const httrackp *opt; + char *dir; + bundle_entry *items; + size_t count; + struct bundle_set *next; +} bundle_set; + +static htsmutex bundle_lock = HTSMUTEX_INIT; +static bundle_set *bundle_sets = NULL; + +/* Call with bundle_lock held. */ +static bundle_set *bundle_set_of(const httrackp *opt, hts_boolean create) { + bundle_set *set; + + for (set = bundle_sets; set != NULL; set = set->next) { + if (set->opt == opt) + return set; + } + if (!create) + return NULL; + set = calloct(1, sizeof(*set)); + if (set != NULL) { + set->opt = opt; + set->next = bundle_sets; + bundle_sets = set; + } + return set; +} + +static hts_boolean strlist_add(char ***list, size_t *count, const char *s) { + char **const grown = realloct(*list, (*count + 1) * sizeof(char *)); + + if (grown == NULL) + return HTS_FALSE; + *list = grown; + grown[*count] = strdupt(s); + if (grown[*count] == NULL) + return HTS_FALSE; + (*count)++; + return HTS_TRUE; +} + +static void strlist_free(char ***list, size_t *count) { + size_t i; + + for (i = 0; i < *count; i++) + freet((*list)[i]); + freet(*list); + *count = 0; +} + +static void bundle_entry_free(bundle_entry *b) { + strlist_free(&b->named, &b->nnamed); + strlist_free(&b->urls, &b->nurls); + strlist_free(&b->scope, &b->nscope); + freet(b->ua); + StringFree(b->headers); + if (b->jar != NULL) + bauth_free(b->jar); + freet(b->jar); +} + +hts_boolean hts_bundle_name_valid(const char *name) { + const size_t len = strlen(name); + + if (len == 0 || len >= HTS_BUNDLE_NAME_MAX || name[0] == '.' || + strstr(name, "..") != NULL) + return HTS_FALSE; + return strspn(name, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789._-") == len; +} + +hts_bundle_suffix hts_bundle_cut(char *url, char *name, size_t name_size) { + char *const at = strstr(url, HTS_BUNDLE_SUFFIX); + const char *given; + + if (at == NULL) + return HTS_BUNDLE_NONE; + given = at + strlen(HTS_BUNDLE_SUFFIX); + if (!hts_bundle_name_valid(given) || strlen(given) >= name_size) + return HTS_BUNDLE_BAD; + strlcpybuff(name, given, name_size); + *at = '\0'; + return HTS_BUNDLE_NAMED; +} + +void hts_bundle_reset(httrackp *opt) { + bundle_set **link; + + hts_mutexlock(&bundle_lock); + for (link = &bundle_sets; *link != NULL; link = &(*link)->next) { + if ((*link)->opt == opt) { + bundle_set *set = *link; + size_t i; + + *link = set->next; + for (i = 0; i < set->count; i++) + bundle_entry_free(&set->items[i]); + freet(set->items); + freet(set->dir); + freet(set); + break; + } + } + hts_mutexrelease(&bundle_lock); +} + +hts_boolean hts_bundle_set_dir(httrackp *opt, const char *dir) { + hts_boolean ok = HTS_FALSE; + bundle_set *set; + + hts_mutexlock(&bundle_lock); + set = bundle_set_of(opt, HTS_TRUE); + if (set != NULL) { + freet(set->dir); + set->dir = strdupt(dir); + ok = set->dir != NULL; + } + hts_mutexrelease(&bundle_lock); + return ok; +} + +hts_boolean hts_bundle_name(httrackp *opt, const char *name, const char *url) { + hts_boolean ok = HTS_FALSE; + bundle_set *set; + + hts_mutexlock(&bundle_lock); + set = bundle_set_of(opt, HTS_TRUE); + if (set != NULL) { + bundle_entry *b = NULL; + size_t i; + + for (i = 0; i < set->count && b == NULL; i++) { + if (strcmp(set->items[i].name, name) == 0) + b = &set->items[i]; + } + if (b == NULL) { + bundle_entry *const grown = + realloct(set->items, (set->count + 1) * sizeof(bundle_entry)); + + if (grown != NULL) { + set->items = grown; + b = &grown[set->count++]; + memset(b, 0, sizeof(*b)); + StringInit(b->headers); + strcpybuff(b->name, name); + } + } + ok = b != NULL && strlist_add(&b->named, &b->nnamed, url); + } + hts_mutexrelease(&bundle_lock); + return ok; +} + +/* Read one line of fp into line, without its LF or a CR before it. Returns 1 + for a line, 0 at the end of the file, -1 for a line too long or holding a + NUL. */ +static int bundle_readline(FILE *fp, char *line, size_t size) { + size_t n = 0; + int c; + hts_boolean bad = HTS_FALSE; + + while ((c = fgetc(fp)) != EOF && c != '\n') { + if (c == '\0' || n + 1 >= size) + bad = HTS_TRUE; + else + line[n++] = (char) c; + } + if (c == EOF && n == 0 && !bad) + return 0; + if (n > 0 && line[n - 1] == '\r') + n--; + line[n] = '\0'; + return bad ? -1 : 1; +} + +/* Is line the version line of a bundle this engine reads? */ +static hts_boolean bundle_version(const char *line, hts_boolean *known) { + static const char magic[] = "# HTTrack bundle "; + const char *p = line; + size_t digits; + + if (strncmp(p, magic, sizeof(magic) - 1) != 0) + return HTS_FALSE; + p += sizeof(magic) - 1; + digits = strspn(p, "0123456789"); + if (digits == 0) + return HTS_FALSE; + *known = digits == 1 && p[0] == '1'; + p += digits; + if (*p == '.') { + p++; + digits = strspn(p, "0123456789"); + if (digits == 0) + return HTS_FALSE; + p += digits; + } + return *p == '\0'; +} + +/* Copy into site the domain a scope gets for host: its registrable domain, or + host itself when it has none and alone is HTS_TRUE. */ +static hts_boolean bundle_site(const char *host, hts_boolean alone, char *site, + size_t size) { + const char *reg = NULL; + + if (!hts_host_is_address(host, strlen(host))) + reg = hts_psl_registrable_domain(host); + if (reg == NULL && !alone) + return HTS_FALSE; + if (reg == NULL) + reg = host; + if (strlen(reg) >= size) + return HTS_FALSE; + strlcpybuff(site, reg, size); + return HTS_TRUE; +} + +static hts_boolean bundle_in_scope(const bundle_entry *b, const char *host) { + size_t i; + + for (i = 0; i < b->nscope; i++) { + if (cookie_domain_match(b->scope[i], host)) + return HTS_TRUE; + } + return HTS_FALSE; +} + +static hts_boolean bundle_scope_add(bundle_entry *b, const char *domain) { + size_t i; + + for (i = 0; i < b->nscope; i++) { + if (strcmp(b->scope[i], domain) == 0) + return HTS_TRUE; + } + return strlist_add(&b->scope, &b->nscope, domain); +} + +/* Add the scope of a URL naming b or listed in it. */ +static hts_boolean bundle_scope_url(bundle_entry *b, const char *url) { + char host[256]; + char site[256]; + hts_boolean https; + + return cookie_url_host(url, host, sizeof(host), &https) && + bundle_site(host, HTS_TRUE, site, sizeof(site)) && + bundle_scope_add(b, site); +} + +#define BUNDLE_FAIL(...) \ + do { \ + snprintf(err, err_size, __VA_ARGS__); \ + return HTS_FALSE; \ + } while (0) + +/* Read b's header lines up to "cookies", then its jar. */ +static hts_boolean bundle_parse(httrackp *opt, bundle_entry *b, FILE *fp, + hts_boolean *scope_given, char *err, + size_t err_size) { + char BIGSTK line[BUNDLE_LINE_MAX]; + int lineno = 0; + int got; + hts_boolean known; + + got = bundle_readline(fp, line, sizeof(line)); + lineno++; + { + static const char bom[] = "\xEF\xBB\xBF"; + const char *const first = + got > 0 && strncmp(line, bom, sizeof(bom) - 1) == 0 + ? line + sizeof(bom) - 1 + : line; + + if (got <= 0 || !bundle_version(first, &known)) + BUNDLE_FAIL("%s: not an HTTrack bundle, the first line must be " + "\"# HTTrack bundle 1\"", + b->file); + } + if (!known) + BUNDLE_FAIL("%s: this bundle version is not supported: %s", b->file, line); + *scope_given = HTS_FALSE; + while ((got = bundle_readline(fp, line, sizeof(line))) != 0) { + const char *value; + size_t klen; + + lineno++; + if (got < 0) + BUNDLE_FAIL("%s, line %d: line too long or holding a NUL", b->file, + lineno); + if (line[0] == '\0' || line[0] == '#') + continue; + klen = strcspn(line, " \t"); + value = line + klen; + value += strspn(value, " \t"); + if (klen == 7 && strncmp(line, "cookies", klen) == 0) { + int refused; + + if (*value != '\0') + BUNDLE_FAIL("%s, line %d: \"cookies\" takes no value", b->file, lineno); + b->jar = calloct(1, sizeof(t_cookie)); + if (b->jar == NULL) + BUNDLE_FAIL("%s: out of memory", b->file); + b->jar->max_len = BUNDLE_JAR_MAX; + refused = cookie_load_stream(opt, b->jar, fp, b->file); + if (refused == COOKIE_LOAD_KEEP) + BUNDLE_FAIL("%s: out of memory", b->file); + if (refused != 0) + BUNDLE_FAIL("%s: %d cookie line(s) could not be read, see the log", + b->file, refused); + return HTS_TRUE; + } + if (*value == '\0') + BUNDLE_FAIL("%s, line %d: no value", b->file, lineno); + if (klen == 3 && strncmp(line, "url", klen) == 0) { + char host[256]; + hts_boolean https; + + if (strlen(value) >= HTS_URLMAXSIZE || + strstr(value, HTS_BUNDLE_SUFFIX) != NULL || + !cookie_url_host(value, host, sizeof(host), &https) || + host[0] == '\0') + BUNDLE_FAIL("%s, line %d: invalid url: %.64s", b->file, lineno, value); + if (!strlist_add(&b->urls, &b->nurls, value)) + BUNDLE_FAIL("%s: out of memory", b->file); + } else if (klen == 2 && strncmp(line, "ua", klen) == 0) { + if (b->ua != NULL) + BUNDLE_FAIL("%s, line %d: ua given twice", b->file, lineno); + b->ua = strdupt(value); + if (b->ua == NULL) + BUNDLE_FAIL("%s: out of memory", b->file); + } else if (klen == 6 && strncmp(line, "header", klen) == 0) { + const char *const why = + http_bundle_header_refusal(StringBuff(b->headers), value); + + if (why != NULL) + BUNDLE_FAIL("%s, line %d: refused header (%s): %.64s", b->file, lineno, + why, value); + StringCat(b->headers, value); + StringCat(b->headers, "\r\n"); + } else if (klen == 5 && strncmp(line, "scope", klen) == 0) { + char dom[256]; + + if (value[0] == '.') + value++; + if (strlen(value) >= sizeof(dom) || + strspn(value, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789.-") != strlen(value) || + value[0] == '\0') + BUNDLE_FAIL("%s, line %d: invalid scope: %.64s", b->file, lineno, + value); + strcpybuff(dom, value); + hts_lowcase(dom); + if (!hts_host_is_address(dom, strlen(dom)) && + hts_psl_registrable_domain(dom) == NULL) + BUNDLE_FAIL("%s, line %d: scope %s is a public suffix", b->file, lineno, + dom); + if (!*scope_given) { + strlist_free(&b->scope, &b->nscope); + *scope_given = HTS_TRUE; + } + if (!bundle_scope_add(b, dom)) + BUNDLE_FAIL("%s: out of memory", b->file); + } else { + hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: unknown key %.*s", + b->file, lineno, (int) klen, line); + } + } + return HTS_TRUE; +} + +/* Set b's default scope, then drop its cookies outside the scope. */ +static hts_boolean bundle_scope(httrackp *opt, bundle_entry *b, + hts_boolean scope_given) { + size_t i; + char *rec; + + if (!scope_given) { + for (i = 0; i < b->nnamed; i++) { + if (!bundle_scope_url(b, b->named[i])) + return HTS_FALSE; + } + for (i = 0; i < b->nurls; i++) { + if (!bundle_scope_url(b, b->urls[i])) + return HTS_FALSE; + } + for (rec = b->jar != NULL ? b->jar->data : NULL; + rec != NULL && *rec != '\0'; rec = cookie_nextfield(rec)) { + char buffer[8192]; + const char *dom = cookie_get(buffer, rec, 0); + char site[256]; + + if (dom[0] == '.') + dom++; + if (bundle_site(dom, HTS_FALSE, site, sizeof(site)) && + !bundle_scope_add(b, site)) + return HTS_FALSE; + } + } + rec = b->jar != NULL ? b->jar->data : NULL; + while (rec != NULL && *rec != '\0') { + char buffer[8192]; + const char *dom = cookie_get(buffer, rec, 0); + + if (dom[0] == '.') + dom++; + if (bundle_in_scope(b, dom)) { + rec = cookie_nextfield(rec); + } else { + char name[8192]; + + hts_log_print(opt, LOG_WARNING, + "Bundle %s: cookie '%.64s' for %s is outside the bundle's " + "scope, refused", + b->file, cookie_get(name, rec, 5), dom); + cookie_delete(rec, b->jar->max_len - (size_t) (rec - b->jar->data), + (size_t) (cookie_nextfield(rec) - rec)); + } + } + return HTS_TRUE; +} + +static hts_boolean bundle_scopes_overlap(const bundle_entry *a, + const bundle_entry *b) { + size_t i, j; + + for (i = 0; i < a->nscope; i++) { + for (j = 0; j < b->nscope; j++) { + if (cookie_domain_match(a->scope[i], b->scope[j]) || + cookie_domain_match(b->scope[j], a->scope[i])) + return HTS_TRUE; + } + } + return HTS_FALSE; +} + +static hts_boolean bundle_same_identity(const bundle_entry *a, + const bundle_entry *b) { + if ((a->ua == NULL) != (b->ua == NULL)) + return HTS_FALSE; + if (a->ua != NULL && strcmp(a->ua, b->ua) != 0) + return HTS_FALSE; + return strcmp(StringBuff(a->headers), StringBuff(b->headers)) == 0; +} + +static hts_boolean bundle_load(httrackp *opt, bundle_set *set, bundle_entry *b, + char *err, size_t err_size) { + const char *const dir = + set->dir != NULL ? set->dir : StringBuff(opt->path_log); + const size_t dlen = strlen(dir); + hts_boolean scope_given = HTS_FALSE; + hts_boolean ok; + FILE *fp; + + if (!slprintfbuff( + b->file, sizeof(b->file), "%s%s%s.hts", dir, + dlen != 0 && dir[dlen - 1] != '/' && dir[dlen - 1] != '\\' ? "/" : "", + b->name)) + BUNDLE_FAIL("bundle %s: the folder path is too long", b->name); + fp = FOPEN(b->file, "rb"); + if (fp == NULL) + BUNDLE_FAIL("%s: cannot read the login bundle: %s", b->file, + strerror(errno)); + ok = bundle_parse(opt, b, fp, &scope_given, err, err_size); + fclose(fp); + if (!ok) + return HTS_FALSE; + if (!bundle_scope(opt, b, scope_given)) + BUNDLE_FAIL("%s: cannot work out the bundle's scope", b->file); + return HTS_TRUE; +} + +hts_boolean hts_bundle_load_all(httrackp *opt, char *err, size_t err_size) { + hts_boolean ok = HTS_TRUE; + bundle_set *set; + size_t i, j; + + err[0] = '\0'; + hts_mutexlock(&bundle_lock); + set = bundle_set_of(opt, HTS_FALSE); + for (i = 0; set != NULL && i < set->count && ok; i++) + ok = bundle_load(opt, set, &set->items[i], err, err_size); + for (i = 0; set != NULL && i < set->count && ok; i++) { + for (j = i + 1; j < set->count && ok; j++) { + bundle_entry *const a = &set->items[i]; + bundle_entry *const b = &set->items[j]; + char name[256]; + + if (bundle_scopes_overlap(a, b) && !bundle_same_identity(a, b)) { + snprintf(err, err_size, + "%s and %s cover the same site with a different user agent " + "or headers", + a->file, b->file); + ok = HTS_FALSE; + } else if (a->jar != NULL && b->jar != NULL && + cookie_jars_conflict(a->jar, b->jar, name, sizeof(name))) { + snprintf(err, err_size, + "%s and %s hold two values of the cookie '%.64s'", a->file, + b->file, name); + ok = HTS_FALSE; + } + } + } + hts_mutexrelease(&bundle_lock); + return ok; +} + +char *hts_bundle_urls(httrackp *opt) { + String all = STRING_EMPTY; + bundle_set *set; + size_t i, j; + + hts_mutexlock(&bundle_lock); + set = bundle_set_of(opt, HTS_FALSE); + for (i = 0; set != NULL && i < set->count; i++) { + for (j = 0; j < set->items[i].nurls; j++) { + StringCat(all, set->items[i].urls[j]); + StringCat(all, "\n"); + } + } + hts_mutexrelease(&bundle_lock); + return StringNotEmpty(all) ? StringAcquire(&all) : NULL; +} + +int hts_bundle_cookie_count(httrackp *opt) { + int count = 0; + bundle_set *set; + size_t i; + + hts_mutexlock(&bundle_lock); + set = bundle_set_of(opt, HTS_FALSE); + for (i = 0; set != NULL && i < set->count; i++) { + char *rec; + + for (rec = set->items[i].jar != NULL ? set->items[i].jar->data : NULL; + rec != NULL && *rec != '\0'; rec = cookie_nextfield(rec)) + count++; + } + hts_mutexrelease(&bundle_lock); + return count; +} + +int hts_bundle_cookies(httrackp *opt, t_cookie *jar) { + int refused = 0; + bundle_set *set; + size_t i; + + hts_mutexlock(&bundle_lock); + set = bundle_set_of(opt, HTS_FALSE); + for (i = 0; set != NULL && i < set->count; i++) { + if (set->items[i].jar != NULL) + refused += cookie_merge(jar, set->items[i].jar); + } + hts_mutexrelease(&bundle_lock); + return refused; +} + +hts_boolean hts_bundle_identity(httrackp *opt, const char *adr, const char **ua, + const char **headers) { + hts_boolean changed = HTS_FALSE; + char host[256]; + hts_boolean https; + bundle_set *set; + size_t i; + + hts_mutexlock(&bundle_lock); + set = bundle_set_of(opt, HTS_FALSE); + if (set != NULL && set->count != 0 && + cookie_url_host(adr, host, sizeof(host), &https)) { + for (i = 0; i < set->count; i++) { + const bundle_entry *const b = &set->items[i]; + + if (bundle_in_scope(b, host)) { + if (b->ua != NULL) { + *ua = b->ua; + changed = HTS_TRUE; + } + if (StringNotEmpty(b->headers)) + *headers = StringBuff(b->headers); + break; + } + } + } + hts_mutexrelease(&bundle_lock); + return changed; +} + +void hts_bundle_dump(httrackp *opt) { + bundle_set *set; + size_t i, j; + + hts_mutexlock(&bundle_lock); + set = bundle_set_of(opt, HTS_FALSE); + for (i = 0; set != NULL && i < set->count; i++) { + const bundle_entry *const b = &set->items[i]; + char *rec; + + printf("bundle %s\n", b->name); + if (b->ua != NULL) + printf("ua %s\n", b->ua); + for (j = 0; j < b->nurls; j++) + printf("url %s\n", b->urls[j]); + for (j = 0; j < b->nscope; j++) + printf("scope %s\n", b->scope[j]); + if (StringNotEmpty(b->headers)) { + const char *p; + + for (p = StringBuff(b->headers); *p != '\0';) { + const size_t len = strcspn(p, "\r\n"); + + printf("header %.*s\n", (int) len, p); + p += len; + p += strspn(p, "\r\n"); + } + } + for (rec = b->jar != NULL ? b->jar->data : NULL; + rec != NULL && *rec != '\0'; rec = cookie_nextfield(rec)) { + char dom[8192], name[8192], value[8192]; + + printf("cookie %s %s=%s\n", cookie_get(dom, rec, 0), + cookie_get(name, rec, 5), cookie_get(value, rec, 6)); + } + } + hts_mutexrelease(&bundle_lock); +} diff --git a/src/htsbundle.h b/src/htsbundle.h new file mode 100644 index 000000000..3eb8b2c0b --- /dev/null +++ b/src/htsbundle.h @@ -0,0 +1,97 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) 2026 Xavier Roche and other contributors + +SPDX-License-Identifier: GPL-3.0-or-later + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Ethical use: we kindly ask that you NOT use this software to harvest email +addresses or to collect any other private information about people. Doing so +would dishonor our work and waste the many hours we have spent on it. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: htsbundle.h subroutines: */ +/* login bundles named by a start URL */ +/* ------------------------------------------------------------ */ + +#ifndef HTSBUNDLE_DEFH +#define HTSBUNDLE_DEFH + +#ifdef HTS_INTERNAL_BYTECODE + +#include + +#include "htsglobal.h" +#include "htsbauth.h" + +/* A start URL ending with this names the bundle file .hts. */ +#define HTS_BUNDLE_SUFFIX "#httrack-bundle=" + +/* A bundle name holds fewer bytes than this. */ +#define HTS_BUNDLE_NAME_MAX 128 + +typedef enum { + HTS_BUNDLE_NONE, /* no suffix */ + HTS_BUNDLE_NAMED, /* a valid name, cut off the URL */ + HTS_BUNDLE_BAD /* a suffix whose name is refused, URL left alone */ +} hts_bundle_suffix; + +/* Is name a bundle name: [A-Za-z0-9._-], not starting with '.' and no ".."? */ +hts_boolean hts_bundle_name_valid(const char *name); + +/* Cut the bundle suffix off url in place and copy its name into name. */ +hts_bundle_suffix hts_bundle_cut(char *url, char *name, size_t name_size); + +/* Forget the bundles and the --bundle-dir recorded for opt. */ +void hts_bundle_reset(httrackp *opt); + +/* Read names from dir rather than the project folder. HTS_FALSE when out of + memory. */ +hts_boolean hts_bundle_set_dir(httrackp *opt, const char *dir); + +/* Record that the start URL url (suffix already cut) names bundle name. A + name given twice is one bundle. HTS_FALSE when out of memory. */ +hts_boolean hts_bundle_name(httrackp *opt, const char *name, const char *url); + +/* Read every named bundle, then check scopes and identities. HTS_FALSE after + writing why into err, naming the file: the run must stop. */ +hts_boolean hts_bundle_load_all(httrackp *opt, char *err, size_t err_size); + +/* The bundles' url lines, each followed by a LF, in a malloct'd string the + caller freet's. NULL when there are none. */ +char *hts_bundle_urls(httrackp *opt); + +/* How many cookies the loaded bundles hold. */ +int hts_bundle_cookie_count(httrackp *opt); + +/* Store the bundles' cookies into jar. Returns how many it refused. */ +int hts_bundle_cookies(httrackp *opt, t_cookie *jar); + +/* When adr's host is in a bundle's scope, point *ua and *headers at that + bundle's, for each it gives. Returns HTS_TRUE when it changed *ua. The + strings live until hts_bundle_reset. */ +hts_boolean hts_bundle_identity(httrackp *opt, const char *adr, const char **ua, + const char **headers); + +/* Print each loaded bundle (UA, headers, scope, cookies) for the self-tests. */ +void hts_bundle_dump(httrackp *opt); + +#endif + +#endif diff --git a/src/htsbundle_selftest.c b/src/htsbundle_selftest.c new file mode 100644 index 000000000..539775cd6 --- /dev/null +++ b/src/htsbundle_selftest.c @@ -0,0 +1,129 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) 2026 Xavier Roche and other contributors + +SPDX-License-Identifier: GPL-3.0-or-later + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Ethical use: we kindly ask that you NOT use this software to harvest email +addresses or to collect any other private information about people. Doing so +would dishonor our work and waste the many hours we have spent on it. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: htsbundle_selftest.c subroutines: */ +/* self-tests for the login bundles */ +/* ------------------------------------------------------------ */ + +#include "htsselftest_int.h" + +#include "htsbundle.h" + +/* -#test=bundlecut : the bundle a start URL names, and the URL left. */ +static int st_bundlecut(httrackp *opt, int argc, char **argv) { + char url[HTS_URLMAXSIZE * 2]; + char name[HTS_BUNDLE_NAME_MAX]; + + (void) opt; + if (argc != 1 || strlen(argv[0]) >= sizeof(url)) { + fprintf(stderr, "bundlecut: takes \n"); + return 1; + } + strcpybuff(url, argv[0]); + switch (hts_bundle_cut(url, name, sizeof(name))) { + case HTS_BUNDLE_NONE: + printf("none\n"); + break; + case HTS_BUNDLE_BAD: + printf("bad\n"); + break; + case HTS_BUNDLE_NAMED: + printf("%s %s\n", name, url); + break; + } + return 0; +} + +/* -#test=bundle

... [host:]...: load the bundles the URLs + name from dir, print them, then the UA and headers each host gets. */ +static int st_bundle(httrackp *opt, int argc, char **argv) { + FILE *const log = opt->log; + char err[1024]; + int i; + + if (argc < 2) { + fprintf(stderr, "bundle: takes ... [host:]...\n"); + return 1; + } + hts_bundle_reset(opt); + assertf(hts_bundle_set_dir(opt, argv[0])); + opt->log = stderr; /* warnings stay out of the graded output */ + for (i = 1; i < argc; i++) { + char url[HTS_URLMAXSIZE * 2]; + char name[HTS_BUNDLE_NAME_MAX]; + + if (strncmp(argv[i], "host:", 5) == 0 || strlen(argv[i]) >= sizeof(url)) + continue; + strcpybuff(url, argv[i]); + if (hts_bundle_cut(url, name, sizeof(name)) == HTS_BUNDLE_NAMED) + assertf(hts_bundle_name(opt, name, url)); + } + if (!hts_bundle_load_all(opt, err, sizeof(err))) { + /* the folder is the caller's, so the message is graded without it */ + const size_t dlen = strlen(argv[0]); + const char *p = err; + const char *at; + + printf("refused: "); + while ((at = strstr(p, argv[0])) != NULL) { + printf("%.*s", (int) (at - p), p); + p = at + dlen; + } + printf("%s\n", p); + hts_bundle_reset(opt); + opt->log = log; + return 0; + } + hts_bundle_dump(opt); + for (i = 1; i < argc; i++) { + if (strncmp(argv[i], "host:", 5) == 0) { + const char *ua = "global"; + const char *headers = "global"; + const char *p; + + (void) hts_bundle_identity(opt, argv[i] + 5, &ua, &headers); + printf("%s ua=%s headers=", argv[i] + 5, ua); + /* one line per host, so the block's line ends print as '|' */ + for (p = headers; *p != '\0'; p++) { + if (*p != '\r') + putchar(*p == '\n' ? '|' : *p); + } + putchar('\n'); + } + } + hts_bundle_reset(opt); + opt->log = log; + return 0; +} + +const struct selftest_entry selftests_bundle[] = { + {"bundlecut", "", "the login bundle a start URL names", st_bundlecut}, + {"bundle", " ... [host:]...", + "load login bundles and print the identity each host gets", st_bundle}, + {NULL, NULL, NULL, NULL}, +}; diff --git a/src/htscore.c b/src/htscore.c index fc198a27c..e5ebd13fe 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com /* File defs */ #include "htscore.h" +#include "htsbundle.h" /* after htscore.h, which is what pulls in config.h and so HAVE_PATHS_H */ #ifndef _WIN32 @@ -783,14 +784,9 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { opt->cookie = &cookie; cookie.max_len = 30000; // max len strcpybuff(cookie.data, ""); - if (strnotempty(StringBuff(opt->cookies_file))) { - // loaded once the start URLs are known, and left alone until then - cookies_file_refused = COOKIE_LOAD_KEEP; - } else { - cookies_file_refused = cookie_load( - opt, opt->cookie, StringBuff(opt->path_log), "cookies.txt"); - cookie_load(opt, opt->cookie, "", "cookies.txt"); - } + /* loaded over the login bundles once the start URLs are known, and left + alone until then */ + cookies_file_refused = COOKIE_LOAD_KEEP; } else { opt->cookie = NULL; if (strnotempty(StringBuff(opt->cookies_file))) @@ -1050,6 +1046,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { /* load URL file list */ /* OPTIMIZED for fast load */ + hts_boolean bundle_refused = HTS_FALSE; /* a -%L line named a bad one */ + if (StringNotEmpty(opt->filelist)) { char *filelist_buff = NULL; size_t filelist_sz = 0; @@ -1092,6 +1090,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { int n = 0; int lineno = 0; char BIGSTK line[HTS_URLMAXSIZE * 2]; + char name[HTS_BUNDLE_NAME_MAX]; while(filelist_ptr < filelist_sz) { int adv; @@ -1132,6 +1131,22 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { StringBuff(opt->filelist), lineno, line); continue; } + switch (hts_bundle_cut(line, name, sizeof(name))) { + case HTS_BUNDLE_NONE: + break; + case HTS_BUNDLE_BAD: + hts_log_print(opt, LOG_ERROR, + "\"%s\", line %d: invalid login bundle name: %s", + StringBuff(opt->filelist), lineno, line); + bundle_refused = HTS_TRUE; + continue; + case HTS_BUNDLE_NAMED: + if (!hts_bundle_name(opt, name, line)) { + hts_log_print(opt, LOG_ERROR, "Login bundle: out of memory"); + bundle_refused = HTS_TRUE; + } + break; + } if (line[0]) { n++; if (strstr(line, ":/") == NULL) { @@ -1152,6 +1167,85 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { } } + /* the login bundles: their url lines are start URLs, and their cookies go + first in the jar, so what the server refreshed since wins */ + { + char err[1024]; + char *extra; + + err[0] = '\0'; + if (bundle_refused || !hts_bundle_load_all(opt, err, sizeof(err))) { + if (err[0] != '\0') { + hts_log_print(opt, LOG_ERROR, "Login bundle: %s", err); + fprintf(stderr, "Login bundle: %s\n", err); + } else { + fprintf(stderr, "Login bundle: %s refused, see the log\n", + StringBuff(opt->filelist)); + } + freet(primary); + XH_extuninit; + return 0; + } + extra = hts_bundle_urls(opt); + if (extra != NULL) { + size_t lines = 0; + size_t room; + const char *p; + char *grown; + + for (p = extra; *p != '\0'; p++) { + if (*p == '\n') + lines++; + } + /* "http://" for each line, and the NUL */ + room = strlen(extra) + lines * 7 + 1; + grown = room < (size_t) -1 - primarybuff.cap + ? (char *) realloct(primary, primarybuff.cap + room) + : NULL; + if (grown == NULL) { + printf("PANIC! : Not enough memory [%d]\n", __LINE__); + freet(extra); + freet(primary); + XH_extuninit; + return 0; + } + primary = grown; + primarybuff.buf = grown; + primarybuff.cap += room; + for (p = extra; *p != '\0';) { + const size_t len = strcspn(p, "\n"); + + if (strstr(p, ":/") == NULL || strstr(p, ":/") > p + len) + htsbuff_cat(&primarybuff, "http://"); + htsbuff_catn(&primarybuff, p, len); + htsbuff_cat(&primarybuff, "\n"); + p += len; + if (*p == '\n') + p++; + } + freet(extra); + } + if (opt->accept_cookie) { + const int refused = hts_bundle_cookies(opt, opt->cookie); + + if (refused != 0) { + hts_log_print(opt, LOG_ERROR, + "Login bundle: the jar is full, %d cookie(s) left out", + refused); + fprintf(stderr, + "Login bundle: the jar is full, %d cookie(s) left out\n", + refused); + freet(primary); + XH_extuninit; + return 0; + } + } else if (hts_bundle_cookie_count(opt) != 0) { + hts_log_print(opt, LOG_WARNING, + "Cookies are disabled, so the login bundles' cookies " + "are not sent"); + } + } + // the --cookies-file is the whole jar, because the crawl writes it back if (opt->accept_cookie && strnotempty(StringBuff(opt->cookies_file))) { char BIGSTK start_url[HTS_URLMAXSIZE * 2]; @@ -1162,6 +1256,10 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { strlncatbuff(start_url, primary, sizeof(start_url), first); cookies_file_refused = cookie_load_from( opt, opt->cookie, "", StringBuff(opt->cookies_file), start_url); + } else if (opt->accept_cookie) { + cookies_file_refused = cookie_load( + opt, opt->cookie, StringBuff(opt->path_log), "cookies.txt"); + cookie_load(opt, opt->cookie, "", "cookies.txt"); } // lien primaire diff --git a/src/htscoremain.c b/src/htscoremain.c index 5f37eb582..7c57a9bba 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -58,6 +58,7 @@ Please visit our Website: http://www.httrack.com #include "htsmd5.h" #include "htsthread.h" #include "htsfetch.h" +#include "htsbundle.h" #include /* hts_self_path() */ @@ -568,6 +569,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { int httrack_logmode = 3; // ONE log file ensureUrlCapacity(url, url_sz, 65536); + hts_bundle_reset(opt); /* a previous run's bundles are not this one's */ // Create options _DEBUG_HEAD = 0; // pas de debuggage en têtes @@ -2531,6 +2533,29 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { #endif break; + case 'b': // --bundle-dir: where #httrack-bundle= names resolve + /* a vetted pair's value may begin with '-' (#1425) */ + if ((na + 1 >= argc) || + (argv[na + 1][0] == '-' && !optparam_dash_ok(&x_cmd, na))) { + HTS_PANIC_PRINTF("Option bundle-dir needs a blank space and " + "a folder"); + printf("Example: --bundle-dir \"/home/me/logins\"\n"); + htsmain_free(); + return -1; + } + na++; + if (strlen(argv[na]) >= HTS_URLMAXSIZE) { + HTS_PANIC_PRINTF("Bundle folder path too long"); + htsmain_free(); + return -1; + } + if (!hts_bundle_set_dir(opt, argv[na])) { + HTS_PANIC_PRINTF("Not enough memory"); + htsmain_free(); + return -1; + } + break; + case 'm': { // --mptcp: Multipath TCP where the system has it int on = 1; @@ -2853,6 +2878,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { } else { // URL/filters char catbuff[CATBUFF_SIZE]; + /* argv itself goes to doit.log, so the bundle suffix is cut here */ + char word[HTS_CDLMAXSIZE]; const size_t urlSize = strlen(argv[na]); const size_t capa = strlen(url) + urlSize + 32; @@ -2867,15 +2894,38 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { fprintf(stderr, "* Warning: filter %s holds a quote, kept as is\n", argv[na]); assertf(urlSize < HTS_URLMAXSIZE); + strcpybuff(word, argv[na]); + if (argv[na][0] != '+' && argv[na][0] != '-') { + char name[HTS_BUNDLE_NAME_MAX]; + + switch (hts_bundle_cut(word, name, sizeof(name))) { + case HTS_BUNDLE_NONE: + break; + case HTS_BUNDLE_BAD: + cmdl_panic_value(opt, + "Invalid login bundle name (letters, digits, " + "'.', '_' and '-' only)", + argv[na]); + htsmain_free(); + return -1; + case HTS_BUNDLE_NAMED: + if (!hts_bundle_name(opt, name, word)) { + HTS_PANIC_PRINTF("Not enough memory"); + htsmain_free(); + return -1; + } + break; + } + } if (urlSize < HTS_URLMAXSIZE) { ensureUrlCapacity(url, url_sz, capa); if (strnotempty(url)) strlcatbuff(url, " ", url_sz); // separator space /* a rule is decoded once, where the engine splits the rules */ append_escape_spc_url( - argv[na][0] == '+' || argv[na][0] == '-' - ? argv[na] - : unescape_http_unharm(catbuff, sizeof(catbuff), argv[na], 1), + word[0] == '+' || word[0] == '-' + ? word + : unescape_http_unharm(catbuff, sizeof(catbuff), word, 1), url, url_sz); } } // if argv=- etc. diff --git a/src/htshelp.c b/src/htshelp.c index 015d25061..d09717f61 100644 --- a/src/htshelp.c +++ b/src/htshelp.c @@ -594,6 +594,10 @@ void help(const char *app, int more) { infomsg("Spider options:"); infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)"); infomsg(" %K use a Netscape cookies.txt as the jar, rewritten at the end"); + infomsg(" @b folder holding the login bundles that start URLs name"); + infomsg(" #httrack-bundle= crawls with the user agent,"); + infomsg(" headers and cookies of .hts, by default in the project " + "folder"); infomsg(" %Y explain which filter rule accepts or rejects a URL, then exit"); infomsg (" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)"); diff --git a/src/htslib.c b/src/htslib.c index 235ec5720..e41fdd685 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -36,6 +36,7 @@ Please visit our Website: http://www.httrack.com // Fichier librairie .c #include "htscore.h" +#include "htsbundle.h" #include "htssitemap.h" #include "htswarc.h" #include "htschanges.h" @@ -1156,6 +1157,19 @@ const char *http_custom_header_warning(const char *headers, const char *line) { return NULL; } +const char *http_bundle_header_refusal(const char *headers, const char *line) { + static const char *const engine[] = {"Host", "Cookie"}; + const char *const why = http_custom_header_warning(headers, line); + + if (why != NULL) + return why; + if (header_name_in(line, engine, sizeof(engine) / sizeof(engine[0]))) + return "Host and Cookie are the engine's"; + if (strfield(line, "Proxy-")) + return "a proxy header is not for the site"; + return NULL; +} + /* Dropping a line drops its folded continuations too: kept, that value would attach to whichever header precedes it in the request. */ static void append_custom_headers(buff_struct *bstr, const char *headers) { @@ -7574,6 +7588,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) { /* An FTP worker reads opt for its whole run, and not every caller drains its threads the way httrack.c does before getting here. */ ftp_stop_workers(); + hts_bundle_reset(opt); /* Alocated callbacks */ diff --git a/src/htslib.h b/src/htslib.h index 20755dca5..a3caba3df 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -328,6 +328,11 @@ hts_boolean hts_body_missing_unexpectedly(const htsblk *r); typed. headers is the block so far. */ const char *http_custom_header_warning(const char *headers, const char *line); +/* Why a login bundle's header line is refused, or NULL when it is allowed: + the -%X warnings, plus Host, Cookie and Proxy-*. headers is the block so + far. */ +const char *http_bundle_header_refusal(const char *headers, const char *line); + /* Append the custom header block to dst (capacity dst_size), dropping any line whose field dst carries, a malformed or framing line, an earlier copy of Host, Cookie or Authorization, and a leading folded line. Field names ignore diff --git a/src/htsselftest.c b/src/htsselftest.c index 1c8083fbd..b0dc0e573 100644 --- a/src/htsselftest.c +++ b/src/htsselftest.c @@ -223,7 +223,7 @@ static const struct selftest_entry *const selftest_tables[] = { selftests_filters, selftests_header, selftests_io, selftests_mime, selftests_name, selftests_net, selftests_opt, selftests_parse, selftests_tools, selftests_warc, selftests_wizard, selftests_lines, - selftests_fetch, selftests_psl, + selftests_fetch, selftests_psl, selftests_bundle, }; static void list_selftests(void) { diff --git a/src/htsselftest_int.h b/src/htsselftest_int.h index a3f9b89a0..e15a3505e 100644 --- a/src/htsselftest_int.h +++ b/src/htsselftest_int.h @@ -134,5 +134,6 @@ extern const struct selftest_entry selftests_warc[]; extern const struct selftest_entry selftests_wizard[]; extern const struct selftest_entry selftests_fetch[]; extern const struct selftest_entry selftests_psl[]; +extern const struct selftest_entry selftests_bundle[]; #endif diff --git a/src/libhttrack.vcxproj b/src/libhttrack.vcxproj index f219a6e6d..44b27da14 100644 --- a/src/libhttrack.vcxproj +++ b/src/libhttrack.vcxproj @@ -141,6 +141,7 @@ + @@ -156,6 +157,7 @@ + diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test new file mode 100644 index 000000000..dc9821cee --- /dev/null +++ b/tests/655_engine-bundle.test @@ -0,0 +1,151 @@ +#!/bin/bash +# +# Login bundles (#httrack-bundle=NAME): the name a start URL carries, the file +# format, the scope each bundle covers, and the bundles refused at startup. + +set -euo pipefail + +# shellcheck source=tests/testlib.sh +. "${0%"${0##*/}"}./testlib.sh" + +tmpdir=$(mktemp -d) +cleanup_push rm -rf "$tmpdir" +dir=$(nativepath "$tmpdir") + +bundle() { # bundle NAME LINE... -- writes $tmpdir/NAME.hts + local name=$1 + shift + printf '%s\n' "$@" >"$tmpdir/$name.hts" +} +tab=$(printf '\t') +jar() { # jar DOMAIN NAME VALUE -- one Netscape jar line + printf '%s' ".$1${tab}TRUE${tab}/${tab}FALSE${tab}0${tab}$2${tab}$3" +} + +selftest_queue 'login1 https://example.com/members/' bundlecut \ + 'https://example.com/members/#httrack-bundle=login1' +selftest_queue 'a.b_c-9 http://x/' bundlecut 'http://x/#httrack-bundle=a.b_c-9' +selftest_queue none bundlecut 'http://x/#top' +selftest_queue bad bundlecut 'http://x/#httrack-bundle=' +selftest_queue bad bundlecut 'http://x/#httrack-bundle=../x' +selftest_queue bad bundlecut 'http://x/#httrack-bundle=a..b' +selftest_queue bad bundlecut 'http://x/#httrack-bundle=.hidden' +selftest_queue bad bundlecut 'http://x/#httrack-bundle=a/b' +selftest_queue bad bundlecut 'http://x/#httrack-bundle=a\b' +selftest_queue bad bundlecut 'http://x/#httrack-bundle=a%2fb' +selftest_queue bad bundlecut 'http://x/#httrack-bundle=a b' + +# The default scope is the registrable domains of the naming URL, the url lines +# and the cookie domains. Only the hosts in it get the UA and the headers. +bundle alpha '# HTTrack bundle 1' 'url https://login.alpha-sso.test/start' \ + 'ua AlphaUA/1.0' 'header Authorization: Bearer t' 'header X-A: 1' \ + 'unknown-key is warned about' cookies "$(jar alpha.test SID s1)" \ + "$(jar alpha-cdn.test C c1)" +selftest_queue_lines "bundle alpha +ua AlphaUA/1.0 +url https://login.alpha-sso.test/start +scope alpha.test +scope alpha-sso.test +scope alpha-cdn.test +header Authorization: Bearer t +header X-A: 1 +cookie .alpha.test SID=s1 +cookie .alpha-cdn.test C=c1 +alpha.test ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1| +https://deep.www.alpha.test:8443 ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1| +login.alpha-sso.test ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1| +cdn.gamma.test ua=global headers=global +notalpha.test ua=global headers=global" bundle "$dir" \ + 'http://www.alpha.test/#httrack-bundle=alpha' host:alpha.test \ + host:https://deep.www.alpha.test:8443 host:login.alpha-sso.test \ + host:cdn.gamma.test host:notalpha.test + +# scope lines replace the default, and a cookie outside them is dropped +bundle narrow '# HTTrack bundle 1.3' 'scope WWW.alpha.test' 'ua N' cookies \ + "$(jar alpha.test SID wide)" "$(jar www.alpha.test K narrow)" +selftest_queue_lines "bundle narrow +ua N +scope www.alpha.test +cookie .www.alpha.test K=narrow +alpha.test ua=global headers=global +a.www.alpha.test ua=N headers=global" bundle "$dir" \ + 'http://www.alpha.test/#httrack-bundle=narrow' host:alpha.test \ + host:a.www.alpha.test + +# An address has no registrable domain, so it scopes to itself alone. +bundle ip '# HTTrack bundle 1' 'ua I' +selftest_queue_lines "bundle ip +ua I +scope 127.0.0.1 +127.0.0.1 ua=I headers=global +127.0.0.2 ua=global headers=global" bundle "$dir" \ + 'http://127.0.0.1:8080/#httrack-bundle=ip' host:127.0.0.1 host:127.0.0.2 + +# Refused: each one stops the run and names the file. The batch reads them all +# at the end, so each case writes its own. +nbad=0 +refused() { # refused WHY LINE... -- a one-bundle file the engine refuses + local why=$1 + shift + nbad=$((nbad + 1)) + bundle "bad$nbad" "$@" + selftest_queue "refused: /bad$nbad.hts$why" bundle "$dir" \ + "http://www.alpha.test/#httrack-bundle=bad$nbad" +} +refused ': this bundle version is not supported: # HTTrack bundle 2' \ + '# HTTrack bundle 2' +refused ': not an HTTrack bundle, the first line must be "# HTTrack bundle 1"' \ + 'ua X' +refused ', line 2: refused header (Host and Cookie are the engine'"'"'s): Host: x' \ + '# HTTrack bundle 1' 'header Host: x' +refused ', line 2: refused header (Host and Cookie are the engine'"'"'s): cookie: a=b' \ + '# HTTrack bundle 1' 'header cookie: a=b' +refused ', line 2: refused header (a proxy header is not for the site): Proxy-Authorization: x' \ + '# HTTrack bundle 1' 'header Proxy-Authorization: x' +refused ', line 2: refused header (framing header ignored): Transfer-Encoding: chunked' \ + '# HTTrack bundle 1' 'header Transfer-Encoding: chunked' +refused ', line 3: refused header (header replaces an earlier one): Authorization: b' \ + '# HTTrack bundle 1' 'header Authorization: a' 'header Authorization: b' +refused ', line 2: refused header (malformed header ignored): no colon' \ + '# HTTrack bundle 1' 'header no colon' +refused ', line 3: ua given twice' '# HTTrack bundle 1' 'ua A' 'ua B' +refused ', line 2: scope co.uk is a public suffix' '# HTTrack bundle 1' \ + 'scope co.uk' +refused ', line 2: invalid url: ftp:///' '# HTTrack bundle 1' 'url ftp:///' +refused ', line 2: no value' '# HTTrack bundle 1' 'ua' +refused ': 1 cookie line(s) could not be read, see the log' \ + '# HTTrack bundle 1' cookies 'not a jar line' +selftest_queue 'refused: /none.hts: cannot read the login bundle: No such file or directory' \ + bundle "$dir" 'http://www.alpha.test/#httrack-bundle=none' + +# Several bundles: one name twice is one bundle, equal identities on one site +# merge, and different ones are refused. +bundle beta '# HTTrack bundle 1' 'ua BetaUA' cookies "$(jar beta.test SID b1)" +bundle alpha2 '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ + 'header Authorization: Bearer t' 'header X-A: 1' cookies \ + "$(jar alpha.test OTHER o)" +bundle alphaua '# HTTrack bundle 1' 'ua SomeoneElse' +bundle alphasid '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ + 'header Authorization: Bearer t' 'header X-A: 1' cookies \ + "$(jar alpha.test SID another-account)" +selftest_queue_lines "bundle beta +ua BetaUA +scope beta.test +cookie .beta.test SID=b1 +www.beta.test ua=BetaUA headers=global" bundle "$dir" \ + 'http://www.beta.test/#httrack-bundle=beta' \ + 'http://beta.test/x#httrack-bundle=beta' host:www.beta.test +selftest_queue_tail " +alpha.test ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1| +beta.test ua=BetaUA headers=global" bundle "$dir" \ + 'http://www.alpha.test/#httrack-bundle=alpha' \ + 'http://alpha.test/#httrack-bundle=alpha2' \ + 'http://beta.test/#httrack-bundle=beta' host:alpha.test host:beta.test +selftest_queue 'refused: /alpha.hts and /alphaua.hts cover the same site with a different user agent or headers' \ + bundle "$dir" 'http://www.alpha.test/#httrack-bundle=alpha' \ + 'http://alpha.test/#httrack-bundle=alphaua' +selftest_queue "refused: /alpha.hts and /alphasid.hts hold two values of the cookie 'SID'" \ + bundle "$dir" 'http://www.alpha.test/#httrack-bundle=alpha' \ + 'http://alpha.test/#httrack-bundle=alphasid' + +selftest_run_queued diff --git a/tests/656_local-bundles.test b/tests/656_local-bundles.test new file mode 100644 index 000000000..788fe17b6 --- /dev/null +++ b/tests/656_local-bundles.test @@ -0,0 +1,112 @@ +#!/bin/bash +# +# Two login bundles and a CDN on a third domain, graded on what the server +# received: each site gets its own UA, Authorization and cookies, and the CDN +# gets the global -F and -%X and none of the bundles' identity. + +set -euo pipefail + +# shellcheck source=tests/testlib.sh +. "${0%"${0##*/}"}./testlib.sh" + +python=$(find_python) || skip "python3 missing" + +server=$(nativepath "$top_srcdir/tests/bundle-server.py") +tmpdir=$(mktemp -d) +serverpid= +log=$tmpdir/crawl.log +fail_dump_var log + +cleanup() { + stop_server "$serverpid" + rm -rf "$tmpdir" +} +cleanup_push cleanup + +"$python" "$server" >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" & +serverpid=$! +port=$(discover_server_port "$tmpdir/srv.out" "$serverpid") || exit 1 + +tab=$(printf '\t') +jar() { # jar DOMAIN NAME VALUE -- one Netscape jar line + printf '%s' ".$1${tab}TRUE${tab}/${tab}FALSE${tab}0${tab}$2${tab}$3" +} +mkdir "$tmpdir/logins" "$tmpdir/one" "$tmpdir/two" +printf '%s\n' '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ + 'header Authorization: Bearer alpha-token' cookies \ + "$(jar alpha.test SID alpha-captured)" "$(jar alpha.test PREF alpha-pref)" \ + >"$tmpdir/logins/alpha.test.hts" +printf '%s\n' '# HTTrack bundle 1' 'ua BetaUA/2.0' \ + 'header Authorization: Bearer beta-token' cookies \ + "$(jar beta.test SID beta-captured)" >"$tmpdir/logins/beta.test.hts" +cp "$tmpdir/logins/alpha.test.hts" "$tmpdir/alpha.orig" +# the project jar loads over the bundle, as a cookie the server refreshed +jar alpha.test SID alpha-refreshed >"$tmpdir/one/cookies.txt" + +crawl() { # crawl DIR [httrack args...] + local dir=$1 rc=0 + shift + run_with_timeout 120 httrack -O "$(nativepath "$dir")" \ + -P "127.0.0.1:$port" -F GlobalUA/9 -%X 'X-Global: 1' \ + '+*cdn.gamma.test/*' -c1 --robots=0 -%v0 -q "$@" >>"$log" 2>&1 || rc=$? + test "$rc" -ne 124 || fail "crawl hit the deadline" + return "$rc" +} + +hit() { # hit WHAT LINE -- the server must have logged LINE exactly + command grep -qxF "HIT $2" "$tmpdir/srv.out" || + fail_dump "$1: no request [HIT $2]" "$tmpdir/srv.out" +} + +crawl "$tmpdir/one" --bundle-dir "$(nativepath "$tmpdir/logins")" \ + 'http://www.alpha.test/#httrack-bundle=alpha.test' \ + 'http://www.beta.test/#httrack-bundle=beta.test' \ + 'http://alpha.test/plain.html' || fail "the bundle crawl failed" + +alpha='ua=AlphaUA/1.0 auth=Bearer alpha-token cookie=PREF=alpha-pref; SID=alpha-refreshed x=-' +hit "start page" "www.alpha.test/ $alpha" +hit "crawled link" "www.alpha.test/page.html $alpha" +hit "plain URL on the bundle's site" "alpha.test/plain.html $alpha" +hit "second bundle" "www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-" +for gif in a.gif b.gif; do + hit "CDN" "cdn.gamma.test/$gif ua=GlobalUA/9 auth=- cookie=- x=1" +done +n=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) +test "$n" -eq 6 || fail_dump "$n requests, expected 6" "$tmpdir/srv.out" + +# --update finds the bundle again: doit.log keeps the name and the folder +doit=$tmpdir/one/hts-cache/doit.log +command grep -qF '#httrack-bundle=alpha.test' "$doit" || + fail_dump "doit.log lost the bundle name" "$doit" +command grep -qF -- "-@b " "$doit" || fail_dump "doit.log lost --bundle-dir" "$doit" +cmp -s "$tmpdir/alpha.orig" "$tmpdir/logins/alpha.test.hts" || + fail "the crawl wrote to the bundle" + +# A -%L list names a bundle too, and it resolves in the project folder. +cp "$tmpdir/logins/beta.test.hts" "$tmpdir/two/" +printf '%s\n' 'http://www.beta.test/#httrack-bundle=beta.test' >"$tmpdir/list.txt" +before=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) +crawl "$tmpdir/two" -%L "$(nativepath "$tmpdir/list.txt")" || + fail "the -%L crawl failed" +after=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) +test "$((after - before))" -eq 2 || + fail_dump "-%L crawl: $((after - before)) requests, expected 2" "$tmpdir/srv.out" +n=$(command grep -cxF "HIT www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-" "$tmpdir/srv.out" || true) +test "$n" -eq 2 || fail_dump "-%L crawl: the beta page did not get its bundle" "$tmpdir/srv.out" + +# A missing bundle stops the run before any request, naming the file. +rc=0 +crawl "$tmpdir/two" 'http://www.alpha.test/#httrack-bundle=nosuch' || rc=$? +test "$rc" -ne 0 || fail "a missing bundle did not stop the run" +command grep -qF "nosuch.hts: cannot read the login bundle" "$log" || + fail_dump "no message naming the missing bundle" "$log" +final=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) +test "$final" -eq "$after" || fail_dump "a missing bundle still crawled" "$tmpdir/srv.out" + +# So does a -%L line naming an invalid one. +printf '%s\n' 'http://www.beta.test/#httrack-bundle=../beta.test' >"$tmpdir/bad.txt" +rc=0 +crawl "$tmpdir/two" -%L "$(nativepath "$tmpdir/bad.txt")" || rc=$? +test "$rc" -ne 0 || fail "an invalid -%L bundle name did not stop the run" +final=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) +test "$final" -eq "$after" || fail_dump "an invalid -%L bundle name still crawled" "$tmpdir/srv.out" diff --git a/tests/bundle-server.py b/tests/bundle-server.py new file mode 100644 index 000000000..634bda134 --- /dev/null +++ b/tests/bundle-server.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Three sites behind one proxy port, for 656_local-bundles. + +httrack reaches it as its -P proxy, so the absolute request URL names the +site. Every request prints one "HIT ua=<...> auth=<...> +cookie=<...> x=<...>" line, "-" for a missing header. Prints "PORT ". +""" + +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + +CDN = "http://cdn.gamma.test" +PAGES = { + "www.alpha.test/": 'p' % CDN, + "www.alpha.test/page.html": "alpha page", + "alpha.test/plain.html": "plain URL on the alpha site", + "www.beta.test/": '' % CDN, + "cdn.gamma.test/a.gif": "GIF89a", + "cdn.gamma.test/b.gif": "GIF89a", +} + +lock = threading.Lock() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.0" + + def log_message(self, *args): + pass + + def do_GET(self): + url = urlsplit(self.path) + key = url.hostname + url.path if url.hostname else self.path + with lock: + print( + "HIT %s ua=%s auth=%s cookie=%s x=%s" + % ( + key, + self.headers.get("User-Agent", "-"), + self.headers.get("Authorization", "-"), + self.headers.get("Cookie", "-"), + self.headers.get("X-Global", "-"), + ), + flush=True, + ) + body = PAGES.get(key) + if body is None: + return self.reply(404, b"no", "text/html") + ctype = "image/gif" if key.endswith(".gif") else "text/html" + if ctype == "text/html": + body = "%s" % body + self.reply(200, body.encode(), ctype) + + def reply(self, code, body, ctype): + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class Server(ThreadingHTTPServer): + request_queue_size = 32 # BSD/macOS drop connections at Python's default 5 + + +# The shell greps these lines anchored, and a native python writes CRLF. +sys.stdout.reconfigure(newline="\n") + +srv = Server(("127.0.0.1", 0), Handler) +print("PORT %d" % srv.server_address[1], flush=True) +srv.serve_forever() From c7195605e622473fe4987e3878cb6677826f152e Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 14:33:54 +0200 Subject: [PATCH 02/10] Rework login bundles after review round 1 Keep the bundle state in two httrackp tail fields (bundle_dir, bundle_state) instead of a global registry and its mutex, and bump the VERSION_INFO revision. Load bundle cookies straight into the crawl's jar through a scope filter in the existing Netscape loader, which -%K and cookies.txt still call without one. Of the bundle and the jar, the newer file wins, and the bundle wins a tie. Three security bugs are fixed: - a refused header's message printed its value, which can be a secret, so it now prints the header name only - a start URL on a public suffix or a single label scoped its whole subtree, so it now scopes to that host alone - a CR or other control byte inside a bundle line reached the wire, so the line is now refused --bundle-dir is long-only and doit.log keeps it absolute. Cookie domains no longer widen the default scope, and cookies outside it are dropped with one summary warning. Header lines go over https alone when every URL naming the bundle is https. A repeated cookie or unknown comment warns instead of stopping the run. Windows device names are refused as bundle names, and scope and url lines are capped at 256. The new prototypes moved out of the installed htsbauth.h, which is back to master's bytes. The cookies.txt load order for runs without a bundle is master's again. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- configure.ac | 4 +- html/httrack.man.html | 81 ++-- man/httrack.1 | 37 +- man/httrack.tmpl | 32 +- src/htsalias.c | 4 +- src/htsback.c | 4 +- src/htsbauth.c | 108 +++-- src/htsbauth.h | 21 - src/htsbundle.c | 745 +++++++++++++++-------------------- src/htsbundle.h | 84 ++-- src/htsbundle_selftest.c | 68 ++-- src/htscore.c | 155 ++++---- src/htscoremain.c | 74 ++-- src/htshelp.c | 5 +- src/htslib.c | 5 +- src/htslines.c | 12 +- src/htslines.h | 3 + src/htsopt.h | 4 + tests/655_engine-bundle.test | 211 +++++++--- tests/656_local-bundles.test | 173 +++++--- tests/Makefile.am | 2 +- tests/bundle-server.py | 13 +- 22 files changed, 967 insertions(+), 878 deletions(-) diff --git a/configure.ac b/configure.ac index 1929749bd..c8820efb2 100644 --- a/configure.ac +++ b/configure.ac @@ -41,6 +41,8 @@ AC_CONFIG_HEADERS(config.h) # header we can install without handing a consumer our PACKAGE_* and HAVE_*. AC_CONFIG_HEADERS([htsfeatures.h:src/htsfeatures.h.in]) AM_INIT_AUTOMAKE([subdir-objects]) +# 3:27:0: revision-only bump. httrackp gained two tail fields (bundle_dir, +# bundle_state) for the login bundles; no export came or went. # 3:26:0: revision-only bump. No export came or went and no installed struct # moved, per nm -D and gdb ptype /o against 3.50.4. The installed headers # gained HTS_FALLTHROUGH and two static inline path checks in htssafe.h, and @@ -112,7 +114,7 @@ AM_INIT_AUTOMAKE([subdir-objects]) # moves nothing). Soname stays .so.3: HTTrackQt is the only consumer of the installed # headers, so a libhttrack4 rename isn't worth it. # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.) -VERSION_INFO="3:26:0" +VERSION_INFO="3:27:0" AM_MAINTAINER_MODE AC_USE_SYSTEM_EXTENSIONS diff --git a/html/httrack.man.html b/html/httrack.man.html index 33cd33d87..a91d46ffc 100644 --- a/html/httrack.man.html +++ b/html/httrack.man.html @@ -101,16 +101,15 @@

SYNOPSIS --generate-errors[=N] ] [ -X, --purge-old[=0] ] [ -%p, --preserve ] [ -%T, --utf8-conversion[=0] ] [ -bN, --cookies[=N] ] [ -%K, --cookies-file -] [ -@b, --bundle-dir ] [ -%Y, --why ] [ --u, --check-type[=N] ] [ -j, --parse-java[=N] -] [ -sN, --robots[=N] ] [ -%h, --http-10[=0] ] -[ -%k, --keep-alive[=0] ] [ -%z, ---disable-compression[=0] ] [ -%B, --tolerant[=0] -] [ -%s, --updatehack[=0] ] [ -%u, ---urlhack[=0] ] [ -%A, --assume ] [ -@iN, ---protocol[=N] ] [ -@mN, --mptcp[=0] ] [ -%w, ---disable-module ] [ -F, --user-agent ] [ -%R, ---referer ] [ -%E, --from ] [ -%F, +] [ -%Y, --why ] [ -u, --check-type[=N] ] [ +-j, --parse-java[=N] ] [ -sN, --robots[=N] ] [ +-%h, --http-10[=0] ] [ -%k, --keep-alive[=0] ] +[ -%z, --disable-compression[=0] ] [ -%B, +--tolerant[=0] ] [ -%s, --updatehack[=0] ] [ +-%u, --urlhack[=0] ] [ -%A, --assume ] [ +-@iN, --protocol[=N] ] [ -@mN, --mptcp[=0] ] [ +-%w, --disable-module ] [ -F, --user-agent ] [ +-%R, --referer ] [ -%E, --from ] [ -%F, --footer ] [ -%l, --language ] [ -%a, --accept ] [ -%X, --headers ] [ -%Q, --browser-headers[=0] ] [ -C, --cache[=N] ] [ @@ -882,25 +881,14 @@

Spider options:

use a Netscape cookies.txt as the jar, rewritten at the end (--cookies-file <param>)

- - -

-@b

 - - -

folder holding the login bundles that start URLs name -(--bundle-dir <param>)

<URL>#httrack-bundle=<name> crawls <URL> with the user agent,
-headers and cookies of <name>.hts, by default in the -project folder

+headers and cookies of the login bundle <name>.hts, +read from the project folder or from --bundle-dir +<dir>

@@ -2497,29 +2485,42 @@

FILES https://example.com/members/#httrack-bundle=login1. The name holds letters, digits, ., _ and - only. HTTrack reads the file in the project folder, -or in the folder given to --bundle-dir, and never -writes it. HTTrack reads the suffix on the command line, in -a -%L list and in doit.log, never on a crawled link. -A missing file, or one that does not parse, stops the run -before any request.

+or in the folder given to --bundle-dir <dir>, +and never writes it. doit.log keeps that folder as an +absolute path. HTTrack reads the suffix on the command line, +in a -%L list and in doit.log, never on a crawled +link. A missing file, or one that does not parse, stops the +run before any request.

The first line -is # HTTrack bundle 1. Then come url lines -(more start URLs), at most one ua line, and +is # HTTrack bundle 1. Then come url lines, +each one more start URL, at most one ua line, and header and scope lines. A cookies line ends them, and a Netscape cookie jar follows it.

The bundle’s user agent and headers replace -F and --%X for each host in its scope, and only there. By -default the scope is the registrable domains of the start -URL naming the bundle, of its url lines and of its cookies. -Scope lines replace that default. HTTrack refuses a cookie -outside the scope, and a Host, Cookie, Proxy-* or framing -header. Its cookies load before cookies.txt, so a cookie the -server refreshed wins. Two bundles whose scopes overlap must -give the same user agent, headers and cookie values, or the -run stops.

+-%X for each host in its scope, and only there. A +bundle with header lines but no ua line sends the +-F user agent. When every start URL naming the bundle +is https, its headers go over https alone. By default the +scope is the registrable domains of the start URLs naming +the bundle and of its url lines. A host with no registrable +domain, such as a public suffix, a single label or an +address, covers itself alone. Scope lines replace that +default. HTTrack drops a cookie outside the scope and gives +the count in one warning. HTTrack refuses a Host, Cookie, +Proxy-* or framing header, and a line holding a control +byte. Two bundles whose scopes overlap must give the same +user agent, headers and cookie values, or the run stops.

+ +

The +bundle’s cookies go into the cookie jar, which HTTrack +writes at the end of the crawl. Of the bundle and the jar, +the newer file wins a cookie both hold. A new capture +therefore wins over the jar, and a cookie the server +refreshed wins over an older bundle. On equal times the +bundle wins.

A running mirror also watches its own output directory for these. Creating diff --git a/man/httrack.1 b/man/httrack.1 index b4f0ff7f2..4dc28b01e 100644 --- a/man/httrack.1 +++ b/man/httrack.1 @@ -56,7 +56,6 @@ httrack \- offline browser : copy websites to a local directory [ \fB\-%T, \-\-utf8\-conversion[=0]\fR ] [ \fB\-bN, \-\-cookies[=N]\fR ] [ \fB\-%K, \-\-cookies\-file\fR ] -[ \fB\-@b, \-\-bundle\-dir\fR ] [ \fB\-%Y, \-\-why\fR ] [ \fB\-u, \-\-check\-type[=N]\fR ] [ \fB\-j, \-\-parse\-java[=N]\fR ] @@ -243,12 +242,10 @@ links conversion to UTF\-8 (\-\-utf8\-conversion[=0]) accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N]) .IP \-%K use a Netscape cookies.txt as the jar, rewritten at the end (\-\-cookies\-file ) -.IP \-@b -folder holding the login bundles that start URLs name (\-\-bundle\-dir ) .br #httrack\-bundle= crawls with the user agent, .br -headers and cookies of .hts, by default in the project folder +headers and cookies of the login bundle .hts, read from the project folder or from \-\-bundle\-dir

.IP \-%Y explain which filter rule accepts or rejects a URL, then exit (\-\-why ) .IP \-u @@ -601,8 +598,9 @@ The name holds letters, digits, and .I \- only. HTTrack reads the file in the project folder, or in the folder given to -.IR \-\-bundle\-dir , -and never writes it. HTTrack reads the suffix on the command line, in a +.IR "\-\-bundle\-dir " , +and never writes it. doit.log keeps that folder as an absolute path. HTTrack +reads the suffix on the command line, in a .I \-%L list and in doit.log, never on a crawled link. A missing file, or one that does not parse, stops the run before any request. @@ -611,7 +609,7 @@ The first line is .IR "# HTTrack bundle 1" . Then come .I url -lines (more start URLs), at most one +lines, each one more start URL, at most one .I ua line, and .I header @@ -625,13 +623,24 @@ The bundle's user agent and headers replace .I \-F and .I \-%X -for each host in its scope, and only there. By default the scope is the -registrable domains of the start URL naming the bundle, of its url lines and -of its cookies. Scope lines replace that default. HTTrack refuses a cookie -outside the scope, and a Host, Cookie, Proxy\-* or framing header. Its cookies -load before cookies.txt, so a cookie the server refreshed wins. Two bundles -whose scopes overlap must give the same user agent, headers and cookie values, -or the run stops. +for each host in its scope, and only there. A bundle with header lines but no +.I ua +line sends the +.I \-F +user agent. When every start URL naming the bundle is https, its headers go +over https alone. By default the scope is the registrable domains of the start +URLs naming the bundle and of its url lines. A host with no registrable domain, +such as a public suffix, a single label or an address, covers itself alone. +Scope lines replace that default. HTTrack drops a cookie outside the scope and +gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or +framing header, and a line holding a control byte. Two bundles whose scopes +overlap must give the same user agent, headers and cookie values, or the run +stops. +.IP +The bundle's cookies go into the cookie jar, which HTTrack writes at the end of +the crawl. Of the bundle and the jar, the newer file wins a cookie both hold. +A new capture therefore wins over the jar, and a cookie the server refreshed +wins over an older bundle. On equal times the bundle wins. .PP A running mirror also watches its own output directory for these. Creating one is how a script drives an engine it has no other handle on. On Windows, which diff --git a/man/httrack.tmpl b/man/httrack.tmpl index 6c42d01c0..16189ea9f 100644 --- a/man/httrack.tmpl +++ b/man/httrack.tmpl @@ -81,8 +81,9 @@ The name holds letters, digits, and .I \- only. HTTrack reads the file in the project folder, or in the folder given to -.IR \-\-bundle\-dir , -and never writes it. HTTrack reads the suffix on the command line, in a +.IR "\-\-bundle\-dir " , +and never writes it. doit.log keeps that folder as an absolute path. HTTrack +reads the suffix on the command line, in a .I \-%L list and in doit.log, never on a crawled link. A missing file, or one that does not parse, stops the run before any request. @@ -91,7 +92,7 @@ The first line is .IR "# HTTrack bundle 1" . Then come .I url -lines (more start URLs), at most one +lines, each one more start URL, at most one .I ua line, and .I header @@ -105,13 +106,24 @@ The bundle's user agent and headers replace .I \-F and .I \-%X -for each host in its scope, and only there. By default the scope is the -registrable domains of the start URL naming the bundle, of its url lines and -of its cookies. Scope lines replace that default. HTTrack refuses a cookie -outside the scope, and a Host, Cookie, Proxy\-* or framing header. Its cookies -load before cookies.txt, so a cookie the server refreshed wins. Two bundles -whose scopes overlap must give the same user agent, headers and cookie values, -or the run stops. +for each host in its scope, and only there. A bundle with header lines but no +.I ua +line sends the +.I \-F +user agent. When every start URL naming the bundle is https, its headers go +over https alone. By default the scope is the registrable domains of the start +URLs naming the bundle and of its url lines. A host with no registrable domain, +such as a public suffix, a single label or an address, covers itself alone. +Scope lines replace that default. HTTrack drops a cookie outside the scope and +gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or +framing header, and a line holding a control byte. Two bundles whose scopes +overlap must give the same user agent, headers and cookie values, or the run +stops. +.IP +The bundle's cookies go into the cookie jar, which HTTrack writes at the end of +the crawl. Of the bundle and the jar, the newer file wins a cookie both hold. +A new capture therefore wins over the jar, and a cookie the server refreshed +wins over an older bundle. On equal times the bundle wins. .PP A running mirror also watches its own output directory for these. Creating one is how a script drives an engine it has no other handle on. On Windows, which diff --git a/src/htsalias.c b/src/htsalias.c index 99061caff..67afa64d1 100644 --- a/src/htsalias.c +++ b/src/htsalias.c @@ -165,7 +165,7 @@ const char *hts_optalias[][4] = { "longest Retry-After delay obeyed, in seconds"}, {"mptcp", "-@m", "onoff", "carry connections over Multipath TCP where the system offers it"}, - {"bundle-dir", "-@b", "param1", + {"bundle-dir", "--bundle-dir", "param1", "folder where #httrack-bundle= names are read, not the project folder"}, {"generate-errors", "-o", "level", ""}, {"do-not-generate-errors", "-o0", "single", ""}, @@ -349,7 +349,7 @@ static const char *const dashvalue_opt[] = { "-%L", /* list */ "-%S", /* urllist */ "-%rf", /* warc-file */ - "-@b", /* bundle-dir */ + "--bundle-dir", "-F", /* user-agent */ NULL }; diff --git a/src/htsback.c b/src/htsback.c index f45300b92..458e3ad23 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -3070,9 +3070,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back[p].r.req.headers = StringBuff(opt->headers); back[p].r.req.user_agent_send = opt->user_agent_send; /* a login bundle's site gets the bundle's UA and headers */ - if (hts_bundle_identity(opt, adr, &back[p].r.req.user_agent, - &back[p].r.req.headers)) - back[p].r.req.user_agent_send = 1; + hts_bundle_identity(opt, adr, &back[p].r.req); // et http11 back[p].r.req.http11 = back[p].http11; back[p].r.req.nocompression = opt->nocompression; diff --git a/src/htsbauth.c b/src/htsbauth.c index f64d26fd8..a344ecc43 100644 --- a/src/htsbauth.c +++ b/src/htsbauth.c @@ -43,6 +43,7 @@ Please visit our Website: http://www.httrack.com #include "htslib.h" #include "htscore.h" #include "htspsl.h" +#include "htsbundle.h" #include "htsdate.h" #include "htscookie_selftest.h" #ifdef _WIN32 @@ -652,7 +653,7 @@ static hts_boolean cookie_is_flat(FILE *fp, int *quote) { return flat; } -/* See htsbauth.h. */ +/* See htsbundle.h. */ hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size, hts_boolean *https) { const char *const rest = jump_protocol_const(url); @@ -817,11 +818,43 @@ static int cookie_load_flat(httrackp *opt, t_cookie *cookie, FILE *fp, return COOKIE_LOAD_KEEP; } +/* Should filter let the record name=value for domain and path into cookie? + seen holds this load's records, which may repeat one without a clash. */ +static hts_boolean cookie_admit(cookie_filter *filter, t_cookie *cookie, + t_cookie *seen, const char *name, + const char *value, const char *domain, + const char *path) { + char buffer[8192]; + char host[256]; + const char *other; + + if (!cookie_host(domain, host, sizeof(host)) || + !filter->covers(filter->scope, host[0] == '.' ? host + 1 : host)) { + filter->dropped++; + return HTS_FALSE; + } + other = cookie_find_exact(filter->others->data, name, host, path); + if (other != NULL && + (seen == NULL || + cookie_find_exact(seen->data, name, host, path) == NULL) && + strcmp(cookie_get(buffer, other, 6), value) != 0) { + filter->clash[0] = '\0'; + strlncatbuff(filter->clash, name, sizeof(filter->clash), + sizeof(filter->clash) - 1); + return HTS_FALSE; + } + (void) cookie_set(filter->others, name, value, domain, path, NULL); + return !filter->under || + cookie_find_exact(cookie->data, name, host, path) == NULL; +} + /* Read the Netscape records from fp's position to its end into cookie, and name the file as given in warnings. A UTF-8 BOM is skipped on the first line - when first_line is HTS_TRUE. Returns what cookie_load_from does. */ + when first_line is HTS_TRUE. filter, when not NULL, picks the records. + Returns what cookie_load_from does. */ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, - const char *given, hts_boolean first_line) { + const char *given, hts_boolean first_line, + cookie_filter *filter) { char buffer[8192]; int refused = 0; hts_boolean keep = HTS_FALSE; @@ -926,7 +959,9 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, attr.http_only = is_httponly || tabs == 7; attr.expires = cookie_record_expiry(rec); // an expired one is skipped, because a browser drops it too - if (attr.expires == 0 || attr.expires > (int64_t) time(NULL)) { + if ((attr.expires == 0 || attr.expires > (int64_t) time(NULL)) && + (filter == NULL || cookie_admit(filter, cookie, seen, cook_name, + cook_value, domain, path))) { const hts_boolean repeated = seen != NULL && cookie_host(domain, host, sizeof(host)) && cookie_find_exact(seen->data, cook_name, host, path) != NULL; @@ -952,6 +987,8 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, "%s", cook_name, given); refused++; + if (filter != NULL) + filter->lost++; } else if (seen != NULL && !repeated) { (void) cookie_set(seen, cook_name, "", domain, path, NULL); } @@ -966,6 +1003,8 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, hts_log_print(opt, LOG_WARNING, "Cookies: stopped reading before the end of %s", given); refused++; + if (filter != NULL) + filter->lost++; } freet(seen); return keep ? COOKIE_LOAD_KEEP : refused; @@ -1088,7 +1127,8 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, return ret; } if (fp) { - const int ret = cookie_load_records(opt, cookie, fp, given, HTS_TRUE); + const int ret = + cookie_load_records(opt, cookie, fp, given, HTS_TRUE, NULL); fclose(fp); return ret; @@ -1103,60 +1143,10 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, return -1; } -/* See htsbauth.h. */ -int cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, - const char *name) { - return cookie_load_records(opt, cookie, fp, name, HTS_FALSE); -} - -/* See htsbauth.h. */ -int cookie_merge(t_cookie *dst, t_cookie *src) { - char BIGSTK domain[8192]; - char BIGSTK path[8192]; - char BIGSTK name[8192]; - char BIGSTK value[8192]; - char buffer[8192]; - int refused = 0; - char *a; - - for (a = src->data; *a != '\0'; a = cookie_nextfield(a)) { - cookie_attr attr; - - attr.host_only = cookie_is_host_only(a); - attr.secure = strcmpnocase(cookie_get(buffer, a, 3), "TRUE") == 0; - attr.http_only = - strcmp(cookie_get(buffer, a, 7), COOKIE_HTTPONLY_FIELD) == 0; - attr.expires = cookie_record_expiry(a); - if (cookie_set(dst, cookie_get(name, a, 5), cookie_get(value, a, 6), - cookie_get(domain, a, 0), cookie_get(path, a, 2), - &attr) != 0) - refused++; - } - return refused; -} - -/* See htsbauth.h. */ -hts_boolean cookie_jars_conflict(t_cookie *a, t_cookie *b, char *name, - size_t name_size) { - char BIGSTK domain[8192]; - char BIGSTK path[8192]; - char BIGSTK cook_name[8192]; - char buffer[8192]; - char *rec; - - for (rec = a->data; *rec != '\0'; rec = cookie_nextfield(rec)) { - const char *const other = - cookie_find_exact(b->data, cookie_get(cook_name, rec, 5), - cookie_get(domain, rec, 0), cookie_get(path, rec, 2)); - - if (other != NULL && - strcmp(cookie_get(buffer, rec, 6), cookie_get(domain, other, 6)) != 0) { - name[0] = '\0'; - strlncatbuff(name, cook_name, name_size, name_size - 1); - return HTS_TRUE; - } - } - return HTS_FALSE; +/* See htsbundle.h. */ +void cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, + const char *name, cookie_filter *filter) { + (void) cookie_load_records(opt, cookie, fp, name, HTS_FALSE, filter); } /* Open name for writing, creating it if needed, and return NULL on failure. diff --git a/src/htsbauth.h b/src/htsbauth.h index da304f2b4..c925774bf 100644 --- a/src/htsbauth.h +++ b/src/htsbauth.h @@ -73,7 +73,6 @@ struct t_cookie { /* Library internal definitions */ #ifdef HTS_INTERNAL_BYTECODE -#include #include #ifndef HTS_DEF_FWSTRUCT_httrackp @@ -90,11 +89,6 @@ typedef struct httrackp httrackp; HTS_FALSE means the caller sends no cookie. */ hts_boolean cookie_host(const char *adr, char *dst, size_t dst_size); -/** Copy the host of url, with or without a scheme, into dst as cookie_host - does, and set *https. */ -hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size, - hts_boolean *https); - /** A jar record holds this many bytes of name plus value. */ #define COOKIE_MAX_PAIR 4096 @@ -154,21 +148,6 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *path, const char *name, const char *start_url); #define COOKIE_LOAD_KEEP (-2) -/** Load Netscape jar records from fp's position to its end, as cookie_load - reads a file, naming name in warnings. fp stays open. Returns the count of - lines skipped, or COOKIE_LOAD_KEEP when memory ran out. */ -int cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, - const char *name); - -/** Store every record of src into dst, with its flags and expiry, replacing - a record of the same name, domain and path. Returns how many dst refused. */ -int cookie_merge(t_cookie *dst, t_cookie *src); - -/** Does b hold a record with the name, domain and path of one in a, but - another value? If so, copy that name into name (name_size bytes). */ -hts_boolean cookie_jars_conflict(t_cookie *a, t_cookie *b, char *name, - size_t name_size); - /** Write the jar's cookies to name in Netscape format, owner-only on Unix. The new jar goes to a temporary file first, which then replaces name in one step, so name holds the old or the new jar whatever fails. A symbolic link diff --git a/src/htsbundle.c b/src/htsbundle.c index f613e9a07..17b9abbfe 100644 --- a/src/htsbundle.c +++ b/src/htsbundle.c @@ -36,62 +36,38 @@ Please visit our Website: http://www.httrack.com #include "htscore.h" #include "htslib.h" +#include "htslines.h" #include "htspsl.h" #include "htssafe.h" -#include "htsthread.h" #include /* A bundle line, the jar aside, holds fewer bytes than this. */ #define BUNDLE_LINE_MAX 8192 -/* The jar room the crawl's own jar gets (see httpmirror). */ -#define BUNDLE_JAR_MAX 30000 +/* A bundle holds at most this many url lines, and as many scope entries. */ +#define BUNDLE_LIST_MAX 256 -typedef struct bundle_entry { +typedef struct bundle { char name[HTS_BUNDLE_NAME_MAX]; char file[HTS_URLMAXSIZE * 2]; - char **named; /* the start URLs naming it */ - size_t nnamed; + char **start_urls; /* the start URLs naming it */ + size_t nstart; char **urls; /* its url lines */ size_t nurls; - char **scope; /* lowercase domains, each covering its subdomains */ + /* lowercase domains, each covering its subdomains, or "=host" for a host + alone */ + char **scope; size_t nscope; char *ua; /* NULL when it gives none */ String headers; - t_cookie *jar; /* NULL until loaded */ -} bundle_entry; - -/* The bundles of one opt, kept out of httrackp so its layout is unchanged. */ -typedef struct bundle_set { - const httrackp *opt; - char *dir; - bundle_entry *items; - size_t count; - struct bundle_set *next; -} bundle_set; - -static htsmutex bundle_lock = HTSMUTEX_INIT; -static bundle_set *bundle_sets = NULL; + hts_boolean https_only; /* Are its headers for https alone? */ +} bundle; -/* Call with bundle_lock held. */ -static bundle_set *bundle_set_of(const httrackp *opt, hts_boolean create) { - bundle_set *set; - - for (set = bundle_sets; set != NULL; set = set->next) { - if (set->opt == opt) - return set; - } - if (!create) - return NULL; - set = calloct(1, sizeof(*set)); - if (set != NULL) { - set->opt = opt; - set->next = bundle_sets; - bundle_sets = set; - } - return set; -} +typedef struct hts_bundle_state { + bundle *items; + size_t count; +} hts_bundle_state; static hts_boolean strlist_add(char ***list, size_t *count, const char *s) { char **const grown = realloct(*list, (*count + 1) * sizeof(char *)); @@ -115,243 +91,252 @@ static void strlist_free(char ***list, size_t *count) { *count = 0; } -static void bundle_entry_free(bundle_entry *b) { - strlist_free(&b->named, &b->nnamed); +/* Forget what reading b's file gave. */ +static void bundle_clear(bundle *b) { strlist_free(&b->urls, &b->nurls); strlist_free(&b->scope, &b->nscope); freet(b->ua); - StringFree(b->headers); - if (b->jar != NULL) - bauth_free(b->jar); - freet(b->jar); + StringClear(b->headers); } -hts_boolean hts_bundle_name_valid(const char *name) { +/* Is name a bundle name: [A-Za-z0-9._-], no leading '.', no "..", and no + Windows device name such as CON or com1.txt? */ +static hts_boolean bundle_name_valid(const char *name) { + static const char *const devices[] = {"con", "prn", "aux", "nul"}; const size_t len = strlen(name); + const size_t base = strcspn(name, "."); + char dev[5]; + size_t i; if (len == 0 || len >= HTS_BUNDLE_NAME_MAX || name[0] == '.' || - strstr(name, "..") != NULL) + strstr(name, "..") != NULL || + strspn(name, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789._-") != len) return HTS_FALSE; - return strspn(name, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - "0123456789._-") == len; + if (base < 3 || base > 4) + return HTS_TRUE; + dev[0] = '\0'; + strlncatbuff(dev, name, sizeof(dev), base); + hts_lowcase(dev); + for (i = 0; base == 3 && i < sizeof(devices) / sizeof(devices[0]); i++) { + if (strcmp(dev, devices[i]) == 0) + return HTS_FALSE; + } + return base == 3 || dev[3] < '1' || dev[3] > '9' || + (strncmp(dev, "com", 3) != 0 && strncmp(dev, "lpt", 3) != 0); } hts_bundle_suffix hts_bundle_cut(char *url, char *name, size_t name_size) { char *const at = strstr(url, HTS_BUNDLE_SUFFIX); - const char *given; + const char *text; if (at == NULL) return HTS_BUNDLE_NONE; - given = at + strlen(HTS_BUNDLE_SUFFIX); - if (!hts_bundle_name_valid(given) || strlen(given) >= name_size) + text = at + strlen(HTS_BUNDLE_SUFFIX); + if (!bundle_name_valid(text) || strlen(text) >= name_size) return HTS_BUNDLE_BAD; - strlcpybuff(name, given, name_size); + strlcpybuff(name, text, name_size); *at = '\0'; return HTS_BUNDLE_NAMED; } -void hts_bundle_reset(httrackp *opt) { - bundle_set **link; - - hts_mutexlock(&bundle_lock); - for (link = &bundle_sets; *link != NULL; link = &(*link)->next) { - if ((*link)->opt == opt) { - bundle_set *set = *link; - size_t i; - - *link = set->next; - for (i = 0; i < set->count; i++) - bundle_entry_free(&set->items[i]); - freet(set->items); - freet(set->dir); - freet(set); - break; - } - } - hts_mutexrelease(&bundle_lock); -} - -hts_boolean hts_bundle_set_dir(httrackp *opt, const char *dir) { - hts_boolean ok = HTS_FALSE; - bundle_set *set; +void hts_bundle_free(httrackp *opt) { + hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state; + size_t i; - hts_mutexlock(&bundle_lock); - set = bundle_set_of(opt, HTS_TRUE); - if (set != NULL) { - freet(set->dir); - set->dir = strdupt(dir); - ok = set->dir != NULL; + if (st == NULL) + return; + for (i = 0; i < st->count; i++) { + bundle_clear(&st->items[i]); + strlist_free(&st->items[i].start_urls, &st->items[i].nstart); + StringFree(st->items[i].headers); } - hts_mutexrelease(&bundle_lock); - return ok; + freet(st->items); + freet(opt->bundle_state); } -hts_boolean hts_bundle_name(httrackp *opt, const char *name, const char *url) { - hts_boolean ok = HTS_FALSE; - bundle_set *set; +hts_boolean hts_bundle_add(httrackp *opt, const char *name, const char *url) { + hts_bundle_state *st = (hts_bundle_state *) opt->bundle_state; + bundle *b = NULL; + size_t i; - hts_mutexlock(&bundle_lock); - set = bundle_set_of(opt, HTS_TRUE); - if (set != NULL) { - bundle_entry *b = NULL; - size_t i; + if (st == NULL) { + st = calloct(1, sizeof(*st)); + if (st == NULL) + return HTS_FALSE; + opt->bundle_state = st; + } + for (i = 0; i < st->count && b == NULL; i++) { + if (strcmp(st->items[i].name, name) == 0) + b = &st->items[i]; + } + if (b == NULL) { + bundle *const grown = realloct(st->items, (st->count + 1) * sizeof(bundle)); - for (i = 0; i < set->count && b == NULL; i++) { - if (strcmp(set->items[i].name, name) == 0) - b = &set->items[i]; - } - if (b == NULL) { - bundle_entry *const grown = - realloct(set->items, (set->count + 1) * sizeof(bundle_entry)); - - if (grown != NULL) { - set->items = grown; - b = &grown[set->count++]; - memset(b, 0, sizeof(*b)); - StringInit(b->headers); - strcpybuff(b->name, name); - } - } - ok = b != NULL && strlist_add(&b->named, &b->nnamed, url); + if (grown == NULL) + return HTS_FALSE; + st->items = grown; + b = &grown[st->count++]; + memset(b, 0, sizeof(*b)); + StringInit(b->headers); + strcpybuff(b->name, name); } - hts_mutexrelease(&bundle_lock); - return ok; + return strlist_add(&b->start_urls, &b->nstart, url); } -/* Read one line of fp into line, without its LF or a CR before it. Returns 1 - for a line, 0 at the end of the file, -1 for a line too long or holding a - NUL. */ -static int bundle_readline(FILE *fp, char *line, size_t size) { - size_t n = 0; - int c; - hts_boolean bad = HTS_FALSE; - - while ((c = fgetc(fp)) != EOF && c != '\n') { - if (c == '\0' || n + 1 >= size) - bad = HTS_TRUE; - else - line[n++] = (char) c; +/* Read a line into line without its line end. HTS_FALSE for a line too long, + or holding a control byte other than TAB: a CR would end a header early. */ +static hts_boolean bundle_readline(FILE *fp, char *line, size_t size) { + size_t len; + size_t i; + + if (hts_readline_raw(fp, line, size, &len)) + return HTS_FALSE; + if (len > 0 && line[len - 1] == '\r') + line[--len] = '\0'; + for (i = 0; i < len; i++) { + const unsigned char c = (unsigned char) line[i]; + + if ((c < 0x20 && c != '\t') || c == 0x7f) + return HTS_FALSE; } - if (c == EOF && n == 0 && !bad) - return 0; - if (n > 0 && line[n - 1] == '\r') - n--; - line[n] = '\0'; - return bad ? -1 : 1; + return HTS_TRUE; } -/* Is line the version line of a bundle this engine reads? */ -static hts_boolean bundle_version(const char *line, hts_boolean *known) { +/* The major version a "# HTTrack bundle N[.M]" line gives, or -1 when line is + not one. */ +static int bundle_version_of(const char *line) { static const char magic[] = "# HTTrack bundle "; const char *p = line; size_t digits; + int major; if (strncmp(p, magic, sizeof(magic) - 1) != 0) - return HTS_FALSE; + return -1; p += sizeof(magic) - 1; digits = strspn(p, "0123456789"); - if (digits == 0) - return HTS_FALSE; - *known = digits == 1 && p[0] == '1'; + if (digits == 0 || digits > 3 || (p[0] == '0' && digits > 1)) + return -1; + major = atoi(p); p += digits; if (*p == '.') { - p++; - digits = strspn(p, "0123456789"); + digits = strspn(++p, "0123456789"); if (digits == 0) - return HTS_FALSE; + return -1; p += digits; } - return *p == '\0'; + return *p == '\0' ? major : -1; } -/* Copy into site the domain a scope gets for host: its registrable domain, or - host itself when it has none and alone is HTS_TRUE. */ -static hts_boolean bundle_site(const char *host, hts_boolean alone, char *site, - size_t size) { - const char *reg = NULL; - - if (!hts_host_is_address(host, strlen(host))) - reg = hts_psl_registrable_domain(host); - if (reg == NULL && !alone) - return HTS_FALSE; - if (reg == NULL) - reg = host; - if (strlen(reg) >= size) - return HTS_FALSE; - strlcpybuff(site, reg, size); - return HTS_TRUE; +/* Does the scope entry cover host? */ +static hts_boolean scope_covers(const char *entry, const char *host) { + return entry[0] == '=' ? strcmp(entry + 1, host) == 0 + : cookie_domain_match(entry, host); } -static hts_boolean bundle_in_scope(const bundle_entry *b, const char *host) { +static hts_boolean bundle_in_scope(const void *b_, const char *host) { + const bundle *const b = (const bundle *) b_; size_t i; for (i = 0; i < b->nscope; i++) { - if (cookie_domain_match(b->scope[i], host)) + if (scope_covers(b->scope[i], host)) return HTS_TRUE; } return HTS_FALSE; } -static hts_boolean bundle_scope_add(bundle_entry *b, const char *domain) { +/* Add entry to b's scope. HTS_FALSE when out of memory or full. */ +static hts_boolean bundle_scope_add(bundle *b, const char *entry) { size_t i; for (i = 0; i < b->nscope; i++) { - if (strcmp(b->scope[i], domain) == 0) + if (strcmp(b->scope[i], entry) == 0) return HTS_TRUE; } - return strlist_add(&b->scope, &b->nscope, domain); + return b->nscope < BUNDLE_LIST_MAX && + strlist_add(&b->scope, &b->nscope, entry); } -/* Add the scope of a URL naming b or listed in it. */ -static hts_boolean bundle_scope_url(bundle_entry *b, const char *url) { +/* Add url's registrable domain to b's scope, or its host alone when it has + none: a public suffix, a single label or an address. */ +static hts_boolean bundle_scope_url(bundle *b, const char *url) { char host[256]; - char site[256]; + char entry[sizeof(host) + 1]; hts_boolean https; + const char *reg; - return cookie_url_host(url, host, sizeof(host), &https) && - bundle_site(host, HTS_TRUE, site, sizeof(site)) && - bundle_scope_add(b, site); + if (!cookie_url_host(url, host, sizeof(host), &https)) + return HTS_FALSE; + reg = hts_host_is_address(host, strlen(host)) + ? NULL + : hts_psl_registrable_domain(host); + snprintf(entry, sizeof(entry), "%s%s", reg != NULL ? "" : "=", + reg != NULL ? reg : host); + return bundle_scope_add(b, entry); } +/* Write why into err and return HTS_FALSE, so it needs err and err_size in + scope. */ #define BUNDLE_FAIL(...) \ do { \ snprintf(err, err_size, __VA_ARGS__); \ return HTS_FALSE; \ } while (0) -/* Read b's header lines up to "cookies", then its jar. */ -static hts_boolean bundle_parse(httrackp *opt, bundle_entry *b, FILE *fp, - hts_boolean *scope_given, char *err, - size_t err_size) { +/* Add a scope line's domain to b's scope. */ +static hts_boolean bundle_scope_line(bundle *b, const char *value, int lineno, + char *err, size_t err_size) { + char dom[256]; + char entry[sizeof(dom) + 1]; + + if (value[0] == '.') + value++; + if (value[0] == '\0' || strlen(value) >= sizeof(dom) || + strspn(value, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789.-") != strlen(value)) + BUNDLE_FAIL("%s, line %d: invalid scope: %.64s", b->file, lineno, value); + strcpybuff(dom, value); + hts_lowcase(dom); + if (hts_host_is_address(dom, strlen(dom))) + snprintf(entry, sizeof(entry), "=%s", dom); + else if (hts_psl_registrable_domain(dom) != NULL) + strcpybuff(entry, dom); + else + BUNDLE_FAIL("%s, line %d: scope %s is a public suffix", b->file, lineno, + dom); + if (!bundle_scope_add(b, entry)) + BUNDLE_FAIL("%s, line %d: more than %d scope lines, or out of memory", + b->file, lineno, BUNDLE_LIST_MAX); + return HTS_TRUE; +} + +/* Read b's lines up to "cookies", leaving fp at its jar, and set *jar. */ +static hts_boolean bundle_parse(httrackp *opt, bundle *b, FILE *fp, + hts_boolean *scope_given, hts_boolean *jar, + char *err, size_t err_size) { + static const char bom[] = "\xEF\xBB\xBF"; char BIGSTK line[BUNDLE_LINE_MAX]; - int lineno = 0; - int got; - hts_boolean known; - - got = bundle_readline(fp, line, sizeof(line)); - lineno++; - { - static const char bom[] = "\xEF\xBB\xBF"; - const char *const first = - got > 0 && strncmp(line, bom, sizeof(bom) - 1) == 0 - ? line + sizeof(bom) - 1 - : line; - - if (got <= 0 || !bundle_version(first, &known)) - BUNDLE_FAIL("%s: not an HTTrack bundle, the first line must be " - "\"# HTTrack bundle 1\"", - b->file); - } - if (!known) - BUNDLE_FAIL("%s: this bundle version is not supported: %s", b->file, line); - *scope_given = HTS_FALSE; - while ((got = bundle_readline(fp, line, sizeof(line))) != 0) { + int lineno = 1; + int version; + + if (!bundle_readline(fp, line, sizeof(line))) + BUNDLE_FAIL("%s, line 1: too long or holding a control byte", b->file); + version = bundle_version_of( + strncmp(line, bom, sizeof(bom) - 1) == 0 ? line + sizeof(bom) - 1 : line); + if (version < 0) + BUNDLE_FAIL("%s: not an HTTrack bundle, the first line must be " + "\"# HTTrack bundle 1\"", + b->file); + if (version != 1) + BUNDLE_FAIL("%s: bundle version %d is not supported", b->file, version); + *scope_given = *jar = HTS_FALSE; + while (!feof(fp)) { const char *value; size_t klen; lineno++; - if (got < 0) - BUNDLE_FAIL("%s, line %d: line too long or holding a NUL", b->file, + if (!bundle_readline(fp, line, sizeof(line))) + BUNDLE_FAIL("%s, line %d: too long or holding a control byte", b->file, lineno); if (line[0] == '\0' || line[0] == '#') continue; @@ -359,20 +344,9 @@ static hts_boolean bundle_parse(httrackp *opt, bundle_entry *b, FILE *fp, value = line + klen; value += strspn(value, " \t"); if (klen == 7 && strncmp(line, "cookies", klen) == 0) { - int refused; - if (*value != '\0') BUNDLE_FAIL("%s, line %d: \"cookies\" takes no value", b->file, lineno); - b->jar = calloct(1, sizeof(t_cookie)); - if (b->jar == NULL) - BUNDLE_FAIL("%s: out of memory", b->file); - b->jar->max_len = BUNDLE_JAR_MAX; - refused = cookie_load_stream(opt, b->jar, fp, b->file); - if (refused == COOKIE_LOAD_KEEP) - BUNDLE_FAIL("%s: out of memory", b->file); - if (refused != 0) - BUNDLE_FAIL("%s: %d cookie line(s) could not be read, see the log", - b->file, refused); + *jar = HTS_TRUE; return HTS_TRUE; } if (*value == '\0') @@ -381,11 +355,14 @@ static hts_boolean bundle_parse(httrackp *opt, bundle_entry *b, FILE *fp, char host[256]; hts_boolean https; + /* a url can hold a token, so the message leaves it out */ if (strlen(value) >= HTS_URLMAXSIZE || strstr(value, HTS_BUNDLE_SUFFIX) != NULL || - !cookie_url_host(value, host, sizeof(host), &https) || - host[0] == '\0') - BUNDLE_FAIL("%s, line %d: invalid url: %.64s", b->file, lineno, value); + !cookie_url_host(value, host, sizeof(host), &https)) + BUNDLE_FAIL("%s, line %d: invalid url", b->file, lineno); + if (b->nurls >= BUNDLE_LIST_MAX) + BUNDLE_FAIL("%s, line %d: more than %d url lines", b->file, lineno, + BUNDLE_LIST_MAX); if (!strlist_add(&b->urls, &b->nurls, value)) BUNDLE_FAIL("%s: out of memory", b->file); } else if (klen == 2 && strncmp(line, "ua", klen) == 0) { @@ -397,35 +374,19 @@ static hts_boolean bundle_parse(httrackp *opt, bundle_entry *b, FILE *fp, } else if (klen == 6 && strncmp(line, "header", klen) == 0) { const char *const why = http_bundle_header_refusal(StringBuff(b->headers), value); + /* the value can be a secret, so only the name is shown */ + const size_t name = strcspn(value, ":"); + const int shown = value[name] == ':' && name <= 64 ? (int) name : 0; if (why != NULL) - BUNDLE_FAIL("%s, line %d: refused header (%s): %.64s", b->file, lineno, - why, value); + BUNDLE_FAIL("%s, line %d: refused header%s%.*s: %s", b->file, lineno, + shown != 0 ? " " : "", shown, value, why); StringCat(b->headers, value); StringCat(b->headers, "\r\n"); } else if (klen == 5 && strncmp(line, "scope", klen) == 0) { - char dom[256]; - - if (value[0] == '.') - value++; - if (strlen(value) >= sizeof(dom) || - strspn(value, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - "0123456789.-") != strlen(value) || - value[0] == '\0') - BUNDLE_FAIL("%s, line %d: invalid scope: %.64s", b->file, lineno, - value); - strcpybuff(dom, value); - hts_lowcase(dom); - if (!hts_host_is_address(dom, strlen(dom)) && - hts_psl_registrable_domain(dom) == NULL) - BUNDLE_FAIL("%s, line %d: scope %s is a public suffix", b->file, lineno, - dom); - if (!*scope_given) { - strlist_free(&b->scope, &b->nscope); - *scope_given = HTS_TRUE; - } - if (!bundle_scope_add(b, dom)) - BUNDLE_FAIL("%s: out of memory", b->file); + *scope_given = HTS_TRUE; + if (!bundle_scope_line(b, value, lineno, err, err_size)) + return HTS_FALSE; } else { hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: unknown key %.*s", b->file, lineno, (int) klen, line); @@ -434,122 +395,119 @@ static hts_boolean bundle_parse(httrackp *opt, bundle_entry *b, FILE *fp, return HTS_TRUE; } -/* Set b's default scope, then drop its cookies outside the scope. */ -static hts_boolean bundle_scope(httrackp *opt, bundle_entry *b, - hts_boolean scope_given) { +/* Is the jar file newer than b's file, so its records win? */ +static hts_boolean bundle_under(const bundle *b, const char *jar) { + hts_filetime_t jt, bt; + + return jar != NULL && hts_file_mtime(jar, &jt) && + hts_file_mtime(b->file, &bt) && + (jt.sec > bt.sec || (jt.sec == bt.sec && jt.nsec > bt.nsec)); +} + +/* Read b's file, set its scope, and load its cookies through others. */ +static hts_boolean bundle_load(httrackp *opt, bundle *b, const char *jar, + t_cookie *others, char *err, size_t err_size) { + const char *const dir = StringNotEmpty(opt->bundle_dir) + ? StringBuff(opt->bundle_dir) + : StringBuff(opt->path_log); + const size_t dlen = strlen(dir); + hts_boolean scope_given, has_jar; + cookie_filter filter; size_t i; - char *rec; + FILE *fp; - if (!scope_given) { - for (i = 0; i < b->nnamed; i++) { - if (!bundle_scope_url(b, b->named[i])) - return HTS_FALSE; - } - for (i = 0; i < b->nurls; i++) { - if (!bundle_scope_url(b, b->urls[i])) - return HTS_FALSE; - } - for (rec = b->jar != NULL ? b->jar->data : NULL; - rec != NULL && *rec != '\0'; rec = cookie_nextfield(rec)) { - char buffer[8192]; - const char *dom = cookie_get(buffer, rec, 0); - char site[256]; - - if (dom[0] == '.') - dom++; - if (bundle_site(dom, HTS_FALSE, site, sizeof(site)) && - !bundle_scope_add(b, site)) - return HTS_FALSE; - } + bundle_clear(b); + if (!slprintfbuff( + b->file, sizeof(b->file), "%s%s%s.hts", dir, + dlen != 0 && dir[dlen - 1] != '/' && dir[dlen - 1] != '\\' ? "/" : "", + b->name)) + BUNDLE_FAIL("bundle %s: the folder path is too long", b->name); + fp = FOPEN(b->file, "rb"); + if (fp == NULL) + BUNDLE_FAIL("%s: cannot read the login bundle: %s", b->file, + strerror(errno)); + if (!bundle_parse(opt, b, fp, &scope_given, &has_jar, err, err_size)) { + fclose(fp); + return HTS_FALSE; } - rec = b->jar != NULL ? b->jar->data : NULL; - while (rec != NULL && *rec != '\0') { - char buffer[8192]; - const char *dom = cookie_get(buffer, rec, 0); - - if (dom[0] == '.') - dom++; - if (bundle_in_scope(b, dom)) { - rec = cookie_nextfield(rec); - } else { - char name[8192]; - - hts_log_print(opt, LOG_WARNING, - "Bundle %s: cookie '%.64s' for %s is outside the bundle's " - "scope, refused", - b->file, cookie_get(name, rec, 5), dom); - cookie_delete(rec, b->jar->max_len - (size_t) (rec - b->jar->data), - (size_t) (cookie_nextfield(rec) - rec)); + b->https_only = HTS_TRUE; + for (i = 0; i < b->nstart; i++) + b->https_only = b->https_only && strfield(b->start_urls[i], "https:"); + for (i = 0; !scope_given && i < b->nstart + b->nurls; i++) { + if (!bundle_scope_url(b, i < b->nstart ? b->start_urls[i] + : b->urls[i - b->nstart])) { + fclose(fp); + BUNDLE_FAIL("%s: cannot work out the bundle's scope", b->file); } } + memset(&filter, 0, sizeof(filter)); + filter.covers = bundle_in_scope; + filter.scope = b; + filter.others = others; + filter.under = bundle_under(b, jar); + if (has_jar && opt->cookie != NULL) + cookie_load_stream(opt, opt->cookie, fp, b->file, &filter); + else if (has_jar) + hts_log_print(opt, LOG_WARNING, + "Cookies are disabled, so the cookies of %s are not sent", + b->file); + fclose(fp); + if (filter.dropped != 0) + hts_log_print(opt, LOG_WARNING, + "Bundle %s: %d cookie(s) outside its scope left out; a " + "scope line keeps them", + b->file, filter.dropped); + if (filter.clash[0] != '\0') + BUNDLE_FAIL("%s: cookie '%s' has another value in another bundle", b->file, + filter.clash); + if (filter.lost != 0) + BUNDLE_FAIL("%s: the cookie jar is full, or the file could not be read to " + "its end", + b->file); return HTS_TRUE; } -static hts_boolean bundle_scopes_overlap(const bundle_entry *a, - const bundle_entry *b) { +static hts_boolean bundle_scopes_overlap(const bundle *a, const bundle *b) { size_t i, j; for (i = 0; i < a->nscope; i++) { for (j = 0; j < b->nscope; j++) { - if (cookie_domain_match(a->scope[i], b->scope[j]) || - cookie_domain_match(b->scope[j], a->scope[i])) + const char *const ea = a->scope[i]; + const char *const eb = b->scope[j]; + + if (scope_covers(ea, eb + (eb[0] == '=')) || + scope_covers(eb, ea + (ea[0] == '='))) return HTS_TRUE; } } return HTS_FALSE; } -static hts_boolean bundle_same_identity(const bundle_entry *a, - const bundle_entry *b) { - if ((a->ua == NULL) != (b->ua == NULL)) - return HTS_FALSE; - if (a->ua != NULL && strcmp(a->ua, b->ua) != 0) - return HTS_FALSE; - return strcmp(StringBuff(a->headers), StringBuff(b->headers)) == 0; -} - -static hts_boolean bundle_load(httrackp *opt, bundle_set *set, bundle_entry *b, - char *err, size_t err_size) { - const char *const dir = - set->dir != NULL ? set->dir : StringBuff(opt->path_log); - const size_t dlen = strlen(dir); - hts_boolean scope_given = HTS_FALSE; - hts_boolean ok; - FILE *fp; - - if (!slprintfbuff( - b->file, sizeof(b->file), "%s%s%s.hts", dir, - dlen != 0 && dir[dlen - 1] != '/' && dir[dlen - 1] != '\\' ? "/" : "", - b->name)) - BUNDLE_FAIL("bundle %s: the folder path is too long", b->name); - fp = FOPEN(b->file, "rb"); - if (fp == NULL) - BUNDLE_FAIL("%s: cannot read the login bundle: %s", b->file, - strerror(errno)); - ok = bundle_parse(opt, b, fp, &scope_given, err, err_size); - fclose(fp); - if (!ok) - return HTS_FALSE; - if (!bundle_scope(opt, b, scope_given)) - BUNDLE_FAIL("%s: cannot work out the bundle's scope", b->file); - return HTS_TRUE; +static hts_boolean bundle_same_identity(const bundle *a, const bundle *b) { + return (a->ua == NULL ? b->ua == NULL + : b->ua != NULL && strcmp(a->ua, b->ua) == 0) && + strcmp(StringBuff(a->headers), StringBuff(b->headers)) == 0; } -hts_boolean hts_bundle_load_all(httrackp *opt, char *err, size_t err_size) { +hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar, char *err, + size_t err_size) { + hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state; + t_cookie *others; hts_boolean ok = HTS_TRUE; - bundle_set *set; size_t i, j; - err[0] = '\0'; - hts_mutexlock(&bundle_lock); - set = bundle_set_of(opt, HTS_FALSE); - for (i = 0; set != NULL && i < set->count && ok; i++) - ok = bundle_load(opt, set, &set->items[i], err, err_size); - for (i = 0; set != NULL && i < set->count && ok; i++) { - for (j = i + 1; j < set->count && ok; j++) { - bundle_entry *const a = &set->items[i]; - bundle_entry *const b = &set->items[j]; - char name[256]; + if (st == NULL) + return HTS_TRUE; + others = calloct(1, sizeof(t_cookie)); + if (others == NULL) + BUNDLE_FAIL("login bundles: out of memory"); + others->max_len = sizeof(others->data); + for (i = 0; i < st->count && ok; i++) + ok = bundle_load(opt, &st->items[i], jar, others, err, err_size); + for (i = 0; i < st->count && ok; i++) { + for (j = i + 1; j < st->count && ok; j++) { + const bundle *const a = &st->items[i]; + const bundle *const b = &st->items[j]; if (bundle_scopes_overlap(a, b) && !bundle_same_identity(a, b)) { snprintf(err, err_size, @@ -557,108 +515,55 @@ hts_boolean hts_bundle_load_all(httrackp *opt, char *err, size_t err_size) { "or headers", a->file, b->file); ok = HTS_FALSE; - } else if (a->jar != NULL && b->jar != NULL && - cookie_jars_conflict(a->jar, b->jar, name, sizeof(name))) { - snprintf(err, err_size, - "%s and %s hold two values of the cookie '%.64s'", a->file, - b->file, name); - ok = HTS_FALSE; } } } - hts_mutexrelease(&bundle_lock); + freet(others); return ok; } -char *hts_bundle_urls(httrackp *opt) { - String all = STRING_EMPTY; - bundle_set *set; - size_t i, j; +const char *hts_bundle_url(httrackp *opt, size_t i) { + const hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state; + size_t k; - hts_mutexlock(&bundle_lock); - set = bundle_set_of(opt, HTS_FALSE); - for (i = 0; set != NULL && i < set->count; i++) { - for (j = 0; j < set->items[i].nurls; j++) { - StringCat(all, set->items[i].urls[j]); - StringCat(all, "\n"); - } + for (k = 0; st != NULL && k < st->count; k++) { + if (i < st->items[k].nurls) + return st->items[k].urls[i]; + i -= st->items[k].nurls; } - hts_mutexrelease(&bundle_lock); - return StringNotEmpty(all) ? StringAcquire(&all) : NULL; + return NULL; } -int hts_bundle_cookie_count(httrackp *opt) { - int count = 0; - bundle_set *set; - size_t i; - - hts_mutexlock(&bundle_lock); - set = bundle_set_of(opt, HTS_FALSE); - for (i = 0; set != NULL && i < set->count; i++) { - char *rec; - - for (rec = set->items[i].jar != NULL ? set->items[i].jar->data : NULL; - rec != NULL && *rec != '\0'; rec = cookie_nextfield(rec)) - count++; - } - hts_mutexrelease(&bundle_lock); - return count; -} - -int hts_bundle_cookies(httrackp *opt, t_cookie *jar) { - int refused = 0; - bundle_set *set; - size_t i; - - hts_mutexlock(&bundle_lock); - set = bundle_set_of(opt, HTS_FALSE); - for (i = 0; set != NULL && i < set->count; i++) { - if (set->items[i].jar != NULL) - refused += cookie_merge(jar, set->items[i].jar); - } - hts_mutexrelease(&bundle_lock); - return refused; -} - -hts_boolean hts_bundle_identity(httrackp *opt, const char *adr, const char **ua, - const char **headers) { - hts_boolean changed = HTS_FALSE; +void hts_bundle_identity(httrackp *opt, const char *adr, htsrequest *req) { + const hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state; char host[256]; - hts_boolean https; - bundle_set *set; size_t i; - hts_mutexlock(&bundle_lock); - set = bundle_set_of(opt, HTS_FALSE); - if (set != NULL && set->count != 0 && - cookie_url_host(adr, host, sizeof(host), &https)) { - for (i = 0; i < set->count; i++) { - const bundle_entry *const b = &set->items[i]; - - if (bundle_in_scope(b, host)) { - if (b->ua != NULL) { - *ua = b->ua; - changed = HTS_TRUE; - } - if (StringNotEmpty(b->headers)) - *headers = StringBuff(b->headers); - break; + if (st == NULL || !cookie_host(adr, host, sizeof(host))) + return; + for (i = 0; i < st->count; i++) { + const bundle *const b = &st->items[i]; + + if (bundle_in_scope(b, host)) { + if (b->ua != NULL) { + req->user_agent = b->ua; + req->user_agent_send = 1; } + if (StringNotEmpty(b->headers) && + (!b->https_only || strfield(adr, "https://"))) + req->headers = StringBuff(b->headers); + return; } } - hts_mutexrelease(&bundle_lock); - return changed; } void hts_bundle_dump(httrackp *opt) { - bundle_set *set; + const hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state; size_t i, j; - hts_mutexlock(&bundle_lock); - set = bundle_set_of(opt, HTS_FALSE); - for (i = 0; set != NULL && i < set->count; i++) { - const bundle_entry *const b = &set->items[i]; - char *rec; + for (i = 0; st != NULL && i < st->count; i++) { + const bundle *const b = &st->items[i]; + const char *p; printf("bundle %s\n", b->name); if (b->ua != NULL) @@ -667,24 +572,12 @@ void hts_bundle_dump(httrackp *opt) { printf("url %s\n", b->urls[j]); for (j = 0; j < b->nscope; j++) printf("scope %s\n", b->scope[j]); - if (StringNotEmpty(b->headers)) { - const char *p; - - for (p = StringBuff(b->headers); *p != '\0';) { - const size_t len = strcspn(p, "\r\n"); - - printf("header %.*s\n", (int) len, p); - p += len; - p += strspn(p, "\r\n"); - } - } - for (rec = b->jar != NULL ? b->jar->data : NULL; - rec != NULL && *rec != '\0'; rec = cookie_nextfield(rec)) { - char dom[8192], name[8192], value[8192]; + for (p = StringBuff(b->headers); *p != '\0';) { + const size_t len = strcspn(p, "\r\n"); - printf("cookie %s %s=%s\n", cookie_get(dom, rec, 0), - cookie_get(name, rec, 5), cookie_get(value, rec, 6)); + printf("header %.*s\n", (int) len, p); + p += len; + p += strspn(p, "\r\n"); } } - hts_mutexrelease(&bundle_lock); } diff --git a/src/htsbundle.h b/src/htsbundle.h index 3eb8b2c0b..c10f7cd4c 100644 --- a/src/htsbundle.h +++ b/src/htsbundle.h @@ -36,9 +36,10 @@ Please visit our Website: http://www.httrack.com #ifdef HTS_INTERNAL_BYTECODE #include +#include #include "htsglobal.h" -#include "htsbauth.h" +#include "htsopt.h" /* A start URL ending with this names the bundle file .hts. */ #define HTS_BUNDLE_SUFFIX "#httrack-bundle=" @@ -52,46 +53,61 @@ typedef enum { HTS_BUNDLE_BAD /* a suffix whose name is refused, URL left alone */ } hts_bundle_suffix; -/* Is name a bundle name: [A-Za-z0-9._-], not starting with '.' and no ".."? */ -hts_boolean hts_bundle_name_valid(const char *name); - /* Cut the bundle suffix off url in place and copy its name into name. */ hts_bundle_suffix hts_bundle_cut(char *url, char *name, size_t name_size); -/* Forget the bundles and the --bundle-dir recorded for opt. */ -void hts_bundle_reset(httrackp *opt); - -/* Read names from dir rather than the project folder. HTS_FALSE when out of - memory. */ -hts_boolean hts_bundle_set_dir(httrackp *opt, const char *dir); - /* Record that the start URL url (suffix already cut) names bundle name. A name given twice is one bundle. HTS_FALSE when out of memory. */ -hts_boolean hts_bundle_name(httrackp *opt, const char *name, const char *url); - -/* Read every named bundle, then check scopes and identities. HTS_FALSE after - writing why into err, naming the file: the run must stop. */ -hts_boolean hts_bundle_load_all(httrackp *opt, char *err, size_t err_size); - -/* The bundles' url lines, each followed by a LF, in a malloct'd string the - caller freet's. NULL when there are none. */ -char *hts_bundle_urls(httrackp *opt); - -/* How many cookies the loaded bundles hold. */ -int hts_bundle_cookie_count(httrackp *opt); - -/* Store the bundles' cookies into jar. Returns how many it refused. */ -int hts_bundle_cookies(httrackp *opt, t_cookie *jar); - -/* When adr's host is in a bundle's scope, point *ua and *headers at that - bundle's, for each it gives. Returns HTS_TRUE when it changed *ua. The - strings live until hts_bundle_reset. */ -hts_boolean hts_bundle_identity(httrackp *opt, const char *adr, const char **ua, - const char **headers); - -/* Print each loaded bundle (UA, headers, scope, cookies) for the self-tests. */ +hts_boolean hts_bundle_add(httrackp *opt, const char *name, const char *url); + +/* Read every named bundle from opt->bundle_dir, or the project folder, and + refuse two bundles that overlap with different identities or cookie values. + The cookies go into opt->cookie, under the records already there when the + jar file is newer than the bundle. jar is that file, or NULL. HTS_FALSE + after writing why into err, naming the file: the run must stop. A second + call reads the files again. */ +hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar, char *err, + size_t err_size); + +/* The i-th url line of the loaded bundles, or NULL past the last one. */ +const char *hts_bundle_url(httrackp *opt, size_t i); + +/* Give req the UA and headers of the first bundle whose scope holds adr's + host, where it has them. Its headers go over https alone when every URL + naming it is https. */ +void hts_bundle_identity(httrackp *opt, const char *adr, htsrequest *req); + +/* Forget the bundles named for opt. */ +void hts_bundle_free(httrackp *opt); + +/* Print each loaded bundle (UA, url lines, scope, headers) for the + self-tests. */ void hts_bundle_dump(httrackp *opt); +/* htsbauth.c internals the bundles share. */ + +/* Copy the host of url, with or without a scheme, into dst as cookie_host + does, and set *https. */ +hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size, + hts_boolean *https); + +/* What cookie_load_stream lets into the jar, and the count of what it keeps + out. */ +typedef struct cookie_filter { + hts_boolean (*covers)(const void *scope, const char *host); + const void *scope; + t_cookie *others; /* every bundle's records so far */ + hts_boolean under; /* a record the jar holds already wins */ + int dropped; /* records outside the scope */ + char clash[64]; /* a name others holds with another value, or "" */ + int lost; /* records the jar could not take */ +} cookie_filter; + +/* Load Netscape jar records from fp's position to its end through filter, + as cookie_load reads a file. name is the file name used in warnings. */ +void cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, + const char *name, cookie_filter *filter); + #endif #endif diff --git a/src/htsbundle_selftest.c b/src/htsbundle_selftest.c index 539775cd6..3c3954eb7 100644 --- a/src/htsbundle_selftest.c +++ b/src/htsbundle_selftest.c @@ -59,31 +59,44 @@ static int st_bundlecut(httrackp *opt, int argc, char **argv) { return 0; } -/* -#test=bundle ... [host:]...: load the bundles the URLs - name from dir, print them, then the UA and headers each host gets. */ +/* -#test=bundle ... [host:]... [jar:]: load the bundles + the URLs name from dir over the jar file, print them and the jar, then the + UA and headers each adr gets. */ static int st_bundle(httrackp *opt, int argc, char **argv) { FILE *const log = opt->log; + t_cookie *const prev = opt->cookie; + t_cookie *jar = calloct(1, sizeof(t_cookie)); + const char *jar_file = NULL; char err[1024]; + char *rec; int i; - if (argc < 2) { - fprintf(stderr, "bundle: takes ... [host:]...\n"); + if (argc < 2 || jar == NULL) { + fprintf(stderr, "bundle: takes ... [host:]... " + "[jar:]\n"); + freet(jar); return 1; } - hts_bundle_reset(opt); - assertf(hts_bundle_set_dir(opt, argv[0])); - opt->log = stderr; /* warnings stay out of the graded output */ + jar->max_len = 30000; /* httpmirror's */ + opt->log = stderr; /* warnings stay out of the graded output */ + opt->cookie = jar; + hts_bundle_free(opt); + StringCopy(opt->bundle_dir, argv[0]); for (i = 1; i < argc; i++) { char url[HTS_URLMAXSIZE * 2]; char name[HTS_BUNDLE_NAME_MAX]; - if (strncmp(argv[i], "host:", 5) == 0 || strlen(argv[i]) >= sizeof(url)) - continue; - strcpybuff(url, argv[i]); - if (hts_bundle_cut(url, name, sizeof(name)) == HTS_BUNDLE_NAMED) - assertf(hts_bundle_name(opt, name, url)); + if (strncmp(argv[i], "jar:", 4) == 0) { + jar_file = argv[i] + 4; + (void) cookie_load(opt, jar, "", jar_file); + } else if (strncmp(argv[i], "host:", 5) != 0 && + strlen(argv[i]) < sizeof(url)) { + strcpybuff(url, argv[i]); + if (hts_bundle_cut(url, name, sizeof(name)) == HTS_BUNDLE_NAMED) + assertf(hts_bundle_add(opt, name, url)); + } } - if (!hts_bundle_load_all(opt, err, sizeof(err))) { + if (!hts_bundle_load_all(opt, jar_file, err, sizeof(err))) { /* the folder is the caller's, so the message is graded without it */ const size_t dlen = strlen(argv[0]); const char *p = err; @@ -95,35 +108,44 @@ static int st_bundle(httrackp *opt, int argc, char **argv) { p = at + dlen; } printf("%s\n", p); - hts_bundle_reset(opt); - opt->log = log; - return 0; + goto done; } hts_bundle_dump(opt); + for (rec = jar->data; *rec != '\0'; rec = cookie_nextfield(rec)) { + char dom[8192], cname[8192], value[8192]; + + printf("cookie %s %s=%s\n", cookie_get(dom, rec, 0), + cookie_get(cname, rec, 5), cookie_get(value, rec, 6)); + } for (i = 1; i < argc; i++) { if (strncmp(argv[i], "host:", 5) == 0) { - const char *ua = "global"; - const char *headers = "global"; + htsrequest req; const char *p; - (void) hts_bundle_identity(opt, argv[i] + 5, &ua, &headers); - printf("%s ua=%s headers=", argv[i] + 5, ua); + memset(&req, 0, sizeof(req)); + req.user_agent = req.headers = "global"; + hts_bundle_identity(opt, argv[i] + 5, &req); + printf("%s ua=%s headers=", argv[i] + 5, req.user_agent); /* one line per host, so the block's line ends print as '|' */ - for (p = headers; *p != '\0'; p++) { + for (p = req.headers; *p != '\0'; p++) { if (*p != '\r') putchar(*p == '\n' ? '|' : *p); } putchar('\n'); } } - hts_bundle_reset(opt); +done: + hts_bundle_free(opt); + StringCopy(opt->bundle_dir, ""); + opt->cookie = prev; opt->log = log; + freet(jar); return 0; } const struct selftest_entry selftests_bundle[] = { {"bundlecut", "", "the login bundle a start URL names", st_bundlecut}, - {"bundle", " ... [host:]...", + {"bundle", " ... [host:]... [jar:]", "load login bundles and print the identity each host gets", st_bundle}, {NULL, NULL, NULL, NULL}, }; diff --git a/src/htscore.c b/src/htscore.c index e5ebd13fe..b81561dbc 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -784,9 +784,14 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { opt->cookie = &cookie; cookie.max_len = 30000; // max len strcpybuff(cookie.data, ""); - /* loaded over the login bundles once the start URLs are known, and left - alone until then */ - cookies_file_refused = COOKIE_LOAD_KEEP; + if (strnotempty(StringBuff(opt->cookies_file))) { + // loaded once the start URLs are known, and left alone until then + cookies_file_refused = COOKIE_LOAD_KEEP; + } else { + cookies_file_refused = cookie_load( + opt, opt->cookie, StringBuff(opt->path_log), "cookies.txt"); + cookie_load(opt, opt->cookie, "", "cookies.txt"); + } } else { opt->cookie = NULL; if (strnotempty(StringBuff(opt->cookies_file))) @@ -1136,12 +1141,14 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { break; case HTS_BUNDLE_BAD: hts_log_print(opt, LOG_ERROR, - "\"%s\", line %d: invalid login bundle name: %s", - StringBuff(opt->filelist), lineno, line); + "\"%s\", line %d: invalid login bundle name", + StringBuff(opt->filelist), lineno); + fprintf(stderr, "\"%s\", line %d: invalid login bundle name\n", + StringBuff(opt->filelist), lineno); bundle_refused = HTS_TRUE; continue; case HTS_BUNDLE_NAMED: - if (!hts_bundle_name(opt, name, line)) { + if (!hts_bundle_add(opt, name, line)) { hts_log_print(opt, LOG_ERROR, "Login bundle: out of memory"); bundle_refused = HTS_TRUE; } @@ -1167,85 +1174,6 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { } } - /* the login bundles: their url lines are start URLs, and their cookies go - first in the jar, so what the server refreshed since wins */ - { - char err[1024]; - char *extra; - - err[0] = '\0'; - if (bundle_refused || !hts_bundle_load_all(opt, err, sizeof(err))) { - if (err[0] != '\0') { - hts_log_print(opt, LOG_ERROR, "Login bundle: %s", err); - fprintf(stderr, "Login bundle: %s\n", err); - } else { - fprintf(stderr, "Login bundle: %s refused, see the log\n", - StringBuff(opt->filelist)); - } - freet(primary); - XH_extuninit; - return 0; - } - extra = hts_bundle_urls(opt); - if (extra != NULL) { - size_t lines = 0; - size_t room; - const char *p; - char *grown; - - for (p = extra; *p != '\0'; p++) { - if (*p == '\n') - lines++; - } - /* "http://" for each line, and the NUL */ - room = strlen(extra) + lines * 7 + 1; - grown = room < (size_t) -1 - primarybuff.cap - ? (char *) realloct(primary, primarybuff.cap + room) - : NULL; - if (grown == NULL) { - printf("PANIC! : Not enough memory [%d]\n", __LINE__); - freet(extra); - freet(primary); - XH_extuninit; - return 0; - } - primary = grown; - primarybuff.buf = grown; - primarybuff.cap += room; - for (p = extra; *p != '\0';) { - const size_t len = strcspn(p, "\n"); - - if (strstr(p, ":/") == NULL || strstr(p, ":/") > p + len) - htsbuff_cat(&primarybuff, "http://"); - htsbuff_catn(&primarybuff, p, len); - htsbuff_cat(&primarybuff, "\n"); - p += len; - if (*p == '\n') - p++; - } - freet(extra); - } - if (opt->accept_cookie) { - const int refused = hts_bundle_cookies(opt, opt->cookie); - - if (refused != 0) { - hts_log_print(opt, LOG_ERROR, - "Login bundle: the jar is full, %d cookie(s) left out", - refused); - fprintf(stderr, - "Login bundle: the jar is full, %d cookie(s) left out\n", - refused); - freet(primary); - XH_extuninit; - return 0; - } - } else if (hts_bundle_cookie_count(opt) != 0) { - hts_log_print(opt, LOG_WARNING, - "Cookies are disabled, so the login bundles' cookies " - "are not sent"); - } - } - // the --cookies-file is the whole jar, because the crawl writes it back if (opt->accept_cookie && strnotempty(StringBuff(opt->cookies_file))) { char BIGSTK start_url[HTS_URLMAXSIZE * 2]; @@ -1256,10 +1184,56 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { strlncatbuff(start_url, primary, sizeof(start_url), first); cookies_file_refused = cookie_load_from( opt, opt->cookie, "", StringBuff(opt->cookies_file), start_url); - } else if (opt->accept_cookie) { - cookies_file_refused = cookie_load( - opt, opt->cookie, StringBuff(opt->path_log), "cookies.txt"); - cookie_load(opt, opt->cookie, "", "cookies.txt"); + } + + /* the login bundles: url lines join the start URLs, cookies the jar */ + { + char err[1024]; + char jarbuff[CATBUFF_SIZE]; + const char *jar = !opt->accept_cookie ? NULL + : strnotempty(StringBuff(opt->cookies_file)) + ? StringBuff(opt->cookies_file) + : fconcat(jarbuff, sizeof(jarbuff), + StringBuff(opt->path_log), "cookies.txt"); + const char *extra; + size_t room = 0; + size_t i; + + if (!bundle_refused && !hts_bundle_load_all(opt, jar, err, sizeof(err))) { + hts_log_print(opt, LOG_ERROR, "Login bundle: %s", err); + fprintf(stderr, "Login bundle: %s\n", err); + bundle_refused = HTS_TRUE; + } + for (i = 0; !bundle_refused && (extra = hts_bundle_url(opt, i)) != NULL; + i++) + room += strlen(extra) + sizeof("http://\n"); + if (room != 0) { + char *const grown = + room < (size_t) -1 - primarybuff.cap + ? (char *) realloct(primary, primarybuff.cap + room) + : NULL; + + if (grown == NULL) { + printf("PANIC! : Not enough memory [%d]\n", __LINE__); + bundle_refused = HTS_TRUE; + } else { + primary = primarybuff.buf = grown; + primarybuff.cap += room; + } + } + if (bundle_refused) { + /* the jar may hold half a bundle, so it is not saved */ + cookies_file_refused = COOKIE_LOAD_KEEP; + freet(primary); + XH_extuninit; + return 0; + } + for (i = 0; (extra = hts_bundle_url(opt, i)) != NULL; i++) { + if (strstr(extra, ":/") == NULL) + htsbuff_cat(&primarybuff, "http://"); + htsbuff_cat(&primarybuff, extra); + htsbuff_cat(&primarybuff, "\n"); + } } // lien primaire @@ -4865,6 +4839,9 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) { if (StringNotEmpty(from->accept_kinds)) StringCopyS(to->accept_kinds, from->accept_kinds); + if (StringNotEmpty(from->bundle_dir)) + StringCopyS(to->bundle_dir, from->bundle_dir); + /* An explicit -%a or -%l travels with its flag, so it stays explicit. */ if (from->accept_given) { StringCopyS(to->accept, from->accept); diff --git a/src/htscoremain.c b/src/htscoremain.c index 7c57a9bba..94062f39f 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -466,6 +466,40 @@ static hts_boolean optparam_dash_ok(const cmdl_argv *cmd, int na) { return na + 1 < cmd->argc && cmd->param[na + 1]; } +/* Read the --bundle-dir at NA into opt, made absolute in cmd too, so doit.log + finds the folder from anywhere. HTS_FALSE after the panic message. */ +static hts_boolean bundle_dir_set(httrackp *opt, cmdl_argv *cmd, int na) { + const char *dir = na + 1 < cmd->argc ? cmd->argv[na + 1] : NULL; + char cwd[HTS_URLMAXSIZE]; + String abs = STRING_EMPTY; + char *copy; + + if (dir == NULL || (dir[0] == '-' && !optparam_dash_ok(cmd, na))) { + HTS_PANIC_PRINTF("Option bundle-dir needs a blank space and a folder"); + return HTS_FALSE; + } + if (dir[0] != '/' && dir[0] != '\\' && !(dir[0] != '\0' && dir[1] == ':')) { + if (getcwd(cwd, sizeof(cwd)) == NULL) { + HTS_PANIC_PRINTF("Option bundle-dir: cannot read the current folder"); + return HTS_FALSE; + } + StringCat(abs, cwd); + StringCat(abs, "/"); + } + StringCat(abs, dir); + if (StringLength(abs) >= HTS_URLMAXSIZE || + (copy = hts_arena_strdup(&cmd->tokens, StringBuff(abs))) == NULL) { + StringFree(abs); + HTS_PANIC_PRINTF("Option bundle-dir: folder path too long, or out of " + "memory"); + return HTS_FALSE; + } + StringFree(abs); + cmd->argv[na + 1] = copy; + StringCopy(opt->bundle_dir, copy); + return HTS_TRUE; +} + /* Print the headers -#C lists for `url`, read back out of the ZIP cache. */ static void cmdl_print_cache_entry(httrackp *opt, cache_back *cache, const char *url, int sendb) { @@ -569,7 +603,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { int httrack_logmode = 3; // ONE log file ensureUrlCapacity(url, url_sz, 65536); - hts_bundle_reset(opt); /* a previous run's bundles are not this one's */ + hts_bundle_free(opt); /* a previous run's bundles are not this one's */ // Create options _DEBUG_HEAD = 0; // pas de debuggage en têtes @@ -1101,6 +1135,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { HTS_PANIC_PRINTF(s); htsmain_free(); return -1; + } else if (strfield2(argv[i] + 2, "bundle-dir")) { + i++; /* its value, which the option parser reads */ } // else { @@ -1111,7 +1147,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { htsmain_free(); return -1; } - } } } @@ -1347,6 +1382,14 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { x_cmd.span[prev_opt] = na - prev_opt; prev_opt = hts_rule_would_be_read_as_opt(argv[na]) ? na : 0; + if (strcmp(argv[na], "--bundle-dir") == 0) { + if (!bundle_dir_set(opt, &x_cmd, na)) { + htsmain_free(); + return -1; + } + na++; + continue; + } if (hts_rule_would_be_read_as_opt(argv[na])) { // option com = argv[na] + 1; @@ -2533,29 +2576,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { #endif break; - case 'b': // --bundle-dir: where #httrack-bundle= names resolve - /* a vetted pair's value may begin with '-' (#1425) */ - if ((na + 1 >= argc) || - (argv[na + 1][0] == '-' && !optparam_dash_ok(&x_cmd, na))) { - HTS_PANIC_PRINTF("Option bundle-dir needs a blank space and " - "a folder"); - printf("Example: --bundle-dir \"/home/me/logins\"\n"); - htsmain_free(); - return -1; - } - na++; - if (strlen(argv[na]) >= HTS_URLMAXSIZE) { - HTS_PANIC_PRINTF("Bundle folder path too long"); - htsmain_free(); - return -1; - } - if (!hts_bundle_set_dir(opt, argv[na])) { - HTS_PANIC_PRINTF("Not enough memory"); - htsmain_free(); - return -1; - } - break; - case 'm': { // --mptcp: Multipath TCP where the system has it int on = 1; @@ -2878,7 +2898,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { } else { // URL/filters char catbuff[CATBUFF_SIZE]; - /* argv itself goes to doit.log, so the bundle suffix is cut here */ + /* doit.log keeps argv, so the bundle suffix is cut from a copy */ char word[HTS_CDLMAXSIZE]; const size_t urlSize = strlen(argv[na]); const size_t capa = strlen(url) + urlSize + 32; @@ -2909,7 +2929,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { htsmain_free(); return -1; case HTS_BUNDLE_NAMED: - if (!hts_bundle_name(opt, name, word)) { + if (!hts_bundle_add(opt, name, word)) { HTS_PANIC_PRINTF("Not enough memory"); htsmain_free(); return -1; diff --git a/src/htshelp.c b/src/htshelp.c index d09717f61..b7d8d4a76 100644 --- a/src/htshelp.c +++ b/src/htshelp.c @@ -594,10 +594,9 @@ void help(const char *app, int more) { infomsg("Spider options:"); infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)"); infomsg(" %K use a Netscape cookies.txt as the jar, rewritten at the end"); - infomsg(" @b folder holding the login bundles that start URLs name"); infomsg(" #httrack-bundle= crawls with the user agent,"); - infomsg(" headers and cookies of .hts, by default in the project " - "folder"); + infomsg(" headers and cookies of the login bundle .hts, read from " + "the project folder or from --bundle-dir "); infomsg(" %Y explain which filter rule accepts or rejects a URL, then exit"); infomsg (" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)"); diff --git a/src/htslib.c b/src/htslib.c index e41fdd685..209d89b0b 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -7517,6 +7517,8 @@ HTSEXT_API httrackp *hts_create_opt(void) { StringCopy(opt->accept_kinds, ""); opt->accept_given = HTS_FALSE; opt->lang_given = HTS_FALSE; + StringCopy(opt->bundle_dir, ""); + opt->bundle_state = NULL; StringCopy(opt->headers, ""); StringCopy(opt->mimedefs, "\n"); // aucun filtre mime (\n IMPORTANT) StringClear(opt->mod_blacklist); @@ -7588,7 +7590,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) { /* An FTP worker reads opt for its whole run, and not every caller drains its threads the way httrack.c does before getting here. */ ftp_stop_workers(); - hts_bundle_reset(opt); + hts_bundle_free(opt); /* Alocated callbacks */ @@ -7665,6 +7667,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) { StringFree(opt->lang_iso); StringFree(opt->accept); StringFree(opt->accept_kinds); + StringFree(opt->bundle_dir); StringFree(opt->headers); StringFree(opt->sys_com); StringFree(opt->mimedefs); diff --git a/src/htslines.c b/src/htslines.c index d7a95cfd0..6ca4f175d 100644 --- a/src/htslines.c +++ b/src/htslines.c @@ -37,9 +37,13 @@ Please visit our Website: http://www.httrack.com #include +/* hts_readline_raw() keeps every byte. */ +#define LINE_RAW 0x100 + /* Is c one of the bytes flags drop? */ static hts_boolean line_drops(int c, int flags) { - return c == '\r' || ((flags & HTS_LINE_DROP_TAB) != 0 && c == '\t') || + return (flags & LINE_RAW) != 0 ? HTS_FALSE + : c == '\r' || ((flags & HTS_LINE_DROP_TAB) != 0 && c == '\t') || ((flags & HTS_LINE_DROP_FF) != 0 && c == '\f') || ((flags & HTS_LINE_DROP_NUL) != 0 && c == '\0') ? HTS_TRUE @@ -102,6 +106,12 @@ hts_boolean hts_readline(FILE *fp, char *s, size_t max, int flags) { return readline_end(fp, s, max, flags, &len, &end); } +hts_boolean hts_readline_raw(FILE *fp, char *s, size_t max, size_t *len) { + line_end end; + + return readline_end(fp, s, max, LINE_RAW, len, &end); +} + /* readline_end(), then the trim hts_readline_cpp() does on each line. */ static hts_boolean readline_trim(FILE *fp, char *s, size_t max, int flags, size_t *len, line_end *end) { diff --git a/src/htslines.h b/src/htslines.h index 0f55e440a..99d9832d3 100644 --- a/src/htslines.h +++ b/src/htslines.h @@ -50,6 +50,9 @@ Please visit our Website: http://www.httrack.com when a non-space byte was dropped. At EOF it reads an empty line. */ hts_boolean hts_readline(FILE *fp, char *s, size_t max, int flags); +/* hts_readline() keeping CR and NUL, with the bytes stored in *len. */ +hts_boolean hts_readline_raw(FILE *fp, char *s, size_t max, size_t *len); + /* hts_readline() trimmed and joined after a trailing backslash. Leading spaces and backslashes count against max, so "abc\\\n\n" at 4 is cut. */ hts_boolean hts_readline_cpp(FILE *fp, char *s, size_t max, int flags); diff --git a/src/htsopt.h b/src/htsopt.h index 8fd2d93ce..bf62ca976 100644 --- a/src/htsopt.h +++ b/src/htsopt.h @@ -657,6 +657,10 @@ struct httrackp { line, and the last one wins. Tail: ABI */ hts_boolean accept_given; /**< Did the user give -%a? Tail: ABI */ hts_boolean lang_given; /**< Did the user give -%l? Tail: ABI */ + String bundle_dir; /**< --bundle-dir: where login bundle names resolve, or + empty for the project folder. Tail: ABI */ + void *bundle_state; /**< hts_bundle_state*, or NULL. Live state, so + copy_htsopt must leave it alone. Tail: ABI */ }; /** Running statistics for a mirror. */ diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index dc9821cee..c3f28a531 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -22,47 +22,80 @@ jar() { # jar DOMAIN NAME VALUE -- one Netscape jar line printf '%s' ".$1${tab}TRUE${tab}/${tab}FALSE${tab}0${tab}$2${tab}$3" } +name127=$(repeat_chars 127 a) selftest_queue 'login1 https://example.com/members/' bundlecut \ 'https://example.com/members/#httrack-bundle=login1' selftest_queue 'a.b_c-9 http://x/' bundlecut 'http://x/#httrack-bundle=a.b_c-9' +selftest_queue "$name127 http://x/" bundlecut "http://x/#httrack-bundle=$name127" +selftest_queue 'x http://x/#a' bundlecut 'http://x/#a#httrack-bundle=x' +selftest_queue 'console http://x/' bundlecut 'http://x/#httrack-bundle=console' +selftest_queue 'com10 http://x/' bundlecut 'http://x/#httrack-bundle=com10' selftest_queue none bundlecut 'http://x/#top' -selftest_queue bad bundlecut 'http://x/#httrack-bundle=' -selftest_queue bad bundlecut 'http://x/#httrack-bundle=../x' -selftest_queue bad bundlecut 'http://x/#httrack-bundle=a..b' -selftest_queue bad bundlecut 'http://x/#httrack-bundle=.hidden' -selftest_queue bad bundlecut 'http://x/#httrack-bundle=a/b' -selftest_queue bad bundlecut 'http://x/#httrack-bundle=a\b' -selftest_queue bad bundlecut 'http://x/#httrack-bundle=a%2fb' -selftest_queue bad bundlecut 'http://x/#httrack-bundle=a b' +for bad in '' ../x a..b .hidden a/b 'a\b' a%2fb "${name127}a" CON con.test \ + Nul.x com1 LPT9.hts 'a#httrack-bundle=b'; do + selftest_queue bad bundlecut "http://x/#httrack-bundle=$bad" +done -# The default scope is the registrable domains of the naming URL, the url lines -# and the cookie domains. Only the hosts in it get the UA and the headers. +# The default scope is the registrable domains of the naming URL and the url +# lines. Cookie domains do not widen it: alpha-cdn.test's cookie is dropped. +# The naming URL is https, so the headers go over https alone. bundle alpha '# HTTrack bundle 1' 'url https://login.alpha-sso.test/start' \ 'ua AlphaUA/1.0' 'header Authorization: Bearer t' 'header X-A: 1' \ 'unknown-key is warned about' cookies "$(jar alpha.test SID s1)" \ - "$(jar alpha-cdn.test C c1)" + "$(jar alpha-cdn.test C c1)" "$(jar alpha.test PREF p1)" \ + "$(jar alpha.test SID s2)" '# an unknown comment is warned about' +alpha_auth='ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1|' selftest_queue_lines "bundle alpha ua AlphaUA/1.0 url https://login.alpha-sso.test/start scope alpha.test scope alpha-sso.test -scope alpha-cdn.test header Authorization: Bearer t header X-A: 1 -cookie .alpha.test SID=s1 -cookie .alpha-cdn.test C=c1 -alpha.test ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1| -https://deep.www.alpha.test:8443 ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1| -login.alpha-sso.test ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1| -cdn.gamma.test ua=global headers=global -notalpha.test ua=global headers=global" bundle "$dir" \ - 'http://www.alpha.test/#httrack-bundle=alpha' host:alpha.test \ - host:https://deep.www.alpha.test:8443 host:login.alpha-sso.test \ - host:cdn.gamma.test host:notalpha.test +cookie .alpha.test PREF=p1 +cookie .alpha.test SID=s2 +https://alpha.test $alpha_auth +https://deep.www.alpha.test:8443 $alpha_auth +http://www.alpha.test ua=AlphaUA/1.0 headers=global +https://login.alpha-sso.test $alpha_auth +https://cdn.gamma.test ua=global headers=global +https://notalpha.test ua=global headers=global" bundle "$dir" \ + 'https://www.alpha.test/#httrack-bundle=alpha' host:https://alpha.test \ + host:https://deep.www.alpha.test:8443 host:http://www.alpha.test \ + host:https://login.alpha-sso.test host:https://cdn.gamma.test \ + host:https://notalpha.test -# scope lines replace the default, and a cookie outside them is dropped -bundle narrow '# HTTrack bundle 1.3' 'scope WWW.alpha.test' 'ua N' cookies \ - "$(jar alpha.test SID wide)" "$(jar www.alpha.test K narrow)" +# The registrable domain follows the suffix list, not the last two labels. A +# host with none (a public suffix, a single label) covers itself alone, and so +# does an address. +bundle uk '# HTTrack bundle 1' 'ua UK' 'header X-UK: 1' +selftest_queue_lines "bundle uk +ua UK +scope alpha.co.uk +scope =s3.amazonaws.com +scope =wiki +scope =127.0.0.1 +header X-UK: 1 +a.alpha.co.uk ua=UK headers=X-UK: 1| +other.co.uk ua=global headers=global +s3.amazonaws.com ua=UK headers=X-UK: 1| +attacker.s3.amazonaws.com ua=global headers=global +wiki ua=UK headers=X-UK: 1| +a.wiki ua=global headers=global +127.0.0.1 ua=UK headers=X-UK: 1| +127.0.0.2 ua=global headers=global" bundle "$dir" \ + 'http://www.alpha.co.uk/#httrack-bundle=uk' \ + 'http://s3.amazonaws.com/x#httrack-bundle=uk' \ + 'http://wiki/#httrack-bundle=uk' 'http://127.0.0.1:8080/#httrack-bundle=uk' \ + host:a.alpha.co.uk host:other.co.uk host:s3.amazonaws.com \ + host:attacker.s3.amazonaws.com host:wiki host:a.wiki host:127.0.0.1 \ + host:127.0.0.2 + +# scope lines replace the default, and a cookie outside them is dropped. CRLF +# line ends and a BOM are read. +printf '\357\273\277# HTTrack bundle 1.3\r\nscope WWW.alpha.test\r\nua N\r\ncookies\r\n%s\r\n%s\r\n' \ + "$(jar alpha.test SID wide)" "$(jar www.alpha.test K narrow)" \ + >"$tmpdir/narrow.hts" selftest_queue_lines "bundle narrow ua N scope www.alpha.test @@ -72,14 +105,19 @@ a.www.alpha.test ua=N headers=global" bundle "$dir" \ 'http://www.alpha.test/#httrack-bundle=narrow' host:alpha.test \ host:a.www.alpha.test -# An address has no registrable domain, so it scopes to itself alone. -bundle ip '# HTTrack bundle 1' 'ua I' -selftest_queue_lines "bundle ip -ua I -scope 127.0.0.1 -127.0.0.1 ua=I headers=global -127.0.0.2 ua=global headers=global" bundle "$dir" \ - 'http://127.0.0.1:8080/#httrack-bundle=ip' host:127.0.0.1 host:127.0.0.2 +# Of the bundle and the jar, the newer file wins, and the bundle wins a tie. +order() { # order NAME BUNDLE-DATE JAR-DATE WINNER + bundle "$1" '# HTTrack bundle 1' cookies "$(jar order.test SID bundle)" + jar order.test SID jar >"$tmpdir/$1.txt" + touch -d "$2" "$tmpdir/$1.hts" + touch -d "$3" "$tmpdir/$1.txt" + selftest_queue_tail " +cookie .order.test SID=$4" bundle "$dir" \ + "http://www.order.test/#httrack-bundle=$1" "jar:$dir/$1.txt" +} +order order1 '2026-01-02 10:00:00' '2026-01-01 10:00:00' bundle +order order2 '2026-01-01 10:00:00' '2026-01-02 10:00:00' jar +order order3 '2026-01-01 10:00:00' '2026-01-01 10:00:00' bundle # Refused: each one stops the run and names the file. The batch reads them all # at the end, so each case writes its own. @@ -92,42 +130,75 @@ refused() { # refused WHY LINE... -- a one-bundle file the engine refuses selftest_queue "refused: /bad$nbad.hts$why" bundle "$dir" \ "http://www.alpha.test/#httrack-bundle=bad$nbad" } -refused ': this bundle version is not supported: # HTTrack bundle 2' \ - '# HTTrack bundle 2' -refused ': not an HTTrack bundle, the first line must be "# HTTrack bundle 1"' \ - 'ua X' -refused ', line 2: refused header (Host and Cookie are the engine'"'"'s): Host: x' \ +notbundle=': not an HTTrack bundle, the first line must be "# HTTrack bundle 1"' +refused ': bundle version 2 is not supported' '# HTTrack bundle 2' +refused "$notbundle" 'ua X' +refused "$notbundle" '# HTTrack bundle 1.' +refused "$notbundle" '# HTTrack bundle 1 x' +refused "$notbundle" '# HTTrack bundle 01' +# an empty file, and one holding a BOM alone +: >"$tmpdir/empty.hts" +selftest_queue "refused: /empty.hts$notbundle" bundle "$dir" \ + 'http://www.alpha.test/#httrack-bundle=empty' +printf '\357\273\277' >"$tmpdir/bom.hts" +selftest_queue "refused: /bom.hts$notbundle" bundle "$dir" \ + 'http://www.alpha.test/#httrack-bundle=bom' +# The messages name a refused header, never its value, which can be a secret. +refused ', line 2: refused header Host: Host and Cookie are the engine'"'"'s' \ '# HTTrack bundle 1' 'header Host: x' -refused ', line 2: refused header (Host and Cookie are the engine'"'"'s): cookie: a=b' \ - '# HTTrack bundle 1' 'header cookie: a=b' -refused ', line 2: refused header (a proxy header is not for the site): Proxy-Authorization: x' \ - '# HTTrack bundle 1' 'header Proxy-Authorization: x' -refused ', line 2: refused header (framing header ignored): Transfer-Encoding: chunked' \ +refused ', line 2: refused header cookie: Host and Cookie are the engine'"'"'s' \ + '# HTTrack bundle 1' 'header cookie: secret-cookie' +refused ', line 2: refused header Proxy-Authorization: a proxy header is not for the site' \ + '# HTTrack bundle 1' 'header Proxy-Authorization: secret-proxy' +refused ', line 2: refused header Transfer-Encoding: framing header ignored' \ '# HTTrack bundle 1' 'header Transfer-Encoding: chunked' -refused ', line 3: refused header (header replaces an earlier one): Authorization: b' \ - '# HTTrack bundle 1' 'header Authorization: a' 'header Authorization: b' -refused ', line 2: refused header (malformed header ignored): no colon' \ - '# HTTrack bundle 1' 'header no colon' +refused ', line 3: refused header Authorization: header replaces an earlier one' \ + '# HTTrack bundle 1' 'header Authorization: a' 'header Authorization: secret-b' +refused ', line 2: refused header: malformed header ignored' \ + '# HTTrack bundle 1' 'header Bearer secret-no-colon' +# A CR, NUL or other control byte would split a header on the wire. +ctl=', line 2: too long or holding a control byte' +refused "$ctl" '# HTTrack bundle 1' "$(printf 'ua x\rHost: evil.test')" +refused "$ctl" '# HTTrack bundle 1' \ + "$(printf 'header X-A: 1\rCookie: forged=1\rProxy-Authorization: z')" +refused "$ctl" '# HTTrack bundle 1' "$(printf 'scope a.test\001')" +refused "$ctl" '# HTTrack bundle 1' "ua $(repeat_chars 8200 u)" +nbad=$((nbad + 1)) +printf '# HTTrack bundle 1\nua a\000Host: evil.test\n' >"$tmpdir/bad$nbad.hts" +selftest_queue "refused: /bad$nbad.hts$ctl" bundle "$dir" \ + "http://www.alpha.test/#httrack-bundle=bad$nbad" refused ', line 3: ua given twice' '# HTTrack bundle 1' 'ua A' 'ua B' refused ', line 2: scope co.uk is a public suffix' '# HTTrack bundle 1' \ 'scope co.uk' -refused ', line 2: invalid url: ftp:///' '# HTTrack bundle 1' 'url ftp:///' +refused ', line 2: invalid scope: ' '# HTTrack bundle 1' 'scope .' +refused ', line 2: invalid url' '# HTTrack bundle 1' 'url ftp:///' +refused ', line 2: invalid url' '# HTTrack bundle 1' \ + 'url http://a.test/#httrack-bundle=x' refused ', line 2: no value' '# HTTrack bundle 1' 'ua' -refused ': 1 cookie line(s) could not be read, see the log' \ - '# HTTrack bundle 1' cookies 'not a jar line' +selftest_queue "refused: /uk.hts: cannot work out the bundle's scope" \ + bundle "$dir" 'http:///#httrack-bundle=uk' selftest_queue 'refused: /none.hts: cannot read the login bundle: No such file or directory' \ bundle "$dir" 'http://www.alpha.test/#httrack-bundle=none' +full=() +for i in $(seq 1 400); do + full+=("$(jar alpha.test "C$i" "$(repeat_chars 100 v)")") +done +refused ': the cookie jar is full, or the file could not be read to its end' \ + '# HTTrack bundle 1' cookies "${full[@]}" # Several bundles: one name twice is one bundle, equal identities on one site -# merge, and different ones are refused. +# merge with their cookies, and different ones are refused either way round. bundle beta '# HTTrack bundle 1' 'ua BetaUA' cookies "$(jar beta.test SID b1)" bundle alpha2 '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ 'header Authorization: Bearer t' 'header X-A: 1' cookies \ "$(jar alpha.test OTHER o)" bundle alphaua '# HTTrack bundle 1' 'ua SomeoneElse' +bundle alphaauth '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ + 'header Authorization: Bearer u' 'header X-A: 1' bundle alphasid '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ 'header Authorization: Bearer t' 'header X-A: 1' cookies \ "$(jar alpha.test SID another-account)" +bundle wwwalpha '# HTTrack bundle 1' 'scope www.alpha.test' 'ua Narrow' selftest_queue_lines "bundle beta ua BetaUA scope beta.test @@ -136,16 +207,34 @@ www.beta.test ua=BetaUA headers=global" bundle "$dir" \ 'http://www.beta.test/#httrack-bundle=beta' \ 'http://beta.test/x#httrack-bundle=beta' host:www.beta.test selftest_queue_tail " -alpha.test ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1| +cookie .alpha.test PREF=p1 +cookie .alpha.test SID=s2 +cookie .alpha.test OTHER=o +cookie .beta.test SID=b1 +https://alpha.test $alpha_auth beta.test ua=BetaUA headers=global" bundle "$dir" \ - 'http://www.alpha.test/#httrack-bundle=alpha' \ - 'http://alpha.test/#httrack-bundle=alpha2' \ - 'http://beta.test/#httrack-bundle=beta' host:alpha.test host:beta.test -selftest_queue 'refused: /alpha.hts and /alphaua.hts cover the same site with a different user agent or headers' \ - bundle "$dir" 'http://www.alpha.test/#httrack-bundle=alpha' \ - 'http://alpha.test/#httrack-bundle=alphaua' -selftest_queue "refused: /alpha.hts and /alphasid.hts hold two values of the cookie 'SID'" \ - bundle "$dir" 'http://www.alpha.test/#httrack-bundle=alpha' \ - 'http://alpha.test/#httrack-bundle=alphasid' + 'https://www.alpha.test/#httrack-bundle=alpha' \ + 'https://alpha.test/#httrack-bundle=alpha2' \ + 'http://beta.test/#httrack-bundle=beta' host:https://alpha.test \ + host:beta.test +differ() { # differ A B -- A and B overlap with different identities + selftest_queue "refused: /$1.hts and /$2.hts cover the same site with a different user agent or headers" \ + bundle "$dir" "https://www.alpha.test/#httrack-bundle=$1" \ + "https://alpha.test/#httrack-bundle=$2" +} +differ alpha alphaua +differ alpha alphaauth +differ alpha wwwalpha +differ wwwalpha alpha +selftest_queue "refused: /alphasid.hts: cookie 'SID' has another value in another bundle" \ + bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' \ + 'https://alpha.test/#httrack-bundle=alphasid' selftest_run_queued + +# A bad name on the command line stops the run before it starts. +rc=0 +out=$(httrack -O "$dir/argv" 'http://x.test/#httrack-bundle=../a' 2>&1) || rc=$? +test "$rc" -ne 0 || fail "a bad bundle name on the command line was accepted" +grep -qF 'Invalid login bundle name' <<<"$out" || + fail "no message for a bad bundle name: $out" diff --git a/tests/656_local-bundles.test b/tests/656_local-bundles.test index 788fe17b6..9a664f879 100644 --- a/tests/656_local-bundles.test +++ b/tests/656_local-bundles.test @@ -1,8 +1,9 @@ #!/bin/bash # -# Two login bundles and a CDN on a third domain, graded on what the server -# received: each site gets its own UA, Authorization and cookies, and the CDN -# gets the global -F and -%X and none of the bundles' identity. +# Login bundles in a real crawl, graded on what the server received: each site +# gets its own UA, Authorization and cookies, while a CDN and another user's +# site under a public suffix get the global -F and -%X. --update finds the +# bundles again, and the newer of the bundle and the jar wins. set -euo pipefail @@ -11,6 +12,7 @@ set -euo pipefail python=$(find_python) || skip "python3 missing" +bin=$(httrack_path) server=$(nativepath "$top_srcdir/tests/bundle-server.py") tmpdir=$(mktemp -d) serverpid= @@ -32,81 +34,140 @@ jar() { # jar DOMAIN NAME VALUE -- one Netscape jar line printf '%s' ".$1${tab}TRUE${tab}/${tab}FALSE${tab}0${tab}$2${tab}$3" } mkdir "$tmpdir/logins" "$tmpdir/one" "$tmpdir/two" +alpha=$tmpdir/logins/alpha.test.hts printf '%s\n' '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ 'header Authorization: Bearer alpha-token' cookies \ "$(jar alpha.test SID alpha-captured)" "$(jar alpha.test PREF alpha-pref)" \ - >"$tmpdir/logins/alpha.test.hts" + >"$alpha" printf '%s\n' '# HTTrack bundle 1' 'ua BetaUA/2.0' \ 'header Authorization: Bearer beta-token' cookies \ "$(jar beta.test SID beta-captured)" >"$tmpdir/logins/beta.test.hts" -cp "$tmpdir/logins/alpha.test.hts" "$tmpdir/alpha.orig" -# the project jar loads over the bundle, as a cookie the server refreshed -jar alpha.test SID alpha-refreshed >"$tmpdir/one/cookies.txt" +printf '%s\n' '# HTTrack bundle 1' 'ua GhUA/3.0' \ + 'header Authorization: Bearer gh-token' >"$tmpdir/logins/gh.hts" +cp "$alpha" "$tmpdir/alpha.orig" +jarfile=$tmpdir/one/cookies.txt +jar alpha.test SID alpha-refreshed >"$jarfile" -crawl() { # crawl DIR [httrack args...] - local dir=$1 rc=0 +crawl() { # crawl CWD [httrack args...] + local cwd=$1 rc=0 shift - run_with_timeout 120 httrack -O "$(nativepath "$dir")" \ - -P "127.0.0.1:$port" -F GlobalUA/9 -%X 'X-Global: 1' \ - '+*cdn.gamma.test/*' -c1 --robots=0 -%v0 -q "$@" >>"$log" 2>&1 || rc=$? + (cd "$cwd" && run_with_timeout 120 "$bin" -P "127.0.0.1:$port" \ + -F GlobalUA/9 -%X 'X-Global: 1' '+*cdn.gamma.test/*' \ + '+*github.io/*' -c1 --robots=0 -%v0 -q "$@") >>"$log" 2>&1 || rc=$? test "$rc" -ne 124 || fail "crawl hit the deadline" return "$rc" } - -hit() { # hit WHAT LINE -- the server must have logged LINE exactly - command grep -qxF "HIT $2" "$tmpdir/srv.out" || - fail_dump "$1: no request [HIT $2]" "$tmpdir/srv.out" +hits() { wc -l <"$tmpdir/srv.out"; } +hit() { # hit WHAT FROM LINE -- the server logged LINE after line FROM + tail -n "+$(($2 + 1))" "$tmpdir/srv.out" >"$tmpdir/new.out" + command grep -qxF "HIT $3" "$tmpdir/new.out" || + fail_dump "$1: no request [HIT $3]" "$tmpdir/new.out" +} +dated() { # dated BUNDLE-DATE JAR-DATE + touch -d "$1" "$alpha" + touch -d "$2" "$jarfile" } -crawl "$tmpdir/one" --bundle-dir "$(nativepath "$tmpdir/logins")" \ +# The jar is newer than the bundle, so its SID wins. --bundle-dir is relative. +dated '2026-01-01 10:00:00' '2026-01-02 10:00:00' +crawl "$tmpdir" -O one --bundle-dir logins \ 'http://www.alpha.test/#httrack-bundle=alpha.test' \ 'http://www.beta.test/#httrack-bundle=beta.test' \ - 'http://alpha.test/plain.html' || fail "the bundle crawl failed" - -alpha='ua=AlphaUA/1.0 auth=Bearer alpha-token cookie=PREF=alpha-pref; SID=alpha-refreshed x=-' -hit "start page" "www.alpha.test/ $alpha" -hit "crawled link" "www.alpha.test/page.html $alpha" -hit "plain URL on the bundle's site" "alpha.test/plain.html $alpha" -hit "second bundle" "www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-" + 'http://alpha.test/plain.html' 'http://github.io/#httrack-bundle=gh' || + fail "the bundle crawl failed" +a='ua=AlphaUA/1.0 auth=Bearer alpha-token cookie=PREF=alpha-pref; SID' +hit "start page" 0 "www.alpha.test/ $a=alpha-refreshed x=-" +hit "crawled link" 0 "www.alpha.test/page.html $a=alpha-refreshed x=-" +hit "plain URL on the bundle's site" 0 "alpha.test/plain.html $a=alpha-refreshed x=-" +hit "second bundle" 0 "www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-" for gif in a.gif b.gif; do - hit "CDN" "cdn.gamma.test/$gif ua=GlobalUA/9 auth=- cookie=- x=1" + hit "CDN" 0 "cdn.gamma.test/$gif ua=GlobalUA/9 auth=- cookie=- x=1" done -n=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) -test "$n" -eq 6 || fail_dump "$n requests, expected 6" "$tmpdir/srv.out" - -# --update finds the bundle again: doit.log keeps the name and the folder +hit "suffix host" 0 "github.io/ ua=GhUA/3.0 auth=Bearer gh-token cookie=- x=-" +hit "another user under the suffix" 0 \ + "attacker.github.io/x.html ua=GlobalUA/9 auth=- cookie=- x=1" +n=$(hits) +test "$n" -eq 9 || fail_dump "$n lines, expected PORT and 8 requests" "$tmpdir/srv.out" +cmp -s "$tmpdir/alpha.orig" "$alpha" || fail "the crawl wrote to the bundle" doit=$tmpdir/one/hts-cache/doit.log -command grep -qF '#httrack-bundle=alpha.test' "$doit" || - fail_dump "doit.log lost the bundle name" "$doit" -command grep -qF -- "-@b " "$doit" || fail_dump "doit.log lost --bundle-dir" "$doit" -cmp -s "$tmpdir/alpha.orig" "$tmpdir/logins/alpha.test.hts" || - fail "the crawl wrote to the bundle" +command grep -qF -- '--bundle-dir logins' "$doit" && + fail_dump "doit.log kept a relative --bundle-dir" "$doit" +# The suffix is cut before any URL reaches a file name, a log or the index. +# hts-log.txt echoes the command line in parentheses, as doit.log keeps it. +names=$(cd "$tmpdir/one" && find . -name '*httrack-bundle*') +test -z "$names" || fail "a file name holds the bundle suffix: $names" +for f in index.html hts-log.txt hts-cache/new.txt; do + test -f "$tmpdir/one/$f" || fail "no $f" + command grep -v '^(' "$tmpdir/one/$f" | command grep -F '#httrack-bundle' && + fail_dump "$f holds the bundle suffix" "$tmpdir/one/$f" +done +command grep -qF 'http://www.alpha.test/ ' "$tmpdir/one/hts-log.txt" || + fail_dump "hts-log.txt does not name the start URL" "$tmpdir/one/hts-log.txt" + +refresh() { # the server refreshed SID, as the saved jar holds it + sed "s/${tab}SID${tab}.*/${tab}SID${tab}alpha-server/" "$jarfile" >"$tmpdir/jar.new" + cat "$tmpdir/jar.new" >"$jarfile" + command grep -qF "SID${tab}alpha-server" "$jarfile" || + fail_dump "the jar holds no SID to refresh" "$jarfile" +} +# --update from another folder reads doit.log, so it finds the bundles again. +# The jar the crawl saved is newer: a cookie the server refreshed wins. +update() { # update WANT-SID + local from + from=$(hits) + crawl / -O "$(nativepath "$tmpdir/one")" --update || fail "--update failed" + hit "--update with SID=$1" "$from" "www.alpha.test/ $a=$1 x=-" +} +refresh +dated '2026-01-01 10:00:00' '2026-01-02 10:00:00' +update alpha-server +# a new capture is newer than the jar, so the fresh login wins +dated '2026-01-03 10:00:00' '2026-01-02 10:00:00' +update alpha-captured +# and so it does on a tie, where a coarse clock cannot order the two +refresh +dated '2026-01-04 10:00:00' '2026-01-04 10:00:00' +update alpha-captured # A -%L list names a bundle too, and it resolves in the project folder. cp "$tmpdir/logins/beta.test.hts" "$tmpdir/two/" printf '%s\n' 'http://www.beta.test/#httrack-bundle=beta.test' >"$tmpdir/list.txt" -before=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) -crawl "$tmpdir/two" -%L "$(nativepath "$tmpdir/list.txt")" || +before=$(hits) +crawl "$tmpdir" -O two -%L "$(nativepath "$tmpdir/list.txt")" || fail "the -%L crawl failed" -after=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) -test "$((after - before))" -eq 2 || - fail_dump "-%L crawl: $((after - before)) requests, expected 2" "$tmpdir/srv.out" -n=$(command grep -cxF "HIT www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-" "$tmpdir/srv.out" || true) -test "$n" -eq 2 || fail_dump "-%L crawl: the beta page did not get its bundle" "$tmpdir/srv.out" - -# A missing bundle stops the run before any request, naming the file. -rc=0 -crawl "$tmpdir/two" 'http://www.alpha.test/#httrack-bundle=nosuch' || rc=$? -test "$rc" -ne 0 || fail "a missing bundle did not stop the run" -command grep -qF "nosuch.hts: cannot read the login bundle" "$log" || - fail_dump "no message naming the missing bundle" "$log" -final=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) -test "$final" -eq "$after" || fail_dump "a missing bundle still crawled" "$tmpdir/srv.out" +test "$(($(hits) - before))" -eq 2 || + fail_dump "-%L crawl: $(($(hits) - before)) requests, expected 2" "$tmpdir/srv.out" +hit "-%L" "$before" "www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-" -# So does a -%L line naming an invalid one. +# A bundle that cannot be read stops the run before any request, with a message +# that names it and never shows a refused header's value. A CR would end a +# header early and start another on the wire. +printf '%s\n' '# HTTrack bundle 1' 'header Proxy-Authorization: planted-secret' \ + >"$tmpdir/two/secret.hts" +printf '# HTTrack bundle 1\nua U\rX-Injected: 1\nheader X-A: 1\rCookie: forged=1\n' \ + >"$tmpdir/two/cr.hts" printf '%s\n' 'http://www.beta.test/#httrack-bundle=../beta.test' >"$tmpdir/bad.txt" -rc=0 -crawl "$tmpdir/two" -%L "$(nativepath "$tmpdir/bad.txt")" || rc=$? -test "$rc" -ne 0 || fail "an invalid -%L bundle name did not stop the run" -final=$(command grep -c '^HIT ' "$tmpdir/srv.out" || true) -test "$final" -eq "$after" || fail_dump "an invalid -%L bundle name still crawled" "$tmpdir/srv.out" +stops=0 +stop() { # stop MESSAGE [httrack args...] + local want=$1 rc=0 + shift + : >"$log" + crawl "$tmpdir" -O two "$@" || rc=$? + test "$rc" -ne 0 || fail "the run did not stop: $*" + command grep -qF "$want" "$log" || fail_dump "no message [$want]" "$log" + stops=$((stops + 1)) +} +before=$(hits) +stop 'nosuch.hts: cannot read the login bundle' \ + 'http://www.alpha.test/#httrack-bundle=nosuch' +stop 'line 1: invalid login bundle name' -%L "$(nativepath "$tmpdir/bad.txt")" +stop 'refused header Proxy-Authorization' \ + 'http://www.alpha.test/#httrack-bundle=secret' +stop 'line 2: too long or holding a control byte' \ + 'http://www.alpha.test/#httrack-bundle=cr' +test "$stops" -eq 4 || fail "$stops stops ran, expected 4" +test "$(hits)" -eq "$before" || fail_dump "a refused bundle still crawled" "$tmpdir/srv.out" +for f in "$log" "$tmpdir/two/hts-log.txt"; do + command grep -qF planted-secret "$f" && fail_dump "$f shows the secret" "$f" +done +true diff --git a/tests/Makefile.am b/tests/Makefile.am index d62dc1c3e..f1cf3b066 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -10,7 +10,7 @@ EXTRA_DIST = $(TESTS) renamefail.c unlinkfail.c threadattrfail.c gaistall.c noba proxy-https-server.py socks5-server.py proxy-connect-server.py \ proxytestlib.py tls-stall-server.py warc-validate.py wacz-validate.py \ header-injection-server.py header-injection-check.py \ - postfile-server.py \ + postfile-server.py bundle-server.py \ status-line-server.py retry-after-server.py \ pty-resize.py ttyrun.py test-timeout.sh png-colors.py \ local-crawl.sh local-server.py ftp-server.py testlib.sh proclib.sh \ diff --git a/tests/bundle-server.py b/tests/bundle-server.py index 634bda134..f68915003 100644 --- a/tests/bundle-server.py +++ b/tests/bundle-server.py @@ -3,7 +3,8 @@ httrack reaches it as its -P proxy, so the absolute request URL names the site. Every request prints one "HIT ua=<...> auth=<...> -cookie=<...> x=<...>" line, "-" for a missing header. Prints "PORT ". +cookie=<...> x=<...>" line, "-" for a missing header, with the cookies in +name order. Prints "PORT ". """ import sys @@ -19,6 +20,8 @@ "www.beta.test/": '' % CDN, "cdn.gamma.test/a.gif": "GIF89a", "cdn.gamma.test/b.gif": "GIF89a", + "github.io/": 'x', + "attacker.github.io/x.html": "another user's site", } lock = threading.Lock() @@ -32,7 +35,7 @@ def log_message(self, *args): def do_GET(self): url = urlsplit(self.path) - key = url.hostname + url.path if url.hostname else self.path + key = url.hostname + url.path with lock: print( "HIT %s ua=%s auth=%s cookie=%s x=%s" @@ -40,14 +43,12 @@ def do_GET(self): key, self.headers.get("User-Agent", "-"), self.headers.get("Authorization", "-"), - self.headers.get("Cookie", "-"), + "; ".join(sorted(self.headers.get("Cookie", "-").split("; "))), self.headers.get("X-Global", "-"), ), flush=True, ) - body = PAGES.get(key) - if body is None: - return self.reply(404, b"no", "text/html") + body = PAGES.get(key, "") ctype = "image/gif" if key.endswith(".gif") else "text/html" if ctype == "text/html": body = "%s" % body From 2e844e1893673426cc7e70953d6edbf2aa526c1f Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 15:29:45 +0200 Subject: [PATCH 03/10] Rework login bundles after review round 2 A bundle cookie line with a control byte other than TAB now stops the run. Only the bundle path changed, so -%K and cookies.txt read as on master. No bundle message prints a byte of a bundle line any more. It gives the file, the line number and the reason, and names an unknown key only when the key is all lowercase letters. The https rule follows the design. When a URL naming a bundle or one of its url lines is https, its headers go over https alone. Bundles that merge share the rule, whichever one is listed first. The cookie filter is now an admit callback, and the scope and clash checks live in htsbundle.c. A full comparison jar stops the run, and so does a header block over 8192 bytes. One helper cuts the suffix for the command line and for -%L. The dump function is gone, and the self-test reads the identity each host gets instead. Signed-off-by: Xavier Roche Co-Authored-By: Claude Opus 5.5 --- html/httrack.man.html | 22 +- man/httrack.1 | 15 +- man/httrack.tmpl | 15 +- src/htsbauth.c | 181 ++++++++-------- src/htsbundle.c | 387 +++++++++++++++++++++-------------- src/htsbundle.h | 71 ++++--- src/htsbundle_selftest.c | 74 ++++--- src/htscore.c | 40 ++-- src/htscoremain.c | 60 ++---- src/htslib.c | 1 + src/htslib.h | 5 +- src/htslines.c | 12 +- src/htslines.h | 3 +- tests/655_engine-bundle.test | 245 ++++++++++++++++------ tests/656_local-bundles.test | 33 ++- 15 files changed, 680 insertions(+), 484 deletions(-) diff --git a/html/httrack.man.html b/html/httrack.man.html index a91d46ffc..1c1ba38a0 100644 --- a/html/httrack.man.html +++ b/html/httrack.man.html @@ -2502,17 +2502,23 @@

FILES bundle’s user agent and headers replace -F and -%X for each host in its scope, and only there. A bundle with header lines but no ua line sends the --F user agent. When every start URL naming the bundle -is https, its headers go over https alone. By default the -scope is the registrable domains of the start URLs naming -the bundle and of its url lines. A host with no registrable -domain, such as a public suffix, a single label or an -address, covers itself alone. Scope lines replace that +-F user agent. When any start URL naming the bundle, +or any of its url lines, is https, its headers go over https +alone, and so do those of the bundles it merges with. By +default the scope is the registrable domains of the start +URLs naming the bundle and of its url lines. A host with no +registrable domain, such as a public suffix, a single label +or an address, covers itself alone. Scope lines replace that default. HTTrack drops a cookie outside the scope and gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy-* or framing header, and a line holding a control -byte. Two bundles whose scopes overlap must give the same -user agent, headers and cookie values, or the run stops.

+byte. A second ua line, header lines over 8192 bytes, +or a full cookie jar stops the run too. A repeated cookie +line, an unknown key or an unknown # line only warns. +Messages give the file and the line number, never the +line’s text. Two bundles whose scopes overlap must +give the same user agent, headers and cookie values, or the +run stops.

The bundle’s cookies go into the cookie jar, which HTTrack diff --git a/man/httrack.1 b/man/httrack.1 index 4dc28b01e..3652d1a8b 100644 --- a/man/httrack.1 +++ b/man/httrack.1 @@ -627,15 +627,20 @@ for each host in its scope, and only there. A bundle with header lines but no .I ua line sends the .I \-F -user agent. When every start URL naming the bundle is https, its headers go -over https alone. By default the scope is the registrable domains of the start +user agent. When any start URL naming the bundle, or any of its url lines, is +https, its headers go over https alone, and so do those of the bundles it +merges with. By default the scope is the registrable domains of the start URLs naming the bundle and of its url lines. A host with no registrable domain, such as a public suffix, a single label or an address, covers itself alone. Scope lines replace that default. HTTrack drops a cookie outside the scope and gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or -framing header, and a line holding a control byte. Two bundles whose scopes -overlap must give the same user agent, headers and cookie values, or the run -stops. +framing header, and a line holding a control byte. A second +.I ua +line, header lines over 8192 bytes, or a full cookie jar stops the run too. A +repeated cookie line, an unknown key or an unknown # line only warns. Messages +give the file and the line number, never the line's text. Two bundles whose +scopes overlap must give the same user agent, headers and cookie values, or the +run stops. .IP The bundle's cookies go into the cookie jar, which HTTrack writes at the end of the crawl. Of the bundle and the jar, the newer file wins a cookie both hold. diff --git a/man/httrack.tmpl b/man/httrack.tmpl index 16189ea9f..da966208b 100644 --- a/man/httrack.tmpl +++ b/man/httrack.tmpl @@ -110,15 +110,20 @@ for each host in its scope, and only there. A bundle with header lines but no .I ua line sends the .I \-F -user agent. When every start URL naming the bundle is https, its headers go -over https alone. By default the scope is the registrable domains of the start +user agent. When any start URL naming the bundle, or any of its url lines, is +https, its headers go over https alone, and so do those of the bundles it +merges with. By default the scope is the registrable domains of the start URLs naming the bundle and of its url lines. A host with no registrable domain, such as a public suffix, a single label or an address, covers itself alone. Scope lines replace that default. HTTrack drops a cookie outside the scope and gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or -framing header, and a line holding a control byte. Two bundles whose scopes -overlap must give the same user agent, headers and cookie values, or the run -stops. +framing header, and a line holding a control byte. A second +.I ua +line, header lines over 8192 bytes, or a full cookie jar stops the run too. A +repeated cookie line, an unknown key or an unknown # line only warns. Messages +give the file and the line number, never the line's text. Two bundles whose +scopes overlap must give the same user agent, headers and cookie values, or the +run stops. .IP The bundle's cookies go into the cookie jar, which HTTrack writes at the end of the crawl. Of the bundle and the jar, the newer file wins a cookie both hold. diff --git a/src/htsbauth.c b/src/htsbauth.c index a344ecc43..d844f35e1 100644 --- a/src/htsbauth.c +++ b/src/htsbauth.c @@ -97,8 +97,6 @@ hts_boolean cookie_host(const char *adr, char *dst, size_t dst_size) { return HTS_TRUE; } -static char *cookie_find_exact(char *s, const char *cook_name, - const char *domain, const char *path); static hts_boolean cookie_domain_is_suffix(const char *domain); /* See htsbauth.h. Records are kept longest path first. */ @@ -219,10 +217,10 @@ static hts_boolean cookie_same_domain(const char *a, const char *b) { return strcmpnocase(a, b) == 0; } -/* Return the record with this name, domain and path, or NULL. The host-only - flag is not part of that identity (RFC 6265 5.3 step 11). */ -static char *cookie_find_exact(char *s, const char *cook_name, - const char *domain, const char *path) { +/* See htsbundle.h. The host-only flag is not part of a record's identity + (RFC 6265 5.3 step 11). */ +char *cookie_find_exact(char *s, const char *cook_name, const char *domain, + const char *path) { char buffer[8192]; char *a; @@ -469,19 +467,31 @@ char *cookie_nextfield(char *a) { return a; } +/* Warn about the line just read. A bundle's warning gives the line number and + the reason but no byte of the line, which can hold a secret. */ +#define COOKIE_WARN(reason, ...) \ + do { \ + if (filter != NULL) \ + hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: %s", given, \ + filter->line, reason); \ + else \ + hts_log_print(opt, LOG_WARNING, __VA_ARGS__); \ + } while (0) + /* Copy Netscape-jar field into dst, or refuse the line and say so: a shortened domain or path would silently change where the cookie is sent. */ static hts_boolean cookie_load_field(httrackp *opt, char *dst, size_t dst_size, char *buffer, const char *line, int param, - const char *field_name, const char *file) { + const char *field_name, const char *given, + const cookie_filter *filter) { const char *const value = cookie_get(buffer, line, param); const size_t len = strlen(value); if (len >= dst_size) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a line whose %s field is %d bytes " - "(maximum %d): %s", - field_name, (int) len, (int) dst_size - 1, file); + COOKIE_WARN("a cookie field is too long", + "Cookies: ignoring a line whose %s field is %d bytes " + "(maximum %d): %s", + field_name, (int) len, (int) dst_size - 1, given); return HTS_FALSE; } strlcpybuff(dst, value, dst_size); @@ -654,13 +664,11 @@ static hts_boolean cookie_is_flat(FILE *fp, int *quote) { } /* See htsbundle.h. */ -hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size, - hts_boolean *https) { +hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size) { const char *const rest = jump_protocol_const(url); const size_t len = strcspn(rest, "/?#"); char adr[HTS_URLMAXSIZE]; - *https = strfield(url, "https:") != 0; if (len >= sizeof(adr)) return HTS_FALSE; adr[0] = '\0'; @@ -724,14 +732,13 @@ static int cookie_load_flat(httrackp *opt, t_cookie *cookie, FILE *fp, hts_boolean nul = HTS_FALSE, too_long = HTS_FALSE; hts_boolean line_start = HTS_TRUE; hts_boolean quoted = HTS_FALSE, quote_done = quote == 0; - hts_boolean https = HTS_FALSE; cookie_attr attr; const char *domain; int stored = 0, ignored = 0; size_t i; int c; - if (!cookie_url_host(start_url, host, sizeof(host), &https)) { + if (!cookie_url_host(start_url, host, sizeof(host))) { hts_log_print(opt, LOG_WARNING, "Cookies: %s is a document.cookie string, but there is no " "start URL to bind it to, so it is ignored", @@ -743,7 +750,7 @@ static int cookie_load_flat(httrackp *opt, t_cookie *cookie, FILE *fp, ? NULL : hts_psl_registrable_domain(host); attr.host_only = domain == NULL; - attr.secure = https; + attr.secure = strfield(start_url, "https:") != 0; attr.http_only = HTS_FALSE; attr.expires = 0; if (domain == NULL) @@ -818,36 +825,6 @@ static int cookie_load_flat(httrackp *opt, t_cookie *cookie, FILE *fp, return COOKIE_LOAD_KEEP; } -/* Should filter let the record name=value for domain and path into cookie? - seen holds this load's records, which may repeat one without a clash. */ -static hts_boolean cookie_admit(cookie_filter *filter, t_cookie *cookie, - t_cookie *seen, const char *name, - const char *value, const char *domain, - const char *path) { - char buffer[8192]; - char host[256]; - const char *other; - - if (!cookie_host(domain, host, sizeof(host)) || - !filter->covers(filter->scope, host[0] == '.' ? host + 1 : host)) { - filter->dropped++; - return HTS_FALSE; - } - other = cookie_find_exact(filter->others->data, name, host, path); - if (other != NULL && - (seen == NULL || - cookie_find_exact(seen->data, name, host, path) == NULL) && - strcmp(cookie_get(buffer, other, 6), value) != 0) { - filter->clash[0] = '\0'; - strlncatbuff(filter->clash, name, sizeof(filter->clash), - sizeof(filter->clash) - 1); - return HTS_FALSE; - } - (void) cookie_set(filter->others, name, value, domain, path, NULL); - return !filter->under || - cookie_find_exact(cookie->data, name, host, path) == NULL; -} - /* Read the Netscape records from fp's position to its end into cookie, and name the file as given in warnings. A UTF-8 BOM is skipped on the first line when first_line is HTS_TRUE. filter, when not NULL, picks the records. @@ -871,11 +848,14 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, keep = HTS_TRUE; } // ferror ends the loop too, because a directory never reaches EOF - while (!feof(fp) && !ferror(fp) && strlen(cookie->data) < cookie->max_len) { + while (!feof(fp) && !ferror(fp) && strlen(cookie->data) < cookie->max_len && + (filter == NULL || filter->why == NULL)) { size_t len; // len counts what the buffer cut too, so this catches that line cookie_readline(fp, line, sizeof(line), &len); + if (filter != NULL) + filter->line++; if (first_line && len >= sizeof(cookie_bom) - 1 && strncmp(line, cookie_bom, sizeof(cookie_bom) - 1) == 0) { len -= sizeof(cookie_bom) - 1; @@ -884,11 +864,14 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, } first_line = HTS_FALSE; if (len >= line_max) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring an over-long line " - "(%d bytes, maximum %d): %s", - (int) len, (int) line_max - 1, given); + COOKIE_WARN("a cookie line is too long", + "Cookies: ignoring an over-long line " + "(%d bytes, maximum %d): %s", + (int) len, (int) line_max - 1, given); refused++; + } else if (filter != NULL && hts_bundle_has_control(line, len)) { + // a bundle is written for this run, so a damaged one stops it + filter->why = "a cookie line holds a control byte"; } else if (memchr(line, '\0', len) != NULL || memchr(line, '\r', len) != NULL) { hts_log_print(opt, LOG_WARNING, @@ -904,9 +887,9 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, if (rec[0] == '#') { if (!cookie_known_comment(rec)) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring an unknown comment in %s: %.64s", - given, rec); + COOKIE_WARN("an unknown comment", + "Cookies: ignoring an unknown comment in %s: %.64s", + given, rec); refused++; } } else { @@ -923,36 +906,39 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, for (p = strchr(rec, '\t'); p != NULL; p = strchr(p + 1, '\t')) tabs++; if (tabs < 6 || !strnotempty(cookie_get(buffer, rec, 5))) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a line with fewer than 7 " - "fields or no name: %s", - given); + COOKIE_WARN("a cookie line has fewer than 7 fields or no name", + "Cookies: ignoring a line with fewer than 7 " + "fields or no name: %s", + given); refused++; } else if (tabs > 7 || (tabs == 7 && strcmp(cookie_get(buffer, rec, 7), COOKIE_HTTPONLY_FIELD) != 0)) { // a TAB in the value, because only HTTrack writes an eighth field - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a line with more than 7 " - "fields: %s", - given); + COOKIE_WARN("a cookie line has more than 7 fields", + "Cookies: ignoring a line with more than 7 " + "fields: %s", + given); refused++; } else if (!cookie_is_expiry(cookie_get(buffer, rec, 4))) { - hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a line whose expiry is not a " - "whole number: %s", - given); + COOKIE_WARN("a cookie expiry is not a whole number", + "Cookies: ignoring a line whose expiry is not a " + "whole number: %s", + given); refused++; } else if (cookie_load_field(opt, domain, sizeof(domain), buffer, rec, - 0, "domain", given) && + 0, "domain", given, filter) && cookie_load_field(opt, path, sizeof(path), buffer, rec, 2, - "path", given) && + "path", given, filter) && cookie_load_field(opt, cook_name, sizeof(cook_name), buffer, - rec, 5, "name", given) && + rec, 5, "name", given, filter) && cookie_load_field(opt, cook_value, sizeof(cook_value), - buffer, rec, 6, "value", given)) { + buffer, rec, 6, "value", given, filter)) { cookie_attr attr; char host[256]; + const hts_boolean repeated = + seen != NULL && cookie_host(domain, host, sizeof(host)) && + cookie_find_exact(seen->data, cook_name, host, path) != NULL; attr.host_only = cookie_is_host_only(rec); attr.secure = strcmpnocase(cookie_get(buffer, rec, 3), "TRUE") == 0; @@ -960,35 +946,33 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, attr.expires = cookie_record_expiry(rec); // an expired one is skipped, because a browser drops it too if ((attr.expires == 0 || attr.expires > (int64_t) time(NULL)) && - (filter == NULL || cookie_admit(filter, cookie, seen, cook_name, - cook_value, domain, path))) { - const hts_boolean repeated = - seen != NULL && cookie_host(domain, host, sizeof(host)) && - cookie_find_exact(seen->data, cook_name, host, path) != NULL; - + (filter == NULL || + filter->admit(filter, cookie, repeated, cook_name, cook_value, + domain, path))) { // cookie_set stores it host-only if (!attr.host_only && cookie_domain_is_suffix(domain)) - hts_log_print(opt, LOG_WARNING, - "Cookies: '%.64s' for the public suffix %s is " - "now host-only: %s", - cook_name, domain, given); + COOKIE_WARN("a cookie for a public suffix is now host-only", + "Cookies: '%.64s' for the public suffix %s is " + "now host-only: %s", + cook_name, domain, given); // the last one wins, as curl reads it, but a rewrite loses one if (repeated) { - hts_log_print(opt, LOG_WARNING, - "Cookies: '%.64s' appears twice in %s", cook_name, - given); + COOKIE_WARN("a cookie appears twice", + "Cookies: '%.64s' appears twice in %s", cook_name, + given); refused++; } if (cookie_set(cookie, cook_name, cook_value, domain, path, &attr) != 0) { // the jar caps a field tighter still, and a full one refuses - hts_log_print(opt, LOG_WARNING, - "Cookies: the jar did not store '%.64s' from " - "%s", - cook_name, given); - refused++; if (filter != NULL) - filter->lost++; + filter->why = "the cookie jar did not store a cookie"; + else + hts_log_print(opt, LOG_WARNING, + "Cookies: the jar did not store '%.64s' from " + "%s", + cook_name, given); + refused++; } else if (seen != NULL && !repeated) { (void) cookie_set(seen, cook_name, "", domain, path, NULL); } @@ -999,12 +983,10 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, } } } - if (!feof(fp)) { - hts_log_print(opt, LOG_WARNING, - "Cookies: stopped reading before the end of %s", given); + if (!feof(fp) && (filter == NULL || filter->why == NULL)) { + COOKIE_WARN("stopped reading before the end", + "Cookies: stopped reading before the end of %s", given); refused++; - if (filter != NULL) - filter->lost++; } freet(seen); return keep ? COOKIE_LOAD_KEEP : refused; @@ -1127,8 +1109,8 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, return ret; } if (fp) { - const int ret = - cookie_load_records(opt, cookie, fp, given, HTS_TRUE, NULL); + const int ret = cookie_load_records(opt, cookie, fp, given, + /* first_line */ HTS_TRUE, NULL); fclose(fp); return ret; @@ -1145,8 +1127,9 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, /* See htsbundle.h. */ void cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, - const char *name, cookie_filter *filter) { - (void) cookie_load_records(opt, cookie, fp, name, HTS_FALSE, filter); + const char *given, cookie_filter *filter) { + (void) cookie_load_records(opt, cookie, fp, given, /* first_line */ HTS_FALSE, + filter); } /* Open name for writing, creating it if needed, and return NULL on failure. diff --git a/src/htsbundle.c b/src/htsbundle.c index 17b9abbfe..925290384 100644 --- a/src/htsbundle.c +++ b/src/htsbundle.c @@ -48,6 +48,10 @@ Please visit our Website: http://www.httrack.com /* A bundle holds at most this many url lines, and as many scope entries. */ #define BUNDLE_LIST_MAX 256 +/* A bundle's header lines, with their CRLFs, fit in this many bytes, so the + request still has room for the rest. */ +#define BUNDLE_HEADERS_MAX 8192 + typedef struct bundle { char name[HTS_BUNDLE_NAME_MAX]; char file[HTS_URLMAXSIZE * 2]; @@ -97,47 +101,61 @@ static void bundle_clear(bundle *b) { strlist_free(&b->scope, &b->nscope); freet(b->ua); StringClear(b->headers); + b->https_only = HTS_FALSE; } -/* Is name a bundle name: [A-Za-z0-9._-], no leading '.', no "..", and no - Windows device name such as CON or com1.txt? */ -static hts_boolean bundle_name_valid(const char *name) { +/* Is name a Windows device name, such as CON, aux.txt or com1? */ +static hts_boolean bundle_name_is_device(const char *name) { static const char *const devices[] = {"con", "prn", "aux", "nul"}; - const size_t len = strlen(name); const size_t base = strcspn(name, "."); char dev[5]; size_t i; - if (len == 0 || len >= HTS_BUNDLE_NAME_MAX || name[0] == '.' || - strstr(name, "..") != NULL || - strspn(name, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - "0123456789._-") != len) - return HTS_FALSE; if (base < 3 || base > 4) - return HTS_TRUE; + return HTS_FALSE; dev[0] = '\0'; strlncatbuff(dev, name, sizeof(dev), base); hts_lowcase(dev); - for (i = 0; base == 3 && i < sizeof(devices) / sizeof(devices[0]); i++) { + if (base == 4) + return (strncmp(dev, "com", 3) == 0 || strncmp(dev, "lpt", 3) == 0) && + dev[3] >= '1' && dev[3] <= '9'; + for (i = 0; i < sizeof(devices) / sizeof(devices[0]); i++) { if (strcmp(dev, devices[i]) == 0) - return HTS_FALSE; + return HTS_TRUE; } - return base == 3 || dev[3] < '1' || dev[3] > '9' || - (strncmp(dev, "com", 3) != 0 && strncmp(dev, "lpt", 3) != 0); + return HTS_FALSE; } -hts_bundle_suffix hts_bundle_cut(char *url, char *name, size_t name_size) { +/* Why name is not a bundle name, or NULL when it is one. */ +static const char *bundle_name_refusal(const char *name) { + const size_t len = strlen(name); + + if (len >= HTS_BUNDLE_NAME_MAX) + return "invalid login bundle name: 128 bytes or more"; + if (len == 0 || + strspn(name, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789._-") != len) + return "invalid login bundle name: use letters, digits, '.', '_' and '-'"; + if (name[0] == '.' || strstr(name, "..") != NULL) + return "invalid login bundle name: it starts with '.' or holds \"..\""; + if (bundle_name_is_device(name)) + return "invalid login bundle name: a Windows device name"; + return NULL; +} + +const char *hts_bundle_cut(char *url, char *name, size_t name_size) { char *const at = strstr(url, HTS_BUNDLE_SUFFIX); - const char *text; + const char *why; + name[0] = '\0'; if (at == NULL) - return HTS_BUNDLE_NONE; - text = at + strlen(HTS_BUNDLE_SUFFIX); - if (!bundle_name_valid(text) || strlen(text) >= name_size) - return HTS_BUNDLE_BAD; - strlcpybuff(name, text, name_size); + return NULL; + why = bundle_name_refusal(at + strlen(HTS_BUNDLE_SUFFIX)); + if (why != NULL) + return why; + strlcpybuff(name, at + strlen(HTS_BUNDLE_SUFFIX), name_size); *at = '\0'; - return HTS_BUNDLE_NAMED; + return NULL; } void hts_bundle_free(httrackp *opt) { @@ -155,7 +173,10 @@ void hts_bundle_free(httrackp *opt) { freet(opt->bundle_state); } -hts_boolean hts_bundle_add(httrackp *opt, const char *name, const char *url) { +/* Record that the start URL url names bundle name. HTS_FALSE when out of + memory. */ +static hts_boolean bundle_add(httrackp *opt, const char *name, + const char *url) { hts_bundle_state *st = (hts_bundle_state *) opt->bundle_state; bundle *b = NULL; size_t i; @@ -184,23 +205,38 @@ hts_boolean hts_bundle_add(httrackp *opt, const char *name, const char *url) { return strlist_add(&b->start_urls, &b->nstart, url); } -/* Read a line into line without its line end. HTS_FALSE for a line too long, - or holding a control byte other than TAB: a CR would end a header early. */ -static hts_boolean bundle_readline(FILE *fp, char *line, size_t size) { - size_t len; +const char *hts_bundle_take(httrackp *opt, char *url) { + char name[HTS_BUNDLE_NAME_MAX]; + const char *const why = hts_bundle_cut(url, name, sizeof(name)); + + if (why != NULL || name[0] == '\0') + return why; + return bundle_add(opt, name, url) ? NULL : "out of memory"; +} + +hts_boolean hts_bundle_has_control(const char *s, size_t len) { size_t i; - if (hts_readline_raw(fp, line, size, &len)) - return HTS_FALSE; - if (len > 0 && line[len - 1] == '\r') - line[--len] = '\0'; for (i = 0; i < len; i++) { - const unsigned char c = (unsigned char) line[i]; + const unsigned char c = (unsigned char) s[i]; if ((c < 0x20 && c != '\t') || c == 0x7f) - return HTS_FALSE; + return HTS_TRUE; } - return HTS_TRUE; + return HTS_FALSE; +} + +/* Read a line into line without its line end. NULL, or why the line is + refused: too long, or with a control byte other than TAB, because a CR + would end a header early. */ +static const char *bundle_readline(FILE *fp, char *line, size_t size) { + size_t len; + + if (hts_readline_raw(fp, line, size, &len)) + return "too long"; + if (len > 0 && line[len - 1] == '\r') + line[--len] = '\0'; + return hts_bundle_has_control(line, len) ? "holds a control byte" : NULL; } /* The major version a "# HTTrack bundle N[.M]" line gives, or -1 when line is @@ -234,8 +270,7 @@ static hts_boolean scope_covers(const char *entry, const char *host) { : cookie_domain_match(entry, host); } -static hts_boolean bundle_in_scope(const void *b_, const char *host) { - const bundle *const b = (const bundle *) b_; +static hts_boolean bundle_in_scope(const bundle *b, const char *host) { size_t i; for (i = 0; i < b->nscope; i++) { @@ -245,28 +280,35 @@ static hts_boolean bundle_in_scope(const void *b_, const char *host) { return HTS_FALSE; } -/* Add entry to b's scope. HTS_FALSE when out of memory or full. */ -static hts_boolean bundle_scope_add(bundle *b, const char *entry) { +/* Add entry to b's scope. NULL, or why it could not. */ +static const char *bundle_scope_add(bundle *b, const char *entry) { size_t i; for (i = 0; i < b->nscope; i++) { if (strcmp(b->scope[i], entry) == 0) - return HTS_TRUE; + return NULL; } - return b->nscope < BUNDLE_LIST_MAX && - strlist_add(&b->scope, &b->nscope, entry); + if (b->nscope >= BUNDLE_LIST_MAX) + return "more than 256 scope domains"; + return strlist_add(&b->scope, &b->nscope, entry) ? NULL : "out of memory"; } -/* Add url's registrable domain to b's scope, or its host alone when it has - none: a public suffix, a single label or an address. */ -static hts_boolean bundle_scope_url(bundle *b, const char *url) { +/* Note that url names b. An https one makes b's headers https-only, and + unless scope lines replace it, the URL's registrable domain joins the scope, + or its host alone when it has none: a public suffix, a single label or an + address. NULL, or why it could not. */ +static const char *bundle_note_url(bundle *b, const char *url, + hts_boolean scope_given) { char host[256]; char entry[sizeof(host) + 1]; - hts_boolean https; const char *reg; - if (!cookie_url_host(url, host, sizeof(host), &https)) - return HTS_FALSE; + if (strfield(url, "https:")) + b->https_only = HTS_TRUE; + if (scope_given) + return NULL; + if (!cookie_url_host(url, host, sizeof(host))) + return "cannot work out the bundle's scope"; reg = hts_host_is_address(host, strlen(host)) ? NULL : hts_psl_registrable_domain(host); @@ -275,8 +317,8 @@ static hts_boolean bundle_scope_url(bundle *b, const char *url) { return bundle_scope_add(b, entry); } -/* Write why into err and return HTS_FALSE, so it needs err and err_size in - scope. */ +/* Write the reason into err and return HTS_FALSE. The caller needs err and + err_size. */ #define BUNDLE_FAIL(...) \ do { \ snprintf(err, err_size, __VA_ARGS__); \ @@ -288,13 +330,14 @@ static hts_boolean bundle_scope_line(bundle *b, const char *value, int lineno, char *err, size_t err_size) { char dom[256]; char entry[sizeof(dom) + 1]; + const char *why; if (value[0] == '.') value++; if (value[0] == '\0' || strlen(value) >= sizeof(dom) || strspn(value, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" "0123456789.-") != strlen(value)) - BUNDLE_FAIL("%s, line %d: invalid scope: %.64s", b->file, lineno, value); + BUNDLE_FAIL("%s, line %d: invalid scope", b->file, lineno); strcpybuff(dom, value); hts_lowcase(dom); if (hts_host_is_address(dom, strlen(dom))) @@ -302,25 +345,31 @@ static hts_boolean bundle_scope_line(bundle *b, const char *value, int lineno, else if (hts_psl_registrable_domain(dom) != NULL) strcpybuff(entry, dom); else - BUNDLE_FAIL("%s, line %d: scope %s is a public suffix", b->file, lineno, - dom); - if (!bundle_scope_add(b, entry)) - BUNDLE_FAIL("%s, line %d: more than %d scope lines, or out of memory", - b->file, lineno, BUNDLE_LIST_MAX); + BUNDLE_FAIL("%s, line %d: the scope is a public suffix", b->file, lineno); + why = bundle_scope_add(b, entry); + if (why != NULL) + BUNDLE_FAIL("%s, line %d: %s", b->file, lineno, why); return HTS_TRUE; } -/* Read b's lines up to "cookies", leaving fp at its jar, and set *jar. */ +/* Is the klen bytes at line the key key? */ +static hts_boolean key_is(const char *line, size_t klen, const char *key) { + return klen == strlen(key) && strncmp(line, key, klen) == 0; +} + +/* Read b's lines up to "cookies", leaving fp at its jar. *cookies_line gets + the number of that line, or 0 when b has no cookies. */ static hts_boolean bundle_parse(httrackp *opt, bundle *b, FILE *fp, - hts_boolean *scope_given, hts_boolean *jar, + hts_boolean *scope_given, int *cookies_line, char *err, size_t err_size) { static const char bom[] = "\xEF\xBB\xBF"; char BIGSTK line[BUNDLE_LINE_MAX]; int lineno = 1; int version; + const char *why = bundle_readline(fp, line, sizeof(line)); - if (!bundle_readline(fp, line, sizeof(line))) - BUNDLE_FAIL("%s, line 1: too long or holding a control byte", b->file); + if (why != NULL) + BUNDLE_FAIL("%s, line 1: %s", b->file, why); version = bundle_version_of( strncmp(line, bom, sizeof(bom) - 1) == 0 ? line + sizeof(bom) - 1 : line); if (version < 0) @@ -329,90 +378,137 @@ static hts_boolean bundle_parse(httrackp *opt, bundle *b, FILE *fp, b->file); if (version != 1) BUNDLE_FAIL("%s: bundle version %d is not supported", b->file, version); - *scope_given = *jar = HTS_FALSE; + *scope_given = HTS_FALSE; + *cookies_line = 0; while (!feof(fp)) { const char *value; size_t klen; lineno++; - if (!bundle_readline(fp, line, sizeof(line))) - BUNDLE_FAIL("%s, line %d: too long or holding a control byte", b->file, - lineno); + why = bundle_readline(fp, line, sizeof(line)); + if (why != NULL) + BUNDLE_FAIL("%s, line %d: %s", b->file, lineno, why); if (line[0] == '\0' || line[0] == '#') continue; klen = strcspn(line, " \t"); value = line + klen; value += strspn(value, " \t"); - if (klen == 7 && strncmp(line, "cookies", klen) == 0) { + if (key_is(line, klen, "cookies")) { if (*value != '\0') BUNDLE_FAIL("%s, line %d: \"cookies\" takes no value", b->file, lineno); - *jar = HTS_TRUE; + *cookies_line = lineno; return HTS_TRUE; } + if (!key_is(line, klen, "url") && !key_is(line, klen, "ua") && + !key_is(line, klen, "header") && !key_is(line, klen, "scope")) { + /* a key in lowercase letters is no secret, so it alone is named */ + const hts_boolean shown = + klen <= 32 && strspn(line, "abcdefghijklmnopqrstuvwxyz") == klen; + + hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: unknown key%s%.*s", + b->file, lineno, shown ? " " : "", shown ? (int) klen : 0, + line); + continue; + } if (*value == '\0') BUNDLE_FAIL("%s, line %d: no value", b->file, lineno); - if (klen == 3 && strncmp(line, "url", klen) == 0) { + if (key_is(line, klen, "url")) { char host[256]; - hts_boolean https; - /* a url can hold a token, so the message leaves it out */ if (strlen(value) >= HTS_URLMAXSIZE || strstr(value, HTS_BUNDLE_SUFFIX) != NULL || - !cookie_url_host(value, host, sizeof(host), &https)) + !cookie_url_host(value, host, sizeof(host))) BUNDLE_FAIL("%s, line %d: invalid url", b->file, lineno); if (b->nurls >= BUNDLE_LIST_MAX) BUNDLE_FAIL("%s, line %d: more than %d url lines", b->file, lineno, BUNDLE_LIST_MAX); if (!strlist_add(&b->urls, &b->nurls, value)) BUNDLE_FAIL("%s: out of memory", b->file); - } else if (klen == 2 && strncmp(line, "ua", klen) == 0) { + } else if (key_is(line, klen, "ua")) { if (b->ua != NULL) BUNDLE_FAIL("%s, line %d: ua given twice", b->file, lineno); b->ua = strdupt(value); if (b->ua == NULL) BUNDLE_FAIL("%s: out of memory", b->file); - } else if (klen == 6 && strncmp(line, "header", klen) == 0) { - const char *const why = - http_bundle_header_refusal(StringBuff(b->headers), value); - /* the value can be a secret, so only the name is shown */ - const size_t name = strcspn(value, ":"); - const int shown = value[name] == ':' && name <= 64 ? (int) name : 0; - + } else if (key_is(line, klen, "header")) { + why = http_bundle_header_refusal(StringBuff(b->headers), value); if (why != NULL) - BUNDLE_FAIL("%s, line %d: refused header%s%.*s: %s", b->file, lineno, - shown != 0 ? " " : "", shown, value, why); + BUNDLE_FAIL("%s, line %d: refused header: %s", b->file, lineno, why); + if (strlen(value) + 2 > BUNDLE_HEADERS_MAX - StringLength(b->headers)) + BUNDLE_FAIL("%s, line %d: the header lines pass %d bytes", b->file, + lineno, BUNDLE_HEADERS_MAX); StringCat(b->headers, value); StringCat(b->headers, "\r\n"); - } else if (klen == 5 && strncmp(line, "scope", klen) == 0) { + } else { *scope_given = HTS_TRUE; if (!bundle_scope_line(b, value, lineno, err, err_size)) return HTS_FALSE; - } else { - hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: unknown key %.*s", - b->file, lineno, (int) klen, line); } } return HTS_TRUE; } -/* Is the jar file newer than b's file, so its records win? */ -static hts_boolean bundle_under(const bundle *b, const char *jar) { +/* Is jar_file newer than b's file, so its records win? */ +static hts_boolean bundle_jar_wins(const bundle *b, const char *jar_file) { hts_filetime_t jt, bt; - return jar != NULL && hts_file_mtime(jar, &jt) && + return jar_file != NULL && hts_file_mtime(jar_file, &jt) && hts_file_mtime(b->file, &bt) && (jt.sec > bt.sec || (jt.sec == bt.sec && jt.nsec > bt.nsec)); } -/* Read b's file, set its scope, and load its cookies through others. */ -static hts_boolean bundle_load(httrackp *opt, bundle *b, const char *jar, - t_cookie *others, char *err, size_t err_size) { +/* What bundle_admit needs about the bundle being read. */ +typedef struct bundle_jar { + const bundle *b; + t_cookie *all_bundles; /* every bundle's records so far */ + hts_boolean jar_wins; /* A record the jar holds already wins. */ + int dropped; /* records outside the scope */ +} bundle_jar; + +/* The cookie_filter admit callback: keep a record in the bundle's scope, and + stop on one another bundle gives another value. */ +static hts_boolean bundle_admit(cookie_filter *filter, t_cookie *cookie, + hts_boolean repeat, const char *name, + const char *value, const char *domain, + const char *path) { + bundle_jar *const ctx = (bundle_jar *) filter->ctx; + char buffer[8192]; + char host[256]; + const char *other; + + if (!cookie_host(domain, host, sizeof(host)) || + !bundle_in_scope(ctx->b, host[0] == '.' ? host + 1 : host)) { + ctx->dropped++; + return HTS_FALSE; + } + other = cookie_find_exact(ctx->all_bundles->data, name, host, path); + if (other != NULL && !repeat && + strcmp(cookie_get(buffer, other, 6), value) != 0) { + filter->why = "a cookie has another value in another bundle"; + return HTS_FALSE; + } + if (cookie_set(ctx->all_bundles, name, value, domain, path, NULL) != 0) { + filter->why = "too many cookies to compare with the other bundles"; + return HTS_FALSE; + } + return !ctx->jar_wins || + cookie_find_exact(cookie->data, name, host, path) == NULL; +} + +/* Read b's file, set its scope, and load its cookies, comparing them with + all_bundles. */ +static hts_boolean bundle_load(httrackp *opt, bundle *b, const char *jar_file, + t_cookie *all_bundles, char *err, + size_t err_size) { const char *const dir = StringNotEmpty(opt->bundle_dir) ? StringBuff(opt->bundle_dir) : StringBuff(opt->path_log); const size_t dlen = strlen(dir); - hts_boolean scope_given, has_jar; + hts_boolean scope_given; + int cookies_line; + bundle_jar ctx; cookie_filter filter; + const char *why = NULL; size_t i; FILE *fp; @@ -426,44 +522,40 @@ static hts_boolean bundle_load(httrackp *opt, bundle *b, const char *jar, if (fp == NULL) BUNDLE_FAIL("%s: cannot read the login bundle: %s", b->file, strerror(errno)); - if (!bundle_parse(opt, b, fp, &scope_given, &has_jar, err, err_size)) { + if (!bundle_parse(opt, b, fp, &scope_given, &cookies_line, err, err_size)) { fclose(fp); return HTS_FALSE; } - b->https_only = HTS_TRUE; - for (i = 0; i < b->nstart; i++) - b->https_only = b->https_only && strfield(b->start_urls[i], "https:"); - for (i = 0; !scope_given && i < b->nstart + b->nurls; i++) { - if (!bundle_scope_url(b, i < b->nstart ? b->start_urls[i] - : b->urls[i - b->nstart])) { - fclose(fp); - BUNDLE_FAIL("%s: cannot work out the bundle's scope", b->file); - } + for (i = 0; i < b->nstart && why == NULL; i++) + why = bundle_note_url(b, b->start_urls[i], scope_given); + for (i = 0; i < b->nurls && why == NULL; i++) + why = bundle_note_url(b, b->urls[i], scope_given); + if (why != NULL) { + fclose(fp); + BUNDLE_FAIL("%s: %s", b->file, why); } + memset(&ctx, 0, sizeof(ctx)); + ctx.b = b; + ctx.all_bundles = all_bundles; + ctx.jar_wins = bundle_jar_wins(b, jar_file); memset(&filter, 0, sizeof(filter)); - filter.covers = bundle_in_scope; - filter.scope = b; - filter.others = others; - filter.under = bundle_under(b, jar); - if (has_jar && opt->cookie != NULL) + filter.admit = bundle_admit; + filter.ctx = &ctx; + filter.line = cookies_line; + if (cookies_line != 0 && opt->cookie != NULL) cookie_load_stream(opt, opt->cookie, fp, b->file, &filter); - else if (has_jar) + else if (cookies_line != 0) hts_log_print(opt, LOG_WARNING, "Cookies are disabled, so the cookies of %s are not sent", b->file); fclose(fp); - if (filter.dropped != 0) + if (ctx.dropped != 0) hts_log_print(opt, LOG_WARNING, "Bundle %s: %d cookie(s) outside its scope left out; a " "scope line keeps them", - b->file, filter.dropped); - if (filter.clash[0] != '\0') - BUNDLE_FAIL("%s: cookie '%s' has another value in another bundle", b->file, - filter.clash); - if (filter.lost != 0) - BUNDLE_FAIL("%s: the cookie jar is full, or the file could not be read to " - "its end", - b->file); + b->file, ctx.dropped); + if (filter.why != NULL) + BUNDLE_FAIL("%s, line %d: %s", b->file, filter.line, filter.why); return HTS_TRUE; } @@ -489,21 +581,43 @@ static hts_boolean bundle_same_identity(const bundle *a, const bundle *b) { strcmp(StringBuff(a->headers), StringBuff(b->headers)) == 0; } -hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar, char *err, +/* Bundles that overlap have merged, so an https URL in any of them makes the + headers of all of them https-only. */ +static void bundle_merge_https(hts_bundle_state *st) { + hts_boolean changed = HTS_TRUE; + size_t i, j; + + while (changed) { + changed = HTS_FALSE; + for (i = 0; i < st->count; i++) { + for (j = 0; j < st->count; j++) { + bundle *const a = &st->items[i]; + const bundle *const b = &st->items[j]; + + if (!a->https_only && b->https_only && bundle_scopes_overlap(a, b)) { + a->https_only = HTS_TRUE; + changed = HTS_TRUE; + } + } + } + } +} + +hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar_file, char *err, size_t err_size) { hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state; - t_cookie *others; + t_cookie *all_bundles; hts_boolean ok = HTS_TRUE; size_t i, j; if (st == NULL) return HTS_TRUE; - others = calloct(1, sizeof(t_cookie)); - if (others == NULL) + all_bundles = calloct(1, sizeof(t_cookie)); + if (all_bundles == NULL) BUNDLE_FAIL("login bundles: out of memory"); - others->max_len = sizeof(others->data); + all_bundles->max_len = sizeof(all_bundles->data); for (i = 0; i < st->count && ok; i++) - ok = bundle_load(opt, &st->items[i], jar, others, err, err_size); + ok = bundle_load(opt, &st->items[i], jar_file, all_bundles, err, err_size); for (i = 0; i < st->count && ok; i++) { for (j = i + 1; j < st->count && ok; j++) { const bundle *const a = &st->items[i]; @@ -518,7 +632,9 @@ hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar, char *err, } } } - freet(others); + if (ok) + bundle_merge_https(st); + freet(all_bundles); return ok; } @@ -547,7 +663,7 @@ void hts_bundle_identity(httrackp *opt, const char *adr, htsrequest *req) { if (bundle_in_scope(b, host)) { if (b->ua != NULL) { req->user_agent = b->ua; - req->user_agent_send = 1; + req->user_agent_send = HTS_TRUE; } if (StringNotEmpty(b->headers) && (!b->https_only || strfield(adr, "https://"))) @@ -556,28 +672,3 @@ void hts_bundle_identity(httrackp *opt, const char *adr, htsrequest *req) { } } } - -void hts_bundle_dump(httrackp *opt) { - const hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state; - size_t i, j; - - for (i = 0; st != NULL && i < st->count; i++) { - const bundle *const b = &st->items[i]; - const char *p; - - printf("bundle %s\n", b->name); - if (b->ua != NULL) - printf("ua %s\n", b->ua); - for (j = 0; j < b->nurls; j++) - printf("url %s\n", b->urls[j]); - for (j = 0; j < b->nscope; j++) - printf("scope %s\n", b->scope[j]); - for (p = StringBuff(b->headers); *p != '\0';) { - const size_t len = strcspn(p, "\r\n"); - - printf("header %.*s\n", (int) len, p); - p += len; - p += strspn(p, "\r\n"); - } - } -} diff --git a/src/htsbundle.h b/src/htsbundle.h index c10f7cd4c..c1c9a940b 100644 --- a/src/htsbundle.h +++ b/src/htsbundle.h @@ -47,66 +47,65 @@ Please visit our Website: http://www.httrack.com /* A bundle name holds fewer bytes than this. */ #define HTS_BUNDLE_NAME_MAX 128 -typedef enum { - HTS_BUNDLE_NONE, /* no suffix */ - HTS_BUNDLE_NAMED, /* a valid name, cut off the URL */ - HTS_BUNDLE_BAD /* a suffix whose name is refused, URL left alone */ -} hts_bundle_suffix; +/* Cut the bundle suffix off url in place and copy its name into name, or "" + when url has none. NULL, or why the name is refused, url then left alone. */ +const char *hts_bundle_cut(char *url, char *name, size_t name_size); -/* Cut the bundle suffix off url in place and copy its name into name. */ -hts_bundle_suffix hts_bundle_cut(char *url, char *name, size_t name_size); - -/* Record that the start URL url (suffix already cut) names bundle name. A - name given twice is one bundle. HTS_FALSE when out of memory. */ -hts_boolean hts_bundle_add(httrackp *opt, const char *name, const char *url); +/* hts_bundle_cut(), then record that url names the bundle. A name given twice + is one bundle. NULL, or why the run must stop. */ +const char *hts_bundle_take(httrackp *opt, char *url); /* Read every named bundle from opt->bundle_dir, or the project folder, and refuse two bundles that overlap with different identities or cookie values. - The cookies go into opt->cookie, under the records already there when the - jar file is newer than the bundle. jar is that file, or NULL. HTS_FALSE - after writing why into err, naming the file: the run must stop. A second - call reads the files again. */ -hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar, char *err, + The cookies go into opt->cookie, under the records already there when + jar_file (or NULL) is newer than the bundle. HTS_FALSE after writing why + into err, naming the file: the run must stop. A second call reads the files + again. */ +hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar_file, char *err, size_t err_size); /* The i-th url line of the loaded bundles, or NULL past the last one. */ const char *hts_bundle_url(httrackp *opt, size_t i); /* Give req the UA and headers of the first bundle whose scope holds adr's - host, where it has them. Its headers go over https alone when every URL - naming it is https. */ + host, where it has them. Its headers go over https alone when any URL of + the bundles merged with it is https. */ void hts_bundle_identity(httrackp *opt, const char *adr, htsrequest *req); /* Forget the bundles named for opt. */ void hts_bundle_free(httrackp *opt); -/* Print each loaded bundle (UA, url lines, scope, headers) for the - self-tests. */ -void hts_bundle_dump(httrackp *opt); +/* Do the len bytes at s hold a control byte other than TAB? */ +hts_boolean hts_bundle_has_control(const char *s, size_t len); -/* htsbauth.c internals the bundles share. */ +/* The bundles share these htsbauth.c internals. */ /* Copy the host of url, with or without a scheme, into dst as cookie_host - does, and set *https. */ -hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size, - hts_boolean *https); + does. */ +hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size); + +/* The record of s named cook_name for exactly domain and path, or NULL. */ +char *cookie_find_exact(char *s, const char *cook_name, const char *domain, + const char *path); -/* What cookie_load_stream lets into the jar, and the count of what it keeps - out. */ +/* This filter picks what cookie_load_stream lets into the jar. */ typedef struct cookie_filter { - hts_boolean (*covers)(const void *scope, const char *host); - const void *scope; - t_cookie *others; /* every bundle's records so far */ - hts_boolean under; /* a record the jar holds already wins */ - int dropped; /* records outside the scope */ - char clash[64]; /* a name others holds with another value, or "" */ - int lost; /* records the jar could not take */ + /* Should the record go into cookie? repeat says this load had it already. + Setting why stops the load. */ + hts_boolean (*admit)(struct cookie_filter *filter, t_cookie *cookie, + hts_boolean repeat, const char *name, const char *value, + const char *domain, const char *path); + void *ctx; + int line; /* the number of the last line read */ + const char *why; /* NULL, or why the load stopped at line */ } cookie_filter; /* Load Netscape jar records from fp's position to its end through filter, - as cookie_load reads a file. name is the file name used in warnings. */ + as cookie_load reads a file. given names the file in warnings, which name + the line and none of its bytes. A line with a control byte other than TAB + stops the load. */ void cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, - const char *name, cookie_filter *filter); + const char *given, cookie_filter *filter); #endif diff --git a/src/htsbundle_selftest.c b/src/htsbundle_selftest.c index 3c3954eb7..b43f4eb92 100644 --- a/src/htsbundle_selftest.c +++ b/src/htsbundle_selftest.c @@ -38,6 +38,7 @@ Please visit our Website: http://www.httrack.com static int st_bundlecut(httrackp *opt, int argc, char **argv) { char url[HTS_URLMAXSIZE * 2]; char name[HTS_BUNDLE_NAME_MAX]; + const char *why; (void) opt; if (argc != 1 || strlen(argv[0]) >= sizeof(url)) { @@ -45,35 +46,48 @@ static int st_bundlecut(httrackp *opt, int argc, char **argv) { return 1; } strcpybuff(url, argv[0]); - switch (hts_bundle_cut(url, name, sizeof(name))) { - case HTS_BUNDLE_NONE: + why = hts_bundle_cut(url, name, sizeof(name)); + if (why != NULL) + printf("bad: %s\n", why); + else if (name[0] == '\0') printf("none\n"); - break; - case HTS_BUNDLE_BAD: - printf("bad\n"); - break; - case HTS_BUNDLE_NAMED: + else printf("%s %s\n", name, url); - break; - } return 0; } -/* -#test=bundle

... [host:]... [jar:]: load the bundles - the URLs name from dir over the jar file, print them and the jar, then the - UA and headers each adr gets. */ +/* Print err with each "/" cut to "/", since the folder is the caller's. */ +static void print_without_dir(const char *err, const char *dir) { + const size_t dlen = strlen(dir); + const char *p = err; + const char *at; + + while ((at = strstr(p, dir)) != NULL) { + const hts_boolean whole = at[dlen] == '/' || at[dlen] == '\\'; + + printf("%.*s", (int) (at - p) + (whole ? 0 : (int) dlen), p); + p = at + dlen; + } + printf("%s\n", p); +} + +/* -#test=bundle ... [host:]... [jar:] [twice]: load the + bundles the URLs name from dir over the jar file, twice when asked, then + print their url lines, the jar, and the UA and headers each adr gets. */ static int st_bundle(httrackp *opt, int argc, char **argv) { FILE *const log = opt->log; t_cookie *const prev = opt->cookie; t_cookie *jar = calloct(1, sizeof(t_cookie)); const char *jar_file = NULL; + int loads = 1; char err[1024]; + const char *url; char *rec; int i; if (argc < 2 || jar == NULL) { fprintf(stderr, "bundle: takes ... [host:]... " - "[jar:]\n"); + "[jar:] [twice]\n"); freet(jar); return 1; } @@ -83,34 +97,28 @@ static int st_bundle(httrackp *opt, int argc, char **argv) { hts_bundle_free(opt); StringCopy(opt->bundle_dir, argv[0]); for (i = 1; i < argc; i++) { - char url[HTS_URLMAXSIZE * 2]; - char name[HTS_BUNDLE_NAME_MAX]; + char buff[HTS_URLMAXSIZE * 2]; if (strncmp(argv[i], "jar:", 4) == 0) { jar_file = argv[i] + 4; (void) cookie_load(opt, jar, "", jar_file); + } else if (strcmp(argv[i], "twice") == 0) { + loads = 2; } else if (strncmp(argv[i], "host:", 5) != 0 && - strlen(argv[i]) < sizeof(url)) { - strcpybuff(url, argv[i]); - if (hts_bundle_cut(url, name, sizeof(name)) == HTS_BUNDLE_NAMED) - assertf(hts_bundle_add(opt, name, url)); + strlen(argv[i]) < sizeof(buff)) { + strcpybuff(buff, argv[i]); + assertf(hts_bundle_take(opt, buff) == NULL); } } - if (!hts_bundle_load_all(opt, jar_file, err, sizeof(err))) { - /* the folder is the caller's, so the message is graded without it */ - const size_t dlen = strlen(argv[0]); - const char *p = err; - const char *at; - - printf("refused: "); - while ((at = strstr(p, argv[0])) != NULL) { - printf("%.*s", (int) (at - p), p); - p = at + dlen; + for (i = 0; i < loads; i++) { + if (!hts_bundle_load_all(opt, jar_file, err, sizeof(err))) { + printf("refused: "); + print_without_dir(err, argv[0]); + goto done; } - printf("%s\n", p); - goto done; } - hts_bundle_dump(opt); + for (i = 0; (url = hts_bundle_url(opt, i)) != NULL; i++) + printf("url %s\n", url); for (rec = jar->data; *rec != '\0'; rec = cookie_nextfield(rec)) { char dom[8192], cname[8192], value[8192]; @@ -145,7 +153,7 @@ static int st_bundle(httrackp *opt, int argc, char **argv) { const struct selftest_entry selftests_bundle[] = { {"bundlecut", "", "the login bundle a start URL names", st_bundlecut}, - {"bundle", " ... [host:]... [jar:]", + {"bundle", " ... [host:]... [jar:] [twice]", "load login bundles and print the identity each host gets", st_bundle}, {NULL, NULL, NULL, NULL}, }; diff --git a/src/htscore.c b/src/htscore.c index b81561dbc..95910c3f2 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -1095,7 +1095,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { int n = 0; int lineno = 0; char BIGSTK line[HTS_URLMAXSIZE * 2]; - char name[HTS_BUNDLE_NAME_MAX]; + const char *bundle_why; while(filelist_ptr < filelist_sz) { int adv; @@ -1136,23 +1136,14 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { StringBuff(opt->filelist), lineno, line); continue; } - switch (hts_bundle_cut(line, name, sizeof(name))) { - case HTS_BUNDLE_NONE: - break; - case HTS_BUNDLE_BAD: - hts_log_print(opt, LOG_ERROR, - "\"%s\", line %d: invalid login bundle name", - StringBuff(opt->filelist), lineno); - fprintf(stderr, "\"%s\", line %d: invalid login bundle name\n", - StringBuff(opt->filelist), lineno); + bundle_why = hts_bundle_take(opt, line); + if (bundle_why != NULL) { + hts_log_print(opt, LOG_ERROR, "\"%s\", line %d: %s", + StringBuff(opt->filelist), lineno, bundle_why); + fprintf(stderr, "\"%s\", line %d: %s\n", StringBuff(opt->filelist), + lineno, bundle_why); bundle_refused = HTS_TRUE; continue; - case HTS_BUNDLE_NAMED: - if (!hts_bundle_add(opt, name, line)) { - hts_log_print(opt, LOG_ERROR, "Login bundle: out of memory"); - bundle_refused = HTS_TRUE; - } - break; } if (line[0]) { n++; @@ -1190,16 +1181,18 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { { char err[1024]; char jarbuff[CATBUFF_SIZE]; - const char *jar = !opt->accept_cookie ? NULL - : strnotempty(StringBuff(opt->cookies_file)) - ? StringBuff(opt->cookies_file) - : fconcat(jarbuff, sizeof(jarbuff), - StringBuff(opt->path_log), "cookies.txt"); + const char *jar_file = + !opt->accept_cookie ? NULL + : strnotempty(StringBuff(opt->cookies_file)) + ? StringBuff(opt->cookies_file) + : fconcat(jarbuff, sizeof(jarbuff), StringBuff(opt->path_log), + "cookies.txt"); const char *extra; size_t room = 0; size_t i; - if (!bundle_refused && !hts_bundle_load_all(opt, jar, err, sizeof(err))) { + if (!bundle_refused && + !hts_bundle_load_all(opt, jar_file, err, sizeof(err))) { hts_log_print(opt, LOG_ERROR, "Login bundle: %s", err); fprintf(stderr, "Login bundle: %s\n", err); bundle_refused = HTS_TRUE; @@ -4839,9 +4832,6 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) { if (StringNotEmpty(from->accept_kinds)) StringCopyS(to->accept_kinds, from->accept_kinds); - if (StringNotEmpty(from->bundle_dir)) - StringCopyS(to->bundle_dir, from->bundle_dir); - /* An explicit -%a or -%l travels with its flag, so it stays explicit. */ if (from->accept_given) { StringCopyS(to->accept, from->accept); diff --git a/src/htscoremain.c b/src/htscoremain.c index 94062f39f..f520426fe 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -466,37 +466,26 @@ static hts_boolean optparam_dash_ok(const cmdl_argv *cmd, int na) { return na + 1 < cmd->argc && cmd->param[na + 1]; } -/* Read the --bundle-dir at NA into opt, made absolute in cmd too, so doit.log - finds the folder from anywhere. HTS_FALSE after the panic message. */ +/* Store the --bundle-dir at na in opt and cmd as an absolute path, so doit.log + finds it from anywhere. HTS_FALSE after a panic. */ static hts_boolean bundle_dir_set(httrackp *opt, cmdl_argv *cmd, int na) { - const char *dir = na + 1 < cmd->argc ? cmd->argv[na + 1] : NULL; - char cwd[HTS_URLMAXSIZE]; - String abs = STRING_EMPTY; + const char *const dir = na + 1 < cmd->argc ? cmd->argv[na + 1] : NULL; + const hts_boolean relative = dir != NULL && dir[0] != '/' && dir[0] != '\\' && + !(dir[0] != '\0' && dir[1] == ':'); + char cwd[HTS_URLMAXSIZE] = ""; + char abs[HTS_URLMAXSIZE]; char *copy; - if (dir == NULL || (dir[0] == '-' && !optparam_dash_ok(cmd, na))) { + if (dir == NULL || (dir[0] == '-' && !optparam_dash_ok(cmd, na)) || + (relative && getcwd(cwd, sizeof(cwd)) == NULL) || + !slprintfbuff(abs, sizeof(abs), "%s%s%s", cwd, relative ? "/" : "", + dir) || + (copy = hts_arena_strdup(&cmd->tokens, abs)) == NULL) { HTS_PANIC_PRINTF("Option bundle-dir needs a blank space and a folder"); return HTS_FALSE; } - if (dir[0] != '/' && dir[0] != '\\' && !(dir[0] != '\0' && dir[1] == ':')) { - if (getcwd(cwd, sizeof(cwd)) == NULL) { - HTS_PANIC_PRINTF("Option bundle-dir: cannot read the current folder"); - return HTS_FALSE; - } - StringCat(abs, cwd); - StringCat(abs, "/"); - } - StringCat(abs, dir); - if (StringLength(abs) >= HTS_URLMAXSIZE || - (copy = hts_arena_strdup(&cmd->tokens, StringBuff(abs))) == NULL) { - StringFree(abs); - HTS_PANIC_PRINTF("Option bundle-dir: folder path too long, or out of " - "memory"); - return HTS_FALSE; - } - StringFree(abs); cmd->argv[na + 1] = copy; - StringCopy(opt->bundle_dir, copy); + StringCopy(opt->bundle_dir, abs); return HTS_TRUE; } @@ -603,7 +592,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { int httrack_logmode = 3; // ONE log file ensureUrlCapacity(url, url_sz, 65536); - hts_bundle_free(opt); /* a previous run's bundles are not this one's */ + /* a previous run's bundles are not this one's */ + hts_bundle_free(opt); + StringCopy(opt->bundle_dir, ""); // Create options _DEBUG_HEAD = 0; // pas de debuggage en têtes @@ -2916,25 +2907,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { assertf(urlSize < HTS_URLMAXSIZE); strcpybuff(word, argv[na]); if (argv[na][0] != '+' && argv[na][0] != '-') { - char name[HTS_BUNDLE_NAME_MAX]; + const char *const why = hts_bundle_take(opt, word); - switch (hts_bundle_cut(word, name, sizeof(name))) { - case HTS_BUNDLE_NONE: - break; - case HTS_BUNDLE_BAD: - cmdl_panic_value(opt, - "Invalid login bundle name (letters, digits, " - "'.', '_' and '-' only)", - argv[na]); + if (why != NULL) { + cmdl_panic_value(opt, why, argv[na]); htsmain_free(); return -1; - case HTS_BUNDLE_NAMED: - if (!hts_bundle_add(opt, name, word)) { - HTS_PANIC_PRINTF("Not enough memory"); - htsmain_free(); - return -1; - } - break; } } if (urlSize < HTS_URLMAXSIZE) { diff --git a/src/htslib.c b/src/htslib.c index 209d89b0b..90301febf 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -1157,6 +1157,7 @@ const char *http_custom_header_warning(const char *headers, const char *line) { return NULL; } +/* The -%X warnings, plus Host, Cookie and Proxy-*. */ const char *http_bundle_header_refusal(const char *headers, const char *line) { static const char *const engine[] = {"Host", "Cookie"}; const char *const why = http_custom_header_warning(headers, line); diff --git a/src/htslib.h b/src/htslib.h index a3caba3df..eef12713b 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -328,9 +328,8 @@ hts_boolean hts_body_missing_unexpectedly(const htsblk *r); typed. headers is the block so far. */ const char *http_custom_header_warning(const char *headers, const char *line); -/* Why a login bundle's header line is refused, or NULL when it is allowed: - the -%X warnings, plus Host, Cookie and Proxy-*. headers is the block so - far. */ +/* Why a bundle header line is refused, or NULL when allowed. headers is the + block so far. */ const char *http_bundle_header_refusal(const char *headers, const char *line); /* Append the custom header block to dst (capacity dst_size), dropping any line diff --git a/src/htslines.c b/src/htslines.c index 6ca4f175d..4305746eb 100644 --- a/src/htslines.c +++ b/src/htslines.c @@ -42,12 +42,10 @@ Please visit our Website: http://www.httrack.com /* Is c one of the bytes flags drop? */ static hts_boolean line_drops(int c, int flags) { - return (flags & LINE_RAW) != 0 ? HTS_FALSE - : c == '\r' || ((flags & HTS_LINE_DROP_TAB) != 0 && c == '\t') || - ((flags & HTS_LINE_DROP_FF) != 0 && c == '\f') || - ((flags & HTS_LINE_DROP_NUL) != 0 && c == '\0') - ? HTS_TRUE - : HTS_FALSE; + return (flags & LINE_RAW) == 0 && + (c == '\r' || ((flags & HTS_LINE_DROP_TAB) != 0 && c == '\t') || + ((flags & HTS_LINE_DROP_FF) != 0 && c == '\f') || + ((flags & HTS_LINE_DROP_NUL) != 0 && c == '\0')); } /* How a line ends once trimmed, which decides whether it continues. */ @@ -91,7 +89,7 @@ static hts_boolean readline_end(FILE *fp, char *s, size_t max, int flags, } if (j + 1 < max) s[j++] = (char) c; - else if (!is_realspace(c)) + else if ((flags & LINE_RAW) != 0 || !is_realspace(c)) cut = HTS_TRUE; } s[j] = '\0'; diff --git a/src/htslines.h b/src/htslines.h index 99d9832d3..8bb2edef6 100644 --- a/src/htslines.h +++ b/src/htslines.h @@ -50,7 +50,8 @@ Please visit our Website: http://www.httrack.com when a non-space byte was dropped. At EOF it reads an empty line. */ hts_boolean hts_readline(FILE *fp, char *s, size_t max, int flags); -/* hts_readline() keeping CR and NUL, with the bytes stored in *len. */ +/* hts_readline(fp, s, max, 0) keeping CR and NUL, and storing the byte count + in len. HTS_TRUE when the line was cut. */ hts_boolean hts_readline_raw(FILE *fp, char *s, size_t max, size_t *len); /* hts_readline() trimmed and joined after a trailing backslash. Leading spaces diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index c3f28a531..6eedcacd2 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -28,30 +28,37 @@ selftest_queue 'login1 https://example.com/members/' bundlecut \ selftest_queue 'a.b_c-9 http://x/' bundlecut 'http://x/#httrack-bundle=a.b_c-9' selftest_queue "$name127 http://x/" bundlecut "http://x/#httrack-bundle=$name127" selftest_queue 'x http://x/#a' bundlecut 'http://x/#a#httrack-bundle=x' -selftest_queue 'console http://x/' bundlecut 'http://x/#httrack-bundle=console' -selftest_queue 'com10 http://x/' bundlecut 'http://x/#httrack-bundle=com10' +for ok in console com10 com0 lpt10 nul1; do + selftest_queue "$ok http://x/" bundlecut "http://x/#httrack-bundle=$ok" +done selftest_queue none bundlecut 'http://x/#top' -for bad in '' ../x a..b .hidden a/b 'a\b' a%2fb "${name127}a" CON con.test \ - Nul.x com1 LPT9.hts 'a#httrack-bundle=b'; do - selftest_queue bad bundlecut "http://x/#httrack-bundle=$bad" +badname='bad: invalid login bundle name: ' +for bad in '' ../x a/b 'a\b' a%2fb 'a#httrack-bundle=b'; do + selftest_queue "${badname}use letters, digits, '.', '_' and '-'" \ + bundlecut "http://x/#httrack-bundle=$bad" +done +for bad in ..x a..b .hidden; do + selftest_queue "${badname}it starts with '.' or holds \"..\"" \ + bundlecut "http://x/#httrack-bundle=$bad" +done +selftest_queue "${badname}128 bytes or more" bundlecut \ + "http://x/#httrack-bundle=${name127}a" +for bad in CON con.test Nul.x prn aux com1 COM9 lpt1 LPT9.hts; do + selftest_queue "${badname}a Windows device name" bundlecut \ + "http://x/#httrack-bundle=$bad" done # The default scope is the registrable domains of the naming URL and the url # lines. Cookie domains do not widen it: alpha-cdn.test's cookie is dropped. # The naming URL is https, so the headers go over https alone. bundle alpha '# HTTrack bundle 1' 'url https://login.alpha-sso.test/start' \ - 'ua AlphaUA/1.0' 'header Authorization: Bearer t' 'header X-A: 1' \ - 'unknown-key is warned about' cookies "$(jar alpha.test SID s1)" \ - "$(jar alpha-cdn.test C c1)" "$(jar alpha.test PREF p1)" \ - "$(jar alpha.test SID s2)" '# an unknown comment is warned about' + 'ua AlphaUA/1.0' 'header Authorization: Bearer t' '# a note' \ + 'header X-A: 1' 'unknown-key is warned about' 'nakedkey' cookies \ + "$(jar alpha.test SID s1)" "$(jar alpha-cdn.test C c1)" \ + "$(jar alpha.test PREF p1)" "$(jar alpha.test SID s2)" \ + '# an unknown comment is warned about' alpha_auth='ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1|' -selftest_queue_lines "bundle alpha -ua AlphaUA/1.0 -url https://login.alpha-sso.test/start -scope alpha.test -scope alpha-sso.test -header Authorization: Bearer t -header X-A: 1 +selftest_queue_lines "url https://login.alpha-sso.test/start cookie .alpha.test PREF=p1 cookie .alpha.test SID=s2 https://alpha.test $alpha_auth @@ -59,24 +66,60 @@ https://deep.www.alpha.test:8443 $alpha_auth http://www.alpha.test ua=AlphaUA/1.0 headers=global https://login.alpha-sso.test $alpha_auth https://cdn.gamma.test ua=global headers=global -https://notalpha.test ua=global headers=global" bundle "$dir" \ +https://notalpha.test ua=global headers=global +https://alpha-cdn.test ua=global headers=global" bundle "$dir" \ 'https://www.alpha.test/#httrack-bundle=alpha' host:https://alpha.test \ host:https://deep.www.alpha.test:8443 host:http://www.alpha.test \ host:https://login.alpha-sso.test host:https://cdn.gamma.test \ - host:https://notalpha.test + host:https://notalpha.test host:https://alpha-cdn.test + +# A second load on one opt reads the files again rather than adding to them. +selftest_queue_lines "url https://login.alpha-sso.test/start +cookie .alpha.test PREF=p1 +cookie .alpha.test SID=s2 +https://alpha.test $alpha_auth" bundle "$dir" \ + 'https://www.alpha.test/#httrack-bundle=alpha' host:https://alpha.test \ + twice + +# Any https URL of the bundle, naming it or a url line, keeps its headers off +# plain http. +bundle mixed '# HTTrack bundle 1' 'url http://www.mixed.test/' 'ua M' \ + 'header X-M: 1' +for urls in 'https://mixed.test/ http://mixed.test/' \ + 'http://mixed.test/ https://mixed.test/'; do + read -r u1 u2 <<<"$urls" + selftest_queue_lines "url http://www.mixed.test/ +http://mixed.test ua=M headers=global +http://www.mixed.test ua=M headers=global +https://www.mixed.test ua=M headers=X-M: 1|" bundle "$dir" \ + "$u1#httrack-bundle=mixed" "$u2#httrack-bundle=mixed" \ + host:http://mixed.test host:http://www.mixed.test \ + host:https://www.mixed.test +done +bundle mixed2 '# HTTrack bundle 1' 'url https://www.mixed.test/' 'ua M' \ + 'header X-M: 1' +selftest_queue_tail " +http://mixed.test ua=M headers=global" bundle "$dir" \ + 'http://mixed.test/#httrack-bundle=mixed2' host:http://mixed.test +bundle plainm '# HTTrack bundle 1' 'ua M' 'header X-M: 1' +selftest_queue "http://mixed.test ua=M headers=X-M: 1|" bundle "$dir" \ + 'http://mixed.test/#httrack-bundle=plainm' host:http://mixed.test +# Bundles that merge share the rule, whichever is listed first. +for pair in 'plainm mixed2' 'mixed2 plainm'; do + read -r u1 u2 <<<"$pair" + selftest_queue_tail " +http://mixed.test ua=M headers=global +https://mixed.test ua=M headers=X-M: 1|" bundle "$dir" \ + "http://mixed.test/#httrack-bundle=$u1" \ + "http://www.mixed.test/#httrack-bundle=$u2" host:http://mixed.test \ + host:https://mixed.test +done # The registrable domain follows the suffix list, not the last two labels. A # host with none (a public suffix, a single label) covers itself alone, and so # does an address. bundle uk '# HTTrack bundle 1' 'ua UK' 'header X-UK: 1' -selftest_queue_lines "bundle uk -ua UK -scope alpha.co.uk -scope =s3.amazonaws.com -scope =wiki -scope =127.0.0.1 -header X-UK: 1 -a.alpha.co.uk ua=UK headers=X-UK: 1| +selftest_queue_lines "a.alpha.co.uk ua=UK headers=X-UK: 1| other.co.uk ua=global headers=global s3.amazonaws.com ua=UK headers=X-UK: 1| attacker.s3.amazonaws.com ua=global headers=global @@ -92,18 +135,18 @@ a.wiki ua=global headers=global host:127.0.0.2 # scope lines replace the default, and a cookie outside them is dropped. CRLF -# line ends and a BOM are read. -printf '\357\273\277# HTTrack bundle 1.3\r\nscope WWW.alpha.test\r\nua N\r\ncookies\r\n%s\r\n%s\r\n' \ +# line ends and a BOM are read. TAB is the one control byte a line may hold. +printf '\357\273\277# HTTrack bundle 1.3\r\nscope WWW.alpha.test\r\nscope .beta.test\r\nscope 127.0.0.1\r\nua\tN\tx\r\nheader X-N:\t1\r\ncookies\r\n%s\r\n%s\r\n' \ "$(jar alpha.test SID wide)" "$(jar www.alpha.test K narrow)" \ >"$tmpdir/narrow.hts" -selftest_queue_lines "bundle narrow -ua N -scope www.alpha.test -cookie .www.alpha.test K=narrow +selftest_queue_lines "cookie .www.alpha.test K=narrow alpha.test ua=global headers=global -a.www.alpha.test ua=N headers=global" bundle "$dir" \ +a.www.alpha.test ua=N${tab}x headers=X-N:${tab}1| +a.beta.test ua=N${tab}x headers=X-N:${tab}1| +127.0.0.1 ua=N${tab}x headers=X-N:${tab}1| +127.0.0.2 ua=global headers=global" bundle "$dir" \ 'http://www.alpha.test/#httrack-bundle=narrow' host:alpha.test \ - host:a.www.alpha.test + host:a.www.alpha.test host:a.beta.test host:127.0.0.1 host:127.0.0.2 # Of the bundle and the jar, the newer file wins, and the bundle wins a tie. order() { # order NAME BUNDLE-DATE JAR-DATE WINNER @@ -111,16 +154,20 @@ order() { # order NAME BUNDLE-DATE JAR-DATE WINNER jar order.test SID jar >"$tmpdir/$1.txt" touch -d "$2" "$tmpdir/$1.hts" touch -d "$3" "$tmpdir/$1.txt" - selftest_queue_tail " -cookie .order.test SID=$4" bundle "$dir" \ + selftest_queue "cookie .order.test SID=$4" bundle "$dir" \ "http://www.order.test/#httrack-bundle=$1" "jar:$dir/$1.txt" } order order1 '2026-01-02 10:00:00' '2026-01-01 10:00:00' bundle order order2 '2026-01-01 10:00:00' '2026-01-02 10:00:00' jar order order3 '2026-01-01 10:00:00' '2026-01-01 10:00:00' bundle +# half a second apart in the same second: a seconds-only compare calls it a tie +if touch -d '2026-01-01 10:00:00.5' "$tmpdir/subsecond" 2>/dev/null && + test "$tmpdir/subsecond" -nt "$tmpdir/order3.txt"; then + order order4 '2026-01-01 10:00:00' '2026-01-01 10:00:00.5' jar +fi -# Refused: each one stops the run and names the file. The batch reads them all -# at the end, so each case writes its own. +# Refused: each one stops the run and names the file and line, never a byte of +# the line. The batch reads them all at the end, so each case writes its own. nbad=0 refused() { # refused WHY LINE... -- a one-bundle file the engine refuses local why=$1 @@ -143,34 +190,56 @@ selftest_queue "refused: /empty.hts$notbundle" bundle "$dir" \ printf '\357\273\277' >"$tmpdir/bom.hts" selftest_queue "refused: /bom.hts$notbundle" bundle "$dir" \ 'http://www.alpha.test/#httrack-bundle=bom' -# The messages name a refused header, never its value, which can be a secret. -refused ', line 2: refused header Host: Host and Cookie are the engine'"'"'s' \ +refused ", line 2: refused header: Host and Cookie are the engine's" \ '# HTTrack bundle 1' 'header Host: x' -refused ', line 2: refused header cookie: Host and Cookie are the engine'"'"'s' \ +refused ", line 2: refused header: Host and Cookie are the engine's" \ '# HTTrack bundle 1' 'header cookie: secret-cookie' -refused ', line 2: refused header Proxy-Authorization: a proxy header is not for the site' \ +refused ', line 2: refused header: a proxy header is not for the site' \ '# HTTrack bundle 1' 'header Proxy-Authorization: secret-proxy' -refused ', line 2: refused header Transfer-Encoding: framing header ignored' \ +refused ', line 2: refused header: framing header ignored' \ '# HTTrack bundle 1' 'header Transfer-Encoding: chunked' -refused ', line 3: refused header Authorization: header replaces an earlier one' \ +refused ', line 3: refused header: header replaces an earlier one' \ '# HTTrack bundle 1' 'header Authorization: a' 'header Authorization: secret-b' refused ', line 2: refused header: malformed header ignored' \ '# HTTrack bundle 1' 'header Bearer secret-no-colon' -# A CR, NUL or other control byte would split a header on the wire. -ctl=', line 2: too long or holding a control byte' +refused ', line 3: the header lines pass 8192 bytes' '# HTTrack bundle 1' \ + "header X-A: $(repeat_chars 5000 a)" "header X-B: $(repeat_chars 5000 b)" +# A control byte would split a header on the wire. The CR before the LF goes. +ctl=', line 2: holds a control byte' refused "$ctl" '# HTTrack bundle 1' "$(printf 'ua x\rHost: evil.test')" refused "$ctl" '# HTTrack bundle 1' \ "$(printf 'header X-A: 1\rCookie: forged=1\rProxy-Authorization: z')" refused "$ctl" '# HTTrack bundle 1' "$(printf 'scope a.test\001')" -refused "$ctl" '# HTTrack bundle 1' "ua $(repeat_chars 8200 u)" +refused "$ctl" '# HTTrack bundle 1' "$(printf 'ua a\177b')" +refused ', line 1: holds a control byte' "$(printf '# HTTrack bundle 1\rua x')" nbad=$((nbad + 1)) printf '# HTTrack bundle 1\nua a\000Host: evil.test\n' >"$tmpdir/bad$nbad.hts" selftest_queue "refused: /bad$nbad.hts$ctl" bundle "$dir" \ "http://www.alpha.test/#httrack-bundle=bad$nbad" +# A line holds 8191 bytes at most, its line end aside. +bundle long '# HTTrack bundle 1' "ua $(repeat_chars 8188 u)" +selftest_queue '' bundle "$dir" 'http://x.test/#httrack-bundle=long' +refused ', line 2: too long' '# HTTrack bundle 1' "ua $(repeat_chars 8189 u)" +nbad=$((nbad + 1)) +printf '# HTTrack bundle 1\nua %s\r\v\f\n' "$(repeat_chars 8188 u)" \ + >"$tmpdir/bad$nbad.hts" +selftest_queue "refused: /bad$nbad.hts, line 2: too long" bundle "$dir" \ + "http://www.alpha.test/#httrack-bundle=bad$nbad" +# In the jar, every control byte but TAB stops the run too. +for oct in 000 001 002 003 004 005 006 007 010 013 014 015 016 017 020 021 \ + 022 023 024 025 026 027 030 031 032 033 034 035 036 037 177; do + nbad=$((nbad + 1)) + { + printf '# HTTrack bundle 1\ncookies\n' + printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\tv%bx\n' "\\0$oct" + } >"$tmpdir/bad$nbad.hts" + selftest_queue "refused: /bad$nbad.hts, line 3: a cookie line holds a control byte" \ + bundle "$dir" "http://www.alpha.test/#httrack-bundle=bad$nbad" +done refused ', line 3: ua given twice' '# HTTrack bundle 1' 'ua A' 'ua B' -refused ', line 2: scope co.uk is a public suffix' '# HTTrack bundle 1' \ +refused ', line 2: the scope is a public suffix' '# HTTrack bundle 1' \ 'scope co.uk' -refused ', line 2: invalid scope: ' '# HTTrack bundle 1' 'scope .' +refused ', line 2: invalid scope' '# HTTrack bundle 1' 'scope .' refused ', line 2: invalid url' '# HTTrack bundle 1' 'url ftp:///' refused ', line 2: invalid url' '# HTTrack bundle 1' \ 'url http://a.test/#httrack-bundle=x' @@ -183,33 +252,46 @@ full=() for i in $(seq 1 400); do full+=("$(jar alpha.test "C$i" "$(repeat_chars 100 v)")") done -refused ': the cookie jar is full, or the file could not be read to its end' \ +refused ', line 227: the cookie jar did not store a cookie' \ '# HTTrack bundle 1' cookies "${full[@]}" +# 256 url lines and 256 scope domains at most +urls=() +scopes=() +for i in $(seq 1 257); do + urls+=("url http://x.test/$i") + scopes+=("scope h$i.test") +done +bundle urls256 '# HTTrack bundle 1' "${urls[@]:0:256}" +selftest_queue_tail ' +url http://x.test/256' bundle "$dir" 'http://x.test/#httrack-bundle=urls256' +refused ', line 258: more than 256 url lines' '# HTTrack bundle 1' "${urls[@]}" +refused ', line 258: more than 256 scope domains' '# HTTrack bundle 1' \ + "${scopes[@]}" # Several bundles: one name twice is one bundle, equal identities on one site # merge with their cookies, and different ones are refused either way round. bundle beta '# HTTrack bundle 1' 'ua BetaUA' cookies "$(jar beta.test SID b1)" bundle alpha2 '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ 'header Authorization: Bearer t' 'header X-A: 1' cookies \ - "$(jar alpha.test OTHER o)" -bundle alphaua '# HTTrack bundle 1' 'ua SomeoneElse' + "$(jar alpha.test OTHER o)" "$(jar alpha.test SID s2)" +bundle alphaua '# HTTrack bundle 1' 'ua SomeoneElse' \ + 'header Authorization: Bearer t' 'header X-A: 1' bundle alphaauth '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ 'header Authorization: Bearer u' 'header X-A: 1' bundle alphasid '# HTTrack bundle 1' 'ua AlphaUA/1.0' \ 'header Authorization: Bearer t' 'header X-A: 1' cookies \ "$(jar alpha.test SID another-account)" bundle wwwalpha '# HTTrack bundle 1' 'scope www.alpha.test' 'ua Narrow' -selftest_queue_lines "bundle beta -ua BetaUA -scope beta.test -cookie .beta.test SID=b1 +bundle loop1 '# HTTrack bundle 1' 'scope 127.0.0.1' 'ua One' +bundle loop2 '# HTTrack bundle 1' 'ua Two' +selftest_queue_lines "cookie .beta.test SID=b1 www.beta.test ua=BetaUA headers=global" bundle "$dir" \ 'http://www.beta.test/#httrack-bundle=beta' \ 'http://beta.test/x#httrack-bundle=beta' host:www.beta.test selftest_queue_tail " cookie .alpha.test PREF=p1 -cookie .alpha.test SID=s2 cookie .alpha.test OTHER=o +cookie .alpha.test SID=s2 cookie .beta.test SID=b1 https://alpha.test $alpha_auth beta.test ua=BetaUA headers=global" bundle "$dir" \ @@ -217,24 +299,61 @@ beta.test ua=BetaUA headers=global" bundle "$dir" \ 'https://alpha.test/#httrack-bundle=alpha2' \ 'http://beta.test/#httrack-bundle=beta' host:https://alpha.test \ host:beta.test -differ() { # differ A B -- A and B overlap with different identities - selftest_queue "refused: /$1.hts and /$2.hts cover the same site with a different user agent or headers" \ +differ() { # differ A B [C] -- A and the last one overlap with other identities + local last=${3:-$2} more=() + test -z "${3:-}" || more=("http://beta.test/#httrack-bundle=$2") + selftest_queue "refused: /$1.hts and /$last.hts cover the same site with a different user agent or headers" \ bundle "$dir" "https://www.alpha.test/#httrack-bundle=$1" \ - "https://alpha.test/#httrack-bundle=$2" + "${more[@]}" "https://alpha.test/#httrack-bundle=$last" } differ alpha alphaua differ alpha alphaauth differ alpha wwwalpha differ wwwalpha alpha -selftest_queue "refused: /alphasid.hts: cookie 'SID' has another value in another bundle" \ +differ alpha beta alphaua +selftest_queue "refused: /loop1.hts and /loop2.hts cover the same site with a different user agent or headers" \ + bundle "$dir" 'http://x.test/#httrack-bundle=loop1' \ + 'http://127.0.0.1/#httrack-bundle=loop2' +selftest_queue "refused: /alphasid.hts, line 6: a cookie has another value in another bundle" \ bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' \ 'https://alpha.test/#httrack-bundle=alphasid' selftest_run_queued +# Warnings name the line, and never a byte of it that could be a secret: a +# key in lowercase letters alone is named. +out=$(httrack -#test=bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' 2>&1) +for want in 'alpha.hts, line 7: unknown key$' 'alpha.hts, line 8: unknown key nakedkey$' \ + 'alpha.hts, line 13: a cookie appears twice$' 'alpha.hts, line 14: an unknown comment$' \ + 'alpha.hts: 1 cookie\(s\) outside its scope left out'; do + grep -qE "$want" <<<"$out" || fail "no warning [$want]: $out" +done +secret=ZQXSECRET +sec() { # sec NAME LINE... -- a bundle whose lines hold the secret + local name=$1 + shift + printf '%s\n' '# HTTrack bundle 1' "$@" >"$tmpdir/$name.hts" +} +sec s1 "header Authorization Bearer $secret:x" +sec s2 "X-Api-Key:$secret" "$secret" "url ftp:///$secret" +sec s3 "scope $secret!" +sec s4 "header X-A: $secret" "header Proxy-$secret: 1" +sec s5 cookies "#$secret" "$(jar alpha.test "$secret" 1)" \ + "$(jar alpha.test "$secret" 2)" "$(jar alpha.test "$secret" "$(repeat_chars 8000 x)")" \ + "$(jar alpha.test "$secret" 1)$(printf '\001')" +sec control "url http://www.alpha.test/?t=$secret" +for s in s1 s2 s3 s4 s5 control; do + out=$(httrack -#test=bundle "$dir" "https://www.alpha.test/#httrack-bundle=$s" 2>&1) + case $s in + control) grep -qF "$secret" <<<"$out" || fail "the probe cannot see a secret: $out" ;; + *) grep -qF "$secret" <<<"$out" && fail "$s shows the secret: $out" ;; + esac + grep -qF "/$s.hts" <<<"$out" || test $s = control || fail "$s: no message: $out" +done + # A bad name on the command line stops the run before it starts. rc=0 -out=$(httrack -O "$dir/argv" 'http://x.test/#httrack-bundle=../a' 2>&1) || rc=$? +out=$(httrack -O "$dir/argv" 'http://x.test/#httrack-bundle=..a' 2>&1) || rc=$? test "$rc" -ne 0 || fail "a bad bundle name on the command line was accepted" -grep -qF 'Invalid login bundle name' <<<"$out" || +grep -qF "invalid login bundle name: it starts with '.' or holds \"..\"" <<<"$out" || fail "no message for a bad bundle name: $out" diff --git a/tests/656_local-bundles.test b/tests/656_local-bundles.test index 9a664f879..ac2ca46cb 100644 --- a/tests/656_local-bundles.test +++ b/tests/656_local-bundles.test @@ -94,13 +94,18 @@ command grep -qF -- '--bundle-dir logins' "$doit" && fail_dump "doit.log kept a relative --bundle-dir" "$doit" # The suffix is cut before any URL reaches a file name, a log or the index. # hts-log.txt echoes the command line in parentheses, as doit.log keeps it. -names=$(cd "$tmpdir/one" && find . -name '*httrack-bundle*') -test -z "$names" || fail "a file name holds the bundle suffix: $names" -for f in index.html hts-log.txt hts-cache/new.txt; do - test -f "$tmpdir/one/$f" || fail "no $f" - command grep -v '^(' "$tmpdir/one/$f" | command grep -F '#httrack-bundle' && - fail_dump "$f holds the bundle suffix" "$tmpdir/one/$f" -done +no_suffix() { # no_suffix PROJECT + local names f + names=$(cd "$1" && find . -name '*httrack-bundle*') + test -z "$names" || fail "a file name holds the bundle suffix: $names" + for f in index.html hts-log.txt hts-cache/new.txt; do + test -f "$1/$f" || fail "no $1/$f" + if command grep -v '^(' "$1/$f" | command grep -F '#httrack-bundle'; then + fail_dump "$f holds the bundle suffix" "$1/$f" + fi + done +} +no_suffix "$tmpdir/one" command grep -qF 'http://www.alpha.test/ ' "$tmpdir/one/hts-log.txt" || fail_dump "hts-log.txt does not name the start URL" "$tmpdir/one/hts-log.txt" @@ -138,6 +143,7 @@ crawl "$tmpdir" -O two -%L "$(nativepath "$tmpdir/list.txt")" || test "$(($(hits) - before))" -eq 2 || fail_dump "-%L crawl: $(($(hits) - before)) requests, expected 2" "$tmpdir/srv.out" hit "-%L" "$before" "www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-" +no_suffix "$tmpdir/two" # A bundle that cannot be read stops the run before any request, with a message # that names it and never shows a refused header's value. A CR would end a @@ -146,6 +152,8 @@ printf '%s\n' '# HTTrack bundle 1' 'header Proxy-Authorization: planted-secret' >"$tmpdir/two/secret.hts" printf '# HTTrack bundle 1\nua U\rX-Injected: 1\nheader X-A: 1\rCookie: forged=1\n' \ >"$tmpdir/two/cr.hts" +printf '%s\n' '# HTTrack bundle 1' cookies "$(jar beta.test SID "v$(printf '\033')x")" \ + >"$tmpdir/two/esc.hts" printf '%s\n' 'http://www.beta.test/#httrack-bundle=../beta.test' >"$tmpdir/bad.txt" stops=0 stop() { # stop MESSAGE [httrack args...] @@ -160,14 +168,19 @@ stop() { # stop MESSAGE [httrack args...] before=$(hits) stop 'nosuch.hts: cannot read the login bundle' \ 'http://www.alpha.test/#httrack-bundle=nosuch' +command grep -qxF 'Login bundle: two/nosuch.hts: cannot read the login bundle: No such file or directory' "$log" || + fail_dump "the message changed" "$log" stop 'line 1: invalid login bundle name' -%L "$(nativepath "$tmpdir/bad.txt")" -stop 'refused header Proxy-Authorization' \ +stop 'secret.hts, line 2: refused header: a proxy header is not for the site' \ 'http://www.alpha.test/#httrack-bundle=secret' -stop 'line 2: too long or holding a control byte' \ +stop 'cr.hts, line 2: holds a control byte' \ 'http://www.alpha.test/#httrack-bundle=cr' -test "$stops" -eq 4 || fail "$stops stops ran, expected 4" +stop 'esc.hts, line 3: a cookie line holds a control byte' \ + 'http://www.beta.test/#httrack-bundle=esc' +test "$stops" -eq 5 || fail "$stops stops ran, expected 5" test "$(hits)" -eq "$before" || fail_dump "a refused bundle still crawled" "$tmpdir/srv.out" for f in "$log" "$tmpdir/two/hts-log.txt"; do + test -f "$f" || fail "no $f" command grep -qF planted-secret "$f" && fail_dump "$f shows the secret" "$f" done true From bf452e32c43e6ef16748a8a76472e569086865b6 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 18:18:14 +0200 Subject: [PATCH 04/10] Rework login bundles after review round 3 A bundle cookie line that is too long, or that holds a control byte anywhere, now stops the run. Before, a long line with a control byte only warned. Under -b0 the cookie section is still read into a scratch jar, so a bundle is valid or not whatever -b0 says. -%K and cookies.txt load as before. The https rule now merges bundles with a union-find in the existing pair loop, rather than a fixpoint that was cubic on scope chains. A header line limit counts the line without its CR, so LF and CRLF files agree. A cookie over 4096 bytes names that as the reason. htscore.c gets two helpers for the bundle start, COOKIE_WARN becomes a function, bundle_cut is static, and the tests drop GNU touch -d. Test 656 adds a redirect out of the bundle's site and back, and a userinfo URL that spells the site's name. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- html/httrack.man.html | 18 ++--- man/httrack.1 | 12 +++- man/httrack.tmpl | 12 +++- src/htsbauth.c | 71 ++++++++++++------- src/htsbundle.c | 126 ++++++++++++++++++-------------- src/htsbundle.h | 25 +++---- src/htsbundle_selftest.c | 38 ++++++---- src/htscore.c | 107 +++++++++++++++------------- src/htscoremain.c | 5 +- src/htsopt.h | 5 +- tests/655_engine-bundle.test | 134 ++++++++++++++++++++++++++++++----- tests/656_local-bundles.test | 64 +++++++++++------ tests/bundle-server.py | 15 +++- 13 files changed, 419 insertions(+), 213 deletions(-) diff --git a/html/httrack.man.html b/html/httrack.man.html index 1c1ba38a0..34cfe4581 100644 --- a/html/httrack.man.html +++ b/html/httrack.man.html @@ -2511,14 +2511,16 @@

FILES or an address, covers itself alone. Scope lines replace that default. HTTrack drops a cookie outside the scope and gives the count in one warning. HTTrack refuses a Host, Cookie, -Proxy-* or framing header, and a line holding a control -byte. A second ua line, header lines over 8192 bytes, -or a full cookie jar stops the run too. A repeated cookie -line, an unknown key or an unknown # line only warns. -Messages give the file and the line number, never the -line’s text. Two bundles whose scopes overlap must -give the same user agent, headers and cookie values, or the -run stops.

+Proxy-* or framing header, a scope line naming a public +suffix, and a line that holds a control byte or is too long, +in the cookie jar too and even under -b0. A second +ua line, header lines over 8192 bytes, or a full +cookie jar stops the run too. A repeated cookie line or an +unknown key only warns, and so does an unknown # line in the +cookie jar. Before the cookies line, a # line is a +comment. Messages give the file and the line number, never a +value. Two bundles whose scopes overlap must give the same +user agent, headers and cookie values, or the run stops.

The bundle’s cookies go into the cookie jar, which HTTrack diff --git a/man/httrack.1 b/man/httrack.1 index 3652d1a8b..ef0a8194c 100644 --- a/man/httrack.1 +++ b/man/httrack.1 @@ -634,11 +634,17 @@ URLs naming the bundle and of its url lines. A host with no registrable domain, such as a public suffix, a single label or an address, covers itself alone. Scope lines replace that default. HTTrack drops a cookie outside the scope and gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or -framing header, and a line holding a control byte. A second +framing header, a scope line naming a public suffix, and a line that holds a +control byte or is too long, in the cookie jar too and even under +.IR \-b0 . +A second .I ua line, header lines over 8192 bytes, or a full cookie jar stops the run too. A -repeated cookie line, an unknown key or an unknown # line only warns. Messages -give the file and the line number, never the line's text. Two bundles whose +repeated cookie line or an unknown key only warns, and so does an unknown # +line in the cookie jar. Before the +.I cookies +line, a # line is a comment. Messages give the file and the line number, never +a value. Two bundles whose scopes overlap must give the same user agent, headers and cookie values, or the run stops. .IP diff --git a/man/httrack.tmpl b/man/httrack.tmpl index da966208b..723d93aa6 100644 --- a/man/httrack.tmpl +++ b/man/httrack.tmpl @@ -117,11 +117,17 @@ URLs naming the bundle and of its url lines. A host with no registrable domain, such as a public suffix, a single label or an address, covers itself alone. Scope lines replace that default. HTTrack drops a cookie outside the scope and gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or -framing header, and a line holding a control byte. A second +framing header, a scope line naming a public suffix, and a line that holds a +control byte or is too long, in the cookie jar too and even under +.IR \-b0 . +A second .I ua line, header lines over 8192 bytes, or a full cookie jar stops the run too. A -repeated cookie line, an unknown key or an unknown # line only warns. Messages -give the file and the line number, never the line's text. Two bundles whose +repeated cookie line or an unknown key only warns, and so does an unknown # +line in the cookie jar. Before the +.I cookies +line, a # line is a comment. Messages give the file and the line number, never +a value. Two bundles whose scopes overlap must give the same user agent, headers and cookie values, or the run stops. .IP diff --git a/src/htsbauth.c b/src/htsbauth.c index d844f35e1..19ce70e3f 100644 --- a/src/htsbauth.c +++ b/src/htsbauth.c @@ -36,6 +36,7 @@ Please visit our Website: http://www.httrack.com #include "htsbauth.h" #include +#include #include /* specific definitions */ @@ -467,16 +468,26 @@ char *cookie_nextfield(char *a) { return a; } -/* Warn about the line just read. A bundle's warning gives the line number and - the reason but no byte of the line, which can hold a secret. */ -#define COOKIE_WARN(reason, ...) \ - do { \ - if (filter != NULL) \ - hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: %s", given, \ - filter->line, reason); \ - else \ - hts_log_print(opt, LOG_WARNING, __VA_ARGS__); \ - } while (0) +/* Warn about the line just read, with fmt for -%K and cookies.txt. A bundle's + warning gives the line number and reason, never a byte that can be secret. */ +static void cookie_warn(httrackp *opt, const cookie_filter *filter, + const char *given, const char *reason, const char *fmt, + ...) HTS_PRINTF_FUN(5, 6); + +static void cookie_warn(httrackp *opt, const cookie_filter *filter, + const char *given, const char *reason, const char *fmt, + ...) { + va_list args; + + if (filter != NULL) { + hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: %s", given, + filter->line, reason); + return; + } + va_start(args, fmt); + hts_log_vprint(opt, LOG_WARNING, fmt, args); + va_end(args); +} /* Copy Netscape-jar field into dst, or refuse the line and say so: a shortened domain or path would silently change where the cookie is sent. */ @@ -488,7 +499,7 @@ static hts_boolean cookie_load_field(httrackp *opt, char *dst, size_t dst_size, const size_t len = strlen(value); if (len >= dst_size) { - COOKIE_WARN("a cookie field is too long", + cookie_warn(opt, filter, given, "a cookie field is too long", "Cookies: ignoring a line whose %s field is %d bytes " "(maximum %d): %s", field_name, (int) len, (int) dst_size - 1, given); @@ -863,15 +874,19 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, strlen(line) + 1 - (sizeof(cookie_bom) - 1)); } first_line = HTS_FALSE; - if (len >= line_max) { - COOKIE_WARN("a cookie line is too long", - "Cookies: ignoring an over-long line " - "(%d bytes, maximum %d): %s", - (int) len, (int) line_max - 1, given); - refused++; - } else if (filter != NULL && hts_bundle_has_control(line, len)) { - // a bundle is written for this run, so a damaged one stops it + // a bundle is written for this run, so a damaged one stops it + if (filter != NULL && + hts_bundle_has_control(line, + len < sizeof(line) ? len : sizeof(line) - 1)) { filter->why = "a cookie line holds a control byte"; + } else if (filter != NULL && len >= line_max) { + filter->why = "a cookie line is too long"; + } else if (len >= line_max) { + hts_log_print(opt, LOG_WARNING, + "Cookies: ignoring an over-long line " + "(%d bytes, maximum %d): %s", + (int) len, (int) line_max - 1, given); + refused++; } else if (memchr(line, '\0', len) != NULL || memchr(line, '\r', len) != NULL) { hts_log_print(opt, LOG_WARNING, @@ -887,7 +902,7 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, if (rec[0] == '#') { if (!cookie_known_comment(rec)) { - COOKIE_WARN("an unknown comment", + cookie_warn(opt, filter, given, "an unknown comment", "Cookies: ignoring an unknown comment in %s: %.64s", given, rec); refused++; @@ -906,7 +921,8 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, for (p = strchr(rec, '\t'); p != NULL; p = strchr(p + 1, '\t')) tabs++; if (tabs < 6 || !strnotempty(cookie_get(buffer, rec, 5))) { - COOKIE_WARN("a cookie line has fewer than 7 fields or no name", + cookie_warn(opt, filter, given, + "a cookie line has fewer than 7 fields or no name", "Cookies: ignoring a line with fewer than 7 " "fields or no name: %s", given); @@ -915,13 +931,15 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, (tabs == 7 && strcmp(cookie_get(buffer, rec, 7), COOKIE_HTTPONLY_FIELD) != 0)) { // a TAB in the value, because only HTTrack writes an eighth field - COOKIE_WARN("a cookie line has more than 7 fields", + cookie_warn(opt, filter, given, + "a cookie line has more than 7 fields", "Cookies: ignoring a line with more than 7 " "fields: %s", given); refused++; } else if (!cookie_is_expiry(cookie_get(buffer, rec, 4))) { - COOKIE_WARN("a cookie expiry is not a whole number", + cookie_warn(opt, filter, given, + "a cookie expiry is not a whole number", "Cookies: ignoring a line whose expiry is not a " "whole number: %s", given); @@ -951,13 +969,14 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, domain, path))) { // cookie_set stores it host-only if (!attr.host_only && cookie_domain_is_suffix(domain)) - COOKIE_WARN("a cookie for a public suffix is now host-only", + cookie_warn(opt, filter, given, + "a cookie for a public suffix is now host-only", "Cookies: '%.64s' for the public suffix %s is " "now host-only: %s", cook_name, domain, given); // the last one wins, as curl reads it, but a rewrite loses one if (repeated) { - COOKIE_WARN("a cookie appears twice", + cookie_warn(opt, filter, given, "a cookie appears twice", "Cookies: '%.64s' appears twice in %s", cook_name, given); refused++; @@ -984,7 +1003,7 @@ static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp, } } if (!feof(fp) && (filter == NULL || filter->why == NULL)) { - COOKIE_WARN("stopped reading before the end", + cookie_warn(opt, filter, given, "stopped reading before the end", "Cookies: stopped reading before the end of %s", given); refused++; } diff --git a/src/htsbundle.c b/src/htsbundle.c index 925290384..0c31a765e 100644 --- a/src/htsbundle.c +++ b/src/htsbundle.c @@ -65,7 +65,8 @@ typedef struct bundle { size_t nscope; char *ua; /* NULL when it gives none */ String headers; - hts_boolean https_only; /* Are its headers for https alone? */ + hts_boolean https_only; /* Are its headers for https alone? */ + hts_boolean scope_given; /* Do scope lines replace the default scope? */ } bundle; typedef struct hts_bundle_state { @@ -102,6 +103,7 @@ static void bundle_clear(bundle *b) { freet(b->ua); StringClear(b->headers); b->https_only = HTS_FALSE; + b->scope_given = HTS_FALSE; } /* Is name a Windows device name, such as CON, aux.txt or com1? */ @@ -143,7 +145,9 @@ static const char *bundle_name_refusal(const char *name) { return NULL; } -const char *hts_bundle_cut(char *url, char *name, size_t name_size) { +/* Cut the bundle suffix off url in place and copy its name into name, or "" + when url has none. NULL, or why the name is refused and url is unchanged. */ +static const char *bundle_cut(char *url, char *name, size_t name_size) { char *const at = strstr(url, HTS_BUNDLE_SUFFIX); const char *why; @@ -207,7 +211,7 @@ static hts_boolean bundle_add(httrackp *opt, const char *name, const char *hts_bundle_take(httrackp *opt, char *url) { char name[HTS_BUNDLE_NAME_MAX]; - const char *const why = hts_bundle_cut(url, name, sizeof(name)); + const char *const why = bundle_cut(url, name, sizeof(name)); if (why != NULL || name[0] == '\0') return why; @@ -226,16 +230,18 @@ hts_boolean hts_bundle_has_control(const char *s, size_t len) { return HTS_FALSE; } -/* Read a line into line without its line end. NULL, or why the line is - refused: too long, or with a control byte other than TAB, because a CR - would end a header early. */ -static const char *bundle_readline(FILE *fp, char *line, size_t size) { +/* Read a line of fewer than BUNDLE_LINE_MAX bytes, its line end aside, into + line. NULL, or why the line is refused: too long, or with a control byte + other than TAB, because a CR would end a header early. */ +static const char *bundle_readline(FILE *fp, char line[BUNDLE_LINE_MAX + 1]) { size_t len; - if (hts_readline_raw(fp, line, size, &len)) + if (hts_readline_raw(fp, line, BUNDLE_LINE_MAX + 1, &len)) return "too long"; if (len > 0 && line[len - 1] == '\r') line[--len] = '\0'; + if (len >= BUNDLE_LINE_MAX) + return "too long"; return hts_bundle_has_control(line, len) ? "holds a control byte" : NULL; } @@ -293,22 +299,19 @@ static const char *bundle_scope_add(bundle *b, const char *entry) { return strlist_add(&b->scope, &b->nscope, entry) ? NULL : "out of memory"; } -/* Note that url names b. An https one makes b's headers https-only, and - unless scope lines replace it, the URL's registrable domain joins the scope, - or its host alone when it has none: a public suffix, a single label or an - address. NULL, or why it could not. */ -static const char *bundle_note_url(bundle *b, const char *url, - hts_boolean scope_given) { +/* Add url's https flag and default scope to b. NULL, or why it could not. */ +static const char *bundle_note_url(bundle *b, const char *url) { char host[256]; char entry[sizeof(host) + 1]; const char *reg; if (strfield(url, "https:")) b->https_only = HTS_TRUE; - if (scope_given) + if (b->scope_given) return NULL; if (!cookie_url_host(url, host, sizeof(host))) return "cannot work out the bundle's scope"; + // a host with no registrable domain covers itself alone reg = hts_host_is_address(host, strlen(host)) ? NULL : hts_psl_registrable_domain(host); @@ -360,13 +363,12 @@ static hts_boolean key_is(const char *line, size_t klen, const char *key) { /* Read b's lines up to "cookies", leaving fp at its jar. *cookies_line gets the number of that line, or 0 when b has no cookies. */ static hts_boolean bundle_parse(httrackp *opt, bundle *b, FILE *fp, - hts_boolean *scope_given, int *cookies_line, - char *err, size_t err_size) { + int *cookies_line, char *err, size_t err_size) { static const char bom[] = "\xEF\xBB\xBF"; - char BIGSTK line[BUNDLE_LINE_MAX]; + char BIGSTK line[BUNDLE_LINE_MAX + 1]; int lineno = 1; int version; - const char *why = bundle_readline(fp, line, sizeof(line)); + const char *why = bundle_readline(fp, line); if (why != NULL) BUNDLE_FAIL("%s, line 1: %s", b->file, why); @@ -378,14 +380,13 @@ static hts_boolean bundle_parse(httrackp *opt, bundle *b, FILE *fp, b->file); if (version != 1) BUNDLE_FAIL("%s: bundle version %d is not supported", b->file, version); - *scope_given = HTS_FALSE; *cookies_line = 0; while (!feof(fp)) { const char *value; size_t klen; lineno++; - why = bundle_readline(fp, line, sizeof(line)); + why = bundle_readline(fp, line); if (why != NULL) BUNDLE_FAIL("%s, line %d: %s", b->file, lineno, why); if (line[0] == '\0' || line[0] == '#') @@ -440,7 +441,7 @@ static hts_boolean bundle_parse(httrackp *opt, bundle *b, FILE *fp, StringCat(b->headers, value); StringCat(b->headers, "\r\n"); } else { - *scope_given = HTS_TRUE; + b->scope_given = HTS_TRUE; if (!bundle_scope_line(b, value, lineno, err, err_size)) return HTS_FALSE; } @@ -481,12 +482,19 @@ static hts_boolean bundle_admit(cookie_filter *filter, t_cookie *cookie, ctx->dropped++; return HTS_FALSE; } + if (strlen(name) > COOKIE_MAX_PAIR || + strlen(value) > COOKIE_MAX_PAIR - strlen(name)) { + filter->why = "a cookie name and value pass 4096 bytes"; + return HTS_FALSE; + } other = cookie_find_exact(ctx->all_bundles->data, name, host, path); if (other != NULL && !repeat && strcmp(cookie_get(buffer, other, 6), value) != 0) { filter->why = "a cookie has another value in another bundle"; return HTS_FALSE; } + /* unreachable while all_bundles outsizes opt->cookie; a silent skip would + stop the clash check */ if (cookie_set(ctx->all_bundles, name, value, domain, path, NULL) != 0) { filter->why = "too many cookies to compare with the other bundles"; return HTS_FALSE; @@ -504,8 +512,8 @@ static hts_boolean bundle_load(httrackp *opt, bundle *b, const char *jar_file, ? StringBuff(opt->bundle_dir) : StringBuff(opt->path_log); const size_t dlen = strlen(dir); - hts_boolean scope_given; int cookies_line; + t_cookie *scratch = NULL; bundle_jar ctx; cookie_filter filter; const char *why = NULL; @@ -522,14 +530,14 @@ static hts_boolean bundle_load(httrackp *opt, bundle *b, const char *jar_file, if (fp == NULL) BUNDLE_FAIL("%s: cannot read the login bundle: %s", b->file, strerror(errno)); - if (!bundle_parse(opt, b, fp, &scope_given, &cookies_line, err, err_size)) { + if (!bundle_parse(opt, b, fp, &cookies_line, err, err_size)) { fclose(fp); return HTS_FALSE; } for (i = 0; i < b->nstart && why == NULL; i++) - why = bundle_note_url(b, b->start_urls[i], scope_given); + why = bundle_note_url(b, b->start_urls[i]); for (i = 0; i < b->nurls && why == NULL; i++) - why = bundle_note_url(b, b->urls[i], scope_given); + why = bundle_note_url(b, b->urls[i]); if (why != NULL) { fclose(fp); BUNDLE_FAIL("%s: %s", b->file, why); @@ -542,12 +550,22 @@ static hts_boolean bundle_load(httrackp *opt, bundle *b, const char *jar_file, filter.admit = bundle_admit; filter.ctx = &ctx; filter.line = cookies_line; - if (cookies_line != 0 && opt->cookie != NULL) - cookie_load_stream(opt, opt->cookie, fp, b->file, &filter); - else if (cookies_line != 0) + if (cookies_line != 0 && opt->cookie == NULL) { hts_log_print(opt, LOG_WARNING, "Cookies are disabled, so the cookies of %s are not sent", b->file); + // read all the same, so -b0 does not decide whether a bundle is valid + scratch = calloct(1, sizeof(t_cookie)); + if (scratch == NULL) { + fclose(fp); + BUNDLE_FAIL("%s: out of memory", b->file); + } + scratch->max_len = sizeof(scratch->data); + } + if (cookies_line != 0) + cookie_load_stream(opt, scratch != NULL ? scratch : opt->cookie, fp, + b->file, &filter); + freet(scratch); fclose(fp); if (ctx.dropped != 0) hts_log_print(opt, LOG_WARNING, @@ -581,41 +599,33 @@ static hts_boolean bundle_same_identity(const bundle *a, const bundle *b) { strcmp(StringBuff(a->headers), StringBuff(b->headers)) == 0; } -/* Bundles that overlap have merged, so an https URL in any of them makes the - headers of all of them https-only. */ -static void bundle_merge_https(hts_bundle_state *st) { - hts_boolean changed = HTS_TRUE; - size_t i, j; - - while (changed) { - changed = HTS_FALSE; - for (i = 0; i < st->count; i++) { - for (j = 0; j < st->count; j++) { - bundle *const a = &st->items[i]; - const bundle *const b = &st->items[j]; - - if (!a->https_only && b->https_only && bundle_scopes_overlap(a, b)) { - a->https_only = HTS_TRUE; - changed = HTS_TRUE; - } - } - } - } +/* The bundle that stands for the merged group holding i. */ +static size_t bundle_group(size_t *group, size_t i) { + while (group[i] != i) + i = group[i] = group[group[i]]; + return i; } hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar_file, char *err, size_t err_size) { hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state; t_cookie *all_bundles; + size_t *group; hts_boolean ok = HTS_TRUE; size_t i, j; if (st == NULL) return HTS_TRUE; all_bundles = calloct(1, sizeof(t_cookie)); - if (all_bundles == NULL) + group = calloct(st->count, sizeof(size_t)); + if (all_bundles == NULL || group == NULL) { + freet(all_bundles); + freet(group); BUNDLE_FAIL("login bundles: out of memory"); + } all_bundles->max_len = sizeof(all_bundles->data); + for (i = 0; i < st->count; i++) + group[i] = i; for (i = 0; i < st->count && ok; i++) ok = bundle_load(opt, &st->items[i], jar_file, all_bundles, err, err_size); for (i = 0; i < st->count && ok; i++) { @@ -623,17 +633,27 @@ hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar_file, char *err, const bundle *const a = &st->items[i]; const bundle *const b = &st->items[j]; - if (bundle_scopes_overlap(a, b) && !bundle_same_identity(a, b)) { + if (!bundle_scopes_overlap(a, b)) + continue; + if (!bundle_same_identity(a, b)) { snprintf(err, err_size, "%s and %s cover the same site with a different user agent " "or headers", a->file, b->file); ok = HTS_FALSE; + } else { + group[bundle_group(group, j)] = bundle_group(group, i); } } } - if (ok) - bundle_merge_https(st); + // an https URL in any merged bundle makes the headers of all https-only + for (i = 0; i < st->count; i++) { + if (st->items[i].https_only) + st->items[bundle_group(group, i)].https_only = HTS_TRUE; + } + for (i = 0; i < st->count; i++) + st->items[i].https_only = st->items[bundle_group(group, i)].https_only; + freet(group); freet(all_bundles); return ok; } diff --git a/src/htsbundle.h b/src/htsbundle.h index c1c9a940b..53480bf42 100644 --- a/src/htsbundle.h +++ b/src/htsbundle.h @@ -47,29 +47,24 @@ Please visit our Website: http://www.httrack.com /* A bundle name holds fewer bytes than this. */ #define HTS_BUNDLE_NAME_MAX 128 -/* Cut the bundle suffix off url in place and copy its name into name, or "" - when url has none. NULL, or why the name is refused, url then left alone. */ -const char *hts_bundle_cut(char *url, char *name, size_t name_size); - -/* hts_bundle_cut(), then record that url names the bundle. A name given twice - is one bundle. NULL, or why the run must stop. */ +/* Cut the bundle suffix off url in place and record that url names that + bundle. A name given twice is one bundle. NULL, or why the run must stop. */ const char *hts_bundle_take(httrackp *opt, char *url); /* Read every named bundle from opt->bundle_dir, or the project folder, and refuse two bundles that overlap with different identities or cookie values. - The cookies go into opt->cookie, under the records already there when - jar_file (or NULL) is newer than the bundle. HTS_FALSE after writing why - into err, naming the file: the run must stop. A second call reads the files - again. */ + The cookies go into opt->cookie. A jar_file newer than the bundle keeps its + records. A NULL one never does. HTS_FALSE after writing why into err, naming + the file, and then the run must stop. A second call reads the files again. */ hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar_file, char *err, size_t err_size); /* The i-th url line of the loaded bundles, or NULL past the last one. */ const char *hts_bundle_url(httrackp *opt, size_t i); -/* Give req the UA and headers of the first bundle whose scope holds adr's - host, where it has them. Its headers go over https alone when any URL of - the bundles merged with it is https. */ +/* Give req the UA and the headers of the first bundle whose scope holds adr's + host, when that bundle has them. Its headers go over https alone when any URL + of the bundles merged with it is https. */ void hts_bundle_identity(httrackp *opt, const char *adr, htsrequest *req); /* Forget the bundles named for opt. */ @@ -102,8 +97,8 @@ typedef struct cookie_filter { /* Load Netscape jar records from fp's position to its end through filter, as cookie_load reads a file. given names the file in warnings, which name - the line and none of its bytes. A line with a control byte other than TAB - stops the load. */ + the line and none of its bytes. A line with a control byte other than TAB, + or one too long, stops the load. */ void cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp, const char *given, cookie_filter *filter); diff --git a/src/htsbundle_selftest.c b/src/htsbundle_selftest.c index b43f4eb92..ce03ef467 100644 --- a/src/htsbundle_selftest.c +++ b/src/htsbundle_selftest.c @@ -36,23 +36,34 @@ Please visit our Website: http://www.httrack.com /* -#test=bundlecut : the bundle a start URL names, and the URL left. */ static int st_bundlecut(httrackp *opt, int argc, char **argv) { + static const char nodir[] = "/nonexistent-bundlecut/"; char url[HTS_URLMAXSIZE * 2]; - char name[HTS_BUNDLE_NAME_MAX]; + char err[1024]; const char *why; - (void) opt; if (argc != 1 || strlen(argv[0]) >= sizeof(url)) { fprintf(stderr, "bundlecut: takes \n"); return 1; } strcpybuff(url, argv[0]); - why = hts_bundle_cut(url, name, sizeof(name)); - if (why != NULL) + hts_bundle_free(opt); + why = hts_bundle_take(opt, url); + if (why != NULL) { printf("bad: %s\n", why); - else if (name[0] == '\0') + } else if (opt->bundle_state == NULL) { printf("none\n"); - else - printf("%s %s\n", name, url); + } else { + /* the name shows in the file a load from a missing folder cannot open */ + StringCopy(opt->bundle_dir, nodir); + assertf(!hts_bundle_load_all(opt, NULL, err, sizeof(err)) && + strncmp(err, nodir, sizeof(nodir) - 1) == 0 && + strstr(err, ".hts: ") != NULL); + printf("%.*s %s\n", + (int) (strstr(err, ".hts: ") - err) - (int) (sizeof(nodir) - 1), + err + sizeof(nodir) - 1, url); + StringCopy(opt->bundle_dir, ""); + hts_bundle_free(opt); + } return 0; } @@ -71,9 +82,10 @@ static void print_without_dir(const char *err, const char *dir) { printf("%s\n", p); } -/* -#test=bundle

... [host:]... [jar:] [twice]: load the - bundles the URLs name from dir over the jar file, twice when asked, then - print their url lines, the jar, and the UA and headers each adr gets. */ +/* -#test=bundle ... [host:]... [jar:] [twice] [b0]: load + the bundles the URLs name from dir over the jar file, twice when asked, with + cookies off for b0, then print their url lines, the jar, and the UA and + headers each adr gets. */ static int st_bundle(httrackp *opt, int argc, char **argv) { FILE *const log = opt->log; t_cookie *const prev = opt->cookie; @@ -87,7 +99,7 @@ static int st_bundle(httrackp *opt, int argc, char **argv) { if (argc < 2 || jar == NULL) { fprintf(stderr, "bundle: takes ... [host:]... " - "[jar:] [twice]\n"); + "[jar:] [twice] [b0]\n"); freet(jar); return 1; } @@ -104,6 +116,8 @@ static int st_bundle(httrackp *opt, int argc, char **argv) { (void) cookie_load(opt, jar, "", jar_file); } else if (strcmp(argv[i], "twice") == 0) { loads = 2; + } else if (strcmp(argv[i], "b0") == 0) { + opt->cookie = NULL; } else if (strncmp(argv[i], "host:", 5) != 0 && strlen(argv[i]) < sizeof(buff)) { strcpybuff(buff, argv[i]); @@ -153,7 +167,7 @@ static int st_bundle(httrackp *opt, int argc, char **argv) { const struct selftest_entry selftests_bundle[] = { {"bundlecut", "", "the login bundle a start URL names", st_bundlecut}, - {"bundle", " ... [host:]... [jar:] [twice]", + {"bundle", " ... [host:]... [jar:] [twice] [b0]", "load login bundles and print the identity each host gets", st_bundle}, {NULL, NULL, NULL, NULL}, }; diff --git a/src/htscore.c b/src/htscore.c index 95910c3f2..e19005f4c 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -679,6 +679,56 @@ static hts_boolean filter_rule_decode_(const char *rule, char *dest, return HTS_TRUE; } +/* The jar whose records can win over a bundle's: the -%K file, or the + project's cookies.txt. NULL under -b0. */ +static const char *bundle_jar_file(httrackp *opt, char buff[CATBUFF_SIZE]) { + if (!opt->accept_cookie) + return NULL; + if (strnotempty(StringBuff(opt->cookies_file))) + return StringBuff(opt->cookies_file); + return fconcat(buff, CATBUFF_SIZE, StringBuff(opt->path_log), "cookies.txt"); +} + +/* Load the login bundles, and add their url lines to the start URLs in + *primary, which grows. HTS_FALSE after saying why the run must stop. */ +static hts_boolean bundle_start(httrackp *opt, char **primary, + htsbuff *primarybuff) { + char err[1024]; + char jarbuff[CATBUFF_SIZE]; + const char *extra; + size_t room = 0; + size_t i; + + if (!hts_bundle_load_all(opt, bundle_jar_file(opt, jarbuff), err, + sizeof(err))) { + hts_log_print(opt, LOG_ERROR, "Login bundle: %s", err); + fprintf(stderr, "Login bundle: %s\n", err); + return HTS_FALSE; + } + for (i = 0; (extra = hts_bundle_url(opt, i)) != NULL; i++) + room += strlen(extra) + sizeof("http://\n"); + if (room != 0) { + char *const grown = + room < (size_t) -1 - primarybuff->cap + ? (char *) realloct(*primary, primarybuff->cap + room) + : NULL; + + if (grown == NULL) { + printf("PANIC! : Not enough memory [%d]\n", __LINE__); + return HTS_FALSE; + } + *primary = primarybuff->buf = grown; + primarybuff->cap += room; + } + for (i = 0; (extra = hts_bundle_url(opt, i)) != NULL; i++) { + if (strstr(extra, ":/") == NULL) + htsbuff_cat(primarybuff, "http://"); + htsbuff_cat(primarybuff, extra); + htsbuff_cat(primarybuff, "\n"); + } + return HTS_TRUE; +} + int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { char *primary = NULL; // première page, contenant les liens à scanner hash_struct hash; // système de hachage, accélère la recherche dans les liens @@ -1177,56 +1227,13 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { opt, opt->cookie, "", StringBuff(opt->cookies_file), start_url); } - /* the login bundles: url lines join the start URLs, cookies the jar */ - { - char err[1024]; - char jarbuff[CATBUFF_SIZE]; - const char *jar_file = - !opt->accept_cookie ? NULL - : strnotempty(StringBuff(opt->cookies_file)) - ? StringBuff(opt->cookies_file) - : fconcat(jarbuff, sizeof(jarbuff), StringBuff(opt->path_log), - "cookies.txt"); - const char *extra; - size_t room = 0; - size_t i; - - if (!bundle_refused && - !hts_bundle_load_all(opt, jar_file, err, sizeof(err))) { - hts_log_print(opt, LOG_ERROR, "Login bundle: %s", err); - fprintf(stderr, "Login bundle: %s\n", err); - bundle_refused = HTS_TRUE; - } - for (i = 0; !bundle_refused && (extra = hts_bundle_url(opt, i)) != NULL; - i++) - room += strlen(extra) + sizeof("http://\n"); - if (room != 0) { - char *const grown = - room < (size_t) -1 - primarybuff.cap - ? (char *) realloct(primary, primarybuff.cap + room) - : NULL; - - if (grown == NULL) { - printf("PANIC! : Not enough memory [%d]\n", __LINE__); - bundle_refused = HTS_TRUE; - } else { - primary = primarybuff.buf = grown; - primarybuff.cap += room; - } - } - if (bundle_refused) { - /* the jar may hold half a bundle, so it is not saved */ - cookies_file_refused = COOKIE_LOAD_KEEP; - freet(primary); - XH_extuninit; - return 0; - } - for (i = 0; (extra = hts_bundle_url(opt, i)) != NULL; i++) { - if (strstr(extra, ":/") == NULL) - htsbuff_cat(&primarybuff, "http://"); - htsbuff_cat(&primarybuff, extra); - htsbuff_cat(&primarybuff, "\n"); - } + // bundle url lines join the start URLs, and bundle cookies go to the jar + if (bundle_refused || !bundle_start(opt, &primary, &primarybuff)) { + /* the jar may hold half a bundle, so it is not saved */ + cookies_file_refused = COOKIE_LOAD_KEEP; + freet(primary); + XH_extuninit; + return 0; } // lien primaire diff --git a/src/htscoremain.c b/src/htscoremain.c index f520426fe..bccac89fe 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -592,7 +592,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { int httrack_logmode = 3; // ONE log file ensureUrlCapacity(url, url_sz, 65536); - /* a previous run's bundles are not this one's */ + /* a previous run's bundles and folder are not this one's, but only a library + caller loads twice on one opt */ hts_bundle_free(opt); StringCopy(opt->bundle_dir, ""); @@ -1127,7 +1128,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { htsmain_free(); return -1; } else if (strfield2(argv[i] + 2, "bundle-dir")) { - i++; /* its value, which the option parser reads */ + i++; /* The option parser reads its value. */ } // else { diff --git a/src/htsopt.h b/src/htsopt.h index bf62ca976..4d44b93f2 100644 --- a/src/htsopt.h +++ b/src/htsopt.h @@ -657,8 +657,9 @@ struct httrackp { line, and the last one wins. Tail: ABI */ hts_boolean accept_given; /**< Did the user give -%a? Tail: ABI */ hts_boolean lang_given; /**< Did the user give -%l? Tail: ABI */ - String bundle_dir; /**< --bundle-dir: where login bundle names resolve, or - empty for the project folder. Tail: ABI */ + String bundle_dir; /**< Where login bundle names resolve, or empty for + the project folder. Only --bundle-dir sets it, and + each run resets it. Tail: ABI */ void *bundle_state; /**< hts_bundle_state*, or NULL. Live state, so copy_htsopt must leave it alone. Tail: ABI */ }; diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index 6eedcacd2..a6bacc4fa 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -115,6 +115,34 @@ https://mixed.test ua=M headers=X-M: 1|" bundle "$dir" \ host:https://mixed.test done +# Merging is transitive: A and C share no site, but both overlap B, so C's +# https URL keeps A's headers off plain http too, in either order. scope lines +# leave the https rule as it is. +for n in a b c; do + lines=('# HTTrack bundle 1' 'ua H' 'header X-H: 1') + case $n in + a) lines+=('scope a.test' 'scope b.test') ;; + b) lines+=('scope b.test' 'scope c.test') ;; + c) lines+=('scope c.test' 'url https://www.c.test/') ;; + esac + bundle "chain$n" "${lines[@]}" +done +chain_hosts=(host:http://a.test host:https://a.test host:http://c.test) +chain_out="http://a.test ua=H headers=global +https://a.test ua=H headers=X-H: 1| +http://c.test ua=H headers=global" +selftest_queue_tail " +$chain_out" bundle "$dir" 'http://a.test/#httrack-bundle=chaina' \ + 'http://b.test/#httrack-bundle=chainb' 'http://c.test/#httrack-bundle=chainc' \ + "${chain_hosts[@]}" +selftest_queue_tail " +$chain_out" bundle "$dir" 'http://c.test/#httrack-bundle=chainc' \ + 'http://b.test/#httrack-bundle=chainb' 'http://a.test/#httrack-bundle=chaina' \ + "${chain_hosts[@]}" +selftest_queue_tail " +http://c.test ua=H headers=global" bundle "$dir" \ + 'http://c.test/#httrack-bundle=chainc' host:http://c.test + # The registrable domain follows the suffix list, not the last two labels. A # host with none (a public suffix, a single label) covers itself alone, and so # does an address. @@ -149,23 +177,31 @@ a.beta.test ua=N${tab}x headers=X-N:${tab}1| host:a.www.alpha.test host:a.beta.test host:127.0.0.1 host:127.0.0.2 # Of the bundle and the jar, the newer file wins, and the bundle wins a tie. -order() { # order NAME BUNDLE-DATE JAR-DATE WINNER +order() { # order NAME BUNDLE-TIME JAR-TIME WINNER -- times as touch -t takes bundle "$1" '# HTTrack bundle 1' cookies "$(jar order.test SID bundle)" jar order.test SID jar >"$tmpdir/$1.txt" - touch -d "$2" "$tmpdir/$1.hts" - touch -d "$3" "$tmpdir/$1.txt" + touch -t "$2" "$tmpdir/$1.hts" + touch -t "$3" "$tmpdir/$1.txt" selftest_queue "cookie .order.test SID=$4" bundle "$dir" \ "http://www.order.test/#httrack-bundle=$1" "jar:$dir/$1.txt" } -order order1 '2026-01-02 10:00:00' '2026-01-01 10:00:00' bundle -order order2 '2026-01-01 10:00:00' '2026-01-02 10:00:00' jar -order order3 '2026-01-01 10:00:00' '2026-01-01 10:00:00' bundle -# half a second apart in the same second: a seconds-only compare calls it a tie -if touch -d '2026-01-01 10:00:00.5' "$tmpdir/subsecond" 2>/dev/null && +order order1 202601021000.00 202601011000.00 bundle +order order2 202601011000.00 202601021000.00 jar +order order3 202601011000.00 202601011000.00 bundle +# Half a second apart in the same second, which a seconds-only compare calls a +# tie. Only a touch that sets fractions can write it. +if touch -d '2026-01-01T10:00:00.5' "$tmpdir/subsecond" 2>/dev/null && test "$tmpdir/subsecond" -nt "$tmpdir/order3.txt"; then - order order4 '2026-01-01 10:00:00' '2026-01-01 10:00:00.5' jar + order order4 202601011000.00 202601011000.00 jar + touch -d '2026-01-01T10:00:00.5' "$tmpdir/order4.txt" fi +# A control byte in a -%K or cookies.txt jar loads as it always did. +printf '.ctl.test\tTRUE\t/\tFALSE\t0\tK\ta\001b\n' >"$tmpdir/ctl.txt" +bundle plain '# HTTrack bundle 1' +selftest_queue "cookie .ctl.test K=a$(printf '\001')b" bundle "$dir" \ + 'http://www.alpha.test/#httrack-bundle=plain' "jar:$dir/ctl.txt" + # Refused: each one stops the run and names the file and line, never a byte of # the line. The batch reads them all at the end, so each case writes its own. nbad=0 @@ -202,8 +238,12 @@ refused ', line 3: refused header: header replaces an earlier one' \ '# HTTrack bundle 1' 'header Authorization: a' 'header Authorization: secret-b' refused ', line 2: refused header: malformed header ignored' \ '# HTTrack bundle 1' 'header Bearer secret-no-colon' +# The header lines hold 8192 bytes at most, each with its CRLF. +bundle hdr8192 '# HTTrack bundle 1' "header X-A: $(repeat_chars 5000 a)" \ + "header X-B: $(repeat_chars 3178 b)" +selftest_queue '' bundle "$dir" 'http://x.test/#httrack-bundle=hdr8192' refused ', line 3: the header lines pass 8192 bytes' '# HTTrack bundle 1' \ - "header X-A: $(repeat_chars 5000 a)" "header X-B: $(repeat_chars 5000 b)" + "header X-A: $(repeat_chars 5000 a)" "header X-B: $(repeat_chars 3179 b)" # A control byte would split a header on the wire. The CR before the LF goes. ctl=', line 2: holds a control byte' refused "$ctl" '# HTTrack bundle 1' "$(printf 'ua x\rHost: evil.test')" @@ -216,16 +256,45 @@ nbad=$((nbad + 1)) printf '# HTTrack bundle 1\nua a\000Host: evil.test\n' >"$tmpdir/bad$nbad.hts" selftest_queue "refused: /bad$nbad.hts$ctl" bundle "$dir" \ "http://www.alpha.test/#httrack-bundle=bad$nbad" -# A line holds 8191 bytes at most, its line end aside. +# A line holds 8191 bytes at most, its LF or CRLF aside. bundle long '# HTTrack bundle 1' "ua $(repeat_chars 8188 u)" selftest_queue '' bundle "$dir" 'http://x.test/#httrack-bundle=long' +printf '# HTTrack bundle 1\nua %s\r\n' "$(repeat_chars 8188 u)" \ + >"$tmpdir/longcrlf.hts" +selftest_queue '' bundle "$dir" 'http://x.test/#httrack-bundle=longcrlf' refused ', line 2: too long' '# HTTrack bundle 1' "ua $(repeat_chars 8189 u)" -nbad=$((nbad + 1)) -printf '# HTTrack bundle 1\nua %s\r\v\f\n' "$(repeat_chars 8188 u)" \ - >"$tmpdir/bad$nbad.hts" -selftest_queue "refused: /bad$nbad.hts, line 2: too long" bundle "$dir" \ - "http://www.alpha.test/#httrack-bundle=bad$nbad" -# In the jar, every control byte but TAB stops the run too. +for end in '\r\v\f' '\v' '\f'; do + nbad=$((nbad + 1)) + printf '# HTTrack bundle 1\nua %s%b\n' "$(repeat_chars 8188 u)" "$end" \ + >"$tmpdir/bad$nbad.hts" + selftest_queue "refused: /bad$nbad.hts, line 2: too long" bundle "$dir" \ + "http://www.alpha.test/#httrack-bundle=bad$nbad" +done +# In the jar, every control byte but TAB stops the run too, in any field and +# under -b0, and so does a line too long to read whole. +jarbad() { # jarbad LINE WHY [b0] -- LINE is the cookie line of a bundle + nbad=$((nbad + 1)) + printf '# HTTrack bundle 1\ncookies\n%s\n' "$1" >"$tmpdir/bad$nbad.hts" + selftest_queue "refused: /bad$nbad.hts, line 3: $2" bundle "$dir" \ + "http://www.alpha.test/#httrack-bundle=bad$nbad" "${@:3}" +} +ctlcookie='a cookie line holds a control byte' +jarbad "$(printf '.al\001pha.test\tTRUE\t/\tFALSE\t0\tSID\tv')" "$ctlcookie" +jarbad "$(printf '.alpha.test\tTRUE\t/\001\tFALSE\t0\tSID\tv')" "$ctlcookie" +jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tS\001ID\tv')" "$ctlcookie" +jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\ta\001b')" "$ctlcookie" b0 +long=$(repeat_chars 8100 x) +jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s\001' "$long")" "$ctlcookie" +for c in '\v' '\f'; do + jarbad "$(printf '%b.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s' "$c" "$long")" "$ctlcookie" +done +jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s' "$long")" 'a cookie line is too long' +# past the bytes the reader keeps, the length alone stops it +jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s%s\001' "$long" "$long")" 'a cookie line is too long' +jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s' "$(repeat_chars 4094 v)")" 'a cookie name and value pass 4096 bytes' +bundle b0ok '# HTTrack bundle 1' 'ua B0' cookies "$(jar alpha.test SID s)" +selftest_queue 'www.alpha.test ua=B0 headers=global' bundle "$dir" \ + 'http://www.alpha.test/#httrack-bundle=b0ok' host:www.alpha.test b0 for oct in 000 001 002 003 004 005 006 007 010 013 014 015 016 017 020 021 \ 022 023 024 025 026 027 030 031 032 033 034 035 036 037 177; do nbad=$((nbad + 1)) @@ -314,6 +383,17 @@ differ alpha beta alphaua selftest_queue "refused: /loop1.hts and /loop2.hts cover the same site with a different user agent or headers" \ bundle "$dir" 'http://x.test/#httrack-bundle=loop1' \ 'http://127.0.0.1/#httrack-bundle=loop2' +# A cookie of the same name on another path or domain is another cookie. +bundle clash1 '# HTTrack bundle 1' cookies "$(jar alpha.test SID one)" +bundle clash2 '# HTTrack bundle 1' cookies \ + ".alpha.test${tab}TRUE${tab}/x${tab}FALSE${tab}0${tab}SID${tab}two" +bundle clash3 '# HTTrack bundle 1' cookies "$(jar www.alpha.test SID three)" +selftest_queue_lines "cookie .alpha.test SID=two +cookie .alpha.test SID=one +cookie .www.alpha.test SID=three" bundle "$dir" \ + 'http://alpha.test/#httrack-bundle=clash1' \ + 'http://www.alpha.test/#httrack-bundle=clash2' \ + 'http://x.alpha.test/#httrack-bundle=clash3' selftest_queue "refused: /alphasid.hts, line 6: a cookie has another value in another bundle" \ bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' \ 'https://alpha.test/#httrack-bundle=alphasid' @@ -328,6 +408,21 @@ for want in 'alpha.hts, line 7: unknown key$' 'alpha.hts, line 8: unknown key na 'alpha.hts: 1 cookie\(s\) outside its scope left out'; do grep -qE "$want" <<<"$out" || fail "no warning [$want]: $out" done +bundle warns '# HTTrack bundle 1' 'headers X: 1' 'abc123 v' \ + "$(repeat_chars 33 k) v" "$(repeat_chars 32 k) v" cookies \ + ".alpha.test${tab}TRUE${tab}/${tab}FALSE${tab}0${tab}A" \ + "$(jar alpha.test B b)${tab}x${tab}y" \ + ".alpha.test${tab}TRUE${tab}/${tab}FALSE${tab}soon${tab}C${tab}c" \ + "$(jar "$(repeat_chars 300 d).alpha.test" D d)" +out=$(httrack -#test=bundle "$dir" 'https://www.alpha.test/#httrack-bundle=warns' 2>&1) +for want in 'line 2: unknown key headers' 'line 3: unknown key' \ + 'line 4: unknown key' "line 5: unknown key $(repeat_chars 32 k)" \ + 'line 7: a cookie line has fewer than 7 fields or no name' \ + 'line 8: a cookie line has more than 7 fields' \ + 'line 9: a cookie expiry is not a whole number' \ + 'line 10: a cookie field is too long'; do + grep -qE "/warns\.hts, $want\$" <<<"$out" || fail "no warning [$want]: $out" +done secret=ZQXSECRET sec() { # sec NAME LINE... -- a bundle whose lines hold the secret local name=$1 @@ -342,13 +437,16 @@ sec s5 cookies "#$secret" "$(jar alpha.test "$secret" 1)" \ "$(jar alpha.test "$secret" 2)" "$(jar alpha.test "$secret" "$(repeat_chars 8000 x)")" \ "$(jar alpha.test "$secret" 1)$(printf '\001')" sec control "url http://www.alpha.test/?t=$secret" -for s in s1 s2 s3 s4 s5 control; do +sec keys "token $secret" +for s in s1 s2 s3 s4 s5 keys control; do out=$(httrack -#test=bundle "$dir" "https://www.alpha.test/#httrack-bundle=$s" 2>&1) case $s in control) grep -qF "$secret" <<<"$out" || fail "the probe cannot see a secret: $out" ;; *) grep -qF "$secret" <<<"$out" && fail "$s shows the secret: $out" ;; esac grep -qF "/$s.hts" <<<"$out" || test $s = control || fail "$s: no message: $out" + test $s != keys || grep -qE '/keys\.hts, line 2: unknown key token$' <<<"$out" || + fail "the key alone is not named: $out" done # A bad name on the command line stops the run before it starts. diff --git a/tests/656_local-bundles.test b/tests/656_local-bundles.test index ac2ca46cb..83689d5ac 100644 --- a/tests/656_local-bundles.test +++ b/tests/656_local-bundles.test @@ -63,13 +63,13 @@ hit() { # hit WHAT FROM LINE -- the server logged LINE after line FROM command grep -qxF "HIT $3" "$tmpdir/new.out" || fail_dump "$1: no request [HIT $3]" "$tmpdir/new.out" } -dated() { # dated BUNDLE-DATE JAR-DATE - touch -d "$1" "$alpha" - touch -d "$2" "$jarfile" +dated() { # dated BUNDLE-TIME JAR-TIME -- times as touch -t takes them + touch -t "$1" "$alpha" + touch -t "$2" "$jarfile" } # The jar is newer than the bundle, so its SID wins. --bundle-dir is relative. -dated '2026-01-01 10:00:00' '2026-01-02 10:00:00' +dated 202601011000.00 202601021000.00 crawl "$tmpdir" -O one --bundle-dir logins \ 'http://www.alpha.test/#httrack-bundle=alpha.test' \ 'http://www.beta.test/#httrack-bundle=beta.test' \ @@ -90,6 +90,7 @@ n=$(hits) test "$n" -eq 9 || fail_dump "$n lines, expected PORT and 8 requests" "$tmpdir/srv.out" cmp -s "$tmpdir/alpha.orig" "$alpha" || fail "the crawl wrote to the bundle" doit=$tmpdir/one/hts-cache/doit.log +test -f "$doit" || fail "no $doit" command grep -qF -- '--bundle-dir logins' "$doit" && fail_dump "doit.log kept a relative --bundle-dir" "$doit" # The suffix is cut before any URL reaches a file name, a log or the index. @@ -124,14 +125,14 @@ update() { # update WANT-SID hit "--update with SID=$1" "$from" "www.alpha.test/ $a=$1 x=-" } refresh -dated '2026-01-01 10:00:00' '2026-01-02 10:00:00' +dated 202601011000.00 202601021000.00 update alpha-server # a new capture is newer than the jar, so the fresh login wins -dated '2026-01-03 10:00:00' '2026-01-02 10:00:00' +dated 202601031000.00 202601021000.00 update alpha-captured # and so it does on a tie, where a coarse clock cannot order the two refresh -dated '2026-01-04 10:00:00' '2026-01-04 10:00:00' +dated 202601041000.00 202601041000.00 update alpha-captured # A -%L list names a bundle too, and it resolves in the project folder. @@ -154,33 +155,56 @@ printf '# HTTrack bundle 1\nua U\rX-Injected: 1\nheader X-A: 1\rCookie: forged=1 >"$tmpdir/two/cr.hts" printf '%s\n' '# HTTrack bundle 1' cookies "$(jar beta.test SID "v$(printf '\033')x")" \ >"$tmpdir/two/esc.hts" -printf '%s\n' 'http://www.beta.test/#httrack-bundle=../beta.test' >"$tmpdir/bad.txt" +printf '%s\n' '# HTTrack bundle 1' cookies \ + "$(jar beta.test SID "$(repeat_chars 8100 v)")" >"$tmpdir/two/long.hts" +badlist=$(nativepath "$tmpdir/bad.txt") +printf '%s\n' 'http://www.beta.test/#httrack-bundle=../beta.test' >"$badlist" stops=0 -stop() { # stop MESSAGE [httrack args...] +stop() { # stop MESSAGE [httrack args...] -- MESSAGE is a whole line of output local want=$1 rc=0 shift : >"$log" crawl "$tmpdir" -O two "$@" || rc=$? test "$rc" -ne 0 || fail "the run did not stop: $*" - command grep -qF "$want" "$log" || fail_dump "no message [$want]" "$log" + command grep -qxF "$want" "$log" || fail_dump "no message [$want]" "$log" stops=$((stops + 1)) } before=$(hits) -stop 'nosuch.hts: cannot read the login bundle' \ +stop 'Login bundle: two/nosuch.hts: cannot read the login bundle: No such file or directory' \ 'http://www.alpha.test/#httrack-bundle=nosuch' -command grep -qxF 'Login bundle: two/nosuch.hts: cannot read the login bundle: No such file or directory' "$log" || - fail_dump "the message changed" "$log" -stop 'line 1: invalid login bundle name' -%L "$(nativepath "$tmpdir/bad.txt")" -stop 'secret.hts, line 2: refused header: a proxy header is not for the site' \ +stop "\"$badlist\", line 1: invalid login bundle name: use letters, digits, '.', '_' and '-'" \ + -%L "$badlist" +stop 'Login bundle: two/secret.hts, line 2: refused header: a proxy header is not for the site' \ 'http://www.alpha.test/#httrack-bundle=secret' -stop 'cr.hts, line 2: holds a control byte' \ +stop 'Login bundle: two/cr.hts, line 2: holds a control byte' \ 'http://www.alpha.test/#httrack-bundle=cr' -stop 'esc.hts, line 3: a cookie line holds a control byte' \ - 'http://www.beta.test/#httrack-bundle=esc' -test "$stops" -eq 5 || fail "$stops stops ran, expected 5" +for b0 in -b1 -b0; do + stop 'Login bundle: two/esc.hts, line 3: a cookie line holds a control byte' \ + "$b0" 'http://www.beta.test/#httrack-bundle=esc' +done +stop 'Login bundle: two/long.hts, line 3: a cookie line is too long' \ + 'http://www.beta.test/#httrack-bundle=long' +test "$stops" -eq 7 || fail "$stops stops ran, expected 7" test "$(hits)" -eq "$before" || fail_dump "a refused bundle still crawled" "$tmpdir/srv.out" for f in "$log" "$tmpdir/two/hts-log.txt"; do test -f "$f" || fail "no $f" command grep -qF planted-secret "$f" && fail_dump "$f shows the secret" "$f" done -true + +# The identity is per request host: it drops on a redirect away from the +# bundle's site and returns on one back. A userinfo that spells the site's name +# gets none of it. +before=$(hits) +crawl "$tmpdir" -O three --bundle-dir logins '+*evil.test/*' \ + '+*sub.alpha.test/*' 'http://www.alpha.test/out#httrack-bundle=alpha.test' \ + 'http://www.alpha.test@evil.test/' || fail "the redirect crawl failed" +a='ua=AlphaUA/1.0 auth=Bearer alpha-token cookie=PREF=alpha-pref; SID=alpha-captured x=-' +g='ua=GlobalUA/9 auth=- cookie=- x=1' +hit "redirect start" "$before" "www.alpha.test/out $a" +hit "redirect out" "$before" "evil.test/back $g" +hit "redirect back" "$before" "sub.alpha.test/home $a" +# the userinfo is the user's own Basic login, sent as given +hit "userinfo" "$before" \ + "evil.test/ ua=GlobalUA/9 auth=Basic d3d3LmFscGhhLnRlc3Q= cookie=- x=1" +test "$(($(hits) - before))" -eq 4 || + fail_dump "redirect crawl: $(($(hits) - before)) requests, expected 4" "$tmpdir/srv.out" diff --git a/tests/bundle-server.py b/tests/bundle-server.py index f68915003..df83c3c6c 100644 --- a/tests/bundle-server.py +++ b/tests/bundle-server.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Three sites behind one proxy port, for 656_local-bundles. +"""Several sites behind one proxy port, for 656_local-bundles. httrack reaches it as its -P proxy, so the absolute request URL names the site. Every request prints one "HIT ua=<...> auth=<...> @@ -22,6 +22,13 @@ "cdn.gamma.test/b.gif": "GIF89a", "github.io/": 'x', "attacker.github.io/x.html": "another user's site", + "evil.test/": "another site", + "sub.alpha.test/home": "back on the alpha site", +} +# out of the bundle's site and back in +REDIRECTS = { + "www.alpha.test/out": "http://evil.test/back", + "evil.test/back": "http://sub.alpha.test/home", } lock = threading.Lock() @@ -48,6 +55,12 @@ def do_GET(self): ), flush=True, ) + if key in REDIRECTS: + self.send_response(302) + self.send_header("Location", REDIRECTS[key]) + self.send_header("Content-Length", "0") + self.end_headers() + return body = PAGES.get(key, "") ctype = "image/gif" if key.endswith(".gif") else "text/html" if ctype == "text/html": From 8f0873dd337f96e1188e27a069b7b88e81d211e1 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 18:51:58 +0200 Subject: [PATCH 05/10] Add floors and edge rows to the login bundle tests Floor the queued rows in 655, accept a 4096-byte cookie pair, time 2000 chained bundles, print a skip line for the sub-second order row, and shorten three comments. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Xavier Roche --- src/htsbundle.c | 4 ++-- src/htscoremain.c | 3 +-- src/htsopt.h | 5 ++--- tests/655_engine-bundle.test | 22 ++++++++++++++++++++++ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/htsbundle.c b/src/htsbundle.c index 0c31a765e..81cb20d48 100644 --- a/src/htsbundle.c +++ b/src/htsbundle.c @@ -145,8 +145,8 @@ static const char *bundle_name_refusal(const char *name) { return NULL; } -/* Cut the bundle suffix off url in place and copy its name into name, or "" - when url has none. NULL, or why the name is refused and url is unchanged. */ +/* Cut the suffix off url into name ("" if none). Returns NULL, or why the name + * is refused. */ static const char *bundle_cut(char *url, char *name, size_t name_size) { char *const at = strstr(url, HTS_BUNDLE_SUFFIX); const char *why; diff --git a/src/htscoremain.c b/src/htscoremain.c index bccac89fe..d6a3044c7 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -592,8 +592,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { int httrack_logmode = 3; // ONE log file ensureUrlCapacity(url, url_sz, 65536); - /* a previous run's bundles and folder are not this one's, but only a library - caller loads twice on one opt */ + /* a library caller may run twice on one opt */ hts_bundle_free(opt); StringCopy(opt->bundle_dir, ""); diff --git a/src/htsopt.h b/src/htsopt.h index 4d44b93f2..150571f91 100644 --- a/src/htsopt.h +++ b/src/htsopt.h @@ -657,9 +657,8 @@ struct httrackp { line, and the last one wins. Tail: ABI */ hts_boolean accept_given; /**< Did the user give -%a? Tail: ABI */ hts_boolean lang_given; /**< Did the user give -%l? Tail: ABI */ - String bundle_dir; /**< Where login bundle names resolve, or empty for - the project folder. Only --bundle-dir sets it, and - each run resets it. Tail: ABI */ + String bundle_dir; /**< --bundle-dir folder, or empty. Reset per run. Tail: + ABI */ void *bundle_state; /**< hts_bundle_state*, or NULL. Live state, so copy_htsopt must leave it alone. Tail: ABI */ }; diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index a6bacc4fa..6d9ce07e1 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -194,6 +194,8 @@ if touch -d '2026-01-01T10:00:00.5' "$tmpdir/subsecond" 2>/dev/null && test "$tmpdir/subsecond" -nt "$tmpdir/order3.txt"; then order order4 202601011000.00 202601011000.00 jar touch -d '2026-01-01T10:00:00.5' "$tmpdir/order4.txt" +else + echo "skipping the sub-second jar order row: touch cannot set fractions" >&2 fi # A control byte in a -%K or cookies.txt jar loads as it always did. @@ -292,6 +294,11 @@ jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s' "$long")" 'a cookie l # past the bytes the reader keeps, the length alone stops it jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s%s\001' "$long" "$long")" 'a cookie line is too long' jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s' "$(repeat_chars 4094 v)")" 'a cookie name and value pass 4096 bytes' +# name and value of exactly 4096 bytes are accepted +ok4096=$(repeat_chars 4093 v) +bundle ok4096 '# HTTrack bundle 1' cookies "$(jar alpha.test SID "$ok4096")" +selftest_queue_lines "cookie .alpha.test SID=$ok4096" bundle "$dir" \ + 'http://www.alpha.test/#httrack-bundle=ok4096' bundle b0ok '# HTTrack bundle 1' 'ua B0' cookies "$(jar alpha.test SID s)" selftest_queue 'www.alpha.test ua=B0 headers=global' bundle "$dir" \ 'http://www.alpha.test/#httrack-bundle=b0ok' host:www.alpha.test b0 @@ -398,6 +405,21 @@ selftest_queue "refused: /alphasid.hts, line 6: a cookie has another value in an bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' \ 'https://alpha.test/#httrack-bundle=alphasid' +# 2000 bundles chained by overlapping scopes load in seconds, not cubic time. +chain=() +for i in $(seq 1 2000); do + printf '# HTTrack bundle 1\nua U\nscope h%d.test\nscope h%d.test\n' \ + "$i" "$((i + 1))" >"$tmpdir/ch$i.hts" + chain+=("http://h$i.test/#httrack-bundle=ch$i") +done +t0=$SECONDS +assert_selftest 'h1.test ua=U headers=global' bundle "$dir" "${chain[@]}" host:h1.test +test "$((SECONDS - t0))" -lt 10 || + fail "2000 chained bundles took $((SECONDS - t0))s to load" + +# The real count of queued rows, so a deleted row cannot pass unseen. +test "${#SELFTEST_WANT[@]}" -eq 139 || + fail "expected 139 queued rows, got ${#SELFTEST_WANT[@]}" selftest_run_queued # Warnings name the line, and never a byte of it that could be a secret: a From 7f7542cb6e417a5741740fac60516cfef515b698 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 18:57:15 +0200 Subject: [PATCH 06/10] Time the bundle chain row against a smaller chain The 2000-chain row now compares its load time with a 250-chain load in the same run, so slow legs do not trip an absolute bound. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Xavier Roche --- tests/655_engine-bundle.test | 46 +++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index 6d9ce07e1..10326da10 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -5,6 +5,8 @@ set -euo pipefail +# TEST_TIMEOUT_AT_LEAST: 300 + # shellcheck source=tests/testlib.sh . "${0%"${0##*/}"}./testlib.sh" @@ -405,17 +407,39 @@ selftest_queue "refused: /alphasid.hts, line 6: a cookie has another value in an bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' \ 'https://alpha.test/#httrack-bundle=alphasid' -# 2000 bundles chained by overlapping scopes load in seconds, not cubic time. -chain=() -for i in $(seq 1 2000); do - printf '# HTTrack bundle 1\nua U\nscope h%d.test\nscope h%d.test\n' \ - "$i" "$((i + 1))" >"$tmpdir/ch$i.hts" - chain+=("http://h$i.test/#httrack-bundle=ch$i") -done -t0=$SECONDS -assert_selftest 'h1.test ua=U headers=global' bundle "$dir" "${chain[@]}" host:h1.test -test "$((SECONDS - t0))" -lt 10 || - fail "2000 chained bundles took $((SECONDS - t0))s to load" +# Chained bundles must load in linear time: 2000 may cost at most 64 times 250 +# (linear is about 8, cubic about 512). Relative, so slow legs pass. +now_ms() { + if perl -MTime::HiRes=time -e 1 2>/dev/null; then + perl -MTime::HiRes=time -e 'printf "%d", time * 1000' + elif command -v python3 >/dev/null; then + python3 -c 'import time; print(int(time.time() * 1000))' + else + return 1 + fi +} +chain_ms() { # chain_ms N -- milliseconds to load N chained bundles + local n=$1 i t0 t1 chain=() + for i in $(seq 1 "$n"); do + printf '# HTTrack bundle 1\nua U\nscope h%d.test\nscope h%d.test\n' \ + "$i" "$((i + 1))" >"$tmpdir/ch$i.hts" + chain+=("http://h$i.test/#httrack-bundle=ch$i") + done + t0=$(now_ms) + assert_selftest 'h1.test ua=U headers=global' bundle "$dir" "${chain[@]}" host:h1.test + t1=$(now_ms) + echo "$((t1 - t0))" +} +if now_ms >/dev/null; then + t250=$(chain_ms 250) + test "$t250" -ge 50 || t250=50 + skip_if_out_of_budget 1 "$((t250 * 8 / 1000 + 1))" "the 2000-bundle chain row" + t2000=$(chain_ms 2000) + test "$t2000" -le "$((t250 * 64))" || + fail "2000 chained bundles took ${t2000}ms against ${t250}ms for 250: not linear" +else + echo "skipping the chained bundle timing row: no perl or python3" >&2 +fi # The real count of queued rows, so a deleted row cannot pass unseen. test "${#SELFTEST_WANT[@]}" -eq 139 || From f3fbf8a6b908cfef0f21d41c4a03a395addb8a7b Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 20:17:49 +0200 Subject: [PATCH 07/10] Make the bundle tests pass on macOS bash 3.2 and on Windows Expand a possibly empty array the bash 3.2 way under set -u, skip the 2000-URL timing row where the Windows command line is too short, and write the bad -%L list through the shell path rather than the native one. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- tests/655_engine-bundle.test | 7 +++++-- tests/656_local-bundles.test | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index 10326da10..01f9cd46a 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -382,7 +382,7 @@ differ() { # differ A B [C] -- A and the last one overlap with other identities test -z "${3:-}" || more=("http://beta.test/#httrack-bundle=$2") selftest_queue "refused: /$1.hts and /$last.hts cover the same site with a different user agent or headers" \ bundle "$dir" "https://www.alpha.test/#httrack-bundle=$1" \ - "${more[@]}" "https://alpha.test/#httrack-bundle=$last" + ${more[@]+"${more[@]}"} "https://alpha.test/#httrack-bundle=$last" } differ alpha alphaua differ alpha alphaauth @@ -430,7 +430,10 @@ chain_ms() { # chain_ms N -- milliseconds to load N chained bundles t1=$(now_ms) echo "$((t1 - t0))" } -if now_ms >/dev/null; then +if target_is_windows; then + # 2000 URLs pass the 32767-character Windows command line + echo "skipping the chained bundle timing row on Windows" >&2 +elif now_ms >/dev/null; then t250=$(chain_ms 250) test "$t250" -ge 50 || t250=50 skip_if_out_of_budget 1 "$((t250 * 8 / 1000 + 1))" "the 2000-bundle chain row" diff --git a/tests/656_local-bundles.test b/tests/656_local-bundles.test index 83689d5ac..3842c842a 100644 --- a/tests/656_local-bundles.test +++ b/tests/656_local-bundles.test @@ -157,8 +157,8 @@ printf '%s\n' '# HTTrack bundle 1' cookies "$(jar beta.test SID "v$(printf '\033 >"$tmpdir/two/esc.hts" printf '%s\n' '# HTTrack bundle 1' cookies \ "$(jar beta.test SID "$(repeat_chars 8100 v)")" >"$tmpdir/two/long.hts" +printf '%s\n' 'http://www.beta.test/#httrack-bundle=../beta.test' >"$tmpdir/bad.txt" badlist=$(nativepath "$tmpdir/bad.txt") -printf '%s\n' 'http://www.beta.test/#httrack-bundle=../beta.test' >"$badlist" stops=0 stop() { # stop MESSAGE [httrack args...] -- MESSAGE is a whole line of output local want=$1 rc=0 From 8ad5e6949324edb653ca87e9497bbf5c35f92d20 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 20:51:44 +0200 Subject: [PATCH 08/10] Count the sub-second row only where it runs in the 655 row floor Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- tests/655_engine-bundle.test | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index 01f9cd46a..82e067f72 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -196,8 +196,10 @@ if touch -d '2026-01-01T10:00:00.5' "$tmpdir/subsecond" 2>/dev/null && test "$tmpdir/subsecond" -nt "$tmpdir/order3.txt"; then order order4 202601011000.00 202601011000.00 jar touch -d '2026-01-01T10:00:00.5' "$tmpdir/order4.txt" + want_rows=139 else echo "skipping the sub-second jar order row: touch cannot set fractions" >&2 + want_rows=138 fi # A control byte in a -%K or cookies.txt jar loads as it always did. @@ -445,8 +447,8 @@ else fi # The real count of queued rows, so a deleted row cannot pass unseen. -test "${#SELFTEST_WANT[@]}" -eq 139 || - fail "expected 139 queued rows, got ${#SELFTEST_WANT[@]}" +test "${#SELFTEST_WANT[@]}" -eq "$want_rows" || + fail "expected $want_rows queued rows, got ${#SELFTEST_WANT[@]}" selftest_run_queued # Warnings name the line, and never a byte of it that could be a secret: a From c908e383a2e6d66b224fdcf79acbb546032dceb2 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 21:37:15 +0200 Subject: [PATCH 09/10] Drop the CR the Windows build writes before matching bundle warnings Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- tests/655_engine-bundle.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index 82e067f72..4493eeb89 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -453,7 +453,7 @@ selftest_run_queued # Warnings name the line, and never a byte of it that could be a secret: a # key in lowercase letters alone is named. -out=$(httrack -#test=bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' 2>&1) +out=$(httrack -#test=bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' 2>&1 | tr -d '\r') for want in 'alpha.hts, line 7: unknown key$' 'alpha.hts, line 8: unknown key nakedkey$' \ 'alpha.hts, line 13: a cookie appears twice$' 'alpha.hts, line 14: an unknown comment$' \ 'alpha.hts: 1 cookie\(s\) outside its scope left out'; do @@ -465,7 +465,7 @@ bundle warns '# HTTrack bundle 1' 'headers X: 1' 'abc123 v' \ "$(jar alpha.test B b)${tab}x${tab}y" \ ".alpha.test${tab}TRUE${tab}/${tab}FALSE${tab}soon${tab}C${tab}c" \ "$(jar "$(repeat_chars 300 d).alpha.test" D d)" -out=$(httrack -#test=bundle "$dir" 'https://www.alpha.test/#httrack-bundle=warns' 2>&1) +out=$(httrack -#test=bundle "$dir" 'https://www.alpha.test/#httrack-bundle=warns' 2>&1 | tr -d '\r') for want in 'line 2: unknown key headers' 'line 3: unknown key' \ 'line 4: unknown key' "line 5: unknown key $(repeat_chars 32 k)" \ 'line 7: a cookie line has fewer than 7 fields or no name' \ @@ -490,7 +490,7 @@ sec s5 cookies "#$secret" "$(jar alpha.test "$secret" 1)" \ sec control "url http://www.alpha.test/?t=$secret" sec keys "token $secret" for s in s1 s2 s3 s4 s5 keys control; do - out=$(httrack -#test=bundle "$dir" "https://www.alpha.test/#httrack-bundle=$s" 2>&1) + out=$(httrack -#test=bundle "$dir" "https://www.alpha.test/#httrack-bundle=$s" 2>&1 | tr -d '\r') case $s in control) grep -qF "$secret" <<<"$out" || fail "the probe cannot see a secret: $out" ;; *) grep -qF "$secret" <<<"$out" && fail "$s shows the secret: $out" ;; From 227481c72b874927bf38786340b0eb94e0584bd7 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Thu, 22 Oct 2026 11:10:23 +0200 Subject: [PATCH 10/10] Teach the log backstop a bundle's header tokens and url-line passwords Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- tests/655_engine-bundle.test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test index 3e537048a..ddcfe462c 100644 --- a/tests/655_engine-bundle.test +++ b/tests/655_engine-bundle.test @@ -503,9 +503,10 @@ done # The log backstop learns a bundle's header token and url-line password. bundle hidden '# HTTrack bundle 1' 'header Authorization: Bearer BUNDLETOKEN123' \ 'url http://u:URLPASSWD9@www.alpha.test/x' -assert_selftest 'hide x *** y' bundle "$dir" \ +hidden_url='url http://u:URLPASSWD9@www.alpha.test/x' +assert_selftest "$hidden_url"$'\n''hide x *** y' bundle "$dir" \ 'https://www.alpha.test/#httrack-bundle=hidden' 'hide:x BUNDLETOKEN123 y' -assert_selftest 'hide see http://u:***@h/' bundle "$dir" \ +assert_selftest "$hidden_url"$'\n''hide see http://u:***@h/' bundle "$dir" \ 'https://www.alpha.test/#httrack-bundle=hidden' 'hide:see http://u:URLPASSWD9@h/' # A bad name on the command line stops the run before it starts.