|
@@ -2465,6 +2477,59 @@ FILES
the Cookie header from DevTools "Copy as cURL",
pasted here.
+<name>.hts
+
+A login bundle, named by a
+start URL that ends with
+#httrack-bundle=<name>, such as
+https://example.com/members/#httrack-bundle=login1.
+The name holds letters, digits, ., _ and
+- only. HTTrack reads the file in the project folder,
+or in the folder given to --bundle-dir <dir>,
+and never writes it. doit.log keeps that folder as an
+absolute path. HTTrack reads the suffix on the command line,
+in a -%L list and in doit.log, never on a crawled
+link. A missing file, or one that does not parse, stops the
+run before any request.
+
+The first line
+is # HTTrack bundle 1. Then come url lines,
+each one more start URL, at most one ua line, and
+header and scope lines. A cookies line
+ends them, and a Netscape cookie jar follows it.
+
+The
+bundle’s user agent and headers replace -F and
+-%X for each host in its scope, and only there. A
+bundle with header lines but no ua line sends the
+-F user agent. When any start URL naming the bundle,
+or any of its url lines, is https, its headers go over https
+alone, and so do those of the bundles it merges with. By
+default the scope is the registrable domains of the start
+URLs naming the bundle and of its url lines. A host with no
+registrable domain, such as a public suffix, a single label
+or an address, covers itself alone. Scope lines replace that
+default. HTTrack drops a cookie outside the scope and gives
+the count in one warning. HTTrack refuses a Host, Cookie,
+Proxy-* or framing header, a scope line naming a public
+suffix, and a line that holds a control byte or is too long,
+in the cookie jar too and even under -b0. A second
+ua line, header lines over 8192 bytes, or a full
+cookie jar stops the run too. A repeated cookie line or an
+unknown key only warns, and so does an unknown # line in the
+cookie jar. Before the cookies line, a # line is a
+comment. Messages give the file and the line number, never a
+value. Two bundles whose scopes overlap must give the same
+user agent, headers and cookie values, or the run stops.
+
+The
+bundle’s cookies go into the cookie jar, which HTTrack
+writes at the end of the crawl. Of the bundle and the jar,
+the newer file wins a cookie both hold. A new capture
+therefore wins over the jar, and a cookie the server
+refreshed wins over an older bundle. On equal times the
+bundle wins.
+
A running mirror
also watches its own output directory for these. Creating
one is how a script drives an engine it has no other handle
diff --git a/man/httrack.1 b/man/httrack.1
index ed2adc446..ef0a8194c 100644
--- a/man/httrack.1
+++ b/man/httrack.1
@@ -3,7 +3,7 @@
.\"
.\" This file is generated by man/makeman.sh; do not edit by hand.
.\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "19 October 2026" "httrack website copier"
+.TH httrack 1 "21 October 2026" "httrack website copier"
.SH NAME
httrack \- offline browser : copy websites to a local directory
.SH SYNOPSIS
@@ -242,6 +242,10 @@ links conversion to UTF\-8 (\-\-utf8\-conversion[=0])
accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
.IP \-%K
use a Netscape cookies.txt as the jar, rewritten at the end (\-\-cookies\-file )
+.br
+#httrack\-bundle= crawls with the user agent,
+.br
+headers and cookies of the login bundle .hts, read from the project folder or from \-\-bundle\-dir
.IP \-%Y
explain which filter rule accepts or rejects a URL, then exit (\-\-why )
.IP \-u
@@ -582,6 +586,72 @@ start URL and its subdomains, or to that host alone when it is an address or a
public suffix. They are Secure when that URL is https. HTTrack never rewrites
such a file. HttpOnly cookies are not in it. A cookies.txt export has them,
and so does the Cookie header from DevTools "Copy as cURL", pasted here.
+.TP
+.I .hts
+A login bundle, named by a start URL that ends with
+.IR #httrack\-bundle= ,
+such as
+.IR https://example.com/members/#httrack\-bundle=login1 .
+The name holds letters, digits,
+.IR . ,
+.I _
+and
+.I \-
+only. HTTrack reads the file in the project folder, or in the folder given to
+.IR "\-\-bundle\-dir " ,
+and never writes it. doit.log keeps that folder as an absolute path. HTTrack
+reads the suffix on the command line, in a
+.I \-%L
+list and in doit.log, never on a crawled link. A missing file, or one that
+does not parse, stops the run before any request.
+.IP
+The first line is
+.IR "# HTTrack bundle 1" .
+Then come
+.I url
+lines, each one more start URL, at most one
+.I ua
+line, and
+.I header
+and
+.I scope
+lines. A
+.I cookies
+line ends them, and a Netscape cookie jar follows it.
+.IP
+The bundle's user agent and headers replace
+.I \-F
+and
+.I \-%X
+for each host in its scope, and only there. A bundle with header lines but no
+.I ua
+line sends the
+.I \-F
+user agent. When any start URL naming the bundle, or any of its url lines, is
+https, its headers go over https alone, and so do those of the bundles it
+merges with. By default the scope is the registrable domains of the start
+URLs naming the bundle and of its url lines. A host with no registrable domain,
+such as a public suffix, a single label or an address, covers itself alone.
+Scope lines replace that default. HTTrack drops a cookie outside the scope and
+gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or
+framing header, a scope line naming a public suffix, and a line that holds a
+control byte or is too long, in the cookie jar too and even under
+.IR \-b0 .
+A second
+.I ua
+line, header lines over 8192 bytes, or a full cookie jar stops the run too. A
+repeated cookie line or an unknown key only warns, and so does an unknown #
+line in the cookie jar. Before the
+.I cookies
+line, a # line is a comment. Messages give the file and the line number, never
+a value. Two bundles whose
+scopes overlap must give the same user agent, headers and cookie values, or the
+run stops.
+.IP
+The bundle's cookies go into the cookie jar, which HTTrack writes at the end of
+the crawl. Of the bundle and the jar, the newer file wins a cookie both hold.
+A new capture therefore wins over the jar, and a cookie the server refreshed
+wins over an older bundle. On equal times the bundle wins.
.PP
A running mirror also watches its own output directory for these. Creating one
is how a script drives an engine it has no other handle on. On Windows, which
diff --git a/man/httrack.tmpl b/man/httrack.tmpl
index 1e3b818a1..723d93aa6 100644
--- a/man/httrack.tmpl
+++ b/man/httrack.tmpl
@@ -69,6 +69,72 @@ start URL and its subdomains, or to that host alone when it is an address or a
public suffix. They are Secure when that URL is https. HTTrack never rewrites
such a file. HttpOnly cookies are not in it. A cookies.txt export has them,
and so does the Cookie header from DevTools "Copy as cURL", pasted here.
+.TP
+.I .hts
+A login bundle, named by a start URL that ends with
+.IR #httrack\-bundle= ,
+such as
+.IR https://example.com/members/#httrack\-bundle=login1 .
+The name holds letters, digits,
+.IR . ,
+.I _
+and
+.I \-
+only. HTTrack reads the file in the project folder, or in the folder given to
+.IR "\-\-bundle\-dir " ,
+and never writes it. doit.log keeps that folder as an absolute path. HTTrack
+reads the suffix on the command line, in a
+.I \-%L
+list and in doit.log, never on a crawled link. A missing file, or one that
+does not parse, stops the run before any request.
+.IP
+The first line is
+.IR "# HTTrack bundle 1" .
+Then come
+.I url
+lines, each one more start URL, at most one
+.I ua
+line, and
+.I header
+and
+.I scope
+lines. A
+.I cookies
+line ends them, and a Netscape cookie jar follows it.
+.IP
+The bundle's user agent and headers replace
+.I \-F
+and
+.I \-%X
+for each host in its scope, and only there. A bundle with header lines but no
+.I ua
+line sends the
+.I \-F
+user agent. When any start URL naming the bundle, or any of its url lines, is
+https, its headers go over https alone, and so do those of the bundles it
+merges with. By default the scope is the registrable domains of the start
+URLs naming the bundle and of its url lines. A host with no registrable domain,
+such as a public suffix, a single label or an address, covers itself alone.
+Scope lines replace that default. HTTrack drops a cookie outside the scope and
+gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or
+framing header, a scope line naming a public suffix, and a line that holds a
+control byte or is too long, in the cookie jar too and even under
+.IR \-b0 .
+A second
+.I ua
+line, header lines over 8192 bytes, or a full cookie jar stops the run too. A
+repeated cookie line or an unknown key only warns, and so does an unknown #
+line in the cookie jar. Before the
+.I cookies
+line, a # line is a comment. Messages give the file and the line number, never
+a value. Two bundles whose
+scopes overlap must give the same user agent, headers and cookie values, or the
+run stops.
+.IP
+The bundle's cookies go into the cookie jar, which HTTrack writes at the end of
+the crawl. Of the bundle and the jar, the newer file wins a cookie both hold.
+A new capture therefore wins over the jar, and a cookie the server refreshed
+wins over an older bundle. On equal times the bundle wins.
.PP
A running mirror also watches its own output directory for these. Creating one
is how a script drives an engine it has no other handle on. On Windows, which
diff --git a/src/Makefile.am b/src/Makefile.am
index 4107093ee..f03f3b885 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -86,6 +86,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
htsback_selftest.c htscache_selftest.c htscharset_selftest.c \
htscmdline_selftest.c \
htscookie_selftest.c htsdns_selftest.c htsfetch_selftest.c htspsl_selftest.c \
+ htsbundle_selftest.c \
htsfilters_selftest.c \
htsheader_selftest.c htsio_selftest.c htslib_selftest.c \
htslines_selftest.c \
@@ -96,14 +97,14 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
htscatchurl.c htsfetch.c htspsl.c htspsl_data.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
htscmdline.c htshelp.c htslib.c htslines.c htsurlport.c htsdate.c htscoremain.c \
htsname.c htsrobots.c htstools.c htsrandom.c htswizard.c \
- htsalias.c htsthread.c htsindex.c htsbauth.c \
+ htsalias.c htsthread.c htsindex.c htsbauth.c htsbundle.c \
htscrashtest.c \
htsmd5.c htsnet.c htscodec.c htswarc.c htschanges.c htssinglefile.c htssitemap.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
htsmodules.c htscharset.c punycode.c htsencoding.c htsescape.c htssniff.c \
md5.c \
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h htsdate.h \
- htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htscookie_selftest.h htsdns_selftest.h htsparse_selftest.h \
+ htsbasenet.h htsbauth.h htsbundle.h htscache.h htscache_selftest.h htscookie_selftest.h htsdns_selftest.h htsparse_selftest.h \
htsselftest.h htsselftest_int.h htsselftest_util.h htscatchurl.h \
htscmdline.h htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
diff --git a/src/htsalias.c b/src/htsalias.c
index aa9b6a7b4..67afa64d1 100644
--- a/src/htsalias.c
+++ b/src/htsalias.c
@@ -165,6 +165,8 @@ const char *hts_optalias[][4] = {
"longest Retry-After delay obeyed, in seconds"},
{"mptcp", "-@m", "onoff",
"carry connections over Multipath TCP where the system offers it"},
+ {"bundle-dir", "--bundle-dir", "param1",
+ "folder where #httrack-bundle= names are read, not the project folder"},
{"generate-errors", "-o", "level", ""},
{"do-not-generate-errors", "-o0", "single", ""},
{"purge-old", "-X", "onoff", ""},
@@ -347,6 +349,7 @@ static const char *const dashvalue_opt[] = {
"-%L", /* list */
"-%S", /* urllist */
"-%rf", /* warc-file */
+ "--bundle-dir",
"-F", /* user-agent */
NULL
};
diff --git a/src/htsback.c b/src/htsback.c
index 63e7bd886..02a53988b 100644
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -36,6 +36,7 @@ Please visit our Website: http://www.httrack.com
/* specific definitions */
#include "htsnet.h"
+#include "htsbundle.h"
#include "htscore.h"
#include "htsio.h"
#include "htswarc.h"
@@ -3089,6 +3090,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
back[p].r.req.accept = StringBuff(opt->accept);
back[p].r.req.headers = StringBuff(opt->headers);
back[p].r.req.user_agent_send = opt->user_agent_send;
+ /* a login bundle's site gets the bundle's UA and headers */
+ hts_bundle_identity(opt, adr, &back[p].r.req);
// et http11
back[p].r.req.http11 = back[p].http11;
back[p].r.req.nocompression = opt->nocompression;
diff --git a/src/htsbauth.c b/src/htsbauth.c
index c6fa2cc1a..502e87aea 100644
--- a/src/htsbauth.c
+++ b/src/htsbauth.c
@@ -36,6 +36,7 @@ Please visit our Website: http://www.httrack.com
#include "htsbauth.h"
#include
+#include
#include
/* specific definitions */
@@ -44,6 +45,7 @@ Please visit our Website: http://www.httrack.com
#include "htslines.h"
#include "htscore.h"
#include "htspsl.h"
+#include "htsbundle.h"
#include "htsdate.h"
#include "htscookie_selftest.h"
#ifdef _WIN32
@@ -97,8 +99,6 @@ hts_boolean cookie_host(const char *adr, char *dst, size_t dst_size) {
return HTS_TRUE;
}
-static char *cookie_find_exact(char *s, const char *cook_name,
- const char *domain, const char *path);
static hts_boolean cookie_domain_is_suffix(const char *domain);
/* See htsbauth.h. Records are kept longest path first. */
@@ -219,10 +219,10 @@ static hts_boolean cookie_same_domain(const char *a, const char *b) {
return strcmpnocase(a, b) == 0;
}
-/* Return the record with this name, domain and path, or NULL. The host-only
- flag is not part of that identity (RFC 6265 5.3 step 11). */
-static char *cookie_find_exact(char *s, const char *cook_name,
- const char *domain, const char *path) {
+/* See htsbundle.h. The host-only flag is not part of a record's identity
+ (RFC 6265 5.3 step 11). */
+char *cookie_find_exact(char *s, const char *cook_name, const char *domain,
+ const char *path) {
char buffer[8192];
char *a;
@@ -452,19 +452,41 @@ char *cookie_nextfield(char *a) {
return a;
}
+/* Warn about the line just read, with fmt for -%K and cookies.txt. A bundle's
+ warning gives the line number and reason, never a byte that can be secret. */
+static void cookie_warn(httrackp *opt, const cookie_filter *filter,
+ const char *given, const char *reason, const char *fmt,
+ ...) HTS_PRINTF_FUN(5, 6);
+
+static void cookie_warn(httrackp *opt, const cookie_filter *filter,
+ const char *given, const char *reason, const char *fmt,
+ ...) {
+ va_list args;
+
+ if (filter != NULL) {
+ hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: %s", given,
+ filter->line, reason);
+ return;
+ }
+ va_start(args, fmt);
+ hts_log_vprint(opt, LOG_WARNING, fmt, args);
+ va_end(args);
+}
+
/* Copy Netscape-jar field into dst, or refuse the line and say so: a
shortened domain or path would silently change where the cookie is sent. */
static hts_boolean cookie_load_field(httrackp *opt, char *dst, size_t dst_size,
char *buffer, const char *line, int param,
- const char *field_name, const char *file) {
+ const char *field_name, const char *given,
+ const cookie_filter *filter) {
const char *const value = cookie_get(buffer, line, param);
const size_t len = strlen(value);
if (len >= dst_size) {
- hts_log_print(opt, LOG_WARNING,
- "Cookies: ignoring a line whose %s field is %d bytes "
- "(maximum %d): %s",
- field_name, (int) len, (int) dst_size - 1, file);
+ cookie_warn(opt, filter, given, "a cookie field is too long",
+ "Cookies: ignoring a line whose %s field is %d bytes "
+ "(maximum %d): %s",
+ field_name, (int) len, (int) dst_size - 1, given);
return HTS_FALSE;
}
strlcpybuff(dst, value, dst_size);
@@ -636,15 +658,12 @@ static hts_boolean cookie_is_flat(FILE *fp, int *quote) {
return flat;
}
-/* Copy the host of url, with or without a scheme, into dst as cookie_host
- does, and set *https. */
-static hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size,
- hts_boolean *https) {
+/* See htsbundle.h. */
+hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size) {
const char *const rest = jump_protocol_const(url);
const size_t len = strcspn(rest, "/?#");
char adr[HTS_URLMAXSIZE];
- *https = strfield(url, "https:") != 0;
if (len >= sizeof(adr))
return HTS_FALSE;
adr[0] = '\0';
@@ -707,14 +726,13 @@ static int cookie_load_flat(httrackp *opt, t_cookie *cookie, FILE *fp,
hts_boolean nul = HTS_FALSE, too_long = HTS_FALSE;
hts_boolean line_start = HTS_TRUE;
hts_boolean quoted = HTS_FALSE, quote_done = quote == 0;
- hts_boolean https = HTS_FALSE;
cookie_attr attr;
const char *domain;
int stored = 0, ignored = 0;
size_t i;
int c;
- if (!cookie_url_host(start_url, host, sizeof(host), &https)) {
+ if (!cookie_url_host(start_url, host, sizeof(host))) {
hts_log_print(opt, LOG_WARNING,
"Cookies: %s is a document.cookie string, but there is no "
"start URL to bind it to, so it is ignored",
@@ -726,7 +744,7 @@ static int cookie_load_flat(httrackp *opt, t_cookie *cookie, FILE *fp,
? NULL
: hts_psl_registrable_domain(host);
attr.host_only = domain == NULL;
- attr.secure = https;
+ attr.secure = strfield(start_url, "https:") != 0;
attr.http_only = HTS_FALSE;
attr.expires = 0;
if (domain == NULL)
@@ -801,6 +819,185 @@ static int cookie_load_flat(httrackp *opt, t_cookie *cookie, FILE *fp,
return COOKIE_LOAD_KEEP;
}
+/* Read the Netscape records from fp's position to its end into cookie, and
+ name the file as given in warnings. A UTF-8 BOM is skipped on the first line
+ when first_line is HTS_TRUE. filter, when not NULL, picks the records.
+ Returns what cookie_load_from does. */
+static int cookie_load_records(httrackp *opt, t_cookie *cookie, FILE *fp,
+ const char *given, hts_boolean first_line,
+ cookie_filter *filter) {
+ char buffer[8192];
+ int refused = 0;
+ hts_boolean keep = HTS_FALSE;
+ char BIGSTK line[8192];
+ const size_t line_max = 8000;
+ int lineno = 0;
+ // this load's records alone, so a line repeating one is caught
+ t_cookie *seen = (t_cookie *) calloct(1, sizeof(t_cookie));
+
+ if (seen != NULL) {
+ seen->max_len = sizeof(seen->data);
+ } else {
+ hts_log_print(opt, LOG_WARNING,
+ "Cookies: out of memory, so %s is left unchanged", given);
+ keep = HTS_TRUE;
+ }
+ // ferror ends the loop too, because a directory never reaches EOF
+ while (!feof(fp) && !ferror(fp) && strlen(cookie->data) < cookie->max_len &&
+ (filter == NULL || filter->why == NULL)) {
+ size_t len;
+
+ // len counts what the buffer cut too, so this catches that line
+ cookie_readline(fp, line, sizeof(line), &len);
+ if (lineno < INT_MAX)
+ lineno++;
+ if (filter != NULL)
+ filter->line++;
+ if (first_line && len >= sizeof(cookie_bom) - 1 &&
+ strncmp(line, cookie_bom, sizeof(cookie_bom) - 1) == 0) {
+ len -= sizeof(cookie_bom) - 1;
+ memmove(line, line + sizeof(cookie_bom) - 1,
+ strlen(line) + 1 - (sizeof(cookie_bom) - 1));
+ }
+ first_line = HTS_FALSE;
+ // a bundle is written for this run, so a damaged one stops it
+ if (filter != NULL &&
+ hts_bundle_has_control(line,
+ len < sizeof(line) ? len : sizeof(line) - 1)) {
+ filter->why = "a cookie line holds a control byte";
+ } else if (filter != NULL && len >= line_max) {
+ filter->why = "a cookie line is too long";
+ } else if (len >= line_max) {
+ hts_log_print(opt, LOG_WARNING,
+ "Cookies: ignoring an over-long line "
+ "(%d bytes, maximum %d): %s",
+ (int) len, (int) line_max - 1, given);
+ refused++;
+ } else if (memchr(line, '\0', len) != NULL ||
+ memchr(line, '\r', len) != NULL) {
+ hts_log_print(opt, LOG_WARNING,
+ "Cookies: ignoring a line holding a NUL or a CR: %s",
+ given);
+ refused++;
+ } else if (line[strspn(line, " \t")] != '\0') {
+ /* curl and browser exports mark an HttpOnly cookie this way */
+ static const char httponly[] = COOKIE_HTTPONLY_PREFIX;
+ const hts_boolean is_httponly =
+ strncmp(line, httponly, sizeof(httponly) - 1) == 0;
+ const char *const rec = is_httponly ? line + sizeof(httponly) - 1 : line;
+
+ if (rec[0] == '#') {
+ if (!cookie_known_comment(rec)) {
+ // no line text, because any part of it can hold a cookie value
+ cookie_warn(opt, filter, given, "an unknown comment",
+ "Cookies: ignoring an unknown comment on line %d of %s",
+ lineno, given);
+ refused++;
+ }
+ } else {
+ char domain[256]; // .netscape.com
+ char path[256]; // /
+ char BIGSTK cook_name[COOKIE_MAX_PAIR + 1]; // MYCOOK
+ char BIGSTK cook_value[8192]; // ID=toto,S=1234
+ const char *p;
+ int tabs = 0;
+
+#if DEBUG_COOK
+ printf("%s\n", rec);
+#endif
+ for (p = strchr(rec, '\t'); p != NULL; p = strchr(p + 1, '\t'))
+ tabs++;
+ if (tabs < 6 || !strnotempty(cookie_get(buffer, rec, 5))) {
+ cookie_warn(opt, filter, given,
+ "a cookie line has fewer than 7 fields or no name",
+ "Cookies: ignoring a line with fewer than 7 "
+ "fields or no name: %s",
+ given);
+ refused++;
+ } else if (tabs > 7 ||
+ (tabs == 7 && strcmp(cookie_get(buffer, rec, 7),
+ COOKIE_HTTPONLY_FIELD) != 0)) {
+ // a TAB in the value, because only HTTrack writes an eighth field
+ cookie_warn(opt, filter, given,
+ "a cookie line has more than 7 fields",
+ "Cookies: ignoring a line with more than 7 "
+ "fields: %s",
+ given);
+ refused++;
+ } else if (!cookie_is_expiry(cookie_get(buffer, rec, 4))) {
+ cookie_warn(opt, filter, given,
+ "a cookie expiry is not a whole number",
+ "Cookies: ignoring a line whose expiry is not a "
+ "whole number: %s",
+ given);
+ refused++;
+ } else if (cookie_load_field(opt, domain, sizeof(domain), buffer, rec,
+ 0, "domain", given, filter) &&
+ cookie_load_field(opt, path, sizeof(path), buffer, rec, 2,
+ "path", given, filter) &&
+ cookie_load_field(opt, cook_name, sizeof(cook_name), buffer,
+ rec, 5, "name", given, filter) &&
+ cookie_load_field(opt, cook_value, sizeof(cook_value),
+ buffer, rec, 6, "value", given, filter)) {
+ cookie_attr attr;
+ char host[256];
+ const hts_boolean repeated =
+ seen != NULL && cookie_host(domain, host, sizeof(host)) &&
+ cookie_find_exact(seen->data, cook_name, host, path) != NULL;
+
+ attr.host_only = cookie_is_host_only(rec);
+ attr.secure = strcmpnocase(cookie_get(buffer, rec, 3), "TRUE") == 0;
+ attr.http_only = is_httponly || tabs == 7;
+ attr.expires = cookie_record_expiry(rec);
+ // an expired one is skipped, because a browser drops it too
+ if ((attr.expires == 0 || attr.expires > (int64_t) time(NULL)) &&
+ (filter == NULL ||
+ filter->admit(filter, cookie, repeated, cook_name, cook_value,
+ domain, path))) {
+ // cookie_set stores it host-only
+ if (!attr.host_only && cookie_domain_is_suffix(domain))
+ cookie_warn(opt, filter, given,
+ "a cookie for a public suffix is now host-only",
+ "Cookies: '%.64s' for the public suffix %s is "
+ "now host-only: %s",
+ cook_name, domain, given);
+ // the last one wins, as curl reads it, but a rewrite loses one
+ if (repeated) {
+ cookie_warn(opt, filter, given, "a cookie appears twice",
+ "Cookies: '%.64s' appears twice in %s", cook_name,
+ given);
+ refused++;
+ }
+ if (cookie_set(cookie, cook_name, cook_value, domain, path,
+ &attr) != 0) {
+ // the jar caps a field tighter still, and a full one refuses
+ if (filter != NULL)
+ filter->why = "the cookie jar did not store a cookie";
+ else
+ hts_log_print(opt, LOG_WARNING,
+ "Cookies: the jar did not store '%.64s' from "
+ "%s",
+ cook_name, given);
+ refused++;
+ } else if (seen != NULL && !repeated) {
+ (void) cookie_set(seen, cook_name, "", domain, path, NULL);
+ }
+ }
+ } else {
+ refused++;
+ }
+ }
+ }
+ }
+ if (!feof(fp) && (filter == NULL || filter->why == NULL)) {
+ cookie_warn(opt, filter, given, "stopped reading before the end",
+ "Cookies: stopped reading before the end of %s", given);
+ refused++;
+ }
+ freet(seen);
+ return keep ? COOKIE_LOAD_KEEP : refused;
+}
+
/* See htsbauth.h. */
int cookie_load(httrackp *opt, t_cookie *cookie, const char *fpath,
const char *name) {
@@ -811,7 +1008,6 @@ int cookie_load(httrackp *opt, t_cookie *cookie, const char *fpath,
int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath,
const char *name, const char *start_url) {
char catbuff[CATBUFF_SIZE];
- char buffer[8192];
// Merge any IE cookies first
#ifdef _WIN32
@@ -897,8 +1093,6 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath,
const char *const given = fconcat(catbuff, sizeof(catbuff), fpath, name);
char file[CATBUFF_SIZE];
char bak[CATBUFF_SIZE];
- int refused = 0;
- hts_boolean keep = HTS_FALSE;
int quote = 0;
FILE *fp;
@@ -921,161 +1115,11 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath,
return ret;
}
if (fp) {
- char BIGSTK line[8192];
- const size_t line_max = 8000;
- hts_boolean first_line = HTS_TRUE;
- int lineno = 0;
- // this load's records alone, so a line repeating one is caught
- t_cookie *seen = (t_cookie *) calloct(1, sizeof(t_cookie));
-
- if (seen != NULL) {
- seen->max_len = sizeof(seen->data);
- } else {
- hts_log_print(opt, LOG_WARNING,
- "Cookies: out of memory, so %s is left unchanged", given);
- keep = HTS_TRUE;
- }
- // ferror ends the loop too, because a directory never reaches EOF
- while (!feof(fp) && !ferror(fp) &&
- strlen(cookie->data) < cookie->max_len) {
- size_t len;
-
- // len counts what the buffer cut too, so this catches that line
- cookie_readline(fp, line, sizeof(line), &len);
- if (lineno < INT_MAX)
- lineno++;
- if (first_line && len >= sizeof(cookie_bom) - 1 &&
- strncmp(line, cookie_bom, sizeof(cookie_bom) - 1) == 0) {
- len -= sizeof(cookie_bom) - 1;
- memmove(line, line + sizeof(cookie_bom) - 1,
- strlen(line) + 1 - (sizeof(cookie_bom) - 1));
- }
- first_line = HTS_FALSE;
- if (len >= line_max) {
- hts_log_print(opt, LOG_WARNING,
- "Cookies: ignoring an over-long line "
- "(%d bytes, maximum %d): %s",
- (int) len, (int) line_max - 1, given);
- refused++;
- } else if (memchr(line, '\0', len) != NULL ||
- memchr(line, '\r', len) != NULL) {
- hts_log_print(opt, LOG_WARNING,
- "Cookies: ignoring a line holding a NUL or a CR: %s",
- given);
- refused++;
- } else if (line[strspn(line, " \t")] != '\0') {
- /* curl and browser exports mark an HttpOnly cookie this way */
- static const char httponly[] = COOKIE_HTTPONLY_PREFIX;
- const hts_boolean is_httponly =
- strncmp(line, httponly, sizeof(httponly) - 1) == 0;
- const char *const rec =
- is_httponly ? line + sizeof(httponly) - 1 : line;
-
- if (rec[0] == '#') {
- if (!cookie_known_comment(rec)) {
- // no line text, because any part of it can hold a cookie value
- hts_log_print(opt, LOG_WARNING,
- "Cookies: ignoring an unknown comment on line %d "
- "of %s",
- lineno, given);
- refused++;
- }
- } else {
- char domain[256]; // .netscape.com
- char path[256]; // /
- char BIGSTK cook_name[COOKIE_MAX_PAIR + 1]; // MYCOOK
- char BIGSTK cook_value[8192]; // ID=toto,S=1234
- const char *p;
- int tabs = 0;
+ const int ret = cookie_load_records(opt, cookie, fp, given,
+ /* first_line */ HTS_TRUE, NULL);
-#if DEBUG_COOK
- printf("%s\n", rec);
-#endif
- for (p = strchr(rec, '\t'); p != NULL; p = strchr(p + 1, '\t'))
- tabs++;
- if (tabs < 6 || !strnotempty(cookie_get(buffer, rec, 5))) {
- hts_log_print(opt, LOG_WARNING,
- "Cookies: ignoring a line with fewer than 7 "
- "fields or no name: %s",
- given);
- refused++;
- } else if (tabs > 7 ||
- (tabs == 7 && strcmp(cookie_get(buffer, rec, 7),
- COOKIE_HTTPONLY_FIELD) != 0)) {
- // a TAB in the value, because only HTTrack writes an eighth field
- hts_log_print(opt, LOG_WARNING,
- "Cookies: ignoring a line with more than 7 "
- "fields: %s",
- given);
- refused++;
- } else if (!cookie_is_expiry(cookie_get(buffer, rec, 4))) {
- hts_log_print(opt, LOG_WARNING,
- "Cookies: ignoring a line whose expiry is not a "
- "whole number: %s",
- given);
- refused++;
- } else if (cookie_load_field(opt, domain, sizeof(domain), buffer,
- rec, 0, "domain", given) &&
- cookie_load_field(opt, path, sizeof(path), buffer, rec,
- 2, "path", given) &&
- cookie_load_field(opt, cook_name, sizeof(cook_name),
- buffer, rec, 5, "name", given) &&
- cookie_load_field(opt, cook_value, sizeof(cook_value),
- buffer, rec, 6, "value", given)) {
- cookie_attr attr;
- char host[256];
-
- attr.host_only = cookie_is_host_only(rec);
- attr.secure =
- strcmpnocase(cookie_get(buffer, rec, 3), "TRUE") == 0;
- attr.http_only = is_httponly || tabs == 7;
- attr.expires = cookie_record_expiry(rec);
- // an expired one is skipped, because a browser drops it too
- if (attr.expires == 0 || attr.expires > (int64_t) time(NULL)) {
- const hts_boolean repeated =
- seen != NULL && cookie_host(domain, host, sizeof(host)) &&
- cookie_find_exact(seen->data, cook_name, host, path) !=
- NULL;
-
- // cookie_set stores it host-only
- if (!attr.host_only && cookie_domain_is_suffix(domain))
- hts_log_print(opt, LOG_WARNING,
- "Cookies: '%.64s' for the public suffix %s is "
- "now host-only: %s",
- cook_name, domain, given);
- // the last one wins, as curl reads it, but a rewrite loses one
- if (repeated) {
- hts_log_print(opt, LOG_WARNING,
- "Cookies: '%.64s' appears twice in %s",
- cook_name, given);
- refused++;
- }
- if (cookie_set(cookie, cook_name, cook_value, domain, path,
- &attr) != 0) {
- // the jar caps a field tighter still, and a full one refuses
- hts_log_print(opt, LOG_WARNING,
- "Cookies: the jar did not store '%.64s' from "
- "%s",
- cook_name, given);
- refused++;
- } else if (seen != NULL && !repeated) {
- (void) cookie_set(seen, cook_name, "", domain, path, NULL);
- }
- }
- } else {
- refused++;
- }
- }
- }
- }
- if (!feof(fp)) {
- hts_log_print(opt, LOG_WARNING,
- "Cookies: stopped reading before the end of %s", given);
- refused++;
- }
fclose(fp);
- freet(seen);
- return keep ? COOKIE_LOAD_KEEP : refused;
+ return ret;
} else if (errno != ENOENT) {
// the file exists, so a rewrite would replace what was never read
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
@@ -1087,6 +1131,13 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath,
return -1;
}
+/* See htsbundle.h. */
+void cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp,
+ const char *given, cookie_filter *filter) {
+ (void) cookie_load_records(opt, cookie, fp, given, /* first_line */ HTS_FALSE,
+ filter);
+}
+
/* Flush fp to the disk and close it. Did every step succeed? */
static hts_boolean cookie_close_synced(FILE *fp) {
// a crash after the rename must not leave an empty file behind it
diff --git a/src/htsbundle.c b/src/htsbundle.c
new file mode 100644
index 000000000..290283810
--- /dev/null
+++ b/src/htsbundle.c
@@ -0,0 +1,700 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: htsbundle.c subroutines: */
+/* login bundles named by a start URL */
+/* ------------------------------------------------------------ */
+
+#define HTS_INTERNAL_BYTECODE
+
+#include "htsbundle.h"
+
+#include "htscore.h"
+#include "htslib.h"
+#include "htslines.h"
+#include "htspsl.h"
+#include "htssafe.h"
+
+#include
+
+/* A bundle line, the jar aside, holds fewer bytes than this. */
+#define BUNDLE_LINE_MAX 8192
+
+/* A bundle holds at most this many url lines, and as many scope entries. */
+#define BUNDLE_LIST_MAX 256
+
+/* A bundle's header lines, with their CRLFs, fit in this many bytes, so the
+ request still has room for the rest. */
+#define BUNDLE_HEADERS_MAX 8192
+
+typedef struct bundle {
+ char name[HTS_BUNDLE_NAME_MAX];
+ char file[HTS_URLMAXSIZE * 2];
+ char **start_urls; /* the start URLs naming it */
+ size_t nstart;
+ char **urls; /* its url lines */
+ size_t nurls;
+ /* lowercase domains, each covering its subdomains, or "=host" for a host
+ alone */
+ char **scope;
+ size_t nscope;
+ char *ua; /* NULL when it gives none */
+ String headers;
+ hts_boolean https_only; /* Are its headers for https alone? */
+ hts_boolean scope_given; /* Do scope lines replace the default scope? */
+} bundle;
+
+typedef struct hts_bundle_state {
+ bundle *items;
+ size_t count;
+} hts_bundle_state;
+
+static hts_boolean strlist_add(char ***list, size_t *count, const char *s) {
+ char **const grown = realloct(*list, (*count + 1) * sizeof(char *));
+
+ if (grown == NULL)
+ return HTS_FALSE;
+ *list = grown;
+ grown[*count] = strdupt(s);
+ if (grown[*count] == NULL)
+ return HTS_FALSE;
+ (*count)++;
+ return HTS_TRUE;
+}
+
+static void strlist_free(char ***list, size_t *count) {
+ size_t i;
+
+ for (i = 0; i < *count; i++)
+ freet((*list)[i]);
+ freet(*list);
+ *count = 0;
+}
+
+/* Forget what reading b's file gave. */
+static void bundle_clear(bundle *b) {
+ strlist_free(&b->urls, &b->nurls);
+ strlist_free(&b->scope, &b->nscope);
+ freet(b->ua);
+ StringClear(b->headers);
+ b->https_only = HTS_FALSE;
+ b->scope_given = HTS_FALSE;
+}
+
+/* Is name a Windows device name, such as CON, aux.txt or com1? */
+static hts_boolean bundle_name_is_device(const char *name) {
+ static const char *const devices[] = {"con", "prn", "aux", "nul"};
+ const size_t base = strcspn(name, ".");
+ char dev[5];
+ size_t i;
+
+ if (base < 3 || base > 4)
+ return HTS_FALSE;
+ dev[0] = '\0';
+ strlncatbuff(dev, name, sizeof(dev), base);
+ hts_lowcase(dev);
+ if (base == 4)
+ return (strncmp(dev, "com", 3) == 0 || strncmp(dev, "lpt", 3) == 0) &&
+ dev[3] >= '1' && dev[3] <= '9';
+ for (i = 0; i < sizeof(devices) / sizeof(devices[0]); i++) {
+ if (strcmp(dev, devices[i]) == 0)
+ return HTS_TRUE;
+ }
+ return HTS_FALSE;
+}
+
+/* Why name is not a bundle name, or NULL when it is one. */
+static const char *bundle_name_refusal(const char *name) {
+ const size_t len = strlen(name);
+
+ if (len >= HTS_BUNDLE_NAME_MAX)
+ return "invalid login bundle name: 128 bytes or more";
+ if (len == 0 ||
+ strspn(name, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+ "0123456789._-") != len)
+ return "invalid login bundle name: use letters, digits, '.', '_' and '-'";
+ if (name[0] == '.' || strstr(name, "..") != NULL)
+ return "invalid login bundle name: it starts with '.' or holds \"..\"";
+ if (bundle_name_is_device(name))
+ return "invalid login bundle name: a Windows device name";
+ return NULL;
+}
+
+/* Cut the suffix off url into name ("" if none). Returns NULL, or why the name
+ * is refused. */
+static const char *bundle_cut(char *url, char *name, size_t name_size) {
+ char *const at = strstr(url, HTS_BUNDLE_SUFFIX);
+ const char *why;
+
+ name[0] = '\0';
+ if (at == NULL)
+ return NULL;
+ why = bundle_name_refusal(at + strlen(HTS_BUNDLE_SUFFIX));
+ if (why != NULL)
+ return why;
+ strlcpybuff(name, at + strlen(HTS_BUNDLE_SUFFIX), name_size);
+ *at = '\0';
+ return NULL;
+}
+
+void hts_bundle_free(httrackp *opt) {
+ hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state;
+ size_t i;
+
+ if (st == NULL)
+ return;
+ for (i = 0; i < st->count; i++) {
+ bundle_clear(&st->items[i]);
+ strlist_free(&st->items[i].start_urls, &st->items[i].nstart);
+ StringFree(st->items[i].headers);
+ }
+ freet(st->items);
+ freet(opt->bundle_state);
+}
+
+/* Record that the start URL url names bundle name. HTS_FALSE when out of
+ memory. */
+static hts_boolean bundle_add(httrackp *opt, const char *name,
+ const char *url) {
+ hts_bundle_state *st = (hts_bundle_state *) opt->bundle_state;
+ bundle *b = NULL;
+ size_t i;
+
+ if (st == NULL) {
+ st = calloct(1, sizeof(*st));
+ if (st == NULL)
+ return HTS_FALSE;
+ opt->bundle_state = st;
+ }
+ for (i = 0; i < st->count && b == NULL; i++) {
+ if (strcmp(st->items[i].name, name) == 0)
+ b = &st->items[i];
+ }
+ if (b == NULL) {
+ bundle *const grown = realloct(st->items, (st->count + 1) * sizeof(bundle));
+
+ if (grown == NULL)
+ return HTS_FALSE;
+ st->items = grown;
+ b = &grown[st->count++];
+ memset(b, 0, sizeof(*b));
+ StringInit(b->headers);
+ strcpybuff(b->name, name);
+ }
+ return strlist_add(&b->start_urls, &b->nstart, url);
+}
+
+const char *hts_bundle_take(httrackp *opt, char *url) {
+ char name[HTS_BUNDLE_NAME_MAX];
+ const char *const why = bundle_cut(url, name, sizeof(name));
+
+ if (why != NULL || name[0] == '\0')
+ return why;
+ return bundle_add(opt, name, url) ? NULL : "out of memory";
+}
+
+hts_boolean hts_bundle_has_control(const char *s, size_t len) {
+ size_t i;
+
+ for (i = 0; i < len; i++) {
+ const unsigned char c = (unsigned char) s[i];
+
+ if ((c < 0x20 && c != '\t') || c == 0x7f)
+ return HTS_TRUE;
+ }
+ return HTS_FALSE;
+}
+
+/* Read a line of fewer than BUNDLE_LINE_MAX bytes, its line end aside, into
+ line. NULL, or why the line is refused: too long, or with a control byte
+ other than TAB, because a CR would end a header early. */
+static const char *bundle_readline(FILE *fp, char line[BUNDLE_LINE_MAX + 1]) {
+ size_t len;
+
+ if (hts_readline_raw(fp, line, BUNDLE_LINE_MAX + 1, &len))
+ return "too long";
+ if (len > 0 && line[len - 1] == '\r')
+ line[--len] = '\0';
+ if (len >= BUNDLE_LINE_MAX)
+ return "too long";
+ return hts_bundle_has_control(line, len) ? "holds a control byte" : NULL;
+}
+
+/* The major version a "# HTTrack bundle N[.M]" line gives, or -1 when line is
+ not one. */
+static int bundle_version_of(const char *line) {
+ static const char magic[] = "# HTTrack bundle ";
+ const char *p = line;
+ size_t digits;
+ int major;
+
+ if (strncmp(p, magic, sizeof(magic) - 1) != 0)
+ return -1;
+ p += sizeof(magic) - 1;
+ digits = strspn(p, "0123456789");
+ if (digits == 0 || digits > 3 || (p[0] == '0' && digits > 1))
+ return -1;
+ major = atoi(p);
+ p += digits;
+ if (*p == '.') {
+ digits = strspn(++p, "0123456789");
+ if (digits == 0)
+ return -1;
+ p += digits;
+ }
+ return *p == '\0' ? major : -1;
+}
+
+/* Does the scope entry cover host? */
+static hts_boolean scope_covers(const char *entry, const char *host) {
+ return entry[0] == '=' ? strcmp(entry + 1, host) == 0
+ : cookie_domain_match(entry, host);
+}
+
+static hts_boolean bundle_in_scope(const bundle *b, const char *host) {
+ size_t i;
+
+ for (i = 0; i < b->nscope; i++) {
+ if (scope_covers(b->scope[i], host))
+ return HTS_TRUE;
+ }
+ return HTS_FALSE;
+}
+
+/* Add entry to b's scope. NULL, or why it could not. */
+static const char *bundle_scope_add(bundle *b, const char *entry) {
+ size_t i;
+
+ for (i = 0; i < b->nscope; i++) {
+ if (strcmp(b->scope[i], entry) == 0)
+ return NULL;
+ }
+ if (b->nscope >= BUNDLE_LIST_MAX)
+ return "more than 256 scope domains";
+ return strlist_add(&b->scope, &b->nscope, entry) ? NULL : "out of memory";
+}
+
+/* Add url's https flag and default scope to b. NULL, or why it could not. */
+static const char *bundle_note_url(bundle *b, const char *url) {
+ char host[256];
+ char entry[sizeof(host) + 1];
+ const char *reg;
+
+ if (strfield(url, "https:"))
+ b->https_only = HTS_TRUE;
+ if (b->scope_given)
+ return NULL;
+ if (!cookie_url_host(url, host, sizeof(host)))
+ return "cannot work out the bundle's scope";
+ // a host with no registrable domain covers itself alone
+ reg = hts_host_is_address(host, strlen(host))
+ ? NULL
+ : hts_psl_registrable_domain(host);
+ snprintf(entry, sizeof(entry), "%s%s", reg != NULL ? "" : "=",
+ reg != NULL ? reg : host);
+ return bundle_scope_add(b, entry);
+}
+
+/* Write the reason into err and return HTS_FALSE. The caller needs err and
+ err_size. */
+#define BUNDLE_FAIL(...) \
+ do { \
+ snprintf(err, err_size, __VA_ARGS__); \
+ return HTS_FALSE; \
+ } while (0)
+
+/* Add a scope line's domain to b's scope. */
+static hts_boolean bundle_scope_line(bundle *b, const char *value, int lineno,
+ char *err, size_t err_size) {
+ char dom[256];
+ char entry[sizeof(dom) + 1];
+ const char *why;
+
+ if (value[0] == '.')
+ value++;
+ if (value[0] == '\0' || strlen(value) >= sizeof(dom) ||
+ strspn(value, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+ "0123456789.-") != strlen(value))
+ BUNDLE_FAIL("%s, line %d: invalid scope", b->file, lineno);
+ strcpybuff(dom, value);
+ hts_lowcase(dom);
+ if (hts_host_is_address(dom, strlen(dom)))
+ snprintf(entry, sizeof(entry), "=%s", dom);
+ else if (hts_psl_registrable_domain(dom) != NULL)
+ strcpybuff(entry, dom);
+ else
+ BUNDLE_FAIL("%s, line %d: the scope is a public suffix", b->file, lineno);
+ why = bundle_scope_add(b, entry);
+ if (why != NULL)
+ BUNDLE_FAIL("%s, line %d: %s", b->file, lineno, why);
+ return HTS_TRUE;
+}
+
+/* Is the klen bytes at line the key key? */
+static hts_boolean key_is(const char *line, size_t klen, const char *key) {
+ return klen == strlen(key) && strncmp(line, key, klen) == 0;
+}
+
+/* Read b's lines up to "cookies", leaving fp at its jar. *cookies_line gets
+ the number of that line, or 0 when b has no cookies. */
+static hts_boolean bundle_parse(httrackp *opt, bundle *b, FILE *fp,
+ int *cookies_line, char *err, size_t err_size) {
+ static const char bom[] = "\xEF\xBB\xBF";
+ char BIGSTK line[BUNDLE_LINE_MAX + 1];
+ int lineno = 1;
+ int version;
+ const char *why = bundle_readline(fp, line);
+
+ if (why != NULL)
+ BUNDLE_FAIL("%s, line 1: %s", b->file, why);
+ version = bundle_version_of(
+ strncmp(line, bom, sizeof(bom) - 1) == 0 ? line + sizeof(bom) - 1 : line);
+ if (version < 0)
+ BUNDLE_FAIL("%s: not an HTTrack bundle, the first line must be "
+ "\"# HTTrack bundle 1\"",
+ b->file);
+ if (version != 1)
+ BUNDLE_FAIL("%s: bundle version %d is not supported", b->file, version);
+ *cookies_line = 0;
+ while (!feof(fp)) {
+ const char *value;
+ size_t klen;
+
+ lineno++;
+ why = bundle_readline(fp, line);
+ if (why != NULL)
+ BUNDLE_FAIL("%s, line %d: %s", b->file, lineno, why);
+ if (line[0] == '\0' || line[0] == '#')
+ continue;
+ klen = strcspn(line, " \t");
+ value = line + klen;
+ value += strspn(value, " \t");
+ if (key_is(line, klen, "cookies")) {
+ if (*value != '\0')
+ BUNDLE_FAIL("%s, line %d: \"cookies\" takes no value", b->file, lineno);
+ *cookies_line = lineno;
+ return HTS_TRUE;
+ }
+ if (!key_is(line, klen, "url") && !key_is(line, klen, "ua") &&
+ !key_is(line, klen, "header") && !key_is(line, klen, "scope")) {
+ /* a key in lowercase letters is no secret, so it alone is named */
+ const hts_boolean shown =
+ klen <= 32 && strspn(line, "abcdefghijklmnopqrstuvwxyz") == klen;
+
+ hts_log_print(opt, LOG_WARNING, "Bundle %s, line %d: unknown key%s%.*s",
+ b->file, lineno, shown ? " " : "", shown ? (int) klen : 0,
+ line);
+ continue;
+ }
+ if (*value == '\0')
+ BUNDLE_FAIL("%s, line %d: no value", b->file, lineno);
+ if (key_is(line, klen, "url")) {
+ char host[256];
+
+ if (strlen(value) >= HTS_URLMAXSIZE ||
+ strstr(value, HTS_BUNDLE_SUFFIX) != NULL ||
+ !cookie_url_host(value, host, sizeof(host)))
+ BUNDLE_FAIL("%s, line %d: invalid url", b->file, lineno);
+ if (b->nurls >= BUNDLE_LIST_MAX)
+ BUNDLE_FAIL("%s, line %d: more than %d url lines", b->file, lineno,
+ BUNDLE_LIST_MAX);
+ if (!strlist_add(&b->urls, &b->nurls, value))
+ BUNDLE_FAIL("%s: out of memory", b->file);
+ } else if (key_is(line, klen, "ua")) {
+ if (b->ua != NULL)
+ BUNDLE_FAIL("%s, line %d: ua given twice", b->file, lineno);
+ b->ua = strdupt(value);
+ if (b->ua == NULL)
+ BUNDLE_FAIL("%s: out of memory", b->file);
+ } else if (key_is(line, klen, "header")) {
+ why = http_bundle_header_refusal(StringBuff(b->headers), value);
+ if (why != NULL)
+ BUNDLE_FAIL("%s, line %d: refused header: %s", b->file, lineno, why);
+ if (strlen(value) + 2 > BUNDLE_HEADERS_MAX - StringLength(b->headers))
+ BUNDLE_FAIL("%s, line %d: the header lines pass %d bytes", b->file,
+ lineno, BUNDLE_HEADERS_MAX);
+ StringCat(b->headers, value);
+ StringCat(b->headers, "\r\n");
+ } else {
+ b->scope_given = HTS_TRUE;
+ if (!bundle_scope_line(b, value, lineno, err, err_size))
+ return HTS_FALSE;
+ }
+ }
+ return HTS_TRUE;
+}
+
+/* Is jar_file newer than b's file, so its records win? */
+static hts_boolean bundle_jar_wins(const bundle *b, const char *jar_file) {
+ hts_filetime_t jt, bt;
+
+ return jar_file != NULL && hts_file_mtime(jar_file, &jt) &&
+ hts_file_mtime(b->file, &bt) &&
+ (jt.sec > bt.sec || (jt.sec == bt.sec && jt.nsec > bt.nsec));
+}
+
+/* What bundle_admit needs about the bundle being read. */
+typedef struct bundle_jar {
+ const bundle *b;
+ t_cookie *all_bundles; /* every bundle's records so far */
+ hts_boolean jar_wins; /* A record the jar holds already wins. */
+ int dropped; /* records outside the scope */
+} bundle_jar;
+
+/* The cookie_filter admit callback: keep a record in the bundle's scope, and
+ stop on one another bundle gives another value. */
+static hts_boolean bundle_admit(cookie_filter *filter, t_cookie *cookie,
+ hts_boolean repeat, const char *name,
+ const char *value, const char *domain,
+ const char *path) {
+ bundle_jar *const ctx = (bundle_jar *) filter->ctx;
+ char buffer[8192];
+ char host[256];
+ const char *other;
+
+ if (!cookie_host(domain, host, sizeof(host)) ||
+ !bundle_in_scope(ctx->b, host[0] == '.' ? host + 1 : host)) {
+ ctx->dropped++;
+ return HTS_FALSE;
+ }
+ if (strlen(name) > COOKIE_MAX_PAIR ||
+ strlen(value) > COOKIE_MAX_PAIR - strlen(name)) {
+ filter->why = "a cookie name and value pass 4096 bytes";
+ return HTS_FALSE;
+ }
+ other = cookie_find_exact(ctx->all_bundles->data, name, host, path);
+ if (other != NULL && !repeat &&
+ strcmp(cookie_get(buffer, other, 6), value) != 0) {
+ filter->why = "a cookie has another value in another bundle";
+ return HTS_FALSE;
+ }
+ /* unreachable while all_bundles outsizes opt->cookie; a silent skip would
+ stop the clash check */
+ if (cookie_set(ctx->all_bundles, name, value, domain, path, NULL) != 0) {
+ filter->why = "too many cookies to compare with the other bundles";
+ return HTS_FALSE;
+ }
+ return !ctx->jar_wins ||
+ cookie_find_exact(cookie->data, name, host, path) == NULL;
+}
+
+/* Read b's file, set its scope, and load its cookies, comparing them with
+ all_bundles. */
+static hts_boolean bundle_load(httrackp *opt, bundle *b, const char *jar_file,
+ t_cookie *all_bundles, char *err,
+ size_t err_size) {
+ const char *const dir = StringNotEmpty(opt->bundle_dir)
+ ? StringBuff(opt->bundle_dir)
+ : StringBuff(opt->path_log);
+ const size_t dlen = strlen(dir);
+ int cookies_line;
+ t_cookie *scratch = NULL;
+ bundle_jar ctx;
+ cookie_filter filter;
+ const char *why = NULL;
+ size_t i;
+ FILE *fp;
+
+ bundle_clear(b);
+ if (!slprintfbuff(
+ b->file, sizeof(b->file), "%s%s%s.hts", dir,
+ dlen != 0 && dir[dlen - 1] != '/' && dir[dlen - 1] != '\\' ? "/" : "",
+ b->name))
+ BUNDLE_FAIL("bundle %s: the folder path is too long", b->name);
+ fp = FOPEN(b->file, "rb");
+ if (fp == NULL)
+ BUNDLE_FAIL("%s: cannot read the login bundle: %s", b->file,
+ strerror(errno));
+ if (!bundle_parse(opt, b, fp, &cookies_line, err, err_size)) {
+ fclose(fp);
+ return HTS_FALSE;
+ }
+ for (i = 0; i < b->nstart && why == NULL; i++)
+ why = bundle_note_url(b, b->start_urls[i]);
+ for (i = 0; i < b->nurls && why == NULL; i++)
+ why = bundle_note_url(b, b->urls[i]);
+ if (why != NULL) {
+ fclose(fp);
+ BUNDLE_FAIL("%s: %s", b->file, why);
+ }
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.b = b;
+ ctx.all_bundles = all_bundles;
+ ctx.jar_wins = bundle_jar_wins(b, jar_file);
+ memset(&filter, 0, sizeof(filter));
+ filter.admit = bundle_admit;
+ filter.ctx = &ctx;
+ filter.line = cookies_line;
+ if (cookies_line != 0 && opt->cookie == NULL) {
+ hts_log_print(opt, LOG_WARNING,
+ "Cookies are disabled, so the cookies of %s are not sent",
+ b->file);
+ // read all the same, so -b0 does not decide whether a bundle is valid
+ scratch = calloct(1, sizeof(t_cookie));
+ if (scratch == NULL) {
+ fclose(fp);
+ BUNDLE_FAIL("%s: out of memory", b->file);
+ }
+ scratch->max_len = sizeof(scratch->data);
+ }
+ if (cookies_line != 0)
+ cookie_load_stream(opt, scratch != NULL ? scratch : opt->cookie, fp,
+ b->file, &filter);
+ freet(scratch);
+ fclose(fp);
+ if (ctx.dropped != 0)
+ hts_log_print(opt, LOG_WARNING,
+ "Bundle %s: %d cookie(s) outside its scope left out; a "
+ "scope line keeps them",
+ b->file, ctx.dropped);
+ if (filter.why != NULL)
+ BUNDLE_FAIL("%s, line %d: %s", b->file, filter.line, filter.why);
+ return HTS_TRUE;
+}
+
+static hts_boolean bundle_scopes_overlap(const bundle *a, const bundle *b) {
+ size_t i, j;
+
+ for (i = 0; i < a->nscope; i++) {
+ for (j = 0; j < b->nscope; j++) {
+ const char *const ea = a->scope[i];
+ const char *const eb = b->scope[j];
+
+ if (scope_covers(ea, eb + (eb[0] == '=')) ||
+ scope_covers(eb, ea + (ea[0] == '=')))
+ return HTS_TRUE;
+ }
+ }
+ return HTS_FALSE;
+}
+
+static hts_boolean bundle_same_identity(const bundle *a, const bundle *b) {
+ return (a->ua == NULL ? b->ua == NULL
+ : b->ua != NULL && strcmp(a->ua, b->ua) == 0) &&
+ strcmp(StringBuff(a->headers), StringBuff(b->headers)) == 0;
+}
+
+/* The bundle that stands for the merged group holding i. */
+static size_t bundle_group(size_t *group, size_t i) {
+ while (group[i] != i)
+ i = group[i] = group[group[i]];
+ return i;
+}
+
+hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar_file, char *err,
+ size_t err_size) {
+ hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state;
+ t_cookie *all_bundles;
+ size_t *group;
+ hts_boolean ok = HTS_TRUE;
+ size_t i, j;
+
+ if (st == NULL)
+ return HTS_TRUE;
+ all_bundles = calloct(1, sizeof(t_cookie));
+ group = calloct(st->count, sizeof(size_t));
+ if (all_bundles == NULL || group == NULL) {
+ freet(all_bundles);
+ freet(group);
+ BUNDLE_FAIL("login bundles: out of memory");
+ }
+ all_bundles->max_len = sizeof(all_bundles->data);
+ for (i = 0; i < st->count; i++)
+ group[i] = i;
+ for (i = 0; i < st->count && ok; i++)
+ ok = bundle_load(opt, &st->items[i], jar_file, all_bundles, err, err_size);
+ // the log backstop hides a bundle's tokens and url-line passwords too
+ for (i = 0; i < st->count && ok; i++) {
+ for (j = 0; j < st->items[i].nurls; j++)
+ hts_secret_add_login(opt, st->items[i].urls[j]);
+ hts_secret_add_headers(opt, StringBuff(st->items[i].headers));
+ }
+ for (i = 0; i < st->count && ok; i++) {
+ for (j = i + 1; j < st->count && ok; j++) {
+ const bundle *const a = &st->items[i];
+ const bundle *const b = &st->items[j];
+
+ if (!bundle_scopes_overlap(a, b))
+ continue;
+ if (!bundle_same_identity(a, b)) {
+ snprintf(err, err_size,
+ "%s and %s cover the same site with a different user agent "
+ "or headers",
+ a->file, b->file);
+ ok = HTS_FALSE;
+ } else {
+ group[bundle_group(group, j)] = bundle_group(group, i);
+ }
+ }
+ }
+ // an https URL in any merged bundle makes the headers of all https-only
+ for (i = 0; i < st->count; i++) {
+ if (st->items[i].https_only)
+ st->items[bundle_group(group, i)].https_only = HTS_TRUE;
+ }
+ for (i = 0; i < st->count; i++)
+ st->items[i].https_only = st->items[bundle_group(group, i)].https_only;
+ freet(group);
+ freet(all_bundles);
+ return ok;
+}
+
+const char *hts_bundle_url(httrackp *opt, size_t i) {
+ const hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state;
+ size_t k;
+
+ for (k = 0; st != NULL && k < st->count; k++) {
+ if (i < st->items[k].nurls)
+ return st->items[k].urls[i];
+ i -= st->items[k].nurls;
+ }
+ return NULL;
+}
+
+void hts_bundle_identity(httrackp *opt, const char *adr, htsrequest *req) {
+ const hts_bundle_state *const st = (hts_bundle_state *) opt->bundle_state;
+ char host[256];
+ size_t i;
+
+ if (st == NULL || !cookie_host(adr, host, sizeof(host)))
+ return;
+ for (i = 0; i < st->count; i++) {
+ const bundle *const b = &st->items[i];
+
+ if (bundle_in_scope(b, host)) {
+ if (b->ua != NULL) {
+ req->user_agent = b->ua;
+ req->user_agent_send = HTS_TRUE;
+ }
+ if (StringNotEmpty(b->headers) &&
+ (!b->https_only || strfield(adr, "https://")))
+ req->headers = StringBuff(b->headers);
+ return;
+ }
+ }
+}
diff --git a/src/htsbundle.h b/src/htsbundle.h
new file mode 100644
index 000000000..53480bf42
--- /dev/null
+++ b/src/htsbundle.h
@@ -0,0 +1,107 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: htsbundle.h subroutines: */
+/* login bundles named by a start URL */
+/* ------------------------------------------------------------ */
+
+#ifndef HTSBUNDLE_DEFH
+#define HTSBUNDLE_DEFH
+
+#ifdef HTS_INTERNAL_BYTECODE
+
+#include
+#include
+
+#include "htsglobal.h"
+#include "htsopt.h"
+
+/* A start URL ending with this names the bundle file .hts. */
+#define HTS_BUNDLE_SUFFIX "#httrack-bundle="
+
+/* A bundle name holds fewer bytes than this. */
+#define HTS_BUNDLE_NAME_MAX 128
+
+/* Cut the bundle suffix off url in place and record that url names that
+ bundle. A name given twice is one bundle. NULL, or why the run must stop. */
+const char *hts_bundle_take(httrackp *opt, char *url);
+
+/* Read every named bundle from opt->bundle_dir, or the project folder, and
+ refuse two bundles that overlap with different identities or cookie values.
+ The cookies go into opt->cookie. A jar_file newer than the bundle keeps its
+ records. A NULL one never does. HTS_FALSE after writing why into err, naming
+ the file, and then the run must stop. A second call reads the files again. */
+hts_boolean hts_bundle_load_all(httrackp *opt, const char *jar_file, char *err,
+ size_t err_size);
+
+/* The i-th url line of the loaded bundles, or NULL past the last one. */
+const char *hts_bundle_url(httrackp *opt, size_t i);
+
+/* Give req the UA and the headers of the first bundle whose scope holds adr's
+ host, when that bundle has them. Its headers go over https alone when any URL
+ of the bundles merged with it is https. */
+void hts_bundle_identity(httrackp *opt, const char *adr, htsrequest *req);
+
+/* Forget the bundles named for opt. */
+void hts_bundle_free(httrackp *opt);
+
+/* Do the len bytes at s hold a control byte other than TAB? */
+hts_boolean hts_bundle_has_control(const char *s, size_t len);
+
+/* The bundles share these htsbauth.c internals. */
+
+/* Copy the host of url, with or without a scheme, into dst as cookie_host
+ does. */
+hts_boolean cookie_url_host(const char *url, char *dst, size_t dst_size);
+
+/* The record of s named cook_name for exactly domain and path, or NULL. */
+char *cookie_find_exact(char *s, const char *cook_name, const char *domain,
+ const char *path);
+
+/* This filter picks what cookie_load_stream lets into the jar. */
+typedef struct cookie_filter {
+ /* Should the record go into cookie? repeat says this load had it already.
+ Setting why stops the load. */
+ hts_boolean (*admit)(struct cookie_filter *filter, t_cookie *cookie,
+ hts_boolean repeat, const char *name, const char *value,
+ const char *domain, const char *path);
+ void *ctx;
+ int line; /* the number of the last line read */
+ const char *why; /* NULL, or why the load stopped at line */
+} cookie_filter;
+
+/* Load Netscape jar records from fp's position to its end through filter,
+ as cookie_load reads a file. given names the file in warnings, which name
+ the line and none of its bytes. A line with a control byte other than TAB,
+ or one too long, stops the load. */
+void cookie_load_stream(httrackp *opt, t_cookie *cookie, FILE *fp,
+ const char *given, cookie_filter *filter);
+
+#endif
+
+#endif
diff --git a/src/htsbundle_selftest.c b/src/htsbundle_selftest.c
new file mode 100644
index 000000000..c7a7d7a5e
--- /dev/null
+++ b/src/htsbundle_selftest.c
@@ -0,0 +1,182 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: htsbundle_selftest.c subroutines: */
+/* self-tests for the login bundles */
+/* ------------------------------------------------------------ */
+
+#include "htsselftest_int.h"
+
+#include "htsbundle.h"
+
+/* -#test=bundlecut : the bundle a start URL names, and the URL left. */
+static int st_bundlecut(httrackp *opt, int argc, char **argv) {
+ static const char nodir[] = "/nonexistent-bundlecut/";
+ char url[HTS_URLMAXSIZE * 2];
+ char err[1024];
+ const char *why;
+
+ if (argc != 1 || strlen(argv[0]) >= sizeof(url)) {
+ fprintf(stderr, "bundlecut: takes \n");
+ return 1;
+ }
+ strcpybuff(url, argv[0]);
+ hts_bundle_free(opt);
+ why = hts_bundle_take(opt, url);
+ if (why != NULL) {
+ printf("bad: %s\n", why);
+ } else if (opt->bundle_state == NULL) {
+ printf("none\n");
+ } else {
+ /* the name shows in the file a load from a missing folder cannot open */
+ StringCopy(opt->bundle_dir, nodir);
+ assertf(!hts_bundle_load_all(opt, NULL, err, sizeof(err)) &&
+ strncmp(err, nodir, sizeof(nodir) - 1) == 0 &&
+ strstr(err, ".hts: ") != NULL);
+ printf("%.*s %s\n",
+ (int) (strstr(err, ".hts: ") - err) - (int) (sizeof(nodir) - 1),
+ err + sizeof(nodir) - 1, url);
+ StringCopy(opt->bundle_dir, "");
+ hts_bundle_free(opt);
+ }
+ return 0;
+}
+
+/* Print err with each "/" cut to "/", since the folder is the caller's. */
+static void print_without_dir(const char *err, const char *dir) {
+ const size_t dlen = strlen(dir);
+ const char *p = err;
+ const char *at;
+
+ while ((at = strstr(p, dir)) != NULL) {
+ const hts_boolean whole = at[dlen] == '/' || at[dlen] == '\\';
+
+ printf("%.*s", (int) (at - p) + (whole ? 0 : (int) dlen), p);
+ p = at + dlen;
+ }
+ printf("%s\n", p);
+}
+
+/* -#test=bundle ... [host:]... [hide:]... [jar:]
+ [twice] [b0]: load the bundles the URLs name from dir over the jar file,
+ twice when asked, with cookies off for b0, then print their url lines, the
+ jar, the UA and headers each adr gets, and each text as the logs show it. */
+static int st_bundle(httrackp *opt, int argc, char **argv) {
+ FILE *const log = opt->log;
+ t_cookie *const prev = opt->cookie;
+ t_cookie *jar = calloct(1, sizeof(t_cookie));
+ const char *jar_file = NULL;
+ int loads = 1;
+ char err[1024];
+ const char *url;
+ char *rec;
+ int i;
+
+ if (argc < 2 || jar == NULL) {
+ fprintf(stderr, "bundle: takes ... [host:]... "
+ "[hide:]... [jar:] [twice] [b0]\n");
+ freet(jar);
+ return 1;
+ }
+ jar->max_len = 30000; /* httpmirror's */
+ opt->log = stderr; /* warnings stay out of the graded output */
+ opt->cookie = jar;
+ hts_bundle_free(opt);
+ StringCopy(opt->bundle_dir, argv[0]);
+ for (i = 1; i < argc; i++) {
+ char buff[HTS_URLMAXSIZE * 2];
+
+ if (strncmp(argv[i], "jar:", 4) == 0) {
+ jar_file = argv[i] + 4;
+ (void) cookie_load(opt, jar, "", jar_file);
+ } else if (strcmp(argv[i], "twice") == 0) {
+ loads = 2;
+ } else if (strcmp(argv[i], "b0") == 0) {
+ opt->cookie = NULL;
+ } else if (strncmp(argv[i], "host:", 5) != 0 &&
+ strncmp(argv[i], "hide:", 5) != 0 &&
+ strlen(argv[i]) < sizeof(buff)) {
+ strcpybuff(buff, argv[i]);
+ assertf(hts_bundle_take(opt, buff) == NULL);
+ }
+ }
+ for (i = 0; i < loads; i++) {
+ if (!hts_bundle_load_all(opt, jar_file, err, sizeof(err))) {
+ printf("refused: ");
+ print_without_dir(err, argv[0]);
+ goto done;
+ }
+ }
+ for (i = 0; (url = hts_bundle_url(opt, i)) != NULL; i++)
+ printf("url %s\n", url);
+ for (rec = jar->data; *rec != '\0'; rec = cookie_nextfield(rec)) {
+ char dom[8192], cname[8192], value[8192];
+
+ printf("cookie %s %s=%s\n", cookie_get(dom, rec, 0),
+ cookie_get(cname, rec, 5), cookie_get(value, rec, 6));
+ }
+ for (i = 1; i < argc; i++) {
+ if (strncmp(argv[i], "host:", 5) == 0) {
+ htsrequest req;
+ const char *p;
+
+ memset(&req, 0, sizeof(req));
+ req.user_agent = req.headers = "global";
+ hts_bundle_identity(opt, argv[i] + 5, &req);
+ printf("%s ua=%s headers=", argv[i] + 5, req.user_agent);
+ /* one line per host, so the block's line ends print as '|' */
+ for (p = req.headers; *p != '\0'; p++) {
+ if (*p != '\r')
+ putchar(*p == '\n' ? '|' : *p);
+ }
+ putchar('\n');
+ } else if (strncmp(argv[i], "hide:", 5) == 0) {
+ char shown[HTS_URLMAXSIZE * 2];
+
+ (void) strclipbuff(shown, sizeof(shown), argv[i] + 5);
+ hts_hide_secrets(opt, shown, sizeof(shown));
+ printf("hide %s\n", shown);
+ }
+ }
+done:
+ hts_bundle_free(opt);
+ StringCopy(opt->bundle_dir, "");
+ opt->cookie = prev;
+ opt->log = log;
+ freet(jar);
+ return 0;
+}
+
+const struct selftest_entry selftests_bundle[] = {
+ {"bundlecut", "", "the login bundle a start URL names", st_bundlecut},
+ {"bundle",
+ " ... [host:]... [hide:]... [jar:] [twice] "
+ "[b0]",
+ "load login bundles and print the identity each host gets", st_bundle},
+ {NULL, NULL, NULL, NULL},
+};
diff --git a/src/htscore.c b/src/htscore.c
index 66d15e86e..bc7d3da82 100644
--- a/src/htscore.c
+++ b/src/htscore.c
@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
/* File defs */
#include "htscore.h"
+#include "htsbundle.h"
/* after htscore.h, which is what pulls in config.h and so HAVE_PATHS_H */
#ifndef _WIN32
@@ -681,6 +682,56 @@ static hts_boolean filter_rule_decode_(const char *rule, char *dest,
return HTS_TRUE;
}
+/* The jar whose records can win over a bundle's: the -%K file, or the
+ project's cookies.txt. NULL under -b0. */
+static const char *bundle_jar_file(httrackp *opt, char buff[CATBUFF_SIZE]) {
+ if (!opt->accept_cookie)
+ return NULL;
+ if (strnotempty(StringBuff(opt->cookies_file)))
+ return StringBuff(opt->cookies_file);
+ return fconcat(buff, CATBUFF_SIZE, StringBuff(opt->path_log), "cookies.txt");
+}
+
+/* Load the login bundles, and add their url lines to the start URLs in
+ *primary, which grows. HTS_FALSE after saying why the run must stop. */
+static hts_boolean bundle_start(httrackp *opt, char **primary,
+ htsbuff *primarybuff) {
+ char err[1024];
+ char jarbuff[CATBUFF_SIZE];
+ const char *extra;
+ size_t room = 0;
+ size_t i;
+
+ if (!hts_bundle_load_all(opt, bundle_jar_file(opt, jarbuff), err,
+ sizeof(err))) {
+ hts_log_print(opt, LOG_ERROR, "Login bundle: %s", err);
+ fprintf(stderr, "Login bundle: %s\n", err);
+ return HTS_FALSE;
+ }
+ for (i = 0; (extra = hts_bundle_url(opt, i)) != NULL; i++)
+ room += strlen(extra) + sizeof("http://\n");
+ if (room != 0) {
+ char *const grown =
+ room < (size_t) -1 - primarybuff->cap
+ ? (char *) realloct(*primary, primarybuff->cap + room)
+ : NULL;
+
+ if (grown == NULL) {
+ printf("PANIC! : Not enough memory [%d]\n", __LINE__);
+ return HTS_FALSE;
+ }
+ *primary = primarybuff->buf = grown;
+ primarybuff->cap += room;
+ }
+ for (i = 0; (extra = hts_bundle_url(opt, i)) != NULL; i++) {
+ if (strstr(extra, ":/") == NULL)
+ htsbuff_cat(primarybuff, "http://");
+ htsbuff_cat(primarybuff, extra);
+ htsbuff_cat(primarybuff, "\n");
+ }
+ return HTS_TRUE;
+}
+
int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) {
char *primary = NULL; // première page, contenant les liens à scanner
hash_struct hash; // système de hachage, accélère la recherche dans les liens
@@ -1054,6 +1105,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) {
/* load URL file list */
/* OPTIMIZED for fast load */
+ hts_boolean bundle_refused = HTS_FALSE; /* a -%L line named a bad one */
+
if (StringNotEmpty(opt->filelist)) {
char *filelist_buff = NULL;
size_t filelist_sz = 0;
@@ -1096,6 +1149,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) {
int n = 0;
int lineno = 0;
char BIGSTK line[HTS_URLMAXSIZE * 2];
+ const char *bundle_why;
while(filelist_ptr < filelist_sz) {
int adv;
@@ -1136,6 +1190,15 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) {
StringBuff(opt->filelist), lineno, adr_nologin(opt, line + 1));
continue;
}
+ bundle_why = hts_bundle_take(opt, line);
+ if (bundle_why != NULL) {
+ hts_log_print(opt, LOG_ERROR, "\"%s\", line %d: %s",
+ StringBuff(opt->filelist), lineno, bundle_why);
+ fprintf(stderr, "\"%s\", line %d: %s\n", StringBuff(opt->filelist),
+ lineno, bundle_why);
+ bundle_refused = HTS_TRUE;
+ continue;
+ }
if (line[0]) {
n++;
if (strstr(line, ":/") == NULL) {
@@ -1168,6 +1231,15 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) {
opt, opt->cookie, "", StringBuff(opt->cookies_file), start_url);
}
+ // bundle url lines join the start URLs, and bundle cookies go to the jar
+ if (bundle_refused || !bundle_start(opt, &primary, &primarybuff)) {
+ /* the jar may hold half a bundle, so it is not saved */
+ cookies_file_refused = COOKIE_LOAD_KEEP;
+ freet(primary);
+ XH_extuninit;
+ return 0;
+ }
+
// lien primaire
if (!hts_record_link(opt, "primary", "/primary",
fslash(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
diff --git a/src/htscoremain.c b/src/htscoremain.c
index 8bf525b98..d456a860e 100644
--- a/src/htscoremain.c
+++ b/src/htscoremain.c
@@ -58,6 +58,7 @@ Please visit our Website: http://www.httrack.com
#include "htsmd5.h"
#include "htsthread.h"
#include "htsfetch.h"
+#include "htsbundle.h"
#include
/* hts_self_path() */
@@ -488,6 +489,29 @@ static hts_boolean optparam_dash_ok(const cmdl_argv *cmd, int na) {
return na + 1 < cmd->argc && cmd->param[na + 1];
}
+/* Store the --bundle-dir at na in opt and cmd as an absolute path, so doit.log
+ finds it from anywhere. HTS_FALSE after a panic. */
+static hts_boolean bundle_dir_set(httrackp *opt, cmdl_argv *cmd, int na) {
+ const char *const dir = na + 1 < cmd->argc ? cmd->argv[na + 1] : NULL;
+ const hts_boolean relative = dir != NULL && dir[0] != '/' && dir[0] != '\\' &&
+ !(dir[0] != '\0' && dir[1] == ':');
+ char cwd[HTS_URLMAXSIZE] = "";
+ char abs[HTS_URLMAXSIZE];
+ char *copy;
+
+ if (dir == NULL || (dir[0] == '-' && !optparam_dash_ok(cmd, na)) ||
+ (relative && getcwd(cwd, sizeof(cwd)) == NULL) ||
+ !slprintfbuff(abs, sizeof(abs), "%s%s%s", cwd, relative ? "/" : "",
+ dir) ||
+ (copy = hts_arena_strdup(&cmd->tokens, abs)) == NULL) {
+ HTS_PANIC_PRINTF("Option bundle-dir needs a blank space and a folder");
+ return HTS_FALSE;
+ }
+ cmd->argv[na + 1] = copy;
+ StringCopy(opt->bundle_dir, abs);
+ return HTS_TRUE;
+}
+
/* Print the headers -#C lists for `url`, read back out of the ZIP cache. */
static void cmdl_print_cache_entry(httrackp *opt, cache_back *cache,
const char *url, int sendb) {
@@ -615,6 +639,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
int httrack_logmode = 3; // ONE log file
ensureUrlCapacity(url, url_sz, 65536);
+ /* a library caller may run twice on one opt */
+ hts_bundle_free(opt);
+ StringCopy(opt->bundle_dir, "");
// Create options
_DEBUG_HEAD = 0; // pas de debuggage en têtes
@@ -1146,6 +1173,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
HTS_PANIC_PRINTF(s);
htsmain_free();
return -1;
+ } else if (strfield2(argv[i] + 2, "bundle-dir")) {
+ i++; /* The option parser reads its value. */
}
//
else {
@@ -1156,7 +1185,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
htsmain_free();
return -1;
}
-
}
}
}
@@ -1392,6 +1420,14 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
x_cmd.span[prev_opt] = na - prev_opt;
prev_opt = hts_rule_would_be_read_as_opt(argv[na]) ? na : 0;
+ if (strcmp(argv[na], "--bundle-dir") == 0) {
+ if (!bundle_dir_set(opt, &x_cmd, na)) {
+ htsmain_free();
+ return -1;
+ }
+ na++;
+ continue;
+ }
if (hts_rule_would_be_read_as_opt(argv[na])) { // option
com = argv[na] + 1;
@@ -2908,6 +2944,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
} else { // URL/filters
char catbuff[CATBUFF_SIZE];
+ /* doit.log keeps argv, so the bundle suffix is cut from a copy */
+ char word[HTS_CDLMAXSIZE];
const size_t urlSize = strlen(argv[na]);
const size_t capa = strlen(url) + urlSize + 32;
@@ -2922,15 +2960,25 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
fprintf(stderr, "* Warning: filter %s holds a quote, kept as is\n",
argv[na]);
assertf(urlSize < HTS_URLMAXSIZE);
+ strcpybuff(word, argv[na]);
+ if (argv[na][0] != '+' && argv[na][0] != '-') {
+ const char *const why = hts_bundle_take(opt, word);
+
+ if (why != NULL) {
+ cmdl_panic_value(opt, why, argv[na]);
+ htsmain_free();
+ return -1;
+ }
+ }
if (urlSize < HTS_URLMAXSIZE) {
ensureUrlCapacity(url, url_sz, capa);
if (strnotempty(url))
strlcatbuff(url, " ", url_sz); // separator space
/* a rule is decoded once, where the engine splits the rules */
append_escape_spc_url(
- argv[na][0] == '+' || argv[na][0] == '-'
- ? argv[na]
- : unescape_http_unharm(catbuff, sizeof(catbuff), argv[na], 1),
+ word[0] == '+' || word[0] == '-'
+ ? word
+ : unescape_http_unharm(catbuff, sizeof(catbuff), word, 1),
url, url_sz);
}
} // if argv=- etc.
diff --git a/src/htshelp.c b/src/htshelp.c
index 015d25061..b7d8d4a76 100644
--- a/src/htshelp.c
+++ b/src/htshelp.c
@@ -594,6 +594,9 @@ void help(const char *app, int more) {
infomsg("Spider options:");
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
infomsg(" %K use a Netscape cookies.txt as the jar, rewritten at the end");
+ infomsg(" #httrack-bundle= crawls with the user agent,");
+ infomsg(" headers and cookies of the login bundle .hts, read from "
+ "the project folder or from --bundle-dir ");
infomsg(" %Y explain which filter rule accepts or rejects a URL, then exit");
infomsg
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
diff --git a/src/htslib.c b/src/htslib.c
index 1f1d5cf49..77fc40c68 100644
--- a/src/htslib.c
+++ b/src/htslib.c
@@ -36,6 +36,7 @@ Please visit our Website: http://www.httrack.com
// Fichier librairie .c
#include "htscore.h"
+#include "htsbundle.h"
#include "htssitemap.h"
#include "htswarc.h"
#include "htschanges.h"
@@ -1156,6 +1157,20 @@ const char *http_custom_header_warning(const char *headers, const char *line) {
return NULL;
}
+/* The -%X warnings, plus Host, Cookie and Proxy-*. */
+const char *http_bundle_header_refusal(const char *headers, const char *line) {
+ static const char *const engine[] = {"Host", "Cookie"};
+ const char *const why = http_custom_header_warning(headers, line);
+
+ if (why != NULL)
+ return why;
+ if (header_name_in(line, engine, sizeof(engine) / sizeof(engine[0])))
+ return "Host and Cookie are the engine's";
+ if (strfield(line, "Proxy-"))
+ return "a proxy header is not for the site";
+ return NULL;
+}
+
/* Dropping a line drops its folded continuations too: kept, that value would
attach to whichever header precedes it in the request. */
static void append_custom_headers(buff_struct *bstr, const char *headers) {
@@ -8120,6 +8135,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
StringCopy(opt->accept_kinds, "");
opt->accept_given = HTS_FALSE;
opt->lang_given = HTS_FALSE;
+ StringCopy(opt->bundle_dir, "");
+ opt->bundle_state = NULL;
StringCopy(opt->headers, "");
StringCopy(opt->mimedefs, "\n"); // aucun filtre mime (\n IMPORTANT)
StringClear(opt->mod_blacklist);
@@ -8191,6 +8208,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
/* An FTP worker reads opt for its whole run, and not every caller drains
its threads the way httrack.c does before getting here. */
ftp_stop_workers();
+ hts_bundle_free(opt);
/* Alocated callbacks */
@@ -8267,6 +8285,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
StringFree(opt->lang_iso);
StringFree(opt->accept);
StringFree(opt->accept_kinds);
+ StringFree(opt->bundle_dir);
StringFree(opt->headers);
StringFree(opt->sys_com);
StringFree(opt->mimedefs);
diff --git a/src/htslib.h b/src/htslib.h
index a210d8471..5ca411df5 100644
--- a/src/htslib.h
+++ b/src/htslib.h
@@ -328,6 +328,10 @@ hts_boolean hts_body_missing_unexpectedly(const htsblk *r);
typed. headers is the block so far. */
const char *http_custom_header_warning(const char *headers, const char *line);
+/* Why a bundle header line is refused, or NULL when allowed. headers is the
+ block so far. */
+const char *http_bundle_header_refusal(const char *headers, const char *line);
+
/* Append the custom header block to dst (capacity dst_size), dropping any line
whose field dst carries, a malformed or framing line, an earlier copy of
Host, Cookie or Authorization, and a leading folded line. Field names ignore
diff --git a/src/htslines.c b/src/htslines.c
index 49c7d1521..4a13fe1f8 100644
--- a/src/htslines.c
+++ b/src/htslines.c
@@ -68,13 +68,15 @@ FILE *hts_fopen_private(const char *name, const char *mode) {
#endif
}
+/* hts_readline_raw() keeps every byte. */
+#define LINE_RAW 0x100
+
/* Is c one of the bytes flags drop? */
static hts_boolean line_drops(int c, int flags) {
- return c == '\r' || ((flags & HTS_LINE_DROP_TAB) != 0 && c == '\t') ||
- ((flags & HTS_LINE_DROP_FF) != 0 && c == '\f') ||
- ((flags & HTS_LINE_DROP_NUL) != 0 && c == '\0')
- ? HTS_TRUE
- : HTS_FALSE;
+ return (flags & LINE_RAW) == 0 &&
+ (c == '\r' || ((flags & HTS_LINE_DROP_TAB) != 0 && c == '\t') ||
+ ((flags & HTS_LINE_DROP_FF) != 0 && c == '\f') ||
+ ((flags & HTS_LINE_DROP_NUL) != 0 && c == '\0'));
}
/* How a line ends once trimmed, which decides whether it continues. */
@@ -118,7 +120,7 @@ static hts_boolean readline_end(FILE *fp, char *s, size_t max, int flags,
}
if (j + 1 < max)
s[j++] = (char) c;
- else if (!is_realspace(c))
+ else if ((flags & LINE_RAW) != 0 || !is_realspace(c))
cut = HTS_TRUE;
}
s[j] = '\0';
@@ -133,6 +135,12 @@ hts_boolean hts_readline(FILE *fp, char *s, size_t max, int flags) {
return readline_end(fp, s, max, flags, &len, &end);
}
+hts_boolean hts_readline_raw(FILE *fp, char *s, size_t max, size_t *len) {
+ line_end end;
+
+ return readline_end(fp, s, max, LINE_RAW, len, &end);
+}
+
/* readline_end(), then the trim hts_readline_cpp() does on each line. */
static hts_boolean readline_trim(FILE *fp, char *s, size_t max, int flags,
size_t *len, line_end *end) {
diff --git a/src/htslines.h b/src/htslines.h
index 47f7fe244..54e350d91 100644
--- a/src/htslines.h
+++ b/src/htslines.h
@@ -54,6 +54,10 @@ FILE *hts_fopen_private(const char *name, const char *mode);
when a non-space byte was dropped. At EOF it reads an empty line. */
hts_boolean hts_readline(FILE *fp, char *s, size_t max, int flags);
+/* hts_readline(fp, s, max, 0) keeping CR and NUL, and storing the byte count
+ in len. HTS_TRUE when the line was cut. */
+hts_boolean hts_readline_raw(FILE *fp, char *s, size_t max, size_t *len);
+
/* hts_readline() trimmed and joined after a trailing backslash. Leading spaces
and backslashes count against max, so "abc\\\n\n" at 4 is cut. */
hts_boolean hts_readline_cpp(FILE *fp, char *s, size_t max, int flags);
diff --git a/src/htsopt.h b/src/htsopt.h
index 1a5dbd9e7..62e7ffa17 100644
--- a/src/htsopt.h
+++ b/src/htsopt.h
@@ -657,8 +657,12 @@ struct httrackp {
line, and the last one wins. Tail: ABI */
hts_boolean accept_given; /**< Did the user give -%a? Tail: ABI */
hts_boolean lang_given; /**< Did the user give -%l? Tail: ABI */
- void *secrets; /**< passwords and tokens the logs hide (htslib.c). Live
- state, so copy_htsopt must leave it alone. Tail: ABI */
+ void *secrets; /**< passwords and tokens the logs hide (htslib.c). Live
+ state, so copy_htsopt must leave it alone. Tail: ABI */
+ String bundle_dir; /**< --bundle-dir folder, or empty. Reset per run. Tail:
+ ABI */
+ void *bundle_state; /**< hts_bundle_state*, or NULL. Live state, so
+ copy_htsopt must leave it alone. Tail: ABI */
};
/** Running statistics for a mirror. */
diff --git a/src/htsselftest.c b/src/htsselftest.c
index 1c8083fbd..b0dc0e573 100644
--- a/src/htsselftest.c
+++ b/src/htsselftest.c
@@ -223,7 +223,7 @@ static const struct selftest_entry *const selftest_tables[] = {
selftests_filters, selftests_header, selftests_io, selftests_mime,
selftests_name, selftests_net, selftests_opt, selftests_parse,
selftests_tools, selftests_warc, selftests_wizard, selftests_lines,
- selftests_fetch, selftests_psl,
+ selftests_fetch, selftests_psl, selftests_bundle,
};
static void list_selftests(void) {
diff --git a/src/htsselftest_int.h b/src/htsselftest_int.h
index a3f9b89a0..e15a3505e 100644
--- a/src/htsselftest_int.h
+++ b/src/htsselftest_int.h
@@ -134,5 +134,6 @@ extern const struct selftest_entry selftests_warc[];
extern const struct selftest_entry selftests_wizard[];
extern const struct selftest_entry selftests_fetch[];
extern const struct selftest_entry selftests_psl[];
+extern const struct selftest_entry selftests_bundle[];
#endif
diff --git a/src/libhttrack.vcxproj b/src/libhttrack.vcxproj
index f219a6e6d..44b27da14 100644
--- a/src/libhttrack.vcxproj
+++ b/src/libhttrack.vcxproj
@@ -141,6 +141,7 @@
+
@@ -156,6 +157,7 @@
+
diff --git a/tests/655_engine-bundle.test b/tests/655_engine-bundle.test
new file mode 100644
index 000000000..ddcfe462c
--- /dev/null
+++ b/tests/655_engine-bundle.test
@@ -0,0 +1,517 @@
+#!/bin/bash
+#
+# Login bundles (#httrack-bundle=NAME): the name a start URL carries, the file
+# format, the scope each bundle covers, and the bundles refused at startup.
+
+set -euo pipefail
+
+# TEST_TIMEOUT_AT_LEAST: 300
+
+# shellcheck source=tests/testlib.sh
+. "${0%"${0##*/}"}./testlib.sh"
+
+tmpdir=$(mktemp -d)
+cleanup_push rm -rf "$tmpdir"
+dir=$(nativepath "$tmpdir")
+
+bundle() { # bundle NAME LINE... -- writes $tmpdir/NAME.hts
+ local name=$1
+ shift
+ printf '%s\n' "$@" >"$tmpdir/$name.hts"
+}
+tab=$(printf '\t')
+jar() { # jar DOMAIN NAME VALUE -- one Netscape jar line
+ printf '%s' ".$1${tab}TRUE${tab}/${tab}FALSE${tab}0${tab}$2${tab}$3"
+}
+
+name127=$(repeat_chars 127 a)
+selftest_queue 'login1 https://example.com/members/' bundlecut \
+ 'https://example.com/members/#httrack-bundle=login1'
+selftest_queue 'a.b_c-9 http://x/' bundlecut 'http://x/#httrack-bundle=a.b_c-9'
+selftest_queue "$name127 http://x/" bundlecut "http://x/#httrack-bundle=$name127"
+selftest_queue 'x http://x/#a' bundlecut 'http://x/#a#httrack-bundle=x'
+for ok in console com10 com0 lpt10 nul1; do
+ selftest_queue "$ok http://x/" bundlecut "http://x/#httrack-bundle=$ok"
+done
+selftest_queue none bundlecut 'http://x/#top'
+badname='bad: invalid login bundle name: '
+for bad in '' ../x a/b 'a\b' a%2fb 'a#httrack-bundle=b'; do
+ selftest_queue "${badname}use letters, digits, '.', '_' and '-'" \
+ bundlecut "http://x/#httrack-bundle=$bad"
+done
+for bad in ..x a..b .hidden; do
+ selftest_queue "${badname}it starts with '.' or holds \"..\"" \
+ bundlecut "http://x/#httrack-bundle=$bad"
+done
+selftest_queue "${badname}128 bytes or more" bundlecut \
+ "http://x/#httrack-bundle=${name127}a"
+for bad in CON con.test Nul.x prn aux com1 COM9 lpt1 LPT9.hts; do
+ selftest_queue "${badname}a Windows device name" bundlecut \
+ "http://x/#httrack-bundle=$bad"
+done
+
+# The default scope is the registrable domains of the naming URL and the url
+# lines. Cookie domains do not widen it: alpha-cdn.test's cookie is dropped.
+# The naming URL is https, so the headers go over https alone.
+bundle alpha '# HTTrack bundle 1' 'url https://login.alpha-sso.test/start' \
+ 'ua AlphaUA/1.0' 'header Authorization: Bearer t' '# a note' \
+ 'header X-A: 1' 'unknown-key is warned about' 'nakedkey' cookies \
+ "$(jar alpha.test SID s1)" "$(jar alpha-cdn.test C c1)" \
+ "$(jar alpha.test PREF p1)" "$(jar alpha.test SID s2)" \
+ '# an unknown comment is warned about'
+alpha_auth='ua=AlphaUA/1.0 headers=Authorization: Bearer t|X-A: 1|'
+selftest_queue_lines "url https://login.alpha-sso.test/start
+cookie .alpha.test PREF=p1
+cookie .alpha.test SID=s2
+https://alpha.test $alpha_auth
+https://deep.www.alpha.test:8443 $alpha_auth
+http://www.alpha.test ua=AlphaUA/1.0 headers=global
+https://login.alpha-sso.test $alpha_auth
+https://cdn.gamma.test ua=global headers=global
+https://notalpha.test ua=global headers=global
+https://alpha-cdn.test ua=global headers=global" bundle "$dir" \
+ 'https://www.alpha.test/#httrack-bundle=alpha' host:https://alpha.test \
+ host:https://deep.www.alpha.test:8443 host:http://www.alpha.test \
+ host:https://login.alpha-sso.test host:https://cdn.gamma.test \
+ host:https://notalpha.test host:https://alpha-cdn.test
+
+# A second load on one opt reads the files again rather than adding to them.
+selftest_queue_lines "url https://login.alpha-sso.test/start
+cookie .alpha.test PREF=p1
+cookie .alpha.test SID=s2
+https://alpha.test $alpha_auth" bundle "$dir" \
+ 'https://www.alpha.test/#httrack-bundle=alpha' host:https://alpha.test \
+ twice
+
+# Any https URL of the bundle, naming it or a url line, keeps its headers off
+# plain http.
+bundle mixed '# HTTrack bundle 1' 'url http://www.mixed.test/' 'ua M' \
+ 'header X-M: 1'
+for urls in 'https://mixed.test/ http://mixed.test/' \
+ 'http://mixed.test/ https://mixed.test/'; do
+ read -r u1 u2 <<<"$urls"
+ selftest_queue_lines "url http://www.mixed.test/
+http://mixed.test ua=M headers=global
+http://www.mixed.test ua=M headers=global
+https://www.mixed.test ua=M headers=X-M: 1|" bundle "$dir" \
+ "$u1#httrack-bundle=mixed" "$u2#httrack-bundle=mixed" \
+ host:http://mixed.test host:http://www.mixed.test \
+ host:https://www.mixed.test
+done
+bundle mixed2 '# HTTrack bundle 1' 'url https://www.mixed.test/' 'ua M' \
+ 'header X-M: 1'
+selftest_queue_tail "
+http://mixed.test ua=M headers=global" bundle "$dir" \
+ 'http://mixed.test/#httrack-bundle=mixed2' host:http://mixed.test
+bundle plainm '# HTTrack bundle 1' 'ua M' 'header X-M: 1'
+selftest_queue "http://mixed.test ua=M headers=X-M: 1|" bundle "$dir" \
+ 'http://mixed.test/#httrack-bundle=plainm' host:http://mixed.test
+# Bundles that merge share the rule, whichever is listed first.
+for pair in 'plainm mixed2' 'mixed2 plainm'; do
+ read -r u1 u2 <<<"$pair"
+ selftest_queue_tail "
+http://mixed.test ua=M headers=global
+https://mixed.test ua=M headers=X-M: 1|" bundle "$dir" \
+ "http://mixed.test/#httrack-bundle=$u1" \
+ "http://www.mixed.test/#httrack-bundle=$u2" host:http://mixed.test \
+ host:https://mixed.test
+done
+
+# Merging is transitive: A and C share no site, but both overlap B, so C's
+# https URL keeps A's headers off plain http too, in either order. scope lines
+# leave the https rule as it is.
+for n in a b c; do
+ lines=('# HTTrack bundle 1' 'ua H' 'header X-H: 1')
+ case $n in
+ a) lines+=('scope a.test' 'scope b.test') ;;
+ b) lines+=('scope b.test' 'scope c.test') ;;
+ c) lines+=('scope c.test' 'url https://www.c.test/') ;;
+ esac
+ bundle "chain$n" "${lines[@]}"
+done
+chain_hosts=(host:http://a.test host:https://a.test host:http://c.test)
+chain_out="http://a.test ua=H headers=global
+https://a.test ua=H headers=X-H: 1|
+http://c.test ua=H headers=global"
+selftest_queue_tail "
+$chain_out" bundle "$dir" 'http://a.test/#httrack-bundle=chaina' \
+ 'http://b.test/#httrack-bundle=chainb' 'http://c.test/#httrack-bundle=chainc' \
+ "${chain_hosts[@]}"
+selftest_queue_tail "
+$chain_out" bundle "$dir" 'http://c.test/#httrack-bundle=chainc' \
+ 'http://b.test/#httrack-bundle=chainb' 'http://a.test/#httrack-bundle=chaina' \
+ "${chain_hosts[@]}"
+selftest_queue_tail "
+http://c.test ua=H headers=global" bundle "$dir" \
+ 'http://c.test/#httrack-bundle=chainc' host:http://c.test
+
+# The registrable domain follows the suffix list, not the last two labels. A
+# host with none (a public suffix, a single label) covers itself alone, and so
+# does an address.
+bundle uk '# HTTrack bundle 1' 'ua UK' 'header X-UK: 1'
+selftest_queue_lines "a.alpha.co.uk ua=UK headers=X-UK: 1|
+other.co.uk ua=global headers=global
+s3.amazonaws.com ua=UK headers=X-UK: 1|
+attacker.s3.amazonaws.com ua=global headers=global
+wiki ua=UK headers=X-UK: 1|
+a.wiki ua=global headers=global
+127.0.0.1 ua=UK headers=X-UK: 1|
+127.0.0.2 ua=global headers=global" bundle "$dir" \
+ 'http://www.alpha.co.uk/#httrack-bundle=uk' \
+ 'http://s3.amazonaws.com/x#httrack-bundle=uk' \
+ 'http://wiki/#httrack-bundle=uk' 'http://127.0.0.1:8080/#httrack-bundle=uk' \
+ host:a.alpha.co.uk host:other.co.uk host:s3.amazonaws.com \
+ host:attacker.s3.amazonaws.com host:wiki host:a.wiki host:127.0.0.1 \
+ host:127.0.0.2
+
+# scope lines replace the default, and a cookie outside them is dropped. CRLF
+# line ends and a BOM are read. TAB is the one control byte a line may hold.
+printf '\357\273\277# HTTrack bundle 1.3\r\nscope WWW.alpha.test\r\nscope .beta.test\r\nscope 127.0.0.1\r\nua\tN\tx\r\nheader X-N:\t1\r\ncookies\r\n%s\r\n%s\r\n' \
+ "$(jar alpha.test SID wide)" "$(jar www.alpha.test K narrow)" \
+ >"$tmpdir/narrow.hts"
+selftest_queue_lines "cookie .www.alpha.test K=narrow
+alpha.test ua=global headers=global
+a.www.alpha.test ua=N${tab}x headers=X-N:${tab}1|
+a.beta.test ua=N${tab}x headers=X-N:${tab}1|
+127.0.0.1 ua=N${tab}x headers=X-N:${tab}1|
+127.0.0.2 ua=global headers=global" bundle "$dir" \
+ 'http://www.alpha.test/#httrack-bundle=narrow' host:alpha.test \
+ host:a.www.alpha.test host:a.beta.test host:127.0.0.1 host:127.0.0.2
+
+# Of the bundle and the jar, the newer file wins, and the bundle wins a tie.
+order() { # order NAME BUNDLE-TIME JAR-TIME WINNER -- times as touch -t takes
+ bundle "$1" '# HTTrack bundle 1' cookies "$(jar order.test SID bundle)"
+ jar order.test SID jar >"$tmpdir/$1.txt"
+ touch -t "$2" "$tmpdir/$1.hts"
+ touch -t "$3" "$tmpdir/$1.txt"
+ selftest_queue "cookie .order.test SID=$4" bundle "$dir" \
+ "http://www.order.test/#httrack-bundle=$1" "jar:$dir/$1.txt"
+}
+order order1 202601021000.00 202601011000.00 bundle
+order order2 202601011000.00 202601021000.00 jar
+order order3 202601011000.00 202601011000.00 bundle
+# Half a second apart in the same second, which a seconds-only compare calls a
+# tie. Only a touch that sets fractions can write it.
+if touch -d '2026-01-01T10:00:00.5' "$tmpdir/subsecond" 2>/dev/null &&
+ test "$tmpdir/subsecond" -nt "$tmpdir/order3.txt"; then
+ order order4 202601011000.00 202601011000.00 jar
+ touch -d '2026-01-01T10:00:00.5' "$tmpdir/order4.txt"
+ want_rows=139
+else
+ echo "skipping the sub-second jar order row: touch cannot set fractions" >&2
+ want_rows=138
+fi
+
+# A control byte in a -%K or cookies.txt jar loads as it always did.
+printf '.ctl.test\tTRUE\t/\tFALSE\t0\tK\ta\001b\n' >"$tmpdir/ctl.txt"
+bundle plain '# HTTrack bundle 1'
+selftest_queue "cookie .ctl.test K=a$(printf '\001')b" bundle "$dir" \
+ 'http://www.alpha.test/#httrack-bundle=plain' "jar:$dir/ctl.txt"
+
+# Refused: each one stops the run and names the file and line, never a byte of
+# the line. The batch reads them all at the end, so each case writes its own.
+nbad=0
+refused() { # refused WHY LINE... -- a one-bundle file the engine refuses
+ local why=$1
+ shift
+ nbad=$((nbad + 1))
+ bundle "bad$nbad" "$@"
+ selftest_queue "refused: /bad$nbad.hts$why" bundle "$dir" \
+ "http://www.alpha.test/#httrack-bundle=bad$nbad"
+}
+notbundle=': not an HTTrack bundle, the first line must be "# HTTrack bundle 1"'
+refused ': bundle version 2 is not supported' '# HTTrack bundle 2'
+refused "$notbundle" 'ua X'
+refused "$notbundle" '# HTTrack bundle 1.'
+refused "$notbundle" '# HTTrack bundle 1 x'
+refused "$notbundle" '# HTTrack bundle 01'
+# an empty file, and one holding a BOM alone
+: >"$tmpdir/empty.hts"
+selftest_queue "refused: /empty.hts$notbundle" bundle "$dir" \
+ 'http://www.alpha.test/#httrack-bundle=empty'
+printf '\357\273\277' >"$tmpdir/bom.hts"
+selftest_queue "refused: /bom.hts$notbundle" bundle "$dir" \
+ 'http://www.alpha.test/#httrack-bundle=bom'
+refused ", line 2: refused header: Host and Cookie are the engine's" \
+ '# HTTrack bundle 1' 'header Host: x'
+refused ", line 2: refused header: Host and Cookie are the engine's" \
+ '# HTTrack bundle 1' 'header cookie: secret-cookie'
+refused ', line 2: refused header: a proxy header is not for the site' \
+ '# HTTrack bundle 1' 'header Proxy-Authorization: secret-proxy'
+refused ', line 2: refused header: framing header ignored' \
+ '# HTTrack bundle 1' 'header Transfer-Encoding: chunked'
+refused ', line 3: refused header: header replaces an earlier one' \
+ '# HTTrack bundle 1' 'header Authorization: a' 'header Authorization: secret-b'
+refused ', line 2: refused header: malformed header ignored' \
+ '# HTTrack bundle 1' 'header Bearer secret-no-colon'
+# The header lines hold 8192 bytes at most, each with its CRLF.
+bundle hdr8192 '# HTTrack bundle 1' "header X-A: $(repeat_chars 5000 a)" \
+ "header X-B: $(repeat_chars 3178 b)"
+selftest_queue '' bundle "$dir" 'http://x.test/#httrack-bundle=hdr8192'
+refused ', line 3: the header lines pass 8192 bytes' '# HTTrack bundle 1' \
+ "header X-A: $(repeat_chars 5000 a)" "header X-B: $(repeat_chars 3179 b)"
+# A control byte would split a header on the wire. The CR before the LF goes.
+ctl=', line 2: holds a control byte'
+refused "$ctl" '# HTTrack bundle 1' "$(printf 'ua x\rHost: evil.test')"
+refused "$ctl" '# HTTrack bundle 1' \
+ "$(printf 'header X-A: 1\rCookie: forged=1\rProxy-Authorization: z')"
+refused "$ctl" '# HTTrack bundle 1' "$(printf 'scope a.test\001')"
+refused "$ctl" '# HTTrack bundle 1' "$(printf 'ua a\177b')"
+refused ', line 1: holds a control byte' "$(printf '# HTTrack bundle 1\rua x')"
+nbad=$((nbad + 1))
+printf '# HTTrack bundle 1\nua a\000Host: evil.test\n' >"$tmpdir/bad$nbad.hts"
+selftest_queue "refused: /bad$nbad.hts$ctl" bundle "$dir" \
+ "http://www.alpha.test/#httrack-bundle=bad$nbad"
+# A line holds 8191 bytes at most, its LF or CRLF aside.
+bundle long '# HTTrack bundle 1' "ua $(repeat_chars 8188 u)"
+selftest_queue '' bundle "$dir" 'http://x.test/#httrack-bundle=long'
+printf '# HTTrack bundle 1\nua %s\r\n' "$(repeat_chars 8188 u)" \
+ >"$tmpdir/longcrlf.hts"
+selftest_queue '' bundle "$dir" 'http://x.test/#httrack-bundle=longcrlf'
+refused ', line 2: too long' '# HTTrack bundle 1' "ua $(repeat_chars 8189 u)"
+for end in '\r\v\f' '\v' '\f'; do
+ nbad=$((nbad + 1))
+ printf '# HTTrack bundle 1\nua %s%b\n' "$(repeat_chars 8188 u)" "$end" \
+ >"$tmpdir/bad$nbad.hts"
+ selftest_queue "refused: /bad$nbad.hts, line 2: too long" bundle "$dir" \
+ "http://www.alpha.test/#httrack-bundle=bad$nbad"
+done
+# In the jar, every control byte but TAB stops the run too, in any field and
+# under -b0, and so does a line too long to read whole.
+jarbad() { # jarbad LINE WHY [b0] -- LINE is the cookie line of a bundle
+ nbad=$((nbad + 1))
+ printf '# HTTrack bundle 1\ncookies\n%s\n' "$1" >"$tmpdir/bad$nbad.hts"
+ selftest_queue "refused: /bad$nbad.hts, line 3: $2" bundle "$dir" \
+ "http://www.alpha.test/#httrack-bundle=bad$nbad" "${@:3}"
+}
+ctlcookie='a cookie line holds a control byte'
+jarbad "$(printf '.al\001pha.test\tTRUE\t/\tFALSE\t0\tSID\tv')" "$ctlcookie"
+jarbad "$(printf '.alpha.test\tTRUE\t/\001\tFALSE\t0\tSID\tv')" "$ctlcookie"
+jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tS\001ID\tv')" "$ctlcookie"
+jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\ta\001b')" "$ctlcookie" b0
+long=$(repeat_chars 8100 x)
+jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s\001' "$long")" "$ctlcookie"
+for c in '\v' '\f'; do
+ jarbad "$(printf '%b.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s' "$c" "$long")" "$ctlcookie"
+done
+jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s' "$long")" 'a cookie line is too long'
+# past the bytes the reader keeps, the length alone stops it
+jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s%s\001' "$long" "$long")" 'a cookie line is too long'
+jarbad "$(printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\t%s' "$(repeat_chars 4094 v)")" 'a cookie name and value pass 4096 bytes'
+# name and value of exactly 4096 bytes are accepted
+ok4096=$(repeat_chars 4093 v)
+bundle ok4096 '# HTTrack bundle 1' cookies "$(jar alpha.test SID "$ok4096")"
+selftest_queue_lines "cookie .alpha.test SID=$ok4096" bundle "$dir" \
+ 'http://www.alpha.test/#httrack-bundle=ok4096'
+bundle b0ok '# HTTrack bundle 1' 'ua B0' cookies "$(jar alpha.test SID s)"
+selftest_queue 'www.alpha.test ua=B0 headers=global' bundle "$dir" \
+ 'http://www.alpha.test/#httrack-bundle=b0ok' host:www.alpha.test b0
+for oct in 000 001 002 003 004 005 006 007 010 013 014 015 016 017 020 021 \
+ 022 023 024 025 026 027 030 031 032 033 034 035 036 037 177; do
+ nbad=$((nbad + 1))
+ {
+ printf '# HTTrack bundle 1\ncookies\n'
+ printf '.alpha.test\tTRUE\t/\tFALSE\t0\tSID\tv%bx\n' "\\0$oct"
+ } >"$tmpdir/bad$nbad.hts"
+ selftest_queue "refused: /bad$nbad.hts, line 3: a cookie line holds a control byte" \
+ bundle "$dir" "http://www.alpha.test/#httrack-bundle=bad$nbad"
+done
+refused ', line 3: ua given twice' '# HTTrack bundle 1' 'ua A' 'ua B'
+refused ', line 2: the scope is a public suffix' '# HTTrack bundle 1' \
+ 'scope co.uk'
+refused ', line 2: invalid scope' '# HTTrack bundle 1' 'scope .'
+refused ', line 2: invalid url' '# HTTrack bundle 1' 'url ftp:///'
+refused ', line 2: invalid url' '# HTTrack bundle 1' \
+ 'url http://a.test/#httrack-bundle=x'
+refused ', line 2: no value' '# HTTrack bundle 1' 'ua'
+selftest_queue "refused: /uk.hts: cannot work out the bundle's scope" \
+ bundle "$dir" 'http:///#httrack-bundle=uk'
+selftest_queue 'refused: /none.hts: cannot read the login bundle: No such file or directory' \
+ bundle "$dir" 'http://www.alpha.test/#httrack-bundle=none'
+full=()
+for i in $(seq 1 400); do
+ full+=("$(jar alpha.test "C$i" "$(repeat_chars 100 v)")")
+done
+refused ', line 227: the cookie jar did not store a cookie' \
+ '# HTTrack bundle 1' cookies "${full[@]}"
+# 256 url lines and 256 scope domains at most
+urls=()
+scopes=()
+for i in $(seq 1 257); do
+ urls+=("url http://x.test/$i")
+ scopes+=("scope h$i.test")
+done
+bundle urls256 '# HTTrack bundle 1' "${urls[@]:0:256}"
+selftest_queue_tail '
+url http://x.test/256' bundle "$dir" 'http://x.test/#httrack-bundle=urls256'
+refused ', line 258: more than 256 url lines' '# HTTrack bundle 1' "${urls[@]}"
+refused ', line 258: more than 256 scope domains' '# HTTrack bundle 1' \
+ "${scopes[@]}"
+
+# Several bundles: one name twice is one bundle, equal identities on one site
+# merge with their cookies, and different ones are refused either way round.
+bundle beta '# HTTrack bundle 1' 'ua BetaUA' cookies "$(jar beta.test SID b1)"
+bundle alpha2 '# HTTrack bundle 1' 'ua AlphaUA/1.0' \
+ 'header Authorization: Bearer t' 'header X-A: 1' cookies \
+ "$(jar alpha.test OTHER o)" "$(jar alpha.test SID s2)"
+bundle alphaua '# HTTrack bundle 1' 'ua SomeoneElse' \
+ 'header Authorization: Bearer t' 'header X-A: 1'
+bundle alphaauth '# HTTrack bundle 1' 'ua AlphaUA/1.0' \
+ 'header Authorization: Bearer u' 'header X-A: 1'
+bundle alphasid '# HTTrack bundle 1' 'ua AlphaUA/1.0' \
+ 'header Authorization: Bearer t' 'header X-A: 1' cookies \
+ "$(jar alpha.test SID another-account)"
+bundle wwwalpha '# HTTrack bundle 1' 'scope www.alpha.test' 'ua Narrow'
+bundle loop1 '# HTTrack bundle 1' 'scope 127.0.0.1' 'ua One'
+bundle loop2 '# HTTrack bundle 1' 'ua Two'
+selftest_queue_lines "cookie .beta.test SID=b1
+www.beta.test ua=BetaUA headers=global" bundle "$dir" \
+ 'http://www.beta.test/#httrack-bundle=beta' \
+ 'http://beta.test/x#httrack-bundle=beta' host:www.beta.test
+selftest_queue_tail "
+cookie .alpha.test PREF=p1
+cookie .alpha.test OTHER=o
+cookie .alpha.test SID=s2
+cookie .beta.test SID=b1
+https://alpha.test $alpha_auth
+beta.test ua=BetaUA headers=global" bundle "$dir" \
+ 'https://www.alpha.test/#httrack-bundle=alpha' \
+ 'https://alpha.test/#httrack-bundle=alpha2' \
+ 'http://beta.test/#httrack-bundle=beta' host:https://alpha.test \
+ host:beta.test
+differ() { # differ A B [C] -- A and the last one overlap with other identities
+ local last=${3:-$2} more=()
+ test -z "${3:-}" || more=("http://beta.test/#httrack-bundle=$2")
+ selftest_queue "refused: /$1.hts and /$last.hts cover the same site with a different user agent or headers" \
+ bundle "$dir" "https://www.alpha.test/#httrack-bundle=$1" \
+ ${more[@]+"${more[@]}"} "https://alpha.test/#httrack-bundle=$last"
+}
+differ alpha alphaua
+differ alpha alphaauth
+differ alpha wwwalpha
+differ wwwalpha alpha
+differ alpha beta alphaua
+selftest_queue "refused: /loop1.hts and /loop2.hts cover the same site with a different user agent or headers" \
+ bundle "$dir" 'http://x.test/#httrack-bundle=loop1' \
+ 'http://127.0.0.1/#httrack-bundle=loop2'
+# A cookie of the same name on another path or domain is another cookie.
+bundle clash1 '# HTTrack bundle 1' cookies "$(jar alpha.test SID one)"
+bundle clash2 '# HTTrack bundle 1' cookies \
+ ".alpha.test${tab}TRUE${tab}/x${tab}FALSE${tab}0${tab}SID${tab}two"
+bundle clash3 '# HTTrack bundle 1' cookies "$(jar www.alpha.test SID three)"
+selftest_queue_lines "cookie .alpha.test SID=two
+cookie .alpha.test SID=one
+cookie .www.alpha.test SID=three" bundle "$dir" \
+ 'http://alpha.test/#httrack-bundle=clash1' \
+ 'http://www.alpha.test/#httrack-bundle=clash2' \
+ 'http://x.alpha.test/#httrack-bundle=clash3'
+selftest_queue "refused: /alphasid.hts, line 6: a cookie has another value in another bundle" \
+ bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' \
+ 'https://alpha.test/#httrack-bundle=alphasid'
+
+# Chained bundles must load in linear time: 2000 may cost at most 64 times 250
+# (linear is about 8, cubic about 512). Relative, so slow legs pass.
+now_ms() {
+ if perl -MTime::HiRes=time -e 1 2>/dev/null; then
+ perl -MTime::HiRes=time -e 'printf "%d", time * 1000'
+ elif command -v python3 >/dev/null; then
+ python3 -c 'import time; print(int(time.time() * 1000))'
+ else
+ return 1
+ fi
+}
+chain_ms() { # chain_ms N -- milliseconds to load N chained bundles
+ local n=$1 i t0 t1 chain=()
+ for i in $(seq 1 "$n"); do
+ printf '# HTTrack bundle 1\nua U\nscope h%d.test\nscope h%d.test\n' \
+ "$i" "$((i + 1))" >"$tmpdir/ch$i.hts"
+ chain+=("http://h$i.test/#httrack-bundle=ch$i")
+ done
+ t0=$(now_ms)
+ assert_selftest 'h1.test ua=U headers=global' bundle "$dir" "${chain[@]}" host:h1.test
+ t1=$(now_ms)
+ echo "$((t1 - t0))"
+}
+if target_is_windows; then
+ # 2000 URLs pass the 32767-character Windows command line
+ echo "skipping the chained bundle timing row on Windows" >&2
+elif now_ms >/dev/null; then
+ t250=$(chain_ms 250)
+ test "$t250" -ge 50 || t250=50
+ skip_if_out_of_budget 1 "$((t250 * 8 / 1000 + 1))" "the 2000-bundle chain row"
+ t2000=$(chain_ms 2000)
+ test "$t2000" -le "$((t250 * 64))" ||
+ fail "2000 chained bundles took ${t2000}ms against ${t250}ms for 250: not linear"
+else
+ echo "skipping the chained bundle timing row: no perl or python3" >&2
+fi
+
+# The real count of queued rows, so a deleted row cannot pass unseen.
+test "${#SELFTEST_WANT[@]}" -eq "$want_rows" ||
+ fail "expected $want_rows queued rows, got ${#SELFTEST_WANT[@]}"
+selftest_run_queued
+
+# Warnings name the line, and never a byte of it that could be a secret: a
+# key in lowercase letters alone is named.
+out=$(httrack -#test=bundle "$dir" 'https://www.alpha.test/#httrack-bundle=alpha' 2>&1 | tr -d '\r')
+for want in 'alpha.hts, line 7: unknown key$' 'alpha.hts, line 8: unknown key nakedkey$' \
+ 'alpha.hts, line 13: a cookie appears twice$' 'alpha.hts, line 14: an unknown comment$' \
+ 'alpha.hts: 1 cookie\(s\) outside its scope left out'; do
+ grep -qE "$want" <<<"$out" || fail "no warning [$want]: $out"
+done
+bundle warns '# HTTrack bundle 1' 'headers X: 1' 'abc123 v' \
+ "$(repeat_chars 33 k) v" "$(repeat_chars 32 k) v" cookies \
+ ".alpha.test${tab}TRUE${tab}/${tab}FALSE${tab}0${tab}A" \
+ "$(jar alpha.test B b)${tab}x${tab}y" \
+ ".alpha.test${tab}TRUE${tab}/${tab}FALSE${tab}soon${tab}C${tab}c" \
+ "$(jar "$(repeat_chars 300 d).alpha.test" D d)"
+out=$(httrack -#test=bundle "$dir" 'https://www.alpha.test/#httrack-bundle=warns' 2>&1 | tr -d '\r')
+for want in 'line 2: unknown key headers' 'line 3: unknown key' \
+ 'line 4: unknown key' "line 5: unknown key $(repeat_chars 32 k)" \
+ 'line 7: a cookie line has fewer than 7 fields or no name' \
+ 'line 8: a cookie line has more than 7 fields' \
+ 'line 9: a cookie expiry is not a whole number' \
+ 'line 10: a cookie field is too long'; do
+ grep -qE "/warns\.hts, $want\$" <<<"$out" || fail "no warning [$want]: $out"
+done
+secret=ZQXSECRET
+sec() { # sec NAME LINE... -- a bundle whose lines hold the secret
+ local name=$1
+ shift
+ printf '%s\n' '# HTTrack bundle 1' "$@" >"$tmpdir/$name.hts"
+}
+sec s1 "header Authorization Bearer $secret:x"
+sec s2 "X-Api-Key:$secret" "$secret" "url ftp:///$secret"
+sec s3 "scope $secret!"
+sec s4 "header X-A: $secret" "header Proxy-$secret: 1"
+sec s5 cookies "#$secret" "$(jar alpha.test "$secret" 1)" \
+ "$(jar alpha.test "$secret" 2)" "$(jar alpha.test "$secret" "$(repeat_chars 8000 x)")" \
+ "$(jar alpha.test "$secret" 1)$(printf '\001')"
+sec control "url http://www.alpha.test/?t=$secret"
+sec keys "token $secret"
+for s in s1 s2 s3 s4 s5 keys control; do
+ out=$(httrack -#test=bundle "$dir" "https://www.alpha.test/#httrack-bundle=$s" 2>&1 | tr -d '\r')
+ case $s in
+ control) grep -qF "$secret" <<<"$out" || fail "the probe cannot see a secret: $out" ;;
+ *) grep -qF "$secret" <<<"$out" && fail "$s shows the secret: $out" ;;
+ esac
+ grep -qF "/$s.hts" <<<"$out" || test $s = control || fail "$s: no message: $out"
+ test $s != keys || grep -qE '/keys\.hts, line 2: unknown key token$' <<<"$out" ||
+ fail "the key alone is not named: $out"
+done
+
+# The log backstop learns a bundle's header token and url-line password.
+bundle hidden '# HTTrack bundle 1' 'header Authorization: Bearer BUNDLETOKEN123' \
+ 'url http://u:URLPASSWD9@www.alpha.test/x'
+hidden_url='url http://u:URLPASSWD9@www.alpha.test/x'
+assert_selftest "$hidden_url"$'\n''hide x *** y' bundle "$dir" \
+ 'https://www.alpha.test/#httrack-bundle=hidden' 'hide:x BUNDLETOKEN123 y'
+assert_selftest "$hidden_url"$'\n''hide see http://u:***@h/' bundle "$dir" \
+ 'https://www.alpha.test/#httrack-bundle=hidden' 'hide:see http://u:URLPASSWD9@h/'
+
+# A bad name on the command line stops the run before it starts.
+rc=0
+out=$(httrack -O "$dir/argv" 'http://x.test/#httrack-bundle=..a' 2>&1) || rc=$?
+test "$rc" -ne 0 || fail "a bad bundle name on the command line was accepted"
+grep -qF "invalid login bundle name: it starts with '.' or holds \"..\"" <<<"$out" ||
+ fail "no message for a bad bundle name: $out"
diff --git a/tests/656_local-bundles.test b/tests/656_local-bundles.test
new file mode 100644
index 000000000..3842c842a
--- /dev/null
+++ b/tests/656_local-bundles.test
@@ -0,0 +1,210 @@
+#!/bin/bash
+#
+# Login bundles in a real crawl, graded on what the server received: each site
+# gets its own UA, Authorization and cookies, while a CDN and another user's
+# site under a public suffix get the global -F and -%X. --update finds the
+# bundles again, and the newer of the bundle and the jar wins.
+
+set -euo pipefail
+
+# shellcheck source=tests/testlib.sh
+. "${0%"${0##*/}"}./testlib.sh"
+
+python=$(find_python) || skip "python3 missing"
+
+bin=$(httrack_path)
+server=$(nativepath "$top_srcdir/tests/bundle-server.py")
+tmpdir=$(mktemp -d)
+serverpid=
+log=$tmpdir/crawl.log
+fail_dump_var log
+
+cleanup() {
+ stop_server "$serverpid"
+ rm -rf "$tmpdir"
+}
+cleanup_push cleanup
+
+"$python" "$server" >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
+serverpid=$!
+port=$(discover_server_port "$tmpdir/srv.out" "$serverpid") || exit 1
+
+tab=$(printf '\t')
+jar() { # jar DOMAIN NAME VALUE -- one Netscape jar line
+ printf '%s' ".$1${tab}TRUE${tab}/${tab}FALSE${tab}0${tab}$2${tab}$3"
+}
+mkdir "$tmpdir/logins" "$tmpdir/one" "$tmpdir/two"
+alpha=$tmpdir/logins/alpha.test.hts
+printf '%s\n' '# HTTrack bundle 1' 'ua AlphaUA/1.0' \
+ 'header Authorization: Bearer alpha-token' cookies \
+ "$(jar alpha.test SID alpha-captured)" "$(jar alpha.test PREF alpha-pref)" \
+ >"$alpha"
+printf '%s\n' '# HTTrack bundle 1' 'ua BetaUA/2.0' \
+ 'header Authorization: Bearer beta-token' cookies \
+ "$(jar beta.test SID beta-captured)" >"$tmpdir/logins/beta.test.hts"
+printf '%s\n' '# HTTrack bundle 1' 'ua GhUA/3.0' \
+ 'header Authorization: Bearer gh-token' >"$tmpdir/logins/gh.hts"
+cp "$alpha" "$tmpdir/alpha.orig"
+jarfile=$tmpdir/one/cookies.txt
+jar alpha.test SID alpha-refreshed >"$jarfile"
+
+crawl() { # crawl CWD [httrack args...]
+ local cwd=$1 rc=0
+ shift
+ (cd "$cwd" && run_with_timeout 120 "$bin" -P "127.0.0.1:$port" \
+ -F GlobalUA/9 -%X 'X-Global: 1' '+*cdn.gamma.test/*' \
+ '+*github.io/*' -c1 --robots=0 -%v0 -q "$@") >>"$log" 2>&1 || rc=$?
+ test "$rc" -ne 124 || fail "crawl hit the deadline"
+ return "$rc"
+}
+hits() { wc -l <"$tmpdir/srv.out"; }
+hit() { # hit WHAT FROM LINE -- the server logged LINE after line FROM
+ tail -n "+$(($2 + 1))" "$tmpdir/srv.out" >"$tmpdir/new.out"
+ command grep -qxF "HIT $3" "$tmpdir/new.out" ||
+ fail_dump "$1: no request [HIT $3]" "$tmpdir/new.out"
+}
+dated() { # dated BUNDLE-TIME JAR-TIME -- times as touch -t takes them
+ touch -t "$1" "$alpha"
+ touch -t "$2" "$jarfile"
+}
+
+# The jar is newer than the bundle, so its SID wins. --bundle-dir is relative.
+dated 202601011000.00 202601021000.00
+crawl "$tmpdir" -O one --bundle-dir logins \
+ 'http://www.alpha.test/#httrack-bundle=alpha.test' \
+ 'http://www.beta.test/#httrack-bundle=beta.test' \
+ 'http://alpha.test/plain.html' 'http://github.io/#httrack-bundle=gh' ||
+ fail "the bundle crawl failed"
+a='ua=AlphaUA/1.0 auth=Bearer alpha-token cookie=PREF=alpha-pref; SID'
+hit "start page" 0 "www.alpha.test/ $a=alpha-refreshed x=-"
+hit "crawled link" 0 "www.alpha.test/page.html $a=alpha-refreshed x=-"
+hit "plain URL on the bundle's site" 0 "alpha.test/plain.html $a=alpha-refreshed x=-"
+hit "second bundle" 0 "www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-"
+for gif in a.gif b.gif; do
+ hit "CDN" 0 "cdn.gamma.test/$gif ua=GlobalUA/9 auth=- cookie=- x=1"
+done
+hit "suffix host" 0 "github.io/ ua=GhUA/3.0 auth=Bearer gh-token cookie=- x=-"
+hit "another user under the suffix" 0 \
+ "attacker.github.io/x.html ua=GlobalUA/9 auth=- cookie=- x=1"
+n=$(hits)
+test "$n" -eq 9 || fail_dump "$n lines, expected PORT and 8 requests" "$tmpdir/srv.out"
+cmp -s "$tmpdir/alpha.orig" "$alpha" || fail "the crawl wrote to the bundle"
+doit=$tmpdir/one/hts-cache/doit.log
+test -f "$doit" || fail "no $doit"
+command grep -qF -- '--bundle-dir logins' "$doit" &&
+ fail_dump "doit.log kept a relative --bundle-dir" "$doit"
+# The suffix is cut before any URL reaches a file name, a log or the index.
+# hts-log.txt echoes the command line in parentheses, as doit.log keeps it.
+no_suffix() { # no_suffix PROJECT
+ local names f
+ names=$(cd "$1" && find . -name '*httrack-bundle*')
+ test -z "$names" || fail "a file name holds the bundle suffix: $names"
+ for f in index.html hts-log.txt hts-cache/new.txt; do
+ test -f "$1/$f" || fail "no $1/$f"
+ if command grep -v '^(' "$1/$f" | command grep -F '#httrack-bundle'; then
+ fail_dump "$f holds the bundle suffix" "$1/$f"
+ fi
+ done
+}
+no_suffix "$tmpdir/one"
+command grep -qF 'http://www.alpha.test/ ' "$tmpdir/one/hts-log.txt" ||
+ fail_dump "hts-log.txt does not name the start URL" "$tmpdir/one/hts-log.txt"
+
+refresh() { # the server refreshed SID, as the saved jar holds it
+ sed "s/${tab}SID${tab}.*/${tab}SID${tab}alpha-server/" "$jarfile" >"$tmpdir/jar.new"
+ cat "$tmpdir/jar.new" >"$jarfile"
+ command grep -qF "SID${tab}alpha-server" "$jarfile" ||
+ fail_dump "the jar holds no SID to refresh" "$jarfile"
+}
+# --update from another folder reads doit.log, so it finds the bundles again.
+# The jar the crawl saved is newer: a cookie the server refreshed wins.
+update() { # update WANT-SID
+ local from
+ from=$(hits)
+ crawl / -O "$(nativepath "$tmpdir/one")" --update || fail "--update failed"
+ hit "--update with SID=$1" "$from" "www.alpha.test/ $a=$1 x=-"
+}
+refresh
+dated 202601011000.00 202601021000.00
+update alpha-server
+# a new capture is newer than the jar, so the fresh login wins
+dated 202601031000.00 202601021000.00
+update alpha-captured
+# and so it does on a tie, where a coarse clock cannot order the two
+refresh
+dated 202601041000.00 202601041000.00
+update alpha-captured
+
+# A -%L list names a bundle too, and it resolves in the project folder.
+cp "$tmpdir/logins/beta.test.hts" "$tmpdir/two/"
+printf '%s\n' 'http://www.beta.test/#httrack-bundle=beta.test' >"$tmpdir/list.txt"
+before=$(hits)
+crawl "$tmpdir" -O two -%L "$(nativepath "$tmpdir/list.txt")" ||
+ fail "the -%L crawl failed"
+test "$(($(hits) - before))" -eq 2 ||
+ fail_dump "-%L crawl: $(($(hits) - before)) requests, expected 2" "$tmpdir/srv.out"
+hit "-%L" "$before" "www.beta.test/ ua=BetaUA/2.0 auth=Bearer beta-token cookie=SID=beta-captured x=-"
+no_suffix "$tmpdir/two"
+
+# A bundle that cannot be read stops the run before any request, with a message
+# that names it and never shows a refused header's value. A CR would end a
+# header early and start another on the wire.
+printf '%s\n' '# HTTrack bundle 1' 'header Proxy-Authorization: planted-secret' \
+ >"$tmpdir/two/secret.hts"
+printf '# HTTrack bundle 1\nua U\rX-Injected: 1\nheader X-A: 1\rCookie: forged=1\n' \
+ >"$tmpdir/two/cr.hts"
+printf '%s\n' '# HTTrack bundle 1' cookies "$(jar beta.test SID "v$(printf '\033')x")" \
+ >"$tmpdir/two/esc.hts"
+printf '%s\n' '# HTTrack bundle 1' cookies \
+ "$(jar beta.test SID "$(repeat_chars 8100 v)")" >"$tmpdir/two/long.hts"
+printf '%s\n' 'http://www.beta.test/#httrack-bundle=../beta.test' >"$tmpdir/bad.txt"
+badlist=$(nativepath "$tmpdir/bad.txt")
+stops=0
+stop() { # stop MESSAGE [httrack args...] -- MESSAGE is a whole line of output
+ local want=$1 rc=0
+ shift
+ : >"$log"
+ crawl "$tmpdir" -O two "$@" || rc=$?
+ test "$rc" -ne 0 || fail "the run did not stop: $*"
+ command grep -qxF "$want" "$log" || fail_dump "no message [$want]" "$log"
+ stops=$((stops + 1))
+}
+before=$(hits)
+stop 'Login bundle: two/nosuch.hts: cannot read the login bundle: No such file or directory' \
+ 'http://www.alpha.test/#httrack-bundle=nosuch'
+stop "\"$badlist\", line 1: invalid login bundle name: use letters, digits, '.', '_' and '-'" \
+ -%L "$badlist"
+stop 'Login bundle: two/secret.hts, line 2: refused header: a proxy header is not for the site' \
+ 'http://www.alpha.test/#httrack-bundle=secret'
+stop 'Login bundle: two/cr.hts, line 2: holds a control byte' \
+ 'http://www.alpha.test/#httrack-bundle=cr'
+for b0 in -b1 -b0; do
+ stop 'Login bundle: two/esc.hts, line 3: a cookie line holds a control byte' \
+ "$b0" 'http://www.beta.test/#httrack-bundle=esc'
+done
+stop 'Login bundle: two/long.hts, line 3: a cookie line is too long' \
+ 'http://www.beta.test/#httrack-bundle=long'
+test "$stops" -eq 7 || fail "$stops stops ran, expected 7"
+test "$(hits)" -eq "$before" || fail_dump "a refused bundle still crawled" "$tmpdir/srv.out"
+for f in "$log" "$tmpdir/two/hts-log.txt"; do
+ test -f "$f" || fail "no $f"
+ command grep -qF planted-secret "$f" && fail_dump "$f shows the secret" "$f"
+done
+
+# The identity is per request host: it drops on a redirect away from the
+# bundle's site and returns on one back. A userinfo that spells the site's name
+# gets none of it.
+before=$(hits)
+crawl "$tmpdir" -O three --bundle-dir logins '+*evil.test/*' \
+ '+*sub.alpha.test/*' 'http://www.alpha.test/out#httrack-bundle=alpha.test' \
+ 'http://www.alpha.test@evil.test/' || fail "the redirect crawl failed"
+a='ua=AlphaUA/1.0 auth=Bearer alpha-token cookie=PREF=alpha-pref; SID=alpha-captured x=-'
+g='ua=GlobalUA/9 auth=- cookie=- x=1'
+hit "redirect start" "$before" "www.alpha.test/out $a"
+hit "redirect out" "$before" "evil.test/back $g"
+hit "redirect back" "$before" "sub.alpha.test/home $a"
+# the userinfo is the user's own Basic login, sent as given
+hit "userinfo" "$before" \
+ "evil.test/ ua=GlobalUA/9 auth=Basic d3d3LmFscGhhLnRlc3Q= cookie=- x=1"
+test "$(($(hits) - before))" -eq 4 ||
+ fail_dump "redirect crawl: $(($(hits) - before)) requests, expected 4" "$tmpdir/srv.out"
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 385e5f07a..badaf345a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -10,7 +10,7 @@ EXTRA_DIST = $(TESTS) renamefail.c unlinkfail.c threadattrfail.c gaistall.c noba
proxy-https-server.py socks5-server.py proxy-connect-server.py \
proxytestlib.py tls-stall-server.py warc-validate.py wacz-validate.py \
header-injection-server.py header-injection-check.py \
- postfile-server.py \
+ postfile-server.py bundle-server.py \
status-line-server.py retry-after-server.py \
pty-resize.py ttyrun.py test-timeout.sh png-colors.py \
local-crawl.sh local-server.py ftp-server.py testlib.sh proclib.sh \
diff --git a/tests/bundle-server.py b/tests/bundle-server.py
new file mode 100644
index 000000000..df83c3c6c
--- /dev/null
+++ b/tests/bundle-server.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Several sites behind one proxy port, for 656_local-bundles.
+
+httrack reaches it as its -P proxy, so the absolute request URL names the
+site. Every request prints one "HIT ua=<...> auth=<...>
+cookie=<...> x=<...>" line, "-" for a missing header, with the cookies in
+name order. Prints "PORT ".
+"""
+
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import urlsplit
+
+CDN = "http://cdn.gamma.test"
+PAGES = {
+ "www.alpha.test/": 'p ' % CDN,
+ "www.alpha.test/page.html": "alpha page",
+ "alpha.test/plain.html": "plain URL on the alpha site",
+ "www.beta.test/": ' ' % CDN,
+ "cdn.gamma.test/a.gif": "GIF89a",
+ "cdn.gamma.test/b.gif": "GIF89a",
+ "github.io/": 'x',
+ "attacker.github.io/x.html": "another user's site",
+ "evil.test/": "another site",
+ "sub.alpha.test/home": "back on the alpha site",
+}
+# out of the bundle's site and back in
+REDIRECTS = {
+ "www.alpha.test/out": "http://evil.test/back",
+ "evil.test/back": "http://sub.alpha.test/home",
+}
+
+lock = threading.Lock()
+
+
+class Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.0"
+
+ def log_message(self, *args):
+ pass
+
+ def do_GET(self):
+ url = urlsplit(self.path)
+ key = url.hostname + url.path
+ with lock:
+ print(
+ "HIT %s ua=%s auth=%s cookie=%s x=%s"
+ % (
+ key,
+ self.headers.get("User-Agent", "-"),
+ self.headers.get("Authorization", "-"),
+ "; ".join(sorted(self.headers.get("Cookie", "-").split("; "))),
+ self.headers.get("X-Global", "-"),
+ ),
+ flush=True,
+ )
+ if key in REDIRECTS:
+ self.send_response(302)
+ self.send_header("Location", REDIRECTS[key])
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+ return
+ body = PAGES.get(key, "")
+ ctype = "image/gif" if key.endswith(".gif") else "text/html"
+ if ctype == "text/html":
+ body = "%s" % body
+ self.reply(200, body.encode(), ctype)
+
+ def reply(self, code, body, ctype):
+ self.send_response(code)
+ self.send_header("Content-Type", ctype)
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+
+class Server(ThreadingHTTPServer):
+ request_queue_size = 32 # BSD/macOS drop connections at Python's default 5
+
+
+# The shell greps these lines anchored, and a native python writes CRLF.
+sys.stdout.reconfigure(newline="\n")
+
+srv = Server(("127.0.0.1", 0), Handler)
+print("PORT %d" % srv.server_address[1], flush=True)
+srv.serve_forever()
diff --git a/tests/check-log-logins.py b/tests/check-log-logins.py
index 3e9a68f50..520e40ac8 100644
--- a/tests/check-log-logins.py
+++ b/tests/check-log-logins.py
@@ -50,6 +50,8 @@
'htshelp.c: \\nThe URL is: \\"%s\\"\\n': "--catchurl hands the URL back",
'htshelp.c: You can capture it through: httrack \\"%s\\"\\n': "--catchurl",
"htsweb.c: URL=http://%s:%d/\\n": "the server's own address",
+ "htsbauth.c: Bundle %s, line %d: %s": "filter->line is a line number",
+ "htsbundle.c: Bundle %s, line %d: unknown key%s%.*s": "a lowercase key alone",
}
|