Read a login bundle named by a start URL

configure.ac

@@ -41,6 +41,8 @@ AC_CONFIG_HEADERS(config.h)
 # header we can install without handing a consumer our PACKAGE_* and HAVE_*.
 AC_CONFIG_HEADERS([htsfeatures.h:src/htsfeatures.h.in])
 AM_INIT_AUTOMAKE([subdir-objects])
+# 3:27:0: revision-only bump. httrackp gained two tail fields (bundle_dir,
+# bundle_state) for the login bundles; no export came or went.
 # 3:26:0: revision-only bump. No export came or went and no installed struct
 # moved, per nm -D and gdb ptype /o against 3.50.4. The installed headers
 # gained HTS_FALLTHROUGH and two static inline path checks in htssafe.h, and
@@ -112,7 +114,7 @@ AM_INIT_AUTOMAKE([subdir-objects])
 # moves nothing). Soname stays .so.3: HTTrackQt is the only consumer of the installed
 # headers, so a libhttrack4 rename isn't worth it.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:26:0"
+VERSION_INFO="3:27:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "19 October 2026" "httrack website copier"
+.TH httrack 1 "21 October 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -242,6 +242,10 @@ links conversion to UTF\-8 (\-\-utf8\-conversion[=0])
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
 .IP \-%K
 use a Netscape cookies.txt as the jar, rewritten at the end (\-\-cookies\-file <param>)
+.br
+<URL>#httrack\-bundle=<name> crawls <URL> with the user agent,
+.br
+headers and cookies of the login bundle <name>.hts, read from the project folder or from \-\-bundle\-dir <dir>
 .IP \-%Y
 explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
@@ -582,6 +586,72 @@ start URL and its subdomains, or to that host alone when it is an address or a
 public suffix. They are Secure when that URL is https. HTTrack never rewrites
 such a file. HttpOnly cookies are not in it. A cookies.txt export has them,
 and so does the Cookie header from DevTools "Copy as cURL", pasted here.
+.TP
+.I <name>.hts
+A login bundle, named by a start URL that ends with
+.IR #httrack\-bundle=<name> ,
+such as
+.IR https://example.com/members/#httrack\-bundle=login1 .
+The name holds letters, digits,
+.IR . ,
+.I _
+and
+.I \-
+only. HTTrack reads the file in the project folder, or in the folder given to
+.IR "\-\-bundle\-dir <dir>" ,
+and never writes it. doit.log keeps that folder as an absolute path. HTTrack
+reads the suffix on the command line, in a
+.I \-%L
+list and in doit.log, never on a crawled link. A missing file, or one that
+does not parse, stops the run before any request.
+.IP
+The first line is
+.IR "# HTTrack bundle 1" .
+Then come
+.I url
+lines, each one more start URL, at most one
+.I ua
+line, and
+.I header
+and
+.I scope
+lines. A
+.I cookies
+line ends them, and a Netscape cookie jar follows it.
+.IP
+The bundle's user agent and headers replace
+.I \-F
+and
+.I \-%X
+for each host in its scope, and only there. A bundle with header lines but no
+.I ua
+line sends the
+.I \-F
+user agent. When any start URL naming the bundle, or any of its url lines, is
+https, its headers go over https alone, and so do those of the bundles it
+merges with. By default the scope is the registrable domains of the start
+URLs naming the bundle and of its url lines. A host with no registrable domain,
+such as a public suffix, a single label or an address, covers itself alone.
+Scope lines replace that default. HTTrack drops a cookie outside the scope and
+gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or
+framing header, a scope line naming a public suffix, and a line that holds a
+control byte or is too long, in the cookie jar too and even under
+.IR \-b0 .
+A second
+.I ua
+line, header lines over 8192 bytes, or a full cookie jar stops the run too. A
+repeated cookie line or an unknown key only warns, and so does an unknown #
+line in the cookie jar. Before the
+.I cookies
+line, a # line is a comment. Messages give the file and the line number, never
+a value. Two bundles whose
+scopes overlap must give the same user agent, headers and cookie values, or the
+run stops.
+.IP
+The bundle's cookies go into the cookie jar, which HTTrack writes at the end of
+the crawl. Of the bundle and the jar, the newer file wins a cookie both hold.
+A new capture therefore wins over the jar, and a cookie the server refreshed
+wins over an older bundle. On equal times the bundle wins.
 .PP
 A running mirror also watches its own output directory for these. Creating one
 is how a script drives an engine it has no other handle on. On Windows, which

man/httrack.tmpl

@@ -69,6 +69,72 @@ start URL and its subdomains, or to that host alone when it is an address or a
 public suffix. They are Secure when that URL is https. HTTrack never rewrites
 such a file. HttpOnly cookies are not in it. A cookies.txt export has them,
 and so does the Cookie header from DevTools "Copy as cURL", pasted here.
+.TP
+.I <name>.hts
+A login bundle, named by a start URL that ends with
+.IR #httrack\-bundle=<name> ,
+such as
+.IR https://example.com/members/#httrack\-bundle=login1 .
+The name holds letters, digits,
+.IR . ,
+.I _
+and
+.I \-
+only. HTTrack reads the file in the project folder, or in the folder given to
+.IR "\-\-bundle\-dir <dir>" ,
+and never writes it. doit.log keeps that folder as an absolute path. HTTrack
+reads the suffix on the command line, in a
+.I \-%L
+list and in doit.log, never on a crawled link. A missing file, or one that
+does not parse, stops the run before any request.
+.IP
+The first line is
+.IR "# HTTrack bundle 1" .
+Then come
+.I url
+lines, each one more start URL, at most one
+.I ua
+line, and
+.I header
+and
+.I scope
+lines. A
+.I cookies
+line ends them, and a Netscape cookie jar follows it.
+.IP
+The bundle's user agent and headers replace
+.I \-F
+and
+.I \-%X
+for each host in its scope, and only there. A bundle with header lines but no
+.I ua
+line sends the
+.I \-F
+user agent. When any start URL naming the bundle, or any of its url lines, is
+https, its headers go over https alone, and so do those of the bundles it
+merges with. By default the scope is the registrable domains of the start
+URLs naming the bundle and of its url lines. A host with no registrable domain,
+such as a public suffix, a single label or an address, covers itself alone.
+Scope lines replace that default. HTTrack drops a cookie outside the scope and
+gives the count in one warning. HTTrack refuses a Host, Cookie, Proxy\-* or
+framing header, a scope line naming a public suffix, and a line that holds a
+control byte or is too long, in the cookie jar too and even under
+.IR \-b0 .
+A second
+.I ua
+line, header lines over 8192 bytes, or a full cookie jar stops the run too. A
+repeated cookie line or an unknown key only warns, and so does an unknown #
+line in the cookie jar. Before the
+.I cookies
+line, a # line is a comment. Messages give the file and the line number, never
+a value. Two bundles whose
+scopes overlap must give the same user agent, headers and cookie values, or the
+run stops.
+.IP
+The bundle's cookies go into the cookie jar, which HTTrack writes at the end of
+the crawl. Of the bundle and the jar, the newer file wins a cookie both hold.
+A new capture therefore wins over the jar, and a cookie the server refreshed
+wins over an older bundle. On equal times the bundle wins.
 .PP
 A running mirror also watches its own output directory for these. Creating one
 is how a script drives an engine it has no other handle on. On Windows, which

src/Makefile.am

@@ -86,6 +86,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsback_selftest.c htscache_selftest.c htscharset_selftest.c \
 	htscmdline_selftest.c \
 	htscookie_selftest.c htsdns_selftest.c htsfetch_selftest.c htspsl_selftest.c \
+	htsbundle_selftest.c \
 	htsfilters_selftest.c \
 	htsheader_selftest.c htsio_selftest.c htslib_selftest.c \
 	htslines_selftest.c \
@@ -96,14 +97,14 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htscatchurl.c htsfetch.c htspsl.c htspsl_data.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
 	htscmdline.c htshelp.c htslib.c htslines.c htsurlport.c htsdate.c htscoremain.c \
 	htsname.c htsrobots.c htstools.c htsrandom.c htswizard.c \
-	htsalias.c htsthread.c htsindex.c htsbauth.c \
+	htsalias.c htsthread.c htsindex.c htsbauth.c htsbundle.c \
 	htscrashtest.c \
 	htsmd5.c htsnet.c htscodec.c htswarc.c htschanges.c htssinglefile.c htssitemap.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
 	htsmodules.c htscharset.c punycode.c htsencoding.c htsescape.c htssniff.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
 	hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h htsdate.h \
-	htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htscookie_selftest.h htsdns_selftest.h htsparse_selftest.h \
+	htsbasenet.h htsbauth.h htsbundle.h htscache.h htscache_selftest.h htscookie_selftest.h htsdns_selftest.h htsparse_selftest.h \
 	htsselftest.h htsselftest_int.h htsselftest_util.h htscatchurl.h  \
 	htscmdline.h htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h  \
 	htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \

src/htsalias.c

@@ -165,6 +165,8 @@ const char *hts_optalias[][4] = {
    "longest Retry-After delay obeyed, in seconds"},
   {"mptcp", "-@m", "onoff",
    "carry connections over Multipath TCP where the system offers it"},
+  {"bundle-dir", "--bundle-dir", "param1",
+   "folder where #httrack-bundle= names are read, not the project folder"},
   {"generate-errors", "-o", "level", ""},
   {"do-not-generate-errors", "-o0", "single", ""},
   {"purge-old", "-X", "onoff", ""},
@@ -347,6 +349,7 @@ static const char *const dashvalue_opt[] = {
   "-%L",  /* list */
   "-%S",  /* urllist */
   "-%rf", /* warc-file */
+  "--bundle-dir",
   "-F",   /* user-agent */
   NULL
 };

src/htsback.c

@@ -36,6 +36,7 @@ Please visit our Website: http://www.httrack.com
 
 /* specific definitions */
 #include "htsnet.h"
+#include "htsbundle.h"
 #include "htscore.h"
 #include "htsio.h"
 #include "htswarc.h"
@@ -3089,6 +3090,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
       back[p].r.req.accept = StringBuff(opt->accept);
       back[p].r.req.headers = StringBuff(opt->headers);
       back[p].r.req.user_agent_send = opt->user_agent_send;
+      /* a login bundle's site gets the bundle's UA and headers */
+      hts_bundle_identity(opt, adr, &back[p].r.req);
       // et http11
       back[p].r.req.http11 = back[p].http11;
       back[p].r.req.nocompression = opt->nocompression;