From ab9064351d861e126fbb25b9f7ecd7d5d3382e00 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 14:43:51 +0200 Subject: [PATCH 01/12] Hide login secrets in the -%H header log hts-ioinfo.txt printed every request and response header in clear, so a user sharing it for a bug report shared their Authorization, Proxy-Authorization and Cookie values, and the Set-Cookie values the server gave them. The FTP control log printed PASS in clear too. fprintfio(), the only writer of header blocks to that log, now replaces the value of those four fields with "". The name is matched as header_name_len() does: case-insensitive, with blanks before the colon. A folded continuation of a hidden field is hidden too. The FTP PASS argument uses the same marker. The bytes sent on the wire do not change. Test 657 crawls with a cookies.txt cookie, user:pass in the URL and two -%X login headers, plus an FTP login, and checks the log against the wire. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- src/htsftp.c | 7 +- src/htslib.c | 57 +++++++++++----- src/htslib.h | 5 ++ tests/657_local-header-log-redact.test | 90 ++++++++++++++++++++++++++ tests/header-injection-server.py | 19 +++++- 5 files changed, 161 insertions(+), 17 deletions(-) create mode 100644 tests/657_local-header-log-redact.test diff --git a/src/htsftp.c b/src/htsftp.c index 338fffa97..936cf7320 100644 --- a/src/htsftp.c +++ b/src/htsftp.c @@ -1414,7 +1414,12 @@ int send_line(T_SOC soc, const char *data) { if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo, "---> %s\x0d\x0a", data); + if (strncmp(data, "PASS ", 5) == 0) { + fprintf(ioinfo, "---> PASS "); + fprint_redacted(ioinfo, strlen(data + 5)); + fprintf(ioinfo, "\x0d\x0a"); + } else + fprintf(ioinfo, "---> %s\x0d\x0a", data); fflush(ioinfo); } } diff --git a/src/htslib.c b/src/htslib.c index 235ec5720..e18417f7b 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -5806,25 +5806,52 @@ int may_unknown2(httrackp * opt, const char *mime, const char *filename) { // -- Utils fichiers -// pretty print for i/o -void fprintfio(FILE * fp, const char *buff, const char *prefix) { - char nl = 1; +void fprint_redacted(FILE *fp, size_t len) { + fprintf(fp, "", (unsigned long) len); +} - while(*buff) { - switch (*buff) { - case 13: - break; - case 10: - fprintf(fp, "\r\n"); - nl = 1; - break; - default: +/* Fields whose value carries a login, hidden from the -%H log. */ +static const char *const secret_fields[] = { + "Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"}; + +void fprintfio(FILE *fp, const char *buff, const char *prefix) { + hts_boolean nl = HTS_TRUE; + hts_boolean secret = HTS_FALSE; + + while (*buff != '\0') { + const size_t len = strcspn(buff, "\r\n"); + + if (len != 0) { + size_t keep = len; + + /* a folded line continues the value above it */ + if (*buff != ' ' && *buff != '\t') + secret = + header_name_in(buff, secret_fields, + sizeof(secret_fields) / sizeof(secret_fields[0])); + if (secret) { + const char *const colon = memchr(buff, ':', len); + + keep = *buff != ' ' && *buff != '\t' && colon != NULL + ? (size_t) (colon - buff) + 1 + : 0; + while (keep < len && (buff[keep] == ' ' || buff[keep] == '\t')) + keep++; + } if (nl) fprintf(fp, "%s", prefix); - nl = 0; - fputc(*buff, fp); + nl = HTS_FALSE; + fwrite(buff, 1, keep, fp); + if (keep < len) + fprint_redacted(fp, len - keep); + } + buff += len; + if (*buff == '\n') { + fprintf(fp, "\r\n"); + nl = HTS_TRUE; } - buff++; + if (*buff != '\0') + buff++; } } diff --git a/src/htslib.h b/src/htslib.h index 20755dca5..c68797a34 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -541,8 +541,13 @@ char *convtolower(char *catbuff, size_t catbuffsize, const char *a); void hts_lowcase(char *s); int multipleStringMatch(const char *s, const char *match); +/* Writes a header block to fp, each line after prefix, with the value of a + field that carries a login replaced by the fprint_redacted() marker. */ void fprintfio(FILE * fp, const char *buff, const char *prefix); +/* Writes the marker that stands for a hidden secret of len bytes. */ +void fprint_redacted(FILE *fp, size_t len); + /* A write to a socket whose peer is gone raises SIGPIPE, and the default action kills the process. The library must not install a handler, because the process belongs to whoever embedded us. Three steps leave the host's diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test new file mode 100644 index 000000000..4631caea2 --- /dev/null +++ b/tests/657_local-header-log-redact.test @@ -0,0 +1,90 @@ +#!/bin/bash +# +# The -%H log (hts-ioinfo.txt) hides the value of every header and FTP command +# that carries a login, while the wire still carries it. + +set -euo pipefail + +# shellcheck source=tests/testlib.sh +. "${0%"${0##*/}"}./testlib.sh" + +require_httrack + +TAB=$'\t' + +python=$(find_python) || skip "python3 missing" + +server=$(nativepath "$top_srcdir/tests/header-injection-server.py") +tmpdir=$(mktemp -d) +serverpid= +wire=$tmpdir/wire + +cleanup() { + stop_server "$serverpid" + rm -rf "$tmpdir" +} +cleanup_push cleanup + +"$python" "$server" "$(nativepath "$wire")" >"$tmpdir/server.out" 2>"$tmpdir/server.err" & +serverpid=$! +port=$(discover_server_port "$tmpdir/server.out" "$serverpid") || exit 1 + +crawl() { # crawl TAG URL [extra httrack args...] + local tag=$1 url=$2 rc=0 + shift 2 + mkdir -p "$tmpdir/$tag" + if test -e "$tmpdir/cookies.txt"; then + cp "$tmpdir/cookies.txt" "$tmpdir/$tag/" + fi + run_with_timeout 90 httrack "$url" -O "$tmpdir/$tag" -c1 --robots=0 \ + --retries=0 --quiet -%v0 -%H "$@" >>"$tmpdir/log" 2>&1 || rc=$? + test "$rc" -eq 0 || fail "$tag: the crawl exited $rc" + cat "$tmpdir/$tag/hts-ioinfo.txt" >>"$tmpdir/ioinfo" +} + +printf '127.0.0.1\tFALSE\t/\tFALSE\t2999999999\tjarsess\tJARSECRET\n' \ + >"$tmpdir/cookies.txt" +crawl jar "http://urluser:URLSECRET@127.0.0.1:$port/setcookie.html" \ + -%X "Proxy-Authorization : Bearer PROXYSECRET" +rm "$tmpdir/cookies.txt" +crawl box "http://127.0.0.1:$port/plain.html" -%X "authorization: Bearer AUTHSECRET" + +basic=$(printf urluser:URLSECRET | base64) +log=$(<"$tmpdir/ioinfo") +sent=$(<"$wire") +grep -q "^>>> X-Visible: shown" <<<"$log" || fail "the response headers were not logged: $log" +# the server sent these, so only the request secrets are on the wire +for secret in JARSECRET PROXYSECRET AUTHSECRET "$basic" "srvsess=SETCOOKIESECRET"; do + grep -qF "$secret" <<<"$sent" || fail "$secret never reached the server" +done +for secret in JARSECRET PROXYSECRET AUTHSECRET "$basic" SETCOOKIESECRET \ + SPACEDSECRET FOLDEDSECRET; do + if grep -qF "$secret" <<<"$log"; then + fail "$secret is in hts-ioinfo.txt: $(grep -F "$secret" <<<"$log")" + fi +done +for line in "<<< Authorization: " \ + "<<< Proxy-Authorization: " \ + "<<< Cookie: " \ + "<<< authorization: " \ + ">>> Set-Cookie: " \ + ">>> set-cookie$TAB: " \ + ">>> $TAB"; do + grep -qF "$line" <<<"$log" || fail "no '$line' in hts-ioinfo.txt: $log" +done +ok "HTTP login headers are redacted in the log and sent in full" + +root="$tmpdir/root" +cmds="$tmpdir/cmds" +mkdir -p "$root" +printf 'body\n' >"$root/f.txt" +ftp_server_start --root "$root" --require-pass --log "$(nativepath "$cmds")" +: >"$tmpdir/ioinfo" +crawl ftp "ftp://ftpuser:FTPSECRET@127.0.0.1:${FTP_PORT}/f.txt" +grep -qxF "PASS FTPSECRET" "$cmds" || fail "the FTP password never reached the server" +log=$(<"$tmpdir/ioinfo") +if grep -qF FTPSECRET <<<"$log"; then + fail "the FTP password is in hts-ioinfo.txt" +fi +grep -qF -- "---> PASS " <<<"$log" || fail "no redacted PASS: $log" +ok "the FTP password is redacted in the log and sent in full" diff --git a/tests/header-injection-server.py b/tests/header-injection-server.py index a36c951df..c15c46c64 100644 --- a/tests/header-injection-server.py +++ b/tests/header-injection-server.py @@ -61,9 +61,24 @@ def body_for(request, port): # the proxied pages link onward too, so a poisoned *host* becomes a referer if b"/p.html" in line: return page('deep') + # sets a cookie and links on, so the next request sends it back (657) + if b"/setcookie.html" in line: + return page('plain') return page("leaf") +def extra_headers(request): + line = request.split(b"\r\n", 1)[0] + if b"/setcookie.html" in line: + # a blank before the colon and a folded line still belong to the field + return ( + b"Set-Cookie: srvsess=SETCOOKIESECRET; Path=/\r\n" + b"set-cookie\t: spaced=SPACEDSECRET;\r\n\tfolded=FOLDEDSECRET\r\n" + b"X-Visible: shown\r\n" + ) + return b"" + + def handle(conn, port, logf, lock, seq): conn.settimeout(10) data = b"" @@ -86,7 +101,9 @@ def handle(conn, port, logf, lock, seq): try: conn.sendall( b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n" - b"Content-Length: %d\r\nConnection: close\r\n\r\n" % len(body) + body + + extra_headers(data) + + b"Content-Length: %d\r\nConnection: close\r\n\r\n" % len(body) + + body ) except OSError: pass From 8a1214a297956d5c93cd29e4ca24a6bb497a4769 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 14:51:48 +0200 Subject: [PATCH 02/12] Log a refused cookie without its value Two cookie-file warnings in hts-log.txt printed up to 64 bytes of the refused line, value included. A pair with no name is now logged without its text. An unknown comment line is logged up to its first TAB, which holds the domain rather than a value. The other warnings already print the name only. Test 657 plants a value in each refused line. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- src/htsbauth.c | 9 +++++---- tests/657_local-header-log-redact.test | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/htsbauth.c b/src/htsbauth.c index caac90c55..70f8bfbce 100644 --- a/src/htsbauth.c +++ b/src/htsbauth.c @@ -688,8 +688,7 @@ static hts_boolean cookie_flat_pair(httrackp *opt, t_cookie *cookie, char *pair, if (eq == NULL || eq == pair) { if (cookie_flat_ignore(ignored)) hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a cookie with no name in %s: %.64s", - given, pair); + "Cookies: ignoring a cookie with no name in %s", given); return HTS_FALSE; } *eq = '\0'; @@ -987,9 +986,11 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, if (rec[0] == '#') { if (!cookie_known_comment(rec)) { + // the first field only, because a later one may be a value hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring an unknown comment in %s: %.64s", - given, rec); + "Cookies: ignoring an unknown comment in %s: %.*s", + given, (int) min(strcspn(rec, "\t"), (size_t) 64), + rec); refused++; } } else { diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index 4631caea2..5be8faa16 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -74,6 +74,24 @@ for line in "<<< Authorization: " \ done ok "HTTP login headers are redacted in the log and sent in full" +# A cookie the loader refuses is logged by its name, never by its value. +printf 'dup=1; dup=DUPSECRET; =NONAMESECRET; BARESECRET\n' >"$tmpdir/flat.txt" +crawl flat "http://127.0.0.1:$port/plain.html" --cookies-file "$tmpdir/flat.txt" +printf '#HTTPOnly_127.0.0.1\tFALSE\t/\tFALSE\t0\tcomname\tCOMMENTSECRET\n' \ + >"$tmpdir/jar.txt" +crawl netscape "http://127.0.0.1:$port/plain.html" --cookies-file "$tmpdir/jar.txt" +log=$(cat "$tmpdir/flat/hts-log.txt" "$tmpdir/netscape/hts-log.txt") +for secret in DUPSECRET NONAMESECRET BARESECRET COMMENTSECRET; do + if grep -qF "$secret" <<<"$log"; then + fail "$secret is in hts-log.txt: $(grep -F "$secret" <<<"$log")" + fi +done +for line in "ignoring a repeated 'dup'" "a cookie with no name" \ + "unknown comment in $tmpdir/jar.txt: #HTTPOnly_127.0.0.1"; do + grep -qF "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" +done +ok "a refused cookie is logged without its value" + root="$tmpdir/root" cmds="$tmpdir/cmds" mkdir -p "$root" From 4ac2e949370573b22cf3399ee3d6610469369d74 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 15:06:30 +0200 Subject: [PATCH 03/12] Mask URL passwords in hts-log.txt and log no cookie comment text hts-log.txt printed URL passwords in several places: the "launched at" line, the command-line echo, and warnings that name a link. The echo also printed -%X login headers in clear. hts_log_vprint() now formats each line and passes it through hts_mask_passwords(), which writes user:***@ in place of user:pass@. The echo goes through the same helper, and then through fprintfio() for the header redaction. doit.log, new.txt and new.zip keep the URL, because --update needs it. A cookie-file comment line with no TAB was still logged whole. The warning now gives the line number and no line text. Test 657 now checks that look-alike headers and the FTP USER stay in clear. It also tests a bare cookie value on its own, and passwords from HTTP, FTP and -P in hts-log.txt. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- src/htsbauth.c | 11 +++-- src/htscoremain.c | 21 +++++++--- src/htslib.c | 52 +++++++++++++++++++++++- src/htslib.h | 4 ++ tests/657_local-header-log-redact.test | 56 +++++++++++++++++++++----- tests/header-injection-server.py | 2 + 6 files changed, 125 insertions(+), 21 deletions(-) diff --git a/src/htsbauth.c b/src/htsbauth.c index 70f8bfbce..cfa664f1e 100644 --- a/src/htsbauth.c +++ b/src/htsbauth.c @@ -940,6 +940,7 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, char BIGSTK line[8192]; const size_t line_max = 8000; hts_boolean first_line = HTS_TRUE; + int lineno = 0; // this load's records alone, so a line repeating one is caught t_cookie *seen = (t_cookie *) calloct(1, sizeof(t_cookie)); @@ -957,6 +958,8 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, // len counts what the buffer cut too, so this catches that line cookie_readline(fp, line, sizeof(line), &len); + if (lineno < INT_MAX) + lineno++; if (first_line && len >= sizeof(cookie_bom) - 1 && strncmp(line, cookie_bom, sizeof(cookie_bom) - 1) == 0) { len -= sizeof(cookie_bom) - 1; @@ -986,11 +989,11 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, if (rec[0] == '#') { if (!cookie_known_comment(rec)) { - // the first field only, because a later one may be a value + // no line text, because any part of it can hold a cookie value hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring an unknown comment in %s: %.*s", - given, (int) min(strcspn(rec, "\t"), (size_t) 64), - rec); + "Cookies: ignoring an unknown comment on line %d " + "of %s", + lineno, given); refused++; } } else { diff --git a/src/htscoremain.c b/src/htscoremain.c index 5f37eb582..e64a0883e 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -3109,18 +3109,29 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { } // fichier log if (opt->log) { + char *masked_url = hts_mask_passwords(url); int i; fprintf(opt->log, "HTTrack" HTTRACK_VERSION "%s launched on %s at %s" LF, - hts_get_version_info(opt), t, url); + hts_get_version_info(opt), t, + masked_url != NULL ? masked_url : "(out of memory)"); + freet(masked_url); fprintf(opt->log, "("); for(i = 0; i < argc; i++) { const char *arg = argv[i]; // already UTF-8 on every platform - if (strchr(arg, ' ') == NULL || strchr(arg, '\"') != NULL) - fprintf(opt->log, "%s ", arg); - else // entre "" (si espace(s) et pas déja de ") - fprintf(opt->log, "\"%s\" ", arg); + char *masked = hts_mask_passwords(arg); + // quoted if it holds a blank and no quote + const hts_boolean quote = + strchr(arg, ' ') != NULL && strchr(arg, '\"') == NULL; + + if (quote) + fputc('"', opt->log); + // a -%X login header gets the -%H log's redaction + if (masked != NULL) + fprintfio(opt->log, masked, ""); + fputs(quote ? "\" " : " ", opt->log); + freet(masked); } fprintf(opt->log, ")" LF); fprintf(opt->log, LF); diff --git a/src/htslib.c b/src/htslib.c index e18417f7b..41671f959 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -5810,10 +5810,41 @@ void fprint_redacted(FILE *fp, size_t len) { fprintf(fp, "", (unsigned long) len); } -/* Fields whose value carries a login, hidden from the -%H log. */ +/* The -%H log hides the value of these fields because they carry logins. + Unlike refused[] and single[], this list governs the log, not the wire. */ static const char *const secret_fields[] = { "Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"}; +char *hts_mask_passwords(const char *s) { + const size_t len = strlen(s); + /* each mask eats at least ':' and '@' and writes 3 bytes */ + char *const dst = (char *) malloct(len * 2 + 1); + size_t i, o = 0, start = 0; + + if (dst == NULL) + return NULL; + for (i = 0; i < len; i++) { + if (s[i] == '@') { + const char *const colon = memchr(s + start, ':', i - start); + + // mailto:user@host names an address, not a login + if (colon != NULL && !strfield(s + start, "mailto:")) { + const size_t user = (size_t) (colon - s) + 1; + + o -= i - user; + memcpy(dst + o, "***", 3); + o += 3; + } + start = i + 1; + } else if (strchr(" \t\r\n\"'<>()/,;=", s[i]) != NULL) { + start = i + 1; + } + dst[o++] = s[i]; + } + dst[o] = '\0'; + return dst; +} + void fprintfio(FILE *fp, const char *buff, const char *prefix) { hts_boolean nl = HTS_TRUE; hts_boolean secret = HTS_FALSE; @@ -7290,8 +7321,25 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_ fspc_count(opt, s_type); if (opt->log != NULL) { + va_list args_len; + int n; + + va_copy(args_len, args); + n = vsnprintf(NULL, 0, format, args_len); + va_end(args_len); fspc(opt, opt->log, s_type); - (void) vfprintf(opt->log, format, args); + if (n >= 0) { + char *line = (char *) malloct((size_t) n + 1); + char *masked = + line != NULL && vsnprintf(line, (size_t) n + 1, format, args) == n + ? hts_mask_passwords(line) + : NULL; + + // never the raw line, which may hold a password + fputs(masked != NULL ? masked : "(log line lost)", opt->log); + freet(masked); + freet(line); + } if ((type & LOG_ERRNO) != 0) { char errbuf[HTS_STRERROR_SIZE]; diff --git a/src/htslib.h b/src/htslib.h index c68797a34..a6125b6e9 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -545,6 +545,10 @@ int multipleStringMatch(const char *s, const char *match); field that carries a login replaced by the fprint_redacted() marker. */ void fprintfio(FILE * fp, const char *buff, const char *prefix); +/* Returns a copy of s, allocated with malloct, in which the password of every + user:pass@ is *** . The user and the host stay. NULL when out of memory. */ +char *hts_mask_passwords(const char *s); + /* Writes the marker that stands for a hidden secret of len bytes. */ void fprint_redacted(FILE *fp, size_t len); diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index 5be8faa16..717eec9b5 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -11,6 +11,7 @@ set -euo pipefail require_httrack TAB=$'\t' +CR=$'\r' python=$(find_python) || skip "python3 missing" @@ -44,15 +45,21 @@ crawl() { # crawl TAG URL [extra httrack args...] printf '127.0.0.1\tFALSE\t/\tFALSE\t2999999999\tjarsess\tJARSECRET\n' \ >"$tmpdir/cookies.txt" +# -%X drops the blank before the colon (#1996), so only a response keeps one crawl jar "http://urluser:URLSECRET@127.0.0.1:$port/setcookie.html" \ -%X "Proxy-Authorization : Bearer PROXYSECRET" rm "$tmpdir/cookies.txt" -crawl box "http://127.0.0.1:$port/plain.html" -%X "authorization: Bearer AUTHSECRET" +crawl box "http://127.0.0.1:$port/plain.html" -%X "authorization: Bearer AUTHSECRET" \ + -%X "Cookie2: VISIBLE3" basic=$(printf urluser:URLSECRET | base64) log=$(<"$tmpdir/ioinfo") sent=$(<"$wire") -grep -q "^>>> X-Visible: shown" <<<"$log" || fail "the response headers were not logged: $log" +# a field whose name only starts like a hidden one stays in clear +for line in ">>> X-Visible: shown" ">>> X-Cookie-Id: VISIBLE2" \ + "<<< Cookie2: VISIBLE3" ">>> Authorization-Info: VISIBLE4"; do + grep -qxF "$line$CR" <<<"$log" || fail "no '$line' in hts-ioinfo.txt: $log" +done # the server sent these, so only the request secrets are on the wire for secret in JARSECRET PROXYSECRET AUTHSECRET "$basic" "srvsess=SETCOOKIESECRET"; do grep -qF "$secret" <<<"$sent" || fail "$secret never reached the server" @@ -75,22 +82,45 @@ done ok "HTTP login headers are redacted in the log and sent in full" # A cookie the loader refuses is logged by its name, never by its value. -printf 'dup=1; dup=DUPSECRET; =NONAMESECRET; BARESECRET\n' >"$tmpdir/flat.txt" +printf 'dup=1; dup=DUPSECRET; =NONAMESECRET\n' >"$tmpdir/flat.txt" crawl flat "http://127.0.0.1:$port/plain.html" --cookies-file "$tmpdir/flat.txt" -printf '#HTTPOnly_127.0.0.1\tFALSE\t/\tFALSE\t0\tcomname\tCOMMENTSECRET\n' \ - >"$tmpdir/jar.txt" +printf 'ok=1; BARESECRET\n' >"$tmpdir/bare.txt" +crawl bare "http://127.0.0.1:$port/plain.html" --cookies-file "$tmpdir/bare.txt" +printf '%s\n' "#HTTPOnly_127.0.0.1${TAB}FALSE$TAB/${TAB}FALSE${TAB}0${TAB}comname${TAB}TABSECRET" \ + "# SID=EQSECRET" "#TOKSECRET" >"$tmpdir/jar.txt" crawl netscape "http://127.0.0.1:$port/plain.html" --cookies-file "$tmpdir/jar.txt" -log=$(cat "$tmpdir/flat/hts-log.txt" "$tmpdir/netscape/hts-log.txt") -for secret in DUPSECRET NONAMESECRET BARESECRET COMMENTSECRET; do +for tag in flat bare netscape; do + log=$(<"$tmpdir/$tag/hts-log.txt") + for secret in DUPSECRET NONAMESECRET BARESECRET TABSECRET EQSECRET TOKSECRET; do + if grep -qF "$secret" <<<"$log"; then + fail "$secret is in hts-log.txt: $(grep -F "$secret" <<<"$log")" + fi + done +done +grep -qF "ignoring a repeated 'dup'" "$tmpdir/flat/hts-log.txt" || fail "no repeated 'dup'" +grep -qF "a cookie with no name" "$tmpdir/flat/hts-log.txt" || fail "no nameless =value" +grep -qF "a cookie with no name" "$tmpdir/bare/hts-log.txt" || fail "no bare value" +for n in 1 2 3; do + grep -qF "unknown comment on line $n of $tmpdir/jar.txt" "$tmpdir/netscape/hts-log.txt" || + fail "no unknown comment on line $n: $(<"$tmpdir/netscape/hts-log.txt")" +done +ok "a refused cookie is logged without its value" + +# A password in a URL, a proxy or a -%X header never reaches hts-log.txt. +crawl proxy "http://127.0.0.1:$port/plain.html" -P "puser:PPWSECRET@127.0.0.1:$port" +grep -qF "Proxy-Authorization: Basic $(printf puser:PPWSECRET | base64)" "$wire" || + fail "the proxy password never reached the proxy" +log=$(cat "$tmpdir/jar/hts-log.txt" "$tmpdir/box/hts-log.txt" "$tmpdir/proxy/hts-log.txt") +for secret in URLSECRET PPWSECRET AUTHSECRET PROXYSECRET; do if grep -qF "$secret" <<<"$log"; then fail "$secret is in hts-log.txt: $(grep -F "$secret" <<<"$log")" fi done -for line in "ignoring a repeated 'dup'" "a cookie with no name" \ - "unknown comment in $tmpdir/jar.txt: #HTTPOnly_127.0.0.1"; do +for line in "urluser:***@127.0.0.1:$port/setcookie.html" "puser:***@127.0.0.1:$port" \ + "authorization: "; do grep -qF "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" done -ok "a refused cookie is logged without its value" +ok "hts-log.txt masks the passwords of URLs, proxies and -%X headers" root="$tmpdir/root" cmds="$tmpdir/cmds" @@ -105,4 +135,10 @@ if grep -qF FTPSECRET <<<"$log"; then fail "the FTP password is in hts-ioinfo.txt" fi grep -qF -- "---> PASS " <<<"$log" || fail "no redacted PASS: $log" +# the user name alone is no secret, so it stays +grep -qF -- "---> USER ftpuser" <<<"$log" || fail "no USER in clear: $log" +if grep -qF FTPSECRET "$tmpdir/ftp/hts-log.txt"; then + fail "the FTP password is in hts-log.txt" +fi +grep -qF "ftpuser:***@" "$tmpdir/ftp/hts-log.txt" || fail "no masked FTP URL in hts-log.txt" ok "the FTP password is redacted in the log and sent in full" diff --git a/tests/header-injection-server.py b/tests/header-injection-server.py index c15c46c64..1c3135d16 100644 --- a/tests/header-injection-server.py +++ b/tests/header-injection-server.py @@ -75,6 +75,8 @@ def extra_headers(request): b"Set-Cookie: srvsess=SETCOOKIESECRET; Path=/\r\n" b"set-cookie\t: spaced=SPACEDSECRET;\r\n\tfolded=FOLDEDSECRET\r\n" b"X-Visible: shown\r\n" + b"X-Cookie-Id: VISIBLE2\r\n" + b"Authorization-Info: VISIBLE4\r\n" ) return b"" From 62061676dad7b444a18858e630922e1638f3502f Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 15:39:52 +0200 Subject: [PATCH 04/12] Mask only the URL passwords the engine reads, in an exact-size copy The old matcher sized its copy at twice the input, but each ":@" grew to ":***@", so 300 pairs in a hostile link overflowed the heap through the 404 log line, and a local argument overflowed the echo. It also stopped at the first '@' where the engine splits at the last one, and masked any ':' before an '@', such as the filter -*:*@*. The new matcher counts first, then copies. It masks the userinfo where jump_identification_const() finds it: after a scheme://, at the start of a token that goes on to a path (the engine logs http addresses with no scheme), on each start URL, and on each -P value, found from the option spans the parser recorded. The maskpw self-test holds the rows, and test 657 crawls a password through hts_log_print. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- src/htscoremain.c | 19 +++- src/htslib.c | 107 ++++++++++++++++++----- src/htslib.h | 14 ++- src/htslib_selftest.c | 116 +++++++++++++++++++++++++ tests/657_local-header-log-redact.test | 18 ++++ 5 files changed, 244 insertions(+), 30 deletions(-) diff --git a/src/htscoremain.c b/src/htscoremain.c index e64a0883e..9f873c94a 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -3109,8 +3109,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { } // fichier log if (opt->log) { - char *masked_url = hts_mask_passwords(url); - int i; + char *masked_url = hts_mask_passwords(url, HTS_MASK_LIST); + int i, opt_at = 0; fprintf(opt->log, "HTTrack" HTTRACK_VERSION "%s launched on %s at %s" LF, @@ -3120,10 +3120,23 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { fprintf(opt->log, "("); for(i = 0; i < argc; i++) { const char *arg = argv[i]; // already UTF-8 on every platform - char *masked = hts_mask_passwords(arg); // quoted if it holds a blank and no quote const hts_boolean quote = strchr(arg, ' ') != NULL && strchr(arg, '\"') == NULL; + hts_boolean addr; // a start URL, or a -P value, is one address + char *masked; + + if (x_cmd.span[i] != 0) { + opt_at = x_cmd.span[i] > 1 ? i : 0; + addr = HTS_FALSE; + } else if (opt_at != 0 && i < opt_at + x_cmd.span[opt_at]) { + addr = cmdl_shortopt_has(argv[opt_at], 'P'); + } else { + opt_at = 0; + addr = i != 0 && arg[0] != '+' && arg[0] != '-'; + } + masked = + hts_mask_passwords(arg, addr ? HTS_MASK_ADDR : HTS_MASK_LINE); if (quote) fputc('"', opt->log); diff --git a/src/htslib.c b/src/htslib.c index 41671f959..3a0db563e 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -5815,33 +5815,92 @@ void fprint_redacted(FILE *fp, size_t len) { static const char *const secret_fields[] = { "Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"}; -char *hts_mask_passwords(const char *s) { - const size_t len = strlen(s); - /* each mask eats at least ':' and '@' and writes 3 bytes */ - char *const dst = (char *) malloct(len * 2 + 1); - size_t i, o = 0, start = 0; +/* Is c allowed in a URL scheme (RFC 3986 3.1)? */ +static hts_boolean scheme_char(char c) { + return isalnum((unsigned char) c) || c == '+' || c == '-' || c == '.'; +} + +/* Finds the password of the userinfo at the authority a: from the first ':' + up to the last '@' before the first '/', the split + jump_identification_const() makes. A token also ends at a blank, and with + path, it must go on to a '/'. Returns HTS_FALSE when there is none. */ +static hts_boolean userinfo_password(const char *a, hts_boolean token, + hts_boolean path, const char **pass, + const char **at) { + const char *end = a + strcspn(a, token ? "/ \t\r\n" : "/"); + const char *colon; - if (dst == NULL) - return NULL; - for (i = 0; i < len; i++) { - if (s[i] == '@') { - const char *const colon = memchr(s + start, ':', i - start); + if (path && *end != '/') + return HTS_FALSE; + while (end > a && end[-1] != '@') + end--; + if (end == a) + return HTS_FALSE; + colon = memchr(a, ':', (size_t) (end - 1 - a)); + if (colon == NULL) + return HTS_FALSE; + *pass = colon + 1; + *at = end - 1; + return HTS_TRUE; +} - // mailto:user@host names an address, not a login - if (colon != NULL && !strfield(s + start, "mailto:")) { - const size_t user = (size_t) (colon - s) + 1; +/* The masked copy: dst (NULL to only count) holds o bytes, and the source is + copied up to p. */ +typedef struct { + char *dst; + size_t o; + const char *p; +} mask_out; - o -= i - user; - memcpy(dst + o, "***", 3); - o += 3; - } - start = i + 1; - } else if (strchr(" \t\r\n\"'<>()/,;=", s[i]) != NULL) { - start = i + 1; - } - dst[o++] = s[i]; +static void mask_copy(mask_out *m, const char *src, size_t n) { + if (m->dst != NULL) + memcpy(m->dst + m->o, src, n); + m->o += n; +} + +/* Masks the password at the authority a, if any, and moves *i past it. */ +static hts_boolean mask_at(mask_out *m, const char *a, hts_boolean token, + hts_boolean path, const char **i) { + const char *pass, *at; + + if (!userinfo_password(a, token, path, &pass, &at)) + return HTS_FALSE; + mask_copy(m, m->p, (size_t) (pass - m->p)); + mask_copy(m, "***", 3); + m->p = *i = at; + return HTS_TRUE; +} + +/* Copies s to dst, or only counts when dst is NULL, with each password the + engine would read out of an address written as ***. Returns the length. */ +static size_t mask_passwords(const char *s, char *dst, hts_mask_mode mode) { + mask_out m = {dst, 0, s}; + const char *i = s; + + if (mode == HTS_MASK_ADDR) + (void) mask_at(&m, jump_protocol_const(s), HTS_FALSE, HTS_FALSE, &i); + while (*i != '\0') { + // an address the engine printed starts a token, often with no scheme + if (mode != HTS_MASK_ADDR && (i == s || strchr(" \t\r\n", i[-1])) && + strchr("+-*", *i) == NULL && + mask_at(&m, jump_protocol_const(i), HTS_TRUE, mode == HTS_MASK_LINE, + &i)) + continue; + if (i != s && strncmp(i, "://", 3) == 0 && scheme_char(i[-1]) && + mask_at(&m, i + 3, HTS_FALSE, HTS_FALSE, &i)) + continue; + i++; } - dst[o] = '\0'; + mask_copy(&m, m.p, strlen(m.p) + 1); + return m.o - 1; +} + +char *hts_mask_passwords(const char *s, hts_mask_mode mode) { + const size_t len = mask_passwords(s, NULL, mode); + char *const dst = (char *) malloct(len + 1); + + if (dst != NULL) + mask_passwords(s, dst, mode); return dst; } @@ -7332,7 +7391,7 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_ char *line = (char *) malloct((size_t) n + 1); char *masked = line != NULL && vsnprintf(line, (size_t) n + 1, format, args) == n - ? hts_mask_passwords(line) + ? hts_mask_passwords(line, HTS_MASK_LINE) : NULL; // never the raw line, which may hold a password diff --git a/src/htslib.h b/src/htslib.h index a6125b6e9..a3b547678 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -545,9 +545,17 @@ int multipleStringMatch(const char *s, const char *match); field that carries a login replaced by the fprint_redacted() marker. */ void fprintfio(FILE * fp, const char *buff, const char *prefix); -/* Returns a copy of s, allocated with malloct, in which the password of every - user:pass@ is *** . The user and the host stay. NULL when out of memory. */ -char *hts_mask_passwords(const char *s); +/* What hts_mask_passwords() reads: */ +typedef enum { + HTS_MASK_LINE, /* a log line, where an address may lack its scheme */ + HTS_MASK_LIST, /* blank-separated start URLs and filters */ + HTS_MASK_ADDR /* one start URL, or a -P value */ +} hts_mask_mode; + +/* Returns a copy of s, allocated with malloct, in which every password the + engine would read out of an address in s is *** . The rest stays byte for + byte. NULL when out of memory. */ +char *hts_mask_passwords(const char *s, hts_mask_mode mode); /* Writes the marker that stands for a hidden secret of len bytes. */ void fprint_redacted(FILE *fp, size_t len); diff --git a/src/htslib_selftest.c b/src/htslib_selftest.c index 3b2d378bf..55098b965 100644 --- a/src/htslib_selftest.c +++ b/src/htslib_selftest.c @@ -1599,6 +1599,119 @@ static int st_gmtime(httrackp *opt, int argc, char **argv) { return err; } +/* Is the masked in exactly want? Prints the difference when it is not. */ +static int maskpw_row(const char *in, const char *want, hts_mask_mode mode) { + char *got = hts_mask_passwords(in, mode); + const int err = got == NULL || strcmp(got, want) != 0; + + if (err) + fprintf(stderr, "maskpw: '%.80s' gave '%.80s', want '%.80s'\n", in, + got != NULL ? got : "(null)", want); + freet(got); + return err; +} + +/* A hostile URL of n ":@" pairs: in a path it stays, in a userinfo it is one + password. */ +static int maskpw_pairs(size_t n) { + const size_t len = 2 * n; + char *pairs = (char *) malloct(len + 1); + char *in = (char *) malloct(len + 32); + int err; + + if (pairs == NULL || in == NULL) { + freet(pairs); + freet(in); + return 1; + } + memset(pairs, ':', len); + for (size_t i = 1; i < len; i += 2) + pairs[i] = '@'; + pairs[len] = '\0'; + snprintf(in, len + 32, "404 at h/x?%s", pairs); + err = maskpw_row(in, in, HTS_MASK_LINE); + snprintf(in, len + 32, "at link a%sh/", pairs); + err |= maskpw_row(in, "at link a:***@h/", HTS_MASK_LINE); + snprintf(in, len + 32, "http://a%sh", pairs); + err |= maskpw_row(in, "http://a:***@h", HTS_MASK_ADDR); + freet(pairs); + freet(in); + return err; +} + +static int st_maskpw(httrackp *opt, int argc, char **argv) { + static const struct { + hts_mask_mode mode; + const char *in, *want; + } rows[] = { + {HTS_MASK_LINE, "", ""}, + {HTS_MASK_LINE, "http://u:p@h/", "http://u:***@h/"}, + // the engine logs an http address with no scheme + {HTS_MASK_LINE, "at link u:p@h:80/x (from v:q@g/y)", + "at link u:***@h:80/x (from v:***@g/y)"}, + {HTS_MASK_LINE, "see (u:p@h/x) 'v:q@g/y' \"w:r@f/z\"", + "see (u:***@h/x) 'v:***@g/y' \"w:***@f/z\""}, + {HTS_MASK_LINE, "at link HTTP://u:p@h:80/x", + "at link HTTP://u:***@h:80/x"}, + {HTS_MASK_LINE, "ftp://u:p@h", "ftp://u:***@h"}, + {HTS_MASK_LINE, "x1+a.b-c://u:p@h/", "x1+a.b-c://u:***@h/"}, + {HTS_MASK_LINE, "http://u@h/", "http://u@h/"}, + {HTS_MASK_LINE, "http://:p@h/", "http://:***@h/"}, + {HTS_MASK_LINE, "http://u:@h/", "http://u:***@h/"}, + // the engine splits the userinfo at the last '@' + {HTS_MASK_LINE, "http://u:p@SECRETB@h/", "http://u:***@h/"}, + {HTS_MASK_LINE, "u:p@SECRETB@h/", "u:***@h/"}, + {HTS_MASK_LINE, "http://user:pw;SECRETA@h/", "http://user:***@h/"}, + {HTS_MASK_LINE, "http://u:a,b=c\"d'e(f) h@h/", "http://u:***@h/"}, + {HTS_MASK_LINE, "http://u:a%2Fb@h/", "http://u:***@h/"}, + // a raw '/' ends the authority, so the engine sends no password + {HTS_MASK_LINE, "http://u:a/b@h/", "http://u:a/b@h/"}, + {HTS_MASK_LINE, "http://h1/ http://u:p@h2/ x:y@z", + "http://h1/ http://u:***@h2/ x:y@z"}, + {HTS_MASK_LINE, "http://h/?r=ftp://u:p@g/", "http://h/?r=ftp://u:***@g/"}, + // no address + {HTS_MASK_LINE, "-*:*@*", "-*:*@*"}, + {HTS_MASK_LINE, "+*:*@*/x", "+*:*@*/x"}, + {HTS_MASK_LINE, "http://h/x?t=10:30@x", "http://h/x?t=10:30@x"}, + {HTS_MASK_LINE, "?t=10:30@x", "?t=10:30@x"}, + {HTS_MASK_LINE, "mailto:a:b@c", "mailto:a:b@c"}, + {HTS_MASK_LINE, "://u:p@h/", "://u:p@h/"}, + {HTS_MASK_LINE, "user x:y@z said /", "user x:y@z said /"}, + {HTS_MASK_LIST, "u:p@h http://v:q@g +*:*@* -*:*@*/x", + "u:***@h http://v:***@g +*:*@* -*:*@*/x"}, + {HTS_MASK_ADDR, "u:p@h:8080", "u:***@h:8080"}, + {HTS_MASK_ADDR, "u:p@q@h:8080", "u:***@h:8080"}, + {HTS_MASK_ADDR, "u:a b@h/", "u:***@h/"}, + {HTS_MASK_ADDR, "http://u:p@h:8080", "http://u:***@h:8080"}, + {HTS_MASK_ADDR, "h:8080", "h:8080"}, + {HTS_MASK_ADDR, "u@h:8080", "u@h:8080"}, + }; + + static const char *const modes[] = {"line", "list", "addr"}; + int err = 0; + size_t i; + + (void) opt; + if (argc >= 2) { + char *got = NULL; + + for (i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) + if (strcmp(argv[0], modes[i]) == 0) + got = hts_mask_passwords(argv[1], (hts_mask_mode) i); + if (got == NULL) + return 1; + printf("%s\n", got); + freet(got); + return 0; + } + for (i = 0; i < sizeof(rows) / sizeof(rows[0]); i++) + err |= maskpw_row(rows[i].in, rows[i].want, rows[i].mode); + err |= maskpw_pairs(300); + err |= maskpw_pairs(10000); + printf("maskpw self-test: %s\n", err ? "FAIL" : "OK"); + return err; +} + /* #806: hts_localtime() must own its output too, same rationale as hts_gmtime() (#794). Reference table computed under TZ=XXX5 (fixed UTC-5, no DST), which the driving .test script sets. */ @@ -3513,6 +3626,9 @@ const struct selftest_entry selftests_lib[] = { "the fields and copies the span primitive makes", st_span}, {"gmtime", "", "hts_gmtime() fills the caller's buffer, not a static (#794)", st_gmtime}, + {"maskpw", "[line|list|addr ]", + "hts-log.txt masks the password of a URL login and nothing else", + st_maskpw}, {"localtime", "", "hts_localtime() and get_filetime_rfc822()'s GMT labelling (#806)", st_localtime}, diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index 717eec9b5..465d7f394 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -122,6 +122,24 @@ for line in "urluser:***@127.0.0.1:$port/setcookie.html" "puser:***@127.0.0.1:$p done ok "hts-log.txt masks the passwords of URLs, proxies and -%X headers" +out=$(httrack -#test=maskpw) || fail "httrack -#test=maskpw exited non-zero: $out" +test "$out" = "maskpw self-test: OK" || fail "expected 'maskpw self-test: OK', got: $out" +# plain.html is reached only through the crawl, so its line comes from hts_log_print +crawl choke "cuser:CHOKESECRET@127.0.0.1:$port/hop.html" "nuser:NOPATHSECRET@127.0.0.1:1" \ + -z "-*:KEEPFILTER@*" +log=$(<"$tmpdir/choke/hts-log.txt") +for secret in CHOKESECRET NOPATHSECRET; do + if grep -qF "$secret" <<<"$log"; then + fail "$secret is in hts-log.txt: $(grep -F "$secret" <<<"$log")" + fi +done +for line in "at cuser:***@127.0.0.1:$port/hop.html nuser:***@127.0.0.1:1 -*:KEEPFILTER@*" \ + " -%H nuser:***@127.0.0.1:1 -z " " -*:KEEPFILTER@* )" \ + "link recorded: cuser:***@127.0.0.1:$port/plain.html"; do + grep -qF -- "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" +done +ok "every hts-log.txt line masks a URL password and keeps the rest" + root="$tmpdir/root" cmds="$tmpdir/cmds" mkdir -p "$root" From 91395414cb43ed14d292e24110f4f73adddda618 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 16:02:17 +0200 Subject: [PATCH 05/12] Log addresses without their login instead of rewriting log lines The previous round rewrote every hts-log.txt line with a heuristic that masked any "a:b@c" it found. It missed debug lines that print a bare address, and it altered text that held no password. Log lines now go out as formatted. Each log call drops the login at the source: an engine address goes through adr_nologin(), and a full link goes through URL_NOLOGIN(). The command-line echo still masks only the start URLs and the -P value, found through the option spans. Test 657 crawls with -z -Z -%H and passwords planted in the start URLs, in links of every form the engine reads, and in -P. It then greps every project file except doit.log, new.txt and new.zip, which --update needs. Signed-off-by: Xavier Roche Co-Authored-By: Claude Opus 5.5 --- src/htsback.c | 293 +++++++++++++---------- src/htscache.c | 41 ++-- src/htscore.c | 80 ++++--- src/htscoremain.c | 7 +- src/htsftp.c | 7 +- src/htslib.c | 93 ++++---- src/htslib.h | 19 +- src/htslib_selftest.c | 104 +++++---- src/htsname.c | 54 +++-- src/htsparse.c | 308 ++++++++++++++----------- src/htsrobots.c | 4 +- src/htssitemap.c | 33 +-- src/htswizard.c | 113 +++++---- tests/657_local-header-log-redact.test | 29 +-- tests/header-injection-server.py | 20 ++ 15 files changed, 685 insertions(+), 520 deletions(-) diff --git a/src/htsback.c b/src/htsback.c index 0c2c139e2..f695eea95 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -172,7 +172,7 @@ hts_boolean back_spool_write(httrackp *opt, const char *filename, opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: open " "error (%s, %s)", - back->url_adr, back->url_fil, filename, + adr_nologin(back->url_adr), back->url_fil, filename, dir_exists(filename) ? "directory exists" : "directory does NOT exist!", fexist_utf8(filename) ? "file already exists!" : "file does not exist"); back_tmpdir_drop(filename); /* filecreate() may have made the directory */ @@ -185,7 +185,7 @@ hts_boolean back_spool_write(httrackp *opt, const char *filename, hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: write " "error", - back->url_adr, back->url_fil, filename); + adr_nologin(back->url_adr), back->url_fil, filename); (void) UNLINK(filename); back_tmpdir_drop(filename); } @@ -341,7 +341,7 @@ static int back_connect_next(httrackp *opt, struct_back *sback, int i) { back[i].status = STATUS_CONNECTING; hts_log_print(opt, LOG_DEBUG, "connect failed, trying next address (%d/%d) for %s", next + 1, - cf->addr_count, back[i].url_adr); + cf->addr_count, adr_nologin(back[i].url_adr)); return 1; } @@ -468,14 +468,15 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr itemback = NULL; } hts_log_print(opt, LOG_WARNING | LOG_ERRNO, - "engine: warning: unserialize error for %s%s (%s)", adr, - fil, sav); + "engine: warning: unserialize error for %s%s (%s)", + adr_nologin(adr), fil, sav); } fclose(fp); } else { hts_log_print(opt, LOG_WARNING | LOG_ERRNO, - "engine: warning: unserialize error for %s%s (%s), file disappeared", - adr, fil, sav); + "engine: warning: unserialize error for %s%s (%s), file " + "disappeared", + adr_nologin(adr), fil, sav); } (void) UNLINK(fileback); back_tmpdir_drop(fileback); @@ -498,8 +499,9 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr return q; } else { hts_log_print(opt, LOG_WARNING, - "engine: warning: unserialize error for %s%s (%s): no more space to wakeup frozen slots", - adr, fil, sav); + "engine: warning: unserialize error for %s%s (%s): no " + "more space to wakeup frozen slots", + adr_nologin(adr), fil, sav); } } } @@ -764,10 +766,12 @@ int back_cleanup_background(httrackp * opt, cache_back * cache, back[i].url_sav, 1); if (checkIndex != -1) { hts_log_print(opt, LOG_WARNING, - "engine: unexpected duplicate file entry: %s%s -> %s (%d '%s') / %s%s -> %s (%d '%s')", - back[checkIndex].url_adr, back[checkIndex].url_fil, - back[checkIndex].url_sav, back[checkIndex].r.statuscode, - back[checkIndex].r.msg, back[i].url_adr, back[i].url_fil, + "engine: unexpected duplicate file entry: %s%s -> %s (%d " + "'%s') / %s%s -> %s (%d '%s')", + adr_nologin(back[checkIndex].url_adr), + back[checkIndex].url_fil, back[checkIndex].url_sav, + back[checkIndex].r.statuscode, back[checkIndex].r.msg, + adr_nologin(back[i].url_adr), back[i].url_fil, back[i].url_sav, back[i].r.statuscode, back[i].r.msg); back_delete(NULL, NULL, sback, checkIndex); #ifdef _DEBUG @@ -807,7 +811,7 @@ int back_cleanup_background(httrackp * opt, cache_back * cache, } else { hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s: %s", - back[i].url_adr, back[i].url_fil, + adr_nologin(back[i].url_adr), back[i].url_fil, named ? "memory full" : "temporary filename too long"); } } @@ -1162,20 +1166,22 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, "truncated chunked transfer (terminating chunk missing, got " LLintP " bytes): file not cached, will be retried on the next" " update (use -%%B to cache anyway): %s%s", - back[p].r.size, back[p].url_adr, back[p].url_fil); + back[p].r.size, adr_nologin(back[p].url_adr), back[p].url_fil); } else if (back_status(&back[p]) == STATUS_READY) { hts_log_print(opt, LOG_WARNING, "incomplete transfer (expected " LLintP " bytes, got " LLintP "): file not cached, will be retried on the next update" " (use -%%B to cache anyway): %s%s", - back[p].r.totalsize, back[p].r.size, back[p].url_adr, - back[p].url_fil); + back[p].r.totalsize, back[p].r.size, + adr_nologin(back[p].url_adr), back[p].url_fil); } else { - hts_log_print(opt, LOG_INFO, - "incomplete file not yet stored in cache (expected " - LLintP " got " LLintP "): %s%s", back[p].r.totalsize, - back[p].r.size, back[p].url_adr, back[p].url_fil); + hts_log_print( + opt, LOG_INFO, + "incomplete file not yet stored in cache (expected " LLintP + " got " LLintP "): %s%s", + back[p].r.totalsize, back[p].r.size, adr_nologin(back[p].url_adr), + back[p].url_fil); } back_finalize_backup(opt, &back[p], HTS_FALSE); /* Keep the surviving copy in new.lst, else the update purge drops the @@ -1468,19 +1474,21 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, HTS_STAT.stat_bytes += back[p].r.size; HTS_STAT.stat_files++; hts_log_print(opt, LOG_TRACE, "added file %s%s => %s", - back[p].url_adr, back[p].url_fil, back[p].url_sav); + adr_nologin(back[p].url_adr), back[p].url_fil, + back[p].url_sav); } if ((!back[p].r.notmodified) && (opt->is_update)) { HTS_STAT.stat_updated_files++; // page modifiée if (back[p].is_update) { - hts_log_print(opt, LOG_INFO, - "engine: transfer-status: link updated: %s%s -> %s", - back[p].url_adr, back[p].url_fil, - back[p].url_sav); + hts_log_print( + opt, LOG_INFO, + "engine: transfer-status: link updated: %s%s -> %s", + adr_nologin(back[p].url_adr), back[p].url_fil, + back[p].url_sav); } else { hts_log_print(opt, LOG_INFO, "engine: transfer-status: link added: %s%s -> %s", - back[p].url_adr, back[p].url_fil, + adr_nologin(back[p].url_adr), back[p].url_fil, back[p].url_sav); } if (cache->txt) { @@ -1491,9 +1499,11 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, } } } else { - hts_log_print(opt, LOG_INFO, - "engine: transfer-status: link recorded: %s%s -> %s", - back[p].url_adr, back[p].url_fil, back[p].url_sav); + hts_log_print( + opt, LOG_INFO, + "engine: transfer-status: link recorded: %s%s -> %s", + adr_nologin(back[p].url_adr), back[p].url_fil, + back[p].url_sav); if (cache->txt) { if (opt->is_update) state = "untouched"; @@ -1502,10 +1512,11 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, } } } else { - hts_log_print(opt, LOG_INFO, - "engine: transfer-status: empty file? (%d, '%s'): %s%s", - back[p].r.statuscode, back[p].r.msg, back[p].url_adr, - back[p].url_fil); + hts_log_print( + opt, LOG_INFO, + "engine: transfer-status: empty file? (%d, '%s'): %s%s", + back[p].r.statuscode, back[p].r.msg, + adr_nologin(back[p].url_adr), back[p].url_fil); if (cache->txt) { state = "empty"; } @@ -1513,8 +1524,8 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, } else { hts_log_print(opt, LOG_INFO, "engine: transfer-status: link error (%d, '%s'): %s%s", - back[p].r.statuscode, back[p].r.msg, back[p].url_adr, - back[p].url_fil); + back[p].r.statuscode, back[p].r.msg, + adr_nologin(back[p].url_adr), back[p].url_fil); if (cache->txt) { state = "error"; } @@ -1552,8 +1563,8 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, } else { /* error */ if (!HTTP_IS_OK(back[p].r.statuscode)) { - hts_log_print(opt, LOG_DEBUG, "redirect to %s%s", back[p].url_adr, - back[p].url_fil); + hts_log_print(opt, LOG_DEBUG, "redirect to %s%s", + adr_nologin(back[p].url_adr), back[p].url_fil); /* Store only header reference */ cache_mayadd(opt, cache, &back[p].r, back[p].url_adr, back[p].url_fil, NULL); @@ -1564,7 +1575,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, "file with unresolved type not cached (%s (%d), size " LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size, - back[p].url_adr, back[p].url_fil); + adr_nologin(back[p].url_adr), back[p].url_fil); } } @@ -2116,7 +2127,8 @@ int back_unserialize_ref(httrackp * opt, const char *adr, const char *fil, errno = 0; if (!ref_get_record(fp, back)) { hts_log_print(opt, LOG_DEBUG, - "Ignoring an unreadable resume reference for %s%s", adr, fil); + "Ignoring an unreadable resume reference for %s%s", + adr_nologin(adr), fil); fclose(fp); back_clear_entry(back); freet(back); @@ -2161,7 +2173,7 @@ int back_maydelete(httrackp * opt, cache_back * cache, struct_back * sback, assertf(back[p].ka_time_start != 0); hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully saved #%d (%s)", - back[p].r.debugid, back[p].url_adr); + back[p].r.debugid, adr_nologin(back[p].url_adr)); return 1; } } @@ -2215,7 +2227,7 @@ void back_maydeletehttp(httrackp * opt, cache_back * cache, struct_back * sback, assertf(back[q].ka_time_start != 0); hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully preserved #%d (%s)", - back[q].r.debugid, back[q].url_adr); + back[q].r.debugid, adr_nologin(back[q].url_adr)); } else { deletehttp(&back[p].r); back[p].r.soc = INVALID_SOCKET; @@ -2482,7 +2494,8 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback, ) { hts_log_print(opt, LOG_DEBUG, "File '%s%s' -> %s not yet saved in cache - saving now", - back[p].url_adr, back[p].url_fil, back[p].url_sav); + adr_nologin(back[p].url_adr), back[p].url_fil, + back[p].url_sav); } if (cache != NULL) { back_finalize(opt, cache, sback, p); @@ -2583,8 +2596,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, if (!test && back_exist(sback, opt, adr, fil, save)) { int already_there = 0; - hts_log_print(opt, LOG_ERROR, "error: back_add(%s,%s,%s) duplicate", adr, - fil, save); + hts_log_print(opt, LOG_ERROR, "error: back_add(%s,%s,%s) duplicate", + adr_nologin(adr), fil, save); } #endif @@ -2662,7 +2675,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back[p].status = STATUS_READY; // terminé back_set_finished(opt, sback, p); hts_log_print(opt, LOG_WARNING, - "File not added due to mirror cancel: %s%s", adr, fil); + "File not added due to mirror cancel: %s%s", + adr_nologin(adr), fil); return 0; } // test "fast header" cache ; that is, tests we did that lead to 3XX/4XX/5XX response codes @@ -2772,13 +2786,15 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "Previous file '%s' not found (erased by " "user ?), ignoring: %s%s", - save, back[p].url_adr, back[p].url_fil); + save, adr_nologin(back[p].url_adr), + back[p].url_fil); } } else { hts_log_print(opt, LOG_WARNING, "Previous file '%s' not found (erased by user " "?), recatching: %s%s", - save, back[p].url_adr, back[p].url_fil); + save, adr_nologin(back[p].url_adr), + back[p].url_fil); } } } // fsize() <= 0 @@ -2823,11 +2839,11 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, if (!test) { hts_log_print(opt, LOG_DEBUG, "File immediately loaded from cache: %s%s", - back[p].url_adr, back[p].url_fil); + adr_nologin(back[p].url_adr), back[p].url_fil); } else { hts_log_print(opt, LOG_DEBUG, "File immediately tested from cache: %s%s", - back[p].url_adr, back[p].url_fil); + adr_nologin(back[p].url_adr), back[p].url_fil); } back[p].r.notmodified = 1; // fichier non modifié // no request was sent at all, so this is never a server 304 (#839) @@ -2969,7 +2985,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, #endif hts_log_print(opt, LOG_DEBUG, "File partially present (" LLintP " bytes): %s%s", - (LLint) sz, back[p].url_adr, back[p].url_fil); + (LLint) sz, adr_nologin(back[p].url_adr), + back[p].url_fil); /* impossible - don't have etag or date if (strnotempty(back[p].r.etag)) { // ETag (RFC2616) @@ -2990,16 +3007,18 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back[p].range_req_size = sz; } else { hts_log_print(opt, LOG_WARNING, - "Could not find timestamp for partially present file, restarting (lost " - LLintP " bytes): %s%s", (LLint) sz, - back[p].url_adr, back[p].url_fil); + "Could not find timestamp for partially present " + "file, restarting (lost " LLintP " bytes): %s%s", + (LLint) sz, adr_nologin(back[p].url_adr), + back[p].url_fil); } } else { hts_log_print(opt, LOG_NOTICE, "File partially present (" LLintP " bytes) retransferred due to lack of cache: %s%s", - (LLint) sz, back[p].url_adr, back[p].url_fil); + (LLint) sz, adr_nologin(back[p].url_adr), + back[p].url_fil); /* Sinon requête normale... */ back[p].http11 = 0; } @@ -3017,7 +3036,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_NOTICE, "HTML file (" LLintP " bytes) retransferred due to lack of cache: %s%s", - (LLint) sz, back[p].url_adr, back[p].url_fil); + (LLint) sz, adr_nologin(back[p].url_adr), + back[p].url_fil); /* Sinon requête normale... */ back[p].http11 = 0; } @@ -3031,7 +3051,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back[p].status = STATUS_READY; // terminé back_set_finished(opt, sback, p); hts_log_print(opt, LOG_WARNING, - "File not added due to mirror cancel: %s%s", adr, fil); + "File not added due to mirror cancel: %s%s", + adr_nologin(adr), fil); return 0; } @@ -3159,9 +3180,9 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, } else { soc = back[p].r.soc; - hts_log_print(opt, LOG_DEBUG, - "(Keep-Alive): successfully linked #%d (for %s%s)", - back[p].r.debugid, back[p].url_adr, back[p].url_fil); + hts_log_print( + opt, LOG_DEBUG, "(Keep-Alive): successfully linked #%d (for %s%s)", + back[p].r.debugid, adr_nologin(back[p].url_adr), back[p].url_fil); } if (opt->timeout > 0) { // gestion du opt->timeout @@ -3256,8 +3277,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, may_be_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil), /* */ - back[i].r.contenttype, back[i].url_adr, back[i].url_fil, - back[i].url_sav); + back[i].r.contenttype, adr_nologin(back[i].url_adr), + back[i].url_fil, back[i].url_sav); } } } @@ -3318,8 +3339,10 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { opt->liens[index]->pass2 = -1; /* DONE! */ } else { hts_log_print(opt, LOG_INFO, - "engine: warning: entry cleaned up, but no trace on heap: %s%s (%s)", - back[i].url_adr, back[i].url_fil, back[i].url_sav); + "engine: warning: entry cleaned up, but no trace on " + "heap: %s%s (%s)", + adr_nologin(back[i].url_adr), back[i].url_fil, + back[i].url_sav); } } HTS_STAT.stat_background++; @@ -3332,7 +3355,7 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { /* recycle the socket, but keep back[i].r.adr in memory */ hts_log_print(opt, LOG_DEBUG, "file %s%s validated (cached, left in memory)", - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), back[i].url_fil); back_maydeletehttp(opt, cache, sback, i); } } @@ -3356,7 +3379,7 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { } hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): live socket #%d (%s) closed (%s)", - back[i].r.debugid, back[i].url_adr, reason); + back[i].r.debugid, adr_nologin(back[i].url_adr), reason); back_delete(opt, cache, sback, i); // delete backing entry } } @@ -4139,9 +4162,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, #if HDEBUG printf("direct-disk: %s\n", back[i].url_sav); #endif - hts_log_print(opt, LOG_DEBUG, - "File received from net to disk: %s%s", - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_DEBUG, + "File received from net to disk: %s%s", + adr_nologin(back[i].url_adr), back[i].url_fil); if (back[i].r.out == NULL) { hts_log_print(opt, LOG_ERROR | LOG_ERRNO, @@ -4166,8 +4190,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, if (back[i].r.out != NULL && opt->cache != 0) { if (back_serialize_ref(opt, &back[i]) != 0) { hts_log_print(opt, LOG_WARNING, - "Could not create temporary reference file for %s%s", - back[i].url_adr, + "Could not create temporary " + "reference file for %s%s", + adr_nologin(back[i].url_adr), back[i].url_fil); } } @@ -4175,7 +4200,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } else { // on coupe tout! hts_log_print(opt, LOG_DEBUG, "File cancelled (non HTML): %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); back[i].status = STATUS_READY; // terminé back_set_finished(opt, sback, i); if (!back[i].testmode) @@ -4343,8 +4369,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print(opt, LOG_WARNING, "Truncated chunked transfer (" LLintP " Bytes, terminating chunk missing) for %s%s", - (LLint) back[i].r.size, back[i].url_adr, - back[i].url_fil); + (LLint) back[i].r.size, + adr_nologin(back[i].url_adr), back[i].url_fil); } } @@ -4368,8 +4394,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Incorrect length (" LLintP "!=" LLintP " expected) for %s%s", (LLint) back[i].r.size, - (LLint) back[i].r.totalsize, back[i].url_adr, - back[i].url_fil); + (LLint) back[i].r.totalsize, + adr_nologin(back[i].url_adr), back[i].url_fil); } } } @@ -4463,7 +4489,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, if (!hts_inmem_size_fits(back[i].r.totalsize)) { hts_log_print(opt, LOG_WARNING, "Chunked resource too large for %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); chunk_size = -1; } else { char *const grown = (char *) realloct( @@ -4475,11 +4502,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, /* r.totalsize already counts this chunk, so dropping the buffer would have the next read write past it. */ - hts_log_print(opt, LOG_ERROR, - "not enough memory (" LLintP - ") for %s%s", - (LLint) back[i].r.totalsize, - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_ERROR, + "not enough memory (" LLintP ") for %s%s", + (LLint) back[i].r.totalsize, + adr_nologin(back[i].url_adr), back[i].url_fil); chunk_size = -1; } } @@ -4490,10 +4517,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, (LLint) back[i].r.totalsize); #endif } else { - hts_log_print(opt, LOG_WARNING, - "Illegal chunk (%s) for %s%s", - back[i].chunk_adr, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, "Illegal chunk (%s) for %s%s", + back[i].chunk_adr, adr_nologin(back[i].url_adr), + back[i].url_fil); } } else { /* STATUS_CHUNK_CR: the CRLF after the data */ if (chunk_data[0] == '\0') { @@ -4507,10 +4534,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, printf("[%d] chunk CRLF seen\n", (int) back[i].r.soc); #endif } else { - hts_log_print(opt, LOG_WARNING, - "illegal chunk CRLF (%s) for %s%s", - back[i].chunk_adr, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, "illegal chunk CRLF (%s) for %s%s", + back[i].chunk_adr, adr_nologin(back[i].url_adr), + back[i].url_fil); #if CHUNKDEBUG==1 printf("[%d] chunk CRLF ERROR!! : '%s'\n", (int) back[i].r.soc, chunk_data); @@ -4520,8 +4547,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } else { hts_log_print(opt, LOG_WARNING, "chunk too big (" LLintP ") for %s%s", - (LLint) back[i].chunk_size, back[i].url_adr, - back[i].url_fil); + (LLint) back[i].chunk_size, + adr_nologin(back[i].url_adr), back[i].url_fil); } // ok, continuer sur le body @@ -4562,9 +4589,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, back_finalize(opt, cache, sback, i); } else { if (back[i].r.statuscode == HTTP_OK) { - hts_log_print(opt, LOG_WARNING, - "unexpected incomplete type with 200 code at %s%s", - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "unexpected incomplete type with 200 code at %s%s", + adr_nologin(back[i].url_adr), back[i].url_fil); } } if (back[i].r.soc != INVALID_SOCKET) { @@ -4598,11 +4626,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } else { // Un warning suffira.. hts_log_print(opt, LOG_WARNING, - "Incorrect length (" LLintP "!=" - LLintP " expected) for %s%s", + "Incorrect length (" LLintP + "!=" LLintP " expected) for %s%s", (LLint) back[i].r.size, (LLint) back[i].r.totalsize, - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); } } } @@ -4685,7 +4714,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, truncated Location, or read its tail as headers */ hts_log_print(opt, LOG_WARNING, "Over-long header dropped for %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); continue; } #if HDEBUG @@ -4736,8 +4766,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, &back[i].r); if (test_head != 1) { hts_log_print(opt, LOG_WARNING, - "External wrapper aborted transfer, breaking connection: %s%s", - back[i].url_adr, back[i].url_fil); + "External wrapper aborted transfer, " + "breaking connection: %s%s", + adr_nologin(back[i].url_adr), + back[i].url_fil); back[i].status = STATUS_READY; // FINI back_set_finished(opt, sback, i); deletehttp(&back[i].r); @@ -4773,7 +4805,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Bot protection challenge at %s%s: the " "server refused the request (cf-mitigated " "header) rather than failing it", - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); hts_log_print(opt, LOG_WARNING, "Solve the challenge in a browser, then pass " "its cookies with --cookies-file and its " @@ -4799,9 +4832,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, back[i].r.msg[0] = '\0'; /* the interim block is not the final response's headers */ freet(back[i].r.headers); - hts_log_print(opt, LOG_DEBUG, - "Status 100 detected for %s%s, continuing headers", - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_DEBUG, + "Status 100 detected for %s%s, continuing headers", + adr_nologin(back[i].url_adr), back[i].url_fil); continue; } @@ -4834,7 +4868,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Kept existing file %s (" LLintP " bytes), matching size and timestamp: %s%s", back[i].url_sav, (LLint) back[i].range_req_size, - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), back[i].url_fil); } } // transform 406 into 200 ; we'll catch embedded links inside the choice page @@ -4851,7 +4885,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Kept existing file %s after error %d (%s) " "on update: %s%s", back[i].url_sav, back[i].r.statuscode, - back[i].r.msg, back[i].url_adr, + back[i].r.msg, adr_nologin(back[i].url_adr), back[i].url_fil); back[i].r.statuscode = HTTP_NOT_MODIFIED; deletehttp(&back[i].r); @@ -4896,14 +4930,17 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Kept existing file %s (" LLintP " bytes), matching the cached size: %s%s", back[i].url_sav, (LLint) len1, - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); } } } else { - hts_log_print(opt, LOG_WARNING, - "File seems complete (same size), but there was a cache read error (%u): %s%s", - r.statuscode, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "File seems complete (same size), but there was " + "a cache read error (%u): %s%s", + r.statuscode, adr_nologin(back[i].url_adr), + back[i].url_fil); } if (r.adr) { freet(r.adr); @@ -4939,7 +4976,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, " bytes), matching the announced " "size: %s%s", back[i].url_sav, (LLint) size, - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); } } } @@ -4987,7 +5025,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "size, range ignored: %s%s", back[i].url_sav, (LLint) back[i].range_req_size, - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); } } } @@ -5035,7 +5074,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, // head: terminé if (back[i].head_request) { hts_log_print(opt, LOG_DEBUG, "Tested file: %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(head request): deletehttp\n"); #endif @@ -5089,9 +5129,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, get_httptype_sized(opt, back[i].r.contenttype, sizeof(back[i].r.contenttype), back[i].url_sav, 1); - hts_log_print(opt, LOG_DEBUG, - "Not-modified status without cache guessed: %s%s", - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_DEBUG, + "Not-modified status without cache guessed: %s%s", + adr_nologin(back[i].url_adr), back[i].url_fil); } } // Status is okay? @@ -5104,7 +5145,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, server_sent_304 ? HTS_FALSE : HTS_TRUE; hts_log_print(opt, LOG_DEBUG, "File loaded after test from cache: %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(back[i].url_adr), + back[i].url_fil); // finalize if (back[i].r.statuscode > 0) { @@ -5139,7 +5181,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print( opt, LOG_NOTICE, "File excluded by MIME type filter (%s): %s%s", - back[i].r.contenttype, back[i].url_adr, + back[i].r.contenttype, adr_nologin(back[i].url_adr), back[i].url_fil); } else { // il faut aller le chercher @@ -5212,7 +5254,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print(opt, LOG_WARNING, "Could not create temporary " "reference file for %s%s", - back[i].url_adr, + adr_nologin(back[i].url_adr), back[i].url_fil); } #if HDEBUG @@ -5285,8 +5327,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Unusable partial-content range for %s%s " "(have " LLintP " bytes, got " LLintP "-" LLintP "/" LLintP "), restarting", - back[i].url_adr, back[i].url_fil, - (LLint) sz, back[i].r.crange_start, + adr_nologin(back[i].url_adr), + back[i].url_fil, (LLint) sz, + back[i].r.crange_start, back[i].r.crange_end, back[i].r.crange); url_savename_refname_remove(opt, back[i].url_adr, back[i].url_fil); @@ -5457,8 +5500,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } } if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) { - hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr, - back[i].url_fil); + hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", + adr_nologin(back[i].url_adr), back[i].url_fil); if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(timeout): deletehttp\n"); diff --git a/src/htscache.c b/src/htscache.c index a71f76c48..4c3e590b4 100644 --- a/src/htscache.c +++ b/src/htscache.c @@ -53,16 +53,18 @@ Please visit our Website: http://www.httrack.com void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, const char *url_adr, const char *url_fil, const char *url_save) { - hts_log_print(opt, LOG_DEBUG, "File checked by cache: %s", url_adr); + hts_log_print(opt, LOG_DEBUG, "File checked by cache: %s", + adr_nologin(url_adr)); // ---stockage en cache--- // stocker dans le cache? if (opt->cache) { if (cache_writable(cache)) { // ensure not a temporary filename (should not happend ?!) if (IS_DELAYED_EXT(url_save)) { - hts_log_print(opt, LOG_WARNING, - "aborted cache validation: %s%s still has temporary name %s", - url_adr, url_fil, url_save); + hts_log_print( + opt, LOG_WARNING, + "aborted cache validation: %s%s still has temporary name %s", + adr_nologin(url_adr), url_fil, url_save); return; } // c'est le seul endroit ou l'on ajoute des elements dans le cache (fichier entier ou header) @@ -105,9 +107,9 @@ void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, strcatbuff(tempo, "\n"); strcatbuff(tempo, r->location); } - hts_log_print(opt, LOG_DEBUG, - "Cached fast-header response: %s%s is %d", - url_adr, url_fil, (int) r->statuscode); + hts_log_print( + opt, LOG_DEBUG, "Cached fast-header response: %s%s is %d", + adr_nologin(url_adr), url_fil, (int) r->statuscode); coucal_add(cache->cached_tests, concat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), url_adr, url_fil), (intptr_t) strdupt(tempo)); @@ -207,9 +209,10 @@ static void cache_zip_write_failed(httrackp *opt, cache_back *cache, /* fatal: stop the mirror, exit non-zero */ hts_store_release_int(&opt->state.exit_xh, -1); } else { - hts_log_print( - opt, LOG_WARNING, "cache write failed (%s: %s), entry not cached: %s%s", - what, hts_get_zerror(zErr, errbuf, sizeof(errbuf)), url_adr, url_fil); + hts_log_print(opt, LOG_WARNING, + "cache write failed (%s: %s), entry not cached: %s%s", what, + hts_get_zerror(zErr, errbuf, sizeof(errbuf)), + adr_nologin(url_adr), url_fil); } } @@ -274,7 +277,7 @@ void cache_keep_previous(httrackp *opt, cache_back *cache, const char *url_adr, URL's file. */ if (r.statuscode > 0 && strcmp(previous_save, url_save) == 0) { hts_log_print(opt, LOG_DEBUG, "keeping the previous cache entry for %s%s", - url_adr, url_fil); + adr_nologin(url_adr), url_fil); cache_add_ex(opt, cache, &r, url_adr, url_fil, url_save, opt->all_in_cache, StringBuff(opt->path_html_utf8), HTS_TRUE); } @@ -336,12 +339,12 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, if (r->is_write && url_save != NULL && strnotempty(url_save)) { hts_log_print(opt, LOG_WARNING, "file too large for the cache, storing headers only: %s%s", - url_adr, url_fil); + adr_nologin(url_adr), url_fil); dataincache = 0; } else { hts_log_print(opt, LOG_WARNING, - "entry too large for the cache, not cached: %s%s", url_adr, - url_fil); + "entry too large for the cache, not cached: %s%s", + adr_nologin(url_adr), url_fil); return; } } @@ -407,7 +410,7 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, if (headersDropped != 0) { hts_log_print(opt, LOG_WARNING, "cached headers too large, %d field(s) dropped: %s%s", - headersDropped, url_adr, url_fil); + headersDropped, adr_nologin(url_adr), url_fil); } /* Filename */ @@ -421,7 +424,7 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, url_fil)) { hts_log_print(opt, LOG_WARNING, "URL too long to be cached, entry not cached: %s%s", - url_adr, url_fil); + adr_nologin(url_adr), url_fil); return; } } @@ -720,7 +723,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, if (!hts_location_is_safe(r.location)) { hts_log_print(opt, LOG_WARNING, "cached Location naming a post token, dropped: %s%s", - adr, fil); + adr_nologin(adr), fil); r.location[0] = '\0'; } @@ -750,7 +753,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "cached filename leaving the mirror, " "not using the cache entry: %s%s", - adr, fil); + adr_nologin(adr), fil); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Cache Read Error : Bad Filename"); } @@ -761,7 +764,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "cached filename too long once rebuilt under '%s', " "not using the cache entry: %s%s", - path_html, adr, fil); + path_html, adr_nologin(adr), fil); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Cache Read Error : Filename Too Long"); } diff --git a/src/htscore.c b/src/htscore.c index fc198a27c..59cc517cb 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -991,7 +991,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { (int) (sizeof(url) - 1), url); hts_log_print(opt, LOG_WARNING, "URL longer than %d bytes, ignored: %.64s...", - (int) (sizeof(url) - 1), url); + (int) (sizeof(url) - 1), URL_NOLOGIN(opt, url)); continue; } @@ -1424,7 +1424,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { } if (heap(ptr) != NULL) { // on a qq chose à récupérer? - hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s", urladr(), urlfil()); + hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s", adr_nologin(urladr()), + urlfil()); #if DEBUG_ROBOTS if (strcmp(urlfil(), "/robots.txt") == 0) { printf("robots.txt detected\n"); @@ -1544,9 +1545,10 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (istoobig(opt, r.totalsize, opt->maxfile_html, opt->maxfile_nonhtml, r.contenttype)) { error = 0; - hts_log_print(opt, LOG_WARNING, - "Big file cancelled according to user's preferences: %s%s", - urladr(), urlfil()); + hts_log_print( + opt, LOG_WARNING, + "Big file cancelled according to user's preferences: %s%s", + adr_nologin(urladr()), urlfil()); } } } @@ -1599,9 +1601,9 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { is_loaded_from_file = 1; \ r.adr = readfile2_inmem(savename(), &r.size); \ if (r.adr != NULL) { \ - hts_log_print(opt, LOG_INFO, "File successfully loaded for parsing: %s%s (%d bytes)",urladr(),urlfil(),(int)r.size); \ + hts_log_print(opt, LOG_INFO, "File successfully loaded for parsing: %s%s (%d bytes)",adr_nologin(urladr()),urlfil(),(int)r.size); \ } else { \ - hts_log_print(opt, LOG_ERROR, "File could not be loaded for parsing: %s%s",urladr(),urlfil()); \ + hts_log_print(opt, LOG_ERROR, "File could not be loaded for parsing: %s%s",adr_nologin(urladr()),urlfil()); \ } \ } \ } while(0) @@ -1655,7 +1657,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { hts_log_print( opt, LOG_ERROR, "Not enough memory to convert %s%s from UCS2 to UTF-8", - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Not enough memory"); error = 1; @@ -1663,7 +1665,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { hts_log_print(opt, LOG_WARNING, "File %s%s converted from UCS2 to UTF-8 (old " "size: %d bytes, new size: %d bytes)", - urladr(), urlfil(), (int) r.size, (int) new_size); + adr_nologin(urladr()), urlfil(), (int) r.size, + (int) new_size); freet(r.adr); r.adr = new_adr; r.size = (LLint) new_size; @@ -1678,8 +1681,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { is_binary = 1; strcpybuff(r.contenttype, "application/octet-stream"); hts_log_print(opt, LOG_WARNING, - "File not parsed, looks like binary: %s%s", urladr(), - urlfil()); + "File not parsed, looks like binary: %s%s", + adr_nologin(urladr()), urlfil()); } /* This hack allows you to avoid problems with parsing '\0' characters */ @@ -1912,13 +1915,13 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (page_charset[0] == '\0') { hts_log_print(opt, LOG_INFO, "Warning: could not detect encoding for: %s%s", - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); } } /* Info for wrappers */ - hts_log_print(opt, LOG_INFO, "engine: check-html: %s%s", urladr(), - urlfil()); + hts_log_print(opt, LOG_INFO, "engine: check-html: %s%s", + adr_nologin(urladr()), urlfil()); { char BIGSTK buff_err_msg[1024]; htsmoduleStruct BIGSTK str; @@ -2017,10 +2020,12 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (strnotempty(infobuff)) { hts_log_print(opt, LOG_INFO, "Note: robots.txt forbidden links for %s are: %s", - urladr(), infobuff); + adr_nologin(urladr()), infobuff); hts_log_print(opt, LOG_NOTICE, - "Note: due to %s remote robots.txt rules, links beginning with these path will be forbidden: %s (see in the options to disable this)", - urladr(), infobuff); + "Note: due to %s remote robots.txt rules, links " + "beginning with these path will be forbidden: %s " + "(see in the options to disable this)", + adr_nologin(urladr()), infobuff); } } /* After robots_parse, so the rules this very body carries already @@ -2078,7 +2083,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { } else { // lien non autorisé! (ex: cgi-bin en html) hts_log_print(opt, LOG_DEBUG, "non-html file ignored after upload at %s : %s", - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); if (r.adr) { freet(r.adr); r.adr = NULL; @@ -2626,8 +2631,8 @@ void host_ban(httrackp * opt, int ptr, back[i].r.soc = INVALID_SOCKET; back[i].r.statuscode = STATUSCODE_TIMEOUT; // timeout (peu importe si c'est un traffic jam) strcpybuff(back[i].r.msg, "Link Cancelled by host control"); - hts_log_print(opt, LOG_DEBUG, "Shutdown: %s%s", back[i].url_adr, - back[i].url_fil); + hts_log_print(opt, LOG_DEBUG, "Shutdown: %s%s", + adr_nologin(back[i].url_adr), back[i].url_fil); } } @@ -2642,8 +2647,8 @@ void host_ban(httrackp * opt, int ptr, l++; if ((l > 0) && (l < 1020)) { // in range if (strfield2(jump_identification_const(heap(i)->adr), host)) { // host - hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", heap(i)->adr, - heap(i)->fil); + hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", + adr_nologin(heap(i)->adr), heap(i)->fil); hts_invalidate_link(opt, i); // invalidate hashtable entry // on efface pas le hash, because si on rencontre le lien, reverif sav.. } @@ -2762,7 +2767,8 @@ static hts_boolean live_cancel_link_(httrackp *opt, struct_back *sback, int i) { /* no cache: a dropped page must not be recorded as fetched */ back_delete(opt, NULL, sback, b); } - hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", heap(i)->adr, heap(i)->fil); + hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", adr_nologin(heap(i)->adr), + heap(i)->fil); hts_invalidate_link(opt, i); /* the main loop takes every cancelled link out of this count */ HTS_STAT.stat_background++; @@ -4841,18 +4847,19 @@ int htsAddLink(htsmoduleStruct * str, char *link) { codebase[0] = '\0'; - hts_log_print(opt, LOG_DEBUG, "(module): adding link : '%s'", link); + hts_log_print(opt, LOG_DEBUG, "(module): adding link : '%s'", + URL_NOLOGIN(opt, link)); // recopie de "creer le lien" // if (!RUN_CALLBACK1(opt, linkdetected, link)) { hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper", - link); + URL_NOLOGIN(opt, link)); return 0; } if (!RUN_CALLBACK2(opt, linkdetected2, link, NULL)) { hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper(2)", - link); + URL_NOLOGIN(opt, link)); return 0; } // adr = c'est la même @@ -4884,14 +4891,14 @@ int htsAddLink(htsmoduleStruct * str, char *link) { strcpybuff(codebase, tempo); // couper host } else { hts_log_print(opt, LOG_ERROR, "Unexpected strstr error in base %s", - codebase); + URL_NOLOGIN(opt, codebase)); } } } if (!((int) strlen(codebase) < HTS_URLMAXSIZE)) { // trop long hts_log_print(opt, LOG_ERROR, "Codebase too long, parsing skipped (%s)", - codebase); + URL_NOLOGIN(opt, codebase)); } { @@ -4943,9 +4950,10 @@ int htsAddLink(htsmoduleStruct * str, char *link) { if (savename()) { if (lienrelatif(tempo, sizeof(tempo), afs.save, savename()) == 0) { - hts_log_print(opt, LOG_DEBUG, - "(module): relative link at %s build with %s and %s: %s", - afs.af.adr, afs.save, savename(), tempo); + hts_log_print( + opt, LOG_DEBUG, + "(module): relative link at %s build with %s and %s: %s", + adr_nologin(afs.af.adr), afs.save, savename(), tempo); if (str->localLink && str->localLinkSize > (int) strlen(tempo) + 1) { strlcpybuff(str->localLink, tempo, str->localLinkSize); @@ -4957,7 +4965,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) { if (forbidden_url) { hts_log_print(opt, LOG_DEBUG, "(module): file not caught: %s", - lien); + URL_NOLOGIN(opt, lien)); if (str->localLink && str->localLinkSize > (int) (strlen(afs.af.adr) + strlen(afs.af.fil) + 8)) { htsbuff lb = htsbuff_ptr(str->localLink, str->localLinkSize); @@ -4970,8 +4978,9 @@ int htsAddLink(htsmoduleStruct * str, char *link) { } // if (r != -1) { - hts_log_print(opt, LOG_DEBUG, "(module): %s%s -> %s (base %s)", afs.af.adr, - afs.af.fil, afs.save, codebase); + hts_log_print(opt, LOG_DEBUG, "(module): %s%s -> %s (base %s)", + adr_nologin(afs.af.adr), afs.af.fil, afs.save, + URL_NOLOGIN(opt, codebase)); // modifié par rapport à l'autre version (cf prio_fix notamment et save2) @@ -5016,9 +5025,8 @@ int htsAddLink(htsmoduleStruct * str, char *link) { heap_top()->retry = opt->retry; hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s", - heap_top()->adr, heap_top()->fil, + adr_nologin(heap_top()->adr), heap_top()->fil, heap_top()->sav); - } } } diff --git a/src/htscoremain.c b/src/htscoremain.c index 9f873c94a..cd8b12a0f 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -3135,14 +3135,13 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { opt_at = 0; addr = i != 0 && arg[0] != '+' && arg[0] != '-'; } - masked = - hts_mask_passwords(arg, addr ? HTS_MASK_ADDR : HTS_MASK_LINE); + masked = addr ? hts_mask_passwords(arg, HTS_MASK_ADDR) : NULL; if (quote) fputc('"', opt->log); // a -%X login header gets the -%H log's redaction - if (masked != NULL) - fprintfio(opt->log, masked, ""); + if (!addr || masked != NULL) + fprintfio(opt->log, addr ? masked : arg, ""); fputs(quote ? "\" " : " ", opt->log); freet(masked); } diff --git a/src/htsftp.c b/src/htsftp.c index 936cf7320..7a1c8de07 100644 --- a/src/htsftp.c +++ b/src/htsftp.c @@ -639,7 +639,7 @@ static hts_boolean ftp_may_resume(httrackp *opt, const lien_back *back, if (back->r.totalsize <= back->range_req_size) { hts_log_print(opt, LOG_DEBUG, "FTP: not resuming %s%s, the local copy is not shorter", - back->url_adr, back->url_fil); + adr_nologin(back->url_adr), back->url_fil); return HTS_FALSE; } /* Equality, not ordering: a copy this code did not stamp carries a local @@ -648,7 +648,7 @@ static hts_boolean ftp_may_resume(httrackp *opt, const lien_back *back, remote_mtime != local_mtime) { hts_log_print(opt, LOG_DEBUG, "FTP: not resuming %s%s, the copy is not dated as the remote", - back->url_adr, back->url_fil); + adr_nologin(back->url_adr), back->url_fil); return HTS_FALSE; } return HTS_TRUE; @@ -881,7 +881,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) { SOCaddr_initport(server, port); strcpybuff(back->info, "connect"); - hts_log_print(opt, LOG_DEBUG, "FTP: connecting to %s:%d", _adr, port); + hts_log_print(opt, LOG_DEBUG, "FTP: connecting to %s:%d", adr_nologin(_adr), + port); if (!ftp_connect(back, soc_ctl, &server, timeout, opt)) { strcpybuff(back->r.msg, "Unable to connect to the server"); diff --git a/src/htslib.c b/src/htslib.c index 3a0db563e..041de4f4f 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -2567,7 +2567,7 @@ void treathead_opt(httrackp *opt, t_cookie *cookie, const char *adr, &attr) != 0) { hts_log_print(opt, LOG_WARNING, "Cookies: the jar did not store '%.64s' set by %s", - cook_name, adr); + cook_name, adr_nologin(adr)); } } } @@ -4367,6 +4367,29 @@ HTSEXT_API const char *jump_identification_const(const char *source) { HTSEXT_API DECLARE_NON_CONST_VERSION(jump_identification) + const char *adr_nologin(const char *adr) { + const char *const host = jump_identification_const(adr); + + return host != jump_protocol_const(adr) ? host : adr; +} + +const char *url_nologin(char *buf, size_t size, const char *url) { + const char *const a = jump_protocol_const(url); + const char *const host = jump_identification_const(url); + size_t n = (size_t) (a - url), m = strlen(host); + + if (a == url || host == a || size == 0) + return url; + if (n > size - 1) + n = size - 1; + if (m > size - 1 - n) + m = size - 1 - n; + memcpy(buf, url, n); + memcpy(buf + n, host, m); + buf[n + m] = '\0'; + return buf; +} + HTSEXT_API const char *jump_normalized_const(const char *source) { if (strcmp(source, "file://") == 0) return source; @@ -5815,23 +5838,15 @@ void fprint_redacted(FILE *fp, size_t len) { static const char *const secret_fields[] = { "Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"}; -/* Is c allowed in a URL scheme (RFC 3986 3.1)? */ -static hts_boolean scheme_char(char c) { - return isalnum((unsigned char) c) || c == '+' || c == '-' || c == '.'; -} - /* Finds the password of the userinfo at the authority a: from the first ':' up to the last '@' before the first '/', the split - jump_identification_const() makes. A token also ends at a blank, and with - path, it must go on to a '/'. Returns HTS_FALSE when there is none. */ + jump_identification_const() makes. A token also ends at a blank. Returns + HTS_FALSE when there is none. */ static hts_boolean userinfo_password(const char *a, hts_boolean token, - hts_boolean path, const char **pass, - const char **at) { + const char **pass, const char **at) { const char *end = a + strcspn(a, token ? "/ \t\r\n" : "/"); const char *colon; - if (path && *end != '/') - return HTS_FALSE; while (end > a && end[-1] != '@') end--; if (end == a) @@ -5858,39 +5873,32 @@ static void mask_copy(mask_out *m, const char *src, size_t n) { m->o += n; } -/* Masks the password at the authority a, if any, and moves *i past it. */ -static hts_boolean mask_at(mask_out *m, const char *a, hts_boolean token, - hts_boolean path, const char **i) { +/* Masks the password of the address at a, if any, and moves *i past it. */ +static void mask_at(mask_out *m, const char *a, hts_boolean token, + const char **i) { const char *pass, *at; - if (!userinfo_password(a, token, path, &pass, &at)) - return HTS_FALSE; + if (!userinfo_password(jump_protocol_const(a), token, &pass, &at)) + return; mask_copy(m, m->p, (size_t) (pass - m->p)); mask_copy(m, "***", 3); m->p = *i = at; - return HTS_TRUE; } -/* Copies s to dst, or only counts when dst is NULL, with each password the - engine would read out of an address written as ***. Returns the length. */ +/* Copies s to dst, or only counts when dst is NULL, with the password of each + address written as ***. Returns the length. */ static size_t mask_passwords(const char *s, char *dst, hts_mask_mode mode) { mask_out m = {dst, 0, s}; const char *i = s; if (mode == HTS_MASK_ADDR) - (void) mask_at(&m, jump_protocol_const(s), HTS_FALSE, HTS_FALSE, &i); - while (*i != '\0') { - // an address the engine printed starts a token, often with no scheme - if (mode != HTS_MASK_ADDR && (i == s || strchr(" \t\r\n", i[-1])) && - strchr("+-*", *i) == NULL && - mask_at(&m, jump_protocol_const(i), HTS_TRUE, mode == HTS_MASK_LINE, - &i)) - continue; - if (i != s && strncmp(i, "://", 3) == 0 && scheme_char(i[-1]) && - mask_at(&m, i + 3, HTS_FALSE, HTS_FALSE, &i)) - continue; - i++; - } + mask_at(&m, s, HTS_FALSE, &i); + else + for (; *i != '\0'; i++) + // a token that is no filter is a start URL + if ((i == s || strchr(" \t\r\n", i[-1]) != NULL) && + strchr(" \t\r\n+-", *i) == NULL) + mask_at(&m, i, HTS_TRUE, &i); mask_copy(&m, m.p, strlen(m.p) + 1); return m.o - 1; } @@ -7380,25 +7388,8 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_ fspc_count(opt, s_type); if (opt->log != NULL) { - va_list args_len; - int n; - - va_copy(args_len, args); - n = vsnprintf(NULL, 0, format, args_len); - va_end(args_len); fspc(opt, opt->log, s_type); - if (n >= 0) { - char *line = (char *) malloct((size_t) n + 1); - char *masked = - line != NULL && vsnprintf(line, (size_t) n + 1, format, args) == n - ? hts_mask_passwords(line, HTS_MASK_LINE) - : NULL; - - // never the raw line, which may hold a password - fputs(masked != NULL ? masked : "(log line lost)", opt->log); - freet(masked); - freet(line); - } + (void) vfprintf(opt->log, format, args); if ((type & LOG_ERRNO) != 0) { char errbuf[HTS_STRERROR_SIZE]; diff --git a/src/htslib.h b/src/htslib.h index a3b547678..33b863204 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -547,16 +547,27 @@ void fprintfio(FILE * fp, const char *buff, const char *prefix); /* What hts_mask_passwords() reads: */ typedef enum { - HTS_MASK_LINE, /* a log line, where an address may lack its scheme */ HTS_MASK_LIST, /* blank-separated start URLs and filters */ HTS_MASK_ADDR /* one start URL, or a -P value */ } hts_mask_mode; -/* Returns a copy of s, allocated with malloct, in which every password the - engine would read out of an address in s is *** . The rest stays byte for - byte. NULL when out of memory. */ +/* Returns a copy of s, allocated with malloct, in which the password of every + start URL or proxy address in s is *** . The rest stays byte for byte. NULL + when out of memory. */ char *hts_mask_passwords(const char *s, hts_mask_mode mode); +/* Returns the engine address adr without its login, for a log line: adr + itself when it has none, else the part after the login. */ +const char *adr_nologin(const char *adr); + +/* Returns url without its login, for a log line: url itself when it has no + scheme or no login, else a copy clipped to size bytes in buf. */ +const char *url_nologin(char *buf, size_t size, const char *url); + +/* url_nologin() into one of the rotating buffers of opt. */ +#define URL_NOLOGIN(OPT, URL) \ + url_nologin(OPT_GET_BUFF(OPT), OPT_GET_BUFF_SIZE(OPT), URL) + /* Writes the marker that stands for a hidden secret of len bytes. */ void fprint_redacted(FILE *fp, size_t len); diff --git a/src/htslib_selftest.c b/src/htslib_selftest.c index 55098b965..a0122edc8 100644 --- a/src/htslib_selftest.c +++ b/src/htslib_selftest.c @@ -1628,10 +1628,10 @@ static int maskpw_pairs(size_t n) { for (size_t i = 1; i < len; i += 2) pairs[i] = '@'; pairs[len] = '\0'; - snprintf(in, len + 32, "404 at h/x?%s", pairs); - err = maskpw_row(in, in, HTS_MASK_LINE); - snprintf(in, len + 32, "at link a%sh/", pairs); - err |= maskpw_row(in, "at link a:***@h/", HTS_MASK_LINE); + snprintf(in, len + 32, "http://h/x?%s", pairs); + err = maskpw_row(in, in, HTS_MASK_LIST); + snprintf(in, len + 32, "a%sh/ -x", pairs); + err |= maskpw_row(in, "a:***@h/ -x", HTS_MASK_LIST); snprintf(in, len + 32, "http://a%sh", pairs); err |= maskpw_row(in, "http://a:***@h", HTS_MASK_ADDR); freet(pairs); @@ -1639,46 +1639,68 @@ static int maskpw_pairs(size_t n) { return err; } +/* Do adr_nologin() and url_nologin() drop the login and nothing else? */ +static int nologin_rows(void) { + static const struct { + const char *in, *want; + } adrs[] = {{"u:p@h:80", "h:80"}, + {"u:p@q@h", "h"}, + {"h:80", "h:80"}, + {"https://h", "https://h"}, + {"https://u:p@h", "h"}}, + urls[] = {{"http://u:p@h/x", "http://h/x"}, + {"//u:p@q@h", "//h"}, + {"HTTPS://u:a b@[::1]:8/", "HTTPS://[::1]:8/"}, + {"http://h/x?a@b", "http://h/x?a@b"}, + {"mailto:a@b", "mailto:a@b"}, + {"u:p@h/x", "u:p@h/x"}, + {"http://u:p@hostname.example/x", "http://hostname.example"}}; + + char buf[24]; + int err = 0; + size_t i; + + for (i = 0; i < sizeof(adrs) / sizeof(adrs[0]); i++) + if (strcmp(adr_nologin(adrs[i].in), adrs[i].want) != 0) { + fprintf(stderr, "adr_nologin: '%s' gave '%s'\n", adrs[i].in, + adr_nologin(adrs[i].in)); + err = 1; + } + for (i = 0; i < sizeof(urls) / sizeof(urls[0]); i++) { + const char *got = url_nologin(buf, sizeof(buf), urls[i].in); + + if (strcmp(got, urls[i].want) != 0) { + fprintf(stderr, "url_nologin: '%s' gave '%s'\n", urls[i].in, got); + err = 1; + } + } + return err; +} + static int st_maskpw(httrackp *opt, int argc, char **argv) { static const struct { hts_mask_mode mode; const char *in, *want; } rows[] = { - {HTS_MASK_LINE, "", ""}, - {HTS_MASK_LINE, "http://u:p@h/", "http://u:***@h/"}, - // the engine logs an http address with no scheme - {HTS_MASK_LINE, "at link u:p@h:80/x (from v:q@g/y)", - "at link u:***@h:80/x (from v:***@g/y)"}, - {HTS_MASK_LINE, "see (u:p@h/x) 'v:q@g/y' \"w:r@f/z\"", - "see (u:***@h/x) 'v:***@g/y' \"w:***@f/z\""}, - {HTS_MASK_LINE, "at link HTTP://u:p@h:80/x", - "at link HTTP://u:***@h:80/x"}, - {HTS_MASK_LINE, "ftp://u:p@h", "ftp://u:***@h"}, - {HTS_MASK_LINE, "x1+a.b-c://u:p@h/", "x1+a.b-c://u:***@h/"}, - {HTS_MASK_LINE, "http://u@h/", "http://u@h/"}, - {HTS_MASK_LINE, "http://:p@h/", "http://:***@h/"}, - {HTS_MASK_LINE, "http://u:@h/", "http://u:***@h/"}, - // the engine splits the userinfo at the last '@' - {HTS_MASK_LINE, "http://u:p@SECRETB@h/", "http://u:***@h/"}, - {HTS_MASK_LINE, "u:p@SECRETB@h/", "u:***@h/"}, - {HTS_MASK_LINE, "http://user:pw;SECRETA@h/", "http://user:***@h/"}, - {HTS_MASK_LINE, "http://u:a,b=c\"d'e(f) h@h/", "http://u:***@h/"}, - {HTS_MASK_LINE, "http://u:a%2Fb@h/", "http://u:***@h/"}, - // a raw '/' ends the authority, so the engine sends no password - {HTS_MASK_LINE, "http://u:a/b@h/", "http://u:a/b@h/"}, - {HTS_MASK_LINE, "http://h1/ http://u:p@h2/ x:y@z", - "http://h1/ http://u:***@h2/ x:y@z"}, - {HTS_MASK_LINE, "http://h/?r=ftp://u:p@g/", "http://h/?r=ftp://u:***@g/"}, - // no address - {HTS_MASK_LINE, "-*:*@*", "-*:*@*"}, - {HTS_MASK_LINE, "+*:*@*/x", "+*:*@*/x"}, - {HTS_MASK_LINE, "http://h/x?t=10:30@x", "http://h/x?t=10:30@x"}, - {HTS_MASK_LINE, "?t=10:30@x", "?t=10:30@x"}, - {HTS_MASK_LINE, "mailto:a:b@c", "mailto:a:b@c"}, - {HTS_MASK_LINE, "://u:p@h/", "://u:p@h/"}, - {HTS_MASK_LINE, "user x:y@z said /", "user x:y@z said /"}, + {HTS_MASK_LIST, "", ""}, {HTS_MASK_LIST, "u:p@h http://v:q@g +*:*@* -*:*@*/x", "u:***@h http://v:***@g +*:*@* -*:*@*/x"}, + {HTS_MASK_LIST, "HTTP://u:p@h:80/x ftp://u:p@h", + "HTTP://u:***@h:80/x ftp://u:***@h"}, + {HTS_MASK_LIST, "http://h1/ http://h/?r=ftp://u:p@g/", + "http://h1/ http://h/?r=ftp://u:p@g/"}, + {HTS_MASK_LIST, "http://h/x?t=10:30@x", "http://h/x?t=10:30@x"}, + {HTS_MASK_ADDR, "http://u:p@h/", "http://u:***@h/"}, + {HTS_MASK_ADDR, "http://u@h/", "http://u@h/"}, + {HTS_MASK_ADDR, "http://:p@h/", "http://:***@h/"}, + {HTS_MASK_ADDR, "http://u:@h/", "http://u:***@h/"}, + // the engine splits the userinfo at the last '@' + {HTS_MASK_ADDR, "http://u:p@SECRETB@h/", "http://u:***@h/"}, + {HTS_MASK_ADDR, "http://user:pw;SECRETA@h/", "http://user:***@h/"}, + {HTS_MASK_ADDR, "http://u:a,b=c\"d'e(f) h@h/", "http://u:***@h/"}, + {HTS_MASK_ADDR, "http://u:a%2Fb@h/", "http://u:***@h/"}, + // a raw '/' ends the authority, so the engine sends no password + {HTS_MASK_ADDR, "http://u:a/b@h/", "http://u:a/b@h/"}, {HTS_MASK_ADDR, "u:p@h:8080", "u:***@h:8080"}, {HTS_MASK_ADDR, "u:p@q@h:8080", "u:***@h:8080"}, {HTS_MASK_ADDR, "u:a b@h/", "u:***@h/"}, @@ -1687,7 +1709,7 @@ static int st_maskpw(httrackp *opt, int argc, char **argv) { {HTS_MASK_ADDR, "u@h:8080", "u@h:8080"}, }; - static const char *const modes[] = {"line", "list", "addr"}; + static const char *const modes[] = {"list", "addr"}; int err = 0; size_t i; @@ -1706,6 +1728,7 @@ static int st_maskpw(httrackp *opt, int argc, char **argv) { } for (i = 0; i < sizeof(rows) / sizeof(rows[0]); i++) err |= maskpw_row(rows[i].in, rows[i].want, rows[i].mode); + err |= nologin_rows(); err |= maskpw_pairs(300); err |= maskpw_pairs(10000); printf("maskpw self-test: %s\n", err ? "FAIL" : "OK"); @@ -3626,8 +3649,9 @@ const struct selftest_entry selftests_lib[] = { "the fields and copies the span primitive makes", st_span}, {"gmtime", "", "hts_gmtime() fills the caller's buffer, not a static (#794)", st_gmtime}, - {"maskpw", "[line|list|addr ]", - "hts-log.txt masks the password of a URL login and nothing else", + {"maskpw", "[list|addr ]", + "the command-line echo masks the password of a URL login and nothing " + "else", st_maskpw}, {"localtime", "", "hts_localtime() and get_filetime_rfc822()'s GMT labelling (#806)", diff --git a/src/htsname.c b/src/htsname.c index 55743ce67..7457f3a58 100644 --- a/src/htsname.c +++ b/src/htsname.c @@ -747,7 +747,7 @@ int url_savename(lien_adrfilsave *const afs, if (r.statuscode != -1) { // cache entry read OK hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s", - adr_complete, fil_complete); + adr_nologin(adr_complete), fil_complete); if (!HTTP_IS_REDIRECT(r.statuscode)) { const sniff_src src = {sback, NULL, adr, fil, previous_save}; @@ -828,7 +828,7 @@ int url_savename(lien_adrfilsave *const afs, current.adr[0] = current.fil[0] = '\0'; opt->state._hts_in_html_parsing = 2; // test hts_log_print(opt, LOG_DEBUG, "Testing link type %s%s", - adr_complete, fil_complete); + adr_nologin(adr_complete), fil_complete); strcpybuff(current.adr, adr_complete); strcpybuff(current.fil, fil_complete); // ajouter dans le backing le fichier en mode test @@ -922,8 +922,10 @@ int url_savename(lien_adrfilsave *const afs, else { methode = BACK_ADD_TEST2; // tester avec GET hts_log_print(opt, LOG_WARNING, - "Loop with HEAD request (during prefetch) at %s%s", - current.adr, current.fil); + "Loop with HEAD request " + "(during prefetch) at %s%s", + adr_nologin(current.adr), + current.fil); } if (!hts_wait_available_socket(sback, opt, cache, ptr)) @@ -932,12 +934,13 @@ int url_savename(lien_adrfilsave *const afs, moved.fil, methode, referer_adr, referer_fil, 1, HTS_FALSE, NULL) != -1) { // OK - hts_log_print(opt, LOG_DEBUG, - "(during prefetch) %s (%d) to link %s at %s%s", - back[b].r.msg, - back[b].r.statuscode, - back[b].r.location, current.adr, - current.fil); + hts_log_print( + opt, LOG_DEBUG, + "(during prefetch) %s (%d) to link %s at " + "%s%s", + back[b].r.msg, back[b].r.statuscode, + URL_NOLOGIN(opt, back[b].r.location), + adr_nologin(current.adr), current.fil); // libérer emplacement backing actuel et attendre le prochain back_maydelete(opt, cache, sback, b); @@ -953,14 +956,17 @@ int url_savename(lien_adrfilsave *const afs, } else { // sinon on fait rien et on s'en va.. // (ftp etc) hts_log_print(opt, LOG_DEBUG, - "Warning: Savename redirect backing error at %s%s", - moved.adr, moved.fil); + "Warning: Savename redirect " + "backing error at %s%s", + adr_nologin(moved.adr), + moved.fil); } } } else { - hts_log_print(opt, LOG_WARNING, - "Unable to test %s%s (loop to same filename)", - adr_complete, fil_complete); + hts_log_print( + opt, LOG_WARNING, + "Unable to test %s%s (loop to same filename)", + adr_nologin(adr_complete), fil_complete); } } @@ -968,7 +974,7 @@ int url_savename(lien_adrfilsave *const afs, } else { // arrêter les frais hts_log_print(opt, LOG_WARNING, "Unable to test %s%s (loop)", - adr_complete, fil_complete); + adr_nologin(adr_complete), fil_complete); } } // ok, leaving } @@ -1020,8 +1026,8 @@ int url_savename(lien_adrfilsave *const afs, printf("error while savename crash adding\n"); #endif hts_log_print(opt, LOG_ERROR, - "Unexpected savename backing error at %s%s", adr, - fil_complete); + "Unexpected savename backing error at %s%s", + adr_nologin(adr), fil_complete); } // restaurer opt->state._hts_in_html_parsing = hihp; @@ -1052,10 +1058,10 @@ int url_savename(lien_adrfilsave *const afs, if ((opt->debug > 1) && (opt->log != NULL)) { if (ext_chg == 1) hts_log_print(opt, LOG_DEBUG, "Changing link extension %s%s to .%s", - adr_complete, fil_complete, ext); + adr_nologin(adr_complete), fil_complete, ext); else hts_log_print(opt, LOG_DEBUG, "Changing link name %s%s to %s", - adr_complete, fil_complete, ext); + adr_nologin(adr_complete), fil_complete, ext); } if (ext_chg == 1) { // Cut the old extension only when it is empty (a bare trailing dot), the @@ -1521,7 +1527,7 @@ int url_savename(lien_adrfilsave *const afs, if (*a != '.') { // agh pas de point strcatbuff(afs->save, ".html"); // préférable! hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s", - adr_complete, fil_complete, afs->save); + adr_nologin(adr_complete), fil_complete, afs->save); } } // effacer pass au besoin pour les autentifications @@ -1666,7 +1672,7 @@ int url_savename(lien_adrfilsave *const afs, } hts_log_print(opt, LOG_DEBUG, "engine: save-name: local name: %s%s -> %s", - adr, fil, afs->save); + adr_nologin(adr), fil, afs->save); /* Ensure that the MANDATORY "temporary" extension is set */ if (ext_chg_delayed) { @@ -1817,13 +1823,13 @@ int url_savename(lien_adrfilsave *const afs, free(saveFinal); } else { hts_log_print(opt, LOG_ERROR, "Could not revert to UTF-8: %s%s", - adr_complete, fil_complete); + adr_nologin(adr_complete), fil_complete); } free(wsave); // log in debug hts_log_print(opt, LOG_DEBUG, "Too long filename shortened: %s%s => %s", - adr_complete, fil_complete, afs->save); + adr_nologin(adr_complete), fil_complete, afs->save); } else { hts_log_print(opt, LOG_ERROR, "Could not read UTF-8: %s", afs->save); } diff --git a/src/htsparse.c b/src/htsparse.c index d3e4074df..b90cf0570 100644 --- a/src/htsparse.c +++ b/src/htsparse.c @@ -892,8 +892,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { char *cAddr = r->adr; int cSize = (int) r->size; - hts_log_print(opt, LOG_DEBUG, "engine: preprocess-html: %s%s", urladr(), - urlfil()); + hts_log_print(opt, LOG_DEBUG, "engine: preprocess-html: %s%s", + adr_nologin(urladr()), urlfil()); if (RUN_CALLBACK4(opt, preprocess, &cAddr, &cSize, urladr(), urlfil()) == 1) { r->adr = cAddr; r->size = cSize; @@ -905,8 +905,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { const char *lastsaved; // adresse du dernier octet sauvé + 1 hts_boolean sf_may_mark = HTS_FALSE; // --single-file: mark this one? - hts_log_print(opt, LOG_DEBUG, "scanning file %s%s (%s)..", urladr(), urlfil(), - savename()); + hts_log_print(opt, LOG_DEBUG, "scanning file %s%s (%s)..", + adr_nologin(urladr()), urlfil(), savename()); /* Hack to avoid NULL char problems with C syntax */ /* Yes, some bogus HTML pages can embed null chars @@ -1897,8 +1897,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { #endif nofollow = 1; // NE PLUS suivre liens dans cette page hts_log_print(opt, LOG_WARNING, - "Link %s%s not scanned (follow robots meta tag)", - urladr(), urlfil()); + "Link %s%s not scanned " + "(follow robots meta tag)", + adr_nologin(urladr()), + urlfil()); } } } @@ -2384,14 +2386,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { // 1: interdit (patcher tout de même adresse) hts_log_print(opt, LOG_DEBUG, "link detected in html (tag): %s", - lien); + URL_NOLOGIN(opt, lien)); // external check if (!RUN_CALLBACK1(opt, linkdetected, lien) || !RUN_CALLBACK2(opt, linkdetected2, lien, intag_start)) { error = 1; // erreur hts_log_print(opt, LOG_ERROR, - "Link %s refused by external wrapper", lien); + "Link %s refused by external wrapper", + URL_NOLOGIN(opt, lien)); } #if HTS_STRIP_DOUBLE_SLASH // supprimer les // en / (sauf pour http://) @@ -2459,7 +2462,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (strchr(lien, ',')) { error = 1; // erreur hts_log_print(opt, LOG_DEBUG, - "link rejected (multiple-archive) %s", lien); + "link rejected (multiple-archive) %s", + URL_NOLOGIN(opt, lien)); } } @@ -2499,7 +2503,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { error = 1; hts_log_print( opt, LOG_DEBUG, - "link rejected (control bytes do not fit) %s", lien); + "link rejected (control bytes do not fit) %s", + URL_NOLOGIN(opt, lien)); } } @@ -2511,8 +2516,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { char *const s = hts_convertStringToUTF8(lien, strlen(lien), charset); if (s != NULL) { hts_log_print(opt, LOG_DEBUG, - "engine: save-name: '%s' charset conversion from '%s' to '%s'", - charset, lien, s); + "engine: save-name: '%s' charset " + "conversion from '%s' to '%s'", + charset, URL_NOLOGIN(opt, lien), + URL_NOLOGIN(opt, s)); /* One byte can become three, and a clipped URL names another resource, so drop the link instead. */ if (strlen(s) < sizeof(lien) - HTS_LINK_TAIL_ROOM) { @@ -2522,7 +2529,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_WARNING, "Link is too long after charset conversion: %s", - lien); + URL_NOLOGIN(opt, lien)); } free(s); } @@ -2531,7 +2538,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { // decode URI entities with UTF-8 charset if (hts_unescapeEntities(lien, lien, strlen(lien) + 1) != 0) { hts_log_print(opt, LOG_WARNING, - "could not decode URI '%s' with charset '%s'", lien, charset); + "could not decode URI '%s' with charset '%s'", + URL_NOLOGIN(opt, lien), charset); } // decode query entities with the page charset, out-of-place @@ -2559,7 +2567,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { strcpybuff(lien, catbuff); } else { hts_log_print(opt, LOG_WARNING, - "could not URL-decode string '%s'", lien); + "could not URL-decode string '%s'", + URL_NOLOGIN(opt, lien)); } // we need to encode query string non-ascii chars, @@ -2576,7 +2585,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { error = 1; hts_log_print(opt, LOG_DEBUG, "link rejected (query does not fit) %s", - lien); + URL_NOLOGIN(opt, lien)); } } } @@ -2666,7 +2675,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { } // si c'est un chemin, alors vérifier (toto/toto.html -> http://www/toto/) if (!error) { - hts_log_print(opt, LOG_DEBUG, "position link check %s", lien); + hts_log_print(opt, LOG_DEBUG, "position link check %s", + URL_NOLOGIN(opt, lien)); if ((p_type == 2) || (p_type == -2)) { // code ou codebase // Vérifier les codebase=applet (au lieu de applet/) @@ -2709,7 +2719,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { error = 1; hts_log_print(opt, LOG_WARNING, "Link is too long once resolved: %s", - lien); + URL_NOLOGIN(opt, lien)); } else { strcpybuff(lien, "http://"); strcatbuff(lien, af2.adr); @@ -2762,9 +2772,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { } break; // base } - hts_log_print(opt, LOG_DEBUG, - "code/codebase link %s base %s", lien, - base); + hts_log_print( + opt, LOG_DEBUG, "code/codebase link %s base %s", + URL_NOLOGIN(opt, lien), URL_NOLOGIN(opt, base)); } } else { @@ -2792,15 +2802,16 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { strcpybuff(lien, tempo); // patcher en considérant base // ** vérifier que ../ fonctionne (ne doit pas arriver mais bon..) - hts_log_print(opt, LOG_DEBUG, - "link modified with code/codebase %s", - lien); + hts_log_print( + opt, LOG_DEBUG, + "link modified with code/codebase %s", + URL_NOLOGIN(opt, lien)); } } else { error = 1; // erreur hts_log_print(opt, LOG_ERROR, "Link %s too long with base href", - lien); + URL_NOLOGIN(opt, lien)); } } else { lien_adrfil baseaf; @@ -2817,14 +2828,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { strcatbuff(tempo, lien); strcpybuff(lien, tempo); // patcher en considérant base - hts_log_print(opt, LOG_DEBUG, - "link modified with code/codebase %s", - lien); + hts_log_print( + opt, LOG_DEBUG, + "link modified with code/codebase %s", + URL_NOLOGIN(opt, lien)); } else { error = 1; // erreur hts_log_print(opt, LOG_ERROR, "Link %s too long with base href", - lien); + URL_NOLOGIN(opt, lien)); } } } @@ -2838,9 +2850,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (!error) { int reponse; - hts_log_print(opt, LOG_DEBUG, - "build relative link %s with %s%s", lien, - relativeurladr(), relativeurlfil()); + hts_log_print( + opt, LOG_DEBUG, "build relative link %s with %s%s", + URL_NOLOGIN(opt, lien), adr_nologin(relativeurladr()), + relativeurlfil()); if ((reponse = ident_url_relatif(lien, relativeurladr(), relativeurlfil(), &afs.af)) < 0) { @@ -2849,22 +2862,25 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print(opt, LOG_WARNING, "Link %s not caught (protocol not " "supported by this build)", - lien); + URL_NOLOGIN(opt, lien)); } else { hts_log_print(opt, LOG_DEBUG, "ident_url_relatif failed for %s with %s%s", - lien, relativeurladr(), relativeurlfil()); + URL_NOLOGIN(opt, lien), + adr_nologin(relativeurladr()), + relativeurlfil()); } } else { - hts_log_print(opt, LOG_DEBUG, - "built relative link %s with %s%s -> %s%s", - lien, relativeurladr(), relativeurlfil(), afs.af.adr, - afs.af.fil); + hts_log_print( + opt, LOG_DEBUG, + "built relative link %s with %s%s -> %s%s", + URL_NOLOGIN(opt, lien), adr_nologin(relativeurladr()), + relativeurlfil(), adr_nologin(afs.af.adr), afs.af.fil); } } else { hts_log_print(opt, LOG_DEBUG, "link %s not build, error detected before", - lien); + URL_NOLOGIN(opt, lien)); afs.af.adr[0] = '\0'; } @@ -2873,7 +2889,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (p_nocatch) { forbidden_url = 1; // interdire récupération du lien hts_log_print(opt, LOG_DEBUG, "link forced external at %s%s", - afs.af.adr, afs.af.fil); + adr_nologin(afs.af.adr), afs.af.fil); } // Tester si un lien doit être accepté ou refusé (wizard) // forbidden_url=1 : lien refusé @@ -2882,7 +2898,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (!p_nocatch) { if (afs.af.adr[0] != '\0') { hts_log_print(opt, LOG_DEBUG, - "wizard link test at %s%s..", afs.af.adr, afs.af.fil); + "wizard link test at %s%s..", + adr_nologin(afs.af.adr), afs.af.fil); forbidden_url = hts_acceptlink(opt, ptr, afs.af.adr, afs.af.fil, intag_name ? intag_name : NULL, @@ -2943,9 +2960,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if ((ptr > 0) && (p_type != 2) && (p_type != -2)) { // tester autorisations? if (!p_nocatch) { if (afs.af.adr[0] != '\0') { - hts_log_print(opt, LOG_DEBUG, - "wizard moved link retest at %s%s..", - afs.af.adr, afs.af.fil); + hts_log_print( + opt, LOG_DEBUG, + "wizard moved link retest at %s%s..", + adr_nologin(afs.af.adr), afs.af.fil); forbidden_url = hts_acceptlink( opt, ptr, afs.af.adr, afs.af.fil, intag_name ? intag_name : NULL, @@ -3008,23 +3026,25 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (forbidden_url != 1) { // le lien va être chargé if ((p_type == 2) || (p_type == -2)) { // base href ou codebase, pas un lien hts_log_print(opt, LOG_DEBUG, "Code/Codebase: %s%s", - afs.af.adr, afs.af.fil); + adr_nologin(afs.af.adr), afs.af.fil); } else if ((opt->getmode & HTS_GETMODE_HTML_FIRST) == 0) { hts_log_print(opt, LOG_DEBUG, "Record: %s%s -> %s", - afs.af.adr, afs.af.fil, afs.save); + adr_nologin(afs.af.adr), afs.af.fil, + afs.save); } else { if (!ishtml(opt, afs.af.fil)) - hts_log_print(opt, LOG_DEBUG, - "Record after: %s%s -> %s", afs.af.adr, afs.af.fil, - afs.save); + hts_log_print( + opt, LOG_DEBUG, "Record after: %s%s -> %s", + adr_nologin(afs.af.adr), afs.af.fil, afs.save); else hts_log_print(opt, LOG_DEBUG, "Record: %s%s -> %s", - afs.af.adr, afs.af.fil, afs.save); + adr_nologin(afs.af.adr), afs.af.fil, + afs.save); } } else - hts_log_print(opt, LOG_DEBUG, "External: %s%s", afs.af.adr, - afs.af.fil); + hts_log_print(opt, LOG_DEBUG, "External: %s%s", + adr_nologin(afs.af.adr), afs.af.fil); } /* FIN log */ @@ -3409,9 +3429,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (ending_p == ')') escape_url_parens(tempo, sizeof(tempo)); } - hts_log_print(opt, LOG_DEBUG, - "relative link at %s build with %s and %s: %s", - afs.af.adr, afs.save, relativesavename(), tempo); + hts_log_print( + opt, LOG_DEBUG, + "relative link at %s build with %s and %s: %s", + adr_nologin(afs.af.adr), afs.save, + relativesavename(), tempo); // lien applet (code) - il faut placer un codebase avant if (p_type == -1) { // que le nom de fichier @@ -3576,10 +3598,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if ((opt->debug > 1) && (opt->log != NULL)) { if (strcmp(afs.af.adr, heap(i)->adr) != 0 || strcmp(afs.af.fil, heap(i)->fil) != 0) { - hts_log_print(opt, LOG_DEBUG, - "merging similar links %s%s and %s%s", - afs.af.adr, afs.af.fil, heap(i)->adr, - heap(i)->fil); + hts_log_print( + opt, LOG_DEBUG, + "merging similar links %s%s and %s%s", + adr_nologin(afs.af.adr), afs.af.fil, + adr_nologin(heap(i)->adr), heap(i)->fil); } } heap(i)->depth = @@ -3633,7 +3656,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { #endif hts_log_print(opt, LOG_DEBUG, "robots.txt added at %s", - afs.af.adr); + adr_nologin(afs.af.adr)); } else { hts_log_print(opt, LOG_ERROR, "Unexpected robots.txt error at %d", @@ -3690,12 +3713,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (!just_test_it) { hts_log_print(opt, LOG_DEBUG, "OK, NOTE: %s%s -> %s", - heap_top()->adr, - heap_top()->fil, - heap_top()->sav); + adr_nologin(heap_top()->adr), + heap_top()->fil, heap_top()->sav); } else { hts_log_print(opt, LOG_DEBUG, "OK, TEST: %s%s", - heap_top()->adr, + adr_nologin(heap_top()->adr), heap_top()->fil); } @@ -3851,7 +3873,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { int cSize = (int) TypedArraySize(output_buffer); hts_log_print(opt, LOG_DEBUG, "engine: postprocess-html: %s%s", - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); if (RUN_CALLBACK4(opt, postprocess, &cAddr, &cSize, urladr(), urlfil()) == 1) { hts_log_print(opt, LOG_DEBUG, "engine: postprocess-html: callback modified data, applying %d bytes", cSize); @@ -3866,7 +3888,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print(opt, LOG_ERROR, "engine: postprocess-html: callback returned %d " "bytes for %s%s, page skipped", - cSize, urladr(), urlfil()); + cSize, adr_nologin(urladr()), urlfil()); HT_SET_SIZE(0); error = 1; } else if (hts_postprocess_reply_inplace( @@ -3885,7 +3907,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (output_oom) { hts_log_print(opt, LOG_ERROR, "Not enough memory to rewrite %s%s, page skipped", - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); error = 1; } else if (TypedArraySize(output_buffer) != 0) { hts_finish_html_file( @@ -4023,7 +4045,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str, // ------------------------------------------------------------ if (HTTP_IS_REDIRECT(r->statuscode)) { - hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, urladr(), urlfil()); + hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, + adr_nologin(urladr()), urlfil()); { char BIGSTK mov_url[HTS_URLMAXSIZE * 2]; @@ -4058,7 +4081,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str, get_it = -1; // ne rien faire hts_log_print(opt, LOG_WARNING, "Can not bear crazy server (%s) for %s%s", r->msg, - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); } else { // mauvaise casse, effacer entrée dans la pile et rejouer une fois get_it = 1; } @@ -4067,14 +4090,14 @@ int hts_mirror_check_moved(htsmoduleStruct * str, // -> on prend à cette adresse, le lien sera enregistré avec lien_record() (hash) hts_log_print(opt, LOG_DEBUG, "wizard link test for moved file at %s%s..", - moved->adr, moved->fil); + adr_nologin(moved->adr), moved->fil); // accepté? if (hts_acceptlink(opt, ptr, moved->adr, moved->fil, NULL, NULL, HTS_FALSE, &set_prio_to, NULL) != 1) { /* nouvelle adresse non refusée ? */ get_it = 1; hts_log_print(opt, LOG_DEBUG, "moved link accepted: %s%s", - moved->adr, moved->fil); + adr_nologin(moved->adr), moved->fil); } } else if (same_savefile) { // A stub would point at itself; follow the redirect instead. @@ -4084,7 +4107,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str, hts_log_print(opt, LOG_WARNING, "Redirect to a same-file alias, fetching real " "content: %s%s -> %s%s", - urladr(), urlfil(), moved->adr, moved->fil); + adr_nologin(urladr()), urlfil(), + adr_nologin(moved->adr), moved->fil); } } /* sinon traité normalement */ } @@ -4094,7 +4118,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str, // et reculer pour mieux sauter hts_log_print(opt, LOG_WARNING, "Warning moved treated for %s%s (real one is %s%s)", - urladr(), urlfil(), moved->adr, moved->fil); + adr_nologin(urladr()), urlfil(), + adr_nologin(moved->adr), moved->fil); // canceller lien actuel error = 1; hts_invalidate_link(opt, ptr); // invalidate hashtable entry @@ -4153,7 +4178,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str, char *const rn = (char *) malloct(rn_size); if (rn != NULL) { hts_log_print(opt, LOG_WARNING, "File has moved from %s%s to %s", - urladr(), urlfil(), mov_url); + adr_nologin(urladr()), urlfil(), + URL_NOLOGIN(opt, mov_url)); if (!opt->mimehtml) { inplace_escape_uri(mov_url, sizeof(mov_url)); } else { @@ -4226,7 +4252,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str, // gone; a failed removal gives up rather than looping. if (had_partial && ref_gone && !fexist_utf8(heap(ptr)->sav)) { hts_log_print(opt, LOG_DEBUG, "Partial file reget (%s) for %s%s", - r->msg, urladr(), urlfil()); + r->msg, adr_nologin(urladr()), urlfil()); if (hts_record_link(opt, heap(ptr)->adr, heap(ptr)->fil, heap(ptr)->sav, "", "", NULL)) { heap_top()->testmode = heap(ptr)->testmode; heap_top()->link_import = 0; @@ -4246,7 +4272,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str, } else { hts_log_print(opt, LOG_WARNING, "Giving up on partial reget (%s) for %s%s", r->msg, - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); error = 1; } @@ -4268,11 +4294,12 @@ int hts_mirror_check_moved(htsmoduleStruct * str, if (opt->hostcontrol) { // timeout et retry épuisés if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) && (heap(ptr)->retry <= 0)) { - hts_log_print(opt, LOG_DEBUG, "Link banned: %s%s", urladr(), urlfil()); + hts_log_print(opt, LOG_DEBUG, "Link banned: %s%s", + adr_nologin(urladr()), urlfil()); host_ban(opt, ptr, sback, jump_identification_const(urladr())); hts_log_print(opt, LOG_DEBUG, - "Info: previous log - link banned: %s%s", urladr(), - urlfil()); + "Info: previous log - link banned: %s%s", + adr_nologin(urladr()), urlfil()); } else can_retry = 1; } else @@ -4281,11 +4308,12 @@ int hts_mirror_check_moved(htsmoduleStruct * str, case STATUSCODE_SLOW: if ((opt->hostcontrol) && (heap(ptr)->retry <= 0)) { // too slow if (opt->hostcontrol & HTS_HOSTCONTROL_BAN_SLOW) { - hts_log_print(opt, LOG_DEBUG, "Link banned: %s%s", urladr(), urlfil()); + hts_log_print(opt, LOG_DEBUG, "Link banned: %s%s", + adr_nologin(urladr()), urlfil()); host_ban(opt, ptr, sback, jump_identification_const(urladr())); hts_log_print(opt, LOG_DEBUG, - "Info: previous log - link banned: %s%s", urladr(), - urlfil()); + "Info: previous log - link banned: %s%s", + adr_nologin(urladr()), urlfil()); } else can_retry = 1; } else @@ -4326,26 +4354,28 @@ int hts_mirror_check_moved(htsmoduleStruct * str, // give up: no free restart left, and no budget (or nothing to retry) if (!restart_whole && ((heap(ptr)->retry <= 0) || (!can_retry))) { if ((opt->retry > 0) && (can_retry)) { - hts_log_print(opt, LOG_ERROR, - "\"%s\" (%d) after %d retries at link %s%s (from %s%s)", - r->msg, r->statuscode, opt->retry, urladr(), urlfil(), - heap(heap(ptr)->precedent)->adr, - heap(heap(ptr)->precedent)->fil); + hts_log_print( + opt, LOG_ERROR, + "\"%s\" (%d) after %d retries at link %s%s (from %s%s)", r->msg, + r->statuscode, opt->retry, adr_nologin(urladr()), urlfil(), + adr_nologin(heap(heap(ptr)->precedent)->adr), + heap(heap(ptr)->precedent)->fil); } else { if (r->statuscode == STATUSCODE_TEST_OK) { // test OK hts_log_print(opt, LOG_INFO, "Test OK at link %s%s (from %s%s)", - urladr(), urlfil(), heap(heap(ptr)->precedent)->adr, + adr_nologin(urladr()), urlfil(), + adr_nologin(heap(heap(ptr)->precedent)->adr), heap(heap(ptr)->precedent)->fil); } else { if (strcmp(urlfil(), "/robots.txt")) { // ne pas afficher d'infos sur robots.txt par défaut hts_log_print(opt, LOG_ERROR, "\"%s\" (%d) at link %s%s (from %s%s)", r->msg, - r->statuscode, urladr(), urlfil(), - heap(heap(ptr)->precedent)->adr, + r->statuscode, adr_nologin(urladr()), urlfil(), + adr_nologin(heap(heap(ptr)->precedent)->adr), heap(heap(ptr)->precedent)->fil); } else { hts_log_print(opt, LOG_DEBUG, "No robots.txt rules at %s", - urladr()); + adr_nologin(urladr())); } } } @@ -4403,14 +4433,16 @@ int hts_mirror_check_moved(htsmoduleStruct * str, hts_log_print(opt, LOG_NOTICE, "Restarting whole file after error %d (%s) at link " "%s%s (from %s%s)", - r->statuscode, r->msg, urladr(), urlfil(), - heap(heap(ptr)->precedent)->adr, + r->statuscode, r->msg, adr_nologin(urladr()), + urlfil(), + adr_nologin(heap(heap(ptr)->precedent)->adr), heap(heap(ptr)->precedent)->fil); } else { hts_log_print(opt, LOG_NOTICE, "Retry after error %d (%s) at link %s%s (from %s%s)", - r->statuscode, r->msg, urladr(), urlfil(), - heap(heap(ptr)->precedent)->adr, + r->statuscode, r->msg, adr_nologin(urladr()), + urlfil(), + adr_nologin(heap(heap(ptr)->precedent)->adr), heap(heap(ptr)->precedent)->fil); } // re-request the file @@ -4431,8 +4463,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str, } } } else { - hts_log_print(opt, LOG_DEBUG, "Info: no robots.txt at %s%s", urladr(), - urlfil()); + hts_log_print(opt, LOG_DEBUG, "Info: no robots.txt at %s%s", + adr_nologin(urladr()), urlfil()); } if (!store_errpage) { if (r->adr) { // désalloc @@ -4592,8 +4624,8 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str, that name is the bug. */ if (forbidden_url == 1 || IS_DELAYED_EXT(add.save)) { hts_log_print(opt, LOG_NOTICE, - "Link %s%s not added after user request", add.af.adr, - add.af.fil); + "Link %s%s not added after user request", + adr_nologin(add.af.adr), add.af.fil); } else if (hash_read(hash, add.save, NULL, HASH_STRUCT_FILENAME) < 0) { // no existing record // record the link @@ -4607,8 +4639,8 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str, heap_top()->premier = heap_top_index(); heap_top()->precedent = heap_top_index(); // - hts_log_print(opt, LOG_INFO, "Link added by user: %s%s", add.af.adr, - add.af.fil); + hts_log_print(opt, LOG_INFO, "Link added by user: %s%s", + adr_nologin(add.af.adr), add.af.fil); // } else if (hts_maxlinks_no_room(opt)) { /* Give the parser's verdict rather than dropping the rest of @@ -4623,12 +4655,12 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str, } else { hts_log_print(opt, LOG_NOTICE, "Existing link %s%s not added after user request", - add.af.adr, add.af.fil); + adr_nologin(add.af.adr), add.af.fil); } } } else { hts_log_print(opt, LOG_ERROR, "Error during URL decoding for %s", - add_url); + URL_NOLOGIN(opt, add_url)); } // ----Fin Ajout---- } @@ -4692,8 +4724,8 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, /* This is not supposed to hapen. */ if (heap(ptr)->pass2 == -1) { - hts_log_print(opt, LOG_WARNING, "Link is already ready %s%s", urladr(), - urlfil()); + hts_log_print(opt, LOG_WARNING, "Link is already ready %s%s", + adr_nologin(urladr()), urlfil()); } /* User interaction */ @@ -4705,7 +4737,8 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, /* Done while processing user interactions ? */ if (heap(ptr)->pass2 == -1) { - hts_log_print(opt, LOG_DEBUG, "Link is now ready %s%s", urladr(), urlfil()); + hts_log_print(opt, LOG_DEBUG, "Link is now ready %s%s", + adr_nologin(urladr()), urlfil()); // We are ready return 2; // goto jump_if_done; } @@ -4724,7 +4757,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, printf("error while crash adding\n"); #endif hts_log_print(opt, LOG_ERROR, "Unexpected backing error for %s%s", - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); } } #if BDEBUG==1 @@ -4974,7 +5007,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, if (b < 0) { hts_log_print(opt, LOG_DEBUG, "link #%d is ready, no more on the stack, skipping: %s%s..", - ptr, urladr(), urlfil()); + ptr, adr_nologin(urladr()), urlfil()); return 2; // goto jump_if_done; } @@ -5088,8 +5121,8 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, char BIGSTK cookie_before[16384]; /* #15: this URL's Cookie header, pre-request */ - hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", afs->af.adr, - afs->af.fil); + hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", + adr_nologin(afs->af.adr), afs->af.fil); /* Follow while type is unknown and redirects occurs */ for(loops = 0, continue_loop = 1; @@ -5132,9 +5165,10 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, new_forbidden_url); if (new_forbidden_url == 1) { *forbidden_url = new_forbidden_url; - hts_log_print(opt, LOG_DEBUG, - "link forbidden because of MIME types restrictions: %s%s", - afs->af.adr, afs->af.fil); + hts_log_print( + opt, LOG_DEBUG, + "link forbidden because of MIME types restrictions: %s%s", + adr_nologin(afs->af.adr), afs->af.fil); break; // exit loop } } @@ -5148,9 +5182,10 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, /* Check if the file was recorded already (necessary for redirects) */ if (hash_read(hash, afs->save, NULL, HASH_STRUCT_FILENAME) >= 0) { if (loops == 0) { /* Should not happend */ - hts_log_print(opt, LOG_ERROR, - "Duplicate entry in hts_wait_delayed() cancelled: %s%s -> %s", - afs->af.adr, afs->af.fil, afs->save); + hts_log_print( + opt, LOG_ERROR, + "Duplicate entry in hts_wait_delayed() cancelled: %s%s -> %s", + adr_nologin(afs->af.adr), afs->af.fil, afs->save); } /* Exit loop (we're done) */ continue_loop = 0; @@ -5208,9 +5243,10 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, *forbidden_url); if (new_forbidden_url == 1) { *forbidden_url = new_forbidden_url; - hts_log_print(opt, LOG_DEBUG, - "link forbidden because of MIME types restrictions: %s%s", - afs->af.adr, afs->af.fil); + hts_log_print( + opt, LOG_DEBUG, + "link forbidden because of MIME types restrictions: %s%s", + adr_nologin(afs->af.adr), afs->af.fil); break; // exit loop } } @@ -5338,7 +5374,8 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, if (ident_url_relatif(mov_url, afs->af.adr, afs->af.fil, &moved) >= 0) { hts_log_print(opt, LOG_DEBUG, "Redirect while resolving type: %s%s -> %s%s", - afs->af.adr, afs->af.fil, moved.adr, moved.fil); + adr_nologin(afs->af.adr), afs->af.fil, + adr_nologin(moved.adr), moved.fil); // si non bouclage sur soi même, ou si test avec GET non testé if (strcmp(moved.adr, afs->af.adr) != 0 || strcmp(moved.fil, afs->af.fil) != 0) { @@ -5360,8 +5397,10 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, * this error, and we'll only issue ONE request */ *forbidden_url = 1; /* Forbidden! */ hts_log_print(opt, LOG_DEBUG, - "link forbidden because of redirect beyond the mirror scope at %s%s -> %s%s", - afs->af.adr, afs->af.fil, moved.adr, moved.fil); + "link forbidden because of redirect beyond " + "the mirror scope at %s%s -> %s%s", + adr_nologin(afs->af.adr), afs->af.fil, + adr_nologin(moved.adr), moved.fil); strcpybuff(afs->af.adr, moved.adr); strcpybuff(afs->af.fil, moved.fil); mov_url[0] = '\0'; @@ -5402,12 +5441,13 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, } else { hts_log_print(opt, LOG_WARNING, "Unable to test %s%s (loop to same filename)", - afs->af.adr, afs->af.fil); + adr_nologin(afs->af.adr), afs->af.fil); } // loop to same location } // ident_url_relatif() } // location } // redirect - hts_log_print(opt, LOG_DEBUG, "Final type for %s%s: '%s'", afs->af.adr, afs->af.fil, + hts_log_print(opt, LOG_DEBUG, "Final type for %s%s: '%s'", + adr_nologin(afs->af.adr), afs->af.fil, delayed_back.r.contenttype); /* If we are done, do additional checks with final type and authorizations */ @@ -5435,9 +5475,10 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, *forbidden_url); if (new_forbidden_url == 1) { *forbidden_url = new_forbidden_url; - hts_log_print(opt, LOG_DEBUG, - "link forbidden because of MIME types restrictions: %s%s", - afs->af.adr, afs->af.fil); + hts_log_print( + opt, LOG_DEBUG, + "link forbidden because of MIME types restrictions: %s%s", + adr_nologin(afs->af.adr), afs->af.fil); break; // exit loop } } @@ -5480,11 +5521,13 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, if (in_error == STATUSCODE_TOO_BIG) { hts_log_print(opt, LOG_INFO, "link not taken because of its size (%d bytes) at %s%s", - (int) in_error_size, afs->af.adr, afs->af.fil); + (int) in_error_size, adr_nologin(afs->af.adr), + afs->af.fil); } else { hts_log_print(opt, LOG_INFO, "link not taken because of error (%d '%s') at %s%s", - in_error, in_error_msg, afs->af.adr, afs->af.fil); + in_error, in_error_msg, adr_nologin(afs->af.adr), + afs->af.fil); } } } @@ -5494,15 +5537,16 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, if (cancelled) { hts_log_print(opt, LOG_DEBUG, "link cancelled, type unknown, left online: %s%s", - afs->af.adr, afs->af.fil); + adr_nologin(afs->af.adr), afs->af.fil); } else if (in_error) { hts_log_print(opt, LOG_WARNING, "link in error (%d '%s'), type unknown, aborting: %s%s", - in_error, in_error_msg, afs->af.adr, afs->af.fil); + in_error, in_error_msg, adr_nologin(afs->af.adr), + afs->af.fil); } else { hts_log_print(opt, LOG_WARNING, "link is probably looping, type unknown, aborting: %s%s", - afs->af.adr, afs->af.fil); + adr_nologin(afs->af.adr), afs->af.fil); } } diff --git a/src/htsrobots.c b/src/htsrobots.c index 87c34d5f3..e787244ea 100644 --- a/src/htsrobots.c +++ b/src/htsrobots.c @@ -258,8 +258,8 @@ void robots_parse(httrackp *opt, robots_wizard *robots, const char *adr, hts_log_print(opt, LOG_WARNING, "robots.txt for %s: %d rule(s) not honoured as written, " "starting with '%s' (rules kept up to %d bytes, lines to %d)", - adr, ndropped, dropped, (int) HTS_ROBOTS_MAX_TOKEN_SIZE, - (int) HTS_ROBOTS_LINE_SIZE); + adr_nologin(adr), ndropped, dropped, + (int) HTS_ROBOTS_MAX_TOKEN_SIZE, (int) HTS_ROBOTS_LINE_SIZE); if (StringNotEmpty(blob)) checkrobots_set(robots, adr, StringBuff(blob)); StringFree(blob); diff --git a/src/htssitemap.c b/src/htssitemap.c index 99f5f8011..245de1cca 100644 --- a/src/htssitemap.c +++ b/src/htssitemap.c @@ -292,7 +292,7 @@ static hts_boolean sitemap_fetch_allowed(httrackp *opt, const char *adr, refused = (jok == -1) ? HTS_TRUE : HTS_FALSE; if (refused) { hts_log_print(opt, LOG_NOTICE, "Sitemap: filter rule #%d refuses %s%s", - jokdepth + 1, adr, fil); + jokdepth + 1, adr_nologin(adr), fil); return HTS_FALSE; } /* A Sitemap: line, or a sitemapindex entry, is the site inviting the fetch; @@ -301,8 +301,8 @@ static hts_boolean sitemap_fetch_allowed(httrackp *opt, const char *adr, if (src == HTS_SITEMAP_SRC_GUESSED && hts_robots_forbids(opt, adr, fil, (jok != 0) ? HTS_TRUE : HTS_FALSE, refused)) { - hts_log_print(opt, LOG_NOTICE, "Sitemap: robots.txt forbids %s%s", adr, - fil); + hts_log_print(opt, LOG_NOTICE, "Sitemap: robots.txt forbids %s%s", + adr_nologin(adr), fil); return HTS_FALSE; } return HTS_TRUE; @@ -319,8 +319,8 @@ static hts_boolean sitemap_queue_(httrackp *opt, const char *adr, if (st == NULL) return HTS_FALSE; if (st->ndocs >= HTS_SITEMAP_MAX_DOCS || level > HTS_SITEMAP_MAX_LEVEL) { - hts_log_print(opt, LOG_WARNING, "Sitemap: cap reached, skipping %s%s", adr, - fil); + hts_log_print(opt, LOG_WARNING, "Sitemap: cap reached, skipping %s%s", + adr_nologin(adr), fil); return HTS_FALSE; } if (strlen(adr) >= sizeof(d->adr) || strlen(fil) >= sizeof(d->fil)) @@ -351,7 +351,7 @@ static hts_boolean sitemap_queue_(httrackp *opt, const char *adr, heap_top()->retry = opt->retry; heap_top()->premier = heap_top_index(); heap_top()->precedent = heap_top_index(); - hts_log_print(opt, LOG_INFO, "Sitemap: queued %s%s", adr, fil); + hts_log_print(opt, LOG_INFO, "Sitemap: queued %s%s", adr_nologin(adr), fil); return HTS_TRUE; } @@ -370,8 +370,8 @@ void hts_sitemap_redirect(httrackp *opt, const char *adr, const char *fil, d->done = HTS_TRUE; /* the body lives at the target now */ /* The engine already queued the target link, so only the marking moves. */ (void) sitemap_queue_(opt, newadr, newfil, d->level, d->src, HTS_FALSE); - hts_log_print(opt, LOG_NOTICE, "Sitemap: %s%s redirects to %s%s", adr, fil, - newadr, newfil); + hts_log_print(opt, LOG_NOTICE, "Sitemap: %s%s redirects to %s%s", + adr_nologin(adr), fil, adr_nologin(newadr), newfil); } void hts_sitemap_seed(httrackp *opt, const char *starturl) { @@ -384,7 +384,8 @@ void hts_sitemap_seed(httrackp *opt, const char *starturl) { } else { strcpybuff(url, StringBuff(opt->sitemap_url)); if (strstr(url, ":/") == NULL) - hts_log_print(opt, LOG_ERROR, "Sitemap URL must be absolute: %s", url); + hts_log_print(opt, LOG_ERROR, "Sitemap URL must be absolute: %s", + URL_NOLOGIN(opt, url)); else if (ident_url_absolute(url, &af) >= 0) (void) sitemap_queue(opt, af.adr, af.fil, 0, HTS_SITEMAP_SRC_USER); } @@ -458,7 +459,7 @@ void hts_sitemap_robots(httrackp *opt, const char *adr, const char *sitemaps) { queued++; } hts_log_print(opt, LOG_NOTICE, "Sitemap: %d sitemap(s) queued for %s", queued, - adr); + adr_nologin(adr)); } hts_boolean hts_sitemap_pending(httrackp *opt, const char *adr, @@ -522,8 +523,8 @@ static hts_boolean sitemap_seed_child(void *arg, const char *url) { return HTS_TRUE; if (!strfield2(af.adr, c->adr)) { hts_log_print(c->opt, LOG_WARNING, - "Sitemap: ignoring off-host child sitemap %s%s", af.adr, - af.fil); + "Sitemap: ignoring off-host child sitemap %s%s", + adr_nologin(af.adr), af.fil); return HTS_TRUE; } if (sitemap_queue(c->opt, af.adr, af.fil, c->level + 1, @@ -574,17 +575,17 @@ void hts_sitemap_ingest(httrackp *opt, htsmoduleStruct *str, const char *adr, sitemap_seed_any, &ctx); heap(anchor)->depth = saved_depth; if (n < 0) { - hts_log_print(opt, LOG_ERROR, "Sitemap: could not decompress %s%s", adr, - fil); + hts_log_print(opt, LOG_ERROR, "Sitemap: could not decompress %s%s", + adr_nologin(adr), fil); return; } if (ctx.is_index) hts_log_print(opt, LOG_NOTICE, "Sitemap: %d of %d child sitemap(s) listed by %s%s", - ctx.accepted, n, adr, fil); + ctx.accepted, n, adr_nologin(adr), fil); else hts_log_print(opt, LOG_NOTICE, "Sitemap: %d of %d URL(s) added from %s%s", - ctx.accepted, n, adr, fil); + ctx.accepted, n, adr_nologin(adr), fil); } void hts_sitemap_free(httrackp *opt) { diff --git a/src/htswizard.c b/src/htswizard.c index 4c9d19933..b70f5b19d 100644 --- a/src/htswizard.c +++ b/src/htswizard.c @@ -409,7 +409,7 @@ void hts_wizard_apply_verdict(httrackp *opt, int n, const char *adr, hts_log_print(opt, LOG_WARNING, "(wizard) could not read your answer at %s%s: link refused, " "and the refusal recorded", - adr, fil); + adr_nologin(adr), fil); break; case 5: /* this directory and below, or the whole host */ @@ -424,7 +424,7 @@ void hts_wizard_apply_verdict(httrackp *opt, int n, const char *adr, hts_log_print(opt, LOG_WARNING, "(wizard) unknown answer %d at %s%s, keeping the computed " "verdict", - n, adr, fil); + n, adr_nologin(adr), fil); break; } if (hts_wizard_scope_answer(n) == HTS_TRUE) @@ -436,7 +436,7 @@ void hts_wizard_apply_verdict(httrackp *opt, int n, const char *adr, hts_log_print(opt, LOG_WARNING, "(wizard) %s has no domain above it, answer %d applied " "to the whole host", - adr, n); + adr_nologin(adr), n); } break; } @@ -486,7 +486,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, // -------------------- PHASE 0 -------------------- /* Infos */ - hts_log_print(opt, LOG_DEBUG, "wizard test begins: %s%s", adr, fil); + hts_log_print(opt, LOG_DEBUG, "wizard test begins: %s%s", adr_nologin(adr), + fil); /* Already exists? Then, we know that we knew that this link had to be known */ if (adr[0] != '\0' && fil[0] != '\0' && opt->hash != NULL @@ -512,8 +513,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if ((opt->getmode & HTS_GETMODE_NONHTML) == 0) { // non on ne doit pas if (!ishtml(opt, fil)) { // non il ne faut pas forbidden_url = 1; // interdire récupération du lien - hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", adr, - fil); + hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", + adr_nologin(adr), fil); } } @@ -523,7 +524,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, || (heap(ptr)->depth <= 1 && !embedded_triggered)) { forbidden_url = 1; // interdire récupération du lien hts_log_print(opt, LOG_DEBUG, - "file from too far level ignored at %s : %s", adr, fil); + "file from too far level ignored at %s : %s", + adr_nologin(adr), fil); } } @@ -541,9 +543,11 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, hts_host_same_alias(hts_host_alias_rules(opt), adr, urladr(), hts_host_alias_collapse_www(opt)); if (meme_adresse) - hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s=%s", adr, urladr()); + hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s=%s", adr_nologin(adr), + adr_nologin(urladr())); else - hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s!=%s", adr, urladr()); + hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s!=%s", adr_nologin(adr), + adr_nologin(urladr())); if (meme_adresse) { // même adresse { // tester interdiction de descendre // MODIFIE : en cas de remontée puis de redescente, il se pouvait qu'on ne puisse pas atteindre certains fichiers @@ -582,7 +586,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "same level link authorized: %s%s", - adr, fil); + adr_nologin(adr), fil); } } // down @@ -601,13 +605,13 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if ((test1) && (test2)) { // on ne peut que descendre if ((opt->seeker & HTS_SEEKER_DOWN) == 0) { forbidden_url = 1; - hts_log_print(opt, LOG_DEBUG, "lower link canceled: %s%s", adr, - fil); + hts_log_print(opt, LOG_DEBUG, "lower link canceled: %s%s", + adr_nologin(adr), fil); } else { // autorisé à priori - NEW if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "lower link authorized: %s%s", - adr, fil); + adr_nologin(adr), fil); } } } else if ((test1) || (test2)) { // on peut descendre pour accéder au lien @@ -615,7 +619,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "lower link authorized: %s%s", - adr, fil); + adr_nologin(adr), fil); } } } @@ -626,13 +630,13 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if ((opt->seeker & HTS_SEEKER_UP) == 0) { forbidden_url = 1; upper_refused = 1; - hts_log_print(opt, LOG_DEBUG, "upper link canceled: %s%s", adr, - fil); + hts_log_print(opt, LOG_DEBUG, "upper link canceled: %s%s", + adr_nologin(adr), fil); } else { // autorisé à monter - NEW if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "upper link authorized: %s%s", - adr, fil); + adr_nologin(adr), fil); } } } else if ((!strncmp(tempo, "../", 3)) || (!strncmp(tempo2, "../", 3))) { // Possible en montant @@ -640,7 +644,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "upper link authorized: %s%s", - adr, fil); + adr_nologin(adr), fil); } } // sinon autorisé en descente } @@ -688,14 +692,14 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!opt->wizard) { // mode non wizard forbidden_url = 1; // pas même domaine hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s", - adr, fil); + adr_nologin(adr), fil); } } else { if (opt->wizard) { // mode wizard ACCEPT_LINK(); // same domain hts_log_print(opt, LOG_DEBUG, "same domain link authorized: %s%s", - adr, fil); + adr_nologin(adr), fil); } } @@ -715,13 +719,14 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!opt->wizard) { // mode non wizard forbidden_url = 1; // pas même .xx hts_log_print(opt, LOG_DEBUG, - "foreign location link canceled: %s%s", adr, fil); + "foreign location link canceled: %s%s", + adr_nologin(adr), fil); } } else { if (opt->wizard) { // mode wizard ACCEPT_LINK(); // same domain hts_log_print(opt, LOG_DEBUG, "same location link authorized: %s%s", - adr, fil); + adr_nologin(adr), fil); } } } else @@ -745,7 +750,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!ishtml(opt, fil)) { // non html ACCEPT_LINK(); // authorize may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner - hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", adr, fil); + hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", + adr_nologin(adr), fil); } } // -------------------- PHASE 3-BIS -------------------- @@ -755,7 +761,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, ACCEPT_LINK(); // authorize may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner hts_log_print(opt, LOG_DEBUG, "near link authorized (friendly tag): %s%s", - adr, fil); + adr_nologin(adr), fil); } // -------------------- PHASE 4 -------------------- @@ -767,8 +773,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (opt->wizard && !hts_wizard_url_fits(adr, fil)) { /* the length, not the path: 2 KB of it is unreadable in a log */ hts_log_print(opt, LOG_DEBUG, - "(wizard) link too long, forbidden: %s (%d bytes)", adr, - (int) strlen(fil)); + "(wizard) link too long, forbidden: %s (%d bytes)", + adr_nologin(adr), (int) strlen(fil)); REFUSE_LINK(); // URL forbidden } else if (opt->wizard) { // le wizard entre en action.. // @@ -822,12 +828,14 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, question = 0; // résolution auto if (question) { hts_log_print(opt, LOG_DEBUG, - "(wizard) ambiguous link accepted (external depth): link %s at %s%s", - l, urladr(), urlfil()); + "(wizard) ambiguous link accepted (external " + "depth): link %s at %s%s", + l, adr_nologin(urladr()), urlfil()); } else { hts_log_print(opt, LOG_DEBUG, - "(wizard) forced to accept link (external depth): link %s at %s%s", - l, urladr(), urlfil()); + "(wizard) forced to accept link (external depth): " + "link %s at %s%s", + l, adr_nologin(urladr()), urlfil()); } } @@ -852,16 +860,18 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, question = 0; // ne pas poser de question, autorisé ACCEPT_LINK(); // URL authorized may_set_prio_to = 0; // clear may-set flag - hts_log_print(opt, LOG_DEBUG, - "(wizard) explicit authorized (%s) link: link %s at %s%s", - mdepth, l, urladr(), urlfil()); + hts_log_print( + opt, LOG_DEBUG, + "(wizard) explicit authorized (%s) link: link %s at %s%s", mdepth, + l, adr_nologin(urladr()), urlfil()); } else if (jok == -1) { // forbidden filters_answer = 1; // décision prise par les filtres question = 0; // ne pas poser de question: REFUSE_LINK(); // URL forbidden - hts_log_print(opt, LOG_DEBUG, - "(wizard) explicit forbidden (%s) link: link %s at %s%s", - mdepth, l, urladr(), urlfil()); + hts_log_print( + opt, LOG_DEBUG, + "(wizard) explicit forbidden (%s) link: link %s at %s%s", mdepth, + l, adr_nologin(urladr()), urlfil()); } // sinon on touche à rien } } @@ -876,7 +886,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, force_mirror = 5; // mirror (5) hts_log_print(opt, LOG_DEBUG, "(wizard) explicit mirror link: link %s at %s%s", l, - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); } } } @@ -888,7 +898,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, REFUSE_LINK(); hts_log_print(opt, LOG_DEBUG, "(wizard) ambiguous forbidden link: link %s at %s%s", l, - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); } } // vérifier robots.txt @@ -904,14 +914,14 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, hts_log_print( opt, LOG_DEBUG, "Warning link followed against robots.txt: link %s at %s%s", l, - adr, fil); + adr_nologin(adr), fil); } } else { REFUSE_LINK(); question = 0; hts_log_print(opt, LOG_DEBUG, - "(robots.txt) forbidden link: link %s at %s%s", l, adr, - fil); + "(robots.txt) forbidden link: link %s at %s%s", l, + adr_nologin(adr), fil); } } @@ -919,11 +929,11 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!forbidden_url) { hts_log_print(opt, LOG_DEBUG, "(wizard) shared foreign domain link: link %s at %s%s", l, - urladr(), urlfil()); + adr_nologin(urladr()), urlfil()); } else { hts_log_print(opt, LOG_DEBUG, "(wizard) cancelled foreign domain link: link %s at %s%s", - l, urladr(), urlfil()); + l, adr_nologin(urladr()), urlfil()); } #if BDEBUG==3 printf("at %s in %s, wizard says: url %s ", urladr(), urlfil(), l); @@ -1009,7 +1019,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, } hts_log_print( opt, LOG_NOTICE, "(wizard) answer '%s' (n=%d) for %s%s: %s%s%s", - s, n, adr, fil, + s, n, adr_nologin(adr), fil, forbidden_url == 1 ? "forbidden" : forbidden_url == 0 ? "allowed" : "no verdict", @@ -1029,7 +1039,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (strfield(adr, "ftp://") == 0) { // PAS ftp! forbidden_url = 1; // oui oui toujours interdit (note: sert à rien car ==1 mais c pour comprendre) *just_test_it = 1; // mais on teste - hts_log_print(opt, LOG_DEBUG, "Testing link %s%s", adr, fil); + hts_log_print(opt, LOG_DEBUG, "Testing link %s%s", adr_nologin(adr), + fil); } } } @@ -1070,12 +1081,12 @@ int hts_acceptmime(httrackp * opt, int ptr, forbidden_url = 0; // URL autorisée hts_log_print(opt, LOG_DEBUG, "(wizard) explicit authorized (%s) link %s%s: mime '%s'", - mdepth, adr, fil, mime); + mdepth, adr_nologin(adr), fil, mime); } else if (jok == -1) { // forbidden forbidden_url = 1; // URL interdite hts_log_print(opt, LOG_DEBUG, "(wizard) explicit forbidden (%s) link %s%s: mime '%s'", - mdepth, adr, fil, mime); + mdepth, adr_nologin(adr), fil, mime); } // sinon on touche à rien } /* userdef test */ @@ -1151,14 +1162,14 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz // log if (jok == 1) { hts_log_print(opt, LOG_DEBUG, - "File confirmed (size test): %s%s (" LLintP ")", adr, fil, - (LLint) (size)); + "File confirmed (size test): %s%s (" LLintP ")", + adr_nologin(adr), fil, (LLint) (size)); } else if (jok == -1) { if (size_flag) { /* interdit à cause de la taille */ hts_log_print(opt, LOG_DEBUG, "File cancelled due to its size: %s%s (" LLintP - ", limit: " LLintP ")", adr, fil, (LLint) (size), - (LLint) (sz)); + ", limit: " LLintP ")", + adr_nologin(adr), fil, (LLint) (size), (LLint) (sz)); } else { jok = 1; } diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index 465d7f394..780516812 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -124,21 +124,24 @@ ok "hts-log.txt masks the passwords of URLs, proxies and -%X headers" out=$(httrack -#test=maskpw) || fail "httrack -#test=maskpw exited non-zero: $out" test "$out" = "maskpw self-test: OK" || fail "expected 'maskpw self-test: OK', got: $out" -# plain.html is reached only through the crawl, so its line comes from hts_log_print -crawl choke "cuser:CHOKESECRET@127.0.0.1:$port/hop.html" "nuser:NOPATHSECRET@127.0.0.1:1" \ - -z "-*:KEEPFILTER@*" -log=$(<"$tmpdir/choke/hts-log.txt") -for secret in CHOKESECRET NOPATHSECRET; do - if grep -qF "$secret" <<<"$log"; then - fail "$secret is in hts-log.txt: $(grep -F "$secret" <<<"$log")" - fi -done -for line in "at cuser:***@127.0.0.1:$port/hop.html nuser:***@127.0.0.1:1 -*:KEEPFILTER@*" \ - " -%H nuser:***@127.0.0.1:1 -z " " -*:KEEPFILTER@* )" \ - "link recorded: cuser:***@127.0.0.1:$port/plain.html"; do +# planted.html links with a password in every form the engine reads one +crawl planted "http://suser:STARTPWA@127.0.0.1:$port/planted.html" \ + "suser:STARTPWB@127.0.0.1:$port/hop.html" -z -Z -r4 -T5 "+*" "-*:KEEPFILTER@*" \ + -%X "X-Note: a:KEEPHEADER@c/d" +crawl planted-proxy "http://127.0.0.1:$port/planted.html" -P "puser:PROXYPW@127.0.0.1:$port" \ + -z -Z -r4 -T5 "+*" +log=$(<"$tmpdir/planted/hts-log.txt") +for line in "at link 127.0.0.1:1/l8.html (from 127.0.0.1:$port/planted.html)" \ + "at http://suser:***@127.0.0.1:$port/planted.html suser:***@127.0.0.1:$port/hop.html" \ + " -*:KEEPFILTER@* -%X \"X-Note: a:KEEPHEADER@c/d\" "; do grep -qF -- "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" done -ok "every hts-log.txt line masks a URL password and keeps the rest" +grep -qF "STARTPWA" "$tmpdir/planted/hts-cache/doit.log" || fail "the grep below cannot see a password" +# doit.log, new.txt and new.zip keep the URLs that --update needs +leaks=$(find "$tmpdir/planted" "$tmpdir/planted-proxy" -type f ! -name doit.log \ + ! -name new.txt ! -name new.zip -exec grep -aoE "(STARTPW|LINKPW|PROXYPW)[A-Z0-9]*" {} + || true) +test -z "$leaks" || fail "a password is in the project: $leaks" +ok "no project file holds a URL or proxy password" root="$tmpdir/root" cmds="$tmpdir/cmds" diff --git a/tests/header-injection-server.py b/tests/header-injection-server.py index 1c3135d16..9cb715d94 100644 --- a/tests/header-injection-server.py +++ b/tests/header-injection-server.py @@ -36,6 +36,24 @@ def root_page(port): ) +# a password in every form the engine reads one (657) +PLANTED = ( + "http://luser:LINKPWA@127.0.0.1:%(p)d/l1.html", + "//luser:LINKPWB@127.0.0.1:%(p)d/l2.html", + "http://luser:LINKPWC1 LINKPWC2@127.0.0.1:%(p)d/l3.html", + "http://luser:LINKPWD;q@127.0.0.1:%(p)d/l4.html", + "http://luser:LINKPWE@LINKPWF@127.0.0.1:%(p)d/l5.html", + "http://luser:LINKPWG%%40q@127.0.0.1:%(p)d/l6.html", + "http://luser:LINKPWH@[::1]:1/l7.html", + "http://luser:LINKPWI@127.0.0.1:%(p)d", + "http://luser:LINKPWJ@127.0.0.1:1/l8.html", +) + + +def planted_page(port): + return page("".join('l' % (u % {"p": port}) for u in PLANTED)) + + # the poisoned page must link onward, or its URL never becomes a Referer POISON_PAGE = page('deep') # the ftp link takes the ftp-through-proxy emission site, which has no Host: @@ -53,6 +71,8 @@ def body_for(request, port): return PROXY_ROOT if line.startswith(b"GET / "): return root_page(port) + if b"/planted.html" in line: + return planted_page(port) if b"/x" in line: return POISON_PAGE # one hop, so the second request of a crawl carries a Referer (328) From a2ad2b03c6b0a3bc578adbe23fd01a6211799331 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 18:44:46 +0200 Subject: [PATCH 06/12] Hide passwords in URLs and -%H headers from the logs Fix the round-4 review of PR #2000. Two kinds of debug line still printed a login. "link #N is ready, skipping" printed urladr() raw. The two JavaScript debug lines printed the first 32 page bytes, which can hold a CSS url() with a login. The JavaScript lines now mask the whole fragment before they cut it, or print "(hidden)" when no scheme lets them find the login. adr_nologin() and url_nologin() now keep the scheme and the user and write user:***@, so a log line that compares two addresses stays readable. They take a rotating buffer of opt only for an address with a password. A crawl without a login logs what it did before. "file://" is its own host, which the suite caught as a crash. The -%H log masks the URL of a Location or Content-Location field. The start URLs on stdout and the --sitemap-url and --referer values now go through the command-line echo masking. Test 660 fails on any log call that prints an adr, url_adr or urladr() outside the helpers, with an allowlist. Test 657 now reaches the skipped image, a CSS url(), a redirect, robots.txt, a sitemap and stdout. Signed-off-by: Xavier Roche Co-Authored-By: Claude Opus 5.5 --- src/htsback.c | 208 ++++++++++--------- src/htscache.c | 24 +-- src/htscore.c | 72 +++---- src/htscoremain.c | 39 ++-- src/htsftp.c | 8 +- src/htslib.c | 109 +++++++--- src/htslib.h | 21 +- src/htslib_selftest.c | 45 +++-- src/htsname.c | 35 ++-- src/htsparse.c | 270 ++++++++++++++----------- src/htsrobots.c | 2 +- src/htssitemap.c | 23 ++- src/htswizard.c | 85 ++++---- tests/657_local-header-log-redact.test | 20 +- tests/660_lint-log-logins.test | 57 ++++++ tests/Makefile.am | 2 +- tests/check-log-logins.py | 98 +++++++++ tests/header-injection-server.py | 31 ++- 18 files changed, 728 insertions(+), 421 deletions(-) create mode 100644 tests/660_lint-log-logins.test create mode 100644 tests/check-log-logins.py diff --git a/src/htsback.c b/src/htsback.c index f695eea95..89e2677f0 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -172,7 +172,7 @@ hts_boolean back_spool_write(httrackp *opt, const char *filename, opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: open " "error (%s, %s)", - adr_nologin(back->url_adr), back->url_fil, filename, + adr_nologin(opt, back->url_adr), back->url_fil, filename, dir_exists(filename) ? "directory exists" : "directory does NOT exist!", fexist_utf8(filename) ? "file already exists!" : "file does not exist"); back_tmpdir_drop(filename); /* filecreate() may have made the directory */ @@ -185,7 +185,7 @@ hts_boolean back_spool_write(httrackp *opt, const char *filename, hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: write " "error", - adr_nologin(back->url_adr), back->url_fil, filename); + adr_nologin(opt, back->url_adr), back->url_fil, filename); (void) UNLINK(filename); back_tmpdir_drop(filename); } @@ -341,7 +341,7 @@ static int back_connect_next(httrackp *opt, struct_back *sback, int i) { back[i].status = STATUS_CONNECTING; hts_log_print(opt, LOG_DEBUG, "connect failed, trying next address (%d/%d) for %s", next + 1, - cf->addr_count, adr_nologin(back[i].url_adr)); + cf->addr_count, adr_nologin(opt, back[i].url_adr)); return 1; } @@ -469,14 +469,14 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr } hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "engine: warning: unserialize error for %s%s (%s)", - adr_nologin(adr), fil, sav); + adr_nologin(opt, adr), fil, sav); } fclose(fp); } else { hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "engine: warning: unserialize error for %s%s (%s), file " "disappeared", - adr_nologin(adr), fil, sav); + adr_nologin(opt, adr), fil, sav); } (void) UNLINK(fileback); back_tmpdir_drop(fileback); @@ -501,7 +501,7 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr hts_log_print(opt, LOG_WARNING, "engine: warning: unserialize error for %s%s (%s): no " "more space to wakeup frozen slots", - adr_nologin(adr), fil, sav); + adr_nologin(opt, adr), fil, sav); } } } @@ -768,10 +768,10 @@ int back_cleanup_background(httrackp * opt, cache_back * cache, hts_log_print(opt, LOG_WARNING, "engine: unexpected duplicate file entry: %s%s -> %s (%d " "'%s') / %s%s -> %s (%d '%s')", - adr_nologin(back[checkIndex].url_adr), + adr_nologin(opt, back[checkIndex].url_adr), back[checkIndex].url_fil, back[checkIndex].url_sav, back[checkIndex].r.statuscode, back[checkIndex].r.msg, - adr_nologin(back[i].url_adr), back[i].url_fil, + adr_nologin(opt, back[i].url_adr), back[i].url_fil, back[i].url_sav, back[i].r.statuscode, back[i].r.msg); back_delete(NULL, NULL, sback, checkIndex); #ifdef _DEBUG @@ -811,7 +811,7 @@ int back_cleanup_background(httrackp * opt, cache_back * cache, } else { hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s: %s", - adr_nologin(back[i].url_adr), back[i].url_fil, + adr_nologin(opt, back[i].url_adr), back[i].url_fil, named ? "memory full" : "temporary filename too long"); } } @@ -1166,7 +1166,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, "truncated chunked transfer (terminating chunk missing, got " LLintP " bytes): file not cached, will be retried on the next" " update (use -%%B to cache anyway): %s%s", - back[p].r.size, adr_nologin(back[p].url_adr), back[p].url_fil); + back[p].r.size, adr_nologin(opt, back[p].url_adr), back[p].url_fil); } else if (back_status(&back[p]) == STATUS_READY) { hts_log_print(opt, LOG_WARNING, "incomplete transfer (expected " LLintP @@ -1174,14 +1174,14 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, "): file not cached, will be retried on the next update" " (use -%%B to cache anyway): %s%s", back[p].r.totalsize, back[p].r.size, - adr_nologin(back[p].url_adr), back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } else { hts_log_print( opt, LOG_INFO, "incomplete file not yet stored in cache (expected " LLintP " got " LLintP "): %s%s", - back[p].r.totalsize, back[p].r.size, adr_nologin(back[p].url_adr), - back[p].url_fil); + back[p].r.totalsize, back[p].r.size, + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } back_finalize_backup(opt, &back[p], HTS_FALSE); /* Keep the surviving copy in new.lst, else the update purge drops the @@ -1474,7 +1474,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, HTS_STAT.stat_bytes += back[p].r.size; HTS_STAT.stat_files++; hts_log_print(opt, LOG_TRACE, "added file %s%s => %s", - adr_nologin(back[p].url_adr), back[p].url_fil, + adr_nologin(opt, back[p].url_adr), back[p].url_fil, back[p].url_sav); } if ((!back[p].r.notmodified) && (opt->is_update)) { @@ -1483,13 +1483,13 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, hts_log_print( opt, LOG_INFO, "engine: transfer-status: link updated: %s%s -> %s", - adr_nologin(back[p].url_adr), back[p].url_fil, + adr_nologin(opt, back[p].url_adr), back[p].url_fil, back[p].url_sav); } else { hts_log_print(opt, LOG_INFO, "engine: transfer-status: link added: %s%s -> %s", - adr_nologin(back[p].url_adr), back[p].url_fil, - back[p].url_sav); + adr_nologin(opt, back[p].url_adr), + back[p].url_fil, back[p].url_sav); } if (cache->txt) { if (back[p].is_update) { @@ -1502,7 +1502,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, hts_log_print( opt, LOG_INFO, "engine: transfer-status: link recorded: %s%s -> %s", - adr_nologin(back[p].url_adr), back[p].url_fil, + adr_nologin(opt, back[p].url_adr), back[p].url_fil, back[p].url_sav); if (cache->txt) { if (opt->is_update) @@ -1516,7 +1516,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, opt, LOG_INFO, "engine: transfer-status: empty file? (%d, '%s'): %s%s", back[p].r.statuscode, back[p].r.msg, - adr_nologin(back[p].url_adr), back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); if (cache->txt) { state = "empty"; } @@ -1525,7 +1525,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, hts_log_print(opt, LOG_INFO, "engine: transfer-status: link error (%d, '%s'): %s%s", back[p].r.statuscode, back[p].r.msg, - adr_nologin(back[p].url_adr), back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); if (cache->txt) { state = "error"; } @@ -1564,7 +1564,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, /* error */ if (!HTTP_IS_OK(back[p].r.statuscode)) { hts_log_print(opt, LOG_DEBUG, "redirect to %s%s", - adr_nologin(back[p].url_adr), back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); /* Store only header reference */ cache_mayadd(opt, cache, &back[p].r, back[p].url_adr, back[p].url_fil, NULL); @@ -1575,7 +1575,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, "file with unresolved type not cached (%s (%d), size " LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size, - adr_nologin(back[p].url_adr), back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } } @@ -2128,7 +2128,7 @@ int back_unserialize_ref(httrackp * opt, const char *adr, const char *fil, if (!ref_get_record(fp, back)) { hts_log_print(opt, LOG_DEBUG, "Ignoring an unreadable resume reference for %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); fclose(fp); back_clear_entry(back); freet(back); @@ -2173,7 +2173,7 @@ int back_maydelete(httrackp * opt, cache_back * cache, struct_back * sback, assertf(back[p].ka_time_start != 0); hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully saved #%d (%s)", - back[p].r.debugid, adr_nologin(back[p].url_adr)); + back[p].r.debugid, adr_nologin(opt, back[p].url_adr)); return 1; } } @@ -2227,7 +2227,7 @@ void back_maydeletehttp(httrackp * opt, cache_back * cache, struct_back * sback, assertf(back[q].ka_time_start != 0); hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully preserved #%d (%s)", - back[q].r.debugid, adr_nologin(back[q].url_adr)); + back[q].r.debugid, adr_nologin(opt, back[q].url_adr)); } else { deletehttp(&back[p].r); back[p].r.soc = INVALID_SOCKET; @@ -2494,7 +2494,7 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback, ) { hts_log_print(opt, LOG_DEBUG, "File '%s%s' -> %s not yet saved in cache - saving now", - adr_nologin(back[p].url_adr), back[p].url_fil, + adr_nologin(opt, back[p].url_adr), back[p].url_fil, back[p].url_sav); } if (cache != NULL) { @@ -2597,7 +2597,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, int already_there = 0; hts_log_print(opt, LOG_ERROR, "error: back_add(%s,%s,%s) duplicate", - adr_nologin(adr), fil, save); + adr_nologin(opt, adr), fil, save); } #endif @@ -2676,7 +2676,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back_set_finished(opt, sback, p); hts_log_print(opt, LOG_WARNING, "File not added due to mirror cancel: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); return 0; } // test "fast header" cache ; that is, tests we did that lead to 3XX/4XX/5XX response codes @@ -2786,14 +2786,14 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "Previous file '%s' not found (erased by " "user ?), ignoring: %s%s", - save, adr_nologin(back[p].url_adr), + save, adr_nologin(opt, back[p].url_adr), back[p].url_fil); } } else { hts_log_print(opt, LOG_WARNING, "Previous file '%s' not found (erased by user " "?), recatching: %s%s", - save, adr_nologin(back[p].url_adr), + save, adr_nologin(opt, back[p].url_adr), back[p].url_fil); } } @@ -2839,11 +2839,11 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, if (!test) { hts_log_print(opt, LOG_DEBUG, "File immediately loaded from cache: %s%s", - adr_nologin(back[p].url_adr), back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } else { hts_log_print(opt, LOG_DEBUG, "File immediately tested from cache: %s%s", - adr_nologin(back[p].url_adr), back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } back[p].r.notmodified = 1; // fichier non modifié // no request was sent at all, so this is never a server 304 (#839) @@ -2985,7 +2985,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, #endif hts_log_print(opt, LOG_DEBUG, "File partially present (" LLintP " bytes): %s%s", - (LLint) sz, adr_nologin(back[p].url_adr), + (LLint) sz, adr_nologin(opt, back[p].url_adr), back[p].url_fil); /* impossible - don't have etag or date @@ -3009,7 +3009,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "Could not find timestamp for partially present " "file, restarting (lost " LLintP " bytes): %s%s", - (LLint) sz, adr_nologin(back[p].url_adr), + (LLint) sz, adr_nologin(opt, back[p].url_adr), back[p].url_fil); } @@ -3017,7 +3017,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_NOTICE, "File partially present (" LLintP " bytes) retransferred due to lack of cache: %s%s", - (LLint) sz, adr_nologin(back[p].url_adr), + (LLint) sz, adr_nologin(opt, back[p].url_adr), back[p].url_fil); /* Sinon requête normale... */ back[p].http11 = 0; @@ -3036,7 +3036,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_NOTICE, "HTML file (" LLintP " bytes) retransferred due to lack of cache: %s%s", - (LLint) sz, adr_nologin(back[p].url_adr), + (LLint) sz, adr_nologin(opt, back[p].url_adr), back[p].url_fil); /* Sinon requête normale... */ back[p].http11 = 0; @@ -3052,7 +3052,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back_set_finished(opt, sback, p); hts_log_print(opt, LOG_WARNING, "File not added due to mirror cancel: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); return 0; } @@ -3180,9 +3180,10 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, } else { soc = back[p].r.soc; - hts_log_print( - opt, LOG_DEBUG, "(Keep-Alive): successfully linked #%d (for %s%s)", - back[p].r.debugid, adr_nologin(back[p].url_adr), back[p].url_fil); + hts_log_print(opt, LOG_DEBUG, + "(Keep-Alive): successfully linked #%d (for %s%s)", + back[p].r.debugid, adr_nologin(opt, back[p].url_adr), + back[p].url_fil); } if (opt->timeout > 0) { // gestion du opt->timeout @@ -3277,7 +3278,7 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, may_be_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil), /* */ - back[i].r.contenttype, adr_nologin(back[i].url_adr), + back[i].r.contenttype, adr_nologin(opt, back[i].url_adr), back[i].url_fil, back[i].url_sav); } } @@ -3341,7 +3342,7 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { hts_log_print(opt, LOG_INFO, "engine: warning: entry cleaned up, but no trace on " "heap: %s%s (%s)", - adr_nologin(back[i].url_adr), back[i].url_fil, + adr_nologin(opt, back[i].url_adr), back[i].url_fil, back[i].url_sav); } } @@ -3355,7 +3356,7 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { /* recycle the socket, but keep back[i].r.adr in memory */ hts_log_print(opt, LOG_DEBUG, "file %s%s validated (cached, left in memory)", - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); back_maydeletehttp(opt, cache, sback, i); } } @@ -3377,9 +3378,9 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { (int) back[i].r.keep_alive_t); reason = buffer; } - hts_log_print(opt, LOG_DEBUG, - "(Keep-Alive): live socket #%d (%s) closed (%s)", - back[i].r.debugid, adr_nologin(back[i].url_adr), reason); + hts_log_print( + opt, LOG_DEBUG, "(Keep-Alive): live socket #%d (%s) closed (%s)", + back[i].r.debugid, adr_nologin(opt, back[i].url_adr), reason); back_delete(opt, cache, sback, i); // delete backing entry } } @@ -4165,7 +4166,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print( opt, LOG_DEBUG, "File received from net to disk: %s%s", - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); if (back[i].r.out == NULL) { hts_log_print(opt, LOG_ERROR | LOG_ERRNO, @@ -4189,18 +4191,19 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, /* create a temporary reference file in case of broken mirror */ if (back[i].r.out != NULL && opt->cache != 0) { if (back_serialize_ref(opt, &back[i]) != 0) { - hts_log_print(opt, LOG_WARNING, - "Could not create temporary " - "reference file for %s%s", - adr_nologin(back[i].url_adr), - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "Could not create temporary " + "reference file for %s%s", + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } } } } else { // on coupe tout! hts_log_print(opt, LOG_DEBUG, "File cancelled (non HTML): %s%s", - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); back[i].status = STATUS_READY; // terminé back_set_finished(opt, sback, i); @@ -4370,7 +4373,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Truncated chunked transfer (" LLintP " Bytes, terminating chunk missing) for %s%s", (LLint) back[i].r.size, - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } } @@ -4390,12 +4394,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, (LLint) back[i].r.totalsize); } else { // Un warning suffira.. - hts_log_print(opt, LOG_WARNING, - "Incorrect length (" LLintP "!=" LLintP - " expected) for %s%s", - (LLint) back[i].r.size, - (LLint) back[i].r.totalsize, - adr_nologin(back[i].url_adr), back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "Incorrect length (" LLintP "!=" LLintP + " expected) for %s%s", + (LLint) back[i].r.size, (LLint) back[i].r.totalsize, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } } @@ -4489,7 +4493,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, if (!hts_inmem_size_fits(back[i].r.totalsize)) { hts_log_print(opt, LOG_WARNING, "Chunked resource too large for %s%s", - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); chunk_size = -1; } else { @@ -4502,11 +4506,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, /* r.totalsize already counts this chunk, so dropping the buffer would have the next read write past it. */ - hts_log_print( - opt, LOG_ERROR, - "not enough memory (" LLintP ") for %s%s", - (LLint) back[i].r.totalsize, - adr_nologin(back[i].url_adr), back[i].url_fil); + hts_log_print(opt, LOG_ERROR, + "not enough memory (" LLintP + ") for %s%s", + (LLint) back[i].r.totalsize, + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); chunk_size = -1; } } @@ -4519,7 +4524,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } else { hts_log_print( opt, LOG_WARNING, "Illegal chunk (%s) for %s%s", - back[i].chunk_adr, adr_nologin(back[i].url_adr), + back[i].chunk_adr, adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } else { /* STATUS_CHUNK_CR: the CRLF after the data */ @@ -4536,7 +4541,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } else { hts_log_print( opt, LOG_WARNING, "illegal chunk CRLF (%s) for %s%s", - back[i].chunk_adr, adr_nologin(back[i].url_adr), + back[i].chunk_adr, adr_nologin(opt, back[i].url_adr), back[i].url_fil); #if CHUNKDEBUG==1 printf("[%d] chunk CRLF ERROR!! : '%s'\n", @@ -4545,10 +4550,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } } } else { - hts_log_print(opt, LOG_WARNING, - "chunk too big (" LLintP ") for %s%s", - (LLint) back[i].chunk_size, - adr_nologin(back[i].url_adr), back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, "chunk too big (" LLintP ") for %s%s", + (LLint) back[i].chunk_size, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } // ok, continuer sur le body @@ -4592,7 +4597,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print( opt, LOG_WARNING, "unexpected incomplete type with 200 code at %s%s", - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } if (back[i].r.soc != INVALID_SOCKET) { @@ -4630,7 +4635,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "!=" LLintP " expected) for %s%s", (LLint) back[i].r.size, (LLint) back[i].r.totalsize, - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } @@ -4714,7 +4719,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, truncated Location, or read its tail as headers */ hts_log_print(opt, LOG_WARNING, "Over-long header dropped for %s%s", - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); continue; } @@ -4768,7 +4773,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print(opt, LOG_WARNING, "External wrapper aborted transfer, " "breaking connection: %s%s", - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); back[i].status = STATUS_READY; // FINI back_set_finished(opt, sback, i); @@ -4805,7 +4810,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Bot protection challenge at %s%s: the " "server refused the request (cf-mitigated " "header) rather than failing it", - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); hts_log_print(opt, LOG_WARNING, "Solve the challenge in a browser, then pass " @@ -4835,7 +4840,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print( opt, LOG_DEBUG, "Status 100 detected for %s%s, continuing headers", - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); continue; } @@ -4868,7 +4873,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Kept existing file %s (" LLintP " bytes), matching size and timestamp: %s%s", back[i].url_sav, (LLint) back[i].range_req_size, - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } // transform 406 into 200 ; we'll catch embedded links inside the choice page @@ -4881,12 +4886,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, back[i].is_update && !back[i].testmode && back[i].range_req_size == 0 && back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) { - hts_log_print(opt, LOG_NOTICE, - "Kept existing file %s after error %d (%s) " - "on update: %s%s", - back[i].url_sav, back[i].r.statuscode, - back[i].r.msg, adr_nologin(back[i].url_adr), - back[i].url_fil); + hts_log_print( + opt, LOG_NOTICE, + "Kept existing file %s after error %d (%s) " + "on update: %s%s", + back[i].url_sav, back[i].r.statuscode, back[i].r.msg, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); back[i].r.statuscode = HTTP_NOT_MODIFIED; deletehttp(&back[i].r); back[i].r.soc = INVALID_SOCKET; @@ -4930,7 +4935,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Kept existing file %s (" LLintP " bytes), matching the cached size: %s%s", back[i].url_sav, (LLint) len1, - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } @@ -4939,7 +4944,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, opt, LOG_WARNING, "File seems complete (same size), but there was " "a cache read error (%u): %s%s", - r.statuscode, adr_nologin(back[i].url_adr), + r.statuscode, adr_nologin(opt, back[i].url_adr), back[i].url_fil); } if (r.adr) { @@ -4976,7 +4981,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, " bytes), matching the announced " "size: %s%s", back[i].url_sav, (LLint) size, - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } @@ -5025,7 +5030,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "size, range ignored: %s%s", back[i].url_sav, (LLint) back[i].range_req_size, - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } @@ -5074,7 +5079,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, // head: terminé if (back[i].head_request) { hts_log_print(opt, LOG_DEBUG, "Tested file: %s%s", - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(head request): deletehttp\n"); @@ -5132,7 +5137,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print( opt, LOG_DEBUG, "Not-modified status without cache guessed: %s%s", - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } // Status is okay? @@ -5145,7 +5150,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, server_sent_304 ? HTS_FALSE : HTS_TRUE; hts_log_print(opt, LOG_DEBUG, "File loaded after test from cache: %s%s", - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil); // finalize @@ -5181,8 +5186,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print( opt, LOG_NOTICE, "File excluded by MIME type filter (%s): %s%s", - back[i].r.contenttype, adr_nologin(back[i].url_adr), - back[i].url_fil); + back[i].r.contenttype, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } else { // il faut aller le chercher // effacer buffer (requète) @@ -5251,11 +5256,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, /* create a temporary reference file in case of * broken mirror */ if (back_serialize_ref(opt, &back[i]) != 0) { - hts_log_print(opt, LOG_WARNING, - "Could not create temporary " - "reference file for %s%s", - adr_nologin(back[i].url_adr), - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "Could not create temporary " + "reference file for %s%s", + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } #if HDEBUG printf("continue interrupted file\n"); @@ -5327,7 +5333,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Unusable partial-content range for %s%s " "(have " LLintP " bytes, got " LLintP "-" LLintP "/" LLintP "), restarting", - adr_nologin(back[i].url_adr), + adr_nologin(opt, back[i].url_adr), back[i].url_fil, (LLint) sz, back[i].r.crange_start, back[i].r.crange_end, back[i].r.crange); @@ -5501,7 +5507,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) { hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(timeout): deletehttp\n"); diff --git a/src/htscache.c b/src/htscache.c index 4c3e590b4..0e7eb2473 100644 --- a/src/htscache.c +++ b/src/htscache.c @@ -54,7 +54,7 @@ void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, const char *url_adr, const char *url_fil, const char *url_save) { hts_log_print(opt, LOG_DEBUG, "File checked by cache: %s", - adr_nologin(url_adr)); + adr_nologin(opt, url_adr)); // ---stockage en cache--- // stocker dans le cache? if (opt->cache) { @@ -64,7 +64,7 @@ void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, hts_log_print( opt, LOG_WARNING, "aborted cache validation: %s%s still has temporary name %s", - adr_nologin(url_adr), url_fil, url_save); + adr_nologin(opt, url_adr), url_fil, url_save); return; } // c'est le seul endroit ou l'on ajoute des elements dans le cache (fichier entier ou header) @@ -109,7 +109,7 @@ void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, } hts_log_print( opt, LOG_DEBUG, "Cached fast-header response: %s%s is %d", - adr_nologin(url_adr), url_fil, (int) r->statuscode); + adr_nologin(opt, url_adr), url_fil, (int) r->statuscode); coucal_add(cache->cached_tests, concat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), url_adr, url_fil), (intptr_t) strdupt(tempo)); @@ -212,7 +212,7 @@ static void cache_zip_write_failed(httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "cache write failed (%s: %s), entry not cached: %s%s", what, hts_get_zerror(zErr, errbuf, sizeof(errbuf)), - adr_nologin(url_adr), url_fil); + adr_nologin(opt, url_adr), url_fil); } } @@ -277,7 +277,7 @@ void cache_keep_previous(httrackp *opt, cache_back *cache, const char *url_adr, URL's file. */ if (r.statuscode > 0 && strcmp(previous_save, url_save) == 0) { hts_log_print(opt, LOG_DEBUG, "keeping the previous cache entry for %s%s", - adr_nologin(url_adr), url_fil); + adr_nologin(opt, url_adr), url_fil); cache_add_ex(opt, cache, &r, url_adr, url_fil, url_save, opt->all_in_cache, StringBuff(opt->path_html_utf8), HTS_TRUE); } @@ -339,12 +339,12 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, if (r->is_write && url_save != NULL && strnotempty(url_save)) { hts_log_print(opt, LOG_WARNING, "file too large for the cache, storing headers only: %s%s", - adr_nologin(url_adr), url_fil); + adr_nologin(opt, url_adr), url_fil); dataincache = 0; } else { hts_log_print(opt, LOG_WARNING, "entry too large for the cache, not cached: %s%s", - adr_nologin(url_adr), url_fil); + adr_nologin(opt, url_adr), url_fil); return; } } @@ -410,7 +410,7 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, if (headersDropped != 0) { hts_log_print(opt, LOG_WARNING, "cached headers too large, %d field(s) dropped: %s%s", - headersDropped, adr_nologin(url_adr), url_fil); + headersDropped, adr_nologin(opt, url_adr), url_fil); } /* Filename */ @@ -424,7 +424,7 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, url_fil)) { hts_log_print(opt, LOG_WARNING, "URL too long to be cached, entry not cached: %s%s", - adr_nologin(url_adr), url_fil); + adr_nologin(opt, url_adr), url_fil); return; } } @@ -723,7 +723,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, if (!hts_location_is_safe(r.location)) { hts_log_print(opt, LOG_WARNING, "cached Location naming a post token, dropped: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); r.location[0] = '\0'; } @@ -753,7 +753,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "cached filename leaving the mirror, " "not using the cache entry: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Cache Read Error : Bad Filename"); } @@ -764,7 +764,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "cached filename too long once rebuilt under '%s', " "not using the cache entry: %s%s", - path_html, adr_nologin(adr), fil); + path_html, adr_nologin(opt, adr), fil); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Cache Read Error : Filename Too Long"); } diff --git a/src/htscore.c b/src/htscore.c index 59cc517cb..96d0e3c04 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -991,7 +991,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { (int) (sizeof(url) - 1), url); hts_log_print(opt, LOG_WARNING, "URL longer than %d bytes, ignored: %.64s...", - (int) (sizeof(url) - 1), URL_NOLOGIN(opt, url)); + (int) (sizeof(url) - 1), url_nologin(opt, url)); continue; } @@ -1402,14 +1402,15 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { ) ) { // sauter si lien annulé (ou fil vide) if (heap(ptr) != NULL && heap(ptr)->pass2 == -1) { - hts_log_print(opt, LOG_DEBUG, "link #%d is ready, skipping: %s%s..", - ptr, ((urladr() != NULL) ? (urladr()) : (" ")), - ((urlfil() != NULL) ? (urlfil()) : (" "))); + hts_log_print( + opt, LOG_DEBUG, "link #%d is ready, skipping: %s%s..", ptr, + ((urladr() != NULL) ? adr_nologin(opt, urladr()) : (" ")), + ((urlfil() != NULL) ? (urlfil()) : (" "))); } else { - hts_log_print(opt, LOG_DEBUG, - "link #%d seems ready, skipping: %s%s..", ptr, - ((urladr() != NULL) ? (urladr()) : (" ")), - ((urlfil() != NULL) ? (urlfil()) : (" "))); + hts_log_print( + opt, LOG_DEBUG, "link #%d seems ready, skipping: %s%s..", ptr, + ((urladr() != NULL) ? adr_nologin(opt, urladr()) : (" ")), + ((urlfil() != NULL) ? (urlfil()) : (" "))); } // remove from stats if (heap(ptr)->pass2 == -1) { @@ -1424,8 +1425,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { } if (heap(ptr) != NULL) { // on a qq chose à récupérer? - hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s", adr_nologin(urladr()), - urlfil()); + hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s", + adr_nologin(opt, urladr()), urlfil()); #if DEBUG_ROBOTS if (strcmp(urlfil(), "/robots.txt") == 0) { printf("robots.txt detected\n"); @@ -1548,7 +1549,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { hts_log_print( opt, LOG_WARNING, "Big file cancelled according to user's preferences: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } } } @@ -1601,9 +1602,9 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { is_loaded_from_file = 1; \ r.adr = readfile2_inmem(savename(), &r.size); \ if (r.adr != NULL) { \ - hts_log_print(opt, LOG_INFO, "File successfully loaded for parsing: %s%s (%d bytes)",adr_nologin(urladr()),urlfil(),(int)r.size); \ + hts_log_print(opt, LOG_INFO, "File successfully loaded for parsing: %s%s (%d bytes)",adr_nologin(opt, urladr()),urlfil(),(int)r.size); \ } else { \ - hts_log_print(opt, LOG_ERROR, "File could not be loaded for parsing: %s%s",adr_nologin(urladr()),urlfil()); \ + hts_log_print(opt, LOG_ERROR, "File could not be loaded for parsing: %s%s",adr_nologin(opt, urladr()),urlfil()); \ } \ } \ } while(0) @@ -1657,7 +1658,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { hts_log_print( opt, LOG_ERROR, "Not enough memory to convert %s%s from UCS2 to UTF-8", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Not enough memory"); error = 1; @@ -1665,8 +1666,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { hts_log_print(opt, LOG_WARNING, "File %s%s converted from UCS2 to UTF-8 (old " "size: %d bytes, new size: %d bytes)", - adr_nologin(urladr()), urlfil(), (int) r.size, - (int) new_size); + adr_nologin(opt, urladr()), urlfil(), + (int) r.size, (int) new_size); freet(r.adr); r.adr = new_adr; r.size = (LLint) new_size; @@ -1682,7 +1683,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { strcpybuff(r.contenttype, "application/octet-stream"); hts_log_print(opt, LOG_WARNING, "File not parsed, looks like binary: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } /* This hack allows you to avoid problems with parsing '\0' characters */ @@ -1915,13 +1916,13 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (page_charset[0] == '\0') { hts_log_print(opt, LOG_INFO, "Warning: could not detect encoding for: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } } /* Info for wrappers */ hts_log_print(opt, LOG_INFO, "engine: check-html: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); { char BIGSTK buff_err_msg[1024]; htsmoduleStruct BIGSTK str; @@ -2020,12 +2021,12 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (strnotempty(infobuff)) { hts_log_print(opt, LOG_INFO, "Note: robots.txt forbidden links for %s are: %s", - adr_nologin(urladr()), infobuff); + adr_nologin(opt, urladr()), infobuff); hts_log_print(opt, LOG_NOTICE, "Note: due to %s remote robots.txt rules, links " "beginning with these path will be forbidden: %s " "(see in the options to disable this)", - adr_nologin(urladr()), infobuff); + adr_nologin(opt, urladr()), infobuff); } } /* After robots_parse, so the rules this very body carries already @@ -2083,7 +2084,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { } else { // lien non autorisé! (ex: cgi-bin en html) hts_log_print(opt, LOG_DEBUG, "non-html file ignored after upload at %s : %s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); if (r.adr) { freet(r.adr); r.adr = NULL; @@ -2632,7 +2633,7 @@ void host_ban(httrackp * opt, int ptr, back[i].r.statuscode = STATUSCODE_TIMEOUT; // timeout (peu importe si c'est un traffic jam) strcpybuff(back[i].r.msg, "Link Cancelled by host control"); hts_log_print(opt, LOG_DEBUG, "Shutdown: %s%s", - adr_nologin(back[i].url_adr), back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } @@ -2648,7 +2649,7 @@ void host_ban(httrackp * opt, int ptr, if ((l > 0) && (l < 1020)) { // in range if (strfield2(jump_identification_const(heap(i)->adr), host)) { // host hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", - adr_nologin(heap(i)->adr), heap(i)->fil); + adr_nologin(opt, heap(i)->adr), heap(i)->fil); hts_invalidate_link(opt, i); // invalidate hashtable entry // on efface pas le hash, because si on rencontre le lien, reverif sav.. } @@ -2767,7 +2768,7 @@ static hts_boolean live_cancel_link_(httrackp *opt, struct_back *sback, int i) { /* no cache: a dropped page must not be recorded as fetched */ back_delete(opt, NULL, sback, b); } - hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", adr_nologin(heap(i)->adr), + hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", adr_nologin(opt, heap(i)->adr), heap(i)->fil); hts_invalidate_link(opt, i); /* the main loop takes every cancelled link out of this count */ @@ -4848,18 +4849,18 @@ int htsAddLink(htsmoduleStruct * str, char *link) { codebase[0] = '\0'; hts_log_print(opt, LOG_DEBUG, "(module): adding link : '%s'", - URL_NOLOGIN(opt, link)); + url_nologin(opt, link)); // recopie de "creer le lien" // if (!RUN_CALLBACK1(opt, linkdetected, link)) { hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper", - URL_NOLOGIN(opt, link)); + url_nologin(opt, link)); return 0; } if (!RUN_CALLBACK2(opt, linkdetected2, link, NULL)) { hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper(2)", - URL_NOLOGIN(opt, link)); + url_nologin(opt, link)); return 0; } // adr = c'est la même @@ -4891,14 +4892,14 @@ int htsAddLink(htsmoduleStruct * str, char *link) { strcpybuff(codebase, tempo); // couper host } else { hts_log_print(opt, LOG_ERROR, "Unexpected strstr error in base %s", - URL_NOLOGIN(opt, codebase)); + url_nologin(opt, codebase)); } } } if (!((int) strlen(codebase) < HTS_URLMAXSIZE)) { // trop long hts_log_print(opt, LOG_ERROR, "Codebase too long, parsing skipped (%s)", - URL_NOLOGIN(opt, codebase)); + url_nologin(opt, codebase)); } { @@ -4953,7 +4954,8 @@ int htsAddLink(htsmoduleStruct * str, char *link) { hts_log_print( opt, LOG_DEBUG, "(module): relative link at %s build with %s and %s: %s", - adr_nologin(afs.af.adr), afs.save, savename(), tempo); + adr_nologin(opt, afs.af.adr), afs.save, savename(), + tempo); if (str->localLink && str->localLinkSize > (int) strlen(tempo) + 1) { strlcpybuff(str->localLink, tempo, str->localLinkSize); @@ -4965,7 +4967,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) { if (forbidden_url) { hts_log_print(opt, LOG_DEBUG, "(module): file not caught: %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); if (str->localLink && str->localLinkSize > (int) (strlen(afs.af.adr) + strlen(afs.af.fil) + 8)) { htsbuff lb = htsbuff_ptr(str->localLink, str->localLinkSize); @@ -4979,8 +4981,8 @@ int htsAddLink(htsmoduleStruct * str, char *link) { // if (r != -1) { hts_log_print(opt, LOG_DEBUG, "(module): %s%s -> %s (base %s)", - adr_nologin(afs.af.adr), afs.af.fil, afs.save, - URL_NOLOGIN(opt, codebase)); + adr_nologin(opt, afs.af.adr), afs.af.fil, afs.save, + url_nologin(opt, codebase)); // modifié par rapport à l'autre version (cf prio_fix notamment et save2) @@ -5025,7 +5027,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) { heap_top()->retry = opt->retry; hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s", - adr_nologin(heap_top()->adr), heap_top()->fil, + adr_nologin(opt, heap_top()->adr), heap_top()->fil, heap_top()->sav); } } diff --git a/src/htscoremain.c b/src/htscoremain.c index cd8b12a0f..e1562e5a8 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -385,6 +385,7 @@ static hts_boolean write_progress_lock(httrackp *opt, const char *path, static int hts_main_internal(int argc, char **argv, httrackp * opt); static hts_boolean cmdl_shortopt_has(const char *s, char c); +static hts_boolean cmdl_shortopt_named(const char *s, const char *opt); static void cmdl_doit_line(String *line, const cmdl_argv *cmd); // Main, récupère les paramètres et appelle le robot @@ -2997,6 +2998,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { // un petit lock-file pour indiquer un miroir en cours, ainsi qu'un éventuel fichier log { char t[256]; + char *masked_url; + const char *shown_url; time_local_rfc822(t); // faut bien que ca serve quelque part l'heure RFC1945 arf' @@ -3107,30 +3110,32 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { "engine: could not create hts-in_progress.lock, so" " this mirror cannot be stopped or paused by a file"); } - // fichier log + masked_url = hts_mask_passwords(url, HTS_MASK_LIST); + shown_url = masked_url != NULL ? masked_url : "(out of memory)"; + // hts-log.txt if (opt->log) { - char *masked_url = hts_mask_passwords(url, HTS_MASK_LIST); int i, opt_at = 0; fprintf(opt->log, "HTTrack" HTTRACK_VERSION "%s launched on %s at %s" LF, - hts_get_version_info(opt), t, - masked_url != NULL ? masked_url : "(out of memory)"); - freet(masked_url); + hts_get_version_info(opt), t, shown_url); fprintf(opt->log, "("); for(i = 0; i < argc; i++) { const char *arg = argv[i]; // already UTF-8 on every platform // quoted if it holds a blank and no quote const hts_boolean quote = strchr(arg, ' ') != NULL && strchr(arg, '\"') == NULL; - hts_boolean addr; // a start URL, or a -P value, is one address + // a start URL, or a -P, -%mu or -%R value, is one address + hts_boolean addr; char *masked; if (x_cmd.span[i] != 0) { opt_at = x_cmd.span[i] > 1 ? i : 0; addr = HTS_FALSE; } else if (opt_at != 0 && i < opt_at + x_cmd.span[opt_at]) { - addr = cmdl_shortopt_has(argv[opt_at], 'P'); + addr = cmdl_shortopt_has(argv[opt_at], 'P') || + cmdl_shortopt_named(argv[opt_at], "-%mu") || + cmdl_shortopt_named(argv[opt_at], "-%R"); } else { opt_at = 0; addr = i != 0 && arg[0] != '+' && arg[0] != '-'; @@ -3159,14 +3164,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { HTTRACK_VERSION "%s " HTTRACK_AFF_AUTHORS "" LF, t, hts_get_version_info(opt)); if (opt->wizard == HTS_WIZARD_NONE) { - printf - ("mirroring %s with %d levels, %d sockets,t=%d,s=%d,logm=%d,lnk=%d,mdg=%d\n", - url, opt->depth, opt->maxsoc, opt->travel, opt->seeker, - httrack_logmode, opt->urlmode, opt->getmode); + printf("mirroring %s with %d levels, %d " + "sockets,t=%d,s=%d,logm=%d,lnk=%d,mdg=%d\n", + shown_url, opt->depth, opt->maxsoc, opt->travel, opt->seeker, + httrack_logmode, opt->urlmode, opt->getmode); } else { // the magic wizard - printf("mirroring %s with the wizard help..\n", url); + printf("mirroring %s with the wizard help..\n", shown_url); } } + freet(masked_url); } io_flush; @@ -3458,13 +3464,20 @@ static void cmdl_doit_line(String *line, const cmdl_argv *cmd) { /* Does the short-option cluster s carry c from the main option set (-i, -iC2, -%Mi)? The i of -%i belongs to another set. */ static hts_boolean cmdl_shortopt_has(const char *s, char c) { + const char opt[3] = {'-', c, '\0'}; + + return cmdl_shortopt_named(s, opt); +} + +/* Does the short option cluster s hold the option opt, such as "-%mu"? */ +static hts_boolean cmdl_shortopt_named(const char *s, const char *opt) { char name[8]; size_t at = 1; if (s[0] != '-' || s[1] == '-') return HTS_FALSE; while (optcluster_next(s, &at, name, sizeof(name))) - if (name[1] == c && name[2] == '\0') + if (strcmp(name, opt) == 0) return HTS_TRUE; return HTS_FALSE; } diff --git a/src/htsftp.c b/src/htsftp.c index 7a1c8de07..dee8b47d6 100644 --- a/src/htsftp.c +++ b/src/htsftp.c @@ -639,7 +639,7 @@ static hts_boolean ftp_may_resume(httrackp *opt, const lien_back *back, if (back->r.totalsize <= back->range_req_size) { hts_log_print(opt, LOG_DEBUG, "FTP: not resuming %s%s, the local copy is not shorter", - adr_nologin(back->url_adr), back->url_fil); + adr_nologin(opt, back->url_adr), back->url_fil); return HTS_FALSE; } /* Equality, not ordering: a copy this code did not stamp carries a local @@ -648,7 +648,7 @@ static hts_boolean ftp_may_resume(httrackp *opt, const lien_back *back, remote_mtime != local_mtime) { hts_log_print(opt, LOG_DEBUG, "FTP: not resuming %s%s, the copy is not dated as the remote", - adr_nologin(back->url_adr), back->url_fil); + adr_nologin(opt, back->url_adr), back->url_fil); return HTS_FALSE; } return HTS_TRUE; @@ -881,8 +881,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) { SOCaddr_initport(server, port); strcpybuff(back->info, "connect"); - hts_log_print(opt, LOG_DEBUG, "FTP: connecting to %s:%d", adr_nologin(_adr), - port); + hts_log_print(opt, LOG_DEBUG, "FTP: connecting to %s:%d", + adr_nologin(opt, _adr), port); if (!ftp_connect(back, soc_ctl, &server, timeout, opt)) { strcpybuff(back->r.msg, "Unable to connect to the server"); diff --git a/src/htslib.c b/src/htslib.c index 041de4f4f..e5fde0e27 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -2567,7 +2567,7 @@ void treathead_opt(httrackp *opt, t_cookie *cookie, const char *adr, &attr) != 0) { hts_log_print(opt, LOG_WARNING, "Cookies: the jar did not store '%.64s' set by %s", - cook_name, adr_nologin(adr)); + cook_name, adr_nologin(opt, adr)); } } } @@ -4351,8 +4351,55 @@ char *FUN(char *source) { \ return ret != NULL ? source + ( ret - source ) : NULL; \ } -// retourne le pointeur ou le pointeur + offset si il existe dans la chaine un @ signifiant -// une identification +/* Appends n bytes of s at buf[*len], clipped to size - 1, NUL-terminated. */ +static void append_clipped(char *buf, size_t size, size_t *len, const char *s, + size_t n) { + if (n > size - 1 - *len) + n = size - 1 - *len; + memcpy(buf + *len, s, n); + *len += n; + buf[*len] = '\0'; +} + +/* Returns the ':' before the password in the login of adr, or NULL when there + is no password. A user alone stays, as the command-line echo shows it. */ +static const char *password_colon(const char *adr) { + const char *const scheme = jump_protocol_const(adr); + const char *const host = jump_identification_const(adr); + + // "file://" is its own host, ahead of its scheme's end + return host > scheme ? memchr(scheme, ':', (size_t) (host - scheme)) : NULL; +} + +const char *adr_nologin_r(char *buf, size_t size, const char *adr) { + const char *const colon = password_colon(adr); + size_t len = 0; + + if (colon == NULL || size == 0) + return adr; + buf[0] = '\0'; + append_clipped(buf, size, &len, adr, (size_t) (colon - adr)); + append_clipped(buf, size, &len, ":***@", 5); + append_clipped(buf, size, &len, jump_identification_const(adr), + strlen(jump_identification_const(adr))); + return buf; +} + +const char *url_nologin_r(char *buf, size_t size, const char *url) { + return jump_protocol_const(url) != url ? adr_nologin_r(buf, size, url) : url; +} + +const char *adr_nologin(httrackp *opt, const char *adr) { + return password_colon(adr) != NULL + ? adr_nologin_r(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), adr) + : adr; +} + +const char *url_nologin(httrackp *opt, const char *url) { + return jump_protocol_const(url) != url ? adr_nologin(opt, url) : url; +} + +// Returns source past its login (user:pass@), if any. HTSEXT_API const char *jump_identification_const(const char *source) { const char *a, *trytofind; @@ -4367,29 +4414,6 @@ HTSEXT_API const char *jump_identification_const(const char *source) { HTSEXT_API DECLARE_NON_CONST_VERSION(jump_identification) - const char *adr_nologin(const char *adr) { - const char *const host = jump_identification_const(adr); - - return host != jump_protocol_const(adr) ? host : adr; -} - -const char *url_nologin(char *buf, size_t size, const char *url) { - const char *const a = jump_protocol_const(url); - const char *const host = jump_identification_const(url); - size_t n = (size_t) (a - url), m = strlen(host); - - if (a == url || host == a || size == 0) - return url; - if (n > size - 1) - n = size - 1; - if (m > size - 1 - n) - m = size - 1 - n; - memcpy(buf, url, n); - memcpy(buf + n, host, m); - buf[n + m] = '\0'; - return buf; -} - HTSEXT_API const char *jump_normalized_const(const char *source) { if (strcmp(source, "file://") == 0) return source; @@ -5838,6 +5862,28 @@ void fprint_redacted(FILE *fp, size_t len) { static const char *const secret_fields[] = { "Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"}; +/* The -%H log hides the password of the URL these fields carry. */ +static const char *const url_fields[] = {"Location", "Content-Location"}; + +/* Writes the header line buff, len bytes long, with the password of its URL + value as ***. Returns HTS_FALSE, writing nothing, when out of memory. */ +static hts_boolean fprint_url_field(FILE *fp, const char *buff, size_t len) { + const char *const colon = memchr(buff, ':', len); + size_t keep = colon != NULL ? (size_t) (colon - buff) + 1 : len; + char *line = malloct(2 * len + 8); + + if (line == NULL) + return HTS_FALSE; + while (keep < len && (buff[keep] == ' ' || buff[keep] == '\t')) + keep++; + memcpy(line, buff, len); + line[len] = '\0'; + fwrite(line, 1, keep, fp); + fputs(url_nologin_r(line + len + 1, len + 7, line + keep), fp); + freet(line); + return HTS_TRUE; +} + /* Finds the password of the userinfo at the authority a: from the first ':' up to the last '@' before the first '/', the split jump_identification_const() makes. A token also ends at a blank. Returns @@ -5939,9 +5985,14 @@ void fprintfio(FILE *fp, const char *buff, const char *prefix) { if (nl) fprintf(fp, "%s", prefix); nl = HTS_FALSE; - fwrite(buff, 1, keep, fp); - if (keep < len) - fprint_redacted(fp, len - keep); + if (secret || *buff == ' ' || *buff == '\t' || + !header_name_in(buff, url_fields, + sizeof(url_fields) / sizeof(url_fields[0])) || + !fprint_url_field(fp, buff, len)) { + fwrite(buff, 1, keep, fp); + if (keep < len) + fprint_redacted(fp, len - keep); + } } buff += len; if (*buff == '\n') { diff --git a/src/htslib.h b/src/htslib.h index 33b863204..3f3a1fbfb 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -556,17 +556,20 @@ typedef enum { when out of memory. */ char *hts_mask_passwords(const char *s, hts_mask_mode mode); -/* Returns the engine address adr without its login, for a log line: adr - itself when it has none, else the part after the login. */ -const char *adr_nologin(const char *adr); +/* Log calls never print an address raw: pass adr_nologin() or url_nologin(), + which test 660 checks. */ -/* Returns url without its login, for a log line: url itself when it has no - scheme or no login, else a copy clipped to size bytes in buf. */ -const char *url_nologin(char *buf, size_t size, const char *url); +/* Returns adr with its password as ***, for log lines: adr itself when it has + no password, else a copy clipped to size bytes in buf. */ +const char *adr_nologin_r(char *buf, size_t size, const char *adr); -/* url_nologin() into one of the rotating buffers of opt. */ -#define URL_NOLOGIN(OPT, URL) \ - url_nologin(OPT_GET_BUFF(OPT), OPT_GET_BUFF_SIZE(OPT), URL) +/* Same as adr_nologin_r(), but only a URL with a scheme or "//" has a login. */ +const char *url_nologin_r(char *buf, size_t size, const char *url); + +/* adr_nologin_r() and url_nologin_r() into a rotating buffer of opt, which + they take only when there is a login. */ +const char *adr_nologin(httrackp *opt, const char *adr); +const char *url_nologin(httrackp *opt, const char *url); /* Writes the marker that stands for a hidden secret of len bytes. */ void fprint_redacted(FILE *fp, size_t len); diff --git a/src/htslib_selftest.c b/src/htslib_selftest.c index a0122edc8..8f05bbabb 100644 --- a/src/htslib_selftest.c +++ b/src/htslib_selftest.c @@ -1639,41 +1639,53 @@ static int maskpw_pairs(size_t n) { return err; } -/* Do adr_nologin() and url_nologin() drop the login and nothing else? */ -static int nologin_rows(void) { +/* Do the nologin helpers hide the password and nothing else? */ +static int nologin_rows(httrackp *opt) { static const struct { const char *in, *want; - } adrs[] = {{"u:p@h:80", "h:80"}, - {"u:p@q@h", "h"}, + } adrs[] = {{"u:p@h:80", "u:***@h:80"}, + {"u:p@q@h", "u:***@h"}, + {"u@h", "u@h"}, + {":p@h", ":***@h"}, {"h:80", "h:80"}, + {"file://", "file://"}, {"https://h", "https://h"}, - {"https://u:p@h", "h"}}, - urls[] = {{"http://u:p@h/x", "http://h/x"}, - {"//u:p@q@h", "//h"}, - {"HTTPS://u:a b@[::1]:8/", "HTTPS://[::1]:8/"}, + {"https://u:p@h", "https://u:***@h"}}, + urls[] = {{"http://u:p@h/x", "http://u:***@h/x"}, + {"//u:p@q@h", "//u:***@h"}, + {"HTTPS://u:a b@[::1]:8/", "HTTPS://u:***@[::1]:8/"}, {"http://h/x?a@b", "http://h/x?a@b"}, {"mailto:a@b", "mailto:a@b"}, {"u:p@h/x", "u:p@h/x"}, - {"http://u:p@hostname.example/x", "http://hostname.example"}}; - + {"http://u:p@hostname.example/x", "http://u:***@hostname.e"}}; char buf[24]; + const int index = opt->state.concat.index; int err = 0; size_t i; - for (i = 0; i < sizeof(adrs) / sizeof(adrs[0]); i++) - if (strcmp(adr_nologin(adrs[i].in), adrs[i].want) != 0) { - fprintf(stderr, "adr_nologin: '%s' gave '%s'\n", adrs[i].in, - adr_nologin(adrs[i].in)); + for (i = 0; i < sizeof(adrs) / sizeof(adrs[0]); i++) { + const char *got = adr_nologin_r(buf, sizeof(buf), adrs[i].in); + + if (strcmp(got, adrs[i].want) != 0) { + fprintf(stderr, "adr_nologin: '%s' gave '%s'\n", adrs[i].in, got); err = 1; } + } for (i = 0; i < sizeof(urls) / sizeof(urls[0]); i++) { - const char *got = url_nologin(buf, sizeof(buf), urls[i].in); + const char *got = url_nologin_r(buf, sizeof(buf), urls[i].in); if (strcmp(got, urls[i].want) != 0) { fprintf(stderr, "url_nologin: '%s' gave '%s'\n", urls[i].in, got); err = 1; } } + /* An address without a login is returned as is, and takes no buffer. */ + if (adr_nologin(opt, adrs[4].in) != adrs[4].in || + url_nologin(opt, urls[5].in) != urls[5].in || + opt->state.concat.index != index) { + fprintf(stderr, "nologin: a rotating buffer was taken without a login\n"); + err = 1; + } return err; } @@ -1713,7 +1725,6 @@ static int st_maskpw(httrackp *opt, int argc, char **argv) { int err = 0; size_t i; - (void) opt; if (argc >= 2) { char *got = NULL; @@ -1728,7 +1739,7 @@ static int st_maskpw(httrackp *opt, int argc, char **argv) { } for (i = 0; i < sizeof(rows) / sizeof(rows[0]); i++) err |= maskpw_row(rows[i].in, rows[i].want, rows[i].mode); - err |= nologin_rows(); + err |= nologin_rows(opt); err |= maskpw_pairs(300); err |= maskpw_pairs(10000); printf("maskpw self-test: %s\n", err ? "FAIL" : "OK"); diff --git a/src/htsname.c b/src/htsname.c index 7457f3a58..0332fff0f 100644 --- a/src/htsname.c +++ b/src/htsname.c @@ -747,7 +747,7 @@ int url_savename(lien_adrfilsave *const afs, if (r.statuscode != -1) { // cache entry read OK hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s", - adr_nologin(adr_complete), fil_complete); + adr_nologin(opt, adr_complete), fil_complete); if (!HTTP_IS_REDIRECT(r.statuscode)) { const sniff_src src = {sback, NULL, adr, fil, previous_save}; @@ -828,7 +828,7 @@ int url_savename(lien_adrfilsave *const afs, current.adr[0] = current.fil[0] = '\0'; opt->state._hts_in_html_parsing = 2; // test hts_log_print(opt, LOG_DEBUG, "Testing link type %s%s", - adr_nologin(adr_complete), fil_complete); + adr_nologin(opt, adr_complete), fil_complete); strcpybuff(current.adr, adr_complete); strcpybuff(current.fil, fil_complete); // ajouter dans le backing le fichier en mode test @@ -924,7 +924,7 @@ int url_savename(lien_adrfilsave *const afs, hts_log_print(opt, LOG_WARNING, "Loop with HEAD request " "(during prefetch) at %s%s", - adr_nologin(current.adr), + adr_nologin(opt, current.adr), current.fil); } if (!hts_wait_available_socket(sback, opt, @@ -939,8 +939,9 @@ int url_savename(lien_adrfilsave *const afs, "(during prefetch) %s (%d) to link %s at " "%s%s", back[b].r.msg, back[b].r.statuscode, - URL_NOLOGIN(opt, back[b].r.location), - adr_nologin(current.adr), current.fil); + url_nologin(opt, back[b].r.location), + adr_nologin(opt, current.adr), + current.fil); // libérer emplacement backing actuel et attendre le prochain back_maydelete(opt, cache, sback, b); @@ -958,7 +959,7 @@ int url_savename(lien_adrfilsave *const afs, hts_log_print(opt, LOG_DEBUG, "Warning: Savename redirect " "backing error at %s%s", - adr_nologin(moved.adr), + adr_nologin(opt, moved.adr), moved.fil); } } @@ -966,15 +967,15 @@ int url_savename(lien_adrfilsave *const afs, hts_log_print( opt, LOG_WARNING, "Unable to test %s%s (loop to same filename)", - adr_nologin(adr_complete), fil_complete); + adr_nologin(opt, adr_complete), fil_complete); } } } } else { // arrêter les frais - hts_log_print(opt, LOG_WARNING, - "Unable to test %s%s (loop)", - adr_nologin(adr_complete), fil_complete); + hts_log_print( + opt, LOG_WARNING, "Unable to test %s%s (loop)", + adr_nologin(opt, adr_complete), fil_complete); } } // ok, leaving } @@ -1027,7 +1028,7 @@ int url_savename(lien_adrfilsave *const afs, #endif hts_log_print(opt, LOG_ERROR, "Unexpected savename backing error at %s%s", - adr_nologin(adr), fil_complete); + adr_nologin(opt, adr), fil_complete); } // restaurer opt->state._hts_in_html_parsing = hihp; @@ -1058,10 +1059,10 @@ int url_savename(lien_adrfilsave *const afs, if ((opt->debug > 1) && (opt->log != NULL)) { if (ext_chg == 1) hts_log_print(opt, LOG_DEBUG, "Changing link extension %s%s to .%s", - adr_nologin(adr_complete), fil_complete, ext); + adr_nologin(opt, adr_complete), fil_complete, ext); else hts_log_print(opt, LOG_DEBUG, "Changing link name %s%s to %s", - adr_nologin(adr_complete), fil_complete, ext); + adr_nologin(opt, adr_complete), fil_complete, ext); } if (ext_chg == 1) { // Cut the old extension only when it is empty (a bare trailing dot), the @@ -1527,7 +1528,7 @@ int url_savename(lien_adrfilsave *const afs, if (*a != '.') { // agh pas de point strcatbuff(afs->save, ".html"); // préférable! hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s", - adr_nologin(adr_complete), fil_complete, afs->save); + adr_nologin(opt, adr_complete), fil_complete, afs->save); } } // effacer pass au besoin pour les autentifications @@ -1672,7 +1673,7 @@ int url_savename(lien_adrfilsave *const afs, } hts_log_print(opt, LOG_DEBUG, "engine: save-name: local name: %s%s -> %s", - adr_nologin(adr), fil, afs->save); + adr_nologin(opt, adr), fil, afs->save); /* Ensure that the MANDATORY "temporary" extension is set */ if (ext_chg_delayed) { @@ -1823,13 +1824,13 @@ int url_savename(lien_adrfilsave *const afs, free(saveFinal); } else { hts_log_print(opt, LOG_ERROR, "Could not revert to UTF-8: %s%s", - adr_nologin(adr_complete), fil_complete); + adr_nologin(opt, adr_complete), fil_complete); } free(wsave); // log in debug hts_log_print(opt, LOG_DEBUG, "Too long filename shortened: %s%s => %s", - adr_nologin(adr_complete), fil_complete, afs->save); + adr_nologin(opt, adr_complete), fil_complete, afs->save); } else { hts_log_print(opt, LOG_ERROR, "Could not read UTF-8: %s", afs->save); } diff --git a/src/htsparse.c b/src/htsparse.c index b90cf0570..53cf7dcf2 100644 --- a/src/htsparse.c +++ b/src/htsparse.c @@ -337,6 +337,28 @@ static const char *html_inline_safe(const char *src, char *dst, size_t size) { /* Drop a redirect Location's #fragment: a UA anchor, never part of the fetched * resource (#204). */ +/* Copies the first 32 bytes of the script text s, len bytes long, into buf + for a debug line, with a URL login masked, or "(hidden)" when a login + cannot be told apart. */ +static const char *script_excerpt(httrackp *opt, char *buf, size_t size, + const char *s, size_t len) { + char *copy = malloct(len + 1); + const char *shown; + + buf[0] = '\0'; + if (copy == NULL) + return buf; + memcpy(copy, s, len); + copy[len] = '\0'; + shown = url_nologin(opt, copy); + if (shown == copy && jump_protocol_const(copy) == copy && + strchr(copy, '@') != NULL) + shown = "(hidden)"; + strlncatbuff(buf, shown, size, minimum(strlen(shown), 32)); + freet(copy); + return buf; +} + static void url_drop_fragment(char *const url) { char *const frag = strchr(url, '#'); if (frag != NULL) @@ -893,7 +915,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { int cSize = (int) r->size; hts_log_print(opt, LOG_DEBUG, "engine: preprocess-html: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); if (RUN_CALLBACK4(opt, preprocess, &cAddr, &cSize, urladr(), urlfil()) == 1) { r->adr = cAddr; r->size = cSize; @@ -906,7 +928,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_boolean sf_may_mark = HTS_FALSE; // --single-file: mark this one? hts_log_print(opt, LOG_DEBUG, "scanning file %s%s (%s)..", - adr_nologin(urladr()), urlfil(), savename()); + adr_nologin(opt, urladr()), urlfil(), savename()); /* Hack to avoid NULL char problems with C syntax */ /* Yes, some bogus HTML pages can embed null chars @@ -1654,14 +1676,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { } if ((opt->debug > 1) && (opt->log != NULL)) { - char str[512]; + char str[64]; - str[0] = '\0'; - strncatbuff(str, b, minimum((int) (a - b + 1), 32)); - hts_log_print(opt, LOG_DEBUG, - "active code (%s) detected in javascript: %s", - (check_this_fking_line == - 2) ? "parse" : "pickup", str); + hts_log_print( + opt, LOG_DEBUG, + "active code (%s) detected in javascript: %s", + (check_this_fking_line == 2) ? "parse" + : "pickup", + script_excerpt(opt, str, sizeof(str), b, + (size_t) (a - b + 1))); } } @@ -1899,7 +1922,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print(opt, LOG_WARNING, "Link %s%s not scanned " "(follow robots meta tag)", - adr_nologin(urladr()), + adr_nologin(opt, urladr()), urlfil()); } } @@ -1956,13 +1979,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (hts_js_scan_link(opt, html, r->adr, inscript_tag, inscript_tag_lastc, incss, &jslink)) { if ((opt->debug > 1) && (opt->log != NULL)) { - char str[512]; + char str[64]; - str[0] = '\0'; - strncatbuff(str, html + jslink.offset, - minimum(jslink.length, 32)); hts_log_print(opt, LOG_DEBUG, - "link detected in javascript: %s", str); + "link detected in javascript: %s", + script_excerpt(opt, str, sizeof(str), + html + jslink.offset, + (size_t) jslink.length)); } p = jslink.offset; // p non nul: TRAITER CHAINE COMME // FICHIER @@ -2386,7 +2409,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { // 1: interdit (patcher tout de même adresse) hts_log_print(opt, LOG_DEBUG, "link detected in html (tag): %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); // external check if (!RUN_CALLBACK1(opt, linkdetected, lien) @@ -2394,7 +2417,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { error = 1; // erreur hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } #if HTS_STRIP_DOUBLE_SLASH // supprimer les // en / (sauf pour http://) @@ -2463,7 +2486,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { error = 1; // erreur hts_log_print(opt, LOG_DEBUG, "link rejected (multiple-archive) %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } } @@ -2504,7 +2527,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_DEBUG, "link rejected (control bytes do not fit) %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } } @@ -2518,8 +2541,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print(opt, LOG_DEBUG, "engine: save-name: '%s' charset " "conversion from '%s' to '%s'", - charset, URL_NOLOGIN(opt, lien), - URL_NOLOGIN(opt, s)); + charset, url_nologin(opt, lien), + url_nologin(opt, s)); /* One byte can become three, and a clipped URL names another resource, so drop the link instead. */ if (strlen(s) < sizeof(lien) - HTS_LINK_TAIL_ROOM) { @@ -2529,7 +2552,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_WARNING, "Link is too long after charset conversion: %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } free(s); } @@ -2539,7 +2562,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (hts_unescapeEntities(lien, lien, strlen(lien) + 1) != 0) { hts_log_print(opt, LOG_WARNING, "could not decode URI '%s' with charset '%s'", - URL_NOLOGIN(opt, lien), charset); + url_nologin(opt, lien), charset); } // decode query entities with the page charset, out-of-place @@ -2568,7 +2591,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { } else { hts_log_print(opt, LOG_WARNING, "could not URL-decode string '%s'", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } // we need to encode query string non-ascii chars, @@ -2585,7 +2608,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { error = 1; hts_log_print(opt, LOG_DEBUG, "link rejected (query does not fit) %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } } } @@ -2676,7 +2699,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { // si c'est un chemin, alors vérifier (toto/toto.html -> http://www/toto/) if (!error) { hts_log_print(opt, LOG_DEBUG, "position link check %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); if ((p_type == 2) || (p_type == -2)) { // code ou codebase // Vérifier les codebase=applet (au lieu de applet/) @@ -2719,7 +2742,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { error = 1; hts_log_print(opt, LOG_WARNING, "Link is too long once resolved: %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } else { strcpybuff(lien, "http://"); strcatbuff(lien, af2.adr); @@ -2774,7 +2797,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_DEBUG, "code/codebase link %s base %s", - URL_NOLOGIN(opt, lien), URL_NOLOGIN(opt, base)); + url_nologin(opt, lien), url_nologin(opt, base)); } } else { @@ -2805,13 +2828,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_DEBUG, "link modified with code/codebase %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } } else { error = 1; // erreur hts_log_print(opt, LOG_ERROR, "Link %s too long with base href", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } } else { lien_adrfil baseaf; @@ -2831,12 +2854,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_DEBUG, "link modified with code/codebase %s", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } else { error = 1; // erreur hts_log_print(opt, LOG_ERROR, "Link %s too long with base href", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } } } @@ -2852,8 +2875,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_DEBUG, "build relative link %s with %s%s", - URL_NOLOGIN(opt, lien), adr_nologin(relativeurladr()), - relativeurlfil()); + url_nologin(opt, lien), + adr_nologin(opt, relativeurladr()), relativeurlfil()); if ((reponse = ident_url_relatif(lien, relativeurladr(), relativeurlfil(), &afs.af)) < 0) { @@ -2862,25 +2885,26 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print(opt, LOG_WARNING, "Link %s not caught (protocol not " "supported by this build)", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); } else { hts_log_print(opt, LOG_DEBUG, "ident_url_relatif failed for %s with %s%s", - URL_NOLOGIN(opt, lien), - adr_nologin(relativeurladr()), + url_nologin(opt, lien), + adr_nologin(opt, relativeurladr()), relativeurlfil()); } } else { - hts_log_print( - opt, LOG_DEBUG, - "built relative link %s with %s%s -> %s%s", - URL_NOLOGIN(opt, lien), adr_nologin(relativeurladr()), - relativeurlfil(), adr_nologin(afs.af.adr), afs.af.fil); + hts_log_print(opt, LOG_DEBUG, + "built relative link %s with %s%s -> %s%s", + url_nologin(opt, lien), + adr_nologin(opt, relativeurladr()), + relativeurlfil(), + adr_nologin(opt, afs.af.adr), afs.af.fil); } } else { hts_log_print(opt, LOG_DEBUG, "link %s not build, error detected before", - URL_NOLOGIN(opt, lien)); + url_nologin(opt, lien)); afs.af.adr[0] = '\0'; } @@ -2889,7 +2913,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (p_nocatch) { forbidden_url = 1; // interdire récupération du lien hts_log_print(opt, LOG_DEBUG, "link forced external at %s%s", - adr_nologin(afs.af.adr), afs.af.fil); + adr_nologin(opt, afs.af.adr), afs.af.fil); } // Tester si un lien doit être accepté ou refusé (wizard) // forbidden_url=1 : lien refusé @@ -2899,7 +2923,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (afs.af.adr[0] != '\0') { hts_log_print(opt, LOG_DEBUG, "wizard link test at %s%s..", - adr_nologin(afs.af.adr), afs.af.fil); + adr_nologin(opt, afs.af.adr), afs.af.fil); forbidden_url = hts_acceptlink(opt, ptr, afs.af.adr, afs.af.fil, intag_name ? intag_name : NULL, @@ -2963,7 +2987,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_DEBUG, "wizard moved link retest at %s%s..", - adr_nologin(afs.af.adr), afs.af.fil); + adr_nologin(opt, afs.af.adr), afs.af.fil); forbidden_url = hts_acceptlink( opt, ptr, afs.af.adr, afs.af.fil, intag_name ? intag_name : NULL, @@ -3026,25 +3050,27 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (forbidden_url != 1) { // le lien va être chargé if ((p_type == 2) || (p_type == -2)) { // base href ou codebase, pas un lien hts_log_print(opt, LOG_DEBUG, "Code/Codebase: %s%s", - adr_nologin(afs.af.adr), afs.af.fil); + adr_nologin(opt, afs.af.adr), + afs.af.fil); } else if ((opt->getmode & HTS_GETMODE_HTML_FIRST) == 0) { hts_log_print(opt, LOG_DEBUG, "Record: %s%s -> %s", - adr_nologin(afs.af.adr), afs.af.fil, - afs.save); + adr_nologin(opt, afs.af.adr), + afs.af.fil, afs.save); } else { if (!ishtml(opt, afs.af.fil)) - hts_log_print( - opt, LOG_DEBUG, "Record after: %s%s -> %s", - adr_nologin(afs.af.adr), afs.af.fil, afs.save); + hts_log_print(opt, LOG_DEBUG, + "Record after: %s%s -> %s", + adr_nologin(opt, afs.af.adr), + afs.af.fil, afs.save); else hts_log_print(opt, LOG_DEBUG, "Record: %s%s -> %s", - adr_nologin(afs.af.adr), afs.af.fil, - afs.save); + adr_nologin(opt, afs.af.adr), + afs.af.fil, afs.save); } } else hts_log_print(opt, LOG_DEBUG, "External: %s%s", - adr_nologin(afs.af.adr), afs.af.fil); + adr_nologin(opt, afs.af.adr), afs.af.fil); } /* FIN log */ @@ -3432,7 +3458,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_DEBUG, "relative link at %s build with %s and %s: %s", - adr_nologin(afs.af.adr), afs.save, + adr_nologin(opt, afs.af.adr), afs.save, relativesavename(), tempo); // lien applet (code) - il faut placer un codebase avant @@ -3601,8 +3627,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print( opt, LOG_DEBUG, "merging similar links %s%s and %s%s", - adr_nologin(afs.af.adr), afs.af.fil, - adr_nologin(heap(i)->adr), heap(i)->fil); + adr_nologin(opt, afs.af.adr), afs.af.fil, + adr_nologin(opt, heap(i)->adr), + heap(i)->fil); } } heap(i)->depth = @@ -3654,9 +3681,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { ("robots.txt: added file robots.txt for %s\n", adr); #endif - hts_log_print(opt, LOG_DEBUG, - "robots.txt added at %s", - adr_nologin(afs.af.adr)); + hts_log_print( + opt, LOG_DEBUG, + "robots.txt added at %s", + adr_nologin(opt, afs.af.adr)); } else { hts_log_print(opt, LOG_ERROR, "Unexpected robots.txt error at %d", @@ -3713,11 +3741,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (!just_test_it) { hts_log_print(opt, LOG_DEBUG, "OK, NOTE: %s%s -> %s", - adr_nologin(heap_top()->adr), + adr_nologin(opt, heap_top()->adr), heap_top()->fil, heap_top()->sav); } else { hts_log_print(opt, LOG_DEBUG, "OK, TEST: %s%s", - adr_nologin(heap_top()->adr), + adr_nologin(opt, heap_top()->adr), heap_top()->fil); } @@ -3873,7 +3901,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { int cSize = (int) TypedArraySize(output_buffer); hts_log_print(opt, LOG_DEBUG, "engine: postprocess-html: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); if (RUN_CALLBACK4(opt, postprocess, &cAddr, &cSize, urladr(), urlfil()) == 1) { hts_log_print(opt, LOG_DEBUG, "engine: postprocess-html: callback modified data, applying %d bytes", cSize); @@ -3888,7 +3916,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print(opt, LOG_ERROR, "engine: postprocess-html: callback returned %d " "bytes for %s%s, page skipped", - cSize, adr_nologin(urladr()), urlfil()); + cSize, adr_nologin(opt, urladr()), urlfil()); HT_SET_SIZE(0); error = 1; } else if (hts_postprocess_reply_inplace( @@ -3907,7 +3935,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { if (output_oom) { hts_log_print(opt, LOG_ERROR, "Not enough memory to rewrite %s%s, page skipped", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); error = 1; } else if (TypedArraySize(output_buffer) != 0) { hts_finish_html_file( @@ -4046,7 +4074,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str, if (HTTP_IS_REDIRECT(r->statuscode)) { hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); { char BIGSTK mov_url[HTS_URLMAXSIZE * 2]; @@ -4081,7 +4109,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str, get_it = -1; // ne rien faire hts_log_print(opt, LOG_WARNING, "Can not bear crazy server (%s) for %s%s", r->msg, - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } else { // mauvaise casse, effacer entrée dans la pile et rejouer une fois get_it = 1; } @@ -4090,14 +4118,14 @@ int hts_mirror_check_moved(htsmoduleStruct * str, // -> on prend à cette adresse, le lien sera enregistré avec lien_record() (hash) hts_log_print(opt, LOG_DEBUG, "wizard link test for moved file at %s%s..", - adr_nologin(moved->adr), moved->fil); + adr_nologin(opt, moved->adr), moved->fil); // accepté? if (hts_acceptlink(opt, ptr, moved->adr, moved->fil, NULL, NULL, HTS_FALSE, &set_prio_to, NULL) != 1) { /* nouvelle adresse non refusée ? */ get_it = 1; hts_log_print(opt, LOG_DEBUG, "moved link accepted: %s%s", - adr_nologin(moved->adr), moved->fil); + adr_nologin(opt, moved->adr), moved->fil); } } else if (same_savefile) { // A stub would point at itself; follow the redirect instead. @@ -4107,8 +4135,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str, hts_log_print(opt, LOG_WARNING, "Redirect to a same-file alias, fetching real " "content: %s%s -> %s%s", - adr_nologin(urladr()), urlfil(), - adr_nologin(moved->adr), moved->fil); + adr_nologin(opt, urladr()), urlfil(), + adr_nologin(opt, moved->adr), moved->fil); } } /* sinon traité normalement */ } @@ -4118,8 +4146,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str, // et reculer pour mieux sauter hts_log_print(opt, LOG_WARNING, "Warning moved treated for %s%s (real one is %s%s)", - adr_nologin(urladr()), urlfil(), - adr_nologin(moved->adr), moved->fil); + adr_nologin(opt, urladr()), urlfil(), + adr_nologin(opt, moved->adr), moved->fil); // canceller lien actuel error = 1; hts_invalidate_link(opt, ptr); // invalidate hashtable entry @@ -4178,8 +4206,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str, char *const rn = (char *) malloct(rn_size); if (rn != NULL) { hts_log_print(opt, LOG_WARNING, "File has moved from %s%s to %s", - adr_nologin(urladr()), urlfil(), - URL_NOLOGIN(opt, mov_url)); + adr_nologin(opt, urladr()), urlfil(), + url_nologin(opt, mov_url)); if (!opt->mimehtml) { inplace_escape_uri(mov_url, sizeof(mov_url)); } else { @@ -4252,7 +4280,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str, // gone; a failed removal gives up rather than looping. if (had_partial && ref_gone && !fexist_utf8(heap(ptr)->sav)) { hts_log_print(opt, LOG_DEBUG, "Partial file reget (%s) for %s%s", - r->msg, adr_nologin(urladr()), urlfil()); + r->msg, adr_nologin(opt, urladr()), urlfil()); if (hts_record_link(opt, heap(ptr)->adr, heap(ptr)->fil, heap(ptr)->sav, "", "", NULL)) { heap_top()->testmode = heap(ptr)->testmode; heap_top()->link_import = 0; @@ -4272,7 +4300,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str, } else { hts_log_print(opt, LOG_WARNING, "Giving up on partial reget (%s) for %s%s", r->msg, - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); error = 1; } @@ -4295,11 +4323,11 @@ int hts_mirror_check_moved(htsmoduleStruct * str, if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) && (heap(ptr)->retry <= 0)) { hts_log_print(opt, LOG_DEBUG, "Link banned: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); host_ban(opt, ptr, sback, jump_identification_const(urladr())); hts_log_print(opt, LOG_DEBUG, "Info: previous log - link banned: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } else can_retry = 1; } else @@ -4309,11 +4337,11 @@ int hts_mirror_check_moved(htsmoduleStruct * str, if ((opt->hostcontrol) && (heap(ptr)->retry <= 0)) { // too slow if (opt->hostcontrol & HTS_HOSTCONTROL_BAN_SLOW) { hts_log_print(opt, LOG_DEBUG, "Link banned: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); host_ban(opt, ptr, sback, jump_identification_const(urladr())); hts_log_print(opt, LOG_DEBUG, "Info: previous log - link banned: %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } else can_retry = 1; } else @@ -4357,25 +4385,25 @@ int hts_mirror_check_moved(htsmoduleStruct * str, hts_log_print( opt, LOG_ERROR, "\"%s\" (%d) after %d retries at link %s%s (from %s%s)", r->msg, - r->statuscode, opt->retry, adr_nologin(urladr()), urlfil(), - adr_nologin(heap(heap(ptr)->precedent)->adr), + r->statuscode, opt->retry, adr_nologin(opt, urladr()), urlfil(), + adr_nologin(opt, heap(heap(ptr)->precedent)->adr), heap(heap(ptr)->precedent)->fil); } else { if (r->statuscode == STATUSCODE_TEST_OK) { // test OK hts_log_print(opt, LOG_INFO, "Test OK at link %s%s (from %s%s)", - adr_nologin(urladr()), urlfil(), - adr_nologin(heap(heap(ptr)->precedent)->adr), + adr_nologin(opt, urladr()), urlfil(), + adr_nologin(opt, heap(heap(ptr)->precedent)->adr), heap(heap(ptr)->precedent)->fil); } else { if (strcmp(urlfil(), "/robots.txt")) { // ne pas afficher d'infos sur robots.txt par défaut - hts_log_print(opt, LOG_ERROR, - "\"%s\" (%d) at link %s%s (from %s%s)", r->msg, - r->statuscode, adr_nologin(urladr()), urlfil(), - adr_nologin(heap(heap(ptr)->precedent)->adr), - heap(heap(ptr)->precedent)->fil); + hts_log_print( + opt, LOG_ERROR, "\"%s\" (%d) at link %s%s (from %s%s)", + r->msg, r->statuscode, adr_nologin(opt, urladr()), urlfil(), + adr_nologin(opt, heap(heap(ptr)->precedent)->adr), + heap(heap(ptr)->precedent)->fil); } else { hts_log_print(opt, LOG_DEBUG, "No robots.txt rules at %s", - adr_nologin(urladr())); + adr_nologin(opt, urladr())); } } } @@ -4433,16 +4461,16 @@ int hts_mirror_check_moved(htsmoduleStruct * str, hts_log_print(opt, LOG_NOTICE, "Restarting whole file after error %d (%s) at link " "%s%s (from %s%s)", - r->statuscode, r->msg, adr_nologin(urladr()), + r->statuscode, r->msg, adr_nologin(opt, urladr()), urlfil(), - adr_nologin(heap(heap(ptr)->precedent)->adr), + adr_nologin(opt, heap(heap(ptr)->precedent)->adr), heap(heap(ptr)->precedent)->fil); } else { hts_log_print(opt, LOG_NOTICE, "Retry after error %d (%s) at link %s%s (from %s%s)", - r->statuscode, r->msg, adr_nologin(urladr()), + r->statuscode, r->msg, adr_nologin(opt, urladr()), urlfil(), - adr_nologin(heap(heap(ptr)->precedent)->adr), + adr_nologin(opt, heap(heap(ptr)->precedent)->adr), heap(heap(ptr)->precedent)->fil); } // re-request the file @@ -4464,7 +4492,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str, } } else { hts_log_print(opt, LOG_DEBUG, "Info: no robots.txt at %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } if (!store_errpage) { if (r->adr) { // désalloc @@ -4625,7 +4653,7 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str, if (forbidden_url == 1 || IS_DELAYED_EXT(add.save)) { hts_log_print(opt, LOG_NOTICE, "Link %s%s not added after user request", - adr_nologin(add.af.adr), add.af.fil); + adr_nologin(opt, add.af.adr), add.af.fil); } else if (hash_read(hash, add.save, NULL, HASH_STRUCT_FILENAME) < 0) { // no existing record // record the link @@ -4640,7 +4668,7 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str, heap_top()->precedent = heap_top_index(); // hts_log_print(opt, LOG_INFO, "Link added by user: %s%s", - adr_nologin(add.af.adr), add.af.fil); + adr_nologin(opt, add.af.adr), add.af.fil); // } else if (hts_maxlinks_no_room(opt)) { /* Give the parser's verdict rather than dropping the rest of @@ -4655,12 +4683,12 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str, } else { hts_log_print(opt, LOG_NOTICE, "Existing link %s%s not added after user request", - adr_nologin(add.af.adr), add.af.fil); + adr_nologin(opt, add.af.adr), add.af.fil); } } } else { hts_log_print(opt, LOG_ERROR, "Error during URL decoding for %s", - URL_NOLOGIN(opt, add_url)); + url_nologin(opt, add_url)); } // ----Fin Ajout---- } @@ -4725,7 +4753,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, /* This is not supposed to hapen. */ if (heap(ptr)->pass2 == -1) { hts_log_print(opt, LOG_WARNING, "Link is already ready %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } /* User interaction */ @@ -4738,7 +4766,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, /* Done while processing user interactions ? */ if (heap(ptr)->pass2 == -1) { hts_log_print(opt, LOG_DEBUG, "Link is now ready %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); // We are ready return 2; // goto jump_if_done; } @@ -4757,7 +4785,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, printf("error while crash adding\n"); #endif hts_log_print(opt, LOG_ERROR, "Unexpected backing error for %s%s", - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } } #if BDEBUG==1 @@ -5007,7 +5035,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, if (b < 0) { hts_log_print(opt, LOG_DEBUG, "link #%d is ready, no more on the stack, skipping: %s%s..", - ptr, adr_nologin(urladr()), urlfil()); + ptr, adr_nologin(opt, urladr()), urlfil()); return 2; // goto jump_if_done; } @@ -5122,7 +5150,7 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, cookie_before[16384]; /* #15: this URL's Cookie header, pre-request */ hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", - adr_nologin(afs->af.adr), afs->af.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil); /* Follow while type is unknown and redirects occurs */ for(loops = 0, continue_loop = 1; @@ -5168,7 +5196,7 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, hts_log_print( opt, LOG_DEBUG, "link forbidden because of MIME types restrictions: %s%s", - adr_nologin(afs->af.adr), afs->af.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil); break; // exit loop } } @@ -5185,7 +5213,7 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, hts_log_print( opt, LOG_ERROR, "Duplicate entry in hts_wait_delayed() cancelled: %s%s -> %s", - adr_nologin(afs->af.adr), afs->af.fil, afs->save); + adr_nologin(opt, afs->af.adr), afs->af.fil, afs->save); } /* Exit loop (we're done) */ continue_loop = 0; @@ -5246,7 +5274,7 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, hts_log_print( opt, LOG_DEBUG, "link forbidden because of MIME types restrictions: %s%s", - adr_nologin(afs->af.adr), afs->af.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil); break; // exit loop } } @@ -5374,8 +5402,8 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, if (ident_url_relatif(mov_url, afs->af.adr, afs->af.fil, &moved) >= 0) { hts_log_print(opt, LOG_DEBUG, "Redirect while resolving type: %s%s -> %s%s", - adr_nologin(afs->af.adr), afs->af.fil, - adr_nologin(moved.adr), moved.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil, + adr_nologin(opt, moved.adr), moved.fil); // si non bouclage sur soi même, ou si test avec GET non testé if (strcmp(moved.adr, afs->af.adr) != 0 || strcmp(moved.fil, afs->af.fil) != 0) { @@ -5399,8 +5427,8 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, hts_log_print(opt, LOG_DEBUG, "link forbidden because of redirect beyond " "the mirror scope at %s%s -> %s%s", - adr_nologin(afs->af.adr), afs->af.fil, - adr_nologin(moved.adr), moved.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil, + adr_nologin(opt, moved.adr), moved.fil); strcpybuff(afs->af.adr, moved.adr); strcpybuff(afs->af.fil, moved.fil); mov_url[0] = '\0'; @@ -5441,13 +5469,13 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, } else { hts_log_print(opt, LOG_WARNING, "Unable to test %s%s (loop to same filename)", - adr_nologin(afs->af.adr), afs->af.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil); } // loop to same location } // ident_url_relatif() } // location } // redirect hts_log_print(opt, LOG_DEBUG, "Final type for %s%s: '%s'", - adr_nologin(afs->af.adr), afs->af.fil, + adr_nologin(opt, afs->af.adr), afs->af.fil, delayed_back.r.contenttype); /* If we are done, do additional checks with final type and authorizations */ @@ -5478,7 +5506,7 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, hts_log_print( opt, LOG_DEBUG, "link forbidden because of MIME types restrictions: %s%s", - adr_nologin(afs->af.adr), afs->af.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil); break; // exit loop } } @@ -5521,12 +5549,12 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, if (in_error == STATUSCODE_TOO_BIG) { hts_log_print(opt, LOG_INFO, "link not taken because of its size (%d bytes) at %s%s", - (int) in_error_size, adr_nologin(afs->af.adr), + (int) in_error_size, adr_nologin(opt, afs->af.adr), afs->af.fil); } else { hts_log_print(opt, LOG_INFO, "link not taken because of error (%d '%s') at %s%s", - in_error, in_error_msg, adr_nologin(afs->af.adr), + in_error, in_error_msg, adr_nologin(opt, afs->af.adr), afs->af.fil); } } @@ -5537,16 +5565,16 @@ int hts_wait_delayed(htsmoduleStruct *str, lien_adrfilsave *afs, if (cancelled) { hts_log_print(opt, LOG_DEBUG, "link cancelled, type unknown, left online: %s%s", - adr_nologin(afs->af.adr), afs->af.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil); } else if (in_error) { hts_log_print(opt, LOG_WARNING, "link in error (%d '%s'), type unknown, aborting: %s%s", - in_error, in_error_msg, adr_nologin(afs->af.adr), + in_error, in_error_msg, adr_nologin(opt, afs->af.adr), afs->af.fil); } else { hts_log_print(opt, LOG_WARNING, "link is probably looping, type unknown, aborting: %s%s", - adr_nologin(afs->af.adr), afs->af.fil); + adr_nologin(opt, afs->af.adr), afs->af.fil); } } diff --git a/src/htsrobots.c b/src/htsrobots.c index e787244ea..ee092ff85 100644 --- a/src/htsrobots.c +++ b/src/htsrobots.c @@ -258,7 +258,7 @@ void robots_parse(httrackp *opt, robots_wizard *robots, const char *adr, hts_log_print(opt, LOG_WARNING, "robots.txt for %s: %d rule(s) not honoured as written, " "starting with '%s' (rules kept up to %d bytes, lines to %d)", - adr_nologin(adr), ndropped, dropped, + adr_nologin(opt, adr), ndropped, dropped, (int) HTS_ROBOTS_MAX_TOKEN_SIZE, (int) HTS_ROBOTS_LINE_SIZE); if (StringNotEmpty(blob)) checkrobots_set(robots, adr, StringBuff(blob)); diff --git a/src/htssitemap.c b/src/htssitemap.c index 245de1cca..683de5bb0 100644 --- a/src/htssitemap.c +++ b/src/htssitemap.c @@ -292,7 +292,7 @@ static hts_boolean sitemap_fetch_allowed(httrackp *opt, const char *adr, refused = (jok == -1) ? HTS_TRUE : HTS_FALSE; if (refused) { hts_log_print(opt, LOG_NOTICE, "Sitemap: filter rule #%d refuses %s%s", - jokdepth + 1, adr_nologin(adr), fil); + jokdepth + 1, adr_nologin(opt, adr), fil); return HTS_FALSE; } /* A Sitemap: line, or a sitemapindex entry, is the site inviting the fetch; @@ -302,7 +302,7 @@ static hts_boolean sitemap_fetch_allowed(httrackp *opt, const char *adr, hts_robots_forbids(opt, adr, fil, (jok != 0) ? HTS_TRUE : HTS_FALSE, refused)) { hts_log_print(opt, LOG_NOTICE, "Sitemap: robots.txt forbids %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); return HTS_FALSE; } return HTS_TRUE; @@ -320,7 +320,7 @@ static hts_boolean sitemap_queue_(httrackp *opt, const char *adr, return HTS_FALSE; if (st->ndocs >= HTS_SITEMAP_MAX_DOCS || level > HTS_SITEMAP_MAX_LEVEL) { hts_log_print(opt, LOG_WARNING, "Sitemap: cap reached, skipping %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); return HTS_FALSE; } if (strlen(adr) >= sizeof(d->adr) || strlen(fil) >= sizeof(d->fil)) @@ -351,7 +351,8 @@ static hts_boolean sitemap_queue_(httrackp *opt, const char *adr, heap_top()->retry = opt->retry; heap_top()->premier = heap_top_index(); heap_top()->precedent = heap_top_index(); - hts_log_print(opt, LOG_INFO, "Sitemap: queued %s%s", adr_nologin(adr), fil); + hts_log_print(opt, LOG_INFO, "Sitemap: queued %s%s", adr_nologin(opt, adr), + fil); return HTS_TRUE; } @@ -371,7 +372,7 @@ void hts_sitemap_redirect(httrackp *opt, const char *adr, const char *fil, /* The engine already queued the target link, so only the marking moves. */ (void) sitemap_queue_(opt, newadr, newfil, d->level, d->src, HTS_FALSE); hts_log_print(opt, LOG_NOTICE, "Sitemap: %s%s redirects to %s%s", - adr_nologin(adr), fil, adr_nologin(newadr), newfil); + adr_nologin(opt, adr), fil, adr_nologin(opt, newadr), newfil); } void hts_sitemap_seed(httrackp *opt, const char *starturl) { @@ -385,7 +386,7 @@ void hts_sitemap_seed(httrackp *opt, const char *starturl) { strcpybuff(url, StringBuff(opt->sitemap_url)); if (strstr(url, ":/") == NULL) hts_log_print(opt, LOG_ERROR, "Sitemap URL must be absolute: %s", - URL_NOLOGIN(opt, url)); + url_nologin(opt, url)); else if (ident_url_absolute(url, &af) >= 0) (void) sitemap_queue(opt, af.adr, af.fil, 0, HTS_SITEMAP_SRC_USER); } @@ -459,7 +460,7 @@ void hts_sitemap_robots(httrackp *opt, const char *adr, const char *sitemaps) { queued++; } hts_log_print(opt, LOG_NOTICE, "Sitemap: %d sitemap(s) queued for %s", queued, - adr_nologin(adr)); + adr_nologin(opt, adr)); } hts_boolean hts_sitemap_pending(httrackp *opt, const char *adr, @@ -524,7 +525,7 @@ static hts_boolean sitemap_seed_child(void *arg, const char *url) { if (!strfield2(af.adr, c->adr)) { hts_log_print(c->opt, LOG_WARNING, "Sitemap: ignoring off-host child sitemap %s%s", - adr_nologin(af.adr), af.fil); + adr_nologin(c->opt, af.adr), af.fil); return HTS_TRUE; } if (sitemap_queue(c->opt, af.adr, af.fil, c->level + 1, @@ -576,16 +577,16 @@ void hts_sitemap_ingest(httrackp *opt, htsmoduleStruct *str, const char *adr, heap(anchor)->depth = saved_depth; if (n < 0) { hts_log_print(opt, LOG_ERROR, "Sitemap: could not decompress %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); return; } if (ctx.is_index) hts_log_print(opt, LOG_NOTICE, "Sitemap: %d of %d child sitemap(s) listed by %s%s", - ctx.accepted, n, adr_nologin(adr), fil); + ctx.accepted, n, adr_nologin(opt, adr), fil); else hts_log_print(opt, LOG_NOTICE, "Sitemap: %d of %d URL(s) added from %s%s", - ctx.accepted, n, adr_nologin(adr), fil); + ctx.accepted, n, adr_nologin(opt, adr), fil); } void hts_sitemap_free(httrackp *opt) { diff --git a/src/htswizard.c b/src/htswizard.c index b70f5b19d..96982abce 100644 --- a/src/htswizard.c +++ b/src/htswizard.c @@ -409,7 +409,7 @@ void hts_wizard_apply_verdict(httrackp *opt, int n, const char *adr, hts_log_print(opt, LOG_WARNING, "(wizard) could not read your answer at %s%s: link refused, " "and the refusal recorded", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); break; case 5: /* this directory and below, or the whole host */ @@ -424,7 +424,7 @@ void hts_wizard_apply_verdict(httrackp *opt, int n, const char *adr, hts_log_print(opt, LOG_WARNING, "(wizard) unknown answer %d at %s%s, keeping the computed " "verdict", - n, adr_nologin(adr), fil); + n, adr_nologin(opt, adr), fil); break; } if (hts_wizard_scope_answer(n) == HTS_TRUE) @@ -436,7 +436,7 @@ void hts_wizard_apply_verdict(httrackp *opt, int n, const char *adr, hts_log_print(opt, LOG_WARNING, "(wizard) %s has no domain above it, answer %d applied " "to the whole host", - adr_nologin(adr), n); + adr_nologin(opt, adr), n); } break; } @@ -486,8 +486,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, // -------------------- PHASE 0 -------------------- /* Infos */ - hts_log_print(opt, LOG_DEBUG, "wizard test begins: %s%s", adr_nologin(adr), - fil); + hts_log_print(opt, LOG_DEBUG, "wizard test begins: %s%s", + adr_nologin(opt, adr), fil); /* Already exists? Then, we know that we knew that this link had to be known */ if (adr[0] != '\0' && fil[0] != '\0' && opt->hash != NULL @@ -514,7 +514,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!ishtml(opt, fil)) { // non il ne faut pas forbidden_url = 1; // interdire récupération du lien hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } @@ -525,7 +525,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, forbidden_url = 1; // interdire récupération du lien hts_log_print(opt, LOG_DEBUG, "file from too far level ignored at %s : %s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } @@ -543,11 +543,11 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, hts_host_same_alias(hts_host_alias_rules(opt), adr, urladr(), hts_host_alias_collapse_www(opt)); if (meme_adresse) - hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s=%s", adr_nologin(adr), - adr_nologin(urladr())); + hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s=%s", + adr_nologin(opt, adr), adr_nologin(opt, urladr())); else - hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s!=%s", adr_nologin(adr), - adr_nologin(urladr())); + hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s!=%s", + adr_nologin(opt, adr), adr_nologin(opt, urladr())); if (meme_adresse) { // même adresse { // tester interdiction de descendre // MODIFIE : en cas de remontée puis de redescente, il se pouvait qu'on ne puisse pas atteindre certains fichiers @@ -586,7 +586,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "same level link authorized: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } // down @@ -606,12 +606,12 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if ((opt->seeker & HTS_SEEKER_DOWN) == 0) { forbidden_url = 1; hts_log_print(opt, LOG_DEBUG, "lower link canceled: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } else { // autorisé à priori - NEW if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "lower link authorized: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } } else if ((test1) || (test2)) { // on peut descendre pour accéder au lien @@ -619,7 +619,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "lower link authorized: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } } @@ -631,12 +631,12 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, forbidden_url = 1; upper_refused = 1; hts_log_print(opt, LOG_DEBUG, "upper link canceled: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } else { // autorisé à monter - NEW if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "upper link authorized: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } } else if ((!strncmp(tempo, "../", 3)) || (!strncmp(tempo2, "../", 3))) { // Possible en montant @@ -644,7 +644,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!heap(ptr)->link_import) { // ne résulte pas d'un 'moved' ACCEPT_LINK(); hts_log_print(opt, LOG_DEBUG, "upper link authorized: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } // sinon autorisé en descente } @@ -692,14 +692,14 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!opt->wizard) { // mode non wizard forbidden_url = 1; // pas même domaine hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } else { if (opt->wizard) { // mode wizard ACCEPT_LINK(); // same domain hts_log_print(opt, LOG_DEBUG, "same domain link authorized: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } @@ -720,13 +720,13 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, forbidden_url = 1; // pas même .xx hts_log_print(opt, LOG_DEBUG, "foreign location link canceled: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } else { if (opt->wizard) { // mode wizard ACCEPT_LINK(); // same domain hts_log_print(opt, LOG_DEBUG, "same location link authorized: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } } else @@ -751,7 +751,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, ACCEPT_LINK(); // authorize may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } // -------------------- PHASE 3-BIS -------------------- @@ -761,7 +761,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, ACCEPT_LINK(); // authorize may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner hts_log_print(opt, LOG_DEBUG, "near link authorized (friendly tag): %s%s", - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } // -------------------- PHASE 4 -------------------- @@ -774,7 +774,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, /* the length, not the path: 2 KB of it is unreadable in a log */ hts_log_print(opt, LOG_DEBUG, "(wizard) link too long, forbidden: %s (%d bytes)", - adr_nologin(adr), (int) strlen(fil)); + adr_nologin(opt, adr), (int) strlen(fil)); REFUSE_LINK(); // URL forbidden } else if (opt->wizard) { // le wizard entre en action.. // @@ -830,12 +830,12 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, hts_log_print(opt, LOG_DEBUG, "(wizard) ambiguous link accepted (external " "depth): link %s at %s%s", - l, adr_nologin(urladr()), urlfil()); + l, adr_nologin(opt, urladr()), urlfil()); } else { hts_log_print(opt, LOG_DEBUG, "(wizard) forced to accept link (external depth): " "link %s at %s%s", - l, adr_nologin(urladr()), urlfil()); + l, adr_nologin(opt, urladr()), urlfil()); } } @@ -863,7 +863,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, hts_log_print( opt, LOG_DEBUG, "(wizard) explicit authorized (%s) link: link %s at %s%s", mdepth, - l, adr_nologin(urladr()), urlfil()); + l, adr_nologin(opt, urladr()), urlfil()); } else if (jok == -1) { // forbidden filters_answer = 1; // décision prise par les filtres question = 0; // ne pas poser de question: @@ -871,7 +871,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, hts_log_print( opt, LOG_DEBUG, "(wizard) explicit forbidden (%s) link: link %s at %s%s", mdepth, - l, adr_nologin(urladr()), urlfil()); + l, adr_nologin(opt, urladr()), urlfil()); } // sinon on touche à rien } } @@ -886,7 +886,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, force_mirror = 5; // mirror (5) hts_log_print(opt, LOG_DEBUG, "(wizard) explicit mirror link: link %s at %s%s", l, - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } } } @@ -898,7 +898,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, REFUSE_LINK(); hts_log_print(opt, LOG_DEBUG, "(wizard) ambiguous forbidden link: link %s at %s%s", l, - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } } // vérifier robots.txt @@ -914,14 +914,14 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, hts_log_print( opt, LOG_DEBUG, "Warning link followed against robots.txt: link %s at %s%s", l, - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } else { REFUSE_LINK(); question = 0; hts_log_print(opt, LOG_DEBUG, "(robots.txt) forbidden link: link %s at %s%s", l, - adr_nologin(adr), fil); + adr_nologin(opt, adr), fil); } } @@ -929,11 +929,11 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (!forbidden_url) { hts_log_print(opt, LOG_DEBUG, "(wizard) shared foreign domain link: link %s at %s%s", l, - adr_nologin(urladr()), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } else { hts_log_print(opt, LOG_DEBUG, "(wizard) cancelled foreign domain link: link %s at %s%s", - l, adr_nologin(urladr()), urlfil()); + l, adr_nologin(opt, urladr()), urlfil()); } #if BDEBUG==3 printf("at %s in %s, wizard says: url %s ", urladr(), urlfil(), l); @@ -1019,7 +1019,7 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, } hts_log_print( opt, LOG_NOTICE, "(wizard) answer '%s' (n=%d) for %s%s: %s%s%s", - s, n, adr_nologin(adr), fil, + s, n, adr_nologin(opt, adr), fil, forbidden_url == 1 ? "forbidden" : forbidden_url == 0 ? "allowed" : "no verdict", @@ -1039,8 +1039,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (strfield(adr, "ftp://") == 0) { // PAS ftp! forbidden_url = 1; // oui oui toujours interdit (note: sert à rien car ==1 mais c pour comprendre) *just_test_it = 1; // mais on teste - hts_log_print(opt, LOG_DEBUG, "Testing link %s%s", adr_nologin(adr), - fil); + hts_log_print(opt, LOG_DEBUG, "Testing link %s%s", + adr_nologin(opt, adr), fil); } } } @@ -1081,12 +1081,12 @@ int hts_acceptmime(httrackp * opt, int ptr, forbidden_url = 0; // URL autorisée hts_log_print(opt, LOG_DEBUG, "(wizard) explicit authorized (%s) link %s%s: mime '%s'", - mdepth, adr_nologin(adr), fil, mime); + mdepth, adr_nologin(opt, adr), fil, mime); } else if (jok == -1) { // forbidden forbidden_url = 1; // URL interdite hts_log_print(opt, LOG_DEBUG, "(wizard) explicit forbidden (%s) link %s%s: mime '%s'", - mdepth, adr_nologin(adr), fil, mime); + mdepth, adr_nologin(opt, adr), fil, mime); } // sinon on touche à rien } /* userdef test */ @@ -1163,13 +1163,14 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz if (jok == 1) { hts_log_print(opt, LOG_DEBUG, "File confirmed (size test): %s%s (" LLintP ")", - adr_nologin(adr), fil, (LLint) (size)); + adr_nologin(opt, adr), fil, (LLint) (size)); } else if (jok == -1) { if (size_flag) { /* interdit à cause de la taille */ hts_log_print(opt, LOG_DEBUG, "File cancelled due to its size: %s%s (" LLintP ", limit: " LLintP ")", - adr_nologin(adr), fil, (LLint) (size), (LLint) (sz)); + adr_nologin(opt, adr), fil, (LLint) (size), + (LLint) (sz)); } else { jok = 1; } diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index 780516812..3a327d613 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -125,17 +125,29 @@ ok "hts-log.txt masks the passwords of URLs, proxies and -%X headers" out=$(httrack -#test=maskpw) || fail "httrack -#test=maskpw exited non-zero: $out" test "$out" = "maskpw self-test: OK" || fail "expected 'maskpw self-test: OK', got: $out" # planted.html links with a password in every form the engine reads one -crawl planted "http://suser:STARTPWA@127.0.0.1:$port/planted.html" \ - "suser:STARTPWB@127.0.0.1:$port/hop.html" -z -Z -r4 -T5 "+*" "-*:KEEPFILTER@*" \ - -%X "X-Note: a:KEEPHEADER@c/d" +# not quiet, so stdout and stderr carry the start URLs too +mkdir -p "$tmpdir/planted" +run_with_timeout 90 httrack "http://suser:STARTPWA@127.0.0.1:$port/planted.html" \ + "suser:STARTPWB@127.0.0.1:$port/hop.html" -O "$tmpdir/planted" -c4 -s2 \ + --sitemap-url "http://luser:LINKPWS@127.0.0.1:$port/sitemap.xml" --retries=0 -%v0 \ + -%H -z -Z -r4 -T5 "+*" "-*:KEEPFILTER@*" -%X "X-Note: a:KEEPHEADER@c/d" \ + "$tmpdir/planted/stdout.txt" 2>&1 || fail "planted: the crawl failed" crawl planted-proxy "http://127.0.0.1:$port/planted.html" -P "puser:PROXYPW@127.0.0.1:$port" \ -z -Z -r4 -T5 "+*" log=$(<"$tmpdir/planted/hts-log.txt") -for line in "at link 127.0.0.1:1/l8.html (from 127.0.0.1:$port/planted.html)" \ +for line in "at link luser:***@127.0.0.1:1/l8.html (from suser:***@127.0.0.1:$port/planted.html)" \ + "link detected in javascript: http://luser:***@127.0.0.1:" \ + "is ready, skipping: luser:***@127.0.0.1:$port/i.gif" \ + "Sitemap: queued luser:***@127.0.0.1:$port/sitemap.xml" \ + "robots.txt added at suser:***@127.0.0.1:$port" \ "at http://suser:***@127.0.0.1:$port/planted.html suser:***@127.0.0.1:$port/hop.html" \ " -*:KEEPFILTER@* -%X \"X-Note: a:KEEPHEADER@c/d\" "; do grep -qF -- "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" done +grep -qF ">>> Location: http://ruser:***@127.0.0.1:$port/hop.html" "$tmpdir/planted/hts-ioinfo.txt" || + fail "no masked Location in hts-ioinfo.txt" +grep -qF "mirroring http://suser:***@127.0.0.1:$port/planted.html" "$tmpdir/planted/stdout.txt" || + fail "no masked start URL on stdout: $(<"$tmpdir/planted/stdout.txt")" grep -qF "STARTPWA" "$tmpdir/planted/hts-cache/doit.log" || fail "the grep below cannot see a password" # doit.log, new.txt and new.zip keep the URLs that --update needs leaks=$(find "$tmpdir/planted" "$tmpdir/planted-proxy" -type f ! -name doit.log \ diff --git a/tests/660_lint-log-logins.test b/tests/660_lint-log-logins.test new file mode 100644 index 000000000..195996b3a --- /dev/null +++ b/tests/660_lint-log-logins.test @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Check that no log call prints an engine address with its password. + +set -euo pipefail + +# shellcheck source=tests/testlib.sh +. "${0%"${0##*/}"}./testlib.sh" + +python=$(find_python) || skip "no python3 to run the lint" +checker="${testdir}/check-log-logins.py" +src="${top_srcdir}/src" + +work=$(mktemp -d "${TMPDIR:-/tmp}/log_logins.XXXXXX") || fail "no tmpdir" +cleanup_push rm -rf "$work" + +lint() { "$python" "$checker" "$@" 2>"$work/lint.err"; } + +files=() +for f in "$src"/*.c; do + case "$f" in + *_selftest.c) ;; + *) files+=("$f") ;; + esac +done +test "${#files[@]}" -ge 40 || fail "the lint found only ${#files[@]} files to scan" +out=$(lint "${files[@]}") || fail_dump "a log call prints an address raw: +$out" "$work/lint.err" +grep -qx "${#files[@]} files, 0 hits" "$work/lint.err" || + fail_dump "the lint did not scan every file" "$work/lint.err" + +# Mutant: one wrapped address put back raw. +sed 's/? adr_nologin(opt, urladr()) :/? (urladr()) :/' "$src/htscore.c" >"$work/htscore.c" +cmp -s "$src/htscore.c" "$work/htscore.c" && fail "the htscore.c mutant changed nothing" +lint "$work/htscore.c" >"$work/mut.out" && fail "the lint passed a raw urladr()" +grep -q 'is ready, skipping' "$work/mut.out" || + fail "the mutant was not reported: $(<"$work/mut.out")" + +# Each form must be reported, and the safe ones not. +cat >"$work/forms.c" <<'CEOF' +f() { + hts_log_print(opt, LOG_INFO, "a %s", adr); /* hit */ + hts_log_print(opt, LOG_INFO, "b %s%s", back[i].url_adr, fil); /* hit */ + hts_log_vprint(opt, LOG_INFO, "c %s", urladr()); /* hit */ + fprintf(opt->log, "d %s", r->adr); /* hit */ + hts_log_print(opt, LOG_INFO, "e %s", adr_nologin(opt, adr)); + hts_log_print(opt, LOG_INFO, "f %s", url_nologin(opt, url_adr)); + hts_log_print(opt, LOG_INFO, "g %s", urladr() != NULL ? "x" : "y"); + fprintf(fp, "h %s", adr); + hts_log_print(opt, LOG_INFO, "i adr %s", "adr"); +} +CEOF +lint "$work/forms.c" >"$work/forms.out" && fail "the lint passed the raw forms" +want=$(grep -n '/\* hit \*/' "$work/forms.c" | cut -d: -f1) +assert_eq "$want" "$(cut -d: -f2 "$work/forms.out")" "lines reported" + +echo "log login lint OK" diff --git a/tests/Makefile.am b/tests/Makefile.am index d62dc1c3e..385e5f07a 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -15,7 +15,7 @@ EXTRA_DIST = $(TESTS) renamefail.c unlinkfail.c threadattrfail.c gaistall.c noba pty-resize.py ttyrun.py test-timeout.sh png-colors.py \ local-crawl.sh local-server.py ftp-server.py testlib.sh proclib.sh \ webhttracklib.sh httpclient.py webtestlib.py crawllib.sh arclib.sh \ - guide-image-check.py check-subshell-errexit.py \ + guide-image-check.py check-subshell-errexit.py check-log-logins.py \ ci-windows-suite.sh ci-suite-stage.sh ci-windows-watchdog.ps1 install-headers-sweep.sh \ wsl2-exe-shim.sh \ server.crt server.key \ diff --git a/tests/check-log-logins.py b/tests/check-log-logins.py new file mode 100644 index 000000000..4fb5b6fdf --- /dev/null +++ b/tests/check-log-logins.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""List log calls that print an engine address without hiding its password. + +A log call is hts_log_print(), hts_log_vprint() or fprintf() to a log file. +Its arguments must pass every adr, url_adr or urladr() through adr_nologin() +or url_nologin(). An allowed call is named "file: format" in ALLOW. +""" + +import re +import sys + +CALL = re.compile(r"\b(hts_log_print|hts_log_vprint|fprintf)\s*\(") +LOG_FILE = re.compile(r"fprintf\s*\(\s*(opt->log|log|fp_log)\b") +SAFE = re.compile( + r"\b(adr_nologin|url_nologin|jump_identification_const|jump_identification)" + r"\s*\(" +) +ADDRESS = re.compile(r"\b\w*adr\b") +NULL_TEST = re.compile(r"\b\w*adr\b\s*(\(\s*\))?\s*[!=]=\s*NULL") +# Calls whose address argument is not printed as text, with the reason. +ALLOW = { + "htsback.c: Illegal chunk (%s) for %s%s": "chunk_adr is a chunk size", + "htsback.c: illegal chunk CRLF (%s) for %s%s": "chunk_adr is a chunk size", + "htscore.c: dump 1024 bytes (address %p): ": "a pointer", + "htsparse.c: data attribute is no URL: %.*s": "eadr ends the value", +} + + +def call_end(s, i): + """Returns the index after the ')' closing the call opened before i.""" + depth, quote = 1, None + while depth and i < len(s): + c = s[i] + if quote: + if c == "\\": + i += 1 + elif c == quote: + quote = None + elif c in "\"'": + quote = c + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + i += 1 + return i + + +def strip_literals(s): + return re.sub(r'"(\\.|[^"\\])*"' + r"|'(\\.|[^'\\])*'", '""', s) + + +def strip_safe(s): + while True: + m = SAFE.search(s) + if m is None: + return s + s = s[: m.start()] + s[call_end(s, m.end()) :] + + +def first_format(call): + m = re.search(r'"((\\.|[^"\\])*)"', call) + return m.group(1) if m else "" + + +def main(paths): + hits = 0 + used, names = set(), set() + for path in paths: + with open(path, encoding="latin-1") as f: + src = f.read() + name = path.rsplit("/", 1)[-1] + names.add(name) + for m in CALL.finditer(src): + call = src[m.start() : call_end(src, m.end())] + if m.group(1) == "fprintf" and not LOG_FILE.match(call): + continue + args = strip_safe(strip_literals(call[call.index("(") + 1 :])) + args = NULL_TEST.sub("", args) + if not ADDRESS.search(args): + continue + key = "%s: %s" % (name, first_format(call)) + if key in ALLOW: + used.add(key) + continue + line = src.count("\n", 0, m.start()) + 1 + print("%s:%d: %s" % (path, line, " ".join(call.split()))) + hits += 1 + for key in sorted(ALLOW): + if key.split(":")[0] in names and key not in used: + print("stale ALLOW entry: %s" % key) + hits += 1 + print("%d files, %d hits" % (len(paths), hits), file=sys.stderr) + return 1 if hits else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/header-injection-server.py b/tests/header-injection-server.py index 9cb715d94..94edbade8 100644 --- a/tests/header-injection-server.py +++ b/tests/header-injection-server.py @@ -14,6 +14,7 @@ import socket import sys import threading +import time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -47,11 +48,19 @@ def root_page(port): "http://luser:LINKPWH@[::1]:1/l7.html", "http://luser:LINKPWI@127.0.0.1:%(p)d", "http://luser:LINKPWJ@127.0.0.1:1/l8.html", + "/redir.html", +) +# the image loads while the engine waits on slow.html, so it is found ready +PLANTED_FIRST = ( + 's' + '' + "" ) def planted_page(port): - return page("".join('l' % (u % {"p": port}) for u in PLANTED)) + links = "".join('l' % (u % {"p": port}) for u in PLANTED) + return page(PLANTED_FIRST % {"p": port} + links) # the poisoned page must link onward, or its URL never becomes a Referer @@ -87,8 +96,10 @@ def body_for(request, port): return page("leaf") -def extra_headers(request): +def extra_headers(request, port): line = request.split(b"\r\n", 1)[0] + if b"/redir.html" in line: + return b"Location: http://ruser:LINKPWR@127.0.0.1:%d/hop.html\r\n" % port if b"/setcookie.html" in line: # a blank before the colon and a folded line still belong to the field return ( @@ -119,11 +130,23 @@ def handle(conn, port, logf, lock, seq): logf.flush() with open("%s.%d" % (sys.argv[1], seq[0]), "wb") as reqf: reqf.write(data) + if b"/slow.html" in data.split(b"\r\n", 1)[0]: + time.sleep(2) body = body_for(data, port) + extra = extra_headers(data, port) + status = b"302 Found" if extra.startswith(b"Location:") else b"200 OK" + # a real image, which the engine finishes in the background (657) + image = b".gif " in data.split(b"\r\n", 1)[0] + if image: + body = b"GIF89a" try: conn.sendall( - b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n" - + extra_headers(data) + b"HTTP/1.1 " + + status + + b"\r\nContent-Type: " + + (b"image/gif" if image else b"text/html") + + b"\r\n" + + extra + b"Content-Length: %d\r\nConnection: close\r\n\r\n" % len(body) + body ) From 24979acddbef438ffa4bf191fd0592641ee41d4e Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 19:23:33 +0200 Subject: [PATCH 07/12] Mask URL logins in -%H request lines, Referer and data attributes The -%H log now masks the URL login in a request line, as a proxied ftp:// target carries one, and in a Referer, as -%R sets one. The debug line for a data attribute that is no URL masks it too. The -%v1 console lines mask the login of each file they list. A login with no password is masked as ***@, because a token is often sent as the user name. adr_nologin() with a NULL opt drops the login, as back_delete(NULL, ...) logs one. The lint now also checks printf, fputs and fprintf to stdout or stderr, and skips code that a debug switch compiles out. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- src/htsback.c | 11 +-- src/htslib.c | 120 ++++++++++++++++--------- src/htslib.h | 12 +-- src/htslib_selftest.c | 27 +++--- src/htsparse.c | 39 ++++---- tests/657_local-header-log-redact.test | 28 ++++-- tests/660_lint-log-logins.test | 9 ++ tests/check-log-logins.py | 53 +++++++++-- tests/header-injection-server.py | 1 + 9 files changed, 205 insertions(+), 95 deletions(-) diff --git a/src/htsback.c b/src/htsback.c index 89e2677f0..16778533c 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -5458,12 +5458,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) { if (back_status(&back[i]) == STATUS_READY) { if (back[i].r.statuscode == HTTP_OK) - printf("* %s%s (" LLintP " bytes) - OK\n", back[i].url_adr, - back[i].url_fil, (LLint) back[i].r.size); + printf("* %s%s (" LLintP " bytes) - OK\n", + adr_nologin(opt, back[i].url_adr), back[i].url_fil, + (LLint) back[i].r.size); else - printf("* %s%s (" LLintP " bytes) - %d\n", back[i].url_adr, - back[i].url_fil, (LLint) back[i].r.size, - back[i].r.statuscode); + printf("* %s%s (" LLintP " bytes) - %d\n", + adr_nologin(opt, back[i].url_adr), back[i].url_fil, + (LLint) back[i].r.size, back[i].r.statuscode); fflush(stdout); } } diff --git a/src/htslib.c b/src/htslib.c index e5fde0e27..582742610 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -4361,27 +4361,31 @@ static void append_clipped(char *buf, size_t size, size_t *len, const char *s, buf[*len] = '\0'; } -/* Returns the ':' before the password in the login of adr, or NULL when there - is no password. A user alone stays, as the command-line echo shows it. */ -static const char *password_colon(const char *adr) { - const char *const scheme = jump_protocol_const(adr); - const char *const host = jump_identification_const(adr); - +/* Returns HTS_TRUE when adr has a login, a user alone included. */ +static hts_boolean has_login(const char *adr) { // "file://" is its own host, ahead of its scheme's end - return host > scheme ? memchr(scheme, ':', (size_t) (host - scheme)) : NULL; + return jump_identification_const(adr) > jump_protocol_const(adr) ? HTS_TRUE + : HTS_FALSE; } const char *adr_nologin_r(char *buf, size_t size, const char *adr) { - const char *const colon = password_colon(adr); + const char *const scheme = jump_protocol_const(adr); + const char *const host = jump_identification_const(adr); + const char *colon; size_t len = 0; - if (colon == NULL || size == 0) + if (!has_login(adr) || size == 0) return adr; + colon = memchr(scheme, ':', (size_t) (host - scheme)); buf[0] = '\0'; - append_clipped(buf, size, &len, adr, (size_t) (colon - adr)); - append_clipped(buf, size, &len, ":***@", 5); - append_clipped(buf, size, &len, jump_identification_const(adr), - strlen(jump_identification_const(adr))); + if (colon != NULL) { + append_clipped(buf, size, &len, adr, (size_t) (colon - adr)); + append_clipped(buf, size, &len, ":***@", 5); + } else { + append_clipped(buf, size, &len, adr, (size_t) (scheme - adr)); + append_clipped(buf, size, &len, "***@", 4); + } + append_clipped(buf, size, &len, host, strlen(host)); return buf; } @@ -4390,9 +4394,12 @@ const char *url_nologin_r(char *buf, size_t size, const char *url) { } const char *adr_nologin(httrackp *opt, const char *adr) { - return password_colon(adr) != NULL - ? adr_nologin_r(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), adr) - : adr; + if (!has_login(adr)) + return adr; + // no buffer to mask into: drop the whole login + if (opt == NULL) + return jump_identification_const(adr); + return adr_nologin_r(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), adr); } const char *url_nologin(httrackp *opt, const char *url) { @@ -5862,32 +5869,59 @@ void fprint_redacted(FILE *fp, size_t len) { static const char *const secret_fields[] = { "Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"}; -/* The -%H log hides the password of the URL these fields carry. */ -static const char *const url_fields[] = {"Location", "Content-Location"}; +/* The -%H log hides the login of the URL these fields carry. */ +static const char *const url_fields[] = {"Location", "Content-Location", + "Referer"}; -/* Writes the header line buff, len bytes long, with the password of its URL - value as ***. Returns HTS_FALSE, writing nothing, when out of memory. */ -static hts_boolean fprint_url_field(FILE *fp, const char *buff, size_t len) { - const char *const colon = memchr(buff, ':', len); - size_t keep = colon != NULL ? (size_t) (colon - buff) + 1 : len; - char *line = malloct(2 * len + 8); +/* Writes the line buff, len bytes long, with the login of its URL, from keep + up to end, masked. Returns HTS_FALSE, writing nothing, when out of memory. */ +static hts_boolean fprint_url_span(FILE *fp, const char *buff, size_t len, + size_t keep, size_t end) { + char *line = malloct(2 * end + 8); if (line == NULL) return HTS_FALSE; - while (keep < len && (buff[keep] == ' ' || buff[keep] == '\t')) - keep++; - memcpy(line, buff, len); - line[len] = '\0'; + memcpy(line, buff, end); + line[end] = '\0'; fwrite(line, 1, keep, fp); - fputs(url_nologin_r(line + len + 1, len + 7, line + keep), fp); + fputs(url_nologin_r(line + end + 1, end + 7, line + keep), fp); + fwrite(buff + end, 1, len - end, fp); freet(line); return HTS_TRUE; } -/* Finds the password of the userinfo at the authority a: from the first ':' - up to the last '@' before the first '/', the split - jump_identification_const() makes. A token also ends at a blank. Returns - HTS_FALSE when there is none. */ +/* Writes the header line buff, len bytes long, with the login of its URL + value masked. Returns HTS_FALSE, writing nothing, when out of memory. */ +static hts_boolean fprint_url_field(FILE *fp, const char *buff, size_t len) { + const char *const colon = memchr(buff, ':', len); + size_t keep = colon != NULL ? (size_t) (colon - buff) + 1 : len; + + while (keep < len && (buff[keep] == ' ' || buff[keep] == '\t')) + keep++; + return fprint_url_span(fp, buff, len, keep, len); +} + +/* Writes the request line buff, len bytes long, with the login of its target + masked, as a proxied ftp:// target carries one. Returns HTS_FALSE, writing + nothing, when buff is no "METHOD target HTTP/" line or out of memory. */ +static hts_boolean fprint_request_line(FILE *fp, const char *buff, size_t len) { + const char *const sp = memchr(buff, ' ', len); + const char *version; + + if (sp == NULL) + return HTS_FALSE; + version = memchr(sp + 1, ' ', len - (size_t) (sp + 1 - buff)); + if (version == NULL || (size_t) (buff + len - version) < 6 || + strncmp(version, " HTTP/", 6) != 0) + return HTS_FALSE; + return fprint_url_span(fp, buff, len, (size_t) (sp + 1 - buff), + (size_t) (version - buff)); +} + +/* Finds the secret of the userinfo at the authority a, which ends at the last + '@' before the first '/', the split jump_identification_const() makes: the + password after the first ':', or a user alone. A token also ends at a + blank. Returns HTS_FALSE when there is no userinfo. */ static hts_boolean userinfo_password(const char *a, hts_boolean token, const char **pass, const char **at) { const char *end = a + strcspn(a, token ? "/ \t\r\n" : "/"); @@ -5898,9 +5932,7 @@ static hts_boolean userinfo_password(const char *a, hts_boolean token, if (end == a) return HTS_FALSE; colon = memchr(a, ':', (size_t) (end - 1 - a)); - if (colon == NULL) - return HTS_FALSE; - *pass = colon + 1; + *pass = colon != NULL ? colon + 1 : a; *at = end - 1; return HTS_TRUE; } @@ -5919,7 +5951,7 @@ static void mask_copy(mask_out *m, const char *src, size_t n) { m->o += n; } -/* Masks the password of the address at a, if any, and moves *i past it. */ +/* Masks the login secret of the address at a, if any, and moves *i past it. */ static void mask_at(mask_out *m, const char *a, hts_boolean token, const char **i) { const char *pass, *at; @@ -5931,8 +5963,8 @@ static void mask_at(mask_out *m, const char *a, hts_boolean token, m->p = *i = at; } -/* Copies s to dst, or only counts when dst is NULL, with the password of each - address written as ***. Returns the length. */ +/* Copies s to dst, or only counts when dst is NULL, with the login secret of + each address written as ***. Returns the length. */ static size_t mask_passwords(const char *s, char *dst, hts_mask_mode mode) { mask_out m = {dst, 0, s}; const char *i = s; @@ -5959,6 +5991,7 @@ char *hts_mask_passwords(const char *s, hts_mask_mode mode) { } void fprintfio(FILE *fp, const char *buff, const char *prefix) { + const char *const start = buff; hts_boolean nl = HTS_TRUE; hts_boolean secret = HTS_FALSE; @@ -5985,10 +6018,11 @@ void fprintfio(FILE *fp, const char *buff, const char *prefix) { if (nl) fprintf(fp, "%s", prefix); nl = HTS_FALSE; - if (secret || *buff == ' ' || *buff == '\t' || - !header_name_in(buff, url_fields, - sizeof(url_fields) / sizeof(url_fields[0])) || - !fprint_url_field(fp, buff, len)) { + if (!(buff == start && fprint_request_line(fp, buff, len)) && + (secret || *buff == ' ' || *buff == '\t' || + !header_name_in(buff, url_fields, + sizeof(url_fields) / sizeof(url_fields[0])) || + !fprint_url_field(fp, buff, len))) { fwrite(buff, 1, keep, fp); if (keep < len) fprint_redacted(fp, len - keep); diff --git a/src/htslib.h b/src/htslib.h index 3f3a1fbfb..46e40f1e3 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -551,23 +551,23 @@ typedef enum { HTS_MASK_ADDR /* one start URL, or a -P value */ } hts_mask_mode; -/* Returns a copy of s, allocated with malloct, in which the password of every - start URL or proxy address in s is *** . The rest stays byte for byte. NULL - when out of memory. */ +/* Returns a copy of s, allocated with malloct, in which the password, or the + user alone, of every start URL or proxy address in s is *** . The rest stays + byte for byte. NULL when out of memory. */ char *hts_mask_passwords(const char *s, hts_mask_mode mode); /* Log calls never print an address raw: pass adr_nologin() or url_nologin(), which test 660 checks. */ -/* Returns adr with its password as ***, for log lines: adr itself when it has - no password, else a copy clipped to size bytes in buf. */ +/* Returns adr with its login masked as user:*** or ***, for log lines: adr + itself when it has no login, else a copy clipped to size bytes in buf. */ const char *adr_nologin_r(char *buf, size_t size, const char *adr); /* Same as adr_nologin_r(), but only a URL with a scheme or "//" has a login. */ const char *url_nologin_r(char *buf, size_t size, const char *url); /* adr_nologin_r() and url_nologin_r() into a rotating buffer of opt, which - they take only when there is a login. */ + they take only when there is a login. A NULL opt drops the login. */ const char *adr_nologin(httrackp *opt, const char *adr); const char *url_nologin(httrackp *opt, const char *url); diff --git a/src/htslib_selftest.c b/src/htslib_selftest.c index 8f05bbabb..f8a08c646 100644 --- a/src/htslib_selftest.c +++ b/src/htslib_selftest.c @@ -1643,16 +1643,13 @@ static int maskpw_pairs(size_t n) { static int nologin_rows(httrackp *opt) { static const struct { const char *in, *want; - } adrs[] = {{"u:p@h:80", "u:***@h:80"}, - {"u:p@q@h", "u:***@h"}, - {"u@h", "u@h"}, - {":p@h", ":***@h"}, - {"h:80", "h:80"}, - {"file://", "file://"}, - {"https://h", "https://h"}, - {"https://u:p@h", "https://u:***@h"}}, + } adrs[] = {{"u:p@h:80", "u:***@h:80"}, {"u:p@q@h", "u:***@h"}, + {"u@h", "***@h"}, {":p@h", ":***@h"}, + {"h:80", "h:80"}, {"file://", "file://"}, + {"https://h", "https://h"}, {"https://u:p@h", "https://u:***@h"}}, urls[] = {{"http://u:p@h/x", "http://u:***@h/x"}, {"//u:p@q@h", "//u:***@h"}, + {"http://tok@h/x", "http://***@h/x"}, {"HTTPS://u:a b@[::1]:8/", "HTTPS://u:***@[::1]:8/"}, {"http://h/x?a@b", "http://h/x?a@b"}, {"mailto:a@b", "mailto:a@b"}, @@ -1681,11 +1678,17 @@ static int nologin_rows(httrackp *opt) { } /* An address without a login is returned as is, and takes no buffer. */ if (adr_nologin(opt, adrs[4].in) != adrs[4].in || - url_nologin(opt, urls[5].in) != urls[5].in || + url_nologin(opt, urls[6].in) != urls[6].in || opt->state.concat.index != index) { fprintf(stderr, "nologin: a rotating buffer was taken without a login\n"); err = 1; } + /* Without an opt, as back_delete(NULL, ...) logs, the login is dropped. */ + if (strcmp(adr_nologin(NULL, urls[0].in), "h/x") != 0 || + strcmp(url_nologin(NULL, urls[0].in), "h/x") != 0) { + fprintf(stderr, "nologin: a NULL opt kept the login\n"); + err = 1; + } return err; } @@ -1703,7 +1706,7 @@ static int st_maskpw(httrackp *opt, int argc, char **argv) { "http://h1/ http://h/?r=ftp://u:p@g/"}, {HTS_MASK_LIST, "http://h/x?t=10:30@x", "http://h/x?t=10:30@x"}, {HTS_MASK_ADDR, "http://u:p@h/", "http://u:***@h/"}, - {HTS_MASK_ADDR, "http://u@h/", "http://u@h/"}, + {HTS_MASK_ADDR, "http://u@h/", "http://***@h/"}, {HTS_MASK_ADDR, "http://:p@h/", "http://:***@h/"}, {HTS_MASK_ADDR, "http://u:@h/", "http://u:***@h/"}, // the engine splits the userinfo at the last '@' @@ -1718,7 +1721,9 @@ static int st_maskpw(httrackp *opt, int argc, char **argv) { {HTS_MASK_ADDR, "u:a b@h/", "u:***@h/"}, {HTS_MASK_ADDR, "http://u:p@h:8080", "http://u:***@h:8080"}, {HTS_MASK_ADDR, "h:8080", "h:8080"}, - {HTS_MASK_ADDR, "u@h:8080", "u@h:8080"}, + {HTS_MASK_ADDR, "u@h:8080", "***@h:8080"}, + {HTS_MASK_LIST, "tok@h/ http://tok@g/ +*@g/*", + "***@h/ http://***@g/ +*@g/*"}, }; static const char *const modes[] = {"list", "addr"}; diff --git a/src/htsparse.c b/src/htsparse.c index 53cf7dcf2..2df764260 100644 --- a/src/htsparse.c +++ b/src/htsparse.c @@ -335,13 +335,11 @@ static const char *html_inline_safe(const char *src, char *dst, size_t size) { return dst; } -/* Drop a redirect Location's #fragment: a UA anchor, never part of the fetched - * resource (#204). */ -/* Copies the first 32 bytes of the script text s, len bytes long, into buf +/* Copies the first max bytes of the page text s, len bytes long, into buf for a debug line, with a URL login masked, or "(hidden)" when a login cannot be told apart. */ -static const char *script_excerpt(httrackp *opt, char *buf, size_t size, - const char *s, size_t len) { +static const char *text_excerpt(httrackp *opt, char *buf, size_t size, + const char *s, size_t len, size_t max) { char *copy = malloct(len + 1); const char *shown; @@ -354,11 +352,13 @@ static const char *script_excerpt(httrackp *opt, char *buf, size_t size, if (shown == copy && jump_protocol_const(copy) == copy && strchr(copy, '@') != NULL) shown = "(hidden)"; - strlncatbuff(buf, shown, size, minimum(strlen(shown), 32)); + strlncatbuff(buf, shown, size, minimum(strlen(shown), max)); freet(copy); return buf; } +/* Drop a redirect Location's #fragment: a UA anchor, never part of the fetched + * resource (#204). */ static void url_drop_fragment(char *const url) { char *const frag = strchr(url, '#'); if (frag != NULL) @@ -1683,8 +1683,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { "active code (%s) detected in javascript: %s", (check_this_fking_line == 2) ? "parse" : "pickup", - script_excerpt(opt, str, sizeof(str), b, - (size_t) (a - b + 1))); + text_excerpt(opt, str, sizeof(str), b, + (size_t) (a - b + 1), 32)); } } @@ -1983,9 +1983,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { hts_log_print(opt, LOG_DEBUG, "link detected in javascript: %s", - script_excerpt(opt, str, sizeof(str), - html + jslink.offset, - (size_t) jslink.length)); + text_excerpt(opt, str, sizeof(str), + html + jslink.offset, + (size_t) jslink.length, 32)); } p = jslink.offset; // p non nul: TRAITER CHAINE COMME // FICHIER @@ -2371,8 +2371,14 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) { // a lazy-load placeholder or template is no URL: leave it as is if (ok == 0 && dataattr_p && data_attr_is_not_url(html, (size_t) (eadr - html - 1))) { - hts_log_print(opt, LOG_DEBUG, "data attribute is no URL: %.*s", - (int) (eadr - html - 1), html); + const size_t len = (size_t) (eadr - html - 1); + char *shown = malloct(len + 8); + + if (shown != NULL) + hts_log_print( + opt, LOG_DEBUG, "data attribute is no URL: %s", + text_excerpt(opt, shown, len + 8, html, len, len + 7)); + freet(shown); not_url = HTS_TRUE; } } @@ -5053,11 +5059,12 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str, if (b >= 0) { if (back[b].r.statuscode == HTTP_OK) printf("%d/%d: %s%s (" LLintP " bytes) - OK\n", ptr, opt->lien_tot, - back[b].url_adr, back[b].url_fil, (LLint) back[b].r.size); + adr_nologin(opt, back[b].url_adr), back[b].url_fil, + (LLint) back[b].r.size); else printf("%d/%d: %s%s (" LLintP " bytes) - %d\n", ptr, opt->lien_tot, - back[b].url_adr, back[b].url_fil, (LLint) back[b].r.size, - back[b].r.statuscode); + adr_nologin(opt, back[b].url_adr), back[b].url_fil, + (LLint) back[b].r.size, back[b].r.statuscode); } else { hts_log_print(opt, LOG_ERROR, "Link disappeared"); } diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index 3a327d613..cf1d40fcb 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -128,12 +128,14 @@ test "$out" = "maskpw self-test: OK" || fail "expected 'maskpw self-test: OK', g # not quiet, so stdout and stderr carry the start URLs too mkdir -p "$tmpdir/planted" run_with_timeout 90 httrack "http://suser:STARTPWA@127.0.0.1:$port/planted.html" \ - "suser:STARTPWB@127.0.0.1:$port/hop.html" -O "$tmpdir/planted" -c4 -s2 \ - --sitemap-url "http://luser:LINKPWS@127.0.0.1:$port/sitemap.xml" --retries=0 -%v0 \ + "suser:STARTPWB@127.0.0.1:$port/hop.html" "http://STARTPWT@127.0.0.1:$port/plain.html" \ + -O "$tmpdir/planted" -c4 -s2 \ + --sitemap-url "http://luser:LINKPWS@127.0.0.1:$port/sitemap.xml" --retries=0 -%v1 \ -%H -z -Z -r4 -T5 "+*" "-*:KEEPFILTER@*" -%X "X-Note: a:KEEPHEADER@c/d" \ "$tmpdir/planted/stdout.txt" 2>&1 || fail "planted: the crawl failed" crawl planted-proxy "http://127.0.0.1:$port/planted.html" -P "puser:PROXYPW@127.0.0.1:$port" \ - -z -Z -r4 -T5 "+*" + -z -Z -r4 -T5 "+*" -%R "http://ruser:LINKPWREF@127.0.0.1:$port/" +crawl planted-ftp "ftp://suser:STARTPWF@127.0.0.1:$port/f.txt" -P "puser:PROXYPW@127.0.0.1:$port" log=$(<"$tmpdir/planted/hts-log.txt") for line in "at link luser:***@127.0.0.1:1/l8.html (from suser:***@127.0.0.1:$port/planted.html)" \ "link detected in javascript: http://luser:***@127.0.0.1:" \ @@ -141,16 +143,30 @@ for line in "at link luser:***@127.0.0.1:1/l8.html (from suser:***@127.0.0.1:$po "Sitemap: queued luser:***@127.0.0.1:$port/sitemap.xml" \ "robots.txt added at suser:***@127.0.0.1:$port" \ "at http://suser:***@127.0.0.1:$port/planted.html suser:***@127.0.0.1:$port/hop.html" \ + "data attribute is no URL: http://u:***@127.0.0.1:$port/{w}.jpg" \ + " http://***@127.0.0.1:$port/plain.html" \ " -*:KEEPFILTER@* -%X \"X-Note: a:KEEPHEADER@c/d\" "; do grep -qF -- "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" done grep -qF ">>> Location: http://ruser:***@127.0.0.1:$port/hop.html" "$tmpdir/planted/hts-ioinfo.txt" || fail "no masked Location in hts-ioinfo.txt" -grep -qF "mirroring http://suser:***@127.0.0.1:$port/planted.html" "$tmpdir/planted/stdout.txt" || - fail "no masked start URL on stdout: $(<"$tmpdir/planted/stdout.txt")" +# the page keeps the data-src value, which is no link, but the log must not +grep -qF DATAPW "$tmpdir/planted/127.0.0.1_$port/planted.html" || fail "no data-src in the mirror" +if grep -qF DATAPW <<<"$log"; then + fail "the data-src password is in hts-log.txt: $(grep -F DATAPW <<<"$log")" +fi +grep -qF "<<< Referer: http://ruser:***@127.0.0.1:$port/" "$tmpdir/planted-proxy/hts-ioinfo.txt" || + fail "no masked -%R Referer in hts-ioinfo.txt" +grep -qF "<<< GET ftp://suser:***@127.0.0.1:$port/f.txt HTTP/1." "$tmpdir/planted-ftp/hts-ioinfo.txt" || + fail "no masked proxied ftp:// request line in hts-ioinfo.txt" +stdout=$(<"$tmpdir/planted/stdout.txt") +for line in "mirroring http://suser:***@127.0.0.1:$port/planted.html" \ + "suser:***@127.0.0.1:$port/planted.html ("; do + grep -qF "$line" <<<"$stdout" || fail "no '$line' on stdout: $stdout" +done grep -qF "STARTPWA" "$tmpdir/planted/hts-cache/doit.log" || fail "the grep below cannot see a password" # doit.log, new.txt and new.zip keep the URLs that --update needs -leaks=$(find "$tmpdir/planted" "$tmpdir/planted-proxy" -type f ! -name doit.log \ +leaks=$(find "$tmpdir/planted" "$tmpdir/planted-proxy" "$tmpdir/planted-ftp" -type f ! -name doit.log \ ! -name new.txt ! -name new.zip -exec grep -aoE "(STARTPW|LINKPW|PROXYPW)[A-Z0-9]*" {} + || true) test -z "$leaks" || fail "a password is in the project: $leaks" ok "no project file holds a URL or proxy password" diff --git a/tests/660_lint-log-logins.test b/tests/660_lint-log-logins.test index 195996b3a..cbecb16b6 100644 --- a/tests/660_lint-log-logins.test +++ b/tests/660_lint-log-logins.test @@ -48,6 +48,15 @@ f() { hts_log_print(opt, LOG_INFO, "g %s", urladr() != NULL ? "x" : "y"); fprintf(fp, "h %s", adr); hts_log_print(opt, LOG_INFO, "i adr %s", "adr"); + printf("j %s", adr); /* hit */ + fprintf(stderr, "k %s", heap(i)->adr); /* hit */ + fputs(urladr(), stdout); /* hit */ + fputs(adr, fp); +#if HDEBUG + printf("l %s", adr); +#else + printf("m %s", adr); /* hit */ +#endif } CEOF lint "$work/forms.c" >"$work/forms.out" && fail "the lint passed the raw forms" diff --git a/tests/check-log-logins.py b/tests/check-log-logins.py index 4fb5b6fdf..2e5e5bc2d 100644 --- a/tests/check-log-logins.py +++ b/tests/check-log-logins.py @@ -1,31 +1,51 @@ #!/usr/bin/env python3 """List log calls that print an engine address without hiding its password. -A log call is hts_log_print(), hts_log_vprint() or fprintf() to a log file. -Its arguments must pass every adr, url_adr or urladr() through adr_nologin() -or url_nologin(). An allowed call is named "file: format" in ALLOW. +A log call is hts_log_print(), hts_log_vprint(), printf(), or fprintf() or +fputs() to a log file, stdout or stderr. Its arguments must pass every adr, +url_adr or urladr() through adr_nologin() or url_nologin(). An allowed call +is named "file: format" in ALLOW. Code under "#if 0" or a debug switch such +as "#if HDEBUG" is skipped. A call through a macro wrapper, or an address +held in an alias variable, is not detected. """ import re import sys -CALL = re.compile(r"\b(hts_log_print|hts_log_vprint|fprintf)\s*\(") -LOG_FILE = re.compile(r"fprintf\s*\(\s*(opt->log|log|fp_log)\b") +CALL = re.compile(r"\b(hts_log_print|hts_log_vprint|printf|fprintf|fputs)\s*\(") +LOG_FILE = re.compile(r"(opt->log|log|fp_log|stdout|stderr)\s*$") SAFE = re.compile( r"\b(adr_nologin|url_nologin|jump_identification_const|jump_identification)" r"\s*\(" ) ADDRESS = re.compile(r"\b\w*adr\b") NULL_TEST = re.compile(r"\b\w*adr\b\s*(\(\s*\))?\s*[!=]=\s*NULL") +DEBUG_IF = re.compile(r"\s*#\s*if(def)?\s+(0|\w*DEBUG\w*(\s*==\s*\d+)?)\s*(/[/*].*)?$") # Calls whose address argument is not printed as text, with the reason. ALLOW = { "htsback.c: Illegal chunk (%s) for %s%s": "chunk_adr is a chunk size", "htsback.c: illegal chunk CRLF (%s) for %s%s": "chunk_adr is a chunk size", "htscore.c: dump 1024 bytes (address %p): ": "a pointer", - "htsparse.c: data attribute is no URL: %.*s": "eadr ends the value", + "htscoremain.c: X-Host: %s\\r\\n": "the -#C cache dump lists the entry", + "htscoremain.c: %s\\r\\n": "r.adr is the page body", } +def drop_debug(src): + """Blanks the lines a debug switch compiles out, keeping line numbers.""" + out, stack = [], [] + for line in src.split("\n"): + if re.match(r"\s*#\s*if", line): + stack.append(bool(DEBUG_IF.match(line)) or (stack and stack[-1])) + elif re.match(r"\s*#\s*(else|elif)\b", line) and stack: + stack[-1] = len(stack) > 1 and stack[-2] + elif re.match(r"\s*#\s*endif\b", line) and stack: + stack.pop() + line = "" + out.append("" if stack and stack[-1] else line) + return "\n".join(out) + + def call_end(s, i): """Returns the index after the ')' closing the call opened before i.""" depth, quote = 1, None @@ -58,6 +78,21 @@ def strip_safe(s): s = s[: m.start()] + s[call_end(s, m.end()) :] +def stream(call, name): + """Returns the FILE argument of an fprintf() or fputs() call.""" + args = strip_literals(call[call.index("(") + 1 : -1]) + depth, parts, cur = 0, [], "" + for c in args: + if c == "," and depth == 0: + parts.append(cur) + cur = "" + continue + depth += {"(": 1, "[": 1, ")": -1, "]": -1}.get(c, 0) + cur += c + parts.append(cur) + return (parts[0] if name == "fprintf" else parts[-1]).strip() + + def first_format(call): m = re.search(r'"((\\.|[^"\\])*)"', call) return m.group(1) if m else "" @@ -68,12 +103,14 @@ def main(paths): used, names = set(), set() for path in paths: with open(path, encoding="latin-1") as f: - src = f.read() + src = drop_debug(f.read()) name = path.rsplit("/", 1)[-1] names.add(name) for m in CALL.finditer(src): call = src[m.start() : call_end(src, m.end())] - if m.group(1) == "fprintf" and not LOG_FILE.match(call): + if m.group(1) in ("fprintf", "fputs") and not LOG_FILE.search( + stream(call, m.group(1)) + ): continue args = strip_safe(strip_literals(call[call.index("(") + 1 :])) args = NULL_TEST.sub("", args) diff --git a/tests/header-injection-server.py b/tests/header-injection-server.py index 94edbade8..a39249b60 100644 --- a/tests/header-injection-server.py +++ b/tests/header-injection-server.py @@ -55,6 +55,7 @@ def root_page(port): 's' '' "" + '' ) From 62a312449c1d258674413884d1cfd9ef364b1ede Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 20:15:23 +0200 Subject: [PATCH 08/12] Mask logins in any URL scheme and in argument messages The log helpers found a login only after a scheme the engine fetches, so an mms:// or gopher:// link printed its password. They now take any "scheme://". These print the login masked now: a quoted -%L line, a refused quoted argument, the --why answer and its echo, and the over-long seed message. The -%v display and WebHTTrack's drop the login. A -%X warning gets the same redaction as the -%H log. Lint 660 now also checks hts-ioinfo.txt writes and the lien, url, link, line and codebase variables, ignoring comments and sizeof(). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- src/htscore.c | 21 ++++++------ src/htscoremain.c | 18 +++++++--- src/htslib.c | 34 ++++++++++++++----- src/htslib.h | 3 +- src/htsstats.h | 16 +++++++++ src/htsweb.c | 11 ++++-- src/httrack.c | 11 ++++-- tests/657_local-header-log-redact.test | 45 ++++++++++++++++++++++++ tests/660_lint-log-logins.test | 22 ++++++++++++ tests/check-log-logins.py | 47 ++++++++++++++++++++++---- tests/header-injection-server.py | 2 ++ 11 files changed, 194 insertions(+), 36 deletions(-) diff --git a/src/htscore.c b/src/htscore.c index 96d0e3c04..96dbaaeb6 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -988,7 +988,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { // prochaine adresse if (!hts_scan_token(&a, url, sizeof(url))) { printf("URL longer than %d bytes, ignored: %.64s...\n", - (int) (sizeof(url) - 1), url); + (int) (sizeof(url) - 1), url_nologin(opt, url)); hts_log_print(opt, LOG_WARNING, "URL longer than %d bytes, ignored: %.64s...", (int) (sizeof(url) - 1), url_nologin(opt, url)); @@ -1013,7 +1013,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { else strcpybuff(url, StringBuff(opt->why_url)); if (ident_url_absolute(url, &af) < 0) { - printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url)); + printf("--why: unable to parse URL %s" LF, + adr_nologin(opt, StringBuff(opt->why_url))); } else { char *const query = af.fil + strcspn(af.fil, "?"); char BIGSTK path[sizeof(af.fil)]; @@ -1030,15 +1031,15 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { filters_match_url(filters, filptr, af.adr, af.fil, &jokDepth); if (jok > 0) - printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1, - filters[jokDepth]); + printf("%s: accepted by rule #%d (%s)" LF, url_nologin(opt, url), + jokDepth + 1, filters[jokDepth]); else if (jok < 0) - printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1, - filters[jokDepth]); + printf("%s: rejected by rule #%d (%s)" LF, url_nologin(opt, url), + jokDepth + 1, filters[jokDepth]); else printf("%s: no filter rule matches; the wizard decides " "(same-site links are followed by default)" LF, - url); + url_nologin(opt, url)); } freet(primary); /* --why answered; no mirror was asked for. The store comes first @@ -1127,9 +1128,9 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { continue; } if (hts_is_quoted(line, strlen(line))) { - hts_log_print(opt, LOG_ERROR, - "\"%s\", line %d: %s is quoted, ignored", - StringBuff(opt->filelist), lineno, line); + hts_log_print( + opt, LOG_ERROR, "\"%s\", line %d: \"%s is quoted, ignored", + StringBuff(opt->filelist), lineno, url_nologin(opt, line + 1)); continue; } if (line[0]) { diff --git a/src/htscoremain.c b/src/htscoremain.c index e1562e5a8..0f7dca7dd 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -312,8 +312,11 @@ static void cmdl_panic_value(httrackp *opt, const char *msg, const char *value) { char s[HTS_CDLMAXSIZE * 2 + 256]; + char shown[HTS_CDLMAXSIZE]; + if (hts_is_quoted(value, strlen(value))) - snprintf(s, sizeof(s), "%s: %s " HTS_QUOTED_HINT, msg, value); + snprintf(s, sizeof(s), "%s: \"%s " HTS_QUOTED_HINT, msg, + adr_nologin_r(shown, sizeof(shown), value + 1)); else snprintf(s, sizeof(s), "%s", msg); HTS_PANIC_PRINTF(s); @@ -2045,8 +2048,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { const char *const why = http_custom_header_warning( StringBuff(opt->headers), argv[na]); - if (why != NULL) - fprintf(stderr, "warning: -%%X %s: %s\n", why, argv[na]); + // with the -%H log's redaction of a login header + if (why != NULL) { + fprintf(stderr, "warning: -%%X %s: ", why); + fprintfio(stderr, argv[na], ""); + fputc('\n', stderr); + } } /* A refused line goes in, so the request drops its folds. */ if (StringNotEmpty(opt->headers) || @@ -3125,7 +3132,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { // quoted if it holds a blank and no quote const hts_boolean quote = strchr(arg, ' ') != NULL && strchr(arg, '\"') == NULL; - // a start URL, or a -P, -%mu or -%R value, is one address + // a start URL, or a -P, -%mu, -%R or --why value, is one address hts_boolean addr; char *masked; @@ -3135,7 +3142,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { } else if (opt_at != 0 && i < opt_at + x_cmd.span[opt_at]) { addr = cmdl_shortopt_has(argv[opt_at], 'P') || cmdl_shortopt_named(argv[opt_at], "-%mu") || - cmdl_shortopt_named(argv[opt_at], "-%R"); + cmdl_shortopt_named(argv[opt_at], "-%R") || + cmdl_shortopt_named(argv[opt_at], "-%Y"); } else { opt_at = 0; addr = i != 0 && arg[0] != '+' && arg[0] != '-'; diff --git a/src/htslib.c b/src/htslib.c index 582742610..bc950546a 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -4361,16 +4361,34 @@ static void append_clipped(char *buf, size_t size, size_t *len, const char *s, buf[*len] = '\0'; } +/* Returns adr past its scheme and "//", the engine's or any "scheme://". */ +static const char *login_start(const char *adr) { + const char *const known = jump_protocol_const(adr); + const char *s = adr; + + if (known != adr || !isalpha((unsigned char) *s)) + return known; + while (isalnum((unsigned char) *s) || *s == '+' || *s == '-' || *s == '.') + s++; + return strncmp(s, "://", 3) == 0 ? s + 3 : adr; +} + +/* Returns adr past its login, or login_start(adr) when it has none. */ +static const char *login_end(const char *adr) { + const char *const start = login_start(adr); + const char *const at = strrchr_limit(start, '@', strchr(start, '/')); + + return at != NULL ? at : start; +} + /* Returns HTS_TRUE when adr has a login, a user alone included. */ static hts_boolean has_login(const char *adr) { - // "file://" is its own host, ahead of its scheme's end - return jump_identification_const(adr) > jump_protocol_const(adr) ? HTS_TRUE - : HTS_FALSE; + return login_end(adr) > login_start(adr) ? HTS_TRUE : HTS_FALSE; } const char *adr_nologin_r(char *buf, size_t size, const char *adr) { - const char *const scheme = jump_protocol_const(adr); - const char *const host = jump_identification_const(adr); + const char *const scheme = login_start(adr); + const char *const host = login_end(adr); const char *colon; size_t len = 0; @@ -4390,7 +4408,7 @@ const char *adr_nologin_r(char *buf, size_t size, const char *adr) { } const char *url_nologin_r(char *buf, size_t size, const char *url) { - return jump_protocol_const(url) != url ? adr_nologin_r(buf, size, url) : url; + return login_start(url) != url ? adr_nologin_r(buf, size, url) : url; } const char *adr_nologin(httrackp *opt, const char *adr) { @@ -4398,12 +4416,12 @@ const char *adr_nologin(httrackp *opt, const char *adr) { return adr; // no buffer to mask into: drop the whole login if (opt == NULL) - return jump_identification_const(adr); + return login_end(adr); return adr_nologin_r(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), adr); } const char *url_nologin(httrackp *opt, const char *url) { - return jump_protocol_const(url) != url ? adr_nologin(opt, url) : url; + return login_start(url) != url ? adr_nologin(opt, url) : url; } // Returns source past its login (user:pass@), if any. diff --git a/src/htslib.h b/src/htslib.h index 46e40f1e3..b3c7dd9b9 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -563,7 +563,8 @@ char *hts_mask_passwords(const char *s, hts_mask_mode mode); itself when it has no login, else a copy clipped to size bytes in buf. */ const char *adr_nologin_r(char *buf, size_t size, const char *adr); -/* Same as adr_nologin_r(), but only a URL with a scheme or "//" has a login. */ +/* Same as adr_nologin_r(), but only a URL with a scheme or "//" has a login. + Any "scheme://" counts, not only the ones the engine fetches. */ const char *url_nologin_r(char *buf, size_t size, const char *url); /* adr_nologin_r() and url_nologin_r() into a rotating buffer of opt, which diff --git a/src/htsstats.h b/src/htsstats.h index d1184e596..ed91681aa 100644 --- a/src/htsstats.h +++ b/src/htsstats.h @@ -33,7 +33,9 @@ Please visit our Website: http://www.httrack.com #ifndef HTS_STATS_DEFH #define HTS_STATS_DEFH +#include #include "htsglobal.h" +#include "httrack-library.h" /* One row of the "in progress" display, shared so the CLI (httrack) and the web GUI (htsserver) cannot drift apart: each fills its own array. */ @@ -60,4 +62,18 @@ struct t_StatsBuffer { int actived; // pour disabled }; +/* Returns where adr resumes past its login, which the display never shows, + and sets *keep to the length of the scheme shown before it. */ +HTS_STATIC const char *stats_skip_login(const char *adr, size_t *keep) { + const char *const host = jump_identification_const(adr); + const char *const sep = strstr(adr, "://"); + + if (host == adr || host[-1] != '@') { + *keep = 0; + return adr; + } + *keep = sep != NULL && sep < host ? (size_t) (sep + 3 - adr) : 0; + return host; +} + #endif diff --git a/src/htsweb.c b/src/htsweb.c index 127611fe9..caa472fac 100644 --- a/src/htsweb.c +++ b/src/htsweb.c @@ -875,9 +875,14 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b // s[0] = '\0'; strcpybuff(StatsBuffer[index].url_sav, back[i].url_sav); // pour cancel - if (strcmp(back[i].url_adr, "file://")) - strcatbuff(s, back[i].url_adr); - else + if (strcmp(back[i].url_adr, "file://")) { + size_t keep; + const char *const host = + stats_skip_login(back[i].url_adr, &keep); + + strncatbuff(s, back[i].url_adr, keep); + strcatbuff(s, host); + } else strcatbuff(s, "localhost"); /* url_adr and url_fil are each as wide as s, and this is the progress display: clip rather than abort */ diff --git a/src/httrack.c b/src/httrack.c index 3930e8c5a..5fb4f5347 100644 --- a/src/httrack.c +++ b/src/httrack.c @@ -658,9 +658,14 @@ static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_b // s[0] = '\0'; strcpybuff(StatsBuffer[index].url_sav, back[i].url_sav); // pour cancel - if (strcmp(back[i].url_adr, "file://")) - strcatbuff(s, back[i].url_adr); - else + if (strcmp(back[i].url_adr, "file://")) { + size_t keep; + const char *const host = + stats_skip_login(back[i].url_adr, &keep); + + strncatbuff(s, back[i].url_adr, keep); + strcatbuff(s, host); + } else strcatbuff(s, "localhost"); if (back[i].url_fil[0] != '/') strcatbuff(s, "/"); diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index cf1d40fcb..add767392 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -144,12 +144,19 @@ for line in "at link luser:***@127.0.0.1:1/l8.html (from suser:***@127.0.0.1:$po "robots.txt added at suser:***@127.0.0.1:$port" \ "at http://suser:***@127.0.0.1:$port/planted.html suser:***@127.0.0.1:$port/hop.html" \ "data attribute is no URL: http://u:***@127.0.0.1:$port/{w}.jpg" \ + "link detected in html (tag): mms://luser:***@127.0.0.1:$port/m.asf" \ + "link detected in html (tag): gopher://luser:***@127.0.0.1:$port/g" \ " http://***@127.0.0.1:$port/plain.html" \ " -*:KEEPFILTER@* -%X \"X-Note: a:KEEPHEADER@c/d\" "; do grep -qF -- "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" done grep -qF ">>> Location: http://ruser:***@127.0.0.1:$port/hop.html" "$tmpdir/planted/hts-ioinfo.txt" || fail "no masked Location in hts-ioinfo.txt" +# the page keeps a link the engine cannot fetch as it is, but the log must not +grep -qF SCHEMEPWM "$tmpdir/planted/127.0.0.1_$port/planted.html" || fail "no mms:// link in the mirror" +if grep -qF SCHEMEPW <<<"$log"; then + fail "an mms:// or gopher:// password is in hts-log.txt: $(grep -F SCHEMEPW <<<"$log")" +fi # the page keeps the data-src value, which is no link, but the log must not grep -qF DATAPW "$tmpdir/planted/127.0.0.1_$port/planted.html" || fail "no data-src in the mirror" if grep -qF DATAPW <<<"$log"; then @@ -171,6 +178,44 @@ leaks=$(find "$tmpdir/planted" "$tmpdir/planted-proxy" "$tmpdir/planted-ftp" -ty test -z "$leaks" || fail "a password is in the project: $leaks" ok "no project file holds a URL or proxy password" +# The messages about arguments and options mask them too. +printf 'http://u:LONGPW@127.0.0.1/%s\n' "$(repeat_chars 2100 a)" >"$tmpdir/long.txt" +out=$(httrack -O "$tmpdir/why" -q --why "http://u:WHYPW@127.0.0.1/x" -%S "$tmpdir/long.txt" \ + "-*x*" 2>&1) || fail "--why exited $?: $out" +printf '"http://u:QUOTEPW@127.0.0.1:%s/plain.html"\n' "$port" >"$tmpdir/list.txt" +crawl opts "http://127.0.0.1:$port/plain.html" -%L "$tmpdir/list.txt" \ + -%X "Authorization: Basic AUTHPWA" -%X "Authorization: Basic AUTHPWB" +rc=0 +httrack "\"http://u:QSTARTPW@127.0.0.1:$port/plain.html\"" -O "$tmpdir/quoted" \ + >>"$tmpdir/log" 2>&1 || rc=$? +test "$rc" -ne 0 || fail "a quoted start URL was accepted" +log="$out +$(cat "$tmpdir/log" "$tmpdir/why/hts-log.txt" "$tmpdir/opts/hts-log.txt")" +for line in "URL longer than 2047 bytes, ignored: http://u:***@127.0.0.1/aaa" \ + "http://u:***@127.0.0.1/x: rejected by rule #1 (-*x*)" " -%Y http://u:***@127.0.0.1/x " \ + "line 1: \"http://u:***@127.0.0.1:$port/plain.html\" is quoted, ignored" \ + "Argument refused: \"http://u:***@127.0.0.1:$port/plain.html\" is quoted"; do + grep -qF -- "$line" <<<"$log" || fail "no '$line' in the messages: $log" +done +grep -qxF "warning: -%X header replaces an earlier one: Authorization: " \ + "$tmpdir/log" || fail "no redacted -%X warning: $(<"$tmpdir/log")" +if grep -E "(LONG|WHY|QUOTE|AUTH|QSTART)PW" <<<"$log"; then + fail "a password is in the messages about arguments" +fi +# the progress display draws only on a terminal +if "$python" -c 'import pty' 2>/dev/null; then + mkdir -p "$tmpdir/vt" + run_with_timeout 90 "$python" -c 'import pty, sys; pty.spawn(sys.argv[1:])' \ + httrack "http://suser:VTPW@127.0.0.1:$port/planted.html" -O "$tmpdir/vt" -c4 \ + --retries=0 -%v -z -r3 -T5 "$tmpdir/vt.out" 2>&1 || + fail "the -%v crawl failed: $(<"$tmpdir/vt.out")" + grep -qF "127.0.0.1:$port/slow.html" "$tmpdir/vt.out" || fail "no -%v display: $(<"$tmpdir/vt.out")" + if grep -aoE "(VTPW|LINKPW)[A-Z0-9]*" "$tmpdir/vt.out"; then + fail "a password is in the -%v display" + fi +fi +ok "the messages about arguments and the progress display mask passwords" + root="$tmpdir/root" cmds="$tmpdir/cmds" mkdir -p "$root" diff --git a/tests/660_lint-log-logins.test b/tests/660_lint-log-logins.test index cbecb16b6..6e9326166 100644 --- a/tests/660_lint-log-logins.test +++ b/tests/660_lint-log-logins.test @@ -36,6 +36,20 @@ lint "$work/htscore.c" >"$work/mut.out" && fail "the lint passed a raw urladr()" grep -q 'is ready, skipping' "$work/mut.out" || fail "the mutant was not reported: $(<"$work/mut.out")" +# Mutants: a raw address into hts-ioinfo.txt, and a raw lien. +sed '/"\[%d\] request for %s%s:/{n;s/jump_identification_const(adr)/adr/;}' \ + "$src/htslib.c" >"$work/htslib.c" +cmp -s "$src/htslib.c" "$work/htslib.c" && fail "the htslib.c mutant changed nothing" +lint "$work/htslib.c" >"$work/mut.out" && fail "the lint passed a raw adr to ioinfo" +grep -q 'request for' "$work/mut.out" || + fail "the ioinfo mutant was not reported: $(<"$work/mut.out")" +sed '/"link detected in html (tag): %s",/{n;s/url_nologin(opt, lien)/lien/;}' \ + "$src/htsparse.c" >"$work/htsparse.c" +cmp -s "$src/htsparse.c" "$work/htsparse.c" && fail "the htsparse.c mutant changed nothing" +lint "$work/htsparse.c" >"$work/mut.out" && fail "the lint passed a raw lien" +grep -q 'link detected in html' "$work/mut.out" || + fail "the lien mutant was not reported: $(<"$work/mut.out")" + # Each form must be reported, and the safe ones not. cat >"$work/forms.c" <<'CEOF' f() { @@ -52,6 +66,14 @@ f() { fprintf(stderr, "k %s", heap(i)->adr); /* hit */ fputs(urladr(), stdout); /* hit */ fputs(adr, fp); + fprintf(ioinfo, "n %s", adr); /* hit */ + hts_log_print(opt, LOG_INFO, "o %s", lien); /* hit */ + printf("p %s %s", url, link); /* hit */ + fprintf(stderr, "q %s", line); /* hit */ + hts_log_print(opt, LOG_DEBUG, "r %s", codebase); /* hit */ + printf("s %d", (int) sizeof(url)); + /* printf("t %s", lien); */ + printf("u // %s", "lien"); #if HDEBUG printf("l %s", adr); #else diff --git a/tests/check-log-logins.py b/tests/check-log-logins.py index 2e5e5bc2d..70b2a84c4 100644 --- a/tests/check-log-logins.py +++ b/tests/check-log-logins.py @@ -3,7 +3,8 @@ A log call is hts_log_print(), hts_log_vprint(), printf(), or fprintf() or fputs() to a log file, stdout or stderr. Its arguments must pass every adr, -url_adr or urladr() through adr_nologin() or url_nologin(). An allowed call +url_adr, urladr(), lien, url, link, line or codebase through adr_nologin() or +url_nologin(). A log file includes hts-ioinfo.txt. An allowed call is named "file: format" in ALLOW. Code under "#if 0" or a debug switch such as "#if HDEBUG" is skipped. A call through a macro wrapper, or an address held in an alias variable, is not detected. @@ -13,13 +14,16 @@ import sys CALL = re.compile(r"\b(hts_log_print|hts_log_vprint|printf|fprintf|fputs)\s*\(") -LOG_FILE = re.compile(r"(opt->log|log|fp_log|stdout|stderr)\s*$") +LOG_FILE = re.compile(r"(opt->log|log|fp_log|ioinfo|stdout|stderr)\s*$") SAFE = re.compile( r"\b(adr_nologin|url_nologin|jump_identification_const|jump_identification)" r"\s*\(" ) -ADDRESS = re.compile(r"\b\w*adr\b") -NULL_TEST = re.compile(r"\b\w*adr\b\s*(\(\s*\))?\s*[!=]=\s*NULL") +ADDRESS = re.compile(r"\b(\w*adr|lien|url|link|line|codebase)\b") +NULL_TEST = re.compile( + r"\b(\w*adr|lien|url|link|line|codebase)\b\s*(\(\s*\))?\s*[!=]=\s*NULL" +) +SIZEOF = re.compile(r"\bsizeof\s*\(\s*\w+\s*\)") DEBUG_IF = re.compile(r"\s*#\s*if(def)?\s+(0|\w*DEBUG\w*(\s*==\s*\d+)?)\s*(/[/*].*)?$") # Calls whose address argument is not printed as text, with the reason. ALLOW = { @@ -27,6 +31,8 @@ "htsback.c: illegal chunk CRLF (%s) for %s%s": "chunk_adr is a chunk size", "htscore.c: dump 1024 bytes (address %p): ": "a pointer", "htscoremain.c: X-Host: %s\\r\\n": "the -#C cache dump lists the entry", + "htscoremain.c: X-URL: %s\\r\\nX-Cache-Entry-Unreadable: yes\\r\\n\\r\\n": "-#C", + "htscoremain.c: X-URL: %s\\r\\n": "-#C", "htscoremain.c: %s\\r\\n": "r.adr is the page body", } @@ -46,6 +52,35 @@ def drop_debug(src): return "\n".join(out) +def drop_comments(src): + """Blanks comments, keeping line numbers and string literals.""" + out, i, quote = [], 0, None + while i < len(src): + c = src[i] + if quote: + if c == "\\": + out.append(src[i : i + 2]) + i += 2 + continue + if c == quote: + quote = None + elif c in "\"'": + quote = c + elif src.startswith("/*", i): + end = src.find("*/", i + 2) + end = len(src) if end < 0 else end + 2 + out.append(re.sub(r"[^\n]", " ", src[i:end])) + i = end + continue + elif src.startswith("//", i): + end = src.find("\n", i) + i = len(src) if end < 0 else end + continue + out.append(c) + i += 1 + return "".join(out) + + def call_end(s, i): """Returns the index after the ')' closing the call opened before i.""" depth, quote = 1, None @@ -103,7 +138,7 @@ def main(paths): used, names = set(), set() for path in paths: with open(path, encoding="latin-1") as f: - src = drop_debug(f.read()) + src = drop_debug(drop_comments(f.read())) name = path.rsplit("/", 1)[-1] names.add(name) for m in CALL.finditer(src): @@ -113,7 +148,7 @@ def main(paths): ): continue args = strip_safe(strip_literals(call[call.index("(") + 1 :])) - args = NULL_TEST.sub("", args) + args = SIZEOF.sub("", NULL_TEST.sub("", args)) if not ADDRESS.search(args): continue key = "%s: %s" % (name, first_format(call)) diff --git a/tests/header-injection-server.py b/tests/header-injection-server.py index a39249b60..010dcba92 100644 --- a/tests/header-injection-server.py +++ b/tests/header-injection-server.py @@ -48,6 +48,8 @@ def root_page(port): "http://luser:LINKPWH@[::1]:1/l7.html", "http://luser:LINKPWI@127.0.0.1:%(p)d", "http://luser:LINKPWJ@127.0.0.1:1/l8.html", + "mms://luser:SCHEMEPWM@127.0.0.1:%(p)d/m.asf", + "gopher://luser:SCHEMEPWG@127.0.0.1:%(p)d/g", "/redir.html", ) # the image loads while the engine waits on slow.html, so it is found ready From 14cd49d4bbc927065a02ffb3ba27a3ef70b58cee Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 20:34:03 +0200 Subject: [PATCH 09/12] Mask logins in quoted list lines, blank-led links and option messages A quoted -%L line without a scheme now goes through adr_nologin, so its login is masked. A link with blanks before its scheme is masked too, because login_start() skips them. The wizard's relative-link debug line, the rejected --host-alias rule and a -%X Authorization, Proxy-Authorization or Cookie line with no colon no longer print a password. Lint 660 now flags any identifier containing adr, url, lien or link, and puts() and fprintfio() calls. Signed-off-by: Xavier Roche Co-Authored-By: Claude Opus 5.5 --- src/htscore.c | 2 +- src/htscoremain.c | 22 +++++++++++- src/htslib.c | 45 ++++++++++++++++++------ src/htswizard.c | 4 +-- tests/657_local-header-log-redact.test | 21 +++++++++-- tests/660_lint-log-logins.test | 21 +++++++++++ tests/check-log-logins.py | 48 +++++++++++++++++--------- tests/header-injection-server.py | 4 +++ 8 files changed, 133 insertions(+), 34 deletions(-) diff --git a/src/htscore.c b/src/htscore.c index 96dbaaeb6..27420ed54 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -1130,7 +1130,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (hts_is_quoted(line, strlen(line))) { hts_log_print( opt, LOG_ERROR, "\"%s\", line %d: \"%s is quoted, ignored", - StringBuff(opt->filelist), lineno, url_nologin(opt, line + 1)); + StringBuff(opt->filelist), lineno, adr_nologin(opt, line + 1)); continue; } if (line[0]) { diff --git a/src/htscoremain.c b/src/htscoremain.c index 0f7dca7dd..cb876178a 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -322,6 +322,25 @@ static void cmdl_panic_value(httrackp *opt, const char *msg, HTS_PANIC_PRINTF(s); } +/* Prints a --host-alias rule with the login of each of its names masked. */ +static void print_rule_nologin(const char *rule) { + while (*rule != '\0') { + char tok[HTS_CDLMAXSIZE]; + char shown[HTS_CDLMAXSIZE + 8]; + size_t n = strcspn(rule, ",="); + + if (n >= sizeof(tok)) + n = sizeof(tok) - 1; + memcpy(tok, rule, n); + tok[n] = '\0'; + fputs(adr_nologin_r(shown, sizeof(shown), tok), stdout); + rule += n; + if (*rule == ',' || *rule == '=') + putchar(*rule++); + } + putchar('\n'); +} + /* Panic on com, the end of option arg that no option takes. When com is quoted, the message names the option before it. */ static void cmdl_bad_option(httrackp *opt, const char *arg, const char *prefix, @@ -2244,7 +2263,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { if (!hts_host_alias_rule_ok(argv[na])) { HTS_PANIC_PRINTF("Invalid host-alias rule: expected " "alias[,alias...]=canonical-host"); - printf("Rejected: %s\n", argv[na]); + printf("Rejected: "); + print_rule_nologin(argv[na]); printf("Each side names a host, optionally behind its " "scheme; a path is not part of an address\n"); htsmain_free(); diff --git a/src/htslib.c b/src/htslib.c index bc950546a..bcf22c443 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -4361,13 +4361,19 @@ static void append_clipped(char *buf, size_t size, size_t *len, const char *s, buf[*len] = '\0'; } -/* Returns adr past its scheme and "//", the engine's or any "scheme://". */ +/* Returns adr past its blanks, scheme and "//", the engine's or any + "scheme://", or adr when it has no scheme. */ static const char *login_start(const char *adr) { - const char *const known = jump_protocol_const(adr); const char *s = adr; + const char *known; - if (known != adr || !isalpha((unsigned char) *s)) + while (isspace((unsigned char) *s)) + s++; + known = jump_protocol_const(s); + if (known != s) return known; + if (!isalpha((unsigned char) *s)) + return adr; while (isalnum((unsigned char) *s) || *s == '+' || *s == '-' || *s == '.') s++; return strncmp(s, "://", 3) == 0 ? s + 3 : adr; @@ -6008,10 +6014,28 @@ char *hts_mask_passwords(const char *s, hts_mask_mode mode) { return dst; } +/* Returns the length of the secret field name opening line, a line len bytes + long, even with no colon behind it, or 0. */ +static size_t secret_name_len(const char *line, size_t len) { + size_t word, i; + + if (header_name_in(line, secret_fields, + sizeof(secret_fields) / sizeof(secret_fields[0]))) + return header_name_len(line); + if (memchr(line, ':', len) != NULL) + return 0; + word = strcspn(line, " \t\r\n"); + for (i = 0; i < sizeof(secret_fields) / sizeof(secret_fields[0]); i++) + if (header_name_is(line, word, secret_fields[i], strlen(secret_fields[i]))) + return word; + return 0; +} + void fprintfio(FILE *fp, const char *buff, const char *prefix) { const char *const start = buff; hts_boolean nl = HTS_TRUE; hts_boolean secret = HTS_FALSE; + size_t name = 0; while (*buff != '\0') { const size_t len = strcspn(buff, "\r\n"); @@ -6020,16 +6044,17 @@ void fprintfio(FILE *fp, const char *buff, const char *prefix) { size_t keep = len; /* a folded line continues the value above it */ - if (*buff != ' ' && *buff != '\t') - secret = - header_name_in(buff, secret_fields, - sizeof(secret_fields) / sizeof(secret_fields[0])); + if (*buff != ' ' && *buff != '\t') { + name = secret_name_len(buff, len); + secret = name != 0 ? HTS_TRUE : HTS_FALSE; + } if (secret) { const char *const colon = memchr(buff, ':', len); - keep = *buff != ' ' && *buff != '\t' && colon != NULL - ? (size_t) (colon - buff) + 1 - : 0; + /* a line with no colon is redacted after its name */ + keep = *buff == ' ' || *buff == '\t' ? 0 + : colon != NULL ? (size_t) (colon - buff) + 1 + : name; while (keep < len && (buff[keep] == ' ' || buff[keep] == '\t')) keep++; } diff --git a/src/htswizard.c b/src/htswizard.c index 96982abce..7a1efd984 100644 --- a/src/htswizard.c +++ b/src/htswizard.c @@ -565,8 +565,8 @@ static int hts_acceptlink_(httrackp *opt, int ptr, const char *adr, if (lienrelatif(tempo2, sizeof(tempo2), fil, heap(ptr)->fil) == 0) { hts_log_print(opt, LOG_DEBUG, "build relative links to test: %s %s (with %s and %s)", - tempo, tempo2, heap(heap(ptr)->premier)->fil, - heap(ptr)->fil); + url_nologin(opt, tempo), url_nologin(opt, tempo2), + heap(heap(ptr)->premier)->fil, heap(ptr)->fil); // si vient de primary, ne pas tester lienrelatif avec (car host "différent") /*if (heap(heap(ptr)->premier) == 0) { // vient de primary diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index add767392..3caaf3653 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -146,6 +146,8 @@ for line in "at link luser:***@127.0.0.1:1/l8.html (from suser:***@127.0.0.1:$po "data attribute is no URL: http://u:***@127.0.0.1:$port/{w}.jpg" \ "link detected in html (tag): mms://luser:***@127.0.0.1:$port/m.asf" \ "link detected in html (tag): gopher://luser:***@127.0.0.1:$port/g" \ + "link detected in html (tag): gopher://luser:***@127.0.0.1:$port/g2" \ + "build relative links to test: x-y.z+w://luser:***@h.example/a" \ " http://***@127.0.0.1:$port/plain.html" \ " -*:KEEPFILTER@* -%X \"X-Note: a:KEEPHEADER@c/d\" "; do grep -qF -- "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" @@ -182,9 +184,15 @@ ok "no project file holds a URL or proxy password" printf 'http://u:LONGPW@127.0.0.1/%s\n' "$(repeat_chars 2100 a)" >"$tmpdir/long.txt" out=$(httrack -O "$tmpdir/why" -q --why "http://u:WHYPW@127.0.0.1/x" -%S "$tmpdir/long.txt" \ "-*x*" 2>&1) || fail "--why exited $?: $out" -printf '"http://u:QUOTEPW@127.0.0.1:%s/plain.html"\n' "$port" >"$tmpdir/list.txt" +printf '"http://u:QUOTEPWA@127.0.0.1:%s/plain.html"\n"u:QUOTEPWB@127.0.0.1:%s/plain.html"\n' \ + "$port" "$port" >"$tmpdir/list.txt" crawl opts "http://127.0.0.1:$port/plain.html" -%L "$tmpdir/list.txt" \ - -%X "Authorization: Basic AUTHPWA" -%X "Authorization: Basic AUTHPWB" + -%X "Authorization: Basic AUTHPWA" -%X "Authorization: Basic AUTHPWB" \ + -%X "Authorization Basic MALPWA" -%X "proxy-authorization MALPWB" +rc=0 +httrack "http://127.0.0.1:$port/plain.html" -O "$tmpdir/alias" \ + --host-alias "http://u:ALIASPW@a.example/x=b.example" >>"$tmpdir/log" 2>&1 || rc=$? +test "$rc" -ne 0 || fail "a host-alias rule with a path was accepted" rc=0 httrack "\"http://u:QSTARTPW@127.0.0.1:$port/plain.html\"" -O "$tmpdir/quoted" \ >>"$tmpdir/log" 2>&1 || rc=$? @@ -194,12 +202,19 @@ $(cat "$tmpdir/log" "$tmpdir/why/hts-log.txt" "$tmpdir/opts/hts-log.txt")" for line in "URL longer than 2047 bytes, ignored: http://u:***@127.0.0.1/aaa" \ "http://u:***@127.0.0.1/x: rejected by rule #1 (-*x*)" " -%Y http://u:***@127.0.0.1/x " \ "line 1: \"http://u:***@127.0.0.1:$port/plain.html\" is quoted, ignored" \ + "line 2: \"u:***@127.0.0.1:$port/plain.html\" is quoted, ignored" \ + "Rejected: http://u:***@a.example/x=b.example" \ + " -%X \"Authorization \" " \ "Argument refused: \"http://u:***@127.0.0.1:$port/plain.html\" is quoted"; do grep -qF -- "$line" <<<"$log" || fail "no '$line' in the messages: $log" done grep -qxF "warning: -%X header replaces an earlier one: Authorization: " \ "$tmpdir/log" || fail "no redacted -%X warning: $(<"$tmpdir/log")" -if grep -E "(LONG|WHY|QUOTE|AUTH|QSTART)PW" <<<"$log"; then +for line in "Authorization " "proxy-authorization "; do + grep -qxF "warning: -%X malformed header ignored: $line" "$tmpdir/log" || + fail "no redacted '$line' warning: $(<"$tmpdir/log")" +done +if grep -E "(LONG|WHY|QUOTE|AUTH|QSTART|MAL|ALIAS)PW" <<<"$log"; then fail "a password is in the messages about arguments" fi # the progress display draws only on a terminal diff --git a/tests/660_lint-log-logins.test b/tests/660_lint-log-logins.test index 6e9326166..06a2ca12f 100644 --- a/tests/660_lint-log-logins.test +++ b/tests/660_lint-log-logins.test @@ -50,6 +50,21 @@ lint "$work/htsparse.c" >"$work/mut.out" && fail "the lint passed a raw lien" grep -q 'link detected in html' "$work/mut.out" || fail "the lien mutant was not reported: $(<"$work/mut.out")" +# Mutants: a raw --why URL, and a raw lien through puts(). +sed 's/adr_nologin(opt, StringBuff(opt->why_url)))/StringBuff(opt->why_url))/' \ + "$src/htscore.c" >"$work/htscore.c" +cmp -s "$src/htscore.c" "$work/htscore.c" && fail "the --why mutant changed nothing" +lint "$work/htscore.c" >"$work/mut.out" && fail "the lint passed a raw why_url" +grep -q 'unable to parse URL' "$work/mut.out" || + fail "the --why mutant was not reported: $(<"$work/mut.out")" +{ + cat "$src/htsparse.c" + printf 'static void mutant(void) {\n puts(lien);\n}\n' +} >"$work/htsparse.c" +lint "$work/htsparse.c" >"$work/mut.out" && fail "the lint passed puts(lien)" +grep -q 'puts(lien)' "$work/mut.out" || + fail "the puts mutant was not reported: $(<"$work/mut.out")" + # Each form must be reported, and the safe ones not. cat >"$work/forms.c" <<'CEOF' f() { @@ -74,6 +89,12 @@ f() { printf("s %d", (int) sizeof(url)); /* printf("t %s", lien); */ printf("u // %s", "lien"); + puts(lien_in); /* hit */ + fprintfio(stderr, finalurl, ""); /* hit */ + printf("v %s", opt->why_url); /* hit */ + fprintfio(fp, url, ""); + printf("w %s%s -> %s", url_fil, urlfil(), back->url_sav); + fputs(adr_nologin_r(buf, sizeof(buf), adr), stdout); #if HDEBUG printf("l %s", adr); #else diff --git a/tests/check-log-logins.py b/tests/check-log-logins.py index 70b2a84c4..982713f1f 100644 --- a/tests/check-log-logins.py +++ b/tests/check-log-logins.py @@ -1,28 +1,36 @@ #!/usr/bin/env python3 """List log calls that print an engine address without hiding its password. -A log call is hts_log_print(), hts_log_vprint(), printf(), or fprintf() or -fputs() to a log file, stdout or stderr. Its arguments must pass every adr, -url_adr, urladr(), lien, url, link, line or codebase through adr_nologin() or -url_nologin(). A log file includes hts-ioinfo.txt. An allowed call -is named "file: format" in ALLOW. Code under "#if 0" or a debug switch such -as "#if HDEBUG" is skipped. A call through a macro wrapper, or an address -held in an alias variable, is not detected. +A log call is hts_log_print(), hts_log_vprint(), printf(), puts(), or +fprintf(), fputs() or fprintfio() to a log file, stdout or stderr. Its +arguments must pass every identifier containing adr, url, lien or link, and +every line or codebase, through adr_nologin() or url_nologin(). A log file +includes hts-ioinfo.txt. An allowed call is named "file: format" in ALLOW. +Code under "#if 0" or a debug switch such as "#if HDEBUG" is skipped. A call +through a macro wrapper, or an address held in an alias variable, is not +detected. """ import re import sys -CALL = re.compile(r"\b(hts_log_print|hts_log_vprint|printf|fprintf|fputs)\s*\(") +CALL = re.compile( + r"\b(hts_log_print|hts_log_vprint|printf|puts|fprintf|fputs|fprintfio)\s*\(" +) +STREAM_FIRST = ("fprintf", "fprintfio") LOG_FILE = re.compile(r"(opt->log|log|fp_log|ioinfo|stdout|stderr)\s*$") SAFE = re.compile( - r"\b(adr_nologin|url_nologin|jump_identification_const|jump_identification)" - r"\s*\(" + r"\b(adr_nologin|url_nologin|adr_nologin_r|url_nologin_r|jump_identification_const" + r"|jump_identification|text_excerpt)\s*\(" ) -ADDRESS = re.compile(r"\b(\w*adr|lien|url|link|line|codebase)\b") -NULL_TEST = re.compile( - r"\b(\w*adr|lien|url|link|line|codebase)\b\s*(\(\s*\))?\s*[!=]=\s*NULL" +NAME = r"\b(\w*(adr|url|lien|link)\w*|line|codebase)\b" +ADDRESS = re.compile(NAME) +# Names that hold a path, a local file, a count or a mode, never an address. +NOT_ADDRESS = re.compile( + r"\w*fil|\w*_sav\w*|lien_n|lien_tot|maxlink|\w*forbidden_url|urlmode" + r"|catch_url_strerror|shown_url" ) +NULL_TEST = re.compile(NAME + r"\s*(\(\s*\))?\s*[!=]=\s*NULL") SIZEOF = re.compile(r"\bsizeof\s*\(\s*\w+\s*\)") DEBUG_IF = re.compile(r"\s*#\s*if(def)?\s+(0|\w*DEBUG\w*(\s*==\s*\d+)?)\s*(/[/*].*)?$") # Calls whose address argument is not printed as text, with the reason. @@ -34,6 +42,12 @@ "htscoremain.c: X-URL: %s\\r\\nX-Cache-Entry-Unreadable: yes\\r\\n\\r\\n": "-#C", "htscoremain.c: X-URL: %s\\r\\n": "-#C", "htscoremain.c: %s\\r\\n": "r.adr is the page body", + "htsback.c: >>> ": "r.adr is the response header block", + "htscore.c: index redirect omitted: first link too long (%s)": "a local file", + "htshelp.c: \\tProxy's address: \\t%s\\n\\tProxy's port: \\t%d\\n": "a local proxy", + 'htshelp.c: \\nThe URL is: \\"%s\\"\\n': "--catchurl hands the URL back", + 'htshelp.c: You can capture it through: httrack \\"%s\\"\\n': "--catchurl", + "htsweb.c: URL=http://%s:%d/\\n": "the server's own address", } @@ -114,7 +128,7 @@ def strip_safe(s): def stream(call, name): - """Returns the FILE argument of an fprintf() or fputs() call.""" + """Returns the FILE argument of an fprintf(), fprintfio() or fputs() call.""" args = strip_literals(call[call.index("(") + 1 : -1]) depth, parts, cur = 0, [], "" for c in args: @@ -125,7 +139,7 @@ def stream(call, name): depth += {"(": 1, "[": 1, ")": -1, "]": -1}.get(c, 0) cur += c parts.append(cur) - return (parts[0] if name == "fprintf" else parts[-1]).strip() + return (parts[0] if name in STREAM_FIRST else parts[-1]).strip() def first_format(call): @@ -143,13 +157,13 @@ def main(paths): names.add(name) for m in CALL.finditer(src): call = src[m.start() : call_end(src, m.end())] - if m.group(1) in ("fprintf", "fputs") and not LOG_FILE.search( + if m.group(1) in STREAM_FIRST + ("fputs",) and not LOG_FILE.search( stream(call, m.group(1)) ): continue args = strip_safe(strip_literals(call[call.index("(") + 1 :])) args = SIZEOF.sub("", NULL_TEST.sub("", args)) - if not ADDRESS.search(args): + if all(NOT_ADDRESS.fullmatch(a.group(0)) for a in ADDRESS.finditer(args)): continue key = "%s: %s" % (name, first_format(call)) if key in ALLOW: diff --git a/tests/header-injection-server.py b/tests/header-injection-server.py index 010dcba92..e9e9589bf 100644 --- a/tests/header-injection-server.py +++ b/tests/header-injection-server.py @@ -50,6 +50,10 @@ def root_page(port): "http://luser:LINKPWJ@127.0.0.1:1/l8.html", "mms://luser:SCHEMEPWM@127.0.0.1:%(p)d/m.asf", "gopher://luser:SCHEMEPWG@127.0.0.1:%(p)d/g", + " gopher://luser:SCHEMEPWL@127.0.0.1:%(p)d/g2", + "Mms://luser:SCHEMEPWC@127.0.0.1:%(p)d/m2.asf", + "HTTP://luser:LINKPWK@127.0.0.1:1/l9.html", + "x-y.z+w://luser:RELPW@h.example/a", "/redir.html", ) # the image loads while the engine waits on slow.html, so it is found ready From a7d7c691d654e3dfb4b27829e8e6e421edd1e69b Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 21:05:53 +0200 Subject: [PATCH 10/12] Hide the passwords and tokens the engine has seen in every log line The per-site masking finds a login by its place in a URL. As a second layer, the engine now keeps each password and token it learns. They come from the start URLs, -P, --why, every recorded link, the secret -%X values and the Basic tokens it builds. hts-log.txt, hts-ioinfo.txt, the error message and the -%v display then print each of them as ***. A secret of 8 bytes or more is hidden anywhere. A shorter one is hidden only as a login, directly before an '@', so a password such as "index" or "html" leaves index.html alone. The set keeps the first 64 secrets of up to 1 KB each, and logs one warning when a page offers more. With no login, the set is empty and no line is scanned. The set lives in a new tail field, httrackp.secrets, and the front ends reach it through a new export, hts_hide_secrets(). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Xavier Roche --- src/htsback.c | 13 +- src/htscore.c | 8 +- src/htscoremain.c | 31 ++- src/htslib.c | 340 +++++++++++++++++++++++-- src/htslib.h | 37 ++- src/htslib_selftest.c | 138 ++++++++++ src/htsopt.h | 2 + src/htsproxy.c | 1 + src/htsweb.c | 1 + src/httrack-library.h | 5 + src/httrack.c | 1 + tests/657_local-header-log-redact.test | 24 ++ tests/check-log-logins.py | 8 +- tests/header-injection-server.py | 9 + 14 files changed, 586 insertions(+), 32 deletions(-) diff --git a/src/htsback.c b/src/htsback.c index 16778533c..63e7bd886 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -4697,12 +4697,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, #endif if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo, - "[%d] response for %s%s:\r\ncode=%d\r\n", - back[i].r.debugid, - jump_identification_const(back[i].url_adr), - back[i].url_fil, back[i].r.statuscode); - fprintfio(ioinfo, back[i].r.adr, ">>> "); + hts_secret_fprintf( + opt, ioinfo, + "[%d] response for %s%s:\r\ncode=%d\r\n", + back[i].r.debugid, + jump_identification_const(back[i].url_adr), + back[i].url_fil, back[i].r.statuscode); + fprintfio(ioinfo, back[i].r.adr, ">>> ", opt); fprintf(ioinfo, "\r\n"); fflush(ioinfo); } // en-tête diff --git a/src/htscore.c b/src/htscore.c index 27420ed54..66d15e86e 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -385,8 +385,11 @@ int hts_record_link(httrackp * opt, const char *address, const char *file, const char *save, const char *ref_address, const char *ref_file, const char *codebase) { - const int success = - hts_record_link_(opt, address, file, save, ref_address, ref_file, codebase); + int success; + + hts_secret_add_login(opt, address); + success = hts_record_link_(opt, address, file, save, ref_address, ref_file, + codebase); if (!success && opt->maxlink > 0 && opt->lien_tot >= opt->maxlink) { hts_log_print(opt, LOG_PANIC, "Too many links (links=%ld, limit=%ld)", (long int) heap_top_index(), (long int) opt->maxlink); @@ -4516,6 +4519,7 @@ HTSEXT_API int hts_is_exiting(httrackp * opt) { // message d'erreur? char *hts_errmsg(httrackp * opt) { + hts_hide_secrets(opt, opt->state._hts_errmsg, sizeof(opt->state._hts_errmsg)); return opt->state._hts_errmsg; } diff --git a/src/htscoremain.c b/src/htscoremain.c index cb876178a..28f1d188f 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -574,6 +574,32 @@ static void cmdl_print_cache_entry(httrackp *opt, cache_back *cache, freet(r.adr); } +/* Feeds the log backstop the passwords of the blank-separated start URLs, + but not of the +/- filters among them, of the proxy and --why, and the + secret -%X values. */ +static void hts_secret_learn_options(httrackp *opt, const char *urls) { + const char *u = urls; + + while (*u != '\0') { + const size_t n = strcspn(u, " \t\r\n"); + + if (n != 0 && *u != '+' && *u != '-') { + char *word = malloct(n + 1); + + if (word != NULL) { + memcpy(word, u, n); + word[n] = '\0'; + hts_secret_add_login(opt, word); + freet(word); + } + } + u += n != 0 ? n : 1; + } + hts_secret_add_login(opt, StringBuff(opt->proxy.name)); + hts_secret_add_login(opt, StringBuff(opt->why_url)); + hts_secret_add_headers(opt, StringBuff(opt->headers)); +} + static int hts_main_internal(int argc, char **argv, httrackp * opt) { /* command line rebuilt from argv, config files and doit.log */ cmdl_argv x_cmd = {NULL, NULL, NULL, 0, 0, {NULL, 0, 0}}; @@ -2070,7 +2096,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { // with the -%H log's redaction of a login header if (why != NULL) { fprintf(stderr, "warning: -%%X %s: ", why); - fprintfio(stderr, argv[na], ""); + fprintfio(stderr, argv[na], "", opt); fputc('\n', stderr); } } @@ -3137,6 +3163,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { "engine: could not create hts-in_progress.lock, so" " this mirror cannot be stopped or paused by a file"); } + hts_secret_learn_options(opt, url); masked_url = hts_mask_passwords(url, HTS_MASK_LIST); shown_url = masked_url != NULL ? masked_url : "(out of memory)"; // hts-log.txt @@ -3174,7 +3201,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { fputc('"', opt->log); // a -%X login header gets the -%H log's redaction if (!addr || masked != NULL) - fprintfio(opt->log, addr ? masked : arg, ""); + fprintfio(opt->log, addr ? masked : arg, "", opt); fputs(quote ? "\" " : " ", opt->log); freet(masked); } diff --git a/src/htslib.c b/src/htslib.c index bcf22c443..8cf4ede64 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -1702,6 +1702,7 @@ int http_sendhead(httrackp *opt, t_cookie *cookie, int mode, const char *xsend, strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), user_pass)); code64((unsigned char *) user_pass, (int) strlen(user_pass), (unsigned char *) autorisation, 0); + hts_secret_add(opt, autorisation, strlen(autorisation)); print_buffer(&bstr, "Proxy-Authorization: Basic %s"H_CRLF, autorisation); #if HDEBUG @@ -1959,6 +1960,7 @@ int http_sendhead(httrackp *opt, t_cookie *cookie, int mode, const char *xsend, OPT_GET_BUFF_SIZE(opt), user_pass)); code64((unsigned char *) user_pass, (int) strlen(user_pass), (unsigned char *) autorisation, 0); + hts_secret_add(opt, autorisation, strlen(autorisation)); if (strcmp(fil, "/robots.txt")) /* pas robots.txt */ bauth_add(cookie, astart, fil, autorisation); } else { @@ -2021,9 +2023,9 @@ int http_sendhead(httrackp *opt, t_cookie *cookie, int mode, const char *xsend, #endif if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo, "[%d] request for %s%s:\r\n", retour->debugid, - jump_identification_const(adr), fil); - fprintfio(ioinfo, bstr.buffer, "<<< "); + hts_secret_fprintf(opt, ioinfo, "[%d] request for %s%s:\r\n", + retour->debugid, jump_identification_const(adr), fil); + fprintfio(ioinfo, bstr.buffer, "<<< ", opt); fprintf(ioinfo, "\r\n"); fflush(ioinfo); } @@ -5884,6 +5886,255 @@ int may_unknown2(httrackp * opt, const char *mime, const char *filename) { // -- Utils fichiers +/* The backstop: the passwords and tokens the engine has seen, hidden by value + in every log line, behind the per-site masking. */ +typedef struct hts_secret_set { + htsmutex lock; + int count; + hts_boolean full; /* a secret was refused for want of room */ + char *item[HTS_SECRET_MAX]; + size_t len[HTS_SECRET_MAX]; +} hts_secret_set; + +void hts_secret_init(httrackp *opt) { + hts_secret_set *const set = calloct(1, sizeof(hts_secret_set)); + + if (set != NULL) + hts_mutexinit(&set->lock); + opt->secrets = set; +} + +void hts_secret_free(httrackp *opt) { + hts_secret_set *set = (hts_secret_set *) opt->secrets; + int i; + + if (set == NULL) + return; + for (i = 0; i < set->count; i++) + freet(set->item[i]); + hts_mutexfree(&set->lock); + freet(set); + opt->secrets = NULL; +} + +void hts_secret_add(httrackp *opt, const char *s, size_t len) { + hts_secret_set *const set = + opt != NULL ? (hts_secret_set *) opt->secrets : NULL; + hts_boolean warn = HTS_FALSE; + int i; + + if (set == NULL || len == 0 || len > HTS_SECRET_MAXLEN) + return; + hts_mutexlock(&set->lock); + for (i = 0; i < set->count; i++) + if (set->len[i] == len && memcmp(set->item[i], s, len) == 0) + break; + if (i == set->count) { + if (set->count < HTS_SECRET_MAX) { + char *const copy = malloct(len + 1); + + if (copy != NULL) { + memcpy(copy, s, len); + copy[len] = '\0'; + set->item[set->count] = copy; + set->len[set->count] = len; + set->count++; + } + } else if (!set->full) { + set->full = HTS_TRUE; + warn = HTS_TRUE; + } + } + hts_mutexrelease(&set->lock); + // the log line takes the lock again + if (warn) + hts_log_print(opt, LOG_WARNING, + "more than %d passwords seen: the log hides the later ones" + " only where they stand in a URL", + HTS_SECRET_MAX); +} + +void hts_secret_add_login(httrackp *opt, const char *url) { + const char *start, *host, *colon; + + if (url == NULL) + return; + start = login_start(url); + host = login_end(url); + if (host <= start) + return; + colon = memchr(start, ':', (size_t) (host - 1 - start)); + if (colon == NULL) + return; + hts_secret_add(opt, colon + 1, (size_t) (host - 1 - (colon + 1))); + // the Basic login is sent unescaped + if (memchr(colon + 1, '%', (size_t) (host - 1 - (colon + 1))) != NULL && + (size_t) (host - 1 - (colon + 1)) < HTS_SECRET_MAXLEN) { + char pass[HTS_SECRET_MAXLEN + 1]; + char plain[HTS_SECRET_MAXLEN + 1]; + const char *p; + + pass[0] = '\0'; + strncatbuff(pass, colon + 1, (size_t) (host - 1 - (colon + 1))); + p = unescape_http(plain, sizeof(plain), pass); + hts_secret_add(opt, p, strlen(p)); + } +} + +/* May a URL's login hold c? */ +static hts_boolean userinfo_byte(unsigned char c) { + return c != '\0' && (isalnum(c) || strchr("-._~%!$&'()*+,;=", c) != NULL) + ? HTS_TRUE + : HTS_FALSE; +} + +/* Returns the length of the longest secret of set standing at s + i, or 0. A + short one counts only as a login: before an '@', after a ':' or a byte no + login holds. */ +static size_t secret_at(const hts_secret_set *set, const char *s, size_t len, + size_t i) { + size_t best = 0; + int k; + + for (k = 0; k < set->count; k++) { + const size_t n = set->len[k]; + + if (n <= best || n > len - i || memcmp(s + i, set->item[k], n) != 0) + continue; + if (n >= HTS_SECRET_TOKEN || (i + n < len && s[i + n] == '@' && + (i == 0 || s[i - 1] == ':' || + !userinfo_byte((unsigned char) s[i - 1])))) + best = n; + } + return best; +} + +/* Copies the len bytes at s to out, if not NULL, with each secret of set + replaced. Returns the length written. */ +static size_t secret_scan(const hts_secret_set *set, const char *s, size_t len, + char *out) { + size_t i = 0, o = 0; + + while (i < len) { + const size_t n = secret_at(set, s, len, i); + + if (n != 0) { + if (out != NULL) + memcpy(out + o, "***", 3); + o += 3; + i += n; + } else { + if (out != NULL) + out[o] = s[i]; + o++; + i++; + } + } + return o; +} + +/* Returns a copy of the len bytes at s with each secret of opt as ***, + allocated with malloct, its length in *outlen. NULL when no secret is + known, or *outlen set to 0 too when out of memory. */ +static char *secret_mask(httrackp *opt, const char *s, size_t len, + size_t *outlen) { + hts_secret_set *const set = + opt != NULL ? (hts_secret_set *) opt->secrets : NULL; + char *out = NULL; + + *outlen = 1; + if (set == NULL) + return NULL; + hts_mutexlock(&set->lock); + if (set->count != 0) { + *outlen = secret_scan(set, s, len, NULL); + out = malloct(*outlen + 1); + if (out != NULL) { + secret_scan(set, s, len, out); + out[*outlen] = '\0'; + } else + *outlen = 0; + } + hts_mutexrelease(&set->lock); + return out; +} + +void hts_secret_fwrite(httrackp *opt, FILE *fp, const char *s, size_t len) { + size_t n; + char *masked = secret_mask(opt, s, len, &n); + + if (masked != NULL) + fwrite(masked, 1, n, fp); + else if (n == 0) + fputs("", fp); + else + fwrite(s, 1, len, fp); + freet(masked); +} + +/* Does opt know a secret? */ +static hts_boolean secrets_known(httrackp *opt) { + hts_secret_set *const set = + opt != NULL ? (hts_secret_set *) opt->secrets : NULL; + hts_boolean known; + + if (set == NULL) + return HTS_FALSE; + hts_mutexlock(&set->lock); + known = set->count != 0 ? HTS_TRUE : HTS_FALSE; + hts_mutexrelease(&set->lock); + return known; +} + +/* vfprintf() to fp, with each secret of opt as ***. */ +static void secret_vfprintf(httrackp *opt, FILE *fp, const char *format, + va_list args) { + va_list args_copy; + int n; + char *line; + + if (!secrets_known(opt)) { + (void) vfprintf(fp, format, args); + return; + } + va_copy(args_copy, args); + n = vsnprintf(NULL, 0, format, args_copy); + va_end(args_copy); + if (n < 0) + return; + line = malloct((size_t) n + 1); + if (line == NULL) { + fputs("", fp); + return; + } + (void) vsnprintf(line, (size_t) n + 1, format, args); + hts_secret_fwrite(opt, fp, line, (size_t) n); + freet(line); +} + +void hts_secret_fprintf(httrackp *opt, FILE *fp, const char *format, ...) { + va_list args; + + va_start(args, format); + secret_vfprintf(opt, fp, format, args); + va_end(args); +} + +HTSEXT_API void hts_hide_secrets(httrackp *opt, char *s, size_t size) { + size_t n; + char *masked; + + if (size == 0) + return; + masked = secret_mask(opt, s, strlen(s), &n); + if (masked != NULL) { + s[0] = '\0'; + strlncatbuff(s, masked, size, size - 1); + } else if (n == 0) + s[0] = '\0'; + freet(masked); +} + void fprint_redacted(FILE *fp, size_t len) { fprintf(fp, "", (unsigned long) len); } @@ -5899,36 +6150,42 @@ static const char *const url_fields[] = {"Location", "Content-Location", /* Writes the line buff, len bytes long, with the login of its URL, from keep up to end, masked. Returns HTS_FALSE, writing nothing, when out of memory. */ -static hts_boolean fprint_url_span(FILE *fp, const char *buff, size_t len, - size_t keep, size_t end) { +static hts_boolean fprint_url_span(httrackp *opt, FILE *fp, const char *buff, + size_t len, size_t keep, size_t end) { char *line = malloct(2 * end + 8); if (line == NULL) return HTS_FALSE; memcpy(line, buff, end); line[end] = '\0'; - fwrite(line, 1, keep, fp); - fputs(url_nologin_r(line + end + 1, end + 7, line + keep), fp); - fwrite(buff + end, 1, len - end, fp); + { + const char *const url = url_nologin_r(line + end + 1, end + 7, line + keep); + + hts_secret_fwrite(opt, fp, line, keep); + hts_secret_fwrite(opt, fp, url, strlen(url)); + hts_secret_fwrite(opt, fp, buff + end, len - end); + } freet(line); return HTS_TRUE; } /* Writes the header line buff, len bytes long, with the login of its URL value masked. Returns HTS_FALSE, writing nothing, when out of memory. */ -static hts_boolean fprint_url_field(FILE *fp, const char *buff, size_t len) { +static hts_boolean fprint_url_field(httrackp *opt, FILE *fp, const char *buff, + size_t len) { const char *const colon = memchr(buff, ':', len); size_t keep = colon != NULL ? (size_t) (colon - buff) + 1 : len; while (keep < len && (buff[keep] == ' ' || buff[keep] == '\t')) keep++; - return fprint_url_span(fp, buff, len, keep, len); + return fprint_url_span(opt, fp, buff, len, keep, len); } /* Writes the request line buff, len bytes long, with the login of its target masked, as a proxied ftp:// target carries one. Returns HTS_FALSE, writing nothing, when buff is no "METHOD target HTTP/" line or out of memory. */ -static hts_boolean fprint_request_line(FILE *fp, const char *buff, size_t len) { +static hts_boolean fprint_request_line(httrackp *opt, FILE *fp, + const char *buff, size_t len) { const char *const sp = memchr(buff, ' ', len); const char *version; @@ -5938,7 +6195,7 @@ static hts_boolean fprint_request_line(FILE *fp, const char *buff, size_t len) { if (version == NULL || (size_t) (buff + len - version) < 6 || strncmp(version, " HTTP/", 6) != 0) return HTS_FALSE; - return fprint_url_span(fp, buff, len, (size_t) (sp + 1 - buff), + return fprint_url_span(opt, fp, buff, len, (size_t) (sp + 1 - buff), (size_t) (version - buff)); } @@ -6031,7 +6288,54 @@ static size_t secret_name_len(const char *line, size_t len) { return 0; } -void fprintfio(FILE *fp, const char *buff, const char *prefix) { +void hts_secret_add_headers(httrackp *opt, const char *headers) { + const char *line = headers; + + while (line != NULL && *line != '\0') { + const size_t len = strcspn(line, "\r\n"); + const size_t name = len != 0 ? secret_name_len(line, len) : 0; + + if (name != 0) { + const char *const colon = memchr(line, ':', len); + const char *v = colon != NULL ? colon + 1 : line + name; + const char *end = line + len; + + while (v < end && (*v == ' ' || *v == '\t')) + v++; + while (end > v && (end[-1] == ' ' || end[-1] == '\t')) + end--; + hts_secret_add(opt, v, (size_t) (end - v)); + if (header_name_is(line, name, "Cookie", 6) || + header_name_is(line, name, "Set-Cookie", 10)) { + // each cookie value + while (v < end) { + const char *const stop = v + strcspn(v, ";"); + const char *const eq = memchr(v, '=', (size_t) (stop - v)); + const char *const last = stop < end ? stop : end; + + if (eq != NULL && eq < last) + hts_secret_add(opt, eq + 1, (size_t) (last - eq - 1)); + v = last; + while (v < end && (*v == ';' || *v == ' ' || *v == '\t')) + v++; + } + } else { + // the credentials after the scheme, as in "Basic TOKEN" + const char *cred = v + strcspn(v, " \t"); + + while (cred < end && (*cred == ' ' || *cred == '\t')) + cred++; + if (cred < end) + hts_secret_add(opt, cred, (size_t) (end - cred)); + } + } + line += len; + while (*line == '\r' || *line == '\n') + line++; + } +} + +void fprintfio(FILE *fp, const char *buff, const char *prefix, httrackp *opt) { const char *const start = buff; hts_boolean nl = HTS_TRUE; hts_boolean secret = HTS_FALSE; @@ -6061,12 +6365,12 @@ void fprintfio(FILE *fp, const char *buff, const char *prefix) { if (nl) fprintf(fp, "%s", prefix); nl = HTS_FALSE; - if (!(buff == start && fprint_request_line(fp, buff, len)) && + if (!(buff == start && fprint_request_line(opt, fp, buff, len)) && (secret || *buff == ' ' || *buff == '\t' || !header_name_in(buff, url_fields, sizeof(url_fields) / sizeof(url_fields[0])) || - !fprint_url_field(fp, buff, len))) { - fwrite(buff, 1, keep, fp); + !fprint_url_field(opt, fp, buff, len))) { + hts_secret_fwrite(opt, fp, buff, keep); if (keep < len) fprint_redacted(fp, len - keep); } @@ -7517,7 +7821,7 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_ if (opt->log != NULL) { fspc(opt, opt->log, s_type); - (void) vfprintf(opt->log, format, args); + secret_vfprintf(opt, opt->log, format, args); if ((type & LOG_ERRNO) != 0) { char errbuf[HTS_STRERROR_SIZE]; @@ -7652,6 +7956,7 @@ HTSEXT_API httrackp *hts_create_opt(void) { /* mutexes */ hts_mutexinit(&opt->state.lock); + hts_secret_init(opt); /* custom wrappers */ opt->libHandles.count = 0; @@ -7928,6 +8233,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) { StringFree(opt->path_bin); /* mutexes */ + hts_secret_free(opt); hts_mutexfree(&opt->state.lock); /* Free structure */ diff --git a/src/htslib.h b/src/htslib.h index b3c7dd9b9..0f06fe3a7 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -542,8 +542,41 @@ void hts_lowcase(char *s); int multipleStringMatch(const char *s, const char *match); /* Writes a header block to fp, each line after prefix, with the value of a - field that carries a login replaced by the fprint_redacted() marker. */ -void fprintfio(FILE * fp, const char *buff, const char *prefix); + field that carries a login replaced by the fprint_redacted() marker, and + each secret opt knows as ***. opt may be NULL. */ +void fprintfio(FILE *fp, const char *buff, const char *prefix, httrackp *opt); + +/* The backstop behind the per-site masking: opt keeps the first + HTS_SECRET_MAX passwords and tokens it sees, each up to HTS_SECRET_MAXLEN + bytes, and the log writers print each as ***. One of HTS_SECRET_TOKEN bytes + or more is hidden anywhere, a shorter one only as a login, before an '@', + so a password such as "index" leaves index.html alone. */ +#define HTS_SECRET_MAX 64 +#define HTS_SECRET_MAXLEN 1024 +#define HTS_SECRET_TOKEN 8 + +/* Allocates and frees the set of opt->secrets. */ +void hts_secret_init(httrackp *opt); +void hts_secret_free(httrackp *opt); + +/* Adds the len bytes at s to the set of opt. Thread-safe. Once the set is + full, logs one warning and adds nothing more. */ +void hts_secret_add(httrackp *opt, const char *s, size_t len); + +/* Adds the password of the login of url, in any scheme or none, raw and + unescaped. A user alone is not added. url may be NULL. */ +void hts_secret_add_login(httrackp *opt, const char *url); + +/* Adds the values of the Authorization, Proxy-Authorization and Cookie lines + of a -%X header block, which may be NULL. */ +void hts_secret_add_headers(httrackp *opt, const char *headers); + +/* Writes the len bytes at s to fp, with each secret of opt as ***. */ +void hts_secret_fwrite(httrackp *opt, FILE *fp, const char *s, size_t len); + +/* fprintf() to fp, with each secret of opt as ***. */ +void hts_secret_fprintf(httrackp *opt, FILE *fp, const char *format, ...) + HTS_PRINTF_FUN(3, 4); /* What hts_mask_passwords() reads: */ typedef enum { diff --git a/src/htslib_selftest.c b/src/htslib_selftest.c index f8a08c646..82be359c1 100644 --- a/src/htslib_selftest.c +++ b/src/htslib_selftest.c @@ -1692,6 +1692,140 @@ static int nologin_rows(httrackp *opt) { return err; } +/* Reads back the log of o, from offset at, into buf. */ +static void secrets_read(FILE *fp, long at, char *buf, size_t size) { + size_t n; + + fflush(fp); + fseek(fp, at, SEEK_SET); + n = fread(buf, 1, size - 1, fp); + buf[n] = '\0'; + fseek(fp, 0, SEEK_END); +} + +/* Logs line through o and checks the log holds want, and none of bad. */ +static int secrets_row(httrackp *o, FILE *fp, const char *line, + const char *want, const char *bad) { + char got[4096]; + const long at = ftell(fp); + + hts_log_print(o, LOG_NOTICE, "%s", line); + secrets_read(fp, at, got, sizeof(got)); + if (strstr(got, want) == NULL || (bad != NULL && strstr(got, bad) != NULL)) { + fprintf(stderr, "secrets: '%s' logged '%s', want '%s'\n", line, got, want); + return 1; + } + return 0; +} + +/* A hostile page offers 200 passwords, each twice: the first + HTS_SECRET_MAX stay hidden, and one warning says so. */ +static int secrets_full(void) { + httrackp *const o = hts_create_opt(); + FILE *const fp = tmpfile(); + char got[4096]; + const char *w; + int err = 0, i; + + if (o == NULL || fp == NULL) { + fprintf(stderr, "secrets: out of memory\n"); + return 1; + } + o->log = fp; + o->debug = LOG_DEBUG; + for (i = 0; i < 200; i++) { + char url[64]; + + snprintf(url, sizeof(url), "http://u:ZPWCAP%03d@h/", i); + hts_secret_add_login(o, url); + hts_secret_add_login(o, url); + } + hts_log_print(o, LOG_NOTICE, "u:ZPWCAP000@h u:ZPWCAP%03d@h u:ZPWCAP%03d@h", + HTS_SECRET_MAX - 1, HTS_SECRET_MAX); + secrets_read(fp, 0, got, sizeof(got)); + w = strstr(got, "passwords seen"); + if (strstr(got, "u:***@h u:***@h u:ZPWCAP064@h") == NULL || w == NULL || + strstr(w + 1, "passwords seen") != NULL) { + fprintf(stderr, "secrets: full set logged '%s'\n", got); + err = 1; + } + o->log = NULL; + fclose(fp); + hts_free_opt(o); + return err; +} + +static int st_secrets(httrackp *opt, int argc, char **argv) { + httrackp *const o = hts_create_opt(); + FILE *const fp = tmpfile(); + char got[4096]; + char big[HTS_SECRET_MAXLEN + 16]; + long at; + int err = 0; + (void) opt; + (void) argc; + (void) argv; + + if (o == NULL || fp == NULL) { + fprintf(stderr, "secrets: out of memory\n"); + return 1; + } + o->log = fp; + o->debug = LOG_DEBUG; + // none known: the line goes out as given + err |= secrets_row(o, fp, "a u:index@h/index.html", "a u:index@h/index.html", + NULL); + // a short password is hidden only as a login + hts_secret_add_login(o, "http://u:index@h/"); + err |= secrets_row(o, fp, + "a u:index@h/index.html http://index.example/ index@h " + "myindex@h /index", + "a u:***@h/index.html http://index.example/ ***@h " + "myindex@h /index", + NULL); + // a token is hidden anywhere, a short cookie value only as a login + hts_secret_add_headers(o, "X-Keep: ZPWKEEP12345\r\n" + "Authorization: Bearer ZPWTOKEN123\r\n" + "cookie: a=1; sid=ZPWSID9999\r\n" + "Proxy-Authorization Basic ZPWNOCOLON1"); + err |= secrets_row(o, fp, + "GET /x?t=ZPWTOKEN123&s=ZPWSID9999 a=1 ZPWKEEP12345 " + "ZPWNOCOLON1", + "GET /x?t=***&s=*** a=1 ZPWKEEP12345 ***", NULL); + // raw and unescaped, the way Basic sends it + hts_secret_add_login(o, "ftp://u:ZPW%40ESC99@h/"); + err |= secrets_row(o, fp, "ZPW%40ESC99 ZPW@ESC99", "*** ***", NULL); + // a user alone is no password + hts_secret_add_login(o, "http://ZPWUSERONLY@h/"); + err |= secrets_row(o, fp, "ZPWUSERONLY", "ZPWUSERONLY", NULL); + // too long to keep + memset(big, 'Z', sizeof(big) - 1); + big[sizeof(big) - 1] = '\0'; + hts_secret_add(o, big, HTS_SECRET_MAXLEN + 1); + err |= secrets_row(o, fp, big, big, "***"); + // the -%H log and a front end's own buffer + at = ftell(fp); + fprintfio(fp, "GET /?k=ZPWTOKEN123 HTTP/1.1\r\nX-K: ZPWTOKEN123\r\n", "<<< ", + o); + secrets_read(fp, at, got, sizeof(got)); + if (strcmp(got, "<<< GET /?k=*** HTTP/1.1\r\n<<< X-K: ***\r\n") != 0) { + fprintf(stderr, "secrets: fprintfio wrote '%s'\n", got); + err = 1; + } + strcpybuff(got, "h/x?t=ZPWTOKEN123"); + hts_hide_secrets(o, got, sizeof(got)); + if (strcmp(got, "h/x?t=***") != 0) { + fprintf(stderr, "secrets: hts_hide_secrets gave '%s'\n", got); + err = 1; + } + o->log = NULL; + fclose(fp); + hts_free_opt(o); + err |= secrets_full(); + printf("secrets self-test: %s\n", err ? "FAIL" : "OK"); + return err; +} + static int st_maskpw(httrackp *opt, int argc, char **argv) { static const struct { hts_mask_mode mode; @@ -3665,6 +3799,10 @@ const struct selftest_entry selftests_lib[] = { "the fields and copies the span primitive makes", st_span}, {"gmtime", "", "hts_gmtime() fills the caller's buffer, not a static (#794)", st_gmtime}, + {"secrets", "", + "the log hides a password or token it has seen: a token anywhere, a " + "short password only as a login", + st_secrets}, {"maskpw", "[list|addr ]", "the command-line echo masks the password of a URL login and nothing " "else", diff --git a/src/htsopt.h b/src/htsopt.h index 8fd2d93ce..1a5dbd9e7 100644 --- a/src/htsopt.h +++ b/src/htsopt.h @@ -657,6 +657,8 @@ struct httrackp { line, and the last one wins. Tail: ABI */ hts_boolean accept_given; /**< Did the user give -%a? Tail: ABI */ hts_boolean lang_given; /**< Did the user give -%l? Tail: ABI */ + void *secrets; /**< passwords and tokens the logs hide (htslib.c). Live + state, so copy_htsopt must leave it alone. Tail: ABI */ }; /** Running statistics for a mirror. */ diff --git a/src/htsproxy.c b/src/htsproxy.c index 00b01eaac..1fd2773db 100644 --- a/src/htsproxy.c +++ b/src/htsproxy.c @@ -123,6 +123,7 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr, OPT_GET_BUFF_SIZE(opt), user_pass)); code64((unsigned char *) user_pass, (int) strlen(user_pass), (unsigned char *) autorisation, 0); + hts_secret_add(opt, autorisation, strlen(autorisation)); strlcatbuff(req, "Proxy-Authorization: Basic ", sizeof(req)); strlcatbuff(req, autorisation, sizeof(req)); strlcatbuff(req, H_CRLF, sizeof(req)); diff --git a/src/htsweb.c b/src/htsweb.c index caa472fac..ecc1a8c60 100644 --- a/src/htsweb.c +++ b/src/htsweb.c @@ -890,6 +890,7 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b strlncatbuff(s, "/", sizeof(s), sizeof(s) - 1 - strlen(s)); strlncatbuff(s, back[i].url_fil, sizeof(s), sizeof(s) - 1 - strlen(s)); + hts_hide_secrets(opt, s, sizeof(s)); StatsBuffer[index].file[0] = '\0'; { diff --git a/src/httrack-library.h b/src/httrack-library.h index b22b29633..b754718b9 100644 --- a/src/httrack-library.h +++ b/src/httrack-library.h @@ -467,6 +467,11 @@ HTSEXT_API const char *hts_accept_preset(const char *browser, const char *kind); lives. @p opt must not be NULL. */ HTSEXT_API char *hts_errmsg(httrackp *opt); +/** Replace in place each password or token @p opt has seen with ***, so a + front end can show a URL it built itself. @p s is NUL-terminated in a + buffer of @p size bytes, and the result is clipped to it. */ +HTSEXT_API void hts_hide_secrets(httrackp *opt, char *s, size_t size); + /** Get or set the transfer-pause flag. A second argument >= 0 sets it, where nonzero means paused, and a negative one only queries. @return the flag as it now stands. */ diff --git a/src/httrack.c b/src/httrack.c index 5fb4f5347..f46ec269e 100644 --- a/src/httrack.c +++ b/src/httrack.c @@ -670,6 +670,7 @@ static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_b if (back[i].url_fil[0] != '/') strcatbuff(s, "/"); strcatbuff(s, back[i].url_fil); + hts_hide_secrets(opt, s, sizeof(s)); StatsBuffer[index].file[0] = '\0'; { diff --git a/tests/657_local-header-log-redact.test b/tests/657_local-header-log-redact.test index 3caaf3653..26aea5aa1 100644 --- a/tests/657_local-header-log-redact.test +++ b/tests/657_local-header-log-redact.test @@ -124,6 +124,30 @@ ok "hts-log.txt masks the passwords of URLs, proxies and -%X headers" out=$(httrack -#test=maskpw) || fail "httrack -#test=maskpw exited non-zero: $out" test "$out" = "maskpw self-test: OK" || fail "expected 'maskpw self-test: OK', got: $out" +out=$(httrack -#test=secrets) || fail "httrack -#test=secrets exited non-zero: $out" +test "$out" = "secrets self-test: OK" || fail "expected 'secrets self-test: OK', got: $out" +# a known password is hidden by value: a token anywhere, a word only as a login +crawl token "http://u:TOKENPWQ1@127.0.0.1:$port/token.html" -z -Z -r2 \ + -%X "Authorization: Bearer HDRTOKQ3" +crawl word "http://u:html@127.0.0.1:$port/hop.html" -z -Z -r2 +log=$(cat "$tmpdir/token/hts-log.txt" "$tmpdir/token/hts-ioinfo.txt") +for q in k=TOKENPWQ1 l=LINKTOKQ2 h=HDRTOKQ3; do + grep -qF "GET /leaf.html?$q HTTP/1." "$wire" || fail "$q never reached the server" + if grep -qF "${q#*=}" <<<"$log"; then + fail "${q#*=} is in the log: $(grep -F "${q#*=}" <<<"$log")" + fi + for line in "/leaf.html?${q%=*}=***" "<<< GET /leaf.html?${q%=*}=*** HTTP/1."; do + grep -qF -- "$line" <<<"$log" || fail "no '$line' in the log: $log" + done +done +log=$(<"$tmpdir/word/hts-log.txt") +for line in "u:***@127.0.0.1:$port/hop.html" "127.0.0.1:$port/plain.html"; do + grep -qF -- "$line" <<<"$log" || fail "no '$line' in hts-log.txt: $log" +done +if grep -qF "u:html@" <<<"$log"; then + fail "the password html is in hts-log.txt" +fi +ok "the log hides a known token anywhere, and a common word only as a login" # planted.html links with a password in every form the engine reads one # not quiet, so stdout and stderr carry the start URLs too mkdir -p "$tmpdir/planted" diff --git a/tests/check-log-logins.py b/tests/check-log-logins.py index 982713f1f..3e9a68f50 100644 --- a/tests/check-log-logins.py +++ b/tests/check-log-logins.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 """List log calls that print an engine address without hiding its password. -A log call is hts_log_print(), hts_log_vprint(), printf(), puts(), or -fprintf(), fputs() or fprintfio() to a log file, stdout or stderr. Its +A log call is hts_log_print(), hts_log_vprint(), hts_secret_fprintf(), +printf(), puts(), or fprintf(), fputs() or fprintfio() to a log file, stdout +or stderr. Its arguments must pass every identifier containing adr, url, lien or link, and every line or codebase, through adr_nologin() or url_nologin(). A log file includes hts-ioinfo.txt. An allowed call is named "file: format" in ALLOW. @@ -15,7 +16,8 @@ import sys CALL = re.compile( - r"\b(hts_log_print|hts_log_vprint|printf|puts|fprintf|fputs|fprintfio)\s*\(" + r"\b(hts_log_print|hts_log_vprint|hts_secret_fprintf|printf|puts|fprintf|fputs" + r"|fprintfio)\s*\(" ) STREAM_FIRST = ("fprintf", "fprintfio") LOG_FILE = re.compile(r"(opt->log|log|fp_log|ioinfo|stdout|stderr)\s*$") diff --git a/tests/header-injection-server.py b/tests/header-injection-server.py index e9e9589bf..cff5778d5 100644 --- a/tests/header-injection-server.py +++ b/tests/header-injection-server.py @@ -97,6 +97,15 @@ def body_for(request, port): # the proxied pages link onward too, so a poisoned *host* becomes a referer if b"/p.html" in line: return page('deep') + # the passwords of the start URL, of a link and of a -%X header come back + # in a query, which only the log's backstop hides + if b"/token.html" in line: + return page( + 'l' + 't' + 'u' + 'h' % port + ) # sets a cookie and links on, so the next request sends it back (657) if b"/setcookie.html" in line: return page('plain') From d9e7371caa67b668b1233951d205e7d712639040 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 21 Oct 2026 22:09:59 +0200 Subject: [PATCH 11/12] Hide a password only as a login, and a header token anywhere Each secret now has a kind. A URL password is masked only in login position, at any length: followed by '@', with its userinfo starting a word or right after "://". So a password such as "download" or the host name leaves /download/file.html and the Host line alone, and mailto:info@ is not taken for a login. A header token of 8 bytes or more is masked anywhere, and a shorter one is not kept. A secret over 1 KB is skipped with one warning. Signed-off-by: Xavier Roche Co-Authored-By: Claude Opus 5.5 --- src/htslib.c | 80 +++++++++++++++++++------- src/htslib.h | 18 ++++-- src/htslib_selftest.c | 46 +++++++++++---- src/htsproxy.c | 2 +- src/httrack-library.h | 3 +- tests/657_local-header-log-redact.test | 17 +++--- tests/header-injection-server.py | 4 +- 7 files changed, 121 insertions(+), 49 deletions(-) diff --git a/src/htslib.c b/src/htslib.c index 8cf4ede64..e4d000eac 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -1702,7 +1702,8 @@ int http_sendhead(httrackp *opt, t_cookie *cookie, int mode, const char *xsend, strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), user_pass)); code64((unsigned char *) user_pass, (int) strlen(user_pass), (unsigned char *) autorisation, 0); - hts_secret_add(opt, autorisation, strlen(autorisation)); + hts_secret_add(opt, autorisation, strlen(autorisation), + HTS_SECRET_HEADER); print_buffer(&bstr, "Proxy-Authorization: Basic %s"H_CRLF, autorisation); #if HDEBUG @@ -1960,7 +1961,8 @@ int http_sendhead(httrackp *opt, t_cookie *cookie, int mode, const char *xsend, OPT_GET_BUFF_SIZE(opt), user_pass)); code64((unsigned char *) user_pass, (int) strlen(user_pass), (unsigned char *) autorisation, 0); - hts_secret_add(opt, autorisation, strlen(autorisation)); + hts_secret_add(opt, autorisation, strlen(autorisation), + HTS_SECRET_HEADER); if (strcmp(fil, "/robots.txt")) /* pas robots.txt */ bauth_add(cookie, astart, fil, autorisation); } else { @@ -5891,9 +5893,11 @@ int may_unknown2(httrackp * opt, const char *mime, const char *filename) { typedef struct hts_secret_set { htsmutex lock; int count; - hts_boolean full; /* a secret was refused for want of room */ + hts_boolean full; /* a secret was refused for want of room */ + hts_boolean toolong; /* a secret was refused for its length */ char *item[HTS_SECRET_MAX]; size_t len[HTS_SECRET_MAX]; + hts_secret_kind kind[HTS_SECRET_MAX]; } hts_secret_set; void hts_secret_init(httrackp *opt) { @@ -5917,19 +5921,25 @@ void hts_secret_free(httrackp *opt) { opt->secrets = NULL; } -void hts_secret_add(httrackp *opt, const char *s, size_t len) { +void hts_secret_add(httrackp *opt, const char *s, size_t len, + hts_secret_kind kind) { hts_secret_set *const set = opt != NULL ? (hts_secret_set *) opt->secrets : NULL; - hts_boolean warn = HTS_FALSE; + hts_boolean warn = HTS_FALSE, warnlong = HTS_FALSE; int i; - if (set == NULL || len == 0 || len > HTS_SECRET_MAXLEN) + if (set == NULL || len == 0 || + (kind == HTS_SECRET_HEADER && len < HTS_SECRET_TOKEN)) return; hts_mutexlock(&set->lock); for (i = 0; i < set->count; i++) - if (set->len[i] == len && memcmp(set->item[i], s, len) == 0) + if (set->kind[i] == kind && set->len[i] == len && + memcmp(set->item[i], s, len) == 0) break; - if (i == set->count) { + if (len > HTS_SECRET_MAXLEN) { + warnlong = set->toolong ? HTS_FALSE : HTS_TRUE; + set->toolong = HTS_TRUE; + } else if (i == set->count) { if (set->count < HTS_SECRET_MAX) { char *const copy = malloct(len + 1); @@ -5938,6 +5948,7 @@ void hts_secret_add(httrackp *opt, const char *s, size_t len) { copy[len] = '\0'; set->item[set->count] = copy; set->len[set->count] = len; + set->kind[set->count] = kind; set->count++; } } else if (!set->full) { @@ -5952,6 +5963,11 @@ void hts_secret_add(httrackp *opt, const char *s, size_t len) { "more than %d passwords seen: the log hides the later ones" " only where they stand in a URL", HTS_SECRET_MAX); + if (warnlong) + hts_log_print(opt, LOG_WARNING, + "a password or token is over %d bytes, so the log hides it" + " only in a URL login or a known header", + HTS_SECRET_MAXLEN); } void hts_secret_add_login(httrackp *opt, const char *url) { @@ -5966,7 +5982,8 @@ void hts_secret_add_login(httrackp *opt, const char *url) { colon = memchr(start, ':', (size_t) (host - 1 - start)); if (colon == NULL) return; - hts_secret_add(opt, colon + 1, (size_t) (host - 1 - (colon + 1))); + hts_secret_add(opt, colon + 1, (size_t) (host - 1 - (colon + 1)), + HTS_SECRET_LOGIN); // the Basic login is sent unescaped if (memchr(colon + 1, '%', (size_t) (host - 1 - (colon + 1))) != NULL && (size_t) (host - 1 - (colon + 1)) < HTS_SECRET_MAXLEN) { @@ -5977,20 +5994,42 @@ void hts_secret_add_login(httrackp *opt, const char *url) { pass[0] = '\0'; strncatbuff(pass, colon + 1, (size_t) (host - 1 - (colon + 1))); p = unescape_http(plain, sizeof(plain), pass); - hts_secret_add(opt, p, strlen(p)); + hts_secret_add(opt, p, strlen(p), HTS_SECRET_LOGIN); } } -/* May a URL's login hold c? */ -static hts_boolean userinfo_byte(unsigned char c) { - return c != '\0' && (isalnum(c) || strchr("-._~%!$&'()*+,;=", c) != NULL) +/* Does a word start at s + i, after the line start, a blank, a quote, '(' or + '<', or right after "://"? */ +static hts_boolean word_start(const char *s, size_t i) { + return i == 0 || strchr(" \t\r\n\"'(<", s[i - 1]) != NULL || + (i >= 3 && strncmp(s + i - 3, "://", 3) == 0) ? HTS_TRUE : HTS_FALSE; } +/* Does the n bytes at s + i stand as a login, as in "SECRET@" or + "user:SECRET@", whose userinfo starts a word? */ +static hts_boolean login_at(const char *s, size_t len, size_t i, size_t n) { + // schemes whose "name:local@host" is an address, not a login + static const char *const opaque[] = {"mailto", "news", "sip", + "sips", "tel", "xmpp"}; + size_t u = i, k; + + if (i + n >= len || s[i + n] != '@') + return HTS_FALSE; + if (i != 0 && s[i - 1] == ':') { + u = i - 1; + while (u != 0 && strchr("/@ \t\r\n", s[u - 1]) == NULL) + u--; + for (k = 0; k < sizeof(opaque) / sizeof(opaque[0]); k++) + if (strlen(opaque[k]) == i - 1 - u && strfield(s + u, opaque[k])) + return HTS_FALSE; + } + return word_start(s, u); +} + /* Returns the length of the longest secret of set standing at s + i, or 0. A - short one counts only as a login: before an '@', after a ':' or a byte no - login holds. */ + URL password counts only as a login, a header token anywhere. */ static size_t secret_at(const hts_secret_set *set, const char *s, size_t len, size_t i) { size_t best = 0; @@ -6001,9 +6040,7 @@ static size_t secret_at(const hts_secret_set *set, const char *s, size_t len, if (n <= best || n > len - i || memcmp(s + i, set->item[k], n) != 0) continue; - if (n >= HTS_SECRET_TOKEN || (i + n < len && s[i + n] == '@' && - (i == 0 || s[i - 1] == ':' || - !userinfo_byte((unsigned char) s[i - 1])))) + if (set->kind[k] == HTS_SECRET_HEADER || login_at(s, len, i, n)) best = n; } return best; @@ -6304,7 +6341,7 @@ void hts_secret_add_headers(httrackp *opt, const char *headers) { v++; while (end > v && (end[-1] == ' ' || end[-1] == '\t')) end--; - hts_secret_add(opt, v, (size_t) (end - v)); + hts_secret_add(opt, v, (size_t) (end - v), HTS_SECRET_HEADER); if (header_name_is(line, name, "Cookie", 6) || header_name_is(line, name, "Set-Cookie", 10)) { // each cookie value @@ -6314,7 +6351,8 @@ void hts_secret_add_headers(httrackp *opt, const char *headers) { const char *const last = stop < end ? stop : end; if (eq != NULL && eq < last) - hts_secret_add(opt, eq + 1, (size_t) (last - eq - 1)); + hts_secret_add(opt, eq + 1, (size_t) (last - eq - 1), + HTS_SECRET_HEADER); v = last; while (v < end && (*v == ';' || *v == ' ' || *v == '\t')) v++; @@ -6326,7 +6364,7 @@ void hts_secret_add_headers(httrackp *opt, const char *headers) { while (cred < end && (*cred == ' ' || *cred == '\t')) cred++; if (cred < end) - hts_secret_add(opt, cred, (size_t) (end - cred)); + hts_secret_add(opt, cred, (size_t) (end - cred), HTS_SECRET_HEADER); } } line += len; diff --git a/src/htslib.h b/src/htslib.h index 0f06fe3a7..9cb75bc2e 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -548,20 +548,28 @@ void fprintfio(FILE *fp, const char *buff, const char *prefix, httrackp *opt); /* The backstop behind the per-site masking: opt keeps the first HTS_SECRET_MAX passwords and tokens it sees, each up to HTS_SECRET_MAXLEN - bytes, and the log writers print each as ***. One of HTS_SECRET_TOKEN bytes - or more is hidden anywhere, a shorter one only as a login, before an '@', - so a password such as "index" leaves index.html alone. */ + bytes, and the log writers print each as ***. A URL password is hidden only + as a login, before an '@', so a password such as "index" leaves index.html + alone. A header token of HTS_SECRET_TOKEN bytes or more is hidden anywhere. + */ #define HTS_SECRET_MAX 64 #define HTS_SECRET_MAXLEN 1024 #define HTS_SECRET_TOKEN 8 +typedef enum hts_secret_kind { + HTS_SECRET_LOGIN, /* a URL password */ + HTS_SECRET_HEADER /* a header token, kept only from HTS_SECRET_TOKEN bytes */ +} hts_secret_kind; + /* Allocates and frees the set of opt->secrets. */ void hts_secret_init(httrackp *opt); void hts_secret_free(httrackp *opt); /* Adds the len bytes at s to the set of opt. Thread-safe. Once the set is - full, logs one warning and adds nothing more. */ -void hts_secret_add(httrackp *opt, const char *s, size_t len); + full, or for a secret over HTS_SECRET_MAXLEN bytes, logs one warning and + adds nothing. */ +void hts_secret_add(httrackp *opt, const char *s, size_t len, + hts_secret_kind kind); /* Adds the password of the login of url, in any scheme or none, raw and unescaped. A user alone is not added. url may be NULL. */ diff --git a/src/htslib_selftest.c b/src/htslib_selftest.c index 82be359c1..d3242ff0b 100644 --- a/src/htslib_selftest.c +++ b/src/htslib_selftest.c @@ -1760,6 +1760,7 @@ static int st_secrets(httrackp *opt, int argc, char **argv) { FILE *const fp = tmpfile(); char got[4096]; char big[HTS_SECRET_MAXLEN + 16]; + const char *w; long at; int err = 0; (void) opt; @@ -1775,33 +1776,56 @@ static int st_secrets(httrackp *opt, int argc, char **argv) { // none known: the line goes out as given err |= secrets_row(o, fp, "a u:index@h/index.html", "a u:index@h/index.html", NULL); - // a short password is hidden only as a login + // a password is hidden only as a login, at any length hts_secret_add_login(o, "http://u:index@h/"); + hts_secret_add_login(o, "http://u:download@h/"); err |= secrets_row(o, fp, "a u:index@h/index.html http://index.example/ index@h " - "myindex@h /index", + "myindex@h /index (index@h