diff --git a/src/htsback.c b/src/htsback.c index 0c2c139e2..63e7bd886 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -172,7 +172,7 @@ hts_boolean back_spool_write(httrackp *opt, const char *filename, opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: open " "error (%s, %s)", - back->url_adr, back->url_fil, filename, + adr_nologin(opt, back->url_adr), back->url_fil, filename, dir_exists(filename) ? "directory exists" : "directory does NOT exist!", fexist_utf8(filename) ? "file already exists!" : "file does not exist"); back_tmpdir_drop(filename); /* filecreate() may have made the directory */ @@ -185,7 +185,7 @@ hts_boolean back_spool_write(httrackp *opt, const char *filename, hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: write " "error", - back->url_adr, back->url_fil, filename); + adr_nologin(opt, back->url_adr), back->url_fil, filename); (void) UNLINK(filename); back_tmpdir_drop(filename); } @@ -341,7 +341,7 @@ static int back_connect_next(httrackp *opt, struct_back *sback, int i) { back[i].status = STATUS_CONNECTING; hts_log_print(opt, LOG_DEBUG, "connect failed, trying next address (%d/%d) for %s", next + 1, - cf->addr_count, back[i].url_adr); + cf->addr_count, adr_nologin(opt, back[i].url_adr)); return 1; } @@ -468,14 +468,15 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr itemback = NULL; } hts_log_print(opt, LOG_WARNING | LOG_ERRNO, - "engine: warning: unserialize error for %s%s (%s)", adr, - fil, sav); + "engine: warning: unserialize error for %s%s (%s)", + adr_nologin(opt, adr), fil, sav); } fclose(fp); } else { hts_log_print(opt, LOG_WARNING | LOG_ERRNO, - "engine: warning: unserialize error for %s%s (%s), file disappeared", - adr, fil, sav); + "engine: warning: unserialize error for %s%s (%s), file " + "disappeared", + adr_nologin(opt, adr), fil, sav); } (void) UNLINK(fileback); back_tmpdir_drop(fileback); @@ -498,8 +499,9 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr return q; } else { hts_log_print(opt, LOG_WARNING, - "engine: warning: unserialize error for %s%s (%s): no more space to wakeup frozen slots", - adr, fil, sav); + "engine: warning: unserialize error for %s%s (%s): no " + "more space to wakeup frozen slots", + adr_nologin(opt, adr), fil, sav); } } } @@ -764,10 +766,12 @@ int back_cleanup_background(httrackp * opt, cache_back * cache, back[i].url_sav, 1); if (checkIndex != -1) { hts_log_print(opt, LOG_WARNING, - "engine: unexpected duplicate file entry: %s%s -> %s (%d '%s') / %s%s -> %s (%d '%s')", - back[checkIndex].url_adr, back[checkIndex].url_fil, - back[checkIndex].url_sav, back[checkIndex].r.statuscode, - back[checkIndex].r.msg, back[i].url_adr, back[i].url_fil, + "engine: unexpected duplicate file entry: %s%s -> %s (%d " + "'%s') / %s%s -> %s (%d '%s')", + adr_nologin(opt, back[checkIndex].url_adr), + back[checkIndex].url_fil, back[checkIndex].url_sav, + back[checkIndex].r.statuscode, back[checkIndex].r.msg, + adr_nologin(opt, back[i].url_adr), back[i].url_fil, back[i].url_sav, back[i].r.statuscode, back[i].r.msg); back_delete(NULL, NULL, sback, checkIndex); #ifdef _DEBUG @@ -807,7 +811,7 @@ int back_cleanup_background(httrackp * opt, cache_back * cache, } else { hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "engine: warning: serialize error for %s%s: %s", - back[i].url_adr, back[i].url_fil, + adr_nologin(opt, back[i].url_adr), back[i].url_fil, named ? "memory full" : "temporary filename too long"); } } @@ -1162,20 +1166,22 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, "truncated chunked transfer (terminating chunk missing, got " LLintP " bytes): file not cached, will be retried on the next" " update (use -%%B to cache anyway): %s%s", - back[p].r.size, back[p].url_adr, back[p].url_fil); + back[p].r.size, adr_nologin(opt, back[p].url_adr), back[p].url_fil); } else if (back_status(&back[p]) == STATUS_READY) { hts_log_print(opt, LOG_WARNING, "incomplete transfer (expected " LLintP " bytes, got " LLintP "): file not cached, will be retried on the next update" " (use -%%B to cache anyway): %s%s", - back[p].r.totalsize, back[p].r.size, back[p].url_adr, - back[p].url_fil); + back[p].r.totalsize, back[p].r.size, + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } else { - hts_log_print(opt, LOG_INFO, - "incomplete file not yet stored in cache (expected " - LLintP " got " LLintP "): %s%s", back[p].r.totalsize, - back[p].r.size, back[p].url_adr, back[p].url_fil); + hts_log_print( + opt, LOG_INFO, + "incomplete file not yet stored in cache (expected " LLintP + " got " LLintP "): %s%s", + back[p].r.totalsize, back[p].r.size, + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } back_finalize_backup(opt, &back[p], HTS_FALSE); /* Keep the surviving copy in new.lst, else the update purge drops the @@ -1468,20 +1474,22 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, HTS_STAT.stat_bytes += back[p].r.size; HTS_STAT.stat_files++; hts_log_print(opt, LOG_TRACE, "added file %s%s => %s", - back[p].url_adr, back[p].url_fil, back[p].url_sav); + adr_nologin(opt, back[p].url_adr), back[p].url_fil, + back[p].url_sav); } if ((!back[p].r.notmodified) && (opt->is_update)) { HTS_STAT.stat_updated_files++; // page modifiée if (back[p].is_update) { - hts_log_print(opt, LOG_INFO, - "engine: transfer-status: link updated: %s%s -> %s", - back[p].url_adr, back[p].url_fil, - back[p].url_sav); + hts_log_print( + opt, LOG_INFO, + "engine: transfer-status: link updated: %s%s -> %s", + adr_nologin(opt, back[p].url_adr), back[p].url_fil, + back[p].url_sav); } else { hts_log_print(opt, LOG_INFO, "engine: transfer-status: link added: %s%s -> %s", - back[p].url_adr, back[p].url_fil, - back[p].url_sav); + adr_nologin(opt, back[p].url_adr), + back[p].url_fil, back[p].url_sav); } if (cache->txt) { if (back[p].is_update) { @@ -1491,9 +1499,11 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, } } } else { - hts_log_print(opt, LOG_INFO, - "engine: transfer-status: link recorded: %s%s -> %s", - back[p].url_adr, back[p].url_fil, back[p].url_sav); + hts_log_print( + opt, LOG_INFO, + "engine: transfer-status: link recorded: %s%s -> %s", + adr_nologin(opt, back[p].url_adr), back[p].url_fil, + back[p].url_sav); if (cache->txt) { if (opt->is_update) state = "untouched"; @@ -1502,10 +1512,11 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, } } } else { - hts_log_print(opt, LOG_INFO, - "engine: transfer-status: empty file? (%d, '%s'): %s%s", - back[p].r.statuscode, back[p].r.msg, back[p].url_adr, - back[p].url_fil); + hts_log_print( + opt, LOG_INFO, + "engine: transfer-status: empty file? (%d, '%s'): %s%s", + back[p].r.statuscode, back[p].r.msg, + adr_nologin(opt, back[p].url_adr), back[p].url_fil); if (cache->txt) { state = "empty"; } @@ -1513,8 +1524,8 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, } else { hts_log_print(opt, LOG_INFO, "engine: transfer-status: link error (%d, '%s'): %s%s", - back[p].r.statuscode, back[p].r.msg, back[p].url_adr, - back[p].url_fil); + back[p].r.statuscode, back[p].r.msg, + adr_nologin(opt, back[p].url_adr), back[p].url_fil); if (cache->txt) { state = "error"; } @@ -1552,8 +1563,8 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, } else { /* error */ if (!HTTP_IS_OK(back[p].r.statuscode)) { - hts_log_print(opt, LOG_DEBUG, "redirect to %s%s", back[p].url_adr, - back[p].url_fil); + hts_log_print(opt, LOG_DEBUG, "redirect to %s%s", + adr_nologin(opt, back[p].url_adr), back[p].url_fil); /* Store only header reference */ cache_mayadd(opt, cache, &back[p].r, back[p].url_adr, back[p].url_fil, NULL); @@ -1564,7 +1575,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, "file with unresolved type not cached (%s (%d), size " LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size, - back[p].url_adr, back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } } @@ -2116,7 +2127,8 @@ int back_unserialize_ref(httrackp * opt, const char *adr, const char *fil, errno = 0; if (!ref_get_record(fp, back)) { hts_log_print(opt, LOG_DEBUG, - "Ignoring an unreadable resume reference for %s%s", adr, fil); + "Ignoring an unreadable resume reference for %s%s", + adr_nologin(opt, adr), fil); fclose(fp); back_clear_entry(back); freet(back); @@ -2161,7 +2173,7 @@ int back_maydelete(httrackp * opt, cache_back * cache, struct_back * sback, assertf(back[p].ka_time_start != 0); hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully saved #%d (%s)", - back[p].r.debugid, back[p].url_adr); + back[p].r.debugid, adr_nologin(opt, back[p].url_adr)); return 1; } } @@ -2215,7 +2227,7 @@ void back_maydeletehttp(httrackp * opt, cache_back * cache, struct_back * sback, assertf(back[q].ka_time_start != 0); hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully preserved #%d (%s)", - back[q].r.debugid, back[q].url_adr); + back[q].r.debugid, adr_nologin(opt, back[q].url_adr)); } else { deletehttp(&back[p].r); back[p].r.soc = INVALID_SOCKET; @@ -2482,7 +2494,8 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback, ) { hts_log_print(opt, LOG_DEBUG, "File '%s%s' -> %s not yet saved in cache - saving now", - back[p].url_adr, back[p].url_fil, back[p].url_sav); + adr_nologin(opt, back[p].url_adr), back[p].url_fil, + back[p].url_sav); } if (cache != NULL) { back_finalize(opt, cache, sback, p); @@ -2583,8 +2596,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, if (!test && back_exist(sback, opt, adr, fil, save)) { int already_there = 0; - hts_log_print(opt, LOG_ERROR, "error: back_add(%s,%s,%s) duplicate", adr, - fil, save); + hts_log_print(opt, LOG_ERROR, "error: back_add(%s,%s,%s) duplicate", + adr_nologin(opt, adr), fil, save); } #endif @@ -2662,7 +2675,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back[p].status = STATUS_READY; // terminé back_set_finished(opt, sback, p); hts_log_print(opt, LOG_WARNING, - "File not added due to mirror cancel: %s%s", adr, fil); + "File not added due to mirror cancel: %s%s", + adr_nologin(opt, adr), fil); return 0; } // test "fast header" cache ; that is, tests we did that lead to 3XX/4XX/5XX response codes @@ -2772,13 +2786,15 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "Previous file '%s' not found (erased by " "user ?), ignoring: %s%s", - save, back[p].url_adr, back[p].url_fil); + save, adr_nologin(opt, back[p].url_adr), + back[p].url_fil); } } else { hts_log_print(opt, LOG_WARNING, "Previous file '%s' not found (erased by user " "?), recatching: %s%s", - save, back[p].url_adr, back[p].url_fil); + save, adr_nologin(opt, back[p].url_adr), + back[p].url_fil); } } } // fsize() <= 0 @@ -2823,11 +2839,11 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, if (!test) { hts_log_print(opt, LOG_DEBUG, "File immediately loaded from cache: %s%s", - back[p].url_adr, back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } else { hts_log_print(opt, LOG_DEBUG, "File immediately tested from cache: %s%s", - back[p].url_adr, back[p].url_fil); + adr_nologin(opt, back[p].url_adr), back[p].url_fil); } back[p].r.notmodified = 1; // fichier non modifié // no request was sent at all, so this is never a server 304 (#839) @@ -2969,7 +2985,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, #endif hts_log_print(opt, LOG_DEBUG, "File partially present (" LLintP " bytes): %s%s", - (LLint) sz, back[p].url_adr, back[p].url_fil); + (LLint) sz, adr_nologin(opt, back[p].url_adr), + back[p].url_fil); /* impossible - don't have etag or date if (strnotempty(back[p].r.etag)) { // ETag (RFC2616) @@ -2990,16 +3007,18 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back[p].range_req_size = sz; } else { hts_log_print(opt, LOG_WARNING, - "Could not find timestamp for partially present file, restarting (lost " - LLintP " bytes): %s%s", (LLint) sz, - back[p].url_adr, back[p].url_fil); + "Could not find timestamp for partially present " + "file, restarting (lost " LLintP " bytes): %s%s", + (LLint) sz, adr_nologin(opt, back[p].url_adr), + back[p].url_fil); } } else { hts_log_print(opt, LOG_NOTICE, "File partially present (" LLintP " bytes) retransferred due to lack of cache: %s%s", - (LLint) sz, back[p].url_adr, back[p].url_fil); + (LLint) sz, adr_nologin(opt, back[p].url_adr), + back[p].url_fil); /* Sinon requête normale... */ back[p].http11 = 0; } @@ -3017,7 +3036,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_NOTICE, "HTML file (" LLintP " bytes) retransferred due to lack of cache: %s%s", - (LLint) sz, back[p].url_adr, back[p].url_fil); + (LLint) sz, adr_nologin(opt, back[p].url_adr), + back[p].url_fil); /* Sinon requête normale... */ back[p].http11 = 0; } @@ -3031,7 +3051,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, back[p].status = STATUS_READY; // terminé back_set_finished(opt, sback, p); hts_log_print(opt, LOG_WARNING, - "File not added due to mirror cancel: %s%s", adr, fil); + "File not added due to mirror cancel: %s%s", + adr_nologin(opt, adr), fil); return 0; } @@ -3161,7 +3182,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully linked #%d (for %s%s)", - back[p].r.debugid, back[p].url_adr, back[p].url_fil); + back[p].r.debugid, adr_nologin(opt, back[p].url_adr), + back[p].url_fil); } if (opt->timeout > 0) { // gestion du opt->timeout @@ -3256,8 +3278,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache, may_be_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil), /* */ - back[i].r.contenttype, back[i].url_adr, back[i].url_fil, - back[i].url_sav); + back[i].r.contenttype, adr_nologin(opt, back[i].url_adr), + back[i].url_fil, back[i].url_sav); } } } @@ -3318,8 +3340,10 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { opt->liens[index]->pass2 = -1; /* DONE! */ } else { hts_log_print(opt, LOG_INFO, - "engine: warning: entry cleaned up, but no trace on heap: %s%s (%s)", - back[i].url_adr, back[i].url_fil, back[i].url_sav); + "engine: warning: entry cleaned up, but no trace on " + "heap: %s%s (%s)", + adr_nologin(opt, back[i].url_adr), back[i].url_fil, + back[i].url_sav); } } HTS_STAT.stat_background++; @@ -3332,7 +3356,7 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { /* recycle the socket, but keep back[i].r.adr in memory */ hts_log_print(opt, LOG_DEBUG, "file %s%s validated (cached, left in memory)", - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); back_maydeletehttp(opt, cache, sback, i); } } @@ -3354,9 +3378,9 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { (int) back[i].r.keep_alive_t); reason = buffer; } - hts_log_print(opt, LOG_DEBUG, - "(Keep-Alive): live socket #%d (%s) closed (%s)", - back[i].r.debugid, back[i].url_adr, reason); + hts_log_print( + opt, LOG_DEBUG, "(Keep-Alive): live socket #%d (%s) closed (%s)", + back[i].r.debugid, adr_nologin(opt, back[i].url_adr), reason); back_delete(opt, cache, sback, i); // delete backing entry } } @@ -4139,9 +4163,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, #if HDEBUG printf("direct-disk: %s\n", back[i].url_sav); #endif - hts_log_print(opt, LOG_DEBUG, - "File received from net to disk: %s%s", - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_DEBUG, + "File received from net to disk: %s%s", + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); if (back[i].r.out == NULL) { hts_log_print(opt, LOG_ERROR | LOG_ERRNO, @@ -4165,17 +4191,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, /* create a temporary reference file in case of broken mirror */ if (back[i].r.out != NULL && opt->cache != 0) { if (back_serialize_ref(opt, &back[i]) != 0) { - hts_log_print(opt, LOG_WARNING, - "Could not create temporary reference file for %s%s", - back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "Could not create temporary " + "reference file for %s%s", + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } } } } else { // on coupe tout! hts_log_print(opt, LOG_DEBUG, "File cancelled (non HTML): %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); back[i].status = STATUS_READY; // terminé back_set_finished(opt, sback, i); if (!back[i].testmode) @@ -4343,7 +4372,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print(opt, LOG_WARNING, "Truncated chunked transfer (" LLintP " Bytes, terminating chunk missing) for %s%s", - (LLint) back[i].r.size, back[i].url_adr, + (LLint) back[i].r.size, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } @@ -4364,12 +4394,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, (LLint) back[i].r.totalsize); } else { // Un warning suffira.. - hts_log_print(opt, LOG_WARNING, - "Incorrect length (" LLintP "!=" LLintP - " expected) for %s%s", - (LLint) back[i].r.size, - (LLint) back[i].r.totalsize, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "Incorrect length (" LLintP "!=" LLintP + " expected) for %s%s", + (LLint) back[i].r.size, (LLint) back[i].r.totalsize, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } } @@ -4463,7 +4493,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, if (!hts_inmem_size_fits(back[i].r.totalsize)) { hts_log_print(opt, LOG_WARNING, "Chunked resource too large for %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); chunk_size = -1; } else { char *const grown = (char *) realloct( @@ -4479,7 +4510,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "not enough memory (" LLintP ") for %s%s", (LLint) back[i].r.totalsize, - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); chunk_size = -1; } } @@ -4490,10 +4522,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, (LLint) back[i].r.totalsize); #endif } else { - hts_log_print(opt, LOG_WARNING, - "Illegal chunk (%s) for %s%s", - back[i].chunk_adr, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, "Illegal chunk (%s) for %s%s", + back[i].chunk_adr, adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } } else { /* STATUS_CHUNK_CR: the CRLF after the data */ if (chunk_data[0] == '\0') { @@ -4507,10 +4539,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, printf("[%d] chunk CRLF seen\n", (int) back[i].r.soc); #endif } else { - hts_log_print(opt, LOG_WARNING, - "illegal chunk CRLF (%s) for %s%s", - back[i].chunk_adr, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, "illegal chunk CRLF (%s) for %s%s", + back[i].chunk_adr, adr_nologin(opt, back[i].url_adr), + back[i].url_fil); #if CHUNKDEBUG==1 printf("[%d] chunk CRLF ERROR!! : '%s'\n", (int) back[i].r.soc, chunk_data); @@ -4518,10 +4550,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } } } else { - hts_log_print(opt, LOG_WARNING, - "chunk too big (" LLintP ") for %s%s", - (LLint) back[i].chunk_size, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, "chunk too big (" LLintP ") for %s%s", + (LLint) back[i].chunk_size, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } // ok, continuer sur le body @@ -4562,9 +4594,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, back_finalize(opt, cache, sback, i); } else { if (back[i].r.statuscode == HTTP_OK) { - hts_log_print(opt, LOG_WARNING, - "unexpected incomplete type with 200 code at %s%s", - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "unexpected incomplete type with 200 code at %s%s", + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } if (back[i].r.soc != INVALID_SOCKET) { @@ -4598,11 +4631,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } else { // Un warning suffira.. hts_log_print(opt, LOG_WARNING, - "Incorrect length (" LLintP "!=" - LLintP " expected) for %s%s", + "Incorrect length (" LLintP + "!=" LLintP " expected) for %s%s", (LLint) back[i].r.size, (LLint) back[i].r.totalsize, - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } } } @@ -4663,12 +4697,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, #endif if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo, - "[%d] response for %s%s:\r\ncode=%d\r\n", - back[i].r.debugid, - jump_identification_const(back[i].url_adr), - back[i].url_fil, back[i].r.statuscode); - fprintfio(ioinfo, back[i].r.adr, ">>> "); + hts_secret_fprintf( + opt, ioinfo, + "[%d] response for %s%s:\r\ncode=%d\r\n", + back[i].r.debugid, + jump_identification_const(back[i].url_adr), + back[i].url_fil, back[i].r.statuscode); + fprintfio(ioinfo, back[i].r.adr, ">>> ", opt); fprintf(ioinfo, "\r\n"); fflush(ioinfo); } // en-tête @@ -4685,7 +4720,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, truncated Location, or read its tail as headers */ hts_log_print(opt, LOG_WARNING, "Over-long header dropped for %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); continue; } #if HDEBUG @@ -4736,8 +4772,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, &back[i].r); if (test_head != 1) { hts_log_print(opt, LOG_WARNING, - "External wrapper aborted transfer, breaking connection: %s%s", - back[i].url_adr, back[i].url_fil); + "External wrapper aborted transfer, " + "breaking connection: %s%s", + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); back[i].status = STATUS_READY; // FINI back_set_finished(opt, sback, i); deletehttp(&back[i].r); @@ -4773,7 +4811,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Bot protection challenge at %s%s: the " "server refused the request (cf-mitigated " "header) rather than failing it", - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); hts_log_print(opt, LOG_WARNING, "Solve the challenge in a browser, then pass " "its cookies with --cookies-file and its " @@ -4799,9 +4838,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, back[i].r.msg[0] = '\0'; /* the interim block is not the final response's headers */ freet(back[i].r.headers); - hts_log_print(opt, LOG_DEBUG, - "Status 100 detected for %s%s, continuing headers", - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_DEBUG, + "Status 100 detected for %s%s, continuing headers", + adr_nologin(opt, back[i].url_adr), back[i].url_fil); continue; } @@ -4834,7 +4874,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Kept existing file %s (" LLintP " bytes), matching size and timestamp: %s%s", back[i].url_sav, (LLint) back[i].range_req_size, - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } // transform 406 into 200 ; we'll catch embedded links inside the choice page @@ -4847,12 +4887,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, back[i].is_update && !back[i].testmode && back[i].range_req_size == 0 && back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) { - hts_log_print(opt, LOG_NOTICE, - "Kept existing file %s after error %d (%s) " - "on update: %s%s", - back[i].url_sav, back[i].r.statuscode, - back[i].r.msg, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_NOTICE, + "Kept existing file %s after error %d (%s) " + "on update: %s%s", + back[i].url_sav, back[i].r.statuscode, back[i].r.msg, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); back[i].r.statuscode = HTTP_NOT_MODIFIED; deletehttp(&back[i].r); back[i].r.soc = INVALID_SOCKET; @@ -4896,14 +4936,17 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Kept existing file %s (" LLintP " bytes), matching the cached size: %s%s", back[i].url_sav, (LLint) len1, - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } } } else { - hts_log_print(opt, LOG_WARNING, - "File seems complete (same size), but there was a cache read error (%u): %s%s", - r.statuscode, back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "File seems complete (same size), but there was " + "a cache read error (%u): %s%s", + r.statuscode, adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } if (r.adr) { freet(r.adr); @@ -4939,7 +4982,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, " bytes), matching the announced " "size: %s%s", back[i].url_sav, (LLint) size, - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } } } @@ -4987,7 +5031,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "size, range ignored: %s%s", back[i].url_sav, (LLint) back[i].range_req_size, - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } } } @@ -5035,7 +5080,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, // head: terminé if (back[i].head_request) { hts_log_print(opt, LOG_DEBUG, "Tested file: %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(head request): deletehttp\n"); #endif @@ -5089,9 +5135,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, get_httptype_sized(opt, back[i].r.contenttype, sizeof(back[i].r.contenttype), back[i].url_sav, 1); - hts_log_print(opt, LOG_DEBUG, - "Not-modified status without cache guessed: %s%s", - back[i].url_adr, back[i].url_fil); + hts_log_print( + opt, LOG_DEBUG, + "Not-modified status without cache guessed: %s%s", + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } // Status is okay? @@ -5104,7 +5151,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, server_sent_304 ? HTS_FALSE : HTS_TRUE; hts_log_print(opt, LOG_DEBUG, "File loaded after test from cache: %s%s", - back[i].url_adr, back[i].url_fil); + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); // finalize if (back[i].r.statuscode > 0) { @@ -5139,8 +5187,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, hts_log_print( opt, LOG_NOTICE, "File excluded by MIME type filter (%s): %s%s", - back[i].r.contenttype, back[i].url_adr, - back[i].url_fil); + back[i].r.contenttype, + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } else { // il faut aller le chercher // effacer buffer (requète) @@ -5209,11 +5257,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, /* create a temporary reference file in case of * broken mirror */ if (back_serialize_ref(opt, &back[i]) != 0) { - hts_log_print(opt, LOG_WARNING, - "Could not create temporary " - "reference file for %s%s", - back[i].url_adr, - back[i].url_fil); + hts_log_print( + opt, LOG_WARNING, + "Could not create temporary " + "reference file for %s%s", + adr_nologin(opt, back[i].url_adr), + back[i].url_fil); } #if HDEBUG printf("continue interrupted file\n"); @@ -5285,8 +5334,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, "Unusable partial-content range for %s%s " "(have " LLintP " bytes, got " LLintP "-" LLintP "/" LLintP "), restarting", - back[i].url_adr, back[i].url_fil, - (LLint) sz, back[i].r.crange_start, + adr_nologin(opt, back[i].url_adr), + back[i].url_fil, (LLint) sz, + back[i].r.crange_start, back[i].r.crange_end, back[i].r.crange); url_savename_refname_remove(opt, back[i].url_adr, back[i].url_fil); @@ -5409,12 +5459,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) { if (back_status(&back[i]) == STATUS_READY) { if (back[i].r.statuscode == HTTP_OK) - printf("* %s%s (" LLintP " bytes) - OK\n", back[i].url_adr, - back[i].url_fil, (LLint) back[i].r.size); + printf("* %s%s (" LLintP " bytes) - OK\n", + adr_nologin(opt, back[i].url_adr), back[i].url_fil, + (LLint) back[i].r.size); else - printf("* %s%s (" LLintP " bytes) - %d\n", back[i].url_adr, - back[i].url_fil, (LLint) back[i].r.size, - back[i].r.statuscode); + printf("* %s%s (" LLintP " bytes) - %d\n", + adr_nologin(opt, back[i].url_adr), back[i].url_fil, + (LLint) back[i].r.size, back[i].r.statuscode); fflush(stdout); } } @@ -5457,8 +5508,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, } } if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) { - hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr, - back[i].url_fil); + hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", + adr_nologin(opt, back[i].url_adr), back[i].url_fil); if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(timeout): deletehttp\n"); diff --git a/src/htsbauth.c b/src/htsbauth.c index caac90c55..cfa664f1e 100644 --- a/src/htsbauth.c +++ b/src/htsbauth.c @@ -688,8 +688,7 @@ static hts_boolean cookie_flat_pair(httrackp *opt, t_cookie *cookie, char *pair, if (eq == NULL || eq == pair) { if (cookie_flat_ignore(ignored)) hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring a cookie with no name in %s: %.64s", - given, pair); + "Cookies: ignoring a cookie with no name in %s", given); return HTS_FALSE; } *eq = '\0'; @@ -941,6 +940,7 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, char BIGSTK line[8192]; const size_t line_max = 8000; hts_boolean first_line = HTS_TRUE; + int lineno = 0; // this load's records alone, so a line repeating one is caught t_cookie *seen = (t_cookie *) calloct(1, sizeof(t_cookie)); @@ -958,6 +958,8 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, // len counts what the buffer cut too, so this catches that line cookie_readline(fp, line, sizeof(line), &len); + if (lineno < INT_MAX) + lineno++; if (first_line && len >= sizeof(cookie_bom) - 1 && strncmp(line, cookie_bom, sizeof(cookie_bom) - 1) == 0) { len -= sizeof(cookie_bom) - 1; @@ -987,9 +989,11 @@ int cookie_load_from(httrackp *opt, t_cookie *cookie, const char *fpath, if (rec[0] == '#') { if (!cookie_known_comment(rec)) { + // no line text, because any part of it can hold a cookie value hts_log_print(opt, LOG_WARNING, - "Cookies: ignoring an unknown comment in %s: %.64s", - given, rec); + "Cookies: ignoring an unknown comment on line %d " + "of %s", + lineno, given); refused++; } } else { diff --git a/src/htscache.c b/src/htscache.c index a71f76c48..0e7eb2473 100644 --- a/src/htscache.c +++ b/src/htscache.c @@ -53,16 +53,18 @@ Please visit our Website: http://www.httrack.com void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, const char *url_adr, const char *url_fil, const char *url_save) { - hts_log_print(opt, LOG_DEBUG, "File checked by cache: %s", url_adr); + hts_log_print(opt, LOG_DEBUG, "File checked by cache: %s", + adr_nologin(opt, url_adr)); // ---stockage en cache--- // stocker dans le cache? if (opt->cache) { if (cache_writable(cache)) { // ensure not a temporary filename (should not happend ?!) if (IS_DELAYED_EXT(url_save)) { - hts_log_print(opt, LOG_WARNING, - "aborted cache validation: %s%s still has temporary name %s", - url_adr, url_fil, url_save); + hts_log_print( + opt, LOG_WARNING, + "aborted cache validation: %s%s still has temporary name %s", + adr_nologin(opt, url_adr), url_fil, url_save); return; } // c'est le seul endroit ou l'on ajoute des elements dans le cache (fichier entier ou header) @@ -105,9 +107,9 @@ void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, strcatbuff(tempo, "\n"); strcatbuff(tempo, r->location); } - hts_log_print(opt, LOG_DEBUG, - "Cached fast-header response: %s%s is %d", - url_adr, url_fil, (int) r->statuscode); + hts_log_print( + opt, LOG_DEBUG, "Cached fast-header response: %s%s is %d", + adr_nologin(opt, url_adr), url_fil, (int) r->statuscode); coucal_add(cache->cached_tests, concat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), url_adr, url_fil), (intptr_t) strdupt(tempo)); @@ -207,9 +209,10 @@ static void cache_zip_write_failed(httrackp *opt, cache_back *cache, /* fatal: stop the mirror, exit non-zero */ hts_store_release_int(&opt->state.exit_xh, -1); } else { - hts_log_print( - opt, LOG_WARNING, "cache write failed (%s: %s), entry not cached: %s%s", - what, hts_get_zerror(zErr, errbuf, sizeof(errbuf)), url_adr, url_fil); + hts_log_print(opt, LOG_WARNING, + "cache write failed (%s: %s), entry not cached: %s%s", what, + hts_get_zerror(zErr, errbuf, sizeof(errbuf)), + adr_nologin(opt, url_adr), url_fil); } } @@ -274,7 +277,7 @@ void cache_keep_previous(httrackp *opt, cache_back *cache, const char *url_adr, URL's file. */ if (r.statuscode > 0 && strcmp(previous_save, url_save) == 0) { hts_log_print(opt, LOG_DEBUG, "keeping the previous cache entry for %s%s", - url_adr, url_fil); + adr_nologin(opt, url_adr), url_fil); cache_add_ex(opt, cache, &r, url_adr, url_fil, url_save, opt->all_in_cache, StringBuff(opt->path_html_utf8), HTS_TRUE); } @@ -336,12 +339,12 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, if (r->is_write && url_save != NULL && strnotempty(url_save)) { hts_log_print(opt, LOG_WARNING, "file too large for the cache, storing headers only: %s%s", - url_adr, url_fil); + adr_nologin(opt, url_adr), url_fil); dataincache = 0; } else { hts_log_print(opt, LOG_WARNING, - "entry too large for the cache, not cached: %s%s", url_adr, - url_fil); + "entry too large for the cache, not cached: %s%s", + adr_nologin(opt, url_adr), url_fil); return; } } @@ -407,7 +410,7 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, if (headersDropped != 0) { hts_log_print(opt, LOG_WARNING, "cached headers too large, %d field(s) dropped: %s%s", - headersDropped, url_adr, url_fil); + headersDropped, adr_nologin(opt, url_adr), url_fil); } /* Filename */ @@ -421,7 +424,7 @@ static void cache_add_ex(httrackp *opt, cache_back *cache, const htsblk *r, url_fil)) { hts_log_print(opt, LOG_WARNING, "URL too long to be cached, entry not cached: %s%s", - url_adr, url_fil); + adr_nologin(opt, url_adr), url_fil); return; } } @@ -720,7 +723,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, if (!hts_location_is_safe(r.location)) { hts_log_print(opt, LOG_WARNING, "cached Location naming a post token, dropped: %s%s", - adr, fil); + adr_nologin(opt, adr), fil); r.location[0] = '\0'; } @@ -750,7 +753,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "cached filename leaving the mirror, " "not using the cache entry: %s%s", - adr, fil); + adr_nologin(opt, adr), fil); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Cache Read Error : Bad Filename"); } @@ -761,7 +764,7 @@ static htsblk cache_readex_new(httrackp *opt, cache_back *cache, hts_log_print(opt, LOG_WARNING, "cached filename too long once rebuilt under '%s', " "not using the cache entry: %s%s", - path_html, adr, fil); + path_html, adr_nologin(opt, adr), fil); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Cache Read Error : Filename Too Long"); } diff --git a/src/htscore.c b/src/htscore.c index fc198a27c..66d15e86e 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -385,8 +385,11 @@ int hts_record_link(httrackp * opt, const char *address, const char *file, const char *save, const char *ref_address, const char *ref_file, const char *codebase) { - const int success = - hts_record_link_(opt, address, file, save, ref_address, ref_file, codebase); + int success; + + hts_secret_add_login(opt, address); + success = hts_record_link_(opt, address, file, save, ref_address, ref_file, + codebase); if (!success && opt->maxlink > 0 && opt->lien_tot >= opt->maxlink) { hts_log_print(opt, LOG_PANIC, "Too many links (links=%ld, limit=%ld)", (long int) heap_top_index(), (long int) opt->maxlink); @@ -988,10 +991,10 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { // prochaine adresse if (!hts_scan_token(&a, url, sizeof(url))) { printf("URL longer than %d bytes, ignored: %.64s...\n", - (int) (sizeof(url) - 1), url); + (int) (sizeof(url) - 1), url_nologin(opt, url)); hts_log_print(opt, LOG_WARNING, "URL longer than %d bytes, ignored: %.64s...", - (int) (sizeof(url) - 1), url); + (int) (sizeof(url) - 1), url_nologin(opt, url)); continue; } @@ -1013,7 +1016,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { else strcpybuff(url, StringBuff(opt->why_url)); if (ident_url_absolute(url, &af) < 0) { - printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url)); + printf("--why: unable to parse URL %s" LF, + adr_nologin(opt, StringBuff(opt->why_url))); } else { char *const query = af.fil + strcspn(af.fil, "?"); char BIGSTK path[sizeof(af.fil)]; @@ -1030,15 +1034,15 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { filters_match_url(filters, filptr, af.adr, af.fil, &jokDepth); if (jok > 0) - printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1, - filters[jokDepth]); + printf("%s: accepted by rule #%d (%s)" LF, url_nologin(opt, url), + jokDepth + 1, filters[jokDepth]); else if (jok < 0) - printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1, - filters[jokDepth]); + printf("%s: rejected by rule #%d (%s)" LF, url_nologin(opt, url), + jokDepth + 1, filters[jokDepth]); else printf("%s: no filter rule matches; the wizard decides " "(same-site links are followed by default)" LF, - url); + url_nologin(opt, url)); } freet(primary); /* --why answered; no mirror was asked for. The store comes first @@ -1127,9 +1131,9 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { continue; } if (hts_is_quoted(line, strlen(line))) { - hts_log_print(opt, LOG_ERROR, - "\"%s\", line %d: %s is quoted, ignored", - StringBuff(opt->filelist), lineno, line); + hts_log_print( + opt, LOG_ERROR, "\"%s\", line %d: \"%s is quoted, ignored", + StringBuff(opt->filelist), lineno, adr_nologin(opt, line + 1)); continue; } if (line[0]) { @@ -1402,14 +1406,15 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { ) ) { // sauter si lien annulé (ou fil vide) if (heap(ptr) != NULL && heap(ptr)->pass2 == -1) { - hts_log_print(opt, LOG_DEBUG, "link #%d is ready, skipping: %s%s..", - ptr, ((urladr() != NULL) ? (urladr()) : (" ")), - ((urlfil() != NULL) ? (urlfil()) : (" "))); + hts_log_print( + opt, LOG_DEBUG, "link #%d is ready, skipping: %s%s..", ptr, + ((urladr() != NULL) ? adr_nologin(opt, urladr()) : (" ")), + ((urlfil() != NULL) ? (urlfil()) : (" "))); } else { - hts_log_print(opt, LOG_DEBUG, - "link #%d seems ready, skipping: %s%s..", ptr, - ((urladr() != NULL) ? (urladr()) : (" ")), - ((urlfil() != NULL) ? (urlfil()) : (" "))); + hts_log_print( + opt, LOG_DEBUG, "link #%d seems ready, skipping: %s%s..", ptr, + ((urladr() != NULL) ? adr_nologin(opt, urladr()) : (" ")), + ((urlfil() != NULL) ? (urlfil()) : (" "))); } // remove from stats if (heap(ptr)->pass2 == -1) { @@ -1424,7 +1429,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { } if (heap(ptr) != NULL) { // on a qq chose à récupérer? - hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s", urladr(), urlfil()); + hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s", + adr_nologin(opt, urladr()), urlfil()); #if DEBUG_ROBOTS if (strcmp(urlfil(), "/robots.txt") == 0) { printf("robots.txt detected\n"); @@ -1544,9 +1550,10 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (istoobig(opt, r.totalsize, opt->maxfile_html, opt->maxfile_nonhtml, r.contenttype)) { error = 0; - hts_log_print(opt, LOG_WARNING, - "Big file cancelled according to user's preferences: %s%s", - urladr(), urlfil()); + hts_log_print( + opt, LOG_WARNING, + "Big file cancelled according to user's preferences: %s%s", + adr_nologin(opt, urladr()), urlfil()); } } } @@ -1599,9 +1606,9 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { is_loaded_from_file = 1; \ r.adr = readfile2_inmem(savename(), &r.size); \ if (r.adr != NULL) { \ - hts_log_print(opt, LOG_INFO, "File successfully loaded for parsing: %s%s (%d bytes)",urladr(),urlfil(),(int)r.size); \ + hts_log_print(opt, LOG_INFO, "File successfully loaded for parsing: %s%s (%d bytes)",adr_nologin(opt, urladr()),urlfil(),(int)r.size); \ } else { \ - hts_log_print(opt, LOG_ERROR, "File could not be loaded for parsing: %s%s",urladr(),urlfil()); \ + hts_log_print(opt, LOG_ERROR, "File could not be loaded for parsing: %s%s",adr_nologin(opt, urladr()),urlfil()); \ } \ } \ } while(0) @@ -1655,7 +1662,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { hts_log_print( opt, LOG_ERROR, "Not enough memory to convert %s%s from UCS2 to UTF-8", - urladr(), urlfil()); + adr_nologin(opt, urladr()), urlfil()); r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Not enough memory"); error = 1; @@ -1663,7 +1670,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { hts_log_print(opt, LOG_WARNING, "File %s%s converted from UCS2 to UTF-8 (old " "size: %d bytes, new size: %d bytes)", - urladr(), urlfil(), (int) r.size, (int) new_size); + adr_nologin(opt, urladr()), urlfil(), + (int) r.size, (int) new_size); freet(r.adr); r.adr = new_adr; r.size = (LLint) new_size; @@ -1678,8 +1686,8 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { is_binary = 1; strcpybuff(r.contenttype, "application/octet-stream"); hts_log_print(opt, LOG_WARNING, - "File not parsed, looks like binary: %s%s", urladr(), - urlfil()); + "File not parsed, looks like binary: %s%s", + adr_nologin(opt, urladr()), urlfil()); } /* This hack allows you to avoid problems with parsing '\0' characters */ @@ -1912,13 +1920,13 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (page_charset[0] == '\0') { hts_log_print(opt, LOG_INFO, "Warning: could not detect encoding for: %s%s", - urladr(), urlfil()); + adr_nologin(opt, urladr()), urlfil()); } } /* Info for wrappers */ - hts_log_print(opt, LOG_INFO, "engine: check-html: %s%s", urladr(), - urlfil()); + hts_log_print(opt, LOG_INFO, "engine: check-html: %s%s", + adr_nologin(opt, urladr()), urlfil()); { char BIGSTK buff_err_msg[1024]; htsmoduleStruct BIGSTK str; @@ -2017,10 +2025,12 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { if (strnotempty(infobuff)) { hts_log_print(opt, LOG_INFO, "Note: robots.txt forbidden links for %s are: %s", - urladr(), infobuff); + adr_nologin(opt, urladr()), infobuff); hts_log_print(opt, LOG_NOTICE, - "Note: due to %s remote robots.txt rules, links beginning with these path will be forbidden: %s (see in the options to disable this)", - urladr(), infobuff); + "Note: due to %s remote robots.txt rules, links " + "beginning with these path will be forbidden: %s " + "(see in the options to disable this)", + adr_nologin(opt, urladr()), infobuff); } } /* After robots_parse, so the rules this very body carries already @@ -2078,7 +2088,7 @@ int httpmirror(char *url1, httrackp *opt, hts_boolean *completed_out) { } else { // lien non autorisé! (ex: cgi-bin en html) hts_log_print(opt, LOG_DEBUG, "non-html file ignored after upload at %s : %s", - urladr(), urlfil()); + adr_nologin(opt, urladr()), urlfil()); if (r.adr) { freet(r.adr); r.adr = NULL; @@ -2626,8 +2636,8 @@ void host_ban(httrackp * opt, int ptr, back[i].r.soc = INVALID_SOCKET; back[i].r.statuscode = STATUSCODE_TIMEOUT; // timeout (peu importe si c'est un traffic jam) strcpybuff(back[i].r.msg, "Link Cancelled by host control"); - hts_log_print(opt, LOG_DEBUG, "Shutdown: %s%s", back[i].url_adr, - back[i].url_fil); + hts_log_print(opt, LOG_DEBUG, "Shutdown: %s%s", + adr_nologin(opt, back[i].url_adr), back[i].url_fil); } } @@ -2642,8 +2652,8 @@ void host_ban(httrackp * opt, int ptr, l++; if ((l > 0) && (l < 1020)) { // in range if (strfield2(jump_identification_const(heap(i)->adr), host)) { // host - hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", heap(i)->adr, - heap(i)->fil); + hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", + adr_nologin(opt, heap(i)->adr), heap(i)->fil); hts_invalidate_link(opt, i); // invalidate hashtable entry // on efface pas le hash, because si on rencontre le lien, reverif sav.. } @@ -2762,7 +2772,8 @@ static hts_boolean live_cancel_link_(httrackp *opt, struct_back *sback, int i) { /* no cache: a dropped page must not be recorded as fetched */ back_delete(opt, NULL, sback, b); } - hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", heap(i)->adr, heap(i)->fil); + hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", adr_nologin(opt, heap(i)->adr), + heap(i)->fil); hts_invalidate_link(opt, i); /* the main loop takes every cancelled link out of this count */ HTS_STAT.stat_background++; @@ -4508,6 +4519,7 @@ HTSEXT_API int hts_is_exiting(httrackp * opt) { // message d'erreur? char *hts_errmsg(httrackp * opt) { + hts_hide_secrets(opt, opt->state._hts_errmsg, sizeof(opt->state._hts_errmsg)); return opt->state._hts_errmsg; } @@ -4841,18 +4853,19 @@ int htsAddLink(htsmoduleStruct * str, char *link) { codebase[0] = '\0'; - hts_log_print(opt, LOG_DEBUG, "(module): adding link : '%s'", link); + hts_log_print(opt, LOG_DEBUG, "(module): adding link : '%s'", + url_nologin(opt, link)); // recopie de "creer le lien" // if (!RUN_CALLBACK1(opt, linkdetected, link)) { hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper", - link); + url_nologin(opt, link)); return 0; } if (!RUN_CALLBACK2(opt, linkdetected2, link, NULL)) { hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper(2)", - link); + url_nologin(opt, link)); return 0; } // adr = c'est la même @@ -4884,14 +4897,14 @@ int htsAddLink(htsmoduleStruct * str, char *link) { strcpybuff(codebase, tempo); // couper host } else { hts_log_print(opt, LOG_ERROR, "Unexpected strstr error in base %s", - codebase); + url_nologin(opt, codebase)); } } } if (!((int) strlen(codebase) < HTS_URLMAXSIZE)) { // trop long hts_log_print(opt, LOG_ERROR, "Codebase too long, parsing skipped (%s)", - codebase); + url_nologin(opt, codebase)); } { @@ -4943,9 +4956,11 @@ int htsAddLink(htsmoduleStruct * str, char *link) { if (savename()) { if (lienrelatif(tempo, sizeof(tempo), afs.save, savename()) == 0) { - hts_log_print(opt, LOG_DEBUG, - "(module): relative link at %s build with %s and %s: %s", - afs.af.adr, afs.save, savename(), tempo); + hts_log_print( + opt, LOG_DEBUG, + "(module): relative link at %s build with %s and %s: %s", + adr_nologin(opt, afs.af.adr), afs.save, savename(), + tempo); if (str->localLink && str->localLinkSize > (int) strlen(tempo) + 1) { strlcpybuff(str->localLink, tempo, str->localLinkSize); @@ -4957,7 +4972,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) { if (forbidden_url) { hts_log_print(opt, LOG_DEBUG, "(module): file not caught: %s", - lien); + url_nologin(opt, lien)); if (str->localLink && str->localLinkSize > (int) (strlen(afs.af.adr) + strlen(afs.af.fil) + 8)) { htsbuff lb = htsbuff_ptr(str->localLink, str->localLinkSize); @@ -4970,8 +4985,9 @@ int htsAddLink(htsmoduleStruct * str, char *link) { } // if (r != -1) { - hts_log_print(opt, LOG_DEBUG, "(module): %s%s -> %s (base %s)", afs.af.adr, - afs.af.fil, afs.save, codebase); + hts_log_print(opt, LOG_DEBUG, "(module): %s%s -> %s (base %s)", + adr_nologin(opt, afs.af.adr), afs.af.fil, afs.save, + url_nologin(opt, codebase)); // modifié par rapport à l'autre version (cf prio_fix notamment et save2) @@ -5016,9 +5032,8 @@ int htsAddLink(htsmoduleStruct * str, char *link) { heap_top()->retry = opt->retry; hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s", - heap_top()->adr, heap_top()->fil, + adr_nologin(opt, heap_top()->adr), heap_top()->fil, heap_top()->sav); - } } } diff --git a/src/htscoremain.c b/src/htscoremain.c index 5f37eb582..28f1d188f 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -312,13 +312,35 @@ static void cmdl_panic_value(httrackp *opt, const char *msg, const char *value) { char s[HTS_CDLMAXSIZE * 2 + 256]; + char shown[HTS_CDLMAXSIZE]; + if (hts_is_quoted(value, strlen(value))) - snprintf(s, sizeof(s), "%s: %s " HTS_QUOTED_HINT, msg, value); + snprintf(s, sizeof(s), "%s: \"%s " HTS_QUOTED_HINT, msg, + adr_nologin_r(shown, sizeof(shown), value + 1)); else snprintf(s, sizeof(s), "%s", msg); HTS_PANIC_PRINTF(s); } +/* Prints a --host-alias rule with the login of each of its names masked. */ +static void print_rule_nologin(const char *rule) { + while (*rule != '\0') { + char tok[HTS_CDLMAXSIZE]; + char shown[HTS_CDLMAXSIZE + 8]; + size_t n = strcspn(rule, ",="); + + if (n >= sizeof(tok)) + n = sizeof(tok) - 1; + memcpy(tok, rule, n); + tok[n] = '\0'; + fputs(adr_nologin_r(shown, sizeof(shown), tok), stdout); + rule += n; + if (*rule == ',' || *rule == '=') + putchar(*rule++); + } + putchar('\n'); +} + /* Panic on com, the end of option arg that no option takes. When com is quoted, the message names the option before it. */ static void cmdl_bad_option(httrackp *opt, const char *arg, const char *prefix, @@ -385,6 +407,7 @@ static hts_boolean write_progress_lock(httrackp *opt, const char *path, static int hts_main_internal(int argc, char **argv, httrackp * opt); static hts_boolean cmdl_shortopt_has(const char *s, char c); +static hts_boolean cmdl_shortopt_named(const char *s, const char *opt); static void cmdl_doit_line(String *line, const cmdl_argv *cmd); // Main, récupère les paramètres et appelle le robot @@ -551,6 +574,32 @@ static void cmdl_print_cache_entry(httrackp *opt, cache_back *cache, freet(r.adr); } +/* Feeds the log backstop the passwords of the blank-separated start URLs, + but not of the +/- filters among them, of the proxy and --why, and the + secret -%X values. */ +static void hts_secret_learn_options(httrackp *opt, const char *urls) { + const char *u = urls; + + while (*u != '\0') { + const size_t n = strcspn(u, " \t\r\n"); + + if (n != 0 && *u != '+' && *u != '-') { + char *word = malloct(n + 1); + + if (word != NULL) { + memcpy(word, u, n); + word[n] = '\0'; + hts_secret_add_login(opt, word); + freet(word); + } + } + u += n != 0 ? n : 1; + } + hts_secret_add_login(opt, StringBuff(opt->proxy.name)); + hts_secret_add_login(opt, StringBuff(opt->why_url)); + hts_secret_add_headers(opt, StringBuff(opt->headers)); +} + static int hts_main_internal(int argc, char **argv, httrackp * opt) { /* command line rebuilt from argv, config files and doit.log */ cmdl_argv x_cmd = {NULL, NULL, NULL, 0, 0, {NULL, 0, 0}}; @@ -2044,8 +2093,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { const char *const why = http_custom_header_warning( StringBuff(opt->headers), argv[na]); - if (why != NULL) - fprintf(stderr, "warning: -%%X %s: %s\n", why, argv[na]); + // with the -%H log's redaction of a login header + if (why != NULL) { + fprintf(stderr, "warning: -%%X %s: ", why); + fprintfio(stderr, argv[na], "", opt); + fputc('\n', stderr); + } } /* A refused line goes in, so the request drops its folds. */ if (StringNotEmpty(opt->headers) || @@ -2236,7 +2289,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { if (!hts_host_alias_rule_ok(argv[na])) { HTS_PANIC_PRINTF("Invalid host-alias rule: expected " "alias[,alias...]=canonical-host"); - printf("Rejected: %s\n", argv[na]); + printf("Rejected: "); + print_rule_nologin(argv[na]); printf("Each side names a host, optionally behind its " "scheme; a path is not part of an address\n"); htsmain_free(); @@ -2997,6 +3051,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { // un petit lock-file pour indiquer un miroir en cours, ainsi qu'un éventuel fichier log { char t[256]; + char *masked_url; + const char *shown_url; time_local_rfc822(t); // faut bien que ca serve quelque part l'heure RFC1945 arf' @@ -3107,20 +3163,47 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { "engine: could not create hts-in_progress.lock, so" " this mirror cannot be stopped or paused by a file"); } - // fichier log + hts_secret_learn_options(opt, url); + masked_url = hts_mask_passwords(url, HTS_MASK_LIST); + shown_url = masked_url != NULL ? masked_url : "(out of memory)"; + // hts-log.txt if (opt->log) { - int i; + int i, opt_at = 0; fprintf(opt->log, "HTTrack" HTTRACK_VERSION "%s launched on %s at %s" LF, - hts_get_version_info(opt), t, url); + hts_get_version_info(opt), t, shown_url); fprintf(opt->log, "("); for(i = 0; i < argc; i++) { const char *arg = argv[i]; // already UTF-8 on every platform - if (strchr(arg, ' ') == NULL || strchr(arg, '\"') != NULL) - fprintf(opt->log, "%s ", arg); - else // entre "" (si espace(s) et pas déja de ") - fprintf(opt->log, "\"%s\" ", arg); + // quoted if it holds a blank and no quote + const hts_boolean quote = + strchr(arg, ' ') != NULL && strchr(arg, '\"') == NULL; + // a start URL, or a -P, -%mu, -%R or --why value, is one address + hts_boolean addr; + char *masked; + + if (x_cmd.span[i] != 0) { + opt_at = x_cmd.span[i] > 1 ? i : 0; + addr = HTS_FALSE; + } else if (opt_at != 0 && i < opt_at + x_cmd.span[opt_at]) { + addr = cmdl_shortopt_has(argv[opt_at], 'P') || + cmdl_shortopt_named(argv[opt_at], "-%mu") || + cmdl_shortopt_named(argv[opt_at], "-%R") || + cmdl_shortopt_named(argv[opt_at], "-%Y"); + } else { + opt_at = 0; + addr = i != 0 && arg[0] != '+' && arg[0] != '-'; + } + masked = addr ? hts_mask_passwords(arg, HTS_MASK_ADDR) : NULL; + + if (quote) + fputc('"', opt->log); + // a -%X login header gets the -%H log's redaction + if (!addr || masked != NULL) + fprintfio(opt->log, addr ? masked : arg, "", opt); + fputs(quote ? "\" " : " ", opt->log); + freet(masked); } fprintf(opt->log, ")" LF); fprintf(opt->log, LF); @@ -3136,14 +3219,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { HTTRACK_VERSION "%s " HTTRACK_AFF_AUTHORS "" LF, t, hts_get_version_info(opt)); if (opt->wizard == HTS_WIZARD_NONE) { - printf - ("mirroring %s with %d levels, %d sockets,t=%d,s=%d,logm=%d,lnk=%d,mdg=%d\n", - url, opt->depth, opt->maxsoc, opt->travel, opt->seeker, - httrack_logmode, opt->urlmode, opt->getmode); + printf("mirroring %s with %d levels, %d " + "sockets,t=%d,s=%d,logm=%d,lnk=%d,mdg=%d\n", + shown_url, opt->depth, opt->maxsoc, opt->travel, opt->seeker, + httrack_logmode, opt->urlmode, opt->getmode); } else { // the magic wizard - printf("mirroring %s with the wizard help..\n", url); + printf("mirroring %s with the wizard help..\n", shown_url); } } + freet(masked_url); } io_flush; @@ -3435,13 +3519,20 @@ static void cmdl_doit_line(String *line, const cmdl_argv *cmd) { /* Does the short-option cluster s carry c from the main option set (-i, -iC2, -%Mi)? The i of -%i belongs to another set. */ static hts_boolean cmdl_shortopt_has(const char *s, char c) { + const char opt[3] = {'-', c, '\0'}; + + return cmdl_shortopt_named(s, opt); +} + +/* Does the short option cluster s hold the option opt, such as "-%mu"? */ +static hts_boolean cmdl_shortopt_named(const char *s, const char *opt) { char name[8]; size_t at = 1; if (s[0] != '-' || s[1] == '-') return HTS_FALSE; while (optcluster_next(s, &at, name, sizeof(name))) - if (name[1] == c && name[2] == '\0') + if (strcmp(name, opt) == 0) return HTS_TRUE; return HTS_FALSE; } diff --git a/src/htsftp.c b/src/htsftp.c index 338fffa97..dee8b47d6 100644 --- a/src/htsftp.c +++ b/src/htsftp.c @@ -639,7 +639,7 @@ static hts_boolean ftp_may_resume(httrackp *opt, const lien_back *back, if (back->r.totalsize <= back->range_req_size) { hts_log_print(opt, LOG_DEBUG, "FTP: not resuming %s%s, the local copy is not shorter", - back->url_adr, back->url_fil); + adr_nologin(opt, back->url_adr), back->url_fil); return HTS_FALSE; } /* Equality, not ordering: a copy this code did not stamp carries a local @@ -648,7 +648,7 @@ static hts_boolean ftp_may_resume(httrackp *opt, const lien_back *back, remote_mtime != local_mtime) { hts_log_print(opt, LOG_DEBUG, "FTP: not resuming %s%s, the copy is not dated as the remote", - back->url_adr, back->url_fil); + adr_nologin(opt, back->url_adr), back->url_fil); return HTS_FALSE; } return HTS_TRUE; @@ -881,7 +881,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) { SOCaddr_initport(server, port); strcpybuff(back->info, "connect"); - hts_log_print(opt, LOG_DEBUG, "FTP: connecting to %s:%d", _adr, port); + hts_log_print(opt, LOG_DEBUG, "FTP: connecting to %s:%d", + adr_nologin(opt, _adr), port); if (!ftp_connect(back, soc_ctl, &server, timeout, opt)) { strcpybuff(back->r.msg, "Unable to connect to the server"); @@ -1414,7 +1415,12 @@ int send_line(T_SOC soc, const char *data) { if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo, "---> %s\x0d\x0a", data); + if (strncmp(data, "PASS ", 5) == 0) { + fprintf(ioinfo, "---> PASS "); + fprint_redacted(ioinfo, strlen(data + 5)); + fprintf(ioinfo, "\x0d\x0a"); + } else + fprintf(ioinfo, "---> %s\x0d\x0a", data); fflush(ioinfo); } } diff --git a/src/htslib.c b/src/htslib.c index 235ec5720..29184bd29 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -1702,6 +1702,8 @@ int http_sendhead(httrackp *opt, t_cookie *cookie, int mode, const char *xsend, strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), user_pass)); code64((unsigned char *) user_pass, (int) strlen(user_pass), (unsigned char *) autorisation, 0); + hts_secret_add(opt, autorisation, strlen(autorisation), + HTS_SECRET_HEADER); print_buffer(&bstr, "Proxy-Authorization: Basic %s"H_CRLF, autorisation); #if HDEBUG @@ -1959,6 +1961,8 @@ int http_sendhead(httrackp *opt, t_cookie *cookie, int mode, const char *xsend, OPT_GET_BUFF_SIZE(opt), user_pass)); code64((unsigned char *) user_pass, (int) strlen(user_pass), (unsigned char *) autorisation, 0); + hts_secret_add(opt, autorisation, strlen(autorisation), + HTS_SECRET_HEADER); if (strcmp(fil, "/robots.txt")) /* pas robots.txt */ bauth_add(cookie, astart, fil, autorisation); } else { @@ -2021,9 +2025,9 @@ int http_sendhead(httrackp *opt, t_cookie *cookie, int mode, const char *xsend, #endif if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo, "[%d] request for %s%s:\r\n", retour->debugid, - jump_identification_const(adr), fil); - fprintfio(ioinfo, bstr.buffer, "<<< "); + hts_secret_fprintf(opt, ioinfo, "[%d] request for %s%s:\r\n", + retour->debugid, jump_identification_const(adr), fil); + fprintfio(ioinfo, bstr.buffer, "<<< ", opt); fprintf(ioinfo, "\r\n"); fflush(ioinfo); } @@ -2567,7 +2571,7 @@ void treathead_opt(httrackp *opt, t_cookie *cookie, const char *adr, &attr) != 0) { hts_log_print(opt, LOG_WARNING, "Cookies: the jar did not store '%.64s' set by %s", - cook_name, adr); + cook_name, adr_nologin(opt, adr)); } } } @@ -4351,8 +4355,86 @@ char *FUN(char *source) { \ return ret != NULL ? source + ( ret - source ) : NULL; \ } -// retourne le pointeur ou le pointeur + offset si il existe dans la chaine un @ signifiant -// une identification +/* Appends n bytes of s at buf[*len], clipped to size - 1, NUL-terminated. */ +static void append_clipped(char *buf, size_t size, size_t *len, const char *s, + size_t n) { + if (n > size - 1 - *len) + n = size - 1 - *len; + memcpy(buf + *len, s, n); + *len += n; + buf[*len] = '\0'; +} + +/* Returns adr past its blanks, scheme and "//", the engine's or any + "scheme://", or adr when it has no scheme. */ +static const char *login_start(const char *adr) { + const char *s = adr; + const char *known; + + while (isspace((unsigned char) *s)) + s++; + known = jump_protocol_const(s); + if (known != s) + return known; + if (!isalpha((unsigned char) *s)) + return adr; + while (isalnum((unsigned char) *s) || *s == '+' || *s == '-' || *s == '.') + s++; + return strncmp(s, "://", 3) == 0 ? s + 3 : adr; +} + +/* Returns adr past its login, or login_start(adr) when it has none. */ +static const char *login_end(const char *adr) { + const char *const start = login_start(adr); + const char *const at = strrchr_limit(start, '@', strchr(start, '/')); + + return at != NULL ? at : start; +} + +/* Returns HTS_TRUE when adr has a login, a user alone included. */ +static hts_boolean has_login(const char *adr) { + return login_end(adr) > login_start(adr) ? HTS_TRUE : HTS_FALSE; +} + +const char *adr_nologin_r(char *buf, size_t size, const char *adr) { + const char *const scheme = login_start(adr); + const char *const host = login_end(adr); + const char *colon; + size_t len = 0; + + if (!has_login(adr) || size == 0) + return adr; + colon = memchr(scheme, ':', (size_t) (host - scheme)); + buf[0] = '\0'; + if (colon != NULL) { + append_clipped(buf, size, &len, adr, (size_t) (colon - adr)); + append_clipped(buf, size, &len, ":***@", 5); + } else { + append_clipped(buf, size, &len, adr, (size_t) (scheme - adr)); + append_clipped(buf, size, &len, "***@", 4); + } + append_clipped(buf, size, &len, host, strlen(host)); + return buf; +} + +const char *url_nologin_r(char *buf, size_t size, const char *url) { + return login_start(url) != url ? adr_nologin_r(buf, size, url) : url; +} + +const char *adr_nologin(httrackp *opt, const char *adr) { + if (!has_login(adr)) + return adr; + // no buffer to mask into: drop the whole login + if (opt == NULL) + return login_end(adr); + return adr_nologin_r(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), adr); +} + +const char *url_nologin(httrackp *opt, const char *url) { + return login_start(url) != url ? adr_nologin(opt, url) : url; +} + +// Returns source past its login (user:pass@), if any. HTSEXT_API const char *jump_identification_const(const char *source) { const char *a, *trytofind; @@ -5806,25 +5888,526 @@ int may_unknown2(httrackp * opt, const char *mime, const char *filename) { // -- Utils fichiers -// pretty print for i/o -void fprintfio(FILE * fp, const char *buff, const char *prefix) { - char nl = 1; +/* The backstop: the passwords and tokens the engine has seen, hidden by value + in every log line, behind the per-site masking. */ +typedef struct hts_secret_set { + htsmutex lock; + int count; + hts_boolean full; /* a secret was refused for want of room */ + hts_boolean toolong; /* a secret was refused for its length */ + char *item[HTS_SECRET_MAX]; + size_t len[HTS_SECRET_MAX]; + hts_secret_kind kind[HTS_SECRET_MAX]; +} hts_secret_set; - while(*buff) { - switch (*buff) { - case 13: - break; - case 10: - fprintf(fp, "\r\n"); - nl = 1; +void hts_secret_init(httrackp *opt) { + hts_secret_set *const set = calloct(1, sizeof(hts_secret_set)); + + if (set != NULL) + hts_mutexinit(&set->lock); + opt->secrets = set; +} + +void hts_secret_free(httrackp *opt) { + hts_secret_set *set = (hts_secret_set *) opt->secrets; + int i; + + if (set == NULL) + return; + for (i = 0; i < set->count; i++) + freet(set->item[i]); + hts_mutexfree(&set->lock); + freet(set); + opt->secrets = NULL; +} + +void hts_secret_add(httrackp *opt, const char *s, size_t len, + hts_secret_kind kind) { + hts_secret_set *const set = + opt != NULL ? (hts_secret_set *) opt->secrets : NULL; + hts_boolean warn = HTS_FALSE, warnlong = HTS_FALSE; + int i; + + if (set == NULL || len == 0 || + (kind == HTS_SECRET_HEADER && len < HTS_SECRET_TOKEN)) + return; + hts_mutexlock(&set->lock); + for (i = 0; i < set->count; i++) + if (set->kind[i] == kind && set->len[i] == len && + memcmp(set->item[i], s, len) == 0) break; - default: + if (len > HTS_SECRET_MAXLEN) { + warnlong = set->toolong ? HTS_FALSE : HTS_TRUE; + set->toolong = HTS_TRUE; + } else if (i == set->count) { + if (set->count < HTS_SECRET_MAX) { + char *const copy = malloct(len + 1); + + if (copy != NULL) { + memcpy(copy, s, len); + copy[len] = '\0'; + set->item[set->count] = copy; + set->len[set->count] = len; + set->kind[set->count] = kind; + set->count++; + } + } else if (!set->full) { + set->full = HTS_TRUE; + warn = HTS_TRUE; + } + } + hts_mutexrelease(&set->lock); + // the log line takes the lock again + if (warn) + hts_log_print(opt, LOG_WARNING, + "more than %d passwords seen: the log hides the later ones" + " only where they stand in a URL", + HTS_SECRET_MAX); + if (warnlong) + hts_log_print(opt, LOG_WARNING, + "a password or token is over %d bytes, so the log hides it" + " only in a URL login or a known header", + HTS_SECRET_MAXLEN); +} + +void hts_secret_add_login(httrackp *opt, const char *url) { + const char *start, *host, *colon; + + if (url == NULL) + return; + start = login_start(url); + host = login_end(url); + if (host <= start) + return; + colon = memchr(start, ':', (size_t) (host - 1 - start)); + if (colon == NULL) + return; + hts_secret_add(opt, colon + 1, (size_t) (host - 1 - (colon + 1)), + HTS_SECRET_LOGIN); + // the Basic login is sent unescaped + if (memchr(colon + 1, '%', (size_t) (host - 1 - (colon + 1))) != NULL && + (size_t) (host - 1 - (colon + 1)) < HTS_SECRET_MAXLEN) { + char pass[HTS_SECRET_MAXLEN + 1]; + char plain[HTS_SECRET_MAXLEN + 1]; + const char *p; + + pass[0] = '\0'; + strncatbuff(pass, colon + 1, (size_t) (host - 1 - (colon + 1))); + p = unescape_http(plain, sizeof(plain), pass); + hts_secret_add(opt, p, strlen(p), HTS_SECRET_LOGIN); + } +} + +/* Does a word start at s + i, after the line start, a blank, a quote, '(' or + '<', or right after "://"? */ +static hts_boolean word_start(const char *s, size_t i) { + return i == 0 || strchr(" \t\r\n\"'(<", s[i - 1]) != NULL || + (i >= 3 && strncmp(s + i - 3, "://", 3) == 0) + ? HTS_TRUE + : HTS_FALSE; +} + +/* Does the n bytes at s + i stand as a login, as in "SECRET@" or + "user:SECRET@", whose userinfo starts a word? */ +static hts_boolean login_at(const char *s, size_t len, size_t i, size_t n) { + // schemes whose "name:local@host" is an address, not a login + static const char *const opaque[] = {"mailto", "news", "sip", + "sips", "tel", "xmpp"}; + size_t u = i, k; + + if (i + n >= len || s[i + n] != '@') + return HTS_FALSE; + if (i != 0 && s[i - 1] == ':') { + u = i - 1; + while (u != 0 && strchr("/ \t\r\n", s[u - 1]) == NULL) + u--; + for (k = 0; k < sizeof(opaque) / sizeof(opaque[0]); k++) + if (strlen(opaque[k]) == i - 1 - u && strfield(s + u, opaque[k])) + return HTS_FALSE; + } + return word_start(s, u); +} + +/* Returns the length of the longest secret of set standing at s + i, or 0. A + URL password counts only as a login, a header token anywhere. */ +static size_t secret_at(const hts_secret_set *set, const char *s, size_t len, + size_t i) { + size_t best = 0; + int k; + + for (k = 0; k < set->count; k++) { + const size_t n = set->len[k]; + + if (n <= best || n > len - i || memcmp(s + i, set->item[k], n) != 0) + continue; + if (set->kind[k] == HTS_SECRET_HEADER || login_at(s, len, i, n)) + best = n; + } + return best; +} + +/* Copies the len bytes at s to out, if not NULL, with each secret of set + replaced. Returns the length written. */ +static size_t secret_scan(const hts_secret_set *set, const char *s, size_t len, + char *out) { + size_t i = 0, o = 0; + + while (i < len) { + const size_t n = secret_at(set, s, len, i); + + if (n != 0) { + if (out != NULL) + memcpy(out + o, "***", 3); + o += 3; + i += n; + } else { + if (out != NULL) + out[o] = s[i]; + o++; + i++; + } + } + return o; +} + +/* Returns a copy of the len bytes at s with each secret of opt as ***, + allocated with malloct, its length in *outlen. NULL when no secret is + known, or *outlen set to 0 too when out of memory. */ +static char *secret_mask(httrackp *opt, const char *s, size_t len, + size_t *outlen) { + hts_secret_set *const set = + opt != NULL ? (hts_secret_set *) opt->secrets : NULL; + char *out = NULL; + + *outlen = 1; + if (set == NULL) + return NULL; + hts_mutexlock(&set->lock); + if (set->count != 0) { + *outlen = secret_scan(set, s, len, NULL); + out = malloct(*outlen + 1); + if (out != NULL) { + secret_scan(set, s, len, out); + out[*outlen] = '\0'; + } else + *outlen = 0; + } + hts_mutexrelease(&set->lock); + return out; +} + +void hts_secret_fwrite(httrackp *opt, FILE *fp, const char *s, size_t len) { + size_t n; + char *masked = secret_mask(opt, s, len, &n); + + if (masked != NULL) + fwrite(masked, 1, n, fp); + else if (n == 0) + fputs("", fp); + else + fwrite(s, 1, len, fp); + freet(masked); +} + +/* Does opt know a secret? */ +static hts_boolean secrets_known(httrackp *opt) { + hts_secret_set *const set = + opt != NULL ? (hts_secret_set *) opt->secrets : NULL; + hts_boolean known; + + if (set == NULL) + return HTS_FALSE; + hts_mutexlock(&set->lock); + known = set->count != 0 ? HTS_TRUE : HTS_FALSE; + hts_mutexrelease(&set->lock); + return known; +} + +/* vfprintf() to fp, with each secret of opt as ***. */ +static void secret_vfprintf(httrackp *opt, FILE *fp, const char *format, + va_list args) { + va_list args_copy; + int n; + char *line; + + if (!secrets_known(opt)) { + (void) vfprintf(fp, format, args); + return; + } + va_copy(args_copy, args); + n = vsnprintf(NULL, 0, format, args_copy); + va_end(args_copy); + if (n < 0) + return; + line = malloct((size_t) n + 1); + if (line == NULL) { + fputs("", fp); + return; + } + (void) vsnprintf(line, (size_t) n + 1, format, args); + hts_secret_fwrite(opt, fp, line, (size_t) n); + freet(line); +} + +void hts_secret_fprintf(httrackp *opt, FILE *fp, const char *format, ...) { + va_list args; + + va_start(args, format); + secret_vfprintf(opt, fp, format, args); + va_end(args); +} + +HTSEXT_API void hts_hide_secrets(httrackp *opt, char *s, size_t size) { + size_t n; + char *masked; + + if (size == 0) + return; + masked = secret_mask(opt, s, strlen(s), &n); + if (masked != NULL) { + s[0] = '\0'; + strlncatbuff(s, masked, size, size - 1); + } else if (n == 0) + s[0] = '\0'; + freet(masked); +} + +void fprint_redacted(FILE *fp, size_t len) { + fprintf(fp, "", (unsigned long) len); +} + +/* The -%H log hides the value of these fields because they carry logins. + Unlike refused[] and single[], this list governs the log, not the wire. */ +static const char *const secret_fields[] = { + "Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"}; + +/* The -%H log hides the login of the URL these fields carry. */ +static const char *const url_fields[] = {"Location", "Content-Location", + "Referer"}; + +/* Writes the line buff, len bytes long, with the login of its URL, from keep + up to end, masked. Returns HTS_FALSE, writing nothing, when out of memory. */ +static hts_boolean fprint_url_span(httrackp *opt, FILE *fp, const char *buff, + size_t len, size_t keep, size_t end) { + char *line = malloct(2 * end + 8); + + if (line == NULL) + return HTS_FALSE; + memcpy(line, buff, end); + line[end] = '\0'; + { + const char *const url = url_nologin_r(line + end + 1, end + 7, line + keep); + + hts_secret_fwrite(opt, fp, line, keep); + hts_secret_fwrite(opt, fp, url, strlen(url)); + hts_secret_fwrite(opt, fp, buff + end, len - end); + } + freet(line); + return HTS_TRUE; +} + +/* Writes the header line buff, len bytes long, with the login of its URL + value masked. Returns HTS_FALSE, writing nothing, when out of memory. */ +static hts_boolean fprint_url_field(httrackp *opt, FILE *fp, const char *buff, + size_t len) { + const char *const colon = memchr(buff, ':', len); + size_t keep = colon != NULL ? (size_t) (colon - buff) + 1 : len; + + while (keep < len && (buff[keep] == ' ' || buff[keep] == '\t')) + keep++; + return fprint_url_span(opt, fp, buff, len, keep, len); +} + +/* Writes the request line buff, len bytes long, with the login of its target + masked, as a proxied ftp:// target carries one. Returns HTS_FALSE, writing + nothing, when buff is no "METHOD target HTTP/" line or out of memory. */ +static hts_boolean fprint_request_line(httrackp *opt, FILE *fp, + const char *buff, size_t len) { + const char *const sp = memchr(buff, ' ', len); + const char *version; + + if (sp == NULL) + return HTS_FALSE; + version = memchr(sp + 1, ' ', len - (size_t) (sp + 1 - buff)); + if (version == NULL || (size_t) (buff + len - version) < 6 || + strncmp(version, " HTTP/", 6) != 0) + return HTS_FALSE; + return fprint_url_span(opt, fp, buff, len, (size_t) (sp + 1 - buff), + (size_t) (version - buff)); +} + +/* Finds the secret of the userinfo at the authority a, which ends at the last + '@' before the first '/', the split jump_identification_const() makes: the + password after the first ':', or a user alone. A token also ends at a + blank. Returns HTS_FALSE when there is no userinfo. */ +static hts_boolean userinfo_password(const char *a, hts_boolean token, + const char **pass, const char **at) { + const char *end = a + strcspn(a, token ? "/ \t\r\n" : "/"); + const char *colon; + + while (end > a && end[-1] != '@') + end--; + if (end == a) + return HTS_FALSE; + colon = memchr(a, ':', (size_t) (end - 1 - a)); + *pass = colon != NULL ? colon + 1 : a; + *at = end - 1; + return HTS_TRUE; +} + +/* The masked copy: dst (NULL to only count) holds o bytes, and the source is + copied up to p. */ +typedef struct { + char *dst; + size_t o; + const char *p; +} mask_out; + +static void mask_copy(mask_out *m, const char *src, size_t n) { + if (m->dst != NULL) + memcpy(m->dst + m->o, src, n); + m->o += n; +} + +/* Masks the login secret of the address at a, if any, and moves *i past it. */ +static void mask_at(mask_out *m, const char *a, hts_boolean token, + const char **i) { + const char *pass, *at; + + if (!userinfo_password(jump_protocol_const(a), token, &pass, &at)) + return; + mask_copy(m, m->p, (size_t) (pass - m->p)); + mask_copy(m, "***", 3); + m->p = *i = at; +} + +/* Copies s to dst, or only counts when dst is NULL, with the login secret of + each address written as ***. Returns the length. */ +static size_t mask_passwords(const char *s, char *dst, hts_mask_mode mode) { + mask_out m = {dst, 0, s}; + const char *i = s; + + if (mode == HTS_MASK_ADDR) + mask_at(&m, s, HTS_FALSE, &i); + else + for (; *i != '\0'; i++) + // a token that is no filter is a start URL + if ((i == s || strchr(" \t\r\n", i[-1]) != NULL) && + strchr(" \t\r\n+-", *i) == NULL) + mask_at(&m, i, HTS_TRUE, &i); + mask_copy(&m, m.p, strlen(m.p) + 1); + return m.o - 1; +} + +char *hts_mask_passwords(const char *s, hts_mask_mode mode) { + const size_t len = mask_passwords(s, NULL, mode); + char *const dst = (char *) malloct(len + 1); + + if (dst != NULL) + mask_passwords(s, dst, mode); + return dst; +} + +/* Returns the length of the secret field name opening line, a line len bytes + long, even with no colon behind it, or 0. */ +static size_t secret_name_len(const char *line, size_t len) { + size_t word, i; + + if (header_name_in(line, secret_fields, + sizeof(secret_fields) / sizeof(secret_fields[0]))) + return header_name_len(line); + if (memchr(line, ':', len) != NULL) + return 0; + word = strcspn(line, " \t\r\n"); + for (i = 0; i < sizeof(secret_fields) / sizeof(secret_fields[0]); i++) + if (header_name_is(line, word, secret_fields[i], strlen(secret_fields[i]))) + return word; + return 0; +} + +void hts_secret_add_headers(httrackp *opt, const char *headers) { + const char *line = headers; + + while (line != NULL && *line != '\0') { + const size_t len = strcspn(line, "\r\n"); + const size_t name = len != 0 ? secret_name_len(line, len) : 0; + + if (name != 0) { + const char *const colon = memchr(line, ':', len); + const char *v = colon != NULL ? colon + 1 : line + name; + const char *end = line + len; + + while (v < end && (*v == ' ' || *v == '\t')) + v++; + while (end > v && (end[-1] == ' ' || end[-1] == '\t')) + end--; + // a cookie value is often a plain word, and -%H hides the whole header + if (!header_name_is(line, name, "Cookie", 6) && + !header_name_is(line, name, "Set-Cookie", 10)) { + // the whole value, then the credentials after the scheme ("Basic + // TOKEN") + const char *cred = v + strcspn(v, " \t"); + + hts_secret_add(opt, v, (size_t) (end - v), HTS_SECRET_HEADER); + while (cred < end && (*cred == ' ' || *cred == '\t')) + cred++; + if (cred < end) + hts_secret_add(opt, cred, (size_t) (end - cred), HTS_SECRET_HEADER); + } + } + line += len; + while (*line == '\r' || *line == '\n') + line++; + } +} + +void fprintfio(FILE *fp, const char *buff, const char *prefix, httrackp *opt) { + const char *const start = buff; + hts_boolean nl = HTS_TRUE; + hts_boolean secret = HTS_FALSE; + size_t name = 0; + + while (*buff != '\0') { + const size_t len = strcspn(buff, "\r\n"); + + if (len != 0) { + size_t keep = len; + + /* a folded line continues the value above it */ + if (*buff != ' ' && *buff != '\t') { + name = secret_name_len(buff, len); + secret = name != 0 ? HTS_TRUE : HTS_FALSE; + } + if (secret) { + const char *const colon = memchr(buff, ':', len); + + /* a line with no colon is redacted after its name */ + keep = *buff == ' ' || *buff == '\t' ? 0 + : colon != NULL ? (size_t) (colon - buff) + 1 + : name; + while (keep < len && (buff[keep] == ' ' || buff[keep] == '\t')) + keep++; + } if (nl) fprintf(fp, "%s", prefix); - nl = 0; - fputc(*buff, fp); + nl = HTS_FALSE; + if (!(buff == start && fprint_request_line(opt, fp, buff, len)) && + (secret || *buff == ' ' || *buff == '\t' || + !header_name_in(buff, url_fields, + sizeof(url_fields) / sizeof(url_fields[0])) || + !fprint_url_field(opt, fp, buff, len))) { + hts_secret_fwrite(opt, fp, buff, keep); + if (keep < len) + fprint_redacted(fp, len - keep); + } + } + buff += len; + if (*buff == '\n') { + fprintf(fp, "\r\n"); + nl = HTS_TRUE; } - buff++; + if (*buff != '\0') + buff++; } } @@ -7264,7 +7847,7 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_ if (opt->log != NULL) { fspc(opt, opt->log, s_type); - (void) vfprintf(opt->log, format, args); + secret_vfprintf(opt, opt->log, format, args); if ((type & LOG_ERRNO) != 0) { char errbuf[HTS_STRERROR_SIZE]; @@ -7399,6 +7982,7 @@ HTSEXT_API httrackp *hts_create_opt(void) { /* mutexes */ hts_mutexinit(&opt->state.lock); + hts_secret_init(opt); /* custom wrappers */ opt->libHandles.count = 0; @@ -7675,6 +8259,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) { StringFree(opt->path_bin); /* mutexes */ + hts_secret_free(opt); hts_mutexfree(&opt->state.lock); /* Free structure */ diff --git a/src/htslib.h b/src/htslib.h index 20755dca5..9cb75bc2e 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -541,7 +541,80 @@ char *convtolower(char *catbuff, size_t catbuffsize, const char *a); void hts_lowcase(char *s); int multipleStringMatch(const char *s, const char *match); -void fprintfio(FILE * fp, const char *buff, const char *prefix); +/* Writes a header block to fp, each line after prefix, with the value of a + field that carries a login replaced by the fprint_redacted() marker, and + each secret opt knows as ***. opt may be NULL. */ +void fprintfio(FILE *fp, const char *buff, const char *prefix, httrackp *opt); + +/* The backstop behind the per-site masking: opt keeps the first + HTS_SECRET_MAX passwords and tokens it sees, each up to HTS_SECRET_MAXLEN + bytes, and the log writers print each as ***. A URL password is hidden only + as a login, before an '@', so a password such as "index" leaves index.html + alone. A header token of HTS_SECRET_TOKEN bytes or more is hidden anywhere. + */ +#define HTS_SECRET_MAX 64 +#define HTS_SECRET_MAXLEN 1024 +#define HTS_SECRET_TOKEN 8 + +typedef enum hts_secret_kind { + HTS_SECRET_LOGIN, /* a URL password */ + HTS_SECRET_HEADER /* a header token, kept only from HTS_SECRET_TOKEN bytes */ +} hts_secret_kind; + +/* Allocates and frees the set of opt->secrets. */ +void hts_secret_init(httrackp *opt); +void hts_secret_free(httrackp *opt); + +/* Adds the len bytes at s to the set of opt. Thread-safe. Once the set is + full, or for a secret over HTS_SECRET_MAXLEN bytes, logs one warning and + adds nothing. */ +void hts_secret_add(httrackp *opt, const char *s, size_t len, + hts_secret_kind kind); + +/* Adds the password of the login of url, in any scheme or none, raw and + unescaped. A user alone is not added. url may be NULL. */ +void hts_secret_add_login(httrackp *opt, const char *url); + +/* Adds the values of the Authorization, Proxy-Authorization and Cookie lines + of a -%X header block, which may be NULL. */ +void hts_secret_add_headers(httrackp *opt, const char *headers); + +/* Writes the len bytes at s to fp, with each secret of opt as ***. */ +void hts_secret_fwrite(httrackp *opt, FILE *fp, const char *s, size_t len); + +/* fprintf() to fp, with each secret of opt as ***. */ +void hts_secret_fprintf(httrackp *opt, FILE *fp, const char *format, ...) + HTS_PRINTF_FUN(3, 4); + +/* What hts_mask_passwords() reads: */ +typedef enum { + HTS_MASK_LIST, /* blank-separated start URLs and filters */ + HTS_MASK_ADDR /* one start URL, or a -P value */ +} hts_mask_mode; + +/* Returns a copy of s, allocated with malloct, in which the password, or the + user alone, of every start URL or proxy address in s is *** . The rest stays + byte for byte. NULL when out of memory. */ +char *hts_mask_passwords(const char *s, hts_mask_mode mode); + +/* Log calls never print an address raw: pass adr_nologin() or url_nologin(), + which test 660 checks. */ + +/* Returns adr with its login masked as user:*** or ***, for log lines: adr + itself when it has no login, else a copy clipped to size bytes in buf. */ +const char *adr_nologin_r(char *buf, size_t size, const char *adr); + +/* Same as adr_nologin_r(), but only a URL with a scheme or "//" has a login. + Any "scheme://" counts, not only the ones the engine fetches. */ +const char *url_nologin_r(char *buf, size_t size, const char *url); + +/* adr_nologin_r() and url_nologin_r() into a rotating buffer of opt, which + they take only when there is a login. A NULL opt drops the login. */ +const char *adr_nologin(httrackp *opt, const char *adr); +const char *url_nologin(httrackp *opt, const char *url); + +/* Writes the marker that stands for a hidden secret of len bytes. */ +void fprint_redacted(FILE *fp, size_t len); /* A write to a socket whose peer is gone raises SIGPIPE, and the default action kills the process. The library must not install a handler, because diff --git a/src/htslib_selftest.c b/src/htslib_selftest.c index 3b2d378bf..0f36d47f7 100644 --- a/src/htslib_selftest.c +++ b/src/htslib_selftest.c @@ -1599,6 +1599,325 @@ static int st_gmtime(httrackp *opt, int argc, char **argv) { return err; } +/* Is the masked in exactly want? Prints the difference when it is not. */ +static int maskpw_row(const char *in, const char *want, hts_mask_mode mode) { + char *got = hts_mask_passwords(in, mode); + const int err = got == NULL || strcmp(got, want) != 0; + + if (err) + fprintf(stderr, "maskpw: '%.80s' gave '%.80s', want '%.80s'\n", in, + got != NULL ? got : "(null)", want); + freet(got); + return err; +} + +/* A hostile URL of n ":@" pairs: in a path it stays, in a userinfo it is one + password. */ +static int maskpw_pairs(size_t n) { + const size_t len = 2 * n; + char *pairs = (char *) malloct(len + 1); + char *in = (char *) malloct(len + 32); + int err; + + if (pairs == NULL || in == NULL) { + freet(pairs); + freet(in); + return 1; + } + memset(pairs, ':', len); + for (size_t i = 1; i < len; i += 2) + pairs[i] = '@'; + pairs[len] = '\0'; + snprintf(in, len + 32, "http://h/x?%s", pairs); + err = maskpw_row(in, in, HTS_MASK_LIST); + snprintf(in, len + 32, "a%sh/ -x", pairs); + err |= maskpw_row(in, "a:***@h/ -x", HTS_MASK_LIST); + snprintf(in, len + 32, "http://a%sh", pairs); + err |= maskpw_row(in, "http://a:***@h", HTS_MASK_ADDR); + freet(pairs); + freet(in); + return err; +} + +/* Do the nologin helpers hide the password and nothing else? */ +static int nologin_rows(httrackp *opt) { + static const struct { + const char *in, *want; + } adrs[] = {{"u:p@h:80", "u:***@h:80"}, {"u:p@q@h", "u:***@h"}, + {"u@h", "***@h"}, {":p@h", ":***@h"}, + {"h:80", "h:80"}, {"file://", "file://"}, + {"https://h", "https://h"}, {"https://u:p@h", "https://u:***@h"}}, + urls[] = {{"http://u:p@h/x", "http://u:***@h/x"}, + {"//u:p@q@h", "//u:***@h"}, + {"http://tok@h/x", "http://***@h/x"}, + {"HTTPS://u:a b@[::1]:8/", "HTTPS://u:***@[::1]:8/"}, + {"http://h/x?a@b", "http://h/x?a@b"}, + {"mailto:a@b", "mailto:a@b"}, + {"u:p@h/x", "u:p@h/x"}, + {"http://u:p@hostname.example/x", "http://u:***@hostname.e"}}; + char buf[24]; + const int index = opt->state.concat.index; + int err = 0; + size_t i; + + for (i = 0; i < sizeof(adrs) / sizeof(adrs[0]); i++) { + const char *got = adr_nologin_r(buf, sizeof(buf), adrs[i].in); + + if (strcmp(got, adrs[i].want) != 0) { + fprintf(stderr, "adr_nologin: '%s' gave '%s'\n", adrs[i].in, got); + err = 1; + } + } + for (i = 0; i < sizeof(urls) / sizeof(urls[0]); i++) { + const char *got = url_nologin_r(buf, sizeof(buf), urls[i].in); + + if (strcmp(got, urls[i].want) != 0) { + fprintf(stderr, "url_nologin: '%s' gave '%s'\n", urls[i].in, got); + err = 1; + } + } + /* An address without a login is returned as is, and takes no buffer. */ + if (adr_nologin(opt, adrs[4].in) != adrs[4].in || + url_nologin(opt, urls[6].in) != urls[6].in || + opt->state.concat.index != index) { + fprintf(stderr, "nologin: a rotating buffer was taken without a login\n"); + err = 1; + } + /* Without an opt, as back_delete(NULL, ...) logs, the login is dropped. */ + if (strcmp(adr_nologin(NULL, urls[0].in), "h/x") != 0 || + strcmp(url_nologin(NULL, urls[0].in), "h/x") != 0) { + fprintf(stderr, "nologin: a NULL opt kept the login\n"); + err = 1; + } + return err; +} + +/* Reads back the log of o, from offset at, into buf. */ +static void secrets_read(FILE *fp, long at, char *buf, size_t size) { + size_t n; + + fflush(fp); + fseek(fp, at, SEEK_SET); + n = fread(buf, 1, size - 1, fp); + buf[n] = '\0'; + fseek(fp, 0, SEEK_END); +} + +/* Logs line through o and checks the log holds want, and none of bad. */ +static int secrets_row(httrackp *o, FILE *fp, const char *line, + const char *want, const char *bad) { + char got[4096]; + const long at = ftell(fp); + + hts_log_print(o, LOG_NOTICE, "%s", line); + secrets_read(fp, at, got, sizeof(got)); + if (strstr(got, want) == NULL || (bad != NULL && strstr(got, bad) != NULL)) { + fprintf(stderr, "secrets: '%s' logged '%s', want '%s'\n", line, got, want); + return 1; + } + return 0; +} + +/* A hostile page offers 200 passwords, each twice: the first + HTS_SECRET_MAX stay hidden, and one warning says so. */ +static int secrets_full(void) { + httrackp *const o = hts_create_opt(); + FILE *const fp = tmpfile(); + char got[4096]; + const char *w; + int err = 0, i; + + if (o == NULL || fp == NULL) { + fprintf(stderr, "secrets: out of memory\n"); + return 1; + } + o->log = fp; + o->debug = LOG_DEBUG; + for (i = 0; i < 200; i++) { + char url[64]; + + snprintf(url, sizeof(url), "http://u:ZPWCAP%03d@h/", i); + hts_secret_add_login(o, url); + hts_secret_add_login(o, url); + } + hts_log_print(o, LOG_NOTICE, "u:ZPWCAP000@h u:ZPWCAP%03d@h u:ZPWCAP%03d@h", + HTS_SECRET_MAX - 1, HTS_SECRET_MAX); + secrets_read(fp, 0, got, sizeof(got)); + w = strstr(got, "passwords seen"); + if (strstr(got, "u:***@h u:***@h u:ZPWCAP064@h") == NULL || w == NULL || + strstr(w + 1, "passwords seen") != NULL) { + fprintf(stderr, "secrets: full set logged '%s'\n", got); + err = 1; + } + o->log = NULL; + fclose(fp); + hts_free_opt(o); + return err; +} + +static int st_secrets(httrackp *opt, int argc, char **argv) { + httrackp *const o = hts_create_opt(); + FILE *const fp = tmpfile(); + char got[4096]; + char big[HTS_SECRET_MAXLEN + 16]; + const char *w; + long at; + int err = 0; + (void) opt; + (void) argc; + (void) argv; + + if (o == NULL || fp == NULL) { + fprintf(stderr, "secrets: out of memory\n"); + return 1; + } + o->log = fp; + o->debug = LOG_DEBUG; + // none known: the line goes out as given + err |= secrets_row(o, fp, "a u:index@h/index.html", "a u:index@h/index.html", + NULL); + // a password is hidden only as a login, at any length + hts_secret_add_login(o, "http://u:index@h/"); + hts_secret_add_login(o, "http://u:download@h/"); + err |= secrets_row(o, fp, + "a u:index@h/index.html http://index.example/ index@h " + "myindex@h /index (index@h