Minor HTTP improvements

Author: gpotter2

scapy/layers/http.py

@@ -31,7 +31,9 @@
 
 You can turn auto-decompression/auto-compression off with:
 
-    >>> conf.contribs["http"]["auto_compression"] = True
+    >>> conf.contribs["http"]["auto_compression"] = False
+
+(Defaults to True)
 """
 
 # This file is a modified version of the former scapy_http plugin.
@@ -59,11 +61,9 @@
 
 try:
     import brotli
-    is_brotli_available = True
+    _is_brotli_available = True
 except ImportError:
-    is_brotli_available = False
-    log_loading.info("Can't import brotli. Won't be able to decompress "
-                     "data streams compressed with brotli.")
+    _is_brotli_available = False
 
 if "http" not in conf.contribs:
     conf.contribs["http"] = {}
@@ -310,8 +310,14 @@ def post_dissect(self, s):
             elif "compress" in encodings:
                 import lzw
                 s = lzw.decompress(s)
-            elif "br" in encodings and is_brotli_available:
-                s = brotli.decompress(s)
+            elif "br" in encodings:
+                if _is_brotli_available:
+                    s = brotli.decompress(s)
+                else:
+                    log_loading.info(
+                        "Can't import brotli. brotli decompression "
+                        "will be ignored !"
+                    )
         except Exception:
             # Cannot decompress - probably incomplete data
             pass
@@ -330,8 +336,14 @@ def post_build(self, pkt, pay):
         elif "compress" in encodings:
             import lzw
             pay = lzw.compress(pay)
-        elif "br" in encodings and is_brotli_available:
-            pay = brotli.compress(pay)
+        elif "br" in encodings:
+            if _is_brotli_available:
+                pay = brotli.compress(pay)
+            else:
+                log_loading.info(
+                    "Can't import brotli. brotli compression will "
+                    "be ignored !"
+                )
         return pkt + pay
 
     def self_build(self, field_pos_list=None):
@@ -569,7 +581,7 @@ def tcp_reassemble(cls, data, metadata):
                 chunked = ("chunked" in encodings)
                 is_response = isinstance(http_packet.payload, HTTPResponse)
                 if chunked:
-                    detect_end = lambda dat: dat.endswith(b"\r\n\r\n")
+                    detect_end = lambda dat: dat.endswith(b"0\r\n\r\n")
                 # HTTP Requests that do not have any content,
                 # end with a double CRLF
                 elif isinstance(http_packet.payload, HTTPRequest):

scapy/sessions.py

@@ -240,7 +240,7 @@ def _process_packet(self, pkt):
         pay = pkt[TCP].payload
         if isinstance(pay, (NoPayload, conf.padding_layer)):
             return pkt
-        new_data = raw(pay)
+        new_data = pay.original
         # Match packets by a uniqute TCP identifier
         seq = pkt[TCP].seq
         ident = pkt.sprintf(self.fmt)
@@ -256,7 +256,7 @@ def _process_packet(self, pkt):
             pay_class = metadata["pay_class"]
         # Get a relative sequence number for a storage purpose
         relative_seq = metadata.get("relative_seq", None)
-        if not relative_seq:
+        if relative_seq is None:
             relative_seq = metadata["relative_seq"] = seq - 1
         seq = seq - relative_seq
         # Add the data to the buffer

test/regression.uts

@@ -7750,6 +7750,18 @@ for i in range(3, 10):
     assert HTTP not in pkts[i]
     assert H2Frame in pkts[i]
 
+= Test chunked with gzip
+
+conf.contribs["http"]["auto_compression"] = False
+z = b'\x1f\x8b\x08\x00S\\-_\x02\xff\xb3\xc9(\xc9\xcd\xb1\xcb\xcd)\xb0\xd1\x07\xb3\x00\xe6\xedpt\x10\x00\x00\x00'
+a = IP(dst="1.1.1.1", src="2.2.2.2")/TCP(seq=1)/HTTP()/HTTPResponse(Content_Encoding="gzip", Transfer_Encoding="chunked")/(b"5\r\n" + z[:5] + b"\r\n")
+b = IP(dst="1.1.1.1", src="2.2.2.2")/TCP(seq=len(a[TCP].payload)+1)/HTTP()/(hex(len(z[5:])).encode()[2:] + b"\r\n" + z[5:] + b"\r\n0\r\n\r\n")
+xa, xb = IP(raw(a)), IP(raw(b))
+conf.contribs["http"]["auto_compression"] = True
+
+c = sniff(offline=[xa, xb], session=TCPSession)[0]
+assert gzip_decompress(z) == c.load
+
 ############
 ############
 + LLMNR protocol