Minor TLS fixes (#2767)

gpotter2 · web-flow · commit bf0725f13ca7 · 2020-08-18T16:19:16.000+02:00
* Improve doc * TLS TCP decompression & small bugs * Add TLS tests * Fix issue #2767 2527
diff --git a/doc/scapy/usage.rst b/doc/scapy/usage.rst
@@ -758,10 +758,12 @@ Advanced Sniffing - Sniffing Sessions
 Scapy includes some basic Sessions, but it is possible to implement your own.
 Available by default:
 
-- ``IPSession`` -> *defragment IP packets* on-the-flow, to make a stream usable by ``prn``.
-- ``TCPSession`` -> *defragment certain TCP protocols**. Only **HTTP 1.0** currently uses this functionality.
-- ``TLSSession`` -> *matches TLS sessions* on the flow.
-- ``NetflowSession`` -> *resolve Netflow V9 packets* from their NetflowFlowset information objects
+- :py:class:`~scapy.sessions.IPSession` -> *defragment IP packets* on-the-flow, to make a stream usable by ``prn``.
+- :py:class:`~scapy.sessions.TCPSession` -> *defragment certain TCP protocols*. Currently supports:
+   - HTTP 1.0
+   - TLS
+- :py:class:`~scapy.sessions.TLSSession` -> *matches TLS sessions* on the flow.
+- :py:class:`~scapy.sessions.NetflowSession` -> *resolve Netflow V9 packets* from their NetflowFlowset information objects
 
 Those sessions can be used using the ``session=`` parameter of ``sniff()``. Examples::
 
@@ -771,7 +773,34 @@ Those sessions can be used using the ``session=`` parameter of ``sniff()``. Exam
 
 .. note::
    To implement your own Session class, in order to support another flow-based protocol, start by copying a sample from `scapy/sessions.py <https://github.com/secdev/scapy/blob/master/scapy/sessions.py>`_
-   Your custom ``Session`` class only needs to extend the ``DefaultSession`` class, and implement a ``on_packet_received`` function, such as in the example.
+   Your custom ``Session`` class only needs to extend the :py:class:`~scapy.sessions.DefaultSession` class, and implement a ``on_packet_received`` function, such as in the example.
+
+.. note:: Would you need it, you can use: ``class TLS_over_TCP(TLSSession, TCPSession): pass`` to sniff TLS packets that are defragmented.
+
+How to use TCPSession to defragment TCP packets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The layer on which the decompression is applied must be immediately following the TCP layer. You need to implement a class function called ``tcp_reassemble`` that accepts the binary data and a metada dictionary as argument and returns, when full, a packet. Let's study the (pseudo) example of TLS:
+
+.. code::
+
+    class TLS(Packet):
+        [...]
+
+        @classmethod
+        def tcp_reassemble(cls, data, metadata):
+            length = struct.unpack("!H", data[3:5])[0] + 5
+            if len(data) == length:
+                return TLS(data)
+
+
+In this example, we first get the total length of the TLS payload announced by the TLS header, and we compare it to the length of the data. When the data reaches this length, the packet is complete and can be returned. When implementing ``tcp_reassemble``, it's usually a matter of detecting when a packet isn't missing anything else.
+
+The ``data`` argument is bytes and the ``metadata`` argument is a dictionary which keys are as follow:
+
+- ``metadata["pay_class"]``: the TCP payload class (here TLS)
+- ``metadata.get("tcp_psh", False)``: will be present if the PUSH flag is set
+- ``metadata.get("tcp_end", False)``: will be present if the END or RESET flag is set
 
 Filters
 -------
diff --git a/scapy/layers/tls/automaton_cli.py b/scapy/layers/tls/automaton_cli.py
@@ -84,15 +84,21 @@ class TLSClientAutomaton(_TLSAutomaton):
     Rather than with an interruption, the best way to stop this client is by
     typing 'quit'. This won't be a message sent to the server.
 
-    _'mycert' and 'mykey' may be provided as filenames. They will be used in
-    the handshake, should the server ask for client authentication.
-    _'server_name' is the SNI. It does not need to be set.
-    _'client_hello' may hold a TLSClientHello or SSLv2ClientHello to be sent
-    to the server. This is particularly useful for extensions tweaking.
-    _'version' is a quicker way to advertise a protocol version ("sslv2",
-    "tls1", "tls12", etc.) It may be overridden by the previous 'client_hello'.
-    _'data' is a list of raw data to be sent to the server once the handshake
-    has been completed. Both 'stop_server' and 'quit' will work this way.
+    :param server: the server IP or hostname. defaults to 127.0.0.1
+    :param dport: the server port. defaults to 4433
+    :param server_name: the SNI to use. It does not need to be set
+    :param mycert:
+    :param mykey: may be provided as filenames. They will be used in
+        the handshake, should the server ask for client authentication.
+    :param client_hello: may hold a TLSClientHello or SSLv2ClientHello to be
+        sent to the server. This is particularly useful for extensions
+        tweaking.
+    :param version: is a quicker way to advertise a protocol version ("sslv2",
+        "tls1", "tls12", etc.) It may be overridden by the previous
+        'client_hello'.
+    :param data: is a list of raw data to be sent to the server once the
+        handshake has been completed. Both 'stop_server' and 'quit' will
+        work this way.
     """
 
     def parse_args(self, server="127.0.0.1", dport=4433, server_name=None,
diff --git a/scapy/layers/tls/extensions.py b/scapy/layers/tls/extensions.py
@@ -761,7 +761,7 @@ def addfield(self, pkt, s, i):
                 i = self.adjust(pkt, f)
                 if i == 0:  # for correct build if no ext and not explicitly 0
                     v = pkt.tls_session.tls_version
-                    # Xith TLS 1.3, zero lengths are always explicit.
+                    # With TLS 1.3, zero lengths are always explicit.
                     if v is None or v < 0x0304:
                         return s
                     else:
@@ -779,8 +779,8 @@ def i2len(self, pkt, i):
         return len(self.i2m(pkt, i))
 
     def getfield(self, pkt, s):
-        tmp_len = self.length_from(pkt)
-        if tmp_len is None:
+        tmp_len = self.length_from(pkt) or 0
+        if tmp_len <= 0:
             return s, []
         return s[tmp_len:], self.m2i(pkt, s[:tmp_len])
 
diff --git a/scapy/layers/tls/handshake.py b/scapy/layers/tls/handshake.py
@@ -17,9 +17,21 @@
 import struct
 
 from scapy.error import log_runtime, warning
-from scapy.fields import ByteEnumField, ByteField, EnumField, Field, \
-    FieldLenField, IntField, PacketField, PacketListField, ShortField, \
-    StrFixedLenField, StrLenField, ThreeBytesField, UTCTimeField
+from scapy.fields import (
+    ByteEnumField,
+    ByteField,
+    Field,
+    FieldLenField,
+    IntField,
+    PacketField,
+    PacketListField,
+    ShortEnumField,
+    ShortField,
+    StrFixedLenField,
+    StrLenField,
+    ThreeBytesField,
+    UTCTimeField,
+)
 
 from scapy.compat import hex_bytes, orb, raw
 from scapy.config import conf
@@ -481,18 +493,17 @@ class TLSServerHello(_TLSHandshake):
                    _SessionIDField("sid", "",
                                    length_from=lambda pkt: pkt.sidlen),
 
-                   EnumField("cipher", None, _tls_cipher_suites),
+                   ShortEnumField("cipher", None, _tls_cipher_suites),
                    _CompressionMethodsField("comp", [0],
                                             _tls_compression_algs,
                                             itemfmt="B",
                                             length_from=lambda pkt: 1),
 
                    _ExtensionsLenField("extlen", None, length_of="ext"),
                    _ExtensionsField("ext", None,
-                                    length_from=lambda pkt: (pkt.msglen -
-                                                             (pkt.sidlen or 0) -  # noqa: E501
-                                                             38))]
-    # 40)) ]
+                                    length_from=lambda pkt: (
+                                        pkt.msglen - (pkt.sidlen or 0) - 40
+                                    ))]
 
     @classmethod
     def dispatch_hook(cls, _pkt=None, *args, **kargs):
@@ -563,7 +574,7 @@ def tls_session_update(self, msg_str):
     FieldLenField("sidlen", None, length_of="sid", fmt="B"),
     _SessionIDField("sid", "",
                     length_from=lambda pkt: pkt.sidlen),
-    EnumField("cipher", None, _tls_cipher_suites),
+    ShortEnumField("cipher", None, _tls_cipher_suites),
     _CompressionMethodsField("comp", [0],
                              _tls_compression_algs,
                              itemfmt="B",
@@ -1020,7 +1031,7 @@ def build(self, *args, **kargs):
         fval = self.getfieldval("sig")
         if fval is None:
             s = self.tls_session
-            if s.pwcs:
+            if s.pwcs and s.client_random:
                 if not s.pwcs.key_exchange.anonymous:
                     p = self.params
                     if p is None:
@@ -1126,7 +1137,7 @@ class TLSCertificateRequest(_TLSHandshake):
                    SigAndHashAlgsLenField("sig_algs_len", None,
                                           length_of="sig_algs"),
                    SigAndHashAlgsField("sig_algs", [0x0403, 0x0401, 0x0201],
-                                       EnumField("hash_sig", None, _tls_hash_sig),  # noqa: E501
+                                       ShortEnumField("hash_sig", None, _tls_hash_sig),  # noqa: E501
                                        length_from=lambda pkt: pkt.sig_algs_len),  # noqa: E501
                    FieldLenField("certauthlen", None, fmt="!H",
                                  length_of="certauth"),
diff --git a/scapy/layers/tls/keyexchange.py b/scapy/layers/tls/keyexchange.py
@@ -272,7 +272,7 @@ def m2i(self, pkt, m):
         if s.prcs:
             cls = s.prcs.key_exchange.server_kx_msg_cls(m)
             if cls is None:
-                return None, Raw(m[:tmp_len]) / Padding(m[tmp_len:])
+                return Raw(m[:tmp_len]) / Padding(m[tmp_len:])
             return cls(m, tls_session=s)
         else:
             try:
@@ -284,7 +284,7 @@ def m2i(self, pkt, m):
                 cls = _tls_server_ecdh_cls_guess(m)
                 p = cls(m, tls_session=s)
                 if pkcs_os2ip(p.load[:2]) not in _tls_hash_sig:
-                    return None, Raw(m[:tmp_len]) / Padding(m[tmp_len:])
+                    return Raw(m[:tmp_len]) / Padding(m[tmp_len:])
                 return p
 
 
diff --git a/scapy/layers/tls/session.py b/scapy/layers/tls/session.py
@@ -1002,6 +1002,25 @@ def mysummary(self):
         return "TLS %s / %s" % (repr(self.tls_session),
                                 getattr(self, "_name", self.name))
 
+    @classmethod
+    def tcp_reassemble(cls, data, metadata):
+        # Used with TLSSession
+        from scapy.layers.tls.record import TLS
+        from scapy.layers.tls.record_tls13 import TLS13
+        if cls in (TLS, TLS13):
+            length = struct.unpack("!H", data[3:5])[0] + 5
+            if len(data) == length:
+                return cls(data)
+            elif len(data) > length:
+                pkt = cls(data)
+                if hasattr(pkt.payload, "tcp_reassemble"):
+                    if pkt.payload.tcp_reassemble(data[length:], metadata):
+                        return pkt
+                else:
+                    return pkt
+        else:
+            return cls(data)
+
 
 ###############################################################################
 #   Multiple TLS sessions                                                     #
diff --git a/scapy/sendrecv.py b/scapy/sendrecv.py
@@ -752,7 +752,8 @@ class AsyncSniffer(object):
              is displayed.
              --Ex: prn = lambda x: x.summary()
         session: a session = a flow decoder used to handle stream of packets.
-                 e.g: IPSession (to defragment on-the-flow) or NetflowSession
+                 --Ex: session=TCPSession
+                 See below for more details.
         filter: BPF filter to apply.
         lfilter: Python function applied to each packet to determine if
                  further action may be done.
@@ -778,6 +779,9 @@ class AsyncSniffer(object):
     element, a list of elements, or a dict object mapping an element to a
     label (see examples below).
 
+    For more information about the session argument, see
+    https://scapy.rtfd.io/en/latest/usage.html#advanced-sniffing-sniffing-sessions
+
     Examples: synchronous
       >>> sniff(filter="arp")
       >>> sniff(filter="tcp",
diff --git a/scapy/sessions.py b/scapy/sessions.py
@@ -194,10 +194,12 @@ def tcp_reassemble(cls, data, metadata):
             # as you need additional data.
             return None
 
-    A (hard to understand) example can be found in scapy/layers/http.py
+    For more details and a real example, see:
+    https://scapy.readthedocs.io/en/latest/usage.html#how-to-use-tcpsession-to-defragment-tcp-packets
 
     :param app: Whether the socket is on application layer = has no TCP
-                layer. Default to False
+                layer. This is used for instance if you are using a native
+                TCP socket. Default to False
     """
 
     fmt = ('TCP {IP:%IP.src%}{IPv6:%IPv6.src%}:%r,TCP.sport% > ' +
@@ -224,7 +226,8 @@ def _process_packet(self, pkt):
             # Special mode: Application layer. Use on top of TCP
             pay_class = pkt.__class__
             if not hasattr(pay_class, "tcp_reassemble"):
-                # Cannot tcp-reassemble
+                # Being on top of TCP, we have no way of knowing
+                # when a packet ends.
                 return pkt
             self.data += bytes(pkt)
             pkt = pay_class.tcp_reassemble(self.data, self.metadata)
@@ -248,12 +251,16 @@ def _process_packet(self, pkt):
         # Let's guess which class is going to be used
         if "pay_class" not in metadata:
             pay_class = pay.__class__
-            if not hasattr(pay_class, "tcp_reassemble"):
-                # Cannot tcp-reassemble
+            if hasattr(pay_class, "tcp_reassemble"):
+                tcp_reassemble = pay_class.tcp_reassemble
+            else:
+                # We can't know for sure when a packet ends.
+                # Ignore.
                 return pkt
             metadata["pay_class"] = pay_class
+            metadata["tcp_reassemble"] = tcp_reassemble
         else:
-            pay_class = metadata["pay_class"]
+            tcp_reassemble = metadata["tcp_reassemble"]
         # Get a relative sequence number for a storage purpose
         relative_seq = metadata.get("relative_seq", None)
         if relative_seq is None:
@@ -275,10 +282,11 @@ def _process_packet(self, pkt):
         packet = None
         if data.full():
             # Reassemble using all previous packets
-            packet = pay_class.tcp_reassemble(bytes(data), metadata)
+            packet = tcp_reassemble(bytes(data), metadata)
         # Stack the result on top of the previous frames
         if packet:
             data.clear()
+            metadata.clear()
             del self.tcp_frags[ident]
             pay.underlayer.remove_payload()
             if IP in pkt:
diff --git a/test/pcaps/tls_tcp_frag.pcap.gz b/test/pcaps/tls_tcp_frag.pcap.gz
diff --git a/test/tls.uts b/test/tls.uts
diff --git a/test/tls/tests_tls_netaccess.uts b/test/tls/tests_tls_netaccess.uts
