Merge #112: Segregated Witness support

5cd3467 fixed issue in is_witness_scriptpubkey where self[0] is interpretted as str instead of the necessary int (Surya Bakshi)
2b50596 Add segwit block (de)serialization. (#9) (the9ull)
d8a1a0f Closely match Core's (de-)serialization; remove old test case (Bob McElrath)
0c09892 Encode hashtype as signed (following bitcoind) (Bob McElrath)
eca69b7 CTxWitness class, new message types (Bob McElrath)
f4d1d2a BIP143 implementation and test vectors (Bob McElrath)
580f466 Initial segwit implementation, works with P2WPKH addresses (Bob McElrath)

Pull request description:

  This is my initial implementation.  It correctly serializes and deserializes P2WPKH segwit transactions.  I haven't tested the other transaction types.

  Please don't merge this yet, but I wanted to make it available for others to work on.

  TODO
  - ~~BIP 143 tx signing~~
  - ~~`CBlock` modifications to support witness data~~
  - ~~P2WSH and BIP16 embedded scripts~~
  - ~~tests!~~

Tree-SHA512: bc96c263467ae3f27aa86472afb4e29e6c1d81da527832ed153b05faa06ef8152ce518f9c9c1c1fdae45de7f93173c77fa237b160d5cfb7d1068532a8849c9f3

Author: petertodd

.gitignore

@@ -1,4 +1,4 @@
-
+*.sw?
 *.pyc
 
 local*.cfg
@@ -8,3 +8,4 @@ local*.cfg
 build/
 htmlcov/
 python_bitcoinlib.egg-info/
+dist/

bitcoin/core/__init__.py

@@ -24,12 +24,17 @@
     long = int
     _bchr = lambda x: bytes([x])
     _bord = lambda x: x
+    from io import BytesIO as _BytesIO
+else:
+    from cStringIO import StringIO as _BytesIO
 
 import struct
 
 import bitcoin.core
 import bitcoin.core._bignum
 
+from .serialize import *
+
 MAX_SCRIPT_SIZE = 10000
 MAX_SCRIPT_ELEMENT_SIZE = 520
 MAX_SCRIPT_OPCODES = 201
@@ -672,6 +677,31 @@ def is_p2sh(self):
                 _bord(self[1]) == 0x14 and
                 _bord(self[22]) == OP_EQUAL)
 
+    def is_witness_scriptpubkey(self):
+        """Returns true if this is a scriptpubkey signaling segregated witness
+        data. """
+        return 3 <= len(self) <= 42 and CScriptOp(struct.unpack('<b',self[0])[0]).is_small_int()
+
+    def witness_version(self):
+        """Returns the witness version on [0,16]. """
+        return next(iter(self))
+
+    def is_witness_v0_keyhash(self):
+        """Returns true if this is a scriptpubkey for V0 P2WPKH. """
+        return len(self) == 22 and self[0:2] == b'\x00\x14'
+
+    def is_witness_v0_nested_keyhash(self):
+        """Returns true if this is a scriptpubkey for V0 P2WPKH embedded in P2SH. """
+        return len(self) == 23 and self[0:3] == b'\x16\x00\x14'
+
+    def is_witness_v0_scripthash(self):
+        """Returns true if this is a scriptpubkey for V0 P2WSH. """
+        return len(self) == 34 and self[0:2] == b'\x00\x20'
+
+    def is_witness_v0_nested_scripthash(self):
+        """Returns true if this is a scriptpubkey for V0 P2WSH embedded in P2SH. """
+        return len(self) == 23 and self[0:2] == b'\xa9\x14' and self[-1] == b'\x87'
+
     def is_push_only(self):
         """Test if the script only contains pushdata ops
 
@@ -773,6 +803,38 @@ def GetSigOpCount(self, fAccurate):
             lastOpcode = opcode
         return n
 
+class CScriptWitness(ImmutableSerializable):
+    """An encoding of the data elements on the initial stack for (segregated
+        witness)
+    """
+    __slots__ = ['stack']
+
+    def __init__(self, stack=()):
+        object.__setattr__(self, 'stack', stack)
+
+    def __len__(self):
+        return len(self.stack)
+
+    def __iter__(self):
+        return iter(self.stack)
+
+    def __repr__(self):
+        return 'CScriptWitness(' + ','.join("x('%s')" % bitcoin.core.b2x(s) for s in self.stack) + ')'
+
+    def is_null(self):
+        return len(self.stack) == 0
+
+    @classmethod
+    def stream_deserialize(cls, f):
+        n = VarIntSerializer.stream_deserialize(f)
+        stack = tuple(BytesSerializer.stream_deserialize(f) for i in range(n))
+        return cls(stack)
+
+    def stream_serialize(self, f):
+        VarIntSerializer.stream_serialize(len(self.stack), f)
+        for s in self.stack:
+            BytesSerializer.stream_serialize(s, f)
+
 
 SIGHASH_ALL = 1
 SIGHASH_NONE = 2
@@ -894,21 +956,71 @@ def RawSignatureHash(script, txTo, inIdx, hashtype):
         txtmp.vin = []
         txtmp.vin.append(tmp)
 
+    txtmp.wit = bitcoin.core.CTxWitness()
     s = txtmp.serialize()
-    s += struct.pack(b"<I", hashtype)
+    s += struct.pack(b"<i", hashtype)
 
     hash = bitcoin.core.Hash(s)
 
     return (hash, None)
 
+SIGVERSION_BASE = 0
+SIGVERSION_WITNESS_V0 = 1
 
-def SignatureHash(script, txTo, inIdx, hashtype):
+def SignatureHash(script, txTo, inIdx, hashtype, amount=None, sigversion=SIGVERSION_BASE):
     """Calculate a signature hash
 
     'Cooked' version that checks if inIdx is out of bounds - this is *not*
     consensus-correct behavior, but is what you probably want for general
     wallet use.
     """
+
+    if sigversion == SIGVERSION_WITNESS_V0:
+        hashPrevouts = b'\x00'*32
+        hashSequence = b'\x00'*32
+        hashOutputs  = b'\x00'*32
+
+        if not (hashtype & SIGHASH_ANYONECANPAY):
+            serialize_prevouts = bytes()
+            for i in txTo.vin:
+                serialize_prevouts += i.prevout.serialize()
+            hashPrevouts = bitcoin.core.Hash(serialize_prevouts)
+
+        if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
+            serialize_sequence = bytes()
+            for i in txTo.vin:
+                serialize_sequence += struct.pack("<I", i.nSequence)
+            hashSequence = bitcoin.core.Hash(serialize_sequence)
+
+        if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
+            serialize_outputs = bytes()
+            for o in txTo.vout:
+                serialize_outputs += o.serialize()
+            hashOutputs = bitcoin.core.Hash(serialize_outputs)
+        elif ((hashtype & 0x1f) == SIGHASH_SINGLE and inIdx < len(txTo.vout)):
+            serialize_outputs = txTo.vout[inIdx].serialize()
+            hashOutputs = bitcoin.core.Hash(serialize_outputs)
+
+        f = _BytesIO()
+        f.write(struct.pack("<i", txTo.nVersion))
+        f.write(hashPrevouts)
+        f.write(hashSequence)
+        txTo.vin[inIdx].prevout.stream_serialize(f)
+        BytesSerializer.stream_serialize(script, f)
+        f.write(struct.pack("<q", amount))
+        f.write(struct.pack("<I", txTo.vin[inIdx].nSequence))
+        f.write(hashOutputs)
+        f.write(struct.pack("<i", txTo.nLockTime))
+        f.write(struct.pack("<i", hashtype))
+
+        return bitcoin.core.Hash(f.getvalue())
+
+    if script.is_witness_scriptpubkey():
+        print("WARNING: You seem to be attempting to sign a scriptPubKey from an")
+        print("WARNING: output with segregated witness.  This is NOT the correct")
+        print("WARNING: thing to sign. You should pass SignatureHash the corresponding")
+        print("WARNING: P2WPKH or P2WSH script instead.")
+
     (h, err) = RawSignatureHash(script, txTo, inIdx, hashtype)
     if err is not None:
         raise ValueError(err)
@@ -1048,6 +1160,7 @@ def SignatureHash(script, txTo, inIdx, hashtype):
         'CScriptInvalidError',
         'CScriptTruncatedPushDataError',
         'CScript',
+        'CScriptWitness',
         'SIGHASH_ALL',
         'SIGHASH_NONE',
         'SIGHASH_SINGLE',
@@ -1056,4 +1169,7 @@ def SignatureHash(script, txTo, inIdx, hashtype):
         'RawSignatureHash',
         'SignatureHash',
         'IsLowDERSignature',
+
+        'SIGVERSION_BASE',
+        'SIGVERSION_WITNESS_V0',
 )

bitcoin/core/script.py

@@ -87,23 +87,23 @@ class Serializable(object):
 
     __slots__ = []
 
-    def stream_serialize(self, f):
+    def stream_serialize(self, f, **kwargs):
         """Serialize to a stream"""
         raise NotImplementedError
 
     @classmethod
-    def stream_deserialize(cls, f):
+    def stream_deserialize(cls, f, **kwargs):
         """Deserialize from a stream"""
         raise NotImplementedError
 
-    def serialize(self):
+    def serialize(self, params={}):
         """Serialize, returning bytes"""
         f = _BytesIO()
-        self.stream_serialize(f)
+        self.stream_serialize(f, **params)
         return f.getvalue()
 
     @classmethod
-    def deserialize(cls, buf, allow_padding=False):
+    def deserialize(cls, buf, allow_padding=False, params={}):
         """Deserialize bytes, returning an instance
 
         allow_padding - Allow buf to include extra padding. (default False)
@@ -112,7 +112,7 @@ def deserialize(cls, buf, allow_padding=False):
         deserialization DeserializationExtraDataError will be raised.
         """
         fd = _BytesIO(buf)
-        r = cls.stream_deserialize(fd)
+        r = cls.stream_deserialize(fd, **params)
         if not allow_padding:
             padding = fd.read()
             if len(padding) != 0:
@@ -242,17 +242,17 @@ class VectorSerializer(Serializer):
     # and should be rethought at some point.
 
     @classmethod
-    def stream_serialize(cls, inner_cls, objs, f):
+    def stream_serialize(cls, inner_cls, objs, f, inner_params={}):
         VarIntSerializer.stream_serialize(len(objs), f)
         for obj in objs:
-            inner_cls.stream_serialize(obj, f)
+            inner_cls.stream_serialize(obj, f, **inner_params)
 
     @classmethod
-    def stream_deserialize(cls, inner_cls, f):
+    def stream_deserialize(cls, inner_cls, f, inner_params={}):
         n = VarIntSerializer.stream_deserialize(f)
         r = []
         for i in range(n):
-            r.append(inner_cls.stream_deserialize(f))
+            r.append(inner_cls.stream_deserialize(f, **inner_params))
         return r
 
 

bitcoin/core/serialize.py

@@ -35,9 +35,16 @@
 from bitcoin.net import *
 import bitcoin
 
+MSG_WITNESS_FLAG = 1 << 30
+MSG_TYPE_MASK = 0xffffffff >> 2
+
 MSG_TX = 1
 MSG_BLOCK = 2
 MSG_FILTERED_BLOCK = 3
+MSG_CMPCT_BLOCK = 4
+MSG_WITNESS_BLOCK = MSG_BLOCK | MSG_WITNESS_FLAG,
+MSG_WITNESS_TX = MSG_TX | MSG_WITNESS_FLAG,
+MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG,
 
 
 class MsgSerializable(Serializable):
@@ -499,6 +506,12 @@ def __repr__(self):
         'MSG_TX',
         'MSG_BLOCK',
         'MSG_FILTERED_BLOCK',
+        'MSG_CMPCT_BLOCK',
+        'MSG_TYPE_MASK',
+        'MSG_WITNESS_TX',
+        'MSG_WITNESS_BLOCK',
+        'MSG_WITNESS_FLAG',
+        'MSG_FILTERED_WITNESS_BLOCK',
         'MsgSerializable',
         'msg_version',
         'msg_verack',

bitcoin/messages.py

@@ -76,7 +76,8 @@ class CInv(Serializable):
         0: "Error",
         1: "TX",
         2: "Block",
-        3: "FilteredBlock"}
+        3: "FilteredBlock",
+        4: "CompactBlock"}
 
     def __init__(self):
         self.type = 0

bitcoin/net.py

@@ -23,11 +23,6 @@
 [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7a052c840ba73af26755de42cf01cc9e0a49fef0 EQUAL"]],
 "010000000100010000000000000000000000000000000000000000000000000000000000000000000009085768617420697320ffffffff010000000000000000015100000000", true],
 
-["Tests for CheckTransaction()"],
-["No inputs"],
-[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7a052c840ba73af26755de42cf01cc9e0a49fef0 EQUAL"]],
-"0100000000010000000000000000015100000000", true],
-
 ["No outputs"],
 [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x05ab9e14d983742513f0f451e105ffb4198d1dd4 EQUAL"]],
 "01000000010001000000000000000000000000000000000000000000000000000000000000000000006d483045022100f16703104aab4e4088317c862daec83440242411b039d14280e03dd33b487ab802201318a7be236672c5c56083eb7a5a195bc57a40af7923ff8545016cd3b571e2a601232103c40e5d339df3f30bf753e7e04450ae4ef76c9e45587d1d993bdc4cd06f0651c7acffffffff0000000000", true],