fuzzing of DER signatures with hypothesis

use hypothesis to generate malformed signatures by introducing
different changes for different curves made with different hashes

Author: tomato42

.gitignore

@@ -33,6 +33,7 @@ coverage-html
 .tox
 nosetests.xml
 t/
+.hypothesis/
 
 # Translations
 *.mo

build-requirements-2.6.txt

@@ -2,3 +2,4 @@ tox
 coveralls<1.3.0
 idna<2.8
 unittest2
+hypothesis<3

build-requirements-3.3.txt

@@ -3,3 +3,5 @@ pluggy<0.6
 tox<3
 wheel<0.30
 virtualenv==15.2.0
+enum34
+hypothesis<3.44

build-requirements.txt

@@ -1,2 +1,4 @@
 tox
 python-coveralls
+hypothesis
+pytest>=4.6.0

src/ecdsa/test_malformed_sigs.py

@@ -1,51 +1,124 @@
 from __future__ import with_statement, division
 
-import pytest
 import hashlib
+try:
+    from hashlib import algorithms_available
+except ImportError:
+    algorithms_available = [
+        "md5", "sha1", "sha224", "sha256", "sha384", "sha512"]
+from functools import partial
+import pytest
+import sys
+from six import binary_type
+import hypothesis.strategies as st
+from hypothesis import note, assume, given, settings
 
-from six import b, binary_type
-from .keys import SigningKey, VerifyingKey
+from .keys import SigningKey
 from .keys import BadSignatureError
 from .util import sigencode_der, sigencode_string
 from .util import sigdecode_der, sigdecode_string
-from .curves import curves, NIST256p, NIST521p
+from .curves import curves, NIST256p
 
-der_sigs = []
-example_data = b("some data to sign")
 
-# Just NIST256p with SHA256 is 560 test cases, all curves with all hashes is
-# few thousand slow test cases; execute the most interesting only
+example_data = b"some data to sign"
+"""Since the data is hashed for processing, really any string will do."""
 
-#for curve in curves:
-for curve in [NIST521p]:
-    #for hash_alg in ["md5", "sha1", "sha224", "sha256", "sha384", "sha512"]:
-    for hash_alg in ["sha256"]:
-        key = SigningKey.generate(curve)
-        signature = key.sign(example_data, hashfunc=getattr(hashlib, hash_alg),
-                             sigencode=sigencode_der)
-        for pos in range(len(signature)):
-            for xor in (1<<i for i in range(8)):
-                der_sigs.append(pytest.param(
-                    key.verifying_key, hash_alg,
-                    signature, pos, xor,
-                    id="{0}-{1}-pos-{2}-xor-{3}".format(
-                        curve.name, hash_alg, pos, xor)))
-
-
-@pytest.mark.parametrize("verifying_key,hash_alg,signature,pos,xor", der_sigs)
-def test_fuzzed_der_signatures(verifying_key, hash_alg, signature, pos, xor):
-    # check if a malformed DER encoded signature causes the same exception
-    # to be raised irrespective of the type of error
-    sig = bytearray(signature)
-    sig[pos] ^= xor
-    sig = binary_type(sig)
 
-    try:
-        verifying_key.verify(sig, example_data, getattr(hashlib, hash_alg),
-                             sigdecode_der)
-        assert False
-    except BadSignatureError:
-        assert True
+hash_and_size = [(name, hashlib.new(name).digest_size)
+                 for name in algorithms_available]
+"""Pairs of hash names and their output sizes.
+Needed for pairing with curves as we don't support hashes
+bigger than order sizes of curves."""
+
+
+keys_and_sigs = []
+"""Name of the curve+hash combination, VerifyingKey and DER signature."""
+
+
+# for hypothesis strategy shrinking we want smallest curves and hashes first
+for curve in sorted(curves, key=lambda x: x.baselen):
+    for hash_alg in [name for name, size in
+                     sorted(hash_and_size, key=lambda x: x[1])
+                     if 0 < size <= curve.baselen]:
+        sk = SigningKey.generate(
+            curve,
+            hashfunc=partial(hashlib.new, hash_alg))
+
+        keys_and_sigs.append(
+            ("{0} {1}".format(curve, hash_alg),
+             sk.verifying_key,
+             sk.sign(example_data, sigencode=sigencode_der)))
+
+
+# first make sure that the signatures can be verified
+@pytest.mark.parametrize(
+    "verifying_key,signature",
+    [pytest.param(vk, sig, id=name) for name, vk, sig in keys_and_sigs])
+def test_signatures(verifying_key, signature):
+    assert verifying_key.verify(signature, example_data,
+                                sigdecode=sigdecode_der)
+
+
+@st.composite
+def st_fuzzed_sig(draw):
+    """
+    Hypothesis strategy that generates pairs of VerifyingKey and malformed
+    signatures created by fuzzing of a valid signature.
+    """
+    name, verifying_key, old_sig = draw(st.sampled_from(keys_and_sigs))
+    note("Configuration: {0}".format(name))
+
+    sig = bytearray(old_sig)
+
+    # decide which bytes should be removed
+    to_remove = draw(st.lists(
+        st.integers(min_value=0, max_value=len(sig)-1),
+        unique=True))
+    to_remove.sort()
+    for i in reversed(to_remove):
+        del sig[i]
+    note("Remove bytes: {0}".format(to_remove))
+
+    # decide which bytes of the original signature should be changed
+    xors = draw(st.dictionaries(
+        st.integers(min_value=0, max_value=len(sig)-1),
+        st.integers(min_value=1, max_value=255)))
+    for i, val in xors.items():
+        sig[i] ^= val
+    note("xors: {0}".format(xors))
+
+    # decide where new data should be inserted
+    insert_pos = draw(st.integers(min_value=0, max_value=len(sig)))
+    # NIST521p signature is about 140 bytes long, test slightly longer
+    insert_data = draw(st.binary(max_size=256))
+
+    sig = sig[:insert_pos] + insert_data + sig[insert_pos:]
+    note("Inserted at position {0} bytes: {1!r}"
+         .format(insert_pos, insert_data))
+
+    sig = bytes(sig)
+    # make sure that there was performed at least one mutation on the data
+    assume(to_remove or xors or insert_data)
+    # and that the mutations didn't cancel each-other out
+    assume(sig != old_sig)
+
+    return verifying_key, sig
+
+
+params = {}
+# not supported in hypothesis 2.0.0
+if sys.version_info >= (2, 7):
+    # deadline=5s because NIST521p are slow to verify
+    params["deadline"] = 5000
+
+
+@settings(**params)
+@given(st_fuzzed_sig())
+def test_fuzzed_der_signatures(args):
+    verifying_key, sig = args
+
+    with pytest.raises(BadSignatureError):
+        verifying_key.verify(sig, example_data, sigdecode=sigdecode_der)
 
 
 ####

tox.ini

@@ -6,9 +6,14 @@ envlist = py26, py27, py33, py34, py35, py36, py37, py38, py, pypy, pypy3
 deps =
      py{33}: py<1.5
      py{33}: pytest<3.3
+     py{33}: enum34
+     py{33}: hypothesis<3.44
      py{26}: unittest2
+     py{26}: hypothesis<3
      py{26,27,34,35,36,37,38,py,py3}: pytest
+     py{27,34,35,36,37,38,py,py3}: hypothesis
      py: pytest
+     py: hypothesis
      py{33}: wheel<0.30
      coverage
 commands = coverage run --branch -m pytest {posargs:src/ecdsa}