Merge pull request #344 from tlsfuzzer/non-prime-order-curve

handle non-prime order curves more gracefully

src/ecdsa/ellipticcurve.py

@@ -633,7 +633,7 @@ def __eq__(self, other):
         """
         x1, y1, z1 = self.__coords
         if other is INFINITY:
-            return not y1 or not z1
+            return not z1
         if isinstance(other, Point):
             x2, y2, z2 = other.x(), other.y(), 1
         elif isinstance(other, PointJacobi):
@@ -723,11 +723,13 @@ def scale(self):
 
     def to_affine(self):
         """Return point in affine form."""
-        _, y, z = self.__coords
-        if not y or not z:
+        _, _, z = self.__coords
+        p = self.__curve.p()
+        if not (z % p):
             return INFINITY
         self.scale()
         x, y, z = self.__coords
+        assert z == 1
         return Point(self.__curve, x, y, self.__order)
 
     @staticmethod
@@ -759,7 +761,7 @@ def _double_with_z_1(self, X1, Y1, p, a):
         # http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian.html#doubling-mdbl-2007-bl
         XX, YY = X1 * X1 % p, Y1 * Y1 % p
         if not YY:
-            return 0, 0, 1
+            return 0, 0, 0
         YYYY = YY * YY % p
         S = 2 * ((X1 + YY) ** 2 - XX - YYYY) % p
         M = 3 * XX + a
@@ -773,13 +775,13 @@ def _double(self, X1, Y1, Z1, p, a):
         """Add a point to itself, arbitrary z."""
         if Z1 == 1:
             return self._double_with_z_1(X1, Y1, p, a)
-        if not Y1 or not Z1:
-            return 0, 0, 1
+        if not Z1:
+            return 0, 0, 0
         # after:
         # http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian.html#doubling-dbl-2007-bl
         XX, YY = X1 * X1 % p, Y1 * Y1 % p
         if not YY:
-            return 0, 0, 1
+            return 0, 0, 0
         YYYY = YY * YY % p
         ZZ = Z1 * Z1 % p
         S = 2 * ((X1 + YY) ** 2 - XX - YYYY) % p
@@ -795,14 +797,14 @@ def double(self):
         """Add a point to itself."""
         X1, Y1, Z1 = self.__coords
 
-        if not Y1:
+        if not Z1:
             return INFINITY
 
         p, a = self.__curve.p(), self.__curve.a()
 
         X3, Y3, Z3 = self._double(X1, Y1, Z1, p, a)
 
-        if not Y3 or not Z3:
+        if not Z3:
             return INFINITY
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
 
@@ -886,10 +888,10 @@ def __radd__(self, other):
 
     def _add(self, X1, Y1, Z1, X2, Y2, Z2, p):
         """add two points, select fastest method."""
-        if not Y1 or not Z1:
-            return X2, Y2, Z2
-        if not Y2 or not Z2:
-            return X1, Y1, Z1
+        if not Z1:
+            return X2 % p, Y2 % p, Z2 % p
+        if not Z2:
+            return X1 % p, Y1 % p, Z1 % p
         if Z1 == Z2:
             if Z1 == 1:
                 return self._add_with_z_1(X1, Y1, X2, Y2, p)
@@ -917,7 +919,7 @@ def __add__(self, other):
 
         X3, Y3, Z3 = self._add(X1, Y1, Z1, X2, Y2, Z2, p)
 
-        if not Y3 or not Z3:
+        if not Z3:
             return INFINITY
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
 
@@ -927,7 +929,7 @@ def __rmul__(self, other):
 
     def _mul_precompute(self, other):
         """Multiply point by integer with precomputation table."""
-        X3, Y3, Z3, p = 0, 0, 1, self.__curve.p()
+        X3, Y3, Z3, p = 0, 0, 0, self.__curve.p()
         _add = self._add
         for X2, Y2 in self.__precompute:
             if other % 2:
@@ -940,7 +942,7 @@ def _mul_precompute(self, other):
             else:
                 other //= 2
 
-        if not Y3 or not Z3:
+        if not Z3:
             return INFINITY
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
 
@@ -959,7 +961,7 @@ def __mul__(self, other):
 
         self = self.scale()
         X2, Y2, _ = self.__coords
-        X3, Y3, Z3 = 0, 0, 1
+        X3, Y3, Z3 = 0, 0, 0
         p, a = self.__curve.p(), self.__curve.a()
         _double = self._double
         _add = self._add
@@ -972,7 +974,7 @@ def __mul__(self, other):
             elif i > 0:
                 X3, Y3, Z3 = _add(X3, Y3, Z3, X2, Y2, 1, p)
 
-        if not Y3 or not Z3:
+        if not Z3:
             return INFINITY
 
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
@@ -1001,7 +1003,7 @@ def mul_add(self, self_mul, other, other_mul):
             other_mul = other_mul % self.__order
 
         # (X3, Y3, Z3) is the accumulator
-        X3, Y3, Z3 = 0, 0, 1
+        X3, Y3, Z3 = 0, 0, 0
         p, a = self.__curve.p(), self.__curve.a()
 
         # as we have 6 unique points to work with, we can't scale all of them,
@@ -1025,7 +1027,7 @@ def mul_add(self, self_mul, other, other_mul):
         # when the self and other sum to infinity, we need to add them
         # one by one to get correct result but as that's very unlikely to
         # happen in regular operation, we don't need to optimise this case
-        if not pApB_Y or not pApB_Z:
+        if not pApB_Z:
             return self * self_mul + other * other_mul
 
         # gmp object creation has cumulatively higher overhead than the
@@ -1070,7 +1072,7 @@ def mul_add(self, self_mul, other, other_mul):
                     assert B > 0
                     X3, Y3, Z3 = _add(X3, Y3, Z3, pApB_X, pApB_Y, pApB_Z, p)
 
-        if not Y3 or not Z3:
+        if not Z3:
             return INFINITY
 
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
@@ -1154,6 +1156,8 @@ def __eq__(self, other):
 
         Note: only points that lay on the same curve can be equal.
         """
+        if other is INFINITY:
+            return self.__x is None or self.__y is None
         if isinstance(other, Point):
             return (
                 self.__curve == other.__curve
@@ -1220,7 +1224,12 @@ def leftmost_bit(x):
         # From X9.62 D.3.2:
 
         e3 = 3 * e
-        negative_self = Point(self.__curve, self.__x, -self.__y, self.__order)
+        negative_self = Point(
+            self.__curve,
+            self.__x,
+            (-self.__y) % self.__curve.p(),
+            self.__order,
+        )
         i = leftmost_bit(e3) // 2
         result = self
         # print("Multiplying %s by %d (e3 = %d):" % (self, other, e3))
@@ -1247,7 +1256,6 @@ def __str__(self):
 
     def double(self):
         """Return a new point that is twice the old."""
-
         if self == INFINITY:
             return INFINITY
 
@@ -1261,6 +1269,9 @@ def double(self):
             * numbertheory.inverse_mod(2 * self.__y, p)
         ) % p
 
+        if not l:
+            return INFINITY
+
         x3 = (l * l - 2 * self.__x) % p
         y3 = (l * (self.__x - x3) - self.__y) % p
 

src/ecdsa/test_ellipticcurve.py

@@ -83,6 +83,11 @@ def test_inequality_curves(self):
         c192 = CurveFp(p, -3, b)
         self.assertNotEqual(self.c_23, c192)
 
+    def test_inequality_curves_by_b_only(self):
+        a = CurveFp(23, 1, 0)
+        b = CurveFp(23, 1, 1)
+        self.assertNotEqual(a, b)
+
     def test_usability_in_a_hashed_collection_curves(self):
         {self.c_23: None}
 
@@ -184,6 +189,33 @@ def test_double(self):
         self.assertEqual(p3.x(), x3)
         self.assertEqual(p3.y(), y3)
 
+    def test_double_to_infinity(self):
+        p1 = Point(self.c_23, 11, 20)
+        p2 = p1.double()
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2.double()
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
+    def test_add_self_to_infinity(self):
+        p1 = Point(self.c_23, 11, 20)
+        p2 = p1 + p1
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2 + p2
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
+    def test_mul_to_infinity(self):
+        p1 = Point(self.c_23, 11, 20)
+        p2 = p1 * 2
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2 * 2
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
     def test_multiply(self):
         x1, y1, m, x3, y3 = (3, 10, 2, 7, 12)
         p1 = Point(self.c_23, x1, y1)
@@ -224,6 +256,12 @@ def test_inequality_points_diff_types(self):
         c = CurveFp(100, -3, 100)
         self.assertNotEqual(self.g_23, c)
 
+    def test_inequality_diff_y(self):
+        p1 = Point(self.c_23, 6, 4)
+        p2 = Point(self.c_23, 6, 19)
+
+        self.assertNotEqual(p1, p2)
+
     def test_to_bytes_from_bytes(self):
         p = Point(self.c_23, 3, 10)