Ed448: Reject small order public keys

2 files changed, 74 insertions, 33 deletions

diff --git a/crypto/src/math/ec/rfc8032/Ed448.cs b/crypto/src/math/ec/rfc8032/Ed448.cs
index 75629c293..cc189615b 100644
--- a/crypto/src/math/ec/rfc8032/Ed448.cs
+++ b/crypto/src/math/ec/rfc8032/Ed448.cs
@@ -190,15 +190,6 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032
             }
             return false;
         }
-
-        private static bool CheckScalarVar(ReadOnlySpan<byte> s, Span<uint> n)
-        {
-            if (s[ScalarBytes - 1] != 0x00)
-                return false;
-
-            DecodeScalar(s, n);
-            return !Nat.Gte(ScalarUints, n, L);
-        }
 #else
         private static bool CheckPointVar(byte[] p)
         {
@@ -215,7 +206,54 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032
             }
             return false;
         }
+#endif
+
+
+        private static bool CheckPointFullVar(byte[] p)
+        {
+            if ((p[PointBytes - 1] & 0x7F) != 0x00)
+                return false;
+
+            uint y13 = Codec.Decode32(p, 52);
+
+            uint t0 = y13;
+            uint t1 = y13 ^ P[13];
+
+            for (int i = CoordUints - 2; i > 0; --i)
+            {
+                uint yi = Codec.Decode32(p, i * 4);
+
+                // Reject non-canonical encodings (i.e. >= P)
+                if (t1 == 0 && yi > P[i])
+                    return false;
+
+                t0 |= yi;
+                t1 |= yi ^ P[i];
+            }
 
+            uint y0 = Codec.Decode32(p, 0);
+
+            // Reject 0 and 1
+            if (t0 == 0 && y0 <= 1U)
+                return false;
+
+            // Reject P - 1 and non-canonical encodings (i.e. >= P)
+            if (t1 == 0 && y0 >= (P[0] - 1U))
+                return false;
+
+            return true;
+        }
+
+#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER
+        private static bool CheckScalarVar(ReadOnlySpan<byte> s, Span<uint> n)
+        {
+            if (s[ScalarBytes - 1] != 0x00)
+                return false;
+
+            DecodeScalar(s, n);
+            return !Nat.Gte(ScalarUints, n, L);
+        }
+#else
         private static bool CheckScalarVar(byte[] s, uint[] n)
         {
             if (s[ScalarBytes - 1] != 0x00)
@@ -246,7 +284,7 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032
         private static bool DecodePointVar(byte[] p, int pOff, bool negate, ref PointProjective r)
         {
             byte[] py = Copy(p, pOff, PointBytes);
-            if (!CheckPointVar(py))
+            if (!CheckPointFullVar(py))
                 return false;
 
             int x_0 = (py[PointBytes - 1] & 0x80) >> 7;
@@ -2042,13 +2080,6 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032
             if (!DecodePointVar(pk, pkOff, false, ref p))
                 return false;
 
-            F.Normalize(p.x);
-            F.Normalize(p.y);
-            F.Normalize(p.z);
-
-            if (IsNeutralElementVar(p.x, p.y, p.z))
-                return false;
-
             Init(out PointProjective r);
             ScalarMultOrderVar(ref p, ref r);
 
diff --git a/crypto/test/src/math/ec/rfc8032/test/Ed448Test.cs b/crypto/test/src/math/ec/rfc8032/test/Ed448Test.cs
index b4797af26..a26ef5579 100644
--- a/crypto/test/src/math/ec/rfc8032/test/Ed448Test.cs
+++ b/crypto/test/src/math/ec/rfc8032/test/Ed448Test.cs
@@ -14,8 +14,6 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032.Tests
     {
         private static readonly SecureRandom Random = new SecureRandom();
 
-        private static readonly byte[] Neutral = Hex.DecodeStrict("010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000");
-
         [SetUp]
         public void SetUp()
         {
@@ -467,8 +465,6 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032.Tests
         [Test]
         public void TestPublicKeyValidationFull()
         {
-            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Neutral, 0));
-
             byte[] sk = new byte[Ed448.SecretKeySize];
             byte[] pk = new byte[Ed448.PublicKeySize];
 
@@ -479,13 +475,21 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032.Tests
                 Assert.IsTrue(Ed448.ValidatePublicKeyFull(pk, 0));
             }
 
+            // Small order points (canonical encodings)
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
+
+            // Small order points (non-canonical encodings)
             Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
 
-            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
-            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
-            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
-            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
-            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
+            // Non-canonical encodings
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("00000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("00000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
             Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0));
             Assert.IsFalse(Ed448.ValidatePublicKeyFull(Hex.DecodeStrict("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000081"), 0));
 
@@ -515,8 +519,6 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032.Tests
         [Test]
         public void TestPublicKeyValidationPartial()
         {
-            Assert.IsTrue(Ed448.ValidatePublicKeyPartial(Neutral, 0));
-
             byte[] sk = new byte[Ed448.SecretKeySize];
             byte[] pk = new byte[Ed448.PublicKeySize];
 
@@ -527,13 +529,21 @@ namespace Org.BouncyCastle.Math.EC.Rfc8032.Tests
                 Assert.IsTrue(Ed448.ValidatePublicKeyPartial(pk, 0));
             }
 
+            // Small order points (canonical encodings)
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
+
+            // Small order points (non-canonical encodings)
             Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
 
-            Assert.IsTrue (Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
-            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
-            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
-            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
-            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
+            // Non-canonical encodings
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("00000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00"), 0));
+            Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("00000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF80"), 0));
             Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0));
             Assert.IsFalse(Ed448.ValidatePublicKeyPartial(Hex.DecodeStrict("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000081"), 0));