Implement Karatsuba multiply/square on 512-bit numbers and use as basis for P-521 multiply/square

1 files changed, 438 insertions, 18 deletions

diff --git a/crypto/src/math/ec/custom/sec/Nat256.cs b/crypto/src/math/ec/custom/sec/Nat256.cs
index 40e287b9b..3bd329251 100644
--- a/crypto/src/math/ec/custom/sec/Nat256.cs
+++ b/crypto/src/math/ec/custom/sec/Nat256.cs
@@ -39,6 +39,36 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             return (uint)c;
         }
 
+        public static uint Add(uint[] x, int xOff, uint[] y, int yOff, uint[] z, int zOff)
+        {
+            ulong c = 0;
+            c += (ulong)x[xOff + 0] + y[yOff + 0];
+            z[zOff + 0] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 1] + y[yOff + 1];
+            z[zOff + 1] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 2] + y[yOff + 2];
+            z[zOff + 2] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 3] + y[yOff + 3];
+            z[zOff + 3] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 4] + y[yOff + 4];
+            z[zOff + 4] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 5] + y[yOff + 5];
+            z[zOff + 5] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 6] + y[yOff + 6];
+            z[zOff + 6] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 7] + y[yOff + 7];
+            z[zOff + 7] = (uint)c;
+            c >>= 32;
+            return (uint)c;
+        }
+
         public static uint AddBothTo(uint[] x, uint[] y, uint[] z)
         {
             ulong c = 0;
@@ -69,6 +99,36 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             return (uint)c;
         }
 
+        public static uint AddBothTo(uint[] x, int xOff, uint[] y, int yOff, uint[] z, int zOff)
+        {
+            ulong c = 0;
+            c += (ulong)x[xOff + 0] + y[yOff + 0] + z[zOff + 0];
+            z[zOff + 0] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 1] + y[yOff + 1] + z[zOff + 1];
+            z[zOff + 1] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 2] + y[yOff + 2] + z[zOff + 2];
+            z[zOff + 2] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 3] + y[yOff + 3] + z[zOff + 3];
+            z[zOff + 3] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 4] + y[yOff + 4] + z[zOff + 4];
+            z[zOff + 4] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 5] + y[yOff + 5] + z[zOff + 5];
+            z[zOff + 5] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 6] + y[yOff + 6] + z[zOff + 6];
+            z[zOff + 6] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 7] + y[yOff + 7] + z[zOff + 7];
+            z[zOff + 7] = (uint)c;
+            c >>= 32;
+            return (uint)c;
+        }
+
         // TODO Re-write to allow full range for x?
         public static uint AddDWord(ulong x, uint[] z, int zOff)
         {
@@ -95,33 +155,70 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             return (uint)c;
         }
 
-        public static uint AddToExt(uint[] x, int xOff, uint[] zz, int zzOff)
+        public static uint AddTo(uint[] x, int xOff, uint[] z, int zOff, uint cIn)
+        {
+            ulong c = cIn;
+            c += (ulong)x[xOff + 0] + z[zOff + 0];
+            z[zOff + 0] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 1] + z[zOff + 1];
+            z[zOff + 1] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 2] + z[zOff + 2];
+            z[zOff + 2] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 3] + z[zOff + 3];
+            z[zOff + 3] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 4] + z[zOff + 4];
+            z[zOff + 4] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 5] + z[zOff + 5];
+            z[zOff + 5] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 6] + z[zOff + 6];
+            z[zOff + 6] = (uint)c;
+            c >>= 32;
+            c += (ulong)x[xOff + 7] + z[zOff + 7];
+            z[zOff + 7] = (uint)c;
+            c >>= 32;
+            return (uint)c;
+        }
+
+        public static uint AddToEachOther(uint[] u, int uOff, uint[] v, int vOff)
         {
-            Debug.Assert(zzOff <= 8);
             ulong c = 0;
-            c += (ulong)x[xOff + 0] + zz[zzOff + 0];
-            zz[zzOff + 0] = (uint)c;
+            c += (ulong)u[uOff + 0] + v[vOff + 0];
+            u[uOff + 0] = (uint)c;
+            v[vOff + 0] = (uint)c;
             c >>= 32;
-            c += (ulong)x[xOff + 1] + zz[zzOff + 1];
-            zz[zzOff + 1] = (uint)c;
+            c += (ulong)u[uOff + 1] + v[vOff + 1];
+            u[uOff + 1] = (uint)c;
+            v[vOff + 1] = (uint)c;
             c >>= 32;
-            c += (ulong)x[xOff + 2] + zz[zzOff + 2];
-            zz[zzOff + 2] = (uint)c;
+            c += (ulong)u[uOff + 2] + v[vOff + 2];
+            u[uOff + 2] = (uint)c;
+            v[vOff + 2] = (uint)c;
             c >>= 32;
-            c += (ulong)x[xOff + 3] + zz[zzOff + 3];
-            zz[zzOff + 3] = (uint)c;
+            c += (ulong)u[uOff + 3] + v[vOff + 3];
+            u[uOff + 3] = (uint)c;
+            v[vOff + 3] = (uint)c;
             c >>= 32;
-            c += (ulong)x[xOff + 4] + zz[zzOff + 4];
-            zz[zzOff + 4] = (uint)c;
+            c += (ulong)u[uOff + 4] + v[vOff + 4];
+            u[uOff + 4] = (uint)c;
+            v[vOff + 4] = (uint)c;
             c >>= 32;
-            c += (ulong)x[xOff + 5] + zz[zzOff + 5];
-            zz[zzOff + 5] = (uint)c;
+            c += (ulong)u[uOff + 5] + v[vOff + 5];
+            u[uOff + 5] = (uint)c;
+            v[vOff + 5] = (uint)c;
             c >>= 32;
-            c += (ulong)x[xOff + 6] + zz[zzOff + 6];
-            zz[zzOff + 6] = (uint)c;
+            c += (ulong)u[uOff + 6] + v[vOff + 6];
+            u[uOff + 6] = (uint)c;
+            v[vOff + 6] = (uint)c;
             c >>= 32;
-            c += (ulong)x[xOff + 7] + zz[zzOff + 7];
-            zz[zzOff + 7] = (uint)c;
+            c += (ulong)u[uOff + 7] + v[vOff + 7];
+            u[uOff + 7] = (uint)c;
+            v[vOff + 7] = (uint)c;
             c >>= 32;
             return (uint)c;
         }
@@ -158,6 +255,20 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             return -1;
         }
 
+        public static bool Diff(uint[] x, int xOff, uint[] y, int yOff, uint[] z, int zOff)
+        {
+            bool pos = Gte(x, xOff, y, yOff);
+            if (pos)
+            {
+                Sub(x, xOff, y, yOff, z, zOff);
+            }
+            else
+            {
+                Sub(y, yOff, x, xOff, z, zOff);
+            }
+            return pos;
+        }
+
         public static uint[] FromBigInteger(BigInteger x)
         {
             if (x.SignValue < 0 || x.BitLength > 256)
@@ -201,6 +312,19 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             return true;
         }
 
+        public static bool Gte(uint[] x, int xOff, uint[] y, int yOff)
+        {
+            for (int i = 7; i >= 0; --i)
+            {
+                uint x_i = x[xOff + i], y_i = y[yOff + i];
+                if (x_i < y_i)
+                    return false;
+                if (x_i > y_i)
+                    return true;
+            }
+            return true;
+        }
+
         public static bool GteExt(uint[] xx, uint[] yy)
         {
             for (int i = 15; i >= 0; --i)
@@ -351,6 +475,125 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             }
         }
 
+        public static void Mul(uint[] x, int xOff, uint[] y, int yOff, uint[] zz, int zzOff)
+        {
+            ulong y_0 = y[yOff + 0];
+            ulong y_1 = y[yOff + 1];
+            ulong y_2 = y[yOff + 2];
+            ulong y_3 = y[yOff + 3];
+            ulong y_4 = y[yOff + 4];
+            ulong y_5 = y[yOff + 5];
+            ulong y_6 = y[yOff + 6];
+            ulong y_7 = y[yOff + 7];
+
+            {
+                ulong c = 0, x_0 = x[xOff + 0];
+                c += x_0 * y_0;
+                zz[zzOff + 0] = (uint)c;
+                c >>= 32;
+                c += x_0 * y_1;
+                zz[zzOff + 1] = (uint)c;
+                c >>= 32;
+                c += x_0 * y_2;
+                zz[zzOff + 2] = (uint)c;
+                c >>= 32;
+                c += x_0 * y_3;
+                zz[zzOff + 3] = (uint)c;
+                c >>= 32;
+                c += x_0 * y_4;
+                zz[zzOff + 4] = (uint)c;
+                c >>= 32;
+                c += x_0 * y_5;
+                zz[zzOff + 5] = (uint)c;
+                c >>= 32;
+                c += x_0 * y_6;
+                zz[zzOff + 6] = (uint)c;
+                c >>= 32;
+                c += x_0 * y_7;
+                zz[zzOff + 7] = (uint)c;
+                c >>= 32;
+                zz[zzOff + 8] = (uint)c;
+            }
+
+            for (int i = 1; i < 8; ++i)
+            {
+                ++zzOff;
+                ulong c = 0, x_i = x[xOff + i];
+                c += x_i * y_0 + zz[zzOff + 0];
+                zz[zzOff + 0] = (uint)c;
+                c >>= 32;
+                c += x_i * y_1 + zz[zzOff + 1];
+                zz[zzOff + 1] = (uint)c;
+                c >>= 32;
+                c += x_i * y_2 + zz[zzOff + 2];
+                zz[zzOff + 2] = (uint)c;
+                c >>= 32;
+                c += x_i * y_3 + zz[zzOff + 3];
+                zz[zzOff + 3] = (uint)c;
+                c >>= 32;
+                c += x_i * y_4 + zz[zzOff + 4];
+                zz[zzOff + 4] = (uint)c;
+                c >>= 32;
+                c += x_i * y_5 + zz[zzOff + 5];
+                zz[zzOff + 5] = (uint)c;
+                c >>= 32;
+                c += x_i * y_6 + zz[zzOff + 6];
+                zz[zzOff + 6] = (uint)c;
+                c >>= 32;
+                c += x_i * y_7 + zz[zzOff + 7];
+                zz[zzOff + 7] = (uint)c;
+                c >>= 32;
+                zz[zzOff + 8] = (uint)c;
+            }
+        }
+
+        public static uint MulAdd(uint[] x, int xOff, uint[] y, int yOff, uint[] zz, int zzOff)
+        {
+            ulong y_0 = y[yOff + 0];
+            ulong y_1 = y[yOff + 1];
+            ulong y_2 = y[yOff + 2];
+            ulong y_3 = y[yOff + 3];
+            ulong y_4 = y[yOff + 4];
+            ulong y_5 = y[yOff + 5];
+            ulong y_6 = y[yOff + 6];
+            ulong y_7 = y[yOff + 7];
+
+            ulong zc = 0;
+            for (int i = 0; i < 8; ++i)
+            {
+                ulong c = 0, x_i = x[xOff + i];
+                c += x_i * y_0 + zz[zzOff + 0];
+                zz[zzOff + 0] = (uint)c;
+                c >>= 32;
+                c += x_i * y_1 + zz[zzOff + 1];
+                zz[zzOff + 1] = (uint)c;
+                c >>= 32;
+                c += x_i * y_2 + zz[zzOff + 2];
+                zz[zzOff + 2] = (uint)c;
+                c >>= 32;
+                c += x_i * y_3 + zz[zzOff + 3];
+                zz[zzOff + 3] = (uint)c;
+                c >>= 32;
+                c += x_i * y_4 + zz[zzOff + 4];
+                zz[zzOff + 4] = (uint)c;
+                c >>= 32;
+                c += x_i * y_5 + zz[zzOff + 5];
+                zz[zzOff + 5] = (uint)c;
+                c >>= 32;
+                c += x_i * y_6 + zz[zzOff + 6];
+                zz[zzOff + 6] = (uint)c;
+                c >>= 32;
+                c += x_i * y_7 + zz[zzOff + 7];
+                zz[zzOff + 7] = (uint)c;
+                c >>= 32;
+                c += zc + zz[zzOff + 8];
+                zz[zzOff + 8] = (uint)c;
+                zc = c >> 32;
+                ++zzOff;
+            }
+            return (uint)zc;
+        }
+
         public static ulong Mul33AddExt(uint w, uint[] xx, int xxOff, uint[] yy, int yyOff, uint[] zz, int zzOff)
         {
             Debug.Assert(w >> 31 == 0);
@@ -539,6 +782,17 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             return c >> 31;
         }
 
+        public static uint ShiftUpBit(uint[] x, int xOff, int xLen, uint c)
+        {
+            for (int i = 0; i < xLen; ++i)
+            {
+                uint next = x[xOff + i];
+                x[xOff + i] = (next << 1) | (c >> 31);
+                c = next;
+            }
+            return c >> 31;
+        }
+
         public static uint ShiftUpBit(uint[] x, uint c, uint[] z)
         {
             for (int i = 0; i < 8; ++i)
@@ -686,6 +940,142 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             ShiftUpBit(zz, 16, (uint)x_0 << 31);
         }
 
+        public static void Square(uint[] x, int xOff, uint[] zz, int zzOff)
+        {
+            ulong x_0 = x[xOff + 0];
+            ulong zz_1;
+
+            {
+                uint c = 0;
+                int i = 7, j = 16;
+                do
+                {
+                    ulong xVal = x[xOff + i--];
+                    ulong p = xVal * xVal;
+                    zz[zzOff + --j] = (c << 31) | (uint)(p >> 33);
+                    zz[zzOff + --j] = (uint)(p >> 1);
+                    c = (uint)p;
+                }
+                while (i > 0);
+
+                {
+                    ulong p = x_0 * x_0;
+                    zz_1 = (ulong)(c << 31) | (p >> 33);
+                    zz[zzOff + 0] = (uint)(p >> 1);
+                }
+            }
+
+            ulong x_1 = x[xOff + 1];
+            ulong zz_2 = zz[zzOff + 2];
+
+            {
+                zz_1 += x_1 * x_0;
+                zz[zzOff + 1] = (uint)zz_1;
+                zz_2 += zz_1 >> 32;
+            }
+
+            ulong x_2 = x[xOff + 2];
+            ulong zz_3 = zz[zzOff + 3];
+            ulong zz_4 = zz[zzOff + 4];
+            {
+                zz_2 += x_2 * x_0;
+                zz[zzOff + 2] = (uint)zz_2;
+                zz_3 += (zz_2 >> 32) + x_2 * x_1;
+                zz_4 += zz_3 >> 32;
+                zz_3 &= M;
+            }
+
+            ulong x_3 = x[xOff + 3];
+            ulong zz_5 = zz[zzOff + 5];
+            ulong zz_6 = zz[zzOff + 6];
+            {
+                zz_3 += x_3 * x_0;
+                zz[zzOff + 3] = (uint)zz_3;
+                zz_4 += (zz_3 >> 32) + x_3 * x_1;
+                zz_5 += (zz_4 >> 32) + x_3 * x_2;
+                zz_4 &= M;
+                zz_6 += zz_5 >> 32;
+                zz_5 &= M;
+            }
+
+            ulong x_4 = x[xOff + 4];
+            ulong zz_7 = zz[zzOff + 7];
+            ulong zz_8 = zz[zzOff + 8];
+            {
+                zz_4 += x_4 * x_0;
+                zz[zzOff + 4] = (uint)zz_4;
+                zz_5 += (zz_4 >> 32) + x_4 * x_1;
+                zz_6 += (zz_5 >> 32) + x_4 * x_2;
+                zz_5 &= M;
+                zz_7 += (zz_6 >> 32) + x_4 * x_3;
+                zz_6 &= M;
+                zz_8 += zz_7 >> 32;
+                zz_7 &= M;
+            }
+
+            ulong x_5 = x[xOff + 5];
+            ulong zz_9 = zz[zzOff + 9];
+            ulong zz_10 = zz[zzOff + 10];
+            {
+                zz_5 += x_5 * x_0;
+                zz[zzOff + 5] = (uint)zz_5;
+                zz_6 += (zz_5 >> 32) + x_5 * x_1;
+                zz_7 += (zz_6 >> 32) + x_5 * x_2;
+                zz_6 &= M;
+                zz_8 += (zz_7 >> 32) + x_5 * x_3;
+                zz_7 &= M;
+                zz_9 += (zz_8 >> 32) + x_5 * x_4;
+                zz_8 &= M;
+                zz_10 += zz_9 >> 32;
+                zz_9 &= M;
+            }
+
+            ulong x_6 = x[xOff + 6];
+            ulong zz_11 = zz[zzOff + 11];
+            ulong zz_12 = zz[zzOff + 12];
+            {
+                zz_6 += x_6 * x_0;
+                zz[zzOff + 6] = (uint)zz_6;
+                zz_7 += (zz_6 >> 32) + x_6 * x_1;
+                zz_8 += (zz_7 >> 32) + x_6 * x_2;
+                zz_7 &= M;
+                zz_9 += (zz_8 >> 32) + x_6 * x_3;
+                zz_8 &= M;
+                zz_10 += (zz_9 >> 32) + x_6 * x_4;
+                zz_9 &= M;
+                zz_11 += (zz_10 >> 32) + x_6 * x_5;
+                zz_10 &= M;
+                zz_12 += zz_11 >> 32;
+                zz_11 &= M;
+            }
+
+            ulong x_7 = x[xOff + 7];
+            ulong zz_13 = zz[zzOff + 13];
+            ulong zz_14 = zz[zzOff + 14];
+            {
+                zz_7 += x_7 * x_0;
+                zz[zzOff + 7] = (uint)zz_7;
+                zz_8 += (zz_7 >> 32) + x_7 * x_1;
+                zz_9 += (zz_8 >> 32) + x_7 * x_2;
+                zz_10 += (zz_9 >> 32) + x_7 * x_3;
+                zz_11 += (zz_10 >> 32) + x_7 * x_4;
+                zz_12 += (zz_11 >> 32) + x_7 * x_5;
+                zz_13 += (zz_12 >> 32) + x_7 * x_6;
+                zz_14 += zz_13 >> 32;
+            }
+
+            zz[zzOff + 8] = (uint)zz_8;
+            zz[zzOff + 9] = (uint)zz_9;
+            zz[zzOff + 10] = (uint)zz_10;
+            zz[zzOff + 11] = (uint)zz_11;
+            zz[zzOff + 12] = (uint)zz_12;
+            zz[zzOff + 13] = (uint)zz_13;
+            zz[zzOff + 14] = (uint)zz_14;
+            zz[zzOff + 15] += (uint)(zz_14 >> 32);
+
+            ShiftUpBit(zz, zzOff, 16, (uint)x_0 << 31);
+        }
+
         public static uint SquareWordAddExt(uint[] x, int xPos, uint[] zz)
         {
             Debug.Assert(xPos > 0 && xPos < 8);
@@ -731,6 +1121,36 @@ namespace Org.BouncyCastle.Math.EC.Custom.Sec
             return (int)c;
         }
 
+        public static int Sub(uint[] x, int xOff, uint[] y, int yOff, uint[] z, int zOff)
+        {
+            long c = 0;
+            c += (long)x[xOff + 0] - y[yOff + 0];
+            z[zOff + 0] = (uint)c;
+            c >>= 32;
+            c += (long)x[xOff + 1] - y[yOff + 1];
+            z[zOff + 1] = (uint)c;
+            c >>= 32;
+            c += (long)x[xOff + 2] - y[yOff + 2];
+            z[zOff + 2] = (uint)c;
+            c >>= 32;
+            c += (long)x[xOff + 3] - y[yOff + 3];
+            z[zOff + 3] = (uint)c;
+            c >>= 32;
+            c += (long)x[xOff + 4] - y[yOff + 4];
+            z[zOff + 4] = (uint)c;
+            c >>= 32;
+            c += (long)x[xOff + 5] - y[yOff + 5];
+            z[zOff + 5] = (uint)c;
+            c >>= 32;
+            c += (long)x[xOff + 6] - y[yOff + 6];
+            z[zOff + 6] = (uint)c;
+            c >>= 32;
+            c += (long)x[xOff + 7] - y[yOff + 7];
+            z[zOff + 7] = (uint)c;
+            c >>= 32;
+            return (int)c;
+        }
+
         public static int SubBothFrom(uint[] x, uint[] y, uint[] z)
         {
             long c = 0;