GCM perf. opts.

5 files changed, 423 insertions, 376 deletions

diff --git a/crypto/src/crypto/modes/GCMBlockCipher.cs b/crypto/src/crypto/modes/GCMBlockCipher.cs
index 9250097cd..ac54e9762 100644
--- a/crypto/src/crypto/modes/GCMBlockCipher.cs
+++ b/crypto/src/crypto/modes/GCMBlockCipher.cs
@@ -1,4 +1,9 @@
 using System;
+#if NETCOREAPP3_0_OR_GREATER
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+#endif
 
 using Org.BouncyCastle.Crypto.Modes.Gcm;
 using Org.BouncyCastle.Crypto.Parameters;
@@ -48,6 +53,7 @@ namespace Org.BouncyCastle.Crypto.Modes
         private byte[]		macBlock;
         private byte[]      S, S_at, S_atPre;
         private byte[]      counter;
+        private uint        counter32;
         private uint        blocksRemaining;
         private int         bufOff;
         private ulong		totalLength;
@@ -210,6 +216,7 @@ namespace Org.BouncyCastle.Crypto.Modes
             this.atLength = 0;
             this.atLengthPre = 0;
             this.counter = Arrays.Clone(J0);
+            this.counter32 = Pack.BE_To_UInt32(counter, 12);
             this.blocksRemaining = uint.MaxValue - 1; // page 8, len(P) <= 2^39 - 256, 1 block used by tag
             this.bufOff = 0;
             this.totalLength = 0;
@@ -273,17 +280,35 @@ namespace Org.BouncyCastle.Crypto.Modes
         {
             CheckStatus();
 
-            for (int i = 0; i < len; ++i)
+            if (atBlockPos > 0)
             {
-                atBlock[atBlockPos] = inBytes[inOff + i];
-                if (++atBlockPos == BlockSize)
+                int available = BlockSize - atBlockPos;
+                if (len < available)
                 {
-                    // Hash each block as it fills
-                    gHASHBlock(S_at, atBlock);
-                    atBlockPos = 0;
-                    atLength += BlockSize;
+                    Array.Copy(inBytes, inOff, atBlock, atBlockPos, len);
+                    atBlockPos += len;
+                    return;
                 }
+
+                Array.Copy(inBytes, inOff, atBlock, atBlockPos, available);
+                gHASHBlock(S_at, atBlock);
+                atLength += BlockSize;
+                inOff += available;
+                len -= available;
+                //atBlockPos = 0;
+            }
+
+            int inLimit = inOff + len - BlockSize;
+
+            while (inOff <= inLimit)
+            {
+                gHASHBlock(S_at, inBytes, inOff);
+                atLength += BlockSize;
+                inOff += BlockSize;
             }
+
+            atBlockPos = BlockSize + inLimit - inOff;
+            Array.Copy(inBytes, inOff, atBlock, 0, atBlockPos);
         }
 
         private void InitCipher()
@@ -307,23 +332,21 @@ namespace Org.BouncyCastle.Crypto.Modes
             }
         }
 
-        public virtual int ProcessByte(
-            byte	input,
-            byte[]	output,
-            int		outOff)
+        public virtual int ProcessByte(byte	input, byte[] output, int outOff)
         {
             CheckStatus();
 
             bufBlock[bufOff] = input;
             if (++bufOff == bufBlock.Length)
             {
-                ProcessBlock(bufBlock, 0, output, outOff);
                 if (forEncryption)
                 {
+                    EncryptBlock(bufBlock, 0, output, outOff);
                     bufOff = 0;
                 }
                 else
                 {
+                    DecryptBlock(bufBlock, 0, output, outOff);
                     Array.Copy(bufBlock, BlockSize, bufBlock, 0, macSize);
                     bufOff = macSize;
                 }
@@ -332,12 +355,7 @@ namespace Org.BouncyCastle.Crypto.Modes
             return 0;
         }
 
-        public virtual int ProcessBytes(
-            byte[]	input,
-            int		inOff,
-            int		len,
-            byte[]	output,
-            int		outOff)
+        public virtual int ProcessBytes(byte[] input, int inOff, int len, byte[] output, int outOff)
         {
             CheckStatus();
 
@@ -347,49 +365,95 @@ namespace Org.BouncyCastle.Crypto.Modes
 
             if (forEncryption)
             {
-                if (bufOff != 0)
+                if (bufOff > 0)
                 {
-                    while (len > 0)
+                    int available = BlockSize - bufOff;
+                    if (len < available)
                     {
-                        --len;
-                        bufBlock[bufOff] = input[inOff++];
-                        if (++bufOff == BlockSize)
-                        {
-                            ProcessBlock(bufBlock, 0, output, outOff);
-                            bufOff = 0;
-                            resultLen += BlockSize;
-                            break;
-                        }
+                        Array.Copy(input, inOff, bufBlock, bufOff, len);
+                        bufOff += len;
+                        return 0;
                     }
+
+                    Array.Copy(input, inOff, bufBlock, bufOff, available);
+                    EncryptBlock(bufBlock, 0, output, outOff);
+                    inOff += available;
+                    len -= available;
+                    resultLen = BlockSize;
+                    //bufOff = 0;
                 }
 
-                while (len >= BlockSize)
+                int inLimit1 = inOff + len - BlockSize;
+                int inLimit2 = inLimit1 - BlockSize;
+
+                while (inOff <= inLimit2)
                 {
-                    ProcessBlock(input, inOff, output, outOff + resultLen);
-                    inOff += BlockSize;
-                    len -= BlockSize;
-                    resultLen += BlockSize;
+                    EncryptBlocks2(input, inOff, output, outOff + resultLen);
+                    inOff += BlockSize * 2;
+                    resultLen += BlockSize * 2;
                 }
 
-                if (len > 0)
+                if (inOff <= inLimit1)
                 {
-                    Array.Copy(input, inOff, bufBlock, 0, len);
-                    bufOff = len;
+                    EncryptBlock(input, inOff, output, outOff + resultLen);
+                    inOff += BlockSize;
+                    resultLen += BlockSize;
                 }
+
+                bufOff = BlockSize + inLimit1 - inOff;
+                Array.Copy(input, inOff, bufBlock, 0, bufOff);
             }
             else
             {
-                for (int i = 0; i < len; ++i)
+                int available = bufBlock.Length - bufOff;
+                if (len < available)
+                {
+                    Array.Copy(input, inOff, bufBlock, bufOff, len);
+                    bufOff += len;
+                    return 0;
+                }
+
+                if (bufOff >= BlockSize)
                 {
-                    bufBlock[bufOff] = input[inOff + i];
-                    if (++bufOff == bufBlock.Length)
+                    DecryptBlock(bufBlock, 0, output, outOff);
+                    Array.Copy(bufBlock, BlockSize, bufBlock, 0, bufOff -= BlockSize);
+                    resultLen = BlockSize;
+
+                    available += BlockSize;
+                    if (len < available)
                     {
-                        ProcessBlock(bufBlock, 0, output, outOff + resultLen);
-                        Array.Copy(bufBlock, BlockSize, bufBlock, 0, macSize);
-                        bufOff = macSize;
-                        resultLen += BlockSize;
+                        Array.Copy(input, inOff, bufBlock, bufOff, len);
+                        bufOff += len;
+                        return resultLen;
                     }
                 }
+
+                int inLimit1 = inOff + len - bufBlock.Length;
+                int inLimit2 = inLimit1 - BlockSize;
+
+                available = BlockSize - bufOff;
+                Array.Copy(input, inOff, bufBlock, bufOff, available);
+                DecryptBlock(bufBlock, 0, output, outOff + resultLen);
+                inOff += available;
+                resultLen += BlockSize;
+                //bufOff = 0;
+
+                while (inOff <= inLimit2)
+                {
+                    DecryptBlocks2(input, inOff, output, outOff + resultLen);
+                    inOff += BlockSize * 2;
+                    resultLen += BlockSize * 2;
+                }
+
+                if (inOff <= inLimit1)
+                {
+                    DecryptBlock(input, inOff, output, outOff + resultLen);
+                    inOff += BlockSize;
+                    resultLen += BlockSize;
+                }
+
+                bufOff = bufBlock.Length + inLimit1 - inOff;
+                Array.Copy(input, inOff, bufBlock, 0, bufOff);
             }
 
             return resultLen;
@@ -525,6 +589,7 @@ namespace Org.BouncyCastle.Crypto.Modes
             atLength = 0;
             atLengthPre = 0;
             counter = Arrays.Clone(J0);
+            counter32 = Pack.BE_To_UInt32(counter, 12);
             blocksRemaining = uint.MaxValue - 1;
             bufOff = 0;
             totalLength = 0;
@@ -552,9 +617,9 @@ namespace Org.BouncyCastle.Crypto.Modes
             }
         }
 
-        private void ProcessBlock(byte[] buf, int bufOff, byte[] output, int outOff)
+        private void DecryptBlock(byte[] inBuf, int inOff, byte[] outBuf, int outOff)
         {
-            Check.OutputLength(output, outOff, BlockSize, "Output buffer too short");
+            Check.OutputLength(outBuf, outOff, BlockSize, "Output buffer too short");
 
             if (totalLength == 0)
             {
@@ -562,23 +627,280 @@ namespace Org.BouncyCastle.Crypto.Modes
             }
 
             byte[] ctrBlock = new byte[BlockSize];
+
             GetNextCtrBlock(ctrBlock);
+#if NETCOREAPP3_0_OR_GREATER
+            if (Sse2.IsSupported && Unsafe.SizeOf<Vector128<byte>>() == BlockSize)
+            {
+                var t0 = Unsafe.ReadUnaligned<Vector128<byte>>(ref inBuf[inOff]);
+                var t1 = Unsafe.ReadUnaligned<Vector128<byte>>(ref ctrBlock[0]);
+                var t2 = Unsafe.ReadUnaligned<Vector128<byte>>(ref S[0]);
 
-            if (forEncryption)
+                t1 = Sse2.Xor(t1, t0);
+                t2 = Sse2.Xor(t2, t0);
+
+                Unsafe.WriteUnaligned(ref outBuf[outOff], t1);
+                Unsafe.WriteUnaligned(ref S[0], t2);
+            }
+            else
+#endif
+            {
+                for (int i = 0; i < BlockSize; i += 4)
+                {
+                    byte c0 = inBuf[inOff + i + 0];
+                    byte c1 = inBuf[inOff + i + 1];
+                    byte c2 = inBuf[inOff + i + 2];
+                    byte c3 = inBuf[inOff + i + 3];
+
+                    S[i + 0] ^= c0;
+                    S[i + 1] ^= c1;
+                    S[i + 2] ^= c2;
+                    S[i + 3] ^= c3;
+
+                    outBuf[outOff + i + 0] = (byte)(c0 ^ ctrBlock[i + 0]);
+                    outBuf[outOff + i + 1] = (byte)(c1 ^ ctrBlock[i + 1]);
+                    outBuf[outOff + i + 2] = (byte)(c2 ^ ctrBlock[i + 2]);
+                    outBuf[outOff + i + 3] = (byte)(c3 ^ ctrBlock[i + 3]);
+                }
+            }
+            multiplier.MultiplyH(S);
+
+            totalLength += BlockSize;
+        }
+
+        private void DecryptBlocks2(byte[] inBuf, int inOff, byte[] outBuf, int outOff)
+        {
+            Check.OutputLength(outBuf, outOff, BlockSize * 2, "Output buffer too short");
+
+            if (totalLength == 0)
             {
-                GcmUtilities.Xor(ctrBlock, buf, bufOff);
-                gHASHBlock(S, ctrBlock);
-                Array.Copy(ctrBlock, 0, output, outOff, BlockSize);
+                InitCipher();
+            }
+
+            byte[] ctrBlock = new byte[BlockSize];
+
+            GetNextCtrBlock(ctrBlock);
+#if NETCOREAPP3_0_OR_GREATER
+            if (Sse2.IsSupported && Unsafe.SizeOf<Vector128<byte>>() == BlockSize)
+            {
+                var t0 = Unsafe.ReadUnaligned<Vector128<byte>>(ref inBuf[inOff]);
+                var t1 = Unsafe.ReadUnaligned<Vector128<byte>>(ref ctrBlock[0]);
+                var t2 = Unsafe.ReadUnaligned<Vector128<byte>>(ref S[0]);
+
+                t1 = Sse2.Xor(t1, t0);
+                t2 = Sse2.Xor(t2, t0);
+
+                Unsafe.WriteUnaligned(ref outBuf[outOff], t1);
+                Unsafe.WriteUnaligned(ref S[0], t2);
             }
             else
+#endif
             {
-                gHASHBlock(S, buf, bufOff);
-                GcmUtilities.Xor(ctrBlock, 0, buf, bufOff, output, outOff);
+                for (int i = 0; i < BlockSize; i += 4)
+                {
+                    byte c0 = inBuf[inOff + i + 0];
+                    byte c1 = inBuf[inOff + i + 1];
+                    byte c2 = inBuf[inOff + i + 2];
+                    byte c3 = inBuf[inOff + i + 3];
+
+                    S[i + 0] ^= c0;
+                    S[i + 1] ^= c1;
+                    S[i + 2] ^= c2;
+                    S[i + 3] ^= c3;
+
+                    outBuf[outOff + i + 0] = (byte)(c0 ^ ctrBlock[i + 0]);
+                    outBuf[outOff + i + 1] = (byte)(c1 ^ ctrBlock[i + 1]);
+                    outBuf[outOff + i + 2] = (byte)(c2 ^ ctrBlock[i + 2]);
+                    outBuf[outOff + i + 3] = (byte)(c3 ^ ctrBlock[i + 3]);
+                }
             }
+            multiplier.MultiplyH(S);
+
+            inOff += BlockSize;
+            outOff += BlockSize;
+
+            GetNextCtrBlock(ctrBlock);
+#if NETCOREAPP3_0_OR_GREATER
+            if (Sse2.IsSupported && Unsafe.SizeOf<Vector128<byte>>() == BlockSize)
+            {
+                var t0 = Unsafe.ReadUnaligned<Vector128<byte>>(ref inBuf[inOff]);
+                var t1 = Unsafe.ReadUnaligned<Vector128<byte>>(ref ctrBlock[0]);
+                var t2 = Unsafe.ReadUnaligned<Vector128<byte>>(ref S[0]);
+
+                t1 = Sse2.Xor(t1, t0);
+                t2 = Sse2.Xor(t2, t0);
+
+                Unsafe.WriteUnaligned(ref outBuf[outOff], t1);
+                Unsafe.WriteUnaligned(ref S[0], t2);
+            }
+            else
+#endif
+            {
+                for (int i = 0; i < BlockSize; i += 4)
+                {
+                    byte c0 = inBuf[inOff + i + 0];
+                    byte c1 = inBuf[inOff + i + 1];
+                    byte c2 = inBuf[inOff + i + 2];
+                    byte c3 = inBuf[inOff + i + 3];
+
+                    S[i + 0] ^= c0;
+                    S[i + 1] ^= c1;
+                    S[i + 2] ^= c2;
+                    S[i + 3] ^= c3;
+
+                    outBuf[outOff + i + 0] = (byte)(c0 ^ ctrBlock[i + 0]);
+                    outBuf[outOff + i + 1] = (byte)(c1 ^ ctrBlock[i + 1]);
+                    outBuf[outOff + i + 2] = (byte)(c2 ^ ctrBlock[i + 2]);
+                    outBuf[outOff + i + 3] = (byte)(c3 ^ ctrBlock[i + 3]);
+                }
+            }
+            multiplier.MultiplyH(S);
+
+            totalLength += BlockSize * 2;
+        }
+
+        private void EncryptBlock(byte[] inBuf, int inOff, byte[] outBuf, int outOff)
+        {
+            Check.OutputLength(outBuf, outOff, BlockSize, "Output buffer too short");
+
+            if (totalLength == 0)
+            {
+                InitCipher();
+            }
+
+            byte[] ctrBlock = new byte[BlockSize];
+
+            GetNextCtrBlock(ctrBlock);
+#if NETCOREAPP3_0_OR_GREATER
+            if (Sse2.IsSupported && Unsafe.SizeOf<Vector128<byte>>() == BlockSize)
+            {
+                var t0 = Unsafe.ReadUnaligned<Vector128<byte>>(ref inBuf[inOff]);
+                var t1 = Unsafe.ReadUnaligned<Vector128<byte>>(ref ctrBlock[0]);
+                var t2 = Unsafe.ReadUnaligned<Vector128<byte>>(ref S[0]);
+
+                t1 = Sse2.Xor(t1, t0);
+                t2 = Sse2.Xor(t2, t1);
+
+                Unsafe.WriteUnaligned(ref outBuf[outOff], t1);
+                Unsafe.WriteUnaligned(ref S[0], t2);
+            }
+            else
+#endif
+            {
+                for (int i = 0; i < BlockSize; i += 4)
+                {
+                    byte c0 = (byte)(ctrBlock[i + 0] ^ inBuf[inOff + i + 0]);
+                    byte c1 = (byte)(ctrBlock[i + 1] ^ inBuf[inOff + i + 1]);
+                    byte c2 = (byte)(ctrBlock[i + 2] ^ inBuf[inOff + i + 2]);
+                    byte c3 = (byte)(ctrBlock[i + 3] ^ inBuf[inOff + i + 3]);
+
+                    S[i + 0] ^= c0;
+                    S[i + 1] ^= c1;
+                    S[i + 2] ^= c2;
+                    S[i + 3] ^= c3;
+
+                    outBuf[outOff + i + 0] = c0;
+                    outBuf[outOff + i + 1] = c1;
+                    outBuf[outOff + i + 2] = c2;
+                    outBuf[outOff + i + 3] = c3;
+                }
+            }
+            multiplier.MultiplyH(S);
 
             totalLength += BlockSize;
         }
 
+        private void EncryptBlocks2(byte[] inBuf, int inOff, byte[] outBuf, int outOff)
+        {
+            Check.OutputLength(outBuf, outOff, BlockSize * 2, "Output buffer too short");
+
+            if (totalLength == 0)
+            {
+                InitCipher();
+            }
+
+            byte[] ctrBlock = new byte[BlockSize];
+
+            GetNextCtrBlock(ctrBlock);
+#if NETCOREAPP3_0_OR_GREATER
+            if (Sse2.IsSupported && Unsafe.SizeOf<Vector128<byte>>() == BlockSize)
+            {
+                var t0 = Unsafe.ReadUnaligned<Vector128<byte>>(ref inBuf[inOff]);
+                var t1 = Unsafe.ReadUnaligned<Vector128<byte>>(ref ctrBlock[0]);
+                var t2 = Unsafe.ReadUnaligned<Vector128<byte>>(ref S[0]);
+
+                t1 = Sse2.Xor(t1, t0);
+                t2 = Sse2.Xor(t2, t1);
+
+                Unsafe.WriteUnaligned(ref outBuf[outOff], t1);
+                Unsafe.WriteUnaligned(ref S[0], t2);
+            }
+            else
+#endif
+            {
+                for (int i = 0; i < BlockSize; i += 4)
+                {
+                    byte c0 = (byte)(ctrBlock[i + 0] ^ inBuf[inOff + i + 0]);
+                    byte c1 = (byte)(ctrBlock[i + 1] ^ inBuf[inOff + i + 1]);
+                    byte c2 = (byte)(ctrBlock[i + 2] ^ inBuf[inOff + i + 2]);
+                    byte c3 = (byte)(ctrBlock[i + 3] ^ inBuf[inOff + i + 3]);
+
+                    S[i + 0] ^= c0;
+                    S[i + 1] ^= c1;
+                    S[i + 2] ^= c2;
+                    S[i + 3] ^= c3;
+
+                    outBuf[outOff + i + 0] = c0;
+                    outBuf[outOff + i + 1] = c1;
+                    outBuf[outOff + i + 2] = c2;
+                    outBuf[outOff + i + 3] = c3;
+                }
+            }
+            multiplier.MultiplyH(S);
+
+            inOff += BlockSize;
+            outOff += BlockSize;
+
+            GetNextCtrBlock(ctrBlock);
+#if NETCOREAPP3_0_OR_GREATER
+            if (Sse2.IsSupported && Unsafe.SizeOf<Vector128<byte>>() == BlockSize)
+            {
+                var t0 = Unsafe.ReadUnaligned<Vector128<byte>>(ref inBuf[inOff]);
+                var t1 = Unsafe.ReadUnaligned<Vector128<byte>>(ref ctrBlock[0]);
+                var t2 = Unsafe.ReadUnaligned<Vector128<byte>>(ref S[0]);
+
+                t1 = Sse2.Xor(t1, t0);
+                t2 = Sse2.Xor(t2, t1);
+
+                Unsafe.WriteUnaligned(ref outBuf[outOff], t1);
+                Unsafe.WriteUnaligned(ref S[0], t2);
+            }
+            else
+#endif
+            {
+                for (int i = 0; i < BlockSize; i += 4)
+                {
+                    byte c0 = (byte)(ctrBlock[i + 0] ^ inBuf[inOff + i + 0]);
+                    byte c1 = (byte)(ctrBlock[i + 1] ^ inBuf[inOff + i + 1]);
+                    byte c2 = (byte)(ctrBlock[i + 2] ^ inBuf[inOff + i + 2]);
+                    byte c3 = (byte)(ctrBlock[i + 3] ^ inBuf[inOff + i + 3]);
+
+                    S[i + 0] ^= c0;
+                    S[i + 1] ^= c1;
+                    S[i + 2] ^= c2;
+                    S[i + 3] ^= c3;
+
+                    outBuf[outOff + i + 0] = c0;
+                    outBuf[outOff + i + 1] = c1;
+                    outBuf[outOff + i + 2] = c2;
+                    outBuf[outOff + i + 3] = c3;
+                }
+            }
+            multiplier.MultiplyH(S);
+
+            totalLength += BlockSize * 2;
+        }
+
         private void ProcessPartial(byte[] buf, int off, int len, byte[] output, int outOff)
         {
             byte[] ctrBlock = new byte[BlockSize];
@@ -633,11 +955,7 @@ namespace Org.BouncyCastle.Crypto.Modes
 
             blocksRemaining--;
 
-            uint c = 1;
-            c += counter[15]; counter[15] = (byte)c; c >>= 8;
-            c += counter[14]; counter[14] = (byte)c; c >>= 8;
-            c += counter[13]; counter[13] = (byte)c; c >>= 8;
-            c += counter[12]; counter[12] = (byte)c;
+            Pack.UInt32_To_BE(++counter32, counter, 12);
 
             cipher.ProcessBlock(counter, 0, block, 0);
         }
diff --git a/crypto/src/crypto/modes/gcm/GcmUtilities.cs b/crypto/src/crypto/modes/gcm/GcmUtilities.cs
index cf21ace23..4528e172a 100644
--- a/crypto/src/crypto/modes/gcm/GcmUtilities.cs
+++ b/crypto/src/crypto/modes/gcm/GcmUtilities.cs
@@ -1,5 +1,8 @@
 using System;
 using System.Diagnostics;
+#if NETSTANDARD1_0_OR_GREATER || NETCOREAPP1_0_OR_GREATER
+using System.Runtime.CompilerServices;
+#endif
 #if NETCOREAPP3_0_OR_GREATER
 using System.Runtime.Intrinsics;
 using System.Runtime.Intrinsics.X86;
@@ -13,6 +16,12 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
 {
     internal abstract class GcmUtilities
     {
+#if NETCOREAPP3_0_OR_GREATER
+        private static readonly Vector128<byte> EndianMask = Vector128.Create(
+            (byte)0x07, (byte)0x06, (byte)0x05, (byte)0x04, (byte)0x03, (byte)0x02, (byte)0x01, (byte)0x00,
+            (byte)0x0F, (byte)0x0E, (byte)0x0D, (byte)0x0C, (byte)0x0B, (byte)0x0A, (byte)0x09, (byte)0x08);
+#endif
+
         internal struct FieldElement
         {
             internal ulong n0, n1;
@@ -27,128 +36,53 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
             x.n1 = 0UL;
         }
 
-        internal static byte[] OneAsBytes()
-        {
-            byte[] tmp = new byte[16];
-            tmp[0] = 0x80;
-            return tmp;
-        }
-
-        internal static uint[] OneAsUints()
-        {
-            uint[] tmp = new uint[4];
-            tmp[0] = 0x80000000;
-            return tmp;
-        }
-
-        internal static ulong[] OneAsUlongs()
-        {
-            ulong[] tmp = new ulong[2];
-            tmp[0] = 1UL << 63;
-            return tmp;
-        }
-
-        internal static byte[] AsBytes(uint[] x)
-        {
-            return Pack.UInt32_To_BE(x);
-        }
-
-        internal static void AsBytes(uint[] x, byte[] z)
-        {
-            Pack.UInt32_To_BE(x, z, 0);
-        }
-
-        internal static byte[] AsBytes(ulong[] x)
+#if NETSTANDARD1_0_OR_GREATER || NETCOREAPP1_0_OR_GREATER
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
+        internal static void AsBytes(ulong x0, ulong x1, byte[] z)
         {
-            byte[] z = new byte[16];
-            Pack.UInt64_To_BE(x, z, 0);
-            return z;
-        }
+#if NETCOREAPP3_0_OR_GREATER
+            if (Ssse3.IsSupported && BitConverter.IsLittleEndian && Unsafe.SizeOf<Vector128<byte>>() == 16)
+            {
+                var X = Vector128.Create(x0, x1).AsByte();
+                var Z = Ssse3.Shuffle(X, EndianMask);
+                Unsafe.WriteUnaligned(ref z[0], Z);
+                return;
+            }
+#endif
 
-        internal static void AsBytes(ulong[] x, byte[] z)
-        {
-            Pack.UInt64_To_BE(x, z, 0);
+            Pack.UInt64_To_BE(x0, z, 0);
+            Pack.UInt64_To_BE(x1, z, 8);
         }
 
+#if NETSTANDARD1_0_OR_GREATER || NETCOREAPP1_0_OR_GREATER
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
         internal static void AsBytes(ref FieldElement x, byte[] z)
         {
-            Pack.UInt64_To_BE(x.n0, z, 0);
-            Pack.UInt64_To_BE(x.n1, z, 8);
+            AsBytes(x.n0, x.n1, z);
         }
 
+#if NETSTANDARD1_0_OR_GREATER || NETCOREAPP1_0_OR_GREATER
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
         internal static void AsFieldElement(byte[] x, out FieldElement z)
         {
+#if NETCOREAPP3_0_OR_GREATER
+            if (Ssse3.IsSupported && BitConverter.IsLittleEndian && Unsafe.SizeOf<Vector128<byte>>() == 16)
+            {
+                var X = Unsafe.ReadUnaligned<Vector128<byte>>(ref x[0]);
+                var Z = Ssse3.Shuffle(X, EndianMask).AsUInt64();
+                z.n0 = Z.GetElement(0);
+                z.n1 = Z.GetElement(1);
+                return;
+            }
+#endif
+
             z.n0 = Pack.BE_To_UInt64(x, 0);
             z.n1 = Pack.BE_To_UInt64(x, 8);
         }
 
-        internal static uint[] AsUints(byte[] bs)
-        {
-            uint[] output = new uint[4];
-            Pack.BE_To_UInt32(bs, 0, output);
-            return output;
-        }
-
-        internal static void AsUints(byte[] bs, uint[] output)
-        {
-            Pack.BE_To_UInt32(bs, 0, output);
-        }
-
-        internal static ulong[] AsUlongs(byte[] x)
-        {
-            ulong[] z = new ulong[2];
-            Pack.BE_To_UInt64(x, 0, z);
-            return z;
-        }
-
-        internal static void AsUlongs(byte[] x, ulong[] z)
-        {
-            Pack.BE_To_UInt64(x, 0, z);
-        }
-
-        internal static void AsUlongs(byte[] x, ulong[] z, int zOff)
-        {
-            Pack.BE_To_UInt64(x, 0, z, zOff, 2);
-        }
-
-        internal static void Copy(uint[] x, uint[] z)
-        {
-            z[0] = x[0];
-            z[1] = x[1];
-            z[2] = x[2];
-            z[3] = x[3];
-        }
-
-        internal static void Copy(ulong[] x, ulong[] z)
-        {
-            z[0] = x[0];
-            z[1] = x[1];
-        }
-
-        internal static void Copy(ulong[] x, int xOff, ulong[] z, int zOff)
-        {
-            z[zOff + 0] = x[xOff + 0];
-            z[zOff + 1] = x[xOff + 1];
-        }
-
-        internal static void DivideP(ulong[] x, ulong[] z)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong m = (ulong)((long)x0 >> 63);
-            x0 ^= (m & E1UL);
-            z[0] = (x0 << 1) | (x1 >> 63);
-            z[1] = (x1 << 1) | (ulong)(-(long)m);
-        }
-
-        internal static void DivideP(ulong[] x, int xOff, ulong[] z, int zOff)
-        {
-            ulong x0 = x[xOff + 0], x1 = x[xOff + 1];
-            ulong m = (ulong)((long)x0 >> 63);
-            x0 ^= (m & E1UL);
-            z[zOff + 0] = (x0 << 1) | (x1 >> 63);
-            z[zOff + 1] = (x1 << 1) | (ulong)(-(long)m);
-        }
-
         internal static void DivideP(ref FieldElement x, out FieldElement z)
         {
             ulong x0 = x.n0, x1 = x.n1;
@@ -233,90 +167,6 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
             x.n1 = z1;
         }
 
-        internal static void MultiplyP(uint[] x)
-        {
-            uint x0 = x[0], x1 = x[1], x2 = x[2], x3 = x[3];
-            uint m = (uint)((int)(x3 << 31) >> 31);
-            x[0] = (x0 >> 1) ^ (m & E1);
-            x[1] = (x1 >> 1) | (x0 << 31);
-            x[2] = (x2 >> 1) | (x1 << 31);
-            x[3] = (x3 >> 1) | (x2 << 31);
-        }
-
-        internal static void MultiplyP(uint[] x, uint[] z)
-        {
-            uint x0 = x[0], x1 = x[1], x2 = x[2], x3 = x[3];
-            uint m = (uint)((int)(x3 << 31) >> 31);
-            z[0] = (x0 >> 1) ^ (m & E1);
-            z[1] = (x1 >> 1) | (x0 << 31);
-            z[2] = (x2 >> 1) | (x1 << 31);
-            z[3] = (x3 >> 1) | (x2 << 31);
-        }
-
-        internal static void MultiplyP(ulong[] x)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong m = (ulong)((long)(x1 << 63) >> 63);
-            x[0] = (x0 >> 1) ^ (m & E1UL);
-            x[1] = (x1 >> 1) | (x0 << 63);
-        }
-
-        internal static void MultiplyP(ulong[] x, ulong[] z)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong m = (ulong)((long)(x1 << 63) >> 63);
-            z[0] = (x0 >> 1) ^ (m & E1UL);
-            z[1] = (x1 >> 1) | (x0 << 63);
-        }
-
-        internal static void MultiplyP3(ulong[] x, ulong[] z)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong c = x1 << 61;
-            z[0] = (x0 >> 3) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            z[1] = (x1 >> 3) | (x0 << 61);
-        }
-
-        internal static void MultiplyP3(ulong[] x, int xOff, ulong[] z, int zOff)
-        {
-            ulong x0 = x[xOff + 0], x1 = x[xOff + 1];
-            ulong c = x1 << 61;
-            z[zOff + 0] = (x0 >> 3) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            z[zOff + 1] = (x1 >> 3) | (x0 << 61);
-        }
-
-        internal static void MultiplyP4(ulong[] x, ulong[] z)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong c = x1 << 60;
-            z[0] = (x0 >> 4) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            z[1] = (x1 >> 4) | (x0 << 60);
-        }
-
-        internal static void MultiplyP4(ulong[] x, int xOff, ulong[] z, int zOff)
-        {
-            ulong x0 = x[xOff + 0], x1 = x[xOff + 1];
-            ulong c = x1 << 60;
-            z[zOff + 0] = (x0 >> 4) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            z[zOff + 1] = (x1 >> 4) | (x0 << 60);
-        }
-
-        internal static void MultiplyP7(ulong[] x, ulong[] z)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong c = x1 << 57;
-            z[0] = (x0 >> 7) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            z[1] = (x1 >> 7) | (x0 << 57);
-        }
-
-        internal static void MultiplyP7(ulong[] x, int xOff, ulong[] z, int zOff)
-        {
-            ulong x0 = x[xOff + 0], x1 = x[xOff + 1];
-            ulong c = x1 << 57;
-            z[zOff + 0] = (x0 >> 7) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            z[zOff + 1] = (x1 >> 7) | (x0 << 57);
-        }
-
         internal static void MultiplyP7(ref FieldElement x)
         {
             ulong x0 = x.n0, x1 = x.n1;
@@ -325,50 +175,6 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
             x.n1 = (x1 >> 7) | (x0 << 57);
         }
 
-        internal static void MultiplyP8(uint[] x)
-        {
-            uint x0 = x[0], x1 = x[1], x2 = x[2], x3 = x[3];
-            uint c = x3 << 24;
-            x[0] = (x0 >> 8) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            x[1] = (x1 >> 8) | (x0 << 24);
-            x[2] = (x2 >> 8) | (x1 << 24);
-            x[3] = (x3 >> 8) | (x2 << 24);
-        }
-
-        internal static void MultiplyP8(uint[] x, uint[] y)
-        {
-            uint x0 = x[0], x1 = x[1], x2 = x[2], x3 = x[3];
-            uint c = x3 << 24;
-            y[0] = (x0 >> 8) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            y[1] = (x1 >> 8) | (x0 << 24);
-            y[2] = (x2 >> 8) | (x1 << 24);
-            y[3] = (x3 >> 8) | (x2 << 24);
-        }
-
-        internal static void MultiplyP8(ulong[] x)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong c = x1 << 56;
-            x[0] = (x0 >> 8) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            x[1] = (x1 >> 8) | (x0 << 56);
-        }
-
-        internal static void MultiplyP8(ulong[] x, ulong[] y)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong c = x1 << 56;
-            y[0] = (x0 >> 8) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            y[1] = (x1 >> 8) | (x0 << 56);
-        }
-
-        internal static void MultiplyP8(ulong[] x, int xOff, ulong[] y, int yOff)
-        {
-            ulong x0 = x[xOff + 0], x1 = x[xOff + 1];
-            ulong c = x1 << 56;
-            y[yOff + 0] = (x0 >> 8) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            y[yOff + 1] = (x1 >> 8) | (x0 << 56);
-        }
-
         internal static void MultiplyP8(ref FieldElement x)
         {
             ulong x0 = x.n0, x1 = x.n1;
@@ -385,14 +191,6 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
             y.n1 = (x1 >> 8) | (x0 << 56);
         }
 
-        internal static void MultiplyP16(ulong[] x)
-        {
-            ulong x0 = x[0], x1 = x[1];
-            ulong c = x1 << 48;
-            x[0] = (x0 >> 16) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
-            x[1] = (x1 >> 16) | (x0 << 48);
-        }
-
         internal static void MultiplyP16(ref FieldElement x)
         {
             ulong x0 = x.n0, x1 = x.n1;
@@ -448,19 +246,6 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
             while (i < 16);
         }
 
-        internal static void Xor(byte[] x, int xOff, byte[] y, int yOff, byte[] z, int zOff)
-        {
-            int i = 0;
-            do
-            {
-                z[zOff + i] = (byte)(x[xOff + i] ^ y[yOff + i]); ++i;
-                z[zOff + i] = (byte)(x[xOff + i] ^ y[yOff + i]); ++i;
-                z[zOff + i] = (byte)(x[xOff + i] ^ y[yOff + i]); ++i;
-                z[zOff + i] = (byte)(x[xOff + i] ^ y[yOff + i]); ++i;
-            }
-            while (i < 16);
-        }
-
         internal static void Xor(byte[] x, byte[] y, int yOff, int yLen)
         {
             while (--yLen >= 0)
@@ -477,57 +262,10 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
             }
         }
 
-        internal static void Xor(byte[] x, byte[] y, byte[] z)
-        {
-            int i = 0;
-            do
-            {
-                z[i] = (byte)(x[i] ^ y[i]); ++i;
-                z[i] = (byte)(x[i] ^ y[i]); ++i;
-                z[i] = (byte)(x[i] ^ y[i]); ++i;
-                z[i] = (byte)(x[i] ^ y[i]); ++i;
-            }
-            while (i < 16);
-        }
-
-        internal static void Xor(uint[] x, uint[] y)
-        {
-            x[0] ^= y[0];
-            x[1] ^= y[1];
-            x[2] ^= y[2];
-            x[3] ^= y[3];
-        }
-
-        internal static void Xor(uint[] x, uint[] y, uint[] z)
-        {
-            z[0] = x[0] ^ y[0];
-            z[1] = x[1] ^ y[1];
-            z[2] = x[2] ^ y[2];
-            z[3] = x[3] ^ y[3];
-        }
-
-        internal static void Xor(ulong[] x, ulong[] y)
-        {
-            x[0] ^= y[0];
-            x[1] ^= y[1];
-        }
-
-        internal static void Xor(ulong[] x, int xOff, ulong[] y, int yOff)
-        {
-            x[xOff + 0] ^= y[yOff + 0];
-            x[xOff + 1] ^= y[yOff + 1];
-        }
-
-        internal static void Xor(ulong[] x, ulong[] y, ulong[] z)
-        {
-            z[0] = x[0] ^ y[0];
-            z[1] = x[1] ^ y[1];
-        }
-
-        internal static void Xor(ulong[] x, int xOff, ulong[] y, int yOff, ulong[] z, int zOff)
+        internal static void Xor(ref FieldElement x, ref FieldElement y)
         {
-            z[zOff + 0] = x[xOff + 0] ^ y[yOff + 0];
-            z[zOff + 1] = x[xOff + 1] ^ y[yOff + 1];
+            x.n0 ^= y.n0;
+            x.n1 ^= y.n1;
         }
 
         internal static void Xor(ref FieldElement x, ref FieldElement y, out FieldElement z)
@@ -536,12 +274,6 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
             z.n1 = x.n1 ^ y.n1;
         }
 
-        internal static void Xor(ref FieldElement x, ref FieldElement y)
-        {
-            x.n0 ^= y.n0;
-            x.n1 ^= y.n1;
-        }
-
         private static ulong ImplMul64(ulong x, ulong y)
         {
             ulong x0 = x & 0x1111111111111111UL;
diff --git a/crypto/src/crypto/modes/gcm/Tables4kGcmMultiplier.cs b/crypto/src/crypto/modes/gcm/Tables4kGcmMultiplier.cs
index 7867a0b99..117558b45 100644
--- a/crypto/src/crypto/modes/gcm/Tables4kGcmMultiplier.cs
+++ b/crypto/src/crypto/modes/gcm/Tables4kGcmMultiplier.cs
@@ -62,8 +62,7 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
                 z0 = T[pos].n0 ^ (z0 >> 8) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
             }
 
-            Pack.UInt64_To_BE(z0, x, 0);
-            Pack.UInt64_To_BE(z1, x, 8);
+            GcmUtilities.AsBytes(z0, z1, x);
         }
     }
 }
diff --git a/crypto/src/crypto/modes/gcm/Tables64kGcmMultiplier.cs b/crypto/src/crypto/modes/gcm/Tables64kGcmMultiplier.cs
index 364c070e7..52a9d4e82 100644
--- a/crypto/src/crypto/modes/gcm/Tables64kGcmMultiplier.cs
+++ b/crypto/src/crypto/modes/gcm/Tables64kGcmMultiplier.cs
@@ -74,8 +74,7 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
                 z1 ^= t[tPos].n1;
             }
 
-            Pack.UInt64_To_BE(z0, x, 0);
-            Pack.UInt64_To_BE(z1, x, 8);
+            GcmUtilities.AsBytes(z0, z1, x);
         }
     }
 }
diff --git a/crypto/src/crypto/modes/gcm/Tables8kGcmMultiplier.cs b/crypto/src/crypto/modes/gcm/Tables8kGcmMultiplier.cs
index 67a709a75..7fe122526 100644
--- a/crypto/src/crypto/modes/gcm/Tables8kGcmMultiplier.cs
+++ b/crypto/src/crypto/modes/gcm/Tables8kGcmMultiplier.cs
@@ -82,8 +82,7 @@ namespace Org.BouncyCastle.Crypto.Modes.Gcm
                 z0 = T0[uPos].n0 ^ T1[vPos].n0 ^ (z0 >> 16) ^ c ^ (c >> 1) ^ (c >> 2) ^ (c >> 7);
             }
 
-            Pack.UInt64_To_BE(z0, x, 0);
-            Pack.UInt64_To_BE(z1, x, 8);
+            GcmUtilities.AsBytes(z0, z1, x);
         }
     }
 }