1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/security/nss/lib/freebl/ecl/ecp_256_32.c Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,1504 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +/* A 32-bit implementation of the NIST P-256 elliptic curve. */
1.9 +
1.10 +#include <string.h>
1.11 +
1.12 +#include "prtypes.h"
1.13 +#include "mpi.h"
1.14 +#include "mpi-priv.h"
1.15 +#include "ecp.h"
1.16 +
1.17 +typedef PRUint8 u8;
1.18 +typedef PRUint32 u32;
1.19 +typedef PRUint64 u64;
1.20 +
1.21 +/* Our field elements are represented as nine, unsigned 32-bit words. Freebl's
1.22 + * MPI library calls them digits, but here they are called limbs, which is
1.23 + * GMP's terminology.
1.24 + *
1.25 + * The value of an felem (field element) is:
1.26 + * x[0] + (x[1] * 2**29) + (x[2] * 2**57) + ... + (x[8] * 2**228)
1.27 + *
1.28 + * That is, each limb is alternately 29 or 28-bits wide in little-endian
1.29 + * order.
1.30 + *
1.31 + * This means that an felem hits 2**257, rather than 2**256 as we would like. A
1.32 + * 28, 29, ... pattern would cause us to hit 2**256, but that causes problems
1.33 + * when multiplying as terms end up one bit short of a limb which would require
1.34 + * much bit-shifting to correct.
1.35 + *
1.36 + * Finally, the values stored in an felem are in Montgomery form. So the value
1.37 + * |y| is stored as (y*R) mod p, where p is the P-256 prime and R is 2**257.
1.38 + */
1.39 +typedef u32 limb;
1.40 +#define NLIMBS 9
1.41 +typedef limb felem[NLIMBS];
1.42 +
1.43 +static const limb kBottom28Bits = 0xfffffff;
1.44 +static const limb kBottom29Bits = 0x1fffffff;
1.45 +
1.46 +/* kOne is the number 1 as an felem. It's 2**257 mod p split up into 29 and
1.47 + * 28-bit words.
1.48 + */
1.49 +static const felem kOne = {
1.50 + 2, 0, 0, 0xffff800,
1.51 + 0x1fffffff, 0xfffffff, 0x1fbfffff, 0x1ffffff,
1.52 + 0
1.53 +};
1.54 +static const felem kZero = {0};
1.55 +static const felem kP = {
1.56 + 0x1fffffff, 0xfffffff, 0x1fffffff, 0x3ff,
1.57 + 0, 0, 0x200000, 0xf000000,
1.58 + 0xfffffff
1.59 +};
1.60 +static const felem k2P = {
1.61 + 0x1ffffffe, 0xfffffff, 0x1fffffff, 0x7ff,
1.62 + 0, 0, 0x400000, 0xe000000,
1.63 + 0x1fffffff
1.64 +};
1.65 +
1.66 +/* kPrecomputed contains precomputed values to aid the calculation of scalar
1.67 + * multiples of the base point, G. It's actually two, equal length, tables
1.68 + * concatenated.
1.69 + *
1.70 + * The first table contains (x,y) felem pairs for 16 multiples of the base
1.71 + * point, G.
1.72 + *
1.73 + * Index | Index (binary) | Value
1.74 + * 0 | 0000 | 0G (all zeros, omitted)
1.75 + * 1 | 0001 | G
1.76 + * 2 | 0010 | 2**64G
1.77 + * 3 | 0011 | 2**64G + G
1.78 + * 4 | 0100 | 2**128G
1.79 + * 5 | 0101 | 2**128G + G
1.80 + * 6 | 0110 | 2**128G + 2**64G
1.81 + * 7 | 0111 | 2**128G + 2**64G + G
1.82 + * 8 | 1000 | 2**192G
1.83 + * 9 | 1001 | 2**192G + G
1.84 + * 10 | 1010 | 2**192G + 2**64G
1.85 + * 11 | 1011 | 2**192G + 2**64G + G
1.86 + * 12 | 1100 | 2**192G + 2**128G
1.87 + * 13 | 1101 | 2**192G + 2**128G + G
1.88 + * 14 | 1110 | 2**192G + 2**128G + 2**64G
1.89 + * 15 | 1111 | 2**192G + 2**128G + 2**64G + G
1.90 + *
1.91 + * The second table follows the same style, but the terms are 2**32G,
1.92 + * 2**96G, 2**160G, 2**224G.
1.93 + *
1.94 + * This is ~2KB of data.
1.95 + */
1.96 +static const limb kPrecomputed[NLIMBS * 2 * 15 * 2] = {
1.97 + 0x11522878, 0xe730d41, 0xdb60179, 0x4afe2ff, 0x12883add, 0xcaddd88, 0x119e7edc, 0xd4a6eab, 0x3120bee,
1.98 + 0x1d2aac15, 0xf25357c, 0x19e45cdd, 0x5c721d0, 0x1992c5a5, 0xa237487, 0x154ba21, 0x14b10bb, 0xae3fe3,
1.99 + 0xd41a576, 0x922fc51, 0x234994f, 0x60b60d3, 0x164586ae, 0xce95f18, 0x1fe49073, 0x3fa36cc, 0x5ebcd2c,
1.100 + 0xb402f2f, 0x15c70bf, 0x1561925c, 0x5a26704, 0xda91e90, 0xcdc1c7f, 0x1ea12446, 0xe1ade1e, 0xec91f22,
1.101 + 0x26f7778, 0x566847e, 0xa0bec9e, 0x234f453, 0x1a31f21a, 0xd85e75c, 0x56c7109, 0xa267a00, 0xb57c050,
1.102 + 0x98fb57, 0xaa837cc, 0x60c0792, 0xcfa5e19, 0x61bab9e, 0x589e39b, 0xa324c5, 0x7d6dee7, 0x2976e4b,
1.103 + 0x1fc4124a, 0xa8c244b, 0x1ce86762, 0xcd61c7e, 0x1831c8e0, 0x75774e1, 0x1d96a5a9, 0x843a649, 0xc3ab0fa,
1.104 + 0x6e2e7d5, 0x7673a2a, 0x178b65e8, 0x4003e9b, 0x1a1f11c2, 0x7816ea, 0xf643e11, 0x58c43df, 0xf423fc2,
1.105 + 0x19633ffa, 0x891f2b2, 0x123c231c, 0x46add8c, 0x54700dd, 0x59e2b17, 0x172db40f, 0x83e277d, 0xb0dd609,
1.106 + 0xfd1da12, 0x35c6e52, 0x19ede20c, 0xd19e0c0, 0x97d0f40, 0xb015b19, 0x449e3f5, 0xe10c9e, 0x33ab581,
1.107 + 0x56a67ab, 0x577734d, 0x1dddc062, 0xc57b10d, 0x149b39d, 0x26a9e7b, 0xc35df9f, 0x48764cd, 0x76dbcca,
1.108 + 0xca4b366, 0xe9303ab, 0x1a7480e7, 0x57e9e81, 0x1e13eb50, 0xf466cf3, 0x6f16b20, 0x4ba3173, 0xc168c33,
1.109 + 0x15cb5439, 0x6a38e11, 0x73658bd, 0xb29564f, 0x3f6dc5b, 0x53b97e, 0x1322c4c0, 0x65dd7ff, 0x3a1e4f6,
1.110 + 0x14e614aa, 0x9246317, 0x1bc83aca, 0xad97eed, 0xd38ce4a, 0xf82b006, 0x341f077, 0xa6add89, 0x4894acd,
1.111 + 0x9f162d5, 0xf8410ef, 0x1b266a56, 0xd7f223, 0x3e0cb92, 0xe39b672, 0x6a2901a, 0x69a8556, 0x7e7c0,
1.112 + 0x9b7d8d3, 0x309a80, 0x1ad05f7f, 0xc2fb5dd, 0xcbfd41d, 0x9ceb638, 0x1051825c, 0xda0cf5b, 0x812e881,
1.113 + 0x6f35669, 0x6a56f2c, 0x1df8d184, 0x345820, 0x1477d477, 0x1645db1, 0xbe80c51, 0xc22be3e, 0xe35e65a,
1.114 + 0x1aeb7aa0, 0xc375315, 0xf67bc99, 0x7fdd7b9, 0x191fc1be, 0x61235d, 0x2c184e9, 0x1c5a839, 0x47a1e26,
1.115 + 0xb7cb456, 0x93e225d, 0x14f3c6ed, 0xccc1ac9, 0x17fe37f3, 0x4988989, 0x1a90c502, 0x2f32042, 0xa17769b,
1.116 + 0xafd8c7c, 0x8191c6e, 0x1dcdb237, 0x16200c0, 0x107b32a1, 0x66c08db, 0x10d06a02, 0x3fc93, 0x5620023,
1.117 + 0x16722b27, 0x68b5c59, 0x270fcfc, 0xfad0ecc, 0xe5de1c2, 0xeab466b, 0x2fc513c, 0x407f75c, 0xbaab133,
1.118 + 0x9705fe9, 0xb88b8e7, 0x734c993, 0x1e1ff8f, 0x19156970, 0xabd0f00, 0x10469ea7, 0x3293ac0, 0xcdc98aa,
1.119 + 0x1d843fd, 0xe14bfe8, 0x15be825f, 0x8b5212, 0xeb3fb67, 0x81cbd29, 0xbc62f16, 0x2b6fcc7, 0xf5a4e29,
1.120 + 0x13560b66, 0xc0b6ac2, 0x51ae690, 0xd41e271, 0xf3e9bd4, 0x1d70aab, 0x1029f72, 0x73e1c35, 0xee70fbc,
1.121 + 0xad81baf, 0x9ecc49a, 0x86c741e, 0xfe6be30, 0x176752e7, 0x23d416, 0x1f83de85, 0x27de188, 0x66f70b8,
1.122 + 0x181cd51f, 0x96b6e4c, 0x188f2335, 0xa5df759, 0x17a77eb6, 0xfeb0e73, 0x154ae914, 0x2f3ec51, 0x3826b59,
1.123 + 0xb91f17d, 0x1c72949, 0x1362bf0a, 0xe23fddf, 0xa5614b0, 0xf7d8f, 0x79061, 0x823d9d2, 0x8213f39,
1.124 + 0x1128ae0b, 0xd095d05, 0xb85c0c2, 0x1ecb2ef, 0x24ddc84, 0xe35e901, 0x18411a4a, 0xf5ddc3d, 0x3786689,
1.125 + 0x52260e8, 0x5ae3564, 0x542b10d, 0x8d93a45, 0x19952aa4, 0x996cc41, 0x1051a729, 0x4be3499, 0x52b23aa,
1.126 + 0x109f307e, 0x6f5b6bb, 0x1f84e1e7, 0x77a0cfa, 0x10c4df3f, 0x25a02ea, 0xb048035, 0xe31de66, 0xc6ecaa3,
1.127 + 0x28ea335, 0x2886024, 0x1372f020, 0xf55d35, 0x15e4684c, 0xf2a9e17, 0x1a4a7529, 0xcb7beb1, 0xb2a78a1,
1.128 + 0x1ab21f1f, 0x6361ccf, 0x6c9179d, 0xb135627, 0x1267b974, 0x4408bad, 0x1cbff658, 0xe3d6511, 0xc7d76f,
1.129 + 0x1cc7a69, 0xe7ee31b, 0x54fab4f, 0x2b914f, 0x1ad27a30, 0xcd3579e, 0xc50124c, 0x50daa90, 0xb13f72,
1.130 + 0xb06aa75, 0x70f5cc6, 0x1649e5aa, 0x84a5312, 0x329043c, 0x41c4011, 0x13d32411, 0xb04a838, 0xd760d2d,
1.131 + 0x1713b532, 0xbaa0c03, 0x84022ab, 0x6bcf5c1, 0x2f45379, 0x18ae070, 0x18c9e11e, 0x20bca9a, 0x66f496b,
1.132 + 0x3eef294, 0x67500d2, 0xd7f613c, 0x2dbbeb, 0xb741038, 0xe04133f, 0x1582968d, 0xbe985f7, 0x1acbc1a,
1.133 + 0x1a6a939f, 0x33e50f6, 0xd665ed4, 0xb4b7bd6, 0x1e5a3799, 0x6b33847, 0x17fa56ff, 0x65ef930, 0x21dc4a,
1.134 + 0x2b37659, 0x450fe17, 0xb357b65, 0xdf5efac, 0x15397bef, 0x9d35a7f, 0x112ac15f, 0x624e62e, 0xa90ae2f,
1.135 + 0x107eecd2, 0x1f69bbe, 0x77d6bce, 0x5741394, 0x13c684fc, 0x950c910, 0x725522b, 0xdc78583, 0x40eeabb,
1.136 + 0x1fde328a, 0xbd61d96, 0xd28c387, 0x9e77d89, 0x12550c40, 0x759cb7d, 0x367ef34, 0xae2a960, 0x91b8bdc,
1.137 + 0x93462a9, 0xf469ef, 0xb2e9aef, 0xd2ca771, 0x54e1f42, 0x7aaa49, 0x6316abb, 0x2413c8e, 0x5425bf9,
1.138 + 0x1bed3e3a, 0xf272274, 0x1f5e7326, 0x6416517, 0xea27072, 0x9cedea7, 0x6e7633, 0x7c91952, 0xd806dce,
1.139 + 0x8e2a7e1, 0xe421e1a, 0x418c9e1, 0x1dbc890, 0x1b395c36, 0xa1dc175, 0x1dc4ef73, 0x8956f34, 0xe4b5cf2,
1.140 + 0x1b0d3a18, 0x3194a36, 0x6c2641f, 0xe44124c, 0xa2f4eaa, 0xa8c25ba, 0xf927ed7, 0x627b614, 0x7371cca,
1.141 + 0xba16694, 0x417bc03, 0x7c0a7e3, 0x9c35c19, 0x1168a205, 0x8b6b00d, 0x10e3edc9, 0x9c19bf2, 0x5882229,
1.142 + 0x1b2b4162, 0xa5cef1a, 0x1543622b, 0x9bd433e, 0x364e04d, 0x7480792, 0x5c9b5b3, 0xe85ff25, 0x408ef57,
1.143 + 0x1814cfa4, 0x121b41b, 0xd248a0f, 0x3b05222, 0x39bb16a, 0xc75966d, 0xa038113, 0xa4a1769, 0x11fbc6c,
1.144 + 0x917e50e, 0xeec3da8, 0x169d6eac, 0x10c1699, 0xa416153, 0xf724912, 0x15cd60b7, 0x4acbad9, 0x5efc5fa,
1.145 + 0xf150ed7, 0x122b51, 0x1104b40a, 0xcb7f442, 0xfbb28ff, 0x6ac53ca, 0x196142cc, 0x7bf0fa9, 0x957651,
1.146 + 0x4e0f215, 0xed439f8, 0x3f46bd5, 0x5ace82f, 0x110916b6, 0x6db078, 0xffd7d57, 0xf2ecaac, 0xca86dec,
1.147 + 0x15d6b2da, 0x965ecc9, 0x1c92b4c2, 0x1f3811, 0x1cb080f5, 0x2d8b804, 0x19d1c12d, 0xf20bd46, 0x1951fa7,
1.148 + 0xa3656c3, 0x523a425, 0xfcd0692, 0xd44ddc8, 0x131f0f5b, 0xaf80e4a, 0xcd9fc74, 0x99bb618, 0x2db944c,
1.149 + 0xa673090, 0x1c210e1, 0x178c8d23, 0x1474383, 0x10b8743d, 0x985a55b, 0x2e74779, 0x576138, 0x9587927,
1.150 + 0x133130fa, 0xbe05516, 0x9f4d619, 0xbb62570, 0x99ec591, 0xd9468fe, 0x1d07782d, 0xfc72e0b, 0x701b298,
1.151 + 0x1863863b, 0x85954b8, 0x121a0c36, 0x9e7fedf, 0xf64b429, 0x9b9d71e, 0x14e2f5d8, 0xf858d3a, 0x942eea8,
1.152 + 0xda5b765, 0x6edafff, 0xa9d18cc, 0xc65e4ba, 0x1c747e86, 0xe4ea915, 0x1981d7a1, 0x8395659, 0x52ed4e2,
1.153 + 0x87d43b7, 0x37ab11b, 0x19d292ce, 0xf8d4692, 0x18c3053f, 0x8863e13, 0x4c146c0, 0x6bdf55a, 0x4e4457d,
1.154 + 0x16152289, 0xac78ec2, 0x1a59c5a2, 0x2028b97, 0x71c2d01, 0x295851f, 0x404747b, 0x878558d, 0x7d29aa4,
1.155 + 0x13d8341f, 0x8daefd7, 0x139c972d, 0x6b7ea75, 0xd4a9dde, 0xff163d8, 0x81d55d7, 0xa5bef68, 0xb7b30d8,
1.156 + 0xbe73d6f, 0xaa88141, 0xd976c81, 0x7e7a9cc, 0x18beb771, 0xd773cbd, 0x13f51951, 0x9d0c177, 0x1c49a78,
1.157 +};
1.158 +
1.159 +/* Field element operations:
1.160 + */
1.161 +
1.162 +/* NON_ZERO_TO_ALL_ONES returns:
1.163 + * 0xffffffff for 0 < x <= 2**31
1.164 + * 0 for x == 0 or x > 2**31.
1.165 + *
1.166 + * x must be a u32 or an equivalent type such as limb.
1.167 + */
1.168 +#define NON_ZERO_TO_ALL_ONES(x) ((((u32)(x) - 1) >> 31) - 1)
1.169 +
1.170 +/* felem_reduce_carry adds a multiple of p in order to cancel |carry|,
1.171 + * which is a term at 2**257.
1.172 + *
1.173 + * On entry: carry < 2**3, inout[0,2,...] < 2**29, inout[1,3,...] < 2**28.
1.174 + * On exit: inout[0,2,..] < 2**30, inout[1,3,...] < 2**29.
1.175 + */
1.176 +static void felem_reduce_carry(felem inout, limb carry)
1.177 +{
1.178 + const u32 carry_mask = NON_ZERO_TO_ALL_ONES(carry);
1.179 +
1.180 + inout[0] += carry << 1;
1.181 + inout[3] += 0x10000000 & carry_mask;
1.182 + /* carry < 2**3 thus (carry << 11) < 2**14 and we added 2**28 in the
1.183 + * previous line therefore this doesn't underflow.
1.184 + */
1.185 + inout[3] -= carry << 11;
1.186 + inout[4] += (0x20000000 - 1) & carry_mask;
1.187 + inout[5] += (0x10000000 - 1) & carry_mask;
1.188 + inout[6] += (0x20000000 - 1) & carry_mask;
1.189 + inout[6] -= carry << 22;
1.190 + /* This may underflow if carry is non-zero but, if so, we'll fix it in the
1.191 + * next line.
1.192 + */
1.193 + inout[7] -= 1 & carry_mask;
1.194 + inout[7] += carry << 25;
1.195 +}
1.196 +
1.197 +/* felem_sum sets out = in+in2.
1.198 + *
1.199 + * On entry, in[i]+in2[i] must not overflow a 32-bit word.
1.200 + * On exit: out[0,2,...] < 2**30, out[1,3,...] < 2**29
1.201 + */
1.202 +static void felem_sum(felem out, const felem in, const felem in2)
1.203 +{
1.204 + limb carry = 0;
1.205 + unsigned int i;
1.206 + for (i = 0;; i++) {
1.207 + out[i] = in[i] + in2[i];
1.208 + out[i] += carry;
1.209 + carry = out[i] >> 29;
1.210 + out[i] &= kBottom29Bits;
1.211 +
1.212 + i++;
1.213 + if (i == NLIMBS)
1.214 + break;
1.215 +
1.216 + out[i] = in[i] + in2[i];
1.217 + out[i] += carry;
1.218 + carry = out[i] >> 28;
1.219 + out[i] &= kBottom28Bits;
1.220 + }
1.221 +
1.222 + felem_reduce_carry(out, carry);
1.223 +}
1.224 +
1.225 +#define two31m3 (((limb)1) << 31) - (((limb)1) << 3)
1.226 +#define two30m2 (((limb)1) << 30) - (((limb)1) << 2)
1.227 +#define two30p13m2 (((limb)1) << 30) + (((limb)1) << 13) - (((limb)1) << 2)
1.228 +#define two31m2 (((limb)1) << 31) - (((limb)1) << 2)
1.229 +#define two31p24m2 (((limb)1) << 31) + (((limb)1) << 24) - (((limb)1) << 2)
1.230 +#define two30m27m2 (((limb)1) << 30) - (((limb)1) << 27) - (((limb)1) << 2)
1.231 +
1.232 +/* zero31 is 0 mod p.
1.233 + */
1.234 +static const felem zero31 = {
1.235 + two31m3, two30m2, two31m2, two30p13m2,
1.236 + two31m2, two30m2, two31p24m2, two30m27m2,
1.237 + two31m2
1.238 +};
1.239 +
1.240 +/* felem_diff sets out = in-in2.
1.241 + *
1.242 + * On entry: in[0,2,...] < 2**30, in[1,3,...] < 2**29 and
1.243 + * in2[0,2,...] < 2**30, in2[1,3,...] < 2**29.
1.244 + * On exit: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.245 + */
1.246 +static void felem_diff(felem out, const felem in, const felem in2)
1.247 +{
1.248 + limb carry = 0;
1.249 + unsigned int i;
1.250 +
1.251 + for (i = 0;; i++) {
1.252 + out[i] = in[i] - in2[i];
1.253 + out[i] += zero31[i];
1.254 + out[i] += carry;
1.255 + carry = out[i] >> 29;
1.256 + out[i] &= kBottom29Bits;
1.257 +
1.258 + i++;
1.259 + if (i == NLIMBS)
1.260 + break;
1.261 +
1.262 + out[i] = in[i] - in2[i];
1.263 + out[i] += zero31[i];
1.264 + out[i] += carry;
1.265 + carry = out[i] >> 28;
1.266 + out[i] &= kBottom28Bits;
1.267 + }
1.268 +
1.269 + felem_reduce_carry(out, carry);
1.270 +}
1.271 +
1.272 +/* felem_reduce_degree sets out = tmp/R mod p where tmp contains 64-bit words
1.273 + * with the same 29,28,... bit positions as an felem.
1.274 + *
1.275 + * The values in felems are in Montgomery form: x*R mod p where R = 2**257.
1.276 + * Since we just multiplied two Montgomery values together, the result is
1.277 + * x*y*R*R mod p. We wish to divide by R in order for the result also to be
1.278 + * in Montgomery form.
1.279 + *
1.280 + * On entry: tmp[i] < 2**64
1.281 + * On exit: out[0,2,...] < 2**30, out[1,3,...] < 2**29
1.282 + */
1.283 +static void felem_reduce_degree(felem out, u64 tmp[17])
1.284 +{
1.285 + /* The following table may be helpful when reading this code:
1.286 + *
1.287 + * Limb number: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10...
1.288 + * Width (bits): 29| 28| 29| 28| 29| 28| 29| 28| 29| 28| 29
1.289 + * Start bit: 0 | 29| 57| 86|114|143|171|200|228|257|285
1.290 + * (odd phase): 0 | 28| 57| 85|114|142|171|199|228|256|285
1.291 + */
1.292 + limb tmp2[18], carry, x, xMask;
1.293 + unsigned int i;
1.294 +
1.295 + /* tmp contains 64-bit words with the same 29,28,29-bit positions as an
1.296 + * felem. So the top of an element of tmp might overlap with another
1.297 + * element two positions down. The following loop eliminates this
1.298 + * overlap.
1.299 + */
1.300 + tmp2[0] = tmp[0] & kBottom29Bits;
1.301 +
1.302 + /* In the following we use "(limb) tmp[x]" and "(limb) (tmp[x]>>32)" to try
1.303 + * and hint to the compiler that it can do a single-word shift by selecting
1.304 + * the right register rather than doing a double-word shift and truncating
1.305 + * afterwards.
1.306 + */
1.307 + tmp2[1] = ((limb) tmp[0]) >> 29;
1.308 + tmp2[1] |= (((limb) (tmp[0] >> 32)) << 3) & kBottom28Bits;
1.309 + tmp2[1] += ((limb) tmp[1]) & kBottom28Bits;
1.310 + carry = tmp2[1] >> 28;
1.311 + tmp2[1] &= kBottom28Bits;
1.312 +
1.313 + for (i = 2; i < 17; i++) {
1.314 + tmp2[i] = ((limb) (tmp[i - 2] >> 32)) >> 25;
1.315 + tmp2[i] += ((limb) (tmp[i - 1])) >> 28;
1.316 + tmp2[i] += (((limb) (tmp[i - 1] >> 32)) << 4) & kBottom29Bits;
1.317 + tmp2[i] += ((limb) tmp[i]) & kBottom29Bits;
1.318 + tmp2[i] += carry;
1.319 + carry = tmp2[i] >> 29;
1.320 + tmp2[i] &= kBottom29Bits;
1.321 +
1.322 + i++;
1.323 + if (i == 17)
1.324 + break;
1.325 + tmp2[i] = ((limb) (tmp[i - 2] >> 32)) >> 25;
1.326 + tmp2[i] += ((limb) (tmp[i - 1])) >> 29;
1.327 + tmp2[i] += (((limb) (tmp[i - 1] >> 32)) << 3) & kBottom28Bits;
1.328 + tmp2[i] += ((limb) tmp[i]) & kBottom28Bits;
1.329 + tmp2[i] += carry;
1.330 + carry = tmp2[i] >> 28;
1.331 + tmp2[i] &= kBottom28Bits;
1.332 + }
1.333 +
1.334 + tmp2[17] = ((limb) (tmp[15] >> 32)) >> 25;
1.335 + tmp2[17] += ((limb) (tmp[16])) >> 29;
1.336 + tmp2[17] += (((limb) (tmp[16] >> 32)) << 3);
1.337 + tmp2[17] += carry;
1.338 +
1.339 + /* Montgomery elimination of terms:
1.340 + *
1.341 + * Since R is 2**257, we can divide by R with a bitwise shift if we can
1.342 + * ensure that the right-most 257 bits are all zero. We can make that true
1.343 + * by adding multiplies of p without affecting the value.
1.344 + *
1.345 + * So we eliminate limbs from right to left. Since the bottom 29 bits of p
1.346 + * are all ones, then by adding tmp2[0]*p to tmp2 we'll make tmp2[0] == 0.
1.347 + * We can do that for 8 further limbs and then right shift to eliminate the
1.348 + * extra factor of R.
1.349 + */
1.350 + for (i = 0;; i += 2) {
1.351 + tmp2[i + 1] += tmp2[i] >> 29;
1.352 + x = tmp2[i] & kBottom29Bits;
1.353 + xMask = NON_ZERO_TO_ALL_ONES(x);
1.354 + tmp2[i] = 0;
1.355 +
1.356 + /* The bounds calculations for this loop are tricky. Each iteration of
1.357 + * the loop eliminates two words by adding values to words to their
1.358 + * right.
1.359 + *
1.360 + * The following table contains the amounts added to each word (as an
1.361 + * offset from the value of i at the top of the loop). The amounts are
1.362 + * accounted for from the first and second half of the loop separately
1.363 + * and are written as, for example, 28 to mean a value <2**28.
1.364 + *
1.365 + * Word: 3 4 5 6 7 8 9 10
1.366 + * Added in top half: 28 11 29 21 29 28
1.367 + * 28 29
1.368 + * 29
1.369 + * Added in bottom half: 29 10 28 21 28 28
1.370 + * 29
1.371 + *
1.372 + * The value that is currently offset 7 will be offset 5 for the next
1.373 + * iteration and then offset 3 for the iteration after that. Therefore
1.374 + * the total value added will be the values added at 7, 5 and 3.
1.375 + *
1.376 + * The following table accumulates these values. The sums at the bottom
1.377 + * are written as, for example, 29+28, to mean a value < 2**29+2**28.
1.378 + *
1.379 + * Word: 3 4 5 6 7 8 9 10 11 12 13
1.380 + * 28 11 10 29 21 29 28 28 28 28 28
1.381 + * 29 28 11 28 29 28 29 28 29 28
1.382 + * 29 28 21 21 29 21 29 21
1.383 + * 10 29 28 21 28 21 28
1.384 + * 28 29 28 29 28 29 28
1.385 + * 11 10 29 10 29 10
1.386 + * 29 28 11 28 11
1.387 + * 29 29
1.388 + * --------------------------------------------
1.389 + * 30+ 31+ 30+ 31+ 30+
1.390 + * 28+ 29+ 28+ 29+ 21+
1.391 + * 21+ 28+ 21+ 28+ 10
1.392 + * 10 21+ 10 21+
1.393 + * 11 11
1.394 + *
1.395 + * So the greatest amount is added to tmp2[10] and tmp2[12]. If
1.396 + * tmp2[10/12] has an initial value of <2**29, then the maximum value
1.397 + * will be < 2**31 + 2**30 + 2**28 + 2**21 + 2**11, which is < 2**32,
1.398 + * as required.
1.399 + */
1.400 + tmp2[i + 3] += (x << 10) & kBottom28Bits;
1.401 + tmp2[i + 4] += (x >> 18);
1.402 +
1.403 + tmp2[i + 6] += (x << 21) & kBottom29Bits;
1.404 + tmp2[i + 7] += x >> 8;
1.405 +
1.406 + /* At position 200, which is the starting bit position for word 7, we
1.407 + * have a factor of 0xf000000 = 2**28 - 2**24.
1.408 + */
1.409 + tmp2[i + 7] += 0x10000000 & xMask;
1.410 + /* Word 7 is 28 bits wide, so the 2**28 term exactly hits word 8. */
1.411 + tmp2[i + 8] += (x - 1) & xMask;
1.412 + tmp2[i + 7] -= (x << 24) & kBottom28Bits;
1.413 + tmp2[i + 8] -= x >> 4;
1.414 +
1.415 + tmp2[i + 8] += 0x20000000 & xMask;
1.416 + tmp2[i + 8] -= x;
1.417 + tmp2[i + 8] += (x << 28) & kBottom29Bits;
1.418 + tmp2[i + 9] += ((x >> 1) - 1) & xMask;
1.419 +
1.420 + if (i+1 == NLIMBS)
1.421 + break;
1.422 + tmp2[i + 2] += tmp2[i + 1] >> 28;
1.423 + x = tmp2[i + 1] & kBottom28Bits;
1.424 + xMask = NON_ZERO_TO_ALL_ONES(x);
1.425 + tmp2[i + 1] = 0;
1.426 +
1.427 + tmp2[i + 4] += (x << 11) & kBottom29Bits;
1.428 + tmp2[i + 5] += (x >> 18);
1.429 +
1.430 + tmp2[i + 7] += (x << 21) & kBottom28Bits;
1.431 + tmp2[i + 8] += x >> 7;
1.432 +
1.433 + /* At position 199, which is the starting bit of the 8th word when
1.434 + * dealing with a context starting on an odd word, we have a factor of
1.435 + * 0x1e000000 = 2**29 - 2**25. Since we have not updated i, the 8th
1.436 + * word from i+1 is i+8.
1.437 + */
1.438 + tmp2[i + 8] += 0x20000000 & xMask;
1.439 + tmp2[i + 9] += (x - 1) & xMask;
1.440 + tmp2[i + 8] -= (x << 25) & kBottom29Bits;
1.441 + tmp2[i + 9] -= x >> 4;
1.442 +
1.443 + tmp2[i + 9] += 0x10000000 & xMask;
1.444 + tmp2[i + 9] -= x;
1.445 + tmp2[i + 10] += (x - 1) & xMask;
1.446 + }
1.447 +
1.448 + /* We merge the right shift with a carry chain. The words above 2**257 have
1.449 + * widths of 28,29,... which we need to correct when copying them down.
1.450 + */
1.451 + carry = 0;
1.452 + for (i = 0; i < 8; i++) {
1.453 + /* The maximum value of tmp2[i + 9] occurs on the first iteration and
1.454 + * is < 2**30+2**29+2**28. Adding 2**29 (from tmp2[i + 10]) is
1.455 + * therefore safe.
1.456 + */
1.457 + out[i] = tmp2[i + 9];
1.458 + out[i] += carry;
1.459 + out[i] += (tmp2[i + 10] << 28) & kBottom29Bits;
1.460 + carry = out[i] >> 29;
1.461 + out[i] &= kBottom29Bits;
1.462 +
1.463 + i++;
1.464 + out[i] = tmp2[i + 9] >> 1;
1.465 + out[i] += carry;
1.466 + carry = out[i] >> 28;
1.467 + out[i] &= kBottom28Bits;
1.468 + }
1.469 +
1.470 + out[8] = tmp2[17];
1.471 + out[8] += carry;
1.472 + carry = out[8] >> 29;
1.473 + out[8] &= kBottom29Bits;
1.474 +
1.475 + felem_reduce_carry(out, carry);
1.476 +}
1.477 +
1.478 +/* felem_square sets out=in*in.
1.479 + *
1.480 + * On entry: in[0,2,...] < 2**30, in[1,3,...] < 2**29.
1.481 + * On exit: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.482 + */
1.483 +static void felem_square(felem out, const felem in)
1.484 +{
1.485 + u64 tmp[17];
1.486 +
1.487 + tmp[0] = ((u64) in[0]) * in[0];
1.488 + tmp[1] = ((u64) in[0]) * (in[1] << 1);
1.489 + tmp[2] = ((u64) in[0]) * (in[2] << 1) +
1.490 + ((u64) in[1]) * (in[1] << 1);
1.491 + tmp[3] = ((u64) in[0]) * (in[3] << 1) +
1.492 + ((u64) in[1]) * (in[2] << 1);
1.493 + tmp[4] = ((u64) in[0]) * (in[4] << 1) +
1.494 + ((u64) in[1]) * (in[3] << 2) +
1.495 + ((u64) in[2]) * in[2];
1.496 + tmp[5] = ((u64) in[0]) * (in[5] << 1) +
1.497 + ((u64) in[1]) * (in[4] << 1) +
1.498 + ((u64) in[2]) * (in[3] << 1);
1.499 + tmp[6] = ((u64) in[0]) * (in[6] << 1) +
1.500 + ((u64) in[1]) * (in[5] << 2) +
1.501 + ((u64) in[2]) * (in[4] << 1) +
1.502 + ((u64) in[3]) * (in[3] << 1);
1.503 + tmp[7] = ((u64) in[0]) * (in[7] << 1) +
1.504 + ((u64) in[1]) * (in[6] << 1) +
1.505 + ((u64) in[2]) * (in[5] << 1) +
1.506 + ((u64) in[3]) * (in[4] << 1);
1.507 + /* tmp[8] has the greatest value of 2**61 + 2**60 + 2**61 + 2**60 + 2**60,
1.508 + * which is < 2**64 as required.
1.509 + */
1.510 + tmp[8] = ((u64) in[0]) * (in[8] << 1) +
1.511 + ((u64) in[1]) * (in[7] << 2) +
1.512 + ((u64) in[2]) * (in[6] << 1) +
1.513 + ((u64) in[3]) * (in[5] << 2) +
1.514 + ((u64) in[4]) * in[4];
1.515 + tmp[9] = ((u64) in[1]) * (in[8] << 1) +
1.516 + ((u64) in[2]) * (in[7] << 1) +
1.517 + ((u64) in[3]) * (in[6] << 1) +
1.518 + ((u64) in[4]) * (in[5] << 1);
1.519 + tmp[10] = ((u64) in[2]) * (in[8] << 1) +
1.520 + ((u64) in[3]) * (in[7] << 2) +
1.521 + ((u64) in[4]) * (in[6] << 1) +
1.522 + ((u64) in[5]) * (in[5] << 1);
1.523 + tmp[11] = ((u64) in[3]) * (in[8] << 1) +
1.524 + ((u64) in[4]) * (in[7] << 1) +
1.525 + ((u64) in[5]) * (in[6] << 1);
1.526 + tmp[12] = ((u64) in[4]) * (in[8] << 1) +
1.527 + ((u64) in[5]) * (in[7] << 2) +
1.528 + ((u64) in[6]) * in[6];
1.529 + tmp[13] = ((u64) in[5]) * (in[8] << 1) +
1.530 + ((u64) in[6]) * (in[7] << 1);
1.531 + tmp[14] = ((u64) in[6]) * (in[8] << 1) +
1.532 + ((u64) in[7]) * (in[7] << 1);
1.533 + tmp[15] = ((u64) in[7]) * (in[8] << 1);
1.534 + tmp[16] = ((u64) in[8]) * in[8];
1.535 +
1.536 + felem_reduce_degree(out, tmp);
1.537 +}
1.538 +
1.539 +/* felem_mul sets out=in*in2.
1.540 + *
1.541 + * On entry: in[0,2,...] < 2**30, in[1,3,...] < 2**29 and
1.542 + * in2[0,2,...] < 2**30, in2[1,3,...] < 2**29.
1.543 + * On exit: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.544 + */
1.545 +static void felem_mul(felem out, const felem in, const felem in2)
1.546 +{
1.547 + u64 tmp[17];
1.548 +
1.549 + tmp[0] = ((u64) in[0]) * in2[0];
1.550 + tmp[1] = ((u64) in[0]) * (in2[1] << 0) +
1.551 + ((u64) in[1]) * (in2[0] << 0);
1.552 + tmp[2] = ((u64) in[0]) * (in2[2] << 0) +
1.553 + ((u64) in[1]) * (in2[1] << 1) +
1.554 + ((u64) in[2]) * (in2[0] << 0);
1.555 + tmp[3] = ((u64) in[0]) * (in2[3] << 0) +
1.556 + ((u64) in[1]) * (in2[2] << 0) +
1.557 + ((u64) in[2]) * (in2[1] << 0) +
1.558 + ((u64) in[3]) * (in2[0] << 0);
1.559 + tmp[4] = ((u64) in[0]) * (in2[4] << 0) +
1.560 + ((u64) in[1]) * (in2[3] << 1) +
1.561 + ((u64) in[2]) * (in2[2] << 0) +
1.562 + ((u64) in[3]) * (in2[1] << 1) +
1.563 + ((u64) in[4]) * (in2[0] << 0);
1.564 + tmp[5] = ((u64) in[0]) * (in2[5] << 0) +
1.565 + ((u64) in[1]) * (in2[4] << 0) +
1.566 + ((u64) in[2]) * (in2[3] << 0) +
1.567 + ((u64) in[3]) * (in2[2] << 0) +
1.568 + ((u64) in[4]) * (in2[1] << 0) +
1.569 + ((u64) in[5]) * (in2[0] << 0);
1.570 + tmp[6] = ((u64) in[0]) * (in2[6] << 0) +
1.571 + ((u64) in[1]) * (in2[5] << 1) +
1.572 + ((u64) in[2]) * (in2[4] << 0) +
1.573 + ((u64) in[3]) * (in2[3] << 1) +
1.574 + ((u64) in[4]) * (in2[2] << 0) +
1.575 + ((u64) in[5]) * (in2[1] << 1) +
1.576 + ((u64) in[6]) * (in2[0] << 0);
1.577 + tmp[7] = ((u64) in[0]) * (in2[7] << 0) +
1.578 + ((u64) in[1]) * (in2[6] << 0) +
1.579 + ((u64) in[2]) * (in2[5] << 0) +
1.580 + ((u64) in[3]) * (in2[4] << 0) +
1.581 + ((u64) in[4]) * (in2[3] << 0) +
1.582 + ((u64) in[5]) * (in2[2] << 0) +
1.583 + ((u64) in[6]) * (in2[1] << 0) +
1.584 + ((u64) in[7]) * (in2[0] << 0);
1.585 + /* tmp[8] has the greatest value but doesn't overflow. See logic in
1.586 + * felem_square.
1.587 + */
1.588 + tmp[8] = ((u64) in[0]) * (in2[8] << 0) +
1.589 + ((u64) in[1]) * (in2[7] << 1) +
1.590 + ((u64) in[2]) * (in2[6] << 0) +
1.591 + ((u64) in[3]) * (in2[5] << 1) +
1.592 + ((u64) in[4]) * (in2[4] << 0) +
1.593 + ((u64) in[5]) * (in2[3] << 1) +
1.594 + ((u64) in[6]) * (in2[2] << 0) +
1.595 + ((u64) in[7]) * (in2[1] << 1) +
1.596 + ((u64) in[8]) * (in2[0] << 0);
1.597 + tmp[9] = ((u64) in[1]) * (in2[8] << 0) +
1.598 + ((u64) in[2]) * (in2[7] << 0) +
1.599 + ((u64) in[3]) * (in2[6] << 0) +
1.600 + ((u64) in[4]) * (in2[5] << 0) +
1.601 + ((u64) in[5]) * (in2[4] << 0) +
1.602 + ((u64) in[6]) * (in2[3] << 0) +
1.603 + ((u64) in[7]) * (in2[2] << 0) +
1.604 + ((u64) in[8]) * (in2[1] << 0);
1.605 + tmp[10] = ((u64) in[2]) * (in2[8] << 0) +
1.606 + ((u64) in[3]) * (in2[7] << 1) +
1.607 + ((u64) in[4]) * (in2[6] << 0) +
1.608 + ((u64) in[5]) * (in2[5] << 1) +
1.609 + ((u64) in[6]) * (in2[4] << 0) +
1.610 + ((u64) in[7]) * (in2[3] << 1) +
1.611 + ((u64) in[8]) * (in2[2] << 0);
1.612 + tmp[11] = ((u64) in[3]) * (in2[8] << 0) +
1.613 + ((u64) in[4]) * (in2[7] << 0) +
1.614 + ((u64) in[5]) * (in2[6] << 0) +
1.615 + ((u64) in[6]) * (in2[5] << 0) +
1.616 + ((u64) in[7]) * (in2[4] << 0) +
1.617 + ((u64) in[8]) * (in2[3] << 0);
1.618 + tmp[12] = ((u64) in[4]) * (in2[8] << 0) +
1.619 + ((u64) in[5]) * (in2[7] << 1) +
1.620 + ((u64) in[6]) * (in2[6] << 0) +
1.621 + ((u64) in[7]) * (in2[5] << 1) +
1.622 + ((u64) in[8]) * (in2[4] << 0);
1.623 + tmp[13] = ((u64) in[5]) * (in2[8] << 0) +
1.624 + ((u64) in[6]) * (in2[7] << 0) +
1.625 + ((u64) in[7]) * (in2[6] << 0) +
1.626 + ((u64) in[8]) * (in2[5] << 0);
1.627 + tmp[14] = ((u64) in[6]) * (in2[8] << 0) +
1.628 + ((u64) in[7]) * (in2[7] << 1) +
1.629 + ((u64) in[8]) * (in2[6] << 0);
1.630 + tmp[15] = ((u64) in[7]) * (in2[8] << 0) +
1.631 + ((u64) in[8]) * (in2[7] << 0);
1.632 + tmp[16] = ((u64) in[8]) * (in2[8] << 0);
1.633 +
1.634 + felem_reduce_degree(out, tmp);
1.635 +}
1.636 +
1.637 +static void felem_assign(felem out, const felem in)
1.638 +{
1.639 + memcpy(out, in, sizeof(felem));
1.640 +}
1.641 +
1.642 +/* felem_inv calculates |out| = |in|^{-1}
1.643 + *
1.644 + * Based on Fermat's Little Theorem:
1.645 + * a^p = a (mod p)
1.646 + * a^{p-1} = 1 (mod p)
1.647 + * a^{p-2} = a^{-1} (mod p)
1.648 + */
1.649 +static void felem_inv(felem out, const felem in)
1.650 +{
1.651 + felem ftmp, ftmp2;
1.652 + /* each e_I will hold |in|^{2^I - 1} */
1.653 + felem e2, e4, e8, e16, e32, e64;
1.654 + unsigned int i;
1.655 +
1.656 + felem_square(ftmp, in); /* 2^1 */
1.657 + felem_mul(ftmp, in, ftmp); /* 2^2 - 2^0 */
1.658 + felem_assign(e2, ftmp);
1.659 + felem_square(ftmp, ftmp); /* 2^3 - 2^1 */
1.660 + felem_square(ftmp, ftmp); /* 2^4 - 2^2 */
1.661 + felem_mul(ftmp, ftmp, e2); /* 2^4 - 2^0 */
1.662 + felem_assign(e4, ftmp);
1.663 + felem_square(ftmp, ftmp); /* 2^5 - 2^1 */
1.664 + felem_square(ftmp, ftmp); /* 2^6 - 2^2 */
1.665 + felem_square(ftmp, ftmp); /* 2^7 - 2^3 */
1.666 + felem_square(ftmp, ftmp); /* 2^8 - 2^4 */
1.667 + felem_mul(ftmp, ftmp, e4); /* 2^8 - 2^0 */
1.668 + felem_assign(e8, ftmp);
1.669 + for (i = 0; i < 8; i++) {
1.670 + felem_square(ftmp, ftmp);
1.671 + } /* 2^16 - 2^8 */
1.672 + felem_mul(ftmp, ftmp, e8); /* 2^16 - 2^0 */
1.673 + felem_assign(e16, ftmp);
1.674 + for (i = 0; i < 16; i++) {
1.675 + felem_square(ftmp, ftmp);
1.676 + } /* 2^32 - 2^16 */
1.677 + felem_mul(ftmp, ftmp, e16); /* 2^32 - 2^0 */
1.678 + felem_assign(e32, ftmp);
1.679 + for (i = 0; i < 32; i++) {
1.680 + felem_square(ftmp, ftmp);
1.681 + } /* 2^64 - 2^32 */
1.682 + felem_assign(e64, ftmp);
1.683 + felem_mul(ftmp, ftmp, in); /* 2^64 - 2^32 + 2^0 */
1.684 + for (i = 0; i < 192; i++) {
1.685 + felem_square(ftmp, ftmp);
1.686 + } /* 2^256 - 2^224 + 2^192 */
1.687 +
1.688 + felem_mul(ftmp2, e64, e32); /* 2^64 - 2^0 */
1.689 + for (i = 0; i < 16; i++) {
1.690 + felem_square(ftmp2, ftmp2);
1.691 + } /* 2^80 - 2^16 */
1.692 + felem_mul(ftmp2, ftmp2, e16); /* 2^80 - 2^0 */
1.693 + for (i = 0; i < 8; i++) {
1.694 + felem_square(ftmp2, ftmp2);
1.695 + } /* 2^88 - 2^8 */
1.696 + felem_mul(ftmp2, ftmp2, e8); /* 2^88 - 2^0 */
1.697 + for (i = 0; i < 4; i++) {
1.698 + felem_square(ftmp2, ftmp2);
1.699 + } /* 2^92 - 2^4 */
1.700 + felem_mul(ftmp2, ftmp2, e4); /* 2^92 - 2^0 */
1.701 + felem_square(ftmp2, ftmp2); /* 2^93 - 2^1 */
1.702 + felem_square(ftmp2, ftmp2); /* 2^94 - 2^2 */
1.703 + felem_mul(ftmp2, ftmp2, e2); /* 2^94 - 2^0 */
1.704 + felem_square(ftmp2, ftmp2); /* 2^95 - 2^1 */
1.705 + felem_square(ftmp2, ftmp2); /* 2^96 - 2^2 */
1.706 + felem_mul(ftmp2, ftmp2, in); /* 2^96 - 3 */
1.707 +
1.708 + felem_mul(out, ftmp2, ftmp); /* 2^256 - 2^224 + 2^192 + 2^96 - 3 */
1.709 +}
1.710 +
1.711 +/* felem_scalar_3 sets out=3*out.
1.712 + *
1.713 + * On entry: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.714 + * On exit: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.715 + */
1.716 +static void felem_scalar_3(felem out)
1.717 +{
1.718 + limb carry = 0;
1.719 + unsigned int i;
1.720 +
1.721 + for (i = 0;; i++) {
1.722 + out[i] *= 3;
1.723 + out[i] += carry;
1.724 + carry = out[i] >> 29;
1.725 + out[i] &= kBottom29Bits;
1.726 +
1.727 + i++;
1.728 + if (i == NLIMBS)
1.729 + break;
1.730 +
1.731 + out[i] *= 3;
1.732 + out[i] += carry;
1.733 + carry = out[i] >> 28;
1.734 + out[i] &= kBottom28Bits;
1.735 + }
1.736 +
1.737 + felem_reduce_carry(out, carry);
1.738 +}
1.739 +
1.740 +/* felem_scalar_4 sets out=4*out.
1.741 + *
1.742 + * On entry: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.743 + * On exit: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.744 + */
1.745 +static void felem_scalar_4(felem out)
1.746 +{
1.747 + limb carry = 0, next_carry;
1.748 + unsigned int i;
1.749 +
1.750 + for (i = 0;; i++) {
1.751 + next_carry = out[i] >> 27;
1.752 + out[i] <<= 2;
1.753 + out[i] &= kBottom29Bits;
1.754 + out[i] += carry;
1.755 + carry = next_carry + (out[i] >> 29);
1.756 + out[i] &= kBottom29Bits;
1.757 +
1.758 + i++;
1.759 + if (i == NLIMBS)
1.760 + break;
1.761 + next_carry = out[i] >> 26;
1.762 + out[i] <<= 2;
1.763 + out[i] &= kBottom28Bits;
1.764 + out[i] += carry;
1.765 + carry = next_carry + (out[i] >> 28);
1.766 + out[i] &= kBottom28Bits;
1.767 + }
1.768 +
1.769 + felem_reduce_carry(out, carry);
1.770 +}
1.771 +
1.772 +/* felem_scalar_8 sets out=8*out.
1.773 + *
1.774 + * On entry: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.775 + * On exit: out[0,2,...] < 2**30, out[1,3,...] < 2**29.
1.776 + */
1.777 +static void felem_scalar_8(felem out)
1.778 +{
1.779 + limb carry = 0, next_carry;
1.780 + unsigned int i;
1.781 +
1.782 + for (i = 0;; i++) {
1.783 + next_carry = out[i] >> 26;
1.784 + out[i] <<= 3;
1.785 + out[i] &= kBottom29Bits;
1.786 + out[i] += carry;
1.787 + carry = next_carry + (out[i] >> 29);
1.788 + out[i] &= kBottom29Bits;
1.789 +
1.790 + i++;
1.791 + if (i == NLIMBS)
1.792 + break;
1.793 + next_carry = out[i] >> 25;
1.794 + out[i] <<= 3;
1.795 + out[i] &= kBottom28Bits;
1.796 + out[i] += carry;
1.797 + carry = next_carry + (out[i] >> 28);
1.798 + out[i] &= kBottom28Bits;
1.799 + }
1.800 +
1.801 + felem_reduce_carry(out, carry);
1.802 +}
1.803 +
1.804 +/* felem_is_zero_vartime returns 1 iff |in| == 0. It takes a variable amount of
1.805 + * time depending on the value of |in|.
1.806 + */
1.807 +static char felem_is_zero_vartime(const felem in)
1.808 +{
1.809 + limb carry;
1.810 + int i;
1.811 + limb tmp[NLIMBS];
1.812 + felem_assign(tmp, in);
1.813 +
1.814 + /* First, reduce tmp to a minimal form.
1.815 + */
1.816 + do {
1.817 + carry = 0;
1.818 + for (i = 0;; i++) {
1.819 + tmp[i] += carry;
1.820 + carry = tmp[i] >> 29;
1.821 + tmp[i] &= kBottom29Bits;
1.822 +
1.823 + i++;
1.824 + if (i == NLIMBS)
1.825 + break;
1.826 +
1.827 + tmp[i] += carry;
1.828 + carry = tmp[i] >> 28;
1.829 + tmp[i] &= kBottom28Bits;
1.830 + }
1.831 +
1.832 + felem_reduce_carry(tmp, carry);
1.833 + } while (carry);
1.834 +
1.835 + /* tmp < 2**257, so the only possible zero values are 0, p and 2p.
1.836 + */
1.837 + return memcmp(tmp, kZero, sizeof(tmp)) == 0 ||
1.838 + memcmp(tmp, kP, sizeof(tmp)) == 0 ||
1.839 + memcmp(tmp, k2P, sizeof(tmp)) == 0;
1.840 +}
1.841 +
1.842 +/* Group operations:
1.843 + *
1.844 + * Elements of the elliptic curve group are represented in Jacobian
1.845 + * coordinates: (x, y, z). An affine point (x', y') is x'=x/z**2, y'=y/z**3 in
1.846 + * Jacobian form.
1.847 + */
1.848 +
1.849 +/* point_double sets {x_out,y_out,z_out} = 2*{x,y,z}.
1.850 + *
1.851 + * See http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#doubling-dbl-2009-l
1.852 + */
1.853 +static void point_double(felem x_out, felem y_out, felem z_out,
1.854 + const felem x, const felem y, const felem z)
1.855 +{
1.856 + felem delta, gamma, alpha, beta, tmp, tmp2;
1.857 +
1.858 + felem_square(delta, z);
1.859 + felem_square(gamma, y);
1.860 + felem_mul(beta, x, gamma);
1.861 +
1.862 + felem_sum(tmp, x, delta);
1.863 + felem_diff(tmp2, x, delta);
1.864 + felem_mul(alpha, tmp, tmp2);
1.865 + felem_scalar_3(alpha);
1.866 +
1.867 + felem_sum(tmp, y, z);
1.868 + felem_square(tmp, tmp);
1.869 + felem_diff(tmp, tmp, gamma);
1.870 + felem_diff(z_out, tmp, delta);
1.871 +
1.872 + felem_scalar_4(beta);
1.873 + felem_square(x_out, alpha);
1.874 + felem_diff(x_out, x_out, beta);
1.875 + felem_diff(x_out, x_out, beta);
1.876 +
1.877 + felem_diff(tmp, beta, x_out);
1.878 + felem_mul(tmp, alpha, tmp);
1.879 + felem_square(tmp2, gamma);
1.880 + felem_scalar_8(tmp2);
1.881 + felem_diff(y_out, tmp, tmp2);
1.882 +}
1.883 +
1.884 +/* point_add_mixed sets {x_out,y_out,z_out} = {x1,y1,z1} + {x2,y2,1}.
1.885 + * (i.e. the second point is affine.)
1.886 + *
1.887 + * See http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl
1.888 + *
1.889 + * Note that this function does not handle P+P, infinity+P nor P+infinity
1.890 + * correctly.
1.891 + */
1.892 +static void point_add_mixed(felem x_out, felem y_out, felem z_out,
1.893 + const felem x1, const felem y1, const felem z1,
1.894 + const felem x2, const felem y2)
1.895 +{
1.896 + felem z1z1, z1z1z1, s2, u2, h, i, j, r, rr, v, tmp;
1.897 +
1.898 + felem_square(z1z1, z1);
1.899 + felem_sum(tmp, z1, z1);
1.900 +
1.901 + felem_mul(u2, x2, z1z1);
1.902 + felem_mul(z1z1z1, z1, z1z1);
1.903 + felem_mul(s2, y2, z1z1z1);
1.904 + felem_diff(h, u2, x1);
1.905 + felem_sum(i, h, h);
1.906 + felem_square(i, i);
1.907 + felem_mul(j, h, i);
1.908 + felem_diff(r, s2, y1);
1.909 + felem_sum(r, r, r);
1.910 + felem_mul(v, x1, i);
1.911 +
1.912 + felem_mul(z_out, tmp, h);
1.913 + felem_square(rr, r);
1.914 + felem_diff(x_out, rr, j);
1.915 + felem_diff(x_out, x_out, v);
1.916 + felem_diff(x_out, x_out, v);
1.917 +
1.918 + felem_diff(tmp, v, x_out);
1.919 + felem_mul(y_out, tmp, r);
1.920 + felem_mul(tmp, y1, j);
1.921 + felem_diff(y_out, y_out, tmp);
1.922 + felem_diff(y_out, y_out, tmp);
1.923 +}
1.924 +
1.925 +/* point_add sets {x_out,y_out,z_out} = {x1,y1,z1} + {x2,y2,z2}.
1.926 + *
1.927 + * See http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl
1.928 + *
1.929 + * Note that this function does not handle P+P, infinity+P nor P+infinity
1.930 + * correctly.
1.931 + */
1.932 +static void point_add(felem x_out, felem y_out, felem z_out,
1.933 + const felem x1, const felem y1, const felem z1,
1.934 + const felem x2, const felem y2, const felem z2)
1.935 +{
1.936 + felem z1z1, z1z1z1, z2z2, z2z2z2, s1, s2, u1, u2, h, i, j, r, rr, v, tmp;
1.937 +
1.938 + felem_square(z1z1, z1);
1.939 + felem_square(z2z2, z2);
1.940 + felem_mul(u1, x1, z2z2);
1.941 +
1.942 + felem_sum(tmp, z1, z2);
1.943 + felem_square(tmp, tmp);
1.944 + felem_diff(tmp, tmp, z1z1);
1.945 + felem_diff(tmp, tmp, z2z2);
1.946 +
1.947 + felem_mul(z2z2z2, z2, z2z2);
1.948 + felem_mul(s1, y1, z2z2z2);
1.949 +
1.950 + felem_mul(u2, x2, z1z1);
1.951 + felem_mul(z1z1z1, z1, z1z1);
1.952 + felem_mul(s2, y2, z1z1z1);
1.953 + felem_diff(h, u2, u1);
1.954 + felem_sum(i, h, h);
1.955 + felem_square(i, i);
1.956 + felem_mul(j, h, i);
1.957 + felem_diff(r, s2, s1);
1.958 + felem_sum(r, r, r);
1.959 + felem_mul(v, u1, i);
1.960 +
1.961 + felem_mul(z_out, tmp, h);
1.962 + felem_square(rr, r);
1.963 + felem_diff(x_out, rr, j);
1.964 + felem_diff(x_out, x_out, v);
1.965 + felem_diff(x_out, x_out, v);
1.966 +
1.967 + felem_diff(tmp, v, x_out);
1.968 + felem_mul(y_out, tmp, r);
1.969 + felem_mul(tmp, s1, j);
1.970 + felem_diff(y_out, y_out, tmp);
1.971 + felem_diff(y_out, y_out, tmp);
1.972 +}
1.973 +
1.974 +/* point_add_or_double_vartime sets {x_out,y_out,z_out} = {x1,y1,z1} +
1.975 + * {x2,y2,z2}.
1.976 + *
1.977 + * See http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl
1.978 + *
1.979 + * This function handles the case where {x1,y1,z1}={x2,y2,z2}.
1.980 + */
1.981 +static void point_add_or_double_vartime(
1.982 + felem x_out, felem y_out, felem z_out,
1.983 + const felem x1, const felem y1, const felem z1,
1.984 + const felem x2, const felem y2, const felem z2)
1.985 +{
1.986 + felem z1z1, z1z1z1, z2z2, z2z2z2, s1, s2, u1, u2, h, i, j, r, rr, v, tmp;
1.987 + char x_equal, y_equal;
1.988 +
1.989 + felem_square(z1z1, z1);
1.990 + felem_square(z2z2, z2);
1.991 + felem_mul(u1, x1, z2z2);
1.992 +
1.993 + felem_sum(tmp, z1, z2);
1.994 + felem_square(tmp, tmp);
1.995 + felem_diff(tmp, tmp, z1z1);
1.996 + felem_diff(tmp, tmp, z2z2);
1.997 +
1.998 + felem_mul(z2z2z2, z2, z2z2);
1.999 + felem_mul(s1, y1, z2z2z2);
1.1000 +
1.1001 + felem_mul(u2, x2, z1z1);
1.1002 + felem_mul(z1z1z1, z1, z1z1);
1.1003 + felem_mul(s2, y2, z1z1z1);
1.1004 + felem_diff(h, u2, u1);
1.1005 + x_equal = felem_is_zero_vartime(h);
1.1006 + felem_sum(i, h, h);
1.1007 + felem_square(i, i);
1.1008 + felem_mul(j, h, i);
1.1009 + felem_diff(r, s2, s1);
1.1010 + y_equal = felem_is_zero_vartime(r);
1.1011 + if (x_equal && y_equal) {
1.1012 + point_double(x_out, y_out, z_out, x1, y1, z1);
1.1013 + return;
1.1014 + }
1.1015 + felem_sum(r, r, r);
1.1016 + felem_mul(v, u1, i);
1.1017 +
1.1018 + felem_mul(z_out, tmp, h);
1.1019 + felem_square(rr, r);
1.1020 + felem_diff(x_out, rr, j);
1.1021 + felem_diff(x_out, x_out, v);
1.1022 + felem_diff(x_out, x_out, v);
1.1023 +
1.1024 + felem_diff(tmp, v, x_out);
1.1025 + felem_mul(y_out, tmp, r);
1.1026 + felem_mul(tmp, s1, j);
1.1027 + felem_diff(y_out, y_out, tmp);
1.1028 + felem_diff(y_out, y_out, tmp);
1.1029 +}
1.1030 +
1.1031 +/* copy_conditional sets out=in if mask = 0xffffffff in constant time.
1.1032 + *
1.1033 + * On entry: mask is either 0 or 0xffffffff.
1.1034 + */
1.1035 +static void copy_conditional(felem out, const felem in, limb mask)
1.1036 +{
1.1037 + int i;
1.1038 +
1.1039 + for (i = 0; i < NLIMBS; i++) {
1.1040 + const limb tmp = mask & (in[i] ^ out[i]);
1.1041 + out[i] ^= tmp;
1.1042 + }
1.1043 +}
1.1044 +
1.1045 +/* select_affine_point sets {out_x,out_y} to the index'th entry of table.
1.1046 + * On entry: index < 16, table[0] must be zero.
1.1047 + */
1.1048 +static void select_affine_point(felem out_x, felem out_y,
1.1049 + const limb *table, limb index)
1.1050 +{
1.1051 + limb i, j;
1.1052 +
1.1053 + memset(out_x, 0, sizeof(felem));
1.1054 + memset(out_y, 0, sizeof(felem));
1.1055 +
1.1056 + for (i = 1; i < 16; i++) {
1.1057 + limb mask = i ^ index;
1.1058 + mask |= mask >> 2;
1.1059 + mask |= mask >> 1;
1.1060 + mask &= 1;
1.1061 + mask--;
1.1062 + for (j = 0; j < NLIMBS; j++, table++) {
1.1063 + out_x[j] |= *table & mask;
1.1064 + }
1.1065 + for (j = 0; j < NLIMBS; j++, table++) {
1.1066 + out_y[j] |= *table & mask;
1.1067 + }
1.1068 + }
1.1069 +}
1.1070 +
1.1071 +/* select_jacobian_point sets {out_x,out_y,out_z} to the index'th entry of
1.1072 + * table. On entry: index < 16, table[0] must be zero.
1.1073 + */
1.1074 +static void select_jacobian_point(felem out_x, felem out_y, felem out_z,
1.1075 + const limb *table, limb index)
1.1076 +{
1.1077 + limb i, j;
1.1078 +
1.1079 + memset(out_x, 0, sizeof(felem));
1.1080 + memset(out_y, 0, sizeof(felem));
1.1081 + memset(out_z, 0, sizeof(felem));
1.1082 +
1.1083 + /* The implicit value at index 0 is all zero. We don't need to perform that
1.1084 + * iteration of the loop because we already set out_* to zero.
1.1085 + */
1.1086 + table += 3*NLIMBS;
1.1087 +
1.1088 + for (i = 1; i < 16; i++) {
1.1089 + limb mask = i ^ index;
1.1090 + mask |= mask >> 2;
1.1091 + mask |= mask >> 1;
1.1092 + mask &= 1;
1.1093 + mask--;
1.1094 + for (j = 0; j < NLIMBS; j++, table++) {
1.1095 + out_x[j] |= *table & mask;
1.1096 + }
1.1097 + for (j = 0; j < NLIMBS; j++, table++) {
1.1098 + out_y[j] |= *table & mask;
1.1099 + }
1.1100 + for (j = 0; j < NLIMBS; j++, table++) {
1.1101 + out_z[j] |= *table & mask;
1.1102 + }
1.1103 + }
1.1104 +}
1.1105 +
1.1106 +/* get_bit returns the bit'th bit of scalar. */
1.1107 +static char get_bit(const u8 scalar[32], int bit)
1.1108 +{
1.1109 + return ((scalar[bit >> 3]) >> (bit & 7)) & 1;
1.1110 +}
1.1111 +
1.1112 +/* scalar_base_mult sets {nx,ny,nz} = scalar*G where scalar is a little-endian
1.1113 + * number. Note that the value of scalar must be less than the order of the
1.1114 + * group.
1.1115 + */
1.1116 +static void scalar_base_mult(felem nx, felem ny, felem nz, const u8 scalar[32])
1.1117 +{
1.1118 + int i, j;
1.1119 + limb n_is_infinity_mask = -1, p_is_noninfinite_mask, mask;
1.1120 + u32 table_offset;
1.1121 +
1.1122 + felem px, py;
1.1123 + felem tx, ty, tz;
1.1124 +
1.1125 + memset(nx, 0, sizeof(felem));
1.1126 + memset(ny, 0, sizeof(felem));
1.1127 + memset(nz, 0, sizeof(felem));
1.1128 +
1.1129 + /* The loop adds bits at positions 0, 64, 128 and 192, followed by
1.1130 + * positions 32,96,160 and 224 and does this 32 times.
1.1131 + */
1.1132 + for (i = 0; i < 32; i++) {
1.1133 + if (i) {
1.1134 + point_double(nx, ny, nz, nx, ny, nz);
1.1135 + }
1.1136 + table_offset = 0;
1.1137 + for (j = 0; j <= 32; j += 32) {
1.1138 + char bit0 = get_bit(scalar, 31 - i + j);
1.1139 + char bit1 = get_bit(scalar, 95 - i + j);
1.1140 + char bit2 = get_bit(scalar, 159 - i + j);
1.1141 + char bit3 = get_bit(scalar, 223 - i + j);
1.1142 + limb index = bit0 | (bit1 << 1) | (bit2 << 2) | (bit3 << 3);
1.1143 +
1.1144 + select_affine_point(px, py, kPrecomputed + table_offset, index);
1.1145 + table_offset += 30 * NLIMBS;
1.1146 +
1.1147 + /* Since scalar is less than the order of the group, we know that
1.1148 + * {nx,ny,nz} != {px,py,1}, unless both are zero, which we handle
1.1149 + * below.
1.1150 + */
1.1151 + point_add_mixed(tx, ty, tz, nx, ny, nz, px, py);
1.1152 + /* The result of point_add_mixed is incorrect if {nx,ny,nz} is zero
1.1153 + * (a.k.a. the point at infinity). We handle that situation by
1.1154 + * copying the point from the table.
1.1155 + */
1.1156 + copy_conditional(nx, px, n_is_infinity_mask);
1.1157 + copy_conditional(ny, py, n_is_infinity_mask);
1.1158 + copy_conditional(nz, kOne, n_is_infinity_mask);
1.1159 +
1.1160 + /* Equally, the result is also wrong if the point from the table is
1.1161 + * zero, which happens when the index is zero. We handle that by
1.1162 + * only copying from {tx,ty,tz} to {nx,ny,nz} if index != 0.
1.1163 + */
1.1164 + p_is_noninfinite_mask = NON_ZERO_TO_ALL_ONES(index);
1.1165 + mask = p_is_noninfinite_mask & ~n_is_infinity_mask;
1.1166 + copy_conditional(nx, tx, mask);
1.1167 + copy_conditional(ny, ty, mask);
1.1168 + copy_conditional(nz, tz, mask);
1.1169 + /* If p was not zero, then n is now non-zero. */
1.1170 + n_is_infinity_mask &= ~p_is_noninfinite_mask;
1.1171 + }
1.1172 + }
1.1173 +}
1.1174 +
1.1175 +/* point_to_affine converts a Jacobian point to an affine point. If the input
1.1176 + * is the point at infinity then it returns (0, 0) in constant time.
1.1177 + */
1.1178 +static void point_to_affine(felem x_out, felem y_out,
1.1179 + const felem nx, const felem ny, const felem nz) {
1.1180 + felem z_inv, z_inv_sq;
1.1181 + felem_inv(z_inv, nz);
1.1182 + felem_square(z_inv_sq, z_inv);
1.1183 + felem_mul(x_out, nx, z_inv_sq);
1.1184 + felem_mul(z_inv, z_inv, z_inv_sq);
1.1185 + felem_mul(y_out, ny, z_inv);
1.1186 +}
1.1187 +
1.1188 +/* scalar_mult sets {nx,ny,nz} = scalar*{x,y}. */
1.1189 +static void scalar_mult(felem nx, felem ny, felem nz,
1.1190 + const felem x, const felem y, const u8 scalar[32])
1.1191 +{
1.1192 + int i;
1.1193 + felem px, py, pz, tx, ty, tz;
1.1194 + felem precomp[16][3];
1.1195 + limb n_is_infinity_mask, index, p_is_noninfinite_mask, mask;
1.1196 +
1.1197 + /* We precompute 0,1,2,... times {x,y}. */
1.1198 + memset(precomp, 0, sizeof(felem) * 3);
1.1199 + memcpy(&precomp[1][0], x, sizeof(felem));
1.1200 + memcpy(&precomp[1][1], y, sizeof(felem));
1.1201 + memcpy(&precomp[1][2], kOne, sizeof(felem));
1.1202 +
1.1203 + for (i = 2; i < 16; i += 2) {
1.1204 + point_double(precomp[i][0], precomp[i][1], precomp[i][2],
1.1205 + precomp[i / 2][0], precomp[i / 2][1], precomp[i / 2][2]);
1.1206 +
1.1207 + point_add_mixed(precomp[i + 1][0], precomp[i + 1][1], precomp[i + 1][2],
1.1208 + precomp[i][0], precomp[i][1], precomp[i][2], x, y);
1.1209 + }
1.1210 +
1.1211 + memset(nx, 0, sizeof(felem));
1.1212 + memset(ny, 0, sizeof(felem));
1.1213 + memset(nz, 0, sizeof(felem));
1.1214 + n_is_infinity_mask = -1;
1.1215 +
1.1216 + /* We add in a window of four bits each iteration and do this 64 times. */
1.1217 + for (i = 0; i < 64; i++) {
1.1218 + if (i) {
1.1219 + point_double(nx, ny, nz, nx, ny, nz);
1.1220 + point_double(nx, ny, nz, nx, ny, nz);
1.1221 + point_double(nx, ny, nz, nx, ny, nz);
1.1222 + point_double(nx, ny, nz, nx, ny, nz);
1.1223 + }
1.1224 +
1.1225 + index = scalar[31 - i / 2];
1.1226 + if ((i & 1) == 1) {
1.1227 + index &= 15;
1.1228 + } else {
1.1229 + index >>= 4;
1.1230 + }
1.1231 +
1.1232 + /* See the comments in scalar_base_mult about handling infinities. */
1.1233 + select_jacobian_point(px, py, pz, precomp[0][0], index);
1.1234 + point_add(tx, ty, tz, nx, ny, nz, px, py, pz);
1.1235 + copy_conditional(nx, px, n_is_infinity_mask);
1.1236 + copy_conditional(ny, py, n_is_infinity_mask);
1.1237 + copy_conditional(nz, pz, n_is_infinity_mask);
1.1238 +
1.1239 + p_is_noninfinite_mask = NON_ZERO_TO_ALL_ONES(index);
1.1240 + mask = p_is_noninfinite_mask & ~n_is_infinity_mask;
1.1241 + copy_conditional(nx, tx, mask);
1.1242 + copy_conditional(ny, ty, mask);
1.1243 + copy_conditional(nz, tz, mask);
1.1244 + n_is_infinity_mask &= ~p_is_noninfinite_mask;
1.1245 + }
1.1246 +}
1.1247 +
1.1248 +/* Interface with Freebl: */
1.1249 +
1.1250 +/* BYTESWAP_MP_DIGIT_TO_LE swaps the bytes of a mp_digit to
1.1251 + * little-endian order.
1.1252 + */
1.1253 +#ifdef IS_BIG_ENDIAN
1.1254 +#ifdef __APPLE__
1.1255 +#include <libkern/OSByteOrder.h>
1.1256 +#define BYTESWAP32(x) OSSwapInt32(x)
1.1257 +#define BYTESWAP64(x) OSSwapInt64(x)
1.1258 +#else
1.1259 +#define BYTESWAP32(x) \
1.1260 + ((x) >> 24 | (x) >> 8 & 0xff00 | ((x) & 0xff00) << 8 | (x) << 24)
1.1261 +#define BYTESWAP64(x) \
1.1262 + ((x) >> 56 | (x) >> 40 & 0xff00 | \
1.1263 + (x) >> 24 & 0xff0000 | (x) >> 8 & 0xff000000 | \
1.1264 + ((x) & 0xff000000) << 8 | ((x) & 0xff0000) << 24 | \
1.1265 + ((x) & 0xff00) << 40 | (x) << 56)
1.1266 +#endif
1.1267 +
1.1268 +#ifdef MP_USE_UINT_DIGIT
1.1269 +#define BYTESWAP_MP_DIGIT_TO_LE(x) BYTESWAP32(x)
1.1270 +#else
1.1271 +#define BYTESWAP_MP_DIGIT_TO_LE(x) BYTESWAP64(x)
1.1272 +#endif
1.1273 +#endif /* IS_BIG_ENDIAN */
1.1274 +
1.1275 +#ifdef MP_USE_UINT_DIGIT
1.1276 +static const mp_digit kRInvDigits[8] = {
1.1277 + 0x80000000, 1, 0xffffffff, 0,
1.1278 + 0x80000001, 0xfffffffe, 1, 0x7fffffff
1.1279 +};
1.1280 +#else
1.1281 +static const mp_digit kRInvDigits[4] = {
1.1282 + PR_UINT64(0x180000000), 0xffffffff,
1.1283 + PR_UINT64(0xfffffffe80000001), PR_UINT64(0x7fffffff00000001)
1.1284 +};
1.1285 +#endif
1.1286 +#define MP_DIGITS_IN_256_BITS (32/sizeof(mp_digit))
1.1287 +static const mp_int kRInv = {
1.1288 + MP_ZPOS,
1.1289 + MP_DIGITS_IN_256_BITS,
1.1290 + MP_DIGITS_IN_256_BITS,
1.1291 + (mp_digit*) kRInvDigits
1.1292 +};
1.1293 +
1.1294 +static const limb kTwo28 = 0x10000000;
1.1295 +static const limb kTwo29 = 0x20000000;
1.1296 +
1.1297 +/* to_montgomery sets out = R*in. */
1.1298 +static mp_err to_montgomery(felem out, const mp_int *in, const ECGroup *group)
1.1299 +{
1.1300 + /* There are no MPI functions for bitshift operations and we wish to shift
1.1301 + * in 257 bits left so we move the digits 256-bits left and then multiply
1.1302 + * by two.
1.1303 + */
1.1304 + mp_int in_shifted;
1.1305 + int i;
1.1306 + mp_err res;
1.1307 +
1.1308 + mp_init(&in_shifted);
1.1309 + s_mp_pad(&in_shifted, MP_USED(in) + MP_DIGITS_IN_256_BITS);
1.1310 + memcpy(&MP_DIGIT(&in_shifted, MP_DIGITS_IN_256_BITS),
1.1311 + MP_DIGITS(in),
1.1312 + MP_USED(in)*sizeof(mp_digit));
1.1313 + mp_mul_2(&in_shifted, &in_shifted);
1.1314 + MP_CHECKOK(group->meth->field_mod(&in_shifted, &in_shifted, group->meth));
1.1315 +
1.1316 + for (i = 0;; i++) {
1.1317 + out[i] = MP_DIGIT(&in_shifted, 0) & kBottom29Bits;
1.1318 + mp_div_d(&in_shifted, kTwo29, &in_shifted, NULL);
1.1319 +
1.1320 + i++;
1.1321 + if (i == NLIMBS)
1.1322 + break;
1.1323 + out[i] = MP_DIGIT(&in_shifted, 0) & kBottom28Bits;
1.1324 + mp_div_d(&in_shifted, kTwo28, &in_shifted, NULL);
1.1325 + }
1.1326 +
1.1327 +CLEANUP:
1.1328 + mp_clear(&in_shifted);
1.1329 + return res;
1.1330 +}
1.1331 +
1.1332 +/* from_montgomery sets out=in/R. */
1.1333 +static mp_err from_montgomery(mp_int *out, const felem in,
1.1334 + const ECGroup *group)
1.1335 +{
1.1336 + mp_int result, tmp;
1.1337 + mp_err res;
1.1338 + int i;
1.1339 +
1.1340 + mp_init(&result);
1.1341 + mp_init(&tmp);
1.1342 +
1.1343 + MP_CHECKOK(mp_add_d(&tmp, in[NLIMBS-1], &result));
1.1344 + for (i = NLIMBS-2; i >= 0; i--) {
1.1345 + if ((i & 1) == 0) {
1.1346 + MP_CHECKOK(mp_mul_d(&result, kTwo29, &tmp));
1.1347 + } else {
1.1348 + MP_CHECKOK(mp_mul_d(&result, kTwo28, &tmp));
1.1349 + }
1.1350 + MP_CHECKOK(mp_add_d(&tmp, in[i], &result));
1.1351 + }
1.1352 +
1.1353 + MP_CHECKOK(mp_mul(&result, &kRInv, out));
1.1354 + MP_CHECKOK(group->meth->field_mod(out, out, group->meth));
1.1355 +
1.1356 +CLEANUP:
1.1357 + mp_clear(&result);
1.1358 + mp_clear(&tmp);
1.1359 + return res;
1.1360 +}
1.1361 +
1.1362 +/* scalar_from_mp_int sets out_scalar=n, where n < the group order. */
1.1363 +static void scalar_from_mp_int(u8 out_scalar[32], const mp_int *n)
1.1364 +{
1.1365 + /* We require that |n| is less than the order of the group and therefore it
1.1366 + * will fit into |out_scalar|. However, these is a timing side-channel here
1.1367 + * that we cannot avoid: if |n| is sufficiently small it may be one or more
1.1368 + * words too short and we'll copy less data.
1.1369 + */
1.1370 + memset(out_scalar, 0, 32);
1.1371 +#ifdef IS_LITTLE_ENDIAN
1.1372 + memcpy(out_scalar, MP_DIGITS(n), MP_USED(n) * sizeof(mp_digit));
1.1373 +#else
1.1374 + {
1.1375 + mp_size i;
1.1376 + mp_digit swapped[MP_DIGITS_IN_256_BITS];
1.1377 + for (i = 0; i < MP_USED(n); i++) {
1.1378 + swapped[i] = BYTESWAP_MP_DIGIT_TO_LE(MP_DIGIT(n, i));
1.1379 + }
1.1380 + memcpy(out_scalar, swapped, MP_USED(n) * sizeof(mp_digit));
1.1381 + }
1.1382 +#endif
1.1383 +}
1.1384 +
1.1385 +/* ec_GFp_nistp256_base_point_mul sets {out_x,out_y} = nG, where n is < the
1.1386 + * order of the group.
1.1387 + */
1.1388 +static mp_err ec_GFp_nistp256_base_point_mul(const mp_int *n,
1.1389 + mp_int *out_x, mp_int *out_y,
1.1390 + const ECGroup *group)
1.1391 +{
1.1392 + u8 scalar[32];
1.1393 + felem x, y, z, x_affine, y_affine;
1.1394 + mp_err res;
1.1395 +
1.1396 + /* FIXME(agl): test that n < order. */
1.1397 +
1.1398 + scalar_from_mp_int(scalar, n);
1.1399 + scalar_base_mult(x, y, z, scalar);
1.1400 + point_to_affine(x_affine, y_affine, x, y, z);
1.1401 + MP_CHECKOK(from_montgomery(out_x, x_affine, group));
1.1402 + MP_CHECKOK(from_montgomery(out_y, y_affine, group));
1.1403 +
1.1404 +CLEANUP:
1.1405 + return res;
1.1406 +}
1.1407 +
1.1408 +/* ec_GFp_nistp256_point_mul sets {out_x,out_y} = n*{in_x,in_y}, where n is <
1.1409 + * the order of the group.
1.1410 + */
1.1411 +static mp_err ec_GFp_nistp256_point_mul(const mp_int *n,
1.1412 + const mp_int *in_x, const mp_int *in_y,
1.1413 + mp_int *out_x, mp_int *out_y,
1.1414 + const ECGroup *group)
1.1415 +{
1.1416 + u8 scalar[32];
1.1417 + felem x, y, z, x_affine, y_affine, px, py;
1.1418 + mp_err res;
1.1419 +
1.1420 + scalar_from_mp_int(scalar, n);
1.1421 +
1.1422 + MP_CHECKOK(to_montgomery(px, in_x, group));
1.1423 + MP_CHECKOK(to_montgomery(py, in_y, group));
1.1424 +
1.1425 + scalar_mult(x, y, z, px, py, scalar);
1.1426 + point_to_affine(x_affine, y_affine, x, y, z);
1.1427 + MP_CHECKOK(from_montgomery(out_x, x_affine, group));
1.1428 + MP_CHECKOK(from_montgomery(out_y, y_affine, group));
1.1429 +
1.1430 +CLEANUP:
1.1431 + return res;
1.1432 +}
1.1433 +
1.1434 +/* ec_GFp_nistp256_point_mul_vartime sets {out_x,out_y} = n1*G +
1.1435 + * n2*{in_x,in_y}, where n1 and n2 are < the order of the group.
1.1436 + *
1.1437 + * As indicated by the name, this function operates in variable time. This
1.1438 + * is safe because it's used for signature validation which doesn't deal
1.1439 + * with secrets.
1.1440 + */
1.1441 +static mp_err ec_GFp_nistp256_points_mul_vartime(
1.1442 + const mp_int *n1, const mp_int *n2,
1.1443 + const mp_int *in_x, const mp_int *in_y,
1.1444 + mp_int *out_x, mp_int *out_y,
1.1445 + const ECGroup *group)
1.1446 +{
1.1447 + u8 scalar1[32], scalar2[32];
1.1448 + felem x1, y1, z1, x2, y2, z2, x_affine, y_affine, px, py;
1.1449 + mp_err res = MP_OKAY;
1.1450 +
1.1451 + /* If n2 == NULL, this is just a base-point multiplication. */
1.1452 + if (n2 == NULL) {
1.1453 + return ec_GFp_nistp256_base_point_mul(n1, out_x, out_y, group);
1.1454 + }
1.1455 +
1.1456 + /* If n1 == nULL, this is just an arbitary-point multiplication. */
1.1457 + if (n1 == NULL) {
1.1458 + return ec_GFp_nistp256_point_mul(n2, in_x, in_y, out_x, out_y, group);
1.1459 + }
1.1460 +
1.1461 + /* If both scalars are zero, then the result is the point at infinity. */
1.1462 + if (mp_cmp_z(n1) == 0 && mp_cmp_z(n2) == 0) {
1.1463 + mp_zero(out_x);
1.1464 + mp_zero(out_y);
1.1465 + return res;
1.1466 + }
1.1467 +
1.1468 + scalar_from_mp_int(scalar1, n1);
1.1469 + scalar_from_mp_int(scalar2, n2);
1.1470 +
1.1471 + MP_CHECKOK(to_montgomery(px, in_x, group));
1.1472 + MP_CHECKOK(to_montgomery(py, in_y, group));
1.1473 + scalar_base_mult(x1, y1, z1, scalar1);
1.1474 + scalar_mult(x2, y2, z2, px, py, scalar2);
1.1475 +
1.1476 + if (mp_cmp_z(n2) == 0) {
1.1477 + /* If n2 == 0, then {x2,y2,z2} is zero and the result is just
1.1478 + * {x1,y1,z1}. */
1.1479 + } else if (mp_cmp_z(n1) == 0) {
1.1480 + /* If n1 == 0, then {x1,y1,z1} is zero and the result is just
1.1481 + * {x2,y2,z2}. */
1.1482 + memcpy(x1, x2, sizeof(x2));
1.1483 + memcpy(y1, y2, sizeof(y2));
1.1484 + memcpy(z1, z2, sizeof(z2));
1.1485 + } else {
1.1486 + /* This function handles the case where {x1,y1,z1} == {x2,y2,z2}. */
1.1487 + point_add_or_double_vartime(x1, y1, z1, x1, y1, z1, x2, y2, z2);
1.1488 + }
1.1489 +
1.1490 + point_to_affine(x_affine, y_affine, x1, y1, z1);
1.1491 + MP_CHECKOK(from_montgomery(out_x, x_affine, group));
1.1492 + MP_CHECKOK(from_montgomery(out_y, y_affine, group));
1.1493 +
1.1494 +CLEANUP:
1.1495 + return res;
1.1496 +}
1.1497 +
1.1498 +/* Wire in fast point multiplication for named curves. */
1.1499 +mp_err ec_group_set_gfp256_32(ECGroup *group, ECCurveName name)
1.1500 +{
1.1501 + if (name == ECCurve_NIST_P256) {
1.1502 + group->base_point_mul = &ec_GFp_nistp256_base_point_mul;
1.1503 + group->point_mul = &ec_GFp_nistp256_point_mul;
1.1504 + group->points_mul = &ec_GFp_nistp256_points_mul_vartime;
1.1505 + }
1.1506 + return MP_OKAY;
1.1507 +}
