michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: /*
michael@0:  * RSA PKCS#1 v2.1 (RFC 3447) operations
michael@0:  */
michael@0: 
michael@0: #ifdef FREEBL_NO_DEPEND
michael@0: #include "stubs.h"
michael@0: #endif
michael@0: 
michael@0: #include "secerr.h"
michael@0: 
michael@0: #include "blapi.h"
michael@0: #include "secitem.h"
michael@0: #include "blapii.h"
michael@0: 
michael@0: #define RSA_BLOCK_MIN_PAD_LEN            8
michael@0: #define RSA_BLOCK_FIRST_OCTET            0x00
michael@0: #define RSA_BLOCK_PRIVATE_PAD_OCTET      0xff
michael@0: #define RSA_BLOCK_AFTER_PAD_OCTET        0x00
michael@0: 
michael@0: /*
michael@0:  * RSA block types
michael@0:  *
michael@0:  * The values of RSA_BlockPrivate and RSA_BlockPublic are fixed.
michael@0:  * The value of RSA_BlockRaw isn't fixed by definition, but we are keeping
michael@0:  * the value that NSS has been using in the past.
michael@0:  */
michael@0: typedef enum {
michael@0:     RSA_BlockPrivate = 1,   /* pad for a private-key operation */
michael@0:     RSA_BlockPublic = 2,    /* pad for a public-key operation */
michael@0:     RSA_BlockRaw = 4        /* simply justify the block appropriately */
michael@0: } RSA_BlockType;
michael@0: 
michael@0: /* Needed for RSA-PSS functions */
michael@0: static const unsigned char eightZeros[] = { 0, 0, 0, 0, 0, 0, 0, 0 };
michael@0: 
michael@0: /* Constant time comparison of a single byte.
michael@0:  * Returns 1 iff a == b, otherwise returns 0.
michael@0:  * Note: For ranges of bytes, use constantTimeCompare.
michael@0:  */
michael@0: static unsigned char constantTimeEQ8(unsigned char a, unsigned char b) {
michael@0:     unsigned char c = ~((a - b) | (b - a));
michael@0:     c >>= 7;
michael@0:     return c;
michael@0: }
michael@0: 
michael@0: /* Constant time comparison of a range of bytes.
michael@0:  * Returns 1 iff len bytes of a are identical to len bytes of b, otherwise
michael@0:  * returns 0.
michael@0:  */
michael@0: static unsigned char constantTimeCompare(const unsigned char *a,
michael@0:                                          const unsigned char *b,
michael@0:                                          unsigned int len) {
michael@0:     unsigned char tmp = 0;
michael@0:     unsigned int i;
michael@0:     for (i = 0; i < len; ++i, ++a, ++b)
michael@0:         tmp |= *a ^ *b;
michael@0:     return constantTimeEQ8(0x00, tmp);
michael@0: }
michael@0: 
michael@0: /* Constant time conditional.
michael@0:  * Returns a if c is 1, or b if c is 0. The result is undefined if c is
michael@0:  * not 0 or 1.
michael@0:  */
michael@0: static unsigned int constantTimeCondition(unsigned int c,
michael@0:                                           unsigned int a,
michael@0:                                           unsigned int b)
michael@0: {
michael@0:     return (~(c - 1) & a) | ((c - 1) & b);
michael@0: }
michael@0: 
michael@0: static unsigned int
michael@0: rsa_modulusLen(SECItem * modulus)
michael@0: {
michael@0:     unsigned char byteZero = modulus->data[0];
michael@0:     unsigned int modLen = modulus->len - !byteZero;
michael@0:     return modLen;
michael@0: }
michael@0: 
michael@0: /*
michael@0:  * Format one block of data for public/private key encryption using
michael@0:  * the rules defined in PKCS #1.
michael@0:  */
michael@0: static unsigned char *
michael@0: rsa_FormatOneBlock(unsigned modulusLen,
michael@0:                    RSA_BlockType blockType,
michael@0:                    SECItem * data)
michael@0: {
michael@0:     unsigned char *block;
michael@0:     unsigned char *bp;
michael@0:     int padLen;
michael@0:     int i, j;
michael@0:     SECStatus rv;
michael@0: 
michael@0:     block = (unsigned char *) PORT_Alloc(modulusLen);
michael@0:     if (block == NULL)
michael@0:         return NULL;
michael@0: 
michael@0:     bp = block;
michael@0: 
michael@0:     /*
michael@0:      * All RSA blocks start with two octets:
michael@0:      *	0x00 || BlockType
michael@0:      */
michael@0:     *bp++ = RSA_BLOCK_FIRST_OCTET;
michael@0:     *bp++ = (unsigned char) blockType;
michael@0: 
michael@0:     switch (blockType) {
michael@0: 
michael@0:       /*
michael@0:        * Blocks intended for private-key operation.
michael@0:        */
michael@0:       case RSA_BlockPrivate:	 /* preferred method */
michael@0:         /*
michael@0:          * 0x00 || BT || Pad || 0x00 || ActualData
michael@0:          *   1      1   padLen    1      data->len
michael@0:          * Pad is either all 0x00 or all 0xff bytes, depending on blockType.
michael@0:          */
michael@0:         padLen = modulusLen - data->len - 3;
michael@0:         PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
michael@0:         if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
michael@0:             PORT_Free(block);
michael@0:             return NULL;
michael@0:         }
michael@0:         PORT_Memset(bp, RSA_BLOCK_PRIVATE_PAD_OCTET, padLen);
michael@0:         bp += padLen;
michael@0:         *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
michael@0:         PORT_Memcpy(bp, data->data, data->len);
michael@0:         break;
michael@0: 
michael@0:       /*
michael@0:        * Blocks intended for public-key operation.
michael@0:        */
michael@0:       case RSA_BlockPublic:
michael@0:         /*
michael@0:          * 0x00 || BT || Pad || 0x00 || ActualData
michael@0:          *   1      1   padLen    1      data->len
michael@0:          * Pad is all non-zero random bytes.
michael@0:          *
michael@0:          * Build the block left to right.
michael@0:          * Fill the entire block from Pad to the end with random bytes.
michael@0:          * Use the bytes after Pad as a supply of extra random bytes from
michael@0:          * which to find replacements for the zero bytes in Pad.
michael@0:          * If we need more than that, refill the bytes after Pad with
michael@0:          * new random bytes as necessary.
michael@0:          */
michael@0:         padLen = modulusLen - (data->len + 3);
michael@0:         PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
michael@0:         if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
michael@0:             PORT_Free(block);
michael@0:             return NULL;
michael@0:         }
michael@0:         j = modulusLen - 2;
michael@0:         rv = RNG_GenerateGlobalRandomBytes(bp, j);
michael@0:         if (rv == SECSuccess) {
michael@0:             for (i = 0; i < padLen; ) {
michael@0:                 unsigned char repl;
michael@0:                 /* Pad with non-zero random data. */
michael@0:                 if (bp[i] != RSA_BLOCK_AFTER_PAD_OCTET) {
michael@0:                     ++i;
michael@0:                     continue;
michael@0:                 }
michael@0:                 if (j <= padLen) {
michael@0:                     rv = RNG_GenerateGlobalRandomBytes(bp + padLen,
michael@0:                                           modulusLen - (2 + padLen));
michael@0:                     if (rv != SECSuccess)
michael@0:                         break;
michael@0:                     j = modulusLen - 2;
michael@0:                 }
michael@0:                 do {
michael@0:                     repl = bp[--j];
michael@0:                 } while (repl == RSA_BLOCK_AFTER_PAD_OCTET && j > padLen);
michael@0:                 if (repl != RSA_BLOCK_AFTER_PAD_OCTET) {
michael@0:                     bp[i++] = repl;
michael@0:                 }
michael@0:             }
michael@0:         }
michael@0:         if (rv != SECSuccess) {
michael@0:             PORT_Free(block);
michael@0:             PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
michael@0:             return NULL;
michael@0:         }
michael@0:         bp += padLen;
michael@0:         *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
michael@0:         PORT_Memcpy(bp, data->data, data->len);
michael@0:         break;
michael@0: 
michael@0:       default:
michael@0:         PORT_Assert(0);
michael@0:         PORT_Free(block);
michael@0:         return NULL;
michael@0:     }
michael@0: 
michael@0:     return block;
michael@0: }
michael@0: 
michael@0: static SECStatus
michael@0: rsa_FormatBlock(SECItem * result,
michael@0:                 unsigned modulusLen,
michael@0:                 RSA_BlockType blockType,
michael@0:                 SECItem * data)
michael@0: {
michael@0:     switch (blockType) {
michael@0:       case RSA_BlockPrivate:
michael@0:       case RSA_BlockPublic:
michael@0:         /*
michael@0:          * 0x00 || BT || Pad || 0x00 || ActualData
michael@0:          *
michael@0:          * The "3" below is the first octet + the second octet + the 0x00
michael@0:          * octet that always comes just before the ActualData.
michael@0:          */
michael@0:         PORT_Assert(data->len <= (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN)));
michael@0: 
michael@0:         result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
michael@0:         if (result->data == NULL) {
michael@0:             result->len = 0;
michael@0:             return SECFailure;
michael@0:         }
michael@0:         result->len = modulusLen;
michael@0: 
michael@0:         break;
michael@0: 
michael@0:       case RSA_BlockRaw:
michael@0:         /*
michael@0:          * Pad || ActualData
michael@0:          * Pad is zeros. The application is responsible for recovering
michael@0:          * the actual data.
michael@0:          */
michael@0:         if (data->len > modulusLen ) {
michael@0:             return SECFailure;
michael@0:         }
michael@0:         result->data = (unsigned char*)PORT_ZAlloc(modulusLen);
michael@0:         result->len = modulusLen;
michael@0:         PORT_Memcpy(result->data + (modulusLen - data->len),
michael@0:                     data->data, data->len);
michael@0:         break;
michael@0: 
michael@0:       default:
michael@0:         PORT_Assert(0);
michael@0:         result->data = NULL;
michael@0:         result->len = 0;
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     return SECSuccess;
michael@0: }
michael@0: 
michael@0: /*
michael@0:  * Mask generation function MGF1 as defined in PKCS #1 v2.1 / RFC 3447.
michael@0:  */
michael@0: static SECStatus
michael@0: MGF1(HASH_HashType hashAlg,
michael@0:      unsigned char * mask,
michael@0:      unsigned int maskLen,
michael@0:      const unsigned char * mgfSeed,
michael@0:      unsigned int mgfSeedLen)
michael@0: {
michael@0:     unsigned int digestLen;
michael@0:     PRUint32 counter;
michael@0:     PRUint32 rounds;
michael@0:     unsigned char * tempHash;
michael@0:     unsigned char * temp;
michael@0:     const SECHashObject * hash;
michael@0:     void * hashContext;
michael@0:     unsigned char C[4];
michael@0: 
michael@0:     hash = HASH_GetRawHashObject(hashAlg);
michael@0:     if (hash == NULL)
michael@0:         return SECFailure;
michael@0: 
michael@0:     hashContext = (*hash->create)();
michael@0:     rounds = (maskLen + hash->length - 1) / hash->length;
michael@0:     for (counter = 0; counter < rounds; counter++) {
michael@0:         C[0] = (unsigned char)((counter >> 24) & 0xff);
michael@0:         C[1] = (unsigned char)((counter >> 16) & 0xff);
michael@0:         C[2] = (unsigned char)((counter >> 8) & 0xff);
michael@0:         C[3] = (unsigned char)(counter & 0xff);
michael@0: 
michael@0:         /* This could be optimized when the clone functions in
michael@0:          * rawhash.c are implemented. */
michael@0:         (*hash->begin)(hashContext);
michael@0:         (*hash->update)(hashContext, mgfSeed, mgfSeedLen);
michael@0:         (*hash->update)(hashContext, C, sizeof C);
michael@0: 
michael@0:         tempHash = mask + counter * hash->length;
michael@0:         if (counter != (rounds - 1)) {
michael@0:             (*hash->end)(hashContext, tempHash, &digestLen, hash->length);
michael@0:         } else { /* we're in the last round and need to cut the hash */
michael@0:             temp = (unsigned char *)PORT_Alloc(hash->length);
michael@0:             (*hash->end)(hashContext, temp, &digestLen, hash->length);
michael@0:             PORT_Memcpy(tempHash, temp, maskLen - counter * hash->length);
michael@0:             PORT_Free(temp);
michael@0:         }
michael@0:     }
michael@0:     (*hash->destroy)(hashContext, PR_TRUE);
michael@0: 
michael@0:     return SECSuccess;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_SignRaw(RSAPrivateKey * key,
michael@0:             unsigned char * output,
michael@0:             unsigned int * outputLen,
michael@0:             unsigned int maxOutputLen,
michael@0:             const unsigned char * data,
michael@0:             unsigned int dataLen)
michael@0: {
michael@0:     SECStatus rv = SECSuccess;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     SECItem formatted;
michael@0:     SECItem unformatted;
michael@0: 
michael@0:     if (maxOutputLen < modulusLen)
michael@0:         return SECFailure;
michael@0: 
michael@0:     unformatted.len  = dataLen;
michael@0:     unformatted.data = (unsigned char*)data;
michael@0:     formatted.data   = NULL;
michael@0:     rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
michael@0:     if (rv != SECSuccess)
michael@0:         goto done;
michael@0: 
michael@0:     rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
michael@0:     *outputLen = modulusLen;
michael@0: 
michael@0: done:
michael@0:     if (formatted.data != NULL)
michael@0:         PORT_ZFree(formatted.data, modulusLen);
michael@0:     return rv;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_CheckSignRaw(RSAPublicKey * key,
michael@0:                  const unsigned char * sig,
michael@0:                  unsigned int sigLen,
michael@0:                  const unsigned char * hash,
michael@0:                  unsigned int hashLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     unsigned char * buffer;
michael@0: 
michael@0:     if (sigLen != modulusLen)
michael@0:         goto failure;
michael@0:     if (hashLen > modulusLen)
michael@0:         goto failure;
michael@0: 
michael@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
michael@0:     if (!buffer)
michael@0:         goto failure;
michael@0: 
michael@0:     rv = RSA_PublicKeyOp(key, buffer, sig);
michael@0:     if (rv != SECSuccess)
michael@0:         goto loser;
michael@0: 
michael@0:     /*
michael@0:      * make sure we get the same results
michael@0:      */
michael@0:     /* XXX(rsleevi): Constant time */
michael@0:     /* NOTE: should we verify the leading zeros? */
michael@0:     if (PORT_Memcmp(buffer + (modulusLen - hashLen), hash, hashLen) != 0)
michael@0:         goto loser;
michael@0: 
michael@0:     PORT_Free(buffer);
michael@0:     return SECSuccess;
michael@0: 
michael@0: loser:
michael@0:     PORT_Free(buffer);
michael@0: failure:
michael@0:     return SECFailure;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_CheckSignRecoverRaw(RSAPublicKey * key,
michael@0:                         unsigned char * data,
michael@0:                         unsigned int * dataLen,
michael@0:                         unsigned int maxDataLen,
michael@0:                         const unsigned char * sig,
michael@0:                         unsigned int sigLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0: 
michael@0:     if (sigLen != modulusLen)
michael@0:         goto failure;
michael@0:     if (maxDataLen < modulusLen)
michael@0:         goto failure;
michael@0: 
michael@0:     rv = RSA_PublicKeyOp(key, data, sig);
michael@0:     if (rv != SECSuccess)
michael@0:         goto failure;
michael@0: 
michael@0:     *dataLen = modulusLen;
michael@0:     return SECSuccess;
michael@0: 
michael@0: failure:
michael@0:     return SECFailure;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_EncryptRaw(RSAPublicKey * key,
michael@0:                unsigned char * output,
michael@0:                unsigned int * outputLen,
michael@0:                unsigned int maxOutputLen,
michael@0:                const unsigned char * input,
michael@0:                unsigned int inputLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     SECItem formatted;
michael@0:     SECItem unformatted;
michael@0: 
michael@0:     formatted.data = NULL;
michael@0:     if (maxOutputLen < modulusLen)
michael@0:         goto failure;
michael@0: 
michael@0:     unformatted.len  = inputLen;
michael@0:     unformatted.data = (unsigned char*)input;
michael@0:     formatted.data   = NULL;
michael@0:     rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
michael@0:     if (rv != SECSuccess)
michael@0:         goto failure;
michael@0: 
michael@0:     rv = RSA_PublicKeyOp(key, output, formatted.data);
michael@0:     if (rv != SECSuccess)
michael@0:         goto failure;
michael@0: 
michael@0:     PORT_ZFree(formatted.data, modulusLen);
michael@0:     *outputLen = modulusLen;
michael@0:     return SECSuccess;
michael@0: 
michael@0: failure:
michael@0:     if (formatted.data != NULL)
michael@0:         PORT_ZFree(formatted.data, modulusLen);
michael@0:     return SECFailure;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_DecryptRaw(RSAPrivateKey * key,
michael@0:                unsigned char * output,
michael@0:                unsigned int * outputLen,
michael@0:                unsigned int maxOutputLen,
michael@0:                const unsigned char * input,
michael@0:                unsigned int inputLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0: 
michael@0:     if (modulusLen > maxOutputLen)
michael@0:         goto failure;
michael@0:     if (inputLen != modulusLen)
michael@0:         goto failure;
michael@0: 
michael@0:     rv = RSA_PrivateKeyOp(key, output, input);
michael@0:     if (rv != SECSuccess)
michael@0:         goto failure;
michael@0: 
michael@0:     *outputLen = modulusLen;
michael@0:     return SECSuccess;
michael@0: 
michael@0: failure:
michael@0:     return SECFailure;
michael@0: }
michael@0: 
michael@0: /*
michael@0:  * Decodes an EME-OAEP encoded block, validating the encoding in constant
michael@0:  * time.
michael@0:  * Described in RFC 3447, section 7.1.2.
michael@0:  * input contains the encoded block, after decryption.
michael@0:  * label is the optional value L that was associated with the message.
michael@0:  * On success, the original message and message length will be stored in
michael@0:  * output and outputLen.
michael@0:  */
michael@0: static SECStatus
michael@0: eme_oaep_decode(unsigned char * output,
michael@0:                 unsigned int * outputLen,
michael@0:                 unsigned int maxOutputLen,
michael@0:                 const unsigned char * input,
michael@0:                 unsigned int inputLen,
michael@0:                 HASH_HashType hashAlg,
michael@0:                 HASH_HashType maskHashAlg,
michael@0:                 const unsigned char * label,
michael@0:                 unsigned int labelLen)
michael@0: {
michael@0:     const SECHashObject * hash;
michael@0:     void * hashContext;
michael@0:     SECStatus rv = SECFailure;
michael@0:     unsigned char labelHash[HASH_LENGTH_MAX];
michael@0:     unsigned int i;
michael@0:     unsigned int maskLen;
michael@0:     unsigned int paddingOffset;
michael@0:     unsigned char * mask = NULL;
michael@0:     unsigned char * tmpOutput = NULL;
michael@0:     unsigned char isGood;
michael@0:     unsigned char foundPaddingEnd;
michael@0: 
michael@0:     hash = HASH_GetRawHashObject(hashAlg);
michael@0: 
michael@0:     /* 1.c */
michael@0:     if (inputLen < (hash->length * 2) + 2) {
michael@0:         PORT_SetError(SEC_ERROR_INPUT_LEN);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     /* Step 3.a - Generate lHash */
michael@0:     hashContext = (*hash->create)();
michael@0:     if (hashContext == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     (*hash->begin)(hashContext);
michael@0:     if (labelLen > 0)
michael@0:         (*hash->update)(hashContext, label, labelLen);
michael@0:     (*hash->end)(hashContext, labelHash, &i, sizeof(labelHash));
michael@0:     (*hash->destroy)(hashContext, PR_TRUE);
michael@0: 
michael@0:     tmpOutput = (unsigned char*)PORT_Alloc(inputLen);
michael@0:     if (tmpOutput == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         goto done;
michael@0:     }
michael@0: 
michael@0:     maskLen = inputLen - hash->length - 1;
michael@0:     mask = (unsigned char*)PORT_Alloc(maskLen);
michael@0:     if (mask == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         goto done;
michael@0:     }
michael@0: 
michael@0:     PORT_Memcpy(tmpOutput, input, inputLen);
michael@0: 
michael@0:     /* 3.c - Generate seedMask */
michael@0:     MGF1(maskHashAlg, mask, hash->length, &tmpOutput[1 + hash->length],
michael@0:          inputLen - hash->length - 1);
michael@0:     /* 3.d - Unmask seed */
michael@0:     for (i = 0; i < hash->length; ++i)
michael@0:         tmpOutput[1 + i] ^= mask[i];
michael@0: 
michael@0:     /* 3.e - Generate dbMask */
michael@0:     MGF1(maskHashAlg, mask, maskLen, &tmpOutput[1], hash->length);
michael@0:     /* 3.f - Unmask DB */
michael@0:     for (i = 0; i < maskLen; ++i)
michael@0:         tmpOutput[1 + hash->length + i] ^= mask[i];
michael@0: 
michael@0:     /* 3.g - Compare Y, lHash, and PS in constant time
michael@0:      * Warning: This code is timing dependent and must not disclose which of
michael@0:      * these were invalid.
michael@0:      */
michael@0:     paddingOffset = 0;
michael@0:     isGood = 1;
michael@0:     foundPaddingEnd = 0;
michael@0: 
michael@0:     /* Compare Y */
michael@0:     isGood &= constantTimeEQ8(0x00, tmpOutput[0]);
michael@0: 
michael@0:     /* Compare lHash and lHash' */
michael@0:     isGood &= constantTimeCompare(&labelHash[0],
michael@0:                                   &tmpOutput[1 + hash->length],
michael@0:                                   hash->length);
michael@0: 
michael@0:     /* Compare that the padding is zero or more zero octets, followed by a
michael@0:      * 0x01 octet */
michael@0:     for (i = 1 + (hash->length * 2); i < inputLen; ++i) {
michael@0:         unsigned char isZero = constantTimeEQ8(0x00, tmpOutput[i]);
michael@0:         unsigned char isOne = constantTimeEQ8(0x01, tmpOutput[i]);
michael@0:         /* non-constant time equivalent:
michael@0:          * if (tmpOutput[i] == 0x01 && !foundPaddingEnd)
michael@0:          *     paddingOffset = i;
michael@0:          */
michael@0:         paddingOffset = constantTimeCondition(isOne & ~foundPaddingEnd, i,
michael@0:                                               paddingOffset);
michael@0:         /* non-constant time equivalent:
michael@0:          * if (tmpOutput[i] == 0x01)
michael@0:          *    foundPaddingEnd = true;
michael@0:          *
michael@0:          * Note: This may yield false positives, as it will be set whenever
michael@0:          * a 0x01 byte is encountered. If there was bad padding (eg:
michael@0:          * 0x03 0x02 0x01), foundPaddingEnd will still be set to true, and
michael@0:          * paddingOffset will still be set to 2.
michael@0:          */
michael@0:         foundPaddingEnd = constantTimeCondition(isOne, 1, foundPaddingEnd);
michael@0:         /* non-constant time equivalent:
michael@0:          * if (tmpOutput[i] != 0x00 && tmpOutput[i] != 0x01 &&
michael@0:          *     !foundPaddingEnd) {
michael@0:          *    isGood = false;
michael@0:          * }
michael@0:          *
michael@0:          * Note: This may yield false positives, as a message (and padding)
michael@0:          * that is entirely zeros will result in isGood still being true. Thus
michael@0:          * it's necessary to check foundPaddingEnd is positive below.
michael@0:          */
michael@0:         isGood = constantTimeCondition(~foundPaddingEnd & ~isZero, 0, isGood);
michael@0:     }
michael@0: 
michael@0:     /* While both isGood and foundPaddingEnd may have false positives, they
michael@0:      * cannot BOTH have false positives. If both are not true, then an invalid
michael@0:      * message was received. Note, this comparison must still be done in constant
michael@0:      * time so as not to leak either condition.
michael@0:      */
michael@0:     if (!(isGood & foundPaddingEnd)) {
michael@0:         PORT_SetError(SEC_ERROR_BAD_DATA);
michael@0:         goto done;
michael@0:     }
michael@0: 
michael@0:     /* End timing dependent code */
michael@0: 
michael@0:     ++paddingOffset; /* Skip the 0x01 following the end of PS */
michael@0: 
michael@0:     *outputLen = inputLen - paddingOffset;
michael@0:     if (*outputLen > maxOutputLen) {
michael@0:         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
michael@0:         goto done;
michael@0:     }
michael@0: 
michael@0:     if (*outputLen)
michael@0:         PORT_Memcpy(output, &tmpOutput[paddingOffset], *outputLen);
michael@0:     rv = SECSuccess;
michael@0: 
michael@0: done:
michael@0:     if (mask)
michael@0:         PORT_ZFree(mask, maskLen);
michael@0:     if (tmpOutput)
michael@0:         PORT_ZFree(tmpOutput, inputLen);
michael@0:     return rv;
michael@0: }
michael@0: 
michael@0: /*
michael@0:  * Generate an EME-OAEP encoded block for encryption
michael@0:  * Described in RFC 3447, section 7.1.1
michael@0:  * We use input instead of M for the message to be encrypted
michael@0:  * label is the optional value L to be associated with the message.
michael@0:  */
michael@0: static SECStatus
michael@0: eme_oaep_encode(unsigned char * em,
michael@0:                 unsigned int emLen,
michael@0:                 const unsigned char * input,
michael@0:                 unsigned int inputLen,
michael@0:                 HASH_HashType hashAlg,
michael@0:                 HASH_HashType maskHashAlg,
michael@0:                 const unsigned char * label,
michael@0:                 unsigned int labelLen,
michael@0:                 const unsigned char * seed,
michael@0:                 unsigned int seedLen)
michael@0: {
michael@0:     const SECHashObject * hash;
michael@0:     void * hashContext;
michael@0:     SECStatus rv;
michael@0:     unsigned char * mask;
michael@0:     unsigned int reservedLen;
michael@0:     unsigned int dbMaskLen;
michael@0:     unsigned int i;
michael@0: 
michael@0:     hash = HASH_GetRawHashObject(hashAlg);
michael@0:     PORT_Assert(seed == NULL || seedLen == hash->length);
michael@0: 
michael@0:     /* Step 1.b */
michael@0:     reservedLen = (2 * hash->length) + 2;
michael@0:     if (emLen < reservedLen || inputLen > (emLen - reservedLen)) {
michael@0:         PORT_SetError(SEC_ERROR_INPUT_LEN);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     /*
michael@0:      * From RFC 3447, Section 7.1
michael@0:      *                      +----------+---------+-------+
michael@0:      *                 DB = |  lHash   |    PS   |   M   |
michael@0:      *                      +----------+---------+-------+
michael@0:      *                                     |
michael@0:      *           +----------+              V
michael@0:      *           |   seed   |--> MGF ---> xor
michael@0:      *           +----------+              |
michael@0:      *                 |                   |
michael@0:      *        +--+     V                   |
michael@0:      *        |00|    xor <----- MGF <-----|
michael@0:      *        +--+     |                   |
michael@0:      *          |      |                   |
michael@0:      *          V      V                   V
michael@0:      *        +--+----------+----------------------------+
michael@0:      *  EM =  |00|maskedSeed|          maskedDB          |
michael@0:      *        +--+----------+----------------------------+
michael@0:      *
michael@0:      * We use mask to hold the result of the MGF functions, and all other
michael@0:      * values are generated in their final resting place.
michael@0:      */
michael@0:     *em = 0x00;
michael@0: 
michael@0:     /* Step 2.a - Generate lHash */
michael@0:     hashContext = (*hash->create)();
michael@0:     if (hashContext == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     (*hash->begin)(hashContext);
michael@0:     if (labelLen > 0)
michael@0:         (*hash->update)(hashContext, label, labelLen);
michael@0:     (*hash->end)(hashContext, &em[1 + hash->length], &i, hash->length);
michael@0:     (*hash->destroy)(hashContext, PR_TRUE);
michael@0: 
michael@0:     /* Step 2.b - Generate PS */
michael@0:     if (emLen - reservedLen - inputLen > 0) {
michael@0:         PORT_Memset(em + 1 + (hash->length * 2), 0x00,
michael@0:                     emLen - reservedLen - inputLen);
michael@0:     }
michael@0: 
michael@0:     /* Step 2.c. - Generate DB
michael@0:      * DB = lHash || PS || 0x01 || M
michael@0:      * Note that PS and lHash have already been placed into em at their
michael@0:      * appropriate offsets. This just copies M into place
michael@0:      */
michael@0:     em[emLen - inputLen - 1] = 0x01;
michael@0:     if (inputLen)
michael@0:         PORT_Memcpy(em + emLen - inputLen, input, inputLen);
michael@0: 
michael@0:     if (seed == NULL) {
michael@0:         /* Step 2.d - Generate seed */
michael@0:         rv = RNG_GenerateGlobalRandomBytes(em + 1, hash->length);
michael@0:         if (rv != SECSuccess) {
michael@0:             return rv;
michael@0:         }
michael@0:     } else {
michael@0:         /* For Known Answer Tests, copy the supplied seed. */
michael@0:         PORT_Memcpy(em + 1, seed, seedLen);
michael@0:     }
michael@0: 
michael@0:     /* Step 2.e - Generate dbMask*/
michael@0:     dbMaskLen = emLen - hash->length - 1;
michael@0:     mask = (unsigned char*)PORT_Alloc(dbMaskLen);
michael@0:     if (mask == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     MGF1(maskHashAlg, mask, dbMaskLen, em + 1, hash->length);
michael@0:     /* Step 2.f - Compute maskedDB*/
michael@0:     for (i = 0; i < dbMaskLen; ++i)
michael@0:         em[1 + hash->length + i] ^= mask[i];
michael@0: 
michael@0:     /* Step 2.g - Generate seedMask */
michael@0:     MGF1(maskHashAlg, mask, hash->length, &em[1 + hash->length], dbMaskLen);
michael@0:     /* Step 2.h - Compute maskedSeed */
michael@0:     for (i = 0; i < hash->length; ++i)
michael@0:         em[1 + i] ^= mask[i];
michael@0: 
michael@0:     PORT_ZFree(mask, dbMaskLen);
michael@0:     return SECSuccess;
michael@0: }
michael@0: 
michael@0: SECStatus
michael@0: RSA_EncryptOAEP(RSAPublicKey * key,
michael@0:                 HASH_HashType hashAlg,
michael@0:                 HASH_HashType maskHashAlg,
michael@0:                 const unsigned char * label,
michael@0:                 unsigned int labelLen,
michael@0:                 const unsigned char * seed,
michael@0:                 unsigned int seedLen,
michael@0:                 unsigned char * output,
michael@0:                 unsigned int * outputLen,
michael@0:                 unsigned int maxOutputLen,
michael@0:                 const unsigned char * input,
michael@0:                 unsigned int inputLen)
michael@0: {
michael@0:     SECStatus rv = SECFailure;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     unsigned char * oaepEncoded = NULL;
michael@0: 
michael@0:     if (maxOutputLen < modulusLen) {
michael@0:         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
michael@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     if ((labelLen == 0 && label != NULL) ||
michael@0:         (labelLen > 0 && label == NULL)) {
michael@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
michael@0:     if (oaepEncoded == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     rv = eme_oaep_encode(oaepEncoded, modulusLen, input, inputLen,
michael@0:                          hashAlg, maskHashAlg, label, labelLen, seed, seedLen);
michael@0:     if (rv != SECSuccess)
michael@0:         goto done;
michael@0: 
michael@0:     rv = RSA_PublicKeyOp(key, output, oaepEncoded);
michael@0:     if (rv != SECSuccess)
michael@0:         goto done;
michael@0:     *outputLen = modulusLen;
michael@0: 
michael@0: done:
michael@0:     PORT_Free(oaepEncoded);
michael@0:     return rv;
michael@0: }
michael@0: 
michael@0: SECStatus
michael@0: RSA_DecryptOAEP(RSAPrivateKey * key,
michael@0:                 HASH_HashType hashAlg,
michael@0:                 HASH_HashType maskHashAlg,
michael@0:                 const unsigned char * label,
michael@0:                 unsigned int labelLen,
michael@0:                 unsigned char * output,
michael@0:                 unsigned int * outputLen,
michael@0:                 unsigned int maxOutputLen,
michael@0:                 const unsigned char * input,
michael@0:                 unsigned int inputLen)
michael@0: {
michael@0:     SECStatus rv = SECFailure;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     unsigned char * oaepEncoded = NULL;
michael@0: 
michael@0:     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
michael@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     if (inputLen != modulusLen) {
michael@0:         PORT_SetError(SEC_ERROR_INPUT_LEN);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     if ((labelLen == 0 && label != NULL) ||
michael@0:         (labelLen > 0 && label == NULL)) {
michael@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
michael@0:     if (oaepEncoded == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     rv = RSA_PrivateKeyOpDoubleChecked(key, oaepEncoded, input);
michael@0:     if (rv != SECSuccess) {
michael@0:         goto done;
michael@0:     }
michael@0:     rv = eme_oaep_decode(output, outputLen, maxOutputLen, oaepEncoded,
michael@0:                          modulusLen, hashAlg, maskHashAlg, label,
michael@0:                          labelLen);
michael@0: 
michael@0: done:
michael@0:     if (oaepEncoded)
michael@0:         PORT_ZFree(oaepEncoded, modulusLen);
michael@0:     return rv;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_EncryptBlock(RSAPublicKey * key,
michael@0:                  unsigned char * output,
michael@0:                  unsigned int * outputLen,
michael@0:                  unsigned int maxOutputLen,
michael@0:                  const unsigned char * input,
michael@0:                  unsigned int inputLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     SECItem formatted;
michael@0:     SECItem unformatted;
michael@0: 
michael@0:     formatted.data = NULL;
michael@0:     if (maxOutputLen < modulusLen)
michael@0:         goto failure;
michael@0: 
michael@0:     unformatted.len  = inputLen;
michael@0:     unformatted.data = (unsigned char*)input;
michael@0:     formatted.data   = NULL;
michael@0:     rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPublic,
michael@0:                          &unformatted);
michael@0:     if (rv != SECSuccess)
michael@0:         goto failure;
michael@0: 
michael@0:     rv = RSA_PublicKeyOp(key, output, formatted.data);
michael@0:     if (rv != SECSuccess)
michael@0:         goto failure;
michael@0: 
michael@0:     PORT_ZFree(formatted.data, modulusLen);
michael@0:     *outputLen = modulusLen;
michael@0:     return SECSuccess;
michael@0: 
michael@0: failure:
michael@0:     if (formatted.data != NULL)
michael@0:         PORT_ZFree(formatted.data, modulusLen);
michael@0:     return SECFailure;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_DecryptBlock(RSAPrivateKey * key,
michael@0:                  unsigned char * output,
michael@0:                  unsigned int * outputLen,
michael@0:                  unsigned int maxOutputLen,
michael@0:                  const unsigned char * input,
michael@0:                  unsigned int inputLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     unsigned int i;
michael@0:     unsigned char * buffer;
michael@0: 
michael@0:     if (inputLen != modulusLen)
michael@0:         goto failure;
michael@0: 
michael@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
michael@0:     if (!buffer)
michael@0:         goto failure;
michael@0: 
michael@0:     rv = RSA_PrivateKeyOp(key, buffer, input);
michael@0:     if (rv != SECSuccess)
michael@0:         goto loser;
michael@0: 
michael@0:     /* XXX(rsleevi): Constant time */
michael@0:     if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
michael@0:         buffer[1] != (unsigned char)RSA_BlockPublic) {
michael@0:         goto loser;
michael@0:     }
michael@0:     *outputLen = 0;
michael@0:     for (i = 2; i < modulusLen; i++) {
michael@0:         if (buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET) {
michael@0:             *outputLen = modulusLen - i - 1;
michael@0:             break;
michael@0:         }
michael@0:     }
michael@0:     if (*outputLen == 0)
michael@0:         goto loser;
michael@0:     if (*outputLen > maxOutputLen)
michael@0:         goto loser;
michael@0: 
michael@0:     PORT_Memcpy(output, buffer + modulusLen - *outputLen, *outputLen);
michael@0: 
michael@0:     PORT_Free(buffer);
michael@0:     return SECSuccess;
michael@0: 
michael@0: loser:
michael@0:     PORT_Free(buffer);
michael@0: failure:
michael@0:     return SECFailure;
michael@0: }
michael@0: 
michael@0: /*
michael@0:  * Encode a RSA-PSS signature.
michael@0:  * Described in RFC 3447, section 9.1.1.
michael@0:  * We use mHash instead of M as input.
michael@0:  * emBits from the RFC is just modBits - 1, see section 8.1.1.
michael@0:  * We only support MGF1 as the MGF.
michael@0:  *
michael@0:  * NOTE: this code assumes modBits is a multiple of 8.
michael@0:  */
michael@0: static SECStatus
michael@0: emsa_pss_encode(unsigned char * em,
michael@0:                 unsigned int emLen,
michael@0:                 const unsigned char * mHash,
michael@0:                 HASH_HashType hashAlg,
michael@0:                 HASH_HashType maskHashAlg,
michael@0:                 const unsigned char * salt,
michael@0:                 unsigned int saltLen)
michael@0: {
michael@0:     const SECHashObject * hash;
michael@0:     void * hash_context;
michael@0:     unsigned char * dbMask;
michael@0:     unsigned int dbMaskLen;
michael@0:     unsigned int i;
michael@0:     SECStatus rv;
michael@0: 
michael@0:     hash = HASH_GetRawHashObject(hashAlg);
michael@0:     dbMaskLen = emLen - hash->length - 1;
michael@0: 
michael@0:     /* Step 3 */
michael@0:     if (emLen < hash->length + saltLen + 2) {
michael@0:         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     /* Step 4 */
michael@0:     if (salt == NULL) {
michael@0:         rv = RNG_GenerateGlobalRandomBytes(&em[dbMaskLen - saltLen], saltLen);
michael@0:         if (rv != SECSuccess) {
michael@0:             return rv;
michael@0:         }
michael@0:     } else {
michael@0:         PORT_Memcpy(&em[dbMaskLen - saltLen], salt, saltLen);
michael@0:     }
michael@0: 
michael@0:     /* Step 5 + 6 */
michael@0:     /* Compute H and store it at its final location &em[dbMaskLen]. */
michael@0:     hash_context = (*hash->create)();
michael@0:     if (hash_context == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     (*hash->begin)(hash_context);
michael@0:     (*hash->update)(hash_context, eightZeros, 8);
michael@0:     (*hash->update)(hash_context, mHash, hash->length);
michael@0:     (*hash->update)(hash_context, &em[dbMaskLen - saltLen], saltLen);
michael@0:     (*hash->end)(hash_context, &em[dbMaskLen], &i, hash->length);
michael@0:     (*hash->destroy)(hash_context, PR_TRUE);
michael@0: 
michael@0:     /* Step 7 + 8 */
michael@0:     PORT_Memset(em, 0, dbMaskLen - saltLen - 1);
michael@0:     em[dbMaskLen - saltLen - 1] = 0x01;
michael@0: 
michael@0:     /* Step 9 */
michael@0:     dbMask = (unsigned char *)PORT_Alloc(dbMaskLen);
michael@0:     if (dbMask == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     MGF1(maskHashAlg, dbMask, dbMaskLen, &em[dbMaskLen], hash->length);
michael@0: 
michael@0:     /* Step 10 */
michael@0:     for (i = 0; i < dbMaskLen; i++)
michael@0:         em[i] ^= dbMask[i];
michael@0:     PORT_Free(dbMask);
michael@0: 
michael@0:     /* Step 11 */
michael@0:     em[0] &= 0x7f;
michael@0: 
michael@0:     /* Step 12 */
michael@0:     em[emLen - 1] = 0xbc;
michael@0: 
michael@0:     return SECSuccess;
michael@0: }
michael@0: 
michael@0: /*
michael@0:  * Verify a RSA-PSS signature.
michael@0:  * Described in RFC 3447, section 9.1.2.
michael@0:  * We use mHash instead of M as input.
michael@0:  * emBits from the RFC is just modBits - 1, see section 8.1.2.
michael@0:  * We only support MGF1 as the MGF.
michael@0:  *
michael@0:  * NOTE: this code assumes modBits is a multiple of 8.
michael@0:  */
michael@0: static SECStatus
michael@0: emsa_pss_verify(const unsigned char * mHash,
michael@0:                 const unsigned char * em,
michael@0:                 unsigned int emLen,
michael@0:                 HASH_HashType hashAlg,
michael@0:                 HASH_HashType maskHashAlg,
michael@0:                 unsigned int saltLen)
michael@0: {
michael@0:     const SECHashObject * hash;
michael@0:     void * hash_context;
michael@0:     unsigned char * db;
michael@0:     unsigned char * H_;  /* H' from the RFC */
michael@0:     unsigned int i;
michael@0:     unsigned int dbMaskLen;
michael@0:     SECStatus rv;
michael@0: 
michael@0:     hash = HASH_GetRawHashObject(hashAlg);
michael@0:     dbMaskLen = emLen - hash->length - 1;
michael@0: 
michael@0:     /* Step 3 + 4 + 6 */
michael@0:     if ((emLen < (hash->length + saltLen + 2)) ||
michael@0:         (em[emLen - 1] != 0xbc) ||
michael@0:         ((em[0] & 0x80) != 0)) {
michael@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     /* Step 7 */
michael@0:     db = (unsigned char *)PORT_Alloc(dbMaskLen);
michael@0:     if (db == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     /* &em[dbMaskLen] points to H, used as mgfSeed */
michael@0:     MGF1(maskHashAlg, db, dbMaskLen, &em[dbMaskLen], hash->length);
michael@0: 
michael@0:     /* Step 8 */
michael@0:     for (i = 0; i < dbMaskLen; i++) {
michael@0:         db[i] ^= em[i];
michael@0:     }
michael@0: 
michael@0:     /* Step 9 */
michael@0:     db[0] &= 0x7f;
michael@0: 
michael@0:     /* Step 10 */
michael@0:     for (i = 0; i < (dbMaskLen - saltLen - 1); i++) {
michael@0:         if (db[i] != 0) {
michael@0:             PORT_Free(db);
michael@0:             PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
michael@0:             return SECFailure;
michael@0:         }
michael@0:     }
michael@0:     if (db[dbMaskLen - saltLen - 1] != 0x01) {
michael@0:         PORT_Free(db);
michael@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     /* Step 12 + 13 */
michael@0:     H_ = (unsigned char *)PORT_Alloc(hash->length);
michael@0:     if (H_ == NULL) {
michael@0:         PORT_Free(db);
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     hash_context = (*hash->create)();
michael@0:     if (hash_context == NULL) {
michael@0:         PORT_Free(db);
michael@0:         PORT_Free(H_);
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     (*hash->begin)(hash_context);
michael@0:     (*hash->update)(hash_context, eightZeros, 8);
michael@0:     (*hash->update)(hash_context, mHash, hash->length);
michael@0:     (*hash->update)(hash_context, &db[dbMaskLen - saltLen], saltLen);
michael@0:     (*hash->end)(hash_context, H_, &i, hash->length);
michael@0:     (*hash->destroy)(hash_context, PR_TRUE);
michael@0: 
michael@0:     PORT_Free(db);
michael@0: 
michael@0:     /* Step 14 */
michael@0:     if (PORT_Memcmp(H_, &em[dbMaskLen], hash->length) != 0) {
michael@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
michael@0:         rv = SECFailure;
michael@0:     } else {
michael@0:         rv = SECSuccess;
michael@0:     }
michael@0: 
michael@0:     PORT_Free(H_);
michael@0:     return rv;
michael@0: }
michael@0: 
michael@0: SECStatus
michael@0: RSA_SignPSS(RSAPrivateKey * key,
michael@0:             HASH_HashType hashAlg,
michael@0:             HASH_HashType maskHashAlg,
michael@0:             const unsigned char * salt,
michael@0:             unsigned int saltLength,
michael@0:             unsigned char * output,
michael@0:             unsigned int * outputLen,
michael@0:             unsigned int maxOutputLen,
michael@0:             const unsigned char * input,
michael@0:             unsigned int inputLen)
michael@0: {
michael@0:     SECStatus rv = SECSuccess;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     unsigned char *pssEncoded = NULL;
michael@0: 
michael@0:     if (maxOutputLen < modulusLen) {
michael@0:         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
michael@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     pssEncoded = (unsigned char *)PORT_Alloc(modulusLen);
michael@0:     if (pssEncoded == NULL) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0:     rv = emsa_pss_encode(pssEncoded, modulusLen, input, hashAlg,
michael@0:                          maskHashAlg, salt, saltLength);
michael@0:     if (rv != SECSuccess)
michael@0:         goto done;
michael@0: 
michael@0:     rv = RSA_PrivateKeyOpDoubleChecked(key, output, pssEncoded);
michael@0:     *outputLen = modulusLen;
michael@0: 
michael@0: done:
michael@0:     PORT_Free(pssEncoded);
michael@0:     return rv;
michael@0: }
michael@0: 
michael@0: SECStatus
michael@0: RSA_CheckSignPSS(RSAPublicKey * key,
michael@0:                  HASH_HashType hashAlg,
michael@0:                  HASH_HashType maskHashAlg,
michael@0:                  unsigned int saltLength,
michael@0:                  const unsigned char * sig,
michael@0:                  unsigned int sigLen,
michael@0:                  const unsigned char * hash,
michael@0:                  unsigned int hashLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     unsigned char * buffer;
michael@0: 
michael@0:     if (sigLen != modulusLen) {
michael@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
michael@0:         PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen);
michael@0:     if (!buffer) {
michael@0:         PORT_SetError(SEC_ERROR_NO_MEMORY);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     rv = RSA_PublicKeyOp(key, buffer, sig);
michael@0:     if (rv != SECSuccess) {
michael@0:         PORT_Free(buffer);
michael@0:         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
michael@0:         return SECFailure;
michael@0:     }
michael@0: 
michael@0:     rv = emsa_pss_verify(hash, buffer, modulusLen, hashAlg,
michael@0:                          maskHashAlg, saltLength);
michael@0:     PORT_Free(buffer);
michael@0: 
michael@0:     return rv;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_Sign(RSAPrivateKey * key,
michael@0:          unsigned char * output,
michael@0:          unsigned int * outputLen,
michael@0:          unsigned int maxOutputLen,
michael@0:          const unsigned char * input,
michael@0:          unsigned int inputLen)
michael@0: {
michael@0:     SECStatus rv = SECSuccess;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     SECItem formatted;
michael@0:     SECItem unformatted;
michael@0: 
michael@0:     if (maxOutputLen < modulusLen)
michael@0:         return SECFailure;
michael@0: 
michael@0:     unformatted.len  = inputLen;
michael@0:     unformatted.data = (unsigned char*)input;
michael@0:     formatted.data   = NULL;
michael@0:     rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPrivate,
michael@0:                          &unformatted);
michael@0:     if (rv != SECSuccess)
michael@0:         goto done;
michael@0: 
michael@0:     rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
michael@0:     *outputLen = modulusLen;
michael@0: 
michael@0:     goto done;
michael@0: 
michael@0: done:
michael@0:     if (formatted.data != NULL)
michael@0:         PORT_ZFree(formatted.data, modulusLen);
michael@0:     return rv;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_CheckSign(RSAPublicKey * key,
michael@0:               const unsigned char * sig,
michael@0:               unsigned int sigLen,
michael@0:               const unsigned char * data,
michael@0:               unsigned int dataLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     unsigned int i;
michael@0:     unsigned char * buffer;
michael@0: 
michael@0:     if (sigLen != modulusLen)
michael@0:         goto failure;
michael@0:     /*
michael@0:      * 0x00 || BT || Pad || 0x00 || ActualData
michael@0:      *
michael@0:      * The "3" below is the first octet + the second octet + the 0x00
michael@0:      * octet that always comes just before the ActualData.
michael@0:      */
michael@0:     if (dataLen > modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN))
michael@0:         goto failure;
michael@0: 
michael@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
michael@0:     if (!buffer)
michael@0:         goto failure;
michael@0: 
michael@0:     rv = RSA_PublicKeyOp(key, buffer, sig);
michael@0:     if (rv != SECSuccess)
michael@0:         goto loser;
michael@0: 
michael@0:     /*
michael@0:      * check the padding that was used
michael@0:      */
michael@0:     if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
michael@0:         buffer[1] != (unsigned char)RSA_BlockPrivate) {
michael@0:         goto loser;
michael@0:     }
michael@0:     for (i = 2; i < modulusLen - dataLen - 1; i++) {
michael@0:         if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET)
michael@0:             goto loser;
michael@0:     }
michael@0:     if (buffer[i] != RSA_BLOCK_AFTER_PAD_OCTET)
michael@0:         goto loser;
michael@0: 
michael@0:     /*
michael@0:      * make sure we get the same results
michael@0:      */
michael@0:     if (PORT_Memcmp(buffer + modulusLen - dataLen, data, dataLen) != 0)
michael@0:         goto loser;
michael@0: 
michael@0:     PORT_Free(buffer);
michael@0:     return SECSuccess;
michael@0: 
michael@0: loser:
michael@0:     PORT_Free(buffer);
michael@0: failure:
michael@0:     return SECFailure;
michael@0: }
michael@0: 
michael@0: /* XXX Doesn't set error code */
michael@0: SECStatus
michael@0: RSA_CheckSignRecover(RSAPublicKey * key,
michael@0:                      unsigned char * output,
michael@0:                      unsigned int * outputLen,
michael@0:                      unsigned int maxOutputLen,
michael@0:                      const unsigned char * sig,
michael@0:                      unsigned int sigLen)
michael@0: {
michael@0:     SECStatus rv;
michael@0:     unsigned int modulusLen = rsa_modulusLen(&key->modulus);
michael@0:     unsigned int i;
michael@0:     unsigned char * buffer;
michael@0: 
michael@0:     if (sigLen != modulusLen)
michael@0:         goto failure;
michael@0: 
michael@0:     buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
michael@0:     if (!buffer)
michael@0:         goto failure;
michael@0: 
michael@0:     rv = RSA_PublicKeyOp(key, buffer, sig);
michael@0:     if (rv != SECSuccess)
michael@0:         goto loser;
michael@0:     *outputLen = 0;
michael@0: 
michael@0:     /*
michael@0:      * check the padding that was used
michael@0:      */
michael@0:     if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
michael@0:         buffer[1] != (unsigned char)RSA_BlockPrivate) {
michael@0:         goto loser;
michael@0:     }
michael@0:     for (i = 2; i < modulusLen; i++) {
michael@0:         if (buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET) {
michael@0:             *outputLen = modulusLen - i - 1;
michael@0:             break;
michael@0:         }
michael@0:         if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET)
michael@0:             goto loser;
michael@0:     }
michael@0:     if (*outputLen == 0)
michael@0:         goto loser;
michael@0:     if (*outputLen > maxOutputLen)
michael@0:         goto loser;
michael@0: 
michael@0:     PORT_Memcpy(output, buffer + modulusLen - *outputLen, *outputLen);
michael@0: 
michael@0:     PORT_Free(buffer);
michael@0:     return SECSuccess;
michael@0: 
michael@0: loser:
michael@0:     PORT_Free(buffer);
michael@0: failure:
michael@0:     return SECFailure;
michael@0: }