- lib: check return values of RSA_*() functions.

Also fix a memory leak, missing RSA_free().
    Fixes #823.

svn path=/trunk/boinc/; revision=16883

checkin_notes

@@ -134,3 +134,13 @@ David  Jan 11 2009
         sched_plan.cpp
         sched_send.cpp
         server_types.h
+
+David  Jan 12 2009
+    - lib: check return values of RSA_*() functions.
+        Also fix a memory leak, missing RSA_free().
+        Fixes #823.
+
+    lib/
+        crypt.cpp
+        error_numbers.h
+        str_util.cpp

lib/crypt.cpp

@@ -243,7 +243,7 @@ int sscan_key_hex(const char* buf, KEY* key, int size) {
 // The output block must be decrypted in its entirety.
 //
 int encrypt_private(R_RSA_PRIVATE_KEY& key, DATA_BLOCK& in, DATA_BLOCK& out) {
-    int n, modulus_len;
+    int n, modulus_len, retval;
 
     modulus_len = (key.bits+7)/8;
     n = in.len;
@@ -252,17 +252,27 @@ int encrypt_private(R_RSA_PRIVATE_KEY& key, DATA_BLOCK& in, DATA_BLOCK& out) {
     }
     RSA* rp = RSA_new();
     private_to_openssl(key, rp);
-    RSA_private_encrypt(n, in.data, out.data, rp, RSA_PKCS1_PADDING);
+    retval = RSA_private_encrypt(n, in.data, out.data, rp, RSA_PKCS1_PADDING);
+    if (retval < 0) {
+        RSA_free(rp);
+        return ERR_CRYPTO;
+    }
     out.len = RSA_size(rp);
     RSA_free(rp);
     return 0;
 }
 
 int decrypt_public(R_RSA_PUBLIC_KEY& key, DATA_BLOCK& in, DATA_BLOCK& out) {
+    int retval;
     RSA* rp = RSA_new();
     public_to_openssl(key, rp);
-    RSA_public_decrypt(in.len, in.data, out.data, rp, RSA_PKCS1_PADDING);
+    retval = RSA_public_decrypt(in.len, in.data, out.data, rp, RSA_PKCS1_PADDING);
+    if (retval < 0) {
+        RSA_free(rp);
+        return ERR_CRYPTO;
+    }
     out.len = RSA_size(rp);
+    RSA_free(rp);
     return 0;
 }
 

lib/error_numbers.h

@@ -185,6 +185,7 @@
 #define ERR_RMDIR           -227
 #define ERR_SYMLINK         -229
 #define ERR_DB_CONN_LOST    -230
+#define ERR_CRYPTO          -231
 
 // PLEASE: add a text description of your error to 
 // the text description function boincerror() in str_util.C.

lib/str_util.cpp

@@ -735,6 +735,7 @@ const char* boincerror(int which_error) {
         case ERR_RMDIR: return "rmdir() failed";
         case ERR_SYMLINK: return "symlink() failed";
         case ERR_DB_CONN_LOST: return "DB connection lost during enumeration";
+        case ERR_CRYPTO: return "encryption error";
         case 404: return "HTTP file not found";
         case 407: return "HTTP proxy authentication failure";
         case 416: return "HTTP range request error";