android: verify downloaded data against digests from blobrefs

clients/android/src/org/camlistore/BlobVerifier.java

@@ -0,0 +1,78 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.camlistore;
+
+import android.util.Log;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+// Used to verify that the digest of a blob's contents match the digest from its blobref.
+class BlobVerifier {
+    private static final String TAG = "BlobVerifier";
+    private static final String SHA1_PREFIX = "sha1-";
+    private static final String MD5_PREFIX = "md5-";
+
+    private final String mBlobRef;
+
+    // Effectively-final members assigned either in the c'tor or in a method that it calls.
+    private String mExpectedDigest;
+    private MessageDigest mDigester;
+
+    // Initializes a verifier for |blobRef|.
+    BlobVerifier(String blobRef) {
+        mBlobRef = blobRef;
+
+        if (mBlobRef.startsWith(SHA1_PREFIX)) {
+            initForAlgorithm(SHA1_PREFIX, "SHA-1");
+        } else if (mBlobRef.startsWith(MD5_PREFIX)) {
+            initForAlgorithm(MD5_PREFIX, "MD5");
+        }
+    }
+
+    // Update the digest using the blob's bytes.
+    // Can be invoked repeatedly with successive chunks as the blob is being downloaded.
+    public void processBytes(byte[] bytes, int offset, int length) {
+        if (mDigester != null)
+            mDigester.update(bytes, offset, length);
+    }
+
+    // Do the blob's contents match its blobref?
+    // Returns true if the contents are valid or if the blob is using an unknown algorithm.
+    public boolean isBlobValid() {
+        if (mDigester == null)
+            return true;
+
+        final String actualDigest = Util.getHex(mDigester.digest());
+        return actualDigest.equals(mExpectedDigest);
+    }
+
+    // Helper method called by the constructor.
+    // Initializes |mExpectedDigest| and |mDigester| for a blobref starting with |blobRefPrefix|
+    // and that's using |algorithmName| (an algorithm known by MessageDigest).
+    private void initForAlgorithm(String blobRefPrefix, String algorithmName) {
+        if (!mBlobRef.startsWith(blobRefPrefix))
+            throw new RuntimeException("blobref " + mBlobRef + " doesn't start with " + blobRefPrefix);
+        mExpectedDigest = mBlobRef.substring(blobRefPrefix.length(), mBlobRef.length()).toLowerCase();
+
+        try {
+            mDigester = MessageDigest.getInstance(algorithmName);
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

clients/android/src/org/camlistore/DownloadCache.java

@@ -121,23 +121,37 @@ class DownloadCache {
         return file;
     }
 
+    // Status passed to handleDoneWritingTempFile().
+    public enum WriteStatus {
+        // |tempFile| is renamed to indicate that it's fully downloaded and the file's final location is returned.
+        SUCCESS,
+        // |tempFile| is kept on-disk and NULL is returned.
+        FAILURE_KEEP,
+        // |tempFile| is deleted.
+        FAILURE_DELETE,
+    }
+
     // Handle the completion (either successful or not) of a download to a file returned by getTempFileForDownload().
-    // If |successfullyWritten| is true, |tempFile| will be renamed to indicate that it's fully downloaded and the
+    // The exact behavior depends on the value of |status|.
-    // file's final location will be returned.
+    public File handleDoneWritingTempFile(File tempFile, WriteStatus status) {
-    public File handleDoneWritingTempFile(File tempFile, boolean successfullyWritten) {
         Util.assertNotMainThread();
         mLock.lock();
         try {
             while (!mIsReady)
                 try { mIsReadyCondition.await(); } catch (InterruptedException e) {}
-            mUsedBytes += tempFile.length();
             if (!mPinnedPaths.remove(tempFile.getAbsolutePath()))
                 throw new RuntimeException("unknown temp file " + tempFile.getPath());
+
+            if (status == WriteStatus.FAILURE_DELETE) {
+                tempFile.delete();
+            } else {
+                mUsedBytes += tempFile.length();
+            }
         } finally {
             mLock.unlock();
         }
 
-        if (!successfullyWritten)
+        if (status != WriteStatus.SUCCESS)
             return null;
 
         final String name = tempFile.getName();

clients/android/src/org/camlistore/DownloadService.java

@@ -230,6 +230,10 @@ public class DownloadService extends Service {
             // Temporary location in the cache where we write the blob.
             File tempFile = null;
 
+            // Set to true if the downloaded data didn't match the digest from the blobref.
+            // If |tempFile| exists, we need to throw it away.
+            boolean dataIsCorrupt = false;
+
             try {
                 final HttpResponse response = httpClient.execute(req);
                 final HttpEntity entity = response.getEntity();
@@ -266,16 +270,29 @@ public class DownloadService extends Service {
                     throw new RuntimeException("blob " + mBlobRef + " can't be cached but has a file listener");
                 }
 
+                BlobVerifier verifier = new BlobVerifier(mBlobRef);
                 int bytesRead = 0;
                 long totalBytesWritten = 0;
                 byte[] buffer = new byte[BUFFER_SIZE];
                 while ((bytesRead = inputStream.read(buffer)) != -1) {
                     outputStream.write(buffer, 0, bytesRead);
                     totalBytesWritten += bytesRead;
+                    verifier.processBytes(buffer, 0, bytesRead);
                 }
+
+                // This is unnecessary since we verify the digest, but I'm leaving it in since it'll be needed after
+                // support for resuming downloads is added (we don't want to delete a partial download as a result of it
+                // not matching the expected digest).
                 if (contentLength > 0 && totalBytesWritten != contentLength) {
                     Log.e(TAG, "got " + totalBytesWritten + " byte(s) for " + mBlobRef +
                           " but Content-Length header claimed " + contentLength);
+                    if (totalBytesWritten > contentLength)
+                        dataIsCorrupt = true;
+                    return;
+                }
+                if (!verifier.isBlobValid()) {
+                    Log.e(TAG, "blob " + mBlobRef + "'s data doesn't match expected digest");
+                    dataIsCorrupt = true;
                     return;
                 }
 
@@ -301,7 +318,7 @@ public class DownloadService extends Service {
                 }
 
                 if (tempFile != null) {
-                    mBlobFile = mCache.handleDoneWritingTempFile(tempFile, true);
+                    mBlobFile = mCache.handleDoneWritingTempFile(tempFile, DownloadCache.WriteStatus.SUCCESS);
                     tempFile = null;
                 }
             } catch (ClientProtocolException e) {
@@ -315,8 +332,13 @@ public class DownloadService extends Service {
                     try { outputStream.close(); } catch (IOException e) {}
 
                 // Report failure to the cache.
-                if (tempFile != null)
+                if (tempFile != null) {
-                    mCache.handleDoneWritingTempFile(tempFile, false);
+                    DownloadCache.WriteStatus status =
+                        dataIsCorrupt ?
+                        DownloadCache.WriteStatus.FAILURE_DELETE :
+                        DownloadCache.WriteStatus.FAILURE_KEEP;
+                    mCache.handleDoneWritingTempFile(tempFile, status);
+                }
             }
         }
 

clients/android/src/org/camlistore/Util.java

@@ -94,7 +94,7 @@ public class Util {
 
     private static final String HEX = "0123456789abcdef";
 
-    private static String getHex(byte[] raw) {
+    public static String getHex(byte[] raw) {
         if (raw == null) {
             return null;
         }