From c15886898971ffb866fe6a2fce84a868532ce537 Mon Sep 17 00:00:00 2001 From: Aaron Boodman Date: Sat, 4 Jan 2014 10:59:55 -0800 Subject: [PATCH] Move hashing of file uploads into a web worker. We were observing UI jank, which ended up being due to GC. Even with chunking and very small chunk sizes, it was hard to reliably get 30fps on my macbook. Moving to a worker completely solves the problem. We stay at 60fps the entire time the hashing is taking place, no matter how many files/how large, etc. Also, switch to using Closure's crypto support, which has been added since the upload code was originally written, since it hassupport for incremental hashing, javascript typed arrays, and other niceties. Change-Id: I018d6839b2cf037b8d6b03e1a0ea7164bc5a782d --- server/camlistored/ui/Crypto.js | 188 --------------- server/camlistored/ui/SHA1.js | 115 --------- server/camlistored/ui/base64.js | 220 ------------------ server/camlistored/ui/blob.js | 66 ++++++ server/camlistored/ui/blobinfo.html | 5 - server/camlistored/ui/debug.html | 5 - server/camlistored/ui/filetree.html | 5 - server/camlistored/ui/hash_worker.js | 30 +++ server/camlistored/ui/index.html | 6 - server/camlistored/ui/permanode.html | 5 - server/camlistored/ui/server_connection.js | 41 ++-- .../camlistored/ui/worker_message_router.js | 101 ++++++++ 12 files changed, 215 insertions(+), 572 deletions(-) delete mode 100644 server/camlistored/ui/Crypto.js delete mode 100644 server/camlistored/ui/SHA1.js delete mode 100644 server/camlistored/ui/base64.js create mode 100644 server/camlistored/ui/blob.js create mode 100644 server/camlistored/ui/hash_worker.js create mode 100644 server/camlistored/ui/worker_message_router.js diff --git a/server/camlistored/ui/Crypto.js b/server/camlistored/ui/Crypto.js deleted file mode 100644 index 8e1031548..000000000 --- a/server/camlistored/ui/Crypto.js +++ /dev/null @@ -1,188 +0,0 @@ -// From http://code.google.com/p/crypto-js/ -// License: http://www.opensource.org/licenses/bsd-license.php -// -// Copyright (c) 2009, Jeff Mott. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. Redistributions in binary -// form must reproduce the above copyright notice, this list of conditions and -// the following disclaimer in the documentation and/or other materials provided -// with the distribution. Neither the name Crypto-JS nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. THIS SOFTWARE IS PROVIDED -// BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -if (typeof goog != 'undefined' && typeof goog.provide != 'undefined') { - goog.provide('camlistore.Crypto'); -} - -if (typeof Crypto == "undefined" || ! Crypto.util) -{ -(function(){ - -var base64map = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -// Global Crypto object -var Crypto = window.Crypto = {}; - -// Crypto utilities -var util = Crypto.util = { - - // Bit-wise rotate left - rotl: function (n, b) { - return (n << b) | (n >>> (32 - b)); - }, - - // Bit-wise rotate right - rotr: function (n, b) { - return (n << (32 - b)) | (n >>> b); - }, - - // Swap big-endian to little-endian and vice versa - endian: function (n) { - - // If number given, swap endian - if (n.constructor == Number) { - return util.rotl(n, 8) & 0x00FF00FF | - util.rotl(n, 24) & 0xFF00FF00; - } - - // Else, assume array and swap all items - for (var i = 0; i < n.length; i++) - n[i] = util.endian(n[i]); - return n; - - }, - - // Generate an array of any length of random bytes - randomBytes: function (n) { - for (var bytes = []; n > 0; n--) - bytes.push(Math.floor(Math.random() * 256)); - return bytes; - }, - - // Convert a byte array to big-endian 32-bit words - bytesToWords: function (bytes) { - for (var words = [], i = 0, b = 0; i < bytes.length; i++, b += 8) - words[b >>> 5] |= bytes[i] << (24 - b % 32); - return words; - }, - - // Convert big-endian 32-bit words to a byte array - wordsToBytes: function (words) { - for (var bytes = [], b = 0; b < words.length * 32; b += 8) - bytes.push((words[b >>> 5] >>> (24 - b % 32)) & 0xFF); - return bytes; - }, - - // Convert a byte array to a hex string - bytesToHex: function (bytes) { - for (var hex = [], i = 0; i < bytes.length; i++) { - hex.push((bytes[i] >>> 4).toString(16)); - hex.push((bytes[i] & 0xF).toString(16)); - } - return hex.join(""); - }, - - // Convert a hex string to a byte array - hexToBytes: function (hex) { - for (var bytes = [], c = 0; c < hex.length; c += 2) - bytes.push(parseInt(hex.substr(c, 2), 16)); - return bytes; - }, - - // Convert a byte array to a base-64 string - bytesToBase64: function (bytes) { - - // Use browser-native function if it exists - if (typeof btoa == "function") return btoa(Binary.bytesToString(bytes)); - - for(var base64 = [], i = 0; i < bytes.length; i += 3) { - var triplet = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2]; - for (var j = 0; j < 4; j++) { - if (i * 8 + j * 6 <= bytes.length * 8) - base64.push(base64map.charAt((triplet >>> 6 * (3 - j)) & 0x3F)); - else base64.push("="); - } - } - - return base64.join(""); - - }, - - // Convert a base-64 string to a byte array - base64ToBytes: function (base64) { - - // Use browser-native function if it exists - if (typeof atob == "function") return Binary.stringToBytes(atob(base64)); - - // Remove non-base-64 characters - base64 = base64.replace(/[^A-Z0-9+\/]/ig, ""); - - for (var bytes = [], i = 0, imod4 = 0; i < base64.length; imod4 = ++i % 4) { - if (imod4 == 0) continue; - bytes.push(((base64map.indexOf(base64.charAt(i - 1)) & (Math.pow(2, -2 * imod4 + 8) - 1)) << (imod4 * 2)) | - (base64map.indexOf(base64.charAt(i)) >>> (6 - imod4 * 2))); - } - - return bytes; - - } - -}; - -// Crypto mode namespace -Crypto.mode = {}; - -// Crypto character encodings -var charenc = Crypto.charenc = {}; - -// UTF-8 encoding -var UTF8 = charenc.UTF8 = { - - // Convert a string to a byte array - stringToBytes: function (str) { - return Binary.stringToBytes(unescape(encodeURIComponent(str))); - }, - - // Convert a byte array to a string - bytesToString: function (bytes) { - return decodeURIComponent(escape(Binary.bytesToString(bytes))); - } - -}; - -// Binary encoding -var Binary = charenc.Binary = { - - // Convert a string to a byte array - stringToBytes: function (str) { - for (var bytes = [], i = 0; i < str.length; i++) - bytes.push(str.charCodeAt(i)); - return bytes; - }, - - // Convert a byte array to a string - bytesToString: function (bytes) { - for (var str = [], i = 0; i < bytes.length; i++) - str.push(String.fromCharCode(bytes[i])); - return str.join(""); - } - -}; - -})(); -} diff --git a/server/camlistored/ui/SHA1.js b/server/camlistored/ui/SHA1.js deleted file mode 100644 index c0659be82..000000000 --- a/server/camlistored/ui/SHA1.js +++ /dev/null @@ -1,115 +0,0 @@ -// From http://code.google.com/p/crypto-js/ -// License: http://www.opensource.org/licenses/bsd-license.php -// -// Copyright (c) 2009, Jeff Mott. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. Redistributions in binary -// form must reproduce the above copyright notice, this list of conditions and -// the following disclaimer in the documentation and/or other materials provided -// with the distribution. Neither the name Crypto-JS nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. THIS SOFTWARE IS PROVIDED -// BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -if (typeof goog != 'undefined' && typeof goog.provide != 'undefined') { - goog.provide('camlistore.SHA1'); - - goog.require('camlistore.Crypto'); -} - -(function(){ - -// Shortcuts -var C = Crypto, - util = C.util, - charenc = C.charenc, - UTF8 = charenc.UTF8, - Binary = charenc.Binary; - -// Public API -var SHA1 = C.SHA1 = function (message, options) { - var digestbytes = util.wordsToBytes(SHA1._sha1(message)); - return options && options.asBytes ? digestbytes : - options && options.asString ? Binary.bytesToString(digestbytes) : - util.bytesToHex(digestbytes); -}; - -// The core -SHA1._sha1 = function (message) { - - // Convert to byte array - if (message.constructor == String) message = UTF8.stringToBytes(message); - /* else, assume byte array already */ - - var m = util.bytesToWords(message), - l = message.length * 8, - w = [], - H0 = 1732584193, - H1 = -271733879, - H2 = -1732584194, - H3 = 271733878, - H4 = -1009589776; - - // Padding - m[l >> 5] |= 0x80 << (24 - l % 32); - m[((l + 64 >>> 9) << 4) + 15] = l; - - for (var i = 0; i < m.length; i += 16) { - - var a = H0, - b = H1, - c = H2, - d = H3, - e = H4; - - for (var j = 0; j < 80; j++) { - - if (j < 16) w[j] = m[i + j]; - else { - var n = w[j-3] ^ w[j-8] ^ w[j-14] ^ w[j-16]; - w[j] = (n << 1) | (n >>> 31); - } - - var t = ((H0 << 5) | (H0 >>> 27)) + H4 + (w[j] >>> 0) + ( - j < 20 ? (H1 & H2 | ~H1 & H3) + 1518500249 : - j < 40 ? (H1 ^ H2 ^ H3) + 1859775393 : - j < 60 ? (H1 & H2 | H1 & H3 | H2 & H3) - 1894007588 : - (H1 ^ H2 ^ H3) - 899497514); - - H4 = H3; - H3 = H2; - H2 = (H1 << 30) | (H1 >>> 2); - H1 = H0; - H0 = t; - - } - - H0 += a; - H1 += b; - H2 += c; - H3 += d; - H4 += e; - - } - - return [H0, H1, H2, H3, H4]; - -}; - -// Package private blocksize -SHA1._blocksize = 16; - -})(); diff --git a/server/camlistored/ui/base64.js b/server/camlistored/ui/base64.js deleted file mode 100644 index 8609ea7e8..000000000 --- a/server/camlistored/ui/base64.js +++ /dev/null @@ -1,220 +0,0 @@ -/* -Copyright (c) 2008 Fred Palmer fred.palmer_at_gmail.com - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. -*/ - -if (typeof goog != 'undefined' && typeof goog.provide != 'undefined') { - goog.provide('camlistore.base64'); -} - -/** - * @constructor - */ -function StringBuffer() -{ - this.buffer = []; -} - -StringBuffer.prototype.append = function append(string) -{ - this.buffer.push(string); - return this; -}; - -StringBuffer.prototype.toString = function toString() -{ - return this.buffer.join(""); -}; - -var Base64 = -{ - codex : "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=", - - encode : function (input) - { - var output = new StringBuffer(); - - var enumerator = new Utf8EncodeEnumerator(input); - while (enumerator.moveNext()) - { - var chr1 = enumerator.current; - - enumerator.moveNext(); - var chr2 = enumerator.current; - - enumerator.moveNext(); - var chr3 = enumerator.current; - - var enc1 = chr1 >> 2; - var enc2 = ((chr1 & 3) << 4) | (chr2 >> 4); - var enc3 = ((chr2 & 15) << 2) | (chr3 >> 6); - var enc4 = chr3 & 63; - - if (isNaN(chr2)) - { - enc3 = enc4 = 64; - } - else if (isNaN(chr3)) - { - enc4 = 64; - } - - output.append(this.codex.charAt(enc1) + this.codex.charAt(enc2) + this.codex.charAt(enc3) + this.codex.charAt(enc4)); - } - - return output.toString(); - }, - - decode : function (input) - { - // TypedArray usage added by brett@haxor.com 11/27/2010 - var size = 0; - var buffer = new ArrayBuffer(input.length); - var output = new Uint8Array(buffer, 0); - - var enumerator = new Base64DecodeEnumerator(input); - while (enumerator.moveNext()) { - output[size++] = enumerator.current; - } - - // There is nothing in the TypedArray spec to copy/subset a buffer, - // so we have to do a copy to ensure that typedarray.buffer is the - // correct length when passed to XmlHttpRequest methods, etc. - var outputBuffer = new ArrayBuffer(size); - var outputArray = new Uint8Array(outputBuffer, 0); - for (var i = 0; i < size; i++) { - outputArray[i] = output[i]; - } - return outputArray; - } -} - - -/** - * @constructor - */ -function Utf8EncodeEnumerator(input) -{ - this._input = input; - this._index = -1; - this._buffer = []; -} - -Utf8EncodeEnumerator.prototype = -{ - current: Number.NaN, - - moveNext: function() - { - if (this._buffer.length > 0) - { - this.current = this._buffer.shift(); - return true; - } - else if (this._index >= (this._input.length - 1)) - { - this.current = Number.NaN; - return false; - } - else - { - var charCode = this._input.charCodeAt(++this._index); - - // "\r\n" -> "\n" - // - if ((charCode == 13) && (this._input.charCodeAt(this._index + 1) == 10)) - { - charCode = 10; - this._index += 2; - } - - if (charCode < 128) - { - this.current = charCode; - } - else if ((charCode > 127) && (charCode < 2048)) - { - this.current = (charCode >> 6) | 192; - this._buffer.push((charCode & 63) | 128); - } - else - { - this.current = (charCode >> 12) | 224; - this._buffer.push(((charCode >> 6) & 63) | 128); - this._buffer.push((charCode & 63) | 128); - } - - return true; - } - } -} - -/** - * @constructor - */ -function Base64DecodeEnumerator(input) -{ - this._input = input; - this._index = -1; - this._buffer = []; -} - -Base64DecodeEnumerator.prototype = -{ - current: 64, - - moveNext: function() - { - if (this._buffer.length > 0) - { - this.current = this._buffer.shift(); - return true; - } - else if (this._index >= (this._input.length - 1)) - { - this.current = 64; - return false; - } - else - { - var enc1 = Base64.codex.indexOf(this._input.charAt(++this._index)); - var enc2 = Base64.codex.indexOf(this._input.charAt(++this._index)); - var enc3 = Base64.codex.indexOf(this._input.charAt(++this._index)); - var enc4 = Base64.codex.indexOf(this._input.charAt(++this._index)); - - var chr1 = (enc1 << 2) | (enc2 >> 4); - var chr2 = ((enc2 & 15) << 4) | (enc3 >> 2); - var chr3 = ((enc3 & 3) << 6) | enc4; - - this.current = chr1; - - if (enc3 != 64) - this._buffer.push(chr2); - - if (enc4 != 64) - this._buffer.push(chr3); - - return true; - } - } -}; diff --git a/server/camlistored/ui/blob.js b/server/camlistored/ui/blob.js new file mode 100644 index 000000000..df19434a8 --- /dev/null +++ b/server/camlistored/ui/blob.js @@ -0,0 +1,66 @@ +/* +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +goog.provide('camlistore.blob'); + +goog.require('goog.crypt'); +goog.require('goog.crypt.Sha1'); + +// Returns the Camlistore blobref for hash object. The only supported hash function is currently sha1, but more might be added later. +// @param {!goog.crypt.Hash} hash +// @returns {!string} +camlistore.blob.refFromHash = function(hash) { + if (hash instanceof goog.crypt.Sha1) { + return 'sha1-' + goog.crypt.byteArrayToHex(hash.digest()); + } + throw new Error('Unsupported hash function type'); +}; + +// Returns the Camlistore blobref for a string using the currently recommended hash function. +// @param {!string} str +// @returns {!string} +camlistore.blob.refFromString = function(str) { + var hash = camlistore.blob.createHash(); + hash.update(str); + return camlistore.blob.refFromHash(hash); +}; + +// Returns the Camlistore blobref for a DOM blob (different from Camlistore blob) using the currently recommended hash function. This function currently only works within workers. +// @param {Blob} blob +// @returns {!string} +camlistore.blob.refFromDOMBlob = function(blob) { + if (!goog.global.FileReaderSync) { + // TODO(aa): If necessary, we can also implement this using FileReader for use on the main thread. But beware that should not be done for very large objects without checking the effect on framerate carefully. + throw new Error('FileReaderSync not available. Perhaps we are on the main thread?'); + } + + var fr = new FileReaderSync(); + var hash = camlistore.blob.createHash(); + var chunkSize = 1024 * 1024; + for (var start = 0; start < blob.size; start += chunkSize) { + var end = Math.min(start + chunkSize, blob.size); + var slice = blob.slice(start, end); + hash.update(new Uint8Array(fr.readAsArrayBuffer(slice))); + } + + return camlistore.blob.refFromHash(hash); +}; + +// Creates an instance of the currently recommened hash function. +// @return {!goog.crypt.Hash'} +camlistore.blob.createHash = function() { + return new goog.crypt.Sha1(); +}; diff --git a/server/camlistored/ui/blobinfo.html b/server/camlistored/ui/blobinfo.html index 43a90e4ab..f3b810ee4 100644 --- a/server/camlistored/ui/blobinfo.html +++ b/server/camlistored/ui/blobinfo.html @@ -5,11 +5,6 @@ - - - - - - - - - - diff --git a/server/camlistored/ui/filetree.html b/server/camlistored/ui/filetree.html index 38efaf608..5ed4e84cc 100644 --- a/server/camlistored/ui/filetree.html +++ b/server/camlistored/ui/filetree.html @@ -5,11 +5,6 @@ - - - - - diff --git a/server/camlistored/ui/hash_worker.js b/server/camlistored/ui/hash_worker.js new file mode 100644 index 000000000..0bb9ec792 --- /dev/null +++ b/server/camlistored/ui/hash_worker.js @@ -0,0 +1,30 @@ +/* +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// These two lines are required setup to make goog.require() work throughout the codebase. +var CLOSURE_BASE_PATH = 'closure/goog/'; +importScripts('closure/goog/bootstrap/webworkers.js', 'closure/goog/base.js', 'deps.js'); + +goog.require('camlistore.blob'); +goog.require('camlistore.WorkerMessageRouter'); + +// This is a simple webworker that expects to receive a single message containing a file, and sends back that file's sha1 hash. +// We do this in a worker because we observed that doing it on the main thread decreased the framerate significantly, even when chunking, and even when the chunk sizes were as small as 32k. + +var router = new camlistore.WorkerMessageRouter(goog.global); +router.registerHandler('ref', function(msg, sendReply) { + sendReply(camlistore.blob.refFromDOMBlob(msg)); +}); diff --git a/server/camlistored/ui/index.html b/server/camlistored/ui/index.html index 9a0722d23..67f89647f 100644 --- a/server/camlistored/ui/index.html +++ b/server/camlistored/ui/index.html @@ -11,12 +11,6 @@ - - - - - - diff --git a/server/camlistored/ui/permanode.html b/server/camlistored/ui/permanode.html index cd061570e..6701e6c6a 100644 --- a/server/camlistored/ui/permanode.html +++ b/server/camlistored/ui/permanode.html @@ -5,11 +5,6 @@ - - - - - diff --git a/server/camlistored/ui/server_connection.js b/server/camlistored/ui/server_connection.js index c6aae010d..654abd2bd 100644 --- a/server/camlistored/ui/server_connection.js +++ b/server/camlistored/ui/server_connection.js @@ -1,13 +1,14 @@ goog.provide('camlistore.ServerConnection'); -goog.require('camlistore.base64'); -goog.require('camlistore.SHA1'); goog.require('goog.string'); goog.require('goog.net.XhrIo'); goog.require('goog.Uri'); // because goog.net.XhrIo forgot to include it. goog.require('goog.debug.ErrorHandler'); // because goog.net.Xhrio forgot to include it. goog.require('goog.uri.utils'); + +goog.require('camlistore.blob'); goog.require('camlistore.ServerType'); +goog.require('camlistore.WorkerMessageRouter'); // @fileoverview Connection to the blob server and API for the RPCs it provides. All blob index UI code should use this connection to contact the server. // @param {camlistore.ServerType.DiscoveryDocument} config Discovery document for the current server. @@ -16,6 +17,15 @@ goog.require('camlistore.ServerType'); camlistore.ServerConnection = function(config, opt_sendXhr) { this.config_ = config; this.sendXhr_ = opt_sendXhr || goog.net.XhrIo.send; + this.worker_ = null; +}; + +camlistore.ServerConnection.prototype.getWorker_ = function() { + if (!this.worker_) { + var r = new Date().getTime(); // For cachebusting the worker. Sigh. We need content stamping. + this.worker_ = new camlistore.WorkerMessageRouter(new Worker('hash_worker.js?r=' + r)); + } + return this.worker_; }; camlistore.ServerConnection.prototype.getConfig = function() { @@ -286,7 +296,7 @@ camlistore.ServerConnection.prototype.handlePost_ = function(success, opt_fail, // @param {Function} success Success callback. // @param {?Function} opt_fail Optional fail callback. camlistore.ServerConnection.prototype.uploadString_ = function(s, success, opt_fail) { - var blobref = "sha1-" + Crypto.SHA1(s); + var blobref = camlistore.blob.refFromString(s); var parts = [s]; var bb = new Blob(parts); var fd = new FormData(); @@ -443,27 +453,12 @@ camlistore.ServerConnection.prototype.newDelAttributeClaim = function(permanode, // @param {?Function} opt_fail Optional fail callback. // @param {?Function} opt_onContentsRef Optional callback to set contents during upload. camlistore.ServerConnection.prototype.uploadFile = function(file, success, opt_fail, opt_onContentsRef) { - var fr = new FileReader(); - var onload = function() { - var dataurl = fr.result; - var comma = dataurl.indexOf(","); - if (comma != -1) { - var b64 = dataurl.substring(comma + 1); - var arrayBuffer = Base64.decode(b64).buffer; - var hash = Crypto.SHA1(new Uint8Array(arrayBuffer, 0)); - - var contentsRef = "sha1-" + hash; - if (opt_onContentsRef) { - opt_onContentsRef(contentsRef); - } - this.camliUploadFileHelper_(file, contentsRef, success, this.safeFail_(opt_fail)); + this.getWorker_().sendMessage('ref', file, function(ref) { + if (opt_onContentsRef) { + opt_onContentsRef(ref); } - }; - fr.onload = goog.bind(onload, this); - fr.onerror = function() { - console.log("FileReader onerror: " + fr.error + " code=" + fr.error.code); - }; - fr.readAsDataURL(file); + this.camliUploadFileHelper_(file, ref, success, this.safeFail_(opt_fail)); + }.bind(this)); }; // camliUploadFileHelper uploads the provided file with contents blobref contentsBlobRef diff --git a/server/camlistored/ui/worker_message_router.js b/server/camlistored/ui/worker_message_router.js new file mode 100644 index 000000000..b05fcbcc7 --- /dev/null +++ b/server/camlistored/ui/worker_message_router.js @@ -0,0 +1,101 @@ +/* +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +goog.provide('camlistore.WorkerMessageRouter'); + +goog.require('goog.string'); + +// Convenience for sending request/response style messages to and from workers. +// @param {!Worker} worker The DOM worker to wrap. +// @constructor +camlistore.WorkerMessageRouter = function(worker) { + this.worker_ = worker; + this.nextMessageId_ = 1; + + // name->handler - See registerHandler() + // @type Object. + this.handlers_ = {}; + + // messageid->callback - See sendMessage() + // @type Object. + this.pendingMessages_ = {}; + + this.worker_.addEventListener('message', this.handleMessage_.bind(this)); +}; + +// Send a message over the worker, optionally expecting a response. +// @param {!string} name The name of the message to send. +// @param {!*} msg The message content +// @param {?function(*)} opt_callback The function to receive the response. +camlistore.WorkerMessageRouter.prototype.sendMessage = function(name, msg, opt_callback) { + var messageId = 0; + if (opt_callback) { + messageId = this.nextMessageId_++; + this.pendingMessages_[messageId] = opt_callback; + } + this.worker_.postMessage({ + messageId: messageId, + name: name, + message: msg + }); +}; + +// Registers a function to handle a particular named message type. +// @param {!string} name The name of the message type to handle. +// @param {!function(*, function(*))} handler The function to call to return the reply to the client. +camlistore.WorkerMessageRouter.prototype.registerHandler = function(name, handler) { + this.handlers_[name] = handler; +}; + +camlistore.WorkerMessageRouter.prototype.handleMessage_ = function(e) { + if (!goog.isObject(e.data) || !goog.isDef(e.data.messageId)) { + return; + } + + if (goog.isDef(e.data.name)) { + this.handleRequest_(e.data); + } else { + this.handleReply_(e.data); + } +}; + +camlistore.WorkerMessageRouter.prototype.handleRequest_ = function(request) { + var handler = this.handlers_[request.name]; + if (!handler) { + throw new Error(goog.string.subs('No registered handler with name: %s', request.name)); + } + + var sendReply = function(reply) { + if (!request.messageId) { + return; + } + this.worker_.postMessage({ + messageId: request.messageId, + message: reply + }); + }.bind(this); + + handler(request.message, sendReply); +}; + +camlistore.WorkerMessageRouter.prototype.handleReply_ = function(reply) { + var callback = this.pendingMessages_[reply.messageId]; + if (!callback) { + throw new Error('Could not find callback for pending message: %s', reply.messageId); + } + delete this.pendingMessages_[reply.messageId]; + callback(reply.message); +};