Improved upload speed

Added upload multi threads Improved web workers: added onload event Fixed unauthorised box
11 years ago · 9f9827c7a6
7 changed files with 107 additions and 55 deletions
--- a/app/js/controllers.js
+++ b/app/js/controllers.js
@ -467,7 +467,6 @@ angular.module('myApp.controllers', ['myApp.i18n'])
        peersInDialogs = {},
        contactsShown;

-    MtpApiManager.invokeApi('account.updateStatus', {offline: false});
    $scope.$on('dialogs_need_more', function () {
      // console.log('on need more');
      showMoreDialogs();
@ -629,7 +628,8 @@ angular.module('myApp.controllers', ['myApp.i18n'])

        if (error.code == 401) {
          MtpApiManager.logOut()['finally'](function () {
-            $location.url('/login');
+            location.hash = '/login';
+            AppRuntimeManager.reload();
          });
          error.handled = true;
        }
--- a/app/js/lib/bin_utils.js
+++ b/app/js/lib/bin_utils.js
@ -175,6 +175,18 @@ function convertToUint8Array(bytes) {
  return new Uint8Array(bytes);
 }

+function convertToByteArray(bytes) {
+  if (Array.isArray(bytes)) {
+    return bytes;
+  }
+  bytes = convertToUint8Array(bytes);
+  var newBytes = [];
+  for (var i = 0, len = bytes.length; i < len; i++) {
+    newBytes.push(bytes[i]);
+  }
+  return newBytes;
+}
+
 function bytesFromArrayBuffer (buffer) {
  var len = buffer.byteLength,
      byteView = new Uint8Array(buffer),
--- a/app/js/lib/crypto_worker.js
+++ b/app/js/lib/crypto_worker.js
@ -46,3 +46,5 @@ onmessage = function (e) {

  postMessage({taskID: taskID, result: result});
 }
+
+postMessage('ready');
--- a/app/js/lib/mtproto.js
+++ b/app/js/lib/mtproto.js
@ -988,13 +988,32 @@ angular.module('izhukov.mtproto', ['izhukov.utils'])
        currentTime = tsNow(),
        hasApiCall = false,
        hasHttpWait = false,
+        lengthOverflow = false,
+        singlesCount = 0,
        self = this;

    angular.forEach(this.pendingMessages, function (value, messageID) {
      if (!value || value >= currentTime) {
        if (message = self.sentMessages[messageID]) {
+          var messageByteLength = (message.body.byteLength || message.body.length) + 32;
+          if (!message.notContentRelated &&
+              lengthOverflow) {
+            return;
+          }
+          if (!message.notContentRelated &&
+              messagesByteLen &&
+              messagesByteLen + messageByteLength > 655360) { // 640 Kb
+            lengthOverflow = true;
+            return;
+          }
+          if (message.singleInRequest) {
+            singlesCount++;
+            if (singlesCount > 1) {
+              return;
+            }
+          }
          messages.push(message);
-          messagesByteLen += (message.body.byteLength || message.body.length) + 32;
+          messagesByteLen += messageByteLength;
          if (message.isAPI) {
            hasApiCall = true;
          }
@ -1111,6 +1130,10 @@ angular.module('izhukov.mtproto', ['izhukov.utils'])

      self.toggleOffline(true);
    });
+
+    if (lengthOverflow || singlesCount > 1) {
+      this.sheduleRequest()
+    }
  };

  MtpNetworker.prototype.getEncryptedMessage = function (bytes) {
--- a/app/js/lib/mtproto_wrapper.js
+++ b/app/js/lib/mtproto_wrapper.js
@ -234,14 +234,12 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto'])

  var cachedFs = false;
  var cachedFsPromise = false;
-  var apiUploadPromise = $q.when();
  var cachedSavePromises = {};
  var cachedDownloadPromises = {};
  var cachedDownloads = {};

  var downloadPulls = {};
  var downloadActives = {};
-  var downloadLimit = 5;

  function downloadRequest(dcID, cb, activeDelta) {
    if (downloadPulls[dcID] === undefined) {
@ -251,7 +249,9 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto'])
    var downloadPull = downloadPulls[dcID];
    var deferred = $q.defer();
    downloadPull.push({cb: cb, deferred: deferred, activeDelta: activeDelta});
-    downloadCheck(dcID);
+    setZeroTimeout(function () {
+      downloadCheck(dcID);
+    });

    return deferred.promise;
  };
@ -260,6 +260,7 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto'])

  function downloadCheck(dcID) {
    var downloadPull = downloadPulls[dcID];
+    var downloadLimit = dcID == 'upload' ? 17 : 5;

    if (downloadActives[dcID] >= downloadLimit || !downloadPull || !downloadPull.length) {
      return false;
@ -507,15 +508,23 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto'])
  }

  function uploadFile (file) {
-    var fileSize = file.size,
-        // partSize = fileSize > 102400 ? 65536 : 4096,
-        // partSize = fileSize > 102400 ? 524288 : 4096,
-        partSize = fileSize > 102400 ? 524288 : 32768,
-        isBigFile = fileSize >= 10485760,
-        totalParts = Math.ceil(fileSize / partSize),
-        canceled = false,
-        resolved = false,
-        doneParts = 0;
+    var fileSize    = file.size,
+        isBigFile   = fileSize >= 10485760,
+        canceled    = false,
+        resolved    = false,
+        doneParts   = 0,
+        partSize    = 262144, // 256 Kb
+        activeDelta = 2;
+
+    if (fileSize > 67108864) {
+      partSize = 524288;
+      activeDelta = 4;
+    }
+    else if (fileSize < 102400) {
+      partSize = 32768;
+      activeDelta = 1;
+    }
+    var totalParts = Math.ceil(fileSize / partSize);

    if (totalParts > 1500) {
      return $q.reject({type: 'FILE_TOO_BIG'});
@ -526,6 +535,7 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto'])
        errorHandler = function (error) {
          // console.error('Up Error', error);
          deferred.reject(error);
+          canceled = true;
          errorHandler = angular.noop;
        },
        part = 0,
@ -539,35 +549,34 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto'])
        };


-    var fileReadPromise = $q.when();
-
    for (offset = 0; offset < fileSize; offset += partSize) {
      (function (offset, part) {
-        fileReadPromise = fileReadPromise.then(function () {
-          var fileReadDeferred = $q.defer();
+        downloadRequest('upload', function () {
+          var uploadDeferred = $q.defer();

          var reader = new FileReader();
          var blob = file.slice(offset, offset + partSize);

          reader.onloadend = function (e) {
-            if (canceled || e.target.readyState != FileReader.DONE) {
+            if (canceled) {
+              uploadDeferred.reject();
              return;
            }
-            var apiCurPromise = apiUploadPromise = apiUploadPromise.then(function () {
-              return MtpApiManager.invokeApi(isBigFile ? 'upload.saveBigFilePart' : 'upload.saveFilePart', {
-                file_id: fileID,
-                file_part: part,
-                file_total_parts: totalParts,
-                bytes: e.target.result
-              }, {
-                startMaxLength: partSize + 256,
-                fileUpload: true
-              });
-            }, errorHandler);
-
-            apiCurPromise.then(function (result) {
+            if (e.target.readyState != FileReader.DONE) {
+              return;
+            }
+            MtpApiManager.invokeApi(isBigFile ? 'upload.saveBigFilePart' : 'upload.saveFilePart', {
+              file_id: fileID,
+              file_part: part,
+              file_total_parts: totalParts,
+              bytes: e.target.result
+            }, {
+              startMaxLength: partSize + 256,
+              fileUpload: true,
+              singleInRequest: true
+            }).then(function (result) {
              doneParts++;
-              fileReadDeferred.resolve();
+              uploadDeferred.resolve();
              if (doneParts >= totalParts) {
                deferred.resolve(resultInputFile);
                resolved = true;
@ -580,8 +589,8 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto'])

          reader.readAsArrayBuffer(blob);

-          return fileReadDeferred.promise;
-        });
+          return uploadDeferred.promise;
+        }, activeDelta);
      })(offset, part++);
    }

--- a/app/js/lib/ng_utils.js
+++ b/app/js/lib/ng_utils.js
@ -484,12 +484,12 @@ angular.module('izhukov.utils', [])

 .service('CryptoWorker', function ($timeout, $q) {

-  var worker = window.Worker && new Worker('js/lib/crypto_worker.js') || false,
+  var webWorker = false,
+      naClEmbed = false,
      taskID = 0,
      awaiting = {},
      webCrypto = window.crypto && (window.crypto.subtle || window.crypto.webkitSubtle) || window.msCrypto && window.msCrypto.subtle,
      useSha1Crypto = webCrypto && webCrypto.digest !== undefined,
-      aesNaClEmbed = false,
      finalizeTask = function (taskID, result) {
        var deferred = awaiting[taskID];
        if (deferred !== undefined) {
@ -502,7 +502,7 @@ angular.module('izhukov.utils', [])
  if (navigator.mimeTypes['application/x-pnacl'] !== undefined) {
    var listener = $('<div id="nacl_listener"><embed id="mtproto_crypto" width="0" height="0" src="nacl/mtproto_crypto.nmf?'+Math.random()+'" type="application/x-pnacl" /></div>').appendTo($('body'))[0];
    listener.addEventListener('load', function (e) {
-      aesNaClEmbed = listener.firstChild;
+      naClEmbed = listener.firstChild;
      console.log(dT(), 'NaCl ready');
    }, true);
    listener.addEventListener('message', function (e) {
@ -513,13 +513,18 @@ angular.module('izhukov.utils', [])
    }, true);
  }

-  if (worker) {
-    worker.onmessage = function (e) {
-      finalizeTask(e.data.taskID, e.data.result);
+  if (window.Worker) {
+    var tmpWorker = new Worker('js/lib/crypto_worker.js');
+    tmpWorker.onmessage = function (e) {
+      if (!webWorker) {
+        webWorker = tmpWorker;
+      } else {
+        finalizeTask(e.data.taskID, e.data.result);
+      }
    };
-    worker.onerror = function(error) {
+    tmpWorker.onerror = function(error) {
      console.error('CW error', error, error.stack);
-      worker = false;
+      webWorker = false;
    };
  }

@ -531,7 +536,7 @@ angular.module('izhukov.utils', [])

    params.task = task;
    params.taskID = taskID;
-    (embed || worker).postMessage(params);
+    (embed || webWorker).postMessage(params);

    taskID++;

@ -561,34 +566,35 @@ angular.module('izhukov.utils', [])
      });
    },
    aesEncrypt: function (bytes, keyBytes, ivBytes) {
-      if (aesNaClEmbed) {
+      if (naClEmbed) {
        return performTaskWorker('aes-encrypt', {
          bytes: addPadding(convertToArrayBuffer(bytes)),
          keyBytes: convertToArrayBuffer(keyBytes),
          ivBytes: convertToArrayBuffer(ivBytes)
-        }, aesNaClEmbed);
+        }, naClEmbed);
      }
      return $timeout(function () {
        return convertToArrayBuffer(aesEncryptSync(bytes, keyBytes, ivBytes));
      });
    },
    aesDecrypt: function (encryptedBytes, keyBytes, ivBytes) {
-      if (aesNaClEmbed) {
+      if (naClEmbed) {
        return performTaskWorker('aes-decrypt', {
          encryptedBytes: addPadding(convertToArrayBuffer(encryptedBytes)),
          keyBytes: convertToArrayBuffer(keyBytes),
          ivBytes: convertToArrayBuffer(ivBytes)
-        }, aesNaClEmbed);
+        }, naClEmbed);
      }
      return $timeout(function () {
        return convertToArrayBuffer(aesDecryptSync(encryptedBytes, keyBytes, ivBytes));
      });
    },
    factorize: function (bytes) {
-      if (aesNaClEmbed && bytes.length <= 8) {
-        return performTaskWorker('factorize', {bytes: bytes}, aesNaClEmbed);
+      bytes = convertToByteArray(bytes);
+      if (naClEmbed && bytes.length <= 8) {
+        return performTaskWorker('factorize', {bytes: bytes}, naClEmbed);
      }
-      if (worker) {
+      if (webWorker) {
        return performTaskWorker('factorize', {bytes: bytes});
      }
      return $timeout(function () {
@ -596,7 +602,7 @@ angular.module('izhukov.utils', [])
      });
    },
    modPow: function (x, y, m) {
-      if (worker) {
+      if (webWorker) {
        return performTaskWorker('mod-pow', {
          x: x,
          y: y,
--- a/app/js/services.js
+++ b/app/js/services.js
@ -850,7 +850,7 @@ angular.module('myApp.services', ['myApp.i18n'])
      offset: offset || 0,
      limit: limit || 0,
      max_id: maxID || 0
-    }).then(function (historyResult) {
+    }, {noErrorBox: true}).then(function (historyResult) {
      AppUsersManager.saveApiUsers(historyResult.users);
      AppChatsManager.saveApiChats(historyResult.chats);
      saveMessages(historyResult.messages);
@ -3130,7 +3130,7 @@ angular.module('myApp.services', ['myApp.i18n'])

  function attach () {
    MtpNetworkerFactory.setUpdatesProcessor(processUpdateMessage);
-    MtpApiManager.invokeApi('updates.getState', {noErrorBox: true}).then(function (stateResult) {
+    MtpApiManager.invokeApi('updates.getState', {}, {noErrorBox: true}).then(function (stateResult) {
      curState.seq = stateResult.seq;
      curState.pts = stateResult.pts;
      curState.date = stateResult.date;