From 82949d81188758ea798cf0e72628cfd22ac603be Mon Sep 17 00:00:00 2001 From: Jarrett Cruger Date: Wed, 21 Dec 2016 17:19:27 -0500 Subject: [PATCH 1/3] [fix] aws storage client - Update the aws-sdk - Use built in upload rather than external module - expose concurrency/queueSize and partSize configurability - Add ability to abort upload --- lib/pkgcloud/amazon/storage/client/files.js | 39 +++++++++++++-------- lib/pkgcloud/amazon/storage/client/index.js | 4 --- package.json | 3 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/lib/pkgcloud/amazon/storage/client/files.js b/lib/pkgcloud/amazon/storage/client/files.js index f43cc52aa..b008fd77a 100644 --- a/lib/pkgcloud/amazon/storage/client/files.js +++ b/lib/pkgcloud/amazon/storage/client/files.js @@ -52,6 +52,19 @@ exports.upload = function (options) { Key: options.remote instanceof base.File ? options.remote.name : options.remote }; + // + // Allow for configuration of upload parameters + // + var uploadOptions = {}; + + if (options.concurrency || options.queueSize) { + uploadOptions.queueSize = options.concurrency || options.queueSize; + } + + if (options.partSize) { + uploadOptions.partSize = options.partSize; + } + if (options.cacheControl) { s3Options.CacheControl = options.cacheControl; } @@ -74,25 +87,23 @@ exports.upload = function (options) { s3Options.ServerSideEncryption = options.ServerSideEncryption; } - var proxyStream = through(), - writableStream = self.s3Stream.upload(s3Options); + var proxyStream = through(); - // we need a proxy stream so we can always return a file model - // via the 'success' event - writableStream.on('uploaded', function(details) { - proxyStream.emit('success', new storage.File(self, details)); - }); + s3Options.Body = proxyStream; + var upload = this.s3.upload(s3Options, uploadOptions); - writableStream.on('error', function(err) { - proxyStream.emit('error', err); - }); + // + // To allow aborting of upload if we want + // + proxyStream.abort = upload.abort.bind(upload); - writableStream.on('data', function (chunk) { - proxyStream.emit('data', chunk); + upload.send(function (err, details) { + if (err) return proxyStream.emit('error', err); + // we need a proxy stream so we can always return a file model + // via the 'success' event + proxyStream.emit('success', new storage.File(self, details)); }); - proxyStream.pipe(writableStream); - return proxyStream; }; diff --git a/lib/pkgcloud/amazon/storage/client/index.js b/lib/pkgcloud/amazon/storage/client/index.js index b3d8fd950..f8d7a6e94 100644 --- a/lib/pkgcloud/amazon/storage/client/index.js +++ b/lib/pkgcloud/amazon/storage/client/index.js @@ -7,7 +7,6 @@ var util = require('util'), AWS = require('aws-sdk'), - s3Stream = require('s3-upload-stream'), amazon = require('../../client'), _ = require('lodash'); @@ -18,9 +17,6 @@ var Client = exports.Client = function (options) { _.extend(this, require('./files')); this.s3 = new AWS.S3(this._awsConfig); - - // configure the s3Stream - this.s3Stream = s3Stream(this.s3); }; util.inherits(Client, amazon.Client); diff --git a/package.json b/package.json index a306656f8..2f8342d68 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ ], "dependencies": { "async": "0.9.x", - "aws-sdk": "^2.2.43", + "aws-sdk": "~2.7.17", "errs": "0.3.x", "eventemitter2": "0.4.x", "fast-json-patch": "0.5.x", @@ -67,7 +67,6 @@ "mime": "1.2.x", "qs": "1.2.x", "request": "2.40.x", - "s3-upload-stream": "~1.0.7", "through2": "0.6.x", "url-join": "0.0.x", "xml2js": "0.1.x" From 54be3a02c8de84a62f9c4ce3a4b36892498d1b15 Mon Sep 17 00:00:00 2001 From: Jarrett Cruger Date: Wed, 21 Dec 2016 17:32:16 -0500 Subject: [PATCH 2/3] [fix] lint --- lib/pkgcloud/amazon/storage/client/files.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pkgcloud/amazon/storage/client/files.js b/lib/pkgcloud/amazon/storage/client/files.js index b008fd77a..e3bec6366 100644 --- a/lib/pkgcloud/amazon/storage/client/files.js +++ b/lib/pkgcloud/amazon/storage/client/files.js @@ -98,7 +98,7 @@ exports.upload = function (options) { proxyStream.abort = upload.abort.bind(upload); upload.send(function (err, details) { - if (err) return proxyStream.emit('error', err); + if (err) { return proxyStream.emit('error', err); } // we need a proxy stream so we can always return a file model // via the 'success' event proxyStream.emit('success', new storage.File(self, details)); From 32728c342cebae9188182205288ff2bbb6c01256 Mon Sep 17 00:00:00 2001 From: Jarrett Cruger Date: Mon, 15 May 2017 15:05:39 -0400 Subject: [PATCH 3/3] [dist] update aws-sdk --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2f8342d68..9c596431e 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ ], "dependencies": { "async": "0.9.x", - "aws-sdk": "~2.7.17", + "aws-sdk": "~2.50.0", "errs": "0.3.x", "eventemitter2": "0.4.x", "fast-json-patch": "0.5.x",