From 111bd7f234671eccc73ef3245daac034c7b7b2dd Mon Sep 17 00:00:00 2001 From: nicola Date: Wed, 20 Jan 2016 15:35:50 -0500 Subject: [PATCH 1/3] attempt to have writeStreams --- lib/handlers/post.js | 70 +++++++++++++++------- lib/handlers/put.js | 2 +- lib/identity-provider.js | 4 +- lib/ldp-middleware.js | 29 ++++----- lib/ldp.js | 34 ++++++++--- package.json | 1 + test/ldp.js | 7 ++- test/resources/acl/append-acl/abc.ttl | 2 +- test/resources/acl/append-acl/abc2.ttl.acl | 9 +-- test/resources/acl/read-acl/.acl | 11 +--- 10 files changed, 101 insertions(+), 68 deletions(-) diff --git a/lib/handlers/post.js b/lib/handlers/post.js index 92182319b..f0fb4c052 100644 --- a/lib/handlers/post.js +++ b/lib/handlers/post.js @@ -1,5 +1,7 @@ module.exports = handler +var Busboy = require('busboy') + var debug = require('../debug').handlers var header = require('../header') var patch = require('./patch') @@ -16,48 +18,72 @@ function handler (req, res, next) { return patch(req, res, next) } + // Handle container path var containerPath = req.path - debug('POST -- On parent: ' + containerPath) - - // Not a container if (containerPath[containerPath.length - 1] !== '/') { containerPath += '/' } - debug('POST -- Content Type: ' + contentType) - - var linkHeader = header.parseMetadataFromHeader(req.get('Link')) - + // Check if container exists ldp.exists(req.hostname, containerPath, function (err, stats) { if (err) { return next(error(err, 'Container not valid')) } + // Check if container is a directory if (!stats.isDirectory()) { debug('POST -- Path is not a container') - res.set('Allow', 'GET,HEAD,PUT,DELETE') return next(error(405, 'Requested resource is not a container')) } - // TODO: possibly package this in ldp.post - ldp.getAvailablePath(req.hostname, containerPath, req.get('Slug'), function (resourcePath) { - debug('POST -- Will create at: ' + resourcePath) - var meta = '' - if (linkHeader.isBasicContainer) { - resourcePath += '/' - meta = ldp.suffixMeta - } + // Dispatch to the right handler + if (contentType === 'multipart/form-data') { + multi(req, res, next) + } else { + one(req, res, next) + } + }) + + function multi () { + var busboy = new Busboy({ headers: req.headers }) - ldp.put(req.hostname, resourcePath + meta, req.text, function (err) { + busboy.on('file', function (fieldname, file, filename, encoding, mimetype) { + ldp.post( + req.hostname, + containerPath, + filename, + file, + false, + function (err) { + if (err) { + return busboy.emit(err) + } + }) + }) + busboy.on('error', function (err) { + next(err) + }) + busboy.on('finish', function () { + res.sendStatus(200) + }) + } + + function one () { + var linkHeader = header.parseMetadataFromHeader(req.get('Link')) + + ldp.post( + req.hostname, + containerPath, + req.get('Slug'), + req, + linkHeader.isBasicContainer, + function (err, resourcePath) { if (err) { - return next(err) + next(err) } - header.addLinks(res, linkHeader) res.set('Location', resourcePath) res.sendStatus(201) - return next() }) - }) - }) + } } diff --git a/lib/handlers/put.js b/lib/handlers/put.js index 3f3866184..fe01a538a 100644 --- a/lib/handlers/put.js +++ b/lib/handlers/put.js @@ -7,7 +7,7 @@ function handler (req, res, next) { debug('PUT -- originalUrl: ' + req.originalUrl) res.header('MS-Author-Via', 'SPARQL') - ldp.put(req.hostname, req.path, req.text, function (err) { + ldp.put(req.hostname, req.path, req, function (err) { if (err) { debug('PUT -- Write error: ' + err.message) err.message = 'Can\'t write file: ' + err.message diff --git a/lib/identity-provider.js b/lib/identity-provider.js index de1732dbc..15be510d7 100644 --- a/lib/identity-provider.js +++ b/lib/identity-provider.js @@ -18,6 +18,7 @@ var forge = require('node-forge') var asn1 = forge.asn1 var pki = forge.pki var parse = require('./utils').parse +var stringToStream = require('./utils').stringToStream function defaultBuildURI (account, host, port) { var hostAndPort = (host || 'localhost') + (port || '') @@ -51,10 +52,11 @@ IdentityProvider.prototype.putGraph = function (uri, graph) { var self = this return function (callback) { var content = $rdf.serialize(content, graph, uri, 'text/turtle') + var stream = stringToStream(content) var parsed = url.parse(uri) var host = parsed.hostname var path = parsed.path - return self.store.put(host, path, content, callback) + return self.store.put(host, path, stream, callback) } } diff --git a/lib/ldp-middleware.js b/lib/ldp-middleware.js index 7572b6f12..2cdc0e724 100644 --- a/lib/ldp-middleware.js +++ b/lib/ldp-middleware.js @@ -12,6 +12,20 @@ var del = require('./handlers/delete') var patch = require('./handlers/patch') var errorPages = require('./handlers/error-pages') +function rawBody (req, res, next) { + getRawBody(req, { + length: req.headers['content-length'], + encoding: 'utf-8' + }, + function (err, string) { + if (err) { + return next(err) + } + req.text = string + next() + }) +} + function LdpMiddleware (corsSettings) { var router = express.Router('/') @@ -20,24 +34,11 @@ function LdpMiddleware (corsSettings) { if (corsSettings) { router.use(corsSettings) } - router.use('/*', function (req, res, next) { - getRawBody(req, { - length: req.headers['content-length'], - encoding: 'utf-8' // typer.parse(req.headers['content-type']).parameters.charset - }, - function (err, string) { - if (err) { - return next(err) - } - req.text = string - next() - }) - }) router.use('/*', login.loginHandler) router.get('/*', acl.allow('Read'), get) router.post('/*', acl.allow('Append'), post) - router.patch('/*', acl.allow('Append'), patch) + router.patch('/*', acl.allow('Append'), rawBody, patch) router.put('/*', acl.allow('Append'), put) router.delete('/*', acl.allow('Write'), del) diff --git a/lib/ldp.js b/lib/ldp.js index 3c47d6fca..dd6690016 100644 --- a/lib/ldp.js +++ b/lib/ldp.js @@ -311,7 +311,30 @@ LDP.prototype.listContainer = function (filename, uri, containerData, contentTyp }) } -LDP.prototype.put = function (host, resourcePath, contents, callback) { +LDP.prototype.post = function (hostname, containerPath, slug, stream, container, callback) { + var ldp = this + + debug.handlers('POST -- On parent: ' + containerPath) + + // TODO: possibly package this in ldp.post + ldp.getAvailablePath(hostname, containerPath, slug, function (resourcePath) { + debug.handlers('POST -- Will create at: ' + resourcePath) + var meta = '' + + if (container) { + resourcePath += '/' + meta = ldp.suffixMeta + } + + ldp.put(hostname, resourcePath + meta, stream, function (err) { + if (err) callback(err) + + callback(null, resourcePath) + }) + }) +} + +LDP.prototype.put = function (host, resourcePath, stream, callback) { var ldp = this var root = !ldp.idp ? ldp.root : ldp.root + host + '/' var filePath = utils.uriToFilename(resourcePath, root, host) @@ -326,14 +349,7 @@ LDP.prototype.put = function (host, resourcePath, contents, callback) { debug.handlers('PUT -- Error creating directory: ' + err) return callback(error(err)) } - return fs.writeFile(filePath, contents, function (err) { - if (err) { - debug.handlers('PUT -- Error writing file: ' + err) - return callback(error(err)) - } - // Success! - return callback(null) - }) + return stream.pipe(fs.createWriteStream(filePath)) }) } diff --git a/package.json b/package.json index f541dd8aa..a5d5d78ea 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "dependencies": { "async": "^1.3.0", "body-parser": "^1.14.2", + "busboy": "^0.2.12", "cors": "^2.7.1", "debug": "^2.2.0", "express": "^4.13.3", diff --git a/test/ldp.js b/test/ldp.js index 808e016b3..3ca297e2a 100644 --- a/test/ldp.js +++ b/test/ldp.js @@ -2,6 +2,7 @@ var assert = require('chai').assert var $rdf = require('rdflib') var ns = require('../lib/vocab/ns.js').ns var LDP = require('../lib/ldp') +var stringToStream = require('./utils').stringToStream // Helper functions for the FS var rm = require('./test-utils').rm @@ -68,7 +69,8 @@ describe('LDP', function () { describe('put', function () { it('should write a file in an existing dir', function (done) { - ldp.put('localhost', '/resources/testPut.txt', 'hello world', function (err) { + var stream = stringToStream('hello world') + ldp.put('localhost', '/resources/testPut.txt', stream, function (err) { assert.notOk(err) var found = read('testPut.txt') rm('testPut.txt') @@ -78,7 +80,8 @@ describe('LDP', function () { }) it('should fail if a trailing `/` is passed', function (done) { - ldp.put('localhost', '/resources/', 'hello world', function (err) { + var stream = stringToStream('hello world') + ldp.put('localhost', '/resources/', stream, function (err) { assert.equal(err.status, 409) done() }) diff --git a/test/resources/acl/append-acl/abc.ttl b/test/resources/acl/append-acl/abc.ttl index e5df20eef..a6e141486 100644 --- a/test/resources/acl/append-acl/abc.ttl +++ b/test/resources/acl/append-acl/abc.ttl @@ -1 +1 @@ - . +[object Object] \ No newline at end of file diff --git a/test/resources/acl/append-acl/abc2.ttl.acl b/test/resources/acl/append-acl/abc2.ttl.acl index 8b7c28f18..a6e141486 100644 --- a/test/resources/acl/append-acl/abc2.ttl.acl +++ b/test/resources/acl/append-acl/abc2.ttl.acl @@ -1,8 +1 @@ -<#Owner> - <./abc2.ttl>; - ; - , , . -<#Restricted> - <./abc2.ttl>; - ; - , . +[object Object] \ No newline at end of file diff --git a/test/resources/acl/read-acl/.acl b/test/resources/acl/read-acl/.acl index c1ff5f067..a6e141486 100644 --- a/test/resources/acl/read-acl/.acl +++ b/test/resources/acl/read-acl/.acl @@ -1,10 +1 @@ -<#Owner> - a ; - <./>; - ; - , , . -<#Public> - a ; - <./>; - ; - . \ No newline at end of file +[object Object] \ No newline at end of file From fb4d2925b0d52e6327a1a6e07b7ddd5a787fdebf Mon Sep 17 00:00:00 2001 From: nicola Date: Thu, 21 Jan 2016 19:36:57 -0500 Subject: [PATCH 2/3] solved most of the issues excluding the acl's --- lib/handlers/patch.js | 12 ++++++++ lib/handlers/post.js | 44 +++++++++++++++------------ lib/handlers/put.js | 9 +++--- lib/ldp-middleware.js | 17 +---------- lib/ldp.js | 8 ++++- test/http.js | 10 +++++- test/ldp.js | 5 +-- test/resources/acl/append-acl/abc.ttl | 2 +- 8 files changed, 62 insertions(+), 45 deletions(-) diff --git a/lib/handlers/patch.js b/lib/handlers/patch.js index 7b3e4f9e5..a5fc343ec 100644 --- a/lib/handlers/patch.js +++ b/lib/handlers/patch.js @@ -9,6 +9,18 @@ var utils = require('../utils.js') var error = require('../http-error') function handler (req, res, next) { + req.setEncoding('utf8') + req.text = '' + req.on('data', function (chunk) { + req.text += chunk + }) + + req.on('end', function () { + patchHandler(req, res, next) + }) +} + +function patchHandler (req, res, next) { var ldp = req.app.locals.ldp debug('PATCH -- ' + req.originalUrl) debug('PATCH -- text length: ' + (req.text ? req.text.length : 'undefined2')) diff --git a/lib/handlers/post.js b/lib/handlers/post.js index f0fb4c052..d20ef74ce 100644 --- a/lib/handlers/post.js +++ b/lib/handlers/post.js @@ -1,8 +1,8 @@ module.exports = handler var Busboy = require('busboy') - -var debug = require('../debug').handlers +var each = require('async').each +var debug = require('debug')('ldnode:post') var header = require('../header') var patch = require('./patch') var error = require('../http-error') @@ -14,7 +14,7 @@ function handler (req, res, next) { // Handle SPARQL(-update?) query if (contentType === 'application/sparql' || contentType === 'application/sparql-update') { - debug('POST -- Handling sparql query via PATCH') + debug('switching to sparql query') return patch(req, res, next) } @@ -32,7 +32,7 @@ function handler (req, res, next) { // Check if container is a directory if (!stats.isDirectory()) { - debug('POST -- Path is not a container') + debug('path is not a container, 405!') return next(error(405, 'Requested resource is not a container')) } @@ -45,32 +45,35 @@ function handler (req, res, next) { }) function multi () { + debug('receving multiple files') var busboy = new Busboy({ headers: req.headers }) + var files = [] busboy.on('file', function (fieldname, file, filename, encoding, mimetype) { - ldp.post( - req.hostname, - containerPath, - filename, - file, - false, - function (err) { - if (err) { - return busboy.emit(err) - } - }) - }) - busboy.on('error', function (err) { - next(err) + debug('one file received via multipart: ' + filename) + files.push({stream: file, name: filename}) }) busboy.on('finish', function () { - res.sendStatus(200) + each( + files, + function (file, callback) { + ldp.post( + req.hostname, + containerPath, + file.filename, + file.stream, + false, + callback) + }, function (err) { + debug('done storing files' + (err ? 'with error' + err.message : 'with no error')) + res.sendStatus(err ? 500 : 200) + }) }) } function one () { + debug('receving one file') var linkHeader = header.parseMetadataFromHeader(req.get('Link')) - ldp.post( req.hostname, containerPath, @@ -87,3 +90,4 @@ function handler (req, res, next) { }) } } + diff --git a/lib/handlers/put.js b/lib/handlers/put.js index fe01a538a..bcd70d464 100644 --- a/lib/handlers/put.js +++ b/lib/handlers/put.js @@ -1,22 +1,23 @@ module.exports = handler -var debug = require('../debug').handlers +var debug = require('debug')('put') function handler (req, res, next) { var ldp = req.app.locals.ldp - debug('PUT -- originalUrl: ' + req.originalUrl) + debug(req.originalUrl) res.header('MS-Author-Via', 'SPARQL') ldp.put(req.hostname, req.path, req, function (err) { if (err) { - debug('PUT -- Write error: ' + err.message) + debug('error putting the file:' + err.message) err.message = 'Can\'t write file: ' + err.message return next(err) } - debug('PUT -- Write Ok. Bytes written: ' + req.text.length) + debug('succeded putting the file') res.sendStatus(201) return next() }) } + diff --git a/lib/ldp-middleware.js b/lib/ldp-middleware.js index 2cdc0e724..d5365d515 100644 --- a/lib/ldp-middleware.js +++ b/lib/ldp-middleware.js @@ -1,7 +1,6 @@ module.exports = LdpMiddleware var express = require('express') -var getRawBody = require('raw-body') var header = require('./header') var acl = require('./acl') var login = require('./login') @@ -12,20 +11,6 @@ var del = require('./handlers/delete') var patch = require('./handlers/patch') var errorPages = require('./handlers/error-pages') -function rawBody (req, res, next) { - getRawBody(req, { - length: req.headers['content-length'], - encoding: 'utf-8' - }, - function (err, string) { - if (err) { - return next(err) - } - req.text = string - next() - }) -} - function LdpMiddleware (corsSettings) { var router = express.Router('/') @@ -38,7 +23,7 @@ function LdpMiddleware (corsSettings) { router.use('/*', login.loginHandler) router.get('/*', acl.allow('Read'), get) router.post('/*', acl.allow('Append'), post) - router.patch('/*', acl.allow('Append'), rawBody, patch) + router.patch('/*', acl.allow('Append'), patch) router.put('/*', acl.allow('Append'), put) router.delete('/*', acl.allow('Write'), del) diff --git a/lib/ldp.js b/lib/ldp.js index dd6690016..bbb36f2ef 100644 --- a/lib/ldp.js +++ b/lib/ldp.js @@ -349,7 +349,13 @@ LDP.prototype.put = function (host, resourcePath, stream, callback) { debug.handlers('PUT -- Error creating directory: ' + err) return callback(error(err)) } - return stream.pipe(fs.createWriteStream(filePath)) + var file = stream.pipe(fs.createWriteStream(filePath)) + file.on('error', function () { + callback(error(500)) + }) + file.on('finish', function () { + callback(null) + }) }) } diff --git a/test/http.js b/test/http.js index 53b429f95..9683b3cf9 100644 --- a/test/http.js +++ b/test/http.js @@ -351,7 +351,15 @@ describe('HTTP APIs', function () { .set('slug', 'loans') .set('link', '; rel="type"') .send(postRequest2Body) - .expect(201, done) + .expect(201) + .end(function (err) { + if (err) return done(err) + var stats = fs.statSync(__dirname + '/resources/loans/') + if (!stats.isDirectory()) { + return done(new Error('Cannot read file just created')) + } + done() + }) }) it('should be able to access container', function (done) { server.get('/loans') diff --git a/test/ldp.js b/test/ldp.js index 3ca297e2a..194695573 100644 --- a/test/ldp.js +++ b/test/ldp.js @@ -2,7 +2,7 @@ var assert = require('chai').assert var $rdf = require('rdflib') var ns = require('../lib/vocab/ns.js').ns var LDP = require('../lib/ldp') -var stringToStream = require('./utils').stringToStream +var stringToStream = require('../lib/utils').stringToStream // Helper functions for the FS var rm = require('./test-utils').rm @@ -90,7 +90,8 @@ describe('LDP', function () { describe('delete', function () { it('should delete a file in an existing dir', function (done) { - ldp.put('localhost', '/resources/testPut.txt', 'hello world', function (err) { + var stream = stringToStream('hello world') + ldp.put('localhost', '/resources/testPut.txt', stream, function (err) { assert.notOk(err) fs.stat(ldp.root + '/resources/testPut.txt', function (err) { if (err) { diff --git a/test/resources/acl/append-acl/abc.ttl b/test/resources/acl/append-acl/abc.ttl index a6e141486..e5df20eef 100644 --- a/test/resources/acl/append-acl/abc.ttl +++ b/test/resources/acl/append-acl/abc.ttl @@ -1 +1 @@ -[object Object] \ No newline at end of file + . From a0b8264ee47921eb881760596230ff407ec84374 Mon Sep 17 00:00:00 2001 From: nicola Date: Thu, 21 Jan 2016 19:43:22 -0500 Subject: [PATCH 3/3] fixing debug issue --- lib/handlers/put.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/handlers/put.js b/lib/handlers/put.js index bcd70d464..7bd2b4c41 100644 --- a/lib/handlers/put.js +++ b/lib/handlers/put.js @@ -1,6 +1,6 @@ module.exports = handler -var debug = require('debug')('put') +var debug = require('debug')('ldnode:put') function handler (req, res, next) { var ldp = req.app.locals.ldp