chore: return empty buffer for files with no data

Part of making the hases emitted by js-ipfs the same as go-ipfs meant unsetting
the data attribute of UnixFS metadata objects if the data buffer had 0 length.

ipfs.files.get converts a pull stream of files into a readable stream of files
and all of the content pull streams too. If an empty buffer is not emitted by
the file content pull stream for file with no data then no file is written
out to the filesystem.

This was a fun bug to track down.

This module used to emit empty buffers for UnixFS metadata with no data
property but I removed it as part of #208 as there was no test that failed
without it so here we reinstate it and add a test.

Author: achingbrain

package.json

@@ -60,7 +60,7 @@
     "cids": "~0.5.3",
     "deep-extend": "~0.6.0",
     "ipfs-unixfs": "~0.1.14",
-    "ipld": "^0.17.2",
+    "ipld": "~0.17.2",
     "ipld-dag-pb": "~0.14.4",
     "left-pad": "^1.3.0",
     "lodash": "^4.17.10",

src/exporter/file.js

@@ -63,15 +63,11 @@ function streamBytes (dag, node, fileSize, offset, length) {
   let streamPosition = 0
 
   function getData ({ node, start }) {
-    if (!node || !node.data) {
-      return
-    }
-
     try {
       const file = UnixFS.unmarshal(node.data)
 
       if (!file.data) {
-        return
+        return Buffer.alloc(0)
       }
 
       const block = extractDataFromBlock(file.data, start, offset, end)

test/exporter.js

@@ -14,6 +14,12 @@ const CID = require('cids')
 const loadFixture = require('aegir/fixtures')
 const doUntil = require('async/doUntil')
 const waterfall = require('async/waterfall')
+const series = require('async/series')
+const fs = require('fs')
+const path = require('path')
+const push = require('pull-pushable')
+const toPull = require('stream-to-pull-stream')
+const toStream = require('pull-stream-to-stream')
 
 const unixFSEngine = require('./../src')
 const exporter = unixFSEngine.exporter
@@ -64,6 +70,54 @@ module.exports = (repo) => {
       })
     }
 
+    function addTestDirectory ({directory, strategy = 'balanced', maxChunkSize}, callback) {
+      const input = push()
+      const dirName = path.basename(directory)
+
+      pull(
+        input,
+        pull.map((file) => {
+          const ipfsPath = `${path.join(dirName, file.substring(directory.length))}`
+
+          return {
+            path: ipfsPath,
+            content: toPull.source(fs.createReadStream(file))
+          }
+        }),
+        importer(ipld, {
+          strategy,
+          maxChunkSize
+        }),
+        pull.collect(callback)
+      )
+
+      const listFiles = (directory, depth, stream, cb) => {
+        waterfall([
+          (done) => fs.stat(directory, done),
+          (stats, done) => {
+            if (stats.isDirectory()) {
+              return waterfall([
+                (done) => fs.readdir(directory, done),
+                (children, done) => {
+                  series(
+                    children.map(child => (next) => listFiles(path.join(directory, child), depth + 1, stream, next)),
+                    done
+                  )
+                }
+              ], done)
+            }
+
+            stream.push(directory)
+            done()
+          }
+        ], cb)
+      }
+
+      listFiles(directory, 0, input, () => {
+        input.end()
+      })
+    }
+
     function checkBytesThatSpanBlocks (strategy, cb) {
       const bytesInABlock = 262144
       const bytes = Buffer.alloc(bytesInABlock + 100, 0)
@@ -517,6 +571,55 @@ module.exports = (repo) => {
       checkBytesThatSpanBlocks('trickle', done)
     })
 
+    it('exports a directory containing an empty file whose content gets turned into a ReadableStream', function (done) {
+      // replicates the behaviour of ipfs.files.get
+      waterfall([
+        (cb) => addTestDirectory({
+          directory: path.join(__dirname, 'fixtures', 'dir-with-empty-files')
+        }, cb),
+        (result, cb) => {
+          const dir = result.pop()
+
+          pull(
+            exporter(dir.multihash, ipld),
+            pull.map((file) => {
+              if (file.content) {
+                file.content = toStream.source(file.content)
+                file.content.pause()
+              }
+
+              return file
+            }),
+            pull.collect((error, files) => {
+              if (error) {
+                return cb(error)
+              }
+
+              series(
+                files
+                  .filter(file => Boolean(file.content))
+                  .map(file => {
+                    return (done) => {
+                      if (file.content) {
+                        file.content
+                          .pipe(toStream.sink(pull.collect((error, bufs) => {
+                            expect(error).to.not.exist()
+                            expect(bufs.length).to.equal(1)
+                            expect(bufs[0].length).to.equal(0)
+
+                            done()
+                          })))
+                      }
+                    }
+                  }),
+                cb
+              )
+            })
+          )
+        }
+      ], done)
+    })
+
     // TODO: This needs for the stores to have timeouts,
     // otherwise it is impossible to predict if a file doesn't
     // really exist