Add support for XZ compressed images (#376)

This PR introduces `etcher-image-stream`:

https://github.com/resin-io-modules/etcher-image-stream

a module that will handle support for decompression, URL streaming, and
any other way to get a source of data to write in Etcher.

Most of the changes in this PR are because XZ decompression includes a
native dependency (no pure JS implementations out there for now), so we
had to tweak various things for the dependency to work correctly on
Etcher/Electron.

See: https://github.com/resin-io/etcher/issues/325
Signed-off-by: Juan Cruz Viotti <jviottidc@gmail.com>

.travis.yml

@@ -2,7 +2,7 @@ language: cpp
 sudo: false
 
 env:
-  - NODE_VERSION="4"
+  - NODE_VERSION="5"
 os:
   - linux
   - osx

CONTRIBUTING.md

@@ -10,7 +10,9 @@ Thanks for your interest in contributing to this project! This document aims to
 
 ## Running locally
 
-You can manually run the application with the following steps:
+- Install [NodeJS v5.10.0](https://nodejs.org/en/).
+
+Sadly we need to enforce the same NodeJS version that the Electron version we use is running to avoid module version mismatches when building native dependencies (`electron-rebuild` doesn't seem to be enough).
 
 - Clone the repository.
 

appveyor.yml

@@ -11,13 +11,13 @@ cache:
 # what combinations to test
 environment:
   matrix:
-    - nodejs_version: 4
+    - nodejs_version: 5
 
 install:
   - ps: Install-Product node $env:nodejs_version x64
   - npm -g install npm@2
   - set PATH=%APPDATA%\npm;%PATH%
-  - npm install
+  - npm install --build-from-source
 
 build: off
 

lib/gui/partials/main.html

@@ -8,7 +8,7 @@
       <div class="space-vertical-large">
         <div ng-hide="app.selection.hasImage()">
           <button class="btn btn-primary btn-brick" ng-click="app.openImageSelector()">Select image</button>
-          <p class="step-footer">.img, .iso, or <span class="step-footer-underline" uib-tooltip=".zip">compressed images</span></p>
+          <p class="step-footer">.img, .iso, or <span class="step-footer-underline" uib-tooltip=".zip, .xz">compressed images</span></p>
         </div>
         <div ng-show="app.selection.hasImage()">
           <div ng-bind="app.selection.getImage() | basename"></div>

lib/src/writer.js

@@ -17,10 +17,9 @@
 'use strict';
 
 const imageWrite = require('etcher-image-write');
-const zipImage = require('resin-zip-image');
+const imageStream = require('etcher-image-stream');
 const Bluebird = require('bluebird');
 const umount = Bluebird.promisifyAll(require('umount'));
-const fs = require('fs');
 const os = require('os');
 const isWindows = os.platform() === 'win32';
 
@@ -28,35 +27,6 @@ if (isWindows) {
   var removedrive = Bluebird.promisifyAll(require('removedrive'));
 }
 
-/**
- * @summary Get image readable stream
- * @function
- * @private
- *
- * @description
- * This function adds a custom `.length` property
- * to the stream which equals the image size in bytes.
- *
- * @param {String} image - path to image
- * @returns {ReadableStream} image stream
- *
- * @example
- * const stream = writer.getImageStream('foo/bar/baz.img');
- */
-exports.getImageStream = function(image) {
-  if (zipImage.isZip(image)) {
-    if (!zipImage.isValidZipImage(image)) {
-      return Bluebird.reject(new Error('Invalid zip image'));
-    }
-
-    return zipImage.extractImage(image);
-  }
-
-  let stream = fs.createReadStream(image);
-  stream.length = fs.statSync(image).size;
-  return Bluebird.resolve(stream);
-};
-
 /**
  * @summary Write an image to a disk drive
  * @function
@@ -66,7 +36,7 @@ exports.getImageStream = function(image) {
  * See https://github.com/resin-io-modules/etcher-image-write for information
  * about the `state` object passed to `onProgress` callback.
  *
- * @param {String} image - path to image
+ * @param {String} imagePath - path to image
  * @param {Object} drive - drive
  * @param {Object} options - options
  * @param {Boolean} [options.unmountOnSuccess=false] - unmount on success
@@ -88,13 +58,14 @@ exports.getImageStream = function(image) {
  *   console.log('Done!');
  * });
  */
-exports.writeImage = function(image, drive, options, onProgress) {
+exports.writeImage = function(imagePath, drive, options, onProgress) {
   return umount.umountAsync(drive.device).then(function() {
-    return exports.getImageStream(image);
+    return imageStream.getFromFilePath(imagePath);
-  }).then(function(stream) {
+  }).then(function(image) {
-    return imageWrite.write(drive.device, stream, {
+    return imageWrite.write(drive.device, image.stream, {
       check: options.validateWriteOnSuccess,
-      size: stream.length
+      size: image.size,
+      transform: image.transform
     });
   }).then(function(writer) {
     return new Bluebird(function(resolve, reject) {

package.json

@@ -10,9 +10,7 @@
     "url": "git@github.com:resin-io/etcher.git"
   },
   "scripts": {
-    "test:main": "electron-mocha --recursive tests/src -R progress",
+    "test": "electron-mocha --recursive --renderer tests/gui -R progress",
-    "test:gui": "electron-mocha --recursive --renderer tests/gui -R progress",
-    "test": "npm run-script test:main && npm run-script test:gui",
     "start": "electron lib/start.js"
   },
   "author": "Juan Cruz Viotti <juan@resin.io>",
@@ -65,7 +63,6 @@
     "resin-cli-errors": "^1.2.0",
     "resin-cli-form": "^1.4.1",
     "resin-cli-visuals": "^1.2.8",
-    "resin-zip-image": "^1.1.2",
     "sudo-prompt": "^3.1.0",
     "trackjs": "^2.1.16",
     "umount": "^1.1.3",
@@ -85,8 +82,6 @@
     "gulp-sass": "^2.0.4",
     "jshint": "^2.9.1",
     "jshint-stylish": "^2.0.1",
-    "mochainon": "^1.0.0",
+    "mochainon": "^1.0.0"
-    "rindle": "^1.3.0",
-    "tmp": "0.0.28"
   }
 }

tests/src/writer.spec.js

@@ -1,89 +0,0 @@
-'use strict';
-
-const m = require('mochainon');
-const ReadableStream = require('stream').Readable;
-const Bluebird = require('bluebird');
-const fs = Bluebird.promisifyAll(require('fs'));
-const path = require('path');
-const tmp = require('tmp');
-const rindle = require('rindle');
-const writer = require('../../lib/src/writer');
-
-describe('Writer:', function() {
-
-  describe('.getImageStream()', function() {
-
-    describe('given a valid image file', function() {
-
-      beforeEach(function() {
-        this.image = path.join(__dirname, '..', 'utils', 'data.random');
-      });
-
-      it('should return a readable stream', function(done) {
-        writer.getImageStream(this.image).then(function(stream) {
-          m.chai.expect(stream).to.be.an.instanceof(ReadableStream);
-        }).nodeify(done);
-      });
-
-      it('should append a .length property with the correct size', function(done) {
-        writer.getImageStream(this.image).then(function(stream) {
-          m.chai.expect(stream.length).to.equal(2097152);
-        }).nodeify(done);
-      });
-
-    });
-
-    describe('given a valid image zip', function() {
-
-      beforeEach(function() {
-        this.image = path.join(__dirname, '..', 'utils', 'data.zip');
-      });
-
-      it('should return a readable stream', function(done) {
-        writer.getImageStream(this.image).then(function(stream) {
-          m.chai.expect(stream).to.be.an.instanceof(ReadableStream);
-        }).nodeify(done);
-      });
-
-      it('should append a .length property with the correct size', function(done) {
-        writer.getImageStream(this.image).then(function(stream) {
-          m.chai.expect(stream.length).to.equal(2097152);
-        }).nodeify(done);
-      });
-
-      it('should pipe the image from the zip', function(done) {
-        const tmpFile = tmp.tmpNameSync();
-        const image = path.join(__dirname, '..', 'utils', 'data.random');
-        const output = fs.createWriteStream(tmpFile);
-
-        writer.getImageStream(this.image).then(function(stream) {
-          return stream.pipe(output);
-        }).then(rindle.wait).then(function() {
-          return Bluebird.props({
-            output: fs.readFileAsync(tmpFile),
-            data: fs.readFileAsync(image)
-          });
-        }).then(function(results) {
-          m.chai.expect(results.output).to.deep.equal(results.data);
-          return fs.unlinkAsync(tmpFile);
-        }).nodeify(done);
-      });
-
-    });
-
-    describe('given an invalid image zip', function() {
-
-      beforeEach(function() {
-        this.image = path.join(__dirname, '..', 'utils', 'invalid.zip');
-      });
-
-      it('should be rejected with an error', function() {
-        const promise = writer.getImageStream(this.image);
-        m.chai.expect(promise).to.be.rejectedWith('Invalid zip image');
-      });
-
-    });
-
-  });
-
-});