Update etcher-sdk and use it in the cli

lib/cli/etcher.js

@@ -21,7 +21,8 @@ const Bluebird = require('bluebird')
 const visuals = require('resin-cli-visuals')
 const form = require('resin-cli-form')
 const bytes = require('pretty-bytes')
-const ImageWriter = require('../sdk/writer')
+const sdk = require('etcher-sdk')
+
 const utils = require('./utils')
 const options = require('./options')
 const messages = require('../shared/messages')
@@ -72,70 +73,90 @@ permissions.isElevated().then((elevated) => {
     })
   }
 
-  const progressBars = {
-    write: new visuals.Progress('Flashing'),
-    check: new visuals.Progress('Validating')
+  const progressBars = new Map()
+  let lastStateType
+
+  const onProgress = (state) => {
+    state.message = state.active > 1
+      ? `${bytes(state.totalSpeed)}/s total, ${bytes(state.speed)}/s x ${state.active}`
+      : `${bytes(state.totalSpeed)}/s`
+
+    state.message = `${state.type === 'flashing' ? 'Flashing' : 'Validating'}: ${state.message}`
+
+    if (state.percentage === undefined) {
+      state.message += ` - ${bytes(state.bytes)} written`
+    }
+
+    // Update progress bar
+    let progressBar = progressBars.get(state.type)
+    if (progressBar === undefined) {
+      // Stop the spinner if there is one
+      if ((lastStateType !== undefined) && (lastStateType !== state.type)) {
+        const spinner = progressBars.get(lastStateType)
+        if ((spinner !== undefined) && (spinner instanceof visuals.Spinner)) {
+          console.log()
+          spinner.stop()
+        }
+      }
+      if (state.percentage === undefined) {
+        progressBar = new visuals.Spinner(state.message)
+        progressBar.start()
+      } else {
+        progressBar = new visuals.Progress(state.message)
+        progressBar.update(state)
+      }
+      progressBars.set(state.type, progressBar)
+    } else {
+      if (progressBar instanceof visuals.Spinner) {
+        progressBar.spinner.setSpinnerTitle(state.message)
+      } else {
+        progressBar.update(state)
+      }
+    }
+    lastStateType = state.type
   }
 
-  return new Bluebird((resolve, reject) => {
-    /**
-     * @summary Progress update handler
-     * @param {Object} state - progress state
-     * @private
-     * @example
-     * writer.on('progress', onProgress)
-     */
-    const onProgress = (state) => {
-      state.message = state.active > 1
-        ? `${bytes(state.totalSpeed)}/s total, ${bytes(state.speed)}/s x ${state.active}`
-        : `${bytes(state.totalSpeed)}/s`
-
-      state.message = `${state.type === 'write' ? 'Flashing' : 'Validating'}: ${state.message}`
-
-      // Update progress bar
-      progressBars[state.type].update(state)
-    }
-
-    const writer = new ImageWriter({
-      verify: options.check,
-      unmountOnSuccess: options.unmount,
-      checksumAlgorithms: options.check ? [ 'xxhash' ] : []
-    })
-
-    /**
-     * @summary Finish handler
-     * @private
-     * @example
-     * writer.on('finish', onFinish)
-     */
-    const onFinish = function () {
-      resolve(Array.from(writer.destinations.values()))
-    }
-
-    writer.on('progress', onProgress)
-    writer.on('error', reject)
-    writer.on('finish', onFinish)
-
-    // NOTE: Drive can be (String|Array)
-    const destinations = [].concat(answers.drive)
-
-    writer.write(imagePath, destinations)
+  const adapter = new sdk.scanner.adapters.BlockDeviceAdapter(() => {
+    return options.unmount
   })
-}).then((results) => {
+  const scanner = new sdk.scanner.Scanner([ adapter ])
+  return new Promise((resolve, reject) => {
+    scanner.on('ready', resolve)
+    scanner.on('error', reject)
+    scanner.start()
+  })
+  .then(() => {
+    return (new sdk.sourceDestination.File(imagePath, sdk.sourceDestination.File.OpenFlags.Read)).getInnerSource()
+  })
+  .then((innerSource) => {
+    // NOTE: Drive can be (String|Array)
+    const destinations = [].concat(answers.drive).map((device) => {
+      const drive = scanner.getBy('device', device)
+      if (drive === undefined) {
+        throw new Error(`No such drive ${device}`)
+      }
+      return drive
+    })
+    return sdk.multiWrite.pipeSourceToDestinations(
+      innerSource,
+      destinations,
+      (destination, error) => {
+        console.log(`Error "${error}" on ${destination.drive}`)
+      },
+      onProgress,
+      options.check
+    )
+  })
+}).then(({ failures, bytesWritten }) => {
   let exitCode = EXIT_CODES.SUCCESS
 
-  if (options.check) {
+  if (failures.size > 0) {
+    exitCode = EXIT_CODES.GENERAL_ERROR
     console.log('')
-    console.log('Checksums:')
 
-    _.forEach(results, (result) => {
-      if (result.error) {
-        exitCode = EXIT_CODES.GENERAL_ERROR
-        console.log(`  - ${result.device.device}: ${result.error.message}`)
-      } else {
-        console.log(`  - ${result.device.device}: ${result.checksum.xxhash}`)
-      }
-    })
+    for (const [ destination, error ] of failures) {
+      console.log(`  - ${destination.drive}: ${error.message}`)
+    }
   }
 
   process.exit(exitCode)

lib/gui/app/pages/main/templates/main.tpl.html

@@ -119,7 +119,7 @@
       <div class="space-vertical-large">
         <progress-button
           tabindex="3"
-          striped="main.state.getFlashState().type == 'check'"
+          striped="main.state.getFlashState().type == 'verifying'"
           active = "main.state.isFlashing()"
           percentage="main.state.getFlashState().percentage"
           label="flash.getProgressButtonLabel()"

lib/gui/modules/child-writer.js

@@ -93,155 +93,31 @@ function lastMapValue(map) {
   return value
 }
 
-function writeAndValidate(source, destination, verify, onProgress, onFail, onFinish, onError) {
-  let checksum
-  let sparse
-  let sourceMetadata
-  let step = 'flashing'
-  let lastPosition = 0
-  const errors = new Map()  // destination -> error map  TODO: include open and close errors in it
-  const state = {
-    active: destination.destinations.size,
-    flashing: destination.destinations.size,
-    verifying: 0,
-    failed: 0,
-    successful: 0,
-    type: step
-  }
-  function allDestinationsFailed() {
-    return (errors.size === destination.destinations.size)
-  }
-  function updateState() {
-    state.type = step
-    state.failed = errors.size
-    state.active = destination.destinations.size - state.failed
-    if (step === 'flashing') {
-      state.flashing = state.active
-      state.verifying = 0
-    } else if (step === 'check') {
-      state.flashing = 0
-      state.verifying = state.active
-    } else if (step === 'finished') {
-      state.successful = state.active
-    }
-  }
-  function onProgress2(progressEvent) {
-    lastPosition = progressEvent.position
-    let size
-    if (sparse && (sourceMetadata.blockMap !== undefined)) {
-      size = sourceMetadata.blockMap.mappedBlockCount * sourceMetadata.blockMap.blockSize
-      progressEvent.percentage = progressEvent.bytes / size * 100
-    } else {
-      size = sourceMetadata.size
-      progressEvent.percentage = progressEvent.position / size * 100
-    }
-    // NOTE: We need to guard against this becoming Infinity,
-    // because that value isn't transmitted properly over IPC and becomes `null`
-    progressEvent.eta = progressEvent.speed ? (size - progressEvent.bytes) / progressEvent.speed : null
-    progressEvent.totalSpeed = progressEvent.speed * state.active
-    Object.assign(progressEvent, state)
-    onProgress(progressEvent)
-  }
-  function onFail2(error) {
-    errors.set(error.destination, error.error)
-    updateState()
-    onFail(error)
-  }
-  destination.on('fail', onFail2)
-  return Promise.all([ source.getInnerSource(), destination.open() ])
-  .then(([ _source ]) => {
-    source = _source
-    return source.canCreateSparseReadStream()
+function writeAndValidate(source, destinations, verify, onProgress, onFail, onFinish, onError) {
+  return source.getInnerSource()
+  .then((innerSource) => {
+    return sdk.multiWrite.pipeSourceToDestinations(
+      innerSource,
+      destinations,
+      onFail,
+      onProgress,
+      verify,
+    )
   })
-  .then((_sparse) => {
-    sparse = _sparse
-    let sourceStream
-    let destinationStream
-    if (sparse) {
-      // TODO: calculate checksums in source if needed
-      sourceStream = source.createSparseReadStream()
-      destinationStream = destination.createSparseWriteStream()
-    } else {
-      sourceStream = source.createReadStream()
-      destinationStream = destination.createWriteStream()
-    }
-    return Promise.all([ sourceStream, destinationStream, source.getMetadata() ])
-  })
-  .then(([ sourceStream, destinationStream, metadata ]) => {
-    destinationStream.on('fail', onFail2)
-    sourceMetadata = metadata
-    return new Promise((resolve, reject) => {
-      let done = false
-      let hasher
-      sourceStream.on('error', reject)
-      destinationStream.on('progress', onProgress2)
-      if (verify && !sparse) {
-        hasher = sdk.sourceDestination.createHasher()
-        hasher.on('checksum', (cs) => {
-          checksum = cs
-          if (done) {
-            resolve()
-          }
-        })
-        sourceStream.pipe(hasher)
-      }
-      destinationStream.on('done', () => {
-        done = true;
-        if (allDestinationsFailed() && (hasher !== undefined)) {
-          sourceStream.unpipe(hasher)
-          verify = false
-          resolve()
-          return
-        }
-
-        if (sparse || !verify || (checksum !== undefined)) {
-          resolve()
-        }
-      })
-      sourceStream.pipe(destinationStream)
-    })
-  })
-  .then(() => {
-    if (sourceMetadata.size == null) {
-      // This is needed for compressed sources for which we don't know the uncompressed size:
-      // After writing the image, we know the size.
-      sourceMetadata.size = lastPosition
-    }
-    if (verify) {
-      step = 'check'
-      updateState()
-      const verifier = destination.createVerifier(sparse ? sourceMetadata.blockMap : checksum, sourceMetadata.size)  // TODO: ensure blockMap exists
-      verifier.on('progress', onProgress2)
-      verifier.on('fail', onFail2)
-      return new Promise((resolve) => {
-        verifier.on('finish', resolve);
-        verifier.run();
-      });
-    }
-  })
-  .then(() => {
-    step = 'finished'
-    updateState()
-    return Promise.all([ source.close(), destination.close() ])
-  })
-  .then(() => {
+  .then(({ failures, bytesWritten }) => {
     // If all destinations errored, treat the last fail as an error
-    if (allDestinationsFailed()) {
-      const lastError = lastMapValue(errors)
-      throw lastError
+    if (failures.size === destinations.length) {
+      throw lastMapValue(failures)
     }
     const result = {
-      bytesWritten: lastPosition,
+      bytesWritten,
       devices: {
-        failed: state.failed,
-        successful: state.active
+        failed: failures.size,
+        successful: destinations.length - failures.size,
       },
       errors: []
     }
-    if (verify && (checksum !== undefined)) {
-      result.checksum = { xxhash: checksum }
-    }
-    for (const [ destination, error ] of errors) {
+    for (const [ destination, error ] of failures) {
       error.device = destination.drive.device
       result.errors.push(error)
     }
@@ -337,10 +213,10 @@ ipc.connectTo(IPC_SERVER_ID, () => {
      * @example
      * writer.on('fail', onFail)
      */
-    const onFail = (error) => {
+    const onFail = (destination, error) => {
       ipc.of[IPC_SERVER_ID].emit('fail', {
-        device: error.destination.drive,  // TODO: device should be error.destination
-        error: errors.toJSON(error.error)
+        device: destination.drive,  // TODO: device should be destination
+        error: errors.toJSON(error)
       })
     }
 
@@ -348,11 +224,10 @@ ipc.connectTo(IPC_SERVER_ID, () => {
     const dests = options.destinations.map((destination) => {
       return new sdk.sourceDestination.BlockDevice(destination, options.unmountOnSuccess)
     })
-    const destination = new sdk.sourceDestination.MultiDestination(dests)
     const source = new sdk.sourceDestination.File(options.imagePath, sdk.sourceDestination.File.OpenFlags.Read)
     writeAndValidate(
       source,
-      destination,
+      dests,
       options.validateWriteOnSuccess,
       onProgress,
       onFail,

npm-shrinkwrap.json

@@ -35,8 +35,8 @@
       "resolved": "https://registry.npmjs.org/@types/angular/-/angular-1.6.51.tgz"
     },
     "@types/bluebird": {
-      "version": "3.5.20",
-      "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz"
+      "version": "3.5.21",
+      "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.21.tgz"
     },
     "@types/chart.js": {
       "version": "2.7.41",
@@ -669,8 +669,8 @@
       "dev": true
     },
     "aws-sdk": {
-      "version": "2.263.1",
-      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.263.1.tgz",
+      "version": "2.267.1",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.267.1.tgz",
       "dependencies": {
         "ieee754": {
           "version": "1.1.8",
@@ -3068,7 +3068,7 @@
     },
     "etcher-sdk": {
       "version": "0.0.1",
-      "resolved": "git://github.com/resin-io-modules/etcher-sdk.git#9d483eec059a9e149d0f1c2e2746b8a816f87bb4",
+      "resolved": "git://github.com/resin-io-modules/etcher-sdk.git#356e40b2190492cec55fd92d58f0cc218cae4ed2",
       "dependencies": {
         "@types/lodash": {
           "version": "4.14.110",
@@ -3147,8 +3147,8 @@
           "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz"
         },
         "yauzl": {
-          "version": "2.9.2",
-          "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.9.2.tgz"
+          "version": "2.10.0",
+          "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz"
         }
       }
     },
@@ -5787,8 +5787,8 @@
       }
     },
     "node-raspberrypi-usbboot": {
-      "version": "0.0.5",
-      "resolved": "https://registry.npmjs.org/node-raspberrypi-usbboot/-/node-raspberrypi-usbboot-0.0.5.tgz",
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/node-raspberrypi-usbboot/-/node-raspberrypi-usbboot-0.0.6.tgz",
       "dependencies": {
         "@types/node": {
           "version": "6.0.113",

package.json

@@ -63,7 +63,7 @@
     "debug": "3.1.0",
     "drivelist": "6.4.6",
     "electron-is-running-in-asar": "1.0.0",
-    "etcher-sdk": "github:resin-io-modules/etcher-sdk#9d483eec059a9e149d0f1c2e2746b8a816f87bb4",
+    "etcher-sdk": "github:resin-io-modules/etcher-sdk#356e40b2190492cec55fd92d58f0cc218cae4ed2",
     "file-type": "4.1.0",
     "flexboxgrid": "6.3.0",
     "gpt": "1.0.0",