Show raspberry pi usbboot update progress in devices list

lib/gui/app/app.js

@@ -28,8 +28,12 @@ var angular = require('angular')
 
 const electron = require('electron')
 const Bluebird = require('bluebird')
+const sdk = require('etcher-sdk')
+const _ = require('lodash')
 const semver = require('semver')
 const uuidV4 = require('uuid/v4')
+
+>>>>>>> Show raspberry pi usbboot update progress in devices list
 const EXIT_CODES = require('../../shared/exit-codes')
 const messages = require('../../shared/messages')
 const s3Packages = require('../../shared/s3-packages')
@@ -229,35 +233,77 @@ app.run(() => {
   })
 })
 
-app.run(($timeout) => {
-  function updateDrives() {
-    const drives = Array.from(driveScanner.drives)
-    const BLACKLISTED_DRIVES = settings.has('driveBlacklist')
-      ? settings.get('driveBlacklist').split(',')
-      : []
 
+app.run(($timeout) => {
+  const BLACKLISTED_DRIVES = settings.has('driveBlacklist')
+    ? settings.get('driveBlacklist').split(',')
+    : []
+
+  function driveIsAllowed(drive) {
+    return !(
+      BLACKLISTED_DRIVES.includes(drive.devicePath) ||
+      BLACKLISTED_DRIVES.includes(drive.device) ||
+      BLACKLISTED_DRIVES.includes(drive.raw)
+    )
+  }
+
+  function prepareDrive(drive) {
+    if (drive instanceof sdk.sourceDestination.BlockDevice) {
+      return drive.drive
+    } else if (drive instanceof sdk.sourceDestination.UsbbootDrive) {
+      // This is a workaround etcher expecting a device string and a size
+      drive.device = drive.usbDevice.portId
+      drive.size = 0
+      drive.progress = 0
+      drive.on('progress', (progress) => {
+        updateDriveProgress(drive, progress)
+      })
+      return drive
+    }
+  }
+
+  function setDrives(drives) {
+    drives = _.values(drives)
+    availableDrives.setDrives(drives)
     // Safely trigger a digest cycle.
     // In some cases, AngularJS doesn't acknowledge that the
     // available drives list has changed, and incorrectly
     // keeps asking the user to "Connect a drive".
-    $timeout(() => {
+    $timeout()
-      const allowedDrives = drives
-      .filter((drive) => {
-        return !(
-          BLACKLISTED_DRIVES.includes(drive.devicePath) ||
-          BLACKLISTED_DRIVES.includes(drive.device) ||
-          BLACKLISTED_DRIVES.includes(drive.raw)
-	)
-      })
-      .map((drive) => {
-        // TODO: we should be able to use the SourceDestination `drive` directly
-        return drive.drive
-      })
-      availableDrives.setDrives(allowedDrives)
-    })
   }
-  driveScanner.on('attach', updateDrives)
+
-  driveScanner.on('detach', updateDrives)
+  function getDrives() {
+    return _.keyBy(availableDrives.getDrives() || [], 'device')
+  }
+
+  function addDrive(drive) {
+    drive = prepareDrive(drive)
+    if (!driveIsAllowed(drive)) {
+      return
+    }
+    const drives = getDrives()
+    drives[drive.device] = drive
+    setDrives(drives)
+  }
+
+  function removeDrive(drive) {
+    drive = prepareDrive(drive)
+    const drives = getDrives()
+    delete drives[drive.device]
+    setDrives(drives)
+  }
+
+  function updateDriveProgress(drive, progress) {
+    const drives = getDrives()
+    const drive_ = drives[drive.device]
+    if (drive !== undefined) {
+      drive.progress = progress
+      setDrives(drives)
+    }
+  }
+
+  driveScanner.on('attach', addDrive)
+  driveScanner.on('detach', removeDrive)
 
   driveScanner.on('error', (error) => {
     // Stop the drive scanning loop in case of errors,

lib/gui/app/components/file-selector/file-selector/file-selector.jsx

@@ -174,7 +174,7 @@ class FileSelector extends React.PureComponent {
     if (!supportedFormats.isSupportedImage(image.path)) {
       const invalidImageError = errors.createUserError({
         title: 'Invalid image',
-        description: messages.error.invalidImage(image)
+        description: messages.error.invalidImage(image.path)
       })
 
       osDialog.showError(invalidImageError)
@@ -229,7 +229,7 @@ class FileSelector extends React.PureComponent {
       // An easy way so we can quickly identify if we're making use of
       // certain features without printing pages of text to DevTools.
       image.logo = Boolean(image.logo)
-      image.bmap = Boolean(image.bmap)
+      image.blockMap = Boolean(image.blockMap)
 
       analytics.logEvent('Select image', {
         image,

lib/gui/app/models/selection-state.js

@@ -211,9 +211,7 @@ exports.getImageSize = () => {
   return _.get(store.getState().toJS(), [
     'selection',
     'image',
-    'size',
+    'size'
-    'final',
-    'value'
   ])
 }
 

lib/gui/app/models/store.js

@@ -67,8 +67,7 @@ const flashStateNoNilFields = [
  */
 const selectImageNoNilFields = [
   'path',
-  'extension',
+  'extension'
-  'size'
 ]
 
 /**
@@ -406,29 +405,11 @@ const storeReducer = (state = DEFAULT_STATE, action) => {
         }
       }
 
-      if (!_.isPlainObject(action.data.size)) {
-        throw errors.createError({
-          title: `Invalid image size: ${action.data.size}`
-        })
-      }
-
       const MINIMUM_IMAGE_SIZE = 0
 
-      if (!_.isInteger(action.data.size.original) || action.data.size.original < MINIMUM_IMAGE_SIZE) {
+      if ((action.data.size !== undefined) && (action.data.size < MINIMUM_IMAGE_SIZE)) {
         throw errors.createError({
-          title: `Invalid original image size: ${action.data.size.original}`
+          title: `Invalid image size: ${action.data.size}`
-        })
-      }
-
-      if (!_.isInteger(action.data.size.final.value) || action.data.size.final.value < MINIMUM_IMAGE_SIZE) {
-        throw errors.createError({
-          title: `Invalid final image size: ${action.data.size.final.value}`
-        })
-      }
-
-      if (!_.isBoolean(action.data.size.final.estimation)) {
-        throw errors.createError({
-          title: `Invalid final image size estimation flag: ${action.data.size.final.estimation}`
         })
       }
 

lib/gui/app/modules/drive-scanner.js

@@ -20,7 +20,6 @@ const sdk = require('etcher-sdk')
 const process = require('process')
 
 const settings = require('../models/settings')
-const permissions = require('../../../shared/permissions')
 
 function includeSystemDrives() {
   return settings.get('unsafeMode') && !settings.get('disableUnsafeMode')
@@ -30,12 +29,11 @@ const adapters = [
   new sdk.scanner.adapters.BlockDeviceAdapter(includeSystemDrives)
 ]
 
-permissions.isElevated()
+// Can't use permissions.isElevated() here as it returns a promise and we need to set
-.then((isElevated) => {
+// module.exports = scanner right now.
-  if ((process.platform !== 'linux') || isElevated) {
+if ((process.platform !== 'linux') || (process.geteuid() === 0)) {
-    adapters.push(new sdk.scanner.adapters.UsbbootDeviceAdapter())
+  adapters.push(new sdk.scanner.adapters.UsbbootDeviceAdapter())
-  }
+}
-})
 
 const scanner = new sdk.scanner.Scanner(adapters)
 

lib/gui/app/pages/main/controllers/flash.js

@@ -188,6 +188,7 @@ module.exports = function (
         exceptionReporter.report(error)
       }
     }).finally(() => {
+      availableDrives.setDrives([])
       driveScanner.start()
       unsubscribe()
     })

lib/gui/app/pages/main/controllers/image-selection.js

@@ -19,10 +19,11 @@
 const _ = require('lodash')
 const Bluebird = require('bluebird')
 const path = require('path')
+const sdk = require('etcher-sdk')
+
 const store = require('../../../models/store')
 const messages = require('../../../../../shared/messages')
 const errors = require('../../../../../shared/errors')
-const imageStream = require('../../../../../sdk/image-stream')
 const supportedFormats = require('../../../../../shared/supported-formats')
 const analytics = require('../../../modules/analytics')
 const settings = require('../../../models/settings')
@@ -124,7 +125,7 @@ module.exports = function (
       // An easy way so we can quickly identify if we're making use of
       // certain features without printing pages of text to DevTools.
       image.logo = Boolean(image.logo)
-      image.bmap = Boolean(image.bmap)
+      image.blockMap = Boolean(image.blockMap)
 
       return analytics.logEvent('Select image', {
         image,
@@ -145,10 +146,33 @@ module.exports = function (
    * ImageSelectionController.selectImageByPath('path/to/image.img');
    */
   this.selectImageByPath = (imagePath) => {
-    imageStream.getImageMetadata(imagePath)
+    if (!supportedFormats.isSupportedImage(imagePath)) {
-      .then((imageMetadata) => {
+      const invalidImageError = errors.createUserError({
-        $timeout(() => {
+        title: 'Invalid image',
-          this.selectImage(imageMetadata)
+        description: messages.error.invalidImage(imagePath)
+      })
+
+      osDialog.showError(invalidImageError)
+      analytics.logEvent('Invalid image', { path: imagePath })
+      return
+    }
+
+    const source = new sdk.sourceDestination.File(imagePath, sdk.sourceDestination.File.OpenFlags.Read)
+    source.getInnerSource()
+    .then((innerSource) => {
+      return innerSource.getMetadata()
+      .then((metadata) => {
+        return innerSource.getPartitionTable()
+        .then((partitionTable) => {
+          if (partitionTable !== undefined) {
+            metadata.hasMBR = true
+            metadata.partitions = partitionTable.partitions
+          }
+          $timeout(() => {
+            metadata.path = imagePath
+            metadata.extension = path.extname(imagePath).slice(1)
+            this.selectImage(metadata)
+          })
         })
       })
       .catch((error) => {
@@ -156,10 +180,14 @@ module.exports = function (
           title: 'Error opening image',
           description: messages.error.openImage(path.basename(imagePath), error.message)
         })
-
         osDialog.showError(imageError)
         analytics.logException(error)
       })
+      .then(() => {
+        return innerSource.close()
+        .catch(() => {})
+      })
+    })
   }
 
   /**

lib/gui/modules/child-writer.js

@@ -22,9 +22,6 @@ const ipc = require('node-ipc')
 const sdk = require('etcher-sdk')
 const EXIT_CODES = require('../../shared/exit-codes')
 const errors = require('../../shared/errors')
-const ImageWriter = require('../../sdk/writer')
-const BlockWriteStream = require('../../sdk/writer/block-write-stream')
-const BlockReadStream = require('../../sdk/writer/block-read-stream')
 
 ipc.config.id = process.env.IPC_CLIENT_ID
 ipc.config.socketRoot = process.env.IPC_SOCKET_ROOT
@@ -99,13 +96,14 @@ function runVerifier(verifier, onFail) {
 
 function pipeRegularSourceToDestination(source, destination, verify, onProgress, onFail) {
   let checksum
+  let sparse
   let sourceMetadata
   let step = 'flashing'
   let lastPosition = 0
   const errors = new Map()  // destination -> error map
   const state = {
-    active: destination.destinations.length,
+    active: destination.destinations.size,
-    flashing: destination.destinations.length,
+    flashing: destination.destinations.size,
     verifying: 0,
     failed: 0,
     successful: 0,
@@ -114,7 +112,7 @@ function pipeRegularSourceToDestination(source, destination, verify, onProgress,
   function updateState() {
     state.type = step
     state.failed = errors.size
-    state.active = destination.destinations.length - state.failed
+    state.active = destination.destinations.size - state.failed
     if (step === 'flashing') {
       state.flashing = state.active
       state.verifying = 0
@@ -127,28 +125,48 @@ function pipeRegularSourceToDestination(source, destination, verify, onProgress,
   }
   function onProgress2(progressEvent) {
     lastPosition = progressEvent.position
-    progressEvent.percentage = progressEvent.position / sourceMetadata.size * 100
+    let size
+    if (sparse && (sourceMetadata.blockMap !== undefined)) {
+      size = sourceMetadata.blockMap.mappedBlockCount * sourceMetadata.blockMap.blockSize
+      progressEvent.percentage = progressEvent.bytes / size * 100
+    } else {
+      size = sourceMetadata.size
+      progressEvent.percentage = progressEvent.position / size * 100
+    }
     // NOTE: We need to guard against this becoming Infinity,
     // because that value isn't transmitted properly over IPC and becomes `null`
-    progressEvent.eta = progressEvent.speed ? (sourceMetadata.size - progressEvent.position) / progressEvent.speed : null
+    progressEvent.eta = progressEvent.speed ? (size - progressEvent.bytes) / progressEvent.speed : null
     progressEvent.totalSpeed = progressEvent.speed * state.active
     Object.assign(progressEvent, state)
     onProgress(progressEvent)
   }
-  return Promise.all([ source.createReadStream(), destination.createWriteStream(), source.getMetadata() ])
+  return source.canCreateSparseReadStream()
+  .then((_sparse) => {
+    sparse = _sparse
+    let sourceStream
+    let destinationStream
+    if (sparse) {
+      // TODO: calculate checksums in source if needed
+      sourceStream = source.createSparseReadStream()
+      destinationStream = destination.createSparseWriteStream()
+    } else {
+      sourceStream = source.createReadStream()
+      destinationStream = destination.createWriteStream()
+    }
+    return Promise.all([ sourceStream, destinationStream, source.getMetadata() ])
+  })
   .then(([ sourceStream, destinationStream, metadata ]) => {
     destinationStream.on('fail', (error) => {
       errors.set(error.destination, error.error)
       updateState()
       onFail({ device: error.destination.drive, error: error.error })  // TODO: device should be error.destination
-      onProgress2({ eta: 0, speed: 0, position: lastPosition })  // TODO: this is not needed if a success / error screen is shown
     })
     sourceMetadata = metadata
     return new Promise((resolve, reject) => {
       let done = false
       sourceStream.on('error', reject)
       destinationStream.on('progress', onProgress2)
-      if (verify) {
+      if (verify && !sparse) {
         const hasher = sdk.sourceDestination.createHasher()
         hasher.on('checksum', (cs) => {
           checksum = cs
@@ -160,7 +178,7 @@ function pipeRegularSourceToDestination(source, destination, verify, onProgress,
       }
       destinationStream.on('done', () => {
         done = true;
-        if (!verify || (checksum !== undefined)) {
+        if (sparse || !verify || (checksum !== undefined)) {
           resolve()
         }
       })
@@ -176,7 +194,7 @@ function pipeRegularSourceToDestination(source, destination, verify, onProgress,
     if (verify) {
       step = 'check'
       updateState()
-      const verifier = destination.createVerifier(checksum, sourceMetadata.size)
+      const verifier = destination.createVerifier(sparse ? sourceMetadata.blockMap : checksum, sourceMetadata.size)  // TODO: ensure blockMap exists
       verifier.on('progress', onProgress2)
       return runVerifier(verifier, onFail)
     }
@@ -184,7 +202,7 @@ function pipeRegularSourceToDestination(source, destination, verify, onProgress,
   .then(() => {
     step = 'finished'
     updateState()
-    onProgress2({ speed: 0, position: sourceMetadata.size })
+    //onProgress2({ speed: 0, position: sourceMetadata.size })
   })
   .then(() => {
     const result = {
@@ -281,6 +299,8 @@ ipc.connectTo(IPC_SERVER_ID, () => {
       terminate(exitCode)
     }
 
+    ipc.of[IPC_SERVER_ID].on('cancel', onAbort)
+
     /**
      * @summary Error handler
      * @param {Error} error - error
@@ -306,30 +326,22 @@ ipc.connectTo(IPC_SERVER_ID, () => {
       })
     }
 
-    writer = new ImageWriter({ // TODO: remove
-      verify: options.validateWriteOnSuccess,
-      unmountOnSuccess: options.unmountOnSuccess,
-      checksumAlgorithms: options.checksumAlgorithms || []
-    })
-
-    writer.on('abort', onAbort)
-
     const destinations = _.map(options.destinations, 'drive.device')
     const dests = options.destinations.map((destination) => {
-      return new sdk.sourceDestination.BlockDevice(destination)
+      return new sdk.sourceDestination.BlockDevice(destination, options.unmountOnSuccess)
+    })
+    const source = new sdk.sourceDestination.File(options.imagePath, sdk.sourceDestination.File.OpenFlags.Read)
+    source.getInnerSource()
+    .then((innerSource) => {
+      return Bluebird.using(
+        sourceDestinationDisposer(innerSource),
+        sourceDestinationDisposer(new sdk.sourceDestination.MultiDestination(dests)),
+        (innerSource, destination) => {
+          destination.on('fail', onFail)
+          return pipeRegularSourceToDestination(innerSource, destination, options.validateWriteOnSuccess, onProgress, onFail)
+        }
+      )
     })
-    Bluebird.using(
-      sourceDestinationDisposer(new sdk.sourceDestination.File(options.imagePath, sdk.sourceDestination.File.OpenFlags.Read)),
-      sourceDestinationDisposer(new sdk.sourceDestination.MultiDestination(dests)),
-      (source, destination) => {
-        return source.getInnerSource()
-        .then((innerSource) => {
-          return Bluebird.using(sourceDestinationDisposer(innerSource), (innerSource) => {
-            return pipeRegularSourceToDestination(innerSource, destination, options.validateWriteOnSuccess, onProgress, onFail)
-          })
-        })
-      }
-    )
     .then((results) => {
       onFinish(results)
     })
@@ -343,12 +355,6 @@ ipc.connectTo(IPC_SERVER_ID, () => {
     log(`Validate on success: ${options.validateWriteOnSuccess}`)
   })
 
-  ipc.of[IPC_SERVER_ID].on('cancel', () => {
-    if (writer) {
-      writer.abort()
-    }
-  })
-
   ipc.of[IPC_SERVER_ID].on('connect', () => {
     log(`Successfully connected to IPC server: ${IPC_SERVER_ID}, socket root ${ipc.config.socketRoot}`)
     ipc.of[IPC_SERVER_ID].emit('ready', {})

lib/shared/messages.js

@@ -182,8 +182,8 @@ module.exports = {
       ].join(' ')
     },
 
-    invalidImage: (image) => {
+    invalidImage: (imagePath) => {
-      return `${image.path} is not a supported image type.`
+      return `${imagePath} is not a supported image type.`
     },
 
     openImage: (imageBasename, errorMessage) => {

npm-shrinkwrap.json

@@ -58,6 +58,10 @@
       "version": "0.0.30",
       "resolved": "https://registry.npmjs.org/@types/debug/-/debug-0.0.30.tgz"
     },
+    "@types/events": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@types/events/-/events-1.2.0.tgz"
+    },
     "@types/file-type": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/@types/file-type/-/file-type-5.2.1.tgz"
@@ -126,6 +130,10 @@
       "version": "3.4.4",
       "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.4.tgz"
     },
+    "@types/yauzl": {
+      "version": "2.9.0",
+      "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.0.tgz"
+    },
     "7zip-bin": {
       "version": "2.4.1",
       "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-2.4.1.tgz",
@@ -661,8 +669,8 @@
       "dev": true
     },
     "aws-sdk": {
-      "version": "2.260.1",
+      "version": "2.263.1",
-      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.260.1.tgz",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.263.1.tgz",
       "dependencies": {
         "ieee754": {
           "version": "1.1.8",
@@ -3060,7 +3068,7 @@
     },
     "etcher-sdk": {
       "version": "0.0.1",
-      "resolved": "git://github.com/resin-io-modules/etcher-sdk.git#b12e63b49c4a01305a2809b504859a3940927399",
+      "resolved": "git://github.com/resin-io-modules/etcher-sdk.git#bda51535715edb3691b783973801434bb3d78b30",
       "dependencies": {
         "@types/lodash": {
           "version": "4.14.110",
@@ -3086,6 +3094,10 @@
           "version": "3.5.1",
           "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz"
         },
+        "fd-slicer": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz"
+        },
         "file-type": {
           "version": "8.0.0",
           "resolved": "https://registry.npmjs.org/file-type/-/file-type-8.0.0.tgz"
@@ -3129,6 +3141,10 @@
         "xmlbuilder": {
           "version": "9.0.7",
           "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz"
+        },
+        "yauzl": {
+          "version": "2.9.2",
+          "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.9.2.tgz"
         }
       }
     },

package.json

@@ -63,7 +63,7 @@
     "debug": "3.1.0",
     "drivelist": "6.4.6",
     "electron-is-running-in-asar": "1.0.0",
-    "etcher-sdk": "github:resin-io-modules/etcher-sdk#b12e63b49c4a01305a2809b504859a3940927399",
+    "etcher-sdk": "github:resin-io-modules/etcher-sdk#bda51535715edb3691b783973801434bb3d78b30",
     "file-type": "4.1.0",
     "flexboxgrid": "6.3.0",
     "gpt": "1.0.0",