feat(image-stream): Read MBR & GPT in .getImageMetadata() (#1248)

* feat(image-stream): Read MBR & GPT in .getImageMetadata()
* feat(gui): Display warning when image has no MBR
* test(image-stream): Update .isSupportedImage() tests
* feat(image-stream): Normalize MBR & GPT partitions
* test(image-stream): Add partition info
* feat(image-selection): Send missing part table event
* test(image-stream): Add GPT test image

Change-Type: minor

lib/gui/pages/main/controllers/image-selection.js

@@ -81,19 +81,28 @@ module.exports = function(
     }
 
     Bluebird.try(() => {
-      if (!supportedFormats.looksLikeWindowsImage(image.path)) {
+      let message = null;
-        return false;
+
+      if (supportedFormats.looksLikeWindowsImage(image)) {
+        analytics.logEvent('Possibly Windows image', image);
+        message = messages.warning.looksLikeWindowsImage();
+      } else if (supportedFormats.missingPartitionTable(image)) {
+        analytics.logEvent('Missing partition table', image);
+        message = messages.warning.missingPartitionTable();
       }
 
-      analytics.logEvent('Possibly Windows image', image);
+      if (message) {
+        // TODO: `Continue` should be on a red background (dangerous action) instead of `Change`.
+        // We want `X` to act as `Continue`, that's why `Continue` is the `rejectionLabel`
+        return WarningModalService.display({
+          confirmationLabel: 'Change',
+          rejectionLabel: 'Continue',
+          description: message
+        });
+      }
+
+      return false;
 
-      // TODO: `Continue` should be on a red background (dangerous action) instead of `Change`.
-      // We want `X` to act as `Continue`, that's why `Continue` is the `rejectionLabel`
-      return WarningModalService.display({
-        confirmationLabel: 'Change',
-        rejectionLabel: 'Continue',
-        description: messages.warning.looksLikeWindowsImage()
-      });
     }).then((shouldChange) => {
 
       if (shouldChange) {

lib/image-stream/handlers.js

@@ -63,7 +63,7 @@ module.exports = {
           value: options.size
         }
       },
-      transform: Bluebird.resolve(unbzip2Stream())
+      transform: unbzip2Stream()
     });
   },
 
@@ -94,7 +94,7 @@ module.exports = {
             value: uncompressedSize
           }
         },
-        transform: Bluebird.resolve(zlib.createGunzip())
+        transform: zlib.createGunzip()
       });
     });
   },
@@ -215,7 +215,7 @@ module.exports = {
           value: options.size
         }
       },
-      transform: Bluebird.resolve(new PassThroughStream())
+      transform: new PassThroughStream()
     });
   }
 

lib/image-stream/index.js

@@ -24,6 +24,7 @@ const utils = require('./utils');
 const handlers = require('./handlers');
 const supportedFileTypes = require('./supported');
 const errors = require('../shared/errors');
+const parsePartitions = require('./parse-partitions');
 
 /**
  * @summary Get an image stream from a file
@@ -116,16 +117,13 @@ exports.getFromFilePath = (file) => {
  * });
  */
 exports.getImageMetadata = (file) => {
-  return exports.getFromFilePath(file).then((image) => {
+  return exports.getFromFilePath(file)
-
+    .then(parsePartitions)
-    // Since we're not consuming this stream,
+    .then((image) => {
-    // destroy() it, to avoid dangling open file descriptors etc.
+      return _.omitBy(image, (property) => {
-    image.stream.destroy();
+        return property instanceof stream.Stream || _.isNil(property);
-
+      });
-    return _.omitBy(image, (property) => {
-      return property instanceof stream.Stream;
     });
-  });
 };
 
 /**

lib/image-stream/parse-partitions.js

@@ -0,0 +1,212 @@
+/*
+ * Copyright 2017 resin.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const _ = require('lodash');
+const Bluebird = require('bluebird');
+const MBR = require('mbr');
+const GPT = require('gpt');
+
+/**
+ * @summary Maximum number of bytes to read from the stream
+ * @type {Number}
+ * @constant
+ */
+const MAX_STREAM_BYTES = 65536;
+
+/**
+ * @summary Initial number of bytes read
+ * @type {Number}
+ * @constant
+ */
+const INITIAL_LENGTH = 0;
+
+/**
+ * @summary Initial block size
+ * @type {Number}
+ * @constant
+ */
+const INITIAL_BLOCK_SIZE = 512;
+
+/**
+ * @summary Maximum block size to check for
+ * @type {Number}
+ * @constant
+ */
+const MAX_BLOCK_SIZE = 4096;
+
+/**
+ * @summary Attempt to parse the GPT from various block sizes
+ * @function
+ * @private
+ *
+ * @param {Buffer} buffer - Buffer
+ * @returns {GPT|null}
+ *
+ * @example
+ * const gpt = detectGPT(buffer);
+ *
+ * if (gpt != null) {
+ *   // Has a GPT
+ *   console.log('Partitions:', gpt.partitions);
+ * }
+ */
+const detectGPT = (buffer) => {
+
+  let blockSize = INITIAL_BLOCK_SIZE;
+  let gpt = null;
+
+  // Attempt to parse the GPT from several offsets,
+  // as the block size of the image may vary (512,1024,2048,4096);
+  // For example, ISOs will usually have a block size of 4096,
+  // but raw images a block size of 512 bytes
+  while (blockSize <= MAX_BLOCK_SIZE) {
+    gpt = _.attempt(GPT.parse, buffer.slice(blockSize));
+    if (!_.isError(gpt)) {
+      return gpt;
+    }
+    blockSize += blockSize;
+  }
+
+  return null;
+
+};
+
+/**
+ * @summary Attempt to parse the MBR & GPT from a given buffer
+ * @function
+ * @private
+ *
+ * @param {Object} image - Image metadata
+ * @param {Buffer} buffer - Buffer
+ *
+ * @example
+ * parsePartitionTables(image, buffer);
+ *
+ * if (image.hasMBR || image.hasGPT) {
+ *   console.log('Partitions:', image.partitions);
+ * }
+ */
+const parsePartitionTables = (image, buffer) => {
+
+  const mbr = _.attempt(MBR.parse, buffer);
+  let gpt = null;
+
+  if (!_.isError(mbr)) {
+    image.hasMBR = true;
+    gpt = detectGPT(buffer);
+    image.hasGPT = !_.isNil(gpt);
+  }
+
+  // As MBR and GPT partition entries have a different structure,
+  // we normalize them here to make them easier to deal with and
+  // avoid clutter in what's sent to analytics
+  if (image.hasGPT) {
+    image.partitions = _.map(gpt.partitions, (partition) => {
+      return {
+        type: partition.type.toString(),
+        id: partition.guid.toString(),
+        name: partition.name,
+        firstLBA: partition.firstLBA,
+        lastLBA: partition.lastLBA,
+        extended: false
+      };
+    });
+  } else if (image.hasMBR) {
+    image.partitions = _.map(mbr.partitions, (partition) => {
+      return {
+        type: partition.type,
+        id: null,
+        name: null,
+        firstLBA: partition.firstLBA,
+        lastLBA: partition.lastLBA,
+        extended: partition.extended
+      };
+    });
+  }
+
+};
+
+/**
+ * @summary Attempt to read the MBR and GPT from an imagestream
+ * @function
+ * @public
+ * @description
+ * This operation will consume the first `MAX_STREAM_BYTES`
+ * of the stream and then destroy the stream.
+ *
+ * @param {Object} image - image metadata
+ * @returns {Promise}
+ * @fulfil {Object} image
+ * @reject {Error}
+ *
+ * @example
+ * parsePartitions(image)
+ *   .then((image) => {
+ *     console.log('MBR:', image.hasMBR);
+ *     console.log('GPT:', image.hasGPT);
+ *     console.log('Partitions:', image.partitions);
+ *   });
+ */
+module.exports = (image) => {
+  return new Bluebird((resolve, reject) => {
+
+    const chunks = [];
+    let length = INITIAL_LENGTH;
+    let destroyed = false;
+
+    image.hasMBR = false;
+    image.hasGPT = false;
+
+    let stream = image.stream.pipe(image.transform);
+
+    stream.on('error', reject);
+
+    // We need to use the "old" flowing mode here,
+    // as some dependencies don't implement the "readable"
+    // mode properly (i.e. bzip2)
+    stream.on('data', (chunk) => {
+      chunks.push(chunk);
+      length += chunk.length;
+
+      // Once we've read enough bytes, terminate the stream
+      if (length >= MAX_STREAM_BYTES && !destroyed) {
+
+        // Prefer close() over destroy(), as some streams
+        // from dependencies exhibit quirky behavior when destroyed
+        if (image.stream.close) {
+          image.stream.close();
+        } else {
+          image.stream.destroy();
+        }
+
+        // Remove references to stream to allow them being GCed
+        image.stream = null;
+        image.transform = null;
+        stream = null;
+        destroyed = true;
+
+        // Parse the MBR, GPT and partitions from the obtained buffer
+        parsePartitionTables(image, Buffer.concat(chunks));
+        resolve(image);
+
+      }
+    });
+
+  });
+
+};

lib/shared/messages.js

@@ -61,6 +61,12 @@ module.exports = {
       'Unlike other images, Windows images require special processing to be made bootable.',
       'We suggest you use a tool specially designed for this purpose, such as',
       '<a href="https://rufus.akeo.ie">Rufus</a> (Windows) or Boot Camp Assistant (macOS).'
+    ].join(' ')),
+
+    missingPartitionTable: _.template([
+      'It looks like this is not a bootable image.\n\n',
+      'The image does not appear to contain a partition table,',
+      'and might not be recognized or bootable by your device.'
     ].join(' '))
 
   },

npm-shrinkwrap.json

@@ -885,6 +885,11 @@
       "resolved": "https://registry.npmjs.org/chromium-pickle-js/-/chromium-pickle-js-0.1.0.tgz",
       "dev": true
     },
+    "chs": {
+      "version": "1.1.0",
+      "from": "chs@>=1.1.0 <1.2.0",
+      "resolved": "https://registry.npmjs.org/chs/-/chs-1.1.0.tgz"
+    },
     "ci-info": {
       "version": "1.0.0",
       "from": "ci-info@>=1.0.0 <2.0.0",
@@ -2869,6 +2874,11 @@
       "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
       "dev": true
     },
+    "gpt": {
+      "version": "1.0.0",
+      "from": "gpt@latest",
+      "resolved": "https://registry.npmjs.org/gpt/-/gpt-1.0.0.tgz"
+    },
     "graceful-fs": {
       "version": "4.1.11",
       "from": "graceful-fs@>=4.1.2 <5.0.0",
@@ -4619,6 +4629,11 @@
       "resolved": "https://registry.npmjs.org/markdown-utils/-/markdown-utils-0.7.3.tgz",
       "dev": true
     },
+    "mbr": {
+      "version": "1.1.2",
+      "from": "mbr@latest",
+      "resolved": "https://registry.npmjs.org/mbr/-/mbr-1.1.2.tgz"
+    },
     "mem": {
       "version": "1.1.0",
       "from": "mem@>=1.1.0 <2.0.0",

package.json

@@ -49,9 +49,11 @@
     "etcher-image-write": "9.1.3",
     "file-type": "4.1.0",
     "flexboxgrid": "6.3.0",
+    "gpt": "1.0.0",
     "immutable": "3.8.1",
     "lodash": "4.13.1",
     "lzma-native": "1.5.2",
+    "mbr": "1.1.2",
     "mime-types": "2.1.15",
     "mountutils": "1.2.0",
     "nan": "2.3.5",

tests/image-stream/bz2.spec.js

@@ -62,7 +62,10 @@ describe('ImageStream: BZ2', function() {
               estimation: true,
               value: expectedSize
             }
-          }
+          },
+          hasMBR: true,
+          hasGPT: false,
+          partitions: require('./data/images/etcher-test-partitions.json')
         });
       });
     });

tests/image-stream/data/images/etcher-gpt-test-partitions.json

@@ -0,0 +1,26 @@
+[
+  {
+    "type": "E3C9E316-0B5C-4DB8-817D-F92DF00215AE",
+    "id": "F2020024-6D12-43A7-B0AA-0E243771ED00",
+    "name": "Microsoft reserved partition",
+    "firstLBA": 34,
+    "lastLBA": 65569,
+    "extended": false
+  },
+  {
+    "type": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7",
+    "id": "3B781D99-BEFA-41F7-85C7-01346507805C",
+    "name": "Basic data partition",
+    "firstLBA": 65664,
+    "lastLBA": 163967,
+    "extended": false
+  },
+  {
+    "type": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7",
+    "id": "EE0EAF80-24C1-4A41-949E-419676E89AD6",
+    "name": "Basic data partition",
+    "firstLBA": 163968,
+    "lastLBA": 258175,
+    "extended": false
+  }
+]

tests/image-stream/data/images/etcher-test-partitions.json

@@ -0,0 +1,34 @@
+[
+  {
+    "type": 14,
+    "id": null,
+    "name": null,
+    "firstLBA": 128,
+    "lastLBA": 2176,
+    "extended": false
+  },
+  {
+    "type": 14,
+    "id": null,
+    "name": null,
+    "firstLBA": 2176,
+    "lastLBA": 4224,
+    "extended": false
+  },
+  {
+    "type": 0,
+    "id": null,
+    "name": null,
+    "firstLBA": 0,
+    "lastLBA": 0,
+    "extended": false
+  },
+  {
+    "type": 0,
+    "id": null,
+    "name": null,
+    "firstLBA": 0,
+    "lastLBA": 0,
+    "extended": false
+  }
+]

tests/image-stream/dmg.spec.js

@@ -91,7 +91,10 @@ describe('ImageStream: DMG', function() {
                 estimation: false,
                 value: uncompressedSize
               }
-            }
+            },
+            hasMBR: true,
+            hasGPT: false,
+            partitions: require('./data/images/etcher-test-partitions.json')
           });
         });
       });
@@ -129,7 +132,10 @@ describe('ImageStream: DMG', function() {
                 estimation: false,
                 value: uncompressedSize
               }
-            }
+            },
+            hasMBR: true,
+            hasGPT: false,
+            partitions: require('./data/images/etcher-test-partitions.json')
           });
         });
       });

tests/image-stream/gz.spec.js

@@ -57,7 +57,10 @@ describe('ImageStream: GZ', function() {
               estimation: true,
               value: uncompressedSize
             }
-          }
+          },
+          hasMBR: true,
+          hasGPT: false,
+          partitions: require('./data/images/etcher-test-partitions.json')
         });
       });
     });

tests/image-stream/img.spec.js

@@ -40,23 +40,58 @@ describe('ImageStream: IMG', function() {
 
   describe('.getImageMetadata()', function() {
 
-    it('should return the correct metadata', function() {
+    context('Master Boot Record', function() {
-      const image = path.join(IMAGES_PATH, 'etcher-test.img');
-      const expectedSize = fs.statSync(image).size;
 
-      return imageStream.getImageMetadata(image).then((metadata) => {
+      it('should return the correct metadata', function() {
-        m.chai.expect(metadata).to.deep.equal({
+        const image = path.join(IMAGES_PATH, 'etcher-test.img');
-          path: image,
+        const expectedSize = fs.statSync(image).size;
-          extension: 'img',
+
-          size: {
+        return imageStream.getImageMetadata(image).then((metadata) => {
-            original: expectedSize,
+          m.chai.expect(metadata).to.deep.equal({
-            final: {
+            path: image,
-              estimation: false,
+            extension: 'img',
-              value: expectedSize
+            size: {
-            }
+              original: expectedSize,
-          }
+              final: {
+                estimation: false,
+                value: expectedSize
+              }
+            },
+            hasMBR: true,
+            hasGPT: false,
+            partitions: require('./data/images/etcher-test-partitions.json')
+          });
         });
       });
+
+    });
+
+    context('GUID Partition Table', function() {
+
+      it('should return the correct metadata', function() {
+        const image = path.join(IMAGES_PATH, 'etcher-gpt-test.img.gz');
+        const uncompressedSize = 134217728;
+        const expectedSize = fs.statSync(image).size;
+
+        return imageStream.getImageMetadata(image).then((metadata) => {
+          m.chai.expect(metadata).to.deep.equal({
+            path: image,
+            extension: 'img',
+            archiveExtension: 'gz',
+            size: {
+              original: expectedSize,
+              final: {
+                estimation: true,
+                value: uncompressedSize
+              }
+            },
+            hasMBR: true,
+            hasGPT: true,
+            partitions: require('./data/images/etcher-gpt-test-partitions.json')
+          });
+        });
+      });
+
     });
 
   });

tests/image-stream/iso.spec.js

@@ -54,7 +54,10 @@ describe('ImageStream: ISO', function() {
               estimation: false,
               value: expectedSize
             }
-          }
+          },
+          hasMBR: true,
+          hasGPT: false,
+          partitions: require('./data/images/etcher-test-partitions.json')
         });
       });
     });

tests/image-stream/xz.spec.js

@@ -57,7 +57,10 @@ describe('ImageStream: XZ', function() {
               estimation: false,
               value: uncompressedSize
             }
-          }
+          },
+          hasMBR: true,
+          hasGPT: false,
+          partitions: require('./data/images/etcher-test-partitions.json')
         });
       });
     });

tests/image-stream/zip.spec.js

@@ -122,7 +122,10 @@ describe('ImageStream: ZIP', function() {
               estimation: false,
               value: expectedSize
             }
-          }
+          },
+          hasMBR: true,
+          hasGPT: false,
+          partitions: require('./data/images/etcher-test-partitions.json')
         });
       });
     });