feat(GUI): collect archive and image extension in analytics (#1343)

Change-Type: patch

lib/gui/models/store.js

@@ -23,6 +23,7 @@ const persistState = require('redux-localstorage');
 const uuidV4 = require('uuid/v4');
 const constraints = require('../../shared/drive-constraints');
 const errors = require('../../shared/errors');
+const fileExtensions = require('../../shared/file-extensions');
 
 /**
  * @summary Application default state
@@ -331,6 +332,32 @@ const storeReducer = (state = DEFAULT_STATE, action) => {
         });
       }
 
+      if (!action.data.extension) {
+        throw errors.createError({
+          title: 'Missing image extension'
+        });
+      }
+
+      if (!_.isString(action.data.extension)) {
+        throw errors.createError({
+          title: `Invalid image extension: ${action.data.extension}`
+        });
+      }
+
+      if (fileExtensions.getLastFileExtension(action.data.path) !== action.data.extension) {
+        if (!action.data.archiveExtension) {
+          throw errors.createError({
+            title: 'Missing image archive extension'
+          });
+        }
+
+        if (!_.isString(action.data.archiveExtension)) {
+          throw errors.createError({
+            title: `Invalid image archive extension: ${action.data.archiveExtension}`
+          });
+        }
+      }
+
       if (!action.data.size) {
         throw errors.createError({
           title: 'Missing image size'

lib/image-stream/archive.js

@@ -16,13 +16,13 @@
 
 'use strict';
 
-const path = require('path');
 const Bluebird = require('bluebird');
 const _ = require('lodash');
 const PassThroughStream = require('stream').PassThrough;
 const supportedFileTypes = require('./supported');
 const utils = require('./utils');
 const errors = require('../shared/errors');
+const fileExtensions = require('../shared/file-extensions');
 
 /**
  * @summary Archive metadata base path
@@ -172,8 +172,7 @@ exports.extractImage = (archive, hooks) => {
   return hooks.getEntries(archive).then((entries) => {
 
     const imageEntries = _.filter(entries, (entry) => {
-      const extension = _.toLower(_.replace(path.extname(entry.name), '.', ''));
+      return _.includes(IMAGE_EXTENSIONS, fileExtensions.getLastFileExtension(entry.name));
-      return _.includes(IMAGE_EXTENSIONS, extension);
     });
 
     const VALID_NUMBER_OF_IMAGE_ENTRIES = 1;
@@ -205,6 +204,9 @@ exports.extractImage = (archive, hooks) => {
         }
       };
 
+      results.metadata.extension = fileExtensions.getLastFileExtension(imageEntry.name);
+      results.metadata.archiveExtension = fileExtensions.getLastFileExtension(archive);
+
       return results.metadata;
     });
   });

lib/image-stream/handlers.js

@@ -26,6 +26,7 @@ const gzip = require('./gzip');
 const udif = Bluebird.promisifyAll(require('udif'));
 const archive = require('./archive');
 const zipArchiveHooks = require('./archive-hooks/zip');
+const fileExtensions = require('../shared/file-extensions');
 
 /**
  * @summary Image handlers
@@ -50,6 +51,8 @@ module.exports = {
   'application/x-bzip2': (file, options) => {
     return Bluebird.props({
       path: file,
+      archiveExtension: fileExtensions.getLastFileExtension(file),
+      extension: fileExtensions.getPenultimateFileExtension(file),
       stream: fs.createReadStream(file),
       size: {
         original: options.size,
@@ -79,6 +82,8 @@ module.exports = {
     return gzip.getUncompressedSize(file).then((uncompressedSize) => {
       return Bluebird.props({
         path: file,
+        archiveExtension: fileExtensions.getLastFileExtension(file),
+        extension: fileExtensions.getPenultimateFileExtension(file),
         stream: fs.createReadStream(file),
         size: {
           original: options.size,
@@ -113,6 +118,8 @@ module.exports = {
     }).then((metadata) => {
       return {
         path: file,
+        archiveExtension: fileExtensions.getLastFileExtension(file),
+        extension: fileExtensions.getPenultimateFileExtension(file),
         stream: fs.createReadStream(file),
         size: {
           original: options.size,
@@ -143,6 +150,7 @@ module.exports = {
     return udif.getUncompressedSizeAsync(file).then((size) => {
       return {
         path: file,
+        extension: fileExtensions.getLastFileExtension(file),
         stream: udif.createReadStream(file),
         size: {
           original: options.size,
@@ -186,6 +194,7 @@ module.exports = {
   'application/octet-stream': (file, options) => {
     return Bluebird.props({
       path: file,
+      extension: fileExtensions.getLastFileExtension(file),
       stream: fs.createReadStream(file),
       size: {
         original: options.size,

lib/shared/file-extensions.js

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2017 resin.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const _ = require('lodash');
+
+/**
+ * @summary Get the extensions of a file
+ * @function
+ * @public
+ *
+ * @param {String} filePath - file path
+ * @returns {String[]} extensions
+ *
+ * @example
+ * const extensions = fileExtensions.getFileExtensions('path/to/foo.img.gz');
+ * console.log(extensions);
+ * > [ 'img', 'gz' ]
+ */
+exports.getFileExtensions = (filePath) => {
+  return _.chain(filePath)
+    .split('.')
+    .tail()
+    .map(_.toLower)
+    .value();
+};
+
+/**
+ * @summary Get the last file extension
+ * @function
+ * @public
+ *
+ * @param {String} filePath - file path
+ * @returns {(String|Undefined)} last extension
+ *
+ * @example
+ * const extension = fileExtensions.getLastFileExtension('path/to/foo.img.gz');
+ * console.log(extension);
+ * > [ 'gz' ]
+ */
+exports.getLastFileExtension = (filePath) => {
+  return _.last(exports.getFileExtensions(filePath));
+};
+
+/**
+ * @summary Get the penultimate file extension
+ * @function
+ * @public
+ *
+ * @param {String} filePath - file path
+ * @returns {(String|Undefined)} penultimate extension
+ *
+ * @example
+ * const extension = fileExtensions.getLastFileExtension('path/to/foo.img.gz');
+ * console.log(extension);
+ * > [ 'img' ]
+ */
+exports.getPenultimateFileExtension = (filePath) => {
+  return _.last(_.initial(exports.getFileExtensions(filePath)));
+};

lib/shared/supported-formats.js

@@ -19,6 +19,7 @@
 const _ = require('lodash');
 const path = require('path');
 const imageStream = require('../image-stream');
+const fileExtensions = require('./file-extensions');
 
 /**
  * @summary Build an extension list getter from a type
@@ -106,20 +107,20 @@ exports.getAllExtensions = () => {
  * }
  */
 exports.isSupportedImage = (imagePath) => {
-  const extension = _.toLower(_.replace(path.extname(imagePath), '.', ''));
+  const lastExtension = fileExtensions.getLastFileExtension(imagePath);
+  const penultimateExtension = fileExtensions.getPenultimateFileExtension(imagePath);
 
   if (_.some([
-    _.includes(exports.getNonCompressedExtensions(), extension),
+    _.includes(exports.getNonCompressedExtensions(), lastExtension),
-    _.includes(exports.getArchiveExtensions(), extension)
+    _.includes(exports.getArchiveExtensions(), lastExtension)
   ])) {
     return true;
   }
 
-  if (!_.includes(exports.getCompressedExtensions(), extension)) {
+  return _.every([
-    return false;
+    _.includes(exports.getCompressedExtensions(), lastExtension),
-  }
+    _.includes(exports.getNonCompressedExtensions(), penultimateExtension)
-
+  ]);
-  return exports.isSupportedImage(path.basename(imagePath, `.${extension}`));
 };
 
 /**

tests/gui/models/drives.spec.js

@@ -118,6 +118,7 @@ describe('Browser: DrivesModel', function() {
             SelectionStateModel.removeDrive();
             SelectionStateModel.setImage({
               path: this.imagePath,
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {

tests/gui/models/selection-state.spec.js

@@ -221,6 +221,7 @@ describe('Browser: SelectionState', function() {
       beforeEach(function() {
         this.image = {
           path: 'foo.img',
+          extension: 'img',
           size: {
             original: 999999999,
             final: {
@@ -342,6 +343,7 @@ describe('Browser: SelectionState', function() {
         it('should override the image', function() {
           SelectionStateModel.setImage({
             path: 'bar.img',
+            extension: 'img',
             size: {
               original: 999999999,
               final: {
@@ -381,6 +383,7 @@ describe('Browser: SelectionState', function() {
         it('should be able to set an image', function() {
           SelectionStateModel.setImage({
             path: 'foo.img',
+            extension: 'img',
             size: {
               original: 999999999,
               final: {
@@ -396,9 +399,28 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(imageSize).to.equal(999999999);
         });
 
+        it('should be able to set an image with an archive extension', function() {
+          SelectionStateModel.setImage({
+            path: 'foo.zip',
+            extension: 'img',
+            archiveExtension: 'zip',
+            size: {
+              original: 999999999,
+              final: {
+                estimation: false,
+                value: 999999999
+              }
+            }
+          });
+
+          const imagePath = SelectionStateModel.getImagePath();
+          m.chai.expect(imagePath).to.equal('foo.zip');
+        });
+
         it('should throw if no path', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -414,6 +436,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 123,
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -425,10 +448,75 @@ describe('Browser: SelectionState', function() {
           }).to.throw('Invalid image path: 123');
         });
 
+        it('should throw if no extension', function() {
+          m.chai.expect(function() {
+            SelectionStateModel.setImage({
+              path: 'foo.img',
+              size: {
+                original: 999999999,
+                final: {
+                  estimation: false,
+                  value: 999999999
+                }
+              }
+            });
+          }).to.throw('Missing image extension');
+        });
+
+        it('should throw if extension is not a string', function() {
+          m.chai.expect(function() {
+            SelectionStateModel.setImage({
+              path: 'foo.img',
+              extension: 1,
+              size: {
+                original: 999999999,
+                final: {
+                  estimation: false,
+                  value: 999999999
+                }
+              }
+            });
+          }).to.throw('Invalid image extension: 1');
+        });
+
+        it('should throw if the extension doesn\'t match the path and there is no archive extension', function() {
+          m.chai.expect(function() {
+            SelectionStateModel.setImage({
+              path: 'foo.img',
+              extension: 'iso',
+              size: {
+                original: 999999999,
+                final: {
+                  estimation: false,
+                  value: 999999999
+                }
+              }
+            });
+          }).to.throw('Missing image archive extension');
+        });
+
+        it('should throw if the extension doesn\'t match the path and the archive extension is not a string', function() {
+          m.chai.expect(function() {
+            SelectionStateModel.setImage({
+              path: 'foo.img',
+              extension: 'iso',
+              archiveExtension: 1,
+              size: {
+                original: 999999999,
+                final: {
+                  estimation: false,
+                  value: 999999999
+                }
+              }
+            });
+          }).to.throw('Invalid image archive extension: 1');
+        });
+
         it('should throw if no size', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
-              path: 'foo.img'
+              path: 'foo.img',
+              extension: 'img'
             });
           }).to.throw('Missing image size');
         });
@@ -437,6 +525,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: 999999999
             });
           }).to.throw('Invalid image size: 999999999');
@@ -446,6 +535,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: '999999999',
                 final: {
@@ -461,6 +551,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: 999999999.999,
                 final: {
@@ -476,6 +567,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: -1,
                 final: {
@@ -491,6 +583,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -506,6 +599,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -521,6 +615,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -536,6 +631,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -551,6 +647,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -567,6 +664,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -583,6 +681,7 @@ describe('Browser: SelectionState', function() {
           m.chai.expect(function() {
             SelectionStateModel.setImage({
               path: 'foo.img',
+              extension: 'img',
               size: {
                 original: 999999999,
                 final: {
@@ -610,6 +709,7 @@ describe('Browser: SelectionState', function() {
 
           SelectionStateModel.setImage({
             path: 'foo.img',
+            extension: 'img',
             size: {
               original: 9999999999,
               final: {
@@ -638,6 +738,7 @@ describe('Browser: SelectionState', function() {
 
           SelectionStateModel.setImage({
             path: 'foo.img',
+            extension: 'img',
             size: {
               original: 999999999,
               final: {
@@ -680,6 +781,7 @@ describe('Browser: SelectionState', function() {
 
           SelectionStateModel.setImage({
             path: imagePath,
+            extension: 'img',
             size: {
               original: 999999999,
               final: {
@@ -713,6 +815,7 @@ describe('Browser: SelectionState', function() {
 
         SelectionStateModel.setImage({
           path: 'foo.img',
+          extension: 'img',
           size: {
             original: 999999999,
             final: {

tests/gui/pages/main.spec.js

@@ -46,6 +46,7 @@ describe('Browser: MainPage', function() {
 
         SelectionStateModel.setImage({
           path: 'rpi.img',
+          extension: 'img',
           size: {
             original: 99999,
             final: {
@@ -80,6 +81,7 @@ describe('Browser: MainPage', function() {
         SelectionStateModel.clear();
         SelectionStateModel.setImage({
           path: 'rpi.img',
+          extension: 'img',
           size: {
             original: 99999,
             final: {
@@ -133,6 +135,7 @@ describe('Browser: MainPage', function() {
 
         SelectionStateModel.setImage({
           path: 'rpi.img',
+          extension: 'img',
           size: {
             original: 99999,
             final: {
@@ -178,6 +181,7 @@ describe('Browser: MainPage', function() {
 
         SelectionStateModel.setImage({
           path: path.join(__dirname, 'foo', 'bar.img'),
+          extension: 'img',
           size: {
             original: 999999999,
             final: {

tests/image-stream/bz2.spec.js

@@ -48,6 +48,8 @@ describe('ImageStream: BZ2', function() {
       return imageStream.getImageMetadata(image).then((metadata) => {
         m.chai.expect(metadata).to.deep.equal({
           path: image,
+          extension: 'img',
+          archiveExtension: 'bz2',
           size: {
             original: expectedSize,
             final: {

tests/image-stream/dmg.spec.js

@@ -51,6 +51,7 @@ describe('ImageStream: DMG', function() {
         return imageStream.getImageMetadata(image).then((metadata) => {
           m.chai.expect(metadata).to.deep.equal({
             path: image,
+            extension: 'dmg',
             size: {
               original: compressedSize,
               final: {
@@ -88,6 +89,7 @@ describe('ImageStream: DMG', function() {
         return imageStream.getImageMetadata(image).then((metadata) => {
           m.chai.expect(metadata).to.deep.equal({
             path: image,
+            extension: 'dmg',
             size: {
               original: compressedSize,
               final: {

tests/image-stream/gz.spec.js

@@ -49,6 +49,8 @@ describe('ImageStream: GZ', function() {
       return imageStream.getImageMetadata(image).then((metadata) => {
         m.chai.expect(metadata).to.deep.equal({
           path: image,
+          extension: 'img',
+          archiveExtension: 'gz',
           size: {
             original: compressedSize,
             final: {

tests/image-stream/img.spec.js

@@ -47,6 +47,7 @@ describe('ImageStream: IMG', function() {
       return imageStream.getImageMetadata(image).then((metadata) => {
         m.chai.expect(metadata).to.deep.equal({
           path: image,
+          extension: 'img',
           size: {
             original: expectedSize,
             final: {

tests/image-stream/iso.spec.js

@@ -47,6 +47,7 @@ describe('ImageStream: ISO', function() {
       return imageStream.getImageMetadata(image).then((metadata) => {
         m.chai.expect(metadata).to.deep.equal({
           path: image,
+          extension: 'iso',
           size: {
             original: expectedSize,
             final: {

tests/image-stream/tester.js

@@ -60,6 +60,8 @@ exports.extractFromFilePath = function(file, image) {
 
     return imageStream.getFromFilePath(file).then(function(results) {
       m.chai.expect(results.path).to.equal(file);
+      m.chai.expect(_.isString(results.extension)).to.be.true;
+      m.chai.expect(_.isEmpty(_.trim(results.extension))).to.be.false;
 
       if (!_.some([
         results.size.original === fs.statSync(file).size,

tests/image-stream/xz.spec.js

@@ -49,6 +49,8 @@ describe('ImageStream: XZ', function() {
       return imageStream.getImageMetadata(image).then((metadata) => {
         m.chai.expect(metadata).to.deep.equal({
           path: image,
+          extension: 'img',
+          archiveExtension: 'xz',
           size: {
             original: compressedSize,
             final: {

tests/image-stream/zip.spec.js

@@ -72,6 +72,8 @@ describe('ImageStream: ZIP', function() {
       return imageStream.getImageMetadata(image).then((metadata) => {
         m.chai.expect(metadata).to.deep.equal({
           path: image,
+          extension: 'img',
+          archiveExtension: 'zip',
           size: {
             original: expectedSize,
             final: {

tests/shared/file-extensions.spec.js

@@ -0,0 +1,141 @@
+/*
+ * Copyright 2016 resin.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const m = require('mochainon');
+const _ = require('lodash');
+const fileExtensions = require('../../lib/shared/file-extensions');
+
+describe('Shared: fileExtensions', function() {
+
+  describe('.getFileExtensions()', function() {
+
+    _.forEach([
+
+      // No extension
+      {
+        file: 'path/to/filename',
+        extension: []
+      },
+
+      // Type: 'archive'
+      {
+        file: 'path/to/filename.zip',
+        extension: [ 'zip' ]
+      },
+      {
+        file: 'path/to/filename.etch',
+        extension: [ 'etch' ]
+      },
+
+      // Type: 'compressed'
+      {
+        file: 'path/to/filename.img.gz',
+        extension: [ 'img', 'gz' ]
+      },
+      {
+        file: 'path/to/filename.img.bz2',
+        extension: [ 'img', 'bz2' ]
+      },
+      {
+        file: 'path/to/filename.img.xz',
+        extension: [ 'img', 'xz' ]
+      },
+
+      // Type: 'image'
+      {
+        file: 'path/to/filename.img',
+        extension: [ 'img' ]
+      },
+      {
+        file: 'path/to/filename.iso',
+        extension: [ 'iso' ]
+      },
+      {
+        file: 'path/to/filename.dsk',
+        extension: [ 'dsk' ]
+      },
+      {
+        file: 'path/to/filename.hddimg',
+        extension: [ 'hddimg' ]
+      },
+      {
+        file: 'path/to/filename.raw',
+        extension: [ 'raw' ]
+      },
+      {
+        file: 'path/to/filename.dmg',
+        extension: [ 'dmg' ]
+      }
+
+    ], (testCase) => {
+      it(`should return ${testCase.extension} for ${testCase.file}`, function() {
+        m.chai.expect(fileExtensions.getFileExtensions(testCase.file)).to.deep.equal(testCase.extension);
+      });
+    });
+
+    it('should always return lowercase extensions', function() {
+      const filePath = 'foo.IMG.gZ';
+      m.chai.expect(fileExtensions.getFileExtensions(filePath)).to.deep.equal([
+        'img',
+        'gz'
+      ]);
+    });
+
+  });
+
+  describe('.getLastFileExtension()', function() {
+
+    it('should return undefined in the file path has no extension', function() {
+      m.chai.expect(fileExtensions.getLastFileExtension('foo')).to.be.undefined;
+    });
+
+    it('should return the extension if there is only one extension', function() {
+      m.chai.expect(fileExtensions.getLastFileExtension('foo.img')).to.equal('img');
+    });
+
+    it('should return the last extension if there two extensions', function() {
+      m.chai.expect(fileExtensions.getLastFileExtension('foo.img.gz')).to.equal('gz');
+    });
+
+    it('should return the last extension if there are three extensions', function() {
+      m.chai.expect(fileExtensions.getLastFileExtension('foo.bar.img.gz')).to.equal('gz');
+    });
+
+  });
+
+  describe('.getPenultimateFileExtension()', function() {
+
+    it('should return undefined in the file path has no extension', function() {
+      m.chai.expect(fileExtensions.getPenultimateFileExtension('foo')).to.be.undefined;
+    });
+
+    it('should return undefined if there is only one extension', function() {
+      m.chai.expect(fileExtensions.getPenultimateFileExtension('foo.img')).to.be.undefined;
+    });
+
+    it('should return the first extension if there are two extensions', function() {
+      m.chai.expect(fileExtensions.getPenultimateFileExtension('foo.img.gz')).to.equal('img');
+    });
+
+    it('should return the penultimate extension if there are three extensions', function() {
+      m.chai.expect(fileExtensions.getPenultimateFileExtension('foo.bar.img.gz')).to.equal('img');
+    });
+
+  });
+
+});