refactor: remove extended archives extra functionality (#1055)

We currently attempt to read extra metadata from ZIP files, such as a
logo, image name, image url, etc. In order to simplify the metadata
story, we decided that this metadata will not live on the image itself,
but rather on a centralised repo, which greatly simplified our custom
archive extraction logic and allows us to make use of these nice
features even when streaming the image directly from the internet.

We'll be working on bringing back this functionality from a centralised
repo in subsequent commits.

Change-Type: major
Changelog-Entry: Remove extended archives metadata extraction logic.
Signed-off-by: Juan Cruz Viotti <jviotti@openmailbox.org>
This commit is contained in:
Juan Cruz Viotti 2017-01-26 15:21:01 -04:00 committed by GitHub
parent 1bfcee06e2
commit b78473ea0e
17 changed files with 23 additions and 551 deletions

View File

@ -77,7 +77,6 @@ exports.writeImage = (imagePath, drive, options, onProgress) => {
}, {
check: options.validateWriteOnSuccess,
transform: image.transform,
bmap: image.bmap,
bytesToZeroOutFromTheBeginning: image.bytesToZeroOutFromTheBeginning
});
}).then((writer) => {

View File

@ -213,9 +213,7 @@ app.controller('HeaderController', function(SelectionStateModel, OSOpenExternalS
* HeaderController.openHelpPage();
*/
this.openHelpPage = () => {
const DEFAULT_SUPPORT_URL = 'https://github.com/resin-io/etcher/blob/master/SUPPORT.md';
const supportUrl = SelectionStateModel.getImageSupportUrl() || DEFAULT_SUPPORT_URL;
OSOpenExternalService.open(supportUrl);
OSOpenExternalService.open('https://github.com/resin-io/etcher/blob/master/SUPPORT.md');
};
});

View File

@ -142,76 +142,6 @@ SelectionStateModel.service('SelectionStateModel', function(DrivesModel) {
return _.get(Store.getState().toJS(), 'selection.image.size');
};
/**
* @summary Get image url
* @function
* @public
*
* @returns {String} image url
*
* @example
* const imageUrl = SelectionStateModel.getImageUrl();
*/
this.getImageUrl = () => {
return _.get(Store.getState().toJS(), 'selection.image.url');
};
/**
* @summary Get image name
* @function
* @public
*
* @returns {String} image name
*
* @example
* const imageName = SelectionStateModel.getImageName();
*/
this.getImageName = () => {
return _.get(Store.getState().toJS(), 'selection.image.name');
};
/**
* @summary Get image logo
* @function
* @public
*
* @returns {String} image logo
*
* @example
* const imageLogo = SelectionStateModel.getImageLogo();
*/
this.getImageLogo = () => {
return _.get(Store.getState().toJS(), 'selection.image.logo');
};
/**
* @summary Get image support url
* @function
* @public
*
* @returns {String} image support url
*
* @example
* const imageSupportUrl = SelectionStateModel.getImageSupportUrl();
*/
this.getImageSupportUrl = () => {
return _.get(Store.getState().toJS(), 'selection.image.supportUrl');
};
/**
* @summary Get image recommended drive size
* @function
* @public
*
* @returns {String} image recommended drive size
*
* @example
* const imageRecommendedDriveSize = SelectionStateModel.getImageRecommendedDriveSize();
*/
this.getImageRecommendedDriveSize = () => {
return _.get(Store.getState().toJS(), 'selection.image.recommendedDriveSize');
};
/**
* @summary Check if there is a selected drive
* @function

View File

@ -249,28 +249,13 @@ const storeReducer = (state, action) => {
throw new Error(`Invalid image size: ${action.data.size}`);
}
if (action.data.url && !_.isString(action.data.url)) {
throw new Error(`Invalid image url: ${action.data.url}`);
}
if (action.data.name && !_.isString(action.data.name)) {
throw new Error(`Invalid image name: ${action.data.name}`);
}
if (action.data.logo && !_.isString(action.data.logo)) {
throw new Error(`Invalid image logo: ${action.data.logo}`);
}
const selectedDevice = state.getIn([ 'selection', 'drive' ]);
const selectedDrive = state.get('availableDrives').find((drive) => {
return drive.get('device') === selectedDevice;
});
return _.attempt(() => {
if (selectedDrive && !_.every([
constraints.isDriveLargeEnough(selectedDrive.toJS(), action.data),
constraints.isDriveSizeRecommended(selectedDrive.toJS(), action.data)
])) {
if (selectedDrive && !constraints.isDriveLargeEnough(selectedDrive.toJS(), action.data)) {
return storeReducer(state, {
type: ACTIONS.REMOVE_DRIVE
});

View File

@ -66,12 +66,6 @@ module.exports = function(SupportedFormatsModel, SelectionStateModel, AnalyticsS
}
SelectionStateModel.setImage(image);
// An easy way so we can quickly identify if we're making use of
// certain features without printing pages of text to DevTools.
image.logo = Boolean(image.logo);
image.bmap = Boolean(image.bmap);
AnalyticsService.logEvent('Select image', image);
};

View File

@ -43,7 +43,7 @@
<svg-icon
class="center-block"
path="{{ main.selection.getImageLogo() || '../../../assets/image.svg' }}"
path="../../../assets/image.svg"
ng-disabled="main.shouldImageStepBeDisabled()"></svg-icon>
<span
class="icon-caption"
@ -67,9 +67,8 @@
</div>
<div ng-if="main.selection.hasImage()">
<div class="step-selection-text">
<span ng-click="main.external.open(main.selection.getImageUrl())"
ng-class="{ 'text-disabled': main.shouldImageStepBeDisabled() }"
ng-bind="main.selection.getImageName() || main.selection.getImagePath() | basename | middleEllipses:20"
<span ng-class="{ 'text-disabled': main.shouldImageStepBeDisabled() }"
ng-bind="main.selection.getImagePath() | basename | middleEllipses:20"
uib-tooltip="{{ main.selection.getImagePath() | basename }}"></span>
<span class="glyphicon glyphicon-info-sign"

View File

@ -17,20 +17,10 @@
'use strict';
const path = require('path');
const Bluebird = require('bluebird');
const rindle = require('rindle');
const _ = require('lodash');
const PassThroughStream = require('stream').PassThrough;
const supportedFileTypes = require('./supported');
/**
* @summary Archive metadata base path
* @constant
* @private
* @type {String}
*/
const ARCHIVE_METADATA_BASE_PATH = '.meta';
/**
* @summary Image extensions
* @constant
@ -45,103 +35,6 @@ const IMAGE_EXTENSIONS = _.reduce(supportedFileTypes, (accumulator, file) => {
return accumulator;
}, []);
/**
* @summary Extract entry by path
* @function
* @private
*
* @param {String} archive - archive
* @param {String} filePath - entry file path
* @param {Object} options - options
* @param {Object} options.hooks - archive hooks
* @param {Object[]} options.entries - archive entries
* @param {*} [options.default] - entry default value
* @fulfil {*} contents
* @returns {Promise}
*
* extractEntryByPath('my/archive.zip', '_info/logo.svg', {
* hooks: { ... },
* entries: [ ... ],
* default: ''
* }).then((contents) => {
* console.log(contents);
* });
*/
const extractEntryByPath = (archive, filePath, options) => {
const fileEntry = _.find(options.entries, (entry) => {
return _.chain(entry.name)
.split('/')
.tail()
.join('/')
.value() === filePath;
});
if (!fileEntry) {
return Bluebird.resolve(options.default);
}
return options.hooks.extractFile(archive, options.entries, fileEntry.name)
.then(rindle.extract);
};
/**
* @summary Extract archive metadata
* @function
* @private
*
* @param {String} archive - archive
* @param {String} basePath - metadata base path
* @param {Object} options - options
* @param {Object[]} options.entries - archive entries
* @param {Object} options.hooks - archive hooks
* @fulfil {Object} - metadata
* @returns {Promise}
*
* extractArchiveMetadata('my/archive.zip', '.meta', {
* hooks: { ... },
* entries: [ ... ]
* }).then((metadata) => {
* console.log(metadata);
* });
*/
const extractArchiveMetadata = (archive, basePath, options) => {
return Bluebird.props({
logo: extractEntryByPath(archive, `${basePath}/logo.svg`, options),
instructions: extractEntryByPath(archive, `${basePath}/instructions.markdown`, options),
bmap: extractEntryByPath(archive, `${basePath}/image.bmap`, options),
manifest: _.attempt(() => {
return extractEntryByPath(archive, `${basePath}/manifest.json`, {
hooks: options.hooks,
entries: options.entries,
default: '{}'
}).then((manifest) => {
try {
return JSON.parse(manifest);
} catch (parseError) {
const error = new Error('Invalid archive manifest.json');
error.description = 'The archive manifest.json file is not valid JSON.';
throw error;
}
});
})
}).then((results) => {
return {
name: results.manifest.name,
version: results.manifest.version,
url: results.manifest.url,
supportUrl: results.manifest.supportUrl,
releaseNotesUrl: results.manifest.releaseNotesUrl,
checksumType: results.manifest.checksumType,
checksum: results.manifest.checksum,
bytesToZeroOutFromTheBeginning: results.manifest.bytesToZeroOutFromTheBeginning,
recommendedDriveSize: results.manifest.recommendedDriveSize,
logo: results.logo,
bmap: results.bmap,
instructions: results.instructions
};
});
};
/**
* @summary Extract image from archive
* @function
@ -182,25 +75,18 @@ exports.extractImage = (archive, hooks) => {
const imageEntry = _.first(imageEntries);
return Bluebird.props({
imageStream: hooks.extractFile(archive, entries, imageEntry.name),
metadata: extractArchiveMetadata(archive, ARCHIVE_METADATA_BASE_PATH, {
entries,
hooks
})
}).then((results) => {
results.metadata.stream = results.imageStream;
results.metadata.transform = new PassThroughStream();
results.metadata.size = {
original: imageEntry.size,
final: {
estimation: false,
value: imageEntry.size
return hooks.extractFile(archive, entries, imageEntry.name).then((imageStream) => {
return {
stream: imageStream,
transform: new PassThroughStream(),
size: {
original: imageEntry.size,
final: {
estimation: false,
value: imageEntry.size
}
}
};
return results.metadata;
});
});
};

View File

@ -99,10 +99,7 @@ exports.getFromFilePath = (file) => {
* const imageStream = require('./lib/image-stream');
*
* imageStream.getImageMetadata('path/to/rpi.img.xz').then((metadata) => {
* console.log(`The image display name is: ${metada.name}`);
* console.log(`The image url is: ${metada.url}`);
* console.log(`The image support url is: ${metada.supportUrl}`);
* console.log(`The image logo is: ${metada.logo}`);
* console.log(`The image original size is: ${metada.size.original}`);
* });
*/
exports.getImageMetadata = (file) => {

32
npm-shrinkwrap.json generated
View File

@ -512,16 +512,16 @@
"from": "isarray@0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
},
"isstream": {
"version": "0.1.2",
"from": "isstream@>=0.1.1 <0.2.0",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
},
"isexe": {
"version": "1.1.2",
"from": "isexe@>=1.1.1 <2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-1.1.2.tgz"
},
"isstream": {
"version": "0.1.2",
"from": "isstream@>=0.1.1 <0.2.0",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
},
"js-message": {
"version": "1.0.5",
"from": "js-message@>=1.0.5",
@ -1451,23 +1451,6 @@
"from": "restore-cursor@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz"
},
"rindle": {
"version": "1.3.0",
"from": "rindle@>=1.3.0 <2.0.0",
"resolved": "https://registry.npmjs.org/rindle/-/rindle-1.3.0.tgz",
"dependencies": {
"bluebird": {
"version": "2.11.0",
"from": "bluebird@>=2.10.2 <3.0.0",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz"
},
"lodash": {
"version": "3.10.1",
"from": "lodash@>=3.10.1 <4.0.0",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz"
}
}
},
"run-async": {
"version": "0.1.0",
"from": "run-async@>=0.1.0 <0.2.0",
@ -1592,11 +1575,6 @@
"from": "string-template@>=0.2.1 <0.3.0",
"resolved": "https://registry.npmjs.org/string-template/-/string-template-0.2.1.tgz"
},
"string-to-stream": {
"version": "1.1.0",
"from": "string-to-stream@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.1.0.tgz"
},
"string-width": {
"version": "1.0.1",
"from": "string-width@>=1.0.1 <2.0.0",

View File

@ -90,7 +90,6 @@
"redux-localstorage": "^0.4.1",
"resin-cli-form": "^1.4.1",
"resin-cli-visuals": "^1.2.8",
"rindle": "^1.3.0",
"rx": "^4.1.0",
"semver": "^5.1.0",
"sudo-prompt": "^6.1.0",
@ -114,6 +113,7 @@
"jsonfile": "^2.3.1",
"mochainon": "^1.0.0",
"node-sass": "^3.8.0",
"rindle": "^1.3.0",
"tmp": "0.0.31",
"versionist": "^2.1.0"
},

View File

@ -47,26 +47,6 @@ describe('Browser: SelectionState', function() {
m.chai.expect(SelectionStateModel.getImageSize()).to.be.undefined;
});
it('getImageUrl() should return undefined', function() {
m.chai.expect(SelectionStateModel.getImageUrl()).to.be.undefined;
});
it('getImageName() should return undefined', function() {
m.chai.expect(SelectionStateModel.getImageName()).to.be.undefined;
});
it('getImageLogo() should return undefined', function() {
m.chai.expect(SelectionStateModel.getImageLogo()).to.be.undefined;
});
it('getImageSupportUrl() should return undefined', function() {
m.chai.expect(SelectionStateModel.getImageSupportUrl()).to.be.undefined;
});
it('getImageRecommendedDriveSize() should return undefined', function() {
m.chai.expect(SelectionStateModel.getImageRecommendedDriveSize()).to.be.undefined;
});
it('hasDrive() should return false', function() {
const hasDrive = SelectionStateModel.hasDrive();
m.chai.expect(hasDrive).to.be.false;
@ -219,12 +199,7 @@ describe('Browser: SelectionState', function() {
beforeEach(function() {
this.image = {
path: 'foo.img',
size: 999999999,
recommendedDriveSize: 1000000000,
url: 'https://www.raspbian.org',
supportUrl: 'https://www.raspbian.org/forums/',
name: 'Raspbian',
logo: '<svg><text fill="red">Raspbian</text></svg>'
size: 999999999
};
SelectionStateModel.setImage(this.image);
@ -275,51 +250,6 @@ describe('Browser: SelectionState', function() {
});
describe('.getImageUrl()', function() {
it('should return the image url', function() {
const imageUrl = SelectionStateModel.getImageUrl();
m.chai.expect(imageUrl).to.equal('https://www.raspbian.org');
});
});
describe('.getImageName()', function() {
it('should return the image name', function() {
const imageName = SelectionStateModel.getImageName();
m.chai.expect(imageName).to.equal('Raspbian');
});
});
describe('.getImageLogo()', function() {
it('should return the image logo', function() {
const imageLogo = SelectionStateModel.getImageLogo();
m.chai.expect(imageLogo).to.equal('<svg><text fill="red">Raspbian</text></svg>');
});
});
describe('.getImageSupportUrl()', function() {
it('should return the image support url', function() {
const imageSupportUrl = SelectionStateModel.getImageSupportUrl();
m.chai.expect(imageSupportUrl).to.equal('https://www.raspbian.org/forums/');
});
});
describe('.getImageRecommendedDriveSize()', function() {
it('should return the image recommended drive size', function() {
const imageRecommendedDriveSize = SelectionStateModel.getImageRecommendedDriveSize();
m.chai.expect(imageRecommendedDriveSize).to.equal(1000000000);
});
});
describe('.hasImage()', function() {
it('should return true', function() {
@ -410,36 +340,6 @@ describe('Browser: SelectionState', function() {
}).to.throw('Invalid image size: 999999999');
});
it('should throw if url is defined but its not a string', function() {
m.chai.expect(function() {
SelectionStateModel.setImage({
path: 'foo.img',
size: 999999999,
url: 1234
});
}).to.throw('Invalid image url: 1234');
});
it('should throw if name is defined but its not a string', function() {
m.chai.expect(function() {
SelectionStateModel.setImage({
path: 'foo.img',
size: 999999999,
name: 1234
});
}).to.throw('Invalid image name: 1234');
});
it('should throw if logo is defined but its not a string', function() {
m.chai.expect(function() {
SelectionStateModel.setImage({
path: 'foo.img',
size: 999999999,
logo: 1234
});
}).to.throw('Invalid image logo: 1234');
});
it('should de-select a previously selected not-large-enough drive', function() {
DrivesModel.setDrives([
{
@ -462,29 +362,6 @@ describe('Browser: SelectionState', function() {
SelectionStateModel.removeImage();
});
it('should de-select a previously selected not-recommended drive', function() {
DrivesModel.setDrives([
{
device: '/dev/disk1',
name: 'USB Drive',
size: 1200000000,
protected: false
}
]);
SelectionStateModel.setDrive('/dev/disk1');
m.chai.expect(SelectionStateModel.hasDrive()).to.be.true;
SelectionStateModel.setImage({
path: 'foo.img',
size: 999999999,
recommendedDriveSize: 1500000000
});
m.chai.expect(SelectionStateModel.hasDrive()).to.be.false;
SelectionStateModel.removeImage();
});
});
});

View File

@ -1,171 +0,0 @@
/*
* Copyright 2016 resin.io
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const m = require('mochainon');
const path = require('path');
const DATA_PATH = path.join(__dirname, '..', 'data');
const IMAGES_PATH = path.join(DATA_PATH, 'images');
const ZIP_PATH = path.join(DATA_PATH, 'metadata', 'zip');
const tester = require('../tester');
const imageStream = require('../../../lib/image-stream/index');
const testMetadataProperty = (archivePath, propertyName, expectedValue) => {
return imageStream.getFromFilePath(archivePath).then((image) => {
m.chai.expect(image[propertyName]).to.deep.equal(expectedValue);
return imageStream.getImageMetadata(archivePath).then((metadata) => {
m.chai.expect(metadata[propertyName]).to.deep.equal(expectedValue);
});
});
};
describe('ImageStream: Metadata ZIP', function() {
this.timeout(10000);
describe('given an archive with an invalid `manifest.json`', function() {
tester.expectError(
path.join(ZIP_PATH, 'rpi-invalid-manifest.zip'),
'Invalid archive manifest.json');
describe('.getImageMetadata()', function() {
it('should be rejected with an error', function(done) {
const image = path.join(ZIP_PATH, 'rpi-invalid-manifest.zip');
imageStream.getImageMetadata(image).catch((error) => {
m.chai.expect(error).to.be.an.instanceof(Error);
m.chai.expect(error.message).to.equal('Invalid archive manifest.json');
done();
});
});
});
});
describe('given an archive with a `manifest.json`', function() {
const archive = path.join(ZIP_PATH, 'rpi-with-manifest.zip');
tester.extractFromFilePath(
archive,
path.join(IMAGES_PATH, 'raspberrypi.img'));
it('should read the manifest name property', function(done) {
testMetadataProperty(archive, 'name', 'Raspberry Pi').asCallback(done);
});
it('should read the manifest version property', function(done) {
testMetadataProperty(archive, 'version', '1.0.0').asCallback(done);
});
it('should read the manifest url property', function(done) {
testMetadataProperty(archive, 'url', 'https://www.raspberrypi.org').asCallback(done);
});
it('should read the manifest supportUrl property', function(done) {
const expectedValue = 'https://www.raspberrypi.org/forums/';
testMetadataProperty(archive, 'supportUrl', expectedValue).asCallback(done);
});
it('should read the manifest releaseNotesUrl property', function(done) {
const expectedValue = 'http://downloads.raspberrypi.org/raspbian/release_notes.txt';
testMetadataProperty(archive, 'releaseNotesUrl', expectedValue).asCallback(done);
});
it('should read the manifest checksumType property', function(done) {
testMetadataProperty(archive, 'checksumType', 'md5').asCallback(done);
});
it('should read the manifest checksum property', function(done) {
testMetadataProperty(archive, 'checksum', 'add060b285d512f56c175b76b7ef1bee').asCallback(done);
});
it('should read the manifest bytesToZeroOutFromTheBeginning property', function(done) {
testMetadataProperty(archive, 'bytesToZeroOutFromTheBeginning', 512).asCallback(done);
});
it('should read the manifest recommendedDriveSize property', function(done) {
testMetadataProperty(archive, 'recommendedDriveSize', 4294967296).asCallback(done);
});
});
describe('given an archive with a `logo.svg`', function() {
const archive = path.join(ZIP_PATH, 'rpi-with-logo.zip');
const logo = [
'<svg xmlns="http://www.w3.org/2000/svg">',
' <text>Hello World</text>',
'</svg>',
''
].join('\n');
it('should read the logo contents', function(done) {
testMetadataProperty(archive, 'logo', logo).asCallback(done);
});
});
describe('given an archive with a bmap file', function() {
const archive = path.join(ZIP_PATH, 'rpi-with-bmap.zip');
const bmap = [
'<?xml version="1.0" ?>',
'<bmap version="1.3">',
' <ImageSize> 36864 </ImageSize>',
' <BlockSize> 4096 </BlockSize>',
' <BlocksCount> 9 </BlocksCount>',
' <MappedBlocksCount> 4 </MappedBlocksCount>',
' <BmapFileSHA1> d90f372215cbbef8801caca7b1dd7e587b2142cc </BmapFileSHA1>',
' <BlockMap>',
' <Range sha1="193edb53bde599f58369f4e83a6c5d54b96819ce"> 0-1 </Range>',
' <Range sha1="193edb53bde599f58369f4e83a6c5d54b96819ce"> 7-8 </Range>',
' </BlockMap>',
'</bmap>',
''
].join('\n');
it('should read the bmap contents', function(done) {
testMetadataProperty(archive, 'bmap', bmap).asCallback(done);
});
});
describe('given an archive with instructions', function() {
const archive = path.join(ZIP_PATH, 'rpi-with-instructions.zip');
const instructions = [
'# Raspberry Pi Next Steps',
'',
'Lorem ipsum dolor sit amet.',
''
].join('\n');
it('should read the instruction contents', function(done) {
testMetadataProperty(archive, 'instructions', instructions).asCallback(done);
});
});
});