patch: basic E2E tests for macOS

.github/actions/test/action.yml

@@ -46,9 +46,20 @@ runs:
       with:
         python-version: '3.11'
 
+    - name: Setup Virtual Drive on MacOS
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        hdiutil create -size 4096m -layout NONE -o virtual_test_disk.dmg
+        virtual_path=$(hdiutil attach -nomount virtual_test_disk.dmg | awk '{print $1}')
+        echo "TARGET_DRIVE=${virtual_path}" >> $GITHUB_ENV
+        echo "ETCHER_INCLUDE_VIRTUAL_DRIVES=1" >> $GITHUB_ENV
+
     - name: Test release
       shell: bash
       run: |
+        # Build and Test release
+
         ## FIXME: causes issues with `xxhash` which tries to load a debug build which doens't exist and cannot be compiled
         # if [[ '${{ inputs.VERBOSE }}' =~ on|On|Yes|yes|true|True ]]; then
         #   export DEBUG='electron-forge:*,sidecar'
@@ -57,11 +68,29 @@ runs:
         npm ci
         npm run lint
         npm run package
-        npm run wdio # test stage, note that it requires the package to be done first
+        
+        # tests requires the app to already be built
 
+        # # only run e2e tests on Mac as it's the only supported platform atm
+        if [[ '${{ runner.os }}' == 'macOS' ]]; then
+          # run all tests on macOS including E2E
+          # E2E tests can't input the administrative password, therefore the tests need to run as root
+          wget -q -O ${{ env.TEST_SOURCE_FILE }} ${{ env.TEST_SOURCE_URL }}
+          sudo \
+            TARGET_DRIVE=${{ env.TARGET_DRIVE }} \
+            ETCHER_INCLUDE_VIRTUAL_DRIVES=1 \
+            TEST_SOURCE_FILE: $(pwd)/${{ env.TEST_SOURCE_FILE }} \
+            TEST_SOURCE_URL: ${{ env.TEST_SOURCE_URL }} \
+            npm run wdio:ci
+        else
+          npm run wdio:unit
+        fi
+        
       env:
         # https://www.electronjs.org/docs/latest/api/environment-variables
         ELECTRON_NO_ATTACH_CONSOLE: 'true'
+        TEST_SOURCE_URL: 'https://api.balena-cloud.com/download?deviceType=raspberrypi4-64&version=5.2.8&fileType=.zip'
+        TEST_SOURCE_FILE: 'raspberrypi4-64-5.2.8-v16.1.10.img.zip'
 
     - name: Compress custom source
       if: runner.os != 'Windows'

.gitignore

@@ -120,4 +120,10 @@ secrets/WINDOWS_SIGNING.pfx
 
 #local development
 .yalc
-yalc.lock
+yalc.lock
+
+# Test assets
+virtual_test_disk.dmg
+virtual_test_disk.img
+virtual_test_disk.vhd
+screenshots/

lib/gui/app/components/drive-selector/drive-selector.tsx

@@ -419,6 +419,7 @@ export class DriveSelector extends React.Component<
 					primary: !showWarnings,
 					warning: showWarnings,
 					disabled: !hasAvailableDrives(),
+					'data-testid': 'validate-target-button',
 				}}
 				{...props}
 			>

lib/gui/app/components/flash-another/flash-another.tsx

@@ -25,7 +25,7 @@ export interface FlashAnotherProps {
 
 export const FlashAnother = (props: FlashAnotherProps) => {
 	return (
-		<BaseButton primary onClick={props.onClick}>
+		<BaseButton primary data-testid="flash-another" onClick={props.onClick}>
 			{i18next.t('flash.another')}
 		</BaseButton>
 	);

lib/gui/app/components/flash-results/flash-results.tsx

@@ -163,7 +163,7 @@ export function FlashResults({
 					/>
 					<Txt>{middleEllipsis(image, 24)}</Txt>
 				</Flex>
-				<Txt fontSize={24} color="#fff" mb="17px">
+				<Txt data-testid="flash-results" fontSize={24} color="#fff" mb="17px">
 					{allFailed
 						? i18next.t('flash.flashFailed')
 						: i18next.t('flash.flashCompleted')}

lib/gui/app/components/progress-button/progress-button.tsx

@@ -104,7 +104,9 @@ export class ProgressButton extends React.PureComponent<ProgressButtonProps> {
 						}}
 					>
 						<Flex>
-							<Txt color="#fff">{status}&nbsp;</Txt>
+							<Txt data-testid="flash-status" color="#fff">
+								{status}&nbsp;
+							</Txt>
 							<Txt color={colors[type]}>{position}</Txt>
 						</Flex>
 						{type && (
@@ -125,6 +127,7 @@ export class ProgressButton extends React.PureComponent<ProgressButtonProps> {
 				warning={warning}
 				onClick={this.props.callback}
 				disabled={this.props.disabled}
+				data-testid={'flash-now'}
 				style={{
 					marginTop: 30,
 				}}

lib/gui/app/components/source-selector/source-selector.tsx

@@ -165,6 +165,7 @@ const URLSelector = ({
 			cancel={cancel}
 			primaryButtonProps={{
 				disabled: loading || !imageURL,
+				'data-testid': 'source-url-ok',
 			}}
 			action={loading ? <Spinner /> : i18next.t('ok')}
 			done={async () => {
@@ -186,6 +187,7 @@ const URLSelector = ({
 					</Txt>
 					<Input
 						value={imageURL}
+						data-testid="source-url-input"
 						placeholder={i18next.t('source.enterValidURL')}
 						type="text"
 						onChange={(evt: React.ChangeEvent<HTMLInputElement>) =>
@@ -638,6 +640,7 @@ export class SourceSelector extends React.Component<
 							</StepNameButton>
 							{!flashing && !imageLoading && (
 								<ChangeButton
+									data-testid="change-image"
 									plain
 									mb={14}
 									onClick={() => this.reselectSource()}
@@ -655,6 +658,7 @@ export class SourceSelector extends React.Component<
 								disabled={this.state.imageSelectorOpen}
 								primary={this.state.defaultFlowActive}
 								key="Flash from file"
+								data-testid="flash-from-file"
 								flow={{
 									onClick: () => this.openImageSelector(),
 									label: i18next.t('source.fromFile'),
@@ -665,6 +669,7 @@ export class SourceSelector extends React.Component<
 							/>
 							<FlowSelector
 								key="Flash from URL"
+								data-testid="flash-from-url"
 								flow={{
 									onClick: () => this.openURLSelector(),
 									label: i18next.t('source.fromURL'),

lib/gui/app/components/target-selector/target-selector-button.tsx

@@ -150,6 +150,7 @@ export function TargetSelectorButton(props: TargetSelectorProps) {
 			tabIndex={targets.length > 0 ? -1 : 2}
 			disabled={props.disabled}
 			onClick={props.openDriveSelector}
+			data-testid="select-target"
 		>
 			{i18next.t('target.selectTarget')}
 		</StepButton>

lib/gui/app/modules/progress-status.ts

@@ -34,8 +34,6 @@ export function fromFlashState({
 	status: string;
 	position?: string;
 } {
-	console.log(i18next.t('progress.starting'));
-
 	if (type === undefined) {
 		return { status: i18next.t('progress.starting') };
 	} else if (type === 'decompressing') {

lib/gui/app/os/dialog.ts

@@ -23,6 +23,7 @@ import * as settings from '../../../gui/app/models/settings';
 import { SUPPORTED_EXTENSIONS } from '../../../shared/supported-formats';
 import * as i18next from 'i18next';
 
+// FIXME: this is probably useless now
 async function mountSourceDrive() {
 	// sourceDrivePath is the name of the link in /dev/disk/by-path
 	const sourceDrivePath = await settings.get('automountOnFileSelect');
@@ -43,6 +44,17 @@ async function mountSourceDrive() {
  */
 export async function selectImage(): Promise<string | undefined> {
 	await mountSourceDrive();
+
+	// For automated E2E testing, we can't set the source file by interacting with the OS dialog,
+	// so we use an ENV var instead and bypass the dialog. Note that we still need to press the "flash from file" button.
+	if (
+		process.env.TEST_SOURCE_FILE !== undefined &&
+		typeof process.env.TEST_SOURCE_FILE === 'string'
+	) {
+		console.log(`test mode: loading ${process.env.TEST_SOURCE_FILE}`);
+		return process.env.TEST_SOURCE_FILE;
+	}
+
 	const options: electron.OpenDialogOptions = {
 		// This variable is set when running in GNU/Linux from
 		// inside an AppImage, and represents the working directory

lib/util/drive-scanner.ts

@@ -25,6 +25,8 @@ import { geteuid, platform } from 'process';
 const adapters: Adapter[] = [
 	new BlockDeviceAdapter({
 		includeSystemDrives: () => true,
+		includeVirtualDrives: () =>
+			process.env.ETCHER_INCLUDE_VIRTUAL_DRIVES !== 'undefined',
 	}),
 ];
 

package.json

@@ -20,7 +20,9 @@
     "package": "electron-forge package",
     "start": "electron-forge start",
     "make": "electron-forge make",
-    "wdio": "xvfb-maybe wdio run ./wdio.conf.ts"
+    "wdio:unit": "xvfb-maybe wdio run ./wdio.conf.ts --suite gui --suite shared",
+    "wdio:e2e": "xvfb-maybe wdio run ./wdio.conf.ts --suite e2e",
+    "wdio:ci": "xvfb-maybe wdio run ./wdio.conf.ts --suite ci"
   },
   "husky": {
     "hooks": {
@@ -101,7 +103,7 @@
     "tslib": "2.6.2",
     "typescript": "^5.3.3",
     "url-loader": "4.1.1",
-    "wdio-electron-service": "^6.4.1",
+    "wdio-electron-service": "^6.5.0",
     "xvfb-maybe": "^0.2.1"
   },
   "hostDependencies": {

tests/e2e/e2e-common.ts

@@ -0,0 +1,69 @@
+import { browser } from '@wdio/globals';
+
+const prepare = () => {
+	browser.pause(2000);
+};
+
+const itShouldSelectVirtualTarget = () => {
+	it('should "select a virtual target"', async () => {
+		const selectTargetButton = $('button[data-testid="select-target"]');
+		await selectTargetButton.waitForClickable({ timeout: 30000 });
+		await selectTargetButton.click();
+
+		// target drive is set in the github custom test action
+		// if you run the test locally, pass the varibale
+		const targetVirtualDrive = $(`=${process.env.TARGET_DRIVE}`);
+		await targetVirtualDrive.waitForClickable({ timeout: 10000 });
+		await targetVirtualDrive.click();
+
+		const validateTargetButton = $(
+			'button[data-testid="validate-target-button"]',
+		);
+		await validateTargetButton.waitForClickable({ timeout: 10000 });
+		await validateTargetButton.click();
+	});
+};
+
+const itShouldStartFlashing = () => {
+	it('should "start flashing"', async () => {
+		const flashNowButton = $('button[data-testid="flash-now"]');
+		await flashNowButton.waitForClickable({ timeout: 10000 });
+		await flashNowButton.click();
+	});
+};
+
+const itShouldGetTheFlashCompletedScreen = () => {
+	it('should get the "Flash Completed" screen', async () => {
+		const flashResults = $('[data-testid="flash-results"]');
+		// 5' should be enough from CI, but might be too short for local testing on slow connections
+		await flashResults.waitForDisplayed({ timeout: 300000 });
+
+		const flashResultsText = await flashResults.getText();
+		expect(flashResultsText).toBe('Flash Completed!');
+	});
+};
+
+const itShouldGetBackToHomeScreen = () => {
+	it('should get back to the "Home" screen', async () => {
+		const flashAnotherButton = $('button[data-testid="flash-another"]');
+
+		await flashAnotherButton.waitForClickable({ timeout: 10000 });
+		await flashAnotherButton.click();
+
+		// previously flashes image is still seclected, remove it
+		const changeSource = $('button[data-testid="change-image"]');
+
+		await changeSource.waitForClickable({ timeout: 10000 });
+		await changeSource.click();
+
+		// we're good;
+	});
+};
+
+export {
+	prepare,
+	itShouldSelectVirtualTarget,
+	itShouldStartFlashing,
+	itShouldGetTheFlashCompletedScreen,
+	itShouldGetBackToHomeScreen,
+};

tests/e2e/e2e-flash-from-file.spec.ts

@@ -0,0 +1,23 @@
+import '@wdio/globals';
+import {
+	prepare,
+	itShouldSelectVirtualTarget,
+	itShouldStartFlashing,
+	itShouldGetTheFlashCompletedScreen,
+	itShouldGetBackToHomeScreen,
+} from './e2e-common';
+
+describe('Flash From File E2E Test', () => {
+	before(prepare);
+
+	it('should select a file as source', async () => {
+		const flashFromFileButton = $('button[data-testid="flash-from-file"]');
+		await flashFromFileButton.waitForClickable({ timeout: 10000 });
+		await flashFromFileButton.click();
+	});
+
+	itShouldSelectVirtualTarget();
+	itShouldStartFlashing();
+	itShouldGetTheFlashCompletedScreen();
+	itShouldGetBackToHomeScreen();
+});

tests/e2e/e2e-flash-from-url.spec.ts

@@ -0,0 +1,31 @@
+import '@wdio/globals';
+import {
+	prepare,
+	itShouldSelectVirtualTarget,
+	itShouldStartFlashing,
+	itShouldGetTheFlashCompletedScreen,
+	itShouldGetBackToHomeScreen,
+} from './e2e-common';
+
+describe('Flash From URL E2E test', () => {
+	before(prepare);
+
+	it('should select an url as source', async () => {
+		const flashFromUrlButton = $('button[data-testid="flash-from-url"]');
+		await flashFromUrlButton.waitForClickable({ timeout: 10000 });
+		await flashFromUrlButton.click();
+
+		const enterValidUrlInput = $('input[data-testid="source-url-input"]');
+		await enterValidUrlInput.waitForDisplayed({ timeout: 10000 });
+		await enterValidUrlInput.setValue(process.env.TEST_SOURCE_URL as string);
+
+		const sourceUrlOkButton = $('button[data-testid="source-url-ok"]');
+		await sourceUrlOkButton.waitForClickable({ timeout: 10000 });
+		await sourceUrlOkButton.click();
+	});
+
+	itShouldSelectVirtualTarget();
+	itShouldStartFlashing();
+	itShouldGetTheFlashCompletedScreen();
+	itShouldGetBackToHomeScreen();
+});

tests/test.e2e.ts

@@ -1,7 +0,0 @@
-import { browser } from '@wdio/globals';
-
-describe('Electron Testing', () => {
-	it('should print application title', async () => {
-		console.log('Hello', await browser.getTitle(), 'application!');
-	});
-});

wdio.conf.ts

@@ -35,16 +35,36 @@ export const config: Options.Testrunner = {
 	// Patterns to exclude.
 	// FIXME: Remove the following exclusions once the tests are ported to WDIO
 	exclude: [
-		'tests/gui/modules/image-writer.spec.ts',
-		'tests/gui/os/window-progress.spec.ts',
-		'tests/gui/models/available-drives.spec.ts',
-		'tests/gui/models/flash-state.spec.ts',
-		'tests/gui/models/selection-state.spec.ts',
-		'tests/gui/models/settings.spec.ts',
-		'tests/shared/drive-constraints.spec.ts',
-		'tests/shared/messages.spec.ts',
-		'tests/gui/modules/progress-status.spec.ts',
+		'./tests/gui/modules/image-writer.spec.ts',
+		'./tests/gui/os/window-progress.spec.ts',
+		'./tests/gui/models/available-drives.spec.ts',
+		'./tests/gui/models/flash-state.spec.ts',
+		'./tests/gui/models/selection-state.spec.ts',
+		'./tests/gui/models/settings.spec.ts',
+		'./tests/shared/drive-constraints.spec.ts',
+		'./tests/shared/messages.spec.ts',
+		'./tests/gui/modules/progress-status.spec.ts',
 	],
+
+	suites: {
+		'gui': ['./tests/gui/**/*.spec.ts'],
+		'shared': ['./tests/shared/**/*.spec.ts'],
+		'e2e': [
+			[
+				'./tests/e2e/e2e-flash-from-file.spec.ts',
+				'./tests/e2e/e2e-flash-from-url.spec.ts',
+			]
+		],
+		// CI needs to runs e2e tests and other tests sequencially
+		'ci': [
+			[
+				'./tests/e2e/e2e-flash-from-url.spec.ts',
+				'./tests/e2e/e2e-flash-from-file.spec.ts',
+				'./tests/gui/**/*.spec.ts',
+				'./tests/shared/**/*.spec.ts',
+			],
+		]
+	},
 	//
 	// ============
 	// Capabilities
@@ -85,7 +105,7 @@ export const config: Options.Testrunner = {
 	// Define all options that are relevant for the WebdriverIO instance here
 	//
 	// Level of logging verbosity: trace | debug | info | warn | error | silent
-	logLevel: 'info',
+	logLevel: 'warn',
 	//
 	// Set specific log levels per logger
 	// loggers: