[Experimental] Remove client

Makefile

@@ -15,6 +15,7 @@ UPARGS := --force-recreate --remove-orphans
 QEMUCOMPOSEFILE := docker-compose.qemu.yml
 SECUREBOOTCOMPOSEFILE := docker-compose.secureboot.yml
 CLIENTCOMPOSEFILE := docker-compose.client.yml
+WORKERCOMPOSEFILE := docker-compose.worker.yml
 
 # only use the qemu compose file if worker type is qemu
 ifeq ($(WORKER_TYPE),qemu)
@@ -23,12 +24,14 @@ ifeq ($(QEMU_SECUREBOOT),1)
 COMPOSE_FILE := $(COMPOSE_FILE):$(SECUREBOOTCOMPOSEFILE)
 endif
 else
-COMPOSE_FILE := $(CLIENTCOMPOSEFILE)
+COMPOSE_FILE := $(WORKERCOMPOSEFILE)
 endif
 
 # for arm64 hosts we need to set the BALENA_ARCH to pull the correct balenalib worker image
 ifeq ($(shell uname -m),aarch64)
 BALENA_ARCH ?= aarch64
+else ifeq ($(shell uname -m),armv7l)
+BALENA_ARCH ?= armv7hf
 else ifeq ($(shell uname -m),arm64)
 BALENA_ARCH ?= aarch64
 else
@@ -58,6 +61,8 @@ $(DOCKERCOMPOSE):
 	mkdir -p $(shell dirname "$@")
 ifeq ($(shell uname -m),arm64)
 	curl -fsSL "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(shell uname -s | tr '[:upper:]' '[:lower:]')-aarch64" -o $@
+else ifeq ($(shell uname -m),armv7l)
+	curl -fsSL "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(shell uname -s | tr '[:upper:]' '[:lower:]')-armv7" -o $@
 else
 	curl -fsSL "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(shell uname -s | tr '[:upper:]' '[:lower:]')-$(shell uname -m)" -o $@
 endif
@@ -78,11 +83,14 @@ build: $(DOCKERCOMPOSE) ## Build the required images
 	$(DOCKERCOMPOSE) build $(BUILDARGS)
 
 test: $(DOCKERCOMPOSE) build ## Run the test suites
-	$(DOCKERCOMPOSE) up $(UPARGS) --exit-code-from client
+	$(DOCKERCOMPOSE) up $(UPARGS) --exit-code-from core
 
 local-test: ## Alias for 'make test WORKER_TYPE=qemu'
 	$(MAKE) test WORKER_TYPE=qemu
 
+local-test-autokit: ## Alias for 'make test WORKER_TYPE=qemu'
+	$(MAKE) test WORKER_TYPE=autokit
+
 qemu: ## Alias for 'make test WORKER_TYPE=qemu'
 	$(MAKE) test WORKER_TYPE=qemu
 

client/Dockerfile

@@ -17,4 +17,4 @@ VOLUME /usr/src/app/reports
 
 ENTRYPOINT ["/usr/src/app/entry.sh"]
 
-CMD [ "-c", "/usr/src/app/workspace/config" ]
+CMD [ "-c", "/data/workspace/config" ]

core/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14.19.3-bullseye-slim as base
+FROM node:16-bullseye-slim as base
 
 WORKDIR /usr/app
 
@@ -8,23 +8,20 @@ ENV DEBIAN_FRONTEND noninteractive
 # install docker, balena-cli dependencies, and suite dependencies
 # https://github.com/balena-io/balena-cli/blob/master/INSTALL-LINUX.md#additional-dependencies
 # hadolint ignore=DL3008
+
 RUN apt-get update && apt-get install --no-install-recommends -y \
 	bind9-dnsutils \
 	ca-certificates \
 	docker.io \
-	git \
 	iproute2 \
-	jq \
+	wget \
 	openssh-client \
-	socat \
-	rsync \
+	libudev-dev \	
 	unzip \
 	util-linux \
-	wget \
-	vim \
 	build-essential \
 	make \
-	python && \
+	python3 && \
 	apt-get clean && \
 	rm -rf /var/lib/apt/lists/*
 
@@ -36,19 +33,27 @@ RUN if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ] ; \
 	then \
 		wget -q -O balena-cli.zip "https://github.com/balena-io/balena-cli/releases/download/${BALENA_CLI_REF}/balena-cli-v${BALENA_CLI_VERSION}-linux-arm64-standalone.zip" && \
 		unzip balena-cli.zip && rm balena-cli.zip ; \
+	elif [ "$(uname -m)" = "armv7l" ] || [ "$(uname -m)" = "armv7hf" ] ; \
+	then \
+		npm i balena-cli@latest ; \
+		mkdir /usr/app/cli/ ; \
+		mv /usr/app/node_modules/balena-cli/ /usr/app ; \
+		rm -rf /usr/app/node_modules/ ; \
 	else \
 		wget -q -O balena-cli.zip "https://github.com/balena-io/balena-cli/releases/download/${BALENA_CLI_REF}/balena-cli-v${BALENA_CLI_VERSION}-linux-x64-standalone.zip" && \
 		unzip balena-cli.zip && rm balena-cli.zip ; \
 	fi
 
 # Add balena-cli to PATH
 ENV PATH /usr/app/balena-cli:$PATH
+RUN ls /usr/app/balena-cli && ls /usr/app/balena-cli/bin
+ENV PATH $PATH:/usr/app/balena-cli/bin
 
 RUN balena version
 
 COPY package*.json ./
 
-RUN npm ci
+RUN npm ci --only=production
 
 COPY . .
 

core/balena.yml

@@ -0,0 +1,8 @@
+name: leviathan-core
+type: sw.block
+description: 'Leviathan-core is the testrunner for leviathan'
+assets:
+  repository:
+    type: blob.asset
+    data:
+      url: 'https://github.com/balena-os/leviathan'

core/entry.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
 
+# Internet doesn't work in docker-in-docker when in network_mode: host 
+# due to nftables and iptables-legacy conflict. Docker creates rules in
+# iptables-legacy which is different from what the host (nftables) uses 
+# leading to containers without internet. Commands need to run before starting Docker
+# 
+# Check out: https://stackoverflow.com/a/76488849/8522689
+
+echo "Changing iptables to legacy iptables"
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+
 rm -rf /var/run/docker 2>/dev/null || true
 rm -f /var/run/docker.sock 2>/dev/null || true
 rm -f /var/run/docker.pid 2>/dev/null || true

core/lib/common/suiteSubprocess.js

@@ -62,9 +62,10 @@ async function createJsonSummary(suite) {
 
 (async () => {
 	try{
+		console.log(suiteConfig)
 		const suite = new Suite(
 			{
-				suitePath: config.leviathan.uploads.suite,
+				suitePath: suiteConfig.suite,
 				deviceType: suiteConfig.deviceType,
 				imagePath: config.leviathan.uploads.image,
 			},

core/lib/common/worker.js

@@ -95,6 +95,7 @@ module.exports = class Worker {
 		this.directConnect = (
 			this.url.includes(`worker`)
 			|| this.url.includes('unix:')
+			|| this.url.includes('localhost')
 		);
 		if (this.url.includes(`balena-devices.com`)) {
 			// worker is a testbot connected to balena cloud - we ssh into it via the vpn
@@ -121,56 +122,14 @@ module.exports = class Worker {
 			async () => {
 				attempt++;
 				this.logger.log(`Preparing to flash, attempt ${attempt}...`);
-
-				await new Promise(async (resolve, reject) => {
-					const req = rp.post({ uri: `${this.url}/dut/flash` });
-
-					req.catch((error) => {
-						reject(error);
-					});
-					req.finally(() => {
-						if (lastStatus !== 'done') {
-							reject(new Error('Unexpected end of TCP connection'));
-						}
-
-						resolve();
-					});
-
-					let lastStatus;
-					req.on('data', (data) => {
-						const computedLine = RegExp('(.+?): (.*)').exec(data.toString());
-
-						if (computedLine) {
-							if (computedLine[1] === 'error') {
-								req.cancel();
-								reject(new Error(computedLine[2]));
-							}
-
-							if (computedLine[1] === 'progress') {
-								once(() => {
-									this.logger.log('Flashing');
-								});
-								// Hide any errors as the lines we get can be half written
-								const state = JSON.parse(computedLine[2]);
-								if (state != null && isNumber(state.percentage)) {
-									this.logger.status({
-										message: 'Flashing',
-										percentage: state.percentage,
-									});
-								}
-							}
-
-							if (computedLine[1] === 'status') {
-								lastStatus = computedLine[2];
-							}
-						}
-					});
-
-					pipeline(
-						fs.createReadStream(imagePath),
-						createGzip({ level: 6 }),
-						req,
-					);
+				this.logger.log(`Image path being sent: ${imagePath}`)
+				await doRequest({
+					method: 'POST',
+					uri: `${this.url}/dut/flash`,
+					body: {
+						path: imagePath
+					},
+					json: true,
 				});
 				this.logger.log('Flash completed');
 			},
@@ -416,12 +375,8 @@ module.exports = class Worker {
 	async executeCommandInWorker(command, retryOptions={}) {
 		return retry(
 			async () => {
-				let containerId = await this.executeCommandInWorkerHost(
-					`balena ps | grep worker | awk '{print $1}'`,
-				);
-				let result = await this.executeCommandInWorkerHost(
-					`balena exec ${containerId} ${command}`,
-				);
+				let result = await exec(command);
+				console.log(`Exec call: ${command}, Result: ${result}`)
 				return result;
 			},
 			{

core/lib/config.js

@@ -1,13 +1,12 @@
 module.exports = {
   leviathan: {
     artifacts: '/tmp/artifacts',    // To store artifacts meant to be reported as results at the end of the suite
-    downloads: '/data/downloads',   // To store/download assets needed for the suite (non-persistent) 
+    downloads: '/data/workspace/downloads',   // To store/download assets needed for the suite (non-persistent) 
     reports: '/reports/',           // To store/download reports generated from the suite (non-persistent) 
     workdir: '/data',
     uploads: {
-      image: '/data/os.img',
-      config: '/data/config.json',
-      suite: '/data/suite'
+      config:'/data/workspace/config.js',
+      suite: '/data/suites'
     }
   }
 };

core/lib/main-standalone.js

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2017 balena
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const Bluebird = require('bluebird');
+const { fork } = require('child_process');
+const { getFilesFromDirectory } = require('./common/utils');
+const config = require('./config.js');
+const express = require('express');
+const expressWebSocket = require('express-ws');
+const { ensureDir, pathExists, remove } = require('fs-extra');
+const md5 = require('md5-file');
+const { fs, crypto } = require('mz');
+const { basename, join } = require('path');
+const tar = require('tar-fs');
+const pipeline = Bluebird.promisify(require('stream').pipeline);
+const WebSocket = require('ws');
+const { parse } = require('url'); // eslint-disable-line
+const { createGzip, createGunzip } = require('zlib');
+const setReportsHandler = require('./reports');
+const MachineState = require('./state');
+const { createWriteStream } = require('fs');
+
+async function main() {
+    // Handler definitions
+    const stdHandler = data => {
+       console.log(data.toString())
+    }
+    const msgHandler = message => {
+        console.log(message)
+    };
+
+
+    // The reason we need to fork is because many 3rd party libariers output to stdout
+    // so we need to capture that
+    let suite = fork('./lib/common/suiteSubprocess', {
+        stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+    });
+    suiteStarted = true;
+
+    suite.stdout.on('data', stdHandler);
+    suite.stderr.on('data', stdHandler);
+    suite.on('message', msgHandler);
+
+    const suiteExitCode = await new Promise((resolve, reject) => {
+        suite.on('error', reject);
+        suite.on('exit', code => {
+            resolve(code);
+        });
+    });
+
+    console.log(`Suite exit code is: ${suiteExitCode}`);
+    const success = suiteExitCode === 0;
+    if(success){
+        console.log(`Test suite result: PASS`)
+        process.exit(0)
+    } else {
+        console.log(`Test suite result: FAIL`)
+        process.exit(suiteExitCode)
+    }
+
+};
+
+
+main();

docker-compose.worker.yml

@@ -0,0 +1,41 @@
+version: "2"
+
+services:
+  core:
+    privileged: true # preload requires docker-in-docker
+    #build: core
+    image: bh.cr/gh_rcooke_warwick/leviathan-core-armv7hf
+    network_mode: host
+    volumes:
+      - reports-storage:/reports
+      - "${SUITES:-./suites}:/data/suites"
+      - "${REPORTS:-./workspace/reports}:/data/reports"
+      - "${WORKSPACE:-./workspace}:/data/workspace"
+      - docker-fs:/var/lib/docker #docker data directory must be in a volume to avoid running out of space
+    tmpfs:
+      - /var/run # use tmpfs docker-in-docker pid files
+    restart: 'no'
+
+  worker:
+    image: bh.cr/gh_rcooke_warwick/leviathan-worker-amrv7hf
+    privileged: true
+    pid: host
+    network_mode: host
+    ipc: host
+    volumes:
+      - 'reports-storage:/reports'
+      - "${SUITES:-./suites}:/data/suites"
+      - "${REPORTS:-./workspace/reports}:/data/reports"
+      - "${WORKSPACE:-./workspace}:/data/workspace"
+      - /host/run/dbus:/host/run/dbus
+      - /lib/modules:/lib/modules
+    environment:
+      - UDEV=1
+      - BALENA_API_KEY=${BALENA_API_KEY}
+      - WORKER_TYPE=autokit
+      - DIGITAL_RELAY=dummyPower
+      - TESTBOT_DUT_TYPE=raspberrypi4-64
+
+volumes:
+  reports-storage:
+  docker-fs: