From patchwork Mon Jun 3 07:09:19 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Sai.Sathujoda@toshiba-tsip.com X-Patchwork-Id: 13683349 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 69287C25B76 for ; Mon, 3 Jun 2024 07:09:35 +0000 (UTC) Received: from mo-csw.securemx.jp (mo-csw.securemx.jp [210.130.202.152]) by mx.groups.io with SMTP id smtpd.web11.77503.1717398566386274582 for ; Mon, 03 Jun 2024 00:09:26 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: toshiba-tsip.com, ip: 210.130.202.152, mailfrom: sai.sathujoda@toshiba-tsip.com) Received: by mo-csw.securemx.jp (mx-mo-csw1802) id 45379Oem2114503; Mon, 3 Jun 2024 16:09:24 +0900 X-Iguazu-Qid: 2yAbNsrDKYJ7WeydVE X-Iguazu-QSIG: v=2; s=0; t=1717398564; q=2yAbNsrDKYJ7WeydVE; m=oVb7/0rl17Le2Y7veoXd5SxV0/HntfNdxZfS+dCX5mU= Received: from imx12-a.toshiba.co.jp ([38.106.60.135]) by relay.securemx.jp (mx-mr1802) id 45379N85882083 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NOT); Mon, 3 Jun 2024 16:09:23 +0900 From: Sai.Sathujoda@toshiba-tsip.com To: cip-dev@lists.cip-project.org, jan.kiszka@siemens.com Cc: sai ashrith sathujoda , dinesh.kumar@toshiba-tsip.com, kazuhiro3.hayashi@toshiba.co.jp Subject: [isar-cip-core 2/3] submit_lava.sh: Add script to submit the jobs prepared from templates Date: Mon, 3 Jun 2024 12:39:19 +0530 X-TSB-HOP2: ON Message-Id: <20240603070920.2942859-3-Sai.Sathujoda@toshiba-tsip.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20240603070920.2942859-1-Sai.Sathujoda@toshiba-tsip.com> References: <20240603070920.2942859-1-Sai.Sathujoda@toshiba-tsip.com> MIME-Version: 1.0 X-OriginalArrivalTime: 03 Jun 2024 07:09:19.0455 (UTC) FILETIME=[F3FC26F0:01DAB584] List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Mon, 03 Jun 2024 07:09:35 -0000 X-Groupsio-URL: https://lists.cip-project.org/g/cip-dev/message/16081 From: sai ashrith sathujoda This script creates job definitions from the template files included in the test directory and shall submit these jobs to CIP LAVA platform. This script will wait until all the results of all the submitted jobs are obtained. Signed-off-by: sai ashrith sathujoda --- scripts/submit_lava.sh | 458 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 458 insertions(+) create mode 100755 scripts/submit_lava.sh diff --git a/scripts/submit_lava.sh b/scripts/submit_lava.sh new file mode 100755 index 0000000..d825442 --- /dev/null +++ b/scripts/submit_lava.sh @@ -0,0 +1,458 @@ +#!/bin/bash +# Copyright (C) 2024, Renesas Electronics Europe GmbH +# Chris Paterson +# Sai Ashrith +################################################################################ + +set -e + +################################################################################ +LAVA_TEMPLATES="test/templates" +LAVA_JOBS_URL="https://${CIP_LAVA_LAB_SERVER:-lava.ciplatform.org}/scheduler/job" +LAVA_API_URL="https://${CIP_LAVA_LAB_SERVER:-lava.ciplatform.org}/api/v0.2" +LAVACLI_ARGS="--uri https://$CIP_LAVA_LAB_USER:$CIP_LAVA_LAB_TOKEN@${CIP_LAVA_LAB_SERVER:-lava.ciplatform.org}/RPC2" +SQUAD_GROUP="cip-core" +SQUAD_WATCH_JOBS_URL="${CIP_SQUAD_URL}/api/watchjob" +SQUAD_LAVA_BACKEND="${CIP_SQUAD_LAVA_BACKEND:-cip}" + +WORK_DIR=$(pwd) +RESULTS_DIR="$WORK_DIR/results" +ERROR=false +TEST=$1 +COMMIT_REF=$2 +release=$3 + +if [ -z "$SUBMIT_ONLY" ]; then SUBMIT_ONLY=false; fi + +# Create a dictionary to handle image arguments based on architecture +declare -A image_args +image_args[amd64]="-cpu qemu64 -machine q35,accel=tcg -global ICH9-LPC.noreboot=off -device ide-hd,drive=disk -drive if=pflash,format=raw,unit=0,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd -device virtio-net-pci,netdev=net -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_VARS_4M.snakeoil.fd -global ICH9-LPC.disable_s3=1 -global isa-fdc.driveA= -device tpm-tis,tpmdev=tpm0" +image_args[arm64]="-cpu cortex-a57 -machine virt -device virtio-serial-device -device virtconsole,chardev=con -chardev vc,id=con -device virtio-blk-device,drive=disk -device virtio-net-device,netdev=net -device tpm-tis-device,tpmdev=tpm0" +image_args[arm]="-cpu cortex-a15 -machine virt -device virtio-serial-device -device virtconsole,chardev=con -chardev vc,id=con -device virtio-blk-device,drive=disk -device virtio-net-device,netdev=net -device tpm-tis-device,tpmdev=tpm0" + +set_up (){ + echo "Installing dependencies to run this script..." + sudo apt update && sudo apt install -y --no-install-recommends lavacli curl + job_dir="$(mktemp -d)" +} + +clean_up () { + rm -rf "$job_dir" +} + +# This method is called only for arm64 and arm targets while building job definitions +add_firmware_artifacts () { + sed -i "s@#Firmware#@firmware:@g" "$2" + sed -i "s@#Firmware_args#@image_arg: '-bios {firmware}'@g" "$2" + sed -i "s@#Firmware_url#@url: https://s3.eu-central-1.amazonaws.com/download2.cip-project.org/cip-core/next/qemu-${1}/firmware.bin@g" "$2" +} + +# This method creates LAVA job definitions for QEMU amd64, arm64 and armhf +# The created job definitions test SWUpdate, Secureboot and IEC layer +create_jobs () { + if [ "$1" = "IEC_Layer_test" ]; then + for arch in amd64 arm64 arm + do + cp $LAVA_TEMPLATES/IEC_template.yml "${job_dir}"/IEC_${arch}.yml + sed -i "s@#architecture#@${arch}@g" "${job_dir}"/IEC_${arch}.yml + sed -i "s@#imageargs#@${image_args[$arch]}@g" "${job_dir}"/IEC_${arch}.yml + + if [ $arch != amd64 ]; then + add_firmware_artifacts $arch "${job_dir}"/IEC_${arch}.yml + fi + done + + elif [ "$1" = "software_update_test" ]; then + if [ -z "$2" ]; then + for arch in amd64 arm64 arm + do + cp $LAVA_TEMPLATES/swupdate_template.yml "${job_dir}"/swupdate_${arch}.yml + sed -i "s@#architecture#@${arch}@g" "${job_dir}"/swupdate_${arch}.yml + sed -i "s@#imageargs#@${image_args[$arch]}@g" "${job_dir}"/swupdate_${arch}.yml + + if [ $arch != amd64 ]; then + add_firmware_artifacts $arch "${job_dir}"/swupdate_${arch}.yml + fi + done + else + cp "${job_dir}"/swupdate_amd64.yml "${job_dir}"/"${2}".yml + sed -i "s@software update testing@${2}@g" "${job_dir}"/"${2}".yml + sed -i "s@) = 2@) = 0@g" "${job_dir}"/"${2}".yml + if [ "$2" = "kernel_panic" ]; then + sed -i "s@next@maintain-lava-artifact@g" "${job_dir}"/"${2}".yml + sed -i "s@kernel: C:BOOT1:linux.efi@Kernel panic - not syncing: sysrq triggered crash@g" "${job_dir}"/"${2}".yml + else + sed -i "s@kernel: C:BOOT1:linux.efi@Can't open verity rootfs - continuing will lead to a broken trust chain!@g" "${job_dir}"/"${2}".yml + sed -i "s@echo software update is successful!!@dd if=/dev/urandom of=/dev/sdb5 bs=512 count=1@g" "${job_dir}"/"${2}".yml + fi + fi + else + for arch in amd64 arm64 arm + do + cp $LAVA_TEMPLATES/secureboot_template.yml "${job_dir}"/secureboot_${arch}.yml + sed -i "s@#architecture#@${arch}@g" "${job_dir}"/secureboot_${arch}.yml + sed -i "s@#imageargs#@${image_args[$arch]}@g" "${job_dir}"/secureboot_${arch}.yml + + if [ $arch != amd64 ]; then + add_firmware_artifacts $arch "${job_dir}"/secureboot_${arch}.yml + fi + done + fi + + for arch in amd64 arm64 arm + do + sed -i "s@#distribution#@${release}@g" "${job_dir}"/*_${arch}.yml + done +} + +create_cip_core_jobs () { + if [ "$TEST" = "IEC" ]; then + create_jobs IEC_Layer_test + elif [ "$TEST" = "swupdate" ]; then + create_jobs software_update_test + create_jobs software_update_test kernel_panic + create_jobs software_update_test initramfs_crash + else + create_jobs secure_boot_test + fi +} + +# This method attaches SQUAD watch job to the submitted LAVA job +# $1: LAVA Job ID +submit_squad_watch_job(){ +# SQUAD watch job submission + local ret + if [ -z ${CIP_SQUAD_LAB_TOKEN+x} ]; then + echo "SQUAD_LAB_TOKEN not found, omitting SQUAD results reporting!" + return 0 + fi + + if [ "$TEST" = "swupdate" ]; then + squad_project="swupdate-testing" + elif [ "$TEST" = "secure-boot" ]; then + squad_project="secure-boot-testing" + else + squad_project="iec-layer-testing" + fi + + local DEVICE=$2 + local ENV="${DEVICE}_${squad_project}" + local squad_url="$SQUAD_WATCH_JOBS_URL/${SQUAD_GROUP}/${squad_project}/${COMMIT_REF}/${ENV}" + ret=$(curl -s \ + --header "Authorization: token $CIP_SQUAD_LAB_TOKEN" \ + --form backend="$SQUAD_LAVA_BACKEND" \ + --form testjob_id="$1" \ + --form metadata='{"device": "'${DEVICE}'", "CI pipeline": "'${CI_PIPELINE_URL}'", "CI job": "'${CI_JOB_URL}'"}' \ + "$squad_url") + + if [[ $ret != [0-9]* ]] + then + echo "Something went wrong with SQUAD watch job submission. SQUAD returned:" + echo "${ret}" + echo "SQUAD URL: ${squad_url}" + echo "SQUAD Backend: ${SQUAD_LAVA_BACKEND}" + echo "LAVA Job Id: $1" + else + echo "SQUAD watch job submitted successfully as #${ret}." + fi +} + +# $1: Job definition file +submit_job() { + # Make sure yaml file exists + if [ -f "$1" ]; then + echo "Submitting $1 to LAVA master..." + # Catch error that occurs if invalid yaml file is submitted + local ret=$(lavacli $LAVACLI_ARGS jobs submit "$1") || error=true + + if [[ $ret != [0-9]* ]] + then + echo "Something went wrong with job submission. LAVA returned:" + echo "${ret}" + else + echo "Job submitted successfully as #${ret}." + + local lavacli_output=${job_dir}/lavacli_output + lavacli $LAVACLI_ARGS jobs show "${ret}" \ + > "$lavacli_output" + + local status=$(cat "$lavacli_output" \ + | grep "state" \ + | cut -d ":" -f 2 \ + | awk '{$1=$1};1') + STATUS[${ret}]=$status + + local health=$(cat "$lavacli_output" \ + | grep "Health" \ + | cut -d ":" -f 2 \ + | awk '{$1=$1};1') + HEALTH[${ret}]=$health + + local device_type=$(cat "$lavacli_output" \ + | grep "device-type" \ + | cut -d ":" -f 2 \ + | awk '{$1=$1};1') + DEVICE_TYPE[${ret}]=$device_type + + local device=$(cat "$lavacli_output" \ + | grep "device :" \ + | cut -d ":" -f 2 \ + | awk '{$1=$1};1') + DEVICE[${ret}]=$device + + local test=$(cat "$lavacli_output" \ + | grep "description" \ + | rev | cut -d "_" -f 1 | rev) + TEST[${ret}]=$test + + submit_squad_watch_job "${ret}" "${device}" + + JOBS+=("${ret}") + + fi + fi +} + +# $1: Device-type to search for +is_device_online () { + local lavacli_output=${job_dir}/lavacli_output + + # Get list of all devices + lavacli $LAVACLI_ARGS devices list > "$lavacli_output" + + # Count the number of online devices + local count=$(grep "(${1})" "$lavacli_output" | grep -c "Good") + echo "There are currently $count \"${1}\" devices online." + + if [ "$count" -gt 0 ]; then + return 0 + fi + return 1 +} + +submit_jobs () { + local ret=0 + for JOB in "${job_dir}"/*.yml; do + local device=$(grep device_type "$JOB" | cut -d ":" -f 2 | awk '{$1=$1};1') + if is_device_online "$device"; then + submit_job "$JOB" + else + echo "Refusing to submit test job as there are no suitable devices available." + ret=1 + fi + done + return $ret +} + +# This method is added with the intention to check if all the jobs are valid before submit +# If even a single definition is found to be invalid, then no job shall be submitted until +# it is fixed by the maintainer +validate_jobs () { + local ret=0 + for JOB in "${job_dir}"/*.yml; do + if lavacli $LAVACLI_ARGS jobs validate "$JOB"; then + echo "$JOB is a valid definition" + else + echo "$JOB is not a valid definition" + ret=1 + fi + done + return $ret +} + +check_if_all_finished () { + for i in "${JOBS[@]}"; do + if [ "${STATUS[$i]}" != "Finished" ]; then + return 1 + fi + done + return 0 +} + +check_for_test_error () { + for i in "${JOBS[@]}"; do + if [ "${HEALTH[$i]}" != "Complete" ]; then + return 0 + fi + done + return 1 +} + +# $1: LAVA job ID to show results for +get_test_result () { + if [ -n "${1}" ]; then + lavacli "$LAVACLI_ARGS" results "${1}" + fi +} + +get_junit_test_results () { + mkdir -p "${RESULTS_DIR}" + for i in "${JOBS[@]}"; do + curl -s -o "${RESULTS_DIR}"/results_"$i".xml "${LAVA_API_URL}"/jobs/"$i"/junit/ + done +} + +# $1: Test to print before job summaries +# $2: Set to true to print results for each job +print_status () { + if [ -z "${1}" ]; then + # Set default text + local message="Current job status:" + else + local message="${1}" + fi + + echo "------------------------------" + echo "${message}" + echo "------------------------------" + for i in "${JOBS[@]}"; do + echo "Job #$i: ${STATUS[$i]}" + echo "Health: ${HEALTH[$i]}" + echo "Device Type: ${DEVICE_TYPE[$i]}" + echo "Device: ${DEVICE[$i]}" + echo "Test: ${TEST[$i]}" + echo "URL: ${LAVA_JOBS_URL}/$i" + if [ -n "${2}" ]; then + get_test_result "$i" + fi + echo " " + done +} + +print_summary () { + echo "------------------------------" + echo "Job Summary" + echo "------------------------------" + for i in "${JOBS[@]}" + do + echo "Job #${i} ${STATUS[$i]}. Job health: ${HEALTH[$i]}. URL: ${LAVA_JOBS_URL}/${i}" + done +} + +check_status () { + if [ -n "$TEST_TIMEOUT" ]; then + # Current time + timeout time + local end_time=$(date +%s -d "+ $TEST_TIMEOUT min") + fi + + local error=false + + if [ ${#JOBS[@]} -ne 0 ] + then + + print_status "Current job status:" + while true + do + # Get latest status + for i in "${JOBS[@]}" + do + if [ "${STATUS[$i]}" != "Finished" ] + then + local lavacli_output=${job_dir}/lavacli_output + lavacli $LAVACLI_ARGS jobs show "$i" \ + > "$lavacli_output" + + local status=$(cat "$lavacli_output" \ + | grep "state" \ + | cut -d ":" -f 2 \ + | awk '{$1=$1};1') + + local health=$(cat "$lavacli_output" \ + | grep "Health" \ + | cut -d ":" -f 2 \ + | awk '{$1=$1};1') + HEALTH[$i]=$health + + local device_type=$(cat "$lavacli_output" \ + | grep "device-type" \ + | cut -d ":" -f 2 \ + | awk '{$1=$1};1') + DEVICE_TYPE[$i]=$device_type + + local device=$(cat "$lavacli_output" \ + | grep "device :" \ + | cut -d ":" -f 2 \ + | awk '{$1=$1};1') + DEVICE[$i]=$device + + if [ "${STATUS[$i]}" != "$status" ]; then + STATUS[$i]=$status + + # Something has changed + print_status "Current job status:" + else + STATUS[$i]=$status + fi + fi + done + + if check_if_all_finished; then + break + fi + + if [ -n "$TEST_TIMEOUT" ]; then + # Check timeout + local now=$(date +%s) + if [ "$now" -ge "$end_time" ]; then + echo "Timed out waiting for test jobs to complete" + error=true + break + fi + fi + + # Wait to avoid spamming the server too hard + sleep 60 + done + + if check_if_all_finished; then + # Print job outcome + print_status "Final job status:" true + + if check_for_test_error; then + error=true + fi + fi + fi + + if $error; then + echo "---------------------" + echo "Errors during testing" + echo "---------------------" + print_summary + clean_up + return 1 + fi + + echo "-----------------------------------" + echo "All submitted tests were successful" + echo "-----------------------------------" + print_summary + return 0 +} + +set_up +create_cip_core_jobs + +if ! validate_jobs; then + clean_up + exit 1 +fi + +if ! submit_jobs; then + clean_up + exit 1 +fi + +if ! $SUBMIT_ONLY; then + if ! check_status; then + ERROR=true + fi + + get_junit_test_results +fi + +clean_up + +if $ERROR; then + exit 1 +fi