Tweak tests, consolidate pins target validation (#21254)

buildroot/bin/mftest

@@ -21,7 +21,7 @@ Usage: mftest [-t|--env=<env>] [-n|--num=<num>] [-m|--make] [-y|--build=<Y|n>]
 
 OPTIONS
   -t --env         The environment of the test to apply / run. (As named in platformio.ini.)
-  -n --num         The index of the test to run. (In *-tests file order.)
+  -n --num         The index of the test to run. (In file order.)
   -m --make        Use the make / Docker method for the build.
   -y --build       Skip 'Do you want to build this test?' and assume YES.
   -h --help        Print this help.
@@ -204,11 +204,10 @@ fi
 if [[ $TESTENV == '-' ]]; then
   IND=0
   NAMES=()
-  for FILE in $( ls -1 $TESTPATH/*-tests )
+  for FILE in $( ls -1 $TESTPATH/* )
   do
     let IND++
-    TNAME=${FILE/-tests/}
-    TNAME=${TNAME/$TESTPATH\//}
+    TNAME=${FILE/$TESTPATH\//}
     NAMES+=($TNAME)
     (( IND < 10 )) && echo -n " "
     echo " $IND) $TNAME"
@@ -231,7 +230,7 @@ if [[ $TESTENV == '-' ]]; then
 fi
 
 # Get the contents of the test file
-OUT=$( cat $TESTPATH/$TESTENV-tests 2>/dev/null ) || { errout "Can't find test '$TESTENV'." ; exit 1 ; }
+OUT=$( cat $TESTPATH/$TESTENV 2>/dev/null ) || { errout "Can't find test '$TESTENV'." ; exit 1 ; }
 
 # Count up the number of tests
 TESTCOUNT=$( awk "/$ISEXEC/{a++}END{print a}" <<<"$OUT" )
@@ -297,7 +296,7 @@ echo "$OUT" | {
 echo -ne "\033[0m"
 
 # Make clear it's a TEST
-opt_set CUSTOM_MACHINE_NAME "\"$TESTENV-tests ($CHOICE)\""
+opt_set CUSTOM_MACHINE_NAME "\"Test $TESTENV ($CHOICE)\""
 
 # Build the test too?
 if [[ -z "$BUILD_YES" ]]; then

buildroot/bin/run_tests

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+#
+# run_tests
+#
+HERE="$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )"
+TESTS="$HERE/../tests"
+export PATH="$HERE:$TESTS:$PATH"
+
+# exit on first failure
+set -e
+
+exec_test () {
+  printf "\n\033[0;32m[Test $2] \033[0m$3...\n"
+  # Check to see if we should skip tests
+  if [[ -n "$4" ]] ; then
+    if [[ ! "$3" =~ $4 ]] ; then
+      printf "\033[1;33mSkipped\033[0m\n"
+      return 0
+    fi
+  fi
+  if [[ -z "$VERBOSE_PLATFORMIO" ]] ; then
+    silent="--silent"
+  else
+    silent="-v"
+  fi
+  if platformio run --project-dir $1 -e $2 $silent; then
+    printf "\033[0;32mPassed\033[0m\n"
+    return 0
+  else
+    if [[ -n $GIT_RESET_HARD ]]; then
+      git reset --hard HEAD
+    else
+      restore_configs
+    fi
+    printf "\033[0;31mFailed!\033[0m\n"
+    return 1
+  fi
+}
+export -f exec_test
+
+printf "Running \033[0;32m$2\033[0m Tests\n"
+
+if [[ $2 = "ALL" ]]; then
+  tests=("$TESTS"/*)
+  for f in "${tests[@]}"; do
+    testenv=$(basename $f)
+    printf "Running \033[0;32m$f\033[0m Tests\n"
+    exec_test $1 "$testenv --target clean" "Setup Build Environment"
+    if [[ $GIT_RESET_HARD == "true" ]]; then
+      git reset --hard HEAD
+    else
+      restore_configs
+    fi
+  done
+else
+  exec_test $1 "$2 --target clean" "Setup Build Environment"
+  test_name="$3"
+  # If the test name is 1 or 2 digits, treat it as an index
+  if [[ "$test_name" =~ ^[0-9][0-9]?$ ]] ; then
+    # Find the test name that corresponds to that index
+    test_name="$(cat $TESTS/$2 | grep -e '^exec_test' | sed -n "$3p" | sed "s/.*\$1 \$2 \"\([^\"]*\).*/\1/g")"
+    if [[ -z "$test_name" ]] ; then
+      # Fail if none matches
+      printf "\033[0;31mCould not find test \033[0m#$3\033[0;31m in \033[0mbuildroot/tests/$2\n"
+      exit 1
+    else
+      printf "\033[0;32mMatching test \033[0m#$3\033[0;32m: '\033[0m$test_name\033[0;32m'\n"
+    fi
+  fi
+  $TESTS/$2 $1 $2 "$test_name"
+  if [[ $GIT_RESET_HARD == "true" ]]; then
+    git reset --hard HEAD
+  else
+    restore_configs
+  fi
+fi
+printf "\033[0;32mAll tests completed successfully\033[0m\n"