A completely empty test case will always fail. This is an exception to...

A completely empty test case will always fail.  This is an exception to prevent authors subverting the bulk test check they are using test cases!

cli/bulktestall.php

@@ -108,15 +108,16 @@ foreach ($contexts as $contextid => $numstackquestions) {
         list($passed, $failing) = $bulktester->run_all_tests_for_context($testcontext, 'cli', false);
     }
 
+    $allpassed = $allpassed && $passed;
+
     echo "\n";
-    if ($allpassed) {
+    if ($passed) {
         echo "** " . stack_string('stackInstall_testsuite_pass');
     } else {
         echo "** " . stack_string('stackInstall_testsuite_fail');
     }
     echo "\n";
 
-    $allpassed = $allpassed && $passed;
     echo "\n";
     foreach ($failing as $key => $arrvals) {
         if ($arrvals !== array()) {
@@ -128,6 +129,13 @@ foreach ($contexts as $contextid => $numstackquestions) {
 
 echo "\n\n";
 
+if ($allpassed) {
+    echo "** " . stack_string('stackInstall_testsuite_pass');
+} else {
+    echo "** " . stack_string('stackInstall_testsuite_fail');
+}
+echo "\n";
+
 $took = (microtime(true) - $start);
 $rtook = round($took, 5);
 

lang/en/qtype_stack.php

@@ -517,6 +517,7 @@ $string['prtname'] = 'PRT name';
 $string['questiondoesnotuserandomisation'] = 'This question does not use randomisation.';
 $string['questionnotdeployedyet'] = 'No variants of this question have been deployed yet.';
 $string['questionpreview'] = 'Question preview';
+$string['questiontestempty'] = 'Empty question tests are not permitted!';
 $string['questiontests'] = 'Question tests';
 $string['questiontestsfor'] = 'Question tests for seed {$a}';
 $string['questiontestspass'] = 'All question tests passed.';

questiontestrun.php

@@ -342,6 +342,9 @@ foreach ($testresults as $key => $result) {
     echo $OUTPUT->heading(stack_string('testcasexresult',
             array('no' => $key, 'result' => $outcome)), 3);
 
+    if ($result->emptytestcase) {
+        echo html_writer::tag('p', stack_string_error('questiontestempty'));
+    }
     // Display the information about the inputs.
     $inputstable = new html_table();
     $inputstable->head = array(

stack/questiontest.php

@@ -69,6 +69,10 @@ class stack_question_test {
      */
     public function test_question($questionid, $seed, $context) {
 
+        // We don't permit completely empty test cases.
+        // Completely empty test cases always pass, which is spurious in the bulk test.
+        $emptytestcase = true;
+
         // Create a completely clean version of the question usage we will use.
         // Evaluated state is stored in question variables etc.
         $question = question_bank::load_question($questionid);
@@ -99,12 +103,18 @@ class stack_question_test {
                 } else if (array_key_exists($inputname.'_val', $response)) {
                     $inputresponse = $response[$inputname.'_val'];
                 }
+                if ($inputresponse != '') {
+                    $emptytestcase = false;
+                }
                 $results->set_input_state($inputname, $inputresponse, $inputstate->contentsmodified,
                     $inputstate->contentsdisplayed, $inputstate->status, $inputstate->errors);
             }
         }
 
         foreach ($this->expectedresults as $prtname => $expectedresult) {
+            if (implode(' | ', $expectedresult->answernotes) !== 'NULL') {
+                $emptytestcase = false;
+            }
             $result = $question->get_prt_result($prtname, $response, false);
             // Adapted from renderer.php prt_feedback_display.
             $feedback = '';
@@ -138,6 +148,8 @@ class stack_question_test {
 
         }
 
+        $results->emptytestcase = $emptytestcase;
+
         if ($this->testcase) {
             $this->save_result($question, $results);
         }

stack/questiontestresult.php

@@ -67,6 +67,11 @@ class stack_question_test_result {
      */
     public $questionpenalty;
 
+    /**
+     * @bool Store whether this looks like a trivial empty test case.
+     */
+    public $emptytestcase;
+
     /**
      * Constructor
      * @param stack_question_test $testcase the testcase this is the results for.
@@ -204,6 +209,9 @@ class stack_question_test_result {
      * @return bool whether the test passed successfully.
      */
     public function passed() {
+        if ($this->emptytestcase) {
+            return false;
+        }
         foreach ($this->get_prt_states() as $state) {
             if (!$state->testoutcome) {
                 return false;