MDL-32188 question CBM: compute summary stats for CBM behaviours
authorTim Hunt <T.J.Hunt@open.ac.uk>
Mon, 30 Sep 2013 15:26:21 +0000 (16:26 +0100)
committerTim Hunt <T.J.Hunt@open.ac.uk>
Fri, 4 Oct 2013 16:56:36 +0000 (17:56 +0100)
We now compute the average CBM score, accuracy, CBM bonus and enhanced
accuracy, both for the entire quiz, and for just the questions answered.

Note that these calculations must work correctly in the presence of
descriptions, ungraded questions, and manually graded questions. For
example, imagine a essay added at the end of the quiz "Summarise what
you learned attempting this exercise." This might have max mark zero or
non-zero. The CBM statistics just ignores questions like that.

mod/quiz/attemptlib.php
question/behaviour/behaviourbase.php
question/behaviour/behaviourtypebase.php
question/behaviour/deferredcbm/behaviourtype.php
question/behaviour/deferredcbm/lang/en/qbehaviour_deferredcbm.php
question/behaviour/deferredcbm/styles.css
question/behaviour/deferredcbm/tests/behaviourtype_test.php
question/behaviour/immediatecbm/behaviourtype.php
question/engine/questionusage.php

index b4c2c2f..cb05a54 100644 (file)
@@ -759,8 +759,6 @@ class quiz_attempt {
      * The values are arrays with two items, title and content. Each of these
      * will be either a string, or a renderable.
      *
-     * @param question_display_options $options display options. Indicates what types
-     *      of information should, or should not, be returned.
      * @return array as described above.
      */
     public function get_additional_summary_data(question_display_options $options) {
index 0aea965..17aab50 100644 (file)
@@ -656,6 +656,19 @@ abstract class question_cbm {
     );
     /**#@-*/
 
+    /**#@+ @var array upper and lower limits of the optimal window. */
+    protected static $lowlimit = array(
+        self::LOW  => 0,
+        self::MED  => 0.666666666666667,
+        self::HIGH => 0.8,
+    );
+    protected static $highlimit = array(
+        self::LOW  => 0.666666666666667,
+        self::MED  => 0.8,
+        self::HIGH => 1,
+    );
+    /**#@-*/
+
     /**
      * @return int the default certaintly level that should be assuemd if
      * the student does not choose one.
@@ -686,10 +699,39 @@ abstract class question_cbm {
         return get_string('certainty' . $certainty, 'qbehaviour_deferredcbm');
     }
 
+    /**
+     * @param int $certainty one of the LOW/MED/HIGH constants.
+     * @return string a short textual description of this certainty.
+     */
+    public static function get_short_string($certainty) {
+        return get_string('certaintyshort' . $certainty, 'qbehaviour_deferredcbm');
+    }
+
+    /**
+     * Add information about certainty to a response summary.
+     * @param string $summary the response summary.
+     * @param int $certainty the level of certainty to add.
+     */
     public static function summary_with_certainty($summary, $certainty) {
         if (is_null($certainty)) {
             return $summary;
         }
         return $summary . ' [' . self::get_string($certainty) . ']';
     }
+
+    /**
+     * @param int $certainty one of the LOW/MED/HIGH constants.
+     * @return float the lower limit of the optimal probability range for this certainty.
+     */
+    public static function optimal_probablility_low($certainty) {
+        return self::$lowlimit[$certainty];
+    }
+
+    /**
+     * @param int $certainty one of the LOW/MED/HIGH constants.
+     * @return float the upper limit of the optimal probability range for this certainty.
+     */
+    public static function optimal_probablility_high($certainty) {
+        return self::$highlimit[$certainty];
+    }
 }
index f243702..c4684d7 100644 (file)
@@ -86,8 +86,6 @@ abstract class question_behaviour_type {
      * and their unit tests.
      *
      * @param question_usage_by_activity $quba the usage to provide summary data for.
-     * @param question_display_options $options display options. Indicates what types
-     *      of information should, or should not, be returned.
      * @return array as described above.
      */
     public function summarise_usage(question_usage_by_activity $quba,
index d3af9d2..d67f15c 100644 (file)
@@ -38,4 +38,200 @@ class qbehaviour_deferredcbm_type extends qbehaviour_deferredfeedback_type {
     public function adjust_random_guess_score($fraction) {
         return question_cbm::adjust_fraction($fraction, question_cbm::default_certainty());
     }
+
+    public function summarise_usage(question_usage_by_activity $quba, question_display_options $options) {
+        $summarydata = parent::summarise_usage($quba, $options);
+
+        if ($options->marks < question_display_options::MARK_AND_MAX) {
+            return $summarydata;
+        }
+
+        // Prepare accumulators to hold the data we are about to collect.
+        $notansweredcount  = 0;
+        $notansweredweight = 0;
+        $attemptcount = array(
+            question_cbm::HIGH => 0,
+            question_cbm::MED  => 0,
+            question_cbm::LOW  => 0,
+        );
+        $totalweight = array(
+            question_cbm::HIGH => 0,
+            question_cbm::MED  => 0,
+            question_cbm::LOW  => 0,
+        );
+        $totalrawscore = array(
+            question_cbm::HIGH => 0,
+            question_cbm::MED  => 0,
+            question_cbm::LOW  => 0,
+        );
+        $totalcbmscore = array(
+            question_cbm::HIGH => 0,
+            question_cbm::MED  => 0,
+            question_cbm::LOW  => 0,
+        );
+
+        // Loop through the data, and add it to the accumulators.
+        foreach ($quba->get_attempt_iterator() as $qa) {
+            if (strpos($qa->get_behaviour_name(), 'cbm') === false || $qa->get_max_mark() < 0.0000005) {
+                continue;
+            }
+
+            $gradedstep = $qa->get_last_step_with_behaviour_var('_rawfraction');
+
+            if (!$gradedstep->has_behaviour_var('_rawfraction')) {
+                $notansweredcount  += 1;
+                $notansweredweight += $qa->get_max_mark();
+                continue;
+            }
+
+            $certainty = $qa->get_last_behaviour_var('certainty');
+            if (is_null($certainty)) {
+                $certainty = question_cbm::default_certainty();
+            }
+
+            $attemptcount[$certainty]  += 1;
+            $totalweight[$certainty]   += $qa->get_max_mark();
+            $totalrawscore[$certainty] += $qa->get_max_mark() * $gradedstep->get_behaviour_var('_rawfraction');
+            $totalcbmscore[$certainty] += $qa->get_mark();
+        }
+
+        // Hence compute some statistics.
+        $totalquestions   = $notansweredcount + array_sum($attemptcount);
+        $grandtotalweight = $notansweredweight + array_sum($totalweight);
+        $accuracy         = array_sum($totalrawscore) / $grandtotalweight;
+        $averagecbm       = array_sum($totalcbmscore) / $grandtotalweight;
+        $cbmbonus         = $this->calculate_bonus($averagecbm, $accuracy);
+        $accuracyandbonus = $accuracy + $cbmbonus;
+
+        // Now we can start generating some of the summary: overall values.
+        $summarydata['qbehaviour_cbm_entire_quiz_heading'] = array(
+            'title' => '',
+            'content' => html_writer::tag('h3',
+                    get_string('forentirequiz', 'qbehaviour_deferredcbm', $totalquestions),
+                    array('class' => 'qbehaviour_deferredcbm_summary_heading')),
+        );
+        $summarydata['qbehaviour_cbm_entire_quiz_cbm_average'] = array(
+            'title' => get_string('averagecbmmark', 'qbehaviour_deferredcbm'),
+            'content' => format_float($averagecbm, $options->markdp),
+        );
+        $summarydata['qbehaviour_cbm_entire_quiz_accuracy'] = array(
+            'title' => get_string('accuracy', 'qbehaviour_deferredcbm'),
+            'content' => $this->format_probability($accuracy, 1),
+        );
+        $summarydata['qbehaviour_cbm_entire_quiz_cbm_bonus'] = array(
+            'title' => get_string('cbmbonus', 'qbehaviour_deferredcbm'),
+            'content' => $this->format_probability($cbmbonus, 1),
+        );
+        $summarydata['qbehaviour_cbm_entire_quiz_accuracy_and_bonus'] = array(
+            'title' => get_string('accuracyandbonus', 'qbehaviour_deferredcbm'),
+            'content' => $this->format_probability($accuracyandbonus, 1),
+        );
+
+        if ($notansweredcount && array_sum($attemptcount) > 0) {
+            $totalquestions   = array_sum($attemptcount);
+            $grandtotalweight = array_sum($totalweight);
+            $accuracy         = array_sum($totalrawscore) / $grandtotalweight;
+            $averagecbm       = array_sum($totalcbmscore) / $grandtotalweight;
+            $cbmbonus         = $this->calculate_bonus($averagecbm, $accuracy);
+            $accuracyandbonus = $accuracy + $cbmbonus;
+
+            $summarydata['qbehaviour_cbm_answered_quiz_heading'] = array(
+                'title' => '',
+                'content' => html_writer::tag('h3',
+                        get_string('foransweredquestions', 'qbehaviour_deferredcbm', $totalquestions),
+                        array('class' => 'qbehaviour_deferredcbm_summary_heading')),
+            );
+            $summarydata['qbehaviour_cbm_answered_quiz_cbm_average'] = array(
+                'title' => get_string('averagecbmmark', 'qbehaviour_deferredcbm'),
+                'content' => format_float($averagecbm, $options->markdp),
+            );
+            $summarydata['qbehaviour_cbm_answered_quiz_accuracy'] = array(
+                'title' => get_string('accuracy', 'qbehaviour_deferredcbm'),
+                'content' => $this->format_probability($accuracy, 1),
+            );
+            $summarydata['qbehaviour_cbm_answered_quiz_cbm_bonus'] = array(
+                'title' => get_string('cbmbonus', 'qbehaviour_deferredcbm'),
+                'content' => $this->format_probability($cbmbonus, 1),
+            );
+            $summarydata['qbehaviour_cbm_answered_quiz_accuracy_and_bonus'] = array(
+                'title' => get_string('accuracyandbonus', 'qbehaviour_deferredcbm'),
+                'content' => $this->format_probability($accuracyandbonus, 1),
+            );
+        }
+
+        // Now per-certainty level values.
+        $summarydata['qbehaviour_cbm_judgement_heading'] = array(
+            'title' => '',
+            'content' => html_writer::tag('h3', get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
+                    array('class' => 'qbehaviour_deferredcbm_summary_heading')),
+        );
+
+        foreach ($attemptcount as $certainty => $count) {
+            $key   = 'qbehaviour_cbm_judgement' . $certainty;
+            $title = question_cbm::get_short_string($certainty);
+
+            if ($count == 0) {
+                $summarydata[$key] = array(
+                    'title' => $title,
+                    'content' => get_string('noquestions', 'qbehaviour_deferredcbm'),
+                );
+                continue;
+            }
+
+            $lowerlimit = question_cbm::optimal_probablility_low($certainty);
+            $upperlimit = question_cbm::optimal_probablility_high($certainty);
+            $fraction = $totalrawscore[$certainty] / $totalweight[$certainty];
+
+            $a = new stdClass();
+            $a->responses = $count;
+            $a->idealrangelow  = $this->format_probability($lowerlimit);
+            $a->idealrangehigh = $this->format_probability($upperlimit);
+            $a->fraction       = html_writer::tag('span', $this->format_probability($fraction),
+                    array('class' => 'qbehaviour_deferredcbm_actual_percentage'));
+
+            if ($fraction < $lowerlimit - 0.0000005) {
+                if ((pow($fraction - $lowerlimit, 2) * $count) > 0.5) { // Rough indicator of significance: t > 1.5 or 1.8.
+                    $judgement = 'overconfident';
+                } else {
+                    $judgement = 'slightlyoverconfident';
+                }
+            } else if ($fraction > $upperlimit + 0.0000005) {
+                if ((pow($fraction - $upperlimit, 2) * $count) > 0.5) {
+                    $judgement = 'underconfident';
+                } else {
+                    $judgement = 'slightlyunderconfident';
+                }
+            } else {
+                $judgement = 'judgementok';
+            }
+            $a->judgement = html_writer::tag('span', get_string($judgement, 'qbehaviour_deferredcbm'),
+                    array('class' => 'qbehaviour_deferredcbm_' . $judgement));
+
+            $summarydata[$key] = array(
+                'title' => $title,
+                'content' => get_string('judgementsummary', 'qbehaviour_deferredcbm', $a),
+            );
+        }
+
+        return $summarydata;
+    }
+
+    protected function format_probability($probability, $dp = 0) {
+        return format_float($probability * 100, $dp) . '%';
+    }
+
+    public function calculate_bonus($total, $accuracy) {
+        $expectedforaccuracy = max(
+            $accuracy * question_cbm::adjust_fraction(1, question_cbm::LOW) +
+                (1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::LOW),
+            $accuracy * question_cbm::adjust_fraction(1, question_cbm::MED) +
+                (1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::MED),
+            $accuracy * question_cbm::adjust_fraction(1, question_cbm::HIGH) +
+                (1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::HIGH)
+        );
+        // The constant 0.1 here is determinted empirically from looking at lots
+        // for CBM quiz results. See www.ucl.ac.uk/~ucgbarg/tea/IUPS_2013a.pdf.
+        // It approximately maximises the reliability of accuracy + bonus.
+        return 0.1 * ($total - $expectedforaccuracy);
+    }
 }
index a28c25b..586c81c 100644 (file)
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
+$string['accuracy'] = 'Accuracy';
+$string['accuracyandbonus'] = 'Accuracy + Bonus';
 $string['assumingcertainty'] = 'You did not select a certainty. Assuming: {$a}.';
+$string['averagecbmmark'] = 'Average CBM mark';
 $string['basemark'] = 'Base mark {$a}';
+$string['breakdownbycertainty'] = 'Break-down by certainty';
+$string['cbmbonus'] = 'CBM bonus';
 $string['cbmmark'] = 'CBM mark {$a}';
 $string['certainty'] = 'Certainty';
 $string['certainty_help'] = 'Certainty-based marking requires you to indicate how reliable you think your answer is. The available levels are:
@@ -41,5 +46,18 @@ $string['certainty_link'] = 'qbehaviour/deferredcbm/certainty';
 $string['certainty1'] = 'C=1 (Unsure: <67%)';
 $string['certainty2'] = 'C=2 (Mid: >67%)';
 $string['certainty3'] = 'C=3 (Quite sure: >80%)';
-$string['howcertainareyou'] = 'How certain are you? {$a}';
+$string['certaintyshort1'] = 'C=1';
+$string['certaintyshort2'] = 'C=2';
+$string['certaintyshort3'] = 'C=3';
+$string['dontknow'] = 'No idea';
+$string['foransweredquestions'] = 'For just the {$a} answered questions';
+$string['forentirequiz'] = 'For the entire quiz ({$a} questions)';
+$string['judgementok'] = 'OK';
+$string['judgementsummary'] = 'Responses: {$a->responses}. Accuracy: {$a->fraction}. (Optimal range {$a->idealrangelow} to {$a->idealrangehigh}). You were {$a->judgement} using this certainty level.';
+$string['howcertainareyou'] = 'Certainty{$a->help}: {$a->choices}';
+$string['noquestions'] = 'No responses';
+$string['overconfident'] = 'over-confident';
 $string['pluginname'] = 'Deferred feedback with CBM';
+$string['slightlyoverconfident'] = 'a bit over-confident';
+$string['slightlyunderconfident'] = 'a bit under-confident';
+$string['underconfident'] = 'under-confident';
index 99774eb..0e063dc 100644 (file)
@@ -1,3 +1,26 @@
+.qbehaviour_deferredcbm_slightlyunderconfident,
+.qbehaviour_deferredcbm_slightlyoverconfident {
+    font-weight: bold;
+    color: #600;
+}
+.qbehaviour_deferredcbm_underconfident,
+.qbehaviour_deferredcbm_overconfident {
+    font-weight: bold;
+    color: #c00;
+}
+
+.qbehaviour_deferredcbm_judgementok {
+    font-weight: bold;
+    color: #080;
+}
+
+.qbehaviour_deferredcbm_actual_percentage {
+    font-weight: bold;
+}
+
+.qbehaviour_deferredcbm_summary_heading {
+    margin: 0;
+}
 .que.deferredcbm .certaintychoices input[type="radio"] {
     margin-left: 0.5em;
 }
index e060468..d0f44c0 100644 (file)
@@ -61,4 +61,82 @@ class qbehaviour_deferredcbm_type_test extends qbehaviour_walkthrough_test_base
         $this->assertEquals(0, $this->behaviourtype->adjust_random_guess_score(0));
         $this->assertEquals(1, $this->behaviourtype->adjust_random_guess_score(1));
     }
+
+    public function test_summarise_usage_max_mark_1() {
+
+        // Create a usage comprising 3 true-false questions.
+        $this->quba->set_preferred_behaviour('deferredcbm');
+        $this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 3);
+        $this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 3);
+        $this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 3);
+        $this->quba->start_all_questions();
+
+        // Process responses right, high certainty; right, med certainty; wrong, med certainty.
+        $this->quba->process_action(1, array('answer' => 1, '-certainty' => 3));
+        $this->quba->process_action(2, array('answer' => 1, '-certainty' => 2));
+        $this->quba->process_action(3, array('answer' => 0, '-certainty' => 2));
+        $this->quba->finish_all_questions();
+
+        // Get the summary.
+        $summarydata = $this->quba->get_summary_information(new question_display_options());
+
+        // Verify.
+        $this->assertContains(get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement_heading']['content']);
+
+        $this->assertContains('100%',
+                $summarydata['qbehaviour_cbm_judgement3']['content']);
+        $this->assertContains(get_string('judgementok', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement3']['content']);
+
+        $this->assertContains('50%',
+                $summarydata['qbehaviour_cbm_judgement2']['content']);
+        $this->assertContains(get_string('slightlyoverconfident', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement2']['content']);
+
+        $this->assertContains(get_string('noquestions', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement1']['content']);
+    }
+
+    public function test_summarise_usage_max_mark_3() {
+
+        // Create a usage comprising 3 true-false questions.
+        $this->quba->set_preferred_behaviour('deferredcbm');
+        $this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 1);
+        $this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 1);
+        $this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 1);
+        $this->quba->start_all_questions();
+
+        // Process responses right, high certainty; right, med certainty; wrong, med certainty.
+        $this->quba->process_action(1, array('answer' => 1, '-certainty' => 3));
+        $this->quba->process_action(2, array('answer' => 1, '-certainty' => 2));
+        $this->quba->process_action(3, array('answer' => 0, '-certainty' => 2));
+        $this->quba->finish_all_questions();
+
+        // Get the summary.
+        $summarydata = $this->quba->get_summary_information(new question_display_options());
+
+        // Verify.
+        $this->assertContains(get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement_heading']['content']);
+
+        $this->assertContains('100%',
+                $summarydata['qbehaviour_cbm_judgement3']['content']);
+        $this->assertContains(get_string('judgementok', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement3']['content']);
+
+        $this->assertContains('50%',
+                $summarydata['qbehaviour_cbm_judgement2']['content']);
+        $this->assertContains(get_string('slightlyoverconfident', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement2']['content']);
+
+        $this->assertContains(get_string('noquestions', 'qbehaviour_deferredcbm'),
+                $summarydata['qbehaviour_cbm_judgement1']['content']);
+    }
+
+    public function test_calculate_bonus() {
+        $this->assertEquals(0.05,  $this->behaviourtype->calculate_bonus(1, 1/2));
+        $this->assertEquals(-0.01, $this->behaviourtype->calculate_bonus(2, 9/10));
+        $this->assertEquals(0,     $this->behaviourtype->calculate_bonus(3, 1));
+    }
 }
index 2a671e0..01a2ec3 100644 (file)
@@ -25,7 +25,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-require_once(dirname(__FILE__) . '/../immediatefeedback/behaviourtype.php');
+require_once(dirname(__FILE__) . '/../deferredcbm/behaviourtype.php');
 
 
 /**
@@ -34,8 +34,9 @@ require_once(dirname(__FILE__) . '/../immediatefeedback/behaviourtype.php');
  * @copyright  2012 The Open University
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
-class qbehaviour_immediatecbm_type extends qbehaviour_immediatefeedback_type {
-    public function adjust_random_guess_score($fraction) {
-        return question_cbm::adjust_fraction($fraction, question_cbm::default_certainty());
+class qbehaviour_immediatecbm_type extends qbehaviour_deferredcbm_type {
+
+    public function get_unused_display_options() {
+        return array();
     }
 }
index b478f67..b4a1a29 100644 (file)
@@ -340,8 +340,6 @@ class question_usage_by_activity {
      * The values are arrays with two items, title and content. Each of these
      * will be either a string, or a renderable.
      *
-     * @param question_display_options $options display options. Indicates what types
-     *      of information should, or should not, be returned.
      * @return array as described above.
      */
     public function get_summary_information(question_display_options $options) {