57b7dee343a023e4d44d36c2c7031942ce2a33b1
[moodle.git] / mod / quiz / report / statistics / tests / stats_from_steps_walkthrough_test.php
1 <?php
2 // This file is part of Moodle - http://moodle.org/
3 //
4 // Moodle is free software: you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation, either version 3 of the License, or
7 // (at your option) any later version.
8 //
9 // Moodle is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
17 /**
18  * Quiz attempt walk through using data from csv file.
19  *
20  * @package    quiz_statistics
21  * @category   phpunit
22  * @copyright  2013 The Open University
23  * @author     Jamie Pratt <me@jamiep.org>
24  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
25  */
27 defined('MOODLE_INTERNAL') || die();
29 global $CFG;
30 require_once($CFG->dirroot . '/mod/quiz/tests/attempt_walkthrough_from_csv_test.php');
31 require_once($CFG->dirroot . '/mod/quiz/report/default.php');
32 require_once($CFG->dirroot . '/mod/quiz/report/statistics/report.php');
33 require_once($CFG->dirroot . '/mod/quiz/report/reportlib.php');
35 /**
36  * Quiz attempt walk through using data from csv file.
37  *
38  * @package    quiz_statistics
39  * @category   phpunit
40  * @copyright  2013 The Open University
41  * @author     Jamie Pratt <me@jamiep.org>
42  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
43  */
44 class quiz_report_statistics_from_steps_testcase extends mod_quiz_attempt_walkthrough_from_csv_testcase {
46     /**
47      * @var quiz_statistics_report object to do stats calculations.
48      */
49     protected $report;
51     protected function get_full_path_of_csv_file($setname, $test) {
52         // Overridden here so that __DIR__ points to the path of this file.
53         return  __DIR__."/fixtures/{$setname}{$test}.csv";
54     }
56     protected $files = array('questions', 'steps', 'results', 'qstats', 'responsecounts');
58     /**
59      * Create a quiz add questions to it, walk through quiz attempts and then check results.
60      *
61      * @param PHPUnit_Extensions_Database_DataSet_ITable[] of data read from csv file "questionsXX.csv",
62      *                                                                                  "stepsXX.csv" and "resultsXX.csv".
63      * @dataProvider get_data_for_walkthrough
64      */
65     public function test_walkthrough_from_csv($quizsettings, $csvdata) {
67         // CSV data files for these tests were generated using :
68         // https://github.com/jamiepratt/moodle-quiz-tools/tree/master/responsegenerator
70         $this->resetAfterTest(true);
71         question_bank::get_qtype('random')->clear_caches_before_testing();
73         $this->create_quiz($quizsettings, $csvdata['questions']);
75         $attemptids = $this->walkthrough_attempts($csvdata['steps']);
77         $this->check_attempts_results($csvdata['results'], $attemptids);
79         $this->report = new quiz_statistics_report();
80         $whichattempts = QUIZ_GRADEAVERAGE;
81         $groupstudents = array();
82         $questions = $this->report->load_and_initialise_questions_for_calculations($this->quiz);
83         list($quizstats, $questionstats) =
84                         $this->report->get_all_stats_and_analysis($this->quiz, $whichattempts, $groupstudents, $questions);
86         $qubaids = quiz_statistics_qubaids_condition($this->quiz->id, $groupstudents, $whichattempts);
88         // We will create some quiz and question stat calculator instances and some response analyser instances, just in order
89         // to check the last analysed time then returned.
90         $quizcalc = new \quiz_statistics\calculator();
91         // Should not be a delay of more than one second between the calculation of stats above and here.
92         $this->assertTimeCurrent($quizcalc->get_last_calculated_time($qubaids));
94         $qcalc = new \core_question\statistics\questions\calculator($questions);
95         $this->assertTimeCurrent($qcalc->get_last_calculated_time($qubaids));
97         $expectedvariantcounts = array(2 => array(1 => 6,
98                                                     4 => 4,
99                                                     5 => 3,
100                                                     6 => 4,
101                                                     7 => 2,
102                                                     8 => 5,
103                                                     10 => 1));
105         foreach ($questions as $slot => $question) {
106             if (!question_bank::get_qtype($question->qtype, false)->can_analyse_responses()) {
107                 continue;
108             }
109             $responesstats = new \core_question\statistics\responses\analyser($question);
110             $this->assertTimeCurrent($responesstats->get_last_analysed_time($qubaids));
111             $analysis = $responesstats->load_cached($qubaids);
112             $variantsnos = $analysis->get_variant_nos();
113             if (isset($expectedvariantcounts[$slot])) {
114                 // Compare contents, ignore ordering of array, using canonicalize parameter of assertEquals.
115                 $this->assertEquals(array_keys($expectedvariantcounts[$slot]), $variantsnos, '', 0, 10, true);
116             } else {
117                 $this->assertEquals(array(1), $variantsnos);
118             }
119             $totalspervariantno = array();
120             foreach ($variantsnos as $variantno) {
122                 $subpartids = $analysis->get_subpart_ids($variantno);
123                 foreach ($subpartids as $subpartid) {
124                     if (!isset($totalspervariantno[$subpartid])) {
125                         $totalspervariantno[$subpartid] = array();
126                     }
127                     $totalspervariantno[$subpartid][$variantno] = 0;
129                     $subpartanalysis = $analysis->get_analysis_for_subpart($variantno, $subpartid);
130                     $classids = $subpartanalysis->get_response_class_ids();
131                     foreach ($classids as $classid) {
132                         $classanalysis = $subpartanalysis->get_response_class($classid);
133                         $actualresponsecounts = $classanalysis->data_for_question_response_table('', '');
134                         foreach ($actualresponsecounts as $actualresponsecount) {
135                             $totalspervariantno[$subpartid][$variantno] += $actualresponsecount->count;
136                         }
137                     }
138                 }
139             }
140             // Count all counted responses for each part of question and confirm that counted responses, for most question types
141             // are the number of attempts at the question for each question part.
142             if ($slot != 5) {
143                 // Slot 5 holds a multi-choice multiple question.
144                 // Multi-choice multiple is slightly strange. Actual answer counts given for each sub part do not add up to the
145                 // total attempt count.
146                 // This is because each option is counted as a sub part and each option can be off or on in each attempt. Off is
147                 // not counted in response analysis for this question type.
148                 foreach ($totalspervariantno as $totalpervariantno) {
149                     if (isset($expectedvariantcounts[$slot])) {
150                         // If we know how many attempts there are at each variant we can check
151                         // that we have counted the correct amount of responses for each variant.
152                         $this->assertEquals($expectedvariantcounts[$slot],
153                                             $totalpervariantno,
154                                             "Totals responses do not add up in response analysis for slot {$slot}.",
155                                             0,
156                                             10,
157                                             true);
158                     } else {
159                         $this->assertEquals(25,
160                                             array_sum($totalpervariantno),
161                                             "Totals responses do not add up in response analysis for slot {$slot}.");
162                     }
163                 }
164             }
165         }
166         for ($rowno = 0; $rowno < $csvdata['responsecounts']->getRowCount(); $rowno++) {
167             $responsecount = $csvdata['responsecounts']->getRow($rowno);
168             if ($responsecount['randq'] == '') {
169                 $question = $questions[$responsecount['slot']];
170             } else {
171                 $qid = $this->randqids[$responsecount['slot']][$responsecount['randq']];
172                 $question = question_finder::get_instance()->load_question_data($qid);
173             }
174             $this->assert_response_count_equals($question, $qubaids, $responsecount);
175         }
177         // These quiz stats and the question stats found in qstats00.csv were calculated independently in spreadsheet which is
178         // available in open document or excel format here :
179         // https://github.com/jamiepratt/moodle-quiz-tools/tree/master/statsspreadsheet
180         $quizstatsexpected = array(
181             'median' => 4.5,
182             'firstattemptsavg' => 4.617333332,
183             'allattemptsavg' => 4.617333332,
184             'firstattemptscount' => 25,
185             'allattemptscount' => 25,
186             'standarddeviation' => 0.8117265554,
187             'skewness' => -0.092502502,
188             'kurtosis' => -0.7073968557,
189             'cic' => -87.2230935542,
190             'errorratio' => 136.8294900795,
191             'standarderror' => 1.1106813066
192         );
194         foreach ($quizstatsexpected as $statname => $statvalue) {
195             $this->assertEquals($statvalue, $quizstats->$statname, $quizstats->$statname, abs($statvalue) * 1.5e-5);
196         }
198         for ($rowno = 0; $rowno < $csvdata['qstats']->getRowCount(); $rowno++) {
199             $slotqstats = $csvdata['qstats']->getRow($rowno);
200             foreach ($slotqstats as $statname => $slotqstat) {
201                 if ($statname !== 'slot') {
202                     $this->assert_stat_equals($questionstats, $slotqstats['slot'], null, null, $statname, (float)$slotqstat);
203                 }
204             }
205         }
207         $itemstats = array('s' => 12,
208                           'effectiveweight' => null,
209                           'discriminationindex' => 35.803933,
210                           'discriminativeefficiency' => 39.39393939,
211                           'sd' => 0.514928651,
212                           'facility' => 0.583333333,
213                           'maxmark' => 1,
214                           'positions' => '1',
215                           'slot' => null,
216                           'subquestion' => true);
217         foreach ($itemstats as $statname => $expected) {
218             $this->assert_stat_equals($questionstats, 1, null, 'numerical', $statname, $expected);
219         }
221         // These variant's stats are calculated in stats_for_variant_1.xls and stats_for_variant_8.xls
222         // The calculations in the spreadsheets are the same but applied just to the attempts where the variants appeared.
224         $statsforslot2variants = array(1 => array('s' => 6,
225                                                     'effectiveweight' => null,
226                                                     'discriminationindex' => -10.5999788,
227                                                     'discriminativeefficiency' => -14.28571429,
228                                                     'sd' => 0.5477225575,
229                                                     'facility' => 0.50,
230                                                     'maxmark' => 1,
231                                                     'variant' => 1,
232                                                     'slot' => 2,
233                                                     'subquestion' => false),
234                                       8 => array('s' => 5,
235                                                     'effectiveweight' => null,
236                                                     'discriminationindex' => -57.77466679,
237                                                     'discriminativeefficiency' => -71.05263241,
238                                                     'sd' => 0.547722558,
239                                                     'facility' => 0.40,
240                                                     'maxmark' => 1,
241                                                     'variant' => 8,
242                                                     'slot' => 2,
243                                                     'subquestion' => false));
244         foreach ($statsforslot2variants as $variant => $stats) {
245             foreach ($stats as $statname => $expected) {
246                 $this->assert_stat_equals($questionstats, 2, $variant, null, $statname, $expected);
247             }
248         }
249         foreach ($expectedvariantcounts as $slot => $expectedvariantcount) {
250             foreach ($expectedvariantcount as $variantno => $s) {
251                 $this->assertEquals($s, $questionstats->for_slot($slot, $variantno)->s);
252             }
253         }
254     }
256     /**
257      * Check that the stat is as expected within a reasonable tolerance.
258      *
259      * @param \core_question\statistics\questions\all_calculated_for_qubaid_condition $questionstats
260      * @param int                                              $slot
261      * @param int|null                                         $variant if null then not a variant stat.
262      * @param string|null                                      $subqname if null then not an item stat.
263      * @param string                                           $statname
264      * @param float                                            $expected
265      */
266     protected function assert_stat_equals($questionstats, $slot, $variant, $subqname, $statname, $expected) {
268         if ($variant === null && $subqname === null) {
269             $actual = $questionstats->for_slot($slot)->{$statname};
270         } else if ($subqname !== null) {
271             $actual = $questionstats->for_subq($this->randqids[$slot][$subqname])->{$statname};
272         } else {
273             $actual = $questionstats->for_slot($slot, $variant)->{$statname};
274         }
275         if (is_bool($expected) || is_string($expected)) {
276             $this->assertEquals($expected, $actual, "$statname for slot $slot");
277         } else {
278             switch ($statname) {
279                 case 'covariance' :
280                 case 'discriminationindex' :
281                 case 'discriminativeefficiency' :
282                 case 'effectiveweight' :
283                     $precision = 1e-5;
284                     break;
285                 default :
286                     $precision = 1e-6;
287             }
288             $delta = abs($expected) * $precision;
289             $this->assertEquals(floatval($expected), $actual, "$statname for slot $slot", $delta);
290         }
291     }
293     protected function assert_response_count_equals($question, $qubaids, $responsecount) {
294         $responesstats = new \core_question\statistics\responses\analyser($question);
295         $analysis = $responesstats->load_cached($qubaids);
296         if (!isset($responsecount['subpart'])) {
297             $subpart = 1;
298         } else {
299             $subpart = $responsecount['subpart'];
300         }
301         list($subpartid, $responseclassid) = $this->get_response_subpart_and_class_id($question,
302                                                                                       $subpart,
303                                                                                       $responsecount['modelresponse']);
305         $subpartanalysis = $analysis->get_analysis_for_subpart($responsecount['variant'], $subpartid);
306         $responseclassanalysis = $subpartanalysis->get_response_class($responseclassid);
307         $actualresponsecounts = $responseclassanalysis->data_for_question_response_table('', '');
308         if ($responsecount['modelresponse'] !== '[NO RESPONSE]') {
309             foreach ($actualresponsecounts as $actualresponsecount) {
310                 if ($actualresponsecount->response == $responsecount['actualresponse']) {
311                     $this->assertEquals($responsecount['count'], $actualresponsecount->count);
312                     return;
313                 }
314             }
315             throw new coding_exception("Actual response '{$responsecount['actualresponse']}' not found.");
316         } else {
317             $actualresponsecount = array_pop($actualresponsecounts);
318             $this->assertEquals($responsecount['count'], $actualresponsecount->count);
319         }
320     }
322     protected function get_response_subpart_and_class_id($question, $subpart, $modelresponse) {
323         $qtypeobj = question_bank::get_qtype($question->qtype, false);
324         $possibleresponses = $qtypeobj->get_possible_responses($question);
325         $possibleresponsesubpartids = array_keys($possibleresponses);
326         if (!isset($possibleresponsesubpartids[$subpart - 1])) {
327             throw new coding_exception("Subpart '{$subpart}' not found.");
328         }
329         $subpartid = $possibleresponsesubpartids[$subpart - 1];
331         if ($modelresponse == '[NO RESPONSE]') {
332             return array($subpartid, null);
334         } else if ($modelresponse == '[NO MATCH]') {
335             return array($subpartid, 0);
336         }
338         $modelresponses = array();
339         foreach ($possibleresponses[$subpartid] as $responseclassid => $subpartpossibleresponse) {
340             $modelresponses[$responseclassid] = $subpartpossibleresponse->responseclass;
341         }
342         $this->assertContains($modelresponse, $modelresponses);
343         $responseclassid = array_search($modelresponse, $modelresponses);
344         return array($subpartid, $responseclassid);
345     }