MDL-67673 phpunit: Move tests to use new phpunit_dataset
[moodle.git] / mod / quiz / report / statistics / tests / stats_from_steps_walkthrough_test.php
CommitLineData
3652dddd
JP
1<?php
2// This file is part of Moodle - http://moodle.org/
3//
4// Moodle is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// Moodle is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16
17/**
18 * Quiz attempt walk through using data from csv file.
19 *
038014c4
JP
20 * The quiz stats below and the question stats found in qstats00.csv were calculated independently in a spreadsheet which is
21 * available in open document or excel format here :
22 * https://github.com/jamiepratt/moodle-quiz-tools/tree/master/statsspreadsheet
23 *
24 * Similarly the question variant's stats in qstats00.csv are calculated in stats_for_variant_1.xls and stats_for_variant_8.xls
25 * The calculations in the spreadsheets are the same as for the other question stats but applied just to the attempts where the
26 * variants appeared.
27 *
e68e4ccf 28 * @package quiz_statistics
3652dddd
JP
29 * @category phpunit
30 * @copyright 2013 The Open University
31 * @author Jamie Pratt <me@jamiep.org>
32 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
33 */
34
35defined('MOODLE_INTERNAL') || die();
36
37global $CFG;
38require_once($CFG->dirroot . '/mod/quiz/tests/attempt_walkthrough_from_csv_test.php');
39require_once($CFG->dirroot . '/mod/quiz/report/default.php');
40require_once($CFG->dirroot . '/mod/quiz/report/statistics/report.php');
41require_once($CFG->dirroot . '/mod/quiz/report/reportlib.php');
3652dddd 42
3652dddd
JP
43/**
44 * Quiz attempt walk through using data from csv file.
45 *
e68e4ccf 46 * @package quiz_statistics
3652dddd
JP
47 * @category phpunit
48 * @copyright 2013 The Open University
49 * @author Jamie Pratt <me@jamiep.org>
50 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
51 */
8e328617 52class quiz_report_statistics_from_steps_testcase extends mod_quiz_attempt_walkthrough_from_csv_testcase {
3652dddd
JP
53
54 /**
55 * @var quiz_statistics_report object to do stats calculations.
56 */
57 protected $report;
58
59 protected function get_full_path_of_csv_file($setname, $test) {
60 // Overridden here so that __DIR__ points to the path of this file.
61 return __DIR__."/fixtures/{$setname}{$test}.csv";
62 }
63
4922e79f 64 protected $files = array('questions', 'steps', 'results', 'qstats', 'responsecounts');
3652dddd
JP
65
66 /**
67 * Create a quiz add questions to it, walk through quiz attempts and then check results.
68 *
8183def6 69 * @param array $csvdata data read from csv file "questionsXX.csv", "stepsXX.csv" and "resultsXX.csv".
3652dddd
JP
70 * @dataProvider get_data_for_walkthrough
71 */
764f6153 72 public function test_walkthrough_from_csv($quizsettings, $csvdata) {
3652dddd 73
ea6db479 74 $this->create_quiz_simulate_attempts_and_check_results($quizsettings, $csvdata);
3652dddd 75
038014c4
JP
76 $whichattempts = QUIZ_GRADEAVERAGE; // All attempts.
77 $whichtries = question_attempt::ALL_TRIES;
10c4fce5 78 $groupstudentsjoins = new \core\dml\sql_join();
ea6db479 79 list($questions, $quizstats, $questionstats, $qubaids) =
10c4fce5
JB
80 $this->check_stats_calculations_and_response_analysis($csvdata,
81 $whichattempts, $whichtries, $groupstudentsjoins);
038014c4
JP
82 if ($quizsettings['testnumber'] === '00') {
83 $this->check_variants_count_for_quiz_00($questions, $questionstats, $whichtries, $qubaids);
84 $this->check_quiz_stats_for_quiz_00($quizstats);
3652dddd 85 }
038014c4 86 }
3652dddd 87
038014c4
JP
88 /**
89 * Check actual question stats are the same as that found in csv file.
90 *
8183def6 91 * @param $qstats array data from csv file.
038014c4
JP
92 * @param $questionstats \core_question\statistics\questions\all_calculated_for_qubaid_condition Calculated stats.
93 */
94 protected function check_question_stats($qstats, $questionstats) {
8183def6 95 foreach ($qstats as $slotqstats) {
3652dddd 96 foreach ($slotqstats as $statname => $slotqstat) {
038014c4
JP
97 if (!in_array($statname, array('slot', 'subqname')) && $slotqstat !== '') {
98 $this->assert_stat_equals($slotqstat,
99 $questionstats,
100 $slotqstats['slot'],
101 $slotqstats['subqname'],
102 $slotqstats['variant'],
103 $statname);
3652dddd
JP
104 }
105 }
038014c4
JP
106 // Check that sub-question boolean field is correctly set.
107 $this->assert_stat_equals(!empty($slotqstats['subqname']),
108 $questionstats,
109 $slotqstats['slot'],
110 $slotqstats['subqname'],
111 $slotqstats['variant'],
112 'subquestion');
3d6f2466
JP
113 }
114 }
115
116 /**
117 * Check that the stat is as expected within a reasonable tolerance.
118 *
038014c4 119 * @param float|string|bool $expected expected value of stat.
3d6f2466 120 * @param \core_question\statistics\questions\all_calculated_for_qubaid_condition $questionstats
038014c4
JP
121 * @param int $slot
122 * @param string $subqname if empty string then not an item stat.
123 * @param int|string $variant if empty string then not a variantstat.
124 * @param string $statname
3d6f2466 125 */
038014c4 126 protected function assert_stat_equals($expected, $questionstats, $slot, $subqname, $variant, $statname) {
3d6f2466 127
038014c4 128 if ($variant === '' && $subqname === '') {
3d6f2466 129 $actual = $questionstats->for_slot($slot)->{$statname};
038014c4 130 } else if ($subqname !== '') {
3d6f2466
JP
131 $actual = $questionstats->for_subq($this->randqids[$slot][$subqname])->{$statname};
132 } else {
133 $actual = $questionstats->for_slot($slot, $variant)->{$statname};
134 }
038014c4
JP
135 $message = "$statname for slot $slot";
136 if ($expected === '**NULL**') {
137 $this->assertEquals(null, $actual, $message);
138 } else if (is_bool($expected)) {
139 $this->assertEquals($expected, $actual, $message);
140 } else if (is_numeric($expected)) {
3d6f2466
JP
141 switch ($statname) {
142 case 'covariance' :
143 case 'discriminationindex' :
144 case 'discriminativeefficiency' :
145 case 'effectiveweight' :
146 $precision = 1e-5;
147 break;
148 default :
149 $precision = 1e-6;
150 }
151 $delta = abs($expected) * $precision;
3a5641cb 152 $this->assertEqualsWithDelta((float)$expected, $actual, $delta, $message);
038014c4
JP
153 } else {
154 $this->assertEquals($expected, $actual, $message);
3d6f2466 155 }
4922e79f
JP
156 }
157
038014c4 158 protected function assert_response_count_equals($question, $qubaids, $expected, $whichtries) {
4922e79f 159 $responesstats = new \core_question\statistics\responses\analyser($question);
038014c4
JP
160 $analysis = $responesstats->load_cached($qubaids, $whichtries);
161 if (!isset($expected['subpart'])) {
4922e79f
JP
162 $subpart = 1;
163 } else {
038014c4 164 $subpart = $expected['subpart'];
4922e79f
JP
165 }
166 list($subpartid, $responseclassid) = $this->get_response_subpart_and_class_id($question,
167 $subpart,
038014c4 168 $expected['modelresponse']);
4922e79f 169
038014c4 170 $subpartanalysis = $analysis->get_analysis_for_subpart($expected['variant'], $subpartid);
4922e79f
JP
171 $responseclassanalysis = $subpartanalysis->get_response_class($responseclassid);
172 $actualresponsecounts = $responseclassanalysis->data_for_question_response_table('', '');
038014c4
JP
173
174 foreach ($actualresponsecounts as $actualresponsecount) {
58794ac9 175 if ($actualresponsecount->response == $expected['actualresponse'] || count($actualresponsecounts) == 1) {
038014c4
JP
176 $i = 1;
177 $partofanalysis = " slot {$expected['slot']}, rand q '{$expected['randq']}', variant {$expected['variant']}, ".
178 "for expected model response {$expected['modelresponse']}, ".
179 "actual response {$expected['actualresponse']}";
180 while (isset($expected['count'.$i])) {
181 if ($expected['count'.$i] != 0) {
182 $this->assertTrue(isset($actualresponsecount->trycount[$i]),
183 "There is no count at all for try $i on ".$partofanalysis);
184 $this->assertEquals($expected['count'.$i], $actualresponsecount->trycount[$i],
185 "Count for try $i on ".$partofanalysis);
186 }
187 $i++;
4922e79f 188 }
038014c4
JP
189 if (isset($expected['totalcount'])) {
190 $this->assertEquals($expected['totalcount'], $actualresponsecount->totalcount,
191 "Total count on ".$partofanalysis);
192 }
193 return;
4922e79f 194 }
4922e79f 195 }
038014c4 196 throw new coding_exception("Expected response '{$expected['actualresponse']}' not found.");
4922e79f
JP
197 }
198
199 protected function get_response_subpart_and_class_id($question, $subpart, $modelresponse) {
200 $qtypeobj = question_bank::get_qtype($question->qtype, false);
201 $possibleresponses = $qtypeobj->get_possible_responses($question);
202 $possibleresponsesubpartids = array_keys($possibleresponses);
203 if (!isset($possibleresponsesubpartids[$subpart - 1])) {
204 throw new coding_exception("Subpart '{$subpart}' not found.");
205 }
206 $subpartid = $possibleresponsesubpartids[$subpart - 1];
207
208 if ($modelresponse == '[NO RESPONSE]') {
209 return array($subpartid, null);
210
211 } else if ($modelresponse == '[NO MATCH]') {
212 return array($subpartid, 0);
213 }
214
215 $modelresponses = array();
216 foreach ($possibleresponses[$subpartid] as $responseclassid => $subpartpossibleresponse) {
217 $modelresponses[$responseclassid] = $subpartpossibleresponse->responseclass;
4f7516f6 218 }
4922e79f
JP
219 $this->assertContains($modelresponse, $modelresponses);
220 $responseclassid = array_search($modelresponse, $modelresponses);
221 return array($subpartid, $responseclassid);
8e328617
JP
222 }
223
038014c4
JP
224 /**
225 * @param $responsecounts
226 * @param $qubaids
227 * @param $questions
228 * @param $whichtries
229 */
230 protected function check_response_counts($responsecounts, $qubaids, $questions, $whichtries) {
8183def6 231 foreach ($responsecounts as $expected) {
038014c4
JP
232 $defaultsforexpected = array('randq' => '', 'variant' => '1', 'subpart' => '1');
233 foreach ($defaultsforexpected as $key => $expecteddefault) {
234 if (!isset($expected[$key])) {
235 $expected[$key] = $expecteddefault;
236 }
237 }
238 if ($expected['randq'] == '') {
239 $question = $questions[$expected['slot']];
240 } else {
241 $qid = $this->randqids[$expected['slot']][$expected['randq']];
242 $question = question_finder::get_instance()->load_question_data($qid);
243 }
244 $this->assert_response_count_equals($question, $qubaids, $expected, $whichtries);
245 }
246 }
247
248 /**
249 * @param $questions
250 * @param $questionstats
251 * @param $whichtries
252 * @param $qubaids
253 */
254 protected function check_variants_count_for_quiz_00($questions, $questionstats, $whichtries, $qubaids) {
255 $expectedvariantcounts = array(2 => array(1 => 6,
256 4 => 4,
257 5 => 3,
258 6 => 4,
259 7 => 2,
260 8 => 5,
261 10 => 1));
262
263 foreach ($questions as $slot => $question) {
264 if (!question_bank::get_qtype($question->qtype, false)->can_analyse_responses()) {
265 continue;
266 }
267 $responesstats = new \core_question\statistics\responses\analyser($question);
268 $this->assertTimeCurrent($responesstats->get_last_analysed_time($qubaids, $whichtries));
269 $analysis = $responesstats->load_cached($qubaids, $whichtries);
270 $variantsnos = $analysis->get_variant_nos();
271 if (isset($expectedvariantcounts[$slot])) {
272 // Compare contents, ignore ordering of array, using canonicalize parameter of assertEquals.
3a5641cb 273 $this->assertEqualsCanonicalizing(array_keys($expectedvariantcounts[$slot]), $variantsnos);
038014c4
JP
274 } else {
275 $this->assertEquals(array(1), $variantsnos);
276 }
277 $totalspervariantno = array();
278 foreach ($variantsnos as $variantno) {
279
280 $subpartids = $analysis->get_subpart_ids($variantno);
281 foreach ($subpartids as $subpartid) {
282 if (!isset($totalspervariantno[$subpartid])) {
283 $totalspervariantno[$subpartid] = array();
284 }
285 $totalspervariantno[$subpartid][$variantno] = 0;
286
287 $subpartanalysis = $analysis->get_analysis_for_subpart($variantno, $subpartid);
288 $classids = $subpartanalysis->get_response_class_ids();
289 foreach ($classids as $classid) {
290 $classanalysis = $subpartanalysis->get_response_class($classid);
291 $actualresponsecounts = $classanalysis->data_for_question_response_table('', '');
292 foreach ($actualresponsecounts as $actualresponsecount) {
293 $totalspervariantno[$subpartid][$variantno] += $actualresponsecount->totalcount;
294 }
295 }
296 }
297 }
298 // Count all counted responses for each part of question and confirm that counted responses, for most question types
299 // are the number of attempts at the question for each question part.
300 if ($slot != 5) {
301 // Slot 5 holds a multi-choice multiple question.
302 // Multi-choice multiple is slightly strange. Actual answer counts given for each sub part do not add up to the
303 // total attempt count.
304 // This is because each option is counted as a sub part and each option can be off or on in each attempt. Off is
305 // not counted in response analysis for this question type.
306 foreach ($totalspervariantno as $totalpervariantno) {
307 if (isset($expectedvariantcounts[$slot])) {
308 // If we know how many attempts there are at each variant we can check
309 // that we have counted the correct amount of responses for each variant.
3a5641cb 310 $this->assertEqualsCanonicalizing($expectedvariantcounts[$slot],
038014c4 311 $totalpervariantno,
3a5641cb 312 "Totals responses do not add up in response analysis for slot {$slot}.");
038014c4
JP
313 } else {
314 $this->assertEquals(25,
315 array_sum($totalpervariantno),
316 "Totals responses do not add up in response analysis for slot {$slot}.");
317 }
318 }
319 }
320 }
321
322 foreach ($expectedvariantcounts as $slot => $expectedvariantcount) {
323 foreach ($expectedvariantcount as $variantno => $s) {
324 $this->assertEquals($s, $questionstats->for_slot($slot, $variantno)->s);
325 }
326 }
327 }
328
329 /**
330 * @param $quizstats
331 */
332 protected function check_quiz_stats_for_quiz_00($quizstats) {
333 $quizstatsexpected = array(
334 'median' => 4.5,
335 'firstattemptsavg' => 4.617333332,
336 'allattemptsavg' => 4.617333332,
337 'firstattemptscount' => 25,
338 'allattemptscount' => 25,
339 'standarddeviation' => 0.8117265554,
340 'skewness' => -0.092502502,
341 'kurtosis' => -0.7073968557,
342 'cic' => -87.2230935542,
343 'errorratio' => 136.8294900795,
344 'standarderror' => 1.1106813066
345 );
346
347 foreach ($quizstatsexpected as $statname => $statvalue) {
3a5641cb 348 $this->assertEqualsWithDelta($statvalue, $quizstats->$statname, abs($statvalue) * 1.5e-5, $quizstats->$statname);
038014c4
JP
349 }
350 }
351
ea6db479
JP
352 /**
353 * Check the question stats and the response counts used in the statistics report. If the appropriate files exist in fixtures/.
354 *
8183def6 355 * @param array $csvdata Data loaded from csv files for this test.
ea6db479
JP
356 * @param string $whichattempts
357 * @param string $whichtries
10c4fce5 358 * @param \core\dml\sql_join $groupstudentsjoins
ea6db479
JP
359 * @return array with contents 0 => $questions, 1 => $quizstats, 2=> $questionstats, 3=> $qubaids Might be needed for further
360 * testing.
361 */
10c4fce5
JB
362 protected function check_stats_calculations_and_response_analysis($csvdata, $whichattempts, $whichtries,
363 \core\dml\sql_join $groupstudentsjoins) {
ea6db479
JP
364 $this->report = new quiz_statistics_report();
365 $questions = $this->report->load_and_initialise_questions_for_calculations($this->quiz);
366 list($quizstats, $questionstats) = $this->report->get_all_stats_and_analysis($this->quiz,
367 $whichattempts,
368 $whichtries,
10c4fce5 369 $groupstudentsjoins,
ea6db479
JP
370 $questions);
371
10c4fce5 372 $qubaids = quiz_statistics_qubaids_condition($this->quiz->id, $groupstudentsjoins, $whichattempts);
ea6db479
JP
373
374 // We will create some quiz and question stat calculator instances and some response analyser instances, just in order
375 // to check the last analysed time then returned.
376 $quizcalc = new \quiz_statistics\calculator();
377 // Should not be a delay of more than one second between the calculation of stats above and here.
378 $this->assertTimeCurrent($quizcalc->get_last_calculated_time($qubaids));
379
380 $qcalc = new \core_question\statistics\questions\calculator($questions);
381 $this->assertTimeCurrent($qcalc->get_last_calculated_time($qubaids));
382
383 if (isset($csvdata['responsecounts'])) {
384 $this->check_response_counts($csvdata['responsecounts'], $qubaids, $questions, $whichtries);
385 }
386 if (isset($csvdata['qstats'])) {
387 $this->check_question_stats($csvdata['qstats'], $questionstats);
388 return array($questions, $quizstats, $questionstats, $qubaids);
389 }
390 return array($questions, $quizstats, $questionstats, $qubaids);
391 }
392
3652dddd 393}