MDL-43479 quiz response analysis : suppress break down by variants
[moodle.git] / mod / quiz / report / statistics / tests / stats_from_steps_walkthrough_test.php
CommitLineData
3652dddd
JP
1<?php
2// This file is part of Moodle - http://moodle.org/
3//
4// Moodle is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// Moodle is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16
17/**
18 * Quiz attempt walk through using data from csv file.
19 *
e68e4ccf 20 * @package quiz_statistics
3652dddd
JP
21 * @category phpunit
22 * @copyright 2013 The Open University
23 * @author Jamie Pratt <me@jamiep.org>
24 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
25 */
26
27defined('MOODLE_INTERNAL') || die();
28
29global $CFG;
30require_once($CFG->dirroot . '/mod/quiz/tests/attempt_walkthrough_from_csv_test.php');
31require_once($CFG->dirroot . '/mod/quiz/report/default.php');
32require_once($CFG->dirroot . '/mod/quiz/report/statistics/report.php');
33require_once($CFG->dirroot . '/mod/quiz/report/reportlib.php');
3652dddd 34
3652dddd
JP
35/**
36 * Quiz attempt walk through using data from csv file.
37 *
e68e4ccf 38 * @package quiz_statistics
3652dddd
JP
39 * @category phpunit
40 * @copyright 2013 The Open University
41 * @author Jamie Pratt <me@jamiep.org>
42 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
43 */
8e328617 44class quiz_report_statistics_from_steps_testcase extends mod_quiz_attempt_walkthrough_from_csv_testcase {
3652dddd
JP
45
46 /**
47 * @var quiz_statistics_report object to do stats calculations.
48 */
49 protected $report;
50
51 protected function get_full_path_of_csv_file($setname, $test) {
52 // Overridden here so that __DIR__ points to the path of this file.
53 return __DIR__."/fixtures/{$setname}{$test}.csv";
54 }
55
4922e79f 56 protected $files = array('questions', 'steps', 'results', 'qstats', 'responsecounts');
3652dddd
JP
57
58 /**
59 * Create a quiz add questions to it, walk through quiz attempts and then check results.
60 *
61 * @param PHPUnit_Extensions_Database_DataSet_ITable[] of data read from csv file "questionsXX.csv",
62 * "stepsXX.csv" and "resultsXX.csv".
63 * @dataProvider get_data_for_walkthrough
64 */
764f6153 65 public function test_walkthrough_from_csv($quizsettings, $csvdata) {
3652dddd
JP
66
67 // CSV data files for these tests were generated using :
68 // https://github.com/jamiepratt/moodle-quiz-tools/tree/master/responsegenerator
69
70 $this->resetAfterTest(true);
71 question_bank::get_qtype('random')->clear_caches_before_testing();
72
764f6153 73 $this->create_quiz($quizsettings, $csvdata['questions']);
3652dddd
JP
74
75 $attemptids = $this->walkthrough_attempts($csvdata['steps']);
76
77 $this->check_attempts_results($csvdata['results'], $attemptids);
78
487e7e66
JP
79 $this->report = new quiz_statistics_report();
80 $whichattempts = QUIZ_GRADEAVERAGE;
81 $groupstudents = array();
82 $questions = $this->report->load_and_initialise_questions_for_calculations($this->quiz);
c3e2e754 83 list($quizstats, $questionstats) =
8da6fc9d 84 $this->report->get_all_stats_and_analysis($this->quiz, $whichattempts, $groupstudents, $questions);
487e7e66
JP
85
86 $qubaids = quiz_statistics_qubaids_condition($this->quiz->id, $groupstudents, $whichattempts);
87
88 // We will create some quiz and question stat calculator instances and some response analyser instances, just in order
4922e79f 89 // to check the last analysed time then returned.
8da6fc9d 90 $quizcalc = new \quiz_statistics\calculator();
487e7e66 91 // Should not be a delay of more than one second between the calculation of stats above and here.
7d6b28d8 92 $this->assertTimeCurrent($quizcalc->get_last_calculated_time($qubaids));
487e7e66
JP
93
94 $qcalc = new \core_question\statistics\questions\calculator($questions);
7d6b28d8 95 $this->assertTimeCurrent($qcalc->get_last_calculated_time($qubaids));
487e7e66 96
3d6f2466
JP
97 $expectedvariantcounts = array(2 => array(1 => 6,
98 4 => 4,
99 5 => 3,
100 6 => 4,
101 7 => 2,
102 8 => 5,
103 10 => 1));
104
105 foreach ($questions as $slot => $question) {
106 if (!question_bank::get_qtype($question->qtype, false)->can_analyse_responses()) {
487e7e66
JP
107 continue;
108 }
109 $responesstats = new \core_question\statistics\responses\analyser($question);
7d6b28d8 110 $this->assertTimeCurrent($responesstats->get_last_analysed_time($qubaids));
3d6f2466
JP
111 $analysis = $responesstats->load_cached($qubaids);
112 $variantsnos = $analysis->get_variant_nos();
113 if (isset($expectedvariantcounts[$slot])) {
114 // Compare contents, ignore ordering of array, using canonicalize parameter of assertEquals.
115 $this->assertEquals(array_keys($expectedvariantcounts[$slot]), $variantsnos, '', 0, 10, true);
116 } else {
117 $this->assertEquals(array(1), $variantsnos);
118 }
119 $totalspervariantno = array();
120 foreach ($variantsnos as $variantno) {
121
122 $subpartids = $analysis->get_subpart_ids($variantno);
123 foreach ($subpartids as $subpartid) {
124 if (!isset($totalspervariantno[$subpartid])) {
125 $totalspervariantno[$subpartid] = array();
126 }
127 $totalspervariantno[$subpartid][$variantno] = 0;
128
129 $subpartanalysis = $analysis->get_analysis_for_subpart($variantno, $subpartid);
130 $classids = $subpartanalysis->get_response_class_ids();
131 foreach ($classids as $classid) {
132 $classanalysis = $subpartanalysis->get_response_class($classid);
133 $actualresponsecounts = $classanalysis->data_for_question_response_table('', '');
134 foreach ($actualresponsecounts as $actualresponsecount) {
135 $totalspervariantno[$subpartid][$variantno] += $actualresponsecount->count;
136 }
137 }
138 }
139 }
140 // Count all counted responses for each part of question and confirm that counted responses, for most question types
141 // are the number of attempts at the question for each question part.
142 if ($slot != 5) {
143 // Slot 5 holds a multi-choice multiple question.
144 // Multi-choice multiple is slightly strange. Actual answer counts given for each sub part do not add up to the
145 // total attempt count.
146 // This is because each option is counted as a sub part and each option can be off or on in each attempt. Off is
147 // not counted in response analysis for this question type.
148 foreach ($totalspervariantno as $totalpervariantno) {
149 if (isset($expectedvariantcounts[$slot])) {
150 // If we know how many attempts there are at each variant we can check
151 // that we have counted the correct amount of responses for each variant.
152 $this->assertEquals($expectedvariantcounts[$slot],
153 $totalpervariantno,
154 "Totals responses do not add up in response analysis for slot {$slot}.",
155 0,
156 10,
157 true);
158 } else {
159 $this->assertEquals(25,
160 array_sum($totalpervariantno),
161 "Totals responses do not add up in response analysis for slot {$slot}.");
162 }
163 }
164 }
487e7e66 165 }
4922e79f
JP
166 for ($rowno = 0; $rowno < $csvdata['responsecounts']->getRowCount(); $rowno++) {
167 $responsecount = $csvdata['responsecounts']->getRow($rowno);
168 if ($responsecount['randq'] == '') {
169 $question = $questions[$responsecount['slot']];
170 } else {
171 $qid = $this->randqids[$responsecount['slot']][$responsecount['randq']];
172 $question = question_finder::get_instance()->load_question_data($qid);
173 }
174 $this->assert_response_count_equals($question, $qubaids, $responsecount);
175 }
176
3d6f2466 177 // These quiz stats and the question stats found in qstats00.csv were calculated independently in spreadsheet which is
3652dddd
JP
178 // available in open document or excel format here :
179 // https://github.com/jamiepratt/moodle-quiz-tools/tree/master/statsspreadsheet
180 $quizstatsexpected = array(
181 'median' => 4.5,
182 'firstattemptsavg' => 4.617333332,
183 'allattemptsavg' => 4.617333332,
184 'firstattemptscount' => 25,
185 'allattemptscount' => 25,
186 'standarddeviation' => 0.8117265554,
187 'skewness' => -0.092502502,
188 'kurtosis' => -0.7073968557,
189 'cic' => -87.2230935542,
190 'errorratio' => 136.8294900795,
191 'standarderror' => 1.1106813066
192 );
193
194 foreach ($quizstatsexpected as $statname => $statvalue) {
f9dc4bd7 195 $this->assertEquals($statvalue, $quizstats->$statname, $quizstats->$statname, abs($statvalue) * 1.5e-5);
3652dddd
JP
196 }
197
198 for ($rowno = 0; $rowno < $csvdata['qstats']->getRowCount(); $rowno++) {
199 $slotqstats = $csvdata['qstats']->getRow($rowno);
200 foreach ($slotqstats as $statname => $slotqstat) {
201 if ($statname !== 'slot') {
c3e2e754 202 $this->assert_stat_equals($questionstats, $slotqstats['slot'], null, null, $statname, (float)$slotqstat);
3652dddd
JP
203 }
204 }
205 }
8e328617
JP
206
207 $itemstats = array('s' => 12,
208 'effectiveweight' => null,
209 'discriminationindex' => 35.803933,
210 'discriminativeefficiency' => 39.39393939,
211 'sd' => 0.514928651,
212 'facility' => 0.583333333,
213 'maxmark' => 1,
214 'positions' => '1',
215 'slot' => null,
216 'subquestion' => true);
217 foreach ($itemstats as $statname => $expected) {
c3e2e754 218 $this->assert_stat_equals($questionstats, 1, null, 'numerical', $statname, $expected);
8e328617 219 }
4f7516f6 220
4f7516f6
JP
221 // These variant's stats are calculated in stats_for_variant_1.xls and stats_for_variant_8.xls
222 // The calculations in the spreadsheets are the same but applied just to the attempts where the variants appeared.
223
224 $statsforslot2variants = array(1 => array('s' => 6,
225 'effectiveweight' => null,
226 'discriminationindex' => -10.5999788,
227 'discriminativeefficiency' => -14.28571429,
228 'sd' => 0.5477225575,
229 'facility' => 0.50,
230 'maxmark' => 1,
231 'variant' => 1,
232 'slot' => 2,
233 'subquestion' => false),
234 8 => array('s' => 5,
235 'effectiveweight' => null,
236 'discriminationindex' => -57.77466679,
237 'discriminativeefficiency' => -71.05263241,
238 'sd' => 0.547722558,
239 'facility' => 0.40,
240 'maxmark' => 1,
241 'variant' => 8,
242 'slot' => 2,
243 'subquestion' => false));
244 foreach ($statsforslot2variants as $variant => $stats) {
4922e79f
JP
245 foreach ($stats as $statname => $expected) {
246 $this->assert_stat_equals($questionstats, 2, $variant, null, $statname, $expected);
247 }
248 }
3d6f2466
JP
249 foreach ($expectedvariantcounts as $slot => $expectedvariantcount) {
250 foreach ($expectedvariantcount as $variantno => $s) {
251 $this->assertEquals($s, $questionstats->for_slot($slot, $variantno)->s);
252 }
253 }
254 }
255
256 /**
257 * Check that the stat is as expected within a reasonable tolerance.
258 *
259 * @param \core_question\statistics\questions\all_calculated_for_qubaid_condition $questionstats
260 * @param int $slot
261 * @param int|null $variant if null then not a variant stat.
262 * @param string|null $subqname if null then not an item stat.
263 * @param string $statname
264 * @param float $expected
265 */
266 protected function assert_stat_equals($questionstats, $slot, $variant, $subqname, $statname, $expected) {
267
268 if ($variant === null && $subqname === null) {
269 $actual = $questionstats->for_slot($slot)->{$statname};
270 } else if ($subqname !== null) {
271 $actual = $questionstats->for_subq($this->randqids[$slot][$subqname])->{$statname};
272 } else {
273 $actual = $questionstats->for_slot($slot, $variant)->{$statname};
274 }
275 if (is_bool($expected) || is_string($expected)) {
276 $this->assertEquals($expected, $actual, "$statname for slot $slot");
277 } else {
278 switch ($statname) {
279 case 'covariance' :
280 case 'discriminationindex' :
281 case 'discriminativeefficiency' :
282 case 'effectiveweight' :
283 $precision = 1e-5;
284 break;
285 default :
286 $precision = 1e-6;
287 }
288 $delta = abs($expected) * $precision;
289 $this->assertEquals(floatval($expected), $actual, "$statname for slot $slot", $delta);
290 }
4922e79f
JP
291 }
292
293 protected function assert_response_count_equals($question, $qubaids, $responsecount) {
294 $responesstats = new \core_question\statistics\responses\analyser($question);
295 $analysis = $responesstats->load_cached($qubaids);
296 if (!isset($responsecount['subpart'])) {
297 $subpart = 1;
298 } else {
299 $subpart = $responsecount['subpart'];
300 }
301 list($subpartid, $responseclassid) = $this->get_response_subpart_and_class_id($question,
302 $subpart,
303 $responsecount['modelresponse']);
304
305 $subpartanalysis = $analysis->get_analysis_for_subpart($responsecount['variant'], $subpartid);
306 $responseclassanalysis = $subpartanalysis->get_response_class($responseclassid);
307 $actualresponsecounts = $responseclassanalysis->data_for_question_response_table('', '');
308 if ($responsecount['modelresponse'] !== '[NO RESPONSE]') {
309 foreach ($actualresponsecounts as $actualresponsecount) {
310 if ($actualresponsecount->response == $responsecount['actualresponse']) {
311 $this->assertEquals($responsecount['count'], $actualresponsecount->count);
312 return;
313 }
314 }
315 throw new coding_exception("Actual response '{$responsecount['actualresponse']}' not found.");
316 } else {
317 $actualresponsecount = array_pop($actualresponsecounts);
318 $this->assertEquals($responsecount['count'], $actualresponsecount->count);
319 }
320 }
321
322 protected function get_response_subpart_and_class_id($question, $subpart, $modelresponse) {
323 $qtypeobj = question_bank::get_qtype($question->qtype, false);
324 $possibleresponses = $qtypeobj->get_possible_responses($question);
325 $possibleresponsesubpartids = array_keys($possibleresponses);
326 if (!isset($possibleresponsesubpartids[$subpart - 1])) {
327 throw new coding_exception("Subpart '{$subpart}' not found.");
328 }
329 $subpartid = $possibleresponsesubpartids[$subpart - 1];
330
331 if ($modelresponse == '[NO RESPONSE]') {
332 return array($subpartid, null);
333
334 } else if ($modelresponse == '[NO MATCH]') {
335 return array($subpartid, 0);
336 }
337
338 $modelresponses = array();
339 foreach ($possibleresponses[$subpartid] as $responseclassid => $subpartpossibleresponse) {
340 $modelresponses[$responseclassid] = $subpartpossibleresponse->responseclass;
4f7516f6 341 }
4922e79f
JP
342 $this->assertContains($modelresponse, $modelresponses);
343 $responseclassid = array_search($modelresponse, $modelresponses);
344 return array($subpartid, $responseclassid);
8e328617
JP
345 }
346
3652dddd 347}