MDL-64320 analytics: Validate discrete indicators separately
[moodle.git] / analytics / tests / prediction_test.php
CommitLineData
ff656bae
DM
1<?php
2// This file is part of Moodle - http://moodle.org/
3//
4// Moodle is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// Moodle is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16
17/**
18 * Unit tests for evaluation, training and prediction.
19 *
20 * @package core_analytics
21 * @copyright 2017 David MonllaĆ³ {@link http://www.davidmonllao.com}
22 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
23 */
24
25defined('MOODLE_INTERNAL') || die();
26
3a89d0b5 27global $CFG;
ff656bae
DM
28require_once(__DIR__ . '/fixtures/test_indicator_max.php');
29require_once(__DIR__ . '/fixtures/test_indicator_min.php');
30require_once(__DIR__ . '/fixtures/test_indicator_fullname.php');
31require_once(__DIR__ . '/fixtures/test_indicator_random.php');
32require_once(__DIR__ . '/fixtures/test_target_shortname.php');
b3d68794 33require_once(__DIR__ . '/fixtures/test_static_target_shortname.php');
ff656bae 34
00da1e60
DM
35require_once(__DIR__ . '/../../course/lib.php');
36
ff656bae
DM
37/**
38 * Unit tests for evaluation, training and prediction.
39 *
40 * @package core_analytics
41 * @copyright 2017 David MonllaĆ³ {@link http://www.davidmonllao.com}
42 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
43 */
44class core_analytics_prediction_testcase extends advanced_testcase {
45
b3d68794
DM
46 /**
47 * test_static_prediction
48 *
49 * @return void
50 */
51 public function test_static_prediction() {
52 global $DB;
53
54 $this->resetAfterTest(true);
55 $this->setAdminuser();
56
57 $model = $this->add_perfect_model('test_static_target_shortname');
206d7aa9 58 $model->enable('\core\analytics\time_splitting\no_splitting');
b3d68794
DM
59 $this->assertEquals(1, $model->is_enabled());
60 $this->assertEquals(1, $model->is_trained());
61
62 // No training for static models.
63 $results = $model->train();
64 $trainedsamples = $DB->get_records('analytics_train_samples', array('modelid' => $model->get_id()));
65 $this->assertEmpty($trainedsamples);
66 $this->assertEmpty($DB->count_records('analytics_used_files',
67 array('modelid' => $model->get_id(), 'action' => 'trained')));
68
69 // Now we create 2 hidden courses (only hidden courses are getting predictions).
70 $courseparams = array('shortname' => 'aaaaaa', 'fullname' => 'aaaaaa', 'visible' => 0);
71 $course1 = $this->getDataGenerator()->create_course($courseparams);
72 $courseparams = array('shortname' => 'bbbbbb', 'fullname' => 'bbbbbb', 'visible' => 0);
73 $course2 = $this->getDataGenerator()->create_course($courseparams);
74
75 $result = $model->predict();
76
77 // Var $course1 predictions should be 1 == 'a', $course2 predictions should be 0 == 'b'.
78 $correct = array($course1->id => 1, $course2->id => 0);
79 foreach ($result->predictions as $uniquesampleid => $predictiondata) {
80 list($sampleid, $rangeindex) = $model->get_time_splitting()->infer_sample_info($uniquesampleid);
81
82 // The range index is not important here, both ranges prediction will be the same.
83 $this->assertEquals($correct[$sampleid], $predictiondata->prediction);
84 }
85
86 // 1 range for each analysable.
00da1e60 87 $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id()));
b3d68794
DM
88 $this->assertCount(2, $predictedranges);
89 $this->assertEquals(1, $DB->count_records('analytics_used_files',
90 array('modelid' => $model->get_id(), 'action' => 'predicted')));
91 // 2 predictions for each range.
92 $this->assertEquals(2, $DB->count_records('analytics_predictions',
93 array('modelid' => $model->get_id())));
94
95 // No new generated files nor records as there are no new courses available.
96 $model->predict();
00da1e60 97 $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id()));
b3d68794
DM
98 $this->assertCount(2, $predictedranges);
99 $this->assertEquals(1, $DB->count_records('analytics_used_files',
100 array('modelid' => $model->get_id(), 'action' => 'predicted')));
101 $this->assertEquals(2, $DB->count_records('analytics_predictions',
102 array('modelid' => $model->get_id())));
103 }
104
ff656bae 105 /**
413f19bc
DM
106 * test_ml_training_and_prediction
107 *
e499074f 108 * @dataProvider provider_ml_training_and_prediction
ff656bae 109 * @param string $timesplittingid
00da1e60 110 * @param int $predictedrangeindex
0690a271 111 * @param int $nranges
413f19bc 112 * @param string $predictionsprocessorclass
ff656bae
DM
113 * @return void
114 */
0690a271 115 public function test_ml_training_and_prediction($timesplittingid, $predictedrangeindex, $nranges, $predictionsprocessorclass) {
ff656bae
DM
116 global $DB;
117
584ffa4f
DM
118 $this->resetAfterTest(true);
119 $this->setAdminuser();
cad36252
DM
120 set_config('enabled_stores', 'logstore_standard', 'tool_log');
121
ff656bae
DM
122 $ncourses = 10;
123
ff656bae
DM
124 // Generate training data.
125 $params = array(
126 'startdate' => mktime(0, 0, 0, 10, 24, 2015),
127 'enddate' => mktime(0, 0, 0, 2, 24, 2016),
128 );
129 for ($i = 0; $i < $ncourses; $i++) {
130 $name = 'a' . random_string(10);
131 $courseparams = array('shortname' => $name, 'fullname' => $name) + $params;
132 $this->getDataGenerator()->create_course($courseparams);
133 }
134 for ($i = 0; $i < $ncourses; $i++) {
135 $name = 'b' . random_string(10);
136 $courseparams = array('shortname' => $name, 'fullname' => $name) + $params;
137 $this->getDataGenerator()->create_course($courseparams);
138 }
139
140 // We repeat the test for all prediction processors.
141 $predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
142 if ($predictionsprocessor->is_ready() !== true) {
143 $this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
144 }
145
ff656bae 146 $model = $this->add_perfect_model();
ba0a68e2 147 $model->update(true, false, $timesplittingid, get_class($predictionsprocessor));
ff656bae
DM
148
149 // No samples trained yet.
150 $this->assertEquals(0, $DB->count_records('analytics_train_samples', array('modelid' => $model->get_id())));
151
152 $results = $model->train();
b3d68794
DM
153 $this->assertEquals(1, $model->is_enabled());
154 $this->assertEquals(1, $model->is_trained());
ff656bae 155
0690a271
DM
156 // 20 courses * the 3 model indicators * the number of time ranges of this time splitting method.
157 $indicatorcalc = 20 * 3 * $nranges;
158 $this->assertEquals($indicatorcalc, $DB->count_records('analytics_indicator_calc'));
159
ff656bae
DM
160 // 1 training file was created.
161 $trainedsamples = $DB->get_records('analytics_train_samples', array('modelid' => $model->get_id()));
b3d68794 162 $this->assertCount(1, $trainedsamples);
ff656bae 163 $samples = json_decode(reset($trainedsamples)->sampleids, true);
b3d68794 164 $this->assertCount($ncourses * 2, $samples);
ff656bae
DM
165 $this->assertEquals(1, $DB->count_records('analytics_used_files',
166 array('modelid' => $model->get_id(), 'action' => 'trained')));
28c97d1b
DM
167 // Check that analysable files for training are stored under labelled filearea.
168 $fs = get_file_storage();
169 $this->assertCount(1, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
170 \core_analytics\dataset_manager::LABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
171 $this->assertEmpty($fs->get_directory_files(\context_system::instance()->id, 'analytics',
172 \core_analytics\dataset_manager::UNLABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
ff656bae 173
ff656bae
DM
174 $courseparams = $params + array('shortname' => 'aaaaaa', 'fullname' => 'aaaaaa', 'visible' => 0);
175 $course1 = $this->getDataGenerator()->create_course($courseparams);
176 $courseparams = $params + array('shortname' => 'bbbbbb', 'fullname' => 'bbbbbb', 'visible' => 0);
177 $course2 = $this->getDataGenerator()->create_course($courseparams);
178
ff656bae
DM
179 // They will not be skipped for prediction though.
180 $result = $model->predict();
181
413f19bc 182 // Var $course1 predictions should be 1 == 'a', $course2 predictions should be 0 == 'b'.
ff656bae 183 $correct = array($course1->id => 1, $course2->id => 0);
e499074f
DM
184 foreach ($result->predictions as $uniquesampleid => $predictiondata) {
185 list($sampleid, $rangeindex) = $model->get_time_splitting()->infer_sample_info($uniquesampleid);
ff656bae
DM
186
187 // The range index is not important here, both ranges prediction will be the same.
e499074f 188 $this->assertEquals($correct[$sampleid], $predictiondata->prediction);
ff656bae
DM
189 }
190
00da1e60
DM
191 // 1 range will be predicted.
192 $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id()));
193 $this->assertCount(1, $predictedranges);
194 foreach ($predictedranges as $predictedrange) {
195 $this->assertEquals($predictedrangeindex, $predictedrange->rangeindex);
196 $sampleids = json_decode($predictedrange->sampleids, true);
197 $this->assertCount(2, $sampleids);
198 $this->assertContains($course1->id, $sampleids);
199 $this->assertContains($course2->id, $sampleids);
200 }
ff656bae
DM
201 $this->assertEquals(1, $DB->count_records('analytics_used_files',
202 array('modelid' => $model->get_id(), 'action' => 'predicted')));
00da1e60
DM
203 // 2 predictions.
204 $this->assertEquals(2, $DB->count_records('analytics_predictions',
413f19bc 205 array('modelid' => $model->get_id())));
ff656bae 206
28c97d1b
DM
207 // Check that analysable files to get predictions are stored under unlabelled filearea.
208 $this->assertCount(1, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
209 \core_analytics\dataset_manager::LABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
210 $this->assertCount(1, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
211 \core_analytics\dataset_manager::UNLABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
212
ff656bae
DM
213 // No new generated files nor records as there are no new courses available.
214 $model->predict();
00da1e60
DM
215 $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id()));
216 $this->assertCount(1, $predictedranges);
217 foreach ($predictedranges as $predictedrange) {
218 $this->assertEquals($predictedrangeindex, $predictedrange->rangeindex);
219 }
ff656bae
DM
220 $this->assertEquals(1, $DB->count_records('analytics_used_files',
221 array('modelid' => $model->get_id(), 'action' => 'predicted')));
00da1e60
DM
222 $this->assertEquals(2, $DB->count_records('analytics_predictions',
223 array('modelid' => $model->get_id())));
224
225 // New samples that can be used for prediction.
226 $courseparams = $params + array('shortname' => 'cccccc', 'fullname' => 'cccccc', 'visible' => 0);
227 $course3 = $this->getDataGenerator()->create_course($courseparams);
228 $courseparams = $params + array('shortname' => 'dddddd', 'fullname' => 'dddddd', 'visible' => 0);
229 $course4 = $this->getDataGenerator()->create_course($courseparams);
230
231 $result = $model->predict();
232
233 $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id()));
234 $this->assertCount(1, $predictedranges);
235 foreach ($predictedranges as $predictedrange) {
236 $this->assertEquals($predictedrangeindex, $predictedrange->rangeindex);
237 $sampleids = json_decode($predictedrange->sampleids, true);
238 $this->assertCount(4, $sampleids);
239 $this->assertContains($course1->id, $sampleids);
240 $this->assertContains($course2->id, $sampleids);
241 $this->assertContains($course3->id, $sampleids);
242 $this->assertContains($course4->id, $sampleids);
243 }
244 $this->assertEquals(2, $DB->count_records('analytics_used_files',
245 array('modelid' => $model->get_id(), 'action' => 'predicted')));
246 $this->assertEquals(4, $DB->count_records('analytics_predictions',
247 array('modelid' => $model->get_id())));
28c97d1b
DM
248 $this->assertCount(1, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
249 \core_analytics\dataset_manager::LABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
250 $this->assertCount(2, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
251 \core_analytics\dataset_manager::UNLABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
00da1e60
DM
252
253 // New visible course (for training).
254 $course5 = $this->getDataGenerator()->create_course(array('shortname' => 'aaa', 'fullname' => 'aa'));
255 $course6 = $this->getDataGenerator()->create_course();
256 $result = $model->train();
257 $this->assertEquals(2, $DB->count_records('analytics_used_files',
258 array('modelid' => $model->get_id(), 'action' => 'trained')));
28c97d1b
DM
259 $this->assertCount(2, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
260 \core_analytics\dataset_manager::LABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
261 $this->assertCount(2, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
262 \core_analytics\dataset_manager::UNLABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
382a0890
DM
263
264 set_config('enabled_stores', '', 'tool_log');
265 get_log_manager(true);
ff656bae
DM
266 }
267
413f19bc
DM
268 /**
269 * provider_ml_training_and_prediction
270 *
271 * @return array
272 */
e499074f 273 public function provider_ml_training_and_prediction() {
ff656bae 274 $cases = array(
0690a271
DM
275 'no_splitting' => array('\core\analytics\time_splitting\no_splitting', 0, 1),
276 'quarters' => array('\core\analytics\time_splitting\quarters', 3, 4)
ff656bae
DM
277 );
278
279 // We need to test all system prediction processors.
280 return $this->add_prediction_processors($cases);
281 }
282
325b3bdd
DM
283 /**
284 * Test the system classifiers returns.
285 *
286 * This test checks that all mlbackend plugins in the system are able to return proper status codes
287 * even under weird situations.
288 *
289 * @dataProvider provider_ml_classifiers_return
290 * @param int $success
291 * @param int $nsamples
292 * @param int $classes
293 * @param string $predictionsprocessorclass
294 * @return void
295 */
296 public function test_ml_classifiers_return($success, $nsamples, $classes, $predictionsprocessorclass) {
297 $this->resetAfterTest();
298
299 $predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
300 if ($predictionsprocessor->is_ready() !== true) {
301 $this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
302 }
303
304 if ($nsamples % count($classes) != 0) {
305 throw new \coding_exception('The number of samples should be divisible by the number of classes');
306 }
307 $samplesperclass = $nsamples / count($classes);
308
309 // Metadata (we pass 2 classes even if $classes only provides 1 class samples as we want to test
310 // what the backend does in this case.
311 $dataset = "nfeatures,targetclasses,targettype" . PHP_EOL;
312 $dataset .= "3,\"[0,1]\",\"discrete\"" . PHP_EOL;
313
314 // Headers.
315 $dataset .= "feature1,feature2,feature3,target" . PHP_EOL;
316 foreach ($classes as $class) {
317 for ($i = 0; $i < $samplesperclass; $i++) {
318 $dataset .= "1,0,1,$class" . PHP_EOL;
319 }
320 }
321
322 $trainingfile = array(
323 'contextid' => \context_system::instance()->id,
324 'component' => 'analytics',
325 'filearea' => 'labelled',
326 'itemid' => 123,
327 'filepath' => '/',
328 'filename' => 'whocares.csv'
329 );
330 $fs = get_file_storage();
331 $dataset = $fs->create_file_from_string($trainingfile, $dataset);
332
333 // Training should work correctly if at least 1 sample of each class is included.
334 $dir = make_request_directory();
335 $result = $predictionsprocessor->train_classification('whatever', $dataset, $dir);
336
337 switch ($success) {
338 case 'yes':
339 $this->assertEquals(\core_analytics\model::OK, $result->status);
340 break;
341 case 'no':
342 $this->assertNotEquals(\core_analytics\model::OK, $result->status);
343 break;
344 case 'maybe':
345 default:
346 // We just check that an object is returned so we don't have an empty check,
347 // what we really want to check is that an exception was not thrown.
348 $this->assertInstanceOf(\stdClass::class, $result);
349 }
350 }
351
352 /**
353 * test_ml_classifiers_return provider
354 *
355 * We can not be very specific here as test_ml_classifiers_return only checks that
356 * mlbackend plugins behave and expected and control properly backend errors even
357 * under weird situations.
358 *
359 * @return array
360 */
361 public function provider_ml_classifiers_return() {
362 // Using verbose options as the first argument for readability.
363 $cases = array(
364 '1-samples' => array('maybe', 1, [0]),
365 '2-samples-same-class' => array('maybe', 2, [0]),
366 '2-samples-different-classes' => array('yes', 2, [0, 1]),
367 '4-samples-different-classes' => array('yes', 4, [0, 1])
368 );
369
370 // We need to test all system prediction processors.
371 return $this->add_prediction_processors($cases);
372 }
ff656bae
DM
373
374 /**
375 * Basic test to check that prediction processors work as expected.
376 *
e499074f 377 * @dataProvider provider_ml_test_evaluation
413f19bc
DM
378 * @param string $modelquality
379 * @param int $ncourses
380 * @param array $expected
381 * @param string $predictionsprocessorclass
382 * @return void
ff656bae 383 */
e499074f 384 public function test_ml_evaluation($modelquality, $ncourses, $expected, $predictionsprocessorclass) {
ff656bae 385 $this->resetAfterTest(true);
584ffa4f 386 $this->setAdminuser();
cad36252
DM
387 set_config('enabled_stores', 'logstore_standard', 'tool_log');
388
206d7aa9
DM
389 $sometimesplittings = '\core\analytics\time_splitting\weekly,' .
390 '\core\analytics\time_splitting\single_range,' .
391 '\core\analytics\time_splitting\quarters';
ff656bae
DM
392 set_config('timesplittings', $sometimesplittings, 'analytics');
393
394 if ($modelquality === 'perfect') {
395 $model = $this->add_perfect_model();
396 } else if ($modelquality === 'random') {
397 $model = $this->add_random_model();
398 } else {
399 throw new \coding_exception('Only perfect and random accepted as $modelquality values');
400 }
401
ff656bae
DM
402 // Generate training data.
403 $params = array(
404 'startdate' => mktime(0, 0, 0, 10, 24, 2015),
405 'enddate' => mktime(0, 0, 0, 2, 24, 2016),
406 );
407 for ($i = 0; $i < $ncourses; $i++) {
408 $name = 'a' . random_string(10);
409 $params = array('shortname' => $name, 'fullname' => $name) + $params;
410 $this->getDataGenerator()->create_course($params);
411 }
412 for ($i = 0; $i < $ncourses; $i++) {
413 $name = 'b' . random_string(10);
414 $params = array('shortname' => $name, 'fullname' => $name) + $params;
415 $this->getDataGenerator()->create_course($params);
416 }
417
418 // We repeat the test for all prediction processors.
419 $predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
420 if ($predictionsprocessor->is_ready() !== true) {
421 $this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
422 }
423
ba0a68e2 424 $model->update(false, false, false, get_class($predictionsprocessor));
ff656bae
DM
425 $results = $model->evaluate();
426
427 // We check that the returned status includes at least $expectedcode code.
428 foreach ($results as $timesplitting => $result) {
429 $message = 'The returned status code ' . $result->status . ' should include ' . $expected[$timesplitting];
413f19bc
DM
430 $filtered = $result->status & $expected[$timesplitting];
431 $this->assertEquals($expected[$timesplitting], $filtered, $message);
ff656bae 432 }
382a0890
DM
433
434 set_config('enabled_stores', '', 'tool_log');
435 get_log_manager(true);
ff656bae
DM
436 }
437
0690a271
DM
438 /**
439 * test_read_indicator_calculations
440 *
441 * @return void
442 */
443 public function test_read_indicator_calculations() {
444 global $DB;
445
446 $this->resetAfterTest(true);
447
448 $starttime = 123;
449 $endtime = 321;
450 $sampleorigin = 'whatever';
451
452 $indicator = $this->getMockBuilder('test_indicator_max')->setMethods(['calculate_sample'])->getMock();
453 $indicator->expects($this->never())->method('calculate_sample');
454
e0c88997 455 $existingcalcs = array(111 => 1, 222 => -1);
0690a271
DM
456 $sampleids = array(111 => 111, 222 => 222);
457 list($values, $unused) = $indicator->calculate($sampleids, $sampleorigin, $starttime, $endtime, $existingcalcs);
458 }
459
aa1b04dc
DM
460 /**
461 * test_not_null_samples
462 */
463 public function test_not_null_samples() {
464 $this->resetAfterTest(true);
465
466 $classname = '\core\analytics\time_splitting\quarters';
467 $timesplitting = \core_analytics\manager::get_time_splitting($classname);
468 $timesplitting->set_analysable(new \core_analytics\site());
469
470 $ranges = array(
471 array('start' => 111, 'end' => 222, 'time' => 222),
472 array('start' => 222, 'end' => 333, 'time' => 333)
473 );
474 $samples = array(123 => 123, 321 => 321);
475
476 $indicator1 = $this->getMockBuilder('test_indicator_max')
477 ->setMethods(['calculate_sample'])
478 ->getMock();
479 $indicator1->method('calculate_sample')
480 ->willReturn(null);
481
482 $indicator2 = \core_analytics\manager::get_indicator('test_indicator_min');
483
484 // Samples with at least 1 not null value are returned.
485 $params = array(
486 $samples,
487 'whatever',
488 array($indicator1, $indicator2),
489 $ranges
490 );
491 $dataset = phpunit_util::call_internal_method($timesplitting, 'calculate_indicators', $params, $classname);
492 $this->assertArrayHasKey('123-0', $dataset);
493 $this->assertArrayHasKey('123-1', $dataset);
494 $this->assertArrayHasKey('321-0', $dataset);
495 $this->assertArrayHasKey('321-1', $dataset);
496
497 // Samples with only null values are not returned.
498 $params = array(
499 $samples,
500 'whatever',
501 array($indicator1),
502 $ranges
503 );
504 $dataset = phpunit_util::call_internal_method($timesplitting, 'calculate_indicators', $params, $classname);
505 $this->assertArrayNotHasKey('123-0', $dataset);
506 $this->assertArrayNotHasKey('123-1', $dataset);
507 $this->assertArrayNotHasKey('321-0', $dataset);
508 $this->assertArrayNotHasKey('321-1', $dataset);
509 }
510
413f19bc
DM
511 /**
512 * provider_ml_test_evaluation
513 *
514 * @return array
515 */
e499074f 516 public function provider_ml_test_evaluation() {
ff656bae
DM
517
518 $cases = array(
ff656bae
DM
519 'bad' => array(
520 'modelquality' => 'random',
521 'ncourses' => 50,
522 'expectedresults' => array(
523 // The course duration is too much to be processed by in weekly basis.
206d7aa9 524 '\core\analytics\time_splitting\weekly' => \core_analytics\model::NO_DATASET,
325b3bdd
DM
525 '\core\analytics\time_splitting\single_range' => \core_analytics\model::LOW_SCORE,
526 '\core\analytics\time_splitting\quarters' => \core_analytics\model::LOW_SCORE,
ff656bae
DM
527 )
528 ),
529 'good' => array(
530 'modelquality' => 'perfect',
531 'ncourses' => 50,
532 'expectedresults' => array(
533 // The course duration is too much to be processed by in weekly basis.
206d7aa9
DM
534 '\core\analytics\time_splitting\weekly' => \core_analytics\model::NO_DATASET,
535 '\core\analytics\time_splitting\single_range' => \core_analytics\model::OK,
536 '\core\analytics\time_splitting\quarters' => \core_analytics\model::OK,
ff656bae
DM
537 )
538 )
539 );
540 return $this->add_prediction_processors($cases);
541 }
542
413f19bc
DM
543 /**
544 * add_random_model
545 *
546 * @return \core_analytics\model
547 */
ff656bae
DM
548 protected function add_random_model() {
549
550 $target = \core_analytics\manager::get_target('test_target_shortname');
551 $indicators = array('test_indicator_max', 'test_indicator_min', 'test_indicator_random');
552 foreach ($indicators as $key => $indicator) {
553 $indicators[$key] = \core_analytics\manager::get_indicator($indicator);
554 }
555
556 $model = \core_analytics\model::create($target, $indicators);
557
558 // To load db defaults as well.
559 return new \core_analytics\model($model->get_id());
560 }
561
413f19bc
DM
562 /**
563 * add_perfect_model
564 *
b3d68794 565 * @param string $targetclass
413f19bc
DM
566 * @return \core_analytics\model
567 */
b3d68794 568 protected function add_perfect_model($targetclass = 'test_target_shortname') {
ff656bae 569
b3d68794 570 $target = \core_analytics\manager::get_target($targetclass);
ff656bae
DM
571 $indicators = array('test_indicator_max', 'test_indicator_min', 'test_indicator_fullname');
572 foreach ($indicators as $key => $indicator) {
573 $indicators[$key] = \core_analytics\manager::get_indicator($indicator);
574 }
575
576 $model = \core_analytics\model::create($target, $indicators);
577
578 // To load db defaults as well.
579 return new \core_analytics\model($model->get_id());
580 }
581
413f19bc
DM
582 /**
583 * add_prediction_processors
584 *
585 * @param array $cases
586 * @return array
587 */
ff656bae
DM
588 protected function add_prediction_processors($cases) {
589
590 $return = array();
591
592 // We need to test all system prediction processors.
593 $predictionprocessors = \core_analytics\manager::get_all_prediction_processors();
594 foreach ($predictionprocessors as $classfullname => $unused) {
595 foreach ($cases as $key => $case) {
596 $newkey = $key . '-' . $classfullname;
597 $return[$newkey] = $case + array('predictionsprocessorclass' => $classfullname);
598 }
599 }
600
601 return $return;
602 }
603}