MDL-59694 analytics: Track processed analysables
[moodle.git] / analytics / tests / model_test.php
1 <?php
2 // This file is part of Moodle - http://moodle.org/
3 //
4 // Moodle is free software: you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation, either version 3 of the License, or
7 // (at your option) any later version.
8 //
9 // Moodle is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
17 /**
18  * Unit tests for the model.
19  *
20  * @package   core_analytics
21  * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com}
22  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
23  */
25 defined('MOODLE_INTERNAL') || die();
27 require_once(__DIR__ . '/fixtures/test_indicator_max.php');
28 require_once(__DIR__ . '/fixtures/test_indicator_min.php');
29 require_once(__DIR__ . '/fixtures/test_indicator_fullname.php');
30 require_once(__DIR__ . '/fixtures/test_target_shortname.php');
31 require_once(__DIR__ . '/fixtures/test_target_course_level_shortname.php');
32 require_once(__DIR__ . '/fixtures/test_analyser.php');
34 /**
35  * Unit tests for the model.
36  *
37  * @package   core_analytics
38  * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com}
39  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
40  */
41 class analytics_model_testcase extends advanced_testcase {
43     public function setUp() {
45         $this->setAdminUser();
47         $target = \core_analytics\manager::get_target('test_target_shortname');
48         $indicators = array('test_indicator_max', 'test_indicator_min', 'test_indicator_fullname');
49         foreach ($indicators as $key => $indicator) {
50             $indicators[$key] = \core_analytics\manager::get_indicator($indicator);
51         }
53         $this->model = testable_model::create($target, $indicators);
54         $this->modelobj = $this->model->get_model_obj();
55     }
57     public function test_enable() {
58         $this->resetAfterTest(true);
60         $this->assertEquals(0, $this->model->get_model_obj()->enabled);
61         $this->assertEquals(0, $this->model->get_model_obj()->trained);
62         $this->assertEquals('', $this->model->get_model_obj()->timesplitting);
64         $this->model->enable('\core\analytics\time_splitting\quarters');
65         $this->assertEquals(1, $this->model->get_model_obj()->enabled);
66         $this->assertEquals(0, $this->model->get_model_obj()->trained);
67         $this->assertEquals('\core\analytics\time_splitting\quarters', $this->model->get_model_obj()->timesplitting);
68     }
70     public function test_create() {
71         $this->resetAfterTest(true);
73         $target = \core_analytics\manager::get_target('\core\analytics\target\course_dropout');
74         $indicators = array(
75             \core_analytics\manager::get_indicator('\core\analytics\indicator\any_write_action'),
76             \core_analytics\manager::get_indicator('\core\analytics\indicator\read_actions')
77         );
78         $model = \core_analytics\model::create($target, $indicators);
79         $this->assertInstanceOf('\core_analytics\model', $model);
80     }
82     /**
83      * test_delete
84      */
85     public function test_delete() {
86         global $DB;
88         $this->resetAfterTest(true);
89         set_config('enabled_stores', 'logstore_standard', 'tool_log');
91         $coursepredict1 = $this->getDataGenerator()->create_course(array('visible' => 0));
92         $coursepredict2 = $this->getDataGenerator()->create_course(array('visible' => 0));
93         $coursetrain1 = $this->getDataGenerator()->create_course(array('visible' => 1));
94         $coursetrain2 = $this->getDataGenerator()->create_course(array('visible' => 1));
96         $this->model->enable('\core\analytics\time_splitting\no_splitting');
98         $this->model->train();
99         $this->model->predict();
101         // Fake evaluation results record to check that it is actually deleted.
102         $this->add_fake_log();
104         $modeloutputdir = $this->model->get_output_dir(array(), true);
105         $this->assertTrue(is_dir($modeloutputdir));
107         // Generate a prediction action to confirm that it is deleted when there is an important update.
108         $predictions = $DB->get_records('analytics_predictions');
109         $prediction = reset($predictions);
110         $prediction = new \core_analytics\prediction($prediction, array('whatever' => 'not used'));
111         $prediction->action_executed(\core_analytics\prediction::ACTION_FIXED, $this->model->get_target());
113         $this->model->delete();
114         $this->assertEmpty($DB->count_records('analytics_models', array('id' => $this->modelobj->id)));
115         $this->assertEmpty($DB->count_records('analytics_models_log', array('modelid' => $this->modelobj->id)));
116         $this->assertEmpty($DB->count_records('analytics_predictions'));
117         $this->assertEmpty($DB->count_records('analytics_prediction_actions'));
118         $this->assertEmpty($DB->count_records('analytics_train_samples'));
119         $this->assertEmpty($DB->count_records('analytics_predict_samples'));
120         $this->assertEmpty($DB->count_records('analytics_used_files'));
121         $this->assertFalse(is_dir($modeloutputdir));
123         set_config('enabled_stores', '', 'tool_log');
124         get_log_manager(true);
125     }
127     /**
128      * test_clear
129      */
130     public function test_clear() {
131         global $DB;
133         $this->resetAfterTest(true);
134         set_config('enabled_stores', 'logstore_standard', 'tool_log');
136         $coursepredict1 = $this->getDataGenerator()->create_course(array('visible' => 0));
137         $coursepredict2 = $this->getDataGenerator()->create_course(array('visible' => 0));
138         $coursetrain1 = $this->getDataGenerator()->create_course(array('visible' => 1));
139         $coursetrain2 = $this->getDataGenerator()->create_course(array('visible' => 1));
141         $this->model->enable('\core\analytics\time_splitting\no_splitting');
143         $this->model->train();
144         $this->model->predict();
146         // Fake evaluation results record to check that it is actually deleted.
147         $this->add_fake_log();
149         // Generate a prediction action to confirm that it is deleted when there is an important update.
150         $predictions = $DB->get_records('analytics_predictions');
151         $prediction = reset($predictions);
152         $prediction = new \core_analytics\prediction($prediction, array('whatever' => 'not used'));
153         $prediction->action_executed(\core_analytics\prediction::ACTION_FIXED, $this->model->get_target());
155         $modelversionoutputdir = $this->model->get_output_dir();
156         $this->assertTrue(is_dir($modelversionoutputdir));
158         // Update to an empty time splitting method to force clear_model execution.
159         $this->model->update(1, false, '');
160         $this->assertFalse(is_dir($modelversionoutputdir));
162         // Restore previous time splitting method.
163         $this->model->enable('\core\analytics\time_splitting\no_splitting');
165         // Check that most of the stuff got deleted.
166         $this->assertEquals(1, $DB->count_records('analytics_models', array('id' => $this->modelobj->id)));
167         $this->assertEquals(1, $DB->count_records('analytics_models_log', array('modelid' => $this->modelobj->id)));
168         $this->assertEmpty($DB->count_records('analytics_predictions'));
169         $this->assertEmpty($DB->count_records('analytics_prediction_actions'));
170         $this->assertEmpty($DB->count_records('analytics_train_samples'));
171         $this->assertEmpty($DB->count_records('analytics_predict_samples'));
172         $this->assertEmpty($DB->count_records('analytics_used_files'));
174         set_config('enabled_stores', '', 'tool_log');
175         get_log_manager(true);
176     }
178     public function test_model_manager() {
179         $this->resetAfterTest(true);
181         $this->assertCount(3, $this->model->get_indicators());
182         $this->assertInstanceOf('\core_analytics\local\target\binary', $this->model->get_target());
184         // Using evaluation as the model is not yet enabled.
185         $this->model->init_analyser(array('evaluation' => true));
186         $this->assertInstanceOf('\core_analytics\local\analyser\base', $this->model->get_analyser());
188         $this->model->enable('\core\analytics\time_splitting\quarters');
189         $this->assertInstanceOf('\core\analytics\analyser\site_courses', $this->model->get_analyser());
190     }
192     public function test_output_dir() {
193         $this->resetAfterTest(true);
195         $dir = make_request_directory();
196         set_config('modeloutputdir', $dir, 'analytics');
198         $modeldir = $dir . DIRECTORY_SEPARATOR . $this->modelobj->id . DIRECTORY_SEPARATOR . $this->modelobj->version;
199         $this->assertEquals($modeldir, $this->model->get_output_dir());
200         $this->assertEquals($modeldir . DIRECTORY_SEPARATOR . 'testing', $this->model->get_output_dir(array('testing')));
201     }
203     public function test_unique_id() {
204         global $DB;
206         $this->resetAfterTest(true);
208         $originaluniqueid = $this->model->get_unique_id();
210         // Same id across instances.
211         $this->model = new testable_model($this->modelobj);
212         $this->assertEquals($originaluniqueid, $this->model->get_unique_id());
214         // We will restore it later.
215         $originalversion = $this->modelobj->version;
217         // Generates a different id if timemodified changes.
218         $this->modelobj->version = $this->modelobj->version + 10;
219         $DB->update_record('analytics_models', $this->modelobj);
220         $this->model = new testable_model($this->modelobj);
221         $this->assertNotEquals($originaluniqueid, $this->model->get_unique_id());
223         // Restore original timemodified to continue testing.
224         $this->modelobj->version = $originalversion;
225         $DB->update_record('analytics_models', $this->modelobj);
226         // Same when updating through an action that changes the model.
227         $this->model = new testable_model($this->modelobj);
229         $this->model->mark_as_trained();
230         $this->assertEquals($originaluniqueid, $this->model->get_unique_id());
232         $this->model->enable();
233         $this->assertEquals($originaluniqueid, $this->model->get_unique_id());
235         // Wait 1 sec so the timestamp changes.
236         sleep(1);
237         $this->model->enable('\core\analytics\time_splitting\quarters');
238         $this->assertNotEquals($originaluniqueid, $this->model->get_unique_id());
239     }
241     /**
242      * test_exists
243      *
244      * @return void
245      */
246     public function test_exists() {
247         $this->resetAfterTest(true);
249         global $DB;
251         $count = $DB->count_records('analytics_models');
253         // No new models added if the builtin ones already exist.
254         \core_analytics\manager::add_builtin_models();
255         $this->assertCount($count, $DB->get_records('analytics_models'));
257         $target = \core_analytics\manager::get_target('\core\analytics\target\no_teaching');
258         $this->assertTrue(\core_analytics\model::exists($target));
259     }
261     /**
262      * test_model_timelimit
263      *
264      * @return null
265      */
266     public function test_model_timelimit() {
267         global $DB;
269         $this->resetAfterTest(true);
271         set_config('modeltimelimit', 2, 'analytics');
273         $courses = array();
274         for ($i = 0; $i < 5; $i++) {
275             $course = $this->getDataGenerator()->create_course();
276             $analysable = new \core_analytics\course($course);
277             $courses[$analysable->get_id()] = $course;
278         }
280         $target = new test_target_course_level_shortname();
281         $analyser = new test_analyser(1, $target, [], [], []);
283         // Each analysable element takes 1.1 secs, so the max (and likely) number of analysable
284         // elements that will be processed is 2.
285         $analyser->get_analysable_data(false);
286         $params = array('modelid' => 1, 'action' => 'prediction');
287         $this->assertLessThanOrEqual(2, $DB->count_records('analytics_used_analysables', $params));
289         $analyser->get_analysable_data(false);
290         $this->assertLessThanOrEqual(4, $DB->count_records('analytics_used_analysables', $params));
292         // Check that analysable elements have been processed following the analyser order
293         // (course->sortorder here). We can not check this nicely after next get_analysable_data round
294         // because the first analysed element will be analysed again.
295         $analysedelems = $DB->get_records('analytics_used_analysables', $params, 'timeanalysed ASC');
296         // Just a default for the first checked element.
297         $last = (object)['sortorder' => PHP_INT_MAX];
298         foreach ($analysedelems as $analysed) {
299             if ($courses[$analysed->analysableid]->sortorder > $last->sortorder) {
300                 $this->fail('Analysable elements have not been analysed sorted by course sortorder.');
301             }
302             $last = $courses[$analysed->analysableid];
303         }
305         $analyser->get_analysable_data(false);
306         $this->assertGreaterThanOrEqual(5, $DB->count_records('analytics_used_analysables', $params));
308         // New analysable elements are immediately pulled.
309         $this->getDataGenerator()->create_course();
310         $analyser->get_analysable_data(false);
311         $this->assertGreaterThanOrEqual(6, $DB->count_records('analytics_used_analysables', $params));
313         // Training and prediction data do not get mixed.
314         $analyser->get_analysable_data(true);
315         $params = array('modelid' => 1, 'action' => 'training');
316         $this->assertLessThanOrEqual(2, $DB->count_records('analytics_used_analysables', $params));
317     }
319     /**
320      * Generates a model log record.
321      */
322     private function add_fake_log() {
323         global $DB, $USER;
325         $log = new stdClass();
326         $log->modelid = $this->modelobj->id;
327         $log->version = $this->modelobj->version;
328         $log->target = $this->modelobj->target;
329         $log->indicators = $this->modelobj->indicators;
330         $log->score = 1;
331         $log->info = json_encode([]);
332         $log->dir = 'not important';
333         $log->timecreated = time();
334         $log->usermodified = $USER->id;
335         $DB->insert_record('analytics_models_log', $log);
336     }
339 /**
340  * Testable version to change methods' visibility.
341  *
342  * @package   core_analytics
343  * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com}
344  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
345  */
346 class testable_model extends \core_analytics\model {
348     /**
349      * get_output_dir
350      *
351      * @param array $subdirs
352      * @param bool $onlymodelid
353      * @return string
354      */
355     public function get_output_dir($subdirs = array(), $onlymodelid = false) {
356         return parent::get_output_dir($subdirs, $onlymodelid);
357     }
359     /**
360      * init_analyser
361      *
362      * @param array $options
363      * @return void
364      */
365     public function init_analyser($options = array()) {
366         parent::init_analyser($options);
367     }