Merge branch 'MDL-65307_master' of git://github.com/dmonllao/moodle
authorEloy Lafuente (stronk7) <stronk7@moodle.org>
Mon, 15 Apr 2019 18:43:13 +0000 (20:43 +0200)
committerEloy Lafuente (stronk7) <stronk7@moodle.org>
Mon, 15 Apr 2019 18:43:13 +0000 (20:43 +0200)
326 files changed:
admin/settings/analytics.php
admin/settings/courses.php
admin/settings/development.php
admin/settings/subsystems.php
admin/tool/analytics/amd/build/model.min.js
admin/tool/analytics/amd/src/model.js
admin/tool/analytics/classes/output/form/edit_model.php
admin/tool/analytics/classes/output/helper.php
admin/tool/analytics/classes/output/invalid_analysables.php
admin/tool/analytics/classes/output/models_list.php
admin/tool/analytics/classes/task/predict_models.php
admin/tool/analytics/classes/task/train_models.php
admin/tool/analytics/cli/evaluate_model.php
admin/tool/analytics/createmodel.php
admin/tool/analytics/lang/en/tool_analytics.php
admin/tool/analytics/model.php
admin/tool/analytics/templates/evaluation_options.mustache [new file with mode: 0644]
admin/tool/dataprivacy/classes/api.php
admin/tool/dataprivacy/classes/external/data_request_exporter.php
admin/tool/dataprivacy/classes/output/data_registry_page.php
admin/tool/dataprivacy/classes/output/data_requests_table.php
admin/tool/dataprivacy/createdatarequest.php
admin/tool/dataprivacy/createdatarequest_form.php
admin/tool/dataprivacy/db/access.php
admin/tool/dataprivacy/lang/en/tool_dataprivacy.php
admin/tool/dataprivacy/lib.php
admin/tool/dataprivacy/resubmitrequest.php
admin/tool/dataprivacy/tests/api_test.php
admin/tool/dataprivacy/tests/behat/datadelete.feature
admin/tool/dataprivacy/version.php
admin/tool/generator/classes/make_testplan_form.php
admin/tool/generator/classes/testplan_backend.php
admin/tool/generator/upgrade.txt [new file with mode: 0644]
admin/tool/log/classes/helper/buffered_writer.php
admin/tool/log/classes/helper/reader.php
admin/tool/log/classes/local/privacy/helper.php
admin/tool/log/store/database/classes/log/store.php
admin/tool/log/store/database/db/upgrade.php
admin/tool/log/store/database/lang/en/logstore_database.php
admin/tool/log/store/database/settings.php
admin/tool/log/store/database/tests/store_test.php
admin/tool/log/store/database/version.php
admin/tool/log/store/standard/classes/log/store.php
admin/tool/log/store/standard/db/upgrade.php
admin/tool/log/store/standard/lang/en/logstore_standard.php
admin/tool/log/store/standard/settings.php
admin/tool/log/store/standard/tests/store_test.php
admin/tool/log/store/standard/version.php
admin/tool/log/upgrade.txt
admin/tool/mobile/classes/external.php
admin/tool/mobile/db/services.php
admin/tool/mobile/lang/en/tool_mobile.php
admin/tool/mobile/tests/externallib_test.php
admin/tool/mobile/upgrade.txt
admin/tool/recyclebin/classes/category_bin.php
admin/tool/recyclebin/classes/course_bin.php
admin/tool/recyclebin/tests/category_bin_test.php
admin/tool/recyclebin/tests/course_bin_test.php
admin/user/user_bulk_message.php
analytics/classes/analysable.php
analytics/classes/analysis.php [new file with mode: 0644]
analytics/classes/course.php
analytics/classes/dataset_manager.php
analytics/classes/insights_generator.php [new file with mode: 0644]
analytics/classes/local/analyser/base.php
analytics/classes/local/analyser/by_course.php
analytics/classes/local/analyser/sitewide.php
analytics/classes/local/analysis/result.php [new file with mode: 0644]
analytics/classes/local/analysis/result_array.php [new file with mode: 0644]
analytics/classes/local/analysis/result_file.php [new file with mode: 0644]
analytics/classes/local/target/base.php
analytics/classes/local/target/binary.php
analytics/classes/local/target/discrete.php
analytics/classes/local/time_splitting/base.php
analytics/classes/local/time_splitting/periodic.php [new file with mode: 0644]
analytics/classes/local/time_splitting/upcoming_periodic.php [new file with mode: 0644]
analytics/classes/manager.php
analytics/classes/model.php
analytics/classes/prediction_action.php
analytics/classes/user.php [new file with mode: 0644]
analytics/templates/insight_info_message.mustache [new file with mode: 0644]
analytics/templates/insight_info_message_prediction.mustache [new file with mode: 0644]
analytics/tests/dataset_manager_test.php
analytics/tests/fixtures/test_analysis.php [moved from analytics/tests/fixtures/test_analyser.php with 79% similarity]
analytics/tests/fixtures/test_indicator_null.php [new file with mode: 0644]
analytics/tests/fixtures/test_site_users_analyser.php
analytics/tests/fixtures/test_target_shortname.php
analytics/tests/fixtures/test_target_site_users.php
analytics/tests/fixtures/test_timesplitting_seconds.php [new file with mode: 0644]
analytics/tests/fixtures/test_timesplitting_weekly.php [new file with mode: 0644]
analytics/tests/manager_test.php
analytics/tests/model_test.php
analytics/tests/prediction_test.php
analytics/tests/privacy_test.php
analytics/upgrade.txt
auth/email/tests/behat/signup.feature
backup/backup.class.php
backup/backup.php
backup/controller/backup_controller.class.php
backup/controller/restore_controller.class.php
backup/externallib.php [new file with mode: 0644]
backup/restore.php
backup/restorefile.php
backup/tests/async_backup_test.php [new file with mode: 0644]
backup/tests/async_restore_test.php [new file with mode: 0644]
backup/util/dbops/backup_controller_dbops.class.php
backup/util/dbops/restore_controller_dbops.class.php
backup/util/helper/async_helper.class.php [new file with mode: 0644]
backup/util/helper/tests/async_helper_test.php [new file with mode: 0644]
backup/util/includes/backup_includes.php
backup/util/includes/restore_includes.php
backup/util/ui/amd/build/async_backup.min.js [new file with mode: 0644]
backup/util/ui/amd/src/async_backup.js [new file with mode: 0644]
backup/util/ui/renderer.php
backup/util/ui/tests/behat/duplicate_activities.feature
badges/criteria/award_criteria_courseset.php
badges/cron.php [deleted file]
blocks/course_list/block_course_list.php
blocks/html/lib.php
blocks/myoverview/amd/build/view.min.js
blocks/myoverview/amd/src/view.js
blocks/myoverview/block_myoverview.php
blocks/myoverview/classes/output/main.php
blocks/myoverview/lang/en/block_myoverview.php
blocks/myoverview/lib.php
blocks/myoverview/settings.php [new file with mode: 0644]
blocks/myoverview/templates/courses-view.mustache
blocks/myoverview/templates/view-list.mustache
blocks/myoverview/templates/view-summary.mustache
blog/classes/external/post_exporter.php
blog/tests/external_test.php
blog/upgrade.txt
cache/classes/loaders.php
cache/stores/mongodb/addinstanceform.php
calendar/lib.php
calendar/view.php
course/amd/build/actions.min.js
course/amd/src/actions.js
course/classes/analytics/indicator/activities_due.php [new file with mode: 0644]
course/classes/category.php
course/classes/list_element.php
course/classes/management/helper.php
course/classes/management_renderer.php
course/classes/search/course.php
course/classes/search/customfield.php
course/completion_form.php
course/delete.php
course/externallib.php
course/format/renderer.php
course/format/topics/renderer.php
course/index.php
course/info.php
course/lib.php
course/management.php
course/renderer.php
course/search.php
course/tests/behat/behat_course.php
course/tests/behat/course_browsing.feature [new file with mode: 0644]
course/tests/category_test.php
course/upgrade.txt
course/view.php
dataformat/pdf/classes/privacy/provider.php [new file with mode: 0644]
dataformat/pdf/classes/writer.php [new file with mode: 0644]
dataformat/pdf/lang/en/dataformat_pdf.php [new file with mode: 0644]
dataformat/pdf/version.php [new file with mode: 0644]
enrol/externallib.php
enrol/guest/classes/external.php
enrol/index.php
enrol/self/externallib.php
grade/import/csv/classes/load_data.php
grade/import/csv/tests/load_data_test.php
install/lang/ar/admin.php
install/lang/de_wp/langconfig.php [new file with mode: 0644]
install/lang/en_wp/langconfig.php [new file with mode: 0644]
install/lang/es_wp/langconfig.php [new file with mode: 0644]
install/lang/pt/admin.php
lang/en/admin.php
lang/en/analytics.php
lang/en/backup.php
lang/en/course.php
lang/en/error.php
lang/en/moodle.php
lang/en/role.php
lib/accesslib.php
lib/amd/build/form-course-selector.min.js
lib/amd/build/icon_system_fontawesome.min.js
lib/amd/build/icon_system_standard.min.js
lib/amd/src/form-course-selector.js
lib/amd/src/icon_system_fontawesome.js
lib/amd/src/icon_system_standard.js
lib/authlib.php
lib/badgeslib.php
lib/behat/classes/partial_named_selector.php
lib/classes/analytics/analyser/courses.php
lib/classes/analytics/analyser/site_courses.php
lib/classes/analytics/analyser/student_enrolments.php
lib/classes/analytics/analyser/users.php [new file with mode: 0644]
lib/classes/analytics/target/course_dropout.php
lib/classes/analytics/target/course_enrolments.php
lib/classes/analytics/target/course_gradetopass.php [new file with mode: 0644]
lib/classes/analytics/target/no_teaching.php
lib/classes/analytics/time_splitting/upcoming_3_days.php [new file with mode: 0644]
lib/classes/analytics/time_splitting/upcoming_fortnight.php [new file with mode: 0644]
lib/classes/analytics/time_splitting/upcoming_week.php [new file with mode: 0644]
lib/classes/component.php
lib/classes/lock/db_record_lock_factory.php
lib/classes/lock/file_lock_factory.php
lib/classes/lock/postgres_lock_factory.php
lib/classes/message/message.php
lib/classes/plugin_manager.php
lib/classes/progress/db_updater.php [new file with mode: 0644]
lib/classes/task/asynchronous_backup_task.php [new file with mode: 0644]
lib/classes/task/asynchronous_restore_task.php [new file with mode: 0644]
lib/classes/task/badges_cron_task.php
lib/classes/task/badges_message_task.php [new file with mode: 0644]
lib/datalib.php
lib/db/access.php
lib/db/analytics.php
lib/db/install.xml
lib/db/messages.php
lib/db/services.php
lib/db/tasks.php
lib/db/upgrade.php
lib/deprecatedlib.php
lib/filebrowser/file_info_context_coursecat.php
lib/filelib.php
lib/form/course.php
lib/messagelib.php
lib/mlbackend/php/classes/processor.php
lib/moodlelib.php
lib/navigationlib.php
lib/setuplib.php
lib/templates/async_backup_progress.mustache [moved from admin/tool/analytics/templates/evaluation_mode_selection.mustache with 50% similarity]
lib/templates/async_backup_progress_row.mustache [new file with mode: 0644]
lib/templates/async_backup_status.mustache [new file with mode: 0644]
lib/templates/async_restore_progress_row.mustache [new file with mode: 0644]
lib/testing/classes/util.php
lib/tests/analysers_test.php
lib/tests/component_test.php
lib/tests/fixtures/deprecated_analyser.php [new file with mode: 0644]
lib/tests/lock_test.php
lib/tests/targets_test.php
lib/tests/time_splittings_test.php
lib/upgrade.txt
lib/xhprof/readme_moodle.txt
lib/xhprof/xhprof_moodle.php
message/classes/api.php
message/classes/helper.php
message/externallib.php
message/lib.php
message/tests/behat/behat_message.php
message/tests/behat/message_drawer_manage_contacts.feature
message/tests/behat/message_send_messages.feature [new file with mode: 0644]
mod/book/lib.php
mod/book/tests/lib_test.php
mod/book/upgrade.txt
mod/data/classes/external/record_exporter.php
mod/data/tests/externallib_test.php
mod/data/upgrade.txt
mod/forum/backup/moodle2/backup_forum_stepslib.php
mod/forum/backup/moodle2/restore_forum_stepslib.php
mod/forum/classes/local/data_mappers/legacy/forum.php
mod/forum/classes/local/entities/forum.php
mod/forum/classes/local/factories/entity.php
mod/forum/classes/local/managers/capability.php
mod/forum/classes/local/renderers/discussion.php
mod/forum/classes/local/renderers/discussion_list.php
mod/forum/db/access.php
mod/forum/db/install.xml
mod/forum/db/upgrade.php
mod/forum/externallib.php
mod/forum/lang/en/forum.php
mod/forum/lib.php
mod/forum/locallib.php
mod/forum/mod_form.php
mod/forum/tests/entities_forum_test.php
mod/forum/tests/externallib_test.php
mod/forum/tests/lib_test.php
mod/forum/tests/locallib_test.php [new file with mode: 0644]
mod/forum/tests/vaults_post_test.php
mod/forum/upgrade.txt
mod/forum/version.php
mod/glossary/classes/external.php
mod/glossary/tests/external_test.php
mod/glossary/upgrade.txt
mod/lesson/backup/moodle2/backup_lesson_stepslib.php
mod/lesson/backup/moodle2/restore_lesson_stepslib.php
mod/lesson/classes/privacy/provider.php
mod/lesson/continue.php
mod/lesson/editpage.php
mod/lesson/essay.php
mod/lesson/index.php
mod/lesson/lang/en/lesson.php
mod/lesson/lib.php
mod/lesson/locallib.php
mod/lesson/pagetypes/essay.php
mod/lesson/pagetypes/numerical.php
mod/lesson/pagetypes/shortanswer.php
mod/lesson/report.php
mod/lesson/tests/behat/all_other_answers_catch.feature [new file with mode: 0644]
mod/lesson/tests/behat/wrong_answer_continue.feature [new file with mode: 0644]
mod/lesson/upgrade.txt
mod/wiki/classes/external.php
mod/wiki/tests/externallib_test.php
mod/wiki/upgrade.txt
report/insights/classes/output/insight.php
report/insights/classes/output/insights_list.php
report/insights/done.php [new file with mode: 0644]
report/insights/insights.php
report/insights/lang/en/report_insights.php
report/insights/lib.php
report/insights/templates/insight_details.mustache
report/insights/templates/insights_list.mustache
repository/webdav/lib.php
search/classes/manager.php
tag/classes/external.php
tag/classes/external/tag_area_exporter.php [new file with mode: 0644]
tag/classes/external/tag_collection_exporter.php [new file with mode: 0644]
tag/classes/external/tag_item_exporter.php [new file with mode: 0644]
tag/classes/external/util.php [new file with mode: 0644]
tag/tests/external_test.php
theme/boost/scss/moodle/drawer.scss
theme/boost/style/moodle.css
theme/classic/style/moodle.css
user/classes/analytics/target/upcoming_activities_due.php [new file with mode: 0644]
version.php

index 5f43a52..dfbb1d6 100644 (file)
@@ -75,7 +75,7 @@ if ($hassiteconfig) {
             $defaultreader, $options));
 
         // Enable/disable time splitting methods.
-        $alltimesplittings = \core_analytics\manager::get_all_time_splittings();
+        $alltimesplittings = \core_analytics\manager::get_time_splitting_methods_for_evaluation(true);
 
         $timesplittingoptions = array();
         $timesplittingdefaults = array('\core\analytics\time_splitting\quarters_accum',
index f214ab4..99892d6 100644 (file)
@@ -458,4 +458,28 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) {
 
     $ADMIN->add('backups', $temp);
 
+    // Create a page for asynchronous backup and restore configuration and defaults.
+    if (!empty($CFG->enableasyncbackup)) {  // Only add settings if async mode is enable at site level.
+        $temp = new admin_settingpage('asyncgeneralsettings', new lang_string('asyncgeneralsettings', 'backup'));
+
+        $temp->add(new admin_setting_configcheckbox(
+                'backup/backup_async_message_users',
+                new lang_string('asyncemailenable', 'backup'),
+                new lang_string('asyncemailenabledetail', 'backup'), 0));
+
+        $temp->add(new admin_setting_configtext(
+                'backup/backup_async_message_subject',
+                new lang_string('asyncmessagesubject', 'backup'),
+                new lang_string('asyncmessagesubjectdetail', 'backup'),
+                new lang_string('asyncmessagesubjectdefault', 'backup')));
+
+        $temp->add(new admin_setting_confightmleditor(
+                'backup/backup_async_message',
+                new lang_string('asyncmessagebody', 'backup'),
+                new lang_string('asyncmessagebodydetail', 'backup'),
+                new lang_string('asyncmessagebodydefault', 'backup')));
+
+        $ADMIN->add('backups', $temp);
+    }
+
 }
index 61d82c3..58493dd 100644 (file)
@@ -59,6 +59,8 @@ if ($hassiteconfig) { // speedup for non-admins, add all caps used on this page
     $temp->add(new admin_setting_configcheckbox('profilingallowme', new lang_string('profilingallowme', 'admin'), new lang_string('profilingallowme_help', 'admin'), false));
     // Allow PROFILEALL/PROFILEALLSTOP GPC.
     $temp->add(new admin_setting_configcheckbox('profilingallowall', new lang_string('profilingallowall', 'admin'), new lang_string('profilingallowall_help', 'admin'), false));
+    $temp->add(new admin_setting_configtext('profilingslow', new lang_string('profilingslow', 'admin'),
+        new lang_string('profilingslow_help', 'admin'), 0, PARAM_FLOAT));
     // TODO: Allow to skip PHP functions (XHPROF_FLAGS_NO_BUILTINS)
     // TODO: Allow to skip call_user functions (ignored_functions array)
     // Specify the life time (in minutes) of profiling runs.
index e5c660b..0b62749 100644 (file)
@@ -51,4 +51,7 @@ if ($hassiteconfig) { // speedup for non-admins, add all caps used on this page
 
     $optionalsubsystems->add(new admin_setting_configcheckbox('enablecoursepublishing',
         new lang_string('enablecoursepublishing', 'hub'), new lang_string('enablecoursepublishing_help', 'hub'), 0));
+
+    $optionalsubsystems->add(new admin_setting_configcheckbox('enableasyncbackup', new lang_string('enableasyncbackup', 'backup'),
+        new lang_string('enableasyncbackup_help', 'backup'), 0, 1, 0));
 }
index 9e958db..fd1374c 100644 (file)
Binary files a/admin/tool/analytics/amd/build/model.min.js and b/admin/tool/analytics/amd/build/model.min.js differ
index 8d35e86..7a33345 100644 (file)
@@ -101,50 +101,53 @@ define(['jquery', 'core/str', 'core/log', 'core/notification', 'core/modal_facto
         },
 
         /**
-         * Displays a select-evaluation-mode choice.
+         * Displays evaluation mode and time-splitting method choices.
          *
          * @param  {String}  actionId
          * @param  {Boolean} trainedOnlyExternally
          */
-        selectEvaluationMode: function(actionId, trainedOnlyExternally) {
+        selectEvaluationOptions: function(actionId, trainedOnlyExternally, timeSplittingMethods) {
             $('[data-action-id="' + actionId + '"]').on('click', function(ev) {
                 ev.preventDefault();
 
                 var a = $(ev.currentTarget);
 
-                if (!trainedOnlyExternally) {
-                    // We can not evaluate trained models if the model was trained using data from this site.
-                    // Default to evaluate the model configuration if that is the case.
-                    window.location.href = a.attr('href');
-                    return;
-                }
-
                 var stringsPromise = Str.get_strings([
                     {
                         key: 'evaluatemodel',
                         component: 'tool_analytics'
                     }, {
-                        key: 'evaluationmode',
+                        key: 'evaluate',
                         component: 'tool_analytics'
                     }
                 ]);
                 var modalPromise = ModalFactory.create({type: ModalFactory.types.SAVE_CANCEL});
-                var bodyPromise = Templates.render('tool_analytics/evaluation_mode_selection', {});
+                var bodyPromise = Templates.render('tool_analytics/evaluation_options', {
+                    trainedexternally: trainedOnlyExternally,
+                    timesplittingmethods: timeSplittingMethods
+                });
 
                 $.when(stringsPromise, modalPromise).then(function(strings, modal) {
 
 
                     modal.getRoot().on(ModalEvents.hidden, modal.destroy.bind(modal));
 
-                    modal.setTitle(strings[1]);
-                    modal.setSaveButtonText(strings[0]);
+                    modal.setTitle(strings[0]);
+                    modal.setSaveButtonText(strings[1]);
                     modal.setBody(bodyPromise);
 
                     modal.getRoot().on(ModalEvents.save, function() {
+
+                        // Evaluation mode.
                         var evaluationMode = $("input[name='evaluationmode']:checked").val();
                         if (evaluationMode == 'trainedmodel') {
                             a.attr('href', a.attr('href') + '&mode=trainedmodel');
                         }
+
+                        // Selected time-splitting id.
+                        var timeSplittingMethod = $("#id-evaluation-timesplitting").val();
+                        a.attr('href', a.attr('href') + '&timesplitting=' + timeSplittingMethod);
+
                         window.location.href = a.attr('href');
                         return;
                     });
index 2c0be86..b529d50 100644 (file)
@@ -45,13 +45,14 @@ class edit_model extends \moodleform {
 
         $mform = $this->_form;
 
-        if ($this->_customdata['trainedmodel']) {
+        if ($this->_customdata['trainedmodel'] && $this->_customdata['staticmodel'] === false) {
             $message = get_string('edittrainedwarning', 'tool_analytics');
             $mform->addElement('html', $OUTPUT->notification($message, \core\output\notification::NOTIFY_WARNING));
         }
 
         $mform->addElement('advcheckbox', 'enabled', get_string('enabled', 'tool_analytics'));
 
+        // Target.
         if (!empty($this->_customdata['targets'])) {
             $targets = array('' => '');
             foreach ($this->_customdata['targets'] as $classname => $target) {
@@ -64,41 +65,48 @@ class edit_model extends \moodleform {
             $mform->addRule('target', get_string('required'), 'required', null, 'client');
         }
 
-        $indicators = array();
-        foreach ($this->_customdata['indicators'] as $classname => $indicator) {
-            $optionname = \tool_analytics\output\helper::class_to_option($classname);
-            $indicators[$optionname] = $indicator->get_name();
+        // Indicators.
+        if (!$this->_customdata['staticmodel']) {
+            $indicators = array();
+            foreach ($this->_customdata['indicators'] as $classname => $indicator) {
+                $optionname = \tool_analytics\output\helper::class_to_option($classname);
+                $indicators[$optionname] = $indicator->get_name();
+            }
+            $options = array(
+                'multiple' => true
+            );
+            $mform->addElement('autocomplete', 'indicators', get_string('indicators', 'tool_analytics'), $indicators, $options);
+            $mform->setType('indicators', PARAM_ALPHANUMEXT);
+            $mform->addHelpButton('indicators', 'indicators', 'tool_analytics');
         }
-        $options = array(
-            'multiple' => true
-        );
-        $mform->addElement('autocomplete', 'indicators', get_string('indicators', 'tool_analytics'), $indicators, $options);
-        $mform->setType('indicators', PARAM_ALPHANUMEXT);
 
+        // Time-splitting methods.
         $timesplittings = array('' => '');
         foreach ($this->_customdata['timesplittings'] as $classname => $timesplitting) {
             $optionname = \tool_analytics\output\helper::class_to_option($classname);
             $timesplittings[$optionname] = $timesplitting->get_name();
         }
-
         $mform->addElement('select', 'timesplitting', get_string('timesplittingmethod', 'analytics'), $timesplittings);
         $mform->addHelpButton('timesplitting', 'timesplittingmethod', 'analytics');
 
-        $defaultprocessor = \core_analytics\manager::get_predictions_processor_name(
-            \core_analytics\manager::get_predictions_processor()
-        );
-        $predictionprocessors = ['' => get_string('defaultpredictoroption', 'analytics', $defaultprocessor)];
-        foreach ($this->_customdata['predictionprocessors'] as $classname => $predictionsprocessor) {
-            if ($predictionsprocessor->is_ready() !== true) {
-                continue;
+        // Predictions processor.
+        if (!$this->_customdata['staticmodel']) {
+            $defaultprocessor = \core_analytics\manager::get_predictions_processor_name(
+                \core_analytics\manager::get_predictions_processor()
+            );
+            $predictionprocessors = ['' => get_string('defaultpredictoroption', 'analytics', $defaultprocessor)];
+            foreach ($this->_customdata['predictionprocessors'] as $classname => $predictionsprocessor) {
+                if ($predictionsprocessor->is_ready() !== true) {
+                    continue;
+                }
+                $optionname = \tool_analytics\output\helper::class_to_option($classname);
+                $predictionprocessors[$optionname] = \core_analytics\manager::get_predictions_processor_name($predictionsprocessor);
             }
-            $optionname = \tool_analytics\output\helper::class_to_option($classname);
-            $predictionprocessors[$optionname] = \core_analytics\manager::get_predictions_processor_name($predictionsprocessor);
-        }
 
-        $mform->addElement('select', 'predictionsprocessor', get_string('predictionsprocessor', 'analytics'),
-            $predictionprocessors);
-        $mform->addHelpButton('predictionsprocessor', 'predictionsprocessor', 'analytics');
+            $mform->addElement('select', 'predictionsprocessor', get_string('predictionsprocessor', 'analytics'),
+                $predictionprocessors);
+            $mform->addHelpButton('predictionsprocessor', 'predictionsprocessor', 'analytics');
+        }
 
         if (!empty($this->_customdata['id'])) {
             $mform->addElement('hidden', 'id', $this->_customdata['id']);
@@ -129,13 +137,15 @@ class edit_model extends \moodleform {
             }
         }
 
-        if (empty($data['indicators'])) {
-            $errors['indicators'] = get_string('errornoindicators', 'analytics');
-        } else {
-            foreach ($data['indicators'] as $indicator) {
-                $realindicatorname = \tool_analytics\output\helper::option_to_class($indicator);
-                if (\core_analytics\manager::is_valid($realindicatorname, '\core_analytics\local\indicator\base') === false) {
-                    $errors['indicators'] = get_string('errorinvalidindicator', 'analytics', $realindicatorname);
+        if (!$this->_customdata['staticmodel']) {
+            if (empty($data['indicators'])) {
+                $errors['indicators'] = get_string('errornoindicators', 'analytics');
+            } else {
+                foreach ($data['indicators'] as $indicator) {
+                    $realindicatorname = \tool_analytics\output\helper::option_to_class($indicator);
+                    if (\core_analytics\manager::is_valid($realindicatorname, '\core_analytics\local\indicator\base') === false) {
+                        $errors['indicators'] = get_string('errorinvalidindicator', 'analytics', $realindicatorname);
+                    }
                 }
             }
         }
index dba72e5..1db9981 100644 (file)
@@ -45,7 +45,7 @@ class helper {
         // Form field is PARAM_ALPHANUMEXT and we are sending fully qualified class names
         // as option names, but replacing the backslash for a string that is really unlikely
         // to ever be part of a class name.
-        return str_replace('\\', '2015102400ouuu', $class);
+        return str_replace('\\', '__', $class);
     }
 
     /**
@@ -56,7 +56,7 @@ class helper {
      */
     public static function option_to_class($option) {
         // Really unlikely but yeah, I'm a bad booyyy.
-        return str_replace('2015102400ouuu', '\\', $option);
+        return str_replace('__', '\\', $option);
     }
 
     /**
@@ -92,4 +92,17 @@ class helper {
         $PAGE->set_title($title);
         $PAGE->set_heading($title);
     }
+
+    /**
+     * Resets the current page.
+     *
+     * Note that this function can only be used by analytics pages that work at the system context.
+     *
+     * @return null
+     */
+    public static function reset_page() {
+        global $PAGE;
+        $PAGE->reset_theme_and_output();
+        $PAGE->set_context(\context_system::instance());
+    }
 }
index 0bb3902..dbdaa00 100644 (file)
@@ -76,13 +76,17 @@ class invalid_analysables implements \renderable, \templatable {
 
         $offset = $this->page * $this->perpage;
 
-        $analysables = $this->model->get_analyser(['notimesplitting' => true])->get_analysables();
+        $analysables = $this->model->get_analyser(['notimesplitting' => true])->get_analysables_iterator();
 
         $skipped = 0;
         $enoughresults = false;
         $morepages = false;
         $results = array();
-        foreach ($analysables as $key => $analysable) {
+        foreach ($analysables as $analysable) {
+
+            if (!$analysable) {
+                continue;
+            }
 
             $validtraining = $this->model->get_target()->is_valid_analysable($analysable, true);
             if ($validtraining === true) {
@@ -117,8 +121,6 @@ class invalid_analysables implements \renderable, \templatable {
                 $morepages = true;
                 break;
             }
-
-            unset($analysables[$key]);
         }
 
         // Prepare the context object.
index 5152b6d..b0431a3 100644 (file)
@@ -71,6 +71,17 @@ class models_list implements \renderable, \templatable {
             $onlycli = 1;
         }
 
+        // Evaluation options.
+        $timesplittingmethods = [
+            ['id' => 'all', 'text' => get_string('alltimesplittingmethods', 'tool_analytics')],
+        ];
+        foreach (\core_analytics\manager::get_time_splitting_methods_for_evaluation(true) as $timesplitting) {
+            $timesplittingmethods[] = [
+                'id' => \tool_analytics\output\helper::class_to_option($timesplitting->get_id()),
+                'text' => $timesplitting->get_name()->out(),
+            ];
+        }
+
         $data->models = array();
         foreach ($this->models as $model) {
             $modeldata = $model->export();
@@ -192,7 +203,16 @@ class models_list implements \renderable, \templatable {
                 $trainedonlyexternally = !$model->trained_locally() && $model->is_trained();
 
                 $actionid = 'evaluate-' . $model->get_id();
-                $PAGE->requires->js_call_amd('tool_analytics/model', 'selectEvaluationMode', [$actionid, $trainedonlyexternally]);
+
+                $modeltimesplittingmethods = $timesplittingmethods;
+                // Include the current time-splitting method as the default selection method the model already have one.
+                if ($model->get_model_obj()->timesplitting) {
+                    $currenttimesplitting = ['id' => 'current', 'text' => get_string('currenttimesplitting', 'tool_analytics')];
+                    array_unshift($modeltimesplittingmethods, $currenttimesplitting);
+                }
+
+                $evaluateparams = [$actionid, $trainedonlyexternally, $modeltimesplittingmethods];
+                $PAGE->requires->js_call_amd('tool_analytics/model', 'selectEvaluationOptions', $evaluateparams);
                 $urlparams['action'] = 'evaluate';
                 $url = new \moodle_url('model.php', $urlparams);
                 $icon = new \action_menu_link_secondary($url, new \pix_icon('i/calc', get_string('evaluate', 'tool_analytics')),
@@ -210,12 +230,10 @@ class models_list implements \renderable, \templatable {
             }
 
             // Edit model.
-            if (!$model->is_static()) {
-                $urlparams['action'] = 'edit';
-                $url = new \moodle_url('model.php', $urlparams);
-                $icon = new \action_menu_link_secondary($url, new \pix_icon('t/edit', get_string('edit')), get_string('edit'));
-                $actionsmenu->add($icon);
-            }
+            $urlparams['action'] = 'edit';
+            $url = new \moodle_url('model.php', $urlparams);
+            $icon = new \action_menu_link_secondary($url, new \pix_icon('t/edit', get_string('edit')), get_string('edit'));
+            $actionsmenu->add($icon);
 
             // Enable / disable.
             if ($model->is_enabled() || !empty($modeldata->timesplitting)) {
@@ -280,14 +298,17 @@ class models_list implements \renderable, \templatable {
                 $actionsmenu->add($icon);
             }
 
-            $actionid = 'delete-' . $model->get_id();
-            $PAGE->requires->js_call_amd('tool_analytics/model', 'confirmAction', [$actionid, 'delete']);
-            $urlparams['action'] = 'delete';
-            $url = new \moodle_url('model.php', $urlparams);
-            $icon = new \action_menu_link_secondary($url, new \pix_icon('t/delete',
-                get_string('delete', 'tool_analytics')), get_string('delete', 'tool_analytics'),
-                ['data-action-id' => $actionid]);
-            $actionsmenu->add($icon);
+            // Delete model.
+            if (!$model->is_static()) {
+                $actionid = 'delete-' . $model->get_id();
+                $PAGE->requires->js_call_amd('tool_analytics/model', 'confirmAction', [$actionid, 'delete']);
+                $urlparams['action'] = 'delete';
+                $url = new \moodle_url('model.php', $urlparams);
+                $icon = new \action_menu_link_secondary($url, new \pix_icon('t/delete',
+                    get_string('delete', 'tool_analytics')), get_string('delete', 'tool_analytics'),
+                    ['data-action-id' => $actionid]);
+                $actionsmenu->add($icon);
+            }
 
             $modeldata->actions = $actionsmenu->export_for_template($output);
 
index 83894be..91f3c85 100644 (file)
@@ -60,6 +60,10 @@ class predict_models extends \core\task\scheduled_task {
 
         foreach ($models as $model) {
             $result = $model->predict();
+
+            // Reset the page as some indicators may call external functions that overwrite the page context.
+            \tool_analytics\output\helper::reset_page();
+
             if ($result) {
                 echo $OUTPUT->heading(get_string('modelresults', 'tool_analytics', $model->get_target()->get_name()));
                 $renderer = $PAGE->get_renderer('tool_analytics');
index b017e9a..3c0c3c9 100644 (file)
@@ -71,6 +71,10 @@ class train_models extends \core\task\scheduled_task {
             }
 
             $result = $model->train();
+
+            // Reset the page as some indicators may call external functions that overwrite the page context.
+            \tool_analytics\output\helper::reset_page();
+
             if ($result) {
                 echo $OUTPUT->heading(get_string('modelresults', 'tool_analytics', $model->get_target()->get_name()));
 
index 2836ca7..8473aba 100644 (file)
@@ -111,6 +111,9 @@ $analyseroptions = array(
 // Evaluate its suitability to predict accurately.
 $results = $model->evaluate($analyseroptions);
 
+// Reset the page as some indicators may call external functions that overwrite the page context.
+\tool_analytics\output\helper::reset_page();
+
 $renderer = $PAGE->get_renderer('tool_analytics');
 echo $renderer->render_evaluate_results($results, $model->get_analyser()->get_logs());
 
index bf1701c..e5b9b92 100644 (file)
@@ -40,9 +40,10 @@ $targets = array_filter(\core_analytics\manager::get_all_targets(), function($ta
 
 $customdata = array(
     'trainedmodel' => false,
+    'staticmodel' => false,
     'targets' => $targets,
     'indicators' => \core_analytics\manager::get_all_indicators(),
-    'timesplittings' => \core_analytics\manager::get_enabled_time_splitting_methods(),
+    'timesplittings' => \core_analytics\manager::get_all_time_splittings(),
     'predictionprocessors' => \core_analytics\manager::get_all_prediction_processors(),
 );
 $mform = new \tool_analytics\output\form\edit_model(null, $customdata);
index dc0c5c9..34c1585 100644 (file)
@@ -24,6 +24,7 @@
 
 $string['accuracy'] = 'Accuracy';
 $string['allpredictions'] = 'All predictions';
+$string['alltimesplittingmethods'] = 'All time-splitting methods';
 $string['analysingsitedata'] = 'Analysing the site';
 $string['analyticmodels'] = 'Analytics models';
 $string['bettercli'] = 'Evaluating models and generating predictions may involve heavy processing. It is recommended to run these actions from the command line.';
@@ -36,6 +37,7 @@ $string['clienablemodel'] = 'You can enable the model by selecting a time-splitt
 $string['clievaluationandpredictions'] = 'A scheduled task iterates through enabled models and gets predictions. Models evaluation via the web interface is disabled. You can allow these processes to be executed manually via the web interface by disabling the <a href="{$a}">\'onlycli\'</a> analytics setting.';
 $string['clievaluationandpredictionsnoadmin'] = 'A scheduled task iterates through enabled models and gets predictions. Models evaluation via the web interface is disabled. It may be enabled by a site administrator.';
 $string['createmodel'] = 'Create model';
+$string['currenttimesplitting'] = 'Current time-splitting method';
 $string['delete'] = 'Delete';
 $string['deletemodelconfirmation'] = 'Are you sure you want to delete "{$a}"? These changes can not be reverted.';
 $string['disabled'] = 'Disabled';
@@ -46,7 +48,6 @@ $string['errorcantenablenotimesplitting'] = 'You need to select a time-splitting
 $string['errornoenabledandtrainedmodels'] = 'There are no enabled and trained models to predict.';
 $string['errornoenabledmodels'] = 'There are no enabled models to train.';
 $string['errornoexport'] = 'Only trained models can be exported';
-$string['errornostaticedit'] = 'Models based on assumptions cannot be edited.';
 $string['errornostaticevaluated'] = 'Models based on assumptions cannot be evaluated. They are always 100% correct according to how they were defined.';
 $string['errornostaticlog'] = 'Models based on assumptions cannot be evaluated because there is no performance log.';
 $string['erroronlycli'] = 'Execution only allowed via command line';
@@ -78,6 +79,8 @@ $string['getpredictions'] = 'Get predictions';
 $string['goodmodel'] = 'This is a good model for using to obtain predictions. Enable it to start obtaining predictions.';
 $string['importmodel'] = 'Import model';
 $string['indicators'] = 'Indicators';
+$string['indicators_help'] = 'The indicators are what you think will lead to an accurate prediction of the target.';
+$string['indicators_link'] = 'Indicators';
 $string['info'] = 'Info';
 $string['ignoreversionmismatches'] = 'Ignore version mismatches';
 $string['ignoreversionmismatchescheckbox'] = 'Ignore the differences between this site version and the original site version.';
@@ -109,8 +112,10 @@ $string['predictionprocessfinished'] = 'Prediction process finished';
 $string['previouspage'] = 'Previous page';
 $string['samestartdate'] = 'Current start date is good';
 $string['sameenddate'] = 'Current end date is good';
+$string['selecttimesplittingforevaluation'] = 'Select the time-splitting method you want to use to evaluate the model configuration.';
 $string['target'] = 'Target';
 $string['target_help'] = 'The target is what the model will predict.';
+$string['target_link'] = 'Targets';
 $string['timesplittingnotdefined'] = 'Time splitting is not defined.';
 $string['timesplittingnotdefined_help'] = 'You need to select a time-splitting method before enabling the model.';
 $string['trainandpredictmodel'] = 'Training model and calculating predictions';
index 3cca64e..d70fad3 100644 (file)
@@ -103,21 +103,19 @@ switch ($action) {
     case 'delete':
         confirm_sesskey();
 
-        $model->delete();
+        if (!$model->is_static()) {
+            $model->delete();
+        }
         redirect($returnurl);
         break;
 
     case 'edit':
         confirm_sesskey();
 
-        if ($model->is_static()) {
-            echo $OUTPUT->header();
-            throw new moodle_exception('errornostaticedit', 'tool_analytics');
-        }
-
         $customdata = array(
             'id' => $model->get_id(),
             'trainedmodel' => $model->is_trained(),
+            'staticmodel' => $model->is_static(),
             'indicators' => $model->get_potential_indicators(),
             'timesplittings' => \core_analytics\manager::get_all_time_splittings(),
             'predictionprocessors' => \core_analytics\manager::get_all_prediction_processors()
@@ -129,14 +127,22 @@ switch ($action) {
 
         } else if ($data = $mform->get_data()) {
 
-            // Converting option names to class names.
-            $indicators = array();
-            foreach ($data->indicators as $indicator) {
-                $indicatorclass = \tool_analytics\output\helper::option_to_class($indicator);
-                $indicators[] = \core_analytics\manager::get_indicator($indicatorclass);
-            }
             $timesplitting = \tool_analytics\output\helper::option_to_class($data->timesplitting);
-            $predictionsprocessor = \tool_analytics\output\helper::option_to_class($data->predictionsprocessor);
+
+            if (!$model->is_static()) {
+                // Converting option names to class names.
+                $indicators = array();
+                foreach ($data->indicators as $indicator) {
+                    $indicatorclass = \tool_analytics\output\helper::option_to_class($indicator);
+                    $indicators[] = \core_analytics\manager::get_indicator($indicatorclass);
+                }
+                $predictionsprocessor = \tool_analytics\output\helper::option_to_class($data->predictionsprocessor);
+            } else {
+                // These fields can not be modified.
+                $indicators = false;
+                $predictionsprocessor = false;
+            }
+
             $model->update($data->enabled, $indicators, $timesplitting, $predictionsprocessor);
             redirect($returnurl);
         }
@@ -156,8 +162,6 @@ switch ($action) {
     case 'evaluate':
         confirm_sesskey();
 
-        echo $OUTPUT->header();
-
         if ($model->is_static()) {
             throw new moodle_exception('errornostaticevaluate', 'tool_analytics');
         }
@@ -174,8 +178,26 @@ switch ($action) {
         $mode = optional_param('mode', false, PARAM_ALPHANUM);
         if ($mode == 'trainedmodel') {
             $options['mode'] = 'trainedmodel';
+        } else {
+
+            // All is the default in core_analytics\model::evaluate() as well.
+            $timesplitting = optional_param('timesplitting', 'all', PARAM_ALPHANUMEXT);
+            if ($timesplitting === 'current') {
+                $options['timesplitting'] = \core_analytics\manager::get_time_splitting($model->get_model_obj()->timesplitting);
+            } else if ($timesplitting !== 'all') {
+                $options['timesplitting'] = \core_analytics\manager::get_time_splitting(
+                    \tool_analytics\output\helper::option_to_class($timesplitting)
+                );
+            }
         }
+
         $results = $model->evaluate($options);
+
+        // We reset the theme and the output as some indicators may be using external functions
+        // which reset $PAGE.
+        \tool_analytics\output\helper::reset_page();
+        echo $OUTPUT->header();
+
         $renderer = $PAGE->get_renderer('tool_analytics');
         echo $renderer->render_evaluate_results($results, $model->get_analyser()->get_logs());
         break;
@@ -183,8 +205,6 @@ switch ($action) {
     case 'getpredictions':
         confirm_sesskey();
 
-        echo $OUTPUT->header();
-
         if ($onlycli) {
             throw new moodle_exception('erroronlycli', 'tool_analytics');
         }
@@ -202,6 +222,11 @@ switch ($action) {
             $predictlogs = array();
         }
 
+        // We reset the theme and the output as some indicators may be using external functions
+        // which reset $PAGE.
+        \tool_analytics\output\helper::reset_page();
+        echo $OUTPUT->header();
+
         $renderer = $PAGE->get_renderer('tool_analytics');
         echo $renderer->render_get_predictions_results($trainresults, $trainlogs, $predictresults, $predictlogs);
         break;
diff --git a/admin/tool/analytics/templates/evaluation_options.mustache b/admin/tool/analytics/templates/evaluation_options.mustache
new file mode 100644 (file)
index 0000000..9d77c13
--- /dev/null
@@ -0,0 +1,82 @@
+{{!
+    This file is part of Moodle - http://moodle.org/
+
+    Moodle is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Moodle is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+}}
+{{!
+    @template tool_analytics/evaluation_options
+
+    Evaluation selector.
+
+    The purpose of this template is to render the evaluation mode options.
+
+    Classes required for JS:
+    * none
+
+    Data attributes required for JS:
+    * none
+
+    Example context (json):
+    {
+        "trainedexternally": "1",
+        "timesplittingmethods": [
+            {
+                "id": "ou",
+                "name": "Quarters"
+            }, {
+                "id": "yeah",
+                "name": "Tenths"
+            }
+        ]
+    }
+}}
+
+{{#trainedexternally}}
+    <div class="box mb-4">{{#str}} evaluationmodeinfo, tool_analytics {{/str}}</div>
+
+    <div class="custom-control custom-radio">
+        <input class="custom-control-input" type="radio" name="evaluationmode" id="id-mode-trainedmodel" value="trainedmodel" checked>
+        <label class="custom-control-label" for="id-mode-trainedmodel">{{#str}} evaluationmodetrainedmodel, tool_analytics {{/str}}</label>
+    </div>
+
+    <div class="custom-control custom-radio">
+        <input class="custom-control-input" type="radio" name="evaluationmode" id="id-mode-configuration" value="configuration">
+        <label class="custom-control-label" for="id-mode-configuration">{{#str}} evaluationmodeconfiguration, tool_analytics {{/str}}</label>
+    </div>
+{{/trainedexternally}}
+
+{{! Hidden by default if #trainedexternally as the default option is trainedmodel in this case.}}
+<div id="id-evaluation-timesplitting-container" class="m-t-1 {{#trainedexternally}}hidden{{/trainedexternally}}">
+    {{#str}} selecttimesplittingforevaluation, tool_analytics {{/str}}
+    <div>
+        <select id="id-evaluation-timesplitting" name="timesplitting" class="custom-select m-t-1">
+            {{#timesplittingmethods}}
+                <option value="{{id}}">{{text}}</option>
+            {{/timesplittingmethods}}
+        </select>
+    </div>
+</div>
+
+
+{{#js}}
+    require(['jquery'], function($) {
+        $("input[name='evaluationmode']:radio").change(function() {
+            if ($(this).val() == 'configuration') {
+                $('#id-evaluation-timesplitting-container').show();
+            } else {
+                $('#id-evaluation-timesplitting-container').hide();
+            }
+        });
+    });
+{{/js}}
\ No newline at end of file
index 2c18ac7..80cb7ee 100644 (file)
@@ -614,6 +614,12 @@ class api {
             throw new moodle_exception('errorrequestnotwaitingforapproval', 'tool_dataprivacy');
         }
 
+        // Check if current user has permission to approve delete data request.
+        if ($request->get('type') == self::DATAREQUEST_TYPE_DELETE && !self::can_create_data_deletion_request_for_other()) {
+            throw new required_capability_exception(context_system::instance(),
+                'tool/dataprivacy:requestdeleteforotheruser', 'nopermissions', '');
+        }
+
         // Update the status and the DPO.
         $result = self::update_request_status($requestid, self::DATAREQUEST_STATUS_APPROVED, $USER->id);
 
@@ -653,6 +659,12 @@ class api {
             throw new moodle_exception('errorrequestnotwaitingforapproval', 'tool_dataprivacy');
         }
 
+        // Check if current user has permission to reject delete data request.
+        if ($request->get('type') == self::DATAREQUEST_TYPE_DELETE && !self::can_create_data_deletion_request_for_other()) {
+            throw new required_capability_exception(context_system::instance(),
+                'tool/dataprivacy:requestdeleteforotheruser', 'nopermissions', '');
+        }
+
         // Update the status and the DPO.
         return self::update_request_status($requestid, self::DATAREQUEST_STATUS_REJECTED, $USER->id);
     }
@@ -752,6 +764,48 @@ class api {
         return true;
     }
 
+    /**
+     * Check if user has permisson to create data deletion request for themselves.
+     *
+     * @param int|null $userid ID of the user.
+     * @return bool
+     * @throws coding_exception
+     */
+    public static function can_create_data_deletion_request_for_self(int $userid = null): bool {
+        global $USER;
+        $userid = $userid ?: $USER->id;
+        return has_capability('tool/dataprivacy:requestdelete', \context_user::instance($userid), $userid);
+    }
+
+    /**
+     * Check if user has permission to create data deletion request for another user.
+     *
+     * @param int|null $userid ID of the user.
+     * @return bool
+     * @throws coding_exception
+     * @throws dml_exception
+     */
+    public static function can_create_data_deletion_request_for_other(int $userid = null): bool {
+        global $USER;
+        $userid = $userid ?: $USER->id;
+        return has_capability('tool/dataprivacy:requestdeleteforotheruser', context_system::instance(), $userid);
+    }
+
+    /**
+     * Check if parent can create data deletion request for their children.
+     *
+     * @param int $userid ID of a user being requested.
+     * @param int|null $requesterid ID of a user making request.
+     * @return bool
+     * @throws coding_exception
+     */
+    public static function can_create_data_deletion_request_for_children(int $userid, int $requesterid = null): bool {
+        global $USER;
+        $requesterid = $requesterid ?: $USER->id;
+        return has_capability('tool/dataprivacy:makedatadeletionrequestsforchildren', \context_user::instance($userid),
+            $requesterid);
+    }
+
     /**
      * Checks whether a user can download a data request.
      *
index a309c8e..3c5ce09 100644 (file)
@@ -170,6 +170,10 @@ class data_request_exporter extends persistent_exporter {
                 $values['canreview'] = true;
                 // Whether the DPO can approve or deny the request.
                 $values['approvedeny'] = in_array($requesttype, [api::DATAREQUEST_TYPE_EXPORT, api::DATAREQUEST_TYPE_DELETE]);
+                // If the request's type is delete, check if user have permission to approve/deny it.
+                if ($requesttype == api::DATAREQUEST_TYPE_DELETE) {
+                    $values['approvedeny'] = api::can_create_data_deletion_request_for_other();
+                }
                 break;
             case api::DATAREQUEST_STATUS_APPROVED:
                 $values['statuslabelclass'] = 'badge-info';
index 7249adf..2f12f58 100644 (file)
@@ -234,10 +234,6 @@ class data_registry_page implements renderable, templatable {
 
             $coursecontext = \context_course::instance($course->id);
 
-            if (!$course->visible && !has_capability('moodle/course:viewhiddencourses', $coursecontext)) {
-                continue;
-            }
-
             $coursenode = [
                 'text' => shorten_text(format_string($course->shortname, true, ['context' => $coursecontext])),
                 'contextid' => $coursecontext->id,
index b86e464..303c774 100644 (file)
@@ -116,9 +116,17 @@ class data_requests_table extends table_sql {
      *
      * @param stdClass $data The row data.
      * @return string
+     * @throws \moodle_exception
+     * @throws coding_exception
      */
     public function col_select($data) {
         if ($data->status == \tool_dataprivacy\api::DATAREQUEST_STATUS_AWAITING_APPROVAL) {
+            if ($data->type == \tool_dataprivacy\api::DATAREQUEST_TYPE_DELETE
+                && !api::can_create_data_deletion_request_for_other()) {
+                // Don't show checkbox if request's type is delete and user don't have permission.
+                return false;
+            }
+
             $stringdata = [
                 'username' => $data->foruser->fullname,
                 'requesttype' => \core_text::strtolower($data->typenameshort)
@@ -206,6 +214,7 @@ class data_requests_table extends table_sql {
 
         $requestid = $data->id;
         $status = $data->status;
+        $persistent = $this->datarequests[$requestid];
 
         // Prepare actions.
         $actions = [];
@@ -232,6 +241,11 @@ class data_requests_table extends table_sql {
                 }
                 break;
             case api::DATAREQUEST_STATUS_AWAITING_APPROVAL:
+                // Only show "Approve" and "Deny" button for deletion request if current user has permission.
+                if ($persistent->get('type') == api::DATAREQUEST_TYPE_DELETE &&
+                    !api::can_create_data_deletion_request_for_other()) {
+                    break;
+                }
                 // Approve.
                 $actiondata['data-action'] = 'approve';
                 $actiontext = get_string('approverequest', 'tool_dataprivacy');
@@ -253,9 +267,11 @@ class data_requests_table extends table_sql {
         }
 
         if ($this->manage) {
-            $persistent = $this->datarequests[$requestid];
             $canreset = $persistent->is_active() || empty($this->ongoingrequests[$data->foruser->id]->{$data->type});
             $canreset = $canreset && $persistent->is_resettable();
+            // Prevent re-submmit deletion request if current user don't have permission.
+            $canreset = $canreset && ($persistent->get('type') != api::DATAREQUEST_TYPE_DELETE ||
+                    api::can_create_data_deletion_request_for_other());
             if ($canreset) {
                 $reseturl = new moodle_url('/admin/tool/dataprivacy/resubmitrequest.php', [
                         'requestid' => $requestid,
index 12db2ea..b32dc15 100644 (file)
@@ -76,6 +76,19 @@ if ($data = $mform->get_data()) {
         }
     }
 
+    if ($data->type == \tool_dataprivacy\api::DATAREQUEST_TYPE_DELETE) {
+        if ($data->userid == $USER->id) {
+            if (!\tool_dataprivacy\api::can_create_data_deletion_request_for_self()) {
+                throw new moodle_exception('nopermissions', 'error', '',
+                    get_string('errorcannotrequestdeleteforself', 'tool_dataprivacy'));
+            }
+        } else if (!\tool_dataprivacy\api::can_create_data_deletion_request_for_other()
+            && !\tool_dataprivacy\api::can_create_data_deletion_request_for_children($data->userid)) {
+            throw new moodle_exception('nopermissions', 'error', '',
+                get_string('errorcannotrequestdeleteforother', 'tool_dataprivacy'));
+        }
+    }
+
     \tool_dataprivacy\api::create_data_request($data->userid, $data->type, $data->comments);
 
     if ($manage) {
@@ -93,7 +106,7 @@ $PAGE->set_title($title);
 echo $OUTPUT->header();
 echo $OUTPUT->heading($title);
 
-echo $OUTPUT->box_start();
+echo $OUTPUT->box_start('createrequestform');
 $mform->display();
 echo $OUTPUT->box_end();
 
index 4c94a8a..c91213c 100644 (file)
@@ -45,6 +45,7 @@ class tool_dataprivacy_data_request_form extends moodleform {
      * Form definition.
      *
      * @throws coding_exception
+     * @throws dml_exception
      */
     public function definition() {
         global $USER;
@@ -108,6 +109,24 @@ class tool_dataprivacy_data_request_form extends moodleform {
         // Action buttons.
         $this->add_action_buttons();
 
+        $shouldfreeze = false;
+        if ($this->manage) {
+            $shouldfreeze = !api::can_create_data_deletion_request_for_other();
+        } else {
+            $shouldfreeze = !api::can_create_data_deletion_request_for_self();
+            if ($shouldfreeze && !empty($useroptions)) {
+                foreach ($useroptions as $userid => $useroption) {
+                    if (api::can_create_data_deletion_request_for_children($userid)) {
+                        $shouldfreeze = false;
+                        break;
+                    }
+                }
+            }
+        }
+
+        if ($shouldfreeze) {
+            $mform->freeze('type');
+        }
     }
 
     /**
@@ -120,6 +139,7 @@ class tool_dataprivacy_data_request_form extends moodleform {
      * @throws dml_exception
      */
     public function validation($data, $files) {
+        global $USER;
         $errors = [];
 
         $validrequesttypes = [
@@ -134,6 +154,19 @@ class tool_dataprivacy_data_request_form extends moodleform {
             $errors['type'] = get_string('errorrequestalreadyexists', 'tool_dataprivacy');
         }
 
+        // Check if current user can create data deletion request.
+        $userid = $data['userid'];
+        if ($data['type'] == api::DATAREQUEST_TYPE_DELETE) {
+            if ($userid == $USER->id) {
+                if (!api::can_create_data_deletion_request_for_self()) {
+                    $errors['type'] = get_string('errorcannotrequestdeleteforself', 'tool_dataprivacy');
+                }
+            } else if (!api::can_create_data_deletion_request_for_other()
+                && !api::can_create_data_deletion_request_for_children($userid)) {
+                $errors['type'] = get_string('errorcannotrequestdeleteforother', 'tool_dataprivacy');
+            }
+        }
+
         return $errors;
     }
 }
index ad31867..d70f4a5 100644 (file)
@@ -34,6 +34,15 @@ $capabilities = [
         'archetypes' => []
     ],
 
+    // Capability for create new delete data request. Usually given to the site's Protection Officer.
+    'tool/dataprivacy:requestdeleteforotheruser' => [
+        'riskbitmask' => RISK_SPAM | RISK_PERSONAL | RISK_XSS | RISK_DATALOSS,
+        'captype' => 'write',
+        'contextlevel' => CONTEXT_SYSTEM,
+        'archetypes' => [],
+        'clonepermissionsfrom' => 'tool/dataprivacy:managedatarequests'
+    ],
+
     // Capability for managing the data registry. Usually given to the site's Data Protection Officer.
     'tool/dataprivacy:managedataregistry' => [
         'riskbitmask' => RISK_SPAM | RISK_PERSONAL | RISK_XSS | RISK_DATALOSS,
@@ -50,6 +59,15 @@ $capabilities = [
         'archetypes' => []
     ],
 
+    // Capability for parents/guardians to make delete data requests on behalf of their children.
+    'tool/dataprivacy:makedatadeletionrequestsforchildren' => [
+        'riskbitmask' => RISK_SPAM | RISK_PERSONAL,
+        'captype' => 'write',
+        'contextlevel' => CONTEXT_USER,
+        'archetypes' => [],
+        'clonepermissionsfrom' => 'tool/dataprivacy:makedatarequestsforchildren'
+    ],
+
     // Capability for users to download the results of their own data request.
     'tool/dataprivacy:downloadownrequest' => [
         'riskbitmask' => 0,
@@ -67,4 +85,14 @@ $capabilities = [
         'contextlevel' => CONTEXT_USER,
         'archetypes' => []
     ],
+
+    // Capability for users to create delete data request for their own.
+    'tool/dataprivacy:requestdelete' => [
+        'riskbitmask' => RISK_DATALOSS,
+        'captype' => 'write',
+        'contextlevel' => CONTEXT_USER,
+        'archetypes' => [
+            'user' => CAP_ALLOW
+        ]
+    ]
 ];
index bb3fa70..5a0efd4 100644 (file)
@@ -70,6 +70,7 @@ $string['contextlevelname80'] = 'Blocks';
 $string['contextpurposecategorysaved'] = 'Purpose and category saved.';
 $string['contactdpoviaprivacypolicy'] = 'Please contact the privacy officer as described in the privacy policy.';
 $string['createcategory'] = 'Create data category';
+$string['createdeletedatarequest'] = 'Create data deletion request';
 $string['createnewdatarequest'] = 'Create a new data request';
 $string['createpurpose'] = 'Create data purpose';
 $string['creationauto'] = 'Automatically';
@@ -81,6 +82,9 @@ $string['dataprivacy:managedatarequests'] = 'Manage data requests';
 $string['dataprivacy:managedataregistry'] = 'Manage data registry';
 $string['dataprivacy:downloadownrequest'] = 'Download your own exported data';
 $string['dataprivacy:downloadallrequests'] = 'Download exported data for everyone';
+$string['dataprivacy:requestdeleteforotheruser'] = 'Request data deletion on behalf of another user';
+$string['dataprivacy:makedatadeletionrequestsforchildren'] = 'Request data deletion for minors';
+$string['dataprivacy:requestdelete'] = 'Request data deletion for yourself';
 $string['dataregistry'] = 'Data registry';
 $string['dataregistryinfo'] = 'The data registry enables categories (types of data) and purposes (the reasons for processing data) to be set for all content on the site - from users and courses down to activities and blocks. For each purpose, a retention period may be set. When a retention period has expired, the data is flagged and listed for deletion, awaiting admin confirmation.';
 $string['dataretentionexplanation'] = 'This summary shows the default categories and purposes for retaining user data. Certain areas may have more specific categories and purposes than those listed here.';
@@ -124,6 +128,8 @@ $string['editpurposes'] = 'Edit purposes';
 $string['effectiveretentionperiodcourse'] = '{$a} (after the course end date)';
 $string['effectiveretentionperioduser'] = '{$a} (since the last time the user accessed the site)';
 $string['emailsalutation'] = 'Dear {$a},';
+$string['errorcannotrequestdeleteforself'] = 'You don\'t have permission to create deletion request for yourself.';
+$string['errorcannotrequestdeleteforother'] = 'You don\'t have permission to create deletion request for this user.';
 $string['errorinvalidrequestcreationmethod'] = 'Invalid request creation method!';
 $string['errorinvalidrequeststatus'] = 'Invalid request status!';
 $string['errorinvalidrequesttype'] = 'Invalid request type!';
index 8655f1c..c069301 100644 (file)
@@ -77,8 +77,9 @@ function tool_dataprivacy_myprofile_navigation(tree $tree, $user, $iscurrentuser
 
         // Check if the user has an ongoing data deletion request.
         $hasdeleterequest = \tool_dataprivacy\api::has_ongoing_request($user->id, \tool_dataprivacy\api::DATAREQUEST_TYPE_DELETE);
-        // Show data deletion link only if the user doesn't have an ongoing data deletion request.
-        if (!$hasdeleterequest) {
+        // Show data deletion link only if the user doesn't have an ongoing data deletion request and has permission
+        // to create data deletion request.
+        if (!$hasdeleterequest && \tool_dataprivacy\api::can_create_data_deletion_request_for_self()) {
             $deleteparams = ['type' => \tool_dataprivacy\api::DATAREQUEST_TYPE_DELETE];
             $deleteurl = new moodle_url('/admin/tool/dataprivacy/createdatarequest.php', $deleteparams);
             $deletenode = new core_user\output\myprofile\node('privacyandpolicies', 'requestdatadeletion',
index ac7520e..4f3ef55 100644 (file)
@@ -44,6 +44,11 @@ $stringparams = (object) [
     ];
 
 if (null !== $confirm && confirm_sesskey()) {
+    if ($originalrequest->get('type') == \tool_dataprivacy\api::DATAREQUEST_TYPE_DELETE
+        && !\tool_dataprivacy\api::can_create_data_deletion_request_for_other()) {
+        throw new required_capability_exception(context_system::instance(),
+            'tool/dataprivacy:requestdeleteforotheruser', 'nopermissions', '');
+    }
     $originalrequest->resubmit_request();
     redirect($manageurl, get_string('resubmittedrequest', 'tool_dataprivacy', $stringparams));
 }
index 8e60223..f95dedd 100644 (file)
@@ -2109,4 +2109,90 @@ class tool_dataprivacy_api_testcase extends advanced_testcase {
 
         return $request;
     }
+
+    /**
+     * Test user cannot create data deletion request for themselves if they don't have
+     * "tool/dataprivacy:requestdelete" capability.
+     *
+     * @throws coding_exception
+     */
+    public function test_can_create_data_deletion_request_for_self_no() {
+        $this->resetAfterTest();
+        $userid = $this->getDataGenerator()->create_user()->id;
+        $roleid = $this->getDataGenerator()->create_role();
+        assign_capability('tool/dataprivacy:requestdelete', CAP_PROHIBIT, $roleid, context_user::instance($userid));
+        role_assign($roleid, $userid, context_user::instance($userid));
+        $this->setUser($userid);
+        $this->assertFalse(api::can_create_data_deletion_request_for_self());
+    }
+
+    /**
+     * Test user can create data deletion request for themselves if they have
+     * "tool/dataprivacy:requestdelete" capability.
+     *
+     * @throws coding_exception
+     */
+    public function test_can_create_data_deletion_request_for_self_yes() {
+        $this->resetAfterTest();
+        $userid = $this->getDataGenerator()->create_user()->id;
+        $this->setUser($userid);
+        $this->assertTrue(api::can_create_data_deletion_request_for_self());
+    }
+
+    /**
+     * Test user cannot create data deletion request for another user if they
+     * don't have "tool/dataprivacy:requestdeleteforotheruser" capability.
+     *
+     * @throws coding_exception
+     * @throws dml_exception
+     */
+    public function test_can_create_data_deletion_request_for_other_no() {
+        $this->resetAfterTest();
+        $userid = $this->getDataGenerator()->create_user()->id;
+        $this->setUser($userid);
+        $this->assertFalse(api::can_create_data_deletion_request_for_other());
+    }
+
+    /**
+     * Test user can create data deletion request for another user if they
+     * don't have "tool/dataprivacy:requestdeleteforotheruser" capability.
+     *
+     * @throws coding_exception
+     */
+    public function test_can_create_data_deletion_request_for_other_yes() {
+        $this->resetAfterTest();
+        $userid = $this->getDataGenerator()->create_user()->id;
+        $roleid = $this->getDataGenerator()->create_role();
+        $contextsystem = context_system::instance();
+        assign_capability('tool/dataprivacy:requestdeleteforotheruser', CAP_ALLOW, $roleid, $contextsystem);
+        role_assign($roleid, $userid, $contextsystem);
+        $this->setUser($userid);
+        $this->assertTrue(api::can_create_data_deletion_request_for_other($userid));
+    }
+
+    /**
+     * Check parents can create data deletion request for their children but not others.
+     *
+     * @throws coding_exception
+     * @throws dml_exception
+     */
+    public function test_can_create_data_deletion_request_for_children() {
+        $this->resetAfterTest();
+
+        $parent = $this->getDataGenerator()->create_user();
+        $child = $this->getDataGenerator()->create_user();
+        $otheruser = $this->getDataGenerator()->create_user();
+
+        $contextsystem = \context_system::instance();
+        $parentrole = $this->getDataGenerator()->create_role();
+        assign_capability('tool/dataprivacy:makedatarequestsforchildren', CAP_ALLOW,
+            $parentrole, $contextsystem);
+        assign_capability('tool/dataprivacy:makedatadeletionrequestsforchildren', CAP_ALLOW,
+            $parentrole, $contextsystem);
+        role_assign($parentrole, $parent->id, \context_user::instance($child->id));
+
+        $this->setUser($parent);
+        $this->assertTrue(api::can_create_data_deletion_request_for_children($child->id));
+        $this->assertFalse(api::can_create_data_deletion_request_for_children($otheruser->id));
+    }
 }
index e6c7fde..5806102 100644 (file)
@@ -6,18 +6,24 @@ Feature: Data delete from the privacy API
 
   Background:
     Given the following "users" exist:
-      | username | firstname      | lastname |
-      | victim   | Victim User    | 1        |
-      | parent   | Long-suffering | Parent   |
+      | username       | firstname       | lastname |
+      | victim         | Victim User     | 1        |
+      | parent         | Long-suffering  | Parent   |
+      | privacyofficer | Privacy Officer | One      |
     And the following "roles" exist:
       | shortname | name  | archetype |
       | tired     | Tired |           |
     And the following "permission overrides" exist:
-      | capability                                   | permission | role  | contextlevel | reference |
-      | tool/dataprivacy:makedatarequestsforchildren | Allow      | tired | System       |           |
+      | capability                                           | permission | role    | contextlevel | reference |
+      | tool/dataprivacy:makedatarequestsforchildren         | Allow      | tired   | System       |           |
+      | tool/dataprivacy:makedatadeletionrequestsforchildren | Allow      | tired   | System       |           |
+      | tool/dataprivacy:managedatarequests                  | Allow      | manager | System       |           |
     And the following "role assigns" exist:
       | user   | role  | contextlevel | reference |
       | parent | tired | User         | victim    |
+    And the following "system role assigns" exist:
+      | user           | role    | contextlevel |
+      | privacyofficer | manager | User         |
     And the following config values are set as admin:
       | contactdataprotectionofficer | 1  | tool_dataprivacy |
     And the following data privacy "categories" exist:
@@ -26,6 +32,10 @@ Feature: Data delete from the privacy API
     And the following data privacy "purposes" exist:
       | name         | retentionperiod |
       | Site purpose | P10Y           |
+    And the following config values are set as admin:
+      | contactdataprotectionofficer | 1  | tool_dataprivacy |
+      | privacyrequestexpiry         | 55 | tool_dataprivacy |
+      | dporoles                     | 1  | tool_dataprivacy |
     And I set the site category and purpose to "Site category" and "Site purpose"
 
   @javascript
@@ -115,3 +125,112 @@ Feature: Data delete from the privacy API
     And I run all adhoc tasks
     And I reload the page
     And I should see "You don't have any personal data requests"
+
+  @javascript
+  Scenario: As a Privacy Officer, I cannot create data deletion request unless I have permission.
+    Given I log in as "privacyofficer"
+    And I navigate to "Users > Privacy and policies > Data requests" in site administration
+    And I follow "New request"
+    And I open the autocomplete suggestions list
+    And I click on "Victim User 1" item in the autocomplete list
+    Then I should see "Export all of my personal data"
+    And "Type" "select" should not be visible
+    And the following "permission overrides" exist:
+      | capability                                 | permission | role    | contextlevel | reference |
+      | tool/dataprivacy:requestdeleteforotheruser | Allow      | manager | System       |           |
+    And I reload the page
+    And I open the autocomplete suggestions list
+    And I click on "Victim User 1" item in the autocomplete list
+    And "Type" "select" should be visible
+
+  @javascript
+  Scenario: As a student, I cannot create data deletion request unless I have permission.
+    Given I log in as "victim"
+    And I follow "Profile" in the user menu
+    And I follow "Data requests"
+    And I follow "New request"
+    Then "Type" "select" should exist
+    And the following "permission overrides" exist:
+      | capability                     | permission | role | contextlevel | reference |
+      | tool/dataprivacy:requestdelete | Prevent    | user | System       |           |
+    And I reload the page
+    And I should see "Export all of my personal data"
+    And "Type" "select" should not exist
+
+  @javascript
+  Scenario: As a parent, I cannot create data deletion request unless I have permission.
+    Given I log in as "parent"
+    And the following "permission overrides" exist:
+      | capability                                           | permission | role  | contextlevel | reference |
+      | tool/dataprivacy:makedatadeletionrequestsforchildren | Prevent    | tired | System       | victim    |
+    And I follow "Profile" in the user menu
+    And I follow "Data requests"
+    And I follow "New request"
+    And I open the autocomplete suggestions list
+    And I click on "Victim User 1" item in the autocomplete list
+    And I set the field "Type" to "Delete all of my personal data"
+    And I press "Save changes"
+    And I should see "You don't have permission to create deletion request for this user."
+    And the following "permission overrides" exist:
+      | capability                                           | permission | role  | contextlevel | reference |
+      | tool/dataprivacy:makedatadeletionrequestsforchildren | Allow      | tired | System       | victim    |
+      | tool/dataprivacy:requestdelete                       | Prevent    | user  | System       |           |
+    And I open the autocomplete suggestions list
+    And I click on "Long-suffering Parent" item in the autocomplete list
+    And I press "Save changes"
+    And I should see "You don't have permission to create deletion request for yourself."
+
+  @javascript
+  Scenario: As a student, link to create data deletion should not be shown if I don't have permission.
+    Given the following "permission overrides" exist:
+      | capability                     | permission | role | contextlevel | reference |
+      | tool/dataprivacy:requestdelete | Prohibit   | user | System       |           |
+    When I log in as "victim"
+    And I follow "Profile" in the user menu
+    Then I should not see "Delete my account"
+
+  @javascript
+  Scenario: As a Privacy Officer, I cannot Approve to Deny deletion data request without permission.
+    Given the following "permission overrides" exist:
+      | capability                                 | permission | role    | contextlevel | reference |
+      | tool/dataprivacy:requestdeleteforotheruser | Allow      | manager | System       |           |
+    When I log in as "privacyofficer"
+    And I navigate to "Users > Privacy and policies > Data requests" in site administration
+    And I follow "New request"
+    And I open the autocomplete suggestions list
+    And I click on "Victim User 1" item in the autocomplete list
+    And I set the field "Type" to "Delete all of my personal data"
+    And I press "Save changes"
+    And the following "permission overrides" exist:
+      | capability                                 | permission | role    | contextlevel | reference |
+      | tool/dataprivacy:requestdeleteforotheruser | Prohibit   | manager | System       |           |
+    And I reload the page
+    Then ".selectrequests" "css_element" should not exist
+    And I open the action menu in "region-main" "region"
+    And I should not see "Approve request"
+    And I should not see "Deny request"
+    And I choose "View the request" in the open action menu
+    And "Approve" "button" should not exist
+    And "Deny" "button" should not exist
+
+  @javascript
+  Scenario: As a Privacy Officer, I cannot re-submit deletion data request without permission.
+    Given the following "permission overrides" exist:
+      | capability                                 | permission | role    | contextlevel | reference |
+      | tool/dataprivacy:requestdeleteforotheruser | Allow      | manager | System       |           |
+    When I log in as "privacyofficer"
+    And I navigate to "Users > Privacy and policies > Data requests" in site administration
+    And I follow "New request"
+    And I open the autocomplete suggestions list
+    And I click on "Victim User 1" item in the autocomplete list
+    And I set the field "Type" to "Delete all of my personal data"
+    And I press "Save changes"
+    And I open the action menu in "region-main" "region"
+    And I follow "Deny request"
+    And I press "Deny request"
+    And the following "permission overrides" exist:
+      | capability                                 | permission | role    | contextlevel | reference |
+      | tool/dataprivacy:requestdeleteforotheruser | Prohibit   | manager | System       |           |
+    And I reload the page
+    And I open the action menu in "region-main" "region"
+    Then I should not see "Resubmit as new request"
index 411f0ac..8089b41 100644 (file)
@@ -24,6 +24,6 @@
 
 defined('MOODLE_INTERNAL') || die;
 
-$plugin->version   = 2019011500;
+$plugin->version   = 2019040800;
 $plugin->requires  = 2018112800;        // Moodle 3.5dev (Build 2018031600) and upwards.
 $plugin->component = 'tool_dataprivacy';
index 15552d5..ee40e70 100644 (file)
@@ -47,8 +47,7 @@ class tool_generator_make_testplan_form extends moodleform {
             tool_generator_testplan_backend::get_size_choices());
         $mform->setDefault('size', tool_generator_testplan_backend::DEFAULT_SIZE);
 
-        $mform->addElement('select', 'courseid', get_string('targetcourse', 'tool_generator'),
-            tool_generator_testplan_backend::get_course_options());
+        $mform->addElement('course', 'courseid', get_string('targetcourse', 'tool_generator'));
 
         $mform->addElement('advcheckbox', 'updateuserspassword', get_string('updateuserspassword', 'tool_generator'));
         $mform->addHelpButton('updateuserspassword', 'updateuserspassword', 'tool_generator');
index 2e43b4c..6b3c2e4 100644 (file)
@@ -71,25 +71,6 @@ class tool_generator_testplan_backend extends tool_generator_backend {
         return $options;
     }
 
-    /**
-     * Gets the list of courses that can be used used to generate a test.
-     *
-     * @return array The list of options as courseid => name
-     */
-    public static function get_course_options() {
-        $courses = get_courses('all', 'c.sortorder ASC', 'c.id, c.shortname, c.fullname');
-        if (!$courses) {
-            print_error('error_nocourses', 'tool_generator');
-        }
-
-        $options = array();
-        unset($courses[1]);
-        foreach ($courses as $course) {
-            $options[$course->id] = $course->fullname . '(' . $course->shortname . ')';
-        }
-        return $options;
-    }
-
     /**
      * Getter for moodle-performance-comparison project URL.
      *
diff --git a/admin/tool/generator/upgrade.txt b/admin/tool/generator/upgrade.txt
new file mode 100644 (file)
index 0000000..4d5b19f
--- /dev/null
@@ -0,0 +1,6 @@
+This files describes API changes in core libraries and APIs,
+information provided here is intended especially for developers.
+
+=== 3.7 ===
+
+* Function tool_generator_testplan_backend::get_course_options() is removed, the 'course' form element is used instead.
index 96cfd0c..e88a683 100644 (file)
@@ -43,6 +43,9 @@ trait buffered_writer {
     /** @var int $count Counter. */
     protected $count = 0;
 
+    /** @var bool If true, writes JSON instead of PHP serialized data for 'other' field */
+    protected $jsonformat = false;
+
     /**
      * Should the event be ignored (== not logged)?
      * @param \core\event\base $event
@@ -69,7 +72,11 @@ trait buffered_writer {
         // at the same time this lowers memory use because
         // snapshots and custom objects may be garbage collected.
         $entry = $event->get_data();
-        $entry['other'] = serialize($entry['other']);
+        if ($this->jsonformat) {
+            $entry['other'] = json_encode($entry['other']);
+        } else {
+            $entry['other'] = serialize($entry['other']);
+        }
         $entry['origin'] = $PAGE->requestorigin;
         $entry['ip'] = $PAGE->requestip;
         $entry['realuserid'] = \core\session\manager::is_loggedinas() ? $GLOBALS['USER']->realuser : null;
index 93e0af1..cc7e8b8 100644 (file)
@@ -62,6 +62,25 @@ trait reader {
         return $this->store;
     }
 
+    /**
+     * Function decodes the other field into an array using either PHP serialisation or JSON.
+     *
+     * Note that this does not rely on the config setting, it supports both formats, so you can
+     * use it for data before/after making a change to the config setting.
+     *
+     * The return value is usually an array but it can also be null or a boolean or something.
+     *
+     * @param string $other Other value
+     * @return mixed Decoded value
+     */
+    public static function decode_other(string $other) {
+        if ($other === 'N;' || preg_match('~^.:~', $other)) {
+            return unserialize($other);
+        } else {
+            return json_decode($other, true);
+        }
+    }
+
     /**
      * Adds ID column to $sort to make sure events from one request
      * within 1 second are returned in the same order.
index 8a057cf..95c09e9 100644 (file)
@@ -37,6 +37,7 @@ use core_privacy\local\request\transform;
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 class helper {
+    use \tool_log\helper\reader;
 
     /**
      * Returns an event from a standard record.
@@ -49,7 +50,7 @@ class helper {
         $extra = ['origin' => $data->origin, 'ip' => $data->ip, 'realuserid' => $data->realuserid];
         $data = (array) $data;
         $id = $data['id'];
-        $data['other'] = unserialize($data['other']);
+        $data['other'] = self::decode_other($data['other']);
         if ($data['other'] === false) {
             $data['other'] = [];
         }
index 10398df..5c85e3d 100644 (file)
@@ -57,6 +57,9 @@ class store implements \tool_log\log\writer, \core\log\sql_reader {
         $levels = $this->get_config('includelevels', '');
         $this->includeactions = $actions === '' ? array() : explode(',', $actions);
         $this->includelevels = $levels === '' ? array() : explode(',', $levels);
+        // JSON writing defaults to false (table format compatibility with older versions).
+        // Note: This variable is defined in the buffered_writer trait.
+        $this->jsonformat = (bool)$this->get_config('jsonformat', false);
     }
 
     /**
@@ -223,7 +226,7 @@ class store implements \tool_log\log\writer, \core\log\sql_reader {
         $extra = array('origin' => $data->origin, 'ip' => $data->ip, 'realuserid' => $data->realuserid);
         $data = (array)$data;
         $id = $data['id'];
-        $data['other'] = unserialize($data['other']);
+        $data['other'] = self::decode_other($data['other']);
         if ($data['other'] === false) {
             $data['other'] = array();
         }
index 04137ab..b5536ab 100644 (file)
@@ -39,5 +39,12 @@ function xmldb_logstore_database_upgrade($oldversion) {
     // Automatically generated Moodle v3.6.0 release upgrade line.
     // Put any upgrade step following this.
 
+    if ($oldversion < 2019032800) {
+        // For existing installations, set the new jsonformat option to off (no behaviour change).
+        // New installations default to on.
+        set_config('jsonformat', 0, 'logstore_database');
+        upgrade_plugin_savepoint(true, 2019032800, 'logstore', 'database');
+    }
+
     return true;
 }
index 31edf3a..3fd8017 100644 (file)
@@ -39,6 +39,8 @@ $string['includeactions'] = 'Include actions of these types';
 $string['includelevels'] = 'Include actions with these educational levels';
 $string['filters'] = 'Filter logs';
 $string['filters_help'] = 'Enable filters that exclude some actions from being logged.';
+$string['jsonformat'] = 'JSON format';
+$string['jsonformat_desc'] = 'Use standard JSON format instead of PHP serialised data in the \'other\' database field.';
 $string['logguests'] = 'Log guest actions';
 $string['other'] = 'Other';
 $string['participating'] = 'Participating';
index 4e45038..d406fd0 100644 (file)
@@ -59,6 +59,10 @@ if ($hassiteconfig) {
     $settings->add(new admin_setting_configtext('logstore_database/buffersize', get_string('buffersize',
         'logstore_database'), get_string('buffersize_help', 'logstore_database'), 50));
 
+    $settings->add(new admin_setting_configcheckbox('logstore_database/jsonformat',
+            new lang_string('jsonformat', 'logstore_database'),
+            new lang_string('jsonformat_desc', 'logstore_database'), 1));
+
     // Filters.
     $settings->add(new admin_setting_heading('filters', get_string('filters', 'logstore_database'), get_string('filters_help',
         'logstore_database')));
index a05c33f..10dab29 100644 (file)
@@ -28,11 +28,21 @@ require_once(__DIR__ . '/fixtures/event.php');
 require_once(__DIR__ . '/fixtures/store.php');
 
 class logstore_database_store_testcase extends advanced_testcase {
-    public function test_log_writing() {
+    /**
+     * Tests log writing.
+     *
+     * @param bool $jsonformat True to test with JSON format
+     * @dataProvider test_log_writing_provider
+     * @throws moodle_exception
+     */
+    public function test_log_writing(bool $jsonformat) {
         global $DB, $CFG;
         $this->resetAfterTest();
         $this->preventResetByRollback(); // Logging waits till the transaction gets committed.
 
+        // Apply JSON format system setting.
+        set_config('jsonformat', $jsonformat ? 1 : 0, 'logstore_database');
+
         $dbman = $DB->get_manager();
         $this->assertTrue($dbman->table_exists('logstore_standard_log'));
         $DB->delete_records('logstore_standard_log');
@@ -118,7 +128,11 @@ class logstore_database_store_testcase extends advanced_testcase {
 
         $log1 = reset($logs);
         unset($log1->id);
-        $log1->other = unserialize($log1->other);
+        if ($jsonformat) {
+            $log1->other = json_decode($log1->other, true);
+        } else {
+            $log1->other = unserialize($log1->other);
+        }
         $log1 = (array)$log1;
         $data = $event1->get_data();
         $data['origin'] = 'cli';
@@ -145,7 +159,11 @@ class logstore_database_store_testcase extends advanced_testcase {
 
         $log3 = array_shift($logs);
         unset($log3->id);
-        $log3->other = unserialize($log3->other);
+        if ($jsonformat) {
+            $log3->other = json_decode($log3->other, true);
+        } else {
+            $log3->other = unserialize($log3->other);
+        }
         $log3 = (array)$log3;
         $data = $event2->get_data();
         $data['origin'] = 'cli';
@@ -229,6 +247,19 @@ class logstore_database_store_testcase extends advanced_testcase {
         get_log_manager(true);
     }
 
+    /**
+     * Returns different JSON format settings so the test can be run with JSON format either on or
+     * off.
+     *
+     * @return [bool] Array of true/false
+     */
+    public static function test_log_writing_provider(): array {
+        return [
+            [false],
+            [true]
+        ];
+    }
+
     /**
      * Test method is_event_ignored.
      */
index dbf85d7..4159a06 100644 (file)
 /**
  * External database log store.
  *
- * @package    logstore_standard
+ * @package    logstore_database
  * @copyright  2013 Petr Skoda {@link http://skodak.org}
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version = 2018120300; // The current plugin version (Date: YYYYMMDDXX).
+$plugin->version = 2019032800; // The current plugin version (Date: YYYYMMDDXX).
 $plugin->requires = 2018112800; // Requires this Moodle version.
 $plugin->component = 'logstore_database'; // Full name of the plugin (used for diagnostics).
index fe93ff5..c296546 100644 (file)
@@ -38,6 +38,9 @@ class store implements \tool_log\log\writer, \core\log\sql_internal_table_reader
         $this->helper_setup($manager);
         // Log everything before setting is saved for the first time.
         $this->logguests = $this->get_config('logguests', 1);
+        // JSON writing defaults to false (table format compatibility with older versions).
+        // Note: This variable is defined in the buffered_writer trait.
+        $this->jsonformat = (bool)$this->get_config('jsonformat', false);
     }
 
     /**
@@ -120,7 +123,7 @@ class store implements \tool_log\log\writer, \core\log\sql_internal_table_reader
         $extra = array('origin' => $data->origin, 'ip' => $data->ip, 'realuserid' => $data->realuserid);
         $data = (array)$data;
         $id = $data['id'];
-        $data['other'] = unserialize($data['other']);
+        $data['other'] = self::decode_other($data['other']);
         if ($data['other'] === false) {
             $data['other'] = array();
         }
index d6cd26d..e1ac5a2 100644 (file)
@@ -39,5 +39,12 @@ function xmldb_logstore_standard_upgrade($oldversion) {
     // Automatically generated Moodle v3.6.0 release upgrade line.
     // Put any upgrade step following this.
 
+    if ($oldversion < 2019032800) {
+        // For existing installations, set the new jsonformat option to off (no behaviour change).
+        // New installations default to on.
+        set_config('jsonformat', 0, 'logstore_standard');
+        upgrade_plugin_savepoint(true, 2019032800, 'logstore', 'standard');
+    }
+
     return true;
 }
index 14a1d2b..12a4c42 100644 (file)
@@ -23,6 +23,8 @@
  */
 
 $string['buffersize'] = 'Write buffer size';
+$string['jsonformat'] = 'JSON format';
+$string['jsonformat_desc'] = 'Use standard JSON format instead of PHP serialised data in the \'other\' database field.';
 $string['pluginname'] = 'Standard log';
 $string['pluginname_desc'] = 'A log plugin stores log entries in a Moodle database table.';
 $string['privacy:metadata:log'] = 'A collection of past events';
index daa8131..fc4a894 100644 (file)
@@ -30,6 +30,10 @@ if ($hassiteconfig) {
         new lang_string('logguests', 'core_admin'),
         new lang_string('logguests_help', 'core_admin'), 1));
 
+    $settings->add(new admin_setting_configcheckbox('logstore_standard/jsonformat',
+            new lang_string('jsonformat', 'logstore_standard'),
+            new lang_string('jsonformat_desc', 'logstore_standard'), 1));
+
     $options = array(
         0    => new lang_string('neverdeletelogs'),
         1000 => new lang_string('numdays', '', 1000),
index def6684..f2d0531 100644 (file)
@@ -33,11 +33,21 @@ class logstore_standard_store_testcase extends advanced_testcase {
      */
     private $wedisabledgc = false;
 
-    public function test_log_writing() {
+    /**
+     * Tests log writing.
+     *
+     * @param bool $jsonformat True to test with JSON format
+     * @dataProvider test_log_writing_provider
+     * @throws moodle_exception
+     */
+    public function test_log_writing(bool $jsonformat) {
         global $DB;
         $this->resetAfterTest();
         $this->preventResetByRollback(); // Logging waits till the transaction gets committed.
 
+        // Apply JSON format system setting.
+        set_config('jsonformat', $jsonformat ? 1 : 0, 'logstore_standard');
+
         $this->setAdminUser();
         $user1 = $this->getDataGenerator()->create_user();
         $user2 = $this->getDataGenerator()->create_user();
@@ -82,7 +92,11 @@ class logstore_standard_store_testcase extends advanced_testcase {
 
         $log1 = reset($logs);
         unset($log1->id);
-        $log1->other = unserialize($log1->other);
+        if ($jsonformat) {
+            $log1->other = json_decode($log1->other, true);
+        } else {
+            $log1->other = unserialize($log1->other);
+        }
         $log1 = (array)$log1;
         $data = $event1->get_data();
         $data['origin'] = 'cli';
@@ -112,7 +126,11 @@ class logstore_standard_store_testcase extends advanced_testcase {
 
         $log3 = array_shift($logs);
         unset($log3->id);
-        $log3->other = unserialize($log3->other);
+        if ($jsonformat) {
+            $log3->other = json_decode($log3->other, true);
+        } else {
+            $log3->other = unserialize($log3->other);
+        }
         $log3 = (array)$log3;
         $data = $event2->get_data();
         $data['origin'] = 'restore';
@@ -200,6 +218,19 @@ class logstore_standard_store_testcase extends advanced_testcase {
         get_log_manager(true);
     }
 
+    /**
+     * Returns different JSON format settings so the test can be run with JSON format either on or
+     * off.
+     *
+     * @return [bool] Array of true/false
+     */
+    public static function test_log_writing_provider(): array {
+        return [
+            [false],
+            [true]
+        ];
+    }
+
     /**
      * Test logmanager::get_supported_reports returns all reports that require this store.
      */
@@ -332,6 +363,34 @@ class logstore_standard_store_testcase extends advanced_testcase {
         $this->assertEquals(1, $DB->count_records('logstore_standard_log'));
     }
 
+    /**
+     * Tests the decode_other function can cope with both JSON and PHP serialized format.
+     *
+     * @param mixed $value Value to encode and decode
+     * @dataProvider test_decode_other_provider
+     */
+    public function test_decode_other($value) {
+        $this->assertEquals($value, \logstore_standard\log\store::decode_other(serialize($value)));
+        $this->assertEquals($value, \logstore_standard\log\store::decode_other(json_encode($value)));
+    }
+
+    /**
+     * List of possible values for 'other' field.
+     *
+     * I took these types from our logs based on the different first character of PHP serialized
+     * data - my query found only these types. The normal case is an array.
+     *
+     * @return array Array of parameters
+     */
+    public function test_decode_other_provider(): array {
+        return [
+            [['info' => 'd2819896', 'logurl' => 'discuss.php?d=2819896']],
+            [null],
+            ['just a string'],
+            [32768]
+        ];
+    }
+
     /**
      * Disable the garbage collector if it's enabled to ensure we don't adjust memory statistics.
      */
index 89ab35a..2a6b63d 100644 (file)
@@ -24,6 +24,6 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version = 2018120300; // The current plugin version (Date: YYYYMMDDXX).
+$plugin->version = 2019032800; // The current plugin version (Date: YYYYMMDDXX).
 $plugin->requires = 2018112800; // Requires this Moodle version.
 $plugin->component = 'logstore_standard'; // Full name of the plugin (used for diagnostics).
index e1b65e3..abaf3e2 100644 (file)
@@ -2,7 +2,14 @@ This files describes API changes in /admin/tool/log - plugins,
 information provided here is intended especially for developers.
 
 
+=== 3.7 ===
+
+* The new jsonformat option, which defaults to 'on' for a new install (and 'off' for existing installs) means that
+  the 'other' event field is now stored in JSON format instead of PHP serialize format in the database. The system
+  can read data in both formats but if any third-party software directly accesses the database field, it may need
+  to be modified (or require users to turn off jsonformat).
+
 === 3.6 ===
 
 * The legacy log store is in its first stage of deprecation and is due for removal in Moodle 4.0. Please use one of
-  the other log stores such as "standard" and "database".
\ No newline at end of file
+  the other log stores such as "standard" and "database".
index 1795635..e9d1c67 100644 (file)
@@ -262,6 +262,8 @@ class external extends external_api {
     /**
      * Creates an auto-login key for the current user. Is created only in https sites and is restricted by time and ip address.
      *
+     * Please note that it only works if the request comes from the Moodle mobile or desktop app.
+     *
      * @param string $privatetoken the user private token for validating the request
      * @return array with the settings and warnings
      * @since  Moodle 3.2
@@ -284,6 +286,12 @@ class external extends external_api {
             }
         }
 
+        // Only requests from the Moodle mobile or desktop app. This enhances security to avoid any type of XSS attack.
+        // This code goes intentionally here and not inside the check_autologin_prerequisites() function because it
+        // is used by other PHP scripts that can be opened in any browser.
+        if (!\core_useragent::is_moodle_app()) {
+            throw new moodle_exception('apprequired', 'tool_mobile');
+        }
         api::check_autologin_prerequisites($USER->id);
 
         if (isset($_GET['privatetoken']) or empty($privatetoken)) {
index 0b50d68..d53f7b4 100644 (file)
@@ -57,7 +57,8 @@ $functions = array(
         'classname'   => 'tool_mobile\external',
         'methodname'  => 'get_autologin_key',
         'description' => 'Creates an auto-login key for the current user.
-                            Is created only in https sites and is restricted by time and ip address.',
+                            Is created only in https sites and is restricted by time, ip address and only works if the request
+                            comes from the Moodle mobile or desktop app.',
         'type'        => 'write',
         'services'    => array(MOODLE_OFFICIAL_MOBILE_SERVICE),
     ),
index a373878..0b01cd0 100644 (file)
@@ -27,6 +27,7 @@ $string['androidappid'] = 'Android app\'s unique identifier';
 $string['androidappid_desc'] = 'This setting may be left as default unless you have a custom Android app.';
 $string['apppolicy'] = 'App policy URL';
 $string['apppolicy_help'] = 'The URL of a policy for app users which is listed on the About page in the app. If the field is left empty, the site policy URL will be used instead.';
+$string['apprequired'] = 'This functionality is only available when accessed via the Moodle mobile or desktop app.';
 $string['autologinkeygenerationlockout'] = 'Auto-login key generation is blocked. You need to wait 6 minutes between requests.';
 $string['autologinnotallowedtoadmins'] = 'Auto-login is not allowed for site admins.';
 $string['cachedef_plugininfo'] = 'This stores the list of plugins with mobile addons';
index 4045ab1..1fee731 100644 (file)
@@ -213,6 +213,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         // Enable requeriments.
         $_GET['wstoken'] = $token->token;   // Mock parameters.
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+                'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         // Even if we force the password change for the current user we should be able to retrieve the key.
         set_user_preference('auth_forcepasswordchange', 1, $user->id);
 
@@ -240,6 +244,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         global $CFG;
         $this->resetAfterTest(true);
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+            'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         // Need to disable webservices to verify that's checked.
         $CFG->enablewebservices = 0;
         $CFG->enablemobilewebservice = 0;
@@ -256,6 +264,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
     public function test_get_autologin_key_missing_https() {
         global $CFG;
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+            'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         // Need to simulate a non HTTPS site here.
         $CFG->wwwroot = str_replace('https:', 'http:', $CFG->wwwroot);
 
@@ -276,6 +288,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         $this->resetAfterTest(true);
         $this->setAdminUser();
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+            'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         $this->expectException('moodle_exception');
         $this->expectExceptionMessage(get_string('autologinnotallowedtoadmins', 'tool_mobile'));
         $result = external::get_autologin_key('');
@@ -296,6 +312,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         $token = external_generate_token_for_current_user($service);
         $_GET['wstoken'] = $token->token;   // Mock parameters.
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+            'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         $result = external::get_autologin_key($token->privatetoken);
         $result = external_api::clean_returnvalue(external::get_autologin_key_returns(), $result);
 
@@ -311,6 +331,20 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         $result = external::get_autologin_key($token->privatetoken);
     }
 
+    /**
+     * Test get_autologin_key missing app_request.
+     */
+    public function test_get_autologin_key_missing_app_request() {
+        global $CFG;
+
+        $this->resetAfterTest(true);
+        $this->setAdminUser();
+
+        $this->expectException('moodle_exception');
+        $this->expectExceptionMessage(get_string('apprequired', 'tool_mobile'));
+        $result = external::get_autologin_key('');
+    }
+
     /**
      * Test get_content.
      */
index ec720c3..d39aa5f 100644 (file)
@@ -4,6 +4,10 @@ Information provided here is intended especially for developers.
 === 3.7 ===
 
  * New external function tool_mobile::tool_mobile_call_external_function allows calling multiple external functions and returns all responses.
+ * External function tool_mobile::get_autologin_key now only works if the request comes from the Moodle mobile or desktop app.
+   This increases confidence that requests did originate from the mobile app, decreasing the likelihood of an XSS attack.
+   If you want to use this functionality, please override the Web Service via the override_webservice_execution callback although
+   this is not recommended or encouraged.
 
 === 3.5 ===
 
index b08b6a5..6477c4d 100644 (file)
@@ -108,6 +108,16 @@ class category_bin extends base_bin {
 
         require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
 
+        // As far as recycle bin is using MODE_AUTOMATED, it observes the backup_auto_storage
+        // setting (storing backups @ real location. For recycle bin we want to ensure that
+        // backup files are always stored in Moodle file area. In order to achieve that, we
+        // hack the setting here via $CFG->forced_plugin_settings, so it won't interfere other
+        // operations.  See MDL-65218 for more information.
+        // This hack will be removed once recycle bin switches to use its own backup mode, with
+        // own preferences and 100% appart from MODLE_AUTOMATED.
+        // TODO: Remove this as part of MDL-65228.
+        $CFG->forced_plugin_settings['backup'] = ['backup_auto_storage' => 0];
+
         // Backup the course.
         $user = get_admin();
         $controller = new \backup_controller(
@@ -115,11 +125,15 @@ class category_bin extends base_bin {
             $course->id,
             \backup::FORMAT_MOODLE,
             \backup::INTERACTIVE_NO,
-            \backup::MODE_GENERAL,
+            \backup::MODE_AUTOMATED,
             $user->id
         );
         $controller->execute_plan();
 
+        // We don't need the forced setting anymore, hence unsetting it.
+        // TODO: Remove this as part of MDL-65228.
+        unset($CFG->forced_plugin_settings['backup']);
+
         // Grab the result.
         $result = $controller->get_results();
         if (!isset($result['backup_destination'])) {
@@ -233,7 +247,7 @@ class category_bin extends base_bin {
             $tempdir,
             $course->id,
             \backup::INTERACTIVE_NO,
-            \backup::MODE_GENERAL,
+            \backup::MODE_AUTOMATED,
             $user->id,
             \backup::TARGET_NEW_COURSE
         );
index d58d1a7..deec65c 100644 (file)
@@ -112,6 +112,16 @@ class course_bin extends base_bin {
             return;
         }
 
+        // As far as recycle bin is using MODE_AUTOMATED, it observes the backup_auto_storage
+        // setting (storing backups @ real location. For recycle bin we want to ensure that
+        // backup files are always stored in Moodle file area. In order to achieve that, we
+        // hack the setting here via $CFG->forced_plugin_settings, so it won't interfere other
+        // operations.  See MDL-65218 for more information.
+        // This hack will be removed once recycle bin switches to use its own backup mode, with
+        // own preferences and 100% appart from MODLE_AUTOMATED.
+        // TODO: Remove this as part of MDL-65228.
+        $CFG->forced_plugin_settings['backup'] = ['backup_auto_storage' => 0];
+
         // Backup the activity.
         $user = get_admin();
         $controller = new \backup_controller(
@@ -124,6 +134,10 @@ class course_bin extends base_bin {
         );
         $controller->execute_plan();
 
+        // We don't need the forced setting anymore, hence unsetting it.
+        // TODO: Remove this as part of MDL-65228.
+        unset($CFG->forced_plugin_settings['backup']);
+
         // Grab the result.
         $result = $controller->get_results();
         if (!isset($result['backup_destination'])) {
@@ -344,4 +358,4 @@ class course_bin extends base_bin {
         $context = \context_course::instance($this->_courseid);
         return has_capability('tool/recyclebin:deleteitems', $context);
     }
-}
\ No newline at end of file
+}
index bada3d6..0e24c31 100644 (file)
@@ -176,4 +176,116 @@ class tool_recyclebin_category_bin_tests extends advanced_testcase {
         $course = reset($courses);
         $this->assertEquals('Test course 2', $course->fullname);
     }
+
+    /**
+     * Provider for test_course_restore_with_userdata() and test_course_restore_without_userdata()
+     *
+     * Used to verify that recycle bin is immune to various settings. Provides plugin, name, value for
+     * direct usage with set_config()
+     */
+    public function recycle_bin_settings_provider() {
+        return [
+            'backup/backup_auto_storage moodle' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 0],
+            ]],
+
+            'backup/backup_auto_storage external' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 1],
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true],
+            ]],
+
+            'backup/backup_auto_storage mixed' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 2],
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true],
+            ]],
+        ];
+    }
+
+    /**
+     * Tests that user data is restored when course is restored.
+     *
+     * @dataProvider recycle_bin_settings_provider
+     * @param array $settings array of plugin, name, value stdClass().
+     */
+    public function test_course_restore_with_userdata($settings) {
+        global $DB;
+
+        // Force configuration changes from provider.
+        foreach ($settings as $setting) {
+            // Need to create a directory for backup_auto_destination.
+            if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) {
+                $setting->value = make_request_directory();
+            }
+            set_config($setting->name, $setting->value, $setting->plugin);
+        }
+
+        // We want user data to be included for this test.
+        set_config('backup_auto_users', true, 'backup');
+
+        $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student');
+
+        // Delete course.
+        delete_course($this->course, false);
+        $this->assertFalse($DB->record_exists('course', ['id' => $this->course->id]));
+
+        // Verify there is now a backup @ cat recycle bin file area.
+        $recyclebin = new \tool_recyclebin\category_bin($this->course->category);
+        $this->assertEquals(1, count($recyclebin->get_items()));
+
+        // Restore the recycle bin item.
+        $recyclebin->restore_item(current($recyclebin->get_items()));
+
+        // Get the new course.
+        $newcourse = $DB->get_record('course', ['shortname' => $this->course->shortname], '*', MUST_EXIST);
+
+        // Check that it was removed from the recycle bin.
+        $this->assertEquals(0, count($recyclebin->get_items()));
+
+        // Verify that student DOES continue enrolled.
+        $this->assertTrue(is_enrolled(context_course::instance($newcourse->id), $student->id));
+    }
+
+    /**
+     * Tests that user data is not restored when course is restored.
+     *
+     * @dataProvider recycle_bin_settings_provider
+     * @param array $settings array of plugin, name, value stdClass().
+     */
+    public function test_course_restore_without_userdata($settings) {
+        global $DB;
+
+        // Force configuration changes from provider.
+        foreach ($settings as $setting) {
+            // Need to create a directory for backup_auto_destination.
+            if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) {
+                $setting->value = make_request_directory();
+            }
+            set_config($setting->name, $setting->value, $setting->plugin);
+        }
+
+        // We want user data to be included for this test.
+        set_config('backup_auto_users', false, 'backup');
+
+        $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student');
+
+        // Delete course.
+        delete_course($this->course, false);
+        $this->assertFalse($DB->record_exists('course', ['id' => $this->course->id]));
+
+        // Verify there is now a backup @ cat recycle bin file area.
+        $recyclebin = new \tool_recyclebin\category_bin($this->course->category);
+        $this->assertEquals(1, count($recyclebin->get_items()));
+
+        // Restore the recycle bin item.
+        $recyclebin->restore_item(current($recyclebin->get_items()));
+
+        // Get the new course.
+        $newcourse = $DB->get_record('course', ['shortname' => $this->course->shortname], '*', MUST_EXIST);
+
+        // Check that it was removed from the recycle bin.
+        $this->assertEquals(0, count($recyclebin->get_items()));
+
+        // Verify that student DOES NOT continue enrolled.
+        $this->assertFalse(is_enrolled(context_course::instance($newcourse->id), $student->id));
+    }
 }
index 9d99326..44f9db6 100644 (file)
@@ -174,10 +174,46 @@ class tool_recyclebin_course_bin_tests extends advanced_testcase {
         $this->assertEquals($book->name, $deletedbook->name);
     }
 
+    /**
+     * Provider for test_coursemodule_restore_with_userdata() and test_coursemodule_restore_without_userdata()
+     *
+     * Used to verify that recycle bin is immune to various settings. Provides plugin, name, value for
+     * direct usage with set_config()
+     */
+    public function recycle_bin_settings_provider() {
+        return [
+            'backup/backup_auto_storage moodle' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 0],
+            ]],
+
+            'backup/backup_auto_storage external' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 1],
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true],
+            ]],
+
+            'backup/backup_auto_storage mixed' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 2],
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true],
+            ]],
+        ];
+    }
+
     /**
      * Tests that user data is restored when module is restored.
+     *
+     * @dataProvider recycle_bin_settings_provider
+     * @param array $settings array of plugin, name, value stdClass().
      */
-    public function test_coursemodule_restore_with_userdata() {
+    public function test_coursemodule_restore_with_userdata($settings) {
+        // Force configuration changes from provider.
+        foreach ($settings as $setting) {
+            // Need to create a directory for backup_auto_destination.
+            if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) {
+                $setting->value = make_request_directory();
+            }
+            set_config($setting->name, $setting->value, $setting->plugin);
+        }
+
         $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student');
         $this->setUser($student);
 
@@ -211,8 +247,20 @@ class tool_recyclebin_course_bin_tests extends advanced_testcase {
 
     /**
      * Tests that user data is not restored when module is restored.
+     *
+     * @dataProvider recycle_bin_settings_provider
+     * @param array $settings array of plugin, name, value stdClass().
      */
-    public function test_coursemodule_restore_without_userdata() {
+    public function test_coursemodule_restore_without_userdata($settings) {
+        // Force configuration changes from provider.
+        foreach ($settings as $setting) {
+            // Need to create a directory for backup_auto_destination.
+            if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) {
+                $setting->value = make_request_directory();
+            }
+            set_config($setting->name, $setting->value, $setting->plugin);
+        }
+
         $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student');
         $this->setUser($student);
 
index adf90ff..0759217 100644 (file)
@@ -4,7 +4,7 @@ require_once($CFG->libdir.'/adminlib.php');
 require_once($CFG->dirroot.'/message/lib.php');
 require_once('user_message_form.php');
 
-$msg     = optional_param('msg', '', PARAM_CLEANHTML);
+$msg     = optional_param('msg', '', PARAM_RAW);
 $confirm = optional_param('confirm', 0, PARAM_BOOL);
 
 admin_externalpage_setup('userbulk');
@@ -45,6 +45,7 @@ if ($msgform->is_cancelled()) {
     $options->para     = false;
     $options->newlines = true;
     $options->smiley   = false;
+    $options->trusted = trusttext_trusted(\context_system::instance());
 
     $msg = format_text($formdata->messagebody['text'], $formdata->messagebody['format'], $options);
 
index e9dcaae..97faf59 100644 (file)
@@ -29,10 +29,6 @@ defined('MOODLE_INTERNAL') || die();
 /**
  * Any element analysers can analyse.
  *
- * Analysers get_analysers method return all analysable elements in the site;
- * it is important that analysable elements implement lazy loading to avoid
- * big memory footprints. See \core_analytics\course example.
- *
  * @package   core_analytics
  * @copyright 2016 David Monllao {@link http://www.davidmonllao.com}
  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
diff --git a/analytics/classes/analysis.php b/analytics/classes/analysis.php
new file mode 100644 (file)
index 0000000..bcb621c
--- /dev/null
@@ -0,0 +1,894 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Runs an analysis of the site.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Runs an analysis of the site.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class analysis {
+
+    /**
+     * @var \core_analytics\local\analyser\base
+     */
+    private $analyser;
+
+    /**
+     * @var bool Whether to calculate the target or not in this run.
+     */
+    private $includetarget;
+
+    /**
+     * @var \core_analytics\local\analysis\result
+     */
+    private $result;
+
+    /**
+     * @var \core\lock\lock
+     */
+    private $lock;
+
+    /**
+     * Constructor.
+     *
+     * @param \core_analytics\local\analyser\base   $analyser
+     * @param bool                                  $includetarget Whether to calculate the target or not.
+     * @param \core_analytics\local\analysis\result $result
+     */
+    public function __construct(\core_analytics\local\analyser\base $analyser, bool $includetarget,
+            \core_analytics\local\analysis\result $result) {
+        $this->analyser = $analyser;
+        $this->includetarget = $includetarget;
+        $this->result = $result;
+    }
+
+    /**
+     * Runs the analysis.
+     *
+     * @return null
+     */
+    public function run() {
+
+        $options = $this->analyser->get_options();
+
+        // Time limit control.
+        $modeltimelimit = intval(get_config('analytics', 'modeltimelimit'));
+
+        $filesbytimesplitting = array();
+
+        $alreadyprocessedanalysables = $this->get_processed_analysables();
+
+        if ($this->includetarget) {
+            $action = 'training';
+        } else {
+            $action = 'prediction';
+        }
+        $analysables = $this->analyser->get_analysables_iterator($action);
+
+        $inittime = microtime(true);
+        foreach ($analysables as $analysable) {
+            $processed = false;
+
+            if (!$analysable) {
+                continue;
+            }
+
+            $analysableresults = $this->process_analysable($analysable);
+            if ($analysableresults) {
+                $processed = $this->result->add_analysable_results($analysableresults);
+                if (!$processed) {
+                    $errors = array();
+                    foreach ($analysableresults as $timesplittingid => $result) {
+                        $str = '';
+                        if (count($analysableresults) > 1) {
+                            $str .= $timesplittingid . ': ';
+                        }
+                        $str .= $result->message;
+                        $errors[] = $str;
+                    }
+
+                    $a = new \stdClass();
+                    $a->analysableid = $analysable->get_name();
+                    $a->errors = implode(', ', $errors);
+                    $this->analyser->add_log(get_string('analysablenotused', 'analytics', $a));
+                }
+            }
+
+            // Updated regardless of how well the analysis went.
+            if ($this->analyser->get_target()->always_update_analysis_time() || $processed) {
+                $this->update_analysable_analysed_time($alreadyprocessedanalysables, $analysable->get_id());
+            }
+
+            // Apply time limit.
+            if (!$options['evaluation']) {
+                $timespent = microtime(true) - $inittime;
+                if ($modeltimelimit <= $timespent) {
+                    break;
+                }
+            }
+        }
+    }
+
+    /**
+     * Get analysables that have been already processed.
+     *
+     * @return \stdClass[]
+     */
+    protected function get_processed_analysables(): array {
+        global $DB;
+
+        $params = array('modelid' => $this->analyser->get_modelid());
+        $params['action'] = ($this->includetarget) ? 'training' : 'prediction';
+        $select = 'modelid = :modelid and action = :action';
+
+        // Weird select fields ordering for performance (analysableid key matching, analysableid is also unique by modelid).
+        return $DB->get_records_select('analytics_used_analysables', $select,
+            $params, 'timeanalysed DESC', 'analysableid, modelid, action, timeanalysed, id AS primarykey');
+    }
+
+    /**
+     * Processes an analysable
+     *
+     * This method returns the general analysable status, an array of files by time splitting method and
+     * an error message if there is any problem.
+     *
+     * @param \core_analytics\analysable $analysable
+     * @return \stdClass[] Results objects by time splitting method
+     */
+    public function process_analysable(\core_analytics\analysable $analysable): array {
+
+        // Target instances scope is per-analysable (it can't be lower as calculations run once per
+        // analysable, not time splitting method nor time range).
+        $target = call_user_func(array($this->analyser->get_target(), 'instance'));
+
+        // We need to check that the analysable is valid for the target even if we don't include targets
+        // as we still need to discard invalid analysables for the target.
+        $isvalidresult = $target->is_valid_analysable($analysable, $this->includetarget);
+        if ($isvalidresult !== true) {
+            $a = new \stdClass();
+            $a->analysableid = $analysable->get_name();
+            $a->result = $isvalidresult;
+            $this->analyser->add_log(get_string('analysablenotvalidfortarget', 'analytics', $a));
+            return array();
+        }
+
+        // Process all provided time splitting methods.
+        $results = array();
+        foreach ($this->analyser->get_timesplittings() as $timesplitting) {
+
+            $cachedresult = $this->result->retrieve_cached_result($timesplitting, $analysable);
+            if ($cachedresult) {
+                $result = new \stdClass();
+                $result->result = $cachedresult;
+                $results[$timesplitting->get_id()] = $result;
+                continue;
+            }
+
+            $results[$timesplitting->get_id()] = $this->process_time_splitting($timesplitting, $analysable, $target);
+        }
+
+        return $results;
+    }
+
+    /**
+     * Processes the analysable samples using the provided time splitting method.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param \core_analytics\analysable $analysable
+     * @param \core_analytics\local\target\base $target
+     * @return \stdClass Results object.
+     */
+    protected function process_time_splitting(\core_analytics\local\time_splitting\base $timesplitting,
+            \core_analytics\analysable $analysable, \core_analytics\local\target\base $target): \stdClass {
+
+        $options = $this->analyser->get_options();
+
+        $result = new \stdClass();
+
+        if (!$timesplitting->is_valid_analysable($analysable)) {
+            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+            $result->message = get_string('invalidanalysablefortimesplitting', 'analytics',
+                $timesplitting->get_name());
+            return $result;
+        }
+        $timesplitting->set_analysable($analysable);
+
+        if (CLI_SCRIPT && !PHPUNIT_TEST) {
+            mtrace('Analysing id "' . $analysable->get_id() . '" with "' . $timesplitting->get_name() .
+                '" time splitting method...');
+        }
+
+        // What is a sample is defined by the analyser, it can be an enrolment, a course, a user, a question
+        // attempt... it is on what we will base indicators calculations.
+        list($sampleids, $samplesdata) = $this->analyser->get_all_samples($analysable);
+
+        if (count($sampleids) === 0) {
+            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+            $result->message = get_string('nodata', 'analytics');
+            return $result;
+        }
+
+        if ($this->includetarget) {
+            // All ranges are used when we are calculating data for training.
+            $ranges = $timesplitting->get_training_ranges();
+        } else {
+            // The latest range that has not yet been used for prediction (it depends on the time range where we are right now).
+            $ranges = $timesplitting->get_most_recent_prediction_range();
+        }
+
+        // There is no need to keep track of the evaluated samples and ranges as we always evaluate the whole dataset.
+        if ($options['evaluation'] === false) {
+
+            if (empty($ranges)) {
+                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+                $result->message = get_string('noranges', 'analytics');
+                return $result;
+            }
+
+            // We skip all samples that are already part of a training dataset, even if they have not been used for prediction.
+            if (!$target::based_on_assumptions()) {
+                // Targets based on assumptions can not be trained.
+                $this->filter_out_train_samples($sampleids, $timesplitting);
+            }
+
+            if (count($sampleids) === 0) {
+                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+                $result->message = get_string('nonewdata', 'analytics');
+                return $result;
+            }
+
+            // Only when processing data for predictions.
+            if (!$this->includetarget) {
+                // We also filter out samples and ranges that have already been used for predictions.
+                $predictsamplesrecord = $this->filter_out_prediction_samples_and_ranges($sampleids, $ranges, $timesplitting);
+            }
+
+            if (count($sampleids) === 0) {
+                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+                $result->message = get_string('nonewdata', 'analytics');
+                return $result;
+            }
+
+            if (count($ranges) === 0) {
+                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+                $result->message = get_string('nonewranges', 'analytics');
+                return $result;
+            }
+        }
+
+        // Flag the model + analysable + timesplitting as being analysed (prevent concurrent executions).
+        if (!$this->init_analysable_analysis($timesplitting->get_id(), $analysable->get_id())) {
+            // If this model + analysable + timesplitting combination is being analysed we skip this process.
+            $result->status = \core_analytics\model::NO_DATASET;
+            $result->message = get_string('analysisinprogress', 'analytics');
+            return $result;
+        }
+
+        // Remove samples the target consider invalid.
+        try {
+            $target->add_sample_data($samplesdata);
+            $target->filter_out_invalid_samples($sampleids, $analysable, $this->includetarget);
+        } catch (\Throwable $e) {
+            $this->finish_analysable_analysis();
+            throw $e;
+        }
+
+        if (!$sampleids) {
+            $result->status = \core_analytics\model::NO_DATASET;
+            $result->message = get_string('novalidsamples', 'analytics');
+            $this->finish_analysable_analysis();
+            return $result;
+        }
+
+        try {
+            $indicators = $this->analyser->get_indicators();
+            foreach ($indicators as $key => $indicator) {
+                // The analyser attaches the main entities the sample depends on and are provided to the
+                // indicator to calculate the sample.
+                $indicators[$key]->add_sample_data($samplesdata);
+            }
+
+            // Here we start the memory intensive process that will last until $data var is
+            // unset (until the method is finished basically).
+            $data = $this->calculate($timesplitting, $sampleids, $ranges, $target);
+        } catch (\Throwable $e) {
+            $this->finish_analysable_analysis();
+            throw $e;
+        }
+
+        if (!$data) {
+            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+            $result->message = get_string('novaliddata', 'analytics');
+            $this->finish_analysable_analysis();
+            return $result;
+        }
+
+        try {
+            // No need to keep track of analysed stuff when evaluating.
+            if ($options['evaluation'] === false) {
+                // Save the samples that have been already analysed so they are not analysed again in future.
+
+                if ($this->includetarget) {
+                    $this->save_train_samples($sampleids, $timesplitting);
+                } else {
+                    // The variable $predictsamplesrecord will always be set as filter_out_prediction_samples_and_ranges
+                    // will always be called before it (no evaluation mode and no includetarget).
+                    $this->save_prediction_samples($sampleids, $ranges, $timesplitting, $predictsamplesrecord);
+                }
+            }
+
+            // We need to pass all the analysis data.
+            $formattedresult = $this->result->format_result($data, $target, $timesplitting, $analysable);
+
+        } catch (\Throwable $e) {
+            $this->finish_analysable_analysis();
+            throw $e;
+        }
+
+        if (!$formattedresult) {
+            $this->finish_analysable_analysis();
+            throw new \moodle_exception('errorcannotwritedataset', 'analytics');
+        }
+
+        $result->status = \core_analytics\model::OK;
+        $result->message = get_string('successfullyanalysed', 'analytics');
+        $result->result = $formattedresult;
+
+        // Flag the model + analysable + timesplitting as analysed.
+        $this->finish_analysable_analysis();
+
+        return $result;
+    }
+
+    /**
+     * Calculates indicators and targets.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param array $sampleids
+     * @param array $ranges
+     * @param \core_analytics\local\target\base $target
+     * @return array|null
+     */
+    public function calculate(\core_analytics\local\time_splitting\base $timesplitting, array &$sampleids,
+            array $ranges, \core_analytics\local\target\base $target): ?array {
+
+        $calculatedtarget = null;
+        if ($this->includetarget) {
+            // We first calculate the target because analysable data may still be invalid or none
+            // of the analysable samples may be valid.
+            $calculatedtarget = $target->calculate($sampleids, $timesplitting->get_analysable());
+
+            // We remove samples we can not calculate their target.
+            $sampleids = array_filter($sampleids, function($sampleid) use ($calculatedtarget) {
+                if (is_null($calculatedtarget[$sampleid])) {
+                    return false;
+                }
+                return true;
+            });
+        }
+
+        // No need to continue calculating if the target couldn't be calculated for any sample.
+        if (empty($sampleids)) {
+            return null;
+        }
+
+        $dataset = $this->calculate_indicators($timesplitting, $sampleids, $ranges);
+
+        if (empty($dataset)) {
+            return null;
+        }
+
+        // Now that we have the indicators in place we can add the time range indicators (and target if provided) to each of them.
+        $this->fill_dataset($timesplitting, $dataset, $calculatedtarget);
+
+        $this->add_context_metadata($timesplitting, $dataset, $target);
+
+        if (!PHPUNIT_TEST && CLI_SCRIPT) {
+            echo PHP_EOL;
+        }
+
+        return $dataset;
+    }
+
+    /**
+     * Calculates indicators.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param array $sampleids
+     * @param array $ranges
+     * @return array
+     */
+    protected function calculate_indicators(\core_analytics\local\time_splitting\base $timesplitting, array $sampleids,
+            array $ranges): array {
+        global $DB;
+
+        $options = $this->analyser->get_options();
+
+        $dataset = array();
+
+        // Faster to run 1 db query per range.
+        $existingcalculations = array();
+        if ($timesplitting->cache_indicator_calculations()) {
+            foreach ($ranges as $rangeindex => $range) {
+                // Load existing calculations.
+                $existingcalculations[$rangeindex] = \core_analytics\manager::get_indicator_calculations(
+                    $timesplitting->get_analysable(), $range['start'], $range['end'], $this->analyser->get_samples_origin());
+            }
+        }
+
+        // Here we store samples which calculations are not all null.
+        $notnulls = array();
+
+        // Fill the dataset samples with indicators data.
+        $newcalculations = array();
+        foreach ($this->analyser->get_indicators() as $indicator) {
+
+            // Hook to allow indicators to store analysable-dependant data.
+            $indicator->fill_per_analysable_caches($timesplitting->get_analysable());
+
+            // Per-range calculations.
+            foreach ($ranges as $rangeindex => $range) {
+
+                // Indicator instances are per-range.
+                $rangeindicator = clone $indicator;
+
+                $prevcalculations = array();
+                if (!empty($existingcalculations[$rangeindex][$rangeindicator->get_id()])) {
+                    $prevcalculations = $existingcalculations[$rangeindex][$rangeindicator->get_id()];
+                }
+
+                // Calculate the indicator for each sample in this time range.
+                list($samplesfeatures, $newindicatorcalculations, $indicatornotnulls) = $rangeindicator->calculate($sampleids,
+                    $this->analyser->get_samples_origin(), $range['start'], $range['end'], $prevcalculations);
+
+                // Copy the features data to the dataset.
+                foreach ($samplesfeatures as $analysersampleid => $features) {
+
+                    $uniquesampleid = $timesplitting->append_rangeindex($analysersampleid, $rangeindex);
+
+                    if (!isset($notnulls[$uniquesampleid]) && !empty($indicatornotnulls[$analysersampleid])) {
+                        $notnulls[$uniquesampleid] = $uniquesampleid;
+                    }
+
+                    // Init the sample if it is still empty.
+                    if (!isset($dataset[$uniquesampleid])) {
+                        $dataset[$uniquesampleid] = array();
+                    }
+
+                    // Append the features indicator features at the end of the sample.
+                    $dataset[$uniquesampleid] = array_merge($dataset[$uniquesampleid], $features);
+                }
+
+                if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations()) {
+                    $timecreated = time();
+                    foreach ($newindicatorcalculations as $sampleid => $calculatedvalue) {
+                        // Prepare the new calculations to be stored into DB.
+
+                        $indcalc = new \stdClass();
+                        $indcalc->contextid = $timesplitting->get_analysable()->get_context()->id;
+                        $indcalc->starttime = $range['start'];
+                        $indcalc->endtime = $range['end'];
+                        $indcalc->sampleid = $sampleid;
+                        $indcalc->sampleorigin = $this->analyser->get_samples_origin();
+                        $indcalc->indicator = $rangeindicator->get_id();
+                        $indcalc->value = $calculatedvalue;
+                        $indcalc->timecreated = $timecreated;
+                        $newcalculations[] = $indcalc;
+                    }
+                }
+            }
+
+            if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations()) {
+                $batchsize = self::get_insert_batch_size();
+                if (count($newcalculations) > $batchsize) {
+                    // We don't want newcalculations array to grow too much as we already keep the
+                    // system memory busy storing $dataset contents.
+
+                    // Insert from the beginning.
+                    $remaining = array_splice($newcalculations, $batchsize);
+
+                    // Sorry mssql and oracle, this will be slow.
+                    $DB->insert_records('analytics_indicator_calc', $newcalculations);
+                    $newcalculations = $remaining;
+                }
+            }
+        }
+
+        if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations() && $newcalculations) {
+            // Insert the remaining records.
+            $DB->insert_records('analytics_indicator_calc', $newcalculations);
+        }
+
+        // Delete rows where all calculations are null.
+        // We still store the indicator calculation and we still store the sample id as
+        // processed so we don't have to process this sample again, but we exclude it
+        // from the dataset because it is not useful.
+        $nulls = array_diff_key($dataset, $notnulls);
+        foreach ($nulls as $uniqueid => $ignoredvalues) {
+            unset($dataset[$uniqueid]);
+        }
+
+        return $dataset;
+    }
+
+    /**
+     * Adds time range indicators and the target to each sample.
+     *
+     * This will identify the sample as belonging to a specific range.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param array $dataset
+     * @param array|null $calculatedtarget
+     * @return null
+     */
+    protected function fill_dataset(\core_analytics\local\time_splitting\base $timesplitting,
+            array &$dataset, ?array $calculatedtarget = null) {
+
+        $nranges = count($timesplitting->get_distinct_ranges());
+
+        foreach ($dataset as $uniquesampleid => $unmodified) {
+
+            list($analysersampleid, $rangeindex) = $timesplitting->infer_sample_info($uniquesampleid);
+
+            // No need to add range features if this time splitting method only defines one time range.
+            if ($nranges > 1) {
+
+                // 1 column for each range.
+                $timeindicators = array_fill(0, $nranges, 0);
+
+                $timeindicators[$rangeindex] = 1;
+
+                $dataset[$uniquesampleid] = array_merge($timeindicators, $dataset[$uniquesampleid]);
+            }
+
+            if ($calculatedtarget) {
+                // Add this sampleid's calculated target and the end.
+                $dataset[$uniquesampleid][] = $calculatedtarget[$analysersampleid];
+
+            } else {
+                // Add this sampleid, it will be used to identify the prediction that comes back from
+                // the predictions processor.
+                array_unshift($dataset[$uniquesampleid], $uniquesampleid);
+            }
+        }
+    }
+
+    /**
+     * Updates the analysable analysis time.
+     *
+     * @param array $processedanalysables
+     * @param int $analysableid
+     * @return null
+     */
+    protected function update_analysable_analysed_time(array $processedanalysables, int $analysableid) {
+        global $DB;
+
+        if (!empty($processedanalysables[$analysableid])) {
+            $obj = $processedanalysables[$analysableid];
+
+            $obj->id = $obj->primarykey;
+            unset($obj->primarykey);
+
+            $obj->timeanalysed = time();
+            $DB->update_record('analytics_used_analysables', $obj);
+
+        } else {
+
+            $obj = new \stdClass();
+            $obj->modelid = $this->analyser->get_modelid();
+            $obj->action = ($this->includetarget) ? 'training' : 'prediction';
+            $obj->analysableid = $analysableid;
+            $obj->timeanalysed = time();
+
+            $DB->insert_record('analytics_used_analysables', $obj);
+        }
+    }
+
+    /**
+     * Adds dataset context info.
+     *
+     * The final dataset document will look like this:
+     * ----------------------------------------------------
+     * metadata1,metadata2,metadata3,.....
+     * value1, value2, value3,.....
+     *
+     * header1,header2,header3,header4,.....
+     * stud1value1,stud1value2,stud1value3,stud1value4,.....
+     * stud2value1,stud2value2,stud2value3,stud2value4,.....
+     * .....
+     * ----------------------------------------------------
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param array $dataset
+     * @param \core_analytics\local\target\base $target
+     * @return null
+     */
+    protected function add_context_metadata(\core_analytics\local\time_splitting\base $timesplitting, array &$dataset,
+            \core_analytics\local\target\base $target) {
+        $headers = $this->get_headers($timesplitting, $target);
+
+        // This will also reset samples' dataset keys.
+        array_unshift($dataset, $headers);
+    }
+
+    /**
+     * Returns the headers for the csv file based on the indicators and the target.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param \core_analytics\local\target\base $target
+     * @return string[]
+     */
+    public function get_headers(\core_analytics\local\time_splitting\base $timesplitting,
+            \core_analytics\local\target\base $target): array {
+        // 3rd column will contain the indicator ids.
+        $headers = array();
+
+        if (!$this->includetarget) {
+            // The first column is the sampleid.
+            $headers[] = 'sampleid';
+        }
+
+        // We always have 1 column for each time splitting method range, it does not depend on how
+        // many ranges we calculated.
+        $ranges = $timesplitting->get_distinct_ranges();
+        if (count($ranges) > 1) {
+            foreach ($ranges as $rangeindex) {
+                $headers[] = 'range/' . $rangeindex;
+            }
+        }
+
+        // Model indicators.
+        foreach ($this->analyser->get_indicators() as $indicator) {
+            $headers = array_merge($headers, $indicator::get_feature_headers());
+        }
+
+        // The target as well.
+        if ($this->includetarget) {
+            $headers[] = $target->get_id();
+        }
+
+        return $headers;
+    }
+
+    /**
+     * Filters out samples that have already been used for training.
+     *
+     * @param int[] $sampleids
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @return  null
+     */
+    protected function filter_out_train_samples(array &$sampleids, \core_analytics\local\time_splitting\base $timesplitting) {
+        global $DB;
+
+        $params = array('modelid' => $this->analyser->get_modelid(), 'analysableid' => $timesplitting->get_analysable()->get_id(),
+            'timesplitting' => $timesplitting->get_id());
+
+        $trainingsamples = $DB->get_records('analytics_train_samples', $params);
+
+        // Skip each file trained samples.
+        foreach ($trainingsamples as $trainingfile) {
+
+            $usedsamples = json_decode($trainingfile->sampleids, true);
+
+            if (!empty($usedsamples)) {
+                // Reset $sampleids to $sampleids minus this file's $usedsamples.
+                $sampleids = array_diff_key($sampleids, $usedsamples);
+            }
+        }
+    }
+
+    /**
+     * Filters out samples that have already been used for prediction.
+     *
+     * @param int[] $sampleids
+     * @param array $ranges
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @return  \stdClass|null The analytics_predict_samples record or null
+     */
+    protected function filter_out_prediction_samples_and_ranges(array &$sampleids, array &$ranges,
+            \core_analytics\local\time_splitting\base $timesplitting) {
+
+        if (count($ranges) > 1) {
+            throw new \coding_exception('$ranges argument should only contain one range');
+        }
+
+        $rangeindex = key($ranges);
+        $predictedrange = $this->get_predict_samples_record($timesplitting, $rangeindex);
+
+        if (!$predictedrange) {
+            // Nothing to filter out.
+            return null;
+        }
+
+        $predictedrange->sampleids = json_decode($predictedrange->sampleids, true);
+        $missingsamples = array_diff_key($sampleids, $predictedrange->sampleids);
+        if (count($missingsamples) === 0) {
+            // All samples already calculated.
+            unset($ranges[$rangeindex]);
+            return null;
+        }
+
+        // Replace the list of samples by the one excluding samples that already got predictions at this range.
+        $sampleids = $missingsamples;
+
+        return $predictedrange;
+    }
+
+    /**
+     * Returns a predict samples record.
+     *
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  int                                       $rangeindex
+     * @return \stdClass|false
+     */
+    private function get_predict_samples_record(\core_analytics\local\time_splitting\base $timesplitting, int $rangeindex) {
+        global $DB;
+
+        $params = array('modelid' => $this->analyser->get_modelid(), 'analysableid' => $timesplitting->get_analysable()->get_id(),
+            'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex);
+        $predictedrange = $DB->get_record('analytics_predict_samples', $params);
+
+        return $predictedrange;
+    }
+
+    /**
+     * Saves samples that have just been used for training.
+     *
+     * @param int[] $sampleids
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @return null
+     */
+    protected function save_train_samples(array $sampleids, \core_analytics\local\time_splitting\base $timesplitting) {
+        global $DB;
+
+        $trainingsamples = new \stdClass();
+        $trainingsamples->modelid = $this->analyser->get_modelid();
+        $trainingsamples->analysableid = $timesplitting->get_analysable()->get_id();
+        $trainingsamples->timesplitting = $timesplitting->get_id();
+
+        $trainingsamples->sampleids = json_encode($sampleids);
+        $trainingsamples->timecreated = time();
+
+        $DB->insert_record('analytics_train_samples', $trainingsamples);
+    }
+
+    /**
+     * Saves samples that have just been used for prediction.
+     *
+     * @param int[] $sampleids
+     * @param array $ranges
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param \stdClass|null $predictsamplesrecord The existing record or null if there is no record yet.
+     * @return null
+     */
+    protected function save_prediction_samples(array $sampleids, array $ranges,
+            \core_analytics\local\time_splitting\base $timesplitting, ?\stdClass $predictsamplesrecord = null) {
+        global $DB;
+
+        if (count($ranges) > 1) {
+            throw new \coding_exception('$ranges argument should only contain one range');
+        }
+
+        $rangeindex = key($ranges);
+
+        if ($predictsamplesrecord) {
+            // Append the new samples used for prediction.
+            $predictsamplesrecord->sampleids = json_encode($predictsamplesrecord->sampleids + $sampleids);
+            $predictsamplesrecord->timemodified = time();
+            $DB->update_record('analytics_predict_samples', $predictsamplesrecord);
+        } else {
+            $predictsamplesrecord = (object)[
+                'modelid' => $this->analyser->get_modelid(),
+                'analysableid' => $timesplitting->get_analysable()->get_id(),
+                'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex
+            ];
+            $predictsamplesrecord->sampleids = json_encode($sampleids);
+            $predictsamplesrecord->timecreated = time();
+            $predictsamplesrecord->timemodified = $predictsamplesrecord->timecreated;
+            $DB->insert_record('analytics_predict_samples', $predictsamplesrecord);
+        }
+    }
+
+    /**
+     * Flags the analysable element as in-analysis and stores a lock for it.
+     *
+     * @param  string $timesplittingid
+     * @param  int    $analysableid
+     * @return bool Success or not
+     */
+    private function init_analysable_analysis(string $timesplittingid, int $analysableid) {
+
+        // Do not include $this->includetarget as we don't want the same analysable to be analysed for training
+        // and prediction at the same time.
+        $lockkey = 'modelid:' . $this->analyser->get_modelid() . '-analysableid:' . $analysableid .
+            '-timesplitting:' . self::clean_time_splitting_id($timesplittingid);
+
+        // Large timeout as processes may be quite long.
+        $lockfactory = \core\lock\lock_config::get_lock_factory('core_analytics');
+
+        // If it is not ready in 10 secs skip this model + analysable + timesplittingmethod combination
+        // it will attempt it again during next cron run.
+        if (!$this->lock = $lockfactory->get_lock($lockkey, 10)) {
+            return false;
+        }
+        return true;
+    }
+
+
+    /**
+     * Remove all possibly problematic chars from the time splitting method id (id = its full class name).
+     *
+     * @param string $timesplittingid
+     * @return string
+     */
+    public static function clean_time_splitting_id($timesplittingid) {
+        $timesplittingid = str_replace('\\', '-', $timesplittingid);
+        return clean_param($timesplittingid, PARAM_ALPHANUMEXT);
+    }
+
+    /**
+     * Mark the currently analysed analysable+timesplitting as analysed.
+     *
+     * @return null
+     */
+    private function finish_analysable_analysis() {
+        $this->lock->release();
+    }
+
+    /**
+     * Returns the batch size used for insert_records.
+     *
+     * This method tries to find the best batch size without getting
+     * into dml internals. Maximum 1000 records to save memory.
+     *
+     * @return int
+     */
+    private static function get_insert_batch_size(): int {
+        global $DB;
+
+        $dbconfig = $DB->export_dbconfig();
+
+        // 500 is pgsql default so using 1000 is fine, no other db driver uses a hardcoded value.
+        if (empty($dbconfig) || empty($dbconfig->dboptions) || empty($dbconfig->dboptions['bulkinsertsize'])) {
+            return 1000;
+        }
+
+        $bulkinsert = $dbconfig->dboptions['bulkinsertsize'];
+        if ($bulkinsert < 1000) {
+            return $bulkinsert;
+        }
+
+        while ($bulkinsert > 1000) {
+            $bulkinsert = round($bulkinsert / 2, 0);
+        }
+
+        return (int)$bulkinsert;
+    }
+}
index 919f36b..6e372cf 100644 (file)
@@ -132,12 +132,11 @@ class course implements \core_analytics\analysable {
      * Use self::instance() instead to get cached copies of the course. Instances obtained
      * through this constructor will not be cached.
      *
-     * Lazy load of course data, students and teachers.
-     *
-     * @param int|\stdClass $course Course id
+     * @param int|\stdClass $course Course id or mdl_course record
+     * @param \context|null $context
      * @return void
      */
-    public function __construct($course) {
+    public function __construct($course, ?\context $context = null) {
 
         if (is_scalar($course)) {
             $this->course = new \stdClass();
@@ -145,6 +144,10 @@ class course implements \core_analytics\analysable {
         } else {
             $this->course = $course;
         }
+
+        if (!is_null($context)) {
+            $this->coursecontext = $context;
+        }
     }
 
     /**
@@ -153,9 +156,10 @@ class course implements \core_analytics\analysable {
      * Lazy load of course data, students and teachers.
      *
      * @param int|\stdClass $course Course object or course id
+     * @param \context|null $context
      * @return \core_analytics\course
      */
-    public static function instance($course) {
+    public static function instance($course, ?\context $context = null) {
 
         $courseid = $course;
         if (!is_scalar($courseid)) {
@@ -166,7 +170,7 @@ class course implements \core_analytics\analysable {
             return self::$cachedinstance;
         }
 
-        $cachedinstance = new \core_analytics\course($course);
+        $cachedinstance = new \core_analytics\course($course, $context);
         self::$cachedinstance = $cachedinstance;
         self::$cachedid = (int)$courseid;
         return self::$cachedinstance;
index 00b571a..811bf7c 100644 (file)
@@ -109,31 +109,8 @@ class dataset_manager {
         $this->modelid = $modelid;
         $this->analysableid = $analysableid;
         $this->timesplittingid = $timesplittingid;
-        $this->evaluation = $evaluation;
         $this->filearea = $filearea;
-    }
-
-    /**
-     * Mark the analysable as being analysed.
-     *
-     * @return bool Could we get the lock or not.
-     */
-    public function init_process() {
-
-        // Do not include $this->includetarget as we don't want the same analysable to be analysed for training
-        // and prediction at the same time.
-        $lockkey = 'modelid:' . $this->modelid . '-analysableid:' . $this->analysableid .
-            '-timesplitting:' . self::clean_time_splitting_id($this->timesplittingid);
-
-        // Large timeout as processes may be quite long.
-        $lockfactory = \core\lock\lock_config::get_lock_factory('core_analytics');
-
-        // If it is not ready in 10 secs skip this model + analysable + timesplittingmethod combination
-        // it will attempt it again during next cron run.
-        if (!$this->lock = $lockfactory->get_lock($lockkey, 10)) {
-            return false;
-        }
-        return true;
+        $this->evaluation = $evaluation;
     }
 
     /**
@@ -152,7 +129,8 @@ class dataset_manager {
             'filearea' => $this->filearea,
             'itemid' => $this->modelid,
             'contextid' => \context_system::instance()->id,
-            'filepath' => '/analysable/' . $this->analysableid . '/' . self::clean_time_splitting_id($this->timesplittingid) . '/',
+            'filepath' => '/analysable/' . $this->analysableid . '/' .
+                \core_analytics\analysis::clean_time_splitting_id($this->timesplittingid) . '/',
             'filename' => self::get_filename($this->evaluation)
         ];
 
@@ -167,8 +145,7 @@ class dataset_manager {
         $filepath = make_request_directory() . DIRECTORY_SEPARATOR . $filerecord['filename'];
         $fh = fopen($filepath, 'w+');
         if (!$fh) {
-            $this->close_process();
-            throw new \moodle_exception('errorcannotwritedataset', 'analytics', '', $filepath);
+            return false;
         }
         foreach ($data as $line) {
             fputcsv($fh, $line);
@@ -178,15 +155,6 @@ class dataset_manager {
         return $fs->create_file_from_pathname($filerecord, $filepath);
     }
 
-    /**
-     * Mark as analysed.
-     *
-     * @return void
-     */
-    public function close_process() {
-        $this->lock->release();
-    }
-
     /**
      * Returns the previous evaluation file.
      *
@@ -200,8 +168,9 @@ class dataset_manager {
     public static function get_previous_evaluation_file($modelid, $timesplittingid) {
         $fs = get_file_storage();
         // Evaluation data is always labelled.
+        $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';
         return $fs->get_file(\context_system::instance()->id, 'analytics', self::LABELLED_FILEAREA, $modelid,
-            '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/', self::EVALUATION_FILENAME);
+            $filepath, self::EVALUATION_FILENAME);
     }
 
     /**
@@ -233,7 +202,7 @@ class dataset_manager {
         $filesbytimesplitting = array();
         foreach ($timesplittingids as $timesplittingid) {
 
-            $filepath = '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/';
+            $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';
             $files = $fs->get_directory_files(\context_system::instance()->id, 'analytics', $filearea, $modelid, $filepath);
             foreach ($files as $file) {
 
@@ -291,7 +260,8 @@ class dataset_manager {
         // Always evaluation.csv and labelled as it is an evaluation file.
         $filearea = self::LABELLED_FILEAREA;
         $filename = self::get_filename(true);
-        $filepath = '/analysable/' . $analysableid . '/' . self::clean_time_splitting_id($timesplittingid) . '/';
+        $filepath = '/analysable/' . $analysableid . '/' .
+            \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';
         return $fs->get_file(\context_system::instance()->id, 'analytics', $filearea, $modelid, $filepath, $filename);
     }
 
@@ -372,7 +342,7 @@ class dataset_manager {
             'filearea' => $filearea,
             'itemid' => $modelid,
             'contextid' => \context_system::instance()->id,
-            'filepath' => '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/',
+            'filepath' => '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/',
             'filename' => self::get_filename($evaluation)
         ];
 
@@ -393,7 +363,7 @@ class dataset_manager {
         $fs = get_file_storage();
 
         $contextid = \context_system::instance()->id;
-        $filepath = '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/';
+        $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';
 
         $files = $fs->get_directory_files($contextid, 'analytics', self::LABELLED_FILEAREA, $modelid,
             $filepath, true, false);
@@ -467,17 +437,6 @@ class dataset_manager {
         return $fs->delete_area_files(\context_system::instance()->id, 'analytics', false, $modelid);
     }
 
-    /**
-     * Remove all possibly problematic chars from the time splitting method id (id = its full class name).
-     *
-     * @param string $timesplittingid
-     * @return string
-     */
-    protected static function clean_time_splitting_id($timesplittingid) {
-        $timesplittingid = str_replace('\\', '-', $timesplittingid);
-        return clean_param($timesplittingid, PARAM_ALPHANUMEXT);
-    }
-
     /**
      * Returns the file name to be used.
      *
diff --git a/analytics/classes/insights_generator.php b/analytics/classes/insights_generator.php
new file mode 100644 (file)
index 0000000..24fdfaf
--- /dev/null
@@ -0,0 +1,236 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Insights generator.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics;
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once($CFG->dirroot . '/lib/messagelib.php');
+
+/**
+ * Insights generator.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class insights_generator {
+
+    /**
+     * @var int
+     */
+    private $modelid;
+
+    /**
+     * @var \core_analytics\local\target\base
+     */
+    private $target;
+
+    /**
+     * @var int[]
+     */
+    private $contextcourseids;
+
+    /**
+     * Constructor.
+     *
+     * @param int $modelid
+     * @param \core_analytics\local\target\base $target
+     */
+    public function __construct(int $modelid, \core_analytics\local\target\base $target) {
+        $this->modelid = $modelid;
+        $this->target = $target;
+    }
+
+    /**
+     * Generates insight notifications.
+     *
+     * @param array                         $samplecontexts    The contexts these predictions belong to
+     * @param \core_analytics\prediction[]  $predictions       The prediction records
+     * @return  null
+     */
+    public function generate($samplecontexts, $predictions) {
+        global $OUTPUT;
+
+        $analyserclass = $this->target->get_analyser_class();
+
+        // We will need to restore it later.
+        $actuallanguage = current_language();
+
+        if ($analyserclass::one_sample_per_analysable()) {
+
+            // Iterate through the predictions and the users in each prediction (likely to be just one).
+            foreach ($predictions as $prediction) {
+
+                $context = $samplecontexts[$prediction->get_prediction_data()->contextid];
+
+                $users = $this->target->get_insights_users($context);
+                foreach ($users as $user) {
+
+                    $this->set_notification_language($user);
+                    list($insighturl, $fullmessage, $fullmessagehtml) = $this->prediction_info($prediction);
+                    $this->notification($context, $user, $insighturl, $fullmessage, $fullmessagehtml);
+                }
+            }
+
+        } else {
+
+            // Iterate through the context and the users in each context.
+            foreach ($samplecontexts as $context) {
+
+                $users = $this->target->get_insights_users($context);
+                foreach ($users as $user) {
+
+                    $this->set_notification_language($user);
+
+                    $insighturl = $this->target->get_insight_context_url($this->modelid, $context);
+
+                    $fullmessage = get_string('insightinfomessage', 'analytics', $insighturl->out(false));
+                    $fullmessagehtml = $OUTPUT->render_from_template('core_analytics/insight_info_message',
+                        ['url' => $insighturl->out(false)]
+                    );
+
+                    $this->notification($context, $user, $insighturl, $fullmessage, $fullmessagehtml);
+                }
+            }
+        }
+
+        force_current_language($actuallanguage);
+    }
+
+    /**
+     * Generates a insight notification for the user.
+     *
+     * @param  \context    $context
+     * @param  \stdClass   $user
+     * @param  \moodle_url $insighturl    The insight URL
+     * @param  string      $fullmessage
+     * @param  string      $fullmessagehtml
+     * @return null
+     */
+    private function notification(\context $context, \stdClass $user, \moodle_url $insighturl, string $fullmessage, string $fullmessagehtml) {
+
+        $message = new \core\message\message();
+        $message->component = 'moodle';
+        $message->name = 'insights';
+
+        $message->userfrom = \core_user::get_noreply_user();
+        $message->userto = $user;
+
+        $message->subject = $this->target->get_insight_subject($this->modelid, $context);
+
+        // Same than the subject.
+        $message->contexturlname = $message->subject;
+        $message->courseid = $this->get_context_courseid($context);
+
+        $message->fullmessage = $fullmessage;
+        $message->fullmessageformat = FORMAT_PLAIN;
+        $message->fullmessagehtml = $fullmessagehtml;
+        $message->smallmessage = $fullmessage;
+        $message->contexturl = $insighturl->out(false);
+
+        message_send($message);
+    }
+
+    /**
+     * Returns the course context of the provided context reading an internal cache first.
+     *
+     * @param  \context $context
+     * @return int
+     */
+    private function get_context_courseid(\context $context) {
+
+        if (empty($this->contextcourseids[$context->id])) {
+
+            $coursecontext = $context->get_course_context(false);
+            if (!$coursecontext) {
+                // Default to the frontpage course context.
+                $coursecontext = \context_course::instance(SITEID);
+            }
+            $this->contextcourseids[$context->id] = $coursecontext->instanceid;
+        }
+
+        return $this->contextcourseids[$context->id];
+    }
+
+    /**
+     * Extracts info from the prediction for display purposes.
+     *
+     * @param  \core_analytics\prediction $prediction
+     * @return array Three items array with formats [\moodle_url, string, string]
+     */
+    private function prediction_info(\core_analytics\prediction $prediction) {
+        global $OUTPUT;
+
+        $predictionactions = $this->target->prediction_actions($prediction, true, true);
+
+        // For FORMAT_PLAIN.
+        $fullmessageplaintext  = '';
+
+        // For FORMAT_HTML.
+        $messageactions  = [];
+        $insighturl = null;
+        foreach ($predictionactions as $action) {
+            $actionurl = $action->get_url();
+            if (!$actionurl->get_param('forwardurl')) {
+
+                $actiondoneurl = new \moodle_url('/report/insights/done.php');
+                // Set the forward url to the 'done' script.
+                $actionurl->param('forwardurl', $actiondoneurl->out(false));
+            }
+
+            if (empty($insighturl)) {
+                // We use the primary action url as insight url so we log that the user followed the provided link.
+                $insighturl = $action->get_url();
+            }
+            $actiondata = (object)['url' => $action->get_url()->out(false), 'text' => $action->get_text()];
+            $fullmessageplaintext .= get_string('insightinfomessageaction', 'analytics', $actiondata) . PHP_EOL;
+            $messageactions[] = $actiondata;
+        }
+
+        $fullmessagehtml = $OUTPUT->render_from_template('core_analytics/insight_info_message_prediction',
+            ['actions' => $messageactions]);
+        return [$insighturl, $fullmessageplaintext, $fullmessagehtml];
+    }
+
+    /**
+     * Sets the session language to the language used by the notification receiver.
+     *
+     * @param  \stdClass $user The user who will receive the message
+     * @return null
+     */
+    private function set_notification_language($user) {
+        global $CFG;
+
+        // Copied from current_language().
+        if (!empty($user->lang)) {
+            $lang = $user->lang;
+        } else if (isset($CFG->lang)) {
+            $lang = $CFG->lang;
+        } else {
+            $lang = 'en';
+        }
+        force_current_language($lang);
+    }
+}
index 06b7468..1c65e2c 100644 (file)
@@ -47,13 +47,6 @@ abstract class base {
      */
     protected $target;
 
-    /**
-     * A $this->$target copy loaded with the ongoing analysis analysable.
-     *
-     * @var \core_analytics\local\target\base
-     */
-    protected $analysabletarget;
-
     /**
      * The model indicators.
      *
@@ -118,9 +111,37 @@ abstract class base {
      * \core_analytics\local\analyser\by_course and \core_analytics\local\analyser\sitewide are implementing
      * this method returning site courses (by_course) and the whole system (sitewide) as analysables.
      *
+     * @todo MDL-65284 This will be removed in Moodle 4.1
+     * @deprecated
+     * @see get_analysables_iterator
+     * @throws  \coding_exception
      * @return \core_analytics\analysable[] Array of analysable elements using the analysable id as array key.
      */
-    abstract public function get_analysables();
+    public function get_analysables() {
+        // This function should only be called from get_analysables_iterator and we keep it here until Moodle 4.1
+        // for backwards compatibility.
+        throw new \coding_exception('This method is deprecated in favour of get_analysables_iterator.');
+    }
+
+    /**
+     * Returns the list of analysable elements available on the site.
+     *
+     * A relatively complex SQL query should be set so that we take into account which analysable elements
+     * have already been processed and the order in which they have been processed. Helper methods are available
+     * to ease to implementation of get_analysables_iterator: get_iterator_sql and order_sql.
+     *
+     * @param string|null $action 'prediction', 'training' or null if no specific action needed.
+     * @return \Iterator
+     */
+    public function get_analysables_iterator(?string $action = null) {
+
+        debugging('Please overwrite get_analysables_iterator with your own implementation, we only keep this default
+            implementation for backwards compatibility purposes with get_analysables(). note that $action param will
+            be ignored so the analysable elements will be processed using get_analysables order, regardless of the
+            last time they were processed.');
+
+        return new \ArrayIterator($this->get_analysables());
+    }
 
     /**
      * This function returns this analysable list of samples.
@@ -128,7 +149,7 @@ abstract class base {
      * @param \core_analytics\analysable $analysable
      * @return array array[0] = int[] (sampleids) and array[1] = array (samplesdata)
      */
-    abstract protected function get_all_samples(\core_analytics\analysable $analysable);
+    abstract public function get_all_samples(\core_analytics\analysable $analysable);
 
     /**
      * This function returns the samples data from a list of sample ids.
@@ -174,70 +195,46 @@ abstract class base {
     abstract public function sample_description($sampleid, $contextid, $sampledata);
 
     /**
-     * Main analyser method which processes the site analysables.
-     *
-     * @param bool $includetarget
-     * @return \stored_file[]
+     * Model id getter.
+     * @return int
      */
-    public function get_analysable_data($includetarget) {
-        global $DB;
-
-        // Time limit control.
-        $modeltimelimit = intval(get_config('analytics', 'modeltimelimit'));
-
-        $filesbytimesplitting = array();
-
-        list($analysables, $processedanalysables) = $this->get_sorted_analysables($includetarget);
-
-        $inittime = time();
-        foreach ($analysables as $key => $analysable) {
-
-            $files = $this->process_analysable($analysable, $includetarget);
-
-            // Later we will need to aggregate data by time splitting method.
-            foreach ($files as $timesplittingid => $file) {
-                $filesbytimesplitting[$timesplittingid][] = $file;
-            }
-
-            $this->update_analysable_analysed_time($processedanalysables, $analysable->get_id(), $includetarget);
-
-            // Apply time limit.
-            if (!$this->options['evaluation']) {
-                $timespent = time() - $inittime;
-                if ($modeltimelimit <= $timespent) {
-                    break;
-                }
-            }
-
-            unset($analysables[$key]);
-        }
-
-        if ($this->options['evaluation'] === false) {
-            // Look for previous training and prediction files we generated and couldn't be used
-            // by machine learning backends because they weren't big enough.
+    public function get_modelid(): int {
+        return $this->modelid;
+    }
 
-            $pendingfiles = \core_analytics\dataset_manager::get_pending_files($this->modelid, $includetarget,
-                array_keys($filesbytimesplitting));
-            foreach ($pendingfiles as $timesplittingid => $files) {
-                foreach ($files as $file) {
-                    $filesbytimesplitting[$timesplittingid][] = $file;
-                }
-            }
-        }
+    /**
+     * Options getter.
+     * @return array
+     */
+    public function get_options(): array {
+        return $this->options;
+    }
 
-        // We join the datasets by time splitting method.
-        $timesplittingfiles = $this->merge_analysable_files($filesbytimesplitting, $includetarget);
+    /**
+     * Returns the analysed target.
+     *
+     * @return \core_analytics\local\target\base
+     */
+    public function get_target(): \core_analytics\local\target\base {
+        return $this->target;
+    }
 
-        if (!empty($pendingfiles)) {
-            // We must remove them now as they are already part of another dataset.
-            foreach ($pendingfiles as $timesplittingid => $files) {
-                foreach ($files as $file) {
-                    $file->delete();
-                }
-            }
-        }
+    /**
+     * Getter for time splittings.
+     *
+     * @return \core_analytics\local\time_splitting\base
+     */
+    public function get_timesplittings(): array {
+        return $this->timesplittings;
+    }
 
-        return $timesplittingfiles;
+    /**
+     * Getter for indicators.
+     *
+     * @return \core_analytics\local\indicator\base
+     */
+    public function get_indicators(): array {
+        return $this->indicators;
     }
 
     /**
@@ -252,19 +249,39 @@ abstract class base {
     /**
      * Returns labelled data (training and evaluation).
      *
-     * @return array
+     * @return \stored_file[]
      */
     public function get_labelled_data() {
-        return $this->get_analysable_data(true);
+        // Delegates all processing to the analysis.
+        $result = new \core_analytics\local\analysis\result_file($this->get_modelid(), true, $this->get_options());
+        $analysis = new \core_analytics\analysis($this, true, $result);
+        $analysis->run();
+        return $result->get();
     }
 
     /**
      * Returns unlabelled data (prediction).
      *
-     * @return array
+     * @return \stored_file[]
      */
     public function get_unlabelled_data() {
-        return $this->get_analysable_data(false);
+        // Delegates all processing to the analysis.
+        $result = new \core_analytics\local\analysis\result_file($this->get_modelid(), false, $this->get_options());
+        $analysis = new \core_analytics\analysis($this, false, $result);
+        $analysis->run();
+        return $result->get();
+    }
+
+    /**
+     * Returns indicator calculations as an array.
+     * @return array
+     */
+    public function get_static_data() {
+        // Delegates all processing to the analysis.
+        $result = new \core_analytics\local\analysis\result_array($this->get_modelid(), false, $this->get_options());
+        $analysis = new \core_analytics\analysis($this, false, $result);
+        $analysis->run();
+        return $result->get();
     }
 
     /**
@@ -284,36 +301,6 @@ abstract class base {
         }
     }
 
-    /**
-     * Merges analysable dataset files into 1.
-     *
-     * @param array $filesbytimesplitting
-     * @param bool $includetarget
-     * @return \stored_file[]
-     */
-    protected function merge_analysable_files($filesbytimesplitting, $includetarget) {
-
-        $timesplittingfiles = array();
-        foreach ($filesbytimesplitting as $timesplittingid => $files) {
-
-            if ($this->options['evaluation'] === true) {
-                // Delete the previous copy. Only when evaluating.
-                \core_analytics\dataset_manager::delete_previous_evaluation_file($this->modelid, $timesplittingid);
-            }
-
-            // Merge all course files into one.
-            if ($includetarget) {
-                $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA;
-            } else {
-                $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA;
-            }
-            $timesplittingfiles[$timesplittingid] = \core_analytics\dataset_manager::merge_datasets($files,
-                $this->modelid, $timesplittingid, $filearea, $this->options['evaluation']);
-        }
-
-        return $timesplittingfiles;
-    }
-
     /**
      * Checks that this analyser satisfies the provided indicator requirements.
      *
@@ -338,83 +325,6 @@ abstract class base {
         return $missingrequired;
     }
 
-    /**
-     * Processes an analysable
-     *
-     * This method returns the general analysable status, an array of files by time splitting method and
-     * an error message if there is any problem.
-     *
-     * @param \core_analytics\analysable $analysable
-     * @param bool $includetarget
-     * @return \stored_file[] Files by time splitting method
-     */
-    public function process_analysable($analysable, $includetarget) {
-
-        // Default returns.
-        $files = array();
-        $message = null;
-
-        // Target instances scope is per-analysable (it can't be lower as calculations run once per
-        // analysable, not time splitting method nor time range).
-        $this->analysabletarget = call_user_func(array($this->target, 'instance'));
-
-        // We need to check that the analysable is valid for the target even if we don't include targets
-        // as we still need to discard invalid analysables for the target.
-        $result = $this->analysabletarget->is_valid_analysable($analysable, $includetarget);
-        if ($result !== true) {
-            $a = new \stdClass();
-            $a->analysableid = $analysable->get_name();
-            $a->result = $result;
-            $this->add_log(get_string('analysablenotvalidfortarget', 'analytics', $a));
-            return array();
-        }
-
-        // Process all provided time splitting methods.
-        $results = array();
-        foreach ($this->timesplittings as $timesplitting) {
-
-            // For evaluation purposes we don't need to be that strict about how updated the data is,
-            // if this analyser was analysed less that 1 week ago we skip generating a new one. This
-            // helps scale the evaluation process as sites with tons of courses may a lot of time to
-            // complete an evaluation.
-            if (!empty($this->options['evaluation']) && !empty($this->options['reuseprevanalysed'])) {
-
-                $previousanalysis = \core_analytics\dataset_manager::get_evaluation_analysable_file($this->modelid,
-                    $analysable->get_id(), $timesplitting->get_id());
-                // 1 week is a partly random time interval, no need to worry about DST.
-                $boundary = time() - WEEKSECS;
-                if ($previousanalysis && $previousanalysis->get_timecreated() > $boundary) {
-                    // Recover the previous analysed file and avoid generating a new one.
-
-                    // Don't bother filling a result object as it is only useful when there are no files generated.
-                    $files[$timesplitting->get_id()] = $previousanalysis;
-                    continue;
-                }
-            }
-
-            $result = $this->process_time_splitting($timesplitting, $analysable, $includetarget);
-
-            if (!empty($result->file)) {
-                $files[$timesplitting->get_id()] = $result->file;
-            }
-            $results[] = $result;
-        }
-
-        if (empty($files)) {
-            $errors = array();
-            foreach ($results as $timesplittingid => $result) {
-                $errors[] = $timesplittingid . ': ' . $result->message;
-            }
-
-            $a = new \stdClass();
-            $a->analysableid = $analysable->get_name();
-            $a->errors = implode(', ', $errors);
-            $this->add_log(get_string('analysablenotused', 'analytics', $a));
-        }
-
-        return $files;
-    }
-
     /**
      * Adds a register to the analysis log.
      *
@@ -482,427 +392,83 @@ abstract class base {
     }
 
     /**
-     * Processes the analysable samples using the provided time splitting method.
+     * Do this analyser's analysables have 1 single sample each?
      *
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     * @param \core_analytics\analysable $analysable
-     * @param bool $includetarget
-     * @return \stdClass Results object.
-     */
-    protected function process_time_splitting($timesplitting, $analysable, $includetarget = false) {
-
-        $result = new \stdClass();
-
-        if (!$timesplitting->is_valid_analysable($analysable)) {
-            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-            $result->message = get_string('invalidanalysablefortimesplitting', 'analytics',
-                $timesplitting->get_name());
-            return $result;
-        }
-        $timesplitting->set_analysable($analysable);
-
-        if (CLI_SCRIPT && !PHPUNIT_TEST) {
-            mtrace('Analysing id "' . $analysable->get_id() . '" with "' . $timesplitting->get_name() .
-                '" time splitting method...');
-        }
-
-        // What is a sample is defined by the analyser, it can be an enrolment, a course, a user, a question
-        // attempt... it is on what we will base indicators calculations.
-        list($sampleids, $samplesdata) = $this->get_all_samples($analysable);
-
-        if (count($sampleids) === 0) {
-            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-            $result->message = get_string('nodata', 'analytics');
-            return $result;
-        }
-
-        if ($includetarget) {
-            // All ranges are used when we are calculating data for training.
-            $ranges = $timesplitting->get_all_ranges();
-        } else {
-            // The latest range that has not yet been used for prediction (it depends on the time range where we are right now).
-            $ranges = $this->get_most_recent_prediction_range($timesplitting);
-        }
-
-        // There is no need to keep track of the evaluated samples and ranges as we always evaluate the whole dataset.
-        if ($this->options['evaluation'] === false) {
-
-            if (empty($ranges)) {
-                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-                $result->message = get_string('noranges', 'analytics');
-                return $result;
-            }
-
-            // We skip all samples that are already part of a training dataset, even if they have not been used for prediction.
-            $this->filter_out_train_samples($sampleids, $timesplitting);
-
-            if (count($sampleids) === 0) {
-                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-                $result->message = get_string('nonewdata', 'analytics');
-                return $result;
-            }
-
-            // Only when processing data for predictions.
-            if (!$includetarget) {
-                // We also filter out samples and ranges that have already been used for predictions.
-                $this->filter_out_prediction_samples_and_ranges($sampleids, $ranges, $timesplitting);
-            }
-
-            if (count($sampleids) === 0) {
-                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-                $result->message = get_string('nonewdata', 'analytics');
-                return $result;
-            }
-
-            if (count($ranges) === 0) {
-                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-                $result->message = get_string('nonewranges', 'analytics');
-                return $result;
-            }
-        }
-
-        if (!empty($includetarget)) {
-            $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA;
-        } else {
-            $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA;
-        }
-        $dataset = new \core_analytics\dataset_manager($this->modelid, $analysable->get_id(), $timesplitting->get_id(),
-            $filearea, $this->options['evaluation']);
-
-        // Flag the model + analysable + timesplitting as being analysed (prevent concurrent executions).
-        if (!$dataset->init_process()) {
-            // If this model + analysable + timesplitting combination is being analysed we skip this process.
-            $result->status = \core_analytics\model::NO_DATASET;
-            $result->message = get_string('analysisinprogress', 'analytics');
-            return $result;
-        }
-
-        try {
-            // Remove samples the target consider invalid.
-            $this->analysabletarget->add_sample_data($samplesdata);
-            $this->analysabletarget->filter_out_invalid_samples($sampleids, $analysable, $includetarget);
-        } catch (\Throwable $e) {
-            $dataset->close_process();
-            throw $e;
-        }
-
-        if (!$sampleids) {
-            $result->status = \core_analytics\model::NO_DATASET;
-            $result->message = get_string('novalidsamples', 'analytics');
-            $dataset->close_process();
-            return $result;
-        }
-
-        try {
-            foreach ($this->indicators as $key => $indicator) {
-                // The analyser attaches the main entities the sample depends on and are provided to the
-                // indicator to calculate the sample.
-                $this->indicators[$key]->add_sample_data($samplesdata);
-            }
-
-            // Here we start the memory intensive process that will last until $data var is
-            // unset (until the method is finished basically).
-            if ($includetarget) {
-                $data = $timesplitting->calculate($sampleids, $this->get_samples_origin(), $this->indicators, $ranges,
-                    $this->analysabletarget);
-            } else {
-                $data = $timesplitting->calculate($sampleids, $this->get_samples_origin(), $this->indicators, $ranges);
-            }
-        } catch (\Throwable $e) {
-            $dataset->close_process();
-            throw $e;
-        }
-
-        if (!$data) {
-            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-            $result->message = get_string('novaliddata', 'analytics');
-            $dataset->close_process();
-            return $result;
-        }
-
-        try {
-            // Add extra metadata.
-            $this->add_model_metadata($data);
-
-            // Write all calculated data to a file.
-            $file = $dataset->store($data);
-        } catch (\Throwable $e) {
-            $dataset->close_process();
-            throw $e;
-        }
-
-        // Flag the model + analysable + timesplitting as analysed.
-        $dataset->close_process();
-
-        // No need to keep track of analysed stuff when evaluating.
-        if ($this->options['evaluation'] === false) {
-            // Save the samples that have been already analysed so they are not analysed again in future.
-
-            if ($includetarget) {
-                $this->save_train_samples($sampleids, $timesplitting, $file);
-            } else {
-                $this->save_prediction_samples($sampleids, $ranges, $timesplitting);
-            }
-        }
-
-        $result->status = \core_analytics\model::OK;
-        $result->message = get_string('successfullyanalysed', 'analytics');
-        $result->file = $file;
-        return $result;
-    }
-
-    /**
-     * Returns the most recent range that can be used to predict.
+     * Overwrite and return true if your analysables only have
+     * one sample. The insights generated by models using this
+     * analyser will then include the suggested actions in the
+     * notification.
      *
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     * @return array
+     * @return bool
      */
-    protected function get_most_recent_prediction_range($timesplitting) {
-
-        $now = time();
-        $ranges = $timesplitting->get_all_ranges();
-
-        // Opposite order as we are interested in the last range that can be used for prediction.
-        krsort($ranges);
-
-        // We already provided the analysable to the time splitting method, there is no need to feed it back.
-        foreach ($ranges as $rangeindex => $range) {
-            if ($timesplitting->ready_to_predict($range)) {
-                // We need to maintain the same indexes.
-                return array($rangeindex => $range);
-            }
-        }
-
-        return array();
+    public static function one_sample_per_analysable() {
+        return false;
     }
 
     /**
-     * Filters out samples that have already been used for training.
+     * Get the sql of a default implementation of the iterator.
      *
-     * @param int[] $sampleids
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     */
-    protected function filter_out_train_samples(&$sampleids, $timesplitting) {
-        global $DB;
-
-        $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(),
-            'timesplitting' => $timesplitting->get_id());
-
-        $trainingsamples = $DB->get_records('analytics_train_samples', $params);
-
-        // Skip each file trained samples.
-        foreach ($trainingsamples as $trainingfile) {
-
-            $usedsamples = json_decode($trainingfile->sampleids, true);
-
-            if (!empty($usedsamples)) {
-                // Reset $sampleids to $sampleids minus this file's $usedsamples.
-                $sampleids = array_diff_key($sampleids, $usedsamples);
-            }
-        }
-    }
-
-    /**
-     * Filters out samples that have already been used for prediction.
+     * This method only works for analysers that return analysable elements which ids map to a context instance ids.
      *
-     * @param int[] $sampleids
-     * @param array $ranges
-     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param  string      $tablename    The name of the table
+     * @param  int         $contextlevel The context level of the analysable
+     * @param  string|null $action
+     * @param  string|null $tablealias   The table alias
+     * @return array                     [0] => sql and [1] => params array
      */
-    protected function filter_out_prediction_samples_and_ranges(&$sampleids, &$ranges, $timesplitting) {
-        global $DB;
+    protected function get_iterator_sql(string $tablename, int $contextlevel, ?string $action = null, ?string $tablealias = null) {
 
-        if (count($ranges) > 1) {
-            throw new \coding_exception('$ranges argument should only contain one range');
+        if (!$tablealias) {
+            $tablealias = 'analysable';
         }
 
-        $rangeindex = key($ranges);
+        $params = ['contextlevel' => $contextlevel, 'modelid' => $this->get_modelid()];
+        $select = $tablealias . '.*, ' . \context_helper::get_preload_record_columns_sql('ctx');
 
-        $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(),
-            'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex);
-        $predictedrange = $DB->get_record('analytics_predict_samples', $params);
+        // We add the action filter on ON instead of on WHERE because otherwise records are not returned if there are existing
+        // records for another action or model.
+        $usedanalysablesjoin = ' LEFT JOIN {analytics_used_analysables} aua ON ' . $tablealias . '.id = aua.analysableid AND ' .
+            '(aua.modelid = :modelid OR aua.modelid IS NULL)';
 
-        if (!$predictedrange) {
-            // Nothing to filter out.
-            return;
+        if ($action) {
+            $usedanalysablesjoin .= " AND aua.action = :action";
+            $params = $params + ['action' => $action];
         }
 
-        $predictedrange->sampleids = json_decode($predictedrange->sampleids, true);
-        $missingsamples = array_diff_key($sampleids, $predictedrange->sampleids);
-        if (count($missingsamples) === 0) {
-            // All samples already calculated.
-            unset($ranges[$rangeindex]);
-            return;
-        }
-
-        // Replace the list of samples by the one excluding samples that already got predictions at this range.
-        $sampleids = $missingsamples;
-    }
-
-    /**
-     * Saves samples that have just been used for training.
-     *
-     * @param int[] $sampleids
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     * @param \stored_file $file
-     * @return void
-     */
-    protected function save_train_samples($sampleids, $timesplitting, $file) {
-        global $DB;
-
-        $trainingsamples = new \stdClass();
-        $trainingsamples->modelid = $this->modelid;
-        $trainingsamples->analysableid = $timesplitting->get_analysable()->get_id();
-        $trainingsamples->timesplitting = $timesplitting->get_id();
-        $trainingsamples->fileid = $file->get_id();
-
-        $trainingsamples->sampleids = json_encode($sampleids);
-        $trainingsamples->timecreated = time();
+        // Adding the 1 = 1 just to have the WHERE part so that all further conditions added by callers can be
+        // appended to $sql with and ' AND'.
+        $sql = 'SELECT ' . $select . '
+                  FROM {' . $tablename . '} ' . $tablealias . '
+                  ' . $usedanalysablesjoin . '
+                  JOIN {context} ctx ON (ctx.contextlevel = :contextlevel AND ctx.instanceid = ' . $tablealias . '.id)
+                  WHERE 1 = 1';
 
-        $DB->insert_record('analytics_train_samples', $trainingsamples);
+        return [$sql, $params];
     }
 
     /**
-     * Saves samples that have just been used for prediction.
+     * Returns the order by clause.
      *
-     * @param int[] $sampleids
-     * @param array $ranges
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     * @return void
+     * @param  string|null $fieldname  The field name
+     * @param  string      $order      'ASC' or 'DESC'
+     * @param  string|null $tablealias The table alias of the field
+     * @return string
      */
-    protected function save_prediction_samples($sampleids, $ranges, $timesplitting) {
-        global $DB;
+    protected function order_sql(?string $fieldname = null, string $order = 'ASC', ?string $tablealias = null) {
 
-        if (count($ranges) > 1) {
-            throw new \coding_exception('$ranges argument should only contain one range');
+        if (!$tablealias) {
+            $tablealias = 'analysable';
         }
 
-        $rangeindex = key($ranges);
-
-        $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(),
-            'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex);
-        if ($predictionrange = $DB->get_record('analytics_predict_samples', $params)) {
-            // Append the new samples used for prediction.
-            $prevsamples = json_decode($predictionrange->sampleids, true);
-            $predictionrange->sampleids = json_encode($prevsamples + $sampleids);
-            $predictionrange->timemodified = time();
-            $DB->update_record('analytics_predict_samples', $predictionrange);
-        } else {
-            $predictionrange = (object)$params;
-            $predictionrange->sampleids = json_encode($sampleids);
-            $predictionrange->timecreated = time();
-            $predictionrange->timemodified = $predictionrange->timecreated;
-            $DB->insert_record('analytics_predict_samples', $predictionrange);
+        if ($order != 'ASC' && $order != 'DESC') {
+            throw new \coding_exception('The order can only be ASC or DESC');
         }
-    }
 
-    /**
-     * Adds target metadata to the dataset.
-     *
-     * @param array $data
-     * @return void
-     */
-    protected function add_model_metadata(&$data) {
-        global $CFG;
-
-        $metadata = array(
-            'moodleversion' => $CFG->version,
-            'targetcolumn' => $this->analysabletarget->get_id()
-        );
-        if ($this->analysabletarget->is_linear()) {
-            $metadata['targettype'] = 'linear';
-            $metadata['targetmin'] = $this->analysabletarget::get_min_value();
-            $metadata['targetmax'] = $this->analysabletarget::get_max_value();
-        } else {
-            $metadata['targettype'] = 'discrete';
-            $metadata['targetclasses'] = json_encode($this->analysabletarget::get_classes());
-        }
-
-        foreach ($metadata as $varname => $value) {
-            $data[0][] = $varname;
-            $data[1][] = $value;
+        $ordersql = ' ORDER BY (CASE WHEN aua.timeanalysed IS NULL THEN 0 ELSE aua.timeanalysed END) ASC';
+        if ($fieldname) {
+            $ordersql .= ', ' . $tablealias . '.' . $fieldname .' ' . $order;
         }
-    }
 
-    /**
-     * Returns the list of analysables sorted in processing priority order.
-     *
-     * It will first return analysables that have never been analysed before
-     * and it will continue with the ones we have already seen by timeanalysed DESC
-     * order.
-     *
-     * @param bool $includetarget
-     * @return array(0 => \core_analytics\analysable[], 1 => \stdClass[])
-     */
-    protected function get_sorted_analysables($includetarget) {
-
-        $analysables = $this->get_analysables();
-
-        // Get the list of analysables that have been already processed.
-        $processedanalysables = $this->get_processed_analysables($includetarget);
-
-        // We want to start processing analysables we have not yet processed and later continue
-        // with analysables that we already processed.
-        $unseen = array_diff_key($analysables, $processedanalysables);
-
-        // Var $processed first as we want to respect its timeanalysed DESC order so analysables that
-        // have recently been processed are on the bottom of the stack.
-        $seen = array_intersect_key($processedanalysables, $analysables);
-        array_walk($seen, function(&$value, $analysableid) use ($analysables) {
-            // We replace the analytics_used_analysables record by the analysable object.
-            $value = $analysables[$analysableid];
-        });
-
-        return array($unseen + $seen, $processedanalysables);
-    }
-
-    /**
-     * Get analysables that have been already processed.
-     *
-     * @param bool $includetarget
-     * @return \stdClass[]
-     */
-    protected function get_processed_analysables($includetarget) {
-        global $DB;
-
-        $params = array('modelid' => $this->modelid);
-        $params['action'] = ($includetarget) ? 'training' : 'prediction';
-        $select = 'modelid = :modelid and action = :action';
-
-        // Weird select fields ordering for performance (analysableid key matching, analysableid is also unique by modelid).
-        return $DB->get_records_select('analytics_used_analysables', $select,
-            $params, 'timeanalysed DESC', 'analysableid, modelid, action, timeanalysed, id AS primarykey');
-    }
-
-    /**
-     * Updates the analysable analysis time.
-     *
-     * @param array $processedanalysables
-     * @param int $analysableid
-     * @param bool $includetarget
-     * @return null
-     */
-    protected function update_analysable_analysed_time($processedanalysables, $analysableid, $includetarget) {
-        global $DB;
-
-        if (!empty($processedanalysables[$analysableid])) {
-            $obj = $processedanalysables[$analysableid];
-
-            $obj->id = $obj->primarykey;
-            unset($obj->primarykey);
-
-            $obj->timeanalysed = time();
-            $DB->update_record('analytics_used_analysables', $obj);
-
-        } else {
-
-            $obj = new \stdClass();
-            $obj->modelid = $this->modelid;
-            $obj->action = ($includetarget) ? 'training' : 'prediction';
-            $obj->analysableid = $analysableid;
-            $obj->timeanalysed = time();
-
-            $DB->insert_record('analytics_used_analysables', $obj);
-        }
+        return $ordersql;
     }
 }
index bbda69a..99e70c1 100644 (file)
@@ -38,34 +38,43 @@ abstract class by_course extends base {
     /**
      * Return the list of courses to analyse.
      *
-     * @return \core_analytics\course[]
+     * @param string|null $action 'prediction', 'training' or null if no specific action needed.
+     * @return \Iterator
      */
-    public function get_analysables() {
+    public function get_analysables_iterator(?string $action = null) {
+        global $DB;
 
-        // Default to all system courses.
+        list($sql, $params) = $this->get_iterator_sql('course', CONTEXT_COURSE, $action, 'c');
+
+        // This will be updated to filter by context as part of MDL-64739.
         if (!empty($this->options['filter'])) {
             $courses = array();
             foreach ($this->options['filter'] as $courseid) {
                 $courses[$courseid] = new \stdClass();
                 $courses[$courseid]->id = $courseid;
             }
-        } else {
-            // Iterate through all potentially valid courses.
-            $courses = get_courses('all', 'c.sortorder ASC', 'c.id');
-        }
-        unset($courses[SITEID]);
 
-        $analysables = array();
-        foreach ($courses as $course) {
-            // Skip the frontpage course.
-            $analysable = \core_analytics\course::instance($course->id);
-            $analysables[$analysable->get_id()] = $analysable;
+            list($coursesql, $courseparams) = $DB->get_in_or_equal($courses, SQL_PARAMS_NAMED);
+            $sql .= " AND c.id IN $coursesql";
+            $params = $params + $courseparams;
         }
 
-        if (empty($analysables)) {
-            $this->log[] = get_string('nocourses', 'analytics');
+        $ordersql = $this->order_sql('sortorder', 'ASC', 'c');
+
+        $recordset = $DB->get_recordset_sql($sql . $ordersql, $params);
+
+        if (!$recordset->valid()) {
+            $this->add_log(get_string('nocourses', 'analytics'));
+            return new \ArrayIterator([]);
         }
 
-        return $analysables;
+        return new \core\dml\recordset_walk($recordset, function($record) {
+
+            if ($record->id == SITEID) {
+                return false;
+            }
+            $context = \context_helper::preload_from_record($record);
+            return \core_analytics\course::instance($record, $context);
+        });
     }
-}
+}
\ No newline at end of file
index 075b930..da6ea71 100644 (file)
@@ -36,12 +36,13 @@ defined('MOODLE_INTERNAL') || die();
 abstract class sitewide extends base {
 
     /**
-     * Returns one single analysable element, the site.
+     * Return the list of analysables to analyse.
      *
-     * @return \core_analytics\analysable[]
+     * @param string|null $action 'prediction', 'training' or null if no specific action needed.
+     * @return \Iterator
      */
-    public function get_analysables() {
-        $analysable = new \core_analytics\site();
-        return array(SYSCONTEXTID => $analysable);
+    public function get_analysables_iterator(?string $action = null) {
+        // We can safely ignore $action as we have 1 single analysable element in this analyser.
+        return new \ArrayIterator([new \core_analytics\site()]);
     }
 }
diff --git a/analytics/classes/local/analysis/result.php b/analytics/classes/local/analysis/result.php
new file mode 100644 (file)
index 0000000..d829261
--- /dev/null
@@ -0,0 +1,102 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Keeps track of the analysis results.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics\local\analysis;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Keeps track of the analysis results.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class result {
+
+    /**
+     * @var int
+     */
+    protected $modelid;
+
+    /**
+     * @var bool
+     */
+    protected $includetarget;
+
+    /**
+     * @var array Analysis options
+     */
+    protected $options;
+
+    /**
+     * Stores analysis data at instance level.
+     * @param int   $modelid
+     * @param bool  $includetarget
+     * @param array $options
+     */
+    public function __construct(int $modelid, bool $includetarget, array $options) {
+        $this->modelid = $modelid;
+        $this->includetarget = $includetarget;
+        $this->options = $options;
+    }
+
+    /**
+     * Retrieves cached results during evaluation.
+     *
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed It can be in whatever format the result uses.
+     */
+    public function retrieve_cached_result(\core_analytics\local\time_splitting\base $timesplitting,
+        \core_analytics\analysable $analysable) {
+        return false;
+    }
+
+    /**
+     * Stores the analysis results.
+     *
+     * @param  array $results
+     * @return bool            True if anything was successfully analysed
+     */
+    abstract public function add_analysable_results(array $results): bool;
+
+    /**
+     * Formats the result.
+     *
+     * @param  array                                     $data
+     * @param  \core_analytics\local\target\base         $target
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed It can be in whatever format the result uses
+     */
+    abstract public function format_result(array $data, \core_analytics\local\target\base $target,
+            \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable);
+
+    /**
+     * Returns the results of the analysis.
+     * @return array
+     */
+    abstract public function get(): array;
+}
\ No newline at end of file
diff --git a/analytics/classes/local/analysis/result_array.php b/analytics/classes/local/analysis/result_array.php
new file mode 100644 (file)
index 0000000..0a8e21e
--- /dev/null
@@ -0,0 +1,100 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Keeps track of the analysis results by storing the results in an array.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics\local\analysis;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Keeps track of the analysis results by storing the results in an array.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class result_array extends result {
+
+    /**
+     * Stores the analysis results by time-splitting method.
+     * @var array
+     */
+    private $resultsbytimesplitting = [];
+
+    /**
+     * Stores the analysis results.
+     * @param  array $results
+     * @return bool            True if anything was successfully analysed
+     */
+    public function add_analysable_results(array $results): bool {
+
+        $any = false;
+
+        // Process all provided time splitting methods.
+        foreach ($results as $timesplittingid => $result) {
+            if (!empty($result->result)) {
+                if (empty($this->resultsbytimesplitting[$timesplittingid])) {
+                    $this->resultsbytimesplitting[$timesplittingid] = [];
+                }
+                $this->resultsbytimesplitting[$timesplittingid] += $result->result;
+                $any = true;
+            }
+        }
+        if (empty($any)) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Formats the result.
+     *
+     * @param  array                                     $data
+     * @param  \core_analytics\local\target\base         $target
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed The data as it comes
+     */
+    public function format_result(array $data, \core_analytics\local\target\base $target,
+            \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable) {
+        return $data;
+    }
+
+    /**
+     * Returns the results of the analysis.
+     * @return array
+     */
+    public function get(): array {
+
+        // We join the datasets by time splitting method.
+        $timesplittingresults = array();
+        foreach ($this->resultsbytimesplitting as $timesplittingid => $results) {
+            if (empty($timesplittingresults[$timesplittingid])) {
+                $timesplittingresults[$timesplittingid] = [];
+            }
+            $timesplittingresults[$timesplittingid] += $results;
+        }
+
+        return $timesplittingresults;
+    }
+}
\ No newline at end of file
diff --git a/analytics/classes/local/analysis/result_file.php b/analytics/classes/local/analysis/result_file.php
new file mode 100644 (file)
index 0000000..7a6b61e
--- /dev/null
@@ -0,0 +1,228 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Keeps track of the analysis results by storing the results in files.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics\local\analysis;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Keeps track of the analysis results by storing the results in files.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class result_file extends result {
+
+    /**
+     * Stores the analysis results by time-splitting method.
+     * @var array
+     */
+    private $filesbytimesplitting = [];
+
+    /**
+     * Stores the analysis results.
+     * @param  array $results
+     * @return bool            True if anything was successfully analysed
+     */
+    public function add_analysable_results(array $results): bool {
+
+        $any = false;
+
+        // Process all provided time splitting methods.
+        foreach ($results as $timesplittingid => $result) {
+            if (!empty($result->result)) {
+                $this->filesbytimesplitting[$timesplittingid][] = $result->result;
+                $any = true;
+            }
+        }
+
+        if (empty($any)) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Retrieves cached results during evaluation.
+     *
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed A \stored_file in this case.
+     */
+    public function retrieve_cached_result(\core_analytics\local\time_splitting\base $timesplitting,
+        \core_analytics\analysable $analysable) {
+
+        // For evaluation purposes we don't need to be that strict about how updated the data is,
+        // if this analyser was analysed less that 1 week ago we skip generating a new one. This
+        // helps scale the evaluation process as sites with tons of courses may need a lot of time to
+        // complete an evaluation.
+        if (!empty($options['evaluation']) && !empty($options['reuseprevanalysed'])) {
+
+            $previousanalysis = \core_analytics\dataset_manager::get_evaluation_analysable_file($this->analyser->get_modelid(),
+                $analysable->get_id(), $timesplitting->get_id());
+            // 1 week is a partly random time interval, no need to worry about DST.
+            $boundary = time() - WEEKSECS;
+            if ($previousanalysis && $previousanalysis->get_timecreated() > $boundary) {
+                // Recover the previous analysed file and avoid generating a new one.
+                return $previousanalysis;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Formats the result.
+     *
+     * @param  array                                     $data
+     * @param  \core_analytics\local\target\base         $target
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed A \stored_file in this case
+     */
+    public function format_result(array $data, \core_analytics\local\target\base $target,
+            \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable) {
+
+        if (!empty($this->includetarget)) {
+            $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA;
+        } else {
+            $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA;
+        }
+        $dataset = new \core_analytics\dataset_manager($this->modelid, $analysable->get_id(),
+            $timesplitting->get_id(), $filearea, $this->options['evaluation']);
+
+        // Add extra metadata.
+        $this->add_model_metadata($data, $timesplitting, $target);
+
+        // Write all calculated data to a file.
+        if (!$result = $dataset->store($data)) {
+            return false;
+        }
+
+        return $result;
+    }
+
+    /**
+     * Returns the results of the analysis.
+     * @return array
+     */
+    public function get(): array {
+
+        if ($this->options['evaluation'] === false) {
+            // Look for previous training and prediction files we generated and couldn't be used
+            // by machine learning backends because they weren't big enough.
+
+            $pendingfiles = \core_analytics\dataset_manager::get_pending_files($this->modelid, $this->includetarget,
+                array_keys($this->filesbytimesplitting));
+            foreach ($pendingfiles as $timesplittingid => $files) {
+                foreach ($files as $file) {
+                    $this->filesbytimesplitting[$timesplittingid][] = $file;
+                }
+            }
+        }
+
+        // We join the datasets by time splitting method.
+        $timesplittingfiles = array();
+        foreach ($this->filesbytimesplitting as $timesplittingid => $files) {
+
+            if ($this->options['evaluation'] === true) {
+                // Delete the previous copy. Only when evaluating.
+                \core_analytics\dataset_manager::delete_previous_evaluation_file($this->modelid, $timesplittingid);
+            }
+
+            // Merge all course files into one.
+            if ($this->includetarget) {
+                $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA;
+            } else {
+                $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA;
+            }
+            $timesplittingfiles[$timesplittingid] = \core_analytics\dataset_manager::merge_datasets($files,
+                $this->modelid, $timesplittingid, $filearea, $this->options['evaluation']);
+        }
+
+        if (!empty($pendingfiles)) {
+            // We must remove them now as they are already part of another dataset.
+            foreach ($pendingfiles as $timesplittingid => $files) {
+                foreach ($files as $file) {
+                    $file->delete();
+                }
+            }
+        }
+
+        return $timesplittingfiles;
+    }
+
+    /**
+     * Adds target metadata to the dataset.
+     *
+     * The final dataset document will look like this:
+     * ----------------------------------------------------
+     * metadata1,metadata2,metadata3,.....
+     * value1, value2, value3,.....
+     *
+     * header1,header2,header3,header4,.....
+     * stud1value1,stud1value2,stud1value3,stud1value4,.....
+     * stud2value1,stud2value2,stud2value3,stud2value4,.....
+     * .....
+     * ----------------------------------------------------
+     *
+     * @param array $data
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param \core_analytics\local\target\base         $target
+     * @return null
+     */
+    private function add_model_metadata(array &$data, \core_analytics\local\time_splitting\base $timesplitting,
+            \core_analytics\local\target\base $target) {
+        global $CFG;
+
+        // If no target the first column is the sampleid, if target the last column is the target.
+        // This will need to be updated when we support unsupervised learning models.
+        $metadata = array(
+            'timesplitting' => $timesplitting->get_id(),
+            'nfeatures' => count(current($data)) - 1,
+            'moodleversion' => $CFG->version,
+            'targetcolumn' => $target->get_id()
+        );
+        if ($target->is_linear()) {
+            $metadata['targettype'] = 'linear';
+            $metadata['targetmin'] = $target::get_min_value();
+            $metadata['targetmax'] = $target::get_max_value();
+        } else {
+            $metadata['targettype'] = 'discrete';
+            $metadata['targetclasses'] = json_encode($target::get_classes());
+        }
+
+        // The first 2 samples will be used to store metadata about the dataset.
+        $metadatacolumns = [];
+        $metadatavalues = [];
+        foreach ($metadata as $key => $value) {
+            $metadatacolumns[] = $key;
+            $metadatavalues[] = $value;
+        }
+
+        // This will also reset samples' dataset keys.
+        array_unshift($data, $metadatacolumns, $metadatavalues);
+    }
+}
index 142518e..0ea5e8b 100644 (file)
@@ -104,14 +104,29 @@ abstract class base extends \core_analytics\calculable {
         return false;
     }
 
+    /**
+     * Update the last analysis time on analysable processed or always.
+     *
+     * If you overwrite this method to return false the last analysis time
+     * will only be recorded in DB when the element successfully analysed. You can
+     * safely return false for lightweight targets.
+     *
+     * @return bool
+     */
+    public function always_update_analysis_time(): bool {
+        return true;
+    }
+
     /**
      * Suggested actions for a user.
      *
      * @param \core_analytics\prediction $prediction
      * @param bool $includedetailsaction
+     * @param bool $isinsightuser
      * @return \core_analytics\prediction_action[]
      */
-    public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false) {
+    public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false,
+            $isinsightuser = false) {
         global $PAGE;
 
         $predictionid = $prediction->get_prediction_data()->id;
@@ -122,12 +137,12 @@ abstract class base extends \core_analytics\calculable {
 
         if ($includedetailsaction) {
 
-            $predictionurl = new \moodle_url('/report/insights/prediction.php',
-                array('id' => $predictionid));
+            $predictionurl = new \moodle_url('/report/insights/prediction.php', array('id' => $predictionid));
+            $detailstext = $this->get_view_details_text();
 
             $actions[] = new \core_analytics\prediction_action(\core_analytics\prediction::ACTION_PREDICTION_DETAILS, $prediction,
-                $predictionurl, new \pix_icon('t/preview', get_string('viewprediction', 'analytics')),
-                get_string('viewprediction', 'analytics'));
+                $predictionurl, new \pix_icon('t/preview', $detailstext),
+                $detailstext);
         }
 
         // Flag as fixed / solved.
@@ -151,6 +166,25 @@ abstract class base extends \core_analytics\calculable {
         return $actions;
     }
 
+    /**
+     * Returns the view details link text.
+     * @return string
+     */
+    private function get_view_details_text() {
+        if ($this->based_on_assumptions()) {
+            $analyserclass = $this->get_analyser_class();
+            if ($analyserclass::one_sample_per_analysable()) {
+                $detailstext = get_string('viewinsightdetails', 'analytics');
+            } else {
+                $detailstext = get_string('viewdetails', 'analytics');
+            }
+        } else {
+            $detailstext = get_string('viewprediction', 'analytics');
+        }
+
+        return $detailstext;
+    }
+
     /**
      * Callback to execute once a prediction has been returned from the predictions processor.
      *
@@ -173,61 +207,28 @@ abstract class base extends \core_analytics\calculable {
      *
      * @param int $modelid
      * @param \context[] $samplecontexts
+     * @param  \core_analytics\prediction[] $predictions
      * @return void
      */
-    public function generate_insight_notifications($modelid, $samplecontexts) {
-
-        foreach ($samplecontexts as $context) {
-
-            $insightinfo = new