$this->analyser = $analyser;
$this->includetarget = $includetarget;
$this->result = $result;
+
+ // We cache the first time analysables were analysed because time-splitting methods can depend on these info.
+ self::fill_firstanalyses_cache($this->analyser->get_modelid());
}
/**
// Time limit control.
$modeltimelimit = intval(get_config('analytics', 'modeltimelimit'));
- $filesbytimesplitting = array();
-
- $alreadyprocessedanalysables = $this->get_processed_analysables();
-
if ($this->includetarget) {
$action = 'training';
} else {
}
$analysables = $this->analyser->get_analysables_iterator($action);
+ $processedanalysables = $this->get_processed_analysables();
+
$inittime = microtime(true);
foreach ($analysables as $analysable) {
$processed = false;
}
}
- // Updated regardless of how well the analysis went.
- if ($this->analyser->get_target()->always_update_analysis_time() || $processed) {
- $this->update_analysable_analysed_time($alreadyprocessedanalysables, $analysable->get_id());
- }
-
- // Apply time limit.
if (!$options['evaluation']) {
+
+ if (empty($processedanalysables[$analysable->get_id()]) ||
+ $this->analyser->get_target()->always_update_analysis_time() || $processed) {
+ // We store the list of processed analysables even if the target does not always_update_analysis_time(),
+ // what always_update_analysis_time controls is the update of the data.
+ $this->update_analysable_analysed_time($processedanalysables, $analysable->get_id());
+ }
+
+ // Apply time limit.
$timespent = microtime(true) - $inittime;
if ($modeltimelimit <= $timespent) {
break;
// Weird select fields ordering for performance (analysableid key matching, analysableid is also unique by modelid).
return $DB->get_records_select('analytics_used_analysables', $select,
- $params, 'timeanalysed DESC', 'analysableid, modelid, action, timeanalysed, id AS primarykey');
+ $params, 'timeanalysed DESC', 'analysableid, modelid, action, firstanalysis, timeanalysed, id AS primarykey');
}
/**
protected function update_analysable_analysed_time(array $processedanalysables, int $analysableid) {
global $DB;
+ $now = time();
+
if (!empty($processedanalysables[$analysableid])) {
$obj = $processedanalysables[$analysableid];
$obj->id = $obj->primarykey;
unset($obj->primarykey);
- $obj->timeanalysed = time();
+ $obj->timeanalysed = $now;
+
$DB->update_record('analytics_used_analysables', $obj);
} else {
$obj->modelid = $this->analyser->get_modelid();
$obj->action = ($this->includetarget) ? 'training' : 'prediction';
$obj->analysableid = $analysableid;
- $obj->timeanalysed = time();
+ $obj->firstanalysis = $now;
+ $obj->timeanalysed = $now;
+
+ $obj->primarykey = $DB->insert_record('analytics_used_analysables', $obj);
+
+ // Update the cache just in case it is used in the same request.
+ $key = $this->analyser->get_modelid() . '_' . $analysableid;
+ $cache = \cache::make('core', 'modelfirstanalyses');
+ $cache->set($key, $now);
+ }
+ }
+
+ /**
+ * Fills a cache containing the first time each analysable in the provided model was analysed.
+ *
+ * @param int $modelid
+ * @param int|null $analysableid
+ * @return null
+ */
+ public static function fill_firstanalyses_cache(int $modelid, ?int $analysableid = null) {
+ global $DB;
- $DB->insert_record('analytics_used_analysables', $obj);
+ // Using composed keys instead of cache $identifiers because of MDL-65358.
+ $primarykey = $DB->sql_concat($modelid, "'_'", 'analysableid');
+ $sql = "SELECT $primarykey AS id, MIN(firstanalysis) AS firstanalysis
+ FROM {analytics_used_analysables} aua
+ WHERE modelid = :modelid";
+ $params = ['modelid' => $modelid];
+
+ if ($analysableid) {
+ $sql .= " AND analysableid = :analysableid";
+ $params['analysableid'] = $analysableid;
}
+
+ $sql .= " GROUP BY modelid, analysableid ORDER BY analysableid";
+
+ $firstanalyses = $DB->get_records_sql($sql, $params);
+ if ($firstanalyses) {
+ $cache = \cache::make('core', 'modelfirstanalyses');
+
+ $firstanalyses = array_map(function($record) {
+ return $record->firstanalysis;
+ }, $firstanalyses);
+
+ $cache->set_many($firstanalyses);
+ }
+
+ return $firstanalyses;
}
/**
$periodicity = $this->periodicity();
- $now = new \DateTimeImmutable('now', \core_date::get_server_timezone_object());
-
if ($this->analysable->get_end()) {
$end = (new \DateTimeImmutable())->setTimestamp($this->analysable->get_end());
}
- $next = (new \DateTimeImmutable())->setTimestamp($this->analysable->get_start());
+ $next = (new \DateTimeImmutable())->setTimestamp($this->get_first_start());
+
+ $now = new \DateTimeImmutable('now', \core_date::get_server_timezone_object());
$ranges = [];
while ($next < $now &&
'time' => $end
];
}
+
+ /**
+ * Get the start of the first time range.
+ *
+ * @return int A timestamp.
+ */
+ protected function get_first_start() {
+ return $this->analysable->get_start();
+ }
}
public function valid_for_evaluation(): bool {
return false;
}
+
+ /**
+ * Get the start of the first time range.
+ *
+ * Overwriten to start generating predictions about upcoming stuff from time().
+ *
+ * @return int A timestamp.
+ */
+ protected function get_first_start() {
+ global $DB;
+
+ $cache = \cache::make('core', 'modelfirstanalyses');
+
+ $key = $this->modelid . '_' . $this->analysable->get_id();
+ $firstanalysis = $cache->get($key);
+ if (!empty($firstanalysis)) {
+ return $firstanalysis;
+ }
+
+ // This analysable has not yet been analysed, the start is therefore now (-1 so ready_to_predict can be executed).
+ return time() - 1;
+ }
}
--- /dev/null
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Unit tests for the analysis class.
+ *
+ * @package core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Unit tests for the analysis class.
+ *
+ * @package core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class analytics_analysis_testcase extends advanced_testcase {
+
+ /**
+ * Test fill_firstanalyses_cache.
+ * @return null
+ */
+ public function test_fill_firstanalyses_cache() {
+ $this->resetAfterTest();
+
+ $this->insert_used(1, 1, 'training', 123);
+ $this->insert_used(1, 2, 'training', 124);
+ $this->insert_used(1, 1, 'prediction', 125);
+
+ $firstanalyses = \core_analytics\analysis::fill_firstanalyses_cache(1);
+ $this->assertCount(2, $firstanalyses);
+ $this->assertEquals(123, $firstanalyses['1_1']);
+ $this->assertEquals(124, $firstanalyses['1_2']);
+
+ // The cached elements gets refreshed.
+ $this->insert_used(1, 1, 'prediction', 122);
+ $firstanalyses = \core_analytics\analysis::fill_firstanalyses_cache(1, 1);
+ $this->assertCount(1, $firstanalyses);
+ $this->assertEquals(122, $firstanalyses['1_1']);
+ }
+
+ private function insert_used($modelid, $analysableid, $action, $timestamp) {
+ global $DB;
+
+ $obj = new \stdClass();
+ $obj->modelid = $modelid;
+ $obj->action = $action;
+ $obj->analysableid = $analysableid;
+ $obj->firstanalysis = $timestamp;
+ $obj->timeanalysed = $timestamp;
+ $obj->id = $DB->insert_record('analytics_used_analysables', $obj);
+ }
+}
--- /dev/null
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Test time splitting.
+ *
+ * @package core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Test time splitting.
+ *
+ * @package core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class test_timesplitting_upcoming_seconds extends \core_analytics\local\time_splitting\upcoming_periodic {
+
+ /**
+ * Every second.
+ * @return \DateInterval
+ */
+ public function periodicity() {
+ return new \DateInterval('PT1S');
+ }
+
+ /**
+ * Just to comply with the interface.
+ *
+ * @return \lang_string
+ */
+ public static function get_name() : \lang_string {
+ return new \lang_string('error');
+ }
+}
$string['cachedef_htmlpurifier'] = 'HTML Purifier - cleaned content';
$string['cachedef_langmenu'] = 'List of available languages';
$string['cachedef_message_time_last_message_between_users'] = 'Time created for most recent message in a conversation';
+$string['cachedef_modelprocessedanalysables'] = 'Processed analysables in a model';
$string['cachedef_locking'] = 'Locking';
$string['cachedef_message_processors_enabled'] = "Message processors enabled status";
$string['cachedef_contextwithinsights'] = 'Context with insights';
'simpledata' => true,
'ttl' => 1800
),
+
+ // Caches the first time we analysed models' analysables.
+ 'modelfirstanalyses' => array(
+ 'mode' => cache_store::MODE_REQUEST,
+ 'simplekeys' => true,
+ 'simpledata' => true,
+ ),
);
<?xml version="1.0" encoding="UTF-8" ?>
-<XMLDB PATH="lib/db" VERSION="20190403" COMMENT="XMLDB file for core Moodle tables"
+<XMLDB PATH="lib/db" VERSION="20190412" COMMENT="XMLDB file for core Moodle tables"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="../../lib/xmldb/xmldb.xsd"
>
<FIELD NAME="modelid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
<FIELD NAME="action" TYPE="char" LENGTH="50" NOTNULL="true" SEQUENCE="false"/>
<FIELD NAME="analysableid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
+ <FIELD NAME="firstanalysis" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
<FIELD NAME="timeanalysed" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
</FIELDS>
<KEYS>
upgrade_main_savepoint(true, 2019042300.03);
}
+ if ($oldversion < 2019042700.01) {
+
+ // Define field firstanalysis to be added to analytics_used_analysables.
+ $table = new xmldb_table('analytics_used_analysables');
+
+ // Declaring it as null initially (although it is NOT NULL).
+ $field = new xmldb_field('firstanalysis', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'analysableid');
+
+ // Conditionally launch add field firstanalysis.
+ if (!$dbman->field_exists($table, $field)) {
+ $dbman->add_field($table, $field);
+
+ // Set existing values to the current timeanalysed value.
+ $recordset = $DB->get_recordset('analytics_used_analysables');
+ foreach ($recordset as $record) {
+ $record->firstanalysis = $record->timeanalysed;
+ $DB->update_record('analytics_used_analysables', $record);
+ }
+ $recordset->close();
+
+ // Now make the field 'NOT NULL'.
+ $field = new xmldb_field('firstanalysis', XMLDB_TYPE_INTEGER, '10',
+ null, XMLDB_NOTNULL, null, null, 'analysableid');
+ $dbman->change_field_notnull($table, $field);
+ }
+
+ // Main savepoint reached.
+ upgrade_main_savepoint(true, 2019042700.01);
+ }
+
return true;
}
defined('MOODLE_INTERNAL') || die();
require_once(__DIR__ . '/../../analytics/tests/fixtures/test_timesplitting_seconds.php');
+require_once(__DIR__ . '/../../analytics/tests/fixtures/test_timesplitting_upcoming_seconds.php');
require_once(__DIR__ . '/../../analytics/tests/fixtures/test_timesplitting_weekly.php');
require_once(__DIR__ . '/../../lib/enrollib.php');
$range = reset($ranges);
$this->assertEquals(51, key($ranges));
- $upcomingweek = new \core\analytics\time_splitting\upcoming_week();
- $upcomingweek->set_analysable($this->analysable);
- $this->assertCount(1, $upcomingweek->get_distinct_ranges());
-
- $ranges = $upcomingweek->get_all_ranges();
- $this->assertEquals(53, count($ranges));
- $this->assertEquals($this->course->startdate, $ranges[0]['start']);
- $this->assertEquals($this->course->startdate, $ranges[0]['time']);
-
- $this->assertCount(count($ranges), $upcomingweek->get_training_ranges());
-
- $ranges = $upcomingweek->get_most_recent_prediction_range();
- $range = reset($ranges);
- $this->assertEquals(52, key($ranges));
-
// We now use an ongoing course.
$onemonthago = new DateTime('-30 days');
$this->assertLessThan(time(), $range['start']);
$this->assertLessThan(time(), $range['end']);
+ $starttime = time();
+
$upcomingweek = new \core\analytics\time_splitting\upcoming_week();
$upcomingweek->set_analysable($ongoinganalysable);
$this->assertCount(1, $upcomingweek->get_distinct_ranges());
$ranges = $upcomingweek->get_all_ranges();
- $this->assertEquals(5, count($ranges));
- $this->assertCount(4, $upcomingweek->get_training_ranges());
+ $this->assertEquals(1, count($ranges));
+ $range = reset($ranges);
+ $this->assertLessThan(time(), $range['time']);
+ $this->assertLessThan(time(), $range['start']);
+ $this->assertGreaterThan(time(), $range['end']);
+
+ $this->assertCount(0, $upcomingweek->get_training_ranges());
$ranges = $upcomingweek->get_most_recent_prediction_range();
$range = reset($ranges);
- $this->assertEquals(4, key($ranges));
+ $this->assertEquals(0, key($ranges));
$this->assertLessThan(time(), $range['time']);
$this->assertLessThan(time(), $range['start']);
+ // We substract 1 because upcoming_periodic also has that -1 so that predictions
+ // get executed once the first time range is set.
+ $this->assertGreaterThanOrEqual($starttime - 1, $range['time']);
+ $this->assertGreaterThanOrEqual($starttime - 1, $range['start']);
$this->assertGreaterThan(time(), $range['end']);
+ $this->assertNotEmpty($upcomingweek->get_range_by_index(0));
+ $this->assertFalse($upcomingweek->get_range_by_index(1));
+
// We now check how new ranges get added as time passes.
$fewsecsago = new DateTime('-5 seconds');
// We wait for the next range to be added.
usleep(1000000);
+ // We set the analysable again so the time ranges are recalculated.
+ $seconds->set_analysable($analysable);
+
+ $nnewranges = $seconds->get_all_ranges();
+ $nnewtrainingranges = $seconds->get_training_ranges();
+ $newmostrecentrange = $seconds->get_most_recent_prediction_range();
+ $newmostrecentrange = reset($newmostrecentrange);
+ $this->assertGreaterThan($nranges, $nnewranges);
+ $this->assertGreaterThan($ntrainingranges, $nnewtrainingranges);
+ $this->assertGreaterThan($mostrecentrange['time'], $newmostrecentrange['time']);
+
+ $seconds = new test_timesplitting_upcoming_seconds();
+ $seconds->set_analysable($analysable);
+
+ // Store the ranges we just obtained.
+ $nranges = count($seconds->get_all_ranges());
+ $ntrainingranges = count($seconds->get_training_ranges());
+ $mostrecentrange = $seconds->get_most_recent_prediction_range();
+ $mostrecentrange = reset($mostrecentrange);
+
+ // We wait for the next range to be added.
+ usleep(1000000);
+
$seconds->set_analysable($analysable);
$nnewranges = $seconds->get_all_ranges();
$nnewtrainingranges = $seconds->get_training_ranges();
defined('MOODLE_INTERNAL') || die();
-$version = 2019042700.00; // YYYYMMDD = weekly release date of this DEV branch.
+$version = 2019042700.01; // YYYYMMDD = weekly release date of this DEV branch.
// RR = release increments - 00 in DEV branches.
// .XX = incremental changes.
