Merge branch 'MDL-55356-master' of https://github.com/sammarshallou/moodle
authorAndrew Nicols <andrew@nicols.co.uk>
Thu, 12 Oct 2017 03:28:04 +0000 (11:28 +0800)
committerAndrew Nicols <andrew@nicols.co.uk>
Thu, 12 Oct 2017 03:28:04 +0000 (11:28 +0800)
1  2 
lib/db/install.xml
lib/db/upgrade.php
search/classes/manager.php
search/tests/manager_test.php
version.php

          <INDEX NAME="predictionidanduseridandactionname" UNIQUE="false" FIELDS="predictionid, userid, actionname"/>
        </INDEXES>
      </TABLE>
 +    <TABLE NAME="analytics_used_analysables" COMMENT="List of analysables used by each model">
 +      <FIELDS>
 +        <FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
 +        <FIELD NAME="modelid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
 +        <FIELD NAME="action" TYPE="char" LENGTH="50" NOTNULL="true" SEQUENCE="false"/>
 +        <FIELD NAME="analysableid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
 +        <FIELD NAME="timeanalysed" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
 +      </FIELDS>
 +      <KEYS>
 +        <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
 +        <KEY NAME="modelid" TYPE="foreign" FIELDS="modelid" REFTABLE="analytics_models" REFFIELDS="id"/>
 +      </KEYS>
 +      <INDEXES>
 +        <INDEX NAME="modelid-action" UNIQUE="false" FIELDS="modelid, action"/>
 +      </INDEXES>
 +    </TABLE>
+     <TABLE NAME="search_index_requests" COMMENT="Records requests for (re)indexing of specific contexts. Entries will be removed from this table when indexing of that context is complete. (This table is not used for normal time-based indexing of new content.)">
+       <FIELDS>
+         <FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
+         <FIELD NAME="contextid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="Context ID that has been requested for reindexing."/>
+         <FIELD NAME="searcharea" TYPE="char" LENGTH="255" NOTNULL="true" SEQUENCE="false" COMMENT="Set (e.g. 'forum-post') if a specific area is to be reindexed. Blank indicates all areas."/>
+         <FIELD NAME="timerequested" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="Time at which this index update was requested."/>
+         <FIELD NAME="partialarea" TYPE="char" LENGTH="255" NOTNULL="true" SEQUENCE="false" COMMENT="If processing of this context partially completed, set to the area that needs processing next. Blank indicates not processed yet."/>
+         <FIELD NAME="partialtime" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="If processing partially completed, set to the timestamp within the next area where processing should start. 0 indicates not processed yet."/>
+       </FIELDS>
+       <KEYS>
+         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
+         <KEY NAME="contextid" TYPE="foreign" FIELDS="contextid" REFTABLE="context" REFFIELDS="id"/>
+       </KEYS>
+     </TABLE>
    </TABLES>
 -</XMLDB>
 +</XMLDB>
@@@ -2601,104 -2601,31 +2601,129 @@@ function xmldb_main_upgrade($oldversion
          upgrade_main_savepoint(true, 2017092900.00);
      }
  
 -    if ($oldversion < 2017100600.01) {
 +    if ($oldversion < 2017100900.00) {
 +        // Add index on time modified to grade_outcomes_history, grade_categories_history,
 +        // grade_items_history, and scale_history.
 +        $table = new xmldb_table('grade_outcomes_history');
 +        $index = new xmldb_index('timemodified', XMLDB_INDEX_NOTUNIQUE, array('timemodified'));
  
 -        upgrade_main_savepoint(true, 2017100600.01);
 +        if (!$dbman->index_exists($table, $index)) {
 +            $dbman->add_index($table, $index);
 +        }
 +
 +        $table = new xmldb_table('grade_items_history');
 +        $index = new xmldb_index('timemodified', XMLDB_INDEX_NOTUNIQUE, array('timemodified'));
 +
 +        if (!$dbman->index_exists($table, $index)) {
 +            $dbman->add_index($table, $index);
 +        }
 +
 +        $table = new xmldb_table('grade_categories_history');
 +        $index = new xmldb_index('timemodified', XMLDB_INDEX_NOTUNIQUE, array('timemodified'));
 +
 +        if (!$dbman->index_exists($table, $index)) {
 +            $dbman->add_index($table, $index);
 +        }
 +
 +        $table = new xmldb_table('scale_history');
 +        $index = new xmldb_index('timemodified', XMLDB_INDEX_NOTUNIQUE, array('timemodified'));
 +
 +        if (!$dbman->index_exists($table, $index)) {
 +            $dbman->add_index($table, $index);
 +        }
 +
 +        // Main savepoint reached.
 +        upgrade_main_savepoint(true, 2017100900.00);
 +    }
 +
 +    if ($oldversion < 2017101000.00) {
 +
 +        // Define table analytics_used_analysables to be created.
 +        $table = new xmldb_table('analytics_used_analysables');
 +
 +        // Adding fields to table analytics_used_analysables.
 +        $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
 +        $table->add_field('modelid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
 +        $table->add_field('action', XMLDB_TYPE_CHAR, '50', null, XMLDB_NOTNULL, null, null);
 +        $table->add_field('analysableid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
 +        $table->add_field('timeanalysed', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
 +
 +        // Adding keys to table analytics_used_analysables.
 +        $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
 +        $table->add_key('modelid', XMLDB_KEY_FOREIGN, array('modelid'), 'analytics_models', array('id'));
 +
 +        // Adding indexes to table analytics_used_analysables.
 +        $table->add_index('modelid-action', XMLDB_INDEX_NOTUNIQUE, array('modelid', 'action'));
 +
 +        // Conditionally launch create table for analytics_used_analysables.
 +        if (!$dbman->table_exists($table)) {
 +            $dbman->create_table($table);
 +        }
 +
 +        // Main savepoint reached.
 +        upgrade_main_savepoint(true, 2017101000.00);
 +    }
 +
 +    if ($oldversion < 2017101000.01) {
 +        // Define field override to be added to course_modules_completion.
 +        $table = new xmldb_table('course_modules_completion');
 +        $field = new xmldb_field('overrideby', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'viewed');
 +
 +        // Conditionally launch add field override.
 +        if (!$dbman->field_exists($table, $field)) {
 +            $dbman->add_field($table, $field);
 +        }
 +
 +        // Main savepoint reached.
 +        upgrade_main_savepoint(true, 2017101000.01);
 +      }
 +
 +      if ($oldversion < 2017101000.02) {
 +        // Define field 'timestart' to be added to 'analytics_predictions'.
 +        $table = new xmldb_table('analytics_predictions');
 +        $field = new xmldb_field('timestart', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'timecreated');
 +
 +        // Conditionally launch add field 'timestart'.
 +        if (!$dbman->field_exists($table, $field)) {
 +            $dbman->add_field($table, $field);
 +        }
 +
 +        // Define field 'timeend' to be added to 'analytics_predictions'.
 +        $field = new xmldb_field('timeend', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'timestart');
 +
 +        // Conditionally launch add field 'timeend'.
 +        if (!$dbman->field_exists($table, $field)) {
 +            $dbman->add_field($table, $field);
 +        }
 +
 +        // Main savepoint reached.
 +        upgrade_main_savepoint(true, 2017101000.02);
 +    }
 +
++    if ($oldversion < 2017101200.00) {
+         // Define table search_index_requests to be created.
+         $table = new xmldb_table('search_index_requests');
+         // Adding fields to table search_index_requests.
+         $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
+         $table->add_field('contextid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+         $table->add_field('searcharea', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null);
+         $table->add_field('timerequested', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+         $table->add_field('partialarea', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null);
+         $table->add_field('partialtime', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+         // Adding keys to table search_index_requests.
+         $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
+         $table->add_key('contextid', XMLDB_KEY_FOREIGN, array('contextid'), 'context', array('id'));
+         // Conditionally launch create table for search_index_requests.
+         if (!$dbman->table_exists($table)) {
+             $dbman->create_table($table);
+         }
+         // Main savepoint reached.
++        upgrade_main_savepoint(true, 2017101200.00);
+     }
      return true;
  }
Simple merge
@@@ -261,58 -261,106 +261,157 @@@ class search_manager_testcase extends a
          $this->assertFalse(get_config($componentname, $varname . '_partial'));
      }
  
 -    }
 +    /**
 +     * Tests that documents with modified time in the future are NOT indexed (as this would cause
 +     * a problem by preventing it from indexing other documents modified between now and the future
 +     * date).
 +     */
 +    public function test_future_documents() {
 +        $this->resetAfterTest();
 +
 +        // Create a course and a forum.
 +        $generator = $this->getDataGenerator();
 +        $course = $generator->create_course();
 +        $forum = $generator->create_module('forum', ['course' => $course->id]);
 +
 +        // Index everything up to current. Ensure the course is older than current second so it
 +        // definitely doesn't get indexed again next time.
 +        $this->waitForSecond();
 +        $search = testable_core_search::instance();
 +        $search->index(false, 0);
 +        $search->get_engine()->get_and_clear_added_documents();
 +
 +        // Add 2 discussions to the forum, one of which happend just now, but the other is
 +        // incorrectly set to the future.
 +        $now = time();
 +        $userid = get_admin()->id;
 +        $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
 +                'forum' => $forum->id, 'userid' => $userid, 'timemodified' => $now,
 +                'name' => 'Frog']);
 +        $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
 +                'forum' => $forum->id, 'userid' => $userid, 'timemodified' => $now + 100,
 +                'name' => 'Toad']);
 +
 +        // Wait for a second so we're not actually on the same second as the forum post (there's a
 +        // 1 second overlap between indexing; it would get indexed in both checks below otherwise).
 +        $this->waitForSecond();
 +
 +        // Index.
 +        $search->index(false);
 +        $added = $search->get_engine()->get_and_clear_added_documents();
 +        $this->assertCount(1, $added);
 +        $this->assertEquals('Frog', $added[0]->get('title'));
 +
 +        // Check latest time - it should be the same as $now, not the + 100.
 +        $searcharea = $search->get_search_area($this->forumpostareaid);
 +        list($componentname, $varname) = $searcharea->get_config_var_name();
 +        $this->assertEquals($now, get_config($componentname, $varname . '_lastindexrun'));
 +
 +        // Index again - there should be nothing to index this time.
 +        $search->index(false);
 +        $added = $search->get_engine()->get_and_clear_added_documents();
 +        $this->assertCount(0, $added);
 +    }
 +
+     /**
+      * Tests that indexing a specified context works correctly.
+      */
+     public function test_context_indexing() {
+         global $USER;
+         $this->resetAfterTest();
+         $this->setAdminUser();
+         // Create a course and two forums and a page.
+         $generator = $this->getDataGenerator();
+         $course = $generator->create_course();
+         $now = time();
+         $forum1 = $generator->create_module('forum', ['course' => $course->id]);
+         $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
+                 'forum' => $forum1->id, 'userid' => $USER->id, 'timemodified' => $now,
+                 'name' => 'Frog']);
+         $this->waitForSecond();
+         $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
+                 'forum' => $forum1->id, 'userid' => $USER->id, 'timemodified' => $now + 2,
+                 'name' => 'Zombie']);
+         $forum2 = $generator->create_module('forum', ['course' => $course->id]);
+         $this->waitForSecond();
+         $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
+                 'forum' => $forum2->id, 'userid' => $USER->id, 'timemodified' => $now + 1,
+                 'name' => 'Toad']);
+         $generator->create_module('page', ['course' => $course->id]);
+         $generator->create_module('forum', ['course' => $course->id]);
+         // Index forum 1 only.
+         $search = testable_core_search::instance();
+         $buffer = new progress_trace_buffer(new text_progress_trace(), false);
+         $result = $search->index_context(\context_module::instance($forum1->cmid), '', 0, $buffer);
+         $this->assertTrue($result->complete);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         // Confirm that output only processed 1 forum activity and 2 posts.
+         var_dump(strpos($log, "area: Forum - activity information\n  Processed 1 "));
+         $this->assertNotFalse(strpos($log, "area: Forum - activity information\n  Processed 1 "));
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 2 "));
+         // Confirm that some areas for different types of context were skipped.
+         $this->assertNotFalse(strpos($log, "area: Users\n  Skipping"));
+         $this->assertNotFalse(strpos($log, "area: My courses\n  Skipping"));
+         // Confirm that another module area had no results.
+         $this->assertNotFalse(strpos($log, "area: Page\n  No documents"));
+         // Index whole course.
+         $result = $search->index_context(\context_course::instance($course->id), '', 0, $buffer);
+         $this->assertTrue($result->complete);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         // Confirm that output processed 3 forum activities and 3 posts.
+         $this->assertNotFalse(strpos($log, "area: Forum - activity information\n  Processed 3 "));
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 3 "));
+         // The course area was also included this time.
+         $this->assertNotFalse(strpos($log, "area: My courses\n  Processed 1 "));
+         // Confirm that another module area had results too.
+         $this->assertNotFalse(strpos($log, "area: Page\n  Processed 1 "));
+         // Index whole course, but only forum posts.
+         $result = $search->index_context(\context_course::instance($course->id), 'mod_forum-post',
+                 0, $buffer);
+         $this->assertTrue($result->complete);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         // Confirm that output processed 3 posts but not forum activities.
+         $this->assertFalse(strpos($log, "area: Forum - activity information"));
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 3 "));
+         // Set time limit and retry index of whole course, taking 3 tries to complete it.
+         $search->get_engine()->set_add_delay(0.4);
+         $result = $search->index_context(\context_course::instance($course->id), '', 1, $buffer);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         $this->assertFalse($result->complete);
+         $this->assertNotFalse(strpos($log, "area: Forum - activity information\n  Processed 2 "));
+         $result = $search->index_context(\context_course::instance($course->id), '', 1, $buffer,
+                 $result->startfromarea, $result->startfromtime);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         $this->assertNotFalse(strpos($log, "area: Forum - activity information\n  Processed 2 "));
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 2 "));
+         $this->assertFalse($result->complete);
+         $result = $search->index_context(\context_course::instance($course->id), '', 1, $buffer,
+                 $result->startfromarea, $result->startfromtime);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 2 "));
+         $this->assertTrue($result->complete);
      /**
       * Adding this test here as get_areas_user_accesses process is the same, results just depend on the context level.
       *
diff --cc version.php
@@@ -29,7 -29,7 +29,7 @@@
  
  defined('MOODLE_INTERNAL') || die();
  
- $version  = 2017101000.02;              // YYYYMMDD      = weekly release date of this DEV branch.
 -$version  = 2017100600.01;              // YYYYMMDD      = weekly release date of this DEV branch.
++$version  = 2017101200.00;              // YYYYMMDD      = weekly release date of this DEV branch.
                                          //         RR    = release increments - 00 in DEV branches.
                                          //           .XX = incremental changes.