Merge branch 'MDL-55356-master' of https://github.com/sammarshallou/moodle
authorAndrew Nicols <andrew@nicols.co.uk>
Thu, 12 Oct 2017 03:28:04 +0000 (11:28 +0800)
committerAndrew Nicols <andrew@nicols.co.uk>
Thu, 12 Oct 2017 03:28:04 +0000 (11:28 +0800)
1  2 
lib/db/install.xml
lib/db/upgrade.php
search/classes/manager.php
search/tests/manager_test.php
version.php

diff --combined lib/db/install.xml
@@@ -1,5 -1,5 +1,5 @@@
  <?xml version="1.0" encoding="UTF-8" ?>
 -<XMLDB PATH="lib/db" VERSION="20170921" COMMENT="XMLDB file for core Moodle tables"
 +<XMLDB PATH="lib/db" VERSION="20170929" COMMENT="XMLDB file for core Moodle tables"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:noNamespaceSchemaLocation="../../lib/xmldb/xmldb.xsd"
  >
          <FIELD NAME="userid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="ID of user who has (or hasn't) completed the activity."/>
          <FIELD NAME="completionstate" TYPE="int" LENGTH="1" NOTNULL="true" SEQUENCE="false" COMMENT="Whether or not the user has completed the activity. Available states: 0 = not completed [if there's no row in this table, that also counts as 0] 1 = completed 2 = completed, show passed 3 = completed, show failed"/>
          <FIELD NAME="viewed" TYPE="int" LENGTH="1" NOTNULL="false" SEQUENCE="false" COMMENT="Tracks whether or not this activity has been viewed. NULL = we are not tracking viewed for this activity 0 = not viewed 1 = viewed"/>
 +        <FIELD NAME="overrideby" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="Tracks whether this completion state has been set manually to override a previous state."/>
          <FIELD NAME="timemodified" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="Time at which the completion state last changed."/>
        </FIELDS>
        <KEYS>
        </KEYS>
        <INDEXES>
          <INDEX NAME="action" UNIQUE="false" FIELDS="action" COMMENT="insert/update/delete"/>
 +        <INDEX NAME="timemodified" UNIQUE="false" FIELDS="timemodified"/>
        </INDEXES>
      </TABLE>
      <TABLE NAME="stats_daily" COMMENT="to accumulate daily stats">
        </KEYS>
        <INDEXES>
          <INDEX NAME="action" UNIQUE="false" FIELDS="action" COMMENT="insert/update/delete"/>
 +        <INDEX NAME="timemodified" UNIQUE="false" FIELDS="timemodified"/>
        </INDEXES>
      </TABLE>
      <TABLE NAME="grade_categories_history" COMMENT="History of grade_categories">
        </KEYS>
        <INDEXES>
          <INDEX NAME="action" UNIQUE="false" FIELDS="action" COMMENT="insert/update/delete"/>
 +        <INDEX NAME="timemodified" UNIQUE="false" FIELDS="timemodified"/>
        </INDEXES>
      </TABLE>
      <TABLE NAME="grade_items_history" COMMENT="History of grade_items">
        </KEYS>
        <INDEXES>
          <INDEX NAME="action" UNIQUE="false" FIELDS="action" COMMENT="insert/update/delete"/>
 +        <INDEX NAME="timemodified" UNIQUE="false" FIELDS="timemodified"/>
        </INDEXES>
      </TABLE>
      <TABLE NAME="grade_grades_history" COMMENT="History table">
          <FIELD NAME="predictionscore" TYPE="number" LENGTH="10" NOTNULL="true" SEQUENCE="false" DECIMALS="5"/>
          <FIELD NAME="calculations" TYPE="text" NOTNULL="true" SEQUENCE="false"/>
          <FIELD NAME="timecreated" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
 +        <FIELD NAME="timestart" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false"/>
 +        <FIELD NAME="timeend" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false"/>
        </FIELDS>
        <KEYS>
          <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
          <INDEX NAME="predictionidanduseridandactionname" UNIQUE="false" FIELDS="predictionid, userid, actionname"/>
        </INDEXES>
      </TABLE>
 +    <TABLE NAME="analytics_used_analysables" COMMENT="List of analysables used by each model">
 +      <FIELDS>
 +        <FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
 +        <FIELD NAME="modelid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
 +        <FIELD NAME="action" TYPE="char" LENGTH="50" NOTNULL="true" SEQUENCE="false"/>
 +        <FIELD NAME="analysableid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
 +        <FIELD NAME="timeanalysed" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
 +      </FIELDS>
 +      <KEYS>
 +        <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
 +        <KEY NAME="modelid" TYPE="foreign" FIELDS="modelid" REFTABLE="analytics_models" REFFIELDS="id"/>
 +      </KEYS>
 +      <INDEXES>
 +        <INDEX NAME="modelid-action" UNIQUE="false" FIELDS="modelid, action"/>
 +      </INDEXES>
 +    </TABLE>
+     <TABLE NAME="search_index_requests" COMMENT="Records requests for (re)indexing of specific contexts. Entries will be removed from this table when indexing of that context is complete. (This table is not used for normal time-based indexing of new content.)">
+       <FIELDS>
+         <FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
+         <FIELD NAME="contextid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="Context ID that has been requested for reindexing."/>
+         <FIELD NAME="searcharea" TYPE="char" LENGTH="255" NOTNULL="true" SEQUENCE="false" COMMENT="Set (e.g. 'forum-post') if a specific area is to be reindexed. Blank indicates all areas."/>
+         <FIELD NAME="timerequested" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="Time at which this index update was requested."/>
+         <FIELD NAME="partialarea" TYPE="char" LENGTH="255" NOTNULL="true" SEQUENCE="false" COMMENT="If processing of this context partially completed, set to the area that needs processing next. Blank indicates not processed yet."/>
+         <FIELD NAME="partialtime" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="If processing partially completed, set to the timestamp within the next area where processing should start. 0 indicates not processed yet."/>
+       </FIELDS>
+       <KEYS>
+         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
+         <KEY NAME="contextid" TYPE="foreign" FIELDS="contextid" REFTABLE="context" REFFIELDS="id"/>
+       </KEYS>
+     </TABLE>
    </TABLES>
 -</XMLDB>
 +</XMLDB>
diff --combined lib/db/upgrade.php
@@@ -2601,104 -2601,31 +2601,129 @@@ function xmldb_main_upgrade($oldversion
          upgrade_main_savepoint(true, 2017092900.00);
      }
  
 -    if ($oldversion < 2017100600.01) {
 +    if ($oldversion < 2017100900.00) {
 +        // Add index on time modified to grade_outcomes_history, grade_categories_history,
 +        // grade_items_history, and scale_history.
 +        $table = new xmldb_table('grade_outcomes_history');
 +        $index = new xmldb_index('timemodified', XMLDB_INDEX_NOTUNIQUE, array('timemodified'));
  
 -        upgrade_main_savepoint(true, 2017100600.01);
 +        if (!$dbman->index_exists($table, $index)) {
 +            $dbman->add_index($table, $index);
 +        }
 +
 +        $table = new xmldb_table('grade_items_history');
 +        $index = new xmldb_index('timemodified', XMLDB_INDEX_NOTUNIQUE, array('timemodified'));
 +
 +        if (!$dbman->index_exists($table, $index)) {
 +            $dbman->add_index($table, $index);
 +        }
 +
 +        $table = new xmldb_table('grade_categories_history');
 +        $index = new xmldb_index('timemodified', XMLDB_INDEX_NOTUNIQUE, array('timemodified'));
 +
 +        if (!$dbman->index_exists($table, $index)) {
 +            $dbman->add_index($table, $index);
 +        }
 +
 +        $table = new xmldb_table('scale_history');
 +        $index = new xmldb_index('timemodified', XMLDB_INDEX_NOTUNIQUE, array('timemodified'));
 +
 +        if (!$dbman->index_exists($table, $index)) {
 +            $dbman->add_index($table, $index);
 +        }
 +
 +        // Main savepoint reached.
 +        upgrade_main_savepoint(true, 2017100900.00);
 +    }
 +
 +    if ($oldversion < 2017101000.00) {
 +
 +        // Define table analytics_used_analysables to be created.
 +        $table = new xmldb_table('analytics_used_analysables');
 +
 +        // Adding fields to table analytics_used_analysables.
 +        $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
 +        $table->add_field('modelid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
 +        $table->add_field('action', XMLDB_TYPE_CHAR, '50', null, XMLDB_NOTNULL, null, null);
 +        $table->add_field('analysableid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
 +        $table->add_field('timeanalysed', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
 +
 +        // Adding keys to table analytics_used_analysables.
 +        $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
 +        $table->add_key('modelid', XMLDB_KEY_FOREIGN, array('modelid'), 'analytics_models', array('id'));
 +
 +        // Adding indexes to table analytics_used_analysables.
 +        $table->add_index('modelid-action', XMLDB_INDEX_NOTUNIQUE, array('modelid', 'action'));
 +
 +        // Conditionally launch create table for analytics_used_analysables.
 +        if (!$dbman->table_exists($table)) {
 +            $dbman->create_table($table);
 +        }
 +
 +        // Main savepoint reached.
 +        upgrade_main_savepoint(true, 2017101000.00);
 +    }
 +
 +    if ($oldversion < 2017101000.01) {
 +        // Define field override to be added to course_modules_completion.
 +        $table = new xmldb_table('course_modules_completion');
 +        $field = new xmldb_field('overrideby', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'viewed');
 +
 +        // Conditionally launch add field override.
 +        if (!$dbman->field_exists($table, $field)) {
 +            $dbman->add_field($table, $field);
 +        }
 +
 +        // Main savepoint reached.
 +        upgrade_main_savepoint(true, 2017101000.01);
 +      }
 +
 +      if ($oldversion < 2017101000.02) {
 +        // Define field 'timestart' to be added to 'analytics_predictions'.
 +        $table = new xmldb_table('analytics_predictions');
 +        $field = new xmldb_field('timestart', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'timecreated');
 +
 +        // Conditionally launch add field 'timestart'.
 +        if (!$dbman->field_exists($table, $field)) {
 +            $dbman->add_field($table, $field);
 +        }
 +
 +        // Define field 'timeend' to be added to 'analytics_predictions'.
 +        $field = new xmldb_field('timeend', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'timestart');
 +
 +        // Conditionally launch add field 'timeend'.
 +        if (!$dbman->field_exists($table, $field)) {
 +            $dbman->add_field($table, $field);
 +        }
 +
 +        // Main savepoint reached.
 +        upgrade_main_savepoint(true, 2017101000.02);
 +    }
 +
++    if ($oldversion < 2017101200.00) {
+         // Define table search_index_requests to be created.
+         $table = new xmldb_table('search_index_requests');
+         // Adding fields to table search_index_requests.
+         $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
+         $table->add_field('contextid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+         $table->add_field('searcharea', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null);
+         $table->add_field('timerequested', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+         $table->add_field('partialarea', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null);
+         $table->add_field('partialtime', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+         // Adding keys to table search_index_requests.
+         $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
+         $table->add_key('contextid', XMLDB_KEY_FOREIGN, array('contextid'), 'context', array('id'));
+         // Conditionally launch create table for search_index_requests.
+         if (!$dbman->table_exists($table)) {
+             $dbman->create_table($table);
+         }
+         // Main savepoint reached.
++        upgrade_main_savepoint(true, 2017101200.00);
+     }
      return true;
  }
@@@ -157,6 -157,17 +157,17 @@@ class manager 
          return !empty($CFG->enableglobalsearch);
      }
  
+     /**
+      * Returns whether indexing is enabled or not (you can enable indexing even when search is not
+      * enabled at the moment, so as to have it ready for students).
+      *
+      * @return bool True if indexing is enabled.
+      */
+     public static function is_indexing_enabled() {
+         global $CFG;
+         return !empty($CFG->enableglobalsearch) || !empty($CFG->searchindexwhendisabled);
+     }
      /**
       * Returns an instance of the search engine.
       *
       *
       * @param bool $fullindex Whether we should reindex everything or not.
       * @param float $timelimit Time limit in seconds (0 = no time limit)
-      * @param \progress_trace $progress Optional class for tracking progress
+      * @param \progress_trace|null $progress Optional class for tracking progress
       * @throws \moodle_exception
       * @return bool Whether there was any updated document or not.
       */
      public function index($fullindex = false, $timelimit = 0, \progress_trace $progress = null) {
+         global $DB;
          // Cannot combine time limit with reindex.
          if ($timelimit && $fullindex) {
              throw new \coding_exception('Cannot apply time limit when reindexing');
  
              if ($fullindex === true) {
                  $referencestarttime = 0;
+                 // For full index, we delete any queued context index requests, as those will
+                 // obviously be met by the full index.
+                 $DB->delete_records('search_index_requests');
              } else {
                  $partial = get_config($componentconfigname, $varname . '_partial');
                  if ($partial) {
              if ($timelimit) {
                  $options['stopat'] = $stopat;
              }
 -            $iterator = new \core\dml\recordset_walk($recordset, array($searcharea, 'get_document'), $options);
 +            $iterator = new skip_future_documents_iterator(new \core\dml\recordset_walk(
 +                    $recordset, array($searcharea, 'get_document'), $options));
              $result = $this->engine->add_documents($iterator, $searcharea, $options);
 +            $recordset->close();
              if (count($result) === 5) {
                  list($numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial) = $result;
              } else {
          return (bool)$sumdocs;
      }
  
+     /**
+      * Indexes or reindexes a specific context of the system, e.g. one course.
+      *
+      * The function returns an object with field 'complete' (true or false).
+      *
+      * This function supports partial indexing via the time limit parameter. If the time limit
+      * expires, it will return values for $startfromarea and $startfromtime which can be passed
+      * next time to continue indexing.
+      *
+      * @param \context $context Context to restrict index.
+      * @param string $singleareaid If specified, indexes only the given area.
+      * @param float $timelimit Time limit in seconds (0 = no time limit)
+      * @param \progress_trace|null $progress Optional class for tracking progress
+      * @param string $startfromarea Area to start from
+      * @param int $startfromtime Timestamp to start from
+      * @return \stdClass Object indicating success
+      */
+     public function index_context($context, $singleareaid = '', $timelimit = 0,
+             \progress_trace $progress = null, $startfromarea = '', $startfromtime = 0) {
+         if (!$progress) {
+             $progress = new \null_progress_trace();
+         }
+         // Work out time to stop, if limited.
+         if ($timelimit) {
+             // Decide time to stop.
+             $stopat = microtime(true) + $timelimit;
+         }
+         // No PHP time limit.
+         \core_php_time_limit::raise();
+         // Notify the engine that an index starting.
+         $this->engine->index_starting(false);
+         $sumdocs = 0;
+         // Get all search areas, in consistent order.
+         $searchareas = $this->get_search_areas_list(true);
+         ksort($searchareas);
+         // Are we skipping past some that were handled previously?
+         $skipping = $startfromarea ? true : false;
+         foreach ($searchareas as $areaid => $searcharea) {
+             // If we're only processing one area id, skip all the others.
+             if ($singleareaid && $singleareaid !== $areaid) {
+                 continue;
+             }
+             // If we're skipping to a later area, continue through the loop.
+             $referencestarttime = 0;
+             if ($skipping) {
+                 if ($areaid !== $startfromarea) {
+                     continue;
+                 }
+                 // Stop skipping and note the reference start time.
+                 $skipping = false;
+                 $referencestarttime = $startfromtime;
+             }
+             $progress->output('Processing area: ' . $searcharea->get_visible_name());
+             $elapsed = microtime(true);
+             // Get the recordset of all documents from the area for this context.
+             $recordset = $searcharea->get_document_recordset($referencestarttime, $context);
+             if (!$recordset) {
+                 if ($recordset === null) {
+                     $progress->output('Skipping (not relevant to context).', 1);
+                 } else {
+                     $progress->output('Skipping (does not support context indexing).', 1);
+                 }
+                 continue;
+             }
+             // Notify the engine that an area is starting.
+             $this->engine->area_index_starting($searcharea, false);
+             // Work out search options.
+             $options = [];
+             $options['indexfiles'] = $this->engine->file_indexing_enabled() &&
+                     $searcharea->uses_file_indexing();
+             if ($timelimit) {
+                 $options['stopat'] = $stopat;
+             }
+             // Construct iterator which will use get_document on the recordset results.
+             $iterator = new \core\dml\recordset_walk($recordset,
+                     array($searcharea, 'get_document'), $options);
+             // Use this iterator to add documents.
+             $result = $this->engine->add_documents($iterator, $searcharea, $options);
+             if (count($result) === 5) {
+                 list($numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial) = $result;
+             } else {
+                 // Backward compatibility for engines that don't support partial adding.
+                 list($numrecords, $numdocs, $numdocsignored, $lastindexeddoc) = $result;
+                 debugging('engine::add_documents() should return $partial (4-value return is deprecated)',
+                         DEBUG_DEVELOPER);
+                 $partial = false;
+             }
+             if ($numdocs > 0) {
+                 $elapsed = round((microtime(true) - $elapsed), 3);
+                 $progress->output('Processed ' . $numrecords . ' records containing ' . $numdocs .
+                         ' documents, in ' . $elapsed . ' seconds' .
+                         ($partial ? ' (not complete)' : '') . '.', 1);
+             } else {
+                 $progress->output('No documents to index.', 1);
+             }
+             // Notify the engine this area is complete, but don't store any times as this is not
+             // part of the 'normal' search index.
+             if (!$this->engine->area_index_complete($searcharea, $numdocs, false)) {
+                 $progress->output('Engine reported error.', 1);
+             }
+             if ($partial && $timelimit && (microtime(true) >= $stopat)) {
+                 $progress->output('Stopping indexing due to time limit.');
+                 break;
+             }
+         }
+         if ($sumdocs > 0) {
+             $event = \core\event\search_indexed::create(
+                     array('context' => $context));
+             $event->trigger();
+         }
+         $this->engine->index_complete($sumdocs, false);
+         // Indicate in result whether we completed indexing, or only part of it.
+         $result = new \stdClass();
+         if ($partial) {
+             $result->complete = false;
+             $result->startfromarea = $areaid;
+             $result->startfromtime = $lastindexeddoc;
+         } else {
+             $result->complete = true;
+         }
+         return $result;
+     }
      /**
       * Resets areas config.
       *
  
          return false;
      }
+     /**
+      * Requests that a specific context is indexed by the scheduled task. The context will be
+      * added to a queue which is processed by the task.
+      *
+      * This is used after a restore to ensure that restored items are indexed, even though their
+      * modified time will be older than the latest indexed.
+      *
+      * @param \context $context Context to index within
+      * @param string $areaid Area to index, '' = all areas
+      */
+     public static function request_index(\context $context, $areaid = '') {
+         global $DB;
+         // Check through existing requests for this context or any parent context.
+         list ($contextsql, $contextparams) = $DB->get_in_or_equal(
+                 $context->get_parent_context_ids(true));
+         $existing = $DB->get_records_select('search_index_requests',
+                 'contextid ' . $contextsql, $contextparams, '', 'id, searcharea, partialarea');
+         foreach ($existing as $rec) {
+             // If we haven't started processing the existing request yet, and it covers the same
+             // area (or all areas) then that will be sufficient so don't add anything else.
+             if ($rec->partialarea === '' && ($rec->searcharea === $areaid || $rec->searcharea === '')) {
+                 return;
+             }
+         }
+         // No suitable existing request, so add a new one.
+         $newrecord = [ 'contextid' => $context->id, 'searcharea' => $areaid,
+                 'timerequested' => time(), 'partialarea' => '', 'partialtime' => 0 ];
+         $DB->insert_record('search_index_requests', $newrecord);
+     }
+     /**
+      * Processes outstanding index requests. This will take the first item from the queue and
+      * process it, continuing until an optional time limit is reached.
+      *
+      * If there are no index requests, the function will do nothing.
+      *
+      * @param float $timelimit Time limit (0 = none)
+      * @param \progress_trace|null $progress Optional progress indicator
+      */
+     public function process_index_requests($timelimit = 0.0, \progress_trace $progress = null) {
+         global $DB;
+         if (!$progress) {
+             $progress = new \null_progress_trace();
+         }
+         $complete = false;
+         $before = microtime(true);
+         if ($timelimit) {
+             $stopat = $before + $timelimit;
+         }
+         while (true) {
+             // Retrieve first request, using fully defined ordering.
+             $requests = $DB->get_records('search_index_requests', null,
+                     'timerequested, contextid, searcharea',
+                     'id, contextid, searcharea, partialarea, partialtime', 0, 1);
+             if (!$requests) {
+                 // If there are no more requests, stop.
+                 $complete = true;
+                 break;
+             }
+             $request = reset($requests);
+             // Calculate remaining time.
+             $remainingtime = 0;
+             $beforeindex = microtime(true);
+             if ($timelimit) {
+                 $remainingtime = $stopat - $beforeindex;
+             }
+             // Show a message before each request, indicating what will be indexed.
+             $context = \context::instance_by_id($request->contextid);
+             $contextname = $context->get_context_name();
+             if ($request->searcharea) {
+                 $contextname .= ' (search area: ' . $request->searcharea . ')';
+             }
+             $progress->output('Indexing requested context: ' . $contextname);
+             // Actually index the context.
+             $result = $this->index_context($context, $request->searcharea, $remainingtime,
+                     $progress, $request->partialarea, $request->partialtime);
+             // Work out shared part of message.
+             $endmessage = $contextname . ' (' . round(microtime(true) - $beforeindex, 1) . 's)';
+             // Update database table and continue/stop as appropriate.
+             if ($result->complete) {
+                 // If we completed the request, remove it from the table.
+                 $DB->delete_records('search_index_requests', ['id' => $request->id]);
+                 $progress->output('Completed requested context: ' . $endmessage);
+             } else {
+                 // If we didn't complete the request, store the partial details (how far it got).
+                 $DB->update_record('search_index_requests', ['id' => $request->id,
+                         'partialarea' => $result->startfromarea,
+                         'partialtime' => $result->startfromtime]);
+                 $progress->output('Ending requested context: ' . $endmessage);
+                 // The time limit must have expired, so stop looping.
+                 break;
+             }
+         }
+     }
  }
@@@ -261,58 -261,106 +261,157 @@@ class search_manager_testcase extends a
          $this->assertFalse(get_config($componentname, $varname . '_partial'));
      }
  
 -    }
 +    /**
 +     * Tests that documents with modified time in the future are NOT indexed (as this would cause
 +     * a problem by preventing it from indexing other documents modified between now and the future
 +     * date).
 +     */
 +    public function test_future_documents() {
 +        $this->resetAfterTest();
 +
 +        // Create a course and a forum.
 +        $generator = $this->getDataGenerator();
 +        $course = $generator->create_course();
 +        $forum = $generator->create_module('forum', ['course' => $course->id]);
 +
 +        // Index everything up to current. Ensure the course is older than current second so it
 +        // definitely doesn't get indexed again next time.
 +        $this->waitForSecond();
 +        $search = testable_core_search::instance();
 +        $search->index(false, 0);
 +        $search->get_engine()->get_and_clear_added_documents();
 +
 +        // Add 2 discussions to the forum, one of which happend just now, but the other is
 +        // incorrectly set to the future.
 +        $now = time();
 +        $userid = get_admin()->id;
 +        $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
 +                'forum' => $forum->id, 'userid' => $userid, 'timemodified' => $now,
 +                'name' => 'Frog']);
 +        $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
 +                'forum' => $forum->id, 'userid' => $userid, 'timemodified' => $now + 100,
 +                'name' => 'Toad']);
 +
 +        // Wait for a second so we're not actually on the same second as the forum post (there's a
 +        // 1 second overlap between indexing; it would get indexed in both checks below otherwise).
 +        $this->waitForSecond();
 +
 +        // Index.
 +        $search->index(false);
 +        $added = $search->get_engine()->get_and_clear_added_documents();
 +        $this->assertCount(1, $added);
 +        $this->assertEquals('Frog', $added[0]->get('title'));
 +
 +        // Check latest time - it should be the same as $now, not the + 100.
 +        $searcharea = $search->get_search_area($this->forumpostareaid);
 +        list($componentname, $varname) = $searcharea->get_config_var_name();
 +        $this->assertEquals($now, get_config($componentname, $varname . '_lastindexrun'));
 +
 +        // Index again - there should be nothing to index this time.
 +        $search->index(false);
 +        $added = $search->get_engine()->get_and_clear_added_documents();
 +        $this->assertCount(0, $added);
 +    }
 +
+     /**
+      * Tests that indexing a specified context works correctly.
+      */
+     public function test_context_indexing() {
+         global $USER;
+         $this->resetAfterTest();
+         $this->setAdminUser();
+         // Create a course and two forums and a page.
+         $generator = $this->getDataGenerator();
+         $course = $generator->create_course();
+         $now = time();
+         $forum1 = $generator->create_module('forum', ['course' => $course->id]);
+         $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
+                 'forum' => $forum1->id, 'userid' => $USER->id, 'timemodified' => $now,
+                 'name' => 'Frog']);
+         $this->waitForSecond();
+         $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
+                 'forum' => $forum1->id, 'userid' => $USER->id, 'timemodified' => $now + 2,
+                 'name' => 'Zombie']);
+         $forum2 = $generator->create_module('forum', ['course' => $course->id]);
+         $this->waitForSecond();
+         $generator->get_plugin_generator('mod_forum')->create_discussion(['course' => $course->id,
+                 'forum' => $forum2->id, 'userid' => $USER->id, 'timemodified' => $now + 1,
+                 'name' => 'Toad']);
+         $generator->create_module('page', ['course' => $course->id]);
+         $generator->create_module('forum', ['course' => $course->id]);
+         // Index forum 1 only.
+         $search = testable_core_search::instance();
+         $buffer = new progress_trace_buffer(new text_progress_trace(), false);
+         $result = $search->index_context(\context_module::instance($forum1->cmid), '', 0, $buffer);
+         $this->assertTrue($result->complete);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         // Confirm that output only processed 1 forum activity and 2 posts.
+         var_dump(strpos($log, "area: Forum - activity information\n  Processed 1 "));
+         $this->assertNotFalse(strpos($log, "area: Forum - activity information\n  Processed 1 "));
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 2 "));
+         // Confirm that some areas for different types of context were skipped.
+         $this->assertNotFalse(strpos($log, "area: Users\n  Skipping"));
+         $this->assertNotFalse(strpos($log, "area: My courses\n  Skipping"));
+         // Confirm that another module area had no results.
+         $this->assertNotFalse(strpos($log, "area: Page\n  No documents"));
+         // Index whole course.
+         $result = $search->index_context(\context_course::instance($course->id), '', 0, $buffer);
+         $this->assertTrue($result->complete);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         // Confirm that output processed 3 forum activities and 3 posts.
+         $this->assertNotFalse(strpos($log, "area: Forum - activity information\n  Processed 3 "));
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 3 "));
+         // The course area was also included this time.
+         $this->assertNotFalse(strpos($log, "area: My courses\n  Processed 1 "));
+         // Confirm that another module area had results too.
+         $this->assertNotFalse(strpos($log, "area: Page\n  Processed 1 "));
+         // Index whole course, but only forum posts.
+         $result = $search->index_context(\context_course::instance($course->id), 'mod_forum-post',
+                 0, $buffer);
+         $this->assertTrue($result->complete);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         // Confirm that output processed 3 posts but not forum activities.
+         $this->assertFalse(strpos($log, "area: Forum - activity information"));
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 3 "));
+         // Set time limit and retry index of whole course, taking 3 tries to complete it.
+         $search->get_engine()->set_add_delay(0.4);
+         $result = $search->index_context(\context_course::instance($course->id), '', 1, $buffer);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         $this->assertFalse($result->complete);
+         $this->assertNotFalse(strpos($log, "area: Forum - activity information\n  Processed 2 "));
+         $result = $search->index_context(\context_course::instance($course->id), '', 1, $buffer,
+                 $result->startfromarea, $result->startfromtime);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         $this->assertNotFalse(strpos($log, "area: Forum - activity information\n  Processed 2 "));
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 2 "));
+         $this->assertFalse($result->complete);
+         $result = $search->index_context(\context_course::instance($course->id), '', 1, $buffer,
+                 $result->startfromarea, $result->startfromtime);
+         $log = $buffer->get_buffer();
+         $buffer->reset_buffer();
+         $this->assertNotFalse(strpos($log, "area: Forum - posts\n  Processed 2 "));
+         $this->assertTrue($result->complete);
      /**
       * Adding this test here as get_areas_user_accesses process is the same, results just depend on the context level.
       *
          $this->assertTrue(testable_core_search::is_search_area('\\mod_forum\\search\\post'));
          $this->assertTrue(testable_core_search::is_search_area('mod_forum\\search\\post'));
      }
+     /**
+      * Tests the request_index function used for reindexing certain contexts. This only tests
+      * adding things to the request list, it doesn't test that they are actually indexed by the
+      * scheduled task.
+      */
+     public function test_request_index() {
+         global $DB;
+         $this->resetAfterTest();
+         $course1 = $this->getDataGenerator()->create_course();
+         $course1ctx = context_course::instance($course1->id);
+         $course2 = $this->getDataGenerator()->create_course();
+         $course2ctx = context_course::instance($course2->id);
+         $forum1 = $this->getDataGenerator()->create_module('forum', ['course' => $course1->id]);
+         $forum1ctx = context_module::instance($forum1->cmid);
+         $forum2 = $this->getDataGenerator()->create_module('forum', ['course' => $course2->id]);
+         $forum2ctx = context_module::instance($forum2->cmid);
+         // Initially no requests.
+         $this->assertEquals(0, $DB->count_records('search_index_requests'));
+         // Request update for course 1, all areas.
+         \core_search\manager::request_index($course1ctx);
+         // Check all details of entry.
+         $results = array_values($DB->get_records('search_index_requests'));
+         $this->assertCount(1, $results);
+         $this->assertEquals($course1ctx->id, $results[0]->contextid);
+         $this->assertEquals('', $results[0]->searcharea);
+         $now = time();
+         $this->assertLessThanOrEqual($now, $results[0]->timerequested);
+         $this->assertGreaterThan($now - 10, $results[0]->timerequested);
+         $this->assertEquals('', $results[0]->partialarea);
+         $this->assertEquals(0, $results[0]->partialtime);
+         // Request forum 1, all areas; not added as covered by course 1.
+         \core_search\manager::request_index($forum1ctx);
+         $this->assertEquals(1, $DB->count_records('search_index_requests'));
+         // Request forum 1, specific area; not added as covered by course 1 all areas.
+         \core_search\manager::request_index($forum1ctx, 'forum-post');
+         $this->assertEquals(1, $DB->count_records('search_index_requests'));
+         // Request course 1 again, specific area; not added as covered by all areas.
+         \core_search\manager::request_index($course1ctx, 'forum-post');
+         $this->assertEquals(1, $DB->count_records('search_index_requests'));
+         // Request course 1 again, all areas; not needed as covered already.
+         \core_search\manager::request_index($course1ctx);
+         $this->assertEquals(1, $DB->count_records('search_index_requests'));
+         // Request course 2, specific area.
+         \core_search\manager::request_index($course2ctx, 'label-activity');
+         // Note: I'm ordering by ID for convenience - this is dangerous in real code (see MDL-43447)
+         // but in a unit test it shouldn't matter as nobody is using clustered databases for unit
+         // test.
+         $results = array_values($DB->get_records('search_index_requests', null, 'id'));
+         $this->assertCount(2, $results);
+         $this->assertEquals($course1ctx->id, $results[0]->contextid);
+         $this->assertEquals($course2ctx->id, $results[1]->contextid);
+         $this->assertEquals('label-activity', $results[1]->searcharea);
+         // Request forum 2, same specific area; not added.
+         \core_search\manager::request_index($forum2ctx, 'label-activity');
+         $this->assertEquals(2, $DB->count_records('search_index_requests'));
+         // Request forum 2, different specific area; added.
+         \core_search\manager::request_index($forum2ctx, 'forum-post');
+         $this->assertEquals(3, $DB->count_records('search_index_requests'));
+         // Request forum 2, all areas; also added. (Note: This could obviously remove the previous
+         // one, but for simplicity, I didn't make it do that; also it could perhaps cause problems
+         // if we had already begun processing the previous entry.)
+         \core_search\manager::request_index($forum2ctx);
+         $this->assertEquals(4, $DB->count_records('search_index_requests'));
+     }
+     /**
+      * Tests the process_index_requests function.
+      */
+     public function test_process_index_requests() {
+         global $DB;
+         $this->resetAfterTest();
+         $search = testable_core_search::instance();
+         // When there are no index requests, nothing gets logged.
+         $progress = new progress_trace_buffer(new text_progress_trace(), false);
+         $search->process_index_requests(0.0, $progress);
+         $out = $progress->get_buffer();
+         $progress->reset_buffer();
+         $this->assertEquals('', $out);
+         // Set up the course with 3 forums.
+         $generator = $this->getDataGenerator();
+         $course = $generator->create_course(['fullname' => 'TCourse']);
+         $forum1 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum1']);
+         $forum2 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum2']);
+         $forum3 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum3']);
+         // Hack the forums so they have different creation times.
+         $now = time();
+         $DB->set_field('forum', 'timemodified', $now - 3, ['id' => $forum1->id]);
+         $DB->set_field('forum', 'timemodified', $now - 2, ['id' => $forum2->id]);
+         $DB->set_field('forum', 'timemodified', $now - 1, ['id' => $forum3->id]);
+         $forum2time = $now - 2;
+         // Make 2 index requests.
+         $search::request_index(context_course::instance($course->id), 'mod_label-activity');
+         $this->waitForSecond();
+         $search::request_index(context_module::instance($forum1->cmid));
+         // Run with no time limit.
+         $search->process_index_requests(0.0, $progress);
+         $out = $progress->get_buffer();
+         $progress->reset_buffer();
+         // Check that it's done both areas.
+         $this->assertContains(
+                 'Indexing requested context: Course: TCourse (search area: mod_label-activity)',
+                 $out);
+         $this->assertContains(
+                 'Completed requested context: Course: TCourse (search area: mod_label-activity)',
+                 $out);
+         $this->assertContains('Indexing requested context: Forum: TForum1', $out);
+         $this->assertContains('Completed requested context: Forum: TForum1', $out);
+         // Check the requests database table is now empty.
+         $this->assertEquals(0, $DB->count_records('search_index_requests'));
+         // Request indexing the course a couple of times.
+         $search::request_index(context_course::instance($course->id), 'mod_forum-activity');
+         $search::request_index(context_course::instance($course->id), 'mod_forum-post');
+         // Do the processing again with a time limit and indexing delay. The time limit is too
+         // small; because of the way the logic works, this means it will index 2 activities.
+         $search->get_engine()->set_add_delay(0.2);
+         $search->process_index_requests(0.1, $progress);
+         $out = $progress->get_buffer();
+         $progress->reset_buffer();
+         // Confirm the right wrapper information was logged.
+         $this->assertContains(
+                 'Indexing requested context: Course: TCourse (search area: mod_forum-activity)',
+                 $out);
+         $this->assertContains('Stopping indexing due to time limit', $out);
+         $this->assertContains(
+                 'Ending requested context: Course: TCourse (search area: mod_forum-activity)',
+                 $out);
+         // Check the database table has been updated with progress.
+         $records = array_values($DB->get_records('search_index_requests', null, 'searcharea'));
+         $this->assertEquals('mod_forum-activity', $records[0]->partialarea);
+         $this->assertEquals($forum2time, $records[0]->partialtime);
+         // Run again and confirm it now finishes.
+         $search->process_index_requests(0.1, $progress);
+         $out = $progress->get_buffer();
+         $progress->reset_buffer();
+         $this->assertContains(
+                 'Completed requested context: Course: TCourse (search area: mod_forum-activity)',
+                 $out);
+         $this->assertContains(
+                 'Completed requested context: Course: TCourse (search area: mod_forum-post)',
+                 $out);
+         // Confirm table is now empty.
+         $this->assertEquals(0, $DB->count_records('search_index_requests'));
+     }
  }
diff --combined version.php
@@@ -29,7 -29,7 +29,7 @@@
  
  defined('MOODLE_INTERNAL') || die();
  
- $version  = 2017101000.02;              // YYYYMMDD      = weekly release date of this DEV branch.
 -$version  = 2017100600.01;              // YYYYMMDD      = weekly release date of this DEV branch.
++$version  = 2017101200.00;              // YYYYMMDD      = weekly release date of this DEV branch.
                                          //         RR    = release increments - 00 in DEV branches.
                                          //           .XX = incremental changes.